Merge pull request #7 from maximilianiKIT/5-plugin-functionality

5 plugin functionality

Author: maximiliani

.github/workflows/CI.yml

@@ -7,7 +7,7 @@ name: Java CI
 
 on:
   push:
-    branches: [ main ]
+    branches: [ main , dev]
   pull_request:
     branches: "*"
 
@@ -28,13 +28,11 @@ jobs:
         run: sudo apt install -y python3 python3-setuptools python3-pip
       - name: Update pip
         run: pip3 install --upgrade pip
-      - name: Install libraries via pip (xmltodict and wget)
-        run: pip3 install xmltodict wget
       - name: Grant execute permission for gradlew
         run: chmod +x gradlew
       - name: Build with Gradle
         run: ./gradlew clean check jacocoTestReport
       - name: Codecov
         uses: codecov/codecov-action@v1
         with:
-          files: ./build/reports/jacoco/test/jacocoTestReport.xml #optional
+          files: ./build/reports/jacoco/test/jacocoTestReport.xml

.github/workflows/codeql-analysis.yml

@@ -16,7 +16,7 @@ on:
     branches: [ "main", "dev" ]
   pull_request:
     # The branches below must be a subset of the branches above
-    branches: [ "main", "dev" ]
+    branches: "*"
   schedule:
     - cron: '42 9 * * 4'
 
@@ -40,11 +40,17 @@ jobs:
     - name: Checkout repository
       uses: actions/checkout@v3
 
+    - name: Set up OpenJDK
+      uses: actions/setup-java@v1
+      with:
+        java-version: ${{ matrix.jdk }}
+
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
+        java-version: 17
         # If you wish to specify custom queries, you can do so here or in a config file.
         # By default, queries listed here will override any specified in a config file.
         # Prefix the list here with "+" to use these queries and those in the config file.
@@ -57,16 +63,18 @@ jobs:
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
       uses: github/codeql-action/autobuild@v2
+      with:
+        java-version: 17
 
-    # ℹ️ Command-line programs to run using the OS shell.
+      # ℹ️ Command-line programs to run using the OS shell.
     # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
 
     #   If the Autobuild fails above, remove it and uncomment the following three lines. 
     #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
 
-    # - run: |
-    #   echo "Run, Build Application using script"
-    #   ./location_of_script_within_repo/buildscript.sh
+#   #  - run: |
+#       echo "Run, Build Application using script"
+#       ./location_of_script_within_repo/buildscript.sh
 
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@v2

.github/workflows/dependabot.yml

@@ -0,0 +1,11 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"

.gitignore

@@ -234,4 +234,6 @@ gradle-app.setting
 # JDT-specific (Eclipse Java Development Tools)
 .classpath
 
-# End of https://www.toptal.com/developers/gitignore/api/intellij+all,java,macos,gradle,linux
+# End of https://www.toptal.com/developers/gitignore/api/intellij+all,java,macos,gradle,linux
+lib/gemma
+!/plugins/gemma-plugin-0.1.0.jar

README.md

@@ -6,7 +6,6 @@
 
 :warning:
 Not fully tested yet!
-For mapping documents only Gemma is available currently!
 
 The mapping service allows multiple schemas to be uploaded and managed and documents to be mapped over one of them at a time accordingly.
 Therefore, the service is extensible to other mappers, although currently only Gemma, a service that can only map JSON files, is available.
@@ -17,28 +16,24 @@ The REST API is documented at the following link: [http://localhost:8095/swagger
 Dependencies that are needed to build and are not being downloaded via gradle:
 
 - OpenJDK 17
+- Python 3
 
 `./gradlew -Pclean-release build`
 
 ## How to start
 
-> TODO This section is a placeholder. It still needs to be written properly.
+`./gradlew bootRun`
 
 ### Prerequisites
-
-You might want to take a look at testbed4inf, which should make it easy to satisfy those.
-
-- Gemma
-- Python
-
-### Install Python and Gemma
-```
-sudo apt install -y python3 python3-pip 
-pip3 install xmltodict wget
-```
-
-
-## More information
+Please make sure the application.properties file is configured correctly.
+Espacially the following properties (at the end of the file) are important:
+- `mapping-service.pythonLocation=file:///opt/homebrew/bin/python3` \
+Enter the loacation of your python3 installation. You can determine it by typing `which python3` in your terminal.
+- `mapping-service.mappingsLocation:file:///tmp/mapping-service/` \
+Enter the location where you want to store your mappings. This folder will be created if it does not exist yet.
+
+You might want to add a plugin to make the service working. Normally, there should be a gemma-plugin-x.x.x.jar in the plugins folder.
+If not, you can find its source code [here](https://github.com/maximilianiKIT/gemma-plugin).
 
 ## License
 

build.gradle

@@ -10,9 +10,11 @@ plugins {
 
 description = 'Generic mapping service supporting different mapping implementations.'
 group = 'edu.kit.datamanager'
-version = '1.0.0'
 
+println "Running gradle version: $gradle.gradleVersion"
 println "Building ${name} version: ${version}"
+println "JDK version: ${JavaVersion.current()}"
+
 
 configurations {
     asciidoctorExt
@@ -31,43 +33,45 @@ ext {
 }
 
 dependencies {
-    implementation 'org.springframework.boot:spring-boot-starter-data-jpa:2.6.5'
-    implementation 'org.springframework.boot:spring-boot-starter-security:2.6.5'
-    implementation 'org.springframework.boot:spring-boot-starter-validation:2.6.5'
-    implementation 'org.springframework.boot:spring-boot-starter-web:2.6.5'
-    implementation 'org.springframework.boot:spring-boot-starter-actuator:2.6.6'
-    implementation 'org.springdoc:springdoc-openapi-ui:1.6.6'
-    implementation 'org.springdoc:springdoc-openapi-data-rest:1.6.6'
-    implementation 'org.springdoc:springdoc-openapi-webmvc-core:1.6.6'
-    implementation 'org.javers:javers-spring-boot-starter-sql:6.6.3'
+    implementation 'org.eclipse.jgit:org.eclipse.jgit:6.3.0.202209071007-r'
+    implementation 'org.springframework.boot:spring-boot-starter-data-jpa:2.7.3'
+    implementation 'org.springframework.boot:spring-boot-starter-security:2.7.3'
+    implementation 'org.springframework.boot:spring-boot-starter-validation:2.7.3'
+    implementation 'org.springframework.boot:spring-boot-starter-web:2.7.3'
+    implementation 'org.springframework.boot:spring-boot-starter-actuator:2.7.3'
+    implementation 'org.springdoc:springdoc-openapi-ui:1.6.11'
+    implementation 'org.springdoc:springdoc-openapi-data-rest:1.6.11'
+    implementation 'org.springdoc:springdoc-openapi-webmvc-core:1.6.11'
+    implementation 'org.javers:javers-spring-boot-starter-sql:6.6.5'
     implementation 'org.apache.httpcomponents:httpclient:4.5.13'
     implementation 'org.apache.commons:commons-collections4:4.4'
     implementation 'org.json:json:20220320'
     implementation 'com.github.jknack:handlebars:4.3.0'
     implementation 'com.google.guava:guava:31.1-jre'
     implementation 'commons-io:commons-io:2.11.0'
     implementation 'javax.validation:validation-api:2.0.1.Final'
-    implementation 'edu.kit.datamanager:service-base:1.0.1'
+    implementation 'edu.kit.datamanager:service-base:1.0.4'
 
-    testImplementation(platform('org.junit:junit-bom:5.8.2'))
-    testImplementation 'org.junit.jupiter:junit-jupiter:5.8.2'
-    testImplementation 'org.junit.jupiter:junit-jupiter-migrationsupport:5.8.2'
-    testImplementation 'org.springframework.boot:spring-boot-starter-test:2.6.5'
+    testImplementation platform('org.junit:junit-bom:5.9.0')
+    testImplementation 'org.junit.jupiter:junit-jupiter:5.9.0'
+    testImplementation 'org.junit.jupiter:junit-jupiter-migrationsupport:5.9.0'
+    testImplementation 'org.springframework.boot:spring-boot-starter-test:2.7.3'
     testImplementation 'org.springframework.restdocs:spring-restdocs-mockmvc:2.0.6.RELEASE'
-    testImplementation 'org.springframework.security:spring-security-test:5.6.2'
-    testImplementation 'org.springframework:spring-test:5.3.17'
-    testImplementation 'org.mockito:mockito-core:4.3.1'
+    testImplementation 'org.springframework.security:spring-security-test:5.7.3'
+    testImplementation 'org.springframework:spring-test:5.3.22'
+    testImplementation 'org.mockito:mockito-core:4.8.0'
     testImplementation 'org.powermock:powermock-module-junit4:2.0.9'
     testImplementation 'org.powermock:powermock-api-mockito2:2.0.9'
-    testImplementation 'org.junit.vintage:junit-vintage-engine:5.8.2'
+    testImplementation 'net.bytebuddy:byte-buddy:1.12.16'
+    testImplementation 'org.junit.vintage:junit-vintage-engine:5.9.0'
 
-    annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor:2.6.5'
-    annotationProcessor 'org.projectlombok:lombok:1.18.22'
+    annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor:2.7.3'
+    annotationProcessor 'org.projectlombok:lombok:1.18.24'
 
-    compileOnly 'org.projectlombok:lombok:1.18.22'
+    compileOnly 'org.projectlombok:lombok:1.18.24'
 
-    runtimeOnly 'com.h2database:h2:2.1.210'
-    runtimeOnly 'org.postgresql:postgresql:42.3.3'
+    runtimeOnly 'com.h2database:h2:2.1.214'
+    runtimeOnly 'org.postgresql:postgresql:42.5.0'
     runtimeOnly 'org.apache.httpcomponents:httpclient:4.5.13'
 
     asciidoctorExt 'org.springframework.restdocs:spring-restdocs-asciidoctor:2.0.6.RELEASE'
@@ -97,9 +101,20 @@ asciidoctor {
     dependsOn test
 }
 
+jar {
+    manifest {
+        attributes 'Main-Class': 'edu.kit.datamanager.mapping-service.MappingServiceApplication'
+    }
+}
+
 bootJar {
     dependsOn asciidoctor
     from ("${asciidoctor.outputDir}/html5") {
         into 'static/docs'
     }
+    launchScript()
+}
+
+release {
+    tagTemplate = 'v${version}'
 }

gradle/wrapper/gradle-wrapper.properties

@@ -3,3 +3,5 @@ distributionPath=wrapper/dists
 distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
+
+version=1.0.1-SNAPSHOT

src/main/java/edu/kit/datamanager/mappingservice/MappingServiceApplication.java

@@ -1,44 +1,23 @@
 package edu.kit.datamanager.mappingservice;
 
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
-import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import edu.kit.datamanager.mappingservice.configuration.ApplicationProperties;
+import edu.kit.datamanager.mappingservice.plugins.PluginManager;
+import edu.kit.datamanager.mappingservice.util.PythonRunnerUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.InjectionPoint;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.domain.EntityScan;
 import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.Scope;
-import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder;
-import org.springframework.scheduling.annotation.EnableScheduling;
+import org.springframework.context.annotation.Configuration;
 
 @SpringBootApplication
-@EnableScheduling
-@ComponentScan({"edu.kit.datamanager.mappingservice", "edu.kit.datamanager.configuration"})
 @EntityScan("edu.kit.datamanager")
+@Configuration
 public class MappingServiceApplication {
 
-    @Bean
-    @Scope("prototype")
-    public Logger logger(InjectionPoint injectionPoint) {
-        Class<?> targetClass = injectionPoint.getMember().getDeclaringClass();
-        return LoggerFactory.getLogger(targetClass.getCanonicalName());
-    }
-
-    @Bean(name = "OBJECT_MAPPER_BEAN")
-    public ObjectMapper jsonObjectMapper() {
-        return Jackson2ObjectMapperBuilder.json()
-                .serializationInclusion(JsonInclude.Include.NON_EMPTY) // Don’t include null values
-                .featuresToDisable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) //ISODate
-                .modules(new JavaTimeModule())
-                .build();
-    }
+    private static final Logger LOG = LoggerFactory.getLogger(MappingServiceApplication.class);
 
     @Bean
     @ConfigurationProperties("repo")
@@ -48,6 +27,10 @@ public ApplicationProperties applicationProperties() {
 
     public static void main(String[] args) {
         SpringApplication.run(MappingServiceApplication.class, args);
-        System.out.println("Mapping service is running!");
+
+        PluginManager.soleInstance().getListOfAvailableValidators().forEach((value) -> LOG.info("Found validator: " + value));
+        PythonRunnerUtil.printPythonVersion();
+
+        System.out.println("Mapping service is running! Access it at http://localhost:8095");
     }
 }

src/main/java/edu/kit/datamanager/mappingservice/configuration/ApplicationProperties.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2018 Karlsruhe Institute of Technology.
+ * Copyright 2022 Karlsruhe Institute of Technology.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -26,7 +26,10 @@
 import java.net.URL;
 
 /**
+ * This class is used to configure the application.
+ * It reads the values from the application.properties file.
  *
+ * @author maximilianiKIT
  */
 @ConfigurationProperties(prefix = "mapping-service")
 @Component
@@ -38,20 +41,13 @@ public class ApplicationProperties extends GenericPluginProperties {
      * The absolute path to the python interpreter.
      */
     @edu.kit.datamanager.annotations.ExecutableFileURL
-    @Value("${mapping-service.gemma.pythonLocation}")
+    @Value("${mapping-service.pythonLocation}")
     private URL pythonLocation;
 
-    /**
-     * The path to the gemma mapping script 'mapping_single.py'
-     */
-    @edu.kit.datamanager.annotations.LocalFileURL
-    @Value("${mapping-service.gemma.gemmaLocation}")
-    private URL gemmaLocation;
-
     /**
      * The absolute path where the mappings are stored.
      */
     @edu.kit.datamanager.annotations.LocalFolderURL
-    @Value("${mapping-service.mappingsLocation}")
+    @Value("${mapping-service.mappingSchemasLocation}")
     private URL mappingsLocation;
 }