Merge pull request #21 from kit-data-manager/development

PR for next release

Author: ThomasJejkal

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,38 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. iOS]
+ - Browser [e.g. chrome, safari]
+ - Version [e.g. 22]
+
+**Smartphone (please complete the following information):**
+ - Device: [e.g. iPhone6]
+ - OS: [e.g. iOS8.1]
+ - Browser [e.g. stock browser, safari]
+ - Version [e.g. 22]
+
+**Additional context**
+Add any other context about the problem here.

.github/dependabot.yml

@@ -0,0 +1,12 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "gradle" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    target-branch: "dev"
+    schedule:
+      interval: "weekly"

.github/issue-branch.yml

@@ -0,0 +1,19 @@
+# Automatically close issue after a pull request merge
+autoCloseIssue: true
+
+# Override the source branch 
+defaultBranch: 'development'
+
+#Skip branch creation based on issue label 'question'
+branches:
+  - label: question
+    skip: true
+
+# Automatically open a Pull Request
+openPR: true
+
+# Copy attributes from issue
+copyIssueDescriptionToPR: true
+copyIssueLabelsToPR: true
+copyIssueAssigneeToPR: true
+copyIssueMilestoneToPR: true

.github/workflows/branch_on_issue.yml

@@ -0,0 +1,16 @@
+name: Branch on Issue
+
+on:
+  issues:
+    types: [ assigned ]
+  pull_request:
+    types: [ closed ]
+
+jobs:
+  create_issue_branch_job:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Create Issue Branch
+        uses: robvanderleek/create-issue-branch@main
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -4,13 +4,15 @@
 #!gradle/wrapper/gradle-wrapper.jar
 #!**/src/main/**/build/
 #!**/src/test/**/build/
+lib/
 #
 #### Spring boot ###
-#**/application.properties
+**/application.properties
 #
 #### Gemma/python ###
 #/gemma
 #**/__pycache__
+**/output
 #
 #### STS ###
 #.apt_generated
@@ -129,6 +131,7 @@ fabric.properties
 # See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360
 
 .idea/
+.fleet
 
 # Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023
 

README.md

@@ -21,23 +21,61 @@ Dependencies that are needed to build and are not being downloaded via gradle:
 
 - OpenJDK 17
 - Python 3
+- pip (runtime only)
 
 `./gradlew -Pclean-release build`
 
-## How to start
+### Python Location
+
+Currently, mapping-service requires Python to be installed in order to build and to run. At runtime, the Python executable is configured in 
+`application.properties`(see below). For building the mapping-service Python executable is set to `/usr/bin/python3` by default. In case you want to build 
+the mapping-service on a machine on which the Python installation is located elsewhere, e.g., under Windows, you can provide the Python location 
+used at compile time externally, i.e.:
 
-`./gradlew bootRun`
+```
+.\gradlew -Pclean-release "-DpythonExecutable=file:///C:/Python310/python.exe" build
+```
+
+## How to start
 
-### Prerequisites
-Please make sure the application.properties file is configured correctly.
-Espacially the following properties (at the end of the file) are important:
-- `mapping-service.pythonLocation=file:///opt/homebrew/bin/python3` \
-Enter the loacation of your python3 installation. You can determine it by typing `which python3` in your terminal.
+Before you can start the mapping-service, you  first have to create an `application.properties` file in the source folder. As an example you may use `config/application.default.properties`
+and modify it according to your needs. Espacially the following properties (at the end of the file) are important:
+- `spring.datasource.url=jdbc:h2:file:/tmp/mapping-service/database`
+The path points to the location of the database in which your configured mappings are stored.
+- `mapping-service.pythonExecutable=${pythonExecutable:'file:///usr/bin/python3'}` \
+If no pythonExecutable is provided externally (see above) the default `/usr/bin/python3` is used.
+- `mapping-service.pluginLocation=file:///tmp/mapping-service/plugins` \
+The local folder where available plugins are located.
 - `mapping-service.mappingsLocation:file:///tmp/mapping-service/` \
 Enter the location where you want to store your mappings. This folder will be created if it does not exist yet.
 
-You might want to add a plugin to make the service working. Normally, there should be a gemma-plugin-x.x.x.jar in the plugins folder.
-If not, you can find its source code [here](https://github.com/maximilianiKIT/gemma-plugin).
+In order to provide the mapping-service with mapping functionality, there are already some pre-compiled plugins available under in the `plugins` folder of this repository.
+Copy them to your configured `mapping-service.pluginLocation` to make them available to the mapping-service. 
+The source code of the gemma-plugin can be found [here](https://github.com/maximilianiKIT/gemma-plugin). The plugin shows how to integrate Python mappings easily.
+
+There is also the possibility to add new plugins directly at the source tree and create a pluggable Jar out of them. Therefor, check 
+`src/main/java/edu/kit/datamanager/mappingservice/plugins/impl`. Just add your new plugin, e.g., based on the `TestPlugin` example. 
+In order to make the plugin usable by the mapping service, you then have to build a plugin Jar out of it. In order to do that, just call:
+
+```
+./gradlew buildPluginJar
+```
+
+This task creates a file `default-plugins-<VERSION>` at `build/libs` which has to be copied to `mapping-service.pluginLocation` to make it available. 
+
+After doing this, the mapping-service is ready for the first start. This can be achieved by executing:
+
+`java -jar build/lib/mapping-service-<VERSION>.jar`
+
+This assumes, that the command is called from the source folder and that your `application.properties` is located in the same folder. 
+Otherwise, you may use:
+
+`java -jar build/lib/mapping-service-<VERSION>.jar --spring.config.location=/tmp/application.properties`
+
+Ideally, for production use, you place everything (`mapping-service-<VERSION>.jar`, `application.properties`, `mapping-service.pluginLocation`, `mapping-service.mappingsLocation`,
+and `spring.datasource.url`) in a separate folder from where you then call the mapping-service via: 
+
+`java -jar mapping-service-<VERSION>.jar`
 
 ## License
 

build.gradle

@@ -4,6 +4,7 @@ plugins {
     id "org.asciidoctor.jvm.convert" version "3.3.0"
     id "org.owasp.dependencycheck" version "7.0.4.1"
     id 'net.researchgate.release' version '2.8.1'
+    id "com.gorylenko.gradle-git-properties" version "2.4.1"
     id 'java'
     id 'jacoco'
 }
@@ -24,14 +25,19 @@ configurations {
     }
 }
 
+sourceCompatibility = 1.17
+targetCompatibility = 1.17
+
 repositories {
     mavenLocal()
     mavenCentral()
 }
 
 ext {
     set('snippetsDir', file('build/generated-snippets'))
-    applicationProperties = System.getProperty('applicationProperties', 'src/main/resources/application.properties')
+    applicationProperties = System.getProperty('applicationProperties', './src/test/resources/test-config/application-test.properties')
+    pythonExecutable = System.getProperty('pythonExecutable', 'file:///usr/bin/python3')
+    userDir = System.getProperty('user.dir')
 }
 
 dependencies {
@@ -53,6 +59,8 @@ dependencies {
     implementation 'commons-io:commons-io:2.11.0'
     implementation 'javax.validation:validation-api:2.0.1.Final'
     implementation 'edu.kit.datamanager:service-base:1.0.4'
+    // apache
+    implementation "org.apache.tika:tika-core:2.7.0"
 
     testImplementation platform('org.junit:junit-bom:5.9.0')
     testImplementation 'org.junit.jupiter:junit-jupiter:5.9.0'
@@ -82,10 +90,13 @@ dependencies {
 test {
     outputs.dir snippetsDir
     finalizedBy jacocoTestReport
-    print("Running tests with configuration: ${applicationProperties}")
+    println("Running tests with configuration: ${applicationProperties}")
+    println("Running tests with python: ${pythonExecutable}")
+    println("Running tests in directory: ${userDir}")
     environment 'spring.config.location', applicationProperties
-//    environment 'spring.config.location', 'classpath:/test-config/'
+    environment 'pythonExecutable', pythonExecutable
     useJUnitPlatform()
+    testLogging.showStandardStreams = true
 }
 
 jacoco {
@@ -112,6 +123,15 @@ jar {
     }
 }
 
+springBoot {    
+    buildInfo()
+}
+
+bootRun {
+    systemProperty "spring.config.location", "file:$projectDir/"
+    systemProperty "pythonLocation", pythonExecutable
+}
+
 bootJar {
     dependsOn asciidoctor
     from ("${asciidoctor.outputDir}/html5") {
@@ -122,4 +142,11 @@ bootJar {
 
 release {
     tagTemplate = 'v${version}'
-}
+}
+
+task buildPluginJar(type: Jar) {
+ description = 'Bundeling only plugin classes'
+ archiveFileName.set("default-plugins-${version}.jar")
+ from sourceSets.main.output
+ include '**/plugins/impl/*.class'
+}

src/main/resources/application.properties renamed to config/application.default.properties

@@ -6,6 +6,9 @@ server.port=8095
 # The properties max-file-size and max-request-size define the maximum size of files 
 # transferred to and from the repository. Setting them to -1 removes all limits.
 server.compression.enabled=false
+
+# Max sizes of requests and uploaded files. This value may has to be increased for
+# bigger mapping inputs, e.g., while extracting information for a zipped dataset.
 spring.servlet.multipart.max-file-size=100MB
 spring.servlet.multipart.max-request-size=100MB
 # Logging settings
@@ -24,16 +27,17 @@ management.endpoints.web.exposure.include=*
 # Database
 ##################################################
 spring.datasource.driver-class-name=org.h2.Driver
-spring.datasource.url=jdbc:h2:file:///tmp/mapping-service/database
+spring.datasource.url=jdbc:h2:file:e:/tmp/mapping-service/database
 spring.datasource.username=user
 spring.datasource.password=password
 spring.jpa.hibernate.ddl-auto=update
 
 ##################################################
 # Mapping-Service specific settings
 ##################################################
-# Absolute path to the local python interpreter
-mapping-service.pythonLocation=file:///usr/bin/python3
-
-# Absolute path to the local gemma mappings folder
+# Absolute path to the local python interpreter.
+mapping-service.pythonLocation=${pythonLocation:'file:///usr/bin/python3'}
+# Absolute path to the folder where all plugins are located.
+mapping-service.pluginLocation=file:///${user.dir}/plugins
+# Absolute path to the local gemma mappings folder.
 mapping-service.mappingSchemasLocation=file:///tmp/mapping-service/mappingSchemas

echo.bat

@@ -0,0 +1 @@
+echo @1

output

@@ -24,11 +24,16 @@ public ApplicationProperties applicationProperties() {
         return new ApplicationProperties();
     }
 
+    @Bean
+    public PluginManager pluginManager(){
+        return new PluginManager(applicationProperties());
+    }
+    
     public static void main(String[] args) {
         SpringApplication.run(MappingServiceApplication.class, args);
 
-        PluginManager.soleInstance().getListOfAvailableValidators().forEach((value) -> LOG.info("Found validator: " + value));
-        PythonRunnerUtil.printPythonVersion();
+        //pluginManager().getListOfAvailableValidators().forEach((value) -> LOG.info("Found validator: " + value));
+        //PythonRunnerUtil.printPythonVersion();
 
         System.out.println("Mapping service is running! Access it at http://localhost:8095");
     }

src/main/java/edu/kit/datamanager/mappingservice/MappingServiceApplication.java

@@ -17,7 +17,6 @@
 
 import edu.kit.datamanager.annotations.ExecutableFileURL;
 import edu.kit.datamanager.annotations.LocalFolderURL;
-import edu.kit.datamanager.configuration.GenericPluginProperties;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import org.springframework.beans.factory.annotation.Value;
@@ -28,8 +27,8 @@
 import java.net.URL;
 
 /**
- * This class is used to configure the application.
- * It reads the values from the application.properties file.
+ * This class is used to configure the application. It reads the values from the
+ * application.properties file.
  *
  * @author maximilianiKIT
  */
@@ -39,12 +38,20 @@
 @Validated
 @EqualsAndHashCode
 public class ApplicationProperties {
+
     /**
      * The absolute path to the python interpreter.
      */
     @ExecutableFileURL
-    @Value("${mapping-service.pythonLocation}")
-    private URL pythonLocation;
+    @Value("${mapping-service.pythonExecutable}")
+    private URL pythonExecutable;
+
+    /**
+     * The absolute path where the plugins are stored.
+     */
+    @LocalFolderURL
+    @Value("${mapping-service.pluginLocation}")
+    private URL pluginLocation;
 
     /**
      * The absolute path where the mappings are stored.