diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 045dc95b..6e9a3d9a 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -31,7 +31,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.13' - name: Get Python location run: python -c "import os, sys; print(sys.executable)" - name: Update pip @@ -39,11 +39,7 @@ jobs: - name: Grant execute permission for gradlew run: chmod +x gradlew - name: Prepare - run: mkdir -p /tmp/mapping-service/{schemas,plugins} - - name: Copy Plugins - run: cp plugins/* /tmp/mapping-service/plugins - - name: List Plugins - run: ls -la /tmp/mapping-service/plugins + run: mkdir -p /tmp/mapping-service/{schemas} - name: Clean run: ./gradlew clean # - if: matrix.os == 'windows-latest' diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 7bef61a2..ffd14e01 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -41,7 +41,7 @@ jobs: - name: Set up Python3 uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.13' - name: Set up OpenJDK uses: actions/setup-java@v4 with: @@ -101,7 +101,7 @@ jobs: - name: Set up Python3 uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.13' - name: Set up OpenJDK uses: actions/setup-java@v4 diff --git a/.gitignore b/.gitignore index d8c1bede..8f27be30 100644 --- a/.gitignore +++ b/.gitignore @@ -241,3 +241,4 @@ gradle-app.setting lib/gemma !/plugins/gemma-plugin-0.1.0-SNAPSHOT-plain.jar !/plugins/empty-plugin-0.0.0-SNAPSHOT-plain.jar +custom diff --git a/Dockerfile b/Dockerfile index 423c9ce3..c62092f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ ARG SERVICE_ROOT_DIRECTORY_DEFAULT=/spring/ #################################################### # Building environment (java & git) #################################################### -FROM eclipse-temurin:23 AS build-env-java +FROM eclipse-temurin:24 AS build-env-java LABEL maintainer=webmaster@datamanager.kit.edu LABEL stage=build-env @@ -48,7 +48,7 @@ RUN bash ./build.sh $SERVICE_DIRECTORY #################################################### # Runtime environment 4 metastore2 #################################################### -FROM eclipse-temurin:23 AS run-service-mapping-service +FROM eclipse-temurin:24 AS run-service-mapping-service LABEL maintainer=webmaster@datamanager.kit.edu LABEL stage=run diff --git a/HELP.md b/HELP.md index 42f291da..f8f3cfed 100644 --- a/HELP.md +++ b/HELP.md @@ -1,8 +1,3 @@ -# Read Me First -The following was discovered as part of building this project: - -* The original package name 'edu.kit.datamanager.mapping-service' is invalid and this project uses 'edu.kit.datamanager.mappingservice' instead. - # Getting Started ### Reference Documentation diff --git a/README.md b/README.md index 15387a70..b2106c5c 100644 --- a/README.md +++ b/README.md @@ -17,17 +17,17 @@ The REST-API is documented at the following link: [http://\:8095 Dependencies that are needed to build and are not being downloaded via gradle: - OpenJDK 17 -- Python 3 -- pip (runtime only) +- (Optional) Python 3 +- (Optional) pip (runtime only) `./gradlew build` ### Python Location -Currently, mapping-service requires Python to be installed in order to build and to run. At runtime, the Python executable is configured in -`application.properties`(see below). For building the mapping-service Python executable is set to `/usr/bin/python3` by default. In case you want to build -the mapping-service on a machine on which the Python installation is located elsewhere, e.g., under Windows, you can provide the Python location -used at compile time externally, i.e.: +The mapping-service supports plugins running Python code. To provide basic testing for this feature, some tests require configured Python in order to be executed. +While at runtime, the Python executable is configured in application.properties, at build time the Python location may differ depending on the build environment. +By default, '/usr/bin/python' is assumed as Python location. If you are using a different Python installation, e.g., under Windows or MacOS, you may either modify +'build.gradle' (look out for pythonExecutable) or you provide the Python executable as command line argument, e.g., ``` .\gradlew "-DpythonExecutable=file:///C:/Python310/python.exe" build @@ -35,54 +35,40 @@ used at compile time externally, i.e.: ## How to start -Before you can start the mapping-service, you first have to create an `application.properties` file in the source folder. As an example you may use `config/application.default.properties` +Before you can start the mapping-service, you first have to create an `application.properties` file in the source folder. As an example you may use `settings/application.default.properties` and modify it according to your needs. Espacially the following properties (at the end of the file) are important: -- `spring.datasource.url=jdbc:h2:file:/tmp/mapping-service/database` -The path points to the location of the database in which your configured mappings are stored. -- `mapping-service.pythonExecutable=${pythonExecutable:'file:///usr/bin/python3'}` \ -If no pythonExecutable is provided externally (see above) the default `/usr/bin/python3` is used. -- `mapping-service.pluginLocation=file:///tmp/mapping-service/plugins` \ -The local folder where available plugins are located. -- `mapping-service.mappingsLocation:file:///tmp/mapping-service/` \ -Enter the location where you want to store your mappings. This folder will be created if it does not exist yet. -In order to provide the mapping-service with mapping functionality, there are already some pre-compiled plugins available under in the `plugins` folder of this repository. -Copy them to your configured `mapping-service.pluginLocation` to make them available to the mapping-service. -The source code of the gemma-plugin can be found [here](https://github.com/maximilianiKIT/gemma-plugin). The plugin shows how to integrate Python mappings easily. +| Property | Description | Default | +|----------|-------------|---------| +| spring.datasource.url | The path points to the location of the database in which your configured mappings are stored. For production use it is not recommended to use the pre-configured H2 database! | jdbc:h2:file:/tmp/mapping-service/database | +| mapping-service.pythonExecutable | The path to your local Python executable. The default uses the pythonExecutable system property provided via -DpythonExecutable= or file:///usr/bin/python3 if no such system property is provided. | ${pythonExecutable:'file:///usr/bin/python3'} | +| mapping-service.pluginLocation | The local folder from where plugins are loaded. The folder will be created on startup if it does not exist. | None | +| mapping-service.mappingSchemasLocation | The local folder where the mapping files are stored. The folder will be created on startup if it does not exist. | None | +| mapping-service.jobOutput | The local folder where asynchronous mapping execution job outputs are stored. The folder will be created on startup if it does not exist. | None | +| mapping-service.packagesToScan | Packages scanned for mapping plugins in addition to plugins located in mapping-service.pluginLocation. Typically, this property has not the be changed. | edu.kit.datamanager.mappingservice.plugins.impl | +| mapping-service.executionTimeout | The timeout in seconds a plugin process, i.e., Python of Shell, may take before it is assumed to be stale. | 30 | -There is also the possibility to add new plugins directly at the source tree and create a pluggable Jar out of them. Therefor, check -`src/main/java/edu/kit/datamanager/mappingservice/plugins/impl`. Just add your new plugin, e.g., based on the `TestPlugin` example. -In order to make the plugin usable by the mapping service, you then have to build a plugin Jar out of it. In order to do that, just call: +## Starting the Mapping-Service -``` -./gradlew buildPluginJar -``` - -This task creates a file `default-plugins-` at `build/libs` which has to be copied to `mapping-service.pluginLocation` to make it available. - -After doing this, the mapping-service is ready for the first start. This can be achieved by executing: - -`java -jar build/lib/mapping-service-.jar` +The executable jar of the mapping-service is located at 'build/libs/mapping-service-.jar' You should copy it to some dedicated folder, +place 'application.properties' next to it, adapt it according to your needs, and startt he mapping-service by calling: -This assumes, that the command is called from the source folder and that your `application.properties` is located in the same folder. -Otherwise, you may use: - -`java -jar build/lib/mapping-service-.jar --spring.config.location=/tmp/application.properties` +`java -jar mapping-service-.jar` -Ideally, for production use, you place everything (`mapping-service-.jar`, `application.properties`, `mapping-service.pluginLocation`, `mapping-service.mappingsLocation`, -and `spring.datasource.url`) in a separate folder from where you then call the mapping-service via: +If your 'application.properties' is located in another folder, you may use the following call: -`java -jar mapping-service-.jar` +`java -jar mapping-service-.jar --spring.config.location=/myConfigFolder/application.properties` ## Installation -There are three ways to install metaStore2 as a microservice: +There are three ways to install the mapping-service as a system service: + - [Using](#Installation-via-GitHub-Packages) the image available via [GitHub Packages](https://github.com/orgs/kit-data-manager/packages?repo_name=mapping-service) (***recommended***) - [Building](#Build-docker-container-locally) docker image locally - [Building](#Build-and-run-locally) and running locally ## Installation via GitHub Packages ### Prerequisites -In order to run this microservice via docker you'll need: +In order to run the mapping-service via docker you'll need: * [Docker](https://www.docker.com/) @@ -110,7 +96,7 @@ user@localhost:/home/user/mapping-service$ ``` #### Create image -Now you'll have to create an image containing the microservice. This can be done via a script. +Now you'll have to create an image containing the mapping-service. This can be done via a script. On default the created images will be tagged as follows: *'latest tag'-'actual date(yyyy-mm-dd)'* (e.g.: 1.1.0-2023-06-27) @@ -128,7 +114,7 @@ user@localhost:/home/user/mapping-service$ ``` #### Build docker container -After building image you have to create (and start) a container for executing microservice: +After building image you have to create (and start) a container for executing the mapping-service: ``` # If you want to use a specific image you may list all possible tags first. user@localhost:/home/user/mapping-service$ docker images ghcr.io/kit-data-manager/mapping-service --format {{.Tag}} diff --git a/build.gradle b/build.gradle index 5c1cfeae..a4c22f8a 100644 --- a/build.gradle +++ b/build.gradle @@ -1,14 +1,19 @@ plugins { - id 'org.springframework.boot' version '3.4.1' + id 'org.springframework.boot' version '3.4.5' id 'io.spring.dependency-management' version '1.1.7' id 'org.asciidoctor.jvm.convert' version '4.0.4' - id 'io.freefair.maven-publish-java' version '8.11' - id "org.owasp.dependencycheck" version "12.0.1" + id 'io.freefair.maven-publish-java' version '8.13.1' + id "org.owasp.dependencycheck" version "12.1.1" id 'net.researchgate.release' version '3.1.0' - id "com.gorylenko.gradle-git-properties" version "2.4.2" - id 'io.freefair.lombok' version '8.11' + id "com.gorylenko.gradle-git-properties" version "2.5.0" + id 'io.freefair.lombok' version '8.13.1' id 'java' + id 'application' id 'jacoco' + // plugins for release and publishing to maven repo + id "signing" + id "io.github.gradle-nexus.publish-plugin" version "2.0.0" + id 'maven-publish' } description = 'Generic mapping service supporting different mapping implementations.' @@ -45,16 +50,16 @@ repositories { ext { set('snippetsDir', file('build/generated-snippets')) applicationProperties = System.getProperty('applicationProperties', './src/test/resources/test-config/application-test.properties') - pythonExecutable = System.getProperty('pythonExecutable', 'file:///usr/bin/python') + pythonExecutable = System.getProperty('pythonExecutable', "file:///usr/bin/python3") userDir = System.getProperty('user.dir') set('springBootVersion', "3.2.1") - set('springDocVersion', "2.8.3") - set('javersVersion', "7.7.0") + set('springDocVersion', "2.8.8") + set('javersVersion', "7.8.0") set('keycloakVersion', "19.0.0") } dependencies { - implementation 'org.eclipse.jgit:org.eclipse.jgit:7.1.0.202411261347-r' + implementation 'org.eclipse.jgit:org.eclipse.jgit:7.2.1.202505142326-r' implementation 'org.springframework.boot:spring-boot-starter-data-jpa' implementation 'org.springframework.boot:spring-boot-starter-security' implementation "org.springframework.boot:spring-boot-starter-data-rest" @@ -62,6 +67,10 @@ dependencies { implementation 'org.springframework.boot:spring-boot-starter-web' implementation 'org.springframework.boot:spring-boot-starter-actuator' + // monitoring + implementation 'io.micronaut.micrometer:micronaut-micrometer-registry-prometheus:5.10.2' + implementation 'org.springframework.boot:spring-boot-starter-actuator:3.3.4' + // springdoc implementation "org.springdoc:springdoc-openapi-starter-webmvc-ui:${springDocVersion}" implementation "org.springdoc:springdoc-openapi-starter-common:${springDocVersion}" @@ -74,15 +83,20 @@ dependencies { implementation "org.javers:javers-spring-boot-starter-sql:${javersVersion}" implementation 'org.apache.httpcomponents:httpclient:4.5.14' - implementation 'org.apache.commons:commons-collections4:4.4' - implementation 'org.json:json:20250107' + implementation 'org.apache.commons:commons-collections4:4.5.0' + implementation 'org.apache.maven:maven-artifact:3.9.9' + implementation 'org.json:json:20250517' implementation 'com.github.jknack:handlebars:4.4.0' - implementation 'com.google.guava:guava:33.4.0-jre' - implementation 'commons-io:commons-io:2.18.0' + implementation 'com.google.guava:guava:33.4.8-jre' + implementation 'commons-io:commons-io:2.19.0' implementation 'javax.validation:validation-api:2.0.1.Final' - implementation 'edu.kit.datamanager:service-base:1.3.3' + implementation ('edu.kit.datamanager:service-base:1.3.4'){ + //exclude dependency as spring boot includes + //org.glassfish.jaxb:jaxb-core:4.0.5 which leads to a duplication conflict + exclude group: "com.sun.xml.bind" + } // apache - implementation "org.apache.tika:tika-core:2.9.2" + implementation "org.apache.tika:tika-core:3.1.0" testImplementation platform('org.junit:junit-bom') testImplementation 'org.junit.jupiter:junit-jupiter' @@ -94,10 +108,10 @@ dependencies { //testImplementation 'org.springframework:spring-test' testImplementation 'org.springframework.restdocs:spring-restdocs-mockmvc:3.0.3' - testImplementation 'org.mockito:mockito-core:5.15.2' + testImplementation 'org.mockito:mockito-core:5.17.0' testImplementation 'org.powermock:powermock-module-junit4:2.0.9' testImplementation 'org.powermock:powermock-api-mockito2:2.0.9' - testImplementation 'net.bytebuddy:byte-buddy:1.16.1' + testImplementation 'net.bytebuddy:byte-buddy:1.17.5' annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor' @@ -125,7 +139,7 @@ test { } jacoco { - toolVersion = "0.8.12" + toolVersion = "0.8.13" } jacocoTestReport{ @@ -142,27 +156,31 @@ asciidoctor { dependsOn test } -/*jar { - manifest { - attributes 'Main-Class': 'edu.kit.datamanager.mapping-service.MappingServiceApplication' - } - archiveBaseName = 'mapping-service' - // version is defined in file 'gradle.properties' - archiveVersion = System.getenv('version') - // disable plain jar file - enabled = false -}*/ - bootJar { - println 'Create bootable jar...' + dependsOn asciidoctor + //dependsOn 'buildPluginJar' + + from ("${asciidoctor.outputDir}/html5") { + into 'static/docs' + } archiveFileName = "${archiveBaseName.get()}.${archiveExtension.get()}" duplicatesStrategy = DuplicatesStrategy.EXCLUDE manifest { attributes 'Main-Class': 'org.springframework.boot.loader.launch.PropertiesLauncher' } + //exclude '**/plugins/impl/**' + + // from('build/libs/default-plugins-1.1.2-SNAPSHOT.jar') { + // into('BOOT-INF/lib/plugins' ) + // } launchScript() } +if (project.hasProperty('release')) { + println 'Using \'release\' profile for building ' + project.getName() + apply from: 'gradle/profile-deploy.gradle' +} + springBoot { buildInfo() } @@ -172,25 +190,29 @@ bootRun { systemProperty "pythonLocation", pythonExecutable } -bootJar { - dependsOn asciidoctor - from ("${asciidoctor.outputDir}/html5") { - into 'static/docs' - } - launchScript() -} - release { tagTemplate = 'v${version}' } -task buildPluginJar(type: Jar) { - description = 'Bundeling only plugin classes' - archiveFileName.set("default-plugins-${version}.jar") - from sourceSets.main.output - include '**/plugins/impl/*.class' +tasks.named("jar") { + enabled = false } +//task buildPluginJar(type: Jar) { +// description = 'Bundeling plugin core classes' +// archiveFileName.set("plugin-core-${version}.jar") +// from sourceSets.main.output +// include '**/configuration/ApplicationProperties.class', +// '**/exception.BadExitCodeException.class', +// '**/plugins/AbstractPythonMappingPlugin.class', +// '**/plugins/IMappingPlugin.class', +// '**/plugins/MappingPluginException.class', +// '**/plugins/MappingPluginState.class', +// '**/util/PythonRunnerUtil.class', +// '**/util/ShellRunnerUtil.class' +// includeEmptyDirs false +//} + // task for printing project name. // should be the last task inside file task printProjectName { diff --git a/custom/map-valid-document/curl-request.adoc b/custom/map-valid-document/curl-request.adoc deleted file mode 100644 index 8818d020..00000000 --- a/custom/map-valid-document/curl-request.adoc +++ /dev/null @@ -1,7 +0,0 @@ -[source,bash] ----- -$ curl 'http://localhost:8095/api/v1/mappingAdministration/' -i -X POST \ - -H 'Content-Type: multipart/form-data' \ - -F 'record=@record.json;type=application/json' \ - -F 'document=@my_dc4gemma.mapping;type=application/json' ----- \ No newline at end of file diff --git a/custom/map-valid-document/http-request.adoc b/custom/map-valid-document/http-request.adoc deleted file mode 100644 index c236e043..00000000 --- a/custom/map-valid-document/http-request.adoc +++ /dev/null @@ -1,34 +0,0 @@ -[source,http,options="nowrap"] ----- -POST /api/v1/mappingAdministration/ HTTP/1.1 -Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Host: localhost:8095 - ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Content-Disposition: form-data; name=record; filename=record.json -Content-Type: application/json - -{"mappingId":"my_dc","mappingType":"TEST_0.0.0","title":"TITEL","description":"DESCRIPTION","acl":[{"id":null,"sid":"SELF","permission":"READ"},{"id":null,"sid":"test2","permission":"ADMINISTRATE"}],"mappingDocumentUri":null,"documentHash":null} ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Content-Disposition: form-data; name=document; filename=my_dc4gemma.mapping -Content-Type: application/json - -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "http://example.com/product.schema.json", - "title": "Simple Mapping", - "description": "Data resource mapping from json", - "type": "object", - "properties":{ - "Publisher":{ - "path": "publisher", - "type": "string" - }, - "Publication Date":{ - "path": "publicationDate", - "type": "string" - } - } -} ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm-- ----- \ No newline at end of file diff --git a/custom/map-valid-document/http-response.adoc b/custom/map-valid-document/http-response.adoc deleted file mode 100644 index b32b4a1b..00000000 --- a/custom/map-valid-document/http-response.adoc +++ /dev/null @@ -1,8 +0,0 @@ -[source,http,options="nowrap"] ----- -HTTP/1.1 404 Not Found -Vary: Origin -Vary: Access-Control-Request-Method -Vary: Access-Control-Request-Headers - ----- \ No newline at end of file diff --git a/custom/map-valid-document/httpie-request.adoc b/custom/map-valid-document/httpie-request.adoc deleted file mode 100644 index f6687a5c..00000000 --- a/custom/map-valid-document/httpie-request.adoc +++ /dev/null @@ -1,6 +0,0 @@ -[source,bash] ----- -$ http --multipart POST 'http://localhost:8095/api/v1/mappingAdministration/' \ - 'record'@'record.json' \ - 'document'@'my_dc4gemma.mapping' ----- \ No newline at end of file diff --git a/custom/map-valid-document/request-body.adoc b/custom/map-valid-document/request-body.adoc deleted file mode 100644 index d074c300..00000000 --- a/custom/map-valid-document/request-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,form-data,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/map-valid-document/response-body.adoc b/custom/map-valid-document/response-body.adoc deleted file mode 100644 index dab5f81d..00000000 --- a/custom/map-valid-document/response-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/map-with-invalid-id/curl-request.adoc b/custom/map-with-invalid-id/curl-request.adoc deleted file mode 100644 index 8818d020..00000000 --- a/custom/map-with-invalid-id/curl-request.adoc +++ /dev/null @@ -1,7 +0,0 @@ -[source,bash] ----- -$ curl 'http://localhost:8095/api/v1/mappingAdministration/' -i -X POST \ - -H 'Content-Type: multipart/form-data' \ - -F 'record=@record.json;type=application/json' \ - -F 'document=@my_dc4gemma.mapping;type=application/json' ----- \ No newline at end of file diff --git a/custom/map-with-invalid-id/http-request.adoc b/custom/map-with-invalid-id/http-request.adoc deleted file mode 100644 index c236e043..00000000 --- a/custom/map-with-invalid-id/http-request.adoc +++ /dev/null @@ -1,34 +0,0 @@ -[source,http,options="nowrap"] ----- -POST /api/v1/mappingAdministration/ HTTP/1.1 -Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Host: localhost:8095 - ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Content-Disposition: form-data; name=record; filename=record.json -Content-Type: application/json - -{"mappingId":"my_dc","mappingType":"TEST_0.0.0","title":"TITEL","description":"DESCRIPTION","acl":[{"id":null,"sid":"SELF","permission":"READ"},{"id":null,"sid":"test2","permission":"ADMINISTRATE"}],"mappingDocumentUri":null,"documentHash":null} ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Content-Disposition: form-data; name=document; filename=my_dc4gemma.mapping -Content-Type: application/json - -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "http://example.com/product.schema.json", - "title": "Simple Mapping", - "description": "Data resource mapping from json", - "type": "object", - "properties":{ - "Publisher":{ - "path": "publisher", - "type": "string" - }, - "Publication Date":{ - "path": "publicationDate", - "type": "string" - } - } -} ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm-- ----- \ No newline at end of file diff --git a/custom/map-with-invalid-id/http-response.adoc b/custom/map-with-invalid-id/http-response.adoc deleted file mode 100644 index b32b4a1b..00000000 --- a/custom/map-with-invalid-id/http-response.adoc +++ /dev/null @@ -1,8 +0,0 @@ -[source,http,options="nowrap"] ----- -HTTP/1.1 404 Not Found -Vary: Origin -Vary: Access-Control-Request-Method -Vary: Access-Control-Request-Headers - ----- \ No newline at end of file diff --git a/custom/map-with-invalid-id/httpie-request.adoc b/custom/map-with-invalid-id/httpie-request.adoc deleted file mode 100644 index f6687a5c..00000000 --- a/custom/map-with-invalid-id/httpie-request.adoc +++ /dev/null @@ -1,6 +0,0 @@ -[source,bash] ----- -$ http --multipart POST 'http://localhost:8095/api/v1/mappingAdministration/' \ - 'record'@'record.json' \ - 'document'@'my_dc4gemma.mapping' ----- \ No newline at end of file diff --git a/custom/map-with-invalid-id/request-body.adoc b/custom/map-with-invalid-id/request-body.adoc deleted file mode 100644 index d074c300..00000000 --- a/custom/map-with-invalid-id/request-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,form-data,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/map-with-invalid-id/response-body.adoc b/custom/map-with-invalid-id/response-body.adoc deleted file mode 100644 index dab5f81d..00000000 --- a/custom/map-with-invalid-id/response-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/map-with-missing-parameters/curl-request.adoc b/custom/map-with-missing-parameters/curl-request.adoc deleted file mode 100644 index 8818d020..00000000 --- a/custom/map-with-missing-parameters/curl-request.adoc +++ /dev/null @@ -1,7 +0,0 @@ -[source,bash] ----- -$ curl 'http://localhost:8095/api/v1/mappingAdministration/' -i -X POST \ - -H 'Content-Type: multipart/form-data' \ - -F 'record=@record.json;type=application/json' \ - -F 'document=@my_dc4gemma.mapping;type=application/json' ----- \ No newline at end of file diff --git a/custom/map-with-missing-parameters/http-request.adoc b/custom/map-with-missing-parameters/http-request.adoc deleted file mode 100644 index c236e043..00000000 --- a/custom/map-with-missing-parameters/http-request.adoc +++ /dev/null @@ -1,34 +0,0 @@ -[source,http,options="nowrap"] ----- -POST /api/v1/mappingAdministration/ HTTP/1.1 -Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Host: localhost:8095 - ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Content-Disposition: form-data; name=record; filename=record.json -Content-Type: application/json - -{"mappingId":"my_dc","mappingType":"TEST_0.0.0","title":"TITEL","description":"DESCRIPTION","acl":[{"id":null,"sid":"SELF","permission":"READ"},{"id":null,"sid":"test2","permission":"ADMINISTRATE"}],"mappingDocumentUri":null,"documentHash":null} ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Content-Disposition: form-data; name=document; filename=my_dc4gemma.mapping -Content-Type: application/json - -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "http://example.com/product.schema.json", - "title": "Simple Mapping", - "description": "Data resource mapping from json", - "type": "object", - "properties":{ - "Publisher":{ - "path": "publisher", - "type": "string" - }, - "Publication Date":{ - "path": "publicationDate", - "type": "string" - } - } -} ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm-- ----- \ No newline at end of file diff --git a/custom/map-with-missing-parameters/http-response.adoc b/custom/map-with-missing-parameters/http-response.adoc deleted file mode 100644 index b32b4a1b..00000000 --- a/custom/map-with-missing-parameters/http-response.adoc +++ /dev/null @@ -1,8 +0,0 @@ -[source,http,options="nowrap"] ----- -HTTP/1.1 404 Not Found -Vary: Origin -Vary: Access-Control-Request-Method -Vary: Access-Control-Request-Headers - ----- \ No newline at end of file diff --git a/custom/map-with-missing-parameters/httpie-request.adoc b/custom/map-with-missing-parameters/httpie-request.adoc deleted file mode 100644 index f6687a5c..00000000 --- a/custom/map-with-missing-parameters/httpie-request.adoc +++ /dev/null @@ -1,6 +0,0 @@ -[source,bash] ----- -$ http --multipart POST 'http://localhost:8095/api/v1/mappingAdministration/' \ - 'record'@'record.json' \ - 'document'@'my_dc4gemma.mapping' ----- \ No newline at end of file diff --git a/custom/map-with-missing-parameters/request-body.adoc b/custom/map-with-missing-parameters/request-body.adoc deleted file mode 100644 index d074c300..00000000 --- a/custom/map-with-missing-parameters/request-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,form-data,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/map-with-missing-parameters/response-body.adoc b/custom/map-with-missing-parameters/response-body.adoc deleted file mode 100644 index dab5f81d..00000000 --- a/custom/map-with-missing-parameters/response-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/map-without-document/curl-request.adoc b/custom/map-without-document/curl-request.adoc deleted file mode 100644 index 8818d020..00000000 --- a/custom/map-without-document/curl-request.adoc +++ /dev/null @@ -1,7 +0,0 @@ -[source,bash] ----- -$ curl 'http://localhost:8095/api/v1/mappingAdministration/' -i -X POST \ - -H 'Content-Type: multipart/form-data' \ - -F 'record=@record.json;type=application/json' \ - -F 'document=@my_dc4gemma.mapping;type=application/json' ----- \ No newline at end of file diff --git a/custom/map-without-document/http-request.adoc b/custom/map-without-document/http-request.adoc deleted file mode 100644 index c236e043..00000000 --- a/custom/map-without-document/http-request.adoc +++ /dev/null @@ -1,34 +0,0 @@ -[source,http,options="nowrap"] ----- -POST /api/v1/mappingAdministration/ HTTP/1.1 -Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Host: localhost:8095 - ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Content-Disposition: form-data; name=record; filename=record.json -Content-Type: application/json - -{"mappingId":"my_dc","mappingType":"TEST_0.0.0","title":"TITEL","description":"DESCRIPTION","acl":[{"id":null,"sid":"SELF","permission":"READ"},{"id":null,"sid":"test2","permission":"ADMINISTRATE"}],"mappingDocumentUri":null,"documentHash":null} ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Content-Disposition: form-data; name=document; filename=my_dc4gemma.mapping -Content-Type: application/json - -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "http://example.com/product.schema.json", - "title": "Simple Mapping", - "description": "Data resource mapping from json", - "type": "object", - "properties":{ - "Publisher":{ - "path": "publisher", - "type": "string" - }, - "Publication Date":{ - "path": "publicationDate", - "type": "string" - } - } -} ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm-- ----- \ No newline at end of file diff --git a/custom/map-without-document/http-response.adoc b/custom/map-without-document/http-response.adoc deleted file mode 100644 index b32b4a1b..00000000 --- a/custom/map-without-document/http-response.adoc +++ /dev/null @@ -1,8 +0,0 @@ -[source,http,options="nowrap"] ----- -HTTP/1.1 404 Not Found -Vary: Origin -Vary: Access-Control-Request-Method -Vary: Access-Control-Request-Headers - ----- \ No newline at end of file diff --git a/custom/map-without-document/httpie-request.adoc b/custom/map-without-document/httpie-request.adoc deleted file mode 100644 index f6687a5c..00000000 --- a/custom/map-without-document/httpie-request.adoc +++ /dev/null @@ -1,6 +0,0 @@ -[source,bash] ----- -$ http --multipart POST 'http://localhost:8095/api/v1/mappingAdministration/' \ - 'record'@'record.json' \ - 'document'@'my_dc4gemma.mapping' ----- \ No newline at end of file diff --git a/custom/map-without-document/request-body.adoc b/custom/map-without-document/request-body.adoc deleted file mode 100644 index d074c300..00000000 --- a/custom/map-without-document/request-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,form-data,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/map-without-document/response-body.adoc b/custom/map-without-document/response-body.adoc deleted file mode 100644 index dab5f81d..00000000 --- a/custom/map-without-document/response-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-create-mapping-empty-record/curl-request.adoc b/custom/test-create-mapping-empty-record/curl-request.adoc deleted file mode 100644 index 8818d020..00000000 --- a/custom/test-create-mapping-empty-record/curl-request.adoc +++ /dev/null @@ -1,7 +0,0 @@ -[source,bash] ----- -$ curl 'http://localhost:8095/api/v1/mappingAdministration/' -i -X POST \ - -H 'Content-Type: multipart/form-data' \ - -F 'record=@record.json;type=application/json' \ - -F 'document=@my_dc4gemma.mapping;type=application/json' ----- \ No newline at end of file diff --git a/custom/test-create-mapping-empty-record/http-request.adoc b/custom/test-create-mapping-empty-record/http-request.adoc deleted file mode 100644 index 07fb3580..00000000 --- a/custom/test-create-mapping-empty-record/http-request.adoc +++ /dev/null @@ -1,34 +0,0 @@ -[source,http,options="nowrap"] ----- -POST /api/v1/mappingAdministration/ HTTP/1.1 -Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Host: localhost:8095 - ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Content-Disposition: form-data; name=record; filename=record.json -Content-Type: application/json - - ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Content-Disposition: form-data; name=document; filename=my_dc4gemma.mapping -Content-Type: application/json - -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "http://example.com/product.schema.json", - "title": "Simple Mapping", - "description": "Data resource mapping from json", - "type": "object", - "properties":{ - "Publisher":{ - "path": "publisher", - "type": "string" - }, - "Publication Date":{ - "path": "publicationDate", - "type": "string" - } - } -} ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm-- ----- \ No newline at end of file diff --git a/custom/test-create-mapping-empty-record/http-response.adoc b/custom/test-create-mapping-empty-record/http-response.adoc deleted file mode 100644 index 6bc944f3..00000000 --- a/custom/test-create-mapping-empty-record/http-response.adoc +++ /dev/null @@ -1,5 +0,0 @@ -[source,http,options="nowrap"] ----- -HTTP/1.1 400 Bad Request - ----- \ No newline at end of file diff --git a/custom/test-create-mapping-empty-record/httpie-request.adoc b/custom/test-create-mapping-empty-record/httpie-request.adoc deleted file mode 100644 index f6687a5c..00000000 --- a/custom/test-create-mapping-empty-record/httpie-request.adoc +++ /dev/null @@ -1,6 +0,0 @@ -[source,bash] ----- -$ http --multipart POST 'http://localhost:8095/api/v1/mappingAdministration/' \ - 'record'@'record.json' \ - 'document'@'my_dc4gemma.mapping' ----- \ No newline at end of file diff --git a/custom/test-create-mapping-empty-record/request-body.adoc b/custom/test-create-mapping-empty-record/request-body.adoc deleted file mode 100644 index d074c300..00000000 --- a/custom/test-create-mapping-empty-record/request-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,form-data,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-create-mapping-empty-record/response-body.adoc b/custom/test-create-mapping-empty-record/response-body.adoc deleted file mode 100644 index dab5f81d..00000000 --- a/custom/test-create-mapping-empty-record/response-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-create-mapping-no-mapping/curl-request.adoc b/custom/test-create-mapping-no-mapping/curl-request.adoc deleted file mode 100644 index 78b426e8..00000000 --- a/custom/test-create-mapping-no-mapping/curl-request.adoc +++ /dev/null @@ -1,6 +0,0 @@ -[source,bash] ----- -$ curl 'http://localhost:8095/api/v1/mappingAdministration/' -i -X POST \ - -H 'Content-Type: multipart/form-data' \ - -F 'record=@record.json;type=application/json' ----- \ No newline at end of file diff --git a/custom/test-create-mapping-no-mapping/http-request.adoc b/custom/test-create-mapping-no-mapping/http-request.adoc deleted file mode 100644 index e5ae37bd..00000000 --- a/custom/test-create-mapping-no-mapping/http-request.adoc +++ /dev/null @@ -1,13 +0,0 @@ -[source,http,options="nowrap"] ----- -POST /api/v1/mappingAdministration/ HTTP/1.1 -Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Host: localhost:8095 - ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Content-Disposition: form-data; name=record; filename=record.json -Content-Type: application/json - -{"mappingId":"my_dc","mappingType":"GEMMA","title":"TITEL","description":"DESCRIPTION","acl":[{"id":null,"sid":"SELF","permission":"READ"},{"id":null,"sid":"test2","permission":"ADMINISTRATE"}],"mappingDocumentUri":null,"documentHash":null} ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm-- ----- \ No newline at end of file diff --git a/custom/test-create-mapping-no-mapping/http-response.adoc b/custom/test-create-mapping-no-mapping/http-response.adoc deleted file mode 100644 index 6bc944f3..00000000 --- a/custom/test-create-mapping-no-mapping/http-response.adoc +++ /dev/null @@ -1,5 +0,0 @@ -[source,http,options="nowrap"] ----- -HTTP/1.1 400 Bad Request - ----- \ No newline at end of file diff --git a/custom/test-create-mapping-no-mapping/httpie-request.adoc b/custom/test-create-mapping-no-mapping/httpie-request.adoc deleted file mode 100644 index 12eca7ef..00000000 --- a/custom/test-create-mapping-no-mapping/httpie-request.adoc +++ /dev/null @@ -1,5 +0,0 @@ -[source,bash] ----- -$ http --multipart POST 'http://localhost:8095/api/v1/mappingAdministration/' \ - 'record'@'record.json' ----- \ No newline at end of file diff --git a/custom/test-create-mapping-no-mapping/request-body.adoc b/custom/test-create-mapping-no-mapping/request-body.adoc deleted file mode 100644 index d074c300..00000000 --- a/custom/test-create-mapping-no-mapping/request-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,form-data,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-create-mapping-no-mapping/response-body.adoc b/custom/test-create-mapping-no-mapping/response-body.adoc deleted file mode 100644 index dab5f81d..00000000 --- a/custom/test-create-mapping-no-mapping/response-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-create-mapping-no-record/curl-request.adoc b/custom/test-create-mapping-no-record/curl-request.adoc deleted file mode 100644 index 886f0f75..00000000 --- a/custom/test-create-mapping-no-record/curl-request.adoc +++ /dev/null @@ -1,6 +0,0 @@ -[source,bash] ----- -$ curl 'http://localhost:8095/api/v1/mappingAdministration/' -i -X POST \ - -H 'Content-Type: multipart/form-data' \ - -F 'document=@my_dc4gemma.mapping;type=application/json' ----- \ No newline at end of file diff --git a/custom/test-create-mapping-no-record/http-request.adoc b/custom/test-create-mapping-no-record/http-request.adoc deleted file mode 100644 index d99bd7d0..00000000 --- a/custom/test-create-mapping-no-record/http-request.adoc +++ /dev/null @@ -1,29 +0,0 @@ -[source,http,options="nowrap"] ----- -POST /api/v1/mappingAdministration/ HTTP/1.1 -Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Host: localhost:8095 - ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Content-Disposition: form-data; name=document; filename=my_dc4gemma.mapping -Content-Type: application/json - -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "http://example.com/product.schema.json", - "title": "Simple Mapping", - "description": "Data resource mapping from json", - "type": "object", - "properties":{ - "Publisher":{ - "path": "publisher", - "type": "string" - }, - "Publication Date":{ - "path": "publicationDate", - "type": "string" - } - } -} ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm-- ----- \ No newline at end of file diff --git a/custom/test-create-mapping-no-record/http-response.adoc b/custom/test-create-mapping-no-record/http-response.adoc deleted file mode 100644 index 6bc944f3..00000000 --- a/custom/test-create-mapping-no-record/http-response.adoc +++ /dev/null @@ -1,5 +0,0 @@ -[source,http,options="nowrap"] ----- -HTTP/1.1 400 Bad Request - ----- \ No newline at end of file diff --git a/custom/test-create-mapping-no-record/httpie-request.adoc b/custom/test-create-mapping-no-record/httpie-request.adoc deleted file mode 100644 index 61e90bbe..00000000 --- a/custom/test-create-mapping-no-record/httpie-request.adoc +++ /dev/null @@ -1,5 +0,0 @@ -[source,bash] ----- -$ http --multipart POST 'http://localhost:8095/api/v1/mappingAdministration/' \ - 'document'@'my_dc4gemma.mapping' ----- \ No newline at end of file diff --git a/custom/test-create-mapping-no-record/request-body.adoc b/custom/test-create-mapping-no-record/request-body.adoc deleted file mode 100644 index d074c300..00000000 --- a/custom/test-create-mapping-no-record/request-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,form-data,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-create-mapping-no-record/response-body.adoc b/custom/test-create-mapping-no-record/response-body.adoc deleted file mode 100644 index dab5f81d..00000000 --- a/custom/test-create-mapping-no-record/response-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-create-mapping-twice/curl-request.adoc b/custom/test-create-mapping-twice/curl-request.adoc deleted file mode 100644 index 8818d020..00000000 --- a/custom/test-create-mapping-twice/curl-request.adoc +++ /dev/null @@ -1,7 +0,0 @@ -[source,bash] ----- -$ curl 'http://localhost:8095/api/v1/mappingAdministration/' -i -X POST \ - -H 'Content-Type: multipart/form-data' \ - -F 'record=@record.json;type=application/json' \ - -F 'document=@my_dc4gemma.mapping;type=application/json' ----- \ No newline at end of file diff --git a/custom/test-create-mapping-twice/http-request.adoc b/custom/test-create-mapping-twice/http-request.adoc deleted file mode 100644 index 39da1244..00000000 --- a/custom/test-create-mapping-twice/http-request.adoc +++ /dev/null @@ -1,34 +0,0 @@ -[source,http,options="nowrap"] ----- -POST /api/v1/mappingAdministration/ HTTP/1.1 -Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Host: localhost:8095 - ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Content-Disposition: form-data; name=record; filename=record.json -Content-Type: application/json - -{"mappingId":"my_dc","mappingType":"GEMMA","title":"TITEL","description":"DESCRIPTION","acl":[],"mappingDocumentUri":null,"documentHash":null} ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Content-Disposition: form-data; name=document; filename=my_dc4gemma.mapping -Content-Type: application/json - -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "http://example.com/product.schema.json", - "title": "Simple Mapping", - "description": "Data resource mapping from json", - "type": "object", - "properties":{ - "Publisher":{ - "path": "publisher", - "type": "string" - }, - "Publication Date":{ - "path": "publicationDate", - "type": "string" - } - } -} ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm-- ----- \ No newline at end of file diff --git a/custom/test-create-mapping-twice/http-response.adoc b/custom/test-create-mapping-twice/http-response.adoc deleted file mode 100644 index d863daa7..00000000 --- a/custom/test-create-mapping-twice/http-response.adoc +++ /dev/null @@ -1,5 +0,0 @@ -[source,http,options="nowrap"] ----- -HTTP/1.1 409 Conflict - ----- \ No newline at end of file diff --git a/custom/test-create-mapping-twice/httpie-request.adoc b/custom/test-create-mapping-twice/httpie-request.adoc deleted file mode 100644 index f6687a5c..00000000 --- a/custom/test-create-mapping-twice/httpie-request.adoc +++ /dev/null @@ -1,6 +0,0 @@ -[source,bash] ----- -$ http --multipart POST 'http://localhost:8095/api/v1/mappingAdministration/' \ - 'record'@'record.json' \ - 'document'@'my_dc4gemma.mapping' ----- \ No newline at end of file diff --git a/custom/test-create-mapping-twice/request-body.adoc b/custom/test-create-mapping-twice/request-body.adoc deleted file mode 100644 index d074c300..00000000 --- a/custom/test-create-mapping-twice/request-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,form-data,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-create-mapping-twice/response-body.adoc b/custom/test-create-mapping-twice/response-body.adoc deleted file mode 100644 index dab5f81d..00000000 --- a/custom/test-create-mapping-twice/response-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-create-mapping-with-acl/curl-request.adoc b/custom/test-create-mapping-with-acl/curl-request.adoc deleted file mode 100644 index 8818d020..00000000 --- a/custom/test-create-mapping-with-acl/curl-request.adoc +++ /dev/null @@ -1,7 +0,0 @@ -[source,bash] ----- -$ curl 'http://localhost:8095/api/v1/mappingAdministration/' -i -X POST \ - -H 'Content-Type: multipart/form-data' \ - -F 'record=@record.json;type=application/json' \ - -F 'document=@my_dc4gemma.mapping;type=application/json' ----- \ No newline at end of file diff --git a/custom/test-create-mapping-with-acl/http-request.adoc b/custom/test-create-mapping-with-acl/http-request.adoc deleted file mode 100644 index 36c8215d..00000000 --- a/custom/test-create-mapping-with-acl/http-request.adoc +++ /dev/null @@ -1,34 +0,0 @@ -[source,http,options="nowrap"] ----- -POST /api/v1/mappingAdministration/ HTTP/1.1 -Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Host: localhost:8095 - ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Content-Disposition: form-data; name=record; filename=record.json -Content-Type: application/json - -{"mappingId":"my_dc","mappingType":"GEMMA","title":"TITEL","description":"DESCRIPTION","acl":[{"id":null,"sid":"SELF","permission":"READ"},{"id":null,"sid":"test2","permission":"ADMINISTRATE"}],"mappingDocumentUri":null,"documentHash":null} ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Content-Disposition: form-data; name=document; filename=my_dc4gemma.mapping -Content-Type: application/json - -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "http://example.com/product.schema.json", - "title": "Simple Mapping", - "description": "Data resource mapping from json", - "type": "object", - "properties":{ - "Publisher":{ - "path": "publisher", - "type": "string" - }, - "Publication Date":{ - "path": "publicationDate", - "type": "string" - } - } -} ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm-- ----- \ No newline at end of file diff --git a/custom/test-create-mapping-with-acl/http-response.adoc b/custom/test-create-mapping-with-acl/http-response.adoc deleted file mode 100644 index eaedcd55..00000000 --- a/custom/test-create-mapping-with-acl/http-response.adoc +++ /dev/null @@ -1,29 +0,0 @@ -[source,http,options="nowrap"] ----- -HTTP/1.1 201 Created -Location: http://localhost:8095/api/v1/mappingAdministration/my_dc -Content-Type: application/json -Content-Length: 566 - -{ - "mappingId" : "my_dc", - "mappingType" : "GEMMA", - "title" : "TITEL", - "description" : "DESCRIPTION", - "acl" : [ { - "id" : null, - "sid" : "anonymousUser", - "permission" : "ADMINISTRATE" - }, { - "id" : null, - "sid" : "SELF", - "permission" : "READ" - }, { - "id" : null, - "sid" : "test2", - "permission" : "ADMINISTRATE" - } ], - "mappingDocumentUri" : "http://localhost:8095/api/v1/mappingAdministration/my_dc/document", - "documentHash" : "sha256:0b415cfd8c084ea65ec2c9200a85a95402184011d442e5ab343021660420127f" -} ----- \ No newline at end of file diff --git a/custom/test-create-mapping-with-acl/httpie-request.adoc b/custom/test-create-mapping-with-acl/httpie-request.adoc deleted file mode 100644 index f6687a5c..00000000 --- a/custom/test-create-mapping-with-acl/httpie-request.adoc +++ /dev/null @@ -1,6 +0,0 @@ -[source,bash] ----- -$ http --multipart POST 'http://localhost:8095/api/v1/mappingAdministration/' \ - 'record'@'record.json' \ - 'document'@'my_dc4gemma.mapping' ----- \ No newline at end of file diff --git a/custom/test-create-mapping-with-acl/request-body.adoc b/custom/test-create-mapping-with-acl/request-body.adoc deleted file mode 100644 index d074c300..00000000 --- a/custom/test-create-mapping-with-acl/request-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,form-data,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-create-mapping-with-acl/response-body.adoc b/custom/test-create-mapping-with-acl/response-body.adoc deleted file mode 100644 index 6b7f6fa8..00000000 --- a/custom/test-create-mapping-with-acl/response-body.adoc +++ /dev/null @@ -1,24 +0,0 @@ -[source,json,options="nowrap"] ----- -{ - "mappingId" : "my_dc", - "mappingType" : "GEMMA", - "title" : "TITEL", - "description" : "DESCRIPTION", - "acl" : [ { - "id" : null, - "sid" : "anonymousUser", - "permission" : "ADMINISTRATE" - }, { - "id" : null, - "sid" : "SELF", - "permission" : "READ" - }, { - "id" : null, - "sid" : "test2", - "permission" : "ADMINISTRATE" - } ], - "mappingDocumentUri" : "http://localhost:8095/api/v1/mappingAdministration/my_dc/document", - "documentHash" : "sha256:0b415cfd8c084ea65ec2c9200a85a95402184011d442e5ab343021660420127f" -} ----- \ No newline at end of file diff --git a/custom/test-create-mapping-wrong-record/curl-request.adoc b/custom/test-create-mapping-wrong-record/curl-request.adoc deleted file mode 100644 index 8818d020..00000000 --- a/custom/test-create-mapping-wrong-record/curl-request.adoc +++ /dev/null @@ -1,7 +0,0 @@ -[source,bash] ----- -$ curl 'http://localhost:8095/api/v1/mappingAdministration/' -i -X POST \ - -H 'Content-Type: multipart/form-data' \ - -F 'record=@record.json;type=application/json' \ - -F 'document=@my_dc4gemma.mapping;type=application/json' ----- \ No newline at end of file diff --git a/custom/test-create-mapping-wrong-record/http-request.adoc b/custom/test-create-mapping-wrong-record/http-request.adoc deleted file mode 100644 index 3f563e82..00000000 --- a/custom/test-create-mapping-wrong-record/http-request.adoc +++ /dev/null @@ -1,18 +0,0 @@ -[source,http,options="nowrap"] ----- -POST /api/v1/mappingAdministration/ HTTP/1.1 -Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Host: localhost:8095 - ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Content-Disposition: form-data; name=record; filename=record.json -Content-Type: application/json - -{"mappingId":"my_dc","mappingType":null,"title":null,"description":null,"acl":[],"mappingDocumentUri":null,"documentHash":null} ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Content-Disposition: form-data; name=document; filename=my_dc4gemma.mapping -Content-Type: application/json - - ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm-- ----- \ No newline at end of file diff --git a/custom/test-create-mapping-wrong-record/http-response.adoc b/custom/test-create-mapping-wrong-record/http-response.adoc deleted file mode 100644 index 6bc944f3..00000000 --- a/custom/test-create-mapping-wrong-record/http-response.adoc +++ /dev/null @@ -1,5 +0,0 @@ -[source,http,options="nowrap"] ----- -HTTP/1.1 400 Bad Request - ----- \ No newline at end of file diff --git a/custom/test-create-mapping-wrong-record/httpie-request.adoc b/custom/test-create-mapping-wrong-record/httpie-request.adoc deleted file mode 100644 index f6687a5c..00000000 --- a/custom/test-create-mapping-wrong-record/httpie-request.adoc +++ /dev/null @@ -1,6 +0,0 @@ -[source,bash] ----- -$ http --multipart POST 'http://localhost:8095/api/v1/mappingAdministration/' \ - 'record'@'record.json' \ - 'document'@'my_dc4gemma.mapping' ----- \ No newline at end of file diff --git a/custom/test-create-mapping-wrong-record/request-body.adoc b/custom/test-create-mapping-wrong-record/request-body.adoc deleted file mode 100644 index d074c300..00000000 --- a/custom/test-create-mapping-wrong-record/request-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,form-data,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-create-mapping-wrong-record/response-body.adoc b/custom/test-create-mapping-wrong-record/response-body.adoc deleted file mode 100644 index dab5f81d..00000000 --- a/custom/test-create-mapping-wrong-record/response-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-create-mapping/curl-request.adoc b/custom/test-create-mapping/curl-request.adoc deleted file mode 100644 index 8818d020..00000000 --- a/custom/test-create-mapping/curl-request.adoc +++ /dev/null @@ -1,7 +0,0 @@ -[source,bash] ----- -$ curl 'http://localhost:8095/api/v1/mappingAdministration/' -i -X POST \ - -H 'Content-Type: multipart/form-data' \ - -F 'record=@record.json;type=application/json' \ - -F 'document=@my_dc4gemma.mapping;type=application/json' ----- \ No newline at end of file diff --git a/custom/test-create-mapping/http-request.adoc b/custom/test-create-mapping/http-request.adoc deleted file mode 100644 index 36c8215d..00000000 --- a/custom/test-create-mapping/http-request.adoc +++ /dev/null @@ -1,34 +0,0 @@ -[source,http,options="nowrap"] ----- -POST /api/v1/mappingAdministration/ HTTP/1.1 -Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Host: localhost:8095 - ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Content-Disposition: form-data; name=record; filename=record.json -Content-Type: application/json - -{"mappingId":"my_dc","mappingType":"GEMMA","title":"TITEL","description":"DESCRIPTION","acl":[{"id":null,"sid":"SELF","permission":"READ"},{"id":null,"sid":"test2","permission":"ADMINISTRATE"}],"mappingDocumentUri":null,"documentHash":null} ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Content-Disposition: form-data; name=document; filename=my_dc4gemma.mapping -Content-Type: application/json - -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "http://example.com/product.schema.json", - "title": "Simple Mapping", - "description": "Data resource mapping from json", - "type": "object", - "properties":{ - "Publisher":{ - "path": "publisher", - "type": "string" - }, - "Publication Date":{ - "path": "publicationDate", - "type": "string" - } - } -} ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm-- ----- \ No newline at end of file diff --git a/custom/test-create-mapping/http-response.adoc b/custom/test-create-mapping/http-response.adoc deleted file mode 100644 index eaedcd55..00000000 --- a/custom/test-create-mapping/http-response.adoc +++ /dev/null @@ -1,29 +0,0 @@ -[source,http,options="nowrap"] ----- -HTTP/1.1 201 Created -Location: http://localhost:8095/api/v1/mappingAdministration/my_dc -Content-Type: application/json -Content-Length: 566 - -{ - "mappingId" : "my_dc", - "mappingType" : "GEMMA", - "title" : "TITEL", - "description" : "DESCRIPTION", - "acl" : [ { - "id" : null, - "sid" : "anonymousUser", - "permission" : "ADMINISTRATE" - }, { - "id" : null, - "sid" : "SELF", - "permission" : "READ" - }, { - "id" : null, - "sid" : "test2", - "permission" : "ADMINISTRATE" - } ], - "mappingDocumentUri" : "http://localhost:8095/api/v1/mappingAdministration/my_dc/document", - "documentHash" : "sha256:0b415cfd8c084ea65ec2c9200a85a95402184011d442e5ab343021660420127f" -} ----- \ No newline at end of file diff --git a/custom/test-create-mapping/httpie-request.adoc b/custom/test-create-mapping/httpie-request.adoc deleted file mode 100644 index f6687a5c..00000000 --- a/custom/test-create-mapping/httpie-request.adoc +++ /dev/null @@ -1,6 +0,0 @@ -[source,bash] ----- -$ http --multipart POST 'http://localhost:8095/api/v1/mappingAdministration/' \ - 'record'@'record.json' \ - 'document'@'my_dc4gemma.mapping' ----- \ No newline at end of file diff --git a/custom/test-create-mapping/request-body.adoc b/custom/test-create-mapping/request-body.adoc deleted file mode 100644 index d074c300..00000000 --- a/custom/test-create-mapping/request-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,form-data,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-create-mapping/response-body.adoc b/custom/test-create-mapping/response-body.adoc deleted file mode 100644 index 6b7f6fa8..00000000 --- a/custom/test-create-mapping/response-body.adoc +++ /dev/null @@ -1,24 +0,0 @@ -[source,json,options="nowrap"] ----- -{ - "mappingId" : "my_dc", - "mappingType" : "GEMMA", - "title" : "TITEL", - "description" : "DESCRIPTION", - "acl" : [ { - "id" : null, - "sid" : "anonymousUser", - "permission" : "ADMINISTRATE" - }, { - "id" : null, - "sid" : "SELF", - "permission" : "READ" - }, { - "id" : null, - "sid" : "test2", - "permission" : "ADMINISTRATE" - } ], - "mappingDocumentUri" : "http://localhost:8095/api/v1/mappingAdministration/my_dc/document", - "documentHash" : "sha256:0b415cfd8c084ea65ec2c9200a85a95402184011d442e5ab343021660420127f" -} ----- \ No newline at end of file diff --git a/custom/test-delete-mapping-missing-etag/curl-request.adoc b/custom/test-delete-mapping-missing-etag/curl-request.adoc deleted file mode 100644 index 71c4986c..00000000 --- a/custom/test-delete-mapping-missing-etag/curl-request.adoc +++ /dev/null @@ -1,5 +0,0 @@ -[source,bash] ----- -$ curl 'http://localhost:8095/api/v1/mappingAdministration/my_dc' -i -X GET \ - -H 'Accept: application/vnd.datamanager.mapping-record+json' ----- \ No newline at end of file diff --git a/custom/test-delete-mapping-missing-etag/http-request.adoc b/custom/test-delete-mapping-missing-etag/http-request.adoc deleted file mode 100644 index f03c1a0a..00000000 --- a/custom/test-delete-mapping-missing-etag/http-request.adoc +++ /dev/null @@ -1,7 +0,0 @@ -[source,http,options="nowrap"] ----- -GET /api/v1/mappingAdministration/my_dc HTTP/1.1 -Accept: application/vnd.datamanager.mapping-record+json -Host: localhost:8095 - ----- \ No newline at end of file diff --git a/custom/test-delete-mapping-missing-etag/http-response.adoc b/custom/test-delete-mapping-missing-etag/http-response.adoc deleted file mode 100644 index 0dc4f320..00000000 --- a/custom/test-delete-mapping-missing-etag/http-response.adoc +++ /dev/null @@ -1,29 +0,0 @@ -[source,http,options="nowrap"] ----- -HTTP/1.1 200 OK -ETag: "104363025" -Content-Type: application/vnd.datamanager.mapping-record+json -Content-Length: 560 - -{ - "mappingId" : "my_dc", - "mappingType" : "GEMMA", - "title" : "TITEL", - "description" : "DESCRIPTION", - "acl" : [ { - "id" : 40, - "sid" : "anonymousUser", - "permission" : "ADMINISTRATE" - }, { - "id" : 41, - "sid" : "SELF", - "permission" : "READ" - }, { - "id" : 42, - "sid" : "test2", - "permission" : "ADMINISTRATE" - } ], - "mappingDocumentUri" : "http://localhost:8095/api/v1/mappingAdministration/my_dc/document", - "documentHash" : "sha256:0b415cfd8c084ea65ec2c9200a85a95402184011d442e5ab343021660420127f" -} ----- \ No newline at end of file diff --git a/custom/test-delete-mapping-missing-etag/httpie-request.adoc b/custom/test-delete-mapping-missing-etag/httpie-request.adoc deleted file mode 100644 index 351810e7..00000000 --- a/custom/test-delete-mapping-missing-etag/httpie-request.adoc +++ /dev/null @@ -1,5 +0,0 @@ -[source,bash] ----- -$ http GET 'http://localhost:8095/api/v1/mappingAdministration/my_dc' \ - 'Accept:application/vnd.datamanager.mapping-record+json' ----- \ No newline at end of file diff --git a/custom/test-delete-mapping-missing-etag/request-body.adoc b/custom/test-delete-mapping-missing-etag/request-body.adoc deleted file mode 100644 index dab5f81d..00000000 --- a/custom/test-delete-mapping-missing-etag/request-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-delete-mapping-missing-etag/response-body.adoc b/custom/test-delete-mapping-missing-etag/response-body.adoc deleted file mode 100644 index 3129c6d5..00000000 --- a/custom/test-delete-mapping-missing-etag/response-body.adoc +++ /dev/null @@ -1,24 +0,0 @@ -[source,json,options="nowrap"] ----- -{ - "mappingId" : "my_dc", - "mappingType" : "GEMMA", - "title" : "TITEL", - "description" : "DESCRIPTION", - "acl" : [ { - "id" : 40, - "sid" : "anonymousUser", - "permission" : "ADMINISTRATE" - }, { - "id" : 41, - "sid" : "SELF", - "permission" : "READ" - }, { - "id" : 42, - "sid" : "test2", - "permission" : "ADMINISTRATE" - } ], - "mappingDocumentUri" : "http://localhost:8095/api/v1/mappingAdministration/my_dc/document", - "documentHash" : "sha256:0b415cfd8c084ea65ec2c9200a85a95402184011d442e5ab343021660420127f" -} ----- \ No newline at end of file diff --git a/custom/test-delete-mapping-unknown-mapping-id/curl-request.adoc b/custom/test-delete-mapping-unknown-mapping-id/curl-request.adoc deleted file mode 100644 index 890ac553..00000000 --- a/custom/test-delete-mapping-unknown-mapping-id/curl-request.adoc +++ /dev/null @@ -1,5 +0,0 @@ -[source,bash] ----- -$ curl 'http://localhost:8095/api/v1/mappingAdministration/unknownMappingId' -i -X DELETE \ - -H 'If-Match: "104363025"' ----- \ No newline at end of file diff --git a/custom/test-delete-mapping-unknown-mapping-id/http-request.adoc b/custom/test-delete-mapping-unknown-mapping-id/http-request.adoc deleted file mode 100644 index 8113df33..00000000 --- a/custom/test-delete-mapping-unknown-mapping-id/http-request.adoc +++ /dev/null @@ -1,7 +0,0 @@ -[source,http,options="nowrap"] ----- -DELETE /api/v1/mappingAdministration/unknownMappingId HTTP/1.1 -If-Match: "104363025" -Host: localhost:8095 - ----- \ No newline at end of file diff --git a/custom/test-delete-mapping-unknown-mapping-id/http-response.adoc b/custom/test-delete-mapping-unknown-mapping-id/http-response.adoc deleted file mode 100644 index b2e108f3..00000000 --- a/custom/test-delete-mapping-unknown-mapping-id/http-response.adoc +++ /dev/null @@ -1,5 +0,0 @@ -[source,http,options="nowrap"] ----- -HTTP/1.1 204 No Content - ----- \ No newline at end of file diff --git a/custom/test-delete-mapping-unknown-mapping-id/httpie-request.adoc b/custom/test-delete-mapping-unknown-mapping-id/httpie-request.adoc deleted file mode 100644 index a0cc74a7..00000000 --- a/custom/test-delete-mapping-unknown-mapping-id/httpie-request.adoc +++ /dev/null @@ -1,5 +0,0 @@ -[source,bash] ----- -$ http DELETE 'http://localhost:8095/api/v1/mappingAdministration/unknownMappingId' \ - 'If-Match:"104363025"' ----- \ No newline at end of file diff --git a/custom/test-delete-mapping-unknown-mapping-id/request-body.adoc b/custom/test-delete-mapping-unknown-mapping-id/request-body.adoc deleted file mode 100644 index dab5f81d..00000000 --- a/custom/test-delete-mapping-unknown-mapping-id/request-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-delete-mapping-unknown-mapping-id/response-body.adoc b/custom/test-delete-mapping-unknown-mapping-id/response-body.adoc deleted file mode 100644 index dab5f81d..00000000 --- a/custom/test-delete-mapping-unknown-mapping-id/response-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-delete-mapping-wrong-etag/curl-request.adoc b/custom/test-delete-mapping-wrong-etag/curl-request.adoc deleted file mode 100644 index 71c4986c..00000000 --- a/custom/test-delete-mapping-wrong-etag/curl-request.adoc +++ /dev/null @@ -1,5 +0,0 @@ -[source,bash] ----- -$ curl 'http://localhost:8095/api/v1/mappingAdministration/my_dc' -i -X GET \ - -H 'Accept: application/vnd.datamanager.mapping-record+json' ----- \ No newline at end of file diff --git a/custom/test-delete-mapping-wrong-etag/http-request.adoc b/custom/test-delete-mapping-wrong-etag/http-request.adoc deleted file mode 100644 index f03c1a0a..00000000 --- a/custom/test-delete-mapping-wrong-etag/http-request.adoc +++ /dev/null @@ -1,7 +0,0 @@ -[source,http,options="nowrap"] ----- -GET /api/v1/mappingAdministration/my_dc HTTP/1.1 -Accept: application/vnd.datamanager.mapping-record+json -Host: localhost:8095 - ----- \ No newline at end of file diff --git a/custom/test-delete-mapping-wrong-etag/http-response.adoc b/custom/test-delete-mapping-wrong-etag/http-response.adoc deleted file mode 100644 index 86dd8ecb..00000000 --- a/custom/test-delete-mapping-wrong-etag/http-response.adoc +++ /dev/null @@ -1,29 +0,0 @@ -[source,http,options="nowrap"] ----- -HTTP/1.1 200 OK -ETag: "104363025" -Content-Type: application/vnd.datamanager.mapping-record+json -Content-Length: 560 - -{ - "mappingId" : "my_dc", - "mappingType" : "GEMMA", - "title" : "TITEL", - "description" : "DESCRIPTION", - "acl" : [ { - "id" : 38, - "sid" : "SELF", - "permission" : "READ" - }, { - "id" : 39, - "sid" : "test2", - "permission" : "ADMINISTRATE" - }, { - "id" : 37, - "sid" : "anonymousUser", - "permission" : "ADMINISTRATE" - } ], - "mappingDocumentUri" : "http://localhost:8095/api/v1/mappingAdministration/my_dc/document", - "documentHash" : "sha256:0b415cfd8c084ea65ec2c9200a85a95402184011d442e5ab343021660420127f" -} ----- \ No newline at end of file diff --git a/custom/test-delete-mapping-wrong-etag/httpie-request.adoc b/custom/test-delete-mapping-wrong-etag/httpie-request.adoc deleted file mode 100644 index 351810e7..00000000 --- a/custom/test-delete-mapping-wrong-etag/httpie-request.adoc +++ /dev/null @@ -1,5 +0,0 @@ -[source,bash] ----- -$ http GET 'http://localhost:8095/api/v1/mappingAdministration/my_dc' \ - 'Accept:application/vnd.datamanager.mapping-record+json' ----- \ No newline at end of file diff --git a/custom/test-delete-mapping-wrong-etag/request-body.adoc b/custom/test-delete-mapping-wrong-etag/request-body.adoc deleted file mode 100644 index dab5f81d..00000000 --- a/custom/test-delete-mapping-wrong-etag/request-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-delete-mapping-wrong-etag/response-body.adoc b/custom/test-delete-mapping-wrong-etag/response-body.adoc deleted file mode 100644 index afe6b1da..00000000 --- a/custom/test-delete-mapping-wrong-etag/response-body.adoc +++ /dev/null @@ -1,24 +0,0 @@ -[source,json,options="nowrap"] ----- -{ - "mappingId" : "my_dc", - "mappingType" : "GEMMA", - "title" : "TITEL", - "description" : "DESCRIPTION", - "acl" : [ { - "id" : 38, - "sid" : "SELF", - "permission" : "READ" - }, { - "id" : 39, - "sid" : "test2", - "permission" : "ADMINISTRATE" - }, { - "id" : 37, - "sid" : "anonymousUser", - "permission" : "ADMINISTRATE" - } ], - "mappingDocumentUri" : "http://localhost:8095/api/v1/mappingAdministration/my_dc/document", - "documentHash" : "sha256:0b415cfd8c084ea65ec2c9200a85a95402184011d442e5ab343021660420127f" -} ----- \ No newline at end of file diff --git a/custom/test-delete-mapping/curl-request.adoc b/custom/test-delete-mapping/curl-request.adoc deleted file mode 100644 index 71c4986c..00000000 --- a/custom/test-delete-mapping/curl-request.adoc +++ /dev/null @@ -1,5 +0,0 @@ -[source,bash] ----- -$ curl 'http://localhost:8095/api/v1/mappingAdministration/my_dc' -i -X GET \ - -H 'Accept: application/vnd.datamanager.mapping-record+json' ----- \ No newline at end of file diff --git a/custom/test-delete-mapping/http-request.adoc b/custom/test-delete-mapping/http-request.adoc deleted file mode 100644 index f03c1a0a..00000000 --- a/custom/test-delete-mapping/http-request.adoc +++ /dev/null @@ -1,7 +0,0 @@ -[source,http,options="nowrap"] ----- -GET /api/v1/mappingAdministration/my_dc HTTP/1.1 -Accept: application/vnd.datamanager.mapping-record+json -Host: localhost:8095 - ----- \ No newline at end of file diff --git a/custom/test-delete-mapping/http-response.adoc b/custom/test-delete-mapping/http-response.adoc deleted file mode 100644 index f3b256d0..00000000 --- a/custom/test-delete-mapping/http-response.adoc +++ /dev/null @@ -1,5 +0,0 @@ -[source,http,options="nowrap"] ----- -HTTP/1.1 404 Not Found - ----- \ No newline at end of file diff --git a/custom/test-delete-mapping/httpie-request.adoc b/custom/test-delete-mapping/httpie-request.adoc deleted file mode 100644 index 351810e7..00000000 --- a/custom/test-delete-mapping/httpie-request.adoc +++ /dev/null @@ -1,5 +0,0 @@ -[source,bash] ----- -$ http GET 'http://localhost:8095/api/v1/mappingAdministration/my_dc' \ - 'Accept:application/vnd.datamanager.mapping-record+json' ----- \ No newline at end of file diff --git a/custom/test-delete-mapping/request-body.adoc b/custom/test-delete-mapping/request-body.adoc deleted file mode 100644 index dab5f81d..00000000 --- a/custom/test-delete-mapping/request-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-delete-mapping/response-body.adoc b/custom/test-delete-mapping/response-body.adoc deleted file mode 100644 index dab5f81d..00000000 --- a/custom/test-delete-mapping/response-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-get-mapping-by-id-with-invalid-mapping/curl-request.adoc b/custom/test-get-mapping-by-id-with-invalid-mapping/curl-request.adoc deleted file mode 100644 index 36fb7cb1..00000000 --- a/custom/test-get-mapping-by-id-with-invalid-mapping/curl-request.adoc +++ /dev/null @@ -1,5 +0,0 @@ -[source,bash] ----- -$ curl 'http://localhost:8095/api/v1/mappingAdministration/invalidMappingId' -i -X GET \ - -H 'Accept: application/vnd.datamanager.mapping-record+json' ----- \ No newline at end of file diff --git a/custom/test-get-mapping-by-id-with-invalid-mapping/http-request.adoc b/custom/test-get-mapping-by-id-with-invalid-mapping/http-request.adoc deleted file mode 100644 index 07b9f8c0..00000000 --- a/custom/test-get-mapping-by-id-with-invalid-mapping/http-request.adoc +++ /dev/null @@ -1,7 +0,0 @@ -[source,http,options="nowrap"] ----- -GET /api/v1/mappingAdministration/invalidMappingId HTTP/1.1 -Accept: application/vnd.datamanager.mapping-record+json -Host: localhost:8095 - ----- \ No newline at end of file diff --git a/custom/test-get-mapping-by-id-with-invalid-mapping/http-response.adoc b/custom/test-get-mapping-by-id-with-invalid-mapping/http-response.adoc deleted file mode 100644 index f3b256d0..00000000 --- a/custom/test-get-mapping-by-id-with-invalid-mapping/http-response.adoc +++ /dev/null @@ -1,5 +0,0 @@ -[source,http,options="nowrap"] ----- -HTTP/1.1 404 Not Found - ----- \ No newline at end of file diff --git a/custom/test-get-mapping-by-id-with-invalid-mapping/httpie-request.adoc b/custom/test-get-mapping-by-id-with-invalid-mapping/httpie-request.adoc deleted file mode 100644 index f0ed9abb..00000000 --- a/custom/test-get-mapping-by-id-with-invalid-mapping/httpie-request.adoc +++ /dev/null @@ -1,5 +0,0 @@ -[source,bash] ----- -$ http GET 'http://localhost:8095/api/v1/mappingAdministration/invalidMappingId' \ - 'Accept:application/vnd.datamanager.mapping-record+json' ----- \ No newline at end of file diff --git a/custom/test-get-mapping-by-id-with-invalid-mapping/request-body.adoc b/custom/test-get-mapping-by-id-with-invalid-mapping/request-body.adoc deleted file mode 100644 index dab5f81d..00000000 --- a/custom/test-get-mapping-by-id-with-invalid-mapping/request-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-get-mapping-by-id-with-invalid-mapping/response-body.adoc b/custom/test-get-mapping-by-id-with-invalid-mapping/response-body.adoc deleted file mode 100644 index dab5f81d..00000000 --- a/custom/test-get-mapping-by-id-with-invalid-mapping/response-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-get-mapping-by-id/curl-request.adoc b/custom/test-get-mapping-by-id/curl-request.adoc deleted file mode 100644 index 71c4986c..00000000 --- a/custom/test-get-mapping-by-id/curl-request.adoc +++ /dev/null @@ -1,5 +0,0 @@ -[source,bash] ----- -$ curl 'http://localhost:8095/api/v1/mappingAdministration/my_dc' -i -X GET \ - -H 'Accept: application/vnd.datamanager.mapping-record+json' ----- \ No newline at end of file diff --git a/custom/test-get-mapping-by-id/http-request.adoc b/custom/test-get-mapping-by-id/http-request.adoc deleted file mode 100644 index f03c1a0a..00000000 --- a/custom/test-get-mapping-by-id/http-request.adoc +++ /dev/null @@ -1,7 +0,0 @@ -[source,http,options="nowrap"] ----- -GET /api/v1/mappingAdministration/my_dc HTTP/1.1 -Accept: application/vnd.datamanager.mapping-record+json -Host: localhost:8095 - ----- \ No newline at end of file diff --git a/custom/test-get-mapping-by-id/http-response.adoc b/custom/test-get-mapping-by-id/http-response.adoc deleted file mode 100644 index b568d34f..00000000 --- a/custom/test-get-mapping-by-id/http-response.adoc +++ /dev/null @@ -1,29 +0,0 @@ -[source,http,options="nowrap"] ----- -HTTP/1.1 200 OK -ETag: "104363025" -Content-Type: application/vnd.datamanager.mapping-record+json -Content-Length: 560 - -{ - "mappingId" : "my_dc", - "mappingType" : "GEMMA", - "title" : "TITEL", - "description" : "DESCRIPTION", - "acl" : [ { - "id" : 51, - "sid" : "test2", - "permission" : "ADMINISTRATE" - }, { - "id" : 49, - "sid" : "anonymousUser", - "permission" : "ADMINISTRATE" - }, { - "id" : 50, - "sid" : "SELF", - "permission" : "READ" - } ], - "mappingDocumentUri" : "http://localhost:8095/api/v1/mappingAdministration/my_dc/document", - "documentHash" : "sha256:0b415cfd8c084ea65ec2c9200a85a95402184011d442e5ab343021660420127f" -} ----- \ No newline at end of file diff --git a/custom/test-get-mapping-by-id/httpie-request.adoc b/custom/test-get-mapping-by-id/httpie-request.adoc deleted file mode 100644 index 351810e7..00000000 --- a/custom/test-get-mapping-by-id/httpie-request.adoc +++ /dev/null @@ -1,5 +0,0 @@ -[source,bash] ----- -$ http GET 'http://localhost:8095/api/v1/mappingAdministration/my_dc' \ - 'Accept:application/vnd.datamanager.mapping-record+json' ----- \ No newline at end of file diff --git a/custom/test-get-mapping-by-id/request-body.adoc b/custom/test-get-mapping-by-id/request-body.adoc deleted file mode 100644 index dab5f81d..00000000 --- a/custom/test-get-mapping-by-id/request-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-get-mapping-by-id/response-body.adoc b/custom/test-get-mapping-by-id/response-body.adoc deleted file mode 100644 index 0f3b9c01..00000000 --- a/custom/test-get-mapping-by-id/response-body.adoc +++ /dev/null @@ -1,24 +0,0 @@ -[source,json,options="nowrap"] ----- -{ - "mappingId" : "my_dc", - "mappingType" : "GEMMA", - "title" : "TITEL", - "description" : "DESCRIPTION", - "acl" : [ { - "id" : 51, - "sid" : "test2", - "permission" : "ADMINISTRATE" - }, { - "id" : 49, - "sid" : "anonymousUser", - "permission" : "ADMINISTRATE" - }, { - "id" : 50, - "sid" : "SELF", - "permission" : "READ" - } ], - "mappingDocumentUri" : "http://localhost:8095/api/v1/mappingAdministration/my_dc/document", - "documentHash" : "sha256:0b415cfd8c084ea65ec2c9200a85a95402184011d442e5ab343021660420127f" -} ----- \ No newline at end of file diff --git a/custom/test-get-mapping-document-by-id-with-invalid-mapping/curl-request.adoc b/custom/test-get-mapping-document-by-id-with-invalid-mapping/curl-request.adoc deleted file mode 100644 index 443c291b..00000000 --- a/custom/test-get-mapping-document-by-id-with-invalid-mapping/curl-request.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,bash] ----- -$ curl 'http://localhost:8095/api/v1/mappingAdministration/invalidMappingId' -i -X GET ----- \ No newline at end of file diff --git a/custom/test-get-mapping-document-by-id-with-invalid-mapping/http-request.adoc b/custom/test-get-mapping-document-by-id-with-invalid-mapping/http-request.adoc deleted file mode 100644 index a2d8037e..00000000 --- a/custom/test-get-mapping-document-by-id-with-invalid-mapping/http-request.adoc +++ /dev/null @@ -1,6 +0,0 @@ -[source,http,options="nowrap"] ----- -GET /api/v1/mappingAdministration/invalidMappingId HTTP/1.1 -Host: localhost:8095 - ----- \ No newline at end of file diff --git a/custom/test-get-mapping-document-by-id-with-invalid-mapping/http-response.adoc b/custom/test-get-mapping-document-by-id-with-invalid-mapping/http-response.adoc deleted file mode 100644 index f3b256d0..00000000 --- a/custom/test-get-mapping-document-by-id-with-invalid-mapping/http-response.adoc +++ /dev/null @@ -1,5 +0,0 @@ -[source,http,options="nowrap"] ----- -HTTP/1.1 404 Not Found - ----- \ No newline at end of file diff --git a/custom/test-get-mapping-document-by-id-with-invalid-mapping/httpie-request.adoc b/custom/test-get-mapping-document-by-id-with-invalid-mapping/httpie-request.adoc deleted file mode 100644 index 844f1b2c..00000000 --- a/custom/test-get-mapping-document-by-id-with-invalid-mapping/httpie-request.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,bash] ----- -$ http GET 'http://localhost:8095/api/v1/mappingAdministration/invalidMappingId' ----- \ No newline at end of file diff --git a/custom/test-get-mapping-document-by-id-with-invalid-mapping/request-body.adoc b/custom/test-get-mapping-document-by-id-with-invalid-mapping/request-body.adoc deleted file mode 100644 index dab5f81d..00000000 --- a/custom/test-get-mapping-document-by-id-with-invalid-mapping/request-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-get-mapping-document-by-id-with-invalid-mapping/response-body.adoc b/custom/test-get-mapping-document-by-id-with-invalid-mapping/response-body.adoc deleted file mode 100644 index dab5f81d..00000000 --- a/custom/test-get-mapping-document-by-id-with-invalid-mapping/response-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-get-mapping-document-by-id/curl-request.adoc b/custom/test-get-mapping-document-by-id/curl-request.adoc deleted file mode 100644 index 4b167b7c..00000000 --- a/custom/test-get-mapping-document-by-id/curl-request.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,bash] ----- -$ curl 'http://localhost:8095/api/v1/mappingAdministration/my_dc/document' -i -X GET ----- \ No newline at end of file diff --git a/custom/test-get-mapping-document-by-id/http-request.adoc b/custom/test-get-mapping-document-by-id/http-request.adoc deleted file mode 100644 index 4aa6e413..00000000 --- a/custom/test-get-mapping-document-by-id/http-request.adoc +++ /dev/null @@ -1,6 +0,0 @@ -[source,http,options="nowrap"] ----- -GET /api/v1/mappingAdministration/my_dc/document HTTP/1.1 -Host: localhost:8095 - ----- \ No newline at end of file diff --git a/custom/test-get-mapping-document-by-id/http-response.adoc b/custom/test-get-mapping-document-by-id/http-response.adoc deleted file mode 100644 index 528f6003..00000000 --- a/custom/test-get-mapping-document-by-id/http-response.adoc +++ /dev/null @@ -1,26 +0,0 @@ -[source,http,options="nowrap"] ----- -HTTP/1.1 200 OK -ETag: "104363025" -Content-Length: 425 -Accept-Ranges: bytes -Content-Type: application/octet-stream - -{ - "$schema" : "http://json-schema.org/draft-07/schema#", - "$id" : "http://example.com/product.schema.json", - "title" : "Simple Mapping", - "description" : "Data resource mapping from json", - "type" : "object", - "properties" : { - "Publisher" : { - "path" : "publisher", - "type" : "string" - }, - "Publication Date" : { - "path" : "publicationDate", - "type" : "string" - } - } -} ----- \ No newline at end of file diff --git a/custom/test-get-mapping-document-by-id/httpie-request.adoc b/custom/test-get-mapping-document-by-id/httpie-request.adoc deleted file mode 100644 index a954b9fc..00000000 --- a/custom/test-get-mapping-document-by-id/httpie-request.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,bash] ----- -$ http GET 'http://localhost:8095/api/v1/mappingAdministration/my_dc/document' ----- \ No newline at end of file diff --git a/custom/test-get-mapping-document-by-id/request-body.adoc b/custom/test-get-mapping-document-by-id/request-body.adoc deleted file mode 100644 index dab5f81d..00000000 --- a/custom/test-get-mapping-document-by-id/request-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-get-mapping-document-by-id/response-body.adoc b/custom/test-get-mapping-document-by-id/response-body.adoc deleted file mode 100644 index bd0e4ace..00000000 --- a/custom/test-get-mapping-document-by-id/response-body.adoc +++ /dev/null @@ -1,20 +0,0 @@ -[source,octet-stream,options="nowrap"] ----- -{ - "$schema" : "http://json-schema.org/draft-07/schema#", - "$id" : "http://example.com/product.schema.json", - "title" : "Simple Mapping", - "description" : "Data resource mapping from json", - "type" : "object", - "properties" : { - "Publisher" : { - "path" : "publisher", - "type" : "string" - }, - "Publication Date" : { - "path" : "publicationDate", - "type" : "string" - } - } -} ----- \ No newline at end of file diff --git a/custom/test-update-mapping-with-invalid-record/curl-request.adoc b/custom/test-update-mapping-with-invalid-record/curl-request.adoc deleted file mode 100644 index 71c4986c..00000000 --- a/custom/test-update-mapping-with-invalid-record/curl-request.adoc +++ /dev/null @@ -1,5 +0,0 @@ -[source,bash] ----- -$ curl 'http://localhost:8095/api/v1/mappingAdministration/my_dc' -i -X GET \ - -H 'Accept: application/vnd.datamanager.mapping-record+json' ----- \ No newline at end of file diff --git a/custom/test-update-mapping-with-invalid-record/http-request.adoc b/custom/test-update-mapping-with-invalid-record/http-request.adoc deleted file mode 100644 index f03c1a0a..00000000 --- a/custom/test-update-mapping-with-invalid-record/http-request.adoc +++ /dev/null @@ -1,7 +0,0 @@ -[source,http,options="nowrap"] ----- -GET /api/v1/mappingAdministration/my_dc HTTP/1.1 -Accept: application/vnd.datamanager.mapping-record+json -Host: localhost:8095 - ----- \ No newline at end of file diff --git a/custom/test-update-mapping-with-invalid-record/http-response.adoc b/custom/test-update-mapping-with-invalid-record/http-response.adoc deleted file mode 100644 index e04ca78b..00000000 --- a/custom/test-update-mapping-with-invalid-record/http-response.adoc +++ /dev/null @@ -1,29 +0,0 @@ -[source,http,options="nowrap"] ----- -HTTP/1.1 200 OK -ETag: "104363025" -Content-Type: application/vnd.datamanager.mapping-record+json -Content-Length: 560 - -{ - "mappingId" : "my_dc", - "mappingType" : "GEMMA", - "title" : "TITEL", - "description" : "DESCRIPTION", - "acl" : [ { - "id" : 44, - "sid" : "SELF", - "permission" : "READ" - }, { - "id" : 43, - "sid" : "anonymousUser", - "permission" : "ADMINISTRATE" - }, { - "id" : 45, - "sid" : "test2", - "permission" : "ADMINISTRATE" - } ], - "mappingDocumentUri" : "http://localhost:8095/api/v1/mappingAdministration/my_dc/document", - "documentHash" : "sha256:0b415cfd8c084ea65ec2c9200a85a95402184011d442e5ab343021660420127f" -} ----- \ No newline at end of file diff --git a/custom/test-update-mapping-with-invalid-record/httpie-request.adoc b/custom/test-update-mapping-with-invalid-record/httpie-request.adoc deleted file mode 100644 index 351810e7..00000000 --- a/custom/test-update-mapping-with-invalid-record/httpie-request.adoc +++ /dev/null @@ -1,5 +0,0 @@ -[source,bash] ----- -$ http GET 'http://localhost:8095/api/v1/mappingAdministration/my_dc' \ - 'Accept:application/vnd.datamanager.mapping-record+json' ----- \ No newline at end of file diff --git a/custom/test-update-mapping-with-invalid-record/request-body.adoc b/custom/test-update-mapping-with-invalid-record/request-body.adoc deleted file mode 100644 index dab5f81d..00000000 --- a/custom/test-update-mapping-with-invalid-record/request-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-update-mapping-with-invalid-record/response-body.adoc b/custom/test-update-mapping-with-invalid-record/response-body.adoc deleted file mode 100644 index 979c0422..00000000 --- a/custom/test-update-mapping-with-invalid-record/response-body.adoc +++ /dev/null @@ -1,24 +0,0 @@ -[source,json,options="nowrap"] ----- -{ - "mappingId" : "my_dc", - "mappingType" : "GEMMA", - "title" : "TITEL", - "description" : "DESCRIPTION", - "acl" : [ { - "id" : 44, - "sid" : "SELF", - "permission" : "READ" - }, { - "id" : 43, - "sid" : "anonymousUser", - "permission" : "ADMINISTRATE" - }, { - "id" : 45, - "sid" : "test2", - "permission" : "ADMINISTRATE" - } ], - "mappingDocumentUri" : "http://localhost:8095/api/v1/mappingAdministration/my_dc/document", - "documentHash" : "sha256:0b415cfd8c084ea65ec2c9200a85a95402184011d442e5ab343021660420127f" -} ----- \ No newline at end of file diff --git a/custom/test-update-mapping-with-invalid-record2/curl-request.adoc b/custom/test-update-mapping-with-invalid-record2/curl-request.adoc deleted file mode 100644 index 4b167b7c..00000000 --- a/custom/test-update-mapping-with-invalid-record2/curl-request.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,bash] ----- -$ curl 'http://localhost:8095/api/v1/mappingAdministration/my_dc/document' -i -X GET ----- \ No newline at end of file diff --git a/custom/test-update-mapping-with-invalid-record2/http-request.adoc b/custom/test-update-mapping-with-invalid-record2/http-request.adoc deleted file mode 100644 index 4aa6e413..00000000 --- a/custom/test-update-mapping-with-invalid-record2/http-request.adoc +++ /dev/null @@ -1,6 +0,0 @@ -[source,http,options="nowrap"] ----- -GET /api/v1/mappingAdministration/my_dc/document HTTP/1.1 -Host: localhost:8095 - ----- \ No newline at end of file diff --git a/custom/test-update-mapping-with-invalid-record2/http-response.adoc b/custom/test-update-mapping-with-invalid-record2/http-response.adoc deleted file mode 100644 index 9909624a..00000000 --- a/custom/test-update-mapping-with-invalid-record2/http-response.adoc +++ /dev/null @@ -1,26 +0,0 @@ -[source,http,options="nowrap"] ----- -HTTP/1.1 200 OK -ETag: "104363025" -Content-Length: 434 -Accept-Ranges: bytes -Content-Type: application/octet-stream - -{ - "$schema" : "http://json-schema.org/draft-07/schema#", - "$id" : "http://example.com/product.schema.json", - "title" : "Simple Mapping Version 2", - "description" : "Data resource mapping from json", - "type" : "object", - "properties" : { - "Publisher" : { - "path" : "publisher", - "type" : "string" - }, - "PublicationDate" : { - "path" : "publicationDate", - "type" : "string" - } - } -} ----- \ No newline at end of file diff --git a/custom/test-update-mapping-with-invalid-record2/httpie-request.adoc b/custom/test-update-mapping-with-invalid-record2/httpie-request.adoc deleted file mode 100644 index a954b9fc..00000000 --- a/custom/test-update-mapping-with-invalid-record2/httpie-request.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,bash] ----- -$ http GET 'http://localhost:8095/api/v1/mappingAdministration/my_dc/document' ----- \ No newline at end of file diff --git a/custom/test-update-mapping-with-invalid-record2/request-body.adoc b/custom/test-update-mapping-with-invalid-record2/request-body.adoc deleted file mode 100644 index dab5f81d..00000000 --- a/custom/test-update-mapping-with-invalid-record2/request-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-update-mapping-with-invalid-record2/response-body.adoc b/custom/test-update-mapping-with-invalid-record2/response-body.adoc deleted file mode 100644 index 139a1028..00000000 --- a/custom/test-update-mapping-with-invalid-record2/response-body.adoc +++ /dev/null @@ -1,20 +0,0 @@ -[source,octet-stream,options="nowrap"] ----- -{ - "$schema" : "http://json-schema.org/draft-07/schema#", - "$id" : "http://example.com/product.schema.json", - "title" : "Simple Mapping Version 2", - "description" : "Data resource mapping from json", - "type" : "object", - "properties" : { - "Publisher" : { - "path" : "publisher", - "type" : "string" - }, - "PublicationDate" : { - "path" : "publicationDate", - "type" : "string" - } - } -} ----- \ No newline at end of file diff --git a/custom/test-update-mapping-with-wrong-etag/curl-request.adoc b/custom/test-update-mapping-with-wrong-etag/curl-request.adoc deleted file mode 100644 index 51612ec4..00000000 --- a/custom/test-update-mapping-with-wrong-etag/curl-request.adoc +++ /dev/null @@ -1,8 +0,0 @@ -[source,bash] ----- -$ curl 'http://localhost:8095/api/v1/mappingAdministration/my_dc' -i -X PUT \ - -H 'Content-Type: multipart/form-data' \ - -H 'If-Match: wrongEtag' \ - -F 'record=@record.json;type=application/json' \ - -F 'document=@my_dc4gemma.mapping;type=application/json' ----- \ No newline at end of file diff --git a/custom/test-update-mapping-with-wrong-etag/http-request.adoc b/custom/test-update-mapping-with-wrong-etag/http-request.adoc deleted file mode 100644 index 7b7a9a49..00000000 --- a/custom/test-update-mapping-with-wrong-etag/http-request.adoc +++ /dev/null @@ -1,36 +0,0 @@ -[source,http,options="nowrap"] ----- -PUT /api/v1/mappingAdministration/my_dc HTTP/1.1 -Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -If-Match: wrongEtag -Host: localhost:8095 - ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Content-Disposition: form-data; name=record; filename=record.json -Content-Type: application/json - -{"mappingId":"my_dc","mappingType":"GEMMA","title":"TITEL","description":"DESCRIPTION","acl":[{"id":null,"sid":"SELF","permission":"READ"},{"id":null,"sid":"someoneelse","permission":"ADMINISTRATE"}],"mappingDocumentUri":"http://localhost:8095/api/v1/mappingAdministration/my_dc/document","documentHash":"sha256:0b415cfd8c084ea65ec2c9200a85a95402184011d442e5ab343021660420127f"} ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Content-Disposition: form-data; name=document; filename=my_dc4gemma.mapping -Content-Type: application/json - -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "http://example.com/product.schema.json", - "title": "Simple Mapping Version 2", - "description": "Data resource mapping from json", - "type": "object", - "properties":{ - "Publisher":{ - "path": "publisher", - "type": "string" - }, - "PublicationDate":{ - "path": "publicationDate", - "type": "string" - } - } -} - ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm-- ----- \ No newline at end of file diff --git a/custom/test-update-mapping-with-wrong-etag/http-response.adoc b/custom/test-update-mapping-with-wrong-etag/http-response.adoc deleted file mode 100644 index 5549e3fe..00000000 --- a/custom/test-update-mapping-with-wrong-etag/http-response.adoc +++ /dev/null @@ -1,5 +0,0 @@ -[source,http,options="nowrap"] ----- -HTTP/1.1 412 Precondition Failed - ----- \ No newline at end of file diff --git a/custom/test-update-mapping-with-wrong-etag/httpie-request.adoc b/custom/test-update-mapping-with-wrong-etag/httpie-request.adoc deleted file mode 100644 index 51122c72..00000000 --- a/custom/test-update-mapping-with-wrong-etag/httpie-request.adoc +++ /dev/null @@ -1,7 +0,0 @@ -[source,bash] ----- -$ http --multipart PUT 'http://localhost:8095/api/v1/mappingAdministration/my_dc' \ - 'If-Match:wrongEtag' \ - 'record'@'record.json' \ - 'document'@'my_dc4gemma.mapping' ----- \ No newline at end of file diff --git a/custom/test-update-mapping-with-wrong-etag/request-body.adoc b/custom/test-update-mapping-with-wrong-etag/request-body.adoc deleted file mode 100644 index d074c300..00000000 --- a/custom/test-update-mapping-with-wrong-etag/request-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,form-data,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-update-mapping-with-wrong-etag/response-body.adoc b/custom/test-update-mapping-with-wrong-etag/response-body.adoc deleted file mode 100644 index dab5f81d..00000000 --- a/custom/test-update-mapping-with-wrong-etag/response-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-update-mapping-with-wrong-record3/curl-request.adoc b/custom/test-update-mapping-with-wrong-record3/curl-request.adoc deleted file mode 100644 index f6ea3824..00000000 --- a/custom/test-update-mapping-with-wrong-record3/curl-request.adoc +++ /dev/null @@ -1,8 +0,0 @@ -[source,bash] ----- -$ curl 'http://localhost:8095/api/v1/mappingAdministration/unknownMaping' -i -X PUT \ - -H 'Content-Type: multipart/form-data' \ - -H 'If-Match: "104363025"' \ - -F 'record=@record.json;type=application/json' \ - -F 'document=@my_dc4gemma.mapping;type=application/json' ----- \ No newline at end of file diff --git a/custom/test-update-mapping-with-wrong-record3/http-request.adoc b/custom/test-update-mapping-with-wrong-record3/http-request.adoc deleted file mode 100644 index d7f53e6e..00000000 --- a/custom/test-update-mapping-with-wrong-record3/http-request.adoc +++ /dev/null @@ -1,36 +0,0 @@ -[source,http,options="nowrap"] ----- -PUT /api/v1/mappingAdministration/unknownMaping HTTP/1.1 -Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -If-Match: "104363025" -Host: localhost:8095 - ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Content-Disposition: form-data; name=record; filename=record.json -Content-Type: application/json - -{"mappingId":"my_dc","mappingType":"GEMMA","title":"TITEL","description":"DESCRIPTION","acl":[{"id":36,"sid":"test2","permission":"ADMINISTRATE"},{"id":35,"sid":"SELF","permission":"READ"},{"id":34,"sid":"anonymousUser","permission":"ADMINISTRATE"}],"mappingDocumentUri":"http://localhost:8095/api/v1/mappingAdministration/my_dc/document","documentHash":"sha256:0b415cfd8c084ea65ec2c9200a85a95402184011d442e5ab343021660420127f"} ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Content-Disposition: form-data; name=document; filename=my_dc4gemma.mapping -Content-Type: application/json - -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "http://example.com/product.schema.json", - "title": "Simple Mapping Version 2", - "description": "Data resource mapping from json", - "type": "object", - "properties":{ - "Publisher":{ - "path": "publisher", - "type": "string" - }, - "PublicationDate":{ - "path": "publicationDate", - "type": "string" - } - } -} - ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm-- ----- \ No newline at end of file diff --git a/custom/test-update-mapping-with-wrong-record3/http-response.adoc b/custom/test-update-mapping-with-wrong-record3/http-response.adoc deleted file mode 100644 index f3b256d0..00000000 --- a/custom/test-update-mapping-with-wrong-record3/http-response.adoc +++ /dev/null @@ -1,5 +0,0 @@ -[source,http,options="nowrap"] ----- -HTTP/1.1 404 Not Found - ----- \ No newline at end of file diff --git a/custom/test-update-mapping-with-wrong-record3/httpie-request.adoc b/custom/test-update-mapping-with-wrong-record3/httpie-request.adoc deleted file mode 100644 index 0a2f8d2b..00000000 --- a/custom/test-update-mapping-with-wrong-record3/httpie-request.adoc +++ /dev/null @@ -1,7 +0,0 @@ -[source,bash] ----- -$ http --multipart PUT 'http://localhost:8095/api/v1/mappingAdministration/unknownMaping' \ - 'If-Match:"104363025"' \ - 'record'@'record.json' \ - 'document'@'my_dc4gemma.mapping' ----- \ No newline at end of file diff --git a/custom/test-update-mapping-with-wrong-record3/request-body.adoc b/custom/test-update-mapping-with-wrong-record3/request-body.adoc deleted file mode 100644 index d074c300..00000000 --- a/custom/test-update-mapping-with-wrong-record3/request-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,form-data,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-update-mapping-with-wrong-record3/response-body.adoc b/custom/test-update-mapping-with-wrong-record3/response-body.adoc deleted file mode 100644 index dab5f81d..00000000 --- a/custom/test-update-mapping-with-wrong-record3/response-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-update-mapping-without-document/curl-request.adoc b/custom/test-update-mapping-without-document/curl-request.adoc deleted file mode 100644 index 4b167b7c..00000000 --- a/custom/test-update-mapping-without-document/curl-request.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,bash] ----- -$ curl 'http://localhost:8095/api/v1/mappingAdministration/my_dc/document' -i -X GET ----- \ No newline at end of file diff --git a/custom/test-update-mapping-without-document/http-request.adoc b/custom/test-update-mapping-without-document/http-request.adoc deleted file mode 100644 index 4aa6e413..00000000 --- a/custom/test-update-mapping-without-document/http-request.adoc +++ /dev/null @@ -1,6 +0,0 @@ -[source,http,options="nowrap"] ----- -GET /api/v1/mappingAdministration/my_dc/document HTTP/1.1 -Host: localhost:8095 - ----- \ No newline at end of file diff --git a/custom/test-update-mapping-without-document/http-response.adoc b/custom/test-update-mapping-without-document/http-response.adoc deleted file mode 100644 index 528f6003..00000000 --- a/custom/test-update-mapping-without-document/http-response.adoc +++ /dev/null @@ -1,26 +0,0 @@ -[source,http,options="nowrap"] ----- -HTTP/1.1 200 OK -ETag: "104363025" -Content-Length: 425 -Accept-Ranges: bytes -Content-Type: application/octet-stream - -{ - "$schema" : "http://json-schema.org/draft-07/schema#", - "$id" : "http://example.com/product.schema.json", - "title" : "Simple Mapping", - "description" : "Data resource mapping from json", - "type" : "object", - "properties" : { - "Publisher" : { - "path" : "publisher", - "type" : "string" - }, - "Publication Date" : { - "path" : "publicationDate", - "type" : "string" - } - } -} ----- \ No newline at end of file diff --git a/custom/test-update-mapping-without-document/httpie-request.adoc b/custom/test-update-mapping-without-document/httpie-request.adoc deleted file mode 100644 index a954b9fc..00000000 --- a/custom/test-update-mapping-without-document/httpie-request.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,bash] ----- -$ http GET 'http://localhost:8095/api/v1/mappingAdministration/my_dc/document' ----- \ No newline at end of file diff --git a/custom/test-update-mapping-without-document/request-body.adoc b/custom/test-update-mapping-without-document/request-body.adoc deleted file mode 100644 index dab5f81d..00000000 --- a/custom/test-update-mapping-without-document/request-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-update-mapping-without-document/response-body.adoc b/custom/test-update-mapping-without-document/response-body.adoc deleted file mode 100644 index bd0e4ace..00000000 --- a/custom/test-update-mapping-without-document/response-body.adoc +++ /dev/null @@ -1,20 +0,0 @@ -[source,octet-stream,options="nowrap"] ----- -{ - "$schema" : "http://json-schema.org/draft-07/schema#", - "$id" : "http://example.com/product.schema.json", - "title" : "Simple Mapping", - "description" : "Data resource mapping from json", - "type" : "object", - "properties" : { - "Publisher" : { - "path" : "publisher", - "type" : "string" - }, - "Publication Date" : { - "path" : "publicationDate", - "type" : "string" - } - } -} ----- \ No newline at end of file diff --git a/custom/test-update-mapping-without-etag/curl-request.adoc b/custom/test-update-mapping-without-etag/curl-request.adoc deleted file mode 100644 index ea30de0b..00000000 --- a/custom/test-update-mapping-without-etag/curl-request.adoc +++ /dev/null @@ -1,7 +0,0 @@ -[source,bash] ----- -$ curl 'http://localhost:8095/api/v1/mappingAdministration/my_dc' -i -X PUT \ - -H 'Content-Type: multipart/form-data' \ - -F 'record=@record.json;type=application/json' \ - -F 'document=@my_dc4gemma.mapping;type=application/json' ----- \ No newline at end of file diff --git a/custom/test-update-mapping-without-etag/http-request.adoc b/custom/test-update-mapping-without-etag/http-request.adoc deleted file mode 100644 index b47ef2d2..00000000 --- a/custom/test-update-mapping-without-etag/http-request.adoc +++ /dev/null @@ -1,35 +0,0 @@ -[source,http,options="nowrap"] ----- -PUT /api/v1/mappingAdministration/my_dc HTTP/1.1 -Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Host: localhost:8095 - ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Content-Disposition: form-data; name=record; filename=record.json -Content-Type: application/json - -{"mappingId":"my_dc","mappingType":"GEMMA","title":"TITEL","description":"DESCRIPTION","acl":[{"id":null,"sid":"SELF","permission":"READ"},{"id":null,"sid":"someoneelse","permission":"ADMINISTRATE"}],"mappingDocumentUri":"http://localhost:8095/api/v1/mappingAdministration/my_dc/document","documentHash":"sha256:0b415cfd8c084ea65ec2c9200a85a95402184011d442e5ab343021660420127f"} ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Content-Disposition: form-data; name=document; filename=my_dc4gemma.mapping -Content-Type: application/json - -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "http://example.com/product.schema.json", - "title": "Simple Mapping Version 2", - "description": "Data resource mapping from json", - "type": "object", - "properties":{ - "Publisher":{ - "path": "publisher", - "type": "string" - }, - "PublicationDate":{ - "path": "publicationDate", - "type": "string" - } - } -} - ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm-- ----- \ No newline at end of file diff --git a/custom/test-update-mapping-without-etag/http-response.adoc b/custom/test-update-mapping-without-etag/http-response.adoc deleted file mode 100644 index 7448deeb..00000000 --- a/custom/test-update-mapping-without-etag/http-response.adoc +++ /dev/null @@ -1,5 +0,0 @@ -[source,http,options="nowrap"] ----- -HTTP/1.1 428 Precondition Required - ----- \ No newline at end of file diff --git a/custom/test-update-mapping-without-etag/httpie-request.adoc b/custom/test-update-mapping-without-etag/httpie-request.adoc deleted file mode 100644 index 3b932a63..00000000 --- a/custom/test-update-mapping-without-etag/httpie-request.adoc +++ /dev/null @@ -1,6 +0,0 @@ -[source,bash] ----- -$ http --multipart PUT 'http://localhost:8095/api/v1/mappingAdministration/my_dc' \ - 'record'@'record.json' \ - 'document'@'my_dc4gemma.mapping' ----- \ No newline at end of file diff --git a/custom/test-update-mapping-without-etag/request-body.adoc b/custom/test-update-mapping-without-etag/request-body.adoc deleted file mode 100644 index d074c300..00000000 --- a/custom/test-update-mapping-without-etag/request-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,form-data,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-update-mapping-without-etag/response-body.adoc b/custom/test-update-mapping-without-etag/response-body.adoc deleted file mode 100644 index dab5f81d..00000000 --- a/custom/test-update-mapping-without-etag/response-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-update-mapping-without-record/curl-request.adoc b/custom/test-update-mapping-without-record/curl-request.adoc deleted file mode 100644 index baee036f..00000000 --- a/custom/test-update-mapping-without-record/curl-request.adoc +++ /dev/null @@ -1,7 +0,0 @@ -[source,bash] ----- -$ curl 'http://localhost:8095/api/v1/mappingAdministration/my_dc' -i -X PUT \ - -H 'Content-Type: multipart/form-data' \ - -H 'If-Match: "104363025"' \ - -F 'document=@my_dc4gemma.mapping;type=application/json' ----- \ No newline at end of file diff --git a/custom/test-update-mapping-without-record/http-request.adoc b/custom/test-update-mapping-without-record/http-request.adoc deleted file mode 100644 index 2fab7976..00000000 --- a/custom/test-update-mapping-without-record/http-request.adoc +++ /dev/null @@ -1,31 +0,0 @@ -[source,http,options="nowrap"] ----- -PUT /api/v1/mappingAdministration/my_dc HTTP/1.1 -Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -If-Match: "104363025" -Host: localhost:8095 - ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm -Content-Disposition: form-data; name=document; filename=my_dc4gemma.mapping -Content-Type: application/json - -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "http://example.com/product.schema.json", - "title": "Simple Mapping Version 2", - "description": "Data resource mapping from json", - "type": "object", - "properties":{ - "Publisher":{ - "path": "publisher", - "type": "string" - }, - "PublicationDate":{ - "path": "publicationDate", - "type": "string" - } - } -} - ---6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm-- ----- \ No newline at end of file diff --git a/custom/test-update-mapping-without-record/http-response.adoc b/custom/test-update-mapping-without-record/http-response.adoc deleted file mode 100644 index 6bc944f3..00000000 --- a/custom/test-update-mapping-without-record/http-response.adoc +++ /dev/null @@ -1,5 +0,0 @@ -[source,http,options="nowrap"] ----- -HTTP/1.1 400 Bad Request - ----- \ No newline at end of file diff --git a/custom/test-update-mapping-without-record/httpie-request.adoc b/custom/test-update-mapping-without-record/httpie-request.adoc deleted file mode 100644 index 29510d9f..00000000 --- a/custom/test-update-mapping-without-record/httpie-request.adoc +++ /dev/null @@ -1,6 +0,0 @@ -[source,bash] ----- -$ http --multipart PUT 'http://localhost:8095/api/v1/mappingAdministration/my_dc' \ - 'If-Match:"104363025"' \ - 'document'@'my_dc4gemma.mapping' ----- \ No newline at end of file diff --git a/custom/test-update-mapping-without-record/request-body.adoc b/custom/test-update-mapping-without-record/request-body.adoc deleted file mode 100644 index d074c300..00000000 --- a/custom/test-update-mapping-without-record/request-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,form-data,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-update-mapping-without-record/response-body.adoc b/custom/test-update-mapping-without-record/response-body.adoc deleted file mode 100644 index dab5f81d..00000000 --- a/custom/test-update-mapping-without-record/response-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-update-mapping/curl-request.adoc b/custom/test-update-mapping/curl-request.adoc deleted file mode 100644 index 4b167b7c..00000000 --- a/custom/test-update-mapping/curl-request.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,bash] ----- -$ curl 'http://localhost:8095/api/v1/mappingAdministration/my_dc/document' -i -X GET ----- \ No newline at end of file diff --git a/custom/test-update-mapping/http-request.adoc b/custom/test-update-mapping/http-request.adoc deleted file mode 100644 index 4aa6e413..00000000 --- a/custom/test-update-mapping/http-request.adoc +++ /dev/null @@ -1,6 +0,0 @@ -[source,http,options="nowrap"] ----- -GET /api/v1/mappingAdministration/my_dc/document HTTP/1.1 -Host: localhost:8095 - ----- \ No newline at end of file diff --git a/custom/test-update-mapping/http-response.adoc b/custom/test-update-mapping/http-response.adoc deleted file mode 100644 index 9909624a..00000000 --- a/custom/test-update-mapping/http-response.adoc +++ /dev/null @@ -1,26 +0,0 @@ -[source,http,options="nowrap"] ----- -HTTP/1.1 200 OK -ETag: "104363025" -Content-Length: 434 -Accept-Ranges: bytes -Content-Type: application/octet-stream - -{ - "$schema" : "http://json-schema.org/draft-07/schema#", - "$id" : "http://example.com/product.schema.json", - "title" : "Simple Mapping Version 2", - "description" : "Data resource mapping from json", - "type" : "object", - "properties" : { - "Publisher" : { - "path" : "publisher", - "type" : "string" - }, - "PublicationDate" : { - "path" : "publicationDate", - "type" : "string" - } - } -} ----- \ No newline at end of file diff --git a/custom/test-update-mapping/httpie-request.adoc b/custom/test-update-mapping/httpie-request.adoc deleted file mode 100644 index a954b9fc..00000000 --- a/custom/test-update-mapping/httpie-request.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,bash] ----- -$ http GET 'http://localhost:8095/api/v1/mappingAdministration/my_dc/document' ----- \ No newline at end of file diff --git a/custom/test-update-mapping/request-body.adoc b/custom/test-update-mapping/request-body.adoc deleted file mode 100644 index dab5f81d..00000000 --- a/custom/test-update-mapping/request-body.adoc +++ /dev/null @@ -1,4 +0,0 @@ -[source,options="nowrap"] ----- - ----- \ No newline at end of file diff --git a/custom/test-update-mapping/response-body.adoc b/custom/test-update-mapping/response-body.adoc deleted file mode 100644 index 139a1028..00000000 --- a/custom/test-update-mapping/response-body.adoc +++ /dev/null @@ -1,20 +0,0 @@ -[source,octet-stream,options="nowrap"] ----- -{ - "$schema" : "http://json-schema.org/draft-07/schema#", - "$id" : "http://example.com/product.schema.json", - "title" : "Simple Mapping Version 2", - "description" : "Data resource mapping from json", - "type" : "object", - "properties" : { - "Publisher" : { - "path" : "publisher", - "type" : "string" - }, - "PublicationDate" : { - "path" : "publicationDate", - "type" : "string" - } - } -} ----- \ No newline at end of file diff --git a/gradle/profile-deploy.gradle b/gradle/profile-deploy.gradle index d94f2fe8..05ae8e89 100644 --- a/gradle/profile-deploy.gradle +++ b/gradle/profile-deploy.gradle @@ -14,6 +14,57 @@ * limitations under the License. */ + +// Main jar with only selected classes +task selectedClassesJar(type: Jar) { + archiveBaseName = 'mapping-plugin-core' + archiveClassifier = '' + from sourceSets.main.output + include '**/configuration/ApplicationProperties.class', + '**/exception.BadExitCodeException.class', + '**/plugins/AbstractPythonMappingPlugin.class', + '**/plugins/IMappingPlugin.class', + '**/plugins/MappingPluginException.class', + '**/plugins/MappingPluginState.class', + '**/util/PythonRunnerUtil.class', + '**/util/ShellRunnerUtil.class' + includeEmptyDirs false +} + +// Sources jar with only selected classes +task selectedSourcesJar(type: Jar) { + archiveBaseName = 'mapping-plugin-core' + archiveClassifier = 'sources' + from sourceSets.main.allSource + include '**/configuration/ApplicationProperties.java', + '**/exception.BadExitCodeException.java', + '**/plugins/AbstractPythonMappingPlugin.java', + '**/plugins/IMappingPlugin.java', + '**/plugins/MappingPluginException.java', + '**/plugins/MappingPluginState.java', + '**/util/PythonRunnerUtil.java', + '**/util/ShellRunnerUtil.java' + includeEmptyDirs false +} + +// Javadoc jar +task javadocJar(type: Jar) { + archiveBaseName = 'mapping-plugin-core' + archiveClassifier = 'javadoc' + from javadoc +} + +//////////////////////////////////////////////////////////////////////////////// +//for java plugin +//see https://docs.gradle.org/current/userguide/java_plugin.html +//////////////////////////////////////////////////////////////////////////////// +//java { + //package JavaDoc as part of publication +// withJavadocJar() + //package Sources as part of publication +// withSourcesJar() +//} + //////////////////////////////////////////////////////////////////////////////// //for plugin net.researchgate.release //see https://github.com/researchgate/gradle-release @@ -25,4 +76,98 @@ release { versionPropertyFile = 'gradle.properties' //set possible properties which may contain the version versionProperties = ['version', 'mainversion'] + git { + //branch from where to release (default: main) + requireBranch.set('main') + } +} + +//////////////////////////////////////////////////////////////////////////////// +//for javadoc task +//see https://docs.gradle.org/current/dsl/org.gradle.api.tasks.javadoc.Javadoc.html +//////////////////////////////////////////////////////////////////////////////// +javadoc { + if(JavaVersion.current().isJava9Compatible()) { + options.addBooleanOption('html5', true) + } +} + +//////////////////////////////////////////////////////////////////////////////// +//for plugin io.github.gradle-nexus.publish-plugin +//see https://github.com/gradle-nexus/publish-plugin +//////////////////////////////////////////////////////////////////////////////// +nexusPublishing { + repositories { + sonatype() + } +} +//////////////////////////////////////////////////////////////////////////////// +//for plugin maven-publish/io.github.gradle-nexus.publish-plugin +//see https://docs.gradle.org/current/userguide/publishing_maven.html +//////////////////////////////////////////////////////////////////////////////// +publishing { + publications { + //define publication identity, e.g. maven, which will be used + //by the signing plugin, e.g. sign publishing.publications.maven + mavenJava(MavenPublication) { + artifact selectedClassesJar + artifact selectedSourcesJar + artifact javadocJar + + groupId = "edu.kit.datamanager" + artifactId = "mapping-plugin-core" + version = project.version + + pom { + name = project['name'] + description = project['description'] + url = 'http://datamanager.kit.edu' + licenses { + license { + name = 'The Apache License, Version 2.0' + url = 'http://www.apache.org/licenses/LICENSE-2.0.txt' + } + } + developers { + developer { + id = 'Jejkal' + name = 'Thomas Jejkal' + email = 'webmaster@datamanager.kit.edu' + } + } + scm { + connection = 'scm:git:github.com/kit-data-manager/service-base' + developerConnection = 'scm:git:github.com/kit-data-manager/service-base' + url = 'https://github.com/kit-data-manager/service-base' + } + } + } + } +} +//////////////////////////////////////////////////////////////////////////////// +//for plugin signing +//see https://docs.gradle.org/current/userguide/signing_plugin.html +//////////////////////////////////////////////////////////////////////////////// +signing { + //make signing required unless for SNAPSHOT releases or if signing is explicitly skipped + required { !project.version.endsWith("-SNAPSHOT") && !project.hasProperty("skipSigning") } + + //look for property 'signingKey' + if (project.findProperty("signingKey")) { + //If required, read a sub-key specified by its ID in property signingKeyId + //def signingKeyId = findProperty("signingKeyId") + //read property 'signingKey' + def signingKey = findProperty("signingKey") + //read property 'signingPassword' + def signingPassword = findProperty("signingPassword") + //Select to use in-memory ascii-armored keys + useInMemoryPgpKeys(signingKey, signingPassword) + //Only if also using signingKeyId + //useInMemoryPgpKeys(signingKeyId, signingKey, signingPassword) + + //Apply signing to publication identity 'publishing.publications.maven' + sign publishing.publications.maven + }else { + println 'WARNING: No property \'signingKey\' found. Artifact signing will be skipped.' + } } diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index a4b76b95..1b33c55b 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index cea7a793..ca025c83 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.12-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.14-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew b/gradlew index f3b75f3b..23d15a93 100755 --- a/gradlew +++ b/gradlew @@ -114,7 +114,7 @@ case "$( uname )" in #( NONSTOP* ) nonstop=true ;; esac -CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar +CLASSPATH="\\\"\\\"" # Determine the Java command to use to start the JVM. @@ -205,7 +205,7 @@ fi DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Collect all arguments for the java command: -# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, # and any embedded shellness will be escaped. # * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be # treated as '${Hostname}' itself on the command line. @@ -213,7 +213,7 @@ DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ -classpath "$CLASSPATH" \ - org.gradle.wrapper.GradleWrapperMain \ + -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ "$@" # Stop when "xargs" is not available. diff --git a/gradlew.bat b/gradlew.bat index 9b42019c..5eed7ee8 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -70,11 +70,11 @@ goto fail :execute @rem Setup the command line -set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar +set CLASSPATH= @rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* :end @rem End local scope for the variables with windows NT shell diff --git a/plugins/empty-plugin-0.0.0-SNAPSHOT-plain.jar b/plugins/empty-plugin-0.0.0-SNAPSHOT-plain.jar deleted file mode 100644 index 61bdbe1f..00000000 Binary files a/plugins/empty-plugin-0.0.0-SNAPSHOT-plain.jar and /dev/null differ diff --git a/plugins/gemma-plugin-0.1.0-SNAPSHOT-plain.jar b/plugins/gemma-plugin-0.1.0-SNAPSHOT-plain.jar deleted file mode 100644 index bf7f87dd..00000000 Binary files a/plugins/gemma-plugin-0.1.0-SNAPSHOT-plain.jar and /dev/null differ diff --git a/settings.gradle b/settings.gradle index 8b2ea627..cfb1341a 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,5 +1,5 @@ plugins { - id "com.gradle.enterprise" version "3.19" + id "com.gradle.enterprise" version "3.19.2" } //gradleEnterprise { diff --git a/settings/application-default.properties b/settings/application-default.properties index e97d63d8..921214cf 100644 --- a/settings/application-default.properties +++ b/settings/application-default.properties @@ -19,7 +19,7 @@ spring.servlet.multipart.max-file-size=100MB spring.servlet.multipart.max-request-size=100MB # Logging settings logging.level.root=ERROR -logging.level.edu.kit.datamanager=INFO +logging.level.edu.kit.datamanager=DEBUG springdoc.swagger-ui.disable-swagger-default-url=true # Actuator settings info.app.name=Mapping-Service @@ -27,7 +27,14 @@ info.app.description=Generic mapping service supporting different mapping implem info.app.group=edu.kit.datamanager info.app.version=1.0.4 management.endpoint.health.probes.enabled=true -management.endpoints.web.exposure.include=* +management.endpoint.health.enabled: true +management.endpoint.health.show-details: when-authorized +management.endpoint.health.sensitive: true +management.endpoints.web.exposure.include: health,info + +#spring.security.user.name=admin +#spring.security.user.password=secret +#spring.security.user.roles=ADMIN ############################################################################### # Spring Cloud @@ -53,4 +60,17 @@ mapping-service.pythonExecutable=file:///usr/bin/python3 mapping-service.pluginLocation=file://INSTALLATION_DIR/plugins # Absolute path to the local gemma mappings folder. mapping-service.mappingSchemasLocation=file://INSTALLATION_DIR/mappingSchemas +# Folder where job output files for async mapping executions are stored mapping-service.jobOutput=file://INSTALLATION_DIR/jobOutput + +management.metrics.export.prometheus.enabled=true +management.endpoint.metrics.enabled=true + +# Execution timeout for script calls +mapping-service.executionTimeout=30 + +mapping-service.authEnabled:false +mapping-service.mappingAdminRole:MAPPING_ADMIN + +repo.security.enable-csrf=false +repo.security.allowedOriginPattern=* \ No newline at end of file diff --git a/settings/application-docker.properties b/settings/application-docker.properties index 3fb5481f..73ca39fa 100644 --- a/settings/application-docker.properties +++ b/settings/application-docker.properties @@ -47,4 +47,13 @@ mapping-service.pythonExecutable=file:///usr/bin/python3 mapping-service.pluginLocation=file://INSTALLATION_DIR/plugins # Absolute path to the local gemma mappings folder. mapping-service.mappingSchemasLocation=file://INSTALLATION_DIR/mappingSchemas +# Folder where job output files for async mapping executions are stored mapping-service.jobOutput=file://INSTALLATION_DIR/jobOutput +# Execution timeout for script calls +mapping-service.executionTimeout=30 + +mapping-service.authEnabled:false +mapping-service.mappingAdminRole:MAPPING_ADMIN + +repo.security.enable-csrf=false +repo.security.allowedOriginPattern=* \ No newline at end of file diff --git a/src/main/java/edu/kit/datamanager/mappingservice/MappingServiceApplication.java b/src/main/java/edu/kit/datamanager/mappingservice/MappingServiceApplication.java index 77267aae..126e6a7f 100644 --- a/src/main/java/edu/kit/datamanager/mappingservice/MappingServiceApplication.java +++ b/src/main/java/edu/kit/datamanager/mappingservice/MappingServiceApplication.java @@ -1,18 +1,25 @@ package edu.kit.datamanager.mappingservice; +import com.google.common.collect.ImmutableSet; +import com.google.common.reflect.ClassPath; import edu.kit.datamanager.mappingservice.configuration.ApplicationProperties; -import edu.kit.datamanager.mappingservice.exception.MappingJobException; import edu.kit.datamanager.mappingservice.plugins.PluginManager; -import java.io.File; -import java.util.UUID; -import java.util.regex.Matcher; -import java.util.regex.Pattern; +import io.micrometer.core.instrument.MeterRegistry; +import edu.kit.datamanager.mappingservice.plugins.PluginLoader; +import edu.kit.datamanager.mappingservice.util.PythonRunnerUtil; +import edu.kit.datamanager.mappingservice.util.ShellRunnerUtil; +import edu.kit.datamanager.security.filter.KeycloakJwtProperties; +import edu.kit.datamanager.security.filter.KeycloakTokenFilter; +import edu.kit.datamanager.security.filter.KeycloakTokenValidator; +import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.autoconfigure.domain.EntityScan; -import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; @@ -27,21 +34,60 @@ public class MappingServiceApplication { private static final Logger LOG = LoggerFactory.getLogger(MappingServiceApplication.class); + @Autowired + private final MeterRegistry meterRegistry; + + MappingServiceApplication(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + } + @Bean public ApplicationProperties applicationProperties() { return new ApplicationProperties(); } + @Bean + public PluginLoader pluginLoader() { + return new PluginLoader(applicationProperties()); + } + @Bean public PluginManager pluginManager() { - return new PluginManager(applicationProperties()); + PythonRunnerUtil.init(applicationProperties()); + ShellRunnerUtil.init(applicationProperties()); + return new PluginManager(applicationProperties(), pluginLoader(), meterRegistry); + } + + @Bean + public KeycloakJwtProperties keycloakProperties() { + return new KeycloakJwtProperties(); + } + + @Bean + @ConditionalOnProperty( + value = "mapping-service.authEnabled", + havingValue = "true", + matchIfMissing = false) + public KeycloakTokenFilter keycloaktokenFilterBean() throws Exception { + return new KeycloakTokenFilter(KeycloakTokenValidator.builder() + .readTimeout(keycloakProperties().getReadTimeoutms()) + .connectTimeout(keycloakProperties().getConnectTimeoutms()) + .sizeLimit(keycloakProperties().getSizeLimit()) + .jwtLocalSecret("vkfvoswsohwrxgjaxipuiyyjgubggzdaqrcuupbugxtnalhiegkppdgjgwxsmvdb") + .build(keycloakProperties().getJwkUrl(), keycloakProperties().getResource(), keycloakProperties().getJwtClaim())); } - public static void main(String[] args) { - SpringApplication.run(MappingServiceApplication.class, args); + public static void main(String[] args) throws IOException { + ConfigurableApplicationContext ctx = SpringApplication.run(MappingServiceApplication.class, args); - //pluginManager().getListOfAvailableValidators().forEach((value) -> LOG.info("Found validator: " + value)); - //PythonRunnerUtil.printPythonVersion(); - System.out.println("Mapping service is running! Access it at http://localhost:8095"); + PluginManager mgr = ctx.getBean(PluginManager.class); + System.out.println("Found plugins: "); + mgr.getPlugins().forEach((k, v) -> { + System.out.println(String.format(" - %s (%s)", k, v)); + }); + System.out.println("Using Python Version: "); + PythonRunnerUtil.printPythonVersion(); + String port = ctx.getEnvironment().getProperty("server.port"); + System.out.println(String.format("Mapping service is running on port %s.", port)); } } diff --git a/src/main/java/edu/kit/datamanager/mappingservice/configuration/ApplicationProperties.java b/src/main/java/edu/kit/datamanager/mappingservice/configuration/ApplicationProperties.java index 2bd34558..062ac1ba 100644 --- a/src/main/java/edu/kit/datamanager/mappingservice/configuration/ApplicationProperties.java +++ b/src/main/java/edu/kit/datamanager/mappingservice/configuration/ApplicationProperties.java @@ -15,8 +15,8 @@ */ package edu.kit.datamanager.mappingservice.configuration; -import edu.kit.datamanager.annotations.ExecutableFileURL; import edu.kit.datamanager.annotations.LocalFolderURL; +import edu.kit.datamanager.validator.ExecutableFileValidator; import lombok.Data; import lombok.EqualsAndHashCode; import org.springframework.beans.factory.annotation.Value; @@ -42,7 +42,6 @@ public class ApplicationProperties { /** * The absolute path to the python interpreter. */ - @ExecutableFileURL @Value("${mapping-service.pythonExecutable}") private URL pythonExecutable; @@ -59,11 +58,57 @@ public class ApplicationProperties { @LocalFolderURL @Value("${mapping-service.mappingSchemasLocation}") private URL mappingsLocation; - + + /** + * The absolute path where mapping plugin code is checked out into, i.e., + * for Python-based plugins. + */ + @LocalFolderURL + @Value("${mapping-service.codeLocation}") + private URL codeLocation; + /** * The absolute path where job data is stored. */ @LocalFolderURL @Value("${mapping-service.jobOutput}") private URL jobOutputLocation; + + /** + * One or more packages to scan for plugin classes. + */ + @Value("${mapping-service.packagesToScan:edu.kit.datamanager.mappingservice.plugins.impl}") + private String[] packagesToScan; + + @Value("${mapping-service.executionTimeout:30}") + private int executionTimeout; + + /** + * Auth and permission properties + */ + @Value("${mapping-service.authEnabled:FALSE}") + private boolean authEnabled; + @Value("${mapping-service.mappingAdminRole:MAPPING_ADMIN}") + private String mappingAdminRole; + /** + * CORS and CSRF properties + */ + @Value("${repo.security.allowedOriginPattern:*}") + private String allowedOriginPattern; + @Value("${repo.security.allowedMethods:GET,POST,PUT,PATCH,DELETE,OPTIONS}") + private String[] allowedMethods; + @Value("${repo.security.exposedHeaders:Content-Range,ETag,Link}") + private String[] exposedHeaders; + @Value("${repo.security.allowedHeaders:*}") + private String[] allowedHeaders; + + /** + * Check if configured python executable is valid and executable. + * + * @return TRUE if python executable is valid. + */ + public boolean isPythonAvailable() { + return new ExecutableFileValidator().isValid(pythonExecutable, null); + } + } diff --git a/src/main/java/edu/kit/datamanager/mappingservice/configuration/OpenApiDefinitions.java b/src/main/java/edu/kit/datamanager/mappingservice/configuration/OpenApiDefinitions.java index ef51b657..e0378875 100644 --- a/src/main/java/edu/kit/datamanager/mappingservice/configuration/OpenApiDefinitions.java +++ b/src/main/java/edu/kit/datamanager/mappingservice/configuration/OpenApiDefinitions.java @@ -36,7 +36,7 @@ public OpenAPI customOpenAPI() { .components(new Components()) .info(new Info().title("Mapping-Service - RESTful API"). description("This webpage describes the RESTful API of the KIT Data Manager Mapping-Service."). - version("0.1"). + version("1.1.2"). contact( new Contact(). name("KIT Data Manager Support"). diff --git a/src/main/java/edu/kit/datamanager/mappingservice/configuration/StaticResourcesConfiguration.java b/src/main/java/edu/kit/datamanager/mappingservice/configuration/StaticResourcesConfiguration.java index 9ff92c77..948850d3 100644 --- a/src/main/java/edu/kit/datamanager/mappingservice/configuration/StaticResourcesConfiguration.java +++ b/src/main/java/edu/kit/datamanager/mappingservice/configuration/StaticResourcesConfiguration.java @@ -15,7 +15,10 @@ */ package edu.kit.datamanager.mappingservice.configuration; +import edu.kit.datamanager.mappingservice.rest.impl.PreHandleInterceptor; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Configuration; +import org.springframework.web.servlet.config.annotation.InterceptorRegistry; import org.springframework.web.servlet.config.annotation.PathMatchConfigurer; import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry; import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; @@ -28,10 +31,16 @@ */ @Configuration public class StaticResourcesConfiguration implements WebMvcConfigurer { + private final PreHandleInterceptor preHandleInterceptor; private static final String[] CLASSPATH_RESOURCE_LOCATIONS = { "classpath:/static/"}; + @Autowired + public StaticResourcesConfiguration(PreHandleInterceptor preHandleInterceptor) { + this.preHandleInterceptor = preHandleInterceptor; + } + @Override public void addResourceHandlers(ResourceHandlerRegistry registry) { registry.addResourceHandler("/static/**").addResourceLocations(CLASSPATH_RESOURCE_LOCATIONS); @@ -43,4 +52,9 @@ public void configurePathMatch(PathMatchConfigurer configurer) { urlPathHelper.setUrlDecode(false); configurer.setUrlPathHelper(urlPathHelper); } + + @Override + public void addInterceptors(InterceptorRegistry registry) { + registry.addInterceptor(preHandleInterceptor); + } } diff --git a/src/main/java/edu/kit/datamanager/mappingservice/configuration/WebSecurityConfig.java b/src/main/java/edu/kit/datamanager/mappingservice/configuration/WebSecurityConfig.java index 8fb9c842..73e633ac 100644 --- a/src/main/java/edu/kit/datamanager/mappingservice/configuration/WebSecurityConfig.java +++ b/src/main/java/edu/kit/datamanager/mappingservice/configuration/WebSecurityConfig.java @@ -18,6 +18,7 @@ import edu.kit.datamanager.security.filter.KeycloakTokenFilter; import edu.kit.datamanager.security.filter.NoAuthenticationFilter; import java.util.Arrays; +import java.util.List; import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,6 +35,7 @@ import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; import org.springframework.security.config.annotation.web.configuration.WebSecurityCustomizer; import org.springframework.security.config.http.SessionCreationPolicy; +import org.springframework.security.config.Customizer; import org.springframework.security.web.SecurityFilterChain; import org.springframework.security.web.authentication.www.BasicAuthenticationFilter; import org.springframework.security.web.firewall.DefaultHttpFirewall; @@ -52,10 +54,12 @@ public class WebSecurityConfig { private static final Logger logger = LoggerFactory.getLogger(WebSecurityConfig.class); - + @Autowired private Optional keycloaktokenFilterBean; - + @Autowired + private ApplicationProperties applicationProperties; + private static final String[] AUTH_WHITELIST_SWAGGER_UI = { // -- Swagger UI v2 "/v2/api-docs", @@ -68,11 +72,26 @@ public class WebSecurityConfig { // -- Swagger UI v3 (OpenAPI) "/v3/api-docs/**", "/swagger-ui/**" - // other public endpoints of your API may be appended to this array + // other public endpoints of your API may be appended to this array }; @Bean public SecurityFilterChain filterChain(HttpSecurity http) throws Exception { + + List securedEndpointMatchers; + + if (applicationProperties.isAuthEnabled()) { + logger.trace("Authentication is ENABLED. Collecting secured endpoints."); + securedEndpointMatchers = Arrays.asList( + new AntPathRequestMatcher("/api/v1/mappingAdministration/reloadTypes", "GET"), + new AntPathRequestMatcher("/api/v1/mappingAdministration", "PUT"), + new AntPathRequestMatcher("/api/v1/mappingAdministration", "POST") + ); + } else { + logger.trace("Authentication is DISABLED. Not securing endpoints."); + securedEndpointMatchers = Arrays.asList(); + } + HttpSecurity httpSecurity = http.authorizeHttpRequests( authorize -> authorize. requestMatchers(HttpMethod.OPTIONS).permitAll(). @@ -80,14 +99,14 @@ public SecurityFilterChain filterChain(HttpSecurity http) throws Exception { InfoEndpoint.class, HealthEndpoint.class )).permitAll(). - requestMatchers(EndpointRequest.toAnyEndpoint()).hasAnyRole("ANONYMOUS", "ADMIN", "ACTUATOR", "SERVICE_WRITE"). - // requestMatchers(new AntPathRequestMatcher("/oaipmh")).permitAll(). + requestMatchers(EndpointRequest.toAnyEndpoint()).hasAnyRole("ANONYMOUS", "ADMINISTRATOR", "ACTUATOR", "SERVICE_WRITE"). requestMatchers(new AntPathRequestMatcher("/static/**")).permitAll(). requestMatchers(new AntPathRequestMatcher("/error")).permitAll(). - //requestMatchers(new AntPathRequestMatcher("/api/v1/")).permitAll(). + requestMatchers(securedEndpointMatchers.toArray(AntPathRequestMatcher[]::new)).hasRole(applicationProperties.getMappingAdminRole()). //endpoint filters only active if auth is enabled requestMatchers(AUTH_WHITELIST_SWAGGER_UI).permitAll(). anyRequest().authenticated() ). + httpBasic(Customizer.withDefaults()). cors(cors -> cors.configurationSource(corsConfigurationSource())). sessionManagement( session -> session.sessionCreationPolicy(SessionCreationPolicy.STATELESS)); @@ -95,10 +114,22 @@ public SecurityFilterChain filterChain(HttpSecurity http) throws Exception { logger.info("CSRF disabled!"); httpSecurity = httpSecurity.csrf(csrf -> csrf.disable()); - logger.info("Authentication is DISABLED. Adding 'NoAuthenticationFilter' to authentication chain."); - AuthenticationManager defaultAuthenticationManager = http.getSharedObject(AuthenticationManager.class); - httpSecurity = httpSecurity.addFilterAfter(new NoAuthenticationFilter("vkfvoswsohwrxgjaxipuiyyjgubggzdaqrcuupbugxtnalhiegkppdgjgwxsmvdb", defaultAuthenticationManager), BasicAuthenticationFilter.class); - + if (keycloaktokenFilterBean.isPresent()) { + logger.trace("Adding Keycloak filter to filter chain."); + httpSecurity.addFilterAfter(keycloaktokenFilterBean.get(), BasicAuthenticationFilter.class); + } else { + logger.trace("Keycloak not configured. Skip adding keycloak filter to filter chain."); + } + + if (!applicationProperties.isAuthEnabled()) { + logger.info("Adding 'NoAuthenticationFilter' to filter chain."); + AuthenticationManager defaultAuthenticationManager = http.getSharedObject(AuthenticationManager.class); + httpSecurity = httpSecurity.addFilterAfter(new NoAuthenticationFilter("vkfvoswsohwrxgjaxipuiyyjgubggzdaqrcuupbugxtnalhiegkppdgjgwxsmvdb", defaultAuthenticationManager), BasicAuthenticationFilter.class); + } else { + logger.info("Skip adding NoAuthenticationFilter to filter chain."); + } + + logger.trace("Turning off cache control."); httpSecurity.headers(headers -> headers.cacheControl(cache -> cache.disable())); return httpSecurity.build(); @@ -116,30 +147,14 @@ public HttpFirewall allowUrlEncodedSlashHttpFirewall() { return firewall; } - /* @Bean - public FilterRegistrationBean corsFilter() { - final UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource(); - CorsConfiguration config = new CorsConfiguration(); - config.addAllowedOrigin("*"); - config.addAllowedHeader("*"); - config.addAllowedMethod("*"); - config.addExposedHeader("Content-Range"); - config.addExposedHeader("ETag"); - - source.registerCorsConfiguration("/**", config); - FilterRegistrationBean bean = new FilterRegistrationBean<>(new CorsFilter(source)); - bean.setOrder(0); - return bean; - }*/ public CorsConfigurationSource corsConfigurationSource() { CorsConfiguration config = new CorsConfiguration(); config.setAllowCredentials(true); + config.addAllowedOriginPattern(applicationProperties.getAllowedOriginPattern()); + config.setAllowedHeaders(Arrays.asList(applicationProperties.getAllowedHeaders())); + config.setAllowedMethods(Arrays.asList(applicationProperties.getAllowedMethods())); + config.setExposedHeaders(Arrays.asList(applicationProperties.getExposedHeaders())); - config.addAllowedOriginPattern("*"); - config.setAllowedHeaders(Arrays.asList("*")); - config.setAllowedMethods(Arrays.asList("*")); - config.setExposedHeaders(Arrays.asList("*")); - final UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource(); source.registerCorsConfiguration("/**", config); diff --git a/src/main/java/edu/kit/datamanager/mappingservice/exception/BadExitCodeException.java b/src/main/java/edu/kit/datamanager/mappingservice/exception/BadExitCodeException.java new file mode 100644 index 00000000..4fc1964b --- /dev/null +++ b/src/main/java/edu/kit/datamanager/mappingservice/exception/BadExitCodeException.java @@ -0,0 +1,34 @@ +/* + * Copyright 2025 Karlsruhe Institute of Technology. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.kit.datamanager.mappingservice.exception; + +/** + * + * @author jejkal + */ +public class BadExitCodeException extends Exception { + + private int exitCode = 0; + + public BadExitCodeException(int exitCode) { + super("Process exited with code " + exitCode); + this.exitCode = exitCode; + } + + public int getExitCode() { + return exitCode; + } + +} diff --git a/src/main/java/edu/kit/datamanager/mappingservice/exception/DuplicateMappingException.java b/src/main/java/edu/kit/datamanager/mappingservice/exception/DuplicateMappingException.java index 757c7baf..dc34757b 100644 --- a/src/main/java/edu/kit/datamanager/mappingservice/exception/DuplicateMappingException.java +++ b/src/main/java/edu/kit/datamanager/mappingservice/exception/DuplicateMappingException.java @@ -21,7 +21,7 @@ /** * Invalid json format of data. */ -@ResponseStatus(value = HttpStatus.CONFLICT) +@ResponseStatus(value = HttpStatus.CONFLICT, reason = "Duplicate mapping id.") public class DuplicateMappingException extends RuntimeException { /** diff --git a/src/main/java/edu/kit/datamanager/mappingservice/exception/JobIdConflictException.java b/src/main/java/edu/kit/datamanager/mappingservice/exception/JobIdConflictException.java new file mode 100644 index 00000000..d666040d --- /dev/null +++ b/src/main/java/edu/kit/datamanager/mappingservice/exception/JobIdConflictException.java @@ -0,0 +1,62 @@ +/* + * Copyright 2024 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.kit.datamanager.mappingservice.exception; + +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.ResponseStatus; + +/** + * + * @author jejkal + */ +@ResponseStatus(value = HttpStatus.CONFLICT, reason = "Job Id already in use. Please try again later.") +public class JobIdConflictException extends RuntimeException { + + /** + * Default constructor. + */ + public JobIdConflictException() { + super(); + } + + /** + * Constructor with given message and cause. + * + * @param message Message. + * @param cause Cause. + */ + public JobIdConflictException(String message, Throwable cause) { + super(message, cause); + } + + /** + * Constructor with given message. + * + * @param message Message. + */ + public JobIdConflictException(String message) { + super(message); + } + + /** + * Constructor with given message and cause. + * + * @param cause Cause. + */ + public JobIdConflictException(Throwable cause) { + super(cause); + } +} diff --git a/src/main/java/edu/kit/datamanager/mappingservice/exception/JobNotFoundException.java b/src/main/java/edu/kit/datamanager/mappingservice/exception/JobNotFoundException.java index 9859e7f4..b99ec8de 100644 --- a/src/main/java/edu/kit/datamanager/mappingservice/exception/JobNotFoundException.java +++ b/src/main/java/edu/kit/datamanager/mappingservice/exception/JobNotFoundException.java @@ -21,7 +21,7 @@ /** * Job not found. */ -@ResponseStatus(value = HttpStatus.NOT_FOUND) +@ResponseStatus(value = HttpStatus.NOT_FOUND, reason = "No job with provided id found.") public class JobNotFoundException extends RuntimeException { /** diff --git a/src/main/java/edu/kit/datamanager/mappingservice/exception/MappingNotFoundException.java b/src/main/java/edu/kit/datamanager/mappingservice/exception/MappingNotFoundException.java index d1e9cf2d..07ed7f78 100644 --- a/src/main/java/edu/kit/datamanager/mappingservice/exception/MappingNotFoundException.java +++ b/src/main/java/edu/kit/datamanager/mappingservice/exception/MappingNotFoundException.java @@ -21,7 +21,7 @@ /** * Invalid json format of data. */ -@ResponseStatus(value = HttpStatus.NOT_FOUND) +@ResponseStatus(value = HttpStatus.NOT_FOUND, reason = "No mapping found for provided mappingId.") public class MappingNotFoundException extends RuntimeException { /** @@ -35,7 +35,7 @@ public MappingNotFoundException() { * Constructor with given message and cause. * * @param message Message. - * @param cause Cause. + * @param cause Cause. */ public MappingNotFoundException(String message, Throwable cause) { super(message, cause); diff --git a/src/main/java/edu/kit/datamanager/mappingservice/exception/MappingServiceException.java b/src/main/java/edu/kit/datamanager/mappingservice/exception/MappingServiceException.java new file mode 100644 index 00000000..30b9b2ef --- /dev/null +++ b/src/main/java/edu/kit/datamanager/mappingservice/exception/MappingServiceException.java @@ -0,0 +1,61 @@ +/* + * Copyright 2022 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.kit.datamanager.mappingservice.exception; + +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.ResponseStatus; + +/** + * Used for internal server errors that could not be or were not handled. + */ +@ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR) +public class MappingServiceException extends RuntimeException { + + /** + * Default constructor. + */ + public MappingServiceException() { + super(); + } + + /** + * Constructor with given message and cause. + * + * @param message Message. + * @param cause Cause. + */ + public MappingServiceException(String message, Throwable cause) { + super(message, cause); + } + + /** + * Constructor with given message. + * + * @param message Message. + */ + public MappingServiceException(String message) { + super(message); + } + + /** + * Constructor with given message and cause. + * + * @param cause Cause. + */ + public MappingServiceException(Throwable cause) { + super(cause); + } +} diff --git a/src/main/java/edu/kit/datamanager/mappingservice/exception/MappingServiceUserException.java b/src/main/java/edu/kit/datamanager/mappingservice/exception/MappingServiceUserException.java new file mode 100644 index 00000000..2e4111f1 --- /dev/null +++ b/src/main/java/edu/kit/datamanager/mappingservice/exception/MappingServiceUserException.java @@ -0,0 +1,61 @@ +/* + * Copyright 2022 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.kit.datamanager.mappingservice.exception; + +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.ResponseStatus; + +/** + * Used for errors caused by user input. + */ +@ResponseStatus(value = HttpStatus.BAD_REQUEST) +public class MappingServiceUserException extends RuntimeException { + + /** + * Default constructor. + */ + public MappingServiceUserException() { + super(); + } + + /** + * Constructor with given message and cause. + * + * @param message Message. + * @param cause Cause. + */ + public MappingServiceUserException(String message, Throwable cause) { + super(message, cause); + } + + /** + * Constructor with given message. + * + * @param message Message. + */ + public MappingServiceUserException(String message) { + super(message); + } + + /** + * Constructor with given message and cause. + * + * @param cause Cause. + */ + public MappingServiceUserException(Throwable cause) { + super(cause); + } +} diff --git a/src/main/java/edu/kit/datamanager/mappingservice/exception/PluginInitializationFailedException.java b/src/main/java/edu/kit/datamanager/mappingservice/exception/PluginInitializationFailedException.java new file mode 100644 index 00000000..7fdcd033 --- /dev/null +++ b/src/main/java/edu/kit/datamanager/mappingservice/exception/PluginInitializationFailedException.java @@ -0,0 +1,58 @@ +/* + * Copyright 2025 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.kit.datamanager.mappingservice.exception; + +/** + * + * @author jejkal + */ +public class PluginInitializationFailedException extends RuntimeException { + + /** + * Default constructor. + */ + public PluginInitializationFailedException() { + super(); + } + + /** + * Constructor with given message and cause. + * + * @param message Message. + * @param cause Cause. + */ + public PluginInitializationFailedException(String message, Throwable cause) { + super(message, cause); + } + + /** + * Constructor with given message. + * + * @param message Message. + */ + public PluginInitializationFailedException(String message) { + super(message); + } + + /** + * Constructor with given message and cause. + * + * @param cause Cause. + */ + public PluginInitializationFailedException(Throwable cause) { + super(cause); + } +} diff --git a/src/main/java/edu/kit/datamanager/mappingservice/impl/JobManager.java b/src/main/java/edu/kit/datamanager/mappingservice/impl/JobManager.java index 8a058472..7abdcba6 100644 --- a/src/main/java/edu/kit/datamanager/mappingservice/impl/JobManager.java +++ b/src/main/java/edu/kit/datamanager/mappingservice/impl/JobManager.java @@ -62,7 +62,7 @@ public CompletableFuture getJob(String jobId) { * Remove the job with the provided id. Keep in mind, that removing the job * from the JobManager won't remove job outputs. * - * @param The job's id. + * @param jobId The job's id. */ public void removeJob(String jobId) { mapOfJobs.remove(jobId); diff --git a/src/main/java/edu/kit/datamanager/mappingservice/impl/MappingService.java b/src/main/java/edu/kit/datamanager/mappingservice/impl/MappingService.java index c6b83611..4a75fd80 100644 --- a/src/main/java/edu/kit/datamanager/mappingservice/impl/MappingService.java +++ b/src/main/java/edu/kit/datamanager/mappingservice/impl/MappingService.java @@ -15,6 +15,7 @@ */ package edu.kit.datamanager.mappingservice.impl; +import edu.kit.datamanager.exceptions.BadArgumentException; import edu.kit.datamanager.mappingservice.configuration.ApplicationProperties; import edu.kit.datamanager.mappingservice.dao.IMappingRecordDao; import edu.kit.datamanager.mappingservice.domain.JobStatus; @@ -22,13 +23,16 @@ import edu.kit.datamanager.mappingservice.exception.DuplicateMappingException; import edu.kit.datamanager.mappingservice.exception.JobNotFoundException; import edu.kit.datamanager.mappingservice.exception.JobProcessingException; -import edu.kit.datamanager.mappingservice.exception.MappingException; import edu.kit.datamanager.mappingservice.exception.MappingJobException; import edu.kit.datamanager.mappingservice.exception.MappingNotFoundException; +import edu.kit.datamanager.mappingservice.exception.MappingServiceException; +import edu.kit.datamanager.mappingservice.exception.MappingServiceUserException; import edu.kit.datamanager.mappingservice.plugins.MappingPluginException; import edu.kit.datamanager.mappingservice.plugins.MappingPluginState; import edu.kit.datamanager.mappingservice.plugins.PluginManager; import edu.kit.datamanager.mappingservice.util.FileUtil; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; import org.apache.commons.codec.binary.Hex; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; @@ -88,6 +92,7 @@ public class MappingService { private Path jobsOutputDirectory; private ApplicationProperties applicationProperties; + private final MeterRegistry meterRegistry; /** * Logger for this class. @@ -95,8 +100,9 @@ public class MappingService { private final static Logger LOGGER = LoggerFactory.getLogger(MappingService.class); @Autowired - public MappingService(ApplicationProperties applicationProperties) { + public MappingService(ApplicationProperties applicationProperties, MeterRegistry meterRegistry) { this.applicationProperties = applicationProperties; + this.meterRegistry = meterRegistry; init(this.applicationProperties); } @@ -112,6 +118,21 @@ public MappingService(ApplicationProperties applicationProperties) { */ public MappingRecord createMapping(String content, MappingRecord mappingRecord) throws IOException { LOGGER.trace("Creating mapping with id {}.", mappingRecord.getMappingId()); + // Check for valid mapping ID (should contain at least one non whitespace. + String mappingId = mappingRecord.getMappingId(); + if ((mappingId == null) || (mappingId.isBlank())) { + String message = String.format("MappingID shouldn't be empty or contain only whitespaces. You provide '%s'", mappingId); + LOGGER.error(message); + throw new BadArgumentException(message); + } + + String mappingType = mappingRecord.getMappingType(); + if ((mappingType == null) || (mappingType.isBlank() || !pluginManager.getPlugins().containsKey(mappingType))) { + String message = String.format("MappingType shouldn't be empty or contain only whitespaces and must be a registered plugin id. You provide '%s'", mappingType); + LOGGER.error(message); + throw new BadArgumentException(message); + } + Iterable findMapping = mappingRepo.findByMappingIdIn(Collections.singletonList(mappingRecord.getMappingId())); if (findMapping.iterator().hasNext()) { LOGGER.error("Unable to create mapping with id {}. Mapping id is alreadyy used.", mappingRecord.getMappingId()); @@ -147,7 +168,6 @@ public void updateMapping(String content, MappingRecord mappingRecord) throws Ma LOGGER.trace("Persisting mapping record."); mappingRepo.save(mappingRecord); LOGGER.trace("Mapping with id {} successfully updated.", mappingRecord.getMappingId()); - } /** @@ -164,13 +184,13 @@ public void deleteMapping(MappingRecord mappingRecord) { } LOGGER.trace("Deleting mapping with id {}.", mappingRecord.getMappingId()); mappingRecord = findMapping.get(); + mappingRepo.delete(mappingRecord); + LOGGER.trace("Mapping with id {} deleted.", mappingRecord.getMappingId()); try { deleteMappingFile(mappingRecord); } catch (IOException e) { - LOGGER.error("Failed to delete mapping file at " + mappingRecord.getMappingDocumentUri() + ". Please remove it manually.", e); + LOGGER.error(String.format("Failed to delete mapping file at %s. Please remove it manually.", mappingRecord.getMappingDocumentUri()), e); } - mappingRepo.delete(mappingRecord); - LOGGER.trace("Mapping with id {} deleted.", mappingRecord.getMappingId()); } /** @@ -184,7 +204,7 @@ public void deleteMapping(MappingRecord mappingRecord) { public Optional executeMapping(URI contentUrl, String mappingId) throws MappingPluginException { LOGGER.trace("Executing mapping of content {} using mapping with id {}.", contentUrl, mappingId); if (contentUrl == null || mappingId == null) { - throw new MappingPluginException(MappingPluginState.INVALID_INPUT, "Either contentUrl or mappingId are not provided."); + throw new MappingPluginException(MappingPluginState.INVALID_INPUT(), "Either contentUrl or mappingId are not provided."); } Optional returnValue; @@ -194,21 +214,27 @@ public Optional executeMapping(URI contentUrl, String mappingId) throws Ma LOGGER.trace("Searching for mapping with id {}.", mappingId); Optional optionalMappingRecord = mappingRepo.findByMappingId(mappingId); if (optionalMappingRecord.isPresent()) { - LOGGER.trace("Mapping for id {} found. Creating temporary output file."); + LOGGER.trace("Mapping for id {} found.", mappingId); mappingRecord = optionalMappingRecord.get(); + + Counter.builder("mapping_service.plugin_usage").tag("plugin", mappingRecord.getMappingType()).register(meterRegistry).increment(); + Path mappingFile = Paths.get(mappingRecord.getMappingDocumentUri()); // execute mapping Path resultFile; + LOGGER.trace("Preparing temporary output file."); resultFile = FileUtil.createTempFile(mappingId + "_" + srcFile.hashCode(), ".result"); LOGGER.trace("Temporary output file available at {}. Performing mapping.", resultFile); MappingPluginState result = pluginManager.mapFile(mappingRecord.getMappingType(), mappingFile, srcFile, resultFile); LOGGER.trace("Mapping returned with result {}. Returning result file.", result); returnValue = Optional.of(resultFile); // remove downloaded file + LOGGER.trace("Removing user upload."); FileUtil.removeFile(srcFile); + LOGGER.trace("User upload successfully removed."); } else { - LOGGER.error("Unable to find mapping with id {}.", mappingId); - throw new MappingNotFoundException("Unable to find mapping with id " + mappingId + "."); + LOGGER.error("Unable to find mapping for id {}.", mappingId); + throw new MappingNotFoundException(String.format("Unable to find mapping with id %s.", mappingId)); } return returnValue; } @@ -233,45 +259,45 @@ public CompletableFuture executeMappingAsync(String jobId, URI conten if (contentUrl == null || mappingId == null) { task.complete(JobStatus.error(jobId, JobStatus.STATUS.FAILED, "Either contentUrl or mappingId are not provided.")); - } - - Optional returnValue; - Path srcFile = Paths.get(contentUrl); - MappingRecord mappingRecord; - - // Get mapping file - LOGGER.trace("Searching for mapping with id {}.", mappingId); - Optional optionalMappingRecord = mappingRepo.findByMappingId(mappingId); - if (optionalMappingRecord.isPresent()) { - LOGGER.trace("Mapping for id {} found. Creating temporary output file.", mappingId); - mappingRecord = optionalMappingRecord.get(); - Path mappingFile = Paths.get(mappingRecord.getMappingDocumentUri()); - // execute mapping - Path resultFile = getOutputFile(jobId).toPath(); - LOGGER.trace("Temporary output file available at {}. Performing mapping.", resultFile); - try { - MappingPluginState result = pluginManager.mapFile(mappingRecord.getMappingType(), mappingFile, srcFile, resultFile); - - LOGGER.trace("Mapping returned with result {}. Returning result file.", result); - returnValue = Optional.of(resultFile); - LOGGER.trace("Fixing file extension for output {}", returnValue.get()); - Path outputPath = FileUtil.fixFileExtension(returnValue.get()); - LOGGER.trace("Fixed output path: {}", outputPath); - - task.complete(JobStatus.complete(jobId, JobStatus.STATUS.SUCCEEDED, outputPath.toFile())); - } catch (Throwable t) { - task.complete(JobStatus.error(jobId, JobStatus.STATUS.FAILED, t.getMessage())); - } finally { - // remove downloaded file - LOGGER.trace("Removing user upload at {}.", srcFile); - FileUtil.removeFile(srcFile); - LOGGER.trace("User upload successfully removed."); - - } } else { - LOGGER.error("Unable to find mapping with id {}.", mappingId); - task.complete(JobStatus.error(jobId, JobStatus.STATUS.FAILED, "Unable to find mapping with id " + mappingId + ".")); - //throw new MappingNotFoundException("Unable to find mapping with id " + mappingId + "."); + Optional returnValue; + Path srcFile = Paths.get(contentUrl); + MappingRecord mappingRecord; + + // Get mapping file + LOGGER.trace("Searching for mapping with id {}.", mappingId); + Optional optionalMappingRecord = mappingRepo.findByMappingId(mappingId); + if (optionalMappingRecord.isPresent()) { + LOGGER.trace("Mapping for id {} found. Creating temporary output file.", mappingId); + mappingRecord = optionalMappingRecord.get(); + Path mappingFile = Paths.get(mappingRecord.getMappingDocumentUri()); + // execute mapping + Path resultFile = getOutputFile(jobId).toPath(); + LOGGER.trace("Temporary output file available at {}. Performing mapping.", resultFile); + try { + MappingPluginState result = pluginManager.mapFile(mappingRecord.getMappingType(), mappingFile, srcFile, resultFile); + + LOGGER.trace("Mapping returned with result state {}. Returning result file.", result.getState()); + returnValue = Optional.of(resultFile); + LOGGER.trace("Fixing file extension for output {}", returnValue.get()); + Path outputPath = FileUtil.fixFileExtension(returnValue.get()); + LOGGER.trace("Fixed output path: {}", outputPath); + + task.complete(JobStatus.complete(jobId, JobStatus.STATUS.SUCCEEDED, outputPath.toFile())); + } catch (MappingPluginException t) { + LOGGER.error("Asynchronous job execution failed with error.", t); + task.complete(JobStatus.error(jobId, JobStatus.STATUS.FAILED, t.getMessage())); + } finally { + // remove downloaded file + LOGGER.trace("Removing user upload at {}.", srcFile); + FileUtil.removeFile(srcFile); + LOGGER.trace("User upload successfully removed."); + } + } else { + LOGGER.error("Unable to find mapping with id {}.", mappingId); + task.complete(JobStatus.error(jobId, JobStatus.STATUS.FAILED, "Unable to find mapping with id " + mappingId + ".")); + //throw new MappingNotFoundException("Unable to find mapping with id " + mappingId + "."); + } } return task; } @@ -288,7 +314,7 @@ public CompletableFuture executeMappingAsync(String jobId, URI conten public CompletableFuture fetchJobElseThrowException(String jobId) throws JobNotFoundException { CompletableFuture job = fetchJob(jobId); if (null == job) { - LOGGER.error("Job-id {} not found.", jobId); + LOGGER.error("Job with id {} not found.", jobId); throw new JobNotFoundException(JOB_WITH_SUPPLIED_JOB_ID_NOT_FOUND); } return job; @@ -323,15 +349,15 @@ public JobStatus getJobStatus(String jobId) throws Throwable { if (ex != null) { errors[0] = ex.getCause(); } else { - StringBuilder outputFileUri = new StringBuilder("/api/v1/mappingExecution/schedule/"); - outputFileUri.append(jobId).append("/"); - outputFileUri.append("download"); - response.setOutputFileURI(outputFileUri.toString()); + response.setOutputFileURI(String.format("/api/v1/mappingExecution/schedule/%s/download", jobId)); simpleResponses[0] = response; } }); if (errors[0] != null) { + if (errors[0] instanceof MappingPluginException mappingPluginException) { + mappingPluginException.throwMe(); + } throw errors[0]; } @@ -354,18 +380,21 @@ public File getJobOutputFile(String jobId) throws Throwable { CompletableFuture completableFuture = fetchJob(jobId); if (null == completableFuture) { + //return output file in case it still exists (even after the job was removed from the queue) File outputFile = getOutputFile(jobId); if (outputFile.exists()) { return outputFile; } - + //nothing left, return error throw new JobNotFoundException(JOB_WITH_SUPPLIED_JOB_ID_NOT_FOUND); } if (!completableFuture.isDone()) { + LOGGER.trace("Job {} not finished, yet. Returning RUNNING state.", jobId); throw new JobProcessingException("Job is still in progress...", true); } + LOGGER.trace("Obtaining output from job status."); Throwable[] errors = new Throwable[1]; JobStatus[] jobStatus = new JobStatus[1]; completableFuture.whenComplete((response, ex) -> { @@ -377,6 +406,9 @@ public File getJobOutputFile(String jobId) throws Throwable { }); if (errors[0] != null) { + if (errors[0] instanceof MappingPluginException mappingPluginException) { + mappingPluginException.throwMe(); + } throw errors[0]; } @@ -397,35 +429,38 @@ public JobStatus deleteJobAndAssociatedData(String jobId) { CompletableFuture completableFuture = fetchJob(jobId); if (null == completableFuture) { + //delete output file if file still exists (even after the job was removed from the queue) File outputFile = getOutputFile(jobId); if (outputFile.exists()) { outputFile.delete(); } else { - LOGGER.debug("No output file for job {} found. Returning.", jobId); + LOGGER.debug("No output file for job {} found.", jobId); } return JobStatus.status(jobId, JobStatus.STATUS.DELETED); } if (!completableFuture.isDone()) { + LOGGER.trace("Job {} not finished, yet. Returning RUNNING state.", jobId); return JobStatus.status(jobId, JobStatus.STATUS.RUNNING); } + LOGGER.trace("Cleaning up all job artifacts"); completableFuture.whenComplete((response, ex) -> { if (ex != null) { LOGGER.error("Job failed with exception.", ex); } - + final File jobOutput; if (null != response && null != response.getJobOutput()) { - if (response.getJobOutput().exists()) { - response.getJobOutput().delete(); - } + jobOutput = response.getJobOutput(); } else { - File outputFile = getOutputFile(jobId); - if (outputFile.exists()) { - outputFile.delete(); - } + jobOutput = getOutputFile(jobId); } + if (jobOutput.exists()) { + LOGGER.trace(" - Deleting job output"); + jobOutput.delete(); + } + LOGGER.trace("Removing job from queue."); jobManager.removeJob(jobId); }); @@ -442,12 +477,12 @@ public JobStatus deleteJobAndAssociatedData(String jobId) { private File getOutputFile(String jobId) { Matcher m = Pattern.compile("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$").matcher(jobId); if (!m.matches()) { - throw new MappingJobException("Invalid jobId provided."); + throw new MappingJobException(String.format("Invalid jobId %s provided.", jobId)); } - Path outputPath = jobsOutputDirectory.resolve(jobId + ".out").normalize(); + Path outputPath = jobsOutputDirectory.resolve(String.format("%s.out", jobId)).normalize(); if (!outputPath.startsWith(jobsOutputDirectory)) { - throw new IllegalArgumentException("Invalid jobId provided."); + throw new IllegalArgumentException(String.format("Invalid jobId %s provided.", jobId)); } return outputPath.toFile(); } @@ -463,16 +498,20 @@ private void init(ApplicationProperties applicationProperties) { try { mappingsDirectory = Files.createDirectories(new File(applicationProperties.getMappingsLocation().getPath()).getAbsoluteFile().toPath()); } catch (IOException e) { - throw new MappingException("Could not initialize directory '" + applicationProperties.getMappingsLocation() + "' for mapping.", e); + throw new MappingServiceException(String.format("Could not initialize mappings directory '%s' for mapping.", applicationProperties.getMappingsLocation()), e); } try { jobsOutputDirectory = Files.createDirectories(new File(applicationProperties.getJobOutputLocation().getPath()).getAbsoluteFile().toPath()); } catch (IOException e) { - throw new MappingException("Could not initialize directory '" + applicationProperties.getJobOutputLocation() + "' for job outputs.", e); + throw new MappingServiceException(String.format("Could not initialize job output directory '%s'.", applicationProperties.getJobOutputLocation()), e); + } + try { + Files.createDirectories(new File(applicationProperties.getCodeLocation().getPath()).getAbsoluteFile().toPath()); + } catch (IOException e) { + throw new MappingServiceException(String.format("Could not initialize code target directory '%s'.", applicationProperties.getCodeLocation()), e); } - } else { - throw new MappingException("Could not initialize mapping directory due to missing location!"); + throw new MappingServiceException("Cannot configure MappingService due to missing application.properties."); } } @@ -504,14 +543,10 @@ private void saveMappingFile(String content, MappingRecord mapping) throws IOExc } catch (NoSuchAlgorithmException ex) { String message = "Failed to initialize SHA256 MessageDigest."; LOGGER.error(message, ex); - throw new MappingException(message, ex); - } catch (IllegalArgumentException iae) { - String message = "Error: Unkown mapping! (" + mapping.getMappingType() + ")"; - LOGGER.error(message, iae); - throw new MappingException(message, iae); + throw new MappingServiceException(message, ex); } } else { - throw new MappingException("Error saving mapping file! (no content)"); + throw new MappingServiceUserException("Failed to update mapping file. No content or mapping record provided."); } } diff --git a/src/main/java/edu/kit/datamanager/mappingservice/plugins/AbstractPythonMappingPlugin.java b/src/main/java/edu/kit/datamanager/mappingservice/plugins/AbstractPythonMappingPlugin.java new file mode 100644 index 00000000..e0860dcd --- /dev/null +++ b/src/main/java/edu/kit/datamanager/mappingservice/plugins/AbstractPythonMappingPlugin.java @@ -0,0 +1,288 @@ +/* + * Copyright 2025 Karlsruhe Institute of Technology. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.kit.datamanager.mappingservice.plugins; + +import edu.kit.datamanager.mappingservice.configuration.ApplicationProperties; +import edu.kit.datamanager.mappingservice.exception.PluginInitializationFailedException; +import edu.kit.datamanager.mappingservice.util.FileUtil; +import edu.kit.datamanager.mappingservice.util.PythonRunnerUtil; +import edu.kit.datamanager.mappingservice.util.ShellRunnerUtil; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.Properties; +import org.apache.maven.artifact.versioning.ComparableVersion; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * + * @author jejkal + */ +public abstract class AbstractPythonMappingPlugin implements IMappingPlugin { + + private final Logger LOGGER = LoggerFactory.getLogger(AbstractPythonMappingPlugin.class); + + /** + * The plugin name. + */ + private String name; + /** + * The URL of the Git repository where the plugin code is located. + */ + private String repositoryUrl; + + /** + * The tag which should be used to checkout a specific version from + * repositoryUrl. + */ + private String tag; + + /** + * The minimal python version required by the plugin + */ + private String minPython; + + /** + * The folder where the code is checked out from repositoryUrl. + */ + private Path dir; + + private String pluginVenv = "venv/PluginVenv"; + private String venvInterpreter; + + /** + * Default constructor for instantiating a Python-based mapping plugin. It + * is assumed, that the code for the plugin is stored in a Git repository + * located at 'repositoryUrl'. Furthermore, it is assumed, that the plugin + * itself is delivered as a single Jar file which contains at least the + * plugin class and a file PLUGIN_NAME.properties, where PLUGIN_NAME must be + * identical to the 'pluginName' argument and the file must be located in + * the root of the Jar file. The properties file must contain one property + * 'version' which represents an existing Git tag matching a released + * version of the Python plugin code, e.g., version=v1.0.0 + * + * Furthermore, the properties file may contain a minimal Python version + * that is required by the plugin to work. The minimal Python version is set + * via the 'min.python' property, e.g.., min.python=3.10.0 If the minimal + * Python version is not met, the plugin will be ignored. + * + * @param pluginName The name of the plugin. + * @param repositoryUrl The Git repository where the plugin Python code is + * located. + */ + public AbstractPythonMappingPlugin(String pluginName, String repositoryUrl) { + try { + name = pluginName; + this.repositoryUrl = repositoryUrl; + // Get the context class loader + ClassLoader classLoader = this.getClass().getClassLoader(); + // TODO: do we need to make sure that the resource path is somehow related to the current plugin to avoid loading the wrong property file in case of identical property names? + URL resource = classLoader.getResource(pluginName.toLowerCase() + ".properties"); + LOGGER.info("Resource file: {}", resource); + if (resource != null) { + // Load the properties file + try (InputStream input = resource.openStream()) { + Properties properties = new Properties(); + properties.load(input); + tag = properties.getProperty("version"); + minPython = properties.getProperty("min.python"); + } + } else { + System.err.println("Properties file not found!"); + tag = "unavailable"; + } + + if (System.getProperty("os.name").startsWith("Windows")) { + venvInterpreter = pluginVenv + "/Scripts/python.exe"; + } else { + venvInterpreter = pluginVenv + "/bin/python3"; + } + } catch (IOException e) { + throw new PluginInitializationFailedException("Failed to instantiate plugin class.", e); + } + } + + /** + * Abstract method that is supposed to be implemented by each Python mapping + * plugin to gather all information required for starting a Python process + * executing the mapping script. The returned array must contain at least + * the following information: + * + * <ul> <li>The absolute path of the main script. It must start + * with the working dir received as argument, where all checked-out code is + * located.</li> <li>Script-specific parameters to provide + * mappingFile, inputFile, and outputFile to the script execution. Depending + * on the script implementation, the number and kind of required arguments + * may differ.</li> </ul> + * + * Example: In standalone mode, a script is called via `plugin_wrapper.py + * sem -m mappingFile -i inputFile -o outputFile -debug`. In that case, the + * resulting array should look as follows: [workingDir + + * "plugin_wrapper.py", "sem", "-m", mappingFile.toString(), "-i", + * inputFile.toString(), "-o", outputFile.toString(), "-debug"]. + * + * The Python call itself will be added according to the Venv used for + * plugin execution and must not be included. + * + * @param workingDir The working directory, i.e., where the plugin code was + * checked-out into. + * @param mappingFile The file which contains the mapping rules registered + * at the mapping-service and used by the script. + * @param inputFile The file which was uploaded by the user, i.e., the + * source of the mapping process. + * @param outputFile The destination where mapping results must be written + * to in order to allow the mapping-service to return the result to the + * user. + * + * @return A string array containing the single elements of the command line + * call of the script. + */ + public abstract String[] getCommandArray(Path workingDir, Path mappingFile, Path inputFile, Path outputFile); + + @Override + public String name() { + return this.name; + } + + @Override + public String version() { + return this.tag; + } + + @Override + public String description() { + return "Plugin " + name() + ", Version " + version() + ", Implementation: " + uri(); + } + + @Override + public String uri() { + return this.repositoryUrl; + } + + @Override + public void setup(ApplicationProperties applicationProperties) { + LOGGER.trace("Setting up mapping plugin {} {}", name(), version()); + + //testing minimal Python version + if (minPython != null) { + if (!hasMinimalPythonVersion(minPython)) { + throw new PluginInitializationFailedException("Minimal Python version '" + minPython + "' required by plugin not met."); + } + } + + //checkout and install plugin + try { + LOGGER.info("Cloning git repository {}, tag {}", repositoryUrl, tag); + Path path = Paths.get(applicationProperties.getCodeLocation().toURI()); + path = path.resolve(repositoryUrl.trim().replace("https://", "").replace("http://", "").replace(".git", "") + "_" + version()); + LOGGER.info("Target path: {}", path); + dir = FileUtil.cloneGitRepository(repositoryUrl, tag, path.toAbsolutePath().toString()); + // Install Python dependencies + MappingPluginState venvState = PythonRunnerUtil.runPythonScript("-m", "venv", "--system-site-packages", dir + "/" + pluginVenv); + if (MappingPluginState.SUCCESS().getState().equals(venvState.getState())) { + LOGGER.info("Venv for plugin installed successfully. Installing requirements."); + + Path requirementsFile = Paths.get(dir + "/" + "requirements.dist.txt"); + if (requirementsFile.toFile().exists()) { + MappingPluginState requirementsInstallState = ShellRunnerUtil.run(dir + "/" + venvInterpreter, "-m", "pip", "install", "-r", dir + "/" + "requirements.dist.txt"); + if (MappingPluginState.SUCCESS().getState().equals(requirementsInstallState.getState())) { + LOGGER.info("Requirements for plugin installed successfully. Setup complete."); + } else { + throw new PluginInitializationFailedException("Failed to install plugin requirements. Status: " + venvState.getState()); + } + } else { + LOGGER.info("No requirements file found. Skipping dependency installation."); + } + } else { + throw new PluginInitializationFailedException("Venv installation has failed. Status: " + venvState.getState()); + } + } catch (URISyntaxException e) { + throw new PluginInitializationFailedException("Invalid codeLocation configured in application.properties.", e); + } catch (MappingPluginException e) { + throw new PluginInitializationFailedException("Unexpected error during plugin setup.", e); + } + } + + @Override + public MappingPluginState mapFile(Path mappingFile, Path inputFile, Path outputFile) throws MappingPluginException { + long startTime = System.currentTimeMillis(); + LOGGER.trace("Run mapping plugin {} {} on '{}' with mapping '{}' -> '{}'", name(), version(), mappingFile, inputFile, outputFile); + String[] commandArray = getCommandArray(dir, mappingFile, inputFile, outputFile); + List command = new LinkedList<>(); + command.add(dir + "/" + venvInterpreter); + command.addAll(Arrays.asList(commandArray)); + MappingPluginState result = ShellRunnerUtil.run(command.toArray(String[]::new)); + long endTime = System.currentTimeMillis(); + long totalTime = endTime - startTime; + LOGGER.info("Execution time of mapFile: {} milliseconds", totalTime); + return result; + } + + /** + * This method checks if the local Python installation version is larger or + * equal the provided version number. The version should be provided as + * semantic version number, i.e., 3.13.2 + * + * The method will return TRUE if the minimal requirements are met and false + * otherwise. False is also returned if obtaining/parsing the local python + * version fails. for any reason. + * + * @param versionString The semantic version string to compare the local + * Python version against. + * + * @return True if versionString is smaller or equal the local Python + * version, false otherwise. + */ + private boolean hasMinimalPythonVersion(String versionString) { + boolean result = false; + try { + LOGGER.trace("Checking for minimal Python version {}.", versionString); + ByteArrayOutputStream bout = new ByteArrayOutputStream(); + MappingPluginState state = PythonRunnerUtil.runPythonScript("--version", bout, System.err); + + if (!MappingPluginState.StateEnum.SUCCESS.equals(state.getState())) { + LOGGER.error("Failed to obtain Python version. python --version returned with status {}.", state.getState()); + } else { + + LOGGER.trace("Version command output: {}", bout.toString()); + + String[] split = bout.toString().split(" "); + + if (split.length == 2) { + String localPythonVersion = bout.toString().split(" ")[1].trim(); + LOGGER.trace("Obtained local Python version: {}", localPythonVersion); + ComparableVersion localVersion = new ComparableVersion(localPythonVersion); + ComparableVersion minimalVersion = new ComparableVersion(versionString); + result = minimalVersion.compareTo(localVersion) <= 0; + } else { + LOGGER.info("Unexpected Python version output. Unable to check for minimal version."); + } + } + } catch (MappingPluginException e) { + LOGGER.error("Failed to obtain Python version.", e); + } + return result; + } +} diff --git a/src/main/java/edu/kit/datamanager/mappingservice/plugins/IMappingPlugin.java b/src/main/java/edu/kit/datamanager/mappingservice/plugins/IMappingPlugin.java index 5b1e69c9..3110e940 100644 --- a/src/main/java/edu/kit/datamanager/mappingservice/plugins/IMappingPlugin.java +++ b/src/main/java/edu/kit/datamanager/mappingservice/plugins/IMappingPlugin.java @@ -12,16 +12,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package edu.kit.datamanager.mappingservice.plugins; +import edu.kit.datamanager.mappingservice.configuration.ApplicationProperties; import org.springframework.util.MimeType; import java.nio.file.Path; /** - * Interface for mapping plugins. - * Every plugin which implements this interface and is placed in the plugins folder will be loaded and usable via the REST-API. + * Interface for mapping plugins. Every plugin which implements this interface + * and is placed in the plugins folder will be loaded and usable via the + * REST-API. * * @author maximilianiKIT */ @@ -42,15 +43,17 @@ public interface IMappingPlugin { String description(); /** - * The version of the plugin which gets displayed in the UI and is part of the id. + * The version of the plugin which gets displayed in the UI and is part of + * the id. * * @return The version of the plugin. */ String version(); /** - * A URI which refers to the plugin or the technology used by the plugin (e.g. a link to a GitHub repository). - * This URI will be displayed in the UI. + * A URI which refers to the plugin or the technology used by the plugin + * (e.g. a link to a GitHub repository). This URI will be displayed in the + * UI. * * @return The URI of the plugin. */ @@ -71,8 +74,9 @@ public interface IMappingPlugin { MimeType[] outputTypes(); /** - * The id of the plugin which is used to identify the plugin. - * By default, the id is composed of the name and the version of the plugin (e.g. testPlugin_2.1.0). + * The id of the plugin which is used to identify the plugin. By default, + * the id is composed of the name and the version of the plugin (e.g. + * testPlugin_2.1.0). * * @return The id of the plugin. */ @@ -81,19 +85,22 @@ default String id() { } /** - * This method is called when the plugin is loaded. - * It can be used to initialize the plugin and install dependencies. + * This method is called when the plugin is loaded. It can be used to + * initialize the plugin and install dependencies. + * + * @param applicationProperties Properties object containing all + * mapping-service settings. */ - void setup(); + void setup(ApplicationProperties applicationProperties); /** * The method which is called to execute the plugin. * - * @param inputFile The path to the output document. - * @param outputFile The path to the output document. + * @param inputFile The path to the output document. + * @param outputFile The path to the output document. * @param mappingFile The path to the mapping schema. * @return The exit code of the plugin. - * + * * @throws MappingPluginException If the mapping execution fails. */ MappingPluginState mapFile(Path mappingFile, Path inputFile, Path outputFile) throws MappingPluginException; diff --git a/src/main/java/edu/kit/datamanager/mappingservice/plugins/MappingPluginException.java b/src/main/java/edu/kit/datamanager/mappingservice/plugins/MappingPluginException.java index 9937fb42..24cde918 100644 --- a/src/main/java/edu/kit/datamanager/mappingservice/plugins/MappingPluginException.java +++ b/src/main/java/edu/kit/datamanager/mappingservice/plugins/MappingPluginException.java @@ -13,9 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package edu.kit.datamanager.mappingservice.plugins; +import org.springdoc.core.fn.builders.apiresponse.Builder; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.server.ResponseStatusException; + /** * Exception thrown by mapping plugins. * @@ -26,7 +30,7 @@ public class MappingPluginException extends Exception { /** * State of the plugin. */ - private MappingPluginState state; + private MappingPluginState mappingPluginState; /** * Default constructor. @@ -34,8 +38,8 @@ public class MappingPluginException extends Exception { * @param state State of the plugin. */ public MappingPluginException(MappingPluginState state) { - super(state.name()); - this.state = state; + super(state.getState().name()); + this.mappingPluginState = state; } /** @@ -46,7 +50,7 @@ public MappingPluginException(MappingPluginState state) { */ public MappingPluginException(MappingPluginState state, String message) { super(message); - this.state = state; + this.mappingPluginState = state; } /** @@ -58,7 +62,7 @@ public MappingPluginException(MappingPluginState state, String message) { */ public MappingPluginException(MappingPluginState state, String message, Throwable cause) { super(message, cause); - this.state = state; + this.mappingPluginState = state; } /** @@ -66,8 +70,11 @@ public MappingPluginException(MappingPluginState state, String message, Throwabl * * @return The state of the plugin. */ - public MappingPluginState getState() { - return state; + public MappingPluginState getMappingPluginState() { + return mappingPluginState; } -} + public void throwMe() throws ResponseStatusException { + throw new ResponseStatusException(this.mappingPluginState.getState().getHttpStatus(), "Cause: " + this.mappingPluginState.getDetails()); + } +} diff --git a/src/main/java/edu/kit/datamanager/mappingservice/plugins/MappingPluginState.java b/src/main/java/edu/kit/datamanager/mappingservice/plugins/MappingPluginState.java index c622098f..2c75bc5f 100644 --- a/src/main/java/edu/kit/datamanager/mappingservice/plugins/MappingPluginState.java +++ b/src/main/java/edu/kit/datamanager/mappingservice/plugins/MappingPluginState.java @@ -5,46 +5,88 @@ import java.io.Serializable; /** - * State of a mapping plugin. - * This class is used to store the state of a mapping plugin and return an HTTP status code. + * State of a mapping plugin. This class is used to store the state of a mapping + * plugin and return an HTTP status code. * * @author maximilianiKIT */ -public enum MappingPluginState implements Serializable { - SUCCESS(HttpStatus.OK), - NOT_FOUND(HttpStatus.NOT_FOUND), - TIMEOUT, - EXECUTION_ERROR(HttpStatus.INTERNAL_SERVER_ERROR), - INVALID_INPUT, - INCORRECT_MIME_TYPE, - INSUFFICIENT_PRIVILEGES(HttpStatus.INTERNAL_SERVER_ERROR), - UNKNOWN_ERROR(HttpStatus.INTERNAL_SERVER_ERROR); - - private final HttpStatus httpStatus; +public class MappingPluginState implements Serializable { - /** - * This method returns the HTTP status code for the given state. - * - * @return The HTTP status code for the given state. - */ - public HttpStatus getHttpStatus() { - return httpStatus; - } + public enum StateEnum { + SUCCESS(HttpStatus.OK), + NOT_FOUND(HttpStatus.NOT_FOUND), + TIMEOUT(HttpStatus.GATEWAY_TIMEOUT), + EXECUTION_ERROR(HttpStatus.INTERNAL_SERVER_ERROR), + INVALID_INPUT(HttpStatus.BAD_REQUEST), + BAD_EXIT_CODE(HttpStatus.INTERNAL_SERVER_ERROR), + UNKNOWN_ERROR(HttpStatus.INTERNAL_SERVER_ERROR); + + private final HttpStatus httpStatus; + + StateEnum() { + this.httpStatus = HttpStatus.BAD_REQUEST; + } + + StateEnum(HttpStatus status) { + this.httpStatus = status; + } + + public HttpStatus getHttpStatus() { + return httpStatus; + } + + }; + + private final StateEnum state; + private Object details; /** * Lets the state be created with an HTTP status code. * - * @param status The HTTP status code for the given state. + * @param state The state enum. */ - MappingPluginState(HttpStatus status) { - this.httpStatus = status; + public MappingPluginState(StateEnum state) { + this.state = state; } - /** - * Default constructor. - * Sets the default HTTP status code on error to 400. - */ - MappingPluginState() { - this.httpStatus = HttpStatus.BAD_REQUEST; + public static MappingPluginState SUCCESS() { + return new MappingPluginState(StateEnum.SUCCESS); + } + + public static MappingPluginState NOT_FOUND() { + return new MappingPluginState(StateEnum.NOT_FOUND); + } + + public static MappingPluginState TIMEOUT() { + return new MappingPluginState(StateEnum.TIMEOUT); } + + public static MappingPluginState EXECUTION_ERROR() { + return new MappingPluginState(StateEnum.EXECUTION_ERROR); + } + + public static MappingPluginState INVALID_INPUT() { + return new MappingPluginState(StateEnum.INVALID_INPUT); + } + + public static MappingPluginState BAD_EXIT_CODE() { + return new MappingPluginState(StateEnum.BAD_EXIT_CODE); + } + + public static MappingPluginState UNKNOWN_ERROR() { + return new MappingPluginState(StateEnum.UNKNOWN_ERROR); + } + + public StateEnum getState() { + return state; + } + + public void setDetails(Object details) { + this.details = details; + } + + public Object getDetails() { + return details; + } + } diff --git a/src/main/java/edu/kit/datamanager/mappingservice/plugins/PluginLoader.java b/src/main/java/edu/kit/datamanager/mappingservice/plugins/PluginLoader.java index daec7db6..9cfea49e 100644 --- a/src/main/java/edu/kit/datamanager/mappingservice/plugins/PluginLoader.java +++ b/src/main/java/edu/kit/datamanager/mappingservice/plugins/PluginLoader.java @@ -14,6 +14,8 @@ */ package edu.kit.datamanager.mappingservice.plugins; +import edu.kit.datamanager.mappingservice.configuration.ApplicationProperties; +import edu.kit.datamanager.mappingservice.exception.PluginInitializationFailedException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,12 +32,23 @@ import java.util.Map; import java.util.jar.JarEntry; import java.util.jar.JarInputStream; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationContext; +import org.springframework.core.io.Resource; +import org.springframework.core.io.support.PathMatchingResourcePatternResolver; +import org.springframework.core.io.support.ResourcePatternResolver; +import org.springframework.core.type.classreading.CachingMetadataReaderFactory; +import org.springframework.core.type.classreading.MetadataReader; +import org.springframework.core.type.classreading.MetadataReaderFactory; +import org.springframework.stereotype.Component; +import org.springframework.util.ClassUtils; /** * Class for loading plugins. * * @author maximilianiKIT */ +@Component public class PluginLoader { /** @@ -43,9 +56,19 @@ public class PluginLoader { */ static Logger LOG = LoggerFactory.getLogger(PluginLoader.class); - static ClassLoader cl = null; + private ClassLoader cl = null; - public static void unload() { + private ApplicationProperties applicationProperties; + + @Autowired + private ApplicationContext applicationContext; + + @Autowired + public PluginLoader(ApplicationProperties applicationProperties) { + this.applicationProperties = applicationProperties; + } + + public void unload() { cl = null; System.gc(); } @@ -53,37 +76,47 @@ public static void unload() { /** * Load plugins from a given directory. * - * @param plugDir Directory containing plugins. + * @param pluginDir Directory containing plugins. + * @param packagesToScan Packages to scan in addition for plugins. + * * @return Map of plugins. + * * @throws IOException If there is an error with the file system. * @throws MappingPluginException If there is an error with the plugin or * the input. */ - public static Map loadPlugins(File plugDir) throws IOException, MappingPluginException { + public Map loadPlugins(File pluginDir, String[] packagesToScan) throws IOException, MappingPluginException { Map result = new HashMap<>(); - if (plugDir == null || plugDir.getAbsolutePath().isBlank()) { - LOG.warn("Plugin folder " + plugDir + " is null. Unable to load plugins."); + File[] pluginJars = new File[0]; + if (pluginDir == null || pluginDir.getAbsolutePath().isBlank()) { + LOG.warn("Plugin folder {} is not defined. MappingService will only use plugins in classpath.", pluginDir); } else { - File[] plugJars = plugDir.listFiles(new JARFileFilter()); - if (plugJars == null || plugJars.length < 1) { - LOG.warn("Plugin folder " + plugDir + " is empty. Unable to load plugins."); - } else { - cl = new URLClassLoader(PluginLoader.fileArrayToURLArray(plugJars), Thread.currentThread().getContextClassLoader()); - - List> plugClasses = PluginLoader.extractClassesFromJARs(plugJars, cl); - List IMappingPluginList = PluginLoader.createPluggableObjects(plugClasses); - - for (IMappingPlugin i : IMappingPluginList) { - //TODO: Add error handling in case setup of one plugin fails - i.setup(); - result.put(i.id(), i); - } + pluginJars = pluginDir.listFiles(new JARFileFilter()); + } + + if (pluginJars != null && pluginJars.length > 0) { + cl = new URLClassLoader(fileArrayToURLArray(pluginJars), Thread.currentThread().getContextClassLoader()); + } else { + cl = Thread.currentThread().getContextClassLoader(); + } + + List> pluginClasses = extractClassesFromJARs(pluginJars, packagesToScan, cl); + List IMappingPluginList = createPluggableObjects(pluginClasses); + + for (IMappingPlugin i : IMappingPluginList) { + try { + i.setup(applicationProperties); + LOG.trace(" - Adding new plugin {}, v{} to available list", i.name(), i.version()); + result.put(i.id(), i); + } catch (PluginInitializationFailedException re) { + LOG.error("Failed to initialize plugin " + i.name() + ", version " + i.version() + ". Plugin will be ignored.", re); } } + return result; } - private static URL[] fileArrayToURLArray(File[] files) throws MalformedURLException { + private URL[] fileArrayToURLArray(File[] files) throws MalformedURLException { URL[] urls = new URL[files.length]; for (int i = 0; i < files.length; i++) { urls[i] = files[i].toURI().toURL(); @@ -91,17 +124,40 @@ private static URL[] fileArrayToURLArray(File[] files) throws MalformedURLExcept return urls; } - private static List> extractClassesFromJARs(File[] jars, ClassLoader cl) throws IOException, MappingPluginException { + private List> extractClassesFromJARs(File[] jars, String[] packagesToScan, ClassLoader cl) throws IOException, MappingPluginException { LOG.trace("Extracting classes from plugin JARs."); List> classes = new ArrayList<>(); - for (File jar : jars) { - LOG.trace("Processing file {}.", jar.getAbsolutePath()); - classes.addAll(PluginLoader.extractClassesFromJAR(jar, cl)); + if (jars != null) { + for (File jar : jars) { + LOG.trace("Processing file {}.", jar.getAbsolutePath()); + classes.addAll(extractClassesFromJAR(jar, cl)); + } } + LOG.trace("Found {} plugin classes in jar files.", classes.size()); + + if (packagesToScan != null) { + LOG.trace("Extracting classes from classpath."); + int pluginCnt = 0; + + findAllClasses("edu.kit.datamanager.mappingservice", cl); + + for (String pkg : packagesToScan) { + LOG.trace(" - Scanning package {}", pkg); + + List> result = findAllClasses(pkg, cl); + + for (Class res : result) { + classes.add((Class) res); + pluginCnt++; + } + } + LOG.trace("Found {} plugin classes in classpath.", pluginCnt); + } + return classes; } - private static List> extractClassesFromJAR(File jar, ClassLoader cl) throws IOException, MappingPluginException { + private List> extractClassesFromJAR(File jar, ClassLoader cl) throws IOException, MappingPluginException { LOG.trace("Extracting plugin classes from file {}.", jar.getAbsolutePath()); List> classes = new ArrayList<>(); try (JarInputStream jaris = new JarInputStream(new FileInputStream(jar))) { @@ -111,13 +167,13 @@ private static List> extractClassesFromJAR(File jar, Class try { Class cls = cl.loadClass(ent.getName().substring(0, ent.getName().length() - 6).replace('/', '.')); LOG.trace("Checking {}.", cls); - if (PluginLoader.isPluggableClass(cls)) { + if (isPluggableClass(cls)) { LOG.trace("Plugin class found."); classes.add((Class) cls); } } catch (ClassNotFoundException | NoClassDefFoundError e) { LOG.info("Can't load Class " + ent.getName()); - throw new MappingPluginException(MappingPluginState.UNKNOWN_ERROR, "Can't load Class " + ent.getName(), e); + throw new MappingPluginException(MappingPluginState.UNKNOWN_ERROR(), "Can't load Class " + ent.getName(), e); } } } @@ -125,19 +181,12 @@ private static List> extractClassesFromJAR(File jar, Class return classes; } - private static boolean isPluggableClass(Class cls) { - for (Class i : cls.getInterfaces()) { - LOG.trace("Checking {} against {}.", i, IMappingPlugin.class); - LOG.trace("ASSIGN {}", IMappingPlugin.class.isAssignableFrom(cls)); - if (i.equals(IMappingPlugin.class)) { - LOG.trace("IMappingPlugin interface found."); - return true; - } - } - return false; + private boolean isPluggableClass(Class cls) { + //this should be much easier and faster + return IMappingPlugin.class.isAssignableFrom(cls) && !cls.isInterface(); } - private static List createPluggableObjects(List> pluggable) throws MappingPluginException { + private List createPluggableObjects(List> pluggable) throws MappingPluginException { LOG.trace("Instantiating plugins from list: {}", pluggable); List plugs = new ArrayList<>(pluggable.size()); for (Class plug : pluggable) { @@ -146,12 +195,55 @@ private static List createPluggableObjects(List> findAllClasses(String packageName, ClassLoader loader) { + List> result = new ArrayList<>(); + MetadataReaderFactory metadataReaderFactory = new CachingMetadataReaderFactory( + loader); + try { + Resource[] resources = scan(loader, packageName); + for (Resource resource : resources) { + Class clazz = loadClass(loader, metadataReaderFactory, resource); + if (clazz != null && isPluggableClass(clazz)) { + result.add(clazz); + } + } + } catch (IOException ex) { + //throw new IllegalStateException(ex); + return result; + } + return result; + } + + private Resource[] scan(ClassLoader loader, String packageName) throws IOException { + ResourcePatternResolver resolver = new PathMatchingResourcePatternResolver( + loader); + String pattern = ResourcePatternResolver.CLASSPATH_ALL_URL_PREFIX + + ClassUtils.convertClassNameToResourcePath(packageName) + "/**/*.class"; + Resource[] resources = resolver.getResources(pattern); + return resources; + } + + private Class loadClass(ClassLoader loader, MetadataReaderFactory readerFactory, + Resource resource) { + try { + MetadataReader reader = readerFactory.getMetadataReader(resource); + return ClassUtils.forName(reader.getClassMetadata().getClassName(), loader); + } catch (ClassNotFoundException ex) { + return null; + } catch (LinkageError ex) { + return null; + } catch (Throwable ex) { + + return null; + } + } } diff --git a/src/main/java/edu/kit/datamanager/mappingservice/plugins/PluginManager.java b/src/main/java/edu/kit/datamanager/mappingservice/plugins/PluginManager.java index ce9e91d0..25d13b74 100644 --- a/src/main/java/edu/kit/datamanager/mappingservice/plugins/PluginManager.java +++ b/src/main/java/edu/kit/datamanager/mappingservice/plugins/PluginManager.java @@ -15,6 +15,9 @@ package edu.kit.datamanager.mappingservice.plugins; import edu.kit.datamanager.mappingservice.configuration.ApplicationProperties; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; +import edu.kit.datamanager.mappingservice.exception.MappingServiceException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,6 +50,8 @@ public class PluginManager { */ private final ApplicationProperties applicationProperties; + private final PluginLoader pluginLoader; + /** * Map of plugins. */ @@ -59,16 +64,19 @@ public class PluginManager { * instantiation time. */ @Autowired - public PluginManager(ApplicationProperties applicationProperties) { + public PluginManager(ApplicationProperties applicationProperties, PluginLoader pluginLoader, MeterRegistry meterRegistry) { this.applicationProperties = applicationProperties; + this.pluginLoader = pluginLoader; reloadPlugins(); + + Gauge.builder("mapping_service.plugins_total", () -> plugins.size()).register(meterRegistry); } /** * Unload all plugins and reload them from the configured plugin folder. */ public final void unload() { - PluginLoader.unload(); + pluginLoader.unload(); plugins.clear(); } @@ -78,7 +86,7 @@ public final void unload() { public final void reloadPlugins() { unload(); try { - plugins = PluginLoader.loadPlugins(Paths.get(applicationProperties.getPluginLocation().toURI()).toFile()); + plugins = pluginLoader.loadPlugins(Paths.get(applicationProperties.getPluginLocation().toURI()).toFile(), applicationProperties.getPackagesToScan()); } catch (URISyntaxException ex) { LOG.error("Mapping plugin location " + applicationProperties.getPluginLocation() + " cannot be converted to URI", ex); } catch (IOException ioe) { @@ -118,29 +126,33 @@ public final List listPluginIds() { * @param mappingFile Path to the mapping schema. * @param inputFile Path to the input file. * @param outputFile Path where the output is temporarily stored. + * * @return MappingPluginState.SUCCESS if the plugin was executed * successfully. + * * @throws MappingPluginException If there is an error with the plugin or * the input. */ - public final MappingPluginState mapFile(String pluginId, Path mappingFile, Path inputFile, Path outputFile) throws MappingPluginException { + public final MappingPluginState mapFile(String pluginId, Path mappingFile, Path inputFile, Path outputFile) throws MappingServiceException, MappingPluginException { + //The following issues should never happen as they are checked before. + //If they occur, it's a server fault, nothing a user can solve. if (pluginId == null) { - throw new MappingPluginException(MappingPluginState.INVALID_INPUT, "Plugin ID is null."); + throw new MappingServiceException("PluginId is null."); } if (mappingFile == null) { - throw new MappingPluginException(MappingPluginState.INVALID_INPUT, "Path to mapping schema is null."); + throw new MappingServiceException("Path to mapping file is null."); } if (inputFile == null) { - throw new MappingPluginException(MappingPluginState.INVALID_INPUT, "Path to input file is null."); + throw new MappingServiceException("Path to input file is null."); } if (outputFile == null) { - throw new MappingPluginException(MappingPluginState.INVALID_INPUT, "Path to output file is null."); + throw new MappingServiceException("Path to output file is null."); } if (plugins.containsKey(pluginId)) { LOG.trace("Plugin found. Performing mapFile({}, {}, {}).", mappingFile, inputFile, outputFile); return plugins.get(pluginId).mapFile(mappingFile, inputFile, outputFile); } - throw new MappingPluginException(MappingPluginState.NOT_FOUND, "Plugin '" + pluginId + "' not found!"); + throw new MappingPluginException(MappingPluginState.NOT_FOUND(), String.format("Plugin '%s' not found!", pluginId)); } } diff --git a/src/main/java/edu/kit/datamanager/mappingservice/plugins/impl/GemmaPlugin.java b/src/main/java/edu/kit/datamanager/mappingservice/plugins/impl/GemmaPlugin.java index 0724d2af..52ba8fa2 100644 --- a/src/main/java/edu/kit/datamanager/mappingservice/plugins/impl/GemmaPlugin.java +++ b/src/main/java/edu/kit/datamanager/mappingservice/plugins/impl/GemmaPlugin.java @@ -16,24 +16,16 @@ package edu.kit.datamanager.mappingservice.plugins.impl; import edu.kit.datamanager.mappingservice.plugins.*; -import edu.kit.datamanager.mappingservice.util.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.util.MimeType; import org.springframework.util.MimeTypeUtils; import java.nio.file.Path; -public class GemmaPlugin implements IMappingPlugin { +public class GemmaPlugin extends AbstractPythonMappingPlugin { - private final Logger LOGGER = LoggerFactory.getLogger(GemmaPlugin.class); private static final String GEMMA_REPOSITORY = "https://github.com/kit-data-manager/gemma.git"; - private static final String GEMMA_BRANCH = "master"; - private static Path gemmaDir; - private boolean initialized = false; - @Override - public String name() { - return "GEMMA"; + public GemmaPlugin() { + super("GEMMA", GEMMA_REPOSITORY); } @Override @@ -41,16 +33,6 @@ public String description() { return "GEMMA is a tool written in Python that allows to map from JSON and XML to JSON. Furthermore, it allows to map with a mapping schema."; } - @Override - public String version() { - return "1.0.0"; - } - - @Override - public String uri() { - return "https://github.com/kit-data-manager/gemma"; - } - @Override public MimeType[] inputTypes() { return new MimeType[]{MimeTypeUtils.APPLICATION_JSON, MimeTypeUtils.APPLICATION_XML}; @@ -62,26 +44,12 @@ public MimeType[] outputTypes() { } @Override - public void setup() { - LOGGER.info("Checking and installing dependencies for Gemma: gemma, xmltodict, wget"); - try { - PythonRunnerUtil.runPythonScript("-m", "pip", "install", "xmltodict", "wget"); - PythonRunnerUtil.runPythonScript("-m", new LoggerOutputStream(LOGGER, LoggerOutputStream.Level.DEBUG), new LoggerOutputStream(LOGGER, LoggerOutputStream.Level.DEBUG), "pip", "install", "xmltodict", "wget"); - gemmaDir = FileUtil.cloneGitRepository(GEMMA_REPOSITORY, GEMMA_BRANCH); - initialized = true; - } catch (MappingPluginException e) { - LOGGER.error("Failed to setup plugin '" + name() + "' " + version() + ".", e); - } - } - - @Override - public MappingPluginState mapFile(Path mappingFile, Path inputFile, Path outputFile) throws MappingPluginException { - if (initialized) { - LOGGER.trace("Run gemma on '{}' with mapping '{}' -> '{}'", inputFile, mappingFile, outputFile); - return PythonRunnerUtil.runPythonScript(gemmaDir + "/mapping_single.py", mappingFile.toString(), inputFile.toString(), outputFile.toString()); - } else { - LOGGER.error("Plugin '" + name() + "' " + version() + " not initialized. Returning EXECUTION_ERROR."); - return MappingPluginState.EXECUTION_ERROR; - } + public String[] getCommandArray(Path workingDir, Path mappingFile, Path inputFile, Path outputFile) { + return new String[]{ + workingDir + "/mapping_single.py", + mappingFile.toString(), + inputFile.toString(), + outputFile.toString() + }; } } diff --git a/src/main/java/edu/kit/datamanager/mappingservice/plugins/impl/IdentifyPlugin.java b/src/main/java/edu/kit/datamanager/mappingservice/plugins/impl/IdentifyPlugin.java index 204acff9..d0b315e0 100644 --- a/src/main/java/edu/kit/datamanager/mappingservice/plugins/impl/IdentifyPlugin.java +++ b/src/main/java/edu/kit/datamanager/mappingservice/plugins/impl/IdentifyPlugin.java @@ -15,6 +15,8 @@ */ package edu.kit.datamanager.mappingservice.plugins.impl; +import edu.kit.datamanager.mappingservice.configuration.ApplicationProperties; +import edu.kit.datamanager.mappingservice.exception.PluginInitializationFailedException; import edu.kit.datamanager.mappingservice.plugins.IMappingPlugin; import edu.kit.datamanager.mappingservice.plugins.MappingPluginException; import edu.kit.datamanager.mappingservice.plugins.MappingPluginState; @@ -61,33 +63,37 @@ public String uri() { @Override public MimeType[] inputTypes() { - return new MimeType[]{MimeType.valueOf("application/octet-stream")}; + return new MimeType[]{MimeType.valueOf("image/*")}; } @Override public MimeType[] outputTypes() { - return new MimeType[]{MimeType.valueOf("application/octet-stream")}; + return new MimeType[]{MimeType.valueOf("application/*")}; } @Override - public void setup() { + public void setup(ApplicationProperties applicationProperties) { if (Paths.get("/usr/bin/identify").toFile().exists()) { initialized = true; + } else { + throw new PluginInitializationFailedException("Required executable /usr/bin/identify not found."); } } @Override public MappingPluginState mapFile(Path mappingFile, Path inputFile, Path outputFile) throws MappingPluginException { + MappingPluginState result; try { - FileOutputStream fout = new FileOutputStream(outputFile.toFile()); - ShellRunnerUtil.run(fout, fout, "/usr/bin/identify", "-verbose", inputFile.toAbsolutePath().toString()); - fout.flush(); - fout.close(); + try (FileOutputStream fout = new FileOutputStream(outputFile.toFile())) { + result = ShellRunnerUtil.run(fout, fout, "/usr/bin/identify", "-verbose", inputFile.toAbsolutePath().toString()); + fout.flush(); + } } catch (IOException ex) { LOG.error("Failed to execute plugin.", ex); - return MappingPluginState.EXECUTION_ERROR; + result = MappingPluginState.EXECUTION_ERROR(); + result.setDetails(ex.getMessage()); } - return MappingPluginState.SUCCESS; + return result; } } diff --git a/src/main/java/edu/kit/datamanager/mappingservice/plugins/impl/TestPlugin.java b/src/main/java/edu/kit/datamanager/mappingservice/plugins/impl/InOutPlugin.java similarity index 72% rename from src/main/java/edu/kit/datamanager/mappingservice/plugins/impl/TestPlugin.java rename to src/main/java/edu/kit/datamanager/mappingservice/plugins/impl/InOutPlugin.java index 609d1038..23e5d050 100644 --- a/src/main/java/edu/kit/datamanager/mappingservice/plugins/impl/TestPlugin.java +++ b/src/main/java/edu/kit/datamanager/mappingservice/plugins/impl/InOutPlugin.java @@ -15,6 +15,7 @@ */ package edu.kit.datamanager.mappingservice.plugins.impl; +import edu.kit.datamanager.mappingservice.configuration.ApplicationProperties; import edu.kit.datamanager.mappingservice.exception.MappingException; import edu.kit.datamanager.mappingservice.plugins.IMappingPlugin; import edu.kit.datamanager.mappingservice.plugins.MappingPluginException; @@ -31,23 +32,23 @@ * * @author jejkal */ -public class TestPlugin implements IMappingPlugin { +public class InOutPlugin implements IMappingPlugin { - static Logger LOG = LoggerFactory.getLogger(TestPlugin.class); + static Logger LOG = LoggerFactory.getLogger(InOutPlugin.class); @Override public String name() { - return "TestPlugin"; + return "InOutPlugin"; } @Override public String description() { - return "Simple plugin for testing."; + return "Simple plugin for testing just returning the input file."; } @Override public String version() { - return "1.0.0"; + return "1.1.2"; } @Override @@ -57,29 +58,31 @@ public String uri() { @Override public MimeType[] inputTypes() { - return new MimeType[]{MimeType.valueOf("application/octet-stream")}; + return new MimeType[]{MimeType.valueOf("application/*")}; } @Override public MimeType[] outputTypes() { - return new MimeType[]{MimeType.valueOf("application/octet-stream")}; + return new MimeType[]{MimeType.valueOf("application/*")}; } @Override - public void setup() { + public void setup(ApplicationProperties applicationProperties) { //nothing to do here LOG.trace("Plugin {} {} successfully set up.", name(), version()); } @Override public MappingPluginState mapFile(Path mappingFile, Path inputFile, Path outputFile) throws MappingPluginException { + MappingPluginState result = MappingPluginState.SUCCESS(); try { Files.copy(inputFile, outputFile, StandardCopyOption.REPLACE_EXISTING); } catch (IOException | MappingException ex) { LOG.error("Failed to execute plugin.", ex); - return MappingPluginState.EXECUTION_ERROR; + result = MappingPluginState.EXECUTION_ERROR(); + result.setDetails("Failed to copy input to output, probably due to an I/O error."); } - return MappingPluginState.SUCCESS; + return result; } } diff --git a/src/main/java/edu/kit/datamanager/mappingservice/rest/PluginInformation.java b/src/main/java/edu/kit/datamanager/mappingservice/rest/PluginInformation.java index 12e92225..9c9d635e 100644 --- a/src/main/java/edu/kit/datamanager/mappingservice/rest/PluginInformation.java +++ b/src/main/java/edu/kit/datamanager/mappingservice/rest/PluginInformation.java @@ -99,7 +99,7 @@ public PluginInformation(String id, PluginManager manager) throws MappingPluginE Arrays.stream(p.outputTypes()).toList().forEach(mimeType -> outputTypesList.add(mimeType.toString())); this.outputTypes = outputTypesList.toArray(new String[0]); } else { - throw new MappingPluginException(MappingPluginState.NOT_FOUND, "Plugin with id " + id + " not found."); + throw new MappingPluginException(MappingPluginState.NOT_FOUND(), "Plugin with id " + id + " not found."); } } diff --git a/src/main/java/edu/kit/datamanager/mappingservice/rest/impl/MappingAdministrationController.java b/src/main/java/edu/kit/datamanager/mappingservice/rest/impl/MappingAdministrationController.java index cdd7ae98..25d01da3 100644 --- a/src/main/java/edu/kit/datamanager/mappingservice/rest/impl/MappingAdministrationController.java +++ b/src/main/java/edu/kit/datamanager/mappingservice/rest/impl/MappingAdministrationController.java @@ -15,6 +15,7 @@ */ package edu.kit.datamanager.mappingservice.rest.impl; +import com.google.common.base.Strings; import edu.kit.datamanager.entities.PERMISSION; import edu.kit.datamanager.mappingservice.dao.IMappingRecordDao; import edu.kit.datamanager.mappingservice.domain.MappingRecord; @@ -27,6 +28,8 @@ import edu.kit.datamanager.mappingservice.rest.PluginInformation; import edu.kit.datamanager.util.AuthenticationHelper; import edu.kit.datamanager.util.ControllerUtils; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; import io.swagger.v3.core.util.Json; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,7 +59,6 @@ import java.util.List; import java.util.Optional; - /** * Controller for managing mapping files. * @@ -86,10 +88,12 @@ public class MappingAdministrationController implements IMappingAdministrationCo */ private final MappingService mappingService; - public MappingAdministrationController(IMappingRecordDao mappingRecordDao, PluginManager pluginManager, MappingService mappingService) { + public MappingAdministrationController(IMappingRecordDao mappingRecordDao, PluginManager pluginManager, MappingService mappingService, MeterRegistry meterRegistry) { this.mappingRecordDao = mappingRecordDao; this.mappingService = mappingService; this.pluginManager = pluginManager; + + Gauge.builder("mapping_service.schemes_total", mappingRecordDao::count).register(meterRegistry); } @Override @@ -109,7 +113,11 @@ public ResponseEntity createMapping( } catch (IOException ex) { LOG.error("Unable to deserialize mapping record.", ex); return ResponseEntity.status(HttpStatus.BAD_REQUEST).build(); -//.body("Unable to deserialize provided mapping record."); + } + + if (Strings.isNullOrEmpty(mappingRecord.getMappingId()) || Strings.isNullOrEmpty(mappingRecord.getMappingType())) { + LOG.error("Invalid mapping record. Either mappingId or mappingType are null."); + return ResponseEntity.status(HttpStatus.BAD_REQUEST).build(); } LOG.trace("Obtaining caller principle for authorization purposes."); @@ -145,8 +153,7 @@ public ResponseEntity createMapping( mappingRecord = mappingService.createMapping(contentOfFile, mappingRecord); } catch (IOException ioe) { LOG.error("Unable to create mapping for provided inputs.", ioe); - //return ResponseEntity.internalServerError().body("Unable to create mapping for provided inputs."); - return ResponseEntity.status(HttpStatus.BAD_REQUEST).build(); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); } LOG.trace("Mapping successfully persisted. Updating document URI."); diff --git a/src/main/java/edu/kit/datamanager/mappingservice/rest/impl/MappingExecutionController.java b/src/main/java/edu/kit/datamanager/mappingservice/rest/impl/MappingExecutionController.java index ba68bcaa..d290019a 100644 --- a/src/main/java/edu/kit/datamanager/mappingservice/rest/impl/MappingExecutionController.java +++ b/src/main/java/edu/kit/datamanager/mappingservice/rest/impl/MappingExecutionController.java @@ -18,17 +18,23 @@ import edu.kit.datamanager.mappingservice.dao.IMappingRecordDao; import edu.kit.datamanager.mappingservice.domain.JobStatus; import edu.kit.datamanager.mappingservice.domain.MappingRecord; +import edu.kit.datamanager.mappingservice.exception.JobIdConflictException; import edu.kit.datamanager.mappingservice.exception.JobProcessingException; import edu.kit.datamanager.mappingservice.exception.MappingException; import edu.kit.datamanager.mappingservice.exception.MappingExecutionException; import edu.kit.datamanager.mappingservice.exception.MappingJobException; import edu.kit.datamanager.mappingservice.exception.MappingNotFoundException; +import edu.kit.datamanager.mappingservice.exception.MappingServiceException; +import edu.kit.datamanager.mappingservice.exception.MappingServiceUserException; import edu.kit.datamanager.mappingservice.impl.JobManager; import edu.kit.datamanager.mappingservice.impl.MappingService; import edu.kit.datamanager.mappingservice.plugins.MappingPluginException; import edu.kit.datamanager.mappingservice.plugins.MappingPluginState; import edu.kit.datamanager.mappingservice.rest.IMappingExecutionController; import edu.kit.datamanager.mappingservice.util.FileUtil; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.DistributionSummary; +import io.micrometer.core.instrument.MeterRegistry; import org.apache.commons.io.FilenameUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,7 +55,6 @@ import java.util.Optional; import java.util.UUID; import java.util.concurrent.CompletableFuture; -import org.apache.tomcat.util.file.Matcher; import org.springframework.core.io.InputStreamResource; import org.springframework.core.io.Resource; import org.springframework.http.MediaType; @@ -70,18 +75,24 @@ public class MappingExecutionController implements IMappingExecutionController { private final MappingService mappingService; protected JobManager jobManager; private final IMappingRecordDao mappingRecordDao; + private final MeterRegistry meterRegistry; + private final DistributionSummary documentsInSizeMetric; + private final DistributionSummary documentsOutSizeMetric; - public MappingExecutionController(MappingService mappingService, IMappingRecordDao mappingRecordDao, JobManager jobManager) { + public MappingExecutionController(MappingService mappingService, IMappingRecordDao mappingRecordDao, JobManager jobManager, MeterRegistry meterRegistry) { this.mappingService = mappingService; this.mappingRecordDao = mappingRecordDao; this.jobManager = jobManager; + this.meterRegistry = meterRegistry; + this.documentsInSizeMetric = DistributionSummary.builder("mapping_service.documents.input_size").baseUnit("bytes").register(meterRegistry); + this.documentsOutSizeMetric = DistributionSummary.builder("mapping_service.documents.output_size").baseUnit("bytes").register(meterRegistry); } @Override public void mapDocument(MultipartFile document, String mappingID, HttpServletRequest request, HttpServletResponse response, UriComponentsBuilder uriBuilder) { LOG.trace("Performing mapDocument(File#{}, {})", document.getOriginalFilename(), mappingID); - Optional resultPath; + Optional resultPath = Optional.empty(); if (!document.isEmpty() && !mappingID.isBlank()) { LOG.trace("Obtaining mapping for id {}.", mappingID); Optional record = mappingRecordDao.findByMappingId(mappingID); @@ -89,35 +100,35 @@ public void mapDocument(MultipartFile document, String mappingID, HttpServletReq String message = String.format("No mapping found for mapping id %s.", mappingID); LOG.error(message + " Returning HTTP 404."); throw new MappingNotFoundException(message); - //return ResponseEntity.status(HttpStatus.NOT_FOUND).body(message); } - LOG.trace("Receiving mapping input file."); + LOG.trace("Processing mapping input file."); String extension = "." + FilenameUtils.getExtension(document.getOriginalFilename()); - LOG.trace("Found file extension: {}", extension); + LOG.trace(" - Determined file extension: {}", extension); Path inputPath = FileUtil.createTempFile("inputMultipart", extension); - LOG.trace("Writing user upload to: {}", inputPath); + LOG.trace(" - Writing user upload to: {}", inputPath); File inputFile = inputPath.toFile(); try { document.transferTo(inputFile); - LOG.trace("Successfully received user upload."); + LOG.trace("Successfully stored user upload at {}.", inputPath); } catch (IOException e) { - LOG.error("Failed to receive upload from user.", e); + LOG.error("Failed to store user upload.", e); throw new MappingExecutionException("Unable to write user upload to disk."); } try { - LOG.trace("Performing mapping process of file {} via mapping service", inputPath.toString()); + LOG.trace("Performing mapping process of file {} via mapping service", inputPath); resultPath = mappingService.executeMapping(inputFile.toURI(), mappingID); if (resultPath.isPresent()) { LOG.trace("Mapping process finished. Output written to {}.", resultPath.toString()); } else { - throw new MappingPluginException(MappingPluginState.UNKNOWN_ERROR, "Mapping process finished, but no result was returned."); + throw new MappingPluginException(MappingPluginState.UNKNOWN_ERROR(), "Mapping process finished, but no result was returned."); } } catch (MappingPluginException e) { LOG.error("Failed to execute mapping.", e); - throw new MappingExecutionException("Failed to execute mapping with id " + mappingID + " on provided input document."); + e.throwMe(); + //throw new MappingExecutionException("Failed to execute mapping with id " + mappingID + " on provided input document."); } finally { LOG.trace("Removing user upload at {}.", inputFile); FileUtil.removeFile(inputPath); @@ -126,24 +137,24 @@ public void mapDocument(MultipartFile document, String mappingID, HttpServletReq } else { String message = "Either mapping id or input document are missing. Unable to perform mapping."; LOG.error(message); - throw new MappingException(message); + throw new MappingServiceUserException(message); } + Path result = resultPath.get(); if (!Files.exists(result) || !Files.isRegularFile(result) || !Files.isReadable(result)) { String message = "The mapping result expected at path " + result + " is not accessible. This indicates an error of the mapper implementation."; LOG.error(message); - throw new MappingExecutionException(message); + throw new MappingServiceException(message); } - LOG.trace("Determining mime type for mapping result."); + LOG.trace("Determining mime type for mapping result {}.", result); result = FileUtil.fixFileExtension(result); String mimeType = FileUtil.getMimeType(result); - LOG.trace("Determining file extension for mapping result."); + LOG.trace("Mime type {} determined. Identifying file extension.", mimeType); String extension = FileUtil.getExtensionForMimeType(mimeType); - LOG.trace("Using mime type {} and extension {}.", mimeType, extension); - + LOG.trace("Returning result using mime type {} and file extension {}.", mimeType, extension); response.setStatus(HttpStatus.OK.value()); response.setHeader("Content-Type", mimeType); response.setHeader(HttpHeaders.CONTENT_LENGTH, Long.toString(result.toFile().length())); @@ -152,13 +163,17 @@ public void mapDocument(MultipartFile document, String mappingID, HttpServletReq response.setHeader("Expires", "0"); response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment;" + "filename=result" + extension); try { + LOG.trace("Writing file to response output stream."); Files.copy(result, response.getOutputStream()); - } catch (IOException ex) { String message = "Failed to write mapping result file to stream."; LOG.error(message, ex); - throw new MappingExecutionException(message); + throw new MappingServiceException(message); } finally { + Counter.builder("mapping_service.mapping_usage").tag("mappingID", mappingID).register(meterRegistry).increment(); + this.documentsInSizeMetric.record(document.getSize()); + this.documentsOutSizeMetric.record(result.toFile().length()); + LOG.trace("Result file successfully transferred to client. Removing file {} from disk.", result); try { Files.delete(result); @@ -171,14 +186,21 @@ public void mapDocument(MultipartFile document, String mappingID, HttpServletReq @Override public ResponseEntity scheduleMapDocument(String mappingID, MultipartFile document, HttpServletRequest request, HttpServletResponse response, UriComponentsBuilder uriBuilder) throws Throwable { - LOG.trace("Performing mapDocument(File#{}, {})", document.getOriginalFilename(), mappingID); - String jobId = UUID.randomUUID().toString(); + LOG.trace("Performing scheduleMapDocument(File#{}, {})", document.getOriginalFilename(), mappingID); + String jobId = null; + for (int i = 1; i < 4; i++) { + jobId = UUID.randomUUID().toString(); + if (jobManager.getJob(jobId) == null) { + jobId = null; + } + LOG.trace("Duplicated job id detected. Attempt {}/{}", i, 3); + } - if (null != jobManager.getJob(jobId)) { - throw new JobProcessingException("JobId conflict, please retry again.", true); + if (jobId == null) { + throw new JobIdConflictException(); } - LOG.info("Generated job-id {} for this request.", jobId); + LOG.info("Generated job id {} for this request.", jobId); if (!document.isEmpty() && !mappingID.isBlank()) { LOG.trace("Obtaining mapping for id {}.", mappingID); Optional record = mappingRecordDao.findByMappingId(mappingID); @@ -186,7 +208,6 @@ public ResponseEntity scheduleMapDocument(String mappingID, Multipart String message = String.format("No mapping found for mapping id %s.", mappingID); LOG.error(message + " Returning HTTP 404."); throw new MappingNotFoundException(message); - //return ResponseEntity.status(HttpStatus.NOT_FOUND).body(message); } LOG.trace("Receiving mapping input file."); @@ -200,31 +221,27 @@ public ResponseEntity scheduleMapDocument(String mappingID, Multipart LOG.trace("Successfully received user upload."); } catch (IOException e) { LOG.error("Failed to receive upload from user.", e); - throw new JobProcessingException("Unable to write user upload to disk."); + throw new MappingExecutionException("Unable to write user upload to disk."); } try { LOG.trace("Scheduling mapping process of file {} via mapping service", inputPath.toString()); CompletableFuture completableFuture = mappingService.executeMappingAsync(jobId, inputFile.toURI(), mappingID); jobManager.putJob(jobId, completableFuture); - LOG.info("Job-id {} submitted for processing. Returning from controller.", jobId); + LOG.info("Job id {} scheduled for processing. Returning job status.", jobId); return ResponseEntity.ok(JobStatus.status(jobId, JobStatus.STATUS.SUBMITTED)); } catch (MappingPluginException e) { LOG.error("Failed to execute mapping.", e); + return ResponseEntity.status(500).body(JobStatus.error(jobId, JobStatus.STATUS.FAILED, String.format("Failed to schedule mapping with id '%s' on provided input document.", mappingID))); + } finally { LOG.trace("Removing user upload at {}.", inputFile); FileUtil.removeFile(inputPath); LOG.trace("User upload successfully removed."); - return ResponseEntity.status(500).body(JobStatus.error(jobId, JobStatus.STATUS.FAILED, "Failed to execute mapping with id " + mappingID + " on provided input document.")); } - /*finally { - LOG.trace("Removing user upload at {}.", inputFile); - FileUtil.removeFile(inputPath); - LOG.trace("User upload successfully removed."); - }*/ } else { String message = "Either mapping id or input document are missing. Unable to perform mapping."; LOG.error(message); - throw new JobProcessingException(message); + throw new MappingServiceUserException(message); } } diff --git a/src/main/java/edu/kit/datamanager/mappingservice/rest/impl/PreHandleInterceptor.java b/src/main/java/edu/kit/datamanager/mappingservice/rest/impl/PreHandleInterceptor.java new file mode 100644 index 00000000..37dd3906 --- /dev/null +++ b/src/main/java/edu/kit/datamanager/mappingservice/rest/impl/PreHandleInterceptor.java @@ -0,0 +1,59 @@ +package edu.kit.datamanager.mappingservice.rest.impl; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.lang.Nullable; +import org.springframework.stereotype.Service; +import org.springframework.web.servlet.HandlerInterceptor; + +import java.security.MessageDigest; +import java.util.HashSet; + +@Service +public class PreHandleInterceptor implements HandlerInterceptor { + private final HashSet uniqueUsers = new HashSet<>(); + private final Counter counter; + + /** + * Logger for this class. + */ + private final static Logger LOGGER = LoggerFactory.getLogger(PreHandleInterceptor.class); + + @Autowired + PreHandleInterceptor(MeterRegistry meterRegistry) { + Gauge.builder("mapping_service.unique_users", uniqueUsers::size).register(meterRegistry); + counter = Counter.builder("mapping_service.requests_served").register(meterRegistry); + } + + @Override + public boolean preHandle(HttpServletRequest request, @Nullable HttpServletResponse response, @Nullable Object handler) throws Exception { + String forwardedFor = request.getHeader("X-Forwarded-For"); + LOGGER.debug("X-Forwarded-For: {}", forwardedFor); + String clientIp = null; + + if (forwardedFor != null) { + String[] ipList = forwardedFor.split(", "); + if (ipList.length > 0) clientIp = ipList[0]; + LOGGER.debug("Client IP from X-Forwarded-For: {}", clientIp); + } + + String remoteIp = request.getRemoteAddr(); + LOGGER.debug("Client IP from getRemoteAddr: {}", remoteIp); + String ip = clientIp == null ? remoteIp : clientIp; + LOGGER.debug("Using {} for monitoring", ip); + + MessageDigest messageDigest = MessageDigest.getInstance("SHA-256"); + messageDigest.update(ip.getBytes()); + uniqueUsers.add(new String(messageDigest.digest())); + + counter.increment(); + + return true; + } +} diff --git a/src/main/java/edu/kit/datamanager/mappingservice/util/FileUtil.java b/src/main/java/edu/kit/datamanager/mappingservice/util/FileUtil.java index 8eb4a69d..e29796a8 100644 --- a/src/main/java/edu/kit/datamanager/mappingservice/util/FileUtil.java +++ b/src/main/java/edu/kit/datamanager/mappingservice/util/FileUtil.java @@ -17,6 +17,7 @@ import edu.kit.datamanager.clients.SimpleServiceClient; import edu.kit.datamanager.mappingservice.exception.MappingException; +import edu.kit.datamanager.mappingservice.exception.MappingServiceException; import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; import org.eclipse.jgit.api.Git; @@ -259,7 +260,7 @@ private static String guessFileExtension(String filename, byte[] fewKilobytesOfF returnValue = ".xml"; } } - + if (returnValue == null) { // Use tika library to estimate extension LOGGER.trace("Use tika library to estimate extension."); @@ -280,20 +281,10 @@ private static String guessFileExtension(String filename, byte[] fewKilobytesOfF } /** - * This method clones a git repository into the 'lib' folder. - * - * @param repositoryUrl the url of the repository to clone - * @param branch the branch to clone - * @return the path to the cloned repository - */ - public static Path cloneGitRepository(String repositoryUrl, String branch) { - String target = "lib/" + repositoryUrl.trim().replace("https://", "").replace("http://", "").replace(".git", "") + "_" + branch; - return cloneGitRepository(repositoryUrl, branch, target); - } - - /** - * This method clones a git repository into the 'lib' folder. If the folder - * already exists, a pull is performed. + * This method clones a git repository into the provided target folder. If + * the folder already exists, a pull is performed, otherwise it is created + * before. Typically, the target folder should be takes from property + * 'mapping-service.codeLocation' obtained from ApplicationProperties. * * @param repositoryUrl the url of the repository to clone * @param branch the branch to clone @@ -303,21 +294,34 @@ public static Path cloneGitRepository(String repositoryUrl, String branch) { public static Path cloneGitRepository(String repositoryUrl, String branch, String targetFolder) { File target = new File(targetFolder); if (target.exists()) { + Git g = null; try { - Git.open(target).pull().call(); - } catch (IOException | JGitInternalException | GitAPIException e) { - LOGGER.error("Error pulling git repository at '" + target + "'!", e); - throw new MappingException("Error pulling git repository at '" + target + "'!", e); + g = Git.open(target); + LOGGER.trace("Repository already exists at {}. Active branch is: {}", target, g.getRepository().getBranch()); + } catch (IOException e) { + String message = String.format("Folder '%s' already exists but contains not Git repository.", target); + LOGGER.error(message, e); + throw new MappingServiceException("Failed to prepare plugin. Plugin code destination already exists but is empty."); + } finally { + if (g != null) { + g.getRepository().close(); + } } } else { target.mkdirs(); LOGGER.info("Cloning branch '{}' of repository '{}' to '{}'", branch, repositoryUrl, target.getPath()); + Git g = null; try { - Git.cloneRepository().setURI(repositoryUrl).setBranch(branch).setDirectory(target).call(); + g = Git.cloneRepository().setURI(repositoryUrl).setBranch(branch).setDirectory(target).call(); + LOGGER.trace("Repository successfully cloned to {}.", target); } catch (JGitInternalException | GitAPIException e) { LOGGER.error("Error cloning git repository '" + repositoryUrl + "' to '" + target + "'!", e); - throw new MappingException("Error cloning git repository '" + repositoryUrl + "' to '" + target + "'!", e); + throw new MappingServiceException("Failed to prepare plugin. Plugin code destination not accessible."); + } finally { + if (g != null) { + g.getRepository().close(); + } } } return target.toPath(); diff --git a/src/main/java/edu/kit/datamanager/mappingservice/util/PythonRunnerUtil.java b/src/main/java/edu/kit/datamanager/mappingservice/util/PythonRunnerUtil.java index a8a52924..c515f90b 100644 --- a/src/main/java/edu/kit/datamanager/mappingservice/util/PythonRunnerUtil.java +++ b/src/main/java/edu/kit/datamanager/mappingservice/util/PythonRunnerUtil.java @@ -12,7 +12,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package edu.kit.datamanager.mappingservice.util; import edu.kit.datamanager.mappingservice.configuration.ApplicationProperties; @@ -20,7 +19,6 @@ import edu.kit.datamanager.mappingservice.plugins.MappingPluginState; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import java.io.OutputStream; @@ -34,27 +32,27 @@ */ @Component public class PythonRunnerUtil { + private static ApplicationProperties configuration; private static final Logger LOGGER = LoggerFactory.getLogger(PythonRunnerUtil.class); - @Autowired - public PythonRunnerUtil(ApplicationProperties configuration) { + public static void init(ApplicationProperties configuration) { PythonRunnerUtil.configuration = configuration; } /** - * This method prints the python version to Log.info. + * This method prints the python version to System.out. */ public static void printPythonVersion() { try { - LOGGER.info("Configured Python version:"); - PythonRunnerUtil.runPythonScript("--version", new LoggerOutputStream(LOGGER, LoggerOutputStream.Level.INFO), new LoggerOutputStream(LOGGER, LoggerOutputStream.Level.INFO)); + PythonRunnerUtil.runPythonScript("--version", System.out, System.err); } catch (MappingPluginException e) { - e.printStackTrace(); + LOGGER.error("Failed to obtain python version.", e); } } + /** * This method executes an argument/option on the python interpreter. * @@ -70,31 +68,39 @@ public static MappingPluginState runPythonScript(String arg) throws MappingPlugi * This method executes a python script with the given arguments. * * @param script path to the python script to be executed. - * @param args arguments to be passed to the script. + * @param args arguments to be passed to the script. * @return State of the execution. * @throws MappingPluginException if an error occurs. */ public static MappingPluginState runPythonScript(String script, String... args) throws MappingPluginException { - return runPythonScript(script, new LoggerOutputStream(LOGGER, LoggerOutputStream.Level.DEBUG), new LoggerOutputStream(LOGGER, LoggerOutputStream.Level.INFO), args); + return runPythonScript(script, new LoggerOutputStream(LOGGER, LoggerOutputStream.Level.DEBUG), new LoggerOutputStream(LOGGER, LoggerOutputStream.Level.ERROR), args); } /** - * This method executes a python script with the given arguments and redirects the output and errors to the given streams. + * This method executes a python script with the given arguments and + * redirects the output and errors to the given streams. * * @param script path to the python script to be executed. * @param output OutputStream to redirect the output to. - * @param error OutputStream to redirect the errors to. - * @param args arguments to be passed to the script. - * @return State of the execution. + * @param error OutputStream to redirect the errors to. + * @param args arguments to be passed to the script. + * + * @return State of the execution if execution succeeds. Otherwise, a + * MappingPluginException is thrown. + * * @throws MappingPluginException if an error occurs. */ public static MappingPluginState runPythonScript(String script, OutputStream output, OutputStream error, String... args) throws MappingPluginException { - if (configuration == null || configuration.getPythonExecutable()== null) return MappingPluginState.UNKNOWN_ERROR; + if (configuration == null || !configuration.isPythonAvailable()) { + throw new MappingPluginException(MappingPluginState.UNKNOWN_ERROR(), "No Python runtime configured."); + } ArrayList command = new ArrayList<>(); command.add(configuration.getPythonExecutable().getPath()); command.add(script); - Collections.addAll(command, args); - ShellRunnerUtil.run(output, error, command.toArray(new String[0])); - return MappingPluginState.SUCCESS; + + if (args != null) { + Collections.addAll(command, args); + } + return ShellRunnerUtil.run(output, error, command.toArray(String[]::new)); } } diff --git a/src/main/java/edu/kit/datamanager/mappingservice/util/ShellRunnerUtil.java b/src/main/java/edu/kit/datamanager/mappingservice/util/ShellRunnerUtil.java index 4ea28e76..45303408 100644 --- a/src/main/java/edu/kit/datamanager/mappingservice/util/ShellRunnerUtil.java +++ b/src/main/java/edu/kit/datamanager/mappingservice/util/ShellRunnerUtil.java @@ -12,21 +12,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package edu.kit.datamanager.mappingservice.util; import edu.kit.datamanager.mappingservice.configuration.ApplicationProperties; +import edu.kit.datamanager.mappingservice.exception.BadExitCodeException; import edu.kit.datamanager.mappingservice.plugins.MappingPluginException; import edu.kit.datamanager.mappingservice.plugins.MappingPluginState; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.*; -import java.util.List; +import java.util.Scanner; import java.util.concurrent.*; -import java.util.stream.Collectors; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; /** * Utility class for running shell scripts. @@ -35,16 +32,17 @@ */ public class ShellRunnerUtil { - /** - * Time in seconds when the script should throw a timeout exception. - */ - public static final int TIMEOUT = 30; + private static ApplicationProperties configuration; /** * Logger for this class. */ private final static Logger LOGGER = LoggerFactory.getLogger(ShellRunnerUtil.class); + public static void init(ApplicationProperties configuration) { + ShellRunnerUtil.configuration = configuration; + } + /** * This method executes a shell command. * @@ -53,14 +51,15 @@ public class ShellRunnerUtil { * @throws MappingPluginException If an error occurs. */ public static MappingPluginState run(String... command) throws MappingPluginException { - return run(TIMEOUT, command); + return run(configuration.getExecutionTimeout(), command); } /** * This method executes a shell command. * - * @param timeOutInSeconds Time in seconds when the script should throw a timeout exception. - * @param command The command to execute without spaces. + * @param timeOutInSeconds Time in seconds when the script should throw a + * timeout exception. + * @param command The command to execute without spaces. * @return State of the execution. * @throws MappingPluginException If an error occurs. */ @@ -69,87 +68,96 @@ public static MappingPluginState run(int timeOutInSeconds, String... command) th } /** - * This method executes a shell command and writes the output and errors to the given streams. + * This method executes a shell command and writes the output and errors to + * the given streams. * - * @param output OutputStream to redirect the output to. - * @param error OutputStream to redirect the errors to. + * @param output OutputStream to redirect the output to. + * @param error OutputStream to redirect the errors to. * @param command The command to execute without spaces. * @return State of the execution. * @throws MappingPluginException If an error occurs. */ public static MappingPluginState run(OutputStream output, OutputStream error, String... command) throws MappingPluginException { - return run(output, error, TIMEOUT, command); + return run(output, error, configuration.getExecutionTimeout(), command); } - /** - * This method executes a shell command and writes the output and errors to the given streams. + * This method executes a shell command and writes the output and errors to + * the given streams. + * + * @param output OutputStream to redirect the output to. + * @param error OutputStream to redirect the errors to. + * @param timeOutInSeconds Time in seconds when the script should throw a + * timeout exception. + * @param command The command to execute without spaces. + * + * @return State of the execution only if execution was successful. + * Otherwise, MappingPluginException is thrown. * - * @param output OutputStream to redirect the output to. - * @param error OutputStream to redirect the errors to. - * @param timeOutInSeconds Time in seconds when the script should throw a timeout exception. - * @param command The command to execute without spaces. - * @return State of the execution. * @throws MappingPluginException If an error occurs. */ public static MappingPluginState run(OutputStream output, OutputStream error, int timeOutInSeconds, String... command) throws MappingPluginException { - if (output == null) throw new MappingPluginException(MappingPluginState.INVALID_INPUT, "Output stream is null."); - if (error == null) throw new MappingPluginException(MappingPluginState.INVALID_INPUT, "Error stream is null."); - if (timeOutInSeconds <= 0) throw new MappingPluginException(MappingPluginState.INVALID_INPUT, "Timeout is null or negative."); - if (command == null || command.length == 0) throw new MappingPluginException(MappingPluginState.INVALID_INPUT, "No command given."); + if (output == null) { + throw new MappingPluginException(MappingPluginState.INVALID_INPUT(), "Output stream is null."); + } + if (error == null) { + throw new MappingPluginException(MappingPluginState.INVALID_INPUT(), "Error stream is null."); + } + if (timeOutInSeconds <= 0) { + throw new MappingPluginException(MappingPluginState.INVALID_INPUT(), "Execution timeout is leq 0."); + } + if (command == null || command.length == 0) { + throw new MappingPluginException(MappingPluginState.INVALID_INPUT(), "No command given."); + } ExecutorService pool = Executors.newSingleThreadExecutor(); - int result; - MappingPluginState returnValue = MappingPluginState.SUCCESS; + MappingPluginState returnValue = MappingPluginState.SUCCESS(); try { ProcessBuilder pb = new ProcessBuilder(command); Process p = pb.start(); - Future> errorFuture = pool.submit(new ShellRunnerUtil.ProcessReadTask(p.getErrorStream())); - Future> inputFuture = pool.submit(new ShellRunnerUtil.ProcessReadTask(p.getInputStream())); + pipeStream(p.getInputStream(), new PrintStream(output)); + pipeStream(p.getErrorStream(), new PrintStream(error)); - List stdErr = errorFuture.get(timeOutInSeconds, TimeUnit.SECONDS); - List stdOut = inputFuture.get(timeOutInSeconds, TimeUnit.SECONDS); - - for (String line : stdOut) { - LOGGER.trace("[OUT] {}", line); - if (output != null) { - output.write((line + "\n").getBytes()); - } + if (!p.waitFor(timeOutInSeconds, TimeUnit.SECONDS)) { + throw new TimeoutException("Process did not return within " + timeOutInSeconds + " seconds."); } - - for (String line : stdErr) { - LOGGER.trace("[ERR] {}", line); - if (error != null) { - error.write((line + "\n").getBytes()); - } - } - - result = p.waitFor(); - if (result != 0) { - throw new ExecutionException(new Throwable()); + if (p.exitValue() != 0) { + throw new BadExitCodeException(p.exitValue()); } } catch (IOException ioe) { - LOGGER.error("Failed to execute command.", ioe); - returnValue = MappingPluginState.EXECUTION_ERROR; + LOGGER.error("Failed to run command or to access output/error streams.", ioe); + returnValue = MappingPluginState.EXECUTION_ERROR(); } catch (TimeoutException te) { - LOGGER.error("Command did not return in expected timeframe of " + TIMEOUT + " seconds", te); - returnValue = MappingPluginState.TIMEOUT; - } catch (InterruptedException | ExecutionException e) { - LOGGER.error("Failed to execute command due to an unknown exception.", e); - returnValue = MappingPluginState.UNKNOWN_ERROR; + LOGGER.error("Command did not return in expected timeframe of " + timeOutInSeconds + " seconds", te); + returnValue = MappingPluginState.TIMEOUT(); + } catch (InterruptedException e) { + LOGGER.error("Command execution has been interrupted.", e); + returnValue = MappingPluginState.UNKNOWN_ERROR(); + } catch (BadExitCodeException e) { + LOGGER.error("Failed to execute command due to an unexpected exception.", e); + returnValue = MappingPluginState.BAD_EXIT_CODE(); + returnValue.setDetails(e.getExitCode()); } finally { pool.shutdown(); - if (returnValue != MappingPluginState.SUCCESS) throw new MappingPluginException(returnValue); + } + + if (returnValue.getState() != MappingPluginState.SUCCESS().getState()) { + throw new MappingPluginException(returnValue); } return returnValue; } - private record ProcessReadTask(InputStream inputStream) implements Callable> { - @Override - public List call() { - return new BufferedReader(new InputStreamReader(inputStream)).lines().collect(Collectors.toList()); - } + private static void pipeStream(final InputStream src, final OutputStream dest) { + new Thread(() -> { + Scanner sc = new Scanner(src); + try (PrintStream print = new PrintStream(dest)) { + while (sc.hasNextLine()) { + print.println(sc.nextLine()); + } + print.flush(); + } + }).start(); } -} \ No newline at end of file +} diff --git a/src/main/resources/gemma.properties b/src/main/resources/gemma.properties new file mode 100644 index 00000000..2588f31b --- /dev/null +++ b/src/main/resources/gemma.properties @@ -0,0 +1,2 @@ +version=v1.0.0 +min.python=3.0.0 \ No newline at end of file diff --git a/src/main/resources/python/check.py b/src/main/resources/python/check.py new file mode 100644 index 00000000..d30747b7 --- /dev/null +++ b/src/main/resources/python/check.py @@ -0,0 +1,27 @@ +import sys +import platform +import os +import json +import logging + +# input folder and mapping file are ignored +output_folder = sys.argv[3] +# output path obtained from mapping service +outpath = os.path.join(output_folder, 'info.json') + + +def getSystemInfo(): + try: + # Python code execution + info = {'version': sys.version, 'platform-release': platform.release(), 'platform-version': platform.version(), + 'architecture': platform.machine(), 'processor': platform.processor()} + # output writing + with open(outpath, 'w', encoding='utf-8') as j: + json.dump(info, j, indent=2, ensure_ascii=False) + except Exception as e: + # exception logging for later checks and exit code != 0 to propagate error and cause + logging.exception(e) + exit(1) + + +getSystemInfo() diff --git a/src/test/java/edu/kit/datamanager/mappingservice/TestConfig.java b/src/test/java/edu/kit/datamanager/mappingservice/TestConfig.java index 6c8feb62..a61bd5c2 100644 --- a/src/test/java/edu/kit/datamanager/mappingservice/TestConfig.java +++ b/src/test/java/edu/kit/datamanager/mappingservice/TestConfig.java @@ -5,7 +5,10 @@ package edu.kit.datamanager.mappingservice; import edu.kit.datamanager.mappingservice.configuration.ApplicationProperties; +import edu.kit.datamanager.mappingservice.plugins.PluginLoader; import edu.kit.datamanager.mappingservice.plugins.PluginManager; +import io.micrometer.core.instrument.MeterRegistry; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; @@ -17,14 +20,21 @@ @Configuration @ComponentScan("edu.kit.datamanager.mappingservice") public class TestConfig { + @Autowired + private MeterRegistry meterRegistry; @Bean public ApplicationProperties applicationProperties() { return new ApplicationProperties(); } + @Bean + public PluginLoader pluginLoader() { + return new PluginLoader(applicationProperties()); + } + @Bean public PluginManager pluginManager() { - return new PluginManager(applicationProperties()); + return new PluginManager(applicationProperties(), pluginLoader(), meterRegistry); } } diff --git a/src/test/java/edu/kit/datamanager/mappingservice/impl/MappingServiceTest.java b/src/test/java/edu/kit/datamanager/mappingservice/impl/MappingServiceTest.java index c5be7194..c5d23c75 100644 --- a/src/test/java/edu/kit/datamanager/mappingservice/impl/MappingServiceTest.java +++ b/src/test/java/edu/kit/datamanager/mappingservice/impl/MappingServiceTest.java @@ -21,7 +21,9 @@ import edu.kit.datamanager.mappingservice.domain.MappingRecord; import edu.kit.datamanager.mappingservice.exception.MappingException; import edu.kit.datamanager.mappingservice.exception.MappingNotFoundException; +import edu.kit.datamanager.mappingservice.exception.MappingServiceException; import edu.kit.datamanager.mappingservice.plugins.MappingPluginException; +import io.micrometer.core.instrument.MeterRegistry; import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -74,7 +76,6 @@ @TestPropertySource(properties = {"metastore.indexer.mappingsLocation=file:///tmp/metastore2/mapping"}) //@TestInstance(TestInstance.Lifecycle.PER_CLASS) public class MappingServiceTest { - @Autowired ApplicationProperties applicationProperties; @@ -84,6 +85,9 @@ public class MappingServiceTest { @Autowired MappingService mappingService4Test; + @Autowired + MeterRegistry meterRegistry; + private final static String TEMP_DIR_4_MAPPING = "/tmp/mapping-service/"; @BeforeEach @@ -103,7 +107,7 @@ public void setUp() { @Test public void testConstructor() throws URISyntaxException { - new MappingService(applicationProperties); + new MappingService(applicationProperties, meterRegistry); } @Test @@ -115,7 +119,7 @@ public void testConstructorRelativePath() throws IOException, URISyntaxException ap.setMappingsLocation(relativePath); File file = new File(relativePath.getPath()); assertFalse(file.exists()); - new MappingService(ap); + new MappingService(ap, meterRegistry); assertTrue(file.exists()); FileUtils.deleteDirectory(file); assertFalse(file.exists()); @@ -127,9 +131,9 @@ public void testConstructorRelativePath() throws IOException, URISyntaxException @Test public void testConstructorFailing() throws IOException, URISyntaxException { try { - new MappingService(null); - fail(); - } catch (MappingException ie) { + new MappingService(null, meterRegistry); + fail("Expected MappingServiceException"); + } catch (MappingServiceException ie) { assertTrue(true); } //seems to be no problem under Windows and if run as root this is also no issue, so let's skip this test for the moment. diff --git a/src/test/java/edu/kit/datamanager/mappingservice/plugins/MappingPluginExceptionTest.java b/src/test/java/edu/kit/datamanager/mappingservice/plugins/MappingPluginExceptionTest.java index 0cb2482d..5144038b 100644 --- a/src/test/java/edu/kit/datamanager/mappingservice/plugins/MappingPluginExceptionTest.java +++ b/src/test/java/edu/kit/datamanager/mappingservice/plugins/MappingPluginExceptionTest.java @@ -23,18 +23,18 @@ class MappingPluginExceptionTest { @Test void testConstructor() { - MappingPluginException ex = new MappingPluginException(MappingPluginState.UNKNOWN_ERROR); - assertEquals(MappingPluginState.UNKNOWN_ERROR, ex.getState()); - assertEquals(MappingPluginState.UNKNOWN_ERROR.toString(), ex.getMessage()); + MappingPluginException ex = new MappingPluginException(MappingPluginState.UNKNOWN_ERROR()); + assertEquals(MappingPluginState.UNKNOWN_ERROR().getState(), ex.getMappingPluginState().getState()); + assertEquals(MappingPluginState.UNKNOWN_ERROR().getState().toString(), ex.getMessage()); assertNull(ex.getCause()); - ex = new MappingPluginException(MappingPluginState.UNKNOWN_ERROR, "test"); - assertEquals(MappingPluginState.UNKNOWN_ERROR, ex.getState()); + ex = new MappingPluginException(MappingPluginState.UNKNOWN_ERROR(), "test"); + assertEquals(MappingPluginState.UNKNOWN_ERROR().getState(), ex.getMappingPluginState().getState()); assertEquals("test", ex.getMessage()); assertNull(ex.getCause()); - ex = new MappingPluginException(MappingPluginState.UNKNOWN_ERROR, "test", new Exception()); - assertEquals(MappingPluginState.UNKNOWN_ERROR, ex.getState()); + ex = new MappingPluginException(MappingPluginState.UNKNOWN_ERROR(), "test", new Exception()); + assertEquals(MappingPluginState.UNKNOWN_ERROR().getState(), ex.getMappingPluginState().getState()); assertEquals("test", ex.getMessage()); assertNotNull(ex.getCause()); } diff --git a/src/test/java/edu/kit/datamanager/mappingservice/plugins/PluginLoaderTest.java b/src/test/java/edu/kit/datamanager/mappingservice/plugins/PluginLoaderTest.java index 93cbf68b..b30ebdd1 100644 --- a/src/test/java/edu/kit/datamanager/mappingservice/plugins/PluginLoaderTest.java +++ b/src/test/java/edu/kit/datamanager/mappingservice/plugins/PluginLoaderTest.java @@ -16,13 +16,13 @@ import edu.kit.datamanager.mappingservice.configuration.ApplicationProperties; import org.junit.jupiter.api.Test; -import org.springframework.util.MimeTypeUtils; import java.io.File; import java.io.IOException; import java.nio.file.Path; import java.util.Map; import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Assertions; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.fail; @@ -37,10 +37,14 @@ class PluginLoaderTest { @Autowired private PluginManager pluginManager; + + @Autowired + private PluginLoader pluginLoader; @Autowired private ApplicationProperties applicationProperties; + @BeforeEach void setUp() throws Exception { try { @@ -56,23 +60,25 @@ void valid() { System.out.println("Test valid"); Map plugins = null; try { - plugins = PluginLoader.loadPlugins(Path.of(applicationProperties.getPluginLocation().toURI()).toFile()); + plugins = pluginLoader.loadPlugins(Path.of(applicationProperties.getPluginLocation().toURI()).toFile(), applicationProperties.getPackagesToScan()); } catch (Exception e) { fail(e); } - for (var entry : plugins.entrySet()) { - System.out.println(entry.getValue().id()); - } + try { - assertEquals("TEST_0.0.0", plugins.get("TEST_0.0.0").id()); - assertEquals("TEST", plugins.get("TEST_0.0.0").name()); - assertEquals("Hello world! This is a non functional test plugin.", plugins.get("TEST_0.0.0").description()); - assertEquals("0.0.0", plugins.get("TEST_0.0.0").version()); - assertEquals("https://github.com/kit-data-manager/gemma", plugins.get("TEST_0.0.0").uri()); - assertEquals(MimeTypeUtils.APPLICATION_JSON, plugins.get("TEST_0.0.0").inputTypes()[0]); - assertEquals(MimeTypeUtils.APPLICATION_JSON, plugins.get("TEST_0.0.0").outputTypes()[0]); - plugins.get("TEST_0.0.0").setup(); - assertEquals(MappingPluginState.SUCCESS, plugins.get("TEST_0.0.0").mapFile(new File("schema").toPath(), new File("input").toPath(), new File("output").toPath())); + assertEquals("InOutPlugin_1.1.2", plugins.get("InOutPlugin_1.1.2").id()); + assertEquals("InOutPlugin", plugins.get("InOutPlugin_1.1.2").name()); + assertEquals("Simple plugin for testing just returning the input file.", plugins.get("InOutPlugin_1.1.2").description()); + assertEquals("1.1.2", plugins.get("InOutPlugin_1.1.2").version()); + assertEquals("https://github.com/kit-data-manager/mapping-service", plugins.get("InOutPlugin_1.1.2").uri()); + assertEquals("application/*", plugins.get("InOutPlugin_1.1.2").inputTypes()[0].toString()); + assertEquals("application/*", plugins.get("InOutPlugin_1.1.2").outputTypes()[0].toString()); + plugins.get("InOutPlugin_1.1.2").setup(applicationProperties); + File inputFile = new File("/tmp/imputFile"); + if (!inputFile.exists()) { + Assertions.assertTrue(inputFile.createNewFile()); + } + assertEquals(MappingPluginState.SUCCESS().getState(), plugins.get("InOutPlugin_1.1.2").mapFile(new File("schema").toPath(), inputFile.toPath(), new File("output").toPath()).getState()); } catch (Exception e) { fail(e); } @@ -82,7 +88,7 @@ void valid() { void invalidPath() { Map plugins = null; try { - PluginLoader.loadPlugins(new File("./invalid/test")); + pluginLoader.loadPlugins(new File("./invalid/test"), applicationProperties.getPackagesToScan()); } catch (IOException e) { fail(e); } catch (MappingPluginException validationWarning) { @@ -102,7 +108,7 @@ void invalidPath() { void nullInput() { Map plugins = null; try { - plugins = PluginLoader.loadPlugins(null); + plugins = pluginLoader.loadPlugins(null, null); } catch (IOException e) { fail(e); } catch (MappingPluginException validationWarning) { @@ -113,7 +119,7 @@ void nullInput() { void emptyinput() { Map plugins = null; try { - plugins = PluginLoader.loadPlugins(new File("")); + plugins = pluginLoader.loadPlugins(new File(""), null); } catch (IOException e) { fail(e); } catch (MappingPluginException validationWarning) { diff --git a/src/test/java/edu/kit/datamanager/mappingservice/plugins/PluginManagerTest.java b/src/test/java/edu/kit/datamanager/mappingservice/plugins/PluginManagerTest.java index d8832dc2..86819f03 100644 --- a/src/test/java/edu/kit/datamanager/mappingservice/plugins/PluginManagerTest.java +++ b/src/test/java/edu/kit/datamanager/mappingservice/plugins/PluginManagerTest.java @@ -15,6 +15,7 @@ package edu.kit.datamanager.mappingservice.plugins; import edu.kit.datamanager.mappingservice.configuration.ApplicationProperties; +import edu.kit.datamanager.mappingservice.exception.MappingServiceException; import org.junit.jupiter.api.Test; import java.io.File; import java.io.IOException; @@ -39,11 +40,11 @@ class PluginManagerTest { @BeforeEach void setup() throws Exception { - try { + /*try { FileUtils.copyDirectory(Path.of("./plugins").toFile(), Path.of(applicationProperties.getPluginLocation().toURI()).toFile()); } catch (IOException ex) { ex.printStackTrace(); - } + }*/ pluginManager.reloadPlugins(); } @@ -71,37 +72,42 @@ void getListOfAvailableValidators() { void mapFileInvalidParameters() { try { pluginManager.mapFile(null, null, null, null); - } catch (MappingPluginException e) { - assertEquals(MappingPluginState.INVALID_INPUT, e.getState()); - assertEquals("Plugin ID is null.", e.getMessage()); + } catch (MappingServiceException e) { + assertEquals("PluginId is null.", e.getMessage()); + } catch (MappingPluginException ex) { + fail("Expected MappingServiceException"); } try { pluginManager.mapFile("test", null, null, null); - } catch (MappingPluginException e) { - assertEquals(MappingPluginState.INVALID_INPUT, e.getState()); - assertEquals("Path to mapping schema is null.", e.getMessage()); + } catch (MappingServiceException e) { + assertEquals("Path to mapping file is null.", e.getMessage()); + } catch (MappingPluginException ex) { + fail("Expected MappingServiceException"); } try { pluginManager.mapFile("test", new File("test").toPath(), null, null); - } catch (MappingPluginException e) { - assertEquals(MappingPluginState.INVALID_INPUT, e.getState()); + } catch (MappingServiceException e) { assertEquals("Path to input file is null.", e.getMessage()); + } catch (MappingPluginException ex) { + fail("Expected MappingServiceException"); } try { pluginManager.mapFile("test", new File("test").toPath(), new File("testInput").toPath(), null); - } catch (MappingPluginException e) { - assertEquals(MappingPluginState.INVALID_INPUT, e.getState()); + } catch (MappingServiceException e) { assertEquals("Path to output file is null.", e.getMessage()); + } catch (MappingPluginException ex) { + fail("Expected MappingServiceException"); } try { pluginManager.mapFile("test", new File("test").toPath(), new File("testInput").toPath(), new File("testOutput").toPath()); - } catch (MappingPluginException e) { - assertEquals(MappingPluginState.NOT_FOUND, e.getState()); - assertEquals("Plugin 'test' not found!", e.getMessage()); + } catch (MappingServiceException e) { + fail("Expected MappingPluginException"); + } catch (MappingPluginException ex) { + assertEquals("Plugin 'test' not found!", ex.getMessage()); } } @@ -109,12 +115,16 @@ void mapFileInvalidParameters() { void mapFile() { try { File outputFile = new File("/tmp/testOutput"); - pluginManager.mapFile("TEST_0.0.0", new File("mapping-schema").toPath(), new File("input").toPath(), outputFile.toPath()); + File inputFile = new File("/tmp/testInput"); + if (!inputFile.exists()) { + assertTrue(inputFile.createNewFile()); + } + pluginManager.mapFile("InOutPlugin_1.1.2", new File("mapping-schema").toPath(), inputFile.toPath(), outputFile.toPath()); assertTrue(outputFile.exists()); + inputFile.delete(); outputFile.delete(); - } catch (MappingPluginException e) { - e.printStackTrace(); - fail("Mapping failed"); + } catch (MappingPluginException | IOException e) { + fail("Mapping failed", e); } } } diff --git a/src/test/java/edu/kit/datamanager/mappingservice/python/util/PythonUtilsTest.java b/src/test/java/edu/kit/datamanager/mappingservice/python/util/PythonUtilsTest.java index b92c8cbd..1a7bd6ce 100644 --- a/src/test/java/edu/kit/datamanager/mappingservice/python/util/PythonUtilsTest.java +++ b/src/test/java/edu/kit/datamanager/mappingservice/python/util/PythonUtilsTest.java @@ -1,137 +1,131 @@ -///* -// * Copyright 2019 Karlsruhe Institute of Technology. -// * -// * Licensed under the Apache License, Version 2.0 (the "License"); -// * you may not use this file except in compliance with the License. -// * You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ -//package edu.kit.datamanager.mappingservice.python.util; -// -//import org.junit.jupiter.api.BeforeAll; -//import org.junit.jupiter.api.Test; -// -//import java.io.ByteArrayOutputStream; -//import java.io.File; -//import java.io.IOException; -//import java.io.OutputStream; -// -//import static org.junit.jupiter.api.Assertions.*; -// -///** -// * -// */ -//public class PythonUtilsTest { -// -// private static String PYTHON_EXECUTABLE; -// -// public PythonUtilsTest() { -// } -// -// @BeforeAll -// public static void setUpClass() throws IOException { -// // Determine python location -// OutputStream os = new ByteArrayOutputStream(); -// PythonUtils.run("which", "python3", os, null); -// String pythonExecutable = os.toString(); -// os.flush(); -// if (pythonExecutable.trim().isEmpty()) { -// PythonUtils.run("which", "python", os, null); -// pythonExecutable = os.toString(); -// } -// if (pythonExecutable.trim().isEmpty()) { -// throw new IOException("Python seems not to be available!"); -// } -// System.out.println("Location of python: " + pythonExecutable); -// PYTHON_EXECUTABLE = pythonExecutable.trim(); -// } -// -// /** -// */ -// @Test -// public void testRun_Constructor() { -// assertNotNull(new PythonUtils()); -// } -// -// /** -// * Test of run method, of class PythonUtils. -// */ -// @Test -// public void testRun_3args_withWrongPython() { -// System.out.println("testRun_3args_withWrongPython"); -// String pythonLocation = "/usr/bin/invalidpython"; -// String scriptLocation = ""; -// String[] arguments = null; -// int expResult = PythonUtils.PYTHON_NOT_FOUND_ERROR; -// int result = PythonUtils.run(pythonLocation, scriptLocation, arguments); -// assertEquals(Integer.valueOf(expResult), Integer.valueOf(result)); -// } -// -// /** -// * Test of run method, of class PythonUtils. -// */ -// @Test -// public void testRun_3args_withWrongClass() { -// System.out.println("testRun_3args_withWrongClass"); -// String pythonLocation = PYTHON_EXECUTABLE; -// String scriptLocation = new File("src/test/resources/python/invalid.py").getAbsolutePath(); -// String[] arguments = null; -// int expResult = PythonUtils.EXECUTION_ERROR; -// int result = PythonUtils.run(pythonLocation, scriptLocation, arguments); -// assertEquals(Integer.valueOf(expResult), Integer.valueOf(result)); -// } -// -// /** -// * Test of run method, of class PythonUtils. -// */ -// @Test -// public void testRun_3args_withTimeout() { -// System.out.println("testRun_3args_withTimeout"); -// String pythonLocation = PYTHON_EXECUTABLE; -// String scriptLocation = new File("src/test/resources/python/sleep.py").getAbsolutePath(); -// String[] arguments = null; -// int expResult = PythonUtils.TIMEOUT_ERROR; -// int result = PythonUtils.run(pythonLocation, scriptLocation, 1, arguments); -// assertEquals(Integer.valueOf(expResult), Integer.valueOf(result)); -// expResult = PythonUtils.SUCCESS; -// result = PythonUtils.run(pythonLocation, scriptLocation, 5, arguments); -// assertEquals(Integer.valueOf(expResult), Integer.valueOf(result)); -// -// } -// -// /** -// * Test of run method, of class PythonUtils. -// */ -// @Test -// public void testRun_3args_withNoOutputStreams() { -// System.out.println("testRun_3args_withTimeout"); -// String pythonLocation = PYTHON_EXECUTABLE; -// String scriptLocation = new File("src/test/resources/python/printOutput.py").getAbsolutePath(); -// String[] arguments = null; -// int expResult = PythonUtils.SUCCESS; -// int result = PythonUtils.run(pythonLocation, scriptLocation, null, null, arguments); -// assertEquals(Integer.valueOf(expResult), Integer.valueOf(result)); -// } -// -// /** -// * Test of run method, of class PythonUtils. -// */ -// @Test -// public void testRun_3args_withInvalidPython() { -// System.out.println("testRun_3args_withInvalidPython"); -// String pythonLocation = PYTHON_EXECUTABLE; -// String scriptLocation = "/notExistingFile.py"; -// String[] arguments = null; -// int expResult = PythonUtils.EXECUTION_ERROR; -// int result = PythonUtils.run(pythonLocation, scriptLocation, arguments); -// assertEquals(Integer.valueOf(expResult), Integer.valueOf(result)); -// } -// -//} +/* + * Copyright 2019 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.kit.datamanager.mappingservice.python.util; + +import edu.kit.datamanager.mappingservice.configuration.ApplicationProperties; +import edu.kit.datamanager.mappingservice.plugins.MappingPluginException; +import edu.kit.datamanager.mappingservice.plugins.MappingPluginState; +import edu.kit.datamanager.mappingservice.util.PythonRunnerUtil; +import edu.kit.datamanager.mappingservice.util.ShellRunnerUtil; +import org.junit.jupiter.api.Test; + +import java.io.File; +import org.hamcrest.CoreMatchers; +import org.junit.Assume; + +import static org.junit.jupiter.api.Assertions.*; +import org.junit.jupiter.api.BeforeEach; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; + +/** + * + */ +@SpringBootTest +@ActiveProfiles("test") +public class PythonUtilsTest { + + @Autowired + private ApplicationProperties applicationProperties; + + public PythonUtilsTest() { + } + + @BeforeEach + public void setUpClass() { + PythonRunnerUtil.init(applicationProperties); + } + + @Test + public void testPythonAvailable() { + Assume.assumeThat("Python not configured.", applicationProperties.isPythonAvailable(), CoreMatchers.is(true)); + assertTrue(applicationProperties.isPythonAvailable()); + } + + /** + * Test of run method, of class PythonUtils. + */ + @Test + public void testRun_3args_withWrongPython() { + System.out.println("testRun_3args_withWrongPython"); + ApplicationProperties props = new ApplicationProperties(); + props.setPythonExecutable(null); + PythonRunnerUtil.init(props); + String scriptLocation = ""; + String[] arguments = null; + try { + MappingPluginState result = PythonRunnerUtil.runPythonScript(scriptLocation, arguments); + fail("Expected MappingPluginException"); + } catch (MappingPluginException e) { + assertEquals(MappingPluginState.StateEnum.UNKNOWN_ERROR, e.getMappingPluginState().getState()); + } + } + + /** + * Test of run method, of class PythonUtils. + */ + @Test + public void testRun_3args_withWrongClass() { + Assume.assumeThat("Python not configured.", applicationProperties.isPythonAvailable(), CoreMatchers.is(true)); + System.out.println("testRun_3args_withWrongClass"); + String scriptLocation = new File("src/test/resources/python/invalid.py").getAbsolutePath(); + String[] arguments = null; + try { + PythonRunnerUtil.runPythonScript(scriptLocation, arguments); + fail("Expected MappingPluginException"); + } catch (MappingPluginException e) { + assertEquals(MappingPluginState.StateEnum.BAD_EXIT_CODE, e.getMappingPluginState().getState()); + assertEquals(123, e.getMappingPluginState().getDetails()); + } + } + + /** + * Test of run method, of class PythonUtils. + */ + @Test + public void testRun_3args_withTimeout() { + Assume.assumeThat("Python not configured.", applicationProperties.isPythonAvailable(), CoreMatchers.is(true)); + System.out.println("testRun_3args_withTimeout"); + String scriptLocation = new File("src/test/resources/python/sleep.py").getAbsolutePath(); + String[] arguments = null; + ApplicationProperties props = new ApplicationProperties(); + props.setExecutionTimeout(1); + ShellRunnerUtil.init(props); + try { + PythonRunnerUtil.runPythonScript(scriptLocation, arguments); + } catch (MappingPluginException e) { + assertEquals(MappingPluginState.StateEnum.TIMEOUT, e.getMappingPluginState().getState()); + } + + } + + /** + * Test of run method, of class PythonUtils. + */ + @Test + public void testRun_3args_withNoOutputStreams() { + System.out.println("testRun_3args_withTimeout"); + String scriptLocation = new File("src/test/resources/python/printOutput.py").getAbsolutePath(); + String[] arguments = null; + try { + PythonRunnerUtil.runPythonScript(scriptLocation, null, null, arguments); + fail("Expected MappingPluginException"); + } catch (MappingPluginException e) { + assertEquals(MappingPluginState.StateEnum.INVALID_INPUT, e.getMappingPluginState().getState()); + } + } +} diff --git a/src/test/java/edu/kit/datamanager/mappingservice/rest/PluginInformationTest.java b/src/test/java/edu/kit/datamanager/mappingservice/rest/PluginInformationTest.java index 10396646..e9284241 100644 --- a/src/test/java/edu/kit/datamanager/mappingservice/rest/PluginInformationTest.java +++ b/src/test/java/edu/kit/datamanager/mappingservice/rest/PluginInformationTest.java @@ -230,7 +230,7 @@ void testIDConstructor() { new PluginInformation(null, pluginManager); fail("Expected exception"); } catch (MappingPluginException e) { - assertEquals(MappingPluginState.NOT_FOUND, e.getState()); + assertEquals(MappingPluginState.NOT_FOUND(), e.getMappingPluginState()); } } } diff --git a/src/test/java/edu/kit/datamanager/mappingservice/rest/impl/MappingAdministrationControllerTest.java b/src/test/java/edu/kit/datamanager/mappingservice/rest/impl/MappingAdministrationControllerTest.java index 7333e871..611d632f 100644 --- a/src/test/java/edu/kit/datamanager/mappingservice/rest/impl/MappingAdministrationControllerTest.java +++ b/src/test/java/edu/kit/datamanager/mappingservice/rest/impl/MappingAdministrationControllerTest.java @@ -18,7 +18,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import edu.kit.datamanager.entities.PERMISSION; -import edu.kit.datamanager.mappingservice.MappingServiceApplication; import edu.kit.datamanager.mappingservice.dao.IMappingRecordDao; import edu.kit.datamanager.mappingservice.domain.AclEntry; import edu.kit.datamanager.mappingservice.domain.MappingRecord; @@ -26,7 +25,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.migrationsupport.rules.EnableRuleMigrationSupport; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; import org.springframework.boot.test.context.SpringBootTest; @@ -66,21 +64,17 @@ import static org.junit.jupiter.api.Assertions.*; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.extension.RegisterExtension; -import org.junit.runner.RunWith; -import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; -import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.context.annotation.ComponentScan; import org.springframework.restdocs.RestDocumentationContextProvider; import static org.springframework.restdocs.mockmvc.MockMvcRestDocumentation.document; import static org.springframework.restdocs.mockmvc.MockMvcRestDocumentation.documentationConfiguration; import static org.springframework.restdocs.operation.preprocess.Preprocessors.*; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringRunner; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.redirectedUrlPattern; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + /** * */ @@ -98,17 +92,16 @@ public class MappingAdministrationControllerTest { private final static String TEMP_DIR_4_MAPPING = "/tmp/mapping-service/"; private static final String MAPPING_ID = "my_dc"; - private static final String MAPPING_TYPE = "GEMMA"; + private static final String MAPPING_TYPE = "GEMMA_v1.0.0"; private static final String MAPPING_TITLE = "TITEL"; private static final String MAPPING_DESCRIPTION = "DESCRIPTION"; - @RegisterExtension - final RestDocumentationExtension restDocumentation = new RestDocumentationExtension ("custom"); - + final RestDocumentationExtension restDocumentation = new RestDocumentationExtension("custom"); + @Autowired private MockMvc mockMvc; - + @Autowired private WebApplicationContext webApplicationContext; @@ -116,7 +109,7 @@ public class MappingAdministrationControllerTest { private IMappingRecordDao mappingRecordDao; @BeforeEach - public void setUp(RestDocumentationContextProvider restDocumentation) { + public void setUp(RestDocumentationContextProvider restDocumentation) { mappingRecordDao.deleteAll(); try { try (Stream walk = Files.walk(Paths.get(URI.create("file://" + TEMP_DIR_4_MAPPING)))) { @@ -138,6 +131,111 @@ public void setUp(RestDocumentationContextProvider restDocumentation) { .build(); } + /** + * Test of createMapping method, of class MappingAdministrationController. + */ + @Test + public void testCreateMappingWithoutID() throws Exception { + System.out.println("createMapping"); + Path mappingsDir = Paths.get(URI.create("file://" + TEMP_DIR_4_MAPPING)); + + String mappingContent = FileUtils.readFileToString(new File("src/test/resources/mapping/gemma/simple.mapping"), StandardCharsets.UTF_8); + MappingRecord record = new MappingRecord(); + record.setMappingId(null); + record.setMappingType(MAPPING_TYPE); + record.setTitle(MAPPING_TITLE); + record.setDescription(MAPPING_DESCRIPTION); + Set aclEntries = new HashSet<>(); + aclEntries.add(new AclEntry("SELF", PERMISSION.READ)); + aclEntries.add(new AclEntry("test2", PERMISSION.ADMINISTRATE)); + record.setAcl(aclEntries); + ObjectMapper mapper = new ObjectMapper(); + + MockMultipartFile recordFile = new MockMultipartFile("record", "record.json", "application/json", mapper.writeValueAsString(record).getBytes()); + MockMultipartFile mappingFile = new MockMultipartFile("document", "my_dc4gemma.mapping", "application/json", mappingContent.getBytes()); + + //long before = Files.list(mappingsDir).count(); + this.mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/mappingAdministration/"). + file(recordFile). + file(mappingFile)). + andDo(print()). + andExpect(status().isBadRequest()); + + //assertEquals(before+1, Files.list(mappingsDir).count()); + } + + /** + * Test of createMapping method, of class MappingAdministrationController. + */ + @Test + public void testCreateMappingWithWrongID() throws Exception { + System.out.println("createMapping"); + Path mappingsDir = Paths.get(URI.create("file://" + TEMP_DIR_4_MAPPING)); + + String mappingContent = FileUtils.readFileToString(new File("src/test/resources/mapping/gemma/simple.mapping"), StandardCharsets.UTF_8); + MappingRecord record = new MappingRecord(); + record.setMappingId(""); + record.setMappingType(MAPPING_TYPE); + record.setTitle(MAPPING_TITLE); + record.setDescription(MAPPING_DESCRIPTION); + Set aclEntries = new HashSet<>(); + aclEntries.add(new AclEntry("SELF", PERMISSION.READ)); + aclEntries.add(new AclEntry("test2", PERMISSION.ADMINISTRATE)); + record.setAcl(aclEntries); + ObjectMapper mapper = new ObjectMapper(); + + MockMultipartFile recordFile = new MockMultipartFile("record", "record.json", "application/json", mapper.writeValueAsString(record).getBytes()); + MockMultipartFile mappingFile = new MockMultipartFile("document", "my_dc4gemma.mapping", "application/json", mappingContent.getBytes()); + + //long before = Files.list(mappingsDir).count(); + this.mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/mappingAdministration/"). + file(recordFile). + file(mappingFile)). + andDo(print()). + andExpect(status().isBadRequest()); + + record.setMappingId(""); + + recordFile = new MockMultipartFile("record", "record.json", "application/json", mapper.writeValueAsString(record).getBytes()); + + this.mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/mappingAdministration/"). + file(recordFile). + file(mappingFile)). + andDo(print()). + andExpect(status().isBadRequest()); + + record.setMappingId(" "); + + recordFile = new MockMultipartFile("record", "record.json", "application/json", mapper.writeValueAsString(record).getBytes()); + + this.mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/mappingAdministration/"). + file(recordFile). + file(mappingFile)). + andDo(print()). + andExpect(status().isBadRequest()); + + record.setMappingId("\t"); + + recordFile = new MockMultipartFile("record", "record.json", "application/json", mapper.writeValueAsString(record).getBytes()); + + this.mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/mappingAdministration/"). + file(recordFile). + file(mappingFile)). + andDo(print()). + andExpect(status().isBadRequest()); + + record.setMappingId(" "); + + recordFile = new MockMultipartFile("record", "record.json", "application/json", mapper.writeValueAsString(record).getBytes()); + + this.mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/mappingAdministration/"). + file(recordFile). + file(mappingFile)). + andDo(print()). + andExpect(status().isBadRequest()); + //assertEquals(before+1, Files.list(mappingsDir).count()); + } + /** * Test of createMapping method, of class MappingAdministrationController. */ @@ -161,9 +259,7 @@ public void testCreateMapping() throws Exception { MockMultipartFile recordFile = new MockMultipartFile("record", "record.json", "application/json", mapper.writeValueAsString(record).getBytes()); MockMultipartFile mappingFile = new MockMultipartFile("document", "my_dc4gemma.mapping", "application/json", mappingContent.getBytes()); - //long before = Files.list(mappingsDir).count(); - this.mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/mappingAdministration/"). file(recordFile). file(mappingFile)). @@ -299,7 +395,7 @@ public void testCreateMappingWithAcl() throws Exception { MockMultipartFile recordFile = new MockMultipartFile("record", "record.json", "application/json", mapper.writeValueAsString(record).getBytes()); MockMultipartFile mappingFile = new MockMultipartFile("document", "my_dc4gemma.mapping", "application/json", mappingContent.getBytes()); - // assertEquals(0, mappingsDir.list().length); + // assertEquals(0, mappingsDir.list().length); this.mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/mappingAdministration/"). file(recordFile). file(mappingFile)).andDo(print()).andExpect(status().isCreated()).andExpect(redirectedUrlPattern("http://*:*//api/v1/mappingAdministration/*")).andReturn(); @@ -438,9 +534,9 @@ public void testUpdateMapping() throws JsonProcessingException, Exception { aclEntries.add(new AclEntry("SELF", PERMISSION.READ)); aclEntries.add(new AclEntry("someoneelse", PERMISSION.ADMINISTRATE)); record.setAcl(aclEntries); - + int before = mappingsDir.list().length; - + String mappingContent = FileUtils.readFileToString(new File("src/test/resources/mapping/gemma/simple_v2.mapping"), StandardCharsets.UTF_8); MockMultipartFile recordFile = new MockMultipartFile("record", "record.json", "application/json", mapper.writeValueAsString(record).getBytes()); @@ -768,7 +864,7 @@ public void testDeleteMapping() throws JsonProcessingException, Exception { String deleteMappingIdUrl = "/api/v1/mappingAdministration/" + mappingId; result = this.mockMvc.perform(delete(deleteMappingIdUrl).header("If-Match", etag)).andDo(print()).andExpect(status().isNoContent()).andReturn(); - // assertEquals(1, mappingsDir.list().length); + // assertEquals(1, mappingsDir.list().length); String expectedFilename = mappingId + "_" + mappingType + ".mapping"; assertNotEquals(expectedFilename, mappingsDir.list()[0]); result = this.mockMvc.perform(get(getMappingIdUrl).header("Accept", MappingRecord.MAPPING_RECORD_MEDIA_TYPE)).andDo(print()).andExpect(status().isNotFound()).andReturn(); @@ -792,9 +888,9 @@ public void testDeleteMappingUnknownMappingId() throws JsonProcessingException, String deleteMappingIdUrl = "/api/v1/mappingAdministration/" + "unknownMappingId"; result = this.mockMvc.perform(delete(deleteMappingIdUrl).header("If-Match", etag)).andDo(print()).andExpect(status().isNoContent()).andReturn(); - // assertEquals(1, mappingsDir.list().length); + // assertEquals(1, mappingsDir.list().length); String expectedFilename = mappingId + "_" + mappingType + ".mapping"; - assertEquals("my_dc_GEMMA.mapping", expectedFilename); + assertEquals("my_dc_" + MAPPING_TYPE + ".mapping", expectedFilename); assertEquals(1, mappingRecordDao.count()); } @@ -837,9 +933,9 @@ public void testDeleteMappingMissingEtag() throws JsonProcessingException, Excep String deleteMappingIdUrl = "/api/v1/mappingAdministration/" + mappingId; result = this.mockMvc.perform(delete(deleteMappingIdUrl)).andDo(print()).andExpect(status().isPreconditionRequired()).andReturn(); - // assertEquals(1, mappingsDir.list().length); + // assertEquals(1, mappingsDir.list().length); String expectedFilename = mappingId + "_" + mappingType + ".mapping"; - assertEquals("my_dc_GEMMA.mapping", expectedFilename); + assertEquals("my_dc_" + MAPPING_TYPE + ".mapping", expectedFilename); result = this.mockMvc.perform(get(getMappingIdUrl).header("Accept", MappingRecord.MAPPING_RECORD_MEDIA_TYPE)).andDo(print()).andExpect(status().isOk()).andReturn(); assertEquals(1, mappingRecordDao.count()); } @@ -861,9 +957,9 @@ public void testDeleteMappingWrongEtag() throws JsonProcessingException, Excepti String deleteMappingIdUrl = "/api/v1/mappingAdministration/" + mappingId; result = this.mockMvc.perform(delete(deleteMappingIdUrl).header("If-Match", etag)).andDo(print()).andExpect(status().isPreconditionFailed()).andReturn(); - // assertEquals(1, mappingsDir.list().length); + // assertEquals(1, mappingsDir.list().length); String expectedFilename = mappingId + "_" + mappingType + ".mapping"; - assertEquals("my_dc_GEMMA.mapping", expectedFilename); + assertEquals("my_dc_" + MAPPING_TYPE + ".mapping", expectedFilename); result = this.mockMvc.perform(get(getMappingIdUrl).header("Accept", MappingRecord.MAPPING_RECORD_MEDIA_TYPE)).andDo(print()).andExpect(status().isOk()).andReturn(); assertEquals(1, mappingRecordDao.count()); } diff --git a/src/test/java/edu/kit/datamanager/mappingservice/rest/impl/MappingExecutionControllerTest.java b/src/test/java/edu/kit/datamanager/mappingservice/rest/impl/MappingExecutionControllerTest.java index 1c2c5adb..c76c74f8 100644 --- a/src/test/java/edu/kit/datamanager/mappingservice/rest/impl/MappingExecutionControllerTest.java +++ b/src/test/java/edu/kit/datamanager/mappingservice/rest/impl/MappingExecutionControllerTest.java @@ -61,7 +61,7 @@ public class MappingExecutionControllerTest { private final static String TEMP_DIR_4_ALL = "/tmp/mapping-service/"; private final static String TEMP_DIR_4_MAPPING = TEMP_DIR_4_ALL + "mapping/"; private static final String MAPPING_ID = "my_dc"; - private static final String MAPPING_TYPE = "TEST_0.0.0"; + private static final String MAPPING_TYPE = "InOutPlugin_1.1.2"; private static final String MAPPING_URL = "/api/v1/mappingExecution/" + MAPPING_ID; private static final String MAPPING_TITLE = "TITEL"; private static final String MAPPING_DESCRIPTION = "DESCRIPTION"; @@ -155,7 +155,7 @@ void mapValidDocument() throws Exception { this.mockMvc.perform(MockMvcRequestBuilders.multipart(MAPPING_URL).file(mappingFile)). andDo(print()). andExpect(status().isOk()). - andExpect(header().string("content-disposition", "attachment;filename=result.txt")).andReturn(); + andExpect(header().string("content-disposition", "attachment;filename=result.json")).andReturn(); } @Test diff --git a/src/test/java/edu/kit/datamanager/mappingservice/util/FileUtilTest.java b/src/test/java/edu/kit/datamanager/mappingservice/util/FileUtilTest.java index e4df43d7..f409189f 100644 --- a/src/test/java/edu/kit/datamanager/mappingservice/util/FileUtilTest.java +++ b/src/test/java/edu/kit/datamanager/mappingservice/util/FileUtilTest.java @@ -17,6 +17,7 @@ import com.google.common.io.Files; import edu.kit.datamanager.mappingservice.exception.MappingException; +import edu.kit.datamanager.mappingservice.exception.MappingServiceException; import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; @@ -27,8 +28,10 @@ import java.net.URISyntaxException; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Arrays; import java.util.HashSet; import java.util.Optional; +import org.apache.commons.io.filefilter.HiddenFileFilter; import static org.junit.jupiter.api.Assertions.*; @@ -319,34 +322,24 @@ public void testFixFileExtensionWrongFile() { @Test void cloneValidGitRepository() { Path util = null; + try { util = FileUtil.cloneGitRepository("https://github.com/kit-data-manager/mapping-service.git", "main", "/tmp/test"); } catch (Exception e) { fail(e); + } finally { + try { + FileUtils.deleteDirectory(new File("/tmp/test")); + } catch (IOException e) { + } } - try { - FileUtils.deleteDirectory(new File("tmp/test")); - } catch (IOException e) { - } - assertNotNull(util); - util = null; - try { - util = FileUtil.cloneGitRepository("https://github.com/kit-data-manager/mapping-service.git", "main"); - } catch (Exception e) { - fail(e); - } + assertNotNull(util); - try { - FileUtils.deleteDirectory(new File(util.toUri())); - } catch (IOException e) { - } - util = null; } @Test void cloneInvalidGitRepository() { - assertThrows(MappingException.class, () -> FileUtil.cloneGitRepository("test", "test", "test")); - assertThrows(MappingException.class, () -> FileUtil.cloneGitRepository("test", "test")); + assertThrows(MappingServiceException.class, () -> FileUtil.cloneGitRepository("test", "test", "test")); } @AfterEach diff --git a/src/test/java/edu/kit/datamanager/mappingservice/util/ShellRunnerUtilTest.java b/src/test/java/edu/kit/datamanager/mappingservice/util/ShellRunnerUtilTest.java index f3852b35..59478d2c 100644 --- a/src/test/java/edu/kit/datamanager/mappingservice/util/ShellRunnerUtilTest.java +++ b/src/test/java/edu/kit/datamanager/mappingservice/util/ShellRunnerUtilTest.java @@ -27,17 +27,17 @@ class ShellRunnerUtilTest { void runValid() { if (SystemUtils.IS_OS_WINDOWS) { try { - assertEquals(MappingPluginState.SUCCESS, ShellRunnerUtil.run("echo.bat", "test")); - assertEquals(MappingPluginState.SUCCESS, ShellRunnerUtil.run(5, "echo.bat", "test")); - assertEquals(MappingPluginState.SUCCESS, ShellRunnerUtil.run(System.out, System.err, "echo.bat", "test")); + assertEquals(MappingPluginState.SUCCESS().getState(), ShellRunnerUtil.run("echo.bat", "test").getState()); + assertEquals(MappingPluginState.SUCCESS().getState(), ShellRunnerUtil.run(5, "echo.bat", "test").getState()); + assertEquals(MappingPluginState.SUCCESS().getState(), ShellRunnerUtil.run(System.out, System.err, "echo.bat", "test").getState()); } catch (MappingPluginException e) { fail(e); } } else { try { - assertEquals(MappingPluginState.SUCCESS, ShellRunnerUtil.run("echo", "test")); - assertEquals(MappingPluginState.SUCCESS, ShellRunnerUtil.run(5, "echo", "test")); - assertEquals(MappingPluginState.SUCCESS, ShellRunnerUtil.run(System.out, System.err, "echo", "test")); + assertEquals(MappingPluginState.SUCCESS().getState(), ShellRunnerUtil.run("echo", "test").getState()); + assertEquals(MappingPluginState.SUCCESS().getState(), ShellRunnerUtil.run(5, "echo", "test").getState()); + assertEquals(MappingPluginState.SUCCESS().getState(), ShellRunnerUtil.run(System.out, System.err, "echo", "test").getState()); } catch (MappingPluginException e) { fail(e); } diff --git a/src/test/resources/python/invalid.py b/src/test/resources/python/invalid.py new file mode 100644 index 00000000..1d73fff9 --- /dev/null +++ b/src/test/resources/python/invalid.py @@ -0,0 +1,6 @@ +#!/usr/bin/python + + +print("Exit code incoming...") + +exit(123) \ No newline at end of file diff --git a/src/test/resources/python/printOutput.py b/src/test/resources/python/printOutput.py new file mode 100644 index 00000000..30fb1a63 --- /dev/null +++ b/src/test/resources/python/printOutput.py @@ -0,0 +1,6 @@ +#!/usr/bin/python +import time +import sys + +print ("Hello") +sys.stderr.write("Print to stderr\n") \ No newline at end of file diff --git a/src/test/resources/python/sleep.py b/src/test/resources/python/sleep.py new file mode 100644 index 00000000..cecf0d7b --- /dev/null +++ b/src/test/resources/python/sleep.py @@ -0,0 +1,5 @@ +#!/usr/bin/python +import time + +print ("Sleep for 3 seconds") +time.sleep( 3 ) diff --git a/src/test/resources/test-config/application-test.properties b/src/test/resources/test-config/application-test.properties index 808f9e3d..92033e94 100644 --- a/src/test/resources/test-config/application-test.properties +++ b/src/test/resources/test-config/application-test.properties @@ -27,9 +27,9 @@ spring.servlet.multipart.max-request-size=100MB # Logging settings logging.level.root=WARN -logging.level.web=TRACE -logging.level.org.springframework.web=TRACE -logging.level.edu.kit.datamanager=INFO +logging.level.web=WARN +logging.level.org.springframework.web=WARN +logging.level.edu.kit.datamanager=TRACE #springdoc.swagger-ui.disable-swagger-default-url=true # Actuator settings info.app.name=Mapping-Service @@ -64,9 +64,13 @@ eureka.client.enabled: false # Mapping-Service specific settings ################################################## # Absolute path to the local python interpreter -mapping-service.pythonExecutable=${pythonExecutable:'file:///usr/bin/python'}# +mapping-service.pythonExecutable=${pythonExecutable:file:///usr/bin/python} # Absolute path to the folder where all plugins are located mapping-service.pluginLocation=file:///tmp/mapping-service/plugins +mapping-service.codeLocation=file:///tmp/mapping-service/code # Absolute path to the local gemma mappings folder mapping-service.mappingSchemasLocation=file:///tmp/mapping-service/schemas +# Folder where job output files for async mapping executions are stored mapping-service.jobOutput=file:///tmp/mapping-service/jobOutput +# Execution timeout for script calls +mapping-service.executionTimeout=30