Added mappingPLugin/execute to mappingExecution controller, renamed *types* endpoints to *plugins*, updated changelog

Author: ThomasJejkal

CHANGELOG.md

@@ -6,8 +6,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 ### Fixed
+* Cloned Git repositories are now properly closed (only relevant for tests)
+
 ### New Features
+* Added support for JWT authentication (enabled via property *mapping-service.authEnabled*)
+* Integration of Prometheus monitoring
+* Added endpoint /api/v1/mappingExecution/plugins/<PluginID>/execute for direct execution of plugins
+* Added JoltPlugin to list of default plugins
+* New configuration property *mapping-service.mappingAdminRole* can be used to authenticate access to administrative endpoints
+* plugin-core jar released as separate dependency and can be used for implementing own plugins easier 
+
 ### Changed
+* Countless dependency updates
+* Changed plugin loading behaviour (default plugins are now part of mapping-service and not shipped as jar files)
+* Plugin code (Python-based plugins) is now checked out in a configurable folder (mapping-service.codeLocation)
+* Changed base class for Python-based plugins to AbstractPythonMappingPlugin
+* Plugin version should now match the Git release tag used by Python-based plugins
+* Python-based plugins are now creating an own Venv (stored at the codeLocation) to avoid dependency conflicts
+* MappingAdministration endpoints /api/v1/mappingAdministration/types|reloadTypes were renamed to /api/v1/mappingAdministration/plugins|reloadPlugins
+* MappingAdministration /api/v1/mappingAdministration/reloadPlugins (GET) and PUT|POST /api/v1/mappingAdministration are now secured and can only be accessed from localhost (if authentication is disabled) or by users with the group role defined by property *mapping-service.mappingAdminRole*
 
 ## [1.1.1] - date 2025-01-20
 ### Fixed

gradle.properties

@@ -3,6 +3,6 @@ systemProp.jdk.tls.client.protocols="TLSv1,TLSv1.1,TLSv1.2"
 // Name of project
 rootProject.name = 'mapping-service'
 // Version of project
-version=1.1.2-SNAPSHOT
+version=2.0.0-SNAPSHOT
 action.custom-1=allTests
 action.custom-1.args=--configure-on-demand -w -x check 

settings/application-default.properties

@@ -75,4 +75,11 @@ mapping-service.authEnabled:false
 mapping-service.mappingAdminRole:MAPPING_ADMIN
 
 repo.security.enable-csrf=false
-repo.security.allowedOriginPattern=*
+repo.security.allowedOriginPattern=*
+
+keycloakjwt.jwk-url=https://<KeycloakHostname>/realms/<RealmName>/protocol/openid-connect/certs
+keycloakjwt.resource: <KeyCloak Client>
+keycloakjwt.jwt-claim: preferred_username
+keycloak.realm: <RealmName>
+keycloak.auth-server-url: https://<KeycloakHostname>/
+keycloak.resource: <RealmName>

src/main/java/edu/kit/datamanager/mappingservice/configuration/OpenApiDefinitions.java

@@ -36,7 +36,7 @@ public OpenAPI customOpenAPI() {
                 .components(new Components())
                 .info(new Info().title("Mapping-Service - RESTful API").
                         description("This webpage describes the RESTful API of the KIT Data Manager Mapping-Service.").
-                        version("1.1.2").
+                        version("2.0.0").
                         contact(
                                 new Contact().
                                         name("KIT Data Manager Support").

src/main/java/edu/kit/datamanager/mappingservice/configuration/WebSecurityConfig.java

@@ -45,8 +45,6 @@
 import org.springframework.web.cors.CorsConfigurationSource;
 import org.springframework.web.cors.UrlBasedCorsConfigurationSource;
 import org.springframework.web.servlet.handler.HandlerMappingIntrospector;
-
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Optional;
@@ -89,8 +87,7 @@ record SecuredEndpoint(String method, String pattern) {
 
         //special endpoints only accessible by mapping admin (if auth enabled) or from localhost (if auth disabled)
         List<SecuredEndpoint> securedEndpointMatchers = List.of(
-                new SecuredEndpoint("POST", "/api/v1/mappingAdministration/types/*/execute"),
-                new SecuredEndpoint("GET", "/api/v1/mappingAdministration/reloadTypes"),
+                new SecuredEndpoint("GET", "/api/v1/mappingAdministration/reloadPlugins"),
                 new SecuredEndpoint("PUT", "/api/v1/mappingAdministration"),
                 new SecuredEndpoint("POST", "/api/v1/mappingAdministration")
         );

src/main/java/edu/kit/datamanager/mappingservice/domain/MappingRecord.java

@@ -51,7 +51,7 @@ public class MappingRecord implements EtagSupport, Serializable {
     @NotNull(message = "The unique identify of the record.")
     private String mappingId;
 
-    @NotNull(message = "Type of the mapping, e.g. GEMMA, XSLT, handlebars, ....")
+    @NotNull(message = "Id of the plugin used for the mapping, e.g. GEMMA_v1.0.0, XSLT_v2.1.0, handlebars_v2.0.1, ....")
     private String mappingType;
 
     @NotNull(message = "Title of the mapping.")

src/main/java/edu/kit/datamanager/mappingservice/rest/IMappingAdministrationController.java

@@ -25,7 +25,6 @@
 import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import io.swagger.v3.oas.annotations.responses.ApiResponses;
-import jakarta.servlet.http.HttpServletRequest;
 import jakarta.servlet.http.HttpServletResponse;
 import org.springdoc.core.converters.models.PageableAsQueryParam;
 import org.springframework.core.io.Resource;
@@ -51,7 +50,7 @@
 public interface IMappingAdministrationController {
 
     @Operation(summary = "Create a new mapping.", description = "This endpoint allows to create a new mapping and required two parameters: The record metadata, which contains "
-            + "the mapping identifier and mapping type, and the mapping document, which defines the rules for the mapping applied by the given mapping type. ",
+            + "the mapping identifier and mapping plugin id, and the mapping document, which defines the rules for the mapping applied by the given mapping plugin. ",
             responses = {
                 @ApiResponse(responseCode = "201", description = "CREATED is returned only if the record has been validated, persisted and the mapping document was successfully validated and stored.", content = @Content(schema = @Schema(implementation = MappingRecord.class))),
                 @ApiResponse(responseCode = "400", description = "BAD_REQUEST is returned if the provided mapping record or the mapping document are invalid."),
@@ -63,7 +62,7 @@ public interface IMappingAdministrationController {
     ResponseEntity<MappingRecord> createMapping(
             @Parameter(description = "JSON representation of the mapping record.", required = true) @RequestPart(name = "record") final MultipartFile record,
             @Parameter(description = "The mapping document associated with the record. "
-                    + "The format of the document is defined by the mapping type, which is given by the mappingType attribute of the mapping record.", required = true) @RequestPart(name = "document") final MultipartFile document,
+                    + "The format of the document is defined by the mapping plugin, which is given by the mappingPluginId attribute of the mapping record.", required = true) @RequestPart(name = "document") final MultipartFile document,
             final WebRequest request,
             final HttpServletResponse response,
             final UriComponentsBuilder uriBuilder) throws URISyntaxException;
@@ -95,15 +94,15 @@ ResponseEntity<Resource> getMappingDocumentById(
             HttpServletResponse hsr);
 
     @Operation(summary = "Get all mapping records.", description = "List all mapping records in a paginated and/or sorted form. The listing can be "
-            + "refined by providing a typeId in order to return only mapping for a certain mapping type. If not typeId is provided, all mapping "
+            + "refined by providing a mappingPluginId in order to return only mapping for a certain mapping plugin. If no mappingPluginId is provided, all mapping "
             + "records are returned.",
             responses = {
                 @ApiResponse(responseCode = "200", description = "OK and a list of records, which might be empty.", content = @Content(array = @ArraySchema(schema = @Schema(implementation = MappingRecord.class))))})
     @RequestMapping(value = {"/"}, method = {RequestMethod.GET})
     @PageableAsQueryParam
     @ResponseBody
     ResponseEntity<List<MappingRecord>> getMappings(
-            @Parameter(description = "The type identifier linked to a mapping type.") @RequestParam(value = "typeId") String typeId,
+            @Parameter(description = "The plugin identifier linked to a mapping plugin.") @RequestParam(value = "mappingPluginId") String mappingPluginId,
             Pageable pgbl,
             WebRequest wr,
             HttpServletResponse hsr,
@@ -123,7 +122,7 @@ ResponseEntity<MappingRecord> updateMapping(
             @Parameter(description = "The mapping identifier.", required = true) @PathVariable(value = "mappingId") String mappingId,
             @Parameter(description = "JSON representation of the metadata record.") @RequestPart(name = "record") final MultipartFile record,
             @Parameter(description = "The mapping document associated with the record. "
-                    + "The format of the document is defined by the mapping type, which is given by the mappingType attribute of the mapping record.") @RequestPart(name = "document") final MultipartFile document,
+                    + "The format of the document is defined by the mapping plugin, which is given by the mappingPluginId attribute of the mapping record.") @RequestPart(name = "document") final MultipartFile document,
             final WebRequest request,
             final HttpServletResponse response,
             final UriComponentsBuilder uriBuilder
@@ -146,40 +145,17 @@ ResponseEntity<Void> deleteMapping(
     @Operation(summary = "Get all available plugins.",
             responses = {
                 @ApiResponse(responseCode = "200", description = "OK and a list of all plugins will be returned, which might be empty.", content = @Content(array = @ArraySchema(schema = @Schema(implementation = PluginInformation.class))))})
-    @RequestMapping(value = {"/types"}, method = {RequestMethod.GET})
+    @RequestMapping(value = {"/plugins"}, method = {RequestMethod.GET})
     @ResponseBody
     ResponseEntity<List<PluginInformation>> getAvailablePlugins(
             WebRequest wr,
             HttpServletResponse hsr);
 
-    @Operation(summary = "Reload all mapping types.", description = "Reloads all plugins from the plugin directory and updates their dependencies if necessary.",
+    @Operation(summary = "Reload all mapping plugins.", description = "Reloads all plugins from the plugin directory and updates their dependencies if necessary.",
             responses = {
                 @ApiResponse(responseCode = "204", description = "NO_CONTENT is returned on a successful refresh.")})
-    @RequestMapping(value = {"/reloadTypes"}, method = {RequestMethod.GET})
+    @RequestMapping(value = {"/reloadPlugins"}, method = {RequestMethod.GET})
     ResponseEntity<String> reloadAvailablePlugins(
             WebRequest wr,
             HttpServletResponse hsr);
-
-    @Operation(summary = "Map a document directly using the provided plugin.", description = "This endpoint allows the mapping of documents via a file upload. "
-            + "The identifier of the plugin must be passed to this endpoint as parameters together with the document to be mapped and the mapping rules.", responses = {
-            @ApiResponse(responseCode = "200", description = "OK is returned if the mapping was successful. "
-                    + "The result will also be returned in the response."),
-            @ApiResponse(responseCode = "404", description = "NOT_FOUND is returned if no plugin for typeID could be found."),
-            @ApiResponse(responseCode = "400", description = "BAD_REQUEST is returned if a parameter is missing or the mapping could not be performed with the provided input. It is "
-                    + "expected that a mapping plugin accepts a well defined input and produces results for proper inputs. Therefore, only a faulty input "
-                    + "document should be the reason for a mapper to fail."),
-            @ApiResponse(responseCode = "500", description = "INTERNAL_SERVER_ERROR is returned the mapping returned successfully, but the mapping result "
-                    + "is not accessible. This is expected to be an error in the mapping implementation and should be fixed in there.")})
-    @RequestMapping(value = {"/types/{typeID}/execute"}, method = {RequestMethod.POST}, consumes = {MediaType.MULTIPART_FORM_DATA_VALUE})
-    @ResponseBody
-    void runPlugin(
-            @Parameter(description = "The document to be mapped.", required = true) @RequestPart(name = "document") final MultipartFile document,
-            @Parameter(description = "The mapping rules document.", required = true) @RequestPart(name = "mapping") final MultipartFile mapping,
-            @Parameter(description = "The typeID of the plugin to execute.", required = true) @PathVariable(value = "typeID") String typeID,
-            final HttpServletRequest request,
-            final HttpServletResponse response,
-            final UriComponentsBuilder uriBuilder) throws URISyntaxException;
-
-
-
 }

src/main/java/edu/kit/datamanager/mappingservice/rest/IMappingExecutionController.java

@@ -115,4 +115,24 @@ ResponseEntity<JobStatus> getJobStatus(
                 @ApiResponse(responseCode = "500", description = "INTERNAL_SERVER_ERROR if the mapping job has failed.")})
     @DeleteMapping(path = "/schedule/{job-id}")
     ResponseEntity<Void> deleteJobAndAssociatedData(@PathVariable(name = "job-id") String jobId) throws Throwable;
+
+    @Operation(summary = "Map a document directly using the provided plugin.", description = "This endpoint allows the mapping of documents via a file upload. "
+            + "The identifier of the plugin must be passed to this endpoint as parameters together with the document to be mapped and the mapping rules.", responses = {
+            @ApiResponse(responseCode = "200", description = "OK is returned if the mapping was successful. "
+                    + "The result will also be returned in the response."),
+            @ApiResponse(responseCode = "404", description = "NOT_FOUND is returned if no plugin for pluginId could be found."),
+            @ApiResponse(responseCode = "400", description = "BAD_REQUEST is returned if a parameter is missing or the mapping could not be performed with the provided input. It is "
+                    + "expected that a mapping plugin accepts a well defined input and produces results for proper inputs. Therefore, only a faulty input "
+                    + "document should be the reason for a mapper to fail."),
+            @ApiResponse(responseCode = "500", description = "INTERNAL_SERVER_ERROR is returned the mapping returned successfully, but the mapping result "
+                    + "is not accessible. This is expected to be an error in the mapping implementation and should be fixed in there.")})
+    @RequestMapping(value = {"/plugins/{pluginId}/execute"}, method = {RequestMethod.POST}, consumes = {MediaType.MULTIPART_FORM_DATA_VALUE})
+    @ResponseBody
+    void runPlugin(
+            @Parameter(description = "The document to be mapped.", required = true) @RequestPart(name = "document") final MultipartFile document,
+            @Parameter(description = "The mapping rules document.", required = true) @RequestPart(name = "mapping") final MultipartFile mapping,
+            @Parameter(description = "The pluginId of the plugin to execute.", required = true) @PathVariable(value = "pluginId") String pluginId,
+            final HttpServletRequest request,
+            final HttpServletResponse response,
+            final UriComponentsBuilder uriBuilder) throws URISyntaxException;
 }