Fix FDP v2 index API (#637)

dennisvang · web-flow · commit 4893d666dd15 · 2025-03-10T09:23:28.000+01:00
* replace `modificationTime`, `registrationTime`, and `astRetrievalTime` by `updatedAt`, `createdAt`, and `lastRetrievalAt`, respectively. * replace setCreatedAt by setLastRetrievalAt before save * add tests to reproduce issues #633, #643, #644 * set lastRetrievalAt for newIndexEntry test fixture * sort by 'createdAt' instead of 'created' (fixes #643) * add parameterized types in entry/Detail_GET * add IndexEntry.events mapped to IndexEvent.relatedTo with cascade all (fixes #644) * implement sort parameter validation for /index/entries endpoint based on an explicit list of allowed fields IndexEntryController.sortFields * update changelog
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -17,6 +17,7 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 - Separate and reuse github workflows by @dennisvang in #617
 - Prevent duplicate workflow runs by @dennisvang in #628
 - Clean up Dockerfile by @dennisvang in #626
+- Fix index API by @dennisvang in #637 (backward incompatible)
 
 ## [1.17.2]
 
diff --git a/src/main/java/org/fairdatapoint/api/controller/exception/ExceptionControllerAdvice.java b/src/main/java/org/fairdatapoint/api/controller/exception/ExceptionControllerAdvice.java
@@ -242,5 +242,4 @@ private ErrorDTO handleInvalidSparqlQuery(Exception exception) {
         );
         return new ErrorDTO(HttpStatus.BAD_REQUEST, message, details);
     }
-
 }
diff --git a/src/main/java/org/fairdatapoint/api/controller/index/IndexEntryController.java b/src/main/java/org/fairdatapoint/api/controller/index/IndexEntryController.java
@@ -44,6 +44,7 @@
 import org.springframework.http.MediaType;
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.web.bind.annotation.*;
+import org.springframework.web.server.ResponseStatusException;
 
 import java.util.List;
 import java.util.Optional;
@@ -54,6 +55,9 @@
 @RequestMapping("/index/entries")
 @RequiredArgsConstructor
 public class IndexEntryController {
+    private static final List<String> SORT_FIELDS = List.of(
+            "clientUrl", "createdAt", "lastRetrievalAt", "permit", "status", "updatedAt"
+    );
 
     private final IndexEntryService service;
 
@@ -71,6 +75,14 @@ public Page<IndexEntryDTO> getEntriesPage(
             @RequestParam(required = false, defaultValue = "") String state,
             @RequestParam(required = false, defaultValue = "accepted") String permit
     ) {
+        // validate sort query parameters
+        // todo: implement validator for Pageable if used more often
+        pageable.getSort().stream().forEach(order -> {
+            if (!SORT_FIELDS.contains(order.getProperty())) {
+                throw new ResponseStatusException(
+                        HttpStatus.UNPROCESSABLE_ENTITY, "Invalid sort parameter: " + order.getProperty());
+            }
+        });
         return service.getEntriesPageDTOs(pageable, state, permit);
     }
 
diff --git a/src/main/java/org/fairdatapoint/api/dto/index/entry/IndexEntryDTO.java b/src/main/java/org/fairdatapoint/api/dto/index/entry/IndexEntryDTO.java
@@ -52,8 +52,8 @@ public class IndexEntryDTO {
     private IndexEntryPermit permit;
 
     @NotNull
-    private String registrationTime;
+    private String createdAt;
 
     @NotNull
-    private String modificationTime;
+    private String updatedAt;
 }
diff --git a/src/main/java/org/fairdatapoint/api/dto/index/entry/IndexEntryDetailDTO.java b/src/main/java/org/fairdatapoint/api/dto/index/entry/IndexEntryDetailDTO.java
@@ -61,12 +61,12 @@ public class IndexEntryDetailDTO {
     private List<EventDTO> events;
 
     @NotNull
-    private String registrationTime;
+    private String createdAt;
 
     @NotNull
-    private String modificationTime;
+    private String updatedAt;
 
     @NotNull
-    private String lastRetrievalTime;
+    private String lastRetrievalAt;
 
 }
diff --git a/src/main/java/org/fairdatapoint/entity/index/entry/IndexEntry.java b/src/main/java/org/fairdatapoint/entity/index/entry/IndexEntry.java
@@ -28,14 +28,17 @@
 import lombok.*;
 import lombok.experimental.SuperBuilder;
 import org.fairdatapoint.entity.base.BaseEntity;
+import org.fairdatapoint.entity.index.event.IndexEvent;
 import org.hibernate.annotations.JdbcType;
 import org.hibernate.annotations.Type;
 import org.hibernate.dialect.PostgreSQLEnumJdbcType;
 
 import java.time.Duration;
 import java.time.Instant;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 
 @Entity(name = "IndexEntry")
 @Table(name = "index_entry")
@@ -79,6 +82,9 @@ public class IndexEntry extends BaseEntity {
     @Column(name = "metadata", columnDefinition = "jsonb", nullable = false)
     private Map<String, String> metadata = new HashMap<>();
 
+    @OneToMany(mappedBy = "relatedTo", orphanRemoval = true, cascade = CascadeType.ALL, fetch = FetchType.LAZY)
+    private Set<IndexEvent> events = new HashSet<>();
+
     public Duration getLastRetrievalAgo() {
         if (getLastRetrievalAt() == null) {
             return null;
diff --git a/src/main/java/org/fairdatapoint/service/index/entry/IndexEntryMapper.java b/src/main/java/org/fairdatapoint/service/index/entry/IndexEntryMapper.java
@@ -50,6 +50,7 @@ public IndexEntryDTO toDTO(IndexEntry indexEntry, Instant validThreshold) {
                         indexEntry.getLastRetrievalAt(),
                         validThreshold),
                 indexEntry.getPermit(),
+                // convert to ISO 8601
                 indexEntry.getCreatedAt().toString(),
                 indexEntry.getUpdatedAt().toString()
         );
@@ -69,18 +70,19 @@ public IndexEntryDetailDTO toDetailDTO(
                 StreamSupport.stream(events.spliterator(), false)
                         .map(eventMapper::toDTO)
                         .toList(),
+                // convert to ISO 8601
                 indexEntry.getCreatedAt().toString(),
                 indexEntry.getUpdatedAt().toString(),
                 indexEntry.getLastRetrievalAt().toString()
         );
     }
 
     public IndexEntryStateDTO toStateDTO(
-            IndexEntryState state, Instant lastRetrievalTime, Instant validThreshold
+            IndexEntryState state, Instant lastRetrievalAt, Instant validThreshold
     ) {
         return switch (state) {
             case UNKNOWN -> IndexEntryStateDTO.UNKNOWN;
-            case VALID -> lastRetrievalTime.isAfter(validThreshold)
+            case VALID -> lastRetrievalAt.isAfter(validThreshold)
                     ? IndexEntryStateDTO.ACTIVE
                     : IndexEntryStateDTO.INACTIVE;
             case INVALID -> IndexEntryStateDTO.INVALID;
diff --git a/src/main/java/org/fairdatapoint/service/index/entry/IndexEntryService.java b/src/main/java/org/fairdatapoint/service/index/entry/IndexEntryService.java
@@ -190,7 +190,7 @@ private Iterable<IndexEvent> getEvents(UUID indexEntryUuid) {
 
     public Iterable<IndexEvent> getEvents(IndexEntry indexEntry) {
         return eventRepository.getAllByRelatedTo(indexEntry,
-                PageRequest.of(0, PAGE_SIZE, Sort.by(Sort.Direction.DESC, "created")));
+                PageRequest.of(0, PAGE_SIZE, Sort.by(Sort.Direction.DESC, "createdAt")));
     }
 
     @RequiredEnabledIndexFeature
diff --git a/src/main/java/org/fairdatapoint/service/index/event/EventService.java b/src/main/java/org/fairdatapoint/service/index/event/EventService.java
@@ -177,6 +177,7 @@ private void processMetadataRetrieval(IndexEvent event) {
                         event.getPayload().getMetadataRetrieval().setMetadata(metadata.get());
                         event.getRelatedTo().setCurrentMetadata(metadata.get());
                         event.getRelatedTo().setState(IndexEntryState.VALID);
+                        event.getRelatedTo().setLastRetrievalAt(Instant.now());
                         log.info("Storing metadata for {}", clientUrl);
                         indexEntryRepository.save(event.getRelatedTo());
                     }
@@ -201,7 +202,6 @@ private void processMetadataRetrieval(IndexEvent event) {
             log.info("Rate limit reached for {} (skipping metadata retrieval)", clientUrl);
             event.getPayload().getMetadataRetrieval().setError("Rate limit reached (skipping)");
         }
-        event.getRelatedTo().setCreatedAt(Instant.now());
         event.finish();
         final IndexEvent newEvent = eventRepository.save(event);
         indexEntryRepository.save(newEvent.getRelatedTo());
diff --git a/src/test/java/org/fairdatapoint/acceptance/index/entry/Detail_DELETE.java b/src/test/java/org/fairdatapoint/acceptance/index/entry/Detail_DELETE.java
@@ -0,0 +1,94 @@
+/**
+ * The MIT License
+ * Copyright © 2016-2024 FAIR Data Team
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+package org.fairdatapoint.acceptance.index.entry;
+
+import org.fairdatapoint.WebIntegrationTest;
+import org.fairdatapoint.api.dto.index.entry.IndexEntryDTO;
+import org.fairdatapoint.database.db.repository.IndexEntryRepository;
+import org.fairdatapoint.database.db.repository.IndexEventRepository;
+import org.fairdatapoint.entity.index.entry.IndexEntry;
+import org.fairdatapoint.entity.index.event.IndexEvent;
+import org.fairdatapoint.entity.index.event.IndexEventPayload;
+import org.fairdatapoint.entity.index.event.payload.AdminTrigger;
+import org.fairdatapoint.utils.CustomPageImpl;
+import org.fairdatapoint.utils.TestIndexEntryFixtures;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.ParameterizedTypeReference;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.RequestEntity;
+import org.springframework.http.ResponseEntity;
+
+import java.net.URI;
+import java.util.UUID;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.IsEqual.equalTo;
+
+@DisplayName("DELETE /index/entries/{uuid}")
+public class Detail_DELETE extends WebIntegrationTest {
+    @Autowired
+    private IndexEntryRepository indexEntryRepository;
+
+    @Autowired
+    private IndexEventRepository indexEventRepository;
+
+    private final ParameterizedTypeReference<CustomPageImpl<IndexEntryDTO>> responseType =
+            new ParameterizedTypeReference<>() {};
+
+    private URI url(UUID uuid) {
+        return URI.create("/index/entries/" + uuid);
+    }
+
+    @Test
+    @DisplayName("HTTP 204: delete detail with event")
+    public void res204_deleteDetailWithEvent() {
+        // test for issue #644
+
+        // GIVEN: prepare data
+        indexEntryRepository.deleteAll();
+        IndexEntry entry = TestIndexEntryFixtures.entryExample();
+        indexEntryRepository.save(entry);
+        indexEventRepository.deleteAll();
+        // minimal event related to entry
+        // todo: create TestIndexEventFixtures utility if used more often
+        IndexEvent event = new IndexEvent(0, new AdminTrigger());
+        event.setRelatedTo(entry);
+        indexEventRepository.save(event);
+
+        // AND: prepare request
+        RequestEntity<Void> request = RequestEntity
+                .delete(url(entry.getUuid()))
+                .header(HttpHeaders.AUTHORIZATION, ADMIN_TOKEN)
+                .build();
+
+        // WHEN
+        ResponseEntity<CustomPageImpl<IndexEntryDTO>> result = client.exchange(request, responseType);
+
+        // THEN
+        assertThat("Correct response code", result.getStatusCode(), is(equalTo(HttpStatus.NO_CONTENT)));
+    }
+}
diff --git a/src/test/java/org/fairdatapoint/acceptance/index/entry/Detail_GET.java b/src/test/java/org/fairdatapoint/acceptance/index/entry/Detail_GET.java
@@ -0,0 +1,81 @@
+/**
+ * The MIT License
+ * Copyright © 2016-2024 FAIR Data Team
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+package org.fairdatapoint.acceptance.index.entry;
+
+import org.fairdatapoint.WebIntegrationTest;
+import org.fairdatapoint.api.dto.index.entry.IndexEntryDTO;
+import org.fairdatapoint.database.db.repository.IndexEntryRepository;
+import org.fairdatapoint.entity.index.entry.IndexEntry;
+import org.fairdatapoint.utils.CustomPageImpl;
+import org.fairdatapoint.utils.TestIndexEntryFixtures;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.ParameterizedTypeReference;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
+import org.springframework.http.RequestEntity;
+import org.springframework.http.ResponseEntity;
+
+import java.net.URI;
+import java.util.UUID;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.IsEqual.equalTo;
+
+@DisplayName("GET /index/entries/{uuid}")
+public class Detail_GET extends WebIntegrationTest {
+    @Autowired
+    private IndexEntryRepository indexEntryRepository;
+
+    private final ParameterizedTypeReference<CustomPageImpl<IndexEntryDTO>> responseType =
+            new ParameterizedTypeReference<>() {};
+
+    private URI url(UUID uuid) {
+        return URI.create("/index/entries/" + uuid);
+    }
+
+    @Test
+    @DisplayName("HTTP 200: get detail")
+    public void res200_getDetail() {
+        // test for issue #643
+
+        // GIVEN: prepare data
+        indexEntryRepository.deleteAll();
+        IndexEntry entry =  TestIndexEntryFixtures.entryExample();
+        indexEntryRepository.save(entry);
+
+        // AND: prepare request
+        RequestEntity<Void> request = RequestEntity
+                .get(url(entry.getUuid()))
+                .accept(MediaType.APPLICATION_JSON)
+                .build();
+
+        // WHEN
+        ResponseEntity<CustomPageImpl<IndexEntryDTO>> result = client.exchange(request, responseType);
+
+        // THEN
+        assertThat("Correct response code is received", result.getStatusCode(), is(equalTo(HttpStatus.OK)));
+    }
+}
diff --git a/src/test/java/org/fairdatapoint/acceptance/index/entry/List_GET.java b/src/test/java/org/fairdatapoint/acceptance/index/entry/List_GET.java
@@ -31,6 +31,8 @@
 import org.fairdatapoint.utils.TestIndexEntryFixtures;
 import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.ParameterizedTypeReference;
 import org.springframework.http.*;
@@ -77,6 +79,12 @@ private URI urlWithPermit(String permitQuery) {
                 .build().toUri();
     }
 
+    private URI urlWithSorting(String sortQuery) {
+        return UriComponentsBuilder.fromUri(url())
+                .queryParam("sort", sortQuery)
+                .build().toUri();
+    }
+
     @Test
     @DisplayName("HTTP 200: page empty")
     public void res200_pageEmpty() {
@@ -163,6 +171,39 @@ public void res200_pageFew() {
         }
     }
 
+    @ParameterizedTest
+    @CsvSource({
+            "'updatedAt,desc', OK",  // existing field
+            "'modificationTime,desc', UNPROCESSABLE_ENTITY"  // non-existent field
+    })
+    @DisplayName("HTTP 200: page few (sorted)")
+    public void res200_pageFewSorted(String sortQuery, String expectedStatus) {
+        // Test for issue #633
+        //
+        // https://www.rfc-editor.org/rfc/rfc9110.html
+        //
+        // todo: The res200_ naming convention is a bit restrictive and does not cover
+        //  multiple cases. Should we rename to regression tests or something?
+
+        // GIVEN: prepare data
+        indexEntryRepository.deleteAll();
+        List<IndexEntry> entries = TestIndexEntryFixtures.entriesFew();
+        indexEntryRepository.saveAll(entries);
+
+        // AND: prepare request
+        RequestEntity<?> request = RequestEntity
+                .get(urlWithSorting(sortQuery))
+                .accept(MediaType.APPLICATION_JSON)
+                .build();
+
+        // WHEN
+        ResponseEntity<CustomPageImpl<IndexEntryDTO>> result = client.exchange(request, responseType);
+
+        // THEN
+        assertThat("Correct response code is received",
+                result.getStatusCode(), is(equalTo(HttpStatus.valueOf(expectedStatus))));
+    }
+
     @Test
     @DisplayName("HTTP 200: page many (middle)")
     public void res200_pageManyMiddle() {
diff --git a/src/test/java/org/fairdatapoint/utils/TestIndexEntryFixtures.java b/src/test/java/org/fairdatapoint/utils/TestIndexEntryFixtures.java

Original file line number	Diff line number	Diff line change
`@@ -242,5 +242,4 @@ private ErrorDTO handleInvalidSparqlQuery(Exception exception) {`
`242`	`242`	`);`
`243`	`243`	`return new ErrorDTO(HttpStatus.BAD_REQUEST, message, details);`
`244`	`244`	`}`
`245`		`-`
`246`	`245`	`}`
Original file line number	Diff line number	Diff line change
`@@ -52,8 +52,8 @@ public class IndexEntryDTO {`
`52`	`52`	`private IndexEntryPermit permit;`
`53`	`53`
`54`	`54`	`@NotNull`
`55`		`- private String registrationTime;`
	`55`	`+ private String createdAt;`
`56`	`56`
`57`	`57`	`@NotNull`
`58`		`- private String modificationTime;`
	`58`	`+ private String updatedAt;`
`59`	`59`	`}`
Original file line number	Diff line number	Diff line change
`@@ -190,7 +190,7 @@ private Iterable<IndexEvent> getEvents(UUID indexEntryUuid) {`
`190`	`190`
`191`	`191`	`public Iterable<IndexEvent> getEvents(IndexEntry indexEntry) {`
`192`	`192`	`return eventRepository.getAllByRelatedTo(indexEntry,`
`193`		`- PageRequest.of(0, PAGE_SIZE, Sort.by(Sort.Direction.DESC, "created")));`
	`193`	`+ PageRequest.of(0, PAGE_SIZE, Sort.by(Sort.Direction.DESC, "createdAt")));`
`194`	`194`	`}`
`195`	`195`
`196`	`196`	`@RequiredEnabledIndexFeature`