Skip to content

Commit 291c91c

Browse files
nscuronscuro
committed
Implement file storage plugin
Signed-off-by: nscuro <nscuro@protonmail.com>
1 parent c878dd1 commit 291c91c

24 files changed

+2122
-129
lines changed

‎pom.xml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -543,6 +543,12 @@
543543
<version>${lib.testcontainers.version}</version>
544544
<scope>test</scope>
545545
</dependency>
546+
<dependency>
547+
<groupId>org.testcontainers</groupId>
548+
<artifactId>minio</artifactId>
549+
<version>${lib.testcontainers.version}</version>
550+
<scope>test</scope>
551+
</dependency>
546552
<dependency>
547553
<groupId>org.testcontainers</groupId>
548554
<artifactId>postgresql</artifactId>

‎src/main/java/org/dependencytrack/event/BomUploadEvent.java

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,7 @@
2020

2121
import alpine.event.framework.AbstractChainableEvent;
2222
import org.dependencytrack.model.Project;
23-
24-
import java.io.File;
23+
import org.dependencytrack.proto.storage.v1alpha1.FileMetadata;
2524

2625
/**
2726
* Defines an event triggered when a bill-of-material (bom) document is submitted.
@@ -32,18 +31,18 @@
3231
public class BomUploadEvent extends AbstractChainableEvent {
3332

3433
private final Project project;
35-
private final File file;
34+
private final FileMetadata fileMetadata;
3635

37-
public BomUploadEvent(final Project project, final File file) {
36+
public BomUploadEvent(final Project project, final FileMetadata fileMetadata) {
3837
this.project = project;
39-
this.file = file;
38+
this.fileMetadata = fileMetadata;
4039
}
4140

4241
public Project getProject() {
4342
return project;
4443
}
4544

46-
public File getFile() {
47-
return file;
45+
public FileMetadata getFileMetadata() {
46+
return fileMetadata;
4847
}
4948
}

‎src/main/java/org/dependencytrack/resources/v1/BomResource.java

Lines changed: 33 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -34,23 +34,6 @@
3434
import io.swagger.v3.oas.annotations.security.SecurityRequirement;
3535
import io.swagger.v3.oas.annotations.security.SecurityRequirements;
3636
import io.swagger.v3.oas.annotations.tags.Tag;
37-
import jakarta.json.Json;
38-
import jakarta.json.JsonArray;
39-
import jakarta.json.JsonReader;
40-
import jakarta.json.JsonString;
41-
import jakarta.validation.Validator;
42-
import jakarta.ws.rs.Consumes;
43-
import jakarta.ws.rs.DefaultValue;
44-
import jakarta.ws.rs.GET;
45-
import jakarta.ws.rs.POST;
46-
import jakarta.ws.rs.PUT;
47-
import jakarta.ws.rs.Path;
48-
import jakarta.ws.rs.PathParam;
49-
import jakarta.ws.rs.Produces;
50-
import jakarta.ws.rs.QueryParam;
51-
import jakarta.ws.rs.WebApplicationException;
52-
import jakarta.ws.rs.core.MediaType;
53-
import jakarta.ws.rs.core.Response;
5437
import org.apache.commons.io.IOUtils;
5538
import org.apache.commons.io.input.BOMInputStream;
5639
import org.apache.commons.lang3.StringUtils;
@@ -72,22 +55,40 @@
7255
import org.dependencytrack.parser.cyclonedx.CycloneDxValidator;
7356
import org.dependencytrack.parser.cyclonedx.InvalidBomException;
7457
import org.dependencytrack.persistence.QueryManager;
58+
import org.dependencytrack.plugin.PluginManager;
59+
import org.dependencytrack.proto.storage.v1alpha1.FileMetadata;
7560
import org.dependencytrack.resources.v1.problems.InvalidBomProblemDetails;
7661
import org.dependencytrack.resources.v1.problems.ProblemDetails;
7762
import org.dependencytrack.resources.v1.vo.BomSubmitRequest;
7863
import org.dependencytrack.resources.v1.vo.BomUploadResponse;
64+
import org.dependencytrack.storage.FileStorage;
7965
import org.glassfish.jersey.media.multipart.BodyPartEntity;
8066
import org.glassfish.jersey.media.multipart.FormDataBodyPart;
8167
import org.glassfish.jersey.media.multipart.FormDataParam;
8268

69+
import jakarta.json.Json;
70+
import jakarta.json.JsonArray;
71+
import jakarta.json.JsonReader;
72+
import jakarta.json.JsonString;
73+
import jakarta.validation.Validator;
74+
import jakarta.ws.rs.Consumes;
75+
import jakarta.ws.rs.DefaultValue;
76+
import jakarta.ws.rs.GET;
77+
import jakarta.ws.rs.POST;
78+
import jakarta.ws.rs.PUT;
79+
import jakarta.ws.rs.Path;
80+
import jakarta.ws.rs.PathParam;
81+
import jakarta.ws.rs.Produces;
82+
import jakarta.ws.rs.QueryParam;
83+
import jakarta.ws.rs.WebApplicationException;
84+
import jakarta.ws.rs.core.MediaType;
85+
import jakarta.ws.rs.core.Response;
8386
import java.io.ByteArrayInputStream;
84-
import java.io.File;
8587
import java.io.IOException;
8688
import java.io.StringReader;
8789
import java.nio.charset.StandardCharsets;
88-
import java.nio.file.Files;
89-
import java.nio.file.StandardOpenOption;
9090
import java.security.Principal;
91+
import java.time.Instant;
9192
import java.util.Arrays;
9293
import java.util.Base64;
9394
import java.util.List;
@@ -330,7 +331,7 @@ public Response uploadBom(@Parameter(required = true) BomSubmitRequest request)
330331
final String trimmedProjectName = StringUtils.trimToNull(request.getProjectName());
331332
if (request.isLatestProjectVersion()) {
332333
final Project oldLatest = qm.getLatestProjectVersion(trimmedProjectName);
333-
if(oldLatest != null && !qm.hasAccess(super.getPrincipal(), oldLatest)) {
334+
if (oldLatest != null && !qm.hasAccess(super.getPrincipal(), oldLatest)) {
334335
return Response.status(Response.Status.FORBIDDEN)
335336
.entity("Cannot create latest version for project with this name. Access to current latest " +
336337
"version is forbidden!")
@@ -436,7 +437,7 @@ public Response uploadBom(
436437
}
437438
if (isLatest) {
438439
final Project oldLatest = qm.getLatestProjectVersion(trimmedProjectName);
439-
if(oldLatest != null && !qm.hasAccess(super.getPrincipal(), oldLatest)) {
440+
if (oldLatest != null && !qm.hasAccess(super.getPrincipal(), oldLatest)) {
440441
return Response.status(Response.Status.FORBIDDEN)
441442
.entity("Cannot create latest version for project with this name. Access to current latest " +
442443
"version is forbidden!")
@@ -467,17 +468,17 @@ private Response process(QueryManager qm, Project project, String encodedBomData
467468
return Response.status(Response.Status.FORBIDDEN).entity("Access to the specified project is forbidden").build();
468469
}
469470

470-
final File bomFile;
471+
final FileMetadata bomFileMetadata;
471472
try (final var encodedInputStream = new ByteArrayInputStream(encodedBomData.getBytes(StandardCharsets.UTF_8));
472473
final var decodedInputStream = Base64.getDecoder().wrap(encodedInputStream);
473474
final var byteOrderMarkInputStream = new BOMInputStream(decodedInputStream)) {
474-
bomFile = validateAndStoreBom(IOUtils.toByteArray(byteOrderMarkInputStream), project);
475+
bomFileMetadata = validateAndStoreBom(IOUtils.toByteArray(byteOrderMarkInputStream), project);
475476
} catch (IOException e) {
476477
LOGGER.error("An unexpected error occurred while validating or storing a BOM uploaded to project: " + project.getUuid(), e);
477478
return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
478479
}
479480

480-
final BomUploadEvent bomUploadEvent = new BomUploadEvent(qm.detach(Project.class, project.getId()), bomFile);
481+
final BomUploadEvent bomUploadEvent = new BomUploadEvent(qm.detach(Project.class, project.getId()), bomFileMetadata);
481482
qm.createWorkflowSteps(bomUploadEvent.getChainIdentifier());
482483
Event.dispatch(bomUploadEvent);
483484

@@ -500,18 +501,18 @@ private Response process(QueryManager qm, Project project, List<FormDataBodyPart
500501
return Response.status(Response.Status.FORBIDDEN).entity("Access to the specified project is forbidden").build();
501502
}
502503

503-
final File bomFile;
504+
final FileMetadata bomFileMetadata;
504505
try (final var inputStream = bodyPartEntity.getInputStream();
505506
final var byteOrderMarkInputStream = new BOMInputStream(inputStream)) {
506-
bomFile = validateAndStoreBom(IOUtils.toByteArray(byteOrderMarkInputStream), project);
507+
bomFileMetadata = validateAndStoreBom(IOUtils.toByteArray(byteOrderMarkInputStream), project);
507508
} catch (IOException e) {
508509
LOGGER.error("An unexpected error occurred while validating or storing a BOM uploaded to project: " + project.getUuid(), e);
509510
return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
510511
}
511512

512513
// todo: make option to combine all the bom data so components are reconciled in a single pass.
513514
// todo: https://github.com/DependencyTrack/dependency-track/issues/130
514-
final BomUploadEvent bomUploadEvent = new BomUploadEvent(qm.detach(Project.class, project.getId()), bomFile);
515+
final BomUploadEvent bomUploadEvent = new BomUploadEvent(qm.detach(Project.class, project.getId()), bomFileMetadata);
515516

516517
qm.createWorkflowSteps(bomUploadEvent.getChainIdentifier());
517518
Event.dispatch(bomUploadEvent);
@@ -526,21 +527,12 @@ private Response process(QueryManager qm, Project project, List<FormDataBodyPart
526527
return Response.ok().build();
527528
}
528529

529-
private File validateAndStoreBom(final byte[] bomBytes, final Project project) throws IOException {
530+
private FileMetadata validateAndStoreBom(final byte[] bomBytes, final Project project) throws IOException {
530531
validate(bomBytes, project);
531532

532-
// TODO: Store externally so other instances of the API server can pick it up.
533-
// https://github.com/CycloneDX/cyclonedx-bom-repo-server
534-
final java.nio.file.Path tmpPath = Files.createTempFile("dtrack-bom-%s".formatted(project.getUuid()), null);
535-
final File tmpFile = tmpPath.toFile();
536-
tmpFile.deleteOnExit();
537-
538-
LOGGER.debug("Writing BOM for project %s to %s".formatted(project.getUuid(), tmpPath));
539-
try (final var tmpOutputStream = Files.newOutputStream(tmpPath, StandardOpenOption.WRITE)) {
540-
tmpOutputStream.write(bomBytes);
533+
try (final var fileStorage = PluginManager.getInstance().getExtension(FileStorage.class)) {
534+
return fileStorage.store("bom-upload/%s_%s".formatted(Instant.now().toEpochMilli(), project.getUuid()), bomBytes);
541535
}
542-
543-
return tmpFile;
544536
}
545537

546538
static void validate(final byte[] bomBytes, final Project project) {
@@ -634,7 +626,7 @@ private static boolean shouldValidate(final Project project) {
634626
.map(org.dependencytrack.model.Tag::getName)
635627
.anyMatch(validationModeTags::contains);
636628
return (validationMode == BomValidationMode.ENABLED_FOR_TAGS && doTagsMatch)
637-
|| (validationMode == BomValidationMode.DISABLED_FOR_TAGS && !doTagsMatch);
629+
|| (validationMode == BomValidationMode.DISABLED_FOR_TAGS && !doTagsMatch);
638630
}
639631
}
640632
}

‎src/main/java/org/dependencytrack/storage/FileStorage.java

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
/*
2+
* This file is part of Dependency-Track.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*
16+
* SPDX-License-Identifier: Apache-2.0
17+
* Copyright (c) OWASP Foundation. All Rights Reserved.
18+
*/
19+
package org.dependencytrack.storage;
20+
21+
import org.dependencytrack.plugin.api.ExtensionPoint;
22+
import org.dependencytrack.proto.storage.v1alpha1.FileMetadata;
23+
24+
import java.io.FileNotFoundException;
25+
import java.io.IOException;
26+
import java.util.regex.Pattern;
27+
28+
import static java.util.Objects.requireNonNull;
29+
30+
/**
31+
* @since 5.6.0
32+
*/
33+
public interface FileStorage extends ExtensionPoint {
34+
35+
String METADATA_KEY_SHA256_DIGEST = "sha256_digest";
36+
Pattern VALID_NAME_PATTERN = Pattern.compile("[a-zA-Z0-9_/\\-.]+");
37+
38+
/**
39+
* Persist data to a file in storage.
40+
* <p>
41+
* Storage providers may transparently perform additional steps,
42+
* such as encryption and compression.
43+
*
44+
* @param fileName Name of the file. This fileName is not guaranteed to be reflected
45+
* in storage as-is. It may be modified or changed entirely.
46+
* @param content Data to store.
47+
* @return Metadata of the stored file.
48+
* @throws IOException When storing the file failed.
49+
*/
50+
FileMetadata store(final String fileName, final byte[] content) throws IOException;
51+
52+
/**
53+
* Retrieves a file from storage.
54+
* <p>
55+
* Storage providers may transparently perform additional steps,
56+
* such as integrity verification, decryption and decompression.
57+
* <p>
58+
* Trying to retrieve a file from a different storage provider
59+
* is an illegal operation and yields an exception.
60+
*
61+
* @param fileMetadata Metadata of the file to retrieve.
62+
* @return The file's content.
63+
* @throws IOException When retrieving the file failed.
64+
* @throws FileNotFoundException When the requested file was not found.
65+
*/
66+
byte[] get(final FileMetadata fileMetadata) throws IOException;
67+
68+
/**
69+
* Deletes a file from storage.
70+
* <p>
71+
* Trying to delete a file from a different storage provider
72+
* is an illegal operation and yields an exception.
73+
*
74+
* @param fileMetadata Metadata of the file to delete.
75+
* @return {@code true} when the file was deleted, otherwise {@code false}.
76+
* @throws IOException When deleting the file failed.
77+
*/
78+
boolean delete(final FileMetadata fileMetadata) throws IOException;
79+
80+
// TODO: deleteMany. Some remote storage backends support batch deletes.
81+
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html
82+
83+
static void requireValidFileName(final String fileName) {
84+
requireNonNull(fileName, "fileName must not be null");
85+
86+
if (!VALID_NAME_PATTERN.matcher(fileName).matches()) {
87+
throw new IllegalArgumentException("fileName must match pattern: " + VALID_NAME_PATTERN.pattern());
88+
}
89+
}
90+
91+
class ExtensionPointMetadata implements org.dependencytrack.plugin.api.ExtensionPointMetadata<FileStorage> {
92+
93+
@Override
94+
public String name() {
95+
return "file.storage";
96+
}
97+
98+
@Override
99+
public boolean required() {
100+
return true;
101+
}
102+
103+
@Override
104+
public Class<FileStorage> extensionPointClass() {
105+
return FileStorage.class;
106+
}
107+
108+
}
109+
110+
}

‎src/main/java/org/dependencytrack/storage/FileStoragePlugin.java

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
/*
2+
* This file is part of Dependency-Track.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*
16+
* SPDX-License-Identifier: Apache-2.0
17+
* Copyright (c) OWASP Foundation. All Rights Reserved.
18+
*/
19+
package org.dependencytrack.storage;
20+
21+
import org.dependencytrack.plugin.api.ExtensionFactory;
22+
import org.dependencytrack.plugin.api.ExtensionPoint;
23+
import org.dependencytrack.plugin.api.Plugin;
24+
25+
import java.util.Collection;
26+
import java.util.List;
27+
28+
/**
29+
* @since 5.6.0
30+
*/
31+
public final class FileStoragePlugin implements Plugin {
32+
33+
@Override
34+
public Collection<? extends ExtensionFactory<? extends ExtensionPoint>> extensionFactories() {
35+
return List.of(
36+
new LocalFileStorageFactory(),
37+
new MemoryFileStorageFactory(),
38+
new S3FileStorageFactory());
39+
}
40+
41+
}

0 commit comments

Comments
 (0)