Merge pull request #1047 from DependencyTrack/file-storage-plugin

Implement file storage plugin

Author: nscuro

pom.xml

@@ -543,6 +543,12 @@
             <version>${lib.testcontainers.version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>minio</artifactId>
+            <version>${lib.testcontainers.version}</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.testcontainers</groupId>
             <artifactId>postgresql</artifactId>

src/main/java/org/dependencytrack/event/BomUploadEvent.java

@@ -20,8 +20,7 @@
 
 import alpine.event.framework.AbstractChainableEvent;
 import org.dependencytrack.model.Project;
-
-import java.io.File;
+import org.dependencytrack.proto.storage.v1alpha1.FileMetadata;
 
 /**
  * Defines an event triggered when a bill-of-material (bom) document is submitted.
@@ -32,18 +31,18 @@
 public class BomUploadEvent extends AbstractChainableEvent {
 
     private final Project project;
-    private final File file;
+    private final FileMetadata fileMetadata;
 
-    public BomUploadEvent(final Project project, final File file) {
+    public BomUploadEvent(final Project project, final FileMetadata fileMetadata) {
         this.project = project;
-        this.file = file;
+        this.fileMetadata = fileMetadata;
     }
 
     public Project getProject() {
         return project;
     }
 
-    public File getFile() {
-        return file;
+    public FileMetadata getFileMetadata() {
+        return fileMetadata;
     }
 }

src/main/java/org/dependencytrack/resources/v1/BomResource.java

@@ -54,10 +54,13 @@
 import org.dependencytrack.parser.cyclonedx.CycloneDxValidator;
 import org.dependencytrack.parser.cyclonedx.InvalidBomException;
 import org.dependencytrack.persistence.QueryManager;
+import org.dependencytrack.plugin.PluginManager;
+import org.dependencytrack.proto.storage.v1alpha1.FileMetadata;
 import org.dependencytrack.resources.v1.problems.InvalidBomProblemDetails;
 import org.dependencytrack.resources.v1.problems.ProblemDetails;
 import org.dependencytrack.resources.v1.vo.BomSubmitRequest;
 import org.dependencytrack.resources.v1.vo.BomUploadResponse;
+import org.dependencytrack.storage.FileStorage;
 import org.glassfish.jersey.media.multipart.BodyPartEntity;
 import org.glassfish.jersey.media.multipart.FormDataBodyPart;
 import org.glassfish.jersey.media.multipart.FormDataParam;
@@ -80,13 +83,11 @@
 import jakarta.ws.rs.core.MediaType;
 import jakarta.ws.rs.core.Response;
 import java.io.ByteArrayInputStream;
-import java.io.File;
 import java.io.IOException;
 import java.io.StringReader;
 import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.StandardOpenOption;
 import java.security.Principal;
+import java.time.Instant;
 import java.util.Arrays;
 import java.util.Base64;
 import java.util.List;
@@ -465,17 +466,17 @@ private Response process(QueryManager qm, Project project, String encodedBomData
         if (project != null) {
             requireAccess(qm, project);
 
-            final File bomFile;
+            final FileMetadata bomFileMetadata;
             try (final var encodedInputStream = new ByteArrayInputStream(encodedBomData.getBytes(StandardCharsets.UTF_8));
                  final var decodedInputStream = Base64.getDecoder().wrap(encodedInputStream);
                  final var byteOrderMarkInputStream = new BOMInputStream(decodedInputStream)) {
-                bomFile = validateAndStoreBom(IOUtils.toByteArray(byteOrderMarkInputStream), project);
+                bomFileMetadata = validateAndStoreBom(IOUtils.toByteArray(byteOrderMarkInputStream), project);
             } catch (IOException e) {
                 LOGGER.error("An unexpected error occurred while validating or storing a BOM uploaded to project: " + project.getUuid(), e);
                 return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
             }
 
-            final BomUploadEvent bomUploadEvent = new BomUploadEvent(qm.detach(Project.class, project.getId()), bomFile);
+            final BomUploadEvent bomUploadEvent = new BomUploadEvent(qm.detach(Project.class, project.getId()), bomFileMetadata);
             qm.createWorkflowSteps(bomUploadEvent.getChainIdentifier());
             Event.dispatch(bomUploadEvent);
 
@@ -496,18 +497,18 @@ private Response process(QueryManager qm, Project project, List<FormDataBodyPart
             if (project != null) {
                 requireAccess(qm, project);
 
-                final File bomFile;
+                final FileMetadata bomFileMetadata;
                 try (final var inputStream = bodyPartEntity.getInputStream();
                      final var byteOrderMarkInputStream = new BOMInputStream(inputStream)) {
-                    bomFile = validateAndStoreBom(IOUtils.toByteArray(byteOrderMarkInputStream), project, artifactPart.getMediaType());
+                    bomFileMetadata = validateAndStoreBom(IOUtils.toByteArray(byteOrderMarkInputStream), project, artifactPart.getMediaType());
                 } catch (IOException e) {
                     LOGGER.error("An unexpected error occurred while validating or storing a BOM uploaded to project: " + project.getUuid(), e);
                     return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
                 }
 
                 // todo: make option to combine all the bom data so components are reconciled in a single pass.
                 // todo: https://github.com/DependencyTrack/dependency-track/issues/130
-                final BomUploadEvent bomUploadEvent = new BomUploadEvent(qm.detach(Project.class, project.getId()), bomFile);
+                final BomUploadEvent bomUploadEvent = new BomUploadEvent(qm.detach(Project.class, project.getId()), bomFileMetadata);
 
                 qm.createWorkflowSteps(bomUploadEvent.getChainIdentifier());
                 Event.dispatch(bomUploadEvent);
@@ -522,25 +523,22 @@ private Response process(QueryManager qm, Project project, List<FormDataBodyPart
         return Response.ok().build();
     }
 
-    private File validateAndStoreBom(final byte[] bomBytes, final Project project) throws IOException {
+    private FileMetadata validateAndStoreBom(final byte[] bomBytes, final Project project) throws IOException {
         return validateAndStoreBom(bomBytes, project, null);
     }
 
-    private File validateAndStoreBom(final byte[] bomBytes, final Project project, MediaType mediaType) throws IOException {
+    private FileMetadata validateAndStoreBom(final byte[] bomBytes, final Project project, MediaType mediaType) throws IOException {
         validate(bomBytes, project, mediaType);
 
-        // TODO: Store externally so other instances of the API server can pick it up.
-        //   https://github.com/CycloneDX/cyclonedx-bom-repo-server
-        final java.nio.file.Path tmpPath = Files.createTempFile("dtrack-bom-%s".formatted(project.getUuid()), null);
-        final File tmpFile = tmpPath.toFile();
-        tmpFile.deleteOnExit();
-
-        LOGGER.debug("Writing BOM for project %s to %s".formatted(project.getUuid(), tmpPath));
-        try (final var tmpOutputStream = Files.newOutputStream(tmpPath, StandardOpenOption.WRITE)) {
-            tmpOutputStream.write(bomBytes);
+        // TODO: Provide mediaType to FileStorage#store. Should be any of:
+        //   * application/vnd.cyclonedx+json
+        //   * application/vnd.cyclonedx+xml
+        //   * application/x.vnd.cyclonedx+protobuf
+        //  Consider also attaching the detected version, i.e. application/vnd.cyclonedx+xml; version=1.6
+        //  See https://cyclonedx.org/specification/overview/ -> Media Types.
+        try (final var fileStorage = PluginManager.getInstance().getExtension(FileStorage.class)) {
+            return fileStorage.store("bom-upload/%s_%s".formatted(Instant.now().toEpochMilli(), project.getUuid()), bomBytes);
         }
-
-        return tmpFile;
     }
 
     static void validate(final byte[] bomBytes, final Project project) {
@@ -636,7 +634,7 @@ private static boolean shouldValidate(final Project project) {
                     .map(org.dependencytrack.model.Tag::getName)
                     .anyMatch(validationModeTags::contains);
             return (validationMode == BomValidationMode.ENABLED_FOR_TAGS && doTagsMatch)
-                    || (validationMode == BomValidationMode.DISABLED_FOR_TAGS && !doTagsMatch);
+                   || (validationMode == BomValidationMode.DISABLED_FOR_TAGS && !doTagsMatch);
         }
     }
 }

src/main/java/org/dependencytrack/storage/FileStorage.java

@@ -0,0 +1,120 @@
+/*
+ * This file is part of Dependency-Track.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright (c) OWASP Foundation. All Rights Reserved.
+ */
+package org.dependencytrack.storage;
+
+import org.dependencytrack.plugin.api.ExtensionPoint;
+import org.dependencytrack.proto.storage.v1alpha1.FileMetadata;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.regex.Pattern;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * @since 5.6.0
+ */
+public interface FileStorage extends ExtensionPoint {
+
+    Pattern VALID_NAME_PATTERN = Pattern.compile("[a-zA-Z0-9_/\\-.]+");
+
+    /**
+     * Persist data to a file in storage.
+     * <p>
+     * Storage providers may transparently perform additional steps,
+     * such as encryption and compression.
+     *
+     * @param fileName  Name of the file. This fileName is not guaranteed to be reflected
+     *                  in storage as-is. It may be modified or changed entirely.
+     * @param mediaType Media type of the file.
+     * @param content   Data to store.
+     * @return Metadata of the stored file.
+     * @throws IOException When storing the file failed.
+     * @see <a href="https://www.iana.org/assignments/media-types/media-types.xhtml">IANA Media Types</a>
+     */
+    FileMetadata store(final String fileName, final String mediaType, final byte[] content) throws IOException;
+
+    /**
+     * Persist data to a file in storage, assuming the media type to be {@code application/octet-stream}.
+     *
+     * @see #store(String, String, byte[])
+     */
+    default FileMetadata store(final String fileName, final byte[] content) throws IOException {
+        return store(fileName, "application/octet-stream", content);
+    }
+
+    /**
+     * Retrieves a file from storage.
+     * <p>
+     * Storage providers may transparently perform additional steps,
+     * such as integrity verification, decryption and decompression.
+     * <p>
+     * Trying to retrieve a file from a different storage provider
+     * is an illegal operation and yields an exception.
+     *
+     * @param fileMetadata Metadata of the file to retrieve.
+     * @return The file's content.
+     * @throws IOException           When retrieving the file failed.
+     * @throws FileNotFoundException When the requested file was not found.
+     */
+    byte[] get(final FileMetadata fileMetadata) throws IOException;
+
+    /**
+     * Deletes a file from storage.
+     * <p>
+     * Trying to delete a file from a different storage provider
+     * is an illegal operation and yields an exception.
+     *
+     * @param fileMetadata Metadata of the file to delete.
+     * @return {@code true} when the file was deleted, otherwise {@code false}.
+     * @throws IOException When deleting the file failed.
+     */
+    boolean delete(final FileMetadata fileMetadata) throws IOException;
+
+    // TODO: deleteMany. Some remote storage backends support batch deletes.
+    //  https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html
+
+    static void requireValidFileName(final String fileName) {
+        requireNonNull(fileName, "fileName must not be null");
+
+        if (!VALID_NAME_PATTERN.matcher(fileName).matches()) {
+            throw new IllegalArgumentException("fileName must match pattern: " + VALID_NAME_PATTERN.pattern());
+        }
+    }
+
+    class ExtensionPointMetadata implements org.dependencytrack.plugin.api.ExtensionPointMetadata<FileStorage> {
+
+        @Override
+        public String name() {
+            return "file.storage";
+        }
+
+        @Override
+        public boolean required() {
+            return true;
+        }
+
+        @Override
+        public Class<FileStorage> extensionPointClass() {
+            return FileStorage.class;
+        }
+
+    }
+
+}

src/main/java/org/dependencytrack/storage/FileStoragePlugin.java

@@ -0,0 +1,41 @@
+/*
+ * This file is part of Dependency-Track.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright (c) OWASP Foundation. All Rights Reserved.
+ */
+package org.dependencytrack.storage;
+
+import org.dependencytrack.plugin.api.ExtensionFactory;
+import org.dependencytrack.plugin.api.ExtensionPoint;
+import org.dependencytrack.plugin.api.Plugin;
+
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * @since 5.6.0
+ */
+public final class FileStoragePlugin implements Plugin {
+
+    @Override
+    public Collection<? extends ExtensionFactory<? extends ExtensionPoint>> extensionFactories() {
+        return List.of(
+                new LocalFileStorageFactory(),
+                new MemoryFileStorageFactory(),
+                new S3FileStorageFactory());
+    }
+
+}