Skip to content

Commit 401d4fe

Browse files
nscuronscuro
committed
Implement file storage plugin
Signed-off-by: nscuro <nscuro@protonmail.com>
1 parent 61f25c2 commit 401d4fe

16 files changed

+199
-61
lines changed

‎src/main/java/org/dependencytrack/resources/v1/BomResource.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -531,7 +531,7 @@ private FileMetadata validateAndStoreBom(final byte[] bomBytes, final Project pr
531531
validate(bomBytes, project);
532532

533533
try (final var fileStorage = PluginManager.getInstance().getExtension(FileStorage.class)) {
534-
return fileStorage.store("bom-upload/%s/%s".formatted(project.getUuid(), UUID.randomUUID()), bomBytes);
534+
return fileStorage.store("bom-upload/%s_%s".formatted(project.getUuid(), UUID.randomUUID()), bomBytes);
535535
}
536536
}
537537

‎src/main/java/org/dependencytrack/storage/DefaultFileStoragePlugin.java renamed to ‎src/main/java/org/dependencytrack/storage/FileStoragePlugin.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
/**
2929
* @since 5.6.0
3030
*/
31-
public final class DefaultFileStoragePlugin implements Plugin {
31+
public final class FileStoragePlugin implements Plugin {
3232

3333
@Override
3434
public Collection<? extends ExtensionFactory<? extends ExtensionPoint>> extensionFactories() {

‎src/main/java/org/dependencytrack/storage/LocalFileStorage.java

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@
3232
import java.net.URISyntaxException;
3333
import java.nio.file.Files;
3434
import java.nio.file.Path;
35-
import java.nio.file.Paths;
3635
import java.util.Arrays;
3736
import java.util.HexFormat;
3837

@@ -142,16 +141,9 @@ public boolean delete(final FileMetadata fileMetadata) throws IOException {
142141
}
143142

144143
private Path resolveFilePath(final String filePath) {
145-
Path resolvedFilePath = Paths.get(filePath).normalize();
146-
if (resolvedFilePath.isAbsolute()) {
147-
// Ensure that paths are relative such that they can be resolved
148-
// using the configured base directory: /foo/bar -> foo/bar
149-
resolvedFilePath = resolvedFilePath.subpath(0, resolvedFilePath.getNameCount());
150-
}
151-
152-
resolvedFilePath = baseDirPath.resolve(resolvedFilePath).normalize().toAbsolutePath();
144+
final Path resolvedFilePath = baseDirPath.resolve(filePath).normalize().toAbsolutePath();
153145
if (!resolvedFilePath.startsWith(baseDirPath)) {
154-
throw new IllegalStateException("""
146+
throw new IllegalArgumentException("""
155147
The provided filePath %s does not resolve to a path within the \
156148
configured base directory (%s)""".formatted(filePath, baseDirPath));
157149
}
@@ -170,12 +162,14 @@ Path resolveFilePath(final FileMetadata fileMetadata) {
170162
throw new IllegalArgumentException(
171163
"%s: Host portion is not allowed for scheme %s".formatted(locationUri, EXTENSION_NAME));
172164
}
173-
if (locationUri.getPath() == null) {
165+
if (locationUri.getPath() == null || locationUri.getPath().equals("/")) {
174166
throw new IllegalArgumentException(
175167
"%s: Path portion not set; Unable to determine file name".formatted(locationUri));
176168
}
177169

178-
return resolveFilePath(locationUri.getPath());
170+
// The value returned by URI#getPath always has a leading slash.
171+
// Remove it to prevent the path from erroneously be interpreted as absolute.
172+
return resolveFilePath(locationUri.getPath().replaceFirst("^/", ""));
179173
}
180174

181175
}

‎src/main/java/org/dependencytrack/storage/LocalFileStorageFactory.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ public Class<? extends FileStorage> extensionClass() {
7070

7171
@Override
7272
public int priority() {
73-
return 110;
73+
return 100;
7474
}
7575

7676
@Override

‎src/main/java/org/dependencytrack/storage/MemoryFileStorage.java

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@
2626
import java.net.URI;
2727
import java.net.URISyntaxException;
2828
import java.nio.file.NoSuchFileException;
29-
import java.nio.file.Path;
3029
import java.nio.file.Paths;
3130
import java.util.Arrays;
3231
import java.util.HexFormat;
@@ -113,12 +112,7 @@ public boolean delete(final FileMetadata fileMetadata) {
113112
}
114113

115114
private static String normalizeFileName(final String fileName) {
116-
Path filePath = Paths.get(fileName).normalize();
117-
if (filePath.isAbsolute()) {
118-
filePath = filePath.subpath(0, filePath.getNameCount());
119-
}
120-
121-
return filePath.toString();
115+
return Paths.get(fileName).normalize().toString();
122116
}
123117

124118
private static String resolveFileName(final FileMetadata fileMetadata) {
@@ -136,7 +130,9 @@ private static String resolveFileName(final FileMetadata fileMetadata) {
136130
"%s: Path portion not set; Unable to determine file name".formatted(locationUri));
137131
}
138132

139-
return normalizeFileName(locationUri.getPath());
133+
// The value returned by URI#getPath always has a leading slash.
134+
// Remove it to prevent the path from erroneously be interpreted as absolute.
135+
return normalizeFileName(locationUri.getPath().replaceFirst("^/", ""));
140136
}
141137

142138
}

‎src/main/java/org/dependencytrack/storage/MemoryFileStorageFactory.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ public Class<? extends FileStorage> extensionClass() {
4343

4444
@Override
4545
public int priority() {
46-
return 100;
46+
return 110;
4747
}
4848

4949
@Override

‎src/main/java/org/dependencytrack/storage/S3FileStorage.java

Lines changed: 29 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
*/
1919
package org.dependencytrack.storage;
2020

21+
import com.github.luben.zstd.Zstd;
2122
import io.minio.GetObjectArgs;
2223
import io.minio.GetObjectResponse;
2324
import io.minio.MinioClient;
@@ -51,10 +52,18 @@ final class S3FileStorage implements FileStorage {
5152

5253
private final MinioClient s3Client;
5354
private final String bucketName;
54-
55-
S3FileStorage(final MinioClient s3Client, final String bucketName) {
55+
private final int compressionThresholdBytes;
56+
private final int compressionLevel;
57+
58+
S3FileStorage(
59+
final MinioClient s3Client,
60+
final String bucketName,
61+
final int compressionThresholdBytes,
62+
final int compressionLevel) {
5663
this.s3Client = s3Client;
5764
this.bucketName = bucketName;
65+
this.compressionThresholdBytes = compressionThresholdBytes;
66+
this.compressionLevel = compressionLevel;
5867
}
5968

6069
private record S3FileLocation(String bucket, String object) {
@@ -74,7 +83,9 @@ private static S3FileLocation from(final FileMetadata fileMetadata) {
7483
"Path portion of URI %s not set; Unable to determine object name".formatted(locationUri));
7584
}
7685

77-
return new S3FileLocation(locationUri.getHost(), locationUri.getPath());
86+
// The value returned by URI#getPath always has a leading slash.
87+
// Remove it to prevent the path from erroneously be interpreted as absolute.
88+
return new S3FileLocation(locationUri.getHost(), locationUri.getPath().replaceFirst("^/", ""));
7889
}
7990

8091
private URI asURI() {
@@ -99,13 +110,17 @@ public FileMetadata store(final String fileName, final byte[] content) throws IO
99110
final var fileLocation = new S3FileLocation(bucketName, fileName);
100111
final URI locationUri = fileLocation.asURI();
101112

102-
final byte[] contentDigest = DigestUtils.sha256(content);
113+
final byte[] maybeCompressedContent = content.length >= compressionThresholdBytes
114+
? Zstd.compress(content, compressionLevel)
115+
: content;
116+
117+
final byte[] contentDigest = DigestUtils.sha256(maybeCompressedContent);
103118

104119
try {
105120
s3Client.putObject(PutObjectArgs.builder()
106121
.bucket(fileLocation.bucket())
107122
.object(fileLocation.object())
108-
.stream(new ByteArrayInputStream(content), content.length, -1)
123+
.stream(new ByteArrayInputStream(maybeCompressedContent), maybeCompressedContent.length, -1)
109124
.build());
110125
} catch (Exception e) {
111126
if (e instanceof final IOException ioe) {
@@ -132,14 +147,14 @@ public byte[] get(final FileMetadata fileMetadata) throws IOException {
132147
throw new IllegalArgumentException("File metadata does not contain " + METADATA_KEY_SHA256_DIGEST);
133148
}
134149

135-
final byte[] fileContent;
150+
final byte[] maybeCompressedContent;
136151
try {
137152
try (final GetObjectResponse response = s3Client.getObject(
138153
GetObjectArgs.builder()
139154
.bucket(fileLocation.bucket())
140155
.object(fileLocation.object())
141156
.build())) {
142-
fileContent = response.readAllBytes();
157+
maybeCompressedContent = response.readAllBytes();
143158
}
144159
} catch (ErrorResponseException e) {
145160
// https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html#ErrorCodeList
@@ -156,15 +171,20 @@ public byte[] get(final FileMetadata fileMetadata) throws IOException {
156171
throw new IOException(e);
157172
}
158173

159-
final byte[] actualContentDigest = DigestUtils.sha256(fileContent);
174+
final byte[] actualContentDigest = DigestUtils.sha256(maybeCompressedContent);
160175
final byte[] expectedContentDigest = HexFormat.of().parseHex(expectedContentDigestHex);
161176

162177
if (!Arrays.equals(actualContentDigest, expectedContentDigest)) {
163178
throw new IOException("SHA256 digest mismatch: actual=%s, expected=%s".formatted(
164179
HexFormat.of().formatHex(actualContentDigest), expectedContentDigestHex));
165180
}
166181

167-
return fileContent;
182+
final long decompressedSize = Zstd.decompressedSize(maybeCompressedContent);
183+
if (Zstd.decompressedSize(maybeCompressedContent) <= 0) {
184+
return maybeCompressedContent; // Not compressed.
185+
}
186+
187+
return Zstd.decompress(maybeCompressedContent, Math.toIntExact(decompressedSize));
168188
}
169189

170190
@Override

‎src/main/java/org/dependencytrack/storage/S3FileStorageFactory.java

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
*/
1919
package org.dependencytrack.storage;
2020

21+
import alpine.Config;
2122
import io.minio.BucketExistsArgs;
2223
import io.minio.MinioClient;
2324
import org.dependencytrack.plugin.api.ConfigDefinition;
@@ -61,9 +62,21 @@ public final class S3FileStorageFactory implements ExtensionFactory<FileStorage>
6162
ConfigSource.DEPLOYMENT,
6263
/* isRequired */ false,
6364
/* isSecret */ false);
65+
static final ConfigDefinition CONFIG_COMPRESSION_THRESHOLD_BYTES = new ConfigDefinition(
66+
"compression.threshold.bytes",
67+
ConfigSource.DEPLOYMENT,
68+
/* isRequired */ false,
69+
/* isSecret */ false);
70+
static final ConfigDefinition CONFIG_COMPRESSION_LEVEL = new ConfigDefinition(
71+
"compression.level",
72+
ConfigSource.DEPLOYMENT,
73+
/* isRequired */ false,
74+
/* isSecret */ false);
6475

6576
private MinioClient s3Client;
6677
private String bucketName;
78+
private int compressionThresholdBytes;
79+
private int compressionLevel;
6780

6881
@Override
6982
public String extensionName() {
@@ -77,7 +90,7 @@ public Class<? extends FileStorage> extensionClass() {
7790

7891
@Override
7992
public int priority() {
80-
return 130;
93+
return 120;
8194
}
8295

8396
@Override
@@ -95,13 +108,24 @@ public void init(final ConfigRegistry configRegistry) {
95108
optionalRegion.ifPresent(clientBuilder::region);
96109
s3Client = clientBuilder.build();
97110

111+
s3Client.setAppInfo(
112+
Config.getInstance().getApplicationName(),
113+
Config.getInstance().getApplicationVersion());
114+
115+
compressionThresholdBytes = configRegistry.getOptionalValue(CONFIG_COMPRESSION_THRESHOLD_BYTES)
116+
.map(Integer::parseInt)
117+
.orElse(1024);
118+
compressionLevel = configRegistry.getOptionalValue(CONFIG_COMPRESSION_LEVEL)
119+
.map(Integer::parseInt)
120+
.orElse(5);
121+
98122
LOGGER.debug("Verifying existence of bucket {}", bucketName);
99123
requireBucketExists(s3Client, bucketName);
100124
}
101125

102126
@Override
103127
public FileStorage create() {
104-
return new S3FileStorage(s3Client, bucketName);
128+
return new S3FileStorage(s3Client, bucketName, compressionThresholdBytes, compressionLevel);
105129
}
106130

107131
@Override

‎src/main/resources/META-INF/services/org.dependencytrack.plugin.api.Plugin

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
org.dependencytrack.storage.DefaultFileStoragePlugin
1+
org.dependencytrack.storage.FileStoragePlugin

‎src/main/resources/application.properties

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1554,7 +1554,7 @@ task.project.maintenance.lock.min.duration=PT1M
15541554
#
15551555
# @category: Storage
15561556
# @type: boolean
1557-
file.storage.extension.local.enabled=false
1557+
file.storage.extension.local.enabled=true
15581558

15591559
# Defines the local directory where files shall be stored.
15601560
# Has no effect unless file.storage.extension.local.enabled is `true`.
@@ -1586,7 +1586,7 @@ file.storage.extension.local.enabled=false
15861586
#
15871587
# @category: Storage
15881588
# @type: boolean
1589-
file.storage.extension.memory.enabled=true
1589+
file.storage.extension.memory.enabled=false
15901590

15911591
# Whether the s3 file storage extension shall be enabled.
15921592
#
@@ -1629,4 +1629,22 @@ file.storage.extension.s3.enabled=false
16291629
#
16301630
# @category: Storage
16311631
# @type: string
1632-
# file.storage.extension.s3.region=
1632+
# file.storage.extension.s3.region=
1633+
1634+
# Defines the size threshold for files after which they will be compressed.
1635+
# Compression is performed using the zstd algorithm.
1636+
# Has no effect unless file.storage.extension.s3.enabled is `true`.
1637+
#
1638+
# @category: Storage
1639+
# @default: 1024
1640+
# @type: integer
1641+
# file.storage.extension.s3.compression.threshold.bytes=
1642+
1643+
# Defines the zstd compression level to use.
1644+
# Has no effect unless file.storage.extension.s3.enabled is `true`.
1645+
#
1646+
# @category: Storage
1647+
# @default: 5
1648+
# @type: integer
1649+
# @valid-values: [-7..22]
1650+
# file.storage.extension.s3.compression.level=

‎src/test/java/org/dependencytrack/plugin/PluginManagerTest.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.dependencytrack.plugin.api.ExtensionFactory;
2424
import org.dependencytrack.plugin.api.ExtensionPoint;
2525
import org.dependencytrack.plugin.api.Plugin;
26+
import org.dependencytrack.storage.FileStoragePlugin;
2627
import org.junit.Rule;
2728
import org.junit.Test;
2829
import org.junit.contrib.java.lang.system.EnvironmentVariables;
@@ -45,7 +46,9 @@ interface UnknownExtensionPoint extends ExtensionPoint {
4546
@Test
4647
public void testGetLoadedPlugins() {
4748
final List<Plugin> loadedPlugins = PluginManager.getInstance().getLoadedPlugins();
48-
assertThat(loadedPlugins).satisfiesExactly(plugin -> assertThat(plugin).isOfAnyClassIn(DummyPlugin.class));
49+
assertThat(loadedPlugins).satisfiesExactlyInAnyOrder(
50+
plugin -> assertThat(plugin).isOfAnyClassIn(DummyPlugin.class),
51+
plugin -> assertThat(plugin).isInstanceOf(FileStoragePlugin.class));
4952
assertThat(loadedPlugins).isUnmodifiable();
5053
}
5154

‎src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1971,7 +1971,8 @@ private static FileMetadata storeBomFile(final String testFileName) throws Excep
19711971
final byte[] bomBytes = Files.readAllBytes(bomFilePath);
19721972

19731973
try (final var fileStorage = PluginManager.getInstance().getExtension(FileStorage.class)) {
1974-
return fileStorage.store("bom", bomBytes);
1974+
return fileStorage.store(
1975+
"test/%s-%s".formatted(CelPolicyEngineTest.class.getSimpleName(), UUID.randomUUID()), bomBytes);
19751976
}
19761977
}
19771978

0 commit comments

Comments
 (0)