Skip to content

Commit 1650522

Browse files
nscuronscuro
committed
Implement file storage plugin
Signed-off-by: nscuro <nscuro@protonmail.com>
1 parent 782f2f5 commit 1650522

File tree

8 files changed

+178
-44
lines changed

8 files changed

+178
-44
lines changed

‎src/main/java/org/dependencytrack/storage/LocalFileStorage.java

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@
3232
import java.net.URISyntaxException;
3333
import java.nio.file.Files;
3434
import java.nio.file.Path;
35-
import java.nio.file.Paths;
3635
import java.util.Arrays;
3736
import java.util.HexFormat;
3837

@@ -142,16 +141,9 @@ public boolean delete(final FileMetadata fileMetadata) throws IOException {
142141
}
143142

144143
private Path resolveFilePath(final String filePath) {
145-
Path resolvedFilePath = Paths.get(filePath).normalize();
146-
if (resolvedFilePath.isAbsolute()) {
147-
// Ensure that paths are relative such that they can be resolved
148-
// using the configured base directory: /foo/bar -> foo/bar
149-
resolvedFilePath = resolvedFilePath.subpath(0, resolvedFilePath.getNameCount());
150-
}
151-
152-
resolvedFilePath = baseDirPath.resolve(resolvedFilePath).normalize().toAbsolutePath();
144+
final Path resolvedFilePath = baseDirPath.resolve(filePath).normalize().toAbsolutePath();
153145
if (!resolvedFilePath.startsWith(baseDirPath)) {
154-
throw new IllegalStateException("""
146+
throw new IllegalArgumentException("""
155147
The provided filePath %s does not resolve to a path within the \
156148
configured base directory (%s)""".formatted(filePath, baseDirPath));
157149
}
@@ -170,12 +162,14 @@ Path resolveFilePath(final FileMetadata fileMetadata) {
170162
throw new IllegalArgumentException(
171163
"%s: Host portion is not allowed for scheme %s".formatted(locationUri, EXTENSION_NAME));
172164
}
173-
if (locationUri.getPath() == null) {
165+
if (locationUri.getPath() == null || locationUri.getPath().equals("/")) {
174166
throw new IllegalArgumentException(
175167
"%s: Path portion not set; Unable to determine file name".formatted(locationUri));
176168
}
177169

178-
return resolveFilePath(locationUri.getPath());
170+
// The value returned by URI#getPath always has a leading slash.
171+
// Remove it to prevent the path from erroneously be interpreted as absolute.
172+
return resolveFilePath(locationUri.getPath().replaceFirst("^/", ""));
179173
}
180174

181175
}

‎src/main/java/org/dependencytrack/storage/MemoryFileStorage.java

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@
2626
import java.net.URI;
2727
import java.net.URISyntaxException;
2828
import java.nio.file.NoSuchFileException;
29-
import java.nio.file.Path;
3029
import java.nio.file.Paths;
3130
import java.util.Arrays;
3231
import java.util.HexFormat;
@@ -113,12 +112,7 @@ public boolean delete(final FileMetadata fileMetadata) {
113112
}
114113

115114
private static String normalizeFileName(final String fileName) {
116-
Path filePath = Paths.get(fileName).normalize();
117-
if (filePath.isAbsolute()) {
118-
filePath = filePath.subpath(0, filePath.getNameCount());
119-
}
120-
121-
return filePath.toString();
115+
return Paths.get(fileName).normalize().toString();
122116
}
123117

124118
private static String resolveFileName(final FileMetadata fileMetadata) {
@@ -136,7 +130,9 @@ private static String resolveFileName(final FileMetadata fileMetadata) {
136130
"%s: Path portion not set; Unable to determine file name".formatted(locationUri));
137131
}
138132

139-
return normalizeFileName(locationUri.getPath());
133+
// The value returned by URI#getPath always has a leading slash.
134+
// Remove it to prevent the path from erroneously be interpreted as absolute.
135+
return normalizeFileName(locationUri.getPath().replaceFirst("^/", ""));
140136
}
141137

142138
}

‎src/main/java/org/dependencytrack/storage/S3FileStorage.java

Lines changed: 29 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
*/
1919
package org.dependencytrack.storage;
2020

21+
import com.github.luben.zstd.Zstd;
2122
import io.minio.GetObjectArgs;
2223
import io.minio.GetObjectResponse;
2324
import io.minio.MinioClient;
@@ -51,10 +52,18 @@ final class S3FileStorage implements FileStorage {
5152

5253
private final MinioClient s3Client;
5354
private final String bucketName;
54-
55-
S3FileStorage(final MinioClient s3Client, final String bucketName) {
55+
private final int compressionThresholdBytes;
56+
private final int compressionLevel;
57+
58+
S3FileStorage(
59+
final MinioClient s3Client,
60+
final String bucketName,
61+
final int compressionThresholdBytes,
62+
final int compressionLevel) {
5663
this.s3Client = s3Client;
5764
this.bucketName = bucketName;
65+
this.compressionThresholdBytes = compressionThresholdBytes;
66+
this.compressionLevel = compressionLevel;
5867
}
5968

6069
private record S3FileLocation(String bucket, String object) {
@@ -74,7 +83,9 @@ private static S3FileLocation from(final FileMetadata fileMetadata) {
7483
"Path portion of URI %s not set; Unable to determine object name".formatted(locationUri));
7584
}
7685

77-
return new S3FileLocation(locationUri.getHost(), locationUri.getPath());
86+
// The value returned by URI#getPath always has a leading slash.
87+
// Remove it to prevent the path from erroneously be interpreted as absolute.
88+
return new S3FileLocation(locationUri.getHost(), locationUri.getPath().replaceFirst("^/", ""));
7889
}
7990

8091
private URI asURI() {
@@ -99,13 +110,17 @@ public FileMetadata store(final String fileName, final byte[] content) throws IO
99110
final var fileLocation = new S3FileLocation(bucketName, fileName);
100111
final URI locationUri = fileLocation.asURI();
101112

102-
final byte[] contentDigest = DigestUtils.sha256(content);
113+
final byte[] maybeCompressedContent = content.length >= compressionThresholdBytes
114+
? Zstd.compress(content, compressionLevel)
115+
: content;
116+
117+
final byte[] contentDigest = DigestUtils.sha256(maybeCompressedContent);
103118

104119
try {
105120
s3Client.putObject(PutObjectArgs.builder()
106121
.bucket(fileLocation.bucket())
107122
.object(fileLocation.object())
108-
.stream(new ByteArrayInputStream(content), content.length, -1)
123+
.stream(new ByteArrayInputStream(maybeCompressedContent), maybeCompressedContent.length, -1)
109124
.build());
110125
} catch (Exception e) {
111126
if (e instanceof final IOException ioe) {
@@ -132,14 +147,14 @@ public byte[] get(final FileMetadata fileMetadata) throws IOException {
132147
throw new IllegalArgumentException("File metadata does not contain " + METADATA_KEY_SHA256_DIGEST);
133148
}
134149

135-
final byte[] fileContent;
150+
final byte[] maybeCompressedContent;
136151
try {
137152
try (final GetObjectResponse response = s3Client.getObject(
138153
GetObjectArgs.builder()
139154
.bucket(fileLocation.bucket())
140155
.object(fileLocation.object())
141156
.build())) {
142-
fileContent = response.readAllBytes();
157+
maybeCompressedContent = response.readAllBytes();
143158
}
144159
} catch (ErrorResponseException e) {
145160
// https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html#ErrorCodeList
@@ -156,15 +171,20 @@ public byte[] get(final FileMetadata fileMetadata) throws IOException {
156171
throw new IOException(e);
157172
}
158173

159-
final byte[] actualContentDigest = DigestUtils.sha256(fileContent);
174+
final byte[] actualContentDigest = DigestUtils.sha256(maybeCompressedContent);
160175
final byte[] expectedContentDigest = HexFormat.of().parseHex(expectedContentDigestHex);
161176

162177
if (!Arrays.equals(actualContentDigest, expectedContentDigest)) {
163178
throw new IOException("SHA256 digest mismatch: actual=%s, expected=%s".formatted(
164179
HexFormat.of().formatHex(actualContentDigest), expectedContentDigestHex));
165180
}
166181

167-
return fileContent;
182+
final long decompressedSize = Zstd.decompressedSize(maybeCompressedContent);
183+
if (Zstd.decompressedSize(maybeCompressedContent) <= 0) {
184+
return maybeCompressedContent; // Not compressed.
185+
}
186+
187+
return Zstd.decompress(maybeCompressedContent, Math.toIntExact(decompressedSize));
168188
}
169189

170190
@Override

‎src/main/java/org/dependencytrack/storage/S3FileStorageFactory.java

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
*/
1919
package org.dependencytrack.storage;
2020

21+
import alpine.Config;
2122
import io.minio.BucketExistsArgs;
2223
import io.minio.MinioClient;
2324
import org.dependencytrack.plugin.api.ConfigDefinition;
@@ -61,9 +62,21 @@ public final class S3FileStorageFactory implements ExtensionFactory<FileStorage>
6162
ConfigSource.DEPLOYMENT,
6263
/* isRequired */ false,
6364
/* isSecret */ false);
65+
static final ConfigDefinition CONFIG_COMPRESSION_THRESHOLD_BYTES = new ConfigDefinition(
66+
"compression.threshold.bytes",
67+
ConfigSource.DEPLOYMENT,
68+
/* isRequired */ false,
69+
/* isSecret */ false);
70+
static final ConfigDefinition CONFIG_COMPRESSION_LEVEL = new ConfigDefinition(
71+
"compression.level",
72+
ConfigSource.DEPLOYMENT,
73+
/* isRequired */ false,
74+
/* isSecret */ false);
6475

6576
private MinioClient s3Client;
6677
private String bucketName;
78+
private int compressionThresholdBytes;
79+
private int compressionLevel;
6780

6881
@Override
6982
public String extensionName() {
@@ -95,13 +108,24 @@ public void init(final ConfigRegistry configRegistry) {
95108
optionalRegion.ifPresent(clientBuilder::region);
96109
s3Client = clientBuilder.build();
97110

111+
s3Client.setAppInfo(
112+
Config.getInstance().getApplicationName(),
113+
Config.getInstance().getApplicationVersion());
114+
115+
compressionThresholdBytes = configRegistry.getOptionalValue(CONFIG_COMPRESSION_THRESHOLD_BYTES)
116+
.map(Integer::parseInt)
117+
.orElse(1024);
118+
compressionLevel = configRegistry.getOptionalValue(CONFIG_COMPRESSION_LEVEL)
119+
.map(Integer::parseInt)
120+
.orElse(5);
121+
98122
LOGGER.debug("Verifying existence of bucket {}", bucketName);
99123
requireBucketExists(s3Client, bucketName);
100124
}
101125

102126
@Override
103127
public FileStorage create() {
104-
return new S3FileStorage(s3Client, bucketName);
128+
return new S3FileStorage(s3Client, bucketName, compressionThresholdBytes, compressionLevel);
105129
}
106130

107131
@Override

‎src/main/resources/application.properties

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1602,4 +1602,22 @@ file.storage.extension.s3.enabled=false
16021602
#
16031603
# @category: Storage
16041604
# @type: string
1605-
# file.storage.extension.s3.region=
1605+
# file.storage.extension.s3.region=
1606+
1607+
# Defines the size threshold for files after which they will be compressed.
1608+
# Compression is performed using the zstd algorithm.
1609+
# Has no effect unless file.storage.extension.s3.enabled is `true`.
1610+
#
1611+
# @category: Storage
1612+
# @default: 1024
1613+
# @type: integer
1614+
# file.storage.extension.s3.compression.threshold.bytes=
1615+
1616+
# Defines the zstd compression level to use.
1617+
# Has no effect unless file.storage.extension.s3.enabled is `true`.
1618+
#
1619+
# @category: Storage
1620+
# @default: 5
1621+
# @type: integer
1622+
# @valid-values: [-7..22]
1623+
# file.storage.extension.s3.compression.level=

‎src/test/java/org/dependencytrack/storage/LocalFileStorageTest.java

Lines changed: 42 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@
2828
import java.nio.file.Files;
2929
import java.nio.file.NoSuchFileException;
3030
import java.nio.file.Path;
31-
import java.util.Collections;
3231
import java.util.HexFormat;
3332
import java.util.Map;
3433

@@ -70,15 +69,17 @@ public void shouldStoreGetAndDeleteFile() throws Exception {
7069

7170
final FileStorage storage = storageFactory.create();
7271

73-
final FileMetadata fileMetadata = storage.store("foo", "bar".getBytes());
72+
final FileMetadata fileMetadata = storage.store("foo/bar", "baz".getBytes());
7473
assertThat(fileMetadata).isNotNull();
75-
assertThat(fileMetadata.getLocation()).isEqualTo("local:///foo");
74+
assertThat(fileMetadata.getLocation()).isEqualTo("local:///foo/bar");
7675
assertThat(fileMetadata.getStorageMetadataMap()).containsExactly(
77-
Map.entry("sha256_digest", "fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9"));
76+
Map.entry("sha256_digest", "baa5a0964d3320fbc0c6a922140453c8513ea24ab8fd0577034804a967248096"));
77+
78+
assertThat(tempDirPath.resolve("foo/bar")).exists();
7879

7980
final byte[] fileContent = storage.get(fileMetadata);
8081
assertThat(fileContent).isNotNull();
81-
assertThat(fileContent).asString().isEqualTo("bar");
82+
assertThat(fileContent).asString().isEqualTo("baz");
8283

8384
final boolean deleted = storage.delete(fileMetadata);
8485
assertThat(deleted).isTrue();
@@ -136,6 +137,22 @@ public void storeShouldOverwriteExistingFile() throws Exception {
136137
assertThat(storage.get(fileMetadataB)).asString().isEqualTo("qux");
137138
}
138139

140+
@Test
141+
@SuppressWarnings("resource")
142+
public void storeShouldThrowWhenFileNameAttemptsTraversal() {
143+
final var storageFactory = new LocalFileStorageFactory();
144+
storageFactory.init(new MockConfigRegistry(Map.of(
145+
CONFIG_DIRECTORY.name(), tempDirPath.toAbsolutePath().toString())));
146+
147+
final FileStorage storage = storageFactory.create();
148+
149+
assertThatExceptionOfType(IllegalArgumentException.class)
150+
.isThrownBy(() -> storage.store("foo/../../../bar", "bar".getBytes()))
151+
.withMessage("""
152+
The provided filePath foo/../../../bar does not resolve to a path \
153+
within the configured base directory (%s)""", tempDirPath);
154+
}
155+
139156
@Test
140157
@SuppressWarnings("resource")
141158
public void storeShouldThrowWhenFileHasInvalidName() {
@@ -167,6 +184,26 @@ public void getShouldThrowWhenFileLocationHasInvalidScheme() {
167184
.withMessage("foo:///bar: Unexpected scheme foo, expected local");
168185
}
169186

187+
@Test
188+
@SuppressWarnings("resource")
189+
public void getShouldThrowWhenFileNameAttemptsTraversal() {
190+
final var storageFactory = new LocalFileStorageFactory();
191+
storageFactory.init(new MockConfigRegistry(Map.of(
192+
CONFIG_DIRECTORY.name(), tempDirPath.toAbsolutePath().toString())));
193+
194+
final FileStorage storage = storageFactory.create();
195+
196+
assertThatExceptionOfType(IllegalArgumentException.class)
197+
.isThrownBy(() -> storage.get(
198+
FileMetadata.newBuilder()
199+
.setLocation("local:///foo/../../../bar")
200+
.putStorageMetadata("sha256_digest", "some-digest")
201+
.build()))
202+
.withMessage("""
203+
The provided filePath foo/../../../bar does not resolve to a path \
204+
within the configured base directory (%s)""", tempDirPath);
205+
}
206+
170207
@Test
171208
@SuppressWarnings("resource")
172209
public void getShouldThrowWhenFileDoesNotExist() {

‎src/test/java/org/dependencytrack/storage/MemoryFileStorageTest.java

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import java.io.IOException;
2626
import java.nio.file.NoSuchFileException;
2727
import java.util.Collections;
28+
import java.util.Map;
2829

2930
import static org.assertj.core.api.Assertions.assertThat;
3031
import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
@@ -53,14 +54,15 @@ public void shouldStoreGetAndDeleteFile() throws Exception {
5354

5455
final FileStorage storage = storageFactory.create();
5556

56-
final FileMetadata fileMetadata = storage.store("foo", "bar".getBytes());
57+
final FileMetadata fileMetadata = storage.store("foo/bar", "baz".getBytes());
5758
assertThat(fileMetadata).isNotNull();
58-
assertThat(fileMetadata.getLocation()).isEqualTo("memory:///foo");
59-
assertThat(fileMetadata.getStorageMetadataMap()).isEmpty();
59+
assertThat(fileMetadata.getLocation()).isEqualTo("memory:///foo/bar");
60+
assertThat(fileMetadata.getStorageMetadataMap()).containsExactly(
61+
Map.entry("sha256_digest", "baa5a0964d3320fbc0c6a922140453c8513ea24ab8fd0577034804a967248096"));
6062

6163
final byte[] fileContent = storage.get(fileMetadata);
6264
assertThat(fileContent).isNotNull();
63-
assertThat(fileContent).asString().isEqualTo("bar");
65+
assertThat(fileContent).asString().isEqualTo("baz");
6466

6567
final boolean deleted = storage.delete(fileMetadata);
6668
assertThat(deleted).isTrue();

0 commit comments

Comments
 (0)