Skip to content

Commit 8e458bd

Browse files
gopannabdgopannabd
authored
Merge pull request #35 from blackducksoftware/idetect-4535-add-missing-manifest-type-for-oci-index
Improved exception logging and added new manifest type.
2 parents 4dea497 + cabe25e commit 8e458bd

File tree

3 files changed

+107
-38
lines changed

3 files changed

+107
-38
lines changed

‎src/main/java/com/blackduck/integration/blackduck/imageinspector/image/oci/OciImageDirectoryExtractor.java

Lines changed: 86 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
/*
22
* hub-imageinspector-lib
33
*
4-
* Copyright (c) 2024 Black Duck Software, Inc.
4+
* Copyright (c) 2025 Black Duck Software, Inc.
55
*
66
* Use subject to the terms and conditions of the Black Duck Software End User Software License and Maintenance Agreement. All rights reserved worldwide.
77
*/
@@ -20,7 +20,6 @@
2020
import com.blackduck.integration.blackduck.imageinspector.image.oci.model.OciDescriptor;
2121
import com.blackduck.integration.blackduck.imageinspector.image.oci.model.OciImageIndex;
2222
import com.blackduck.integration.blackduck.imageinspector.image.oci.model.OciImageManifest;
23-
import com.blackduck.integration.blackduck.imageinspector.image.common.*;
2423
import org.jetbrains.annotations.Nullable;
2524
import org.slf4j.Logger;
2625
import org.slf4j.LoggerFactory;
@@ -32,6 +31,7 @@
3231
public class OciImageDirectoryExtractor implements ImageDirectoryExtractor {
3332
private static final String INDEX_FILE_NAME = "index.json";
3433
private static final String BLOBS_DIR_NAME = "blobs";
34+
private static final String CONFIG_FIELD_NAME = "\"Config\":\"";
3535

3636
private static final String INDEX_FILE_MEDIA_TYPE = "application/vnd.oci.image.index.v1+json";
3737
private static final String CONFIG_FILE_MEDIA_TYPE = "application/vnd.oci.image.config.v1+json";
@@ -69,7 +69,7 @@ public List<TypedArchiveFile> getLayerArchives(final File imageDir, @Nullable St
6969
File manifestFile = findManifestFile(imageDir, manifestDescriptor);
7070

7171
try {
72-
return parseLayerArchives(manifestFile, blobsDir);
72+
return parseLayerArchives(manifestFile, blobsDir, imageDir);
7373
} catch (IOException e) {
7474
throw new IntegrationException(String.format("Error parsing layer archives from manifest file %s: %s", manifestFile.getAbsolutePath(), e.getMessage()), e);
7575
}
@@ -83,7 +83,7 @@ public FullLayerMapping getLayerMapping(final File imageDir, @Nullable String gi
8383

8484
String manifestRepoTag = manifestDescriptor.getRepoTagString().orElse(null);
8585
if (manifestRepoTag != null && !manifestRepoTag.contains(":") && givenRepo != null) {
86-
if (givenTag.isBlank()) {
86+
if (givenTag != null && givenTag.isBlank()) {
8787
givenTag = "latest";
8888
}
8989
manifestRepoTag = String.format("%s:%s", givenRepo, givenTag);
@@ -94,23 +94,52 @@ public FullLayerMapping getLayerMapping(final File imageDir, @Nullable String gi
9494
File manifestFile = findManifestFile(imageDir, manifestDescriptor);
9595
String manifestFileText;
9696
try {
97+
logger.debug("Path to the manifest file about to be read: {}", manifestFile.toPath().toString());
9798
manifestFileText = fileOperations.readFileToString(manifestFile);
9899
} catch (IOException e) {
99100
throw new IntegrationException(String.format("Unable to parse manifest file %s", manifestFile.getAbsolutePath()));
100101
}
101-
102+
103+
String pathToImageConfigFileFromRoot = null;
104+
List<String> layerInternalIds;
105+
106+
List<String> layerExternalIds;
102107
OciImageManifest imageManifest = gson.fromJson(manifestFileText, OciImageManifest.class);
103-
104-
// If we ever need more detail (os/architecture, history, cmd, etc):
105-
// imageManifest.config.digest has the filename (in the blobs dir) of the file that has that detail
106-
String pathToImageConfigFileFromRoot = findImageConfigFilePath(imageManifest);
107-
List<String> layerInternalIds = imageManifest.getLayers().stream()
108+
if (imageManifest == null || imageManifest.getConfig() == null) {
109+
logger.debug("JSON text is not of Image Manifest type: {}", manifestFileText);
110+
OciImageIndex imageIndex = gson.fromJson(manifestFileText, OciImageIndex.class);
111+
if (imageIndex == null || imageIndex.getManifests() == null) {
112+
throw new IntegrationException("Unable to find a matching manifest with config file");
113+
}
114+
File rootManifestfile = new File(imageDir, "manifest.json");
115+
try {
116+
String rootManifestFileText = fileOperations.readFileToString(rootManifestfile);
117+
pathToImageConfigFileFromRoot = getConfigDigestFromRootManifestText(rootManifestFileText);
118+
logger.debug("configRelativePathFromRoot: \n{}", pathToImageConfigFileFromRoot);
119+
if (pathToImageConfigFileFromRoot == null) {
120+
throw new IntegrationException("Unable to find config in root manifest.");
121+
}
122+
} catch (IOException ex) {
123+
throw new IntegrationException("Unable to find a matching manifest with config file in the root: {}", ex);
124+
}
125+
layerInternalIds = imageIndex.getManifests().stream()
108126
.map(OciDescriptor::getDigest)
109127
.collect(Collectors.toList());
128+
} else {
129+
// If we ever need more detail (os/architecture, history, cmd, etc):
130+
// imageManifest.config.digest has the filename (in the blobs dir) of the file that has that detail
131+
pathToImageConfigFileFromRoot = findImageConfigFilePath(imageManifest.getConfig());
132+
layerInternalIds = imageManifest.getLayers().stream()
133+
.map(OciDescriptor::getDigest)
134+
.collect(Collectors.toList());
135+
}
136+
ManifestLayerMapping manifestLayerMapping = new ManifestLayerMapping(
137+
resolvedRepoTag.getRepo().orElse(""),
138+
resolvedRepoTag.getTag().orElse(""),
139+
pathToImageConfigFileFromRoot,
140+
layerInternalIds);
110141

111-
ManifestLayerMapping manifestLayerMapping = new ManifestLayerMapping(resolvedRepoTag.getRepo().orElse(""), resolvedRepoTag.getTag().orElse(""), pathToImageConfigFileFromRoot, layerInternalIds);
112-
113-
List<String> layerExternalIds = commonImageConfigParser.getExternalLayerIdsFromImageConfigFile(imageDir, pathToImageConfigFileFromRoot);
142+
layerExternalIds = commonImageConfigParser.getExternalLayerIdsFromImageConfigFile(imageDir, pathToImageConfigFileFromRoot);
114143
return new FullLayerMapping(manifestLayerMapping, layerExternalIds);
115144
}
116145

@@ -128,7 +157,7 @@ private File findManifestFile(File imageDir, OciDescriptor manifestDescriptor) t
128157
return findBlob(blobsDir, pathToManifestFile);
129158
}
130159

131-
private ArchiveFileType parseArchiveTypeFromLayerDescriptorMediaType(String mediaType) throws IntegrationException {
160+
private ArchiveFileType parseArchiveTypeFromLayerDescriptorMediaType(String mediaType, String digest) throws IntegrationException {
132161
if (mediaType.contains("nondistributable")) {
133162
//TODO- what do we do with archives "nondistributable" media types? https://github.com/opencontainers/image-spec/blob/main/layer.md#non-distributable-layers
134163
// ac- based on the linked doc, I think we should just treat them normally (as if they were their "distributable" counterparts)
@@ -140,36 +169,59 @@ private ArchiveFileType parseArchiveTypeFromLayerDescriptorMediaType(String medi
140169
} else if (mediaType.endsWith(LAYER_ARCHIVE_TAR_ZSTD_MEDIA_TYPE_SUFFIX)) {
141170
return ArchiveFileType.TAR_ZSTD;
142171
} else {
143-
throw new IntegrationException(String.format("Unrecognized layer media type: %s", mediaType));
172+
throw new IntegrationException(String.format("Possible unsupported input archive file type. Please refer to the relevant Docker Inspector documentation at https://documentation.blackduck.com/bundle/detect/page/packagemgrs/docker/formats.html. Unrecognized media type %s of layer %s.", mediaType, digest));
144173
}
145174
}
146175

147-
private List<TypedArchiveFile> parseLayerArchives(File manifestFile, File blobsDir) throws IOException {
176+
private List<TypedArchiveFile> parseLayerArchives(File manifestFile, File blobsDir, File imageDir) throws IOException {
177+
148178
// Parse manifest file for names + archive formats of layer files
149179
String manifestFileText = fileOperations.readFileToString(manifestFile);
180+
logger.debug("parseLayerArchives - manifestFileText: {}, blobsDir: {}, imageDir: {}", manifestFileText, blobsDir, imageDir);
150181
OciImageManifest imageManifest = gson.fromJson(manifestFileText, OciImageManifest.class);
151-
182+
List<OciDescriptor> layersOrManifests;
152183
List<TypedArchiveFile> layerArchives = new LinkedList<>();
153-
for (OciDescriptor layer : imageManifest.getLayers()) {
184+
if (imageManifest == null || imageManifest.getLayers() == null) {
185+
logger.debug("JSON text did not match Image Manifest: {}\n", manifestFileText);
186+
OciImageIndex imageIndex = gson.fromJson(manifestFileText, OciImageIndex.class);
187+
if (imageIndex == null || imageIndex.getManifests() == null) {
188+
logger.debug("JSON text did not match Image Index either.");
189+
return layerArchives;
190+
} else {
191+
layersOrManifests = imageIndex.getManifests();
192+
}
193+
} else {
194+
layersOrManifests = imageManifest.getLayers();
195+
}
196+
logger.debug("parseLayerArchives - layersOrManifests.size(): {}", layersOrManifests.size());
197+
for (OciDescriptor layer : layersOrManifests) {
154198
String pathToLayerFile = parsePathToBlobFileFromDigest(layer.getDigest());
199+
logger.debug("parseLayerArchives - pathToLayerFile: {}", pathToLayerFile);
155200
File layerFile;
156201
try {
157-
layerFile = findBlob(blobsDir, pathToLayerFile);
202+
if (pathToLayerFile.startsWith(BLOBS_DIR_NAME)) {
203+
logger.debug("imageDir: {}, pathToLayerFile: {}", imageDir, pathToLayerFile);
204+
layerFile = findBlob(imageDir, pathToLayerFile);
205+
} else {
206+
logger.debug("blobsDir: {}, pathToLayerFile: {}", blobsDir, pathToLayerFile);
207+
layerFile = findBlob(blobsDir, pathToLayerFile);
208+
}
158209
} catch (IntegrationException e) {
159210
logger.error(e.getMessage());
160211
continue;
161212
}
162-
213+
logger.debug("parseLayerArchives - layerFile: {}", layerFile);
163214
ArchiveFileType archiveFileType;
164215
try {
165-
archiveFileType = parseArchiveTypeFromLayerDescriptorMediaType(layer.getMediaType());
216+
logger.debug("parseLayerArchives - layer.getMediaType(): {}", layer.getMediaType());
217+
archiveFileType = parseArchiveTypeFromLayerDescriptorMediaType(layer.getMediaType(), layer.getDigest());
166218
} catch (IntegrationException e) {
167-
logger.trace(e.getMessage());
219+
logger.error(e.getMessage());
168220
continue;
169221
}
170222
layerArchives.add(new TypedArchiveFile(archiveFileType, layerFile));
171223
}
172-
224+
logger.debug("parseLayerArchives - layerArchives.size(): {}", layerArchives.size());
173225
return layerArchives;
174226
}
175227

@@ -186,13 +238,20 @@ private File findBlob(File blobsDir, String pathToBlob) throws IntegrationExcept
186238
return blob;
187239
}
188240

189-
private String findImageConfigFilePath(OciImageManifest imageManifest) throws IntegrationException {
190-
OciDescriptor imageConfig = imageManifest.getConfig();
191-
if (imageConfig.getMediaType().equals(CONFIG_FILE_MEDIA_TYPE)) {
241+
private String findImageConfigFilePath(OciDescriptor imageConfig) throws IntegrationException {
242+
if (imageConfig != null && imageConfig.getMediaType() != null && imageConfig.getMediaType().equals(CONFIG_FILE_MEDIA_TYPE)) {
192243
return String.format("%s/%s", BLOBS_DIR_NAME, parsePathToBlobFileFromDigest(imageConfig.getDigest()));
193244
} else {
194245
throw new IntegrationException("Unable to find config file");
195246
}
196247
}
197-
248+
249+
private String getConfigDigestFromRootManifestText(String rootManifestFileText) {
250+
int start = rootManifestFileText.indexOf(CONFIG_FIELD_NAME) + CONFIG_FIELD_NAME.length();
251+
int end = rootManifestFileText.indexOf('"', start + 1);
252+
if (start > -1 && end > start) {
253+
return rootManifestFileText.substring(start, end).replace(":", "/");
254+
}
255+
return null;
256+
}
198257
}

‎src/main/java/com/blackduck/integration/blackduck/imageinspector/image/oci/OciImageLayerSorter.java

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
/*
22
* hub-imageinspector-lib
33
*
4-
* Copyright (c) 2024 Black Duck Software, Inc.
4+
* Copyright (c) 2025 Black Duck Software, Inc.
55
*
66
* Use subject to the terms and conditions of the Black Duck Software End User Software License and Maintenance Agreement. All rights reserved worldwide.
77
*/
@@ -21,8 +21,14 @@ public class OciImageLayerSorter extends ImageLayerSorter {
2121
@Override
2222
protected TypedArchiveFile getLayerArchive(final List<TypedArchiveFile> unOrderedLayerArchives, final String layerInternalId) throws IntegrationException {
2323
TypedArchiveFile layerArchive = null;
24+
if (unOrderedLayerArchives != null) {
25+
logger.debug("layerInternalId: {}, unOrderedLayerArchives size: {}", layerInternalId, unOrderedLayerArchives.size());
26+
} else {
27+
logger.debug("layerInternalId: {}, unOrderedLayerArchives: {}", layerInternalId, unOrderedLayerArchives);
28+
}
2429
for (final TypedArchiveFile candidateLayerTar : unOrderedLayerArchives) {
2530
String candidateId = String.format("%s:%s", candidateLayerTar.getFile().getParentFile().getName(), candidateLayerTar.getFile().getName());
31+
logger.debug("layerInternalId: {}, candidateId: {}", layerInternalId, candidateId);
2632
if (layerInternalId.equals(candidateId)) {
2733
logger.trace(String.format("Found layer archive for layer %s: ", layerInternalId, candidateLayerTar.getFile().getAbsolutePath()));
2834
layerArchive = candidateLayerTar;

‎src/main/java/com/blackduck/integration/blackduck/imageinspector/image/oci/OciManifestDescriptorParser.java

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
/*
22
* hub-imageinspector-lib
33
*
4-
* Copyright (c) 2024 Black Duck Software, Inc.
4+
* Copyright (c) 2025 Black Duck Software, Inc.
55
*
66
* Use subject to the terms and conditions of the Black Duck Software End User Software License and Maintenance Agreement. All rights reserved worldwide.
77
*/
@@ -16,12 +16,13 @@
1616
import org.slf4j.Logger;
1717
import org.slf4j.LoggerFactory;
1818

19+
import java.util.ArrayList;
1920
import java.util.List;
2021
import java.util.Optional;
21-
import java.util.stream.Collectors;
2222

2323
public class OciManifestDescriptorParser {
2424
private static final String MANIFEST_FILE_MEDIA_TYPE = "application/vnd.oci.image.manifest.v1+json";
25+
private static final String INDEX_FILE_MEDIA_TYPE = "application/vnd.oci.image.index.v1+json";
2526
private final Logger logger = LoggerFactory.getLogger(this.getClass());
2627
private final ManifestRepoTagMatcher manifestRepoTagMatcher;
2728

@@ -32,15 +33,18 @@ public OciManifestDescriptorParser(ManifestRepoTagMatcher manifestRepoTagMatcher
3233
public OciDescriptor getManifestDescriptor(OciImageIndex ociImageIndex,
3334
@Nullable String givenRepo, @Nullable String givenTag) throws IntegrationException {
3435
// TODO- Probably also need to select one of multiple based on arch
35-
List<OciDescriptor> trueManifests =
36-
ociImageIndex.getManifests().stream()
37-
.filter(man -> MANIFEST_FILE_MEDIA_TYPE.equals(man.getMediaType()))
38-
.collect(Collectors.toList());
39-
if (trueManifests.size() == 0) {
40-
throw new IntegrationException(String.format("No manifest descriptor with media type %s was found in OCI image index", MANIFEST_FILE_MEDIA_TYPE));
36+
List<OciDescriptor> trueManifests = new ArrayList<>();
37+
for (OciDescriptor ociDescriptor : ociImageIndex.getManifests()) {
38+
logger.debug("Found a media type in manifest: {}", ociDescriptor.getMediaType());
39+
if (MANIFEST_FILE_MEDIA_TYPE.equals(ociDescriptor.getMediaType()) || INDEX_FILE_MEDIA_TYPE.equals(ociDescriptor.getMediaType())) {
40+
trueManifests.add(ociDescriptor);
41+
}
42+
}
43+
if (trueManifests.isEmpty()) {
44+
throw new IntegrationException(String.format("No manifest descriptor with either media type {} or {} was found in OCI image index", INDEX_FILE_MEDIA_TYPE, MANIFEST_FILE_MEDIA_TYPE));
4145
}
4246
if ((trueManifests.size() == 1)) {
43-
logger.debug(String.format("There is only one manifest; inspecting that one; digest=%s", trueManifests.get(0).getDigest()));
47+
logger.debug(String.format("There is only one manifest; inspecting that one; digest={}", trueManifests.get(0).getDigest()));
4448
return trueManifests.get(0);
4549
}
4650
if ((trueManifests.size() > 1) && StringUtils.isBlank(givenRepo)) {
@@ -59,7 +63,7 @@ public OciDescriptor getManifestDescriptor(OciImageIndex ociImageIndex,
5963
.filter(m -> manifestRepoTagMatcher.findMatch(m.getRepoTagString().get(), givenRepoTag).isPresent())
6064
.findFirst();
6165
if (!matchingManifest.isPresent()) {
62-
throw new IntegrationException(String.format("Unable to find manifest matching repo:tag: %s", givenRepoTag));
66+
throw new IntegrationException(String.format("Unable to find manifest matching repo:tag: {}", givenRepoTag));
6367
}
6468
return matchingManifest.get();
6569
}

0 commit comments

Comments
 (0)