From 253a0338fac1179bc9ead2ef7058aea0eec717bc Mon Sep 17 00:00:00 2001 From: devmehtabd Date: Tue, 13 May 2025 09:07:42 -0700 Subject: [PATCH 1/6] Update SBT detector to use new library --- detectable/build.gradle | 1 + .../detectables/sbt/dot/SbtDotExtractor.java | 12 +-- .../sbt/dot/SbtDotOutputParser.java | 4 +- .../sbt/dot/SbtGraphParserTransformer.java | 74 ++++++++++++------- .../sbt/dot/SbtRootNodeFinder.java | 25 ++++--- .../sbt/unit/SbtDotGraphNodeParserTest.java | 19 +++-- .../sbt/unit/SbtRootNodeFinderTest.java | 58 ++++++++------- 7 files changed, 112 insertions(+), 81 deletions(-) diff --git a/detectable/build.gradle b/detectable/build.gradle index 1c52636465..d5b243cc26 100644 --- a/detectable/build.gradle +++ b/detectable/build.gradle @@ -7,6 +7,7 @@ dependencies { implementation 'org.tomlj:tomlj:1.0.0' implementation 'com.moandjiezana.toml:toml4j:0.7.2' implementation 'com.paypal.digraph:digraph-parser:1.0' + implementation 'guru.nidi:graphviz-java:0.18.1' testImplementation 'org.skyscreamer:jsonassert:1.5.0' testImplementation 'org.junit.jupiter:junit-jupiter-api:5.3.1' diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtDotExtractor.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtDotExtractor.java index a270735b39..5cb9ceeb7d 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtDotExtractor.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtDotExtractor.java @@ -5,11 +5,11 @@ import java.util.List; import java.util.Set; -import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.paypal.digraph.parser.GraphParser; +import guru.nidi.graphviz.model.MutableGraph; +import guru.nidi.graphviz.parse.Parser; import com.blackduck.integration.bdio.graph.DependencyGraph; import com.blackduck.integration.bdio.model.dependency.Dependency; import com.blackduck.integration.detectable.ExecutableTarget; @@ -56,13 +56,13 @@ public Extraction extract(File directory, ExecutableTarget sbt, String sbtComman Extraction.Builder extraction = new Extraction.Builder(); for (File dotGraph : dotGraphs) { - GraphParser graphParser = new GraphParser(FileUtils.openInputStream(dotGraph)); - Set rootIDs = sbtRootNodeFinder.determineRootIDs(graphParser); + MutableGraph mutableGraph = new Parser().read(dotGraph); + Set rootIDs = sbtRootNodeFinder.determineRootIDs(mutableGraph); File projectFolder = dotGraph.getParentFile().getParentFile();//typically found in project-folder/target/<>.dot so .parent.parent == project folder if (rootIDs.size() == 1) { String projectId = rootIDs.stream().findFirst().get(); - DependencyGraph graph = sbtGraphParserTransformer.transformDotToGraph(graphParser, projectId); + DependencyGraph graph = sbtGraphParserTransformer.transformDotToGraph(projectId, mutableGraph); Dependency projectDependency = graphNodeParser.nodeToDependency(projectId); extraction.codeLocations(new CodeLocation(graph, projectDependency.getExternalId(), projectFolder)); if (projectFolder.equals(directory)) { @@ -72,7 +72,7 @@ public Extraction extract(File directory, ExecutableTarget sbt, String sbtComman } else { logger.warn("Unable to determine which node was the project in an SBT graph: " + dotGraph.toString()); logger.warn("This may mean you have extraneous dependencies and should consider removing them. The dependencies are: " + String.join(",", rootIDs)); - DependencyGraph graph = sbtGraphParserTransformer.transformDotToGraph(graphParser, rootIDs); + DependencyGraph graph = sbtGraphParserTransformer.transformDotToGraph(rootIDs, mutableGraph); extraction.codeLocations(new CodeLocation(graph, projectFolder)); } } diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtDotOutputParser.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtDotOutputParser.java index dec067c7c0..dec5e13ffa 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtDotOutputParser.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtDotOutputParser.java @@ -30,8 +30,8 @@ public List parseGeneratedGraphFiles(List dotOutput) { @Nullable private String parseDotGraphFromLine(String line) { - final String DOT_PREFIX = "[info] Wrote dependency graph to '"; - if (line.startsWith(DOT_PREFIX)) { + final String DOT_PREFIX = "Wrote dependency graph to '"; + if (line.contains(DOT_PREFIX)) { final String DOT_SUFFIX = "'"; return StringUtils.substringBetween(line, DOT_PREFIX, DOT_SUFFIX); } else { diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtGraphParserTransformer.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtGraphParserTransformer.java index 9bc6df36e5..c4dcdab5e6 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtGraphParserTransformer.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtGraphParserTransformer.java @@ -1,14 +1,16 @@ package com.blackduck.integration.detectable.detectables.sbt.dot; +import java.util.HashSet; +import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import guru.nidi.graphviz.model.Link; +import guru.nidi.graphviz.model.MutableGraph; +import guru.nidi.graphviz.model.MutableNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.paypal.digraph.parser.GraphEdge; -import com.paypal.digraph.parser.GraphElement; -import com.paypal.digraph.parser.GraphParser; import com.blackduck.integration.bdio.graph.BasicDependencyGraph; import com.blackduck.integration.bdio.graph.DependencyGraph; import com.blackduck.integration.bdio.model.dependency.Dependency; @@ -21,26 +23,32 @@ public SbtGraphParserTransformer(SbtDotGraphNodeParser sbtDotGraphNodeParser) { this.sbtDotGraphNodeParser = sbtDotGraphNodeParser; } - public DependencyGraph transformDotToGraph(GraphParser graphParser, String projectNodeId) { + public DependencyGraph transformDotToGraph(String projectNodeId, MutableGraph mutableGraph) { DependencyGraph graph = new BasicDependencyGraph(); - Set evictedIds = graphParser.getEdges().values().stream() - .filter( - edge -> edge.getAttribute("label") != null - && edge.getAttribute("label").toString().toLowerCase().contains("evicted") - ) - .map(GraphEdge::getNode1) - .map(GraphElement::getId) - .collect(Collectors.toSet()); - - for (GraphEdge graphEdge : graphParser.getEdges().values()) { - Dependency parent = sbtDotGraphNodeParser.nodeToDependency(graphEdge.getNode1().getId()); - Dependency child = sbtDotGraphNodeParser.nodeToDependency(graphEdge.getNode2().getId()); - - if (projectNodeId.equals(graphEdge.getNode1().getId())) { - graph.addChildToRoot(child); + Set evictedIds = new HashSet<>(); + mutableGraph.nodes().forEach(node -> { + node.attrs().forEach(attr -> { + if(attr.getKey().equals("label")) { + if(attr.getValue().toString().toLowerCase().contains("evicted")) { + evictedIds.add(node.name().toString()); + } + } + }); + }); + + List links = mutableGraph.nodes().stream().map(MutableNode::links).flatMap(List::stream).collect(Collectors.toList()); + for (Link link : links) { + String parentNode = normalizeDependency(link.asLinkTarget().name().toString()); + String childNode = normalizeDependency(link.asLinkSource().name().toString()); + + Dependency parent = sbtDotGraphNodeParser.nodeToDependency(parentNode); + Dependency child = sbtDotGraphNodeParser.nodeToDependency(childNode); + + if (projectNodeId.equals(parentNode)) { + graph.addDirectDependency(child); } else { - if (!evictedIds.contains(graphEdge.getNode2().getId())) { + if (!evictedIds.contains(childNode)) { graph.addChildWithParent(child, parent); } } @@ -49,18 +57,32 @@ public DependencyGraph transformDotToGraph(GraphParser graphParser, String proje return graph; } - public DependencyGraph transformDotToGraph(GraphParser graphParser, Set projectNodeIds) { + public DependencyGraph transformDotToGraph(Set projectNodeIds, MutableGraph mutableGraph) { DependencyGraph graph = new BasicDependencyGraph(); - for (GraphEdge graphEdge : graphParser.getEdges().values()) { - Dependency parent = sbtDotGraphNodeParser.nodeToDependency(graphEdge.getNode1().getId()); - Dependency child = sbtDotGraphNodeParser.nodeToDependency(graphEdge.getNode2().getId()); - if (projectNodeIds.contains(graphEdge.getNode1().getId())) { - graph.addChildToRoot(parent); + List links = mutableGraph.nodes().stream().map(MutableNode::links).flatMap(List::stream).collect(Collectors.toList()); + for (Link link : links) { + String parentNode = normalizeDependency(link.asLinkTarget().name().toString()); + String childNode = normalizeDependency(link.asLinkSource().name().toString()); + + Dependency parent = sbtDotGraphNodeParser.nodeToDependency(parentNode); + Dependency child = sbtDotGraphNodeParser.nodeToDependency(childNode); + + if (projectNodeIds.contains(parentNode)) { + graph.addDirectDependency(parent); } + graph.addChildWithParent(child, parent); + } return graph; } + + private String normalizeDependency(String dependency) { + if(dependency.startsWith("--")) { + return dependency.substring(2); + } + return dependency; + } } diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtRootNodeFinder.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtRootNodeFinder.java index e300bf477a..24cf82b393 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtRootNodeFinder.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtRootNodeFinder.java @@ -2,15 +2,18 @@ import java.util.HashSet; import java.util.Set; +import java.util.List; import java.util.stream.Collectors; +import guru.nidi.graphviz.attribute.Label; +import guru.nidi.graphviz.model.MutableNode; +import guru.nidi.graphviz.model.MutableGraph; +import guru.nidi.graphviz.model.LinkSource; +import guru.nidi.graphviz.model.Link; import org.apache.commons.collections4.SetUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.paypal.digraph.parser.GraphEdge; -import com.paypal.digraph.parser.GraphElement; -import com.paypal.digraph.parser.GraphParser; import com.blackduck.integration.detectable.detectable.exception.DetectableException; public class SbtRootNodeFinder { @@ -21,12 +24,16 @@ public SbtRootNodeFinder(SbtDotGraphNodeParser sbtDotGraphNodeParser) { this.sbtDotGraphNodeParser = sbtDotGraphNodeParser; } - public Set determineRootIDs(GraphParser graphParser) throws DetectableException { - Set nodeIdsUsedInDestination = graphParser.getEdges().values().stream() - .map(GraphEdge::getNode2) - .map(GraphElement::getId) - .collect(Collectors.toSet()); - Set allNodeIds = new HashSet<>(graphParser.getNodes().keySet()); + public Set determineRootIDs(MutableGraph mutableGraph) throws DetectableException { + Set nodeIdsUsedInDestination = mutableGraph.nodes().stream() + .map(MutableNode::links) + .flatMap(List::stream) + .map(Link::asLinkSource) + .map(LinkSource::name) + .map(Label::value) + .collect(Collectors.toSet()); + + Set allNodeIds = mutableGraph.nodes().stream().map(MutableNode::name).map(Label::value).collect(Collectors.toSet()); return SetUtils.difference(allNodeIds, nodeIdsUsedInDestination); } } diff --git a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/sbt/unit/SbtDotGraphNodeParserTest.java b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/sbt/unit/SbtDotGraphNodeParserTest.java index 73bcc296aa..d8d0f1560b 100644 --- a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/sbt/unit/SbtDotGraphNodeParserTest.java +++ b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/sbt/unit/SbtDotGraphNodeParserTest.java @@ -1,15 +1,16 @@ package com.blackduck.integration.detectable.detectables.sbt.unit; import java.io.ByteArrayInputStream; +import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; -import java.util.Map; +import guru.nidi.graphviz.model.MutableGraph; +import guru.nidi.graphviz.model.MutableNode; +import guru.nidi.graphviz.parse.Parser; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import com.paypal.digraph.parser.GraphNode; -import com.paypal.digraph.parser.GraphParser; import com.blackduck.integration.bdio.model.dependency.Dependency; import com.blackduck.integration.bdio.model.externalid.ExternalId; import com.blackduck.integration.bdio.model.externalid.ExternalIdFactory; @@ -18,7 +19,7 @@ public class SbtDotGraphNodeParserTest { @Test - public void canParseSimpleGraph() { + public void canParseSimpleGraph() throws IOException { String simpleGraph = "digraph \"dependency-graph\" {\n" + " graph[rankdir=\"LR\"]\n" + " edge [\n" @@ -28,15 +29,13 @@ public void canParseSimpleGraph() { + "\n" + "}"; InputStream stream = new ByteArrayInputStream(simpleGraph.getBytes(StandardCharsets.UTF_8)); - GraphParser graphParser = new GraphParser(stream); + MutableGraph mutableGraph = new Parser().read(stream); - Map.Entry node = graphParser.getNodes().entrySet().stream().findFirst().get(); + MutableNode node = mutableGraph.nodes().stream().findFirst().get(); SbtDotGraphNodeParser nodeParser = new SbtDotGraphNodeParser(new ExternalIdFactory()); - Dependency dependencyFromKey = nodeParser.nodeToDependency(node.getKey()); - Dependency dependencyFromId = nodeParser.nodeToDependency(node.getValue().getId()); + Dependency dependency = nodeParser.nodeToDependency(node.name().toString()); - assertDependency(dependencyFromId, "org.scalameta", "scalafmtroot_2.13", "2.7.5-SNAPSHOT"); - assertDependency(dependencyFromKey, "org.scalameta", "scalafmtroot_2.13", "2.7.5-SNAPSHOT"); + assertDependency(dependency, "org.scalameta", "scalafmtroot_2.13", "2.7.5-SNAPSHOT"); } @Test diff --git a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/sbt/unit/SbtRootNodeFinderTest.java b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/sbt/unit/SbtRootNodeFinderTest.java index 2bdfa429fb..75628959ad 100644 --- a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/sbt/unit/SbtRootNodeFinderTest.java +++ b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/sbt/unit/SbtRootNodeFinderTest.java @@ -1,14 +1,16 @@ package com.blackduck.integration.detectable.detectables.sbt.unit; import java.io.ByteArrayInputStream; +import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.Set; +import guru.nidi.graphviz.model.MutableGraph; +import guru.nidi.graphviz.parse.Parser; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import com.paypal.digraph.parser.GraphParser; import com.blackduck.integration.bdio.model.externalid.ExternalIdFactory; import com.blackduck.integration.detectable.detectable.exception.DetectableException; import com.blackduck.integration.detectable.detectables.sbt.dot.SbtDotGraphNodeParser; @@ -16,17 +18,17 @@ public class SbtRootNodeFinderTest { - private GraphParser createGraphParser(String actualGraph) { + private MutableGraph createMutableGraph(String actualGraph) throws IOException { String simpleGraph = "digraph \"dependency-graph\" {\n" - + " graph[rankdir=\"LR\"]\n" - + " edge [\n" - + " arrowtail=\"none\"\n" - + " ]\n" - + actualGraph + "\n" - + "\n" - + "}"; + + " graph[rankdir=\"LR\"]\n" + + " edge [\n" + + " arrowtail=\"none\"\n" + + " ]\n" + + actualGraph + "\n" + + "\n" + + "}"; InputStream stream = new ByteArrayInputStream(simpleGraph.getBytes(StandardCharsets.UTF_8)); - return new GraphParser(stream); + return new Parser().read(stream); } private String node(String org, String name, String version) { @@ -38,47 +40,47 @@ private String edge(String fromOrg, String fromName, String fromVersion, String } @Test - public void projectFoundFromSingleNode() throws DetectableException { - GraphParser graphParser = createGraphParser(node("one-org", "one-name", "one-version")); + public void projectFoundFromSingleNode() throws DetectableException, IOException { + MutableGraph mutableGraph = createMutableGraph(node("one-org", "one-name", "one-version")); SbtRootNodeFinder projectMatcher = new SbtRootNodeFinder(new SbtDotGraphNodeParser(new ExternalIdFactory())); - Set projectId = projectMatcher.determineRootIDs(graphParser); + Set projectId = projectMatcher.determineRootIDs(mutableGraph); Assertions.assertEquals(1, projectId.size()); - Assertions.assertEquals("\"one-org:one-name:one-version\"", projectId.stream().findFirst().get()); + Assertions.assertEquals("one-org:one-name:one-version", projectId.stream().findFirst().get()); } @Test - public void projectFoundFromTwoNodesWhereProjectIsSecond() throws DetectableException { - GraphParser graphParser = createGraphParser(node("two-org", "two-name", "two-version") + + public void projectFoundFromTwoNodesWhereProjectIsSecond() throws DetectableException, IOException { + MutableGraph mutableGraph = createMutableGraph(node("two-org", "two-name", "two-version") + node("one-org", "one-name", "one-version") + edge("one-org", "one-name", "one-version", "two-org", "two-name", "two-version")); SbtRootNodeFinder projectMatcher = new SbtRootNodeFinder(new SbtDotGraphNodeParser(new ExternalIdFactory())); - Set projectId = projectMatcher.determineRootIDs(graphParser); + Set projectId = projectMatcher.determineRootIDs(mutableGraph); Assertions.assertEquals(1, projectId.size()); - Assertions.assertEquals("\"one-org:one-name:one-version\"", projectId.stream().findFirst().get()); + Assertions.assertEquals("one-org:one-name:one-version", projectId.stream().findFirst().get()); } @Test - public void multipleFoundWithNoEdges() throws DetectableException { - GraphParser graphParser = createGraphParser(node("one-org", "one-name", "one-version") + + public void multipleFoundWithNoEdges() throws DetectableException, IOException { + MutableGraph mutableGraph = createMutableGraph(node("one-org", "one-name", "one-version") + node("one-org", "one-name", "two-version")); SbtRootNodeFinder projectMatcher = new SbtRootNodeFinder(new SbtDotGraphNodeParser(new ExternalIdFactory())); - Set projectId = projectMatcher.determineRootIDs(graphParser); + Set projectId = projectMatcher.determineRootIDs(mutableGraph); Assertions.assertEquals(2, projectId.size()); - Assertions.assertTrue(projectId.contains("\"one-org:one-name:one-version\"")); - Assertions.assertTrue(projectId.contains("\"one-org:one-name:two-version\"")); + Assertions.assertTrue(projectId.contains("one-org:one-name:one-version")); + Assertions.assertTrue(projectId.contains("one-org:one-name:two-version")); } @Test - public void multipleFoundWithEdge() throws DetectableException { - GraphParser graphParser = createGraphParser(node("one-org", "one-name", "one-version") + + public void multipleFoundWithEdge() throws DetectableException, IOException { + MutableGraph mutableGraph = createMutableGraph(node("one-org", "one-name", "one-version") + node("one-org", "one-name", "two-version") + node("one-org", "one-name", "three-version") + //should not be reported edge("one-org", "one-name", "one-version", "one-org", "one-name", "three-version")); SbtRootNodeFinder projectMatcher = new SbtRootNodeFinder(new SbtDotGraphNodeParser(new ExternalIdFactory())); - Set projectId = projectMatcher.determineRootIDs(graphParser); + Set projectId = projectMatcher.determineRootIDs(mutableGraph); Assertions.assertEquals(2, projectId.size()); - Assertions.assertTrue(projectId.contains("\"one-org:one-name:one-version\"")); - Assertions.assertTrue(projectId.contains("\"one-org:one-name:two-version\"")); + Assertions.assertTrue(projectId.contains("one-org:one-name:one-version")); + Assertions.assertTrue(projectId.contains("one-org:one-name:two-version")); } } From e6decd273da94151ebb7bd61717b834ef5991825 Mon Sep 17 00:00:00 2001 From: devmehtabd Date: Wed, 14 May 2025 09:44:45 -0700 Subject: [PATCH 2/6] Fix Evicted Ids not excluded from BOM --- .../detectables/sbt/dot/SbtGraphParserTransformer.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtGraphParserTransformer.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtGraphParserTransformer.java index c4dcdab5e6..88303d834d 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtGraphParserTransformer.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtGraphParserTransformer.java @@ -35,6 +35,15 @@ public DependencyGraph transformDotToGraph(String projectNodeId, MutableGraph mu } } }); + node.links().forEach(link -> { + link.attrs().forEach(attr -> { + if(attr.getKey().equals("label")) { + if(attr.getValue().toString().toLowerCase().contains("evicted")) { + evictedIds.add(node.name().toString()); + } + } + }); + }); }); List links = mutableGraph.nodes().stream().map(MutableNode::links).flatMap(List::stream).collect(Collectors.toList()); From 0ee971e85545d1d1e1b7d99eec52a32f026c73e9 Mon Sep 17 00:00:00 2001 From: devmehtabd Date: Fri, 16 May 2025 17:38:48 -0700 Subject: [PATCH 3/6] Add Integration test --- .../battery/docker/SbtEncodingTest.java | 41 +++++++++++++++++++ .../resources/docker/Sbt_Detector.dockerfile | 25 +++++++++++ 2 files changed, 66 insertions(+) create mode 100644 src/test/resources/docker/Sbt_Detector.dockerfile diff --git a/src/test/java/com/blackduck/integration/detect/battery/docker/SbtEncodingTest.java b/src/test/java/com/blackduck/integration/detect/battery/docker/SbtEncodingTest.java index 110acf165f..ff7dd638da 100644 --- a/src/test/java/com/blackduck/integration/detect/battery/docker/SbtEncodingTest.java +++ b/src/test/java/com/blackduck/integration/detect/battery/docker/SbtEncodingTest.java @@ -1,7 +1,13 @@ package com.blackduck.integration.detect.battery.docker; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import com.blackduck.integration.detect.battery.docker.integration.BlackDuckAssertions; +import com.blackduck.integration.detect.battery.docker.integration.BlackDuckTestConnection; +import com.blackduck.integration.detector.base.DetectorType; +import com.blackduck.integration.exception.IntegrationException; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @@ -14,6 +20,9 @@ @Tag("integration") public class SbtEncodingTest { + + public static String ARTIFACTORY_URL = System.getenv().get("SNPS_INTERNAL_ARTIFACTORY"); + @Test void sbtEncoding() throws IOException { try (DetectDockerTestRunner test = new DetectDockerTestRunner("detect-sbt-encoding", "detect-sbt-encoding:1.0.3")) { @@ -28,4 +37,36 @@ void sbtEncoding() throws IOException { dockerAssertions.projectVersion("sbt-simple-project_2.12", "1.0.0-SNAPSHOT"); } } + + @Test + void sbtDetectorTest() throws IOException, IntegrationException { + try (DetectDockerTestRunner test = new DetectDockerTestRunner("detect-sbt-detector", "detect-sbt-detector:1.0.1")) { + Map artifactoryArgs = new HashMap<>(); + artifactoryArgs.put("ARTIFACTORY_URL", ARTIFACTORY_URL); + + BuildDockerImageProvider buildDockerImageProvider = BuildDockerImageProvider.forDockerfilResourceNamed("Sbt_Detector.dockerfile"); + buildDockerImageProvider.setBuildArgs(artifactoryArgs); + test.withImageProvider(buildDockerImageProvider); + + String projectVersion = "sbt-detector:1.0.1"; + BlackDuckTestConnection blackDuckTestConnection = BlackDuckTestConnection.fromEnvironment(); + BlackDuckAssertions blackduckAssertions = blackDuckTestConnection.projectVersionAssertions("sbt-detector", projectVersion); + blackduckAssertions.emptyOnBlackDuck(); + + DetectCommandBuilder commandBuilder = new DetectCommandBuilder().defaults().defaultDirectories(test); + commandBuilder.connectToBlackDuck(blackDuckTestConnection); + commandBuilder.projectNameVersion(blackduckAssertions); + commandBuilder.waitForResults(); + + commandBuilder.property(DetectProperties.DETECT_TOOLS, DetectTool.DETECTOR.toString()); + DockerAssertions dockerAssertions = test.run(commandBuilder); + + dockerAssertions.atLeastOneBdioFile(); + dockerAssertions.logContains("SBT: SUCCESS"); + + blackduckAssertions.checkComponentVersionExists("scala-compiler", "2.12.18"); + blackduckAssertions.checkComponentVersionExists("Apache Log4J API", "2.24.3"); + blackduckAssertions.checkComponentVersionExists("scopt", "3.7.1"); + } + } } diff --git a/src/test/resources/docker/Sbt_Detector.dockerfile b/src/test/resources/docker/Sbt_Detector.dockerfile new file mode 100644 index 0000000000..fc6587709a --- /dev/null +++ b/src/test/resources/docker/Sbt_Detector.dockerfile @@ -0,0 +1,25 @@ +FROM maven:3-jdk-8-alpine + +ENV SRC_DIR=/opt/project/src + +ARG ARTIFACTORY_URL + +# Install git: https://github.com/nodejs/docker-node/issues/586 +RUN apk update && apk upgrade && \ + apk add --no-cache bash git openssh + +#Install SBT +WORKDIR /home/app +RUN wget -q "https://github.com/sbt/sbt/releases/download/v1.10.11/sbt-1.10.11.tgz" +RUN tar -xzf sbt-1.10.11.tgz +RUN rm sbt-1.10.11.tgz +ENV PATH="/home/app/sbt/bin:${PATH}" + +# Set up the test project +RUN mkdir -p ${SRC_DIR} + +RUN wget ${ARTIFACTORY_URL}/artifactory/detect-generic-qa-local/sbt_graphviz_java.zip +RUN unzip sbt_graphviz_java.zip -d /opt/project/src/ +RUN rm sbt_graphviz_java.zip + +RUN cd ${SRC_DIR} \ No newline at end of file From 6ccf0dad774b4a344f7c738b72c68a5912c5c8d2 Mon Sep 17 00:00:00 2001 From: devmehtabd Date: Wed, 21 May 2025 11:22:41 -0400 Subject: [PATCH 4/6] Remove code duplication --- .../detectables/sbt/dot/SbtDotExtractor.java | 3 +- .../sbt/dot/SbtGraphParserTransformer.java | 94 +++++++++++-------- 2 files changed, 54 insertions(+), 43 deletions(-) diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtDotExtractor.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtDotExtractor.java index 5cb9ceeb7d..1dce99adcc 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtDotExtractor.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtDotExtractor.java @@ -60,9 +60,9 @@ public Extraction extract(File directory, ExecutableTarget sbt, String sbtComman Set rootIDs = sbtRootNodeFinder.determineRootIDs(mutableGraph); File projectFolder = dotGraph.getParentFile().getParentFile();//typically found in project-folder/target/<>.dot so .parent.parent == project folder + DependencyGraph graph = sbtGraphParserTransformer.transformDotToGraph(rootIDs, mutableGraph); if (rootIDs.size() == 1) { String projectId = rootIDs.stream().findFirst().get(); - DependencyGraph graph = sbtGraphParserTransformer.transformDotToGraph(projectId, mutableGraph); Dependency projectDependency = graphNodeParser.nodeToDependency(projectId); extraction.codeLocations(new CodeLocation(graph, projectDependency.getExternalId(), projectFolder)); if (projectFolder.equals(directory)) { @@ -72,7 +72,6 @@ public Extraction extract(File directory, ExecutableTarget sbt, String sbtComman } else { logger.warn("Unable to determine which node was the project in an SBT graph: " + dotGraph.toString()); logger.warn("This may mean you have extraneous dependencies and should consider removing them. The dependencies are: " + String.join(",", rootIDs)); - DependencyGraph graph = sbtGraphParserTransformer.transformDotToGraph(rootIDs, mutableGraph); extraction.codeLocations(new CodeLocation(graph, projectFolder)); } } diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtGraphParserTransformer.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtGraphParserTransformer.java index 88303d834d..558a2b710e 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtGraphParserTransformer.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtGraphParserTransformer.java @@ -2,6 +2,7 @@ import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -23,28 +24,11 @@ public SbtGraphParserTransformer(SbtDotGraphNodeParser sbtDotGraphNodeParser) { this.sbtDotGraphNodeParser = sbtDotGraphNodeParser; } - public DependencyGraph transformDotToGraph(String projectNodeId, MutableGraph mutableGraph) { + public DependencyGraph transformDotToGraph(Set projectIds, MutableGraph mutableGraph) { DependencyGraph graph = new BasicDependencyGraph(); - - Set evictedIds = new HashSet<>(); - mutableGraph.nodes().forEach(node -> { - node.attrs().forEach(attr -> { - if(attr.getKey().equals("label")) { - if(attr.getValue().toString().toLowerCase().contains("evicted")) { - evictedIds.add(node.name().toString()); - } - } - }); - node.links().forEach(link -> { - link.attrs().forEach(attr -> { - if(attr.getKey().equals("label")) { - if(attr.getValue().toString().toLowerCase().contains("evicted")) { - evictedIds.add(node.name().toString()); - } - } - }); - }); - }); + String projectNodeId = projectIds.stream().findFirst().get(); + boolean isOneRoot = projectIds.size() == 1; + Set evictedIds = getEvictedIds(mutableGraph); List links = mutableGraph.nodes().stream().map(MutableNode::links).flatMap(List::stream).collect(Collectors.toList()); for (Link link : links) { @@ -54,40 +38,68 @@ public DependencyGraph transformDotToGraph(String projectNodeId, MutableGraph mu Dependency parent = sbtDotGraphNodeParser.nodeToDependency(parentNode); Dependency child = sbtDotGraphNodeParser.nodeToDependency(childNode); - if (projectNodeId.equals(parentNode)) { - graph.addDirectDependency(child); + if(isOneRoot) { + if (projectNodeId.equals(parentNode)) { + graph.addDirectDependency(child); + } else { + if (!evictedIds.contains(childNode)) { + graph.addChildWithParent(child, parent); + } + } } else { - if (!evictedIds.contains(childNode)) { - graph.addChildWithParent(child, parent); + if (projectIds.contains(parentNode)) { + graph.addDirectDependency(parent); } + + graph.addChildWithParent(child, parent); } } return graph; } - public DependencyGraph transformDotToGraph(Set projectNodeIds, MutableGraph mutableGraph) { - DependencyGraph graph = new BasicDependencyGraph(); - - List links = mutableGraph.nodes().stream().map(MutableNode::links).flatMap(List::stream).collect(Collectors.toList()); - for (Link link : links) { - String parentNode = normalizeDependency(link.asLinkTarget().name().toString()); - String childNode = normalizeDependency(link.asLinkSource().name().toString()); + private Set getEvictedIds(MutableGraph mutableGraph) { + Set evictedIds = new HashSet<>(); + mutableGraph.nodes().forEach(node -> { + node.attrs().forEach(attr -> { + addEvictedEntry(attr, node, evictedIds); + }); + node.links().forEach(link -> { + link.attrs().forEach(attr -> { + addEvictedEntry(attr, node, evictedIds); + }); + }); + }); - Dependency parent = sbtDotGraphNodeParser.nodeToDependency(parentNode); - Dependency child = sbtDotGraphNodeParser.nodeToDependency(childNode); + return evictedIds; + } - if (projectNodeIds.contains(parentNode)) { - graph.addDirectDependency(parent); + private void addEvictedEntry(Map.Entry attr, MutableNode node, Set evictedIds) { + if(attr.getKey().equals("label")) { + if(attr.getValue().toString().toLowerCase().contains("evicted")) { + evictedIds.add(node.name().toString()); } - - graph.addChildWithParent(child, parent); - } - - return graph; } +// public DependencyGraph transformDotToGraph(Set projectNodeIds, MutableGraph mutableGraph) { +// DependencyGraph graph = new BasicDependencyGraph(); +// +// List links = mutableGraph.nodes().stream().map(MutableNode::links).flatMap(List::stream).collect(Collectors.toList()); +// for (Link link : links) { +// String parentNode = normalizeDependency(link.asLinkTarget().name().toString()); +// String childNode = normalizeDependency(link.asLinkSource().name().toString()); +// +// Dependency parent = sbtDotGraphNodeParser.nodeToDependency(parentNode); +// Dependency child = sbtDotGraphNodeParser.nodeToDependency(childNode); +// +// +// +// } +// +// return graph; +// } + private String normalizeDependency(String dependency) { if(dependency.startsWith("--")) { return dependency.substring(2); From e99ad40b5f460810f1eb9b3ce0ded51762476777 Mon Sep 17 00:00:00 2001 From: devmehtabd Date: Wed, 21 May 2025 11:24:05 -0400 Subject: [PATCH 5/6] Remove commented code --- .../sbt/dot/SbtGraphParserTransformer.java | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtGraphParserTransformer.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtGraphParserTransformer.java index 558a2b710e..7eae15a279 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtGraphParserTransformer.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtGraphParserTransformer.java @@ -82,24 +82,6 @@ private void addEvictedEntry(Map.Entry attr, MutableNode node, S } } -// public DependencyGraph transformDotToGraph(Set projectNodeIds, MutableGraph mutableGraph) { -// DependencyGraph graph = new BasicDependencyGraph(); -// -// List links = mutableGraph.nodes().stream().map(MutableNode::links).flatMap(List::stream).collect(Collectors.toList()); -// for (Link link : links) { -// String parentNode = normalizeDependency(link.asLinkTarget().name().toString()); -// String childNode = normalizeDependency(link.asLinkSource().name().toString()); -// -// Dependency parent = sbtDotGraphNodeParser.nodeToDependency(parentNode); -// Dependency child = sbtDotGraphNodeParser.nodeToDependency(childNode); -// -// -// -// } -// -// return graph; -// } - private String normalizeDependency(String dependency) { if(dependency.startsWith("--")) { return dependency.substring(2); From 62ac66fa6cd7c5a6c0c83d5507aa226bd72347db Mon Sep 17 00:00:00 2001 From: devmehtabd Date: Wed, 21 May 2025 13:20:12 -0400 Subject: [PATCH 6/6] Address sonarcloud issues --- .../sbt/dot/SbtGraphParserTransformer.java | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtGraphParserTransformer.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtGraphParserTransformer.java index 7eae15a279..7fb651c7c6 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtGraphParserTransformer.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/sbt/dot/SbtGraphParserTransformer.java @@ -9,15 +9,12 @@ import guru.nidi.graphviz.model.Link; import guru.nidi.graphviz.model.MutableGraph; import guru.nidi.graphviz.model.MutableNode; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import com.blackduck.integration.bdio.graph.BasicDependencyGraph; import com.blackduck.integration.bdio.graph.DependencyGraph; import com.blackduck.integration.bdio.model.dependency.Dependency; public class SbtGraphParserTransformer { - private final Logger logger = LoggerFactory.getLogger(this.getClass()); private final SbtDotGraphNodeParser sbtDotGraphNodeParser; public SbtGraphParserTransformer(SbtDotGraphNodeParser sbtDotGraphNodeParser) { @@ -61,24 +58,16 @@ public DependencyGraph transformDotToGraph(Set projectIds, MutableGraph private Set getEvictedIds(MutableGraph mutableGraph) { Set evictedIds = new HashSet<>(); mutableGraph.nodes().forEach(node -> { - node.attrs().forEach(attr -> { - addEvictedEntry(attr, node, evictedIds); - }); - node.links().forEach(link -> { - link.attrs().forEach(attr -> { - addEvictedEntry(attr, node, evictedIds); - }); - }); + node.attrs().forEach(attr -> addEvictedEntry(attr, node, evictedIds)); + node.links().forEach(link -> link.attrs().forEach(attr -> addEvictedEntry(attr, node, evictedIds))); }); return evictedIds; } private void addEvictedEntry(Map.Entry attr, MutableNode node, Set evictedIds) { - if(attr.getKey().equals("label")) { - if(attr.getValue().toString().toLowerCase().contains("evicted")) { - evictedIds.add(node.name().toString()); - } + if(attr.getKey().equals("label") && attr.getValue().toString().toLowerCase().contains("evicted")) { + evictedIds.add(node.name().toString()); } }