Detect property to exclude Cargo dependencies

detectable/src/main/java/com/blackduck/integration/detectable/detectables/cargo/CargoCliDetectable.java

@@ -36,15 +36,17 @@ public class CargoCliDetectable extends Detectable {
     private final CargoResolver cargoResolver;
     private final CargoCliExtractor cargoCliExtractor;
     private final DetectableExecutableRunner executableRunner;
+    private final CargoDetectableOptions cargoDetectableOptions;
     private ExecutableTarget cargoExe;
     private File cargoToml;
 
-    public CargoCliDetectable(DetectableEnvironment environment, FileFinder fileFinder, CargoResolver cargoResolver, CargoCliExtractor cargoCliExtractor, DetectableExecutableRunner executableRunner) {
+    public CargoCliDetectable(DetectableEnvironment environment, FileFinder fileFinder, CargoResolver cargoResolver, CargoCliExtractor cargoCliExtractor, DetectableExecutableRunner executableRunner, CargoDetectableOptions cargoDetectableOptions) {
         super(environment);
         this.fileFinder = fileFinder;
         this.cargoResolver = cargoResolver;
         this.cargoCliExtractor = cargoCliExtractor;
         this.executableRunner = executableRunner;
+        this.cargoDetectableOptions = cargoDetectableOptions;
     }
 
     @Override
@@ -72,7 +74,7 @@ public DetectableResult extractable() throws DetectableException {
     @Override
     public Extraction extract(ExtractionEnvironment extractionEnvironment) throws IOException, DetectableException, MissingExternalIdException, ExecutableRunnerException {
         try {
-            return cargoCliExtractor.extract(environment.getDirectory(), cargoExe, cargoToml);
+            return cargoCliExtractor.extract(environment.getDirectory(), cargoExe, cargoToml, cargoDetectableOptions);
         } catch (Exception e) {
             logger.error("Failed to extract Cargo dependencies.", e);
             return new Extraction.Builder().failure("Cargo extraction failed due to an exception: " + e.getMessage()).build();

detectable/src/main/java/com/blackduck/integration/detectable/detectables/cargo/CargoCliExtractor.java

@@ -16,9 +16,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Optional;
+import java.util.*;
 
 public class CargoCliExtractor {
     private static final List<String> CARGO_TREE_COMMAND = Arrays.asList("tree", "--no-dedupe", "--prefix", "depth");
@@ -32,8 +30,12 @@ public CargoCliExtractor(DetectableExecutableRunner executableRunner, CargoDepen
         this.cargoTomlParser = cargoTomlParser;
     }
 
-    public Extraction extract(File directory, ExecutableTarget cargoExe, File cargoTomlFile) throws ExecutableFailedException, IOException {
-        ExecutableOutput cargoOutput = executableRunner.executeSuccessfully(ExecutableUtils.createFromTarget(directory, cargoExe, CARGO_TREE_COMMAND));
+    public Extraction extract(File directory, ExecutableTarget cargoExe, File cargoTomlFile, CargoDetectableOptions cargoDetectableOptions) throws ExecutableFailedException, IOException {
+        List<String> cargoTreeCommand = new ArrayList<>(CARGO_TREE_COMMAND);
+
+        addEdgeExclusions(cargoTreeCommand, cargoDetectableOptions);
+
+        ExecutableOutput cargoOutput = executableRunner.executeSuccessfully(ExecutableUtils.createFromTarget(directory, cargoExe, cargoTreeCommand));
         List<String> cargoTreeOutput = cargoOutput.getStandardOutputAsList();
 
         DependencyGraph graph = cargoDependencyTransformer.transform(cargoTreeOutput);
@@ -51,4 +53,24 @@ public Extraction extract(File directory, ExecutableTarget cargoExe, File cargoT
             .nameVersionIfPresent(projectNameVersion)
             .build();
     }
+
+    private void addEdgeExclusions(List<String> cargoTreeCommand, CargoDetectableOptions options) {
+        Map<CargoDependencyType, String> exclusionMap = new EnumMap<>(CargoDependencyType.class);
+        exclusionMap.put(CargoDependencyType.NORMAL, "no-normal");
+        exclusionMap.put(CargoDependencyType.BUILD, "no-build");
+        exclusionMap.put(CargoDependencyType.DEV, "no-dev");
+        exclusionMap.put(CargoDependencyType.PROC_MACRO, "no-proc-macro");
+
+        List<String> exclusions = new ArrayList<>();
+        for (Map.Entry<CargoDependencyType, String> entry : exclusionMap.entrySet()) {
+            if (options.getDependencyTypeFilter().shouldExclude(entry.getKey())) {
+                exclusions.add(entry.getValue());
+            }
+        }
+
+        if (!exclusions.isEmpty()) {
+            cargoTreeCommand.add("--edges");
+            cargoTreeCommand.add(String.join(",", exclusions));
+        }
+    }
 }

detectable/src/main/java/com/blackduck/integration/detectable/detectables/cargo/CargoDependencyType.java

@@ -0,0 +1,8 @@
+package com.blackduck.integration.detectable.detectables.cargo;
+
+public enum CargoDependencyType {
+    NORMAL,
+    BUILD,
+    DEV,
+    PROC_MACRO
+}

detectable/src/main/java/com/blackduck/integration/detectable/detectables/cargo/CargoDetectableOptions.java

@@ -0,0 +1,15 @@
+package com.blackduck.integration.detectable.detectables.cargo;
+
+import com.blackduck.integration.detectable.detectable.util.EnumListFilter;
+
+public class CargoDetectableOptions {
+    private final EnumListFilter<CargoDependencyType> dependencyTypeFilter;
+
+    public CargoDetectableOptions(EnumListFilter<CargoDependencyType> dependencyTypeFilter) {
+        this.dependencyTypeFilter = dependencyTypeFilter;
+    }
+
+    public EnumListFilter<CargoDependencyType> getDependencyTypeFilter() {
+        return dependencyTypeFilter;
+    }
+}

detectable/src/main/java/com/blackduck/integration/detectable/detectables/cargo/CargoExtractor.java

@@ -3,11 +3,10 @@
 import java.io.File;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
+import java.util.*;
 import java.util.stream.Collectors;
 
+import com.blackduck.integration.detectable.detectables.cargo.data.CargoLockPackageData;
 import org.apache.commons.io.FileUtils;
 import org.jetbrains.annotations.Nullable;
 
@@ -23,8 +22,11 @@
 import com.blackduck.integration.detectable.detectables.cargo.transform.CargoLockPackageTransformer;
 import com.blackduck.integration.detectable.extraction.Extraction;
 import com.blackduck.integration.util.NameVersion;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class CargoExtractor {
+    private static final Logger logger = LoggerFactory.getLogger(CargoExtractor.class);
     private final CargoTomlParser cargoTomlParser;
     private final CargoLockPackageDataTransformer cargoLockPackageDataTransformer;
     private final CargoLockPackageTransformer cargoLockPackageTransformer;
@@ -39,26 +41,73 @@ public CargoExtractor(
         this.cargoLockPackageTransformer = cargoLockPackageTransformer;
     }
 
-    public Extraction extract(File cargoLockFile, @Nullable File cargoTomlFile) throws IOException, DetectableException, MissingExternalIdException {
+    public Extraction extract(File cargoLockFile, @Nullable File cargoTomlFile, CargoDetectableOptions cargoDetectableOptions) throws IOException, DetectableException, MissingExternalIdException {
         CargoLockData cargoLockData = new Toml().read(cargoLockFile).to(CargoLockData.class);
-        List<CargoLockPackage> packages = cargoLockData.getPackages()
-            .orElse(new ArrayList<>()).stream()
+        List<CargoLockPackageData> cargoLockPackageDataList = cargoLockData.getPackages().orElse(new ArrayList<>());
+        List<CargoLockPackageData> filteredPackages = cargoLockPackageDataList;
+        String cargoTomlContents = FileUtils.readFileToString(cargoTomlFile, StandardCharsets.UTF_8);
+
+        if (cargoDetectableOptions != null) {
+            Map<String, String> excludableDependencyMap = cargoTomlParser.parseDependencyNameVersions(cargoTomlContents, cargoDetectableOptions);
+            filteredPackages = excludeDependencies(cargoLockPackageDataList, excludableDependencyMap);
+        }
+
+        List<CargoLockPackage> packages = filteredPackages.stream()
             .map(cargoLockPackageDataTransformer::transform)
             .collect(Collectors.toList());
 
         DependencyGraph graph = cargoLockPackageTransformer.transformToGraph(packages);
 
         Optional<NameVersion> projectNameVersion = Optional.empty();
         if (cargoTomlFile != null) {
-            String cargoTomlContents = FileUtils.readFileToString(cargoTomlFile, StandardCharsets.UTF_8);
             projectNameVersion = cargoTomlParser.parseNameVersionFromCargoToml(cargoTomlContents);
         }
-
         CodeLocation codeLocation = new CodeLocation(graph); //TODO: Consider for producing a ProjectDependencyGraph
 
         return new Extraction.Builder()
             .success(codeLocation)
             .nameVersionIfPresent(projectNameVersion)
             .build();
     }
+
+    private List<CargoLockPackageData> excludeDependencies(
+        List<CargoLockPackageData> packages,
+        Map<String, String> excludableDependencyMap
+    ) {
+        Set<String> excludedNames = new HashSet<>();
+
+        List<CargoLockPackageData> filtered = packages.stream()
+            .filter(pkg -> {
+                String name = pkg.getName().orElse(null);
+                String version = pkg.getVersion().orElse(null);
+                if (name == null || version == null) return true;
+
+                if (excludableDependencyMap.containsKey(name)) {
+                    String constraint = excludableDependencyMap.get(name);
+                    boolean matches = constraint == null || VersionUtils.versionMatches(constraint, version);
+                    if (matches) {
+                        logger.debug("Excluding package '{}' version '{}' due to constraint '{}'", name, version, constraint);
+                        excludedNames.add(name);
+                        return false;
+                    }
+                }
+
+                return true;
+            })
+            .collect(Collectors.toList());
+
+        return filtered.stream()
+            .map(pkg -> new CargoLockPackageData(
+                pkg.getName().orElse(null),
+                pkg.getVersion().orElse(null),
+                pkg.getSource().orElse(null),
+                pkg.getChecksum().orElse(null),
+                pkg.getDependencies()
+                    .orElse(new ArrayList<>())
+                    .stream()
+                    .filter(dep -> !excludedNames.contains(dep))
+                    .collect(Collectors.toList())
+            ))
+            .collect(Collectors.toList());
+    }
 }

detectable/src/main/java/com/blackduck/integration/detectable/detectables/cargo/CargoLockDetectable.java

@@ -25,14 +25,15 @@ public class CargoLockDetectable extends Detectable {
 
     private final FileFinder fileFinder;
     private final CargoExtractor cargoExtractor;
-
+    private final CargoDetectableOptions cargoDetectableOptions;
     private File cargoLock;
     private File cargoToml;
 
-    public CargoLockDetectable(DetectableEnvironment environment, FileFinder fileFinder, CargoExtractor cargoExtractor) {
+    public CargoLockDetectable(DetectableEnvironment environment, FileFinder fileFinder, CargoExtractor cargoExtractor, CargoDetectableOptions cargoDetectableOptions) {
         super(environment);
         this.fileFinder = fileFinder;
         this.cargoExtractor = cargoExtractor;
+        this.cargoDetectableOptions = cargoDetectableOptions;
     }
 
     @Override
@@ -52,6 +53,6 @@ public DetectableResult extractable() {
 
     @Override
     public Extraction extract(ExtractionEnvironment extractionEnvironment) throws IOException, DetectableException, MissingExternalIdException {
-        return cargoExtractor.extract(cargoLock, cargoToml);
+        return cargoExtractor.extract(cargoLock, cargoToml, cargoDetectableOptions);
     }
 }

detectable/src/main/java/com/blackduck/integration/detectable/detectables/cargo/VersionUtils.java

@@ -14,4 +14,50 @@ public static int compareVersions(String version1, String version2) {
         }
         return 0;
     }
+
+    public static boolean versionMatches(String constraint, String actualVersion) {
+        if (constraint == null || actualVersion == null) {
+            return false;
+        }
+
+        String normalizedActual = normalizeVersion(actualVersion);
+        String normalizedConstraintVersion;
+
+        if (constraint.startsWith(">=")) {
+            normalizedConstraintVersion = normalizeVersion(constraint.substring(2));
+            return compareVersions(normalizedActual, normalizedConstraintVersion) >= 0;
+        } else if (constraint.startsWith(">")) {
+            normalizedConstraintVersion = normalizeVersion(constraint.substring(1));
+            return compareVersions(normalizedActual, normalizedConstraintVersion) > 0;
+        } else if (constraint.startsWith("<=")) {
+            normalizedConstraintVersion = normalizeVersion(constraint.substring(2));
+            return compareVersions(normalizedActual, normalizedConstraintVersion) <= 0;
+        } else if (constraint.startsWith("<")) {
+            normalizedConstraintVersion = normalizeVersion(constraint.substring(1));
+            return compareVersions(normalizedActual, normalizedConstraintVersion) < 0;
+        } else if (constraint.startsWith("=")) {
+            normalizedConstraintVersion = normalizeVersion(constraint.substring(1));
+            return compareVersions(normalizedActual, normalizedConstraintVersion) == 0;
+        } else {
+            // Default to exact match
+            normalizedConstraintVersion = normalizeVersion(constraint);
+            return compareVersions(normalizedActual, normalizedConstraintVersion) == 0;
+        }
+    }
+
+    private static String normalizeVersion(String version) {
+        String[] parts = version.split("\\.");
+        StringBuilder normalized = new StringBuilder();
+        for (int i = 0; i < 3; i++) {
+            if (i < parts.length) {
+                normalized.append(parts[i]);
+            } else {
+                normalized.append("0");
+            }
+            if (i < 2) {
+                normalized.append(".");
+            }
+        }
+        return normalized.toString();
+    }
 }

detectable/src/main/java/com/blackduck/integration/detectable/detectables/cargo/parse/CargoTomlParser.java

@@ -1,16 +1,22 @@
 package com.blackduck.integration.detectable.detectables.cargo.parse;
 
-import java.util.Optional;
+import java.util.*;
 
+import com.blackduck.integration.detectable.detectables.cargo.CargoDependencyType;
+import com.blackduck.integration.detectable.detectables.cargo.CargoDetectableOptions;
 import org.tomlj.Toml;
 import org.tomlj.TomlParseResult;
 
 import com.blackduck.integration.util.NameVersion;
+import org.tomlj.TomlTable;
 
 public class CargoTomlParser {
     private static final String NAME_KEY = "name";
     private static final String VERSION_KEY = "version";
     private static final String PACKAGE_KEY = "package";
+    private static final String NORMAL_DEPENDENCIES_KEY = "dependencies";
+    private static final String BUILD_DEPENDENCIES_KEY = "build-dependencies";
+    private static final String DEV_DEPENDENCIES_KEY = "dev-dependencies";
 
     public Optional<NameVersion> parseNameVersionFromCargoToml(String tomlFileContents) {
         TomlParseResult cargoTomlObject = Toml.parse(tomlFileContents);
@@ -22,4 +28,41 @@ public Optional<NameVersion> parseNameVersionFromCargoToml(String tomlFileConten
         return Optional.empty();
     }
 
+    public Map<String, String> parseDependencyNameVersions(String tomlFileContents, CargoDetectableOptions cargoDetectableOptions) {
+        TomlParseResult toml = Toml.parse(tomlFileContents);
+        Map<String, String> allDeps = new HashMap<>();
+
+        if (cargoDetectableOptions.getDependencyTypeFilter().shouldExclude(CargoDependencyType.NORMAL)) {
+            allDeps.putAll(parseNamedDependenciesFromTable(toml, NORMAL_DEPENDENCIES_KEY));
+        }
+        if (cargoDetectableOptions.getDependencyTypeFilter().shouldExclude(CargoDependencyType.BUILD)) {
+            allDeps.putAll(parseNamedDependenciesFromTable(toml, BUILD_DEPENDENCIES_KEY));
+        }
+        if (cargoDetectableOptions.getDependencyTypeFilter().shouldExclude(CargoDependencyType.DEV)) {
+            allDeps.putAll(parseNamedDependenciesFromTable(toml, DEV_DEPENDENCIES_KEY));
+        }
+
+        return allDeps;
+    }
+
+    private Map<String, String> parseNamedDependenciesFromTable(TomlParseResult toml, String sectionKey) {
+        Map<String, String> deps = new HashMap<>();
+        TomlTable table = toml.getTable(sectionKey);
+        if (table == null) {
+            return deps;
+        }
+
+        for (String key : table.keySet()) {
+            Object value = table.get(key);
+            if (value instanceof String) {
+                deps.put(key, (String) value);
+            } else if (value instanceof TomlTable) {
+                TomlTable dependencyTable = (TomlTable) value;
+                String version = dependencyTable.getString(VERSION_KEY); // May be null
+                deps.put(key, version);
+            }
+        }
+
+        return deps;
+    }
 }

detectable/src/main/java/com/blackduck/integration/detectable/detectables/cargo/transform/CargoLockPackageTransformer.java

@@ -1,6 +1,8 @@
 package com.blackduck.integration.detectable.detectables.cargo.transform;
 
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import com.blackduck.integration.bdio.graph.DependencyGraph;
 import com.blackduck.integration.bdio.graph.builder.LazyExternalIdDependencyGraphBuilder;
@@ -9,6 +11,7 @@
 import com.blackduck.integration.bdio.model.Forge;
 import com.blackduck.integration.bdio.model.dependency.Dependency;
 import com.blackduck.integration.bdio.model.dependency.DependencyFactory;
+import com.blackduck.integration.bdio.model.externalid.ExternalId;
 import com.blackduck.integration.bdio.model.externalid.ExternalIdFactory;
 import com.blackduck.integration.detectable.detectable.exception.DetectableException;
 import com.blackduck.integration.detectable.detectables.cargo.model.CargoLockPackage;
@@ -26,15 +29,21 @@ public DependencyGraph transformToGraph(List<CargoLockPackage> lockPackages) thr
             String parentName = lockPackage.getPackageNameVersion().getName();
             String parentVersion = lockPackage.getPackageNameVersion().getVersion();
             LazyId parentId = LazyId.fromNameAndVersion(parentName, parentVersion);
-            Dependency parentDependency = dependencyFactory.createNameVersionDependency(Forge.CRATES, parentName, parentVersion);
+            ExternalId parentExternalId = externalIdFactory.createNameVersionExternalId(Forge.CRATES, parentName, parentVersion);
 
-            graph.addChildToRoot(parentId);
-            graph.setDependencyInfo(parentId, parentDependency.getName(), parentDependency.getVersion(), parentDependency.getExternalId());
+            graph.setDependencyInfo(parentId, parentName, parentVersion, parentExternalId);
             graph.setDependencyAsAlias(parentId, LazyId.fromName(parentName));
+            graph.addChildToRoot(parentId);
 
             lockPackage.getDependencies().forEach(childPackage -> {
                 if (childPackage.getVersion().isPresent()) {
-                    LazyId childId = LazyId.fromNameAndVersion(childPackage.getName(), childPackage.getVersion().get());
+                    String childName = childPackage.getName();
+                    String childVersion = childPackage.getVersion().get();
+                    LazyId childId = LazyId.fromNameAndVersion(childName, childVersion);
+                    ExternalId childExternalId = externalIdFactory.createNameVersionExternalId(Forge.CRATES, childName, childVersion);
+
+                    graph.setDependencyInfo(childId, childName, childVersion, childExternalId);
+                    graph.setDependencyAsAlias(childId, LazyId.fromName(childName));
                     graph.addChildWithParent(childId, parentId);
                 } else {
                     LazyId childId = LazyId.fromName(childPackage.getName());