Merge pull request #32 from nexB/prepare-098beta1

Prepare release 0.9.8-beta1

Author: pombredanne

CHANGELOG.rst

@@ -2,6 +2,21 @@ Changelog
 =========
 
 
+v0.9.8
+-------
+
+This is a major feature update release with these updates and API breaking changes:
+
+* Add new "--with-nuget-org" command line option to optionally include the
+  nuget.org API feeds to the set of nuget.config-provided endpoints. Note that
+  this is always included if there is no provided or available nuget.config
+
+* Add extra debug tracing with the --debug option
+
+* Do not create API or download URLs using a default. Only use API-provided values.
+
+* Ensure we do not fail with a Package reference with no version.
+
 
 v0.9.7
 -------

build.sh

@@ -17,6 +17,6 @@ dotnet publish \
   --runtime linux-x64 \
   --self-contained true \
   --configuration Release \
-  -p:Version=0.9.7 \
+  -p:Version=0.9.8-beta1 \
   --output build \
   src/nuget-inspector/nuget-inspector.csproj

release.sh

@@ -16,7 +16,7 @@
 rm -rf release/
 mkdir release
 
-VERSION=0.9.7
+VERSION=0.9.8-beta1
 
 TARGET_BASE=nuget-inspector-$(git describe)
 

setup.cfg

@@ -1,7 +1,7 @@
 [bumpversion]
 commit = False
 tag = False
-current_version = 0.9.7
+current_version = 0.9.8-beta1
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-(?P<release>[a-z]+))?
 serialize = 
 	{major}.{minor}.{patch}-{release}

src/nuget-inspector/Config.cs

@@ -8,6 +8,7 @@ public static class Config
     public static bool TRACE_NET = false;
     public static bool TRACE_DEEP = false;
     public static bool TRACE_META = false;
-    public const string NUGET_INSPECTOR_VERSION = "0.9.7";
+    public static bool TRACE_OUTPUT = false;
+    public const string NUGET_INSPECTOR_VERSION = "0.9.8-beta1";
     #pragma warning restore CA2211
 }

src/nuget-inspector/Models.cs

@@ -442,8 +442,7 @@ public void UpdateAttributes(
                 if (
                     string.IsNullOrWhiteSpace(api_data_url)
                     && download_url.StartsWith("https://api.nuget.org/")
-                    && !string.IsNullOrWhiteSpace(synthetic_api_data_url)
-                )
+                    && !string.IsNullOrWhiteSpace(synthetic_api_data_url))
                 {
                     api_data_url = synthetic_api_data_url;
                 }

src/nuget-inspector/NugetApi.cs

@@ -33,7 +33,7 @@ public class NugetApi
     private readonly Dictionary<string, JObject> catalog_entry_by_catalog_url = new();
     private readonly Dictionary<string, List<PackageSearchMetadataRegistration>> psmrs_by_package_name = new();
     private readonly Dictionary<PackageIdentity, PackageSearchMetadataRegistration?> psmr_by_identity = new();
-    private readonly Dictionary<(PackageIdentity, NuGetFramework), PackageDownload> download_by_identity = new();
+    private readonly Dictionary<(PackageIdentity, NuGetFramework), PackageDownload?> download_by_identity = new();
     private readonly Dictionary<(PackageIdentity, NuGetFramework), SourcePackageDependencyInfo> spdi_by_identity = new();
 
     private readonly List<SourceRepository> source_repositories = new();
@@ -44,7 +44,11 @@ public class NugetApi
 
     private readonly NuGetFramework project_framework;
 
-    public NugetApi(string nuget_config_path, string project_root_path, NuGetFramework project_framework)
+    public NugetApi(
+        string nuget_config_path,
+        string project_root_path,
+        NuGetFramework project_framework,
+        bool with_nuget_org)
     {
         this.project_framework = project_framework;
         List<Lazy<INuGetResourceProvider>> providers = new();
@@ -56,7 +60,8 @@ public NugetApi(string nuget_config_path, string project_root_path, NuGetFramewo
 
         PopulateResources(
             providers: providers,
-            settings: settings);
+            settings: settings,
+            with_nuget_org: with_nuget_org);
     }
 
     /// <summary>
@@ -114,13 +119,13 @@ private List<PackageSearchMetadataRegistration> FindPackageVersionsThroughCache(
     /// <returns>PackageSearchMetadataRegistration or null</returns>
     public PackageSearchMetadataRegistration? FindPackageVersion(PackageIdentity pid)
     {
-        if (Config.TRACE_NET)
+        if (Config.TRACE)
             Console.WriteLine($"FindPackageVersion: {pid}");
 
         if (psmr_by_identity.TryGetValue(key: pid, out PackageSearchMetadataRegistration? psmr))
         {
-            if (Config.TRACE_NET)
-                Console.WriteLine($"Metadata Cache hit for '{pid}'");
+            if (Config.TRACE)
+                Console.WriteLine($"    Metadata Cache hit for '{pid}'");
             return psmr;
         }
 
@@ -139,30 +144,32 @@ private List<PackageSearchMetadataRegistration> FindPackageVersionsThroughCache(
 
                 if (psmr != null)
                 {
-                    if (Config.TRACE_NET)
+                    if (Config.TRACE)
                         Console.WriteLine($"    Found metadata for '{pid}' from: {metadata_resources}");
                     psmr_by_identity[pid] = psmr;
                     return psmr;
                 }
             }
             catch (Exception ex)
             {
-                if (Config.TRACE_NET)
-                    Console.WriteLine($"FAILED to Fetch metadata for '{pid}' with: {ex.StackTrace}");
+                if (Config.TRACE)
+                    Console.WriteLine($"    FAILED to Fetch metadata for '{pid}' with: {ex.StackTrace}");
 
                 exceptions.Add(item: ex);
             }
         }
 
-        if (Config.TRACE_NET && exceptions.Any())
-        {
-            Console.WriteLine($"ERROR: No package found for {pid}.");
-            foreach (var ex in exceptions)
-                Console.WriteLine($"    {ex}");
-        }
+        var error_message = $"No package metadata found for {pid}.";
+        foreach (var ex in exceptions)
+            error_message += $"\n    {ex}";
+
+        if (Config.TRACE)
+            Console.WriteLine(error_message);
+
         // cache this null too
         psmr_by_identity[pid] = null;
-        return null;
+
+        throw new Exception(error_message);
     }
 
     /// <summary>
@@ -292,26 +299,34 @@ public static ISettings LoadNugetConfigSettings(
     /// from a feed URL and a nuget.config file path.
     /// These are MetadataResourceList and DependencyInfoResourceList attributes.
     /// </summary>
-    private void PopulateResources(List<Lazy<INuGetResourceProvider>> providers, ISettings settings)
+    private void PopulateResources(
+        List<Lazy<INuGetResourceProvider>> providers,
+        ISettings settings,
+        bool with_nuget_org = false)
     {
         PackageSourceProvider package_source_provider = new(settings: settings);
         List<PackageSource> package_sources = package_source_provider.LoadPackageSources().ToList();
         if (Config.TRACE)
             Console.WriteLine($"\nPopulateResources: Loaded {package_sources.Count} package sources from nuget.config");
 
-        // always nuget.org as last resort
-        var nuget_source = new PackageSource(source: "https://api.nuget.org/v3/index.json", name : "nuget.org");
-        package_sources.Add(nuget_source);
+        if (with_nuget_org || !package_sources.Any())
+        {
+            // Use nuget.org as last resort
+            var nuget_source = new PackageSource(
+                source: "https://api.nuget.org/v3/index.json",
+                name : "nuget.org");
+            package_sources.Add(nuget_source);
+        }
 
         HashSet<string> seen = new();
         foreach (PackageSource package_source in package_sources)
         {
-            var uri = package_source.SourceUri.ToString();
-            if (seen.Contains(uri))
+            var source_url = package_source.SourceUri.ToString();
+            if (seen.Contains(source_url))
                 continue;
             SourceRepository source_repository = new(source: package_source, providers: providers);
             AddSourceRepo(source_repo: source_repository);
-            seen.Add(uri);
+            seen.Add(source_url);
         }
     }
 
@@ -451,24 +466,18 @@ public IEnumerable<PackageDependency> GetPackageDependenciesForPackage(PackageId
             if (spdi != null)
             {
                 download = PackageDownload.FromSpdi(spdi);
+                download_by_identity[(identity, project_framework)] = download;
             }
             else
             {
-                if (Config.TRACE_NET)
-                    Console.WriteLine($"        Crafting plain download for package '{identity}'");
+                if (Config.TRACE)
+                    Console.WriteLine($"        No download info available for package '{identity}'");
 
-                // Last resort we craft a synthetic download URL
-                string name_lower = identity.Id.ToLower();
-                string version_lower = identity.Version.ToString().ToLower();
-                download = new()
-                {
-                    download_url = $"https://api.nuget.org/v3-flatcontainer/{name_lower}/{version_lower}/{name_lower}.{version_lower}.nupkg"
-                };
+                 download_by_identity[(identity, project_framework)] = download;
             }
-            download_by_identity[(identity, project_framework)] = download;
         }
 
-        if (!with_details || (with_details && download.IsEnhanced()))
+        if (!with_details || (with_details && download?.IsEnhanced() == true))
             return download;
 
         // We need to fetch the SHA512 (and the size)
@@ -511,10 +520,12 @@ public IEnumerable<PackageDependency> GetPackageDependenciesForPackage(PackageId
         }
 
         string hash = catalog_entry["packageHash"]!.ToString();
-        download.hash = Convert.ToHexString(Convert.FromBase64String(hash));
-        download.hash_algorithm = catalog_entry["packageHashAlgorithm"]!.ToString();
-        download.size = (int)catalog_entry["packageSize"]!;
-
+        if (download != null)
+        {
+            download.hash = Convert.ToHexString(Convert.FromBase64String(hash));
+            download.hash_algorithm = catalog_entry["packageHashAlgorithm"]!.ToString();
+            download.size = (int)catalog_entry["packageSize"]!;
+        }
         if (Config.TRACE_NET)
             Console.WriteLine($"        download: {download}");
         download_by_identity[(identity, project_framework)] = download;
@@ -656,7 +667,17 @@ public HashSet<SourcePackageDependencyInfo> ResolveDependenciesForPackageReferen
 
         var packages = new List<PackageId>();
         foreach (var targetref in target_references)
-            packages.Add(PackageId.FromReference(targetref));
+        {
+            try
+            {
+                packages.Add(PackageId.FromReference(targetref));
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"   FAILED: targetref: {targetref}");
+                throw new Exception(targetref.ToString(), ex);
+            }
+        }
 
         walk_context.ProjectLibraryProviders.Add(new ProjectLibraryProvider(packages));
 
@@ -913,10 +934,26 @@ public override string ToString()
 
     public static PackageId FromReference(PackageReference reference)
     {
-        bool allow_prerel = reference.HasAllowedVersions && reference.AllowedVersions.MinVersion.IsPrerelease;
+        if (reference == null)
+            throw new ArgumentNullException(nameof(reference));
+        var av = reference.AllowedVersions;
+        bool allow_prerel = false;
+        string? version = null;
+        if (av != null)
+        {
+            var mv  = reference.AllowedVersions.MinVersion;
+            if (mv != null)
+                allow_prerel = mv.IsPrerelease;
+            version = reference.AllowedVersions.ToNormalizedString();
+        }
+        if (version == null && reference.PackageIdentity.Version != null)
+        {
+            version = reference.PackageIdentity.Version.ToString();
+        }
+
         return new PackageId(
-            id:reference.PackageIdentity.Id,
-            version: reference.AllowedVersions.ToNormalizedString(),
+            id: reference.PackageIdentity.Id,
+            version: version ?? "",
             allow_prerelease_versions: allow_prerel
         );
     }

src/nuget-inspector/Options.cs

@@ -26,10 +26,13 @@ public class Options
         description: "Path to a nuget.config file to use, ignoring all other nuget.config.")]
     public string NugetConfigPath = "";
 
+    // If True, return extra metadata details when available such as SHA512
     public bool WithDetails;
     public bool WithFallback;
+    public bool WithNuGetOrg;
     public bool ShowHelp;
     public bool Verbose;
+    public bool Debug;
     public bool ShowVersion;
     public bool ShowAbout;
 
@@ -63,12 +66,18 @@ public List<string> AsCliList()
         if (Verbose)
             options.Add("--verbose");
 
+        if (Debug)
+            options.Add("--debug");
+
         if (WithDetails)
             options.Add("--with-details");
 
         if (WithFallback)
             options.Add("--with-fallback");
 
+        if (WithNuGetOrg)
+            options.Add("--with-nuget-org");
+
         return options;
     }
 
@@ -96,10 +105,15 @@ public List<string> AsCliList()
         command_options.Add(prototype: "with-fallback", description: "Optionally use a plain XML project file parser as fallback from failures.",
             action: value => options.WithDetails = value != null);
 
+        command_options.Add(prototype: "with-nuget-org", description: "Optionally use the officila, public nuget.org API as a fallback in addition to nuget.config-configured API sources.",
+            action: value => options.WithNuGetOrg = value != null);
+
         command_options.Add(prototype: "h|help", description: "Show this message and exit.",
             action: value => options.ShowHelp = value != null);
         command_options.Add(prototype: "v|verbose", description: "Display more verbose output.",
             action: value => options.Verbose = value != null);
+        command_options.Add(prototype: "debug", description: "Display very verbose debug output.",
+            action: value => options.Debug = value != null);
         command_options.Add(prototype: "version", description: "Display nuget-inspector version and exit.",
             action: value => options.ShowVersion = value != null);
         command_options.Add(prototype: "about", description: "Display information about nuget-inspector and exit.",

src/nuget-inspector/Program.cs

@@ -1,6 +1,6 @@
 using System.Diagnostics;
-using System.Xml;
 using Microsoft.Build.Locator;
+using Newtonsoft.Json;
 using NuGet.Frameworks;
 
 namespace NugetInspector;
@@ -99,7 +99,8 @@ private static ExecutionResult ExecuteInspector(Options options)
             var nuget_api_service = new NugetApi(
                 nuget_config_path: options.NugetConfigPath,
                 project_root_path: project_options.ProjectDirectory,
-                project_framework: project_framework);
+                project_framework: project_framework,
+                with_nuget_org: options.WithNuGetOrg);
 
             var scanner = new ProjectScanner(
                 options: project_options,
@@ -134,6 +135,17 @@ private static ExecutionResult ExecuteInspector(Options options)
             var output_formatter = new OutputFormatJson(scan_result: scan_result);
             output_formatter.Write();
 
+            if (Config.TRACE_OUTPUT)
+            {
+                Console.WriteLine("\n=============JSON OUTPUT================");
+                string output = JsonConvert.SerializeObject(
+                    value: output_formatter.scan_output,
+                    formatting: Formatting.Indented);
+                Console.WriteLine(output);
+
+                Console.WriteLine("=======================================\n");
+            }
+
             BasePackage project_package = scan_result.project_package;
 
             bool success = scan_result.Status == ScanResult.ResultStatus.Success;
@@ -246,6 +258,14 @@ private static ParsedOptions ParseCliArgs(string[] args)
             {
                 Config.TRACE = true;
             }
+            if (options.Debug)
+            {
+                Config.TRACE = true;
+                Config.TRACE_DEEP = true;
+                Config.TRACE_META = true;
+                Config.TRACE_NET = true;
+                Config.TRACE_OUTPUT = true;
+            }
 
             if (Config.TRACE_ARGS)
                 Console.WriteLine($"argument: with-details: {options.WithDetails}");