Merge pull request #34 from nexB/return-download-url

Trye harder to return a download url

Author: pombredanne

CHANGELOG.rst

@@ -2,6 +2,18 @@ Changelog
 =========
 
 
+v0.9.9
+-------
+
+This is a minor bug fix release with this update:
+
+* Try harder to return a download URL for a NuGet
+  The NuGet API does not expose a way to get the downlaod URL, so we
+  are fetching and probing in sequence a few places like the NuGet Client
+  does internally.
+
+
+
 v0.9.8
 -------
 

build.sh

@@ -17,6 +17,6 @@ dotnet publish \
   --runtime linux-x64 \
   --self-contained true \
   --configuration Release \
-  -p:Version=0.9.8 \
+  -p:Version=0.9.9 \
   --output build \
   src/nuget-inspector/nuget-inspector.csproj

release.sh

@@ -16,7 +16,7 @@
 rm -rf release/
 mkdir release
 
-VERSION=0.9.8
+VERSION=0.9.9
 
 TARGET_BASE=nuget-inspector-$(git describe)
 

setup.cfg

@@ -1,7 +1,7 @@
 [bumpversion]
 commit = False
 tag = False
-current_version = 0.9.8
+current_version = 0.9.9
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-(?P<release>[a-z]+))?
 serialize = 
 	{major}.{minor}.{patch}-{release}

src/nuget-inspector/Config.cs

@@ -9,6 +9,6 @@ public static class Config
     public static bool TRACE_DEEP = false;
     public static bool TRACE_META = false;
     public static bool TRACE_OUTPUT = false;
-    public const string NUGET_INSPECTOR_VERSION = "0.9.8";
+    public const string NUGET_INSPECTOR_VERSION = "0.9.9";
     #pragma warning restore CA2211
 }

src/nuget-inspector/NugetApi.cs

@@ -33,16 +33,18 @@ public class NugetApi
     private readonly Dictionary<string, JObject?> catalog_entry_by_catalog_url = new();
     private readonly Dictionary<string, List<PackageSearchMetadataRegistration>> psmrs_by_package_name = new();
     private readonly Dictionary<PackageIdentity, PackageSearchMetadataRegistration?> psmr_by_identity = new();
-    private readonly Dictionary<(PackageIdentity, NuGetFramework), PackageDownload?> download_by_identity = new();
+    private readonly Dictionary<PackageIdentity, PackageDownload?> download_by_identity = new();
     private readonly Dictionary<(PackageIdentity, NuGetFramework), SourcePackageDependencyInfo> spdi_by_identity = new();
 
     private readonly List<SourceRepository> source_repositories = new();
     private readonly List<PackageMetadataResource> metadata_resources = new();
     private readonly List<DependencyInfoResource> dependency_info_resources = new();
 
     private readonly ISettings settings;
-
+    private readonly List<Lazy<INuGetResourceProvider>> providers = new();
     private readonly NuGetFramework project_framework;
+    private readonly List<PackageSource> package_sources = new();
+    private readonly NugetLogger nuget_logger = new();
 
     public NugetApi(
         string nuget_config_path,
@@ -51,15 +53,14 @@ public NugetApi(
         bool with_nuget_org)
     {
         this.project_framework = project_framework;
-        List<Lazy<INuGetResourceProvider>> providers = new();
-        providers.AddRange(Repository.Provider.GetCoreV3());
+        this.providers.AddRange(Repository.Provider.GetCoreV3());
 
         settings = LoadNugetConfigSettings(
             nuget_config_path: nuget_config_path,
             project_root_path: project_root_path);
 
         PopulateResources(
-            providers: providers,
+            providers: this.providers,
             settings: settings,
             with_nuget_org: with_nuget_org);
     }
@@ -138,7 +139,7 @@ private List<PackageSearchMetadataRegistration> FindPackageVersionsThroughCache(
                 psmr = (PackageSearchMetadataRegistration)metadata_resource.GetMetadataAsync(
                     package: pid,
                     sourceCacheContext: source_cache_context,
-                    log: new NugetLogger(),
+                    log: nuget_logger,
                     token: CancellationToken.None
                 ).Result;
 
@@ -305,7 +306,8 @@ private void PopulateResources(
         bool with_nuget_org = false)
     {
         PackageSourceProvider package_source_provider = new(settings: settings);
-        List<PackageSource> package_sources = package_source_provider.LoadPackageSources().ToList();
+        package_sources.AddRange(package_source_provider.LoadPackageSources());
+
         if (Config.TRACE)
             Console.WriteLine($"\nPopulateResources: Loaded {package_sources.Count} package sources from nuget.config");
 
@@ -409,7 +411,7 @@ public IEnumerable<PackageDependency> GetPackageDependenciesForPackage(PackageId
                     package: identity,
                     projectFramework: framework,
                     cacheContext: source_cache_context,
-                    log: new NugetLogger(),
+                    log: nuget_logger,
                     token: CancellationToken.None);
 
                 spdi = infoTask.Result;
@@ -436,6 +438,43 @@ public IEnumerable<PackageDependency> GetPackageDependenciesForPackage(PackageId
         return null;
     }
 
+    /// <summary>
+    /// Return the download URL of the package or null.
+    /// This is based on the logic of private NuGet code.
+    /// 1. Try the SourcePackageDependencyInfo.DownloadUri if present.
+    /// 2. Try the registration metadata JObject, since there is no API
+    /// 3. FUTURE: fall back to craft a URL by hand.
+    /// </summary>
+    public string? GetDownloadUrl(PackageIdentity identity)
+    {
+        if (spdi_by_identity.TryGetValue(key: (identity, project_framework), out SourcePackageDependencyInfo? spdi))
+        {
+            var du = spdi.DownloadUri;
+            if (du != null && !string.IsNullOrWhiteSpace(du.ToString()))
+                return du.ToString();
+
+            RegistrationResourceV3 rrv3 = spdi.Source.GetResource<RegistrationResourceV3>(CancellationToken.None);
+            if (rrv3 != null)
+            {
+               var meta = rrv3.GetPackageMetadata(
+                   identity: identity,
+                   cacheContext: source_cache_context,
+                   log: nuget_logger,
+                   token: CancellationToken.None).Result;
+                JToken? content = meta?["packageContent"];
+                if (content != null)
+                    return content.ToString();
+            }
+        }
+        // TODO last resort: Try if we have package base address URL
+        // var base = "TBD";
+        // var name = identity.Id.ToLowerInvariant();
+        // var version = identity.Version.ToNormalizedString().ToLowerInvariant();
+        // return $"{base}/{name}/{version}/{name}.{version}.nupkg";
+
+        return null;
+    }
+
     /// <summary>
     /// Return a PackageDownload for a given package identity.
     /// Cache entries for a given package identity as needed
@@ -450,12 +489,14 @@ public IEnumerable<PackageDependency> GetPackageDependenciesForPackage(PackageId
         if (Config.TRACE_NET)
             Console.WriteLine($"    GetPackageDownload: {identity}, with_details: {with_details} project_framework: {project_framework}");
 
+        PackageDownload? download = null;
         // try the cache
-        if (download_by_identity.TryGetValue((identity, project_framework), out PackageDownload? download))
+        if (download_by_identity.TryGetValue(identity, out download))
         {
             if (Config.TRACE_NET)
                 Console.WriteLine($"        Caching hit for package '{identity}'");
-            return download;
+            if (download != null)
+                return download;
         }
         else
         {
@@ -471,14 +512,16 @@ public IEnumerable<PackageDependency> GetPackageDependenciesForPackage(PackageId
             if (spdi != null)
             {
                 download = PackageDownload.FromSpdi(spdi);
-                download_by_identity[(identity, project_framework)] = download;
             }
-            // else
-            // {
-            //      download_by_identity[(identity, project_framework)] = download;
-            // }
         }
 
+        if (download != null && string.IsNullOrWhiteSpace(download.download_url))
+            download.download_url = GetDownloadUrl(identity) ?? "";
+            download_by_identity[identity] = download;
+
+        if (Config.TRACE_NET)
+            Console.WriteLine($"       Found download: {download}'");
+
         if (!with_details || (with_details && download?.IsEnhanced() == true))
             return download;
 
@@ -527,7 +570,7 @@ public IEnumerable<PackageDependency> GetPackageDependenciesForPackage(PackageId
                 if (Config.TRACE_NET)
                     Console.WriteLine($"        failed to fetch metadata details for: {package_catalog_url}: {ex}");
                 catalog_entry_by_catalog_url[package_catalog_url] = null;
-                return null;
+                return download;
             }
         }
         if (catalog_entry != null)
@@ -542,10 +585,9 @@ public IEnumerable<PackageDependency> GetPackageDependenciesForPackage(PackageId
             }
             if (Config.TRACE_NET)
                 Console.WriteLine($"        download: {download}");
-            download_by_identity[(identity, project_framework)] = download;
             return download;
         }
-        return null;
+        return download;
     }
 
     /// <summary>
@@ -634,7 +676,7 @@ public HashSet<SourcePackageDependencyInfo> ResolveDependenciesForPackageConfig(
             preferredVersions: direct_deps,
             availablePackages: available_dependencies,
             packageSources: source_repositories.Select(s => s.PackageSource),
-            log: new NugetLogger());
+            log: nuget_logger);
 
         var resolver = new PackageResolver();
         resolver.Resolve(context: context, token: CancellationToken.None);
@@ -674,7 +716,6 @@ public HashSet<SourcePackageDependencyInfo> ResolveDependenciesForPackageConfig(
     public HashSet<SourcePackageDependencyInfo> ResolveDependenciesForPackageReference(
         IEnumerable<PackageReference> target_references)
     {
-        var nuget_logger = new NugetLogger();
         var psm = PackageSourceMapping.GetPackageSourceMapping(settings);
         var walk_context = new RemoteWalkContext(
             cacheContext: source_cache_context,
@@ -709,7 +750,8 @@ public HashSet<SourcePackageDependencyInfo> ResolveDependenciesForPackageReferen
             walk_context.RemoteLibraryProviders.Add(provider);
         }
 
-        // we need a fake root lib as this is the only allowed input
+        // We need a fake root lib as there is only one allowed input
+        // This represents the project
         var rootlib = new LibraryRange(
             name: "root_project",
             versionRange: VersionRange.Parse("1.0.0"),
@@ -728,6 +770,7 @@ public HashSet<SourcePackageDependencyInfo> ResolveDependenciesForPackageReferen
 
         var resolved_package_info_by_package_id = new Dictionary<PackageId, ResolvedPackageInfo>();
 
+        // we iterate only inner nodes, because we have only one outer node: the "fake" root project 
         foreach (GraphNode<RemoteResolveResult> inner in resolved_graph.InnerNodes)
         {
             if (Config.TRACE_DEEP)
@@ -759,6 +802,9 @@ public HashSet<SourcePackageDependencyInfo> ResolveDependenciesForPackageReferen
         return flat_dependencies;
     }
 
+    /// <summary>
+    /// Flatten the graph and populate the result a mapping recursively
+    /// </summary>
     public static void FlattenGraph(
         GraphNode<RemoteResolveResult> node,
         Dictionary<PackageId, ResolvedPackageInfo> resolved_package_info_by_package_id)
@@ -819,6 +865,9 @@ public static void FlattenGraph(
         }
     }
 
+    /// <summary>
+    /// Check the dependency for errors, raise exceptions if these are found
+    /// </summary>
     public static void CheckGraphForErrors(GraphNode<RemoteResolveResult> resolved_graph)
     {
         var analysis = resolved_graph.Analyze();
@@ -879,7 +928,10 @@ public class ResolvedPackageInfo
     public RemoteMatch? remote_match;
 }
 
-internal class ProjectLibraryProvider : NuGet.DependencyResolver.IDependencyProvider
+/// <summary>
+/// A dependency provider that collects only the local package references
+/// </summary>
+internal class ProjectLibraryProvider : IDependencyProvider
 {
     private readonly ICollection<PackageId> package_ids;
 

src/nuget-inspector/nuget-inspector.csproj

@@ -23,11 +23,11 @@
         <PackageId>nuget-inspector</PackageId>
         <Product>nuget-inspector</Product>
         <AssemblyName>nuget-inspector</AssemblyName>
-        <Version>0.9.8</Version>
+        <Version>0.9.9</Version>
         <Authors>nexB Inc.</Authors>
         <Company>nexB Inc</Company>
-        <AssemblyVersion>0.9.8.0</AssemblyVersion>
-        <FileVersion>0.9.8.0</FileVersion>
+        <AssemblyVersion>0.9.9.0</AssemblyVersion>
+        <FileVersion>0.9.9.0</FileVersion>
         <Description>A NuGet and Dotnet package dependency resolver</Description>
         <PackageProjectUrl>https://github.com/nexB/nuget-inspector</PackageProjectUrl>
         <PackageLicenseUrl>Apache-2.0 AND MIT</PackageLicenseUrl>

tests/data/complex/ort-4522/xxx.csproj-expected.json

@@ -331,7 +331,7 @@
           "datafile_path": "",
           "dependencies": [],
           "warnings": [
-            "Failed to get remote metadata for name: 'Softwarehelden.NuGet.Deploy' version: '1.0.32'. System.Exception: No package metadata found for Softwarehelden.NuGet.Deploy.1.0.32.\n   at NugetInspector.NugetApi.FindPackageVersion(PackageIdentity pid) in ./nuget-inspector/NugetApi.cs:line 172\n   at NugetInspector.BasePackage.UpdateWithRemoteMetadata(NugetApi nugetApi, Boolean with_details) in ./nuget-inspector/Models.cs:line 333\n   at NugetInspector.BasePackage.Update(NugetApi nugetApi, Boolean with_details) in ./nuget-inspector/Models.cs:line 312"
+            "Failed to get remote metadata for name: 'Softwarehelden.NuGet.Deploy' version: '1.0.32'. System.Exception: No package metadata found for Softwarehelden.NuGet.Deploy.1.0.32.\n   at NugetInspector.NugetApi.FindPackageVersion(PackageIdentity pid) in ./nuget-inspector/NugetApi.cs:line 173\n   at NugetInspector.BasePackage.UpdateWithRemoteMetadata(NugetApi nugetApi, Boolean with_details) in ./nuget-inspector/Models.cs:line 333\n   at NugetInspector.BasePackage.Update(NugetApi nugetApi, Boolean with_details) in ./nuget-inspector/Models.cs:line 312"
           ],
           "errors": []
         }
@@ -621,7 +621,7 @@
       "datafile_path": "",
       "dependencies": [],
       "warnings": [
-        "Failed to get remote metadata for name: 'Softwarehelden.NuGet.Deploy' version: '1.0.32'. System.Exception: No package metadata found for Softwarehelden.NuGet.Deploy.1.0.32.\n   at NugetInspector.NugetApi.FindPackageVersion(PackageIdentity pid) in ./nuget-inspector/NugetApi.cs:line 172\n   at NugetInspector.BasePackage.UpdateWithRemoteMetadata(NugetApi nugetApi, Boolean with_details) in ./nuget-inspector/Models.cs:line 333\n   at NugetInspector.BasePackage.Update(NugetApi nugetApi, Boolean with_details) in ./nuget-inspector/Models.cs:line 312"
+        "Failed to get remote metadata for name: 'Softwarehelden.NuGet.Deploy' version: '1.0.32'. System.Exception: No package metadata found for Softwarehelden.NuGet.Deploy.1.0.32.\n   at NugetInspector.NugetApi.FindPackageVersion(PackageIdentity pid) in ./nuget-inspector/NugetApi.cs:line 173\n   at NugetInspector.BasePackage.UpdateWithRemoteMetadata(NugetApi nugetApi, Boolean with_details) in ./nuget-inspector/Models.cs:line 333\n   at NugetInspector.BasePackage.Update(NugetApi nugetApi, Boolean with_details) in ./nuget-inspector/Models.cs:line 312"
       ],
       "errors": []
     }

tests/data/complex/thirdparty-suites/dependencychecker/DependencyChecker-22983ae/DependencyChecker.Test/TestProjects/NetStandard/NetStandard/NetStandard/NetStandard.csproj-expected.json

@@ -11,7 +11,7 @@
       "notice": "Dependency tree generated with nuget-inspector.\nnuget-inspector is a free software tool from nexB Inc. and others.\nVisit https://github.com/nexB/nuget-inspector/ for support and download.",
       "warnings": [],
       "errors": [
-        "Failed to process project file: /complex/thirdparty-suites/dependencychecker/DependencyChecker-22983ae/DependencyChecker.Test/TestProjects/NetStandard/NetStandard/NetStandard/NetStandard.csproj with:\nSystem.Exception: Failed to resolve graph with: NoVersionNotFound Accepted\n ---> System.NullReferenceException: Object reference not set to an instance of an object.\n   at NugetInspector.NugetApi.FlattenGraph(GraphNode`1 node, Dictionary`2 resolved_package_info_by_package_id) in ./nuget-inspector/NugetApi.cs:line 785\n   --- End of inner exception stack trace ---\n   at NugetInspector.NugetApi.FlattenGraph(GraphNode`1 node, Dictionary`2 resolved_package_info_by_package_id) in ./nuget-inspector/NugetApi.cs:line 818\n   at NugetInspector.NugetApi.ResolveDependenciesForPackageReference(IEnumerable`1 target_references) in ./nuget-inspector/NugetApi.cs:line 736\n   at NugetInspector.ProjectFileProcessor.ResolveUsingLib() in ./nuget-inspector/ProjectFileProcessor.cs:line 445\n   at NugetInspector.ProjectScanner.RunScan() in ./nuget-inspector/ProjectScanner.cs:line 293"
+        "Failed to process project file: /complex/thirdparty-suites/dependencychecker/DependencyChecker-22983ae/DependencyChecker.Test/TestProjects/NetStandard/NetStandard/NetStandard/NetStandard.csproj with:\nSystem.Exception: Failed to resolve graph with: NoVersionNotFound Accepted\n ---> System.NullReferenceException: Object reference not set to an instance of an object.\n   at NugetInspector.NugetApi.FlattenGraph(GraphNode`1 node, Dictionary`2 resolved_package_info_by_package_id) in ./nuget-inspector/NugetApi.cs:line 831\n   --- End of inner exception stack trace ---\n   at NugetInspector.NugetApi.FlattenGraph(GraphNode`1 node, Dictionary`2 resolved_package_info_by_package_id) in ./nuget-inspector/NugetApi.cs:line 864\n   at NugetInspector.NugetApi.ResolveDependenciesForPackageReference(IEnumerable`1 target_references) in ./nuget-inspector/NugetApi.cs:line 779\n   at NugetInspector.ProjectFileProcessor.ResolveUsingLib() in ./nuget-inspector/ProjectFileProcessor.cs:line 445\n   at NugetInspector.ProjectScanner.RunScan() in ./nuget-inspector/ProjectScanner.cs:line 293"
       ]
     }
   ],

tests/data/complex/thirdparty-suites/dependencychecker/DependencyChecker-22983ae/DependencyChecker.Test/TestProjects/net462withoutPackages/DependencyChecker.csproj-expected.json

@@ -88,7 +88,7 @@
           "datafile_path": "",
           "dependencies": [],
           "warnings": [
-            "Failed to get remote metadata for name: 'CommandLine' version: '2.4.3'. System.Exception: No package metadata found for CommandLine.2.4.3.\n   at NugetInspector.NugetApi.FindPackageVersion(PackageIdentity pid) in ./nuget-inspector/NugetApi.cs:line 172\n   at NugetInspector.BasePackage.UpdateWithRemoteMetadata(NugetApi nugetApi, Boolean with_details) in ./nuget-inspector/Models.cs:line 333\n   at NugetInspector.BasePackage.Update(NugetApi nugetApi, Boolean with_details) in ./nuget-inspector/Models.cs:line 312"
+            "Failed to get remote metadata for name: 'CommandLine' version: '2.4.3'. System.Exception: No package metadata found for CommandLine.2.4.3.\n   at NugetInspector.NugetApi.FindPackageVersion(PackageIdentity pid) in ./nuget-inspector/NugetApi.cs:line 173\n   at NugetInspector.BasePackage.UpdateWithRemoteMetadata(NugetApi nugetApi, Boolean with_details) in ./nuget-inspector/Models.cs:line 333\n   at NugetInspector.BasePackage.Update(NugetApi nugetApi, Boolean with_details) in ./nuget-inspector/Models.cs:line 312"
           ],
           "errors": []
         },
@@ -638,7 +638,7 @@
       "datafile_path": "",
       "dependencies": [],
       "warnings": [
-        "Failed to get remote metadata for name: 'CommandLine' version: '2.4.3'. System.Exception: No package metadata found for CommandLine.2.4.3.\n   at NugetInspector.NugetApi.FindPackageVersion(PackageIdentity pid) in ./nuget-inspector/NugetApi.cs:line 172\n   at NugetInspector.BasePackage.UpdateWithRemoteMetadata(NugetApi nugetApi, Boolean with_details) in ./nuget-inspector/Models.cs:line 333\n   at NugetInspector.BasePackage.Update(NugetApi nugetApi, Boolean with_details) in ./nuget-inspector/Models.cs:line 312"
+        "Failed to get remote metadata for name: 'CommandLine' version: '2.4.3'. System.Exception: No package metadata found for CommandLine.2.4.3.\n   at NugetInspector.NugetApi.FindPackageVersion(PackageIdentity pid) in ./nuget-inspector/NugetApi.cs:line 173\n   at NugetInspector.BasePackage.UpdateWithRemoteMetadata(NugetApi nugetApi, Boolean with_details) in ./nuget-inspector/Models.cs:line 333\n   at NugetInspector.BasePackage.Update(NugetApi nugetApi, Boolean with_details) in ./nuget-inspector/Models.cs:line 312"
       ],
       "errors": []
     },