Implement dependency loading (#469)

* Update PackedPluginProvider.cs

* Update PackedPluginProvider.cs

* Load depends properly

* We don't need to load the assembly of a dependency.

* This might fix depends loading...

* Fix

* Update PluginManager.cs

* Okay dependency loading works

* Go through deps.json instead and load up necessary assemblies

* Update PackedPluginProvider.cs

Author: Tides

Obsidian.API/Obsidian.API.csproj

@@ -44,7 +44,7 @@
 		<AdditionalFiles Include="..\Obsidian\Assets\tags.json" Link="Assets\tags.json" />
 		<AdditionalFiles Include="..\Obsidian\Assets\enums.json" Link="Assets\enums.json" />
 		<AdditionalFiles Include="..\Obsidian\Assets\item_components.json" Link="Assets\item_components.json" />
-		
+
 		<AdditionalFiles Include="..\Obsidian\Assets\Codecs\**\*.json" LinkBase="Assets\Codecs" />
 	</ItemGroup>
 
@@ -69,10 +69,18 @@
 	</ItemGroup>
 
 	<ItemGroup>
-		<PackageReference Include="Microsoft.CodeAnalysis.Common" Version="4.11.0" />
-		<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="9.0.0" />
-		<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.0" />
-		<PackageReference Include="SharpNoise" Version="0.12.1.1" />
+		<PackageReference Include="Microsoft.CodeAnalysis.Common" Version="4.11.0">
+			<ExcludeAssets>runtime</ExcludeAssets>
+		</PackageReference>
+		<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="9.0.0">
+			<ExcludeAssets>runtime</ExcludeAssets>
+		</PackageReference>
+		<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.0">
+			<ExcludeAssets>runtime</ExcludeAssets>
+		</PackageReference>
+		<PackageReference Include="SharpNoise" Version="0.12.1.1">
+			<ExcludeAssets>runtime</ExcludeAssets>
+		</PackageReference>
 	</ItemGroup>
 
 	<ItemGroup>

Obsidian.ConsoleApp/config/server.json

@@ -23,7 +23,7 @@
 
   "Logging": {
     "LogLevel": {
-      "Default": "Information",
+      "Default": "Warning",
       "Microsoft": "Warning"
     }
   }

Obsidian/Commands/Modules/MainCommandModule.cs

@@ -131,15 +131,15 @@ public async Task PluginsAsync()
             var info = pluginContainer.Info;
 
             var plugin = new ChatMessage();
-            var colorByState = pluginContainer.Loaded || pluginContainer.IsReady ? HexColor.Green : HexColor.Red;
+            var colorByState = pluginContainer.Loaded ? ChatColor.BrightGreen : ChatColor.Red;
 
             plugin.Text = pluginContainer.Info.Name;
-            plugin.Color = colorByState;
+            plugin.Color = new HexColor(colorByState.Color);
 
             plugin.HoverEvent = new HoverComponent
             {
                 Action = HoverAction.ShowText,
-                Contents = new HoverChatContent { ChatMessage = $"{colorByState}{info.Name}{ChatColor.Reset}\nVersion: {colorByState}{info.Version}{ChatColor.Reset}\nAuthor(s): {colorByState}{info.Authors}{ChatColor.Reset}" }
+                Contents = new HoverChatContent { ChatMessage = $"{colorByState}{info.Name}{ChatColor.Reset}\nVersion: {colorByState}{info.Version}{ChatColor.Reset}\nAuthor(s): {colorByState}{string.Join(", ", info.Authors)}{ChatColor.Reset}" }
             };
 
             if (pluginContainer.Info.ProjectUrl != null)

Obsidian/Plugins/PluginContainer.cs

@@ -31,9 +31,6 @@ public sealed class PluginContainer : IDisposable, IPluginContainer
 
     public required string Source { get; set; }
     public required bool ValidSignature { get; init; }
-
-    public bool HasDependencies { get; private set; } = true;
-    public bool IsReady => HasDependencies;
     public bool Loaded { get; internal set; }
 
     ~PluginContainer()

Obsidian/Plugins/PluginManager.cs

@@ -89,9 +89,7 @@ public PluginManager(IServiceProvider serverProvider, IServer server,
 
     public async Task LoadPluginsAsync()
     {
-        //TODO talk about what format we should support
-        // get directory surrent dll is in
-        var acceptedKeysPath = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "accepted_keys"); 
+        var acceptedKeysPath = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "accepted_keys");
         var acceptedKeyFiles = Directory.GetFiles(acceptedKeysPath);
 
         using var rsa = RSA.Create();
@@ -113,20 +111,20 @@ public async Task LoadPluginsAsync()
             if (pluginContainer is null)
                 continue;
 
-            foreach (var canLoad in waitingForDepend.Where(x => x.IsDependency(pluginContainer.Info.Id)).ToList())
-            {
-                packedPluginProvider.InitializePlugin(canLoad);
-
-                //Add dependency to plugin
-                canLoad.AddDependency(pluginContainer.LoadContext);
+            if (pluginContainer.Plugin is null)
+                waitingForDepend.Add(pluginContainer);
+        }
 
-                await this.HandlePluginAsync(canLoad);
+        foreach (var canLoad in waitingForDepend)
+        {
+            packedPluginProvider.InitializePlugin(canLoad);
+            packedPluginProvider.HandlePlugin(canLoad, canLoad.PluginAssembly);
 
-                waitingForDepend.Remove(canLoad);
-            }
+            var depends = canLoad.Info.Dependencies.Select(x => x.Id).SelectMany(x => this.Plugins.Where(p => p.Info.Id == x));
+            foreach (var depend in depends)
+                canLoad.AddDependency(depend.LoadContext);
 
-            if (pluginContainer.Plugin is null)
-                waitingForDepend.Add(pluginContainer);
+            await this.HandlePluginAsync(canLoad);
         }
 
         DirectoryWatcher.Watch("plugins");
@@ -238,46 +236,62 @@ private void ConfigureInitialServices()
 
     private async ValueTask<PluginContainer> HandlePluginAsync(PluginContainer pluginContainer)
     {
-        //The plugin still hasn't fully loaded. Probably due to it having a hard dependency
+        // If the plugin is already loaded or staged, skip loading it again
+        if (pluginContainer.Loaded || stagedPlugins.Contains(pluginContainer))
+            return pluginContainer;
+
+        // The plugin still hasn't fully loaded. Probably due to it having a hard dependency
         if (pluginContainer.Plugin is null)
             return pluginContainer;
 
-        //Inject first wave of services (services initialized by obsidian e.x IServerConfiguration)
+        // Inject first wave of services (services initialized by Obsidian e.g., IServerConfiguration)
         PluginServiceHandler.InjectServices(this.serverProvider, pluginContainer, this.logger);
 
-        if (pluginContainer.IsReady)
+        // Check if the plugin has dependencies that are not loaded yet
+        var missingDependencies = pluginContainer.Info.Dependencies
+            .Where(dep => !Plugins.Any(p => p.Info.Id == dep.Id && p.Loaded));
+
+        if (missingDependencies.Any())
         {
-            lock (plugins)
+            // Stage the plugin if dependencies are missing
+            lock (stagedPlugins)
             {
-                plugins.Add(pluginContainer);
+                stagedPlugins.Add(pluginContainer);
             }
 
-            pluginContainer.Plugin.ConfigureServices(this.pluginServiceDescriptors);
-            pluginContainer.Plugin.ConfigureRegistry(this.pluginRegistry);
+            logger.LogWarning("Plugin {name} staged, missing dependencies.", pluginContainer.Info.Name);
 
-            pluginContainer.Loaded = true;
+            return pluginContainer;
+        }
 
-            await pluginContainer.Plugin.OnLoadedAsync(this.server);
+        // Now we can load the plugin since all its dependencies are either loaded or are being loaded
+        lock (plugins)
+        {
+            plugins.Add(pluginContainer);
         }
-        else
+
+        pluginContainer.Plugin.ConfigureServices(this.pluginServiceDescriptors);
+        pluginContainer.Plugin.ConfigureRegistry(this.pluginRegistry);
+
+        pluginContainer.Loaded = true;
+
+        await pluginContainer.Plugin.OnLoadedAsync(this.server);
+
+        // Check and resolve any other plugins that were staged and waiting on dependencies
+        foreach (var stagedPlugin in stagedPlugins.ToList())
         {
-            lock (stagedPlugins)
-            {
-                stagedPlugins.Add(pluginContainer);
-            }
+            // Check if all dependencies of this staged plugin are loaded now
+            var missingDepsForStaged = stagedPlugin.Info.Dependencies
+                .Where(dep => !Plugins.Any(p => p.Info.Id == dep.Id && p.Loaded));
 
-            if (logger != null)
+            if (!missingDepsForStaged.Any())
             {
-                var stageMessage = new System.Text.StringBuilder(50);
-                stageMessage.Append($"Plugin {pluginContainer.Info.Name} staged");
-                if (!pluginContainer.HasDependencies)
-                    stageMessage.Append(", missing dependencies");
-
-                logger.LogWarning("{}", stageMessage.ToString());
+                stagedPlugins.Remove(stagedPlugin);
+                await HandlePluginAsync(stagedPlugin); // Recurse and attempt to load this staged plugin now
             }
         }
 
-        logger?.LogInformation("Loading finished!");
+        logger.LogInformation("Loaded {name}.", pluginContainer.Info.Name);
 
         return pluginContainer;
     }

Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs

@@ -1,10 +1,13 @@
 using Microsoft.Extensions.Logging;
 using Obsidian.API.Plugins;
+using Obsidian.Entities;
 using Org.BouncyCastle.Crypto;
 using System.Collections.Frozen;
 using System.IO;
 using System.Reflection;
+using System.Runtime.Loader;
 using System.Security.Cryptography;
+using System.Text.Json;
 
 namespace Obsidian.Plugins.PluginProviders;
 public sealed class PackedPluginProvider(PluginManager pluginManager, ILogger logger)
@@ -52,12 +55,20 @@ public sealed class PackedPluginProvider(PluginManager pluginManager, ILogger lo
         //Can't load until those plugins are loaded
         if (partialContainer.Info.Dependencies.Any(x => x.Required && !this.pluginManager.Plugins.Any(d => d.Info.Id == x.Id)))
         {
-            var str = partialContainer.Info.Dependencies.Length > 1 ? "has multiple hard dependencies." : 
+            var str = partialContainer.Info.Dependencies.Length > 1 ? "has multiple hard dependencies." :
                 $"has a hard dependency on {partialContainer.Info.Dependencies.First().Id}.";
             this.logger.LogWarning("{name} {message}. Will Attempt to load after.", partialContainer.Info.Name, str);
             return partialContainer;
         }
 
+        foreach (var depends in partialContainer.Info.Dependencies)
+        {
+            var plugin = this.pluginManager.Plugins.FirstOrDefault(x => x.Info.Id == depends.Id);
+
+            partialContainer.AddDependency(plugin.LoadContext);
+            this.logger.LogInformation("Added {depends} as a dependency for {name}", plugin.Info.Name, partialContainer.Info.Name);
+        }
+
         var mainAssembly = this.InitializePlugin(partialContainer);
 
         return HandlePlugin(partialContainer, mainAssembly);
@@ -221,29 +232,98 @@ private List<string> ProcessEntries(PluginContainer pluginContainer)
         var pluginAssembly = pluginContainer.LoadContext.Name;
 
         var libsWithSymbols = new List<string>();
-        foreach (var (_, entry) in pluginContainer.FileEntries)
+
+        using var dependenciesData = new MemoryStream(pluginContainer.GetFileData($"{pluginAssembly}.deps.json"));
+        var dependencies = JsonSerializer.Deserialize<DotNetDeps>(dependenciesData, JsonSerializerOptions.Web);
+        var targets = dependencies.Targets[dependencies.RuntimeTarget.Name].Deserialize<Dictionary<string, DotNetTarget>>(JsonSerializerOptions.Web);
+
+        foreach (var (key, target) in targets)
         {
-            var actualBytes = entry.GetData();
+            var deps = target.Dependencies;
+            var runtimes = target.Runtime;
 
-            var name = Path.GetFileNameWithoutExtension(entry.Name);
-            //Don't load this assembly wait
-            if (name == pluginAssembly)
+            if (runtimes == null)//We don't care if its null
                 continue;
 
-            //TODO LOAD OTHER FILES SOMEWHERE
-            if (entry.Name.EndsWith(".dll"))
+            foreach (var (dll, runtimeElement) in runtimes)
             {
-                if (pluginContainer.FileEntries.ContainsKey(entry.Name.Replace(".dll", ".pdb")))
+                var sanitizedDll = dll;
+                var split = dll.Split('/');
+                if (split.Length > 1)
+                    sanitizedDll = split.Last();
+
+                var name = sanitizedDll[..sanitizedDll.IndexOf(".dll")];
+
+                if (name == pluginAssembly || runtimeElement.ToString() == "{}")
+                    continue;
+
+                var runtime = runtimeElement.Deserialize<DependencyRuntime>(JsonSerializerOptions.Web);
+                var assemblyName = new AssemblyName
+                {
+                    Name = name,
+                    Version = new(runtime.AssemblyVersion),
+                };
+
+                var depends = pluginContainer.Info.Dependencies.Select(x => x.Id)
+                    .SelectMany(x => this.pluginManager.Plugins.Where(p => p.Info.Id == x));
+
+                //TODO allow users to define what version of a dependency is allowed/valid e.g version: >1.0.0 or >=1.0.0 or =1.0.0.
+                var dependency = depends.FirstOrDefault(x => x.PluginAssembly.GetName().Name == name && x.Info.Version == assemblyName.Version);
+
+                //we don't need to load this into the context.
+                if (dependency != null)
+                    continue;
+
+                if (pluginContainer.FileEntries.ContainsKey($"${name}.pdb"))
                 {
                     //Library has debug symbols load in last
-                    libsWithSymbols.Add(entry.Name.Replace(".dll", ".pdb"));
+                    libsWithSymbols.Add(name);
                     continue;
                 }
 
-                pluginContainer.LoadContext.LoadAssembly(actualBytes);
+                try
+                {
+                    //Check to see if this assembly already exists in the shared context.
+                    var sharedAssembly = AssemblyLoadContext.Default.LoadFromAssemblyName(assemblyName);
+                    if (sharedAssembly != null)
+                        continue;
+                }
+                catch { }
+
+                var data = pluginContainer.GetFileData(sanitizedDll);
+                pluginContainer.LoadContext.LoadAssembly(data);
             }
         }
 
         return libsWithSymbols;
     }
 }
+
+
+public readonly struct DotNetDeps
+{
+    public required RuntimeTarget RuntimeTarget { get; init; }
+
+    public required Dictionary<string, JsonElement> Targets { get; init; }
+}
+
+public readonly struct DotNetTarget
+{
+    public Dictionary<string, string>? Dependencies { get; init; }
+
+    public Dictionary<string, JsonElement>? Runtime { get; init; }
+}
+
+public readonly struct DependencyRuntime
+{
+    public required string AssemblyVersion { get; init; }
+
+    public required string FileVersion { get; init; }
+}
+
+public readonly struct RuntimeTarget
+{
+    public required string Name { get; init; }
+
+    public required string Signature { get; init; }
+}

Obsidian/Plugins/PluginProviders/PluginLoadContext.cs

@@ -13,17 +13,15 @@ public sealed class PluginLoadContext(string name) : AssemblyLoadContext(name: n
         using var mainStream = new MemoryStream(mainBytes, false);
         using var pbdStream = pbdBytes != null ? new MemoryStream(pbdBytes, false) : null;
 
-        var asm = this.LoadFromStream(mainStream, pbdStream);
-        
-        return asm;
+        return this.LoadFromStream(mainStream, pbdStream);
     }
 
     public void AddDependency(PluginLoadContext context) => this.Dependencies.Add(context);
 
     protected override Assembly? Load(AssemblyName assemblyName)
     {
-        var assembly = this.Assemblies.FirstOrDefault(x => x.GetName() == assemblyName);
+        var assembly = this.Assemblies.FirstOrDefault(x => x.FullName == assemblyName.FullName);
 
-        return assembly ?? this.Dependencies.Select(x => x.Load(assemblyName)).FirstOrDefault(x => x != null);
+        return this.Dependencies.Select(x => x.Load(assemblyName)).FirstOrDefault(x => x != null) ?? assembly;
     }
 }