Add plugin support for other templates languages

Directory.Packages.props

@@ -6,7 +6,8 @@
 		<PackageVersion Include="coverlet.collector" Version="6.0.2" />
 		<PackageVersion Include="Fluid.Core" Version="2.12.0" />
 		<PackageVersion Include="Microsoft.AspNetCore.Razor.Language" Version="6.0.36" />
-		<PackageVersion Include="Microsoft.CodeAnalysis" Version="4.11.0" />
+		<PackageVersion Include="Microsoft.CodeAnalysis" Version="4.12.0" />
+		<PackageVersion Include="Microsoft.CodeAnalysis.CSharp.Scripting" Version="4.12.0" />
 		<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
 		<PackageVersion Include="OrchardCore.DisplayManagement.Liquid" Version="2.0.0" />
 		<PackageVersion Include="xunit" Version="2.9.2" />

NuGet.config

@@ -1,8 +1,8 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <configuration>
   <packageSources>
+    <!-- Ignore global configuration -->
     <clear />
-    <add key="NuGet" value="https://api.nuget.org/v3/index.json" />
-    <add key="OrchardCore" value="https://nuget.cloudsmith.io/orchardcore/preview/v3/index.json" />
+    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
   </packageSources>
 </configuration>

OrchardCoreContrib.PoExtractor.sln

@@ -35,6 +35,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 	ProjectSection(SolutionItems) = preProject
 		Directory.Build.props = Directory.Build.props
 		Directory.Packages.props = Directory.Packages.props
+		README.md = README.md
+		NuGet.config = NuGet.config
 	EndProjectSection
 EndProject
 Global

README.md

@@ -42,6 +42,15 @@ Specifies the code language to extracts translatable strings from. Default: `C#`
 
 Specifies the template engine to extract the translatable strings from. Default: `Razor` & `Liquid` templates.
 
+- **`-p|--plugin {path to CSX file}`**
+
+Specifies a path to a C# script file which can define further project processors. (You can find an example script [here](test/OrchardCoreContrib.PoExtractor.Tests/PluginTestFiles/BasicJsonLocalizationProcessor.csx).) This can be used to process localization from code languages or template engines not supported by the above options. You can have multiple of this switch in one call to load several plugins at once.
+
+When executing the plugins, all _OrchardCoreContrib.PoExtractor_ assemblies are automatically loaded, and two globals are defined:
+
+- `List<IProjectProcessor> projectProcessors`: Add an instance of your custom `IProjectProcessor` implementation type to this list.
+- `List<string> projectFiles`: In the unlikely case that you have to add a new project file type (such as _.fsproj_) add the project file paths to this list.
+
 ## Uninstallation
 
 ```powershell

src/OrchardCoreContrib.PoExtractor.Abstractions/OrchardCoreContrib.PoExtractor.Abstractions.csproj

@@ -2,4 +2,7 @@
 	<PropertyGroup>
 		<RootNamespace>OrchardCoreContrib.PoExtractor</RootNamespace>
 	</PropertyGroup>
+	<ItemGroup>
+		<PackageReference Include="Microsoft.CodeAnalysis.CSharp.Scripting" />
+	</ItemGroup>
 </Project>

src/OrchardCoreContrib.PoExtractor.Abstractions/PluginHelper.cs

@@ -0,0 +1,25 @@
+using System.Reflection;
+using Microsoft.CodeAnalysis.CSharp.Scripting;
+using Microsoft.CodeAnalysis.Scripting;
+
+namespace OrchardCoreContrib.PoExtractor;
+
+public static class PluginHelper
+{
+    public static async Task ProcessPluginsAsync(
+        IList<string> plugins,
+        List<IProjectProcessor> projectProcessors,
+        List<string> projectFiles,
+        IEnumerable<Assembly> assemblies)
+    {
+        var options = ScriptOptions.Default.AddReferences(assemblies);
+
+        foreach (var plugin in plugins)
+        {
+            var code = await File.ReadAllTextAsync(plugin);
+            await CSharpScript.EvaluateAsync(code, options, new PluginContext(projectProcessors, projectFiles));
+        }
+    }
+
+    public record PluginContext(List<IProjectProcessor> projectProcessors, List<string> projectFiles);
+}

src/OrchardCoreContrib.PoExtractor/GetCliOptionsResult.cs

@@ -0,0 +1,9 @@
+namespace OrchardCoreContrib.PoExtractor;
+
+public class GetCliOptionsResult
+{
+    public string Language { get; set; }
+    public string TemplateEngine { get; set; }
+    public string SingleOutputFile { get; set; }
+    public IList<string> Plugins { get; set; } = new List<string>();
+}

src/OrchardCoreContrib.PoExtractor/Program.cs

@@ -3,10 +3,6 @@
 using OrchardCoreContrib.PoExtractor.DotNet.VB;
 using OrchardCoreContrib.PoExtractor.Liquid;
 using OrchardCoreContrib.PoExtractor.Razor;
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
 
 namespace OrchardCoreContrib.PoExtractor;
 
@@ -15,7 +11,7 @@ public class Program
     private static readonly string _defaultLanguage = Language.CSharp;
     private static readonly string _defaultTemplateEngine = TemplateEngine.Both;
 
-    public static void Main(string[] args)
+    public static async Task Main(string[] args)
     {
         if (args.Length < 2 || args.Length > 10 || args.Length % 2 == 1)
         {
@@ -34,9 +30,9 @@ public static void Main(string[] args)
             return;
         }
 
-        (string language, string templateEngine, string singleOutputFile) = GetCliOptions(args);
+        var options = GetCliOptions(args);
 
-        if (language == null || templateEngine == null)
+        if (options.Language == null || options.TemplateEngine == null)
         {
             ShowHelp();
 
@@ -46,7 +42,7 @@ public static void Main(string[] args)
         var projectFiles = new List<string>();
         var projectProcessors = new List<IProjectProcessor>();
 
-        if (language == Language.CSharp)
+        if (options.Language == Language.CSharp)
         {
             projectProcessors.Add(new CSharpProjectProcessor());
 
@@ -63,21 +59,26 @@ public static void Main(string[] args)
                 .OrderBy(f => f));
         }
 
-        if (templateEngine == TemplateEngine.Both)
+        if (options.TemplateEngine == TemplateEngine.Both)
         {
             projectProcessors.Add(new RazorProjectProcessor());
             projectProcessors.Add(new LiquidProjectProcessor());
         }
-        else if (templateEngine == TemplateEngine.Razor)
+        else if (options.TemplateEngine == TemplateEngine.Razor)
         {
             projectProcessors.Add(new RazorProjectProcessor());
         }
-        else if (templateEngine == TemplateEngine.Liquid)
+        else if (options.TemplateEngine == TemplateEngine.Liquid)
         {
             projectProcessors.Add(new LiquidProjectProcessor());
         }
 
-        var isSingleFileOutput = !string.IsNullOrEmpty(singleOutputFile);
+        if (options.Plugins.Count > 0)
+        {
+            await ProcessPluginsAsync(options.Plugins, projectProcessors, projectFiles);
+        }
+
+        var isSingleFileOutput = !string.IsNullOrEmpty(options.SingleOutputFile);
         var localizableStrings = new LocalizableStringCollection();
         foreach (var projectFile in projectFiles)
         {
@@ -116,7 +117,7 @@ public static void Main(string[] args)
         {
             if (localizableStrings.Values.Any())
             {
-                var potPath = Path.Combine(outputPath, singleOutputFile);
+                var potPath = Path.Combine(outputPath, options.SingleOutputFile);
 
                 Directory.CreateDirectory(Path.GetDirectoryName(potPath));
 
@@ -128,11 +129,32 @@ public static void Main(string[] args)
         }
     }
 
-    private static (string language, string templateEngine, string singleOutputFile) GetCliOptions(string[] args)
+    /// <summary>
+    /// A shortcut to <see cref="PluginHelper.ProcessPluginsAsync"/> that gives the script access to all of the
+    /// <c>OrchardCoreContrib.PoExtractor.*</c> assemblies.
+    /// </summary>
+    public static Task ProcessPluginsAsync(
+        IList<string> plugins,
+        List<IProjectProcessor> projectProcessors,
+        List<string> projectFiles) =>
+        PluginHelper.ProcessPluginsAsync(plugins, projectProcessors, projectFiles, [
+            typeof(IProjectProcessor).Assembly, // OrchardCoreContrib.PoExtractor.Abstractions
+            typeof(ExtractingCodeWalker).Assembly, // OrchardCoreContrib.PoExtractor.DotNet
+            typeof(CSharpProjectProcessor).Assembly, // OrchardCoreContrib.PoExtractor.DotNet.CS
+            typeof(VisualBasicProjectProcessor).Assembly, // OrchardCoreContrib.PoExtractor.DotNet.VB
+            typeof(LiquidProjectProcessor).Assembly, // OrchardCoreContrib.PoExtractor.Liquid
+            typeof(RazorProjectProcessor).Assembly, // OrchardCoreContrib.PoExtractor.Razor
+        ]);
+
+    private static GetCliOptionsResult GetCliOptions(string[] args)
     {
-        var language = _defaultLanguage;
-        var templateEngine = _defaultTemplateEngine;
-        string singleOutputFile = null;
+        var result = new GetCliOptionsResult
+        {
+            Language = _defaultLanguage,
+            TemplateEngine = _defaultTemplateEngine,
+            SingleOutputFile = null,
+        };
+
         for (int i = 4; i <= args.Length; i += 2)
         {
             switch (args[i - 2])
@@ -141,31 +163,31 @@ private static (string language, string templateEngine, string singleOutputFile)
                 case "--language":
                     if (args[i - 1].Equals(Language.CSharp, StringComparison.CurrentCultureIgnoreCase))
                     {
-                        language = Language.CSharp;
+                        result.Language = Language.CSharp;
                     }
                     else if (args[i - 1].Equals(Language.VisualBasic, StringComparison.CurrentCultureIgnoreCase))
                     {
-                        language = Language.VisualBasic;
+                        result.Language = Language.VisualBasic;
                     }
                     else
                     {
-                        language = null;
+                        result.Language = null;
                     }
 
                     break;
                 case "-t":
                 case "--template":
                     if (args[i - 1].Equals(TemplateEngine.Razor, StringComparison.CurrentCultureIgnoreCase))
                     {
-                        templateEngine = TemplateEngine.Razor;
+                        result.TemplateEngine = TemplateEngine.Razor;
                     }
                     else if (args[i - 1].Equals(TemplateEngine.Liquid, StringComparison.CurrentCultureIgnoreCase))
                     {
-                        templateEngine = TemplateEngine.Liquid;
+                        result.TemplateEngine = TemplateEngine.Liquid;
                     }
                     else
                     {
-                        templateEngine = null;
+                        result.TemplateEngine = null;
                     }
 
                     break;
@@ -195,18 +217,26 @@ private static (string language, string templateEngine, string singleOutputFile)
                 case "--single":
                     if (!string.IsNullOrEmpty(args[i - 1]))
                     {
-                        singleOutputFile = args[i - 1];
+                        result.SingleOutputFile = args[i - 1];
+                    }
+
+                    break;
+                case "-p":
+                case "--plugin":
+                    if (File.Exists(args[i - 1]))
+                    {
+                        result.Plugins.Add(args[i - 1]);
                     }
 
                     break;
                 default:
-                    language = null;
-                    templateEngine = null;
+                    result.Language = null;
+                    result.TemplateEngine = null;
                     break;
             }
         }
 
-        return (language, templateEngine, singleOutputFile);
+        return result;
     }
 
     private static void ShowHelp()
@@ -226,5 +256,7 @@ private static void ShowHelp()
         Console.WriteLine("  -i, --ignore project1,project2         Ignores extracting PO filed from a given project(s).");
         Console.WriteLine("  --localizer localizer1,localizer2      Specifies the name of the localizer(s) that will be used during the extraction process.");
         Console.WriteLine("  -s, --single <FILE_NAME>               Specifies the single output file.");
+        Console.WriteLine("  -p, --plugin <FILE_NAME>               A path to a C# script (.csx) file which can define further IProjectProcessor");
+        Console.WriteLine("                                         implementations. You can have multiple of this switch in a call.");
     }
 }

test/OrchardCoreContrib.PoExtractor.Tests/OrchardCoreContrib.PoExtractor.Tests.csproj

@@ -15,4 +15,9 @@
 	<ItemGroup>
 		<ProjectReference Include="..\..\src\OrchardCoreContrib.PoExtractor\OrchardCoreContrib.PoExtractor.csproj" />
 	</ItemGroup>
+	<ItemGroup>
+	  <None Update="PluginTestFiles\**\*">
+	    <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+	  </None>
+	</ItemGroup>
 </Project>

test/OrchardCoreContrib.PoExtractor.Tests/PluginTestFiles/BasicJsonLocalizationProcessor.csx

@@ -0,0 +1,77 @@
+using System;
+using System.IO;
+using System.Linq;
+using System.Text.Json.Nodes;
+using OrchardCoreContrib.PoExtractor;
+
+// This example plugin implements processing for a very simplistic subset of the i18next JSON format. It only supports
+// strings and other objects, and the files must be located in i18n/{language}.json. Even though this is only meant as a
+// demo, even this much can be useful in a real life scenario if paired with a backend API that generates the files for
+// other languages using PO files, to centralize the localization tooling.
+public class BasicJsonLocalizationProcessor : IProjectProcessor
+{
+    public void Process(string path, string basePath, LocalizableStringCollection strings)
+    {
+        ArgumentException.ThrowIfNullOrEmpty(path);
+        ArgumentException.ThrowIfNullOrEmpty(basePath);
+        ArgumentNullException.ThrowIfNull(strings);
+
+        var jsonFilePaths = Directory.GetFiles(path, "*.json", SearchOption.AllDirectories)
+            .Where(path => Path.GetFileNameWithoutExtension(path).ToUpperInvariant() is "EN" or "00" or "IV")
+            .Where(path => Path.GetFileName(Path.GetDirectoryName(path))?.ToUpperInvariant() is "I18N")
+            .GroupBy(Path.GetDirectoryName)
+            .Select(group => group
+                .OrderBy(path => Path.GetFileNameWithoutExtension(path).ToUpperInvariant() switch
+                {
+                    "EN" => 0,
+                    "00" => 1,
+                    "IV" => 2,
+                    _ => 3,
+                })
+                .ThenBy(path => path)
+                .First());
+
+        foreach (var jsonFilePath in jsonFilePaths)
+        {
+            try
+            {
+                ProcessJson(
+                    jsonFilePath,
+                    strings,
+                    JObject.Parse(File.ReadAllText(jsonFilePath)),
+                    string.Empty);
+            }
+            catch
+            {
+                Console.WriteLine("Process failed for: {0}", path);
+            }
+        }
+    }
+
+    private static void ProcessJson(string path, LocalizableStringCollection strings, JsonNode json, string prefix)
+    {
+        if (json is JsonObject jsonObject)
+        {
+            foreach (var (name, value) in jsonObject)
+            {
+                var newPrefix = string.IsNullOrEmpty(prefix) ? name : $"{prefix}.{name}";
+                ProcessJson(path, strings, value, newPrefix);
+            }
+
+            return;
+        }
+
+        if (json is JsonValue jsonValue)
+        {
+            var value = jsonValue.GetObjectValue()?.ToString();
+            strings.Add(new()
+            {
+                Context = prefix,
+                Location = new() { SourceFile = path },
+                Text = value,
+            });
+        }
+    }
+}
+
+projectProcessors.Add(new BasicJsonLocalizationProcessor());

test/OrchardCoreContrib.PoExtractor.Tests/PluginTestFiles/i18n/en.json

@@ -0,0 +1,13 @@
+{
+  "about": {
+    "title": "About us",
+    "notes": "Title for main menu"
+  },
+  "home": {
+    "title": "Home page",
+    "context": "Displayed on the main website page"
+  },
+  "admin.login": {
+    "title": "Administrator login"
+  }
+}