Add plugin support for other templates languages

Directory.Packages.props

@@ -7,7 +7,8 @@
     <PackageVersion Include="Fluid.Core" Version="2.12.0" />
     <PackageVersion Include="McMaster.Extensions.CommandLineUtils" Version="4.1.1" />
     <PackageVersion Include="Microsoft.AspNetCore.Razor.Language" Version="6.0.36" />
-    <PackageVersion Include="Microsoft.CodeAnalysis" Version="4.11.0" />
+    <PackageVersion Include="Microsoft.CodeAnalysis" Version="4.12.0" />
+    <PackageVersion Include="Microsoft.CodeAnalysis.CSharp.Scripting" Version="4.12.0" />
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
     <PackageVersion Include="OrchardCore.DisplayManagement.Liquid" Version="2.0.0" />
     <PackageVersion Include="xunit" Version="2.9.2" />

NuGet.config

@@ -1,8 +1,8 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <configuration>
   <packageSources>
+    <!-- Ignore global configuration -->
     <clear />
     <add key="NuGet" value="https://api.nuget.org/v3/index.json" />
-    <add key="OrchardCore" value="https://nuget.cloudsmith.io/orchardcore/preview/v3/index.json" />
   </packageSources>
 </configuration>

OrchardCoreContrib.PoExtractor.sln

@@ -35,6 +35,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 	ProjectSection(SolutionItems) = preProject
 		Directory.Build.props = Directory.Build.props
 		Directory.Packages.props = Directory.Packages.props
+		README.md = README.md
+		NuGet.config = NuGet.config
 	EndProjectSection
 EndProject
 Global

README.md

@@ -42,6 +42,24 @@ Specifies the code language to extracts translatable strings from. Default: `C#`
 
 Specifies the template engine to extract the translatable strings from. Default: `Razor` & `Liquid` templates.
 
+- **`-p|--plugin {path or URL to CSX file}`**
+
+Specifies a path to a C# script file that can define further project processors. (You can find an example script [here](test/OrchardCoreContrib.PoExtractor.Tests/PluginTestFiles/BasicJsonLocalizationProcessor.csx).) This can be used to process localization from code languages or template engines not supported by the above options. You can have multiple of these switches in one call to load several plugins simultaneously. If the argument starts with `https://` then it's treated as a web URL and the script at that address is downloaded into memory and executed instead of a local file.
+
+When executing the plugins, all _OrchardCoreContrib.PoExtractor_ assemblies are automatically loaded, and two globals are defined:
+
+- `List<IProjectProcessor> ProjectProcessors`: Add an instance of your custom `IProjectProcessor` implementation type to this list.
+- `List<string> ProjectFiles`: In the unlikely case that you have to add a new project file type (such as _.fsproj_) add the project file paths to this list.
+
+> [!TIP]
+> You can't import NuGet packages in your script file, but you can import local DLL files using the `#r "path/to/package.dll"` directive. The path can be relative to the script file's location so you can import packages from the build directory of the project you are extracting from. This can be especially useful if you launch the tool as using MSBuild as a post-build action. (For remote scripts loaded with a URL, the path can be relative to the current working directory.) For example:
+> 
+> ```csharp
+> #r "src/Modules/OrchardCore.Commerce/bin/Debug/net8.0/OrchardCore.Commerce.dll"
+> using OrchardCore.Commerce.Constants;
+> Console.WriteLine("Imported resource name: {0}", ResourceNames.ShoppingCart);
+> ```
+
 ## Uninstallation
 
 ```powershell

src/OrchardCoreContrib.PoExtractor/OrchardCoreContrib.PoExtractor.csproj

@@ -34,4 +34,7 @@
 		<ProjectReference Include="..\OrchardCoreContrib.PoExtractor.Liquid\OrchardCoreContrib.PoExtractor.Liquid.csproj" />
 		<ProjectReference Include="..\OrchardCoreContrib.PoExtractor.Razor\OrchardCoreContrib.PoExtractor.Razor.csproj" />
 	</ItemGroup>
+	<ItemGroup>
+		<PackageReference Include="Microsoft.CodeAnalysis.CSharp.Scripting" />
+	</ItemGroup>
 </Project>

src/OrchardCoreContrib.PoExtractor/PluginHelper.cs

@@ -0,0 +1,38 @@
+using Microsoft.CodeAnalysis.CSharp.Scripting;
+using Microsoft.CodeAnalysis.Scripting;
+
+namespace OrchardCoreContrib.PoExtractor;
+
+public static class PluginHelper
+{
+    public static async Task ProcessPluginsAsync(
+        IEnumerable<string> plugins,
+        List<IProjectProcessor> projectProcessors,
+        List<string> projectFiles)
+    {
+        var sharedOptions = ScriptOptions.Default.AddReferences(typeof(Program).Assembly);
+
+        foreach (var plugin in plugins)
+        {
+            string code;
+            ScriptOptions options;
+
+            if (plugin.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
+            {
+                code = await new HttpClient().GetStringAsync(plugin);
+                options = sharedOptions.WithFilePath(Path.Join(
+                    Environment.CurrentDirectory,
+                    Path.GetFileName(new Uri(plugin).AbsolutePath)));
+            }
+            else
+            {
+                code = await File.ReadAllTextAsync(plugin);
+                options = sharedOptions.WithFilePath(Path.GetFullPath(plugin));
+            }
+
+            await CSharpScript.EvaluateAsync(code, options, new PluginContext(projectProcessors, projectFiles));
+        }
+    }
+
+    public record PluginContext(List<IProjectProcessor> ProjectProcessors, List<string> ProjectFiles);
+}

src/OrchardCoreContrib.PoExtractor/Program.cs

@@ -1,4 +1,6 @@
-using McMaster.Extensions.CommandLineUtils;
+using System.ComponentModel.DataAnnotations;
+using McMaster.Extensions.CommandLineUtils;
+using OrchardCore.Modules;
 using OrchardCoreContrib.PoExtractor.DotNet;
 using OrchardCoreContrib.PoExtractor.DotNet.CS;
 using OrchardCoreContrib.PoExtractor.DotNet.VB;
@@ -22,19 +24,29 @@ public static void Main(string[] args)
             .IsRequired();
 
         // Options
-        var language = app.Option("-l|--language <LANGUAGE>", "Specifies the code language to extracts translatable strings from.", CommandOptionType.SingleValue, options =>
+        var language = app.Option("-l|--language <LANGUAGE>", "Specifies the code language to extracts translatable strings from.", CommandOptionType.SingleValue, option =>
         {
-            options.Accepts(cfg => cfg.Values("C#", "VB"));
-            options.DefaultValue = "C#";
+            option.Accepts(cfg => cfg.Values("C#", "VB"));
+            option.DefaultValue = "C#";
         });
-        var template = app.Option("-t|--template <TEMPLATE>", "Specifies the template engine to extract the translatable strings from.", CommandOptionType.SingleValue, options =>
-            options.Accepts(cfg => cfg.Values("Razor", "Liquid"))
+        var template = app.Option("-t|--template <TEMPLATE>", "Specifies the template engine to extract the translatable strings from.", CommandOptionType.SingleValue, option =>
+            option.Accepts(cfg => cfg.Values("Razor", "Liquid"))
         );
         var ignoredProjects = app.Option("-i|--ignore <IGNORED_PROJECTS>", "Ignores extracting PO files from a given project(s).", CommandOptionType.MultipleValue);
         var localizers = app.Option("--localizer <LOCALIZERS>", "Specifies the name of the localizer(s) that will be used during the extraction process.", CommandOptionType.MultipleValue);
         var single = app.Option("-s|--single <FILE_NAME>", "Specifies the single output file.", CommandOptionType.SingleValue);
-
-        app.OnExecute(() =>
+        var plugins = app.Option(
+            "-p|--plugin <FILE_NAME_OR_HTTPS_URL>",
+            "A path or web URL with HTTPS scheme to a C# script (.csx) file which can define further " +
+            "IProjectProcessor implementations. You can have multiple of this switch in a call.",
+            CommandOptionType.MultipleValue,
+            option => option.OnValidate(_ => option
+                .Values
+                .All(item => File.Exists(item) || item.StartsWithOrdinalIgnoreCase("https://"))
+                ? ValidationResult.Success
+                : new ValidationResult("Plugin must be an existing local file or a valid HTTPS URL.")));
+
+        app.OnExecuteAsync(async cancellationToken =>
         {
             if (!Directory.Exists(inputPath.Value))
             {
@@ -84,14 +96,21 @@ public static void Main(string[] args)
                 projectProcessors.Add(new LiquidProjectProcessor());
             }
 
+            if (plugins.Values.Count > 0)
+            {
+                await PluginHelper.ProcessPluginsAsync(plugins.Values, projectProcessors, projectFiles);
+            }
+
             var isSingleFileOutput = !string.IsNullOrEmpty(single.Value());
             var localizableStrings = new LocalizableStringCollection();
             foreach (var projectFile in projectFiles)
             {
                 var projectPath = Path.GetDirectoryName(projectFile);
                 var projectBasePath = Path.GetDirectoryName(projectPath) + Path.DirectorySeparatorChar;
                 var projectRelativePath = projectPath[projectBasePath.Length..];
-                var rootedProject = projectPath[(projectPath.IndexOf(inputPath.Value) + inputPath.Value.Length + 1)..];
+                var rootedProject = projectPath == inputPath.Value
+                    ? projectPath 
+                    : projectPath[(projectPath.IndexOf(inputPath.Value, StringComparison.Ordinal) + inputPath.Value.Length + 1)..];
                 if (IgnoredProject.ToList().Any(p => rootedProject.StartsWith(p)))
                 {
                     continue;

test/OrchardCoreContrib.PoExtractor.Tests/OrchardCoreContrib.PoExtractor.Tests.csproj

@@ -15,4 +15,9 @@
 	<ItemGroup>
 		<ProjectReference Include="..\..\src\OrchardCoreContrib.PoExtractor\OrchardCoreContrib.PoExtractor.csproj" />
 	</ItemGroup>
+	<ItemGroup>
+	  <None Update="PluginTestFiles\**\*">
+	    <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+	  </None>
+	</ItemGroup>
 </Project>

test/OrchardCoreContrib.PoExtractor.Tests/PluginTestFiles/BasicJsonLocalizationProcessor.csx

@@ -0,0 +1,77 @@
+using System;
+using System.IO;
+using System.Linq;
+using System.Text.Json.Nodes;
+using OrchardCoreContrib.PoExtractor;
+
+// This example plugin implements processing for a very simplistic subset of the i18next JSON format. It only supports
+// strings and other objects, and the files must be located in i18n/{language}.json. Even though this is only meant as a
+// demo, even this much can be useful in a real life scenario if paired with a backend API that generates the files for
+// other languages using PO files, to centralize the localization tooling.
+public class BasicJsonLocalizationProcessor : IProjectProcessor
+{
+    public void Process(string path, string basePath, LocalizableStringCollection strings)
+    {
+        ArgumentException.ThrowIfNullOrEmpty(path);
+        ArgumentException.ThrowIfNullOrEmpty(basePath);
+        ArgumentNullException.ThrowIfNull(strings);
+
+        var jsonFilePaths = Directory.GetFiles(path, "*.json", SearchOption.AllDirectories)
+            .Where(path => Path.GetFileNameWithoutExtension(path).ToUpperInvariant() is "EN" or "00" or "IV")
+            .Where(path => Path.GetFileName(Path.GetDirectoryName(path))?.ToUpperInvariant() is "I18N")
+            .GroupBy(Path.GetDirectoryName)
+            .Select(group => group
+                .OrderBy(path => Path.GetFileNameWithoutExtension(path).ToUpperInvariant() switch
+                {
+                    "EN" => 0,
+                    "00" => 1,
+                    "IV" => 2,
+                    _ => 3,
+                })
+                .ThenBy(path => path)
+                .First());
+
+        foreach (var jsonFilePath in jsonFilePaths)
+        {
+            try
+            {
+                ProcessJson(
+                    jsonFilePath,
+                    strings,
+                    JObject.Parse(File.ReadAllText(jsonFilePath)),
+                    string.Empty);
+            }
+            catch
+            {
+                Console.WriteLine("Process failed for: {0}", path);
+            }
+        }
+    }
+
+    private static void ProcessJson(string path, LocalizableStringCollection strings, JsonNode json, string prefix)
+    {
+        if (json is JsonObject jsonObject)
+        {
+            foreach (var (name, value) in jsonObject)
+            {
+                var newPrefix = string.IsNullOrEmpty(prefix) ? name : $"{prefix}.{name}";
+                ProcessJson(path, strings, value, newPrefix);
+            }
+
+            return;
+        }
+
+        if (json is JsonValue jsonValue)
+        {
+            var value = jsonValue.GetObjectValue()?.ToString();
+            strings.Add(new()
+            {
+                Context = prefix,
+                Location = new() { SourceFile = path },
+                Text = value,
+            });
+        }
+    }
+}
+
+ProjectProcessors.Add(new BasicJsonLocalizationProcessor());

test/OrchardCoreContrib.PoExtractor.Tests/PluginTestFiles/i18n/en.json

@@ -0,0 +1,13 @@
+{
+  "about": {
+    "title": "About us",
+    "notes": "Title for main menu"
+  },
+  "home": {
+    "title": "Home page",
+    "context": "Displayed on the main website page"
+  },
+  "admin.login": {
+    "title": "Administrator login"
+  }
+}

test/OrchardCoreContrib.PoExtractor.Tests/PluginTests.cs

@@ -0,0 +1,75 @@
+using System.Text;
+using System.Text.RegularExpressions;
+using Xunit;
+
+namespace OrchardCoreContrib.PoExtractor.Tests;
+
+public partial class PluginTests
+{
+    private const string PluginTestFiles = nameof(PluginTestFiles);
+
+    [Fact]
+    public async Task ProcessPluginsBasicJsonLocalizationProcessor()
+    {
+        // Arrange
+        using var stream = new MemoryStream();
+        var plugins = new[] { Path.Join(PluginTestFiles, "BasicJsonLocalizationProcessor.csx") };
+        var projectProcessors = new  List<IProjectProcessor>();
+        var projectFiles = new List<string> { Path.Join(PluginTestFiles, "OrchardCoreContrib.PoExtractor.Tests.dll") };
+        var localizableStrings = new LocalizableStringCollection();
+
+        // Act
+        await PluginHelper.ProcessPluginsAsync(plugins, projectProcessors, projectFiles);
+        projectProcessors[0].Process(PluginTestFiles, Path.GetFileName(Environment.CurrentDirectory), localizableStrings);
+
+        using (var writer = new PoWriter(stream))
+        {
+            writer.WriteRecord(localizableStrings.Values);
+        }
+
+        // Assert
+        Assert.Single(projectProcessors);
+        Assert.Single(projectFiles);
+        Assert.Equal(5, localizableStrings.Values.Count());
+
+        const string expectedResult = @"
+        #: PluginTestFiles/i18n/en.json:0
+        msgctxt ""about.title""
+        msgid ""About us""
+        msgstr """"
+
+        #: PluginTestFiles/i18n/en.json:0
+        msgctxt ""about.notes""
+        msgid ""Title for main menu""
+        msgstr """"
+
+        #: PluginTestFiles/i18n/en.json:0
+        msgctxt ""home.title""
+        msgid ""Home page""
+        msgstr """"
+
+        #: PluginTestFiles/i18n/en.json:0
+        msgctxt ""home.context""
+        msgid ""Displayed on the main website page""
+        msgstr """"
+
+        #: PluginTestFiles/i18n/en.json:0
+        msgctxt ""admin.login.title""
+        msgid ""Administrator login""
+        msgstr """"";
+        var actualResult = Encoding.UTF8.GetString(stream.ToArray());
+        Assert.Equal(CleanupSpaces(expectedResult), CleanupSpaces(actualResult));
+    }
+
+    private static string CleanupSpaces(string input)
+    {
+        // Trim leading whitespaces.
+        input = TrimLeadingSpacesRegex().Replace(input.Trim(), string.Empty);
+
+        // Make the path OS-specific, so the test works on Windows as well. 
+        return input.Replace('/', Path.DirectorySeparatorChar);
+    }
+
+    [GeneratedRegex(@"^\s+", RegexOptions.Multiline)]
+    private static partial Regex TrimLeadingSpacesRegex();
+}