V16/multiple nodes per file (#829)

* Production 'mode' where we merge all the files together.

* ajust the interface, so we can have nice defaults for new properties.

* clean up the merge function

* make sure individual empty files process the 'current' way (we don't want to slow down existing imports).

* cache the loading of xml files in the clean loop, to stop multiple file loads.

* clean up the import/report methods to make them more readable.

* add incremental update messages so we can smooth out progress even when using a single import file.

* fix typo on exportOnSave settings localization string.

* XML Comments.

* more xml comments

* Update uSync.BackOffice/Hubs/uSyncCallbacks.cs

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update uSync.BackOffice/Configuration/uSyncSettings.cs

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update uSync.BackOffice/Configuration/uSyncSettings.cs

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update uSync.BackOffice/Services/SyncService_Files.cs

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update uSync.BackOffice/Services/SyncFileService.cs

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update uSync.BackOffice/Hubs/uSyncCallbacks.cs

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* make sure we process all single actions (Co-piolt fix)

* Update uSync.BackOffice/SyncHandlers/Interfaces/ISyncHandler.cs

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* fix serializer interface typo.

* remove bool clean from merge public api

* add using alias for core constants.

* Replace File.Exists with call to SyncFileService.FileExists

* Update uSync.BackOffice/Configuration/SyncConfigService.cs

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* typo fixes in the generated schema file.

* Update uSync.BackOffice/Hubs/HubClientService.cs

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update uSync.BackOffice/Services/ISyncFileService.cs

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: KevinJump

uSync.BackOffice.Targets/appsettings-schema.usync.json

@@ -205,7 +205,7 @@
         },
         "DisableNotificationSuppression": {
           "type": "boolean",
-          "description": "turns of use of the Notifications.Supress method, so notifications\nfire after every item is imported.\n            ",
+          "description": "turns off use of the Notifications.Suppress method, so notifications\nfire after every item is imported.\n            ",
           "default": "true"
         },
         "BackgroundNotifications": {
@@ -217,9 +217,37 @@
           "type": "boolean",
           "description": "Move the uSync tree to it's own section in the back office. \n(requires a restart to take effect).\n            ",
           "default": false
+        },
+        "FolderMode": {
+          "description": "What type of mode the folder should work in (default, root, or production)\n            ",
+          "default": "Normal",
+          "oneOf": [
+            {
+              "$ref": "#/definitions/USyncBackOfficeConfigurationSyncFolderMode"
+            }
+          ]
+        },
+        "ProductionFolder": {
+          "type": "string",
+          "description": "location of the 'production' folder to use when in production mode, \nor when creating the production mode files.\n            ",
+          "default": "uSync/production"
         }
       }
     },
+    "USyncBackOfficeConfigurationSyncFolderMode": {
+      "type": "string",
+      "description": "uSync's folder mode - normal, root or production\n            ",
+      "x-enumNames": [
+        "Normal",
+        "Root",
+        "Production"
+      ],
+      "enum": [
+        "Normal",
+        "Root",
+        "Production"
+      ]
+    },
     "USyncuSyncSetsDefinition": {
       "type": "object",
       "properties": {

uSync.BackOffice/Configuration/SyncConfigService.cs

@@ -66,21 +66,24 @@ private class SyncFolderItem
     /// <inheritdoc/>
     public string GetWorkingFolder()
     {
-        var folders = FetchFolders();
-
-        return Settings.IsRootSite
-            ? folders[0].TrimStart('/')
-            : folders.Last().TrimStart('/');
+        var folders = GetFolders();
+        return folders.Last().TrimStart('/');
     }
 
     /// <inheritdoc/>
     public string[] GetFolders()
     {
         var folders = FetchFolders();
 
-        return Settings.IsRootSite
-            ? [folders[0].TrimStart('/')]
-            : [.. folders.Select(x => x.TrimStart('/'))];
+        switch(Settings.FolderMode)
+        {
+            case SyncFolderMode.Root:
+                return [folders[0].TrimStart('/')];
+            case SyncFolderMode.Production:
+                return [Settings.ProductionFolder.TrimStart('/')];
+            default:
+                return [.. folders.Select(x => x.TrimStart('/'))];
+        }
     }
 
     /// <inheritdoc/>

uSync.BackOffice/Configuration/uSyncSettings.cs

@@ -197,26 +197,22 @@ public class uSyncSettings
     public string HideAddOns { get; set; } = "licence";
 
     /// <summary>
-    ///  turns of use of the Notifications.Supress method, so notifications
+    ///  turns off use of the Notifications.Suppress method, so notifications
     ///  fire after every item is imported.
     /// </summary>
     /// <remarks>
-    ///  I am not sure this does what i think it does, it doesn't suppress
-    ///  then fire at the end , it just suppresses them all. 
+    ///  this disables the internal uSync scope provider that delays all 
+    ///  non cancellable notifications until after the import is complete.
     ///  
-    ///  until we have had time to look at this , we will leave this as 
-    ///  disabled by default so all notification messages fire.
+    ///  on v13 this is false, the import happens and then the notifications fire.
     ///  
-    ///  for v13 thius is fine, but for v14, grouping the notifications
-    ///  can causes issues if something fails. 
-    ///   
-    ///  So if a single content import fails then the whole batch doesn't 
-    ///  get published properly (so no content for you :( ) .
+    ///  on v16 the default is true, because some of the notifications appear to 
+    ///  be closely coupled to the save/publish process, and if something goes 
+    ///  wrong in one item's import it can cause a cascade of failures across 
+    ///  everything that might have been imported along with it. 
     ///  
-    ///  there might be something downlever we can do, but it likey means
-    ///  lots of core investigation to find that, for now 'old' school
-    ///  non suppressed notifications should be fine (if a little slower).
-    /// 
+    ///  if the notifications are not suppressed, then if an item fails to import
+    ///  it doesn't stop other items from being imported. 
     /// </remarks>
     [DefaultValue("true")]
     public bool DisableNotificationSuppression { get; set; } = true;
@@ -238,4 +234,39 @@ public class uSyncSettings
     /// </summary>
     [DefaultValue(false)]
     public bool MoveToSection { get; set; } = false;
+
+    /// <summary>
+    ///  What type of mode the folder should work in (default, root, or production)
+    /// </summary>
+    [DefaultValue(SyncFolderMode.Normal)]
+    public SyncFolderMode FolderMode { get; set; } = SyncFolderMode.Normal;
+
+    /// <summary>
+    ///  location of the 'production' folder to use when in production mode, 
+    ///  or when creating the production mode files.
+    /// </summary>
+    [DefaultValue("uSync/production")]
+    public string ProductionFolder { get; set; } = "uSync/production";
 }
+
+/// <summary>
+///  uSync's folder mode - normal, root or production
+/// </summary>
+public enum SyncFolderMode
+{
+    /// <summary>
+    ///  normal - expects individual files in the uSync folder(s)
+    /// </summary>
+    Normal,
+
+    /// <summary>
+    ///  root - will read and write things to the root folder, 
+    /// </summary>
+    Root,
+
+    /// <summary>
+    ///  production - looks in a 'production' folder, expects a single file per handler.
+    /// </summary>
+    Production,
+};
+

uSync.BackOffice/Hubs/HubClientService.cs

@@ -80,5 +80,27 @@ public void PostUpdate(string message, int count, int total)
     ///  get the uSync callbacks for this connection
     /// </summary>
     /// <returns></returns>
-    public uSyncCallbacks Callbacks() => new(this.PostSummary, this.PostUpdate);
+    public uSyncCallbacks Callbacks() => new(this.PostSummary, this.PostUpdate, this.SetCountRange, this.PostIncrementalUpdate);
+
+    private int _start = 0;
+    private int _end = 0;
+
+    /// <summary>
+    ///  set a range (start to end) that we expect the next set of updates to be bound within.
+    /// </summary>
+    public void SetCountRange(int start, int end)
+    {
+        _start = start;
+        _end = end;
+    }
+
+    /// <summary>
+    ///  post an update and increment the counter by one.
+    /// </summary>
+    public void PostIncrementalUpdate(string message)
+    {
+        _start++;
+        if (_start > _end) _end = _start;
+        this.PostUpdate(message, _start, _end);
+    }
 }

uSync.BackOffice/Hubs/uSyncCallbacks.cs

@@ -1,7 +1,30 @@
-using uSync.BackOffice.SyncHandlers.Interfaces;
+using uSync.BackOffice.Models;
+using uSync.BackOffice.SyncHandlers.Interfaces;
 
 namespace uSync.BackOffice;
 
+/// <summary>
+///  Callback event for SignalR hub
+/// </summary>
+public delegate void SyncEventCallback(SyncProgressSummary summary);
+
+/// <summary>
+///  callback delegate for SignalR messaging 
+/// </summary>
+public delegate void SyncUpdateCallback(string message, int count, int total);
+
+/// <summary>
+///  callback delegate to set the start and end range for the update counters.
+/// </summary>
+public delegate void SyncSetUpdateRange(int start, int end);
+
+/// <summary>
+///  callback to send a update message and increment the counter by one so moving the progress bar.
+/// </summary>
+/// <param name="message"></param>
+public delegate void SyncIncrementalUpdateCallback(string message);
+
+
 /// <summary>
 ///  Callback objects used to communicate via SignalR
 /// </summary>
@@ -17,6 +40,16 @@ public class uSyncCallbacks
     /// </summary>
     public SyncUpdateCallback? Update { get; private set; }
 
+    /// <summary>
+    ///  set a start and end range for the counter.
+    /// </summary>
+    public SyncSetUpdateRange? SetRange { get; private set; }
+
+    /// <summary>
+    ///  update and increment callback.
+    /// </summary>
+    public SyncIncrementalUpdateCallback? IncrementalUpdate { get; private set; }
+
     /// <summary>
     ///  generate a new callback object 
     /// </summary>
@@ -25,4 +58,14 @@ public uSyncCallbacks(SyncEventCallback? callback, SyncUpdateCallback? update)
         this.Callback = callback;
         this.Update = update;
     }
+
+    /// <summary>
+    ///  generate a callback object with range and incremental update
+    /// </summary>
+    public uSyncCallbacks(SyncEventCallback? callback, SyncUpdateCallback? update, SyncSetUpdateRange? updateRange, SyncIncrementalUpdateCallback incrementalUpdate)
+        : this(callback, update)
+    {
+        this.SetRange = updateRange;
+        this.IncrementalUpdate = incrementalUpdate;
+    }
 }

uSync.BackOffice/Services/ISyncFileService.cs

@@ -135,6 +135,11 @@ public interface ISyncFileService
     /// <returns></returns>
     Task<XElement> LoadXElementAsync(string file);
 
+    /// <summary>
+    ///  merge all the files in the given folders into a single xml node, that can be bulk imported
+    /// </summary>
+    Task<int> MakeSingleExportFromFolders(string[] folders, string itemType, ISyncTrackerBase? trackerBase, string fileName, string extension);
+
     /// <summary>
     ///  merge a list of files into a single XElement
     /// </summary>

uSync.BackOffice/Services/ISyncService.cs

@@ -156,4 +156,10 @@ public interface ISyncService
     ///  trigger the end of the bulk process
     /// </summary>
     Task FinishBulkProcessAsync(HandlerActions action, IEnumerable<uSyncAction> actions);
+
+    /// <summary>
+    ///  merge the given folders in single 'production' files for each handler.
+    /// </summary>
+    Task<int> MergeExportFolder(string[] paths, IEnumerable<HandlerConfigPair> handlers);
+
 }

uSync.BackOffice/Services/SyncFileService.cs

@@ -242,6 +242,7 @@ public async Task SaveFileAsync(string filename, string content)
     {
         CheckCharacters = false,
         Async = true,
+        IgnoreWhitespace = true,
     };
 
     private static XmlWriterSettings _writerSettings = new XmlWriterSettings
@@ -251,8 +252,6 @@ public async Task SaveFileAsync(string filename, string content)
         Async = true,
         CloseOutput= false,
         Indent = true,
-        
-
     };
 
     /// <inheritdoc/>
@@ -323,7 +322,26 @@ public void CopyFolder(string source, string target)
         {
             File.Copy(file, file.Replace(resolvedSource, resolvedTarget), true);
         }
+    }
+
+    public async Task<int> MakeSingleExportFromFolders(string[] folders, string itemType, ISyncTrackerBase? trackerBase, string filename, string extension)
+    {
+        var merged = await MergeFoldersAsync(folders, extension, trackerBase);
+
+        var megaNode = new XElement(itemType + "s");
+        int count = 0;
+        foreach(var item in merged)
+        {
+            count++;
+            megaNode.Add(new XElement(item.Node));
+        }
+
+        var resolvedTargetFile = GetAbsPath(filename);
+        CreateFoldersForFile(resolvedTargetFile);
+
+        await SaveXElementAsync(megaNode, resolvedTargetFile);
 
+        return count;
     }
 
     /// <inheritdoc/>

uSync.BackOffice/Services/SyncService.cs

@@ -27,12 +27,6 @@
 
 namespace uSync.BackOffice;
 
-
-/// <summary>
-///  Callback event for SignalR hub
-/// </summary>
-public delegate void SyncEventCallback(SyncProgressSummary summary);
-
 /// <summary>
 ///  the service that does all the processing,
 ///  this forms the entry point as an API to 

uSync.BackOffice/Services/SyncService_Files.cs

@@ -1,7 +1,13 @@
-using System;
+using Microsoft.Extensions.Logging;
+
+using System;
+using System.Collections.Generic;
 using System.IO;
 using System.IO.Compression;
 using System.Linq;
+using System.Threading.Tasks;
+
+using uSync.BackOffice.SyncHandlers.Models;
 
 namespace uSync.BackOffice;
 
@@ -104,4 +110,29 @@ private static string CleanPathForZip(string path)
         => Path.GetFullPath(
             path.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar))
             .TrimEnd(Path.DirectorySeparatorChar);
+
+    /// <inheritdoc />
+    public async Task<int> MergeExportFolder(string[] paths, IEnumerable<HandlerConfigPair> handlers)
+    {
+        var totalMerged = 0;
+
+        foreach (var handler in handlers)
+        {
+            var serializerType = handler.Handler.GetSerializerType();
+            var baseTracker = handler.Handler.GetBaseTracker();
+            if (serializerType is null || baseTracker is null)
+            {
+                _logger.LogWarning("Handler {Handler} does not support file merging", handler.Handler.Alias);
+                continue;
+            }
+
+            var folders = paths.Select(x => Path.Combine(x, handler.Handler.DefaultFolder)).ToArray();
+            var targetFileName = Path.Combine(_uSyncConfig.Settings.ProductionFolder,
+                handler.Handler.DefaultFolder + "." + _uSyncConfig.Settings.DefaultExtension);
+
+            totalMerged += await _syncFileService.MakeSingleExportFromFolders(folders, serializerType, baseTracker, targetFileName, _uSyncConfig.Settings.DefaultExtension);
+        }
+
+        return totalMerged;
+    }
 }