V12/ordered nodes

uSync.BackOffice.Targets/appsettings-schema.usync.json

@@ -248,6 +248,10 @@
           "description": "Override the group the handler belongs too.",
           "default": ""
         },
+        "CreateClean": {
+          "type": "boolean",
+          "description": "create a corresponding _clean file for this export \n            "
+        },
         "Settings": {
           "type": "object",
           "description": "Additional settings for the handler",

uSync.BackOffice/Configuration/uSyncHandlerSettings.cs

@@ -50,6 +50,14 @@ public class HandlerSettings
         [DefaultValue("")]
         public string Group { get; set; } = string.Empty;
 
+        /// <summary>
+        ///  create a corresponding _clean file for this export 
+        /// </summary>
+        /// <remarks>
+        ///  the clean file will only get created if the item in question has children.
+        /// </remarks>
+        public bool CreateClean { get; set; } = false;  
+
         /// <summary>
         /// Additional settings for the handler
         /// </summary>

uSync.BackOffice/Services/uSyncService_Single.cs

@@ -1,11 +1,11 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Xml.Linq;
 
 using Microsoft.Extensions.Logging;
 
-using Umbraco.Cms.Core.Scoping;
 using Umbraco.Extensions;
 
 using uSync.BackOffice.Extensions;
@@ -29,11 +29,21 @@ public partial class uSyncService
         public IEnumerable<uSyncAction> ReportPartial(string folder, uSyncPagedImportOptions options, out int total)
         {
             var orderedNodes = LoadOrderedNodes(folder);
+            return ReportPartial(orderedNodes, options, out total); 
+        }
+
+        /// <summary>
+        ///  perform a paged report with the supplied ordered nodes
+        /// </summary>
+        public IEnumerable<uSyncAction> ReportPartial(IList<OrderedNodeInfo> orderedNodes, uSyncPagedImportOptions options, out int total)
+        {
             total = orderedNodes.Count;
 
             var actions = new List<uSyncAction>();
             var lastType = string.Empty;
 
+            var folder = Path.GetDirectoryName(orderedNodes.FirstOrDefault()?.FileName ?? options.RootFolder);
+
             SyncHandlerOptions syncHandlerOptions = HandlerOptionsFromPaged(options);
 
             HandlerConfigPair handlerPair = null;
@@ -47,11 +57,13 @@ public IEnumerable<uSyncAction> ReportPartial(string folder, uSyncPagedImportOpt
                 {
                     lastType = itemType;
                     handlerPair = _handlerFactory.GetValidHandlerByTypeName(itemType, syncHandlerOptions);
+
+                    handlerPair?.Handler.PreCacheFolderKeys(folder, orderedNodes.Select(x => x.Key).ToList());
                 }
 
                 if (handlerPair == null)
                 {
-                    _logger.LogWarning("No handler was found for {alias} ({itemType}) item might not process correctly", itemType);
+                    _logger.LogWarning("No handler for {itemType} {alias}", itemType, item.Node.GetAlias());
                     continue;
                 }
 
@@ -73,12 +85,20 @@ public IEnumerable<uSyncAction> ReportPartial(string folder, uSyncPagedImportOpt
         ///  Perform a paged Import against a given folder 
         /// </summary>
         public IEnumerable<uSyncAction> ImportPartial(string folder, uSyncPagedImportOptions options, out int total)
+        {
+            var orderedNodes = LoadOrderedNodes(folder);
+            return ImportPartial(orderedNodes, options, out total);
+        }
+
+        /// <summary>
+        ///  perform an import of items from the suppled ordered node list. 
+        /// </summary>
+        public IEnumerable<uSyncAction> ImportPartial(IList<OrderedNodeInfo> orderedNodes, uSyncPagedImportOptions options, out int total)
         {
             lock (_importLock)
             {
                 using (var pause = _mutexService.ImportPause(options.PauseDuringImport))
                 {
-                    var orderedNodes = LoadOrderedNodes(folder);
 
                     total = orderedNodes.Count;
 
@@ -99,20 +119,23 @@ public IEnumerable<uSyncAction> ImportPartial(string folder, uSyncPagedImportOpt
                         {
                             foreach (var item in orderedNodes.Skip(options.PageNumber * options.PageSize).Take(options.PageSize))
                             {
+                                if (item.Node == null) 
+                                    item.Node = XElement.Load(item.FileName);
+
                                 var itemType = item.Node.GetItemType();
                                 if (!itemType.InvariantEquals(lastType))
                                 {
                                     lastType = itemType;
                                     handlerPair = _handlerFactory.GetValidHandlerByTypeName(itemType, syncHandlerOptions);
 
-                                    // special case, blueprints looks like IContent items, except they are slightly different
-                                    // so we check for them specifically and get the handler for the entity rather than the object type.
-                                    if (item.Node.IsContent() && item.Node.IsBlueprint())
-                                    {
-                                        lastType = UdiEntityType.DocumentBlueprint;
-                                        handlerPair = _handlerFactory.GetValidHandlerByEntityType(UdiEntityType.DocumentBlueprint);
-                                    }
-                                }
+                            // special case, blueprints looks like IContent items, except they are slightly different
+                            // so we check for them specifically and get the handler for the entity rather than the object type.
+                            if (item.Node.IsContent() && item.Node.IsBlueprint())
+                            {
+                                lastType = UdiEntityType.DocumentBlueprint;
+                                handlerPair = _handlerFactory.GetValidHandlerByEntityType(UdiEntityType.DocumentBlueprint);
+                            }
+                        }
 
                                 if (handlerPair == null)
                                 {
@@ -306,7 +329,7 @@ private SyncHandlerOptions HandlerOptionsFromPaged(uSyncPagedImportOptions optio
         /// <summary>
         ///  Load the xml in a folder in level order so we process the higher level items first.
         /// </summary>
-        private IList<OrderedNodeInfo> LoadOrderedNodes(string folder)
+        public IList<OrderedNodeInfo> LoadOrderedNodes(string folder)
         {
             var files = _syncFileService.GetFiles(folder, $"*.{_uSyncConfig.Settings.DefaultExtension}", true);
 
@@ -322,19 +345,6 @@ private IList<OrderedNodeInfo> LoadOrderedNodes(string folder)
                 .ToList();
         }
 
-        private class OrderedNodeInfo
-        {
-            public OrderedNodeInfo(string filename, XElement node)
-            {
-                this.FileName = filename;
-                this.Node = node;
-            }
-
-            public XElement Node { get; set; }
-            public string FileName { get; set; }
-        }
-
-
         /// <summary>
         ///  calculate the percentage progress we are making between a range. 
         /// </summary>
@@ -343,6 +353,38 @@ public OrderedNodeInfo(string filename, XElement node)
         /// </remarks>
         private int CalculateProgress(int value, int total, int min, int max)
             => (int)(min + (((float)value / total) * (max - min)));
+    }
+
+    /// <summary>
+    ///  detail for a usync file that can be ordered
+    /// </summary>
+    public class OrderedNodeInfo
+    {
+        /// <summary>
+        ///  constructor
+        /// </summary>
+        public OrderedNodeInfo(string filename, XElement node)
+        {
+            FileName = filename;
+            Node = node;
+            Key = node.GetKey();
+        }
+
+        /// <summary>
+        ///  xml element of the node
+        /// </summary>
+        public XElement Node { get; set; }
 
+        /// <summary>
+        ///  the Guid key for this item, so we can cache the list of keys 
+        /// </summary>
+        public Guid Key { get; set; }
+
+        /// <summary>
+        ///  path to the physical file 
+        /// </summary>
+        public string FileName { get; set; }
     }
+
+
 }

uSync.BackOffice/SyncHandlers/Handlers/ContentHandler.cs

@@ -58,6 +58,10 @@ public ContentHandler(
             this.serializer = syncItemFactory.GetSerializer<IContent>("ContentSerializer");
         }
 
+        /// <inheritdoc />
+        protected override bool HasChildren(IContent item)
+            => contentService.HasChildren(item.Id);
+
         /// <summary>
         ///  Get child items 
         /// </summary>

uSync.BackOffice/SyncHandlers/Handlers/ContentHandlerBase.cs

@@ -103,7 +103,7 @@ private bool ImportTrashedItem(XElement node, HandlerSettings config)
         {
             // unless the setting is explicit we don't import trashed items. 
             var trashed = node.Element("Info")?.Element("Trashed").ValueOrDefault(false);
-            if (trashed.GetValueOrDefault(false) && !config.GetSetting("ImportTrashed", true)) return false;
+            if (trashed.GetValueOrDefault(false) && !config.GetSetting("ImportTrashed", false)) return false;
 
             return true;
         }

uSync.BackOffice/SyncHandlers/Handlers/MediaHandler.cs

@@ -51,6 +51,10 @@ public MediaHandler(
             this.mediaService = mediaService;
         }
 
+        /// <inheritdoc />
+        protected override bool HasChildren(IMedia item)
+            => mediaService.HasChildren(item.Id);
+
         /// <inheritdoc/>
         protected override IEnumerable<IEntity> GetChildItems(IEntity parent)
         {

uSync.BackOffice/SyncHandlers/Interfaces/ISyncHandler.cs

@@ -76,7 +76,7 @@ public interface ISyncHandler
         string EntityType { get; }
 
         /// <summary>
-        /// The type name of the items hanled (Item.getType().ToString())
+        /// The type name of the items handled (Item.getType().ToString())
         /// </summary>
         string TypeName { get; }
 
@@ -95,7 +95,7 @@ public interface ISyncHandler
         /// </summary>
         /// <param name="folder">folder to use when exporting</param>
         /// <param name="settings">Handler settings to use for export</param>
-        /// <param name="callback">Callbacks to keep UI uptodate</param>
+        /// <param name="callback">Callbacks to keep UI upto date</param>
         /// <returns>List of actions detailing changes</returns>
         IEnumerable<uSyncAction> ExportAll(string folder, HandlerSettings settings, SyncUpdateCallback callback);
 
@@ -124,7 +124,7 @@ public interface ISyncHandler
         /// <param name="folder">folder to use when Importing</param>
         /// <param name="settings">Handler settings to use for import</param>
         /// <param name="force">Force the import even if the settings haven't changed</param>
-        /// <param name="callback">Callbacks to keep UI uptodate</param>
+        /// <param name="callback">Callbacks to keep UI upto date</param>
         /// <returns>List of actions detailing changes</returns>
         IEnumerable<uSyncAction> ImportAll(string folder, HandlerSettings settings, bool force, SyncUpdateCallback callback);
 
@@ -138,7 +138,7 @@ public interface ISyncHandler
         /// </summary>
         /// <param name="folder">folder to use when reporting</param>
         /// <param name="settings">Handler settings to use for report</param>
-        /// <param name="callback">Callbacks to keep UI uptodate</param>
+        /// <param name="callback">Callbacks to keep UI upto date</param>
         /// <returns>List of actions detailing changes</returns>
         IEnumerable<uSyncAction> Report(string folder, HandlerSettings settings, SyncUpdateCallback callback);
 
@@ -154,14 +154,21 @@ public interface ISyncHandler
         IEnumerable<uSyncAction> ImportSecondPass(uSyncAction action, HandlerSettings settings, uSyncImportOptions options);
 
         /// <summary>
-        ///  default impimentation, roothandler does do this. 
+        ///  default implementation, root handler does do this. 
         /// </summary>
         Udi FindFromNode(XElement node) => null;
 
         /// <summary>
-        ///  is this a current node (roothandler can do this too)
+        ///  is this a current node (root handler can do this too)
         /// </summary>
         ChangeType GetItemStatus(XElement node) => ChangeType.NoChange;
 
+
+        /// <summary>
+        ///  precaches the keys of a folder
+        /// </summary>
+        /// <param name="folder"></param>
+        /// <param name="keys"></param>
+        void PreCacheFolderKeys(string folder, IList<Guid> keys) { }
     }
 }

uSync.BackOffice/SyncHandlers/SyncHandlerBase.cs

@@ -11,6 +11,7 @@
 using Umbraco.Cms.Core.Models.Entities;
 using Umbraco.Cms.Core.Services;
 using Umbraco.Cms.Core.Strings;
+using Umbraco.Extensions;
 
 using uSync.BackOffice.Configuration;
 using uSync.BackOffice.Services;
@@ -50,6 +51,10 @@ public SyncHandlerBase(
             this.entityService = entityService;
         }
 
+        /// <inheritdoc />
+        protected override bool HasChildren(TObject item)
+            => entityService.GetChildren(item.Id).Any();
+
         /// <summary>
         ///  given a folder we calculate what items we can remove, becuase they are 
         ///  not in one the the files in the folder.
@@ -189,7 +194,12 @@ protected override IEnumerable<IEntity> GetChildItems(IEntity parent)
         /// </summary>
         virtual protected IEnumerable<IEntity> GetChildItems(int parent)
         {
-            if (this.itemObjectType != UmbracoObjectTypes.Unknown)
+            if (this.itemObjectType == UmbracoObjectTypes.Unknown)
+                return Enumerable.Empty<IEntity>();
+
+            var cacheKey = $"{GetCacheKeyBase()}_parent_{parent}";
+
+            return runtimeCache.GetCacheItem(cacheKey, () =>
             {
                 if (parent == -1)
                 {
@@ -201,9 +211,7 @@ virtual protected IEnumerable<IEntity> GetChildItems(int parent)
                     // load it, so GetChildren without the object type is quicker. 
                     return entityService.GetChildren(parent);
                 }
-            }
-
-            return Enumerable.Empty<IEntity>();
+            }, null);
         }
 
         /// <summary>