From b92684750e90e6d6862913b4249fba852239f20f Mon Sep 17 00:00:00 2001 From: Kevin Jump Date: Fri, 29 Sep 2023 17:43:32 +0100 Subject: [PATCH 01/11] Allow - ordered node precaching on report/import. --- .../Services/uSyncService_Single.cs | 53 +++++++++++++------ 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/uSync.BackOffice/Services/uSyncService_Single.cs b/uSync.BackOffice/Services/uSyncService_Single.cs index 32cc9bc3..689393c6 100644 --- a/uSync.BackOffice/Services/uSyncService_Single.cs +++ b/uSync.BackOffice/Services/uSyncService_Single.cs @@ -29,6 +29,11 @@ public partial class uSyncService public IEnumerable ReportPartial(string folder, uSyncPagedImportOptions options, out int total) { var orderedNodes = LoadOrderedNodes(folder); + return ReportPartial(orderedNodes, options, out total); + } + + public IEnumerable ReportPartial(IList orderedNodes, uSyncPagedImportOptions options, out int total) + { total = orderedNodes.Count; var actions = new List(); @@ -73,12 +78,17 @@ public IEnumerable ReportPartial(string folder, uSyncPagedImportOpt /// Perform a paged Import against a given folder /// public IEnumerable ImportPartial(string folder, uSyncPagedImportOptions options, out int total) + { + var orderedNodes = LoadOrderedNodes(folder); + return ImportPartial(orderedNodes, options, out total); + } + + public IEnumerable ImportPartial(IList orderedNodes, uSyncPagedImportOptions options, out int total) { lock (_importLock) { using (var pause = _mutexService.ImportPause(options.PauseDuringImport)) { - var orderedNodes = LoadOrderedNodes(folder); total = orderedNodes.Count; @@ -306,7 +316,7 @@ private SyncHandlerOptions HandlerOptionsFromPaged(uSyncPagedImportOptions optio /// /// Load the xml in a folder in level order so we process the higher level items first. /// - private IList LoadOrderedNodes(string folder) + public IList LoadOrderedNodes(string folder) { var files = _syncFileService.GetFiles(folder, $"*.{_uSyncConfig.Settings.DefaultExtension}", true); @@ -322,19 +332,6 @@ private IList LoadOrderedNodes(string folder) .ToList(); } - private class OrderedNodeInfo - { - public OrderedNodeInfo(string filename, XElement node) - { - this.FileName = filename; - this.Node = node; - } - - public XElement Node { get; set; } - public string FileName { get; set; } - } - - /// /// calculate the percentage progress we are making between a range. /// @@ -343,6 +340,32 @@ public OrderedNodeInfo(string filename, XElement node) /// private int CalculateProgress(int value, int total, int min, int max) => (int)(min + (((float)value / total) * (max - min))); + } + /// + /// detail for a usync file that can be ordered + /// + public class OrderedNodeInfo + { + /// + /// constructor + /// + public OrderedNodeInfo(string filename, XElement node) + { + this.FileName = filename; + this.Node = node; + } + + /// + /// xml element of the node + /// + public XElement Node { get; set; } + + /// + /// path to the physical file + /// + public string FileName { get; set; } } + + } From f17dbd37b5aead8e4974af10c9467688ae1a5a9c Mon Sep 17 00:00:00 2001 From: Kevin Jump Date: Fri, 29 Sep 2023 18:23:31 +0100 Subject: [PATCH 02/11] optimized the clear key lookup --- .../Services/uSyncService_Single.cs | 15 +++++++++++---- .../SyncHandlers/Interfaces/ISyncHandler.cs | 8 ++++++++ .../SyncHandlers/SyncHandlerRoot.cs | 17 ++++++++++++++++- 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/uSync.BackOffice/Services/uSyncService_Single.cs b/uSync.BackOffice/Services/uSyncService_Single.cs index 689393c6..2140aea4 100644 --- a/uSync.BackOffice/Services/uSyncService_Single.cs +++ b/uSync.BackOffice/Services/uSyncService_Single.cs @@ -1,11 +1,11 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Xml.Linq; using Microsoft.Extensions.Logging; -using Umbraco.Cms.Core.Scoping; using Umbraco.Extensions; using uSync.BackOffice.Extensions; @@ -39,6 +39,8 @@ public IEnumerable ReportPartial(IList orderedNode var actions = new List(); var lastType = string.Empty; + var folder = Path.GetDirectoryName(orderedNodes.FirstOrDefault()?.FileName ?? options.RootFolder); + SyncHandlerOptions syncHandlerOptions = HandlerOptionsFromPaged(options); HandlerConfigPair handlerPair = null; @@ -60,6 +62,8 @@ public IEnumerable ReportPartial(IList orderedNode continue; } + handlerPair.Handler.PreCacheFolderKeys(folder, orderedNodes.Select(x => x.Key).ToList()); + options.Callbacks?.Update.Invoke(item.Node.GetAlias(), CalculateProgress(index, total, options.ProgressMin, options.ProgressMax), 100); @@ -352,8 +356,9 @@ public class OrderedNodeInfo /// public OrderedNodeInfo(string filename, XElement node) { - this.FileName = filename; - this.Node = node; + FileName = filename; + Node = node; + Key = node.GetKey(); } /// @@ -361,6 +366,8 @@ public OrderedNodeInfo(string filename, XElement node) /// public XElement Node { get; set; } + public Guid Key { get; set; } + /// /// path to the physical file /// diff --git a/uSync.BackOffice/SyncHandlers/Interfaces/ISyncHandler.cs b/uSync.BackOffice/SyncHandlers/Interfaces/ISyncHandler.cs index 76762769..d7af0ae4 100644 --- a/uSync.BackOffice/SyncHandlers/Interfaces/ISyncHandler.cs +++ b/uSync.BackOffice/SyncHandlers/Interfaces/ISyncHandler.cs @@ -163,5 +163,13 @@ public interface ISyncHandler /// ChangeType GetItemStatus(XElement node) => ChangeType.NoChange; + + /// + /// precaches the keys of a folder + /// + /// + /// + void PreCacheFolderKeys(string folder, IList keys) + { } } } diff --git a/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs b/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs index d1d2d16c..d7aac35a 100644 --- a/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs +++ b/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs @@ -609,6 +609,20 @@ protected virtual IEnumerable CleanFolder(string cleanFile, bool re } } + /// + /// pre-populates the cache folder key list. + /// + /// + /// this means if we are calling the process multiple times, + /// we can optimise the key code and only load it once. + /// + public void PreCacheFolderKeys(string folder, IList folderKeys) + { + var cacheKey = $"{GetCacheKeyBase()}_{folder.GetHashCode()}"; + runtimeCache.ClearByKey(cacheKey) ; + runtimeCache.GetCacheItem(cacheKey, () => folderKeys); + } + /// /// Get the GUIDs for all items in a folder /// @@ -624,10 +638,11 @@ protected IList GetFolderKeys(string folder, bool flat) var cacheKey = $"{GetCacheKeyBase()}_{folderKey}"; - logger.LogDebug("Getting Folder Keys : {cacheKey}", cacheKey); return runtimeCache.GetCacheItem(cacheKey, () => { + logger.LogDebug("Getting Folder Keys : {cacheKey}", cacheKey); + // when it's not flat structure we also get the sub folders. (extra defensive get them all) var keys = new List(); var files = syncFileService.GetFiles(folder, $"*.{this.uSyncConfig.Settings.DefaultExtension}", !flat).ToList(); From cda0bd3eeaf78478e5db98f28c484618414943db Mon Sep 17 00:00:00 2001 From: Kevin Jump Date: Tue, 3 Oct 2023 07:50:18 +0100 Subject: [PATCH 03/11] Add ability to create clean as part of export request. --- .../Configuration/uSyncHandlerSettings.cs | 8 +++++ .../Services/uSyncService_Single.cs | 31 ++++++++++++------- .../SyncHandlers/Handlers/ContentHandler.cs | 3 ++ .../SyncHandlers/Handlers/MediaHandler.cs | 3 ++ .../SyncHandlers/Interfaces/ISyncHandler.cs | 15 +++++---- .../SyncHandlers/SyncHandlerBase.cs | 15 ++++++--- .../SyncHandlers/SyncHandlerRoot.cs | 25 ++++++++++++++- 7 files changed, 76 insertions(+), 24 deletions(-) diff --git a/uSync.BackOffice/Configuration/uSyncHandlerSettings.cs b/uSync.BackOffice/Configuration/uSyncHandlerSettings.cs index d1f10da4..4c0df1d2 100644 --- a/uSync.BackOffice/Configuration/uSyncHandlerSettings.cs +++ b/uSync.BackOffice/Configuration/uSyncHandlerSettings.cs @@ -50,6 +50,14 @@ public class HandlerSettings [DefaultValue("")] public string Group { get; set; } = string.Empty; + /// + /// create a corresponding _clean file for this export + /// + /// + /// the clean file will only get created if the item in question has children. + /// + public bool CreateClean { get; set; } = false; + /// /// Additional settings for the handler /// diff --git a/uSync.BackOffice/Services/uSyncService_Single.cs b/uSync.BackOffice/Services/uSyncService_Single.cs index 2140aea4..44f37c9a 100644 --- a/uSync.BackOffice/Services/uSyncService_Single.cs +++ b/uSync.BackOffice/Services/uSyncService_Single.cs @@ -32,6 +32,9 @@ public IEnumerable ReportPartial(string folder, uSyncPagedImportOpt return ReportPartial(orderedNodes, options, out total); } + /// + /// perform a paged report with the supplied ordered nodes + /// public IEnumerable ReportPartial(IList orderedNodes, uSyncPagedImportOptions options, out int total) { total = orderedNodes.Count; @@ -54,16 +57,16 @@ public IEnumerable ReportPartial(IList orderedNode { lastType = itemType; handlerPair = _handlerFactory.GetValidHandlerByTypeName(itemType, syncHandlerOptions); + + handlerPair?.Handler.PreCacheFolderKeys(folder, orderedNodes.Select(x => x.Key).ToList()); } if (handlerPair == null) { - _logger.LogWarning("No handler was found for {alias} ({itemType}) item might not process correctly", itemType); + _logger.LogWarning("No handler for {itemType} {alias}", itemType, item.Node.GetAlias()); continue; } - handlerPair.Handler.PreCacheFolderKeys(folder, orderedNodes.Select(x => x.Key).ToList()); - options.Callbacks?.Update.Invoke(item.Node.GetAlias(), CalculateProgress(index, total, options.ProgressMin, options.ProgressMax), 100); @@ -87,6 +90,9 @@ public IEnumerable ImportPartial(string folder, uSyncPagedImportOpt return ImportPartial(orderedNodes, options, out total); } + /// + /// perform an import of items from the suppled ordered node list. + /// public IEnumerable ImportPartial(IList orderedNodes, uSyncPagedImportOptions options, out int total) { lock (_importLock) @@ -119,14 +125,14 @@ public IEnumerable ImportPartial(IList orderedNode lastType = itemType; handlerPair = _handlerFactory.GetValidHandlerByTypeName(itemType, syncHandlerOptions); - // special case, blueprints looks like IContent items, except they are slightly different - // so we check for them specifically and get the handler for the entity rather than the object type. - if (item.Node.IsContent() && item.Node.IsBlueprint()) - { - lastType = UdiEntityType.DocumentBlueprint; - handlerPair = _handlerFactory.GetValidHandlerByEntityType(UdiEntityType.DocumentBlueprint); - } - } + // special case, blueprints looks like IContent items, except they are slightly different + // so we check for them specifically and get the handler for the entity rather than the object type. + if (item.Node.IsContent() && item.Node.IsBlueprint()) + { + lastType = UdiEntityType.DocumentBlueprint; + handlerPair = _handlerFactory.GetValidHandlerByEntityType(UdiEntityType.DocumentBlueprint); + } + } if (handlerPair == null) { @@ -366,6 +372,9 @@ public OrderedNodeInfo(string filename, XElement node) /// public XElement Node { get; set; } + /// + /// the Guid key for this item, so we can cache the list of keys + /// public Guid Key { get; set; } /// diff --git a/uSync.BackOffice/SyncHandlers/Handlers/ContentHandler.cs b/uSync.BackOffice/SyncHandlers/Handlers/ContentHandler.cs index 8911f335..88aeaf2e 100644 --- a/uSync.BackOffice/SyncHandlers/Handlers/ContentHandler.cs +++ b/uSync.BackOffice/SyncHandlers/Handlers/ContentHandler.cs @@ -58,6 +58,9 @@ public ContentHandler( this.serializer = syncItemFactory.GetSerializer("ContentSerializer"); } + protected override bool HasChildren(IContent item) + => contentService.HasChildren(item.Id); + /// /// Get child items /// diff --git a/uSync.BackOffice/SyncHandlers/Handlers/MediaHandler.cs b/uSync.BackOffice/SyncHandlers/Handlers/MediaHandler.cs index 7498360b..a4304a3b 100644 --- a/uSync.BackOffice/SyncHandlers/Handlers/MediaHandler.cs +++ b/uSync.BackOffice/SyncHandlers/Handlers/MediaHandler.cs @@ -51,6 +51,9 @@ public MediaHandler( this.mediaService = mediaService; } + protected override bool HasChildren(IMedia item) + => mediaService.HasChildren(item.Id); + /// protected override IEnumerable GetChildItems(IEntity parent) { diff --git a/uSync.BackOffice/SyncHandlers/Interfaces/ISyncHandler.cs b/uSync.BackOffice/SyncHandlers/Interfaces/ISyncHandler.cs index d7af0ae4..88c252f4 100644 --- a/uSync.BackOffice/SyncHandlers/Interfaces/ISyncHandler.cs +++ b/uSync.BackOffice/SyncHandlers/Interfaces/ISyncHandler.cs @@ -76,7 +76,7 @@ public interface ISyncHandler string EntityType { get; } /// - /// The type name of the items hanled (Item.getType().ToString()) + /// The type name of the items handled (Item.getType().ToString()) /// string TypeName { get; } @@ -95,7 +95,7 @@ public interface ISyncHandler /// /// folder to use when exporting /// Handler settings to use for export - /// Callbacks to keep UI uptodate + /// Callbacks to keep UI upto date /// List of actions detailing changes IEnumerable ExportAll(string folder, HandlerSettings settings, SyncUpdateCallback callback); @@ -124,7 +124,7 @@ public interface ISyncHandler /// folder to use when Importing /// Handler settings to use for import /// Force the import even if the settings haven't changed - /// Callbacks to keep UI uptodate + /// Callbacks to keep UI upto date /// List of actions detailing changes IEnumerable ImportAll(string folder, HandlerSettings settings, bool force, SyncUpdateCallback callback); @@ -138,7 +138,7 @@ public interface ISyncHandler /// /// folder to use when reporting /// Handler settings to use for report - /// Callbacks to keep UI uptodate + /// Callbacks to keep UI upto date /// List of actions detailing changes IEnumerable Report(string folder, HandlerSettings settings, SyncUpdateCallback callback); @@ -154,12 +154,12 @@ public interface ISyncHandler IEnumerable ImportSecondPass(uSyncAction action, HandlerSettings settings, uSyncImportOptions options); /// - /// default impimentation, roothandler does do this. + /// default implementation, root handler does do this. /// Udi FindFromNode(XElement node) => null; /// - /// is this a current node (roothandler can do this too) + /// is this a current node (root handler can do this too) /// ChangeType GetItemStatus(XElement node) => ChangeType.NoChange; @@ -169,7 +169,6 @@ public interface ISyncHandler /// /// /// - void PreCacheFolderKeys(string folder, IList keys) - { } + void PreCacheFolderKeys(string folder, IList keys) { } } } diff --git a/uSync.BackOffice/SyncHandlers/SyncHandlerBase.cs b/uSync.BackOffice/SyncHandlers/SyncHandlerBase.cs index 06a57561..69a7c4c4 100644 --- a/uSync.BackOffice/SyncHandlers/SyncHandlerBase.cs +++ b/uSync.BackOffice/SyncHandlers/SyncHandlerBase.cs @@ -11,6 +11,7 @@ using Umbraco.Cms.Core.Models.Entities; using Umbraco.Cms.Core.Services; using Umbraco.Cms.Core.Strings; +using Umbraco.Extensions; using uSync.BackOffice.Configuration; using uSync.BackOffice.Services; @@ -50,6 +51,9 @@ public SyncHandlerBase( this.entityService = entityService; } + protected override bool HasChildren(TObject item) + => entityService.GetChildren(item.Id).Any(); + /// /// given a folder we calculate what items we can remove, becuase they are /// not in one the the files in the folder. @@ -189,7 +193,12 @@ protected override IEnumerable GetChildItems(IEntity parent) /// virtual protected IEnumerable GetChildItems(int parent) { - if (this.itemObjectType != UmbracoObjectTypes.Unknown) + if (this.itemObjectType == UmbracoObjectTypes.Unknown) + return Enumerable.Empty(); + + var cacheKey = $"{GetCacheKeyBase()}_parent_{parent}"; + + return runtimeCache.GetCacheItem(cacheKey, () => { if (parent == -1) { @@ -201,9 +210,7 @@ virtual protected IEnumerable GetChildItems(int parent) // load it, so GetChildren without the object type is quicker. return entityService.GetChildren(parent); } - } - - return Enumerable.Empty(); + }, null); } /// diff --git a/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs b/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs index d7aac35a..c2d920e5 100644 --- a/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs +++ b/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs @@ -868,6 +868,11 @@ virtual public IEnumerable Export(TObject item, string folder, Hand { // only write the file to disk if it should be exported. syncFileService.SaveXElement(attempt.Item, filename); + + if (config.CreateClean && HasChildren(item)) + { + CreateCleanFile(GetItemKey(item), filename); + } } else { @@ -880,6 +885,24 @@ virtual public IEnumerable Export(TObject item, string folder, Hand return uSyncActionHelper.SetAction(attempt, filename, GetItemKey(item), this.Alias).AsEnumerableOfOne(); } + protected virtual bool HasChildren(TObject item) + => true; + + private void CreateCleanFile(Guid key, string filename) + { + if (string.IsNullOrWhiteSpace(filename) || key == Guid.Empty) + return; + + var folder = Path.GetDirectoryName(filename); + var name = Path.GetFileNameWithoutExtension(filename); + + var cleanPath = Path.Combine(folder, $"{name}_clean.config"); + + var node = XElementExtensions.MakeEmpty(key, SyncActionType.Clean, $"clean {name} children"); + node.Add(new XAttribute("itemType", serializer.ItemType)); + syncFileService.SaveXElement(node, cleanPath); + } + #endregion #region Reporting @@ -1764,7 +1787,7 @@ private string GetNameFromFileOrNode(string filename, XElement node) => !string.IsNullOrWhiteSpace(filename) ? filename : node.GetAlias(); - private string GetCacheKeyBase() + protected string GetCacheKeyBase() => $"keycache_{this.Alias}_{Thread.CurrentThread.ManagedThreadId}"; private string PrepCaches() From 9531e9069229c7e52e213e9f3b4212b2a365ff9d Mon Sep 17 00:00:00 2001 From: Kevin Jump Date: Tue, 3 Oct 2023 12:14:08 +0100 Subject: [PATCH 04/11] package update --- uSync.BackOffice.Targets/appsettings-schema.usync.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/uSync.BackOffice.Targets/appsettings-schema.usync.json b/uSync.BackOffice.Targets/appsettings-schema.usync.json index 6c64cb33..278bc8f7 100644 --- a/uSync.BackOffice.Targets/appsettings-schema.usync.json +++ b/uSync.BackOffice.Targets/appsettings-schema.usync.json @@ -248,6 +248,10 @@ "description": "Override the group the handler belongs too.", "default": "" }, + "CreateClean": { + "type": "boolean", + "description": "create a corresponding _clean file for this export \n " + }, "Settings": { "type": "object", "description": "Additional settings for the handler", From e03e8c04d13144b93dc4b835835a7052873bc3e7 Mon Sep 17 00:00:00 2001 From: Kevin Jump Date: Tue, 3 Oct 2023 12:13:03 +0100 Subject: [PATCH 05/11] Fixes #548 --- .../Serializers/ContentSerializer.cs | 18 ++++++++++++----- .../Serializers/ContentSerializerBase.cs | 20 ++++++++++++++++--- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/uSync.Core/Serialization/Serializers/ContentSerializer.cs b/uSync.Core/Serialization/Serializers/ContentSerializer.cs index 7f44d2a2..6ace158c 100644 --- a/uSync.Core/Serialization/Serializers/ContentSerializer.cs +++ b/uSync.Core/Serialization/Serializers/ContentSerializer.cs @@ -402,6 +402,9 @@ protected override uSyncChange HandleTrashedState(IContent item, bool trashed) // if the item is trashed, then the change of it's parent // should restore it (as long as we do a move!) + // if this item is new but in the bin, we can't move it to the bin. + + contentService.Move(item, item.ParentId); // clean out any relations for this item (some versions of Umbraco don't do this on a Move) @@ -412,12 +415,16 @@ protected override uSyncChange HandleTrashedState(IContent item, bool trashed) } else if (trashed && !item.Trashed) { + // not already in the recycle bin? + if (item.ParentId > Constants.System.RecycleBinContent) + { + // clean any relations that may be there (stops an error) + CleanRelations(item, "relateParentDocumentOnDelete"); - // clean any relations that may be there (stops an error) - CleanRelations(item, "relateParentDocumentOnDelete"); + // move to the recycle bin + contentService.MoveToRecycleBin(item); + } - // move to the recycle bin - contentService.MoveToRecycleBin(item); return uSyncChange.Update("Moved to Bin", item.Name, "", "Recycle Bin"); } @@ -432,8 +439,9 @@ protected virtual Attempt DoSaveOrPublish(IContent item, XElement node, return Attempt.Succeed("No Changes"); } + var trashed = item.Trashed || (node.Element("Info")?.Element("Trashed").ValueOrDefault(false) ?? false); var publishedNode = node.Element("Info")?.Element("Published"); - if (!item.Trashed && publishedNode != null) + if (!trashed && publishedNode != null) { var schedules = GetSchedules(node.Element("Info")?.Element("Schedule")); diff --git a/uSync.Core/Serialization/Serializers/ContentSerializerBase.cs b/uSync.Core/Serialization/Serializers/ContentSerializerBase.cs index 0300a0d9..c6c30615 100644 --- a/uSync.Core/Serialization/Serializers/ContentSerializerBase.cs +++ b/uSync.Core/Serialization/Serializers/ContentSerializerBase.cs @@ -394,7 +394,17 @@ protected virtual IEnumerable DeserializeBase(TObject item, XElemen item.Level = nodeLevel; } } - + else // trashed. + { + // we need to set the parent to something, + // or the move will fail. + if (item.ParentId == -1) + { + item.ParentId = item is IContent + ? Constants.System.RecycleBinContent + : Constants.System.RecycleBinMedia; + } + } var key = node.GetKey(); if (key != Guid.Empty && item.Key != key) @@ -679,13 +689,17 @@ protected override Attempt FindOrCreate(XElement node) var alias = node.GetAlias(); - var parentKey = node.Attribute(uSyncConstants.Xml.Parent).ValueOrDefault(Guid.Empty); + var parentKey = node.Element(uSyncConstants.Xml.Info) + ?.Element(uSyncConstants.Xml.Parent) + ?.Attribute(uSyncConstants.Xml.Key) + .ValueOrDefault(Guid.Empty) ?? Guid.Empty; + if (parentKey != Guid.Empty) { item = FindItem(alias, parentKey); if (item != null) return Attempt.Succeed(item); } - + // create var parent = default(TObject); From c9e19c160f4dd372a47bd15b002eab7bcc290a3f Mon Sep 17 00:00:00 2001 From: Kevin Jump Date: Tue, 3 Oct 2023 21:42:31 +0100 Subject: [PATCH 06/11] Config breaking import trashed should default to false. --- uSync.BackOffice/SyncHandlers/Handlers/ContentHandlerBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uSync.BackOffice/SyncHandlers/Handlers/ContentHandlerBase.cs b/uSync.BackOffice/SyncHandlers/Handlers/ContentHandlerBase.cs index aaf06223..ec555713 100644 --- a/uSync.BackOffice/SyncHandlers/Handlers/ContentHandlerBase.cs +++ b/uSync.BackOffice/SyncHandlers/Handlers/ContentHandlerBase.cs @@ -103,7 +103,7 @@ private bool ImportTrashedItem(XElement node, HandlerSettings config) { // unless the setting is explicit we don't import trashed items. var trashed = node.Element("Info")?.Element("Trashed").ValueOrDefault(false); - if (trashed.GetValueOrDefault(false) && !config.GetSetting("ImportTrashed", true)) return false; + if (trashed.GetValueOrDefault(false) && !config.GetSetting("ImportTrashed", false)) return false; return true; } From b475be3f673b6bd18725af081fd376de61c5da6c Mon Sep 17 00:00:00 2001 From: Kevin Jump Date: Wed, 4 Oct 2023 13:12:39 +0100 Subject: [PATCH 07/11] Fix - restoring from recycle box - remain 'deleted' --- .../Serializers/ContentSerializer.cs | 14 ++--- .../Serializers/ContentSerializerBase.cs | 53 ++++++++++++++----- .../Serializers/MediaSerializer.cs | 9 ++-- 3 files changed, 53 insertions(+), 23 deletions(-) diff --git a/uSync.Core/Serialization/Serializers/ContentSerializer.cs b/uSync.Core/Serialization/Serializers/ContentSerializer.cs index 6ace158c..b813e19c 100644 --- a/uSync.Core/Serialization/Serializers/ContentSerializer.cs +++ b/uSync.Core/Serialization/Serializers/ContentSerializer.cs @@ -173,7 +173,8 @@ protected override SyncAttempt DeserializeCore(XElement node, SyncSeri if (node.Element("Info") != null) { var trashed = node.Element("Info").Element("Trashed").ValueOrDefault(false); - details.AddNotNull(HandleTrashedState(item, trashed)); + var restoreParent = node.Element("Info").Element("Trashed").Attribute("Parent").ValueOrDefault(Guid.Empty); + details.AddNotNull(HandleTrashedState(item, trashed, restoreParent)); } details.AddNotNull(DeserializeTemplate(item, node)); @@ -395,22 +396,21 @@ private ContentSchedule FindSchedule(ContentScheduleCollection currentSchedules, } - protected override uSyncChange HandleTrashedState(IContent item, bool trashed) + protected override uSyncChange HandleTrashedState(IContent item, bool trashed, Guid restoreParentKey) { if (!trashed && item.Trashed) { // if the item is trashed, then the change of it's parent // should restore it (as long as we do a move!) - // if this item is new but in the bin, we can't move it to the bin. - - contentService.Move(item, item.ParentId); + var restoreParentId = GetRelationParentId(item, restoreParentKey, Constants.Conventions.RelationTypes.RelateParentDocumentOnDeleteAlias); + contentService.Move(item, restoreParentId); // clean out any relations for this item (some versions of Umbraco don't do this on a Move) - CleanRelations(item, "relateParentDocumentOnDelete"); + CleanRelations(item, Constants.Conventions.RelationTypes.RelateDocumentOnCopyAlias); - return uSyncChange.Update("Restored", item.Name, "Recycle Bin", item.ParentId.ToString()); + return uSyncChange.Update("Restored", item.Name, "Recycle Bin", restoreParentKey.ToString()); } else if (trashed && !item.Trashed) diff --git a/uSync.Core/Serialization/Serializers/ContentSerializerBase.cs b/uSync.Core/Serialization/Serializers/ContentSerializerBase.cs index c6c30615..777abd5c 100644 --- a/uSync.Core/Serialization/Serializers/ContentSerializerBase.cs +++ b/uSync.Core/Serialization/Serializers/ContentSerializerBase.cs @@ -371,20 +371,25 @@ protected virtual IEnumerable DeserializeBase(TObject item, XElemen } } - if (item.ParentId != parentId) + if (!item.Trashed) { - changes.AddUpdate(uSyncConstants.Xml.Parent, item.ParentId, parentId); - logger.LogTrace("{Id} Setting Parent {ParentId}", item.Id, parentId); - item.ParentId = parentId; - } + // we change if its not in the bin, + // if its in the bin it will get fixed by handle trashed state. + if (item.ParentId != parentId) + { + changes.AddUpdate(uSyncConstants.Xml.Parent, item.ParentId, parentId); + logger.LogTrace("{Id} Setting Parent {ParentId}", item.Id, parentId); + item.ParentId = parentId; + } - // the following are calculated (not in the file - // because they might change without this node being saved). - if (item.Path != nodePath) - { - changes.AddUpdate(uSyncConstants.Xml.Path, item.Path, nodePath); - logger.LogDebug("{Id} Setting Path {idPath} was {oldPath}", item.Id, nodePath, item.Path); - item.Path = nodePath; + // the following are calculated (not in the file + // because they might change without this node being saved). + if (item.Path != nodePath) + { + changes.AddUpdate(uSyncConstants.Xml.Path, item.Path, nodePath); + logger.LogDebug("{Id} Setting Path {idPath} was {oldPath}", item.Id, nodePath, item.Path); + item.Path = nodePath; + } } if (item.Level != nodeLevel) @@ -636,7 +641,7 @@ protected uSyncChange HandleSortOrder(TObject item, int sortOrder) return null; } - protected abstract uSyncChange HandleTrashedState(TObject item, bool trashed); + protected abstract uSyncChange HandleTrashedState(TObject item, bool trashed, Guid restoreParent); protected string GetExportValue(object value, IPropertyType propertyType, string culture, string segment) { @@ -996,6 +1001,28 @@ protected void CleanRelations(TObject item, string relationType) } + protected int GetRelationParentId(TObject item, Guid restoreParentKey, string relationType) + { + var parentId = -1; + try + { + var deleteRelations = relationService.GetByChild(item, relationType); + if (deleteRelations.Any()) + parentId = deleteRelations.FirstOrDefault()?.ParentId ?? -1; + + if (parentId != -1) return parentId; + return restoreParentKey == Guid.Empty ? -1 : entityService.Get(restoreParentKey)?.Id ?? -1; + } + catch (Exception ex) + { + // unable to find an existing delete relation. + logger.LogWarning(ex, "Error finding restore relation"); + } + + return -1; + + } + private List GetExcludedProperties(SyncSerializerOptions options) { diff --git a/uSync.Core/Serialization/Serializers/MediaSerializer.cs b/uSync.Core/Serialization/Serializers/MediaSerializer.cs index 92d77ea0..889d3468 100644 --- a/uSync.Core/Serialization/Serializers/MediaSerializer.cs +++ b/uSync.Core/Serialization/Serializers/MediaSerializer.cs @@ -65,7 +65,8 @@ protected override SyncAttempt DeserializeCore(XElement node, SyncSerial if (node.Element("Info") != null) { var trashed = node.Element("Info").Element("Trashed").ValueOrDefault(false); - details.AddNotNull( HandleTrashedState(item, trashed)); + var restoreParent = node.Element("Info").Element("Trashed").Attribute("Parent").ValueOrDefault(Guid.Empty); + details.AddNotNull(HandleTrashedState(item, trashed, restoreParent)); } var propertyAttempt = DeserializeProperties(item, node, options); @@ -103,13 +104,15 @@ protected override SyncAttempt DeserializeCore(XElement node, SyncSerial return SyncAttempt.Succeed(item.Name, item, ChangeType.Import, "", true, propertyAttempt.Result); } - protected override uSyncChange HandleTrashedState(IMedia item, bool trashed) + protected override uSyncChange HandleTrashedState(IMedia item, bool trashed, Guid restoreParentKey) { if (!trashed && item.Trashed) { // if the item is trashed, then moving it back to the parent value // restores it. - _mediaService.Move(item, item.ParentId); + + var restoreParentId = GetRelationParentId(item, restoreParentKey, Constants.Conventions.RelationTypes.RelateParentDocumentOnDeleteAlias); + _mediaService.Move(item, restoreParentId); CleanRelations(item, "relateParentMediaFolderOnDelete"); From 8d48156349ad92181291bd3b90c98f0e846931ba Mon Sep 17 00:00:00 2001 From: Kevin Jump Date: Wed, 4 Oct 2023 13:12:57 +0100 Subject: [PATCH 08/11] Tidy - Load nodes can sometimes be blank --- uSync.BackOffice/Services/uSyncService_Single.cs | 3 +++ .../Serialization/Serializers/ContentTypeBaseSerializer.cs | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/uSync.BackOffice/Services/uSyncService_Single.cs b/uSync.BackOffice/Services/uSyncService_Single.cs index 44f37c9a..6113ee54 100644 --- a/uSync.BackOffice/Services/uSyncService_Single.cs +++ b/uSync.BackOffice/Services/uSyncService_Single.cs @@ -119,6 +119,9 @@ public IEnumerable ImportPartial(IList orderedNode { foreach (var item in orderedNodes.Skip(options.PageNumber * options.PageSize).Take(options.PageSize)) { + if (item.Node == null) + item.Node = XElement.Load(item.FileName); + var itemType = item.Node.GetItemType(); if (!itemType.InvariantEquals(lastType)) { diff --git a/uSync.Core/Serialization/Serializers/ContentTypeBaseSerializer.cs b/uSync.Core/Serialization/Serializers/ContentTypeBaseSerializer.cs index 6ec16d26..281f956e 100644 --- a/uSync.Core/Serialization/Serializers/ContentTypeBaseSerializer.cs +++ b/uSync.Core/Serialization/Serializers/ContentTypeBaseSerializer.cs @@ -1126,7 +1126,7 @@ private IPropertyType GetOrCreateProperty(TObject item, if (dataType == null) { - logger.LogWarning("Cannot find underling DataType {key} {alias} for {property} it is likely you are missing a package?", definitionKey, propertyEditorAlias, alias); + logger.LogWarning("Cannot find underling DataType {key} {alias} for {property} - Either your datatypes are out of sync or you are missing a package?", definitionKey, propertyEditorAlias, alias); return null; } From df15012c75581a6d3e109fe1148e1ea7b9283c5f Mon Sep 17 00:00:00 2001 From: Kevin Jump Date: Wed, 4 Oct 2023 15:05:45 +0100 Subject: [PATCH 09/11] Add comments to remove build warnings. --- .../SyncHandlers/Handlers/ContentHandler.cs | 1 + .../SyncHandlers/Handlers/MediaHandler.cs | 1 + uSync.BackOffice/SyncHandlers/SyncHandlerBase.cs | 1 + uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs | 11 +++++++++++ .../Serializers/ContentSerializerBase.cs | 7 ++++++- 5 files changed, 20 insertions(+), 1 deletion(-) diff --git a/uSync.BackOffice/SyncHandlers/Handlers/ContentHandler.cs b/uSync.BackOffice/SyncHandlers/Handlers/ContentHandler.cs index 88aeaf2e..a64cb808 100644 --- a/uSync.BackOffice/SyncHandlers/Handlers/ContentHandler.cs +++ b/uSync.BackOffice/SyncHandlers/Handlers/ContentHandler.cs @@ -58,6 +58,7 @@ public ContentHandler( this.serializer = syncItemFactory.GetSerializer("ContentSerializer"); } + /// protected override bool HasChildren(IContent item) => contentService.HasChildren(item.Id); diff --git a/uSync.BackOffice/SyncHandlers/Handlers/MediaHandler.cs b/uSync.BackOffice/SyncHandlers/Handlers/MediaHandler.cs index a4304a3b..49fbd247 100644 --- a/uSync.BackOffice/SyncHandlers/Handlers/MediaHandler.cs +++ b/uSync.BackOffice/SyncHandlers/Handlers/MediaHandler.cs @@ -51,6 +51,7 @@ public MediaHandler( this.mediaService = mediaService; } + /// protected override bool HasChildren(IMedia item) => mediaService.HasChildren(item.Id); diff --git a/uSync.BackOffice/SyncHandlers/SyncHandlerBase.cs b/uSync.BackOffice/SyncHandlers/SyncHandlerBase.cs index 69a7c4c4..a9a10092 100644 --- a/uSync.BackOffice/SyncHandlers/SyncHandlerBase.cs +++ b/uSync.BackOffice/SyncHandlers/SyncHandlerBase.cs @@ -51,6 +51,7 @@ public SyncHandlerBase( this.entityService = entityService; } + /// protected override bool HasChildren(TObject item) => entityService.GetChildren(item.Id).Any(); diff --git a/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs b/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs index c2d920e5..e2252510 100644 --- a/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs +++ b/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs @@ -885,6 +885,13 @@ virtual public IEnumerable Export(TObject item, string folder, Hand return uSyncActionHelper.SetAction(attempt, filename, GetItemKey(item), this.Alias).AsEnumerableOfOne(); } + /// + /// does this item have any children ? + /// + /// + /// on items where we can check this (quickly) we can reduce the number of checks we might + /// make on child items or cleaning up where we don't need to. + /// protected virtual bool HasChildren(TObject item) => true; @@ -1787,6 +1794,10 @@ private string GetNameFromFileOrNode(string filename, XElement node) => !string.IsNullOrWhiteSpace(filename) ? filename : node.GetAlias(); + /// + /// get thekey for any caches we might call (thread based cache value) + /// + /// protected string GetCacheKeyBase() => $"keycache_{this.Alias}_{Thread.CurrentThread.ManagedThreadId}"; diff --git a/uSync.Core/Serialization/Serializers/ContentSerializerBase.cs b/uSync.Core/Serialization/Serializers/ContentSerializerBase.cs index 777abd5c..e53bc176 100644 --- a/uSync.Core/Serialization/Serializers/ContentSerializerBase.cs +++ b/uSync.Core/Serialization/Serializers/ContentSerializerBase.cs @@ -641,7 +641,12 @@ protected uSyncChange HandleSortOrder(TObject item, int sortOrder) return null; } - protected abstract uSyncChange HandleTrashedState(TObject item, bool trashed, Guid restoreParent); + [Obsolete("Pass in a restore guid for the parent - should relationships be missing")] + protected virtual uSyncChange HandleTrashedState(TObject item, bool trashed) + => uSyncChange.NoChange($"Member/{item.Name}", item.Name); + + protected virtual uSyncChange HandleTrashedState(TObject item, bool trashed, Guid restoreParent) + => uSyncChange.NoChange($"Member/{item.Name}", item.Name); protected string GetExportValue(object value, IPropertyType propertyType, string culture, string segment) { From 066d99126e5a8bfac084ad2d71a895a2adbd5674 Mon Sep 17 00:00:00 2001 From: Kevin Jump Date: Wed, 4 Oct 2023 15:19:32 +0100 Subject: [PATCH 10/11] use correct alias names for delete relationship types. --- uSync.Core/Serialization/Serializers/ContentSerializer.cs | 6 +++--- uSync.Core/Serialization/Serializers/MediaSerializer.cs | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/uSync.Core/Serialization/Serializers/ContentSerializer.cs b/uSync.Core/Serialization/Serializers/ContentSerializer.cs index b813e19c..03d6da04 100644 --- a/uSync.Core/Serialization/Serializers/ContentSerializer.cs +++ b/uSync.Core/Serialization/Serializers/ContentSerializer.cs @@ -404,11 +404,11 @@ protected override uSyncChange HandleTrashedState(IContent item, bool trashed, G // should restore it (as long as we do a move!) - var restoreParentId = GetRelationParentId(item, restoreParentKey, Constants.Conventions.RelationTypes.RelateParentDocumentOnDeleteAlias); + var restoreParentId = GetRelationParentId(item, restoreParentKey, Constants.Conventions.RelationTypes.RelateParentMediaFolderOnDeleteAlias); contentService.Move(item, restoreParentId); // clean out any relations for this item (some versions of Umbraco don't do this on a Move) - CleanRelations(item, Constants.Conventions.RelationTypes.RelateDocumentOnCopyAlias); + CleanRelations(item, Constants.Conventions.RelationTypes.RelateParentMediaFolderOnDeleteAlias); return uSyncChange.Update("Restored", item.Name, "Recycle Bin", restoreParentKey.ToString()); @@ -419,7 +419,7 @@ protected override uSyncChange HandleTrashedState(IContent item, bool trashed, G if (item.ParentId > Constants.System.RecycleBinContent) { // clean any relations that may be there (stops an error) - CleanRelations(item, "relateParentDocumentOnDelete"); + CleanRelations(item, Constants.Conventions.RelationTypes.RelateParentMediaFolderOnDeleteAlias); // move to the recycle bin contentService.MoveToRecycleBin(item); diff --git a/uSync.Core/Serialization/Serializers/MediaSerializer.cs b/uSync.Core/Serialization/Serializers/MediaSerializer.cs index 889d3468..09ca7b12 100644 --- a/uSync.Core/Serialization/Serializers/MediaSerializer.cs +++ b/uSync.Core/Serialization/Serializers/MediaSerializer.cs @@ -111,17 +111,17 @@ protected override uSyncChange HandleTrashedState(IMedia item, bool trashed, Gui // if the item is trashed, then moving it back to the parent value // restores it. - var restoreParentId = GetRelationParentId(item, restoreParentKey, Constants.Conventions.RelationTypes.RelateParentDocumentOnDeleteAlias); + var restoreParentId = GetRelationParentId(item, restoreParentKey, Constants.Conventions.RelationTypes.RelateParentMediaFolderOnDeleteAlias); _mediaService.Move(item, restoreParentId); - CleanRelations(item, "relateParentMediaFolderOnDelete"); + CleanRelations(item, Constants.Conventions.RelationTypes.RelateParentMediaFolderOnDeleteAlias); return uSyncChange.Update("Restored", item.Name, "Recycle Bin", item.ParentId.ToString()); } else if (trashed && !item.Trashed) { // clean any rouge relations - CleanRelations(item, "relateParentMediaFolderOnDelete"); + CleanRelations(item, Constants.Conventions.RelationTypes.RelateParentMediaFolderOnDeleteAlias); // move to the recycle bin _mediaService.MoveToRecycleBin(item); From 8dfa3a16a0697b8a566e256a49b357483cb0b7be Mon Sep 17 00:00:00 2001 From: Kevin Jump Date: Wed, 4 Oct 2023 15:54:47 +0100 Subject: [PATCH 11/11] Relation names constantes --- uSync.Core/Serialization/Serializers/ContentSerializer.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uSync.Core/Serialization/Serializers/ContentSerializer.cs b/uSync.Core/Serialization/Serializers/ContentSerializer.cs index 03d6da04..acc02213 100644 --- a/uSync.Core/Serialization/Serializers/ContentSerializer.cs +++ b/uSync.Core/Serialization/Serializers/ContentSerializer.cs @@ -404,11 +404,11 @@ protected override uSyncChange HandleTrashedState(IContent item, bool trashed, G // should restore it (as long as we do a move!) - var restoreParentId = GetRelationParentId(item, restoreParentKey, Constants.Conventions.RelationTypes.RelateParentMediaFolderOnDeleteAlias); + var restoreParentId = GetRelationParentId(item, restoreParentKey, Constants.Conventions.RelationTypes.RelateParentDocumentOnDeleteAlias); contentService.Move(item, restoreParentId); // clean out any relations for this item (some versions of Umbraco don't do this on a Move) - CleanRelations(item, Constants.Conventions.RelationTypes.RelateParentMediaFolderOnDeleteAlias); + CleanRelations(item, Constants.Conventions.RelationTypes.RelateParentDocumentOnDeleteAlias); return uSyncChange.Update("Restored", item.Name, "Recycle Bin", restoreParentKey.ToString()); @@ -419,7 +419,7 @@ protected override uSyncChange HandleTrashedState(IContent item, bool trashed, G if (item.ParentId > Constants.System.RecycleBinContent) { // clean any relations that may be there (stops an error) - CleanRelations(item, Constants.Conventions.RelationTypes.RelateParentMediaFolderOnDeleteAlias); + CleanRelations(item, Constants.Conventions.RelationTypes.RelateParentDocumentOnDeleteAlias); // move to the recycle bin contentService.MoveToRecycleBin(item);