Repair order in compositions

Avoid changing the order of entries in compositions for common conversions.
(Discover a problem with System.IO.Compression.ZipArchive changing order of entries)

explore/2020-02-02.memory-leak/demo-memory-leak/TestSetup.cs

@@ -8,6 +8,7 @@ public class TestSetup
         static public IReadOnlyCollection<(string filePath, byte[] fileContent)> GetElmAppFromFilePath(
             string filePath) =>
             ElmApp.FilesFilteredForElmApp(Filesystem.GetAllFilesFromDirectory(filePath))
+            .OrderBy(file => file.filePath)
             .ToImmutableList();
     }
 }

implement/PersistentProcess/PersistentProcess.Common/Composition.cs

@@ -200,9 +200,9 @@ static public Component FromTree(TreeComponent tree)
             };
         }
 
-        static public TreeComponent TreeFromSetOfBlobsWithCommonFilePath(
+        static public TreeComponent SortedTreeFromSetOfBlobsWithCommonFilePath(
             IEnumerable<(string path, IImmutableList<byte> blobContent)> blobsWithPath) =>
-            TreeFromSetOfBlobs(
+            SortedTreeFromSetOfBlobs(
                 blobsWithPath.Select(blobWithPath =>
                 {
                     var pathComponents =
@@ -214,75 +214,74 @@ static public TreeComponent TreeFromSetOfBlobsWithCommonFilePath(
                 })
             );
 
-        static public TreeComponent TreeFromSetOfBlobsWithCommonFilePath(
+        static public TreeComponent SortedTreeFromSetOfBlobsWithCommonFilePath(
             IEnumerable<(string path, byte[] blobContent)> blobsWithPath) =>
-            TreeFromSetOfBlobsWithCommonFilePath(
+            SortedTreeFromSetOfBlobsWithCommonFilePath(
                 blobsWithPath.Select(blobWithPath => (blobWithPath.path, (IImmutableList<byte>)blobWithPath.blobContent.ToImmutableList())));
 
-        static public TreeComponent TreeFromSetOfBlobs<PathT>(
+        static public TreeComponent SortedTreeFromSetOfBlobs<PathT>(
             IEnumerable<(IImmutableList<PathT> path, IImmutableList<byte> blobContent)> blobsWithPath,
             Func<PathT, IImmutableList<byte>> mapPathComponent) =>
-            TreeFromSetOfBlobs(
+            SortedTreeFromSetOfBlobs(
                 blobsWithPath.Select(blobWithPath =>
                     (path: (IImmutableList<IImmutableList<byte>>)blobWithPath.path.Select(mapPathComponent).ToImmutableList(),
                     blobContent: blobWithPath.blobContent)));
 
-        static public TreeComponent TreeFromSetOfBlobsWithStringPath(
+        static public TreeComponent SortedTreeFromSetOfBlobsWithStringPath(
             IEnumerable<(IImmutableList<string> path, IImmutableList<byte> blobContent)> blobsWithPath) =>
-            TreeFromSetOfBlobs(
+            SortedTreeFromSetOfBlobs(
                 blobsWithPath, pathComponent => System.Text.Encoding.UTF8.GetBytes(pathComponent).ToImmutableList());
 
-        static public TreeComponent TreeFromSetOfBlobsWithStringPath(
+        static public TreeComponent SortedTreeFromSetOfBlobsWithStringPath(
             IReadOnlyDictionary<IImmutableList<string>, IImmutableList<byte>> blobsWithPath) =>
-            TreeFromSetOfBlobsWithStringPath(
+            SortedTreeFromSetOfBlobsWithStringPath(
                 blobsWithPath.Select(pathAndBlobContent => (path: pathAndBlobContent.Key, blobContent: pathAndBlobContent.Value)));
 
-        static public TreeComponent TreeFromSetOfBlobs(
+        static public TreeComponent SortedTreeFromSetOfBlobs(
             IEnumerable<(IImmutableList<IImmutableList<byte>> path, IImmutableList<byte> blobContent)> blobsWithPath) =>
             new TreeComponent
             {
-                TreeContent = TreeContentFromSetOfBlobs(blobsWithPath)
+                TreeContent = SortedTreeContentFromSetOfBlobs(blobsWithPath)
             };
 
-        static public IImmutableList<(IImmutableList<byte> name, TreeComponent obj)> TreeContentFromSetOfBlobs(
-            IEnumerable<(IImmutableList<IImmutableList<byte>> path, IImmutableList<byte> blobContent)> blobsWithPath)
+        static public IImmutableList<(IImmutableList<byte> name, TreeComponent obj)> SortedTreeContentFromSetOfBlobs(
+            IEnumerable<(IImmutableList<IImmutableList<byte>> path, IImmutableList<byte> blobContent)> blobsWithPath) =>
+            blobsWithPath
+            .Aggregate(
+                (IImmutableList<(IImmutableList<byte> name, TreeComponent obj)>)
+                ImmutableList<(IImmutableList<byte> name, TreeComponent obj)>.Empty,
+                (intermediateResult, nextBlob) => SetBlobAtPathSorted(intermediateResult, nextBlob.path, nextBlob.blobContent));
+
+        static public IImmutableList<(IImmutableList<byte> name, TreeComponent obj)> SetBlobAtPathSorted(
+            IImmutableList<(IImmutableList<byte> name, TreeComponent obj)> treeContentBefore,
+            IImmutableList<IImmutableList<byte>> path,
+            IImmutableList<byte> blobContent)
         {
-            var groupedByDirectory =
-                blobsWithPath
-                .GroupBy(
-                    pathAndContent => 1 < pathAndContent.path.Count ? pathAndContent.path.First() : null,
-                    new ByteListComparer())
-                .ToImmutableList();
+            var pathFirstElement = path.First();
 
-            var currentLevelBlobs =
-                groupedByDirectory
-                .FirstOrDefault(group => group.Key == null)
-                .EmptyIfNull()
-                .Select(pathAndContent =>
-                    (name: pathAndContent.path.First(),
-                    blobContent: new TreeComponent { BlobContent = pathAndContent.blobContent.ToImmutableList() }))
-                .OrderBy(nameAndContent => nameAndContent.name, new ByteListComparer())
-                .ToImmutableList();
+            var componentBefore =
+                treeContentBefore.FirstOrDefault(c => c.name.SequenceEqual(pathFirstElement)).obj;
 
-            var subTrees =
-                groupedByDirectory
-                .Where(group => group.Key != null)
-                .Select(directoryGroup =>
+            var component =
+                path.Count < 2
+                ?
+                new TreeComponent { BlobContent = blobContent }
+                :
+                new TreeComponent
                 {
-                    var blobsWithRelativePaths =
-                        directoryGroup.Select(pathAndContent =>
-                            (path: (IImmutableList<IImmutableList<byte>>)pathAndContent.path.Skip(1).ToImmutableList(),
-                            blobContent: pathAndContent.blobContent));
+                    TreeContent =
+                        SetBlobAtPathSorted(
+                            componentBefore?.TreeContent ?? ImmutableList<(IImmutableList<byte> name, TreeComponent obj)>.Empty,
+                            path.RemoveAt(0),
+                            blobContent)
+                };
 
-                    return (name: (IImmutableList<byte>)directoryGroup.Key.ToImmutableList(), content: new TreeComponent
-                    {
-                        TreeContent = TreeContentFromSetOfBlobs(blobsWithRelativePaths)
-                    });
-                })
-                .OrderBy(nameAndContent => nameAndContent.name, new ByteListComparer())
+            return
+                treeContentBefore
+                .RemoveAll(c => ByteListComparer.CompareStatic(c.name, pathFirstElement) == 0)
+                .Add((pathFirstElement, component))
+                .OrderBy(c => c.name, new ByteListComparer())
                 .ToImmutableList();
-
-            return currentLevelBlobs.AddRange(subTrees).ToImmutableList();
         }
 
         static public Result<String, Component> Deserialize(
@@ -456,7 +455,10 @@ public class ParseAsTreeResult : Result<IImmutableList<(int index, IImmutableLis
 
         public class ByteListComparer : IComparer<IReadOnlyList<byte>>, IEqualityComparer<IReadOnlyList<byte>>
         {
-            public int Compare(IReadOnlyList<byte> x, IReadOnlyList<byte> y)
+            public int Compare(IReadOnlyList<byte> x, IReadOnlyList<byte> y) =>
+                CompareStatic(x, y);
+
+            static public int CompareStatic(IReadOnlyList<byte> x, IReadOnlyList<byte> y)
             {
                 if (x == null && y == null)
                     return 0;

implement/PersistentProcess/PersistentProcess.Common/ElmApp.cs

@@ -52,9 +52,11 @@ static public bool FilePathMatchesPatternOfFilesInElmApp(string filePath) =>
             .Where(file => 0 < file.filePath?.Length && FilePathMatchesPatternOfFilesInElmApp(file.filePath));
 
         static public IImmutableDictionary<IImmutableList<string>, IImmutableList<byte>> ToFlatDictionaryWithPathComparer(
-            IEnumerable<(IImmutableList<string> filePath, IImmutableList<byte> fileContent)> fileList) =>
-            fileList.ToImmutableDictionary(entry => entry.filePath, entry => entry.fileContent)
-            .WithComparers(EnumerableExtension.EqualityComparer<string>());
+            IEnumerable<(IImmutableList<string> filePath, IImmutableList<byte> fileContent)> filesBeforeSorting) =>
+            filesBeforeSorting.ToImmutableSortedDictionary(
+                entry => entry.filePath,
+                entry => entry.fileContent,
+                EnumerableExtension.Comparer<IImmutableList<string>>());
 
         static public IImmutableDictionary<IImmutableList<string>, IImmutableList<byte>> AsCompletelyLoweredElmApp(
             IImmutableDictionary<IImmutableList<string>, IImmutableList<byte>> sourceFiles,

implement/PersistentProcess/PersistentProcess.Common/EnumerableExtension.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
 using System.Linq;
 
 namespace Kalmit
@@ -15,7 +16,9 @@ static public IEnumerable<T> WhereNotNull<T>(this IEnumerable<T> orig) where T :
         static public IEnumerable<T> EmptyIfNull<T>(this IEnumerable<T> orig) =>
             orig ?? Array.Empty<T>();
 
-        static public IEqualityComparer<IEnumerable<T>> EqualityComparer<T>() => new IEnumerableComparer<T>();
+        static public IEqualityComparer<IEnumerable<T>> EqualityComparer<T>() => new IEnumerableEqualityComparer<T>();
+
+        static public IComparer<T> Comparer<T>() where T : IEnumerable<IComparable> => new IEnumerableComparer<T>();
 
         // From https://github.com/morelinq/MoreLINQ/blob/07bd0861658b381ce97c8b44d3b9f2cd3c9bf769/MoreLinq/TakeUntil.cs
         static public IEnumerable<TSource> TakeUntil<TSource>(this IEnumerable<TSource> source, Func<TSource, bool> predicate)
@@ -34,7 +37,7 @@ static public IEnumerable<TSource> TakeUntil<TSource>(this IEnumerable<TSource>
             }
         }
 
-        class IEnumerableComparer<T> : IEqualityComparer<IEnumerable<T>>
+        class IEnumerableEqualityComparer<T> : IEqualityComparer<IEnumerable<T>>
         {
             public bool Equals(IEnumerable<T> x, IEnumerable<T> y)
             {
@@ -43,5 +46,48 @@ public bool Equals(IEnumerable<T> x, IEnumerable<T> y)
 
             public int GetHashCode(IEnumerable<T> obj) => 0;
         }
+
+        class IEnumerableComparer<T> : IComparer<T> where T : IEnumerable<IComparable>
+        {
+            public int Compare([AllowNull] IEnumerable<IComparable> x, [AllowNull] IEnumerable<IComparable> y)
+            {
+                if (x == y)
+                    return 0;
+
+                if (x == null)
+                    return -1;
+
+                if (y == null)
+                    return 1;
+
+                var xEnumerator = x.GetEnumerator();
+                var yEnumerator = y.GetEnumerator();
+
+                while (true)
+                {
+                    var xHasCurrent = xEnumerator.MoveNext();
+                    var yHasCurrent = yEnumerator.MoveNext();
+
+                    if (!xHasCurrent && !yHasCurrent)
+                        return 0;
+
+                    if (!xHasCurrent)
+                        return -1;
+
+                    if (!yHasCurrent)
+                        return 1;
+
+                    var currentComparison = xEnumerator.Current.CompareTo(yEnumerator.Current);
+
+                    if (currentComparison != 0)
+                        return currentComparison;
+                }
+            }
+
+            public int Compare([AllowNull] T x, [AllowNull] T y)
+            {
+                return Compare((IEnumerable<IComparable>)x, (IEnumerable<IComparable>)y);
+            }
+        }
     }
 }

implement/PersistentProcess/PersistentProcess.Common/LoadFromLocalFilesystem.cs

@@ -1,34 +1,26 @@
 using System.Collections.Immutable;
 using System.IO;
 using System.Linq;
-using System.Text;
 
 namespace Kalmit
 {
     static public class LoadFromLocalFilesystem
     {
-        static public Composition.TreeComponent LoadTreeFromPath(string path)
+        static public Composition.TreeComponent LoadSortedTreeFromPath(string path)
         {
             if (File.Exists(path))
                 return new Composition.TreeComponent { BlobContent = File.ReadAllBytes(path).ToImmutableList() };
 
             if (!Directory.Exists(path))
                 return null;
 
-            var treeEntries =
-                Directory.EnumerateFileSystemEntries(path)
-                .Select(fileSystemEntry =>
-                {
-                    var name = (IImmutableList<byte>)Encoding.UTF8.GetBytes(Path.GetRelativePath(path, fileSystemEntry)).ToImmutableList();
-
-                    return (name, LoadTreeFromPath(fileSystemEntry));
-                })
+            var blobs =
+                Filesystem.GetAllFilesFromDirectory(path)
+                .Select(file => (path: (System.Collections.Immutable.IImmutableList<string>)file.name.Split('/', '\\').ToImmutableList(), content: file.content))
                 .ToImmutableList();
 
-            return new Composition.TreeComponent
-            {
-                TreeContent = treeEntries,
-            };
+            return
+                Composition.SortedTreeFromSetOfBlobsWithStringPath(blobs);
         }
     }
 }

implement/PersistentProcess/PersistentProcess.Common/LoadFromPath.cs

@@ -23,7 +23,7 @@ static public class LoadFromPath
             else
             {
                 return Result<string, (TreeComponent tree, bool comesFromLocalFilesystem)>.ok(
-                    (tree: LoadFromLocalFilesystem.LoadTreeFromPath(path), comesFromLocalFilesystem: true));
+                    (tree: LoadFromLocalFilesystem.LoadSortedTreeFromPath(path), comesFromLocalFilesystem: true));
             }
         }
     }

implement/PersistentProcess/PersistentProcess.Common/ZipArchive.cs

@@ -6,6 +6,9 @@
 
 namespace Kalmit
 {
+    /*
+    2020-07-16 Discovered: The roundtrip over `ZipArchiveFromEntries` and `EntriesFromZipArchive` changed the order of entries!
+    */
     static public class ZipArchive
     {
         /// <summary>

implement/PersistentProcess/PersistentProcess.Test/Common.cs

@@ -76,6 +76,7 @@ static public IImmutableDictionary<IImmutableList<string>, IImmutableList<byte>>
                 ElmApp.ToFlatDictionaryWithPathComparer(
                     ElmApp.FilesFilteredForElmApp(
                         Filesystem.GetAllFilesFromDirectory(Path.Combine(PathToExampleElmApps, exampleName)))
+                    .OrderBy(file => file.filePath)
                     .Select(filePathAndContent => ((IImmutableList<string>)filePathAndContent.filePath.Split(new[] { '/', '\\' }).ToImmutableList(), filePathAndContent.fileContent)));
 
         static public IImmutableDictionary<IImmutableList<string>, IImmutableList<byte>> AsLoweredElmApp(

implement/PersistentProcess/PersistentProcess.Test/TestComposition.cs

@@ -172,7 +172,7 @@ public void Composition_from_file_tree()
             foreach (var testCase in testCases)
             {
                 var asComposition = Composition.FromTree(
-                    Composition.TreeFromSetOfBlobsWithCommonFilePath(testCase.input));
+                    Composition.SortedTreeFromSetOfBlobsWithCommonFilePath(testCase.input));
 
                 Assert.AreEqual(testCase.expectedOutput, asComposition);
             }

implement/PersistentProcess/PersistentProcess.Test/TestElmEvaluation.cs

@@ -21,7 +21,7 @@ public void TestElmEvaluationScenarios()
 
                 try
                 {
-                    var appCodeTree = LoadFromLocalFilesystem.LoadTreeFromPath(Path.Combine(scenarioDirectory, "app-code"));
+                    var appCodeTree = LoadFromLocalFilesystem.LoadSortedTreeFromPath(Path.Combine(scenarioDirectory, "app-code"));
 
                     var expectedValueFile = File.ReadAllBytes(Path.Combine(scenarioDirectory, "expected-value.json"));
 

implement/PersistentProcess/PersistentProcess.Test/TestElmWebAppHttpServer.cs

@@ -10,7 +10,7 @@ public class TestElmWebAppHttpServer
     {
         static public Composition.Component AppConfigComponentFromFiles(
             IImmutableDictionary<IImmutableList<string>, IImmutableList<byte>> appFiles) =>
-            Composition.FromTree(Composition.TreeFromSetOfBlobsWithStringPath(appFiles));
+            Composition.FromTree(Composition.SortedTreeFromSetOfBlobsWithStringPath(appFiles));
 
         static public Composition.Component CounterWebApp =>
             AppConfigComponentFromFiles(TestSetup.CounterElmWebApp);

implement/PersistentProcess/PersistentProcess.Test/TestModeledInElm.cs

@@ -23,6 +23,7 @@ static IImmutableDictionary<IImmutableList<string>, IImmutableList<byte>> GetLow
                 ElmApp.ToFlatDictionaryWithPathComparer(
                     ElmApp.FilesFilteredForElmApp(
                         Filesystem.GetAllFilesFromDirectory(FilePathStringFromPath(directoryPath)))
+                        .OrderBy(file => file.filePath)
                         .Select(filePathAndContent => ((IImmutableList<string>)filePathAndContent.filePath.Split(new[] { '/', '\\' }).ToImmutableList(), filePathAndContent.fileContent)));
 
             return

implement/PersistentProcess/PersistentProcess.Test/TestWebHost.cs

@@ -112,7 +112,7 @@ public void Web_host_serves_static_content_from_source_file()
                 defaultAppSourceFiles.SetItem(demoSourceFilePath, staticContent);
 
             var webAppSource =
-                Composition.FromTree(Composition.TreeFromSetOfBlobsWithStringPath(webAppSourceFiles));
+                Composition.FromTree(Composition.SortedTreeFromSetOfBlobsWithStringPath(webAppSourceFiles));
 
             using (var testSetup = WebHostAdminInterfaceTestSetup.Setup(deployAppConfigAndInitElmState: webAppSource))
             {
@@ -163,7 +163,7 @@ void letTimePassInPersistentProcessHost(TimeSpan amount) =>
                     });
 
             using (var testSetup = WebHostAdminInterfaceTestSetup.Setup(
-                deployAppConfigAndInitElmState: Composition.FromTree(Composition.TreeFromSetOfBlobsWithStringPath(webAppConfig)),
+                deployAppConfigAndInitElmState: Composition.FromTree(Composition.SortedTreeFromSetOfBlobsWithStringPath(webAppConfig)),
                 persistentProcessHostDateTime: () => persistentProcessHostDateTime))
             {
                 IEnumerable<string> EnumerateStoredProcessEventsHttpRequestsBodies() =>
@@ -569,7 +569,7 @@ public void Web_host_supports_deploy_app_config_and_init_elm_app_state()
             var webAppConfigZipArchive = ZipArchive.ZipArchiveFromEntries(TestSetup.CounterElmWebApp);
 
             var webAppConfigTree =
-                Composition.TreeFromSetOfBlobsWithCommonFilePath(
+                Composition.SortedTreeFromSetOfBlobsWithCommonFilePath(
                     ZipArchive.EntriesFromZipArchive(webAppConfigZipArchive).ToImmutableList());
 
             using (var testSetup = WebHostAdminInterfaceTestSetup.Setup())
@@ -592,7 +592,7 @@ public void Web_host_supports_deploy_app_config_and_init_elm_app_state()
                         var getAppResponseContent = getAppConfigResponse.Content.ReadAsByteArrayAsync().Result;
 
                         var responseAppConfigTree =
-                            Composition.TreeFromSetOfBlobsWithCommonFilePath(
+                            Composition.SortedTreeFromSetOfBlobsWithCommonFilePath(
                                 ZipArchive.EntriesFromZipArchive(getAppResponseContent).ToImmutableList());
 
                         CollectionAssert.AreEqual(

implement/PersistentProcess/PersistentProcess.WebHost/Program.cs

@@ -2,6 +2,6 @@ namespace Kalmit.PersistentProcess.WebHost
 {
     public class Program
     {
-        static public string AppVersionId => "2020-07-11";
+        static public string AppVersionId => "2020-07-16";
     }
 }

implement/PersistentProcess/PersistentProcess.WebHost/StartupAdminInterface.cs

@@ -375,7 +375,7 @@ IWebHost buildWebHost()
                         }
 
                         var appConfigTree =
-                            Composition.TreeFromSetOfBlobsWithCommonFilePath(
+                            Composition.SortedTreeFromSetOfBlobsWithCommonFilePath(
                                 ZipArchive.EntriesFromZipArchive(webAppConfigZipArchive));
 
                         var appConfigComponent = Composition.FromTree(appConfigTree);