Make migrations easier with database store replication default

Expand the Docker image to default to replicate from the previously used default storage location in cases where the new store container is empty.
pine-vm · Apr 19, 2024 · 67658db · 67658db
1 parent 55b1a77
commit 67658db
Show file tree

Hide file tree

Showing 7 changed files with 140 additions and 46 deletions.
diff --git a/implement/Dockerfile b/implement/Dockerfile
@@ -22,10 +22,15 @@ RUN apt install -y git
 
 COPY ./example-apps/docker-image-default-app /docker-image-default-app/
 
-RUN dotnet "/pine/dotnet/pine.dll"  deploy  /docker-image-default-app/  /pine/process-store  --init-app-state
+RUN dotnet "/pine/dotnet/pine.dll"  deploy  /docker-image-default-app/  /pine-vm/process-store  --init-app-state
 
 WORKDIR /pine
 
-ENTRYPOINT ["dotnet", "/pine/dotnet/pine.dll", "run-server", "--process-store=/pine/process-store"]
+# Mounting a docker volume can shadow the previous state of the process store:
+#
+# docker  volume  create  docker-volume-name
+# docker  run  --mount 'source=docker-volume-name,destination=/pine-vm/process-store'  ....
+
+ENTRYPOINT ["dotnet", "/pine/dotnet/pine.dll", "run-server", "--process-store=/pine-vm/process-store", "--process-store-readonly=/elm-time/process-store"]
 
 # ENV APPSETTING_adminPassword="password-for-admin-interface"
diff --git a/implement/pine/Pine/FileStore.cs b/implement/pine/Pine/FileStore.cs
@@ -289,40 +289,3 @@ public class EmptyFileStoreReader : IFileStoreReader
     public IEnumerable<IImmutableList<string>> ListFilesInDirectory(IImmutableList<string> directoryPath) =>
         [];
 }
-
-public static class FileStoreExtension
-{
-    public static IEnumerable<IImmutableList<string>> ListFiles(this IFileStoreReader fileStore) =>
-        fileStore.ListFilesInDirectory(ImmutableList<string>.Empty);
-
-    public static IFileStoreReader WithMappedPath(
-        this IFileStoreReader originalFileStore, Func<IImmutableList<string>, IImmutableList<string>> pathMap) =>
-        new DelegatingFileStoreReader
-        (
-            GetFileContentDelegate: originalPath => originalFileStore.GetFileContent(pathMap(originalPath)),
-            ListFilesInDirectoryDelegate: originalPath => originalFileStore.ListFilesInDirectory(pathMap(originalPath))
-        );
-
-    public static IFileStoreReader ForSubdirectory(this IFileStoreReader originalFileStore, string directoryName) =>
-        ForSubdirectory(originalFileStore, ImmutableList.Create(directoryName));
-
-    public static IFileStoreReader ForSubdirectory(
-        this IFileStoreReader originalFileStore, IEnumerable<string> directoryPath) =>
-        WithMappedPath(originalFileStore, originalPath => originalPath.InsertRange(0, directoryPath));
-
-    public static IFileStoreWriter WithMappedPath(
-        this IFileStoreWriter originalFileStore, Func<IImmutableList<string>, IImmutableList<string>> pathMap) =>
-        new DelegatingFileStoreWriter
-        (
-            SetFileContentDelegate: pathAndFileContent => originalFileStore.SetFileContent(pathMap(pathAndFileContent.path), pathAndFileContent.fileContent),
-            AppendFileContentDelegate: pathAndFileContent => originalFileStore.AppendFileContent(pathMap(pathAndFileContent.path), pathAndFileContent.fileContent),
-            DeleteFileDelegate: originalPath => originalFileStore.DeleteFile(pathMap(originalPath))
-        );
-
-    public static IFileStoreWriter ForSubdirectory(this IFileStoreWriter originalFileStore, string directoryName) =>
-        ForSubdirectory(originalFileStore, ImmutableList.Create(directoryName));
-
-    public static IFileStoreWriter ForSubdirectory(
-        this IFileStoreWriter originalFileStore, IEnumerable<string> directoryPath) =>
-        WithMappedPath(originalFileStore, originalPath => originalPath.InsertRange(0, directoryPath));
-}
diff --git a/implement/pine/Pine/FileStoreExtension.cs b/implement/pine/Pine/FileStoreExtension.cs
@@ -0,0 +1,98 @@
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+
+namespace Pine;
+
+public static class FileStoreExtension
+{
+    public static IEnumerable<IImmutableList<string>> ListFiles(this IFileStoreReader fileStore) =>
+        fileStore.ListFilesInDirectory([]);
+
+    public static IFileStoreReader WithMappedPath(
+        this IFileStoreReader originalFileStore, Func<IImmutableList<string>, IImmutableList<string>> pathMap) =>
+        new DelegatingFileStoreReader
+        (
+            GetFileContentDelegate: originalPath => originalFileStore.GetFileContent(pathMap(originalPath)),
+            ListFilesInDirectoryDelegate: originalPath => originalFileStore.ListFilesInDirectory(pathMap(originalPath))
+        );
+
+    public static IFileStoreReader ForSubdirectory(this IFileStoreReader originalFileStore, string directoryName) =>
+        ForSubdirectory(originalFileStore, ImmutableList.Create(directoryName));
+
+    public static IFileStoreReader ForSubdirectory(
+        this IFileStoreReader originalFileStore, IEnumerable<string> directoryPath) =>
+        WithMappedPath(originalFileStore, originalPath => originalPath.InsertRange(0, directoryPath));
+
+    public static IFileStoreWriter WithMappedPath(
+        this IFileStoreWriter originalFileStore, Func<IImmutableList<string>, IImmutableList<string>> pathMap) =>
+        new DelegatingFileStoreWriter
+        (
+            SetFileContentDelegate:
+            pathAndFileContent =>
+            originalFileStore.SetFileContent(pathMap(pathAndFileContent.path), pathAndFileContent.fileContent),
+
+            AppendFileContentDelegate:
+            pathAndFileContent =>
+            originalFileStore.AppendFileContent(pathMap(pathAndFileContent.path), pathAndFileContent.fileContent),
+
+            DeleteFileDelegate:
+            originalPath => originalFileStore.DeleteFile(pathMap(originalPath))
+        );
+
+    public static IFileStoreWriter ForSubdirectory(this IFileStoreWriter originalFileStore, string directoryName) =>
+        ForSubdirectory(originalFileStore, ImmutableList.Create(directoryName));
+
+    public static IFileStoreWriter ForSubdirectory(
+        this IFileStoreWriter originalFileStore, IEnumerable<string> directoryPath) =>
+        WithMappedPath(originalFileStore, originalPath => originalPath.InsertRange(0, directoryPath));
+
+    public static IFileStore MergeReader(
+        this IFileStore primary,
+        IFileStoreReader secondary,
+        bool promoteOnReadFileContentFromSecondary)
+    {
+        ReadOnlyMemory<byte>? GetFileContent(IImmutableList<string> path)
+        {
+            var primaryContent = primary.GetFileContent(path);
+
+            if (primaryContent is not null)
+            {
+                return primaryContent;
+            }
+
+            var secondaryContent = secondary.GetFileContent(path);
+
+            if (secondaryContent.HasValue && promoteOnReadFileContentFromSecondary)
+            {
+                primary.SetFileContent(path, secondaryContent.Value);
+            }
+
+            return secondaryContent;
+        }
+
+        var mergedReader =
+            new DelegatingFileStoreReader
+            (
+                GetFileContentDelegate:
+                GetFileContent,
+
+                ListFilesInDirectoryDelegate:
+                path => [.. secondary.ListFilesInDirectory(path), .. primary.ListFilesInDirectory(path)]
+            );
+
+        return new FileStoreFromWriterAndReader(primary, mergedReader);
+    }
+
+    public static IFileStoreReader MergeReader(
+        this IFileStoreReader primary,
+        IFileStoreReader secondary) =>
+        new DelegatingFileStoreReader
+        (
+            GetFileContentDelegate:
+            path => primary.GetFileContent(path) ?? secondary.GetFileContent(path),
+
+            ListFilesInDirectoryDelegate:
+            path => [.. secondary.ListFilesInDirectory(path), .. primary.ListFilesInDirectory(path)]
+        );
+}
diff --git a/implement/pine/Program.cs b/implement/pine/Program.cs
@@ -18,7 +18,7 @@ namespace ElmTime;
 
 public class Program
 {
-    public static string AppVersionId => "0.3.0";
+    public static string AppVersionId => "0.3.1";
 
     private static int AdminInterfaceDefaultPort => 4000;
 
@@ -280,7 +280,18 @@ private static CommandLineApplication AddRunServerCommand(
 
             var adminUrlsDefault = "http://*:" + AdminInterfaceDefaultPort;
 
-            var processStoreOption = runServerCommand.Option("--process-store", "Directory in the file system to contain the process store.", CommandOptionType.SingleValue);
+            var processStoreOption =
+            runServerCommand.Option(
+                "--process-store",
+                "Directory in the file system to contain the process store.",
+                CommandOptionType.SingleValue);
+
+            var processStoreReadonlyOption =
+            runServerCommand.Option(
+                "--process-store-readonly",
+                "If the primary process store is empty at startup, the system will try to replicate from this location.",
+                CommandOptionType.SingleValue);
+
             var deletePreviousProcessOption = runServerCommand.Option("--delete-previous-process", "Delete the previous backend process found in the given store. If you don't use this option, the server restores the process from the persistent store on startup.", CommandOptionType.NoValue);
             var adminUrlsOption = runServerCommand.Option("--admin-urls", "URLs for the admin interface. The default is " + adminUrlsDefault + ".", CommandOptionType.SingleValue);
             var adminPasswordOption = runServerCommand.Option("--admin-password", "Password for the admin interface at '--admin-urls'.", CommandOptionType.SingleValue);
@@ -307,6 +318,7 @@ private static CommandLineApplication AddRunServerCommand(
 
                 var webHost = RunServer.BuildWebHostToRunServer(
                     processStorePath: processStorePath,
+                    processStoreReadonlyPath: processStoreReadonlyOption.Value(),
                     adminInterfaceUrls: adminInterfaceUrls,
                     adminPassword: adminPasswordOption.Value(),
                     publicAppUrls: publicAppUrls,

diff --git a/implement/pine/RunServer.cs b/implement/pine/RunServer.cs
@@ -16,6 +16,7 @@ public class RunServer
 {
     public static IWebHost BuildWebHostToRunServer(
         string? processStorePath,
+        string? processStoreReadonlyPath,
         string? adminInterfaceUrls,
         string? adminPassword,
         IReadOnlyList<string>? publicAppUrls,
@@ -76,6 +77,16 @@ IFileStore buildProcessStoreFileStore()
 
         var processStoreFileStore = buildProcessStoreFileStore();
 
+        if (processStoreReadonlyPath is not null)
+        {
+            Console.WriteLine("Merging read-only process store from '" + processStoreReadonlyPath + "'.");
+
+            processStoreFileStore =
+                processStoreFileStore?.MergeReader(
+                    new FileStoreFromSystemIOFile(processStoreReadonlyPath),
+                    promoteOnReadFileContentFromSecondary: true);
+        }
+
         if (copyProcess is not null)
         {
             var copyFiles =
@@ -90,10 +101,14 @@ IFileStore buildProcessStoreFileStore()
         var javaScriptEngineFactory =
             elmEngineType switch
             {
-                ElmInteractive.ElmEngineType.JavaScript_Jint => JavaScriptEngineJintOptimizedForElmApps.Create,
-                ElmInteractive.ElmEngineType.JavaScript_V8 => new Func<IJavaScriptEngine>(JavaScriptEngineFromJavaScriptEngineSwitcher.ConstructJavaScriptEngine),
+                ElmInteractive.ElmEngineType.JavaScript_Jint =>
+                JavaScriptEngineJintOptimizedForElmApps.Create,
+
+                ElmInteractive.ElmEngineType.JavaScript_V8 =>
+                new Func<IJavaScriptEngine>(JavaScriptEngineFromJavaScriptEngineSwitcher.ConstructJavaScriptEngine),
 
-                object other => throw new NotImplementedException("Engine type not implemented here: " + other)
+                object other =>
+                throw new NotImplementedException("Engine type not implemented here: " + other)
             };
 
         if (deployApp is not null)

diff --git a/implement/pine/Test/SelfTest.cs b/implement/pine/Test/SelfTest.cs
@@ -72,6 +72,7 @@ public static int RunWebServerTest()
         using var webHost =
             RunServer.BuildWebHostToRunServer(
                 processStorePath: null,
+                processStoreReadonlyPath: null,
                 adminInterfaceUrls: null,
                 adminPassword: null,
                 publicAppUrls: ["http://localhost:" + serverHttpPort],

diff --git a/implement/pine/pine.csproj b/implement/pine/pine.csproj
@@ -4,8 +4,8 @@
     <OutputType>Exe</OutputType>
     <TargetFramework>net8.0</TargetFramework>
     <AssemblyName>pine</AssemblyName>
-    <AssemblyVersion>0.3.0</AssemblyVersion>
-    <FileVersion>0.3.0</FileVersion>
+    <AssemblyVersion>0.3.1</AssemblyVersion>
+    <FileVersion>0.3.1</FileVersion>
     <Nullable>enable</Nullable>
     <GenerateEmbeddedFilesManifest>true</GenerateEmbeddedFilesManifest>
   </PropertyGroup>