Skip to content

Commit

Permalink
Offer easier ways for replicating a process
Browse files Browse the repository at this point in the history 
Support archiving process to a file and support replication from such a file.
  • Loading branch information
@Viir
Viir committed May 16, 2020
1 parent 28af30f commit f374fab
Show file tree
Hide file tree
Showing 8 changed files with 93 additions and 38 deletions.
7 changes: 3 additions & 4 deletions implement/PersistentProcess/PersistentProcess.Common/ElmApp.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,9 @@ static public bool FilePathMatchesPatternOfFilesInElmApp(string filePath) =>
.Where(file => 0 < file.filePath?.Length && FilePathMatchesPatternOfFilesInElmApp(file.filePath));

static public IImmutableDictionary<IImmutableList<string>, IImmutableList<byte>> ToFlatDictionaryWithPathComparer(
IEnumerable<(IImmutableList<string> filePath, IImmutableList<byte> fileContent)> fileList) =>
fileList.ToImmutableDictionary(
entry => entry.filePath, entry => entry.fileContent)
.WithComparers(EnumerableExtension.EqualityComparer<string>());
IEnumerable<(IImmutableList<string> filePath, IImmutableList<byte> fileContent)> fileList) =>
fileList.ToImmutableDictionary(entry => entry.filePath, entry => entry.fileContent)
.WithComparers(EnumerableExtension.EqualityComparer<string>());

static public IImmutableDictionary<IImmutableList<string>, IImmutableList<byte>> AsCompletelyLoweredElmApp(
IImmutableDictionary<IImmutableList<string>, IImmutableList<byte>> originalAppFiles,
Expand Down
6 changes: 3 additions & 3 deletions implement/PersistentProcess/PersistentProcess.Test/TestWebHost.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1062,11 +1062,11 @@ public void tooling_supports_replicate_process_from_remote_host()

using (var replicaHost = replicaSetup.StartWebHost())
{
elm_fullstack.Program.replicateProcess(
elm_fullstack.Program.replicateProcessAndLogToConsole(
site: replicaAdminInterfaceUrl,
sitePassword: replicaAdminPassword,
sourceSite: testSetup.AdminWebHostUrl,
sourceSitePassword: originalHostAdminPassword);
source: testSetup.AdminWebHostUrl,
sourcePassword: originalHostAdminPassword);
}
}
}
Expand Down
20 changes: 11 additions & 9 deletions ...nt/PersistentProcess/PersistentProcess.WebHost/PersistentProcessVolatileRepresentation.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,10 +78,11 @@ static Composition.TreeComponent SubtreeElmAppFromAppConfig(Composition.TreeComp
this.lastSetElmAppStateResult = lastSetElmAppStateResult;
}

static public IImmutableDictionary<IImmutableList<string>, IImmutableList<byte>> GetFilesForRestoreProcess(
static public (IImmutableDictionary<IImmutableList<string>, IImmutableList<byte>> files, string lastCompositionLogRecordHashBase16)
GetFilesForRestoreProcess(
IFileStoreReader fileStoreReader)
{
var filesForProcessRestore = new ConcurrentDictionary<IImmutableList<string>, IImmutableList<byte>>();
var filesForProcessRestore = new ConcurrentDictionary<IImmutableList<string>, IImmutableList<byte>>(EnumerableExtension.EqualityComparer<string>());

var recordingReader = new Kalmit.DelegatingFileStoreReader
{
Expand All @@ -105,9 +106,10 @@ dependencies on the store.
*/
using (var restoredProcess = Restore(new ProcessStoreReaderInFileStore(recordingReader), _ => { }))
{
return (
files: filesForProcessRestore.ToImmutableDictionary(EnumerableExtension.EqualityComparer<string>()),
lastCompositionLogRecordHashBase16: restoredProcess.lastCompositionLogRecordHashBase16);
}

return filesForProcessRestore.ToImmutableDictionary();
}

static IEnumerable<(CompositionLogRecordInFile compositionRecord, string compositionRecordHashBase16, LoadedReduction reduction)>
Expand Down Expand Up @@ -171,7 +173,11 @@ static public PersistentProcessVolatileRepresentation Restore(

logger?.Invoke("Begin to restore the process state.");

if (!storeReader.EnumerateSerializedCompositionLogRecordsReverse().Take(1).Any())
var compositionEventsToLatestReductionReversed =
EnumerateCompositionLogRecordsForRestoreProcess(storeReader)
.ToImmutableList();

if (!compositionEventsToLatestReductionReversed.Any())
{
logger?.Invoke("Found no composition record, default to initial state.");

Expand All @@ -182,10 +188,6 @@ static public PersistentProcessVolatileRepresentation Restore(
lastSetElmAppStateResult: null);
}

var compositionEventsToLatestReductionReversed =
EnumerateCompositionLogRecordsForRestoreProcess(storeReader)
.ToImmutableList();

logger?.Invoke("Found " + compositionEventsToLatestReductionReversed.Count + " composition log records to use for restore.");

var firstCompositionEventRecord =
Expand Down
4 changes: 2 additions & 2 deletions implement/PersistentProcess/PersistentProcess.WebHost/ProcessStoreSupportingMigrations.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,8 @@ static public (string parentHashBase16, IEnumerable<(IImmutableList<string> file
var fileStoreWriter = new DelegatingFileStoreWriter
{
SetFileContentDelegate = projectedFiles.Add,
AppendFileContentDelegate = _ => throw new Exception("Unexpeced operation append to file."),
DeleteFileDelegate = _ => throw new Exception("Unexpeced operation delete file."),
AppendFileContentDelegate = _ => throw new Exception("Unexpected operation append to file."),
DeleteFileDelegate = _ => throw new Exception("Unexpected operation delete file."),
};

var processStoreWriter = new ProcessStoreWriterInFileStore(fileStoreWriter);
Expand Down
2 changes: 1 addition & 1 deletion implement/PersistentProcess/PersistentProcess.WebHost/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@ namespace Kalmit.PersistentProcess.WebHost
{
public class Program
{
static public string AppVersionId => "2020-05-12";
static public string AppVersionId => "2020-05-16";
}
}
2 changes: 1 addition & 1 deletion implement/PersistentProcess/PersistentProcess.WebHost/StartupAdminInterface.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -531,7 +531,7 @@ TruncateProcessHistoryReport truncateProcessHistory(TimeSpan productionBlockDura

var filesForRestore =
PersistentProcess.PersistentProcessVolatileRepresentation.GetFilesForRestoreProcess(
processStoreFileStore)
processStoreFileStore).files
.Select(filePathAndContent => filePathAndContent.Key)
.ToImmutableHashSet(EnumerableExtension.EqualityComparer<string>());

Expand Down
86 changes: 70 additions & 16 deletions implement/elm-fullstack/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,11 +109,11 @@ CommandOption verboseLogOptionFromCommand(CommandLineApplication command) =>

if (replicateProcessSource != null)
{
var filesForRestore = readFilesForRestoreProcessFromAdminInterface(
sourceAdminInterface: replicateProcessSource,
sourceAdminRootPassword: replicateProcessAdminPassword);
var replicateFiles =
LoadFilesForRestoreFromSourceAndLogToConsole(
replicateProcessSource, replicateProcessAdminPassword);

foreach (var file in filesForRestore)
foreach (var file in replicateFiles)
processStoreFileStore.SetFileContent(file.Key, file.Value.ToArray());
}

Expand Down Expand Up @@ -249,6 +249,35 @@ CommandOption verboseLogOptionFromCommand(CommandLineApplication command) =>
});
});

app.Command("archive-process", archiveProcessCmd =>
{
archiveProcessCmd.Description = "Copy the files needed to restore the process and store those in a zip-archive.";
archiveProcessCmd.ThrowOnUnexpectedArgument = true;

var siteAndPasswordFromCmd = siteAndSitePasswordOptionsOnCommand(archiveProcessCmd);

archiveProcessCmd.OnExecute(() =>
{
var (site, sitePassword) = siteAndPasswordFromCmd();

Console.WriteLine("Begin reading process history from '" + site + "' ...");

var restoreResult =
readFilesForRestoreProcessFromAdminInterface(site, sitePassword);

Console.WriteLine("Completed reading files to restore process " + restoreResult.lastCompositionLogRecordHashBase16 + ". Read " + restoreResult.files.Count + " files from '" + site + "'.");

var zipArchive = ZipArchive.ZipArchiveFromEntries(restoreResult.files);

var fileName = "process-" + restoreResult.lastCompositionLogRecordHashBase16 + ".zip";
var filePath = Path.Combine(Environment.CurrentDirectory, fileName);

File.WriteAllBytes(filePath, zipArchive);

Console.WriteLine("Saved process archive to file '" + filePath + "'.");
});
});

app.Command("user-secrets", userSecretsCmd =>
{
userSecretsCmd.Description = "Manage passwords for accessing the admin interfaces of servers.";
Expand Down Expand Up @@ -548,7 +577,7 @@ static TruncateProcessHistoryReport truncateProcessHistory(string site, string s
}
}

static IImmutableDictionary<IImmutableList<string>, IImmutableList<byte>> readFilesForRestoreProcessFromAdminInterface(
static (IImmutableDictionary<IImmutableList<string>, IImmutableList<byte>> files, string lastCompositionLogRecordHashBase16) readFilesForRestoreProcessFromAdminInterface(
string sourceAdminInterface,
string sourceAdminRootPassword)
{
Expand Down Expand Up @@ -583,26 +612,51 @@ static IImmutableDictionary<IImmutableList<string>, IImmutableList<byte>> readFi
}
}

static public void replicateProcess(
string site,
string sitePassword,
string sourceSite,
string sourceSitePassword)
static IImmutableDictionary<IImmutableList<string>, IImmutableList<byte>> LoadFilesForRestoreFromSourceAndLogToConsole(
string source, string sourcePassword)
{
Console.WriteLine("Begin reading process history from '" + sourceSite + "' ...");
if (Regex.IsMatch(source, "http(|s)://", RegexOptions.IgnoreCase))
{
Console.WriteLine("Begin reading process history from '" + source + "' ...");

var restoreResult = readFilesForRestoreProcessFromAdminInterface(
sourceAdminInterface: source,
sourceAdminRootPassword: sourcePassword);

Console.WriteLine("Completed reading files to restore process " + restoreResult.lastCompositionLogRecordHashBase16 + ". Read " + restoreResult.files.Count + " files from '" + source + "'.");

return restoreResult.files;
}

var archive = File.ReadAllBytes(source);

var processHistoryfilesFromRemoteHost =
readFilesForRestoreProcessFromAdminInterface(sourceSite, sourceSitePassword);
var zipArchiveEntries = ZipArchive.EntriesFromZipArchive(archive);

return
ElmApp.ToFlatDictionaryWithPathComparer(
Composition.TreeFromSetOfBlobsWithCommonFilePath(zipArchiveEntries)
.EnumerateBlobsTransitive()
.Select(blobPathAndContent => (
fileName: (IImmutableList<string>)blobPathAndContent.path.Select(name => System.Text.Encoding.UTF8.GetString(name.ToArray())).ToImmutableList(),
fileContent: blobPathAndContent.blobContent)));
}

Console.WriteLine("Completed reading part of process history for restore. Read " + processHistoryfilesFromRemoteHost.Count + " files from " + sourceSite + " during restore.");
static public void replicateProcessAndLogToConsole(
string site,
string sitePassword,
string source,
string sourcePassword)
{
var restoreFiles =
LoadFilesForRestoreFromSourceAndLogToConsole(source, sourcePassword);

var processHistoryTree =
Composition.TreeFromSetOfBlobsWithStringPath(processHistoryfilesFromRemoteHost);
Composition.TreeFromSetOfBlobsWithStringPath(restoreFiles);

var processHistoryComponentHash = Composition.GetHash(Composition.FromTree(processHistoryTree));
var processHistoryComponentHashBase16 = CommonConversion.StringBase16FromByteArray(processHistoryComponentHash);

var processHistoryZipArchive = ZipArchive.ZipArchiveFromEntries(processHistoryfilesFromRemoteHost);
var processHistoryZipArchive = ZipArchive.ZipArchiveFromEntries(restoreFiles);

using (var httpClient = new System.Net.Http.HttpClient())
{
Expand Down
4 changes: 2 additions & 2 deletions implement/elm-fullstack/elm-fullstack.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
<TargetFramework>netcoreapp3.1</TargetFramework>
<RootNamespace>elm_fullstack</RootNamespace>
<AssemblyName>elm-fullstack</AssemblyName>
<AssemblyVersion>2020.0512.0.0</AssemblyVersion>
<FileVersion>2020.0512.0.0</FileVersion>
<AssemblyVersion>2020.0516.0.0</AssemblyVersion>
<FileVersion>2020.0516.0.0</FileVersion>
</PropertyGroup>

<ItemGroup>
Expand Down

0 comments on commit f374fab

Please sign in to comment.