diff --git a/implement/PersistentProcess/PersistentProcess.Common/CommonConversion.cs b/implement/PersistentProcess/PersistentProcess.Common/CommonConversion.cs index ccd2467a..6c37620e 100644 --- a/implement/PersistentProcess/PersistentProcess.Common/CommonConversion.cs +++ b/implement/PersistentProcess/PersistentProcess.Common/CommonConversion.cs @@ -45,6 +45,32 @@ static public byte[] DecompressGzip(byte[] compressed) } } + static public byte[] Deflate(byte[] input) + { + using (var deflatedStream = new System.IO.MemoryStream()) + using (var compressor = new System.IO.Compression.DeflateStream( + deflatedStream, System.IO.Compression.CompressionMode.Compress)) + { + compressor.Write(input, 0, input.Length); + compressor.Close(); + return deflatedStream.ToArray(); + } + } + + static public byte[] Inflate(byte[] input) + { + using (var inflatedStream = new System.IO.MemoryStream()) + { + using (var deflateStream = new System.IO.Compression.DeflateStream( + new System.IO.MemoryStream(input), System.IO.Compression.CompressionMode.Decompress)) + { + deflateStream.CopyTo(inflatedStream); + + return inflatedStream.ToArray(); + } + } + } + static public string TimeStringViewForReport(DateTimeOffset time) => time.ToString("yyyy-MM-ddTHH-mm-ss"); } diff --git a/implement/PersistentProcess/PersistentProcess.WebHost/PersistentProcessVolatileRepresentation.cs b/implement/PersistentProcess/PersistentProcess.WebHost/PersistentProcessVolatileRepresentation.cs index 9e0b7a54..4cd618f6 100644 --- a/implement/PersistentProcess/PersistentProcess.WebHost/PersistentProcessVolatileRepresentation.cs +++ b/implement/PersistentProcess/PersistentProcess.WebHost/PersistentProcessVolatileRepresentation.cs @@ -765,59 +765,61 @@ static string applyCommonFormattingToJson(string originalJson) public ProvisionalReductionRecordInFile StoreReductionRecordForCurrentState(IProcessStoreWriter storeWriter) { + string elmAppState = null; + lock (processLock) { if (lastCompositionLogRecordHashBase16 == CompositionLogRecordInFile.compositionLogFirstRecordParentHashBase16) return null; - var elmAppState = lastElmAppVolatileProcess?.GetSerializedState(); - - var elmAppStateBlob = - elmAppState == null - ? - null - : - Encoding.UTF8.GetBytes(elmAppState); - - var elmAppStateComponent = - elmAppStateBlob == null - ? - null - : - Composition.Component.Blob(elmAppStateBlob); + elmAppState = lastElmAppVolatileProcess?.GetSerializedState(); + } - var reductionRecord = - new ProvisionalReductionRecordInFile - { - reducedCompositionHashBase16 = lastCompositionLogRecordHashBase16, - elmAppState = - elmAppStateComponent == null - ? null - : new ValueInFileStructure - { - HashBase16 = CommonConversion.StringBase16FromByteArray(Composition.GetHash(elmAppStateComponent)) - }, - appConfig = - lastAppConfig == null - ? null - : new ValueInFileStructure - { - HashBase16 = CommonConversion.StringBase16FromByteArray( - Composition.GetHash(lastAppConfig.Value.appConfigComponent)), - }, - }; + var elmAppStateBlob = + elmAppState == null + ? + null + : + Encoding.UTF8.GetBytes(elmAppState); + + var elmAppStateComponent = + elmAppStateBlob == null + ? + null + : + Composition.Component.Blob(elmAppStateBlob); + + var reductionRecord = + new ProvisionalReductionRecordInFile + { + reducedCompositionHashBase16 = lastCompositionLogRecordHashBase16, + elmAppState = + elmAppStateComponent == null + ? null + : new ValueInFileStructure + { + HashBase16 = CommonConversion.StringBase16FromByteArray(Composition.GetHash(elmAppStateComponent)) + }, + appConfig = + lastAppConfig == null + ? null + : new ValueInFileStructure + { + HashBase16 = CommonConversion.StringBase16FromByteArray( + Composition.GetHash(lastAppConfig.Value.appConfigComponent)), + }, + }; - var dependencies = - new[] { elmAppStateComponent, lastAppConfig?.appConfigComponent } - .Where(c => null != c).ToImmutableList(); + var dependencies = + new[] { elmAppStateComponent, lastAppConfig?.appConfigComponent } + .Where(c => null != c).ToImmutableList(); - foreach (var dependency in dependencies) - storeWriter.StoreComponent(dependency); + foreach (var dependency in dependencies) + storeWriter.StoreComponent(dependency); - storeWriter.StoreProvisionalReduction(reductionRecord); + storeWriter.StoreProvisionalReduction(reductionRecord); - return reductionRecord; - } + return reductionRecord; } } } diff --git a/implement/PersistentProcess/PersistentProcess.WebHost/ProcessStoreSupportingMigrations.cs b/implement/PersistentProcess/PersistentProcess.WebHost/ProcessStoreSupportingMigrations.cs index ff13bf07..860fdd8e 100644 --- a/implement/PersistentProcess/PersistentProcess.WebHost/ProcessStoreSupportingMigrations.cs +++ b/implement/PersistentProcess/PersistentProcess.WebHost/ProcessStoreSupportingMigrations.cs @@ -195,6 +195,13 @@ public class ProcessStoreInFileStore static protected IImmutableList CompositionHeadHashFilePath => ImmutableList.Create("composition-log-head-hash"); + static protected string ComponentSubdirectory => "component"; + + static protected string DeflatedComponentSubdirectory => "deflated-component"; + + static protected string ProvisionalReductionSubdirectory => "provisional-reduction"; + + static readonly protected Newtonsoft.Json.JsonSerializerSettings recordSerializationSettings = RecordSerializationSettings; static public IImmutableList GetFilePathForComponentInComponentFileStore(string componentHash) => @@ -216,23 +223,41 @@ public class ProcessStoreReaderInFileStore : ProcessStoreInFileStore, IProcessSt protected IFileStoreReader fileStore; // Plain Kalmit component. - protected IFileStoreReader componentFileStore => fileStore.ForSubdirectory("component"); + protected IFileStoreReader componentFileStore => fileStore.ForSubdirectory(ComponentSubdirectory); + + protected IFileStoreReader deflatedComponentFileStore => fileStore.ForSubdirectory(DeflatedComponentSubdirectory); - protected IFileStoreReader provisionalReductionFileStore => fileStore.ForSubdirectory("provisional-reduction"); + protected IFileStoreReader provisionalReductionFileStore => fileStore.ForSubdirectory(ProvisionalReductionSubdirectory); public ProcessStoreReaderInFileStore(IFileStoreReader fileStore) { this.fileStore = fileStore; } - IReadOnlyList LoadComponentSerialRepresentationForHash(IReadOnlyList componentHash) => - componentFileStore.GetFileContent( - GetFilePathForComponentInComponentFileStore(CommonConversion.StringBase16FromByteArray(componentHash.ToArray()))); + byte[] LoadComponentSerialRepresentationForHash(IReadOnlyList componentHash) => + LoadComponentSerialRepresentationForHash(CommonConversion.StringBase16FromByteArray(componentHash.ToArray())); + + byte[] LoadComponentSerialRepresentationForHash(string componentHashBase16) + { + var originalFile = + componentFileStore.GetFileContent(GetFilePathForComponentInComponentFileStore(componentHashBase16)); + + if (originalFile == null) + { + var deflatedFile = + deflatedComponentFileStore.GetFileContent( + GetFilePathForComponentInComponentFileStore(componentHashBase16)); + + if (deflatedFile != null) + return CommonConversion.Inflate(deflatedFile); + } + + return originalFile; + } public Composition.Component LoadComponent(string componentHashBase16) { - var fromComponentStore = componentFileStore.GetFileContent( - GetFilePathForComponentInComponentFileStore(componentHashBase16)); + var fromComponentStore = LoadComponentSerialRepresentationForHash(componentHashBase16); if (fromComponentStore == null) return null; @@ -331,12 +356,16 @@ public IEnumerable EnumerateSerializedCompositionLogRecordsReverse() public class ProcessStoreWriterInFileStore : ProcessStoreInFileStore, IProcessStoreWriter { + static int tryDeflateSizeThreshold => 10_000; + protected IFileStoreWriter fileStore; // Plain Kalmit component. - protected IFileStoreWriter componentFileStore => fileStore.ForSubdirectory("component"); + protected IFileStoreWriter componentFileStore => fileStore.ForSubdirectory(ComponentSubdirectory); + + protected IFileStoreWriter deflatedComponentFileStore => fileStore.ForSubdirectory(DeflatedComponentSubdirectory); - protected IFileStoreWriter provisionalReductionFileStore => fileStore.ForSubdirectory("provisional-reduction"); + protected IFileStoreWriter provisionalReductionFileStore => fileStore.ForSubdirectory(ProvisionalReductionSubdirectory); public ProcessStoreWriterInFileStore(IFileStoreWriter fileStore) { @@ -375,9 +404,28 @@ public void StoreComponent(Composition.Component component) var hashBase16 = CommonConversion.StringBase16FromByteArray(hash); - componentFileStore.SetFileContent( - GetFilePathForComponentInComponentFileStore(hashBase16), - serialRepresentation); + void storeSelf() + { + if (tryDeflateSizeThreshold <= serialRepresentation.Length) + { + var deflated = CommonConversion.Deflate(serialRepresentation); + + if (deflated.Length * 10 < serialRepresentation.Length * 8) + { + deflatedComponentFileStore.SetFileContent( + GetFilePathForComponentInComponentFileStore(hashBase16), + deflated); + + return; + } + } + + componentFileStore.SetFileContent( + GetFilePathForComponentInComponentFileStore(hashBase16), + serialRepresentation); + } + + storeSelf(); foreach (var dependency in dependencies) StoreComponent(dependency); diff --git a/implement/PersistentProcess/PersistentProcess.WebHost/Program.cs b/implement/PersistentProcess/PersistentProcess.WebHost/Program.cs index 57ddfb9a..0e77b7c1 100644 --- a/implement/PersistentProcess/PersistentProcess.WebHost/Program.cs +++ b/implement/PersistentProcess/PersistentProcess.WebHost/Program.cs @@ -2,6 +2,6 @@ namespace Kalmit.PersistentProcess.WebHost { public class Program { - static public string AppVersionId => "2020-11-07"; + static public string AppVersionId => "2020-11-08"; } } diff --git a/implement/elm-fullstack/elm-fullstack.csproj b/implement/elm-fullstack/elm-fullstack.csproj index cd7e58f6..3ff5b1df 100644 --- a/implement/elm-fullstack/elm-fullstack.csproj +++ b/implement/elm-fullstack/elm-fullstack.csproj @@ -5,8 +5,8 @@ netcoreapp3.1 elm_fullstack elm-fullstack - 2020.1107.0.0 - 2020.1107.0.0 + 2020.1108.0.0 + 2020.1108.0.0