Skip to content

Commit

Permalink
Reduce disk usage for process store in production
Browse files Browse the repository at this point in the history 
Reduce the disk usage by applying compression on larger components.
Adapt to recent observations from an app in production: The reductions can account for more than half of the total size, and deflating the reduction blobs often reduces the file to less than 15%.
  • Loading branch information
@Viir
Viir committed Nov 8, 2020
1 parent 68c2f26 commit e6e8d01
Show file tree
Hide file tree
Showing 5 changed files with 134 additions and 58 deletions.
26 changes: 26 additions & 0 deletions implement/PersistentProcess/PersistentProcess.Common/CommonConversion.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,32 @@ static public byte[] DecompressGzip(byte[] compressed)
}
}

static public byte[] Deflate(byte[] input)
{
using (var deflatedStream = new System.IO.MemoryStream())
using (var compressor = new System.IO.Compression.DeflateStream(
deflatedStream, System.IO.Compression.CompressionMode.Compress))
{
compressor.Write(input, 0, input.Length);
compressor.Close();
return deflatedStream.ToArray();
}
}

static public byte[] Inflate(byte[] input)
{
using (var inflatedStream = new System.IO.MemoryStream())
{
using (var deflateStream = new System.IO.Compression.DeflateStream(
new System.IO.MemoryStream(input), System.IO.Compression.CompressionMode.Decompress))
{
deflateStream.CopyTo(inflatedStream);

return inflatedStream.ToArray();
}
}
}

static public string TimeStringViewForReport(DateTimeOffset time) =>
time.ToString("yyyy-MM-ddTHH-mm-ss");
}
Expand Down
88 changes: 45 additions & 43 deletions ...nt/PersistentProcess/PersistentProcess.WebHost/PersistentProcessVolatileRepresentation.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -765,59 +765,61 @@ static string applyCommonFormattingToJson(string originalJson)

public ProvisionalReductionRecordInFile StoreReductionRecordForCurrentState(IProcessStoreWriter storeWriter)
{
string elmAppState = null;

lock (processLock)
{
if (lastCompositionLogRecordHashBase16 == CompositionLogRecordInFile.compositionLogFirstRecordParentHashBase16)
return null;

var elmAppState = lastElmAppVolatileProcess?.GetSerializedState();

var elmAppStateBlob =
elmAppState == null
?
null
:
Encoding.UTF8.GetBytes(elmAppState);

var elmAppStateComponent =
elmAppStateBlob == null
?
null
:
Composition.Component.Blob(elmAppStateBlob);
elmAppState = lastElmAppVolatileProcess?.GetSerializedState();
}

var reductionRecord =
new ProvisionalReductionRecordInFile
{
reducedCompositionHashBase16 = lastCompositionLogRecordHashBase16,
elmAppState =
elmAppStateComponent == null
? null
: new ValueInFileStructure
{
HashBase16 = CommonConversion.StringBase16FromByteArray(Composition.GetHash(elmAppStateComponent))
},
appConfig =
lastAppConfig == null
? null
: new ValueInFileStructure
{
HashBase16 = CommonConversion.StringBase16FromByteArray(
Composition.GetHash(lastAppConfig.Value.appConfigComponent)),
},
};
var elmAppStateBlob =
elmAppState == null
?
null
:
Encoding.UTF8.GetBytes(elmAppState);

var elmAppStateComponent =
elmAppStateBlob == null
?
null
:
Composition.Component.Blob(elmAppStateBlob);

var reductionRecord =
new ProvisionalReductionRecordInFile
{
reducedCompositionHashBase16 = lastCompositionLogRecordHashBase16,
elmAppState =
elmAppStateComponent == null
? null
: new ValueInFileStructure
{
HashBase16 = CommonConversion.StringBase16FromByteArray(Composition.GetHash(elmAppStateComponent))
},
appConfig =
lastAppConfig == null
? null
: new ValueInFileStructure
{
HashBase16 = CommonConversion.StringBase16FromByteArray(
Composition.GetHash(lastAppConfig.Value.appConfigComponent)),
},
};

var dependencies =
new[] { elmAppStateComponent, lastAppConfig?.appConfigComponent }
.Where(c => null != c).ToImmutableList();
var dependencies =
new[] { elmAppStateComponent, lastAppConfig?.appConfigComponent }
.Where(c => null != c).ToImmutableList();

foreach (var dependency in dependencies)
storeWriter.StoreComponent(dependency);
foreach (var dependency in dependencies)
storeWriter.StoreComponent(dependency);

storeWriter.StoreProvisionalReduction(reductionRecord);
storeWriter.StoreProvisionalReduction(reductionRecord);

return reductionRecord;
}
return reductionRecord;
}
}
}
72 changes: 60 additions & 12 deletions implement/PersistentProcess/PersistentProcess.WebHost/ProcessStoreSupportingMigrations.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,13 @@ public class ProcessStoreInFileStore
static protected IImmutableList<string> CompositionHeadHashFilePath =>
ImmutableList.Create("composition-log-head-hash");

static protected string ComponentSubdirectory => "component";

static protected string DeflatedComponentSubdirectory => "deflated-component";

static protected string ProvisionalReductionSubdirectory => "provisional-reduction";


static readonly protected Newtonsoft.Json.JsonSerializerSettings recordSerializationSettings = RecordSerializationSettings;

static public IImmutableList<string> GetFilePathForComponentInComponentFileStore(string componentHash) =>
Expand All @@ -216,23 +223,41 @@ public class ProcessStoreReaderInFileStore : ProcessStoreInFileStore, IProcessSt
protected IFileStoreReader fileStore;

// Plain Kalmit component.
protected IFileStoreReader componentFileStore => fileStore.ForSubdirectory("component");
protected IFileStoreReader componentFileStore => fileStore.ForSubdirectory(ComponentSubdirectory);

protected IFileStoreReader deflatedComponentFileStore => fileStore.ForSubdirectory(DeflatedComponentSubdirectory);

protected IFileStoreReader provisionalReductionFileStore => fileStore.ForSubdirectory("provisional-reduction");
protected IFileStoreReader provisionalReductionFileStore => fileStore.ForSubdirectory(ProvisionalReductionSubdirectory);

public ProcessStoreReaderInFileStore(IFileStoreReader fileStore)
{
this.fileStore = fileStore;
}

IReadOnlyList<byte> LoadComponentSerialRepresentationForHash(IReadOnlyList<byte> componentHash) =>
componentFileStore.GetFileContent(
GetFilePathForComponentInComponentFileStore(CommonConversion.StringBase16FromByteArray(componentHash.ToArray())));
byte[] LoadComponentSerialRepresentationForHash(IReadOnlyList<byte> componentHash) =>
LoadComponentSerialRepresentationForHash(CommonConversion.StringBase16FromByteArray(componentHash.ToArray()));

byte[] LoadComponentSerialRepresentationForHash(string componentHashBase16)
{
var originalFile =
componentFileStore.GetFileContent(GetFilePathForComponentInComponentFileStore(componentHashBase16));

if (originalFile == null)
{
var deflatedFile =
deflatedComponentFileStore.GetFileContent(
GetFilePathForComponentInComponentFileStore(componentHashBase16));

if (deflatedFile != null)
return CommonConversion.Inflate(deflatedFile);
}

return originalFile;
}

public Composition.Component LoadComponent(string componentHashBase16)
{
var fromComponentStore = componentFileStore.GetFileContent(
GetFilePathForComponentInComponentFileStore(componentHashBase16));
var fromComponentStore = LoadComponentSerialRepresentationForHash(componentHashBase16);

if (fromComponentStore == null)
return null;
Expand Down Expand Up @@ -331,12 +356,16 @@ public IEnumerable<byte[]> EnumerateSerializedCompositionLogRecordsReverse()

public class ProcessStoreWriterInFileStore : ProcessStoreInFileStore, IProcessStoreWriter
{
static int tryDeflateSizeThreshold => 10_000;

protected IFileStoreWriter fileStore;

// Plain Kalmit component.
protected IFileStoreWriter componentFileStore => fileStore.ForSubdirectory("component");
protected IFileStoreWriter componentFileStore => fileStore.ForSubdirectory(ComponentSubdirectory);

protected IFileStoreWriter deflatedComponentFileStore => fileStore.ForSubdirectory(DeflatedComponentSubdirectory);

protected IFileStoreWriter provisionalReductionFileStore => fileStore.ForSubdirectory("provisional-reduction");
protected IFileStoreWriter provisionalReductionFileStore => fileStore.ForSubdirectory(ProvisionalReductionSubdirectory);

public ProcessStoreWriterInFileStore(IFileStoreWriter fileStore)
{
Expand Down Expand Up @@ -375,9 +404,28 @@ public void StoreComponent(Composition.Component component)

var hashBase16 = CommonConversion.StringBase16FromByteArray(hash);

componentFileStore.SetFileContent(
GetFilePathForComponentInComponentFileStore(hashBase16),
serialRepresentation);
void storeSelf()
{
if (tryDeflateSizeThreshold <= serialRepresentation.Length)
{
var deflated = CommonConversion.Deflate(serialRepresentation);

if (deflated.Length * 10 < serialRepresentation.Length * 8)
{
deflatedComponentFileStore.SetFileContent(
GetFilePathForComponentInComponentFileStore(hashBase16),
deflated);

return;
}
}

componentFileStore.SetFileContent(
GetFilePathForComponentInComponentFileStore(hashBase16),
serialRepresentation);
}

storeSelf();

foreach (var dependency in dependencies)
StoreComponent(dependency);
Expand Down
2 changes: 1 addition & 1 deletion implement/PersistentProcess/PersistentProcess.WebHost/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@ namespace Kalmit.PersistentProcess.WebHost
{
public class Program
{
static public string AppVersionId => "2020-11-07";
static public string AppVersionId => "2020-11-08";
}
}
4 changes: 2 additions & 2 deletions implement/elm-fullstack/elm-fullstack.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
<TargetFramework>netcoreapp3.1</TargetFramework>
<RootNamespace>elm_fullstack</RootNamespace>
<AssemblyName>elm-fullstack</AssemblyName>
<AssemblyVersion>2020.1107.0.0</AssemblyVersion>
<FileVersion>2020.1107.0.0</FileVersion>
<AssemblyVersion>2020.1108.0.0</AssemblyVersion>
<FileVersion>2020.1108.0.0</FileVersion>
</PropertyGroup>

<PropertyGroup>
Expand Down

0 comments on commit e6e8d01

Please sign in to comment.