From ec1a760dd556eb086f51803511456005bbcbfc59 Mon Sep 17 00:00:00 2001 From: Igor Kotrasinski Date: Wed, 25 Dec 2019 00:21:50 +0100 Subject: [PATCH] Add locks around zstd compress calls The python library is explicitly NOT thread-safe. This will still let us not block on compression. If ever needed, we can ask zstd to compress with multiple threads anyway. Signed-off-by: Igor Kotrasinski --- replayserver/bookkeeping/storage.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/replayserver/bookkeeping/storage.py b/replayserver/bookkeeping/storage.py index e50f5f7..83771ef 100644 --- a/replayserver/bookkeeping/storage.py +++ b/replayserver/bookkeeping/storage.py @@ -2,6 +2,7 @@ import json import zstandard as zstd import asyncio +import threading from replayserver.errors import BookkeepingError from replayserver.logging import short_exc @@ -40,7 +41,11 @@ class ReplaySaver: def __init__(self, paths, database): self._paths = paths self._database = database - self._compressor = zstd.ZstdCompressor(level=10) + # TODO - consider multi-threaded compression if we end up needing more + # performance. + self._compressor = zstd.ZstdCompressor(level=10, + write_checksum=True) + self._compressor_lock = threading.Lock() @classmethod def build(cls, database, config): @@ -94,7 +99,9 @@ def _write_replay(self, rfile, info, data): try: rfile.write(json.dumps(info).encode('UTF-8')) rfile.write(b"\n") - data = self._compressor.compress(data) + # zstandard is explicitly NOT thread-safe + with self._compressor_lock: + data = self._compressor.compress(data) rfile.write(data) # json should always produce ascii, but just in case... except UnicodeEncodeError: