Dev version of protobuf market order book protobuf timed data exports

.github/workflows/ubuntu-monitoring.yml → .github/workflows/ubuntu-special.yml

@@ -1,4 +1,4 @@
-name: Monitoring
+name: Special
 
 on:
   push:
@@ -7,14 +7,14 @@ on:
   pull_request:
 
 jobs:
-  ubuntu-monitoring-build:
-    name: Build on Ubuntu with monitoring support
+  ubuntu-special-build:
+    name: Build on Ubuntu with monitoring / protobuf support
     runs-on: ubuntu-latest
     strategy:
       matrix:
         compiler: [g++-11]
         buildmode: [Debug]
-        build-prometheus-from-source: [0, 1]
+        build-special-from-source: [0, 1]
 
     steps:
       - name: Checkout repository code
@@ -39,15 +39,15 @@ jobs:
           ninja
 
           sudo cmake --install .
-        if: matrix.build-prometheus-from-source == 0
+        if: matrix.build-special-from-source == 0
 
       - name: Create Build Environment
         run: cmake -E make_directory ${{github.workspace}}/build
 
       - name: Configure CMake
         working-directory: ${{github.workspace}}/build
         shell: bash
-        run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{matrix.buildmode}} -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCCT_BUILD_PROMETHEUS_FROM_SRC=${{matrix.build-prometheus-from-source}} -GNinja
+        run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{matrix.buildmode}} -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCCT_BUILD_PROMETHEUS_FROM_SRC=${{matrix.build-special-from-source}} -DCCT_ENABLE_PROTO=${{matrix.build-special-from-source}} -GNinja
 
       - name: Build
         working-directory: ${{github.workspace}}/build

.github/workflows/windows.yml

@@ -19,7 +19,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          vcpkg install curl
+          vcpkg install curl protobuf
 
       - name: End vcpkg install
         run: |

.gitignore

@@ -64,6 +64,7 @@ data/cache
 data/log
 data/secret
 !data/secret/secret_test.json
+data/serialized
 data/static/exchangeconfig.json
 data/static/generalconfig.json
 monitoring/data/grafana/*

CMakeLists.txt

@@ -36,6 +36,7 @@ option(CCT_ENABLE_TESTS "Build the unit tests" ${MAIN_PROJECT})
 option(CCT_BUILD_EXEC "Build an executable instead of a static library" ${MAIN_PROJECT})
 option(CCT_ENABLE_ASAN "Compile with AddressSanitizer" ${CCT_ASAN_BUILD})
 option(CCT_ENABLE_CLANG_TIDY "Compile with clang-tidy checks" OFF)
+option(CCT_ENABLE_PROTO "Compile with protobuf support (to export data to the outside world)" ON)
 option(CCT_BUILD_PROMETHEUS_FROM_SRC "Fetch and build from prometheus-cpp sources" OFF)
 
 set(CCT_DATA_DIR "${CMAKE_CURRENT_SOURCE_DIR}/data" CACHE PATH "Needed data directory for coincenter. Can also be overriden at runtime with this environment variable")
@@ -141,6 +142,35 @@ if(NOT spdlog_FOUND)
   FetchContent_MakeAvailable(spdlog)
 endif()
 
+# protobuf - serialization / deserialization library
+if(CCT_ENABLE_PROTO)
+  find_package(Protobuf CONFIG)
+  if(Protobuf_FOUND)
+    message(STATUS "Linking with protobuf ${protobuf_VERSION}")
+  else()
+    set(PROTOBUF_VERSION v25.3)
+    if (MSVC)
+      # protobuf v25 does not compile with MSVC: https://github.com/protocolbuffers/protobuf/issues/14602
+      set(PROTOBUF_VERSION v26.0-rc3)
+    endif()
+
+    message(STATUS "Compiling protobuf ${PROTOBUF_VERSION} from sources")
+
+    set(protobuf_BUILD_TESTS OFF)
+    set(ABSL_PROPAGATE_CXX_STD ON)
+
+    FetchContent_Declare(
+      protobuf
+      GIT_REPOSITORY https://github.com/protocolbuffers/protobuf.git
+      GIT_TAG ${PROTOBUF_VERSION}
+    )
+    FetchContent_MakeAvailable(protobuf)
+
+    include(${protobuf_SOURCE_DIR}/cmake/protobuf-generate.cmake)
+
+  endif()
+endif()
+
 # Unit Tests
 include(cmake/AddUnitTest.cmake)
 
@@ -179,13 +209,20 @@ if(CCT_ENABLE_PROMETHEUS)
   add_compile_definitions(CCT_ENABLE_PROMETHEUS)
 endif()
 
+if(CCT_ENABLE_PROTO)
+  add_compile_definitions(CCT_ENABLE_PROTO)
+  add_compile_definitions("CCT_PROTOBUF_VERSION=\"${PROTOBUF_VERSION}\"")
+endif()
+
 # Link to sub folders CMakeLists.txt, from the lowest level to the highest level for documentation
 # (beware of cyclic dependencies)
 add_subdirectory(src/tech)
 add_subdirectory(src/monitoring)
 add_subdirectory(src/http-request)
 add_subdirectory(src/objects)
+add_subdirectory(src/serialization)
 add_subdirectory(src/api-objects)
+add_subdirectory(src/trading)
 add_subdirectory(src/api)
 add_subdirectory(src/engine)
 add_subdirectory(src/main)

Dockerfile

@@ -5,6 +5,7 @@ ARG BUILD_MODE=Release
 ARG BUILD_TEST=0
 ARG BUILD_ASAN=0
 ARG BUILD_WITH_PROMETHEUS=1
+ARG BUILD_WITH_PROTOBUF=1
 
 # Install base & build dependencies, needed certificates for curl to work with https
 RUN apt update && \
@@ -41,6 +42,7 @@ RUN cmake -DCMAKE_BUILD_TYPE=${BUILD_MODE} \
     -DCCT_ENABLE_TESTS=${BUILD_TEST} \
     -DCCT_ENABLE_ASAN=${BUILD_ASAN} \
     -DCCT_BUILD_PROMETHEUS_FROM_SRC=${BUILD_WITH_PROMETHEUS} \
+    -DCCT_ENABLE_PROTO=${BUILD_WITH_PROTOBUF} \
     -GNinja .. && \
     ninja && \
     if [ "$BUILD_TEST" = "1" -o "$BUILD_TEST" = "ON" ]; then \
@@ -68,4 +70,4 @@ COPY --from=build /app/bin/coincenter /app/coincenter
 # 'data' directory of host machine can be mounted when launching the container.
 # To do this, you can use --mount option:
 # docker run --mount type=bind,source=<path-to-data-dir-on-host>,target=/app/data sjanel/coincenter
-ENTRYPOINT [ "/app/coincenter" ]
+ENTRYPOINT [ "/app/coincenter" ]

alpine.Dockerfile

@@ -5,9 +5,10 @@ ARG BUILD_MODE=Release
 ARG BUILD_TEST=0
 ARG BUILD_ASAN=0
 ARG BUILD_WITH_PROMETHEUS=1
+ARG BUILD_WITH_PROTOBUF=1
 
 # Install base & build dependencies, needed certificates for curl to work with https
-RUN apk add --update --upgrade --no-cache g++ libc-dev openssl-dev curl-dev cmake ninja git ca-certificates
+RUN apk add --update --upgrade --no-cache linux-headers g++ libc-dev openssl-dev curl-dev cmake ninja git ca-certificates
 
 # Copy source files
 WORKDIR /app/src
@@ -39,6 +40,7 @@ RUN cmake -DCMAKE_BUILD_TYPE=${BUILD_MODE} \
     -DCCT_ENABLE_TESTS=${BUILD_TEST} \
     -DCCT_ENABLE_ASAN=${BUILD_ASAN} \
     -DCCT_BUILD_PROMETHEUS_FROM_SRC=${BUILD_WITH_PROMETHEUS} \
+    -DCCT_ENABLE_PROTO=${BUILD_WITH_PROTOBUF} \
     -GNinja .. && \
     ninja && \
     if [ "$BUILD_TEST" = "1" -o "$BUILD_TEST" = "ON" ]; then \

src/api-objects/include/closed-order.hpp

@@ -21,4 +21,4 @@ class ClosedOrder : public Order {
  private:
   TimePoint _matchedTime;
 };
-}  // namespace cct
+}  // namespace cct

src/api-objects/include/exchangepublicapitypes.hpp

@@ -12,9 +12,12 @@
 #include "publictrade.hpp"
 
 namespace cct {
+
 using MarketSet = FlatSet<Market>;
 using MarketOrderBookMap = std::unordered_map<Market, MarketOrderBook>;
 using MarketPriceMap = std::unordered_map<Market, MonetaryAmount>;
 using MarketsPath = SmallVector<Market, 3>;
 using TradesVector = vector<PublicTrade>;
-}  // namespace cct
+using MarketOrderBookVector = vector<MarketOrderBook>;
+
+}  // namespace cct

src/api-objects/src/opened-order.cpp

@@ -14,4 +14,4 @@ OpenedOrder::OpenedOrder(string id, MonetaryAmount matchedVolume, MonetaryAmount
                          TimePoint placedTime, TradeSide side)
     : Order(std::move(id), matchedVolume, price, placedTime, side), _remainingVolume(remainingVolume) {}
 
-}  // namespace cct
+}  // namespace cct

src/api/common/include/exchangeprivateapi_mock.hpp

@@ -46,4 +46,4 @@ class MockExchangePrivate : public ExchangePrivate {
               (override));
 };
 
-}  // namespace cct::api
+}  // namespace cct::api

src/api/exchanges/include/bithumbprivateapi.hpp

@@ -88,4 +88,4 @@ class BithumbPrivate : public ExchangePrivate {
   CachedResult<DepositWalletFunc, CurrencyCode> _depositWalletsCache;
 };
 }  // namespace api
-}  // namespace cct
+}  // namespace cct

src/api/exchanges/include/huobiprivateapi.hpp

@@ -77,4 +77,4 @@ class HuobiPrivate : public ExchangePrivate {
   CachedResult<DepositWalletFunc, CurrencyCode> _depositWalletsCache;
 };
 }  // namespace api
-}  // namespace cct
+}  // namespace cct

src/api/exchanges/include/krakenprivateapi.hpp

@@ -76,4 +76,4 @@ class KrakenPrivate : public ExchangePrivate {
   CachedResult<DepositWalletFunc, CurrencyCode> _depositWalletsCache;
 };
 }  // namespace api
-}  // namespace cct
+}  // namespace cct

src/api/exchanges/include/kucoinprivateapi.hpp

@@ -67,4 +67,4 @@ class KucoinPrivate : public ExchangePrivate {
   CachedResult<DepositWalletFunc, CurrencyCode> _depositWalletsCache;
 };
 }  // namespace api
-}  // namespace cct
+}  // namespace cct

src/api/exchanges/test/exchangecommonapi_test.hpp

@@ -327,4 +327,4 @@ class TestAPI {
   TEST(TestAPIType##Test, RecentWithdraws) { testAPI.testRecentWithdraws(); } \
   TEST(TestAPIType##Test, Orders) { testAPI.testOrders(); }                   \
   TEST(TestAPIType##Test, Trade) { testAPI.testTrade(); }
-}  // namespace cct::api
+}  // namespace cct::api

src/api/interface/CMakeLists.txt

@@ -3,6 +3,7 @@ aux_source_directory(src API_INTERFACE_SRC)
 
 add_library(coincenter_api-interface STATIC ${API_INTERFACE_SRC})
 target_link_libraries(coincenter_api-interface PUBLIC coincenter_api-exchange)
+target_link_libraries(coincenter_api-interface PUBLIC coincenter_serialization)
 target_link_libraries(coincenter_api-interface PRIVATE coincenter_monitoring)
 
 target_include_directories(coincenter_api-interface PUBLIC include)

src/api/interface/include/exchange.hpp

@@ -1,7 +1,9 @@
 #pragma once
 
+#include <memory>
 #include <optional>
 #include <string_view>
+#include <type_traits>
 
 #include "cct_exception.hpp"
 #include "currencycode.hpp"
@@ -12,22 +14,30 @@
 #include "exchangeprivateapi.hpp"
 #include "exchangepublicapi.hpp"
 #include "exchangepublicapitypes.hpp"
+#include "market.hpp"
 #include "marketorderbook.hpp"
 #include "monetaryamount.hpp"
 #include "monetaryamountbycurrencyset.hpp"
+#include "time-window.hpp"
 
 namespace cct {
+
+class AbstractMarketDataDeserializer;
+class AbstractMarketDataSerializer;
+
 class Exchange {
  public:
   using ExchangePublic = api::ExchangePublic;
 
   /// Builds a Exchange without private exchange. All private requests will be forbidden.
-  Exchange(const ExchangeConfig &exchangeConfig, api::ExchangePublic &exchangePublic);
+  Exchange(std::string_view dataDir, const ExchangeConfig &exchangeConfig, api::ExchangePublic &exchangePublic);
 
   /// Build a Exchange with both private and public exchanges
-  Exchange(const ExchangeConfig &exchangeConfig, api::ExchangePublic &exchangePublic,
+  Exchange(std::string_view dataDir, const ExchangeConfig &exchangeConfig, api::ExchangePublic &exchangePublic,
            api::ExchangePrivate &exchangePrivate);
 
+  ~Exchange();
+
   std::string_view name() const { return _exchangePublic.name(); }
   std::string_view keyName() const { return apiPrivate().keyName(); }
 
@@ -86,16 +96,12 @@ class Exchange {
     return _exchangePublic.queryAllApproximatedOrderBooks(depth);
   }
 
-  MarketOrderBook queryOrderBook(Market mk, int depth = ExchangePublic::kDefaultDepth) {
-    return _exchangePublic.queryOrderBook(mk, depth);
-  }
+  MarketOrderBook queryOrderBook(Market mk, int depth = ExchangePublic::kDefaultDepth);
 
   MonetaryAmount queryLast24hVolume(Market mk) { return _exchangePublic.queryLast24hVolume(mk); }
 
   /// Retrieve an ordered vector of recent last trades
-  TradesVector queryLastTrades(Market mk, int nbTrades = ExchangePublic::kNbLastTradesDefault) {
-    return _exchangePublic.queryLastTrades(mk, nbTrades);
-  }
+  TradesVector queryLastTrades(Market mk, int nbTrades = ExchangePublic::kNbLastTradesDefault);
 
   /// Retrieve the last price of given market.
   MonetaryAmount queryLastPrice(Market mk) { return _exchangePublic.queryLastPrice(mk); }
@@ -108,11 +114,24 @@ class Exchange {
     return name() == exchangeName.name() && (!exchangeName.isKeyNameDefined() || keyName() == exchangeName.keyName());
   }
 
+  MarketSet pullAvailableMarketsForReplay(TimeWindow timeWindow);
+
+  TradesVector pullTradesForReplay(Market market, TimeWindow timeWindow);
+
+  MarketOrderBookVector pullMarketOrderBooksForReplay(Market market, TimeWindow timeWindow);
+
   void updateCacheFile() const;
 
+  using trivially_relocatable = std::true_type;
+
  private:
+  Exchange(std::string_view dataDir, const ExchangeConfig &exchangeConfig, api::ExchangePublic &exchangePublic,
+           api::ExchangePrivate *pExchangePrivate);
+
   api::ExchangePublic &_exchangePublic;
   api::ExchangePrivate *_pExchangePrivate = nullptr;
   const ExchangeConfig &_exchangeConfig;
+  std::unique_ptr<AbstractMarketDataDeserializer> _marketDataDeserializerPtr;
+  std::unique_ptr<AbstractMarketDataSerializer> _marketDataSerializerPtr;
 };
 }  // namespace cct