Dev version of protobuf market order book protobuf timed data exports

.dockerignore

@@ -9,6 +9,7 @@ LICENSE
 **/*secret.json
 data/log
 data/secret
+data/serialized
 !data/secret/secret_test.json
 monitoring
 resources

.github/workflows/ubuntu-monitoring.yml → .github/workflows/ubuntu-special.yml

@@ -1,4 +1,4 @@
-name: Monitoring
+name: Special
 
 on:
   push:
@@ -7,14 +7,14 @@ on:
   pull_request:
 
 jobs:
-  ubuntu-monitoring-build:
-    name: Build on Ubuntu with monitoring support
+  ubuntu-special-build:
+    name: Build on Ubuntu with monitoring / protobuf support
     runs-on: ubuntu-latest
     strategy:
       matrix:
         compiler: [g++-11]
         buildmode: [Debug]
-        build-prometheus-from-source: [0, 1]
+        build-special-from-source: [0, 1]
 
     steps:
       - name: Checkout repository code
@@ -38,15 +38,15 @@ jobs:
           ninja
 
           sudo cmake --install .
-        if: matrix.build-prometheus-from-source == 0
+        if: matrix.build-special-from-source == 0
 
       - name: Create Build Environment
         run: cmake -E make_directory ${{github.workspace}}/build
 
       - name: Configure CMake
         working-directory: ${{github.workspace}}/build
         shell: bash
-        run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{matrix.buildmode}} -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCCT_BUILD_PROMETHEUS_FROM_SRC=${{matrix.build-prometheus-from-source}} -GNinja
+        run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{matrix.buildmode}} -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCCT_BUILD_PROMETHEUS_FROM_SRC=${{matrix.build-special-from-source}} -DCCT_ENABLE_PROTO=${{matrix.build-special-from-source}} -GNinja
 
       - name: Build
         working-directory: ${{github.workspace}}/build

.gitignore

@@ -66,6 +66,7 @@ cscope*
 data/cache
 data/log
 data/secret
+data/serialized
 data/static/exchangeconfig.json
 data/static/generalconfig.json
 monitoring/data/grafana/*

CMakeLists.txt

@@ -36,6 +36,7 @@ option(CCT_ENABLE_TESTS "Build the unit tests" ${MAIN_PROJECT})
 option(CCT_BUILD_EXEC "Build an executable instead of a static library" ${MAIN_PROJECT})
 option(CCT_ENABLE_ASAN "Compile with AddressSanitizer" ${CCT_ASAN_BUILD})
 option(CCT_ENABLE_CLANG_TIDY "Compile with clang-tidy checks" OFF)
+option(CCT_ENABLE_PROTO "Compile with protobuf support (to export data to the outside world)" ON)
 option(CCT_BUILD_PROMETHEUS_FROM_SRC "Fetch and build from prometheus-cpp sources" OFF)
 
 set(CCT_DATA_DIR "${CMAKE_CURRENT_SOURCE_DIR}/data" CACHE PATH "Needed data directory for coincenter. Can also be overriden at runtime with this environment variable")
@@ -135,6 +136,34 @@ else()
   endif()
 endif()
 
+if(CCT_ENABLE_PROTO)
+  find_package(Protobuf CONFIG)
+  if(protobuf_FOUND)
+    message(STATUS "Linking with protobuf ${protobuf_VERSION}")
+  else()
+    set(PROTOBUF_VERSION v25.3)
+    if (MSVC)
+      protobuf v25 does not compile with MSVC: https://github.com/protocolbuffers/protobuf/issues/14602
+      set(PROTOBUF_VERSION v26.0-rc2)
+    endif()
+
+    message(STATUS "Compiling protobuf ${PROTOBUF_VERSION} from sources")
+
+    set(protobuf_BUILD_TESTS OFF)
+    set(ABSL_PROPAGATE_CXX_STD ON)
+
+    FetchContent_Declare(
+      protobuf
+      GIT_REPOSITORY https://github.com/protocolbuffers/protobuf.git
+      GIT_TAG ${PROTOBUF_VERSION}
+    )
+    FetchContent_MakeAvailable(protobuf)
+
+    include(${protobuf_SOURCE_DIR}/cmake/protobuf-generate.cmake)
+
+  endif()
+endif()
+
 # Unit Tests
 
 #[[ Create an executable
@@ -250,13 +279,20 @@ if(CCT_ENABLE_PROMETHEUS)
   add_compile_definitions(CCT_ENABLE_PROMETHEUS)
 endif()
 
+if(CCT_ENABLE_PROTO)
+  add_compile_definitions(CCT_ENABLE_PROTO)
+  add_compile_definitions("CCT_PROTOBUF_VERSION=\"${PROTOBUF_VERSION}\"")
+endif()
+
 # Link to sub folders CMakeLists.txt, from the lowest level to the highest level for documentation
 # (beware of cyclic dependencies)
 add_subdirectory(src/tech)
 add_subdirectory(src/monitoring)
 add_subdirectory(src/http-request)
 add_subdirectory(src/objects)
+add_subdirectory(src/serialization)
 add_subdirectory(src/api-objects)
+add_subdirectory(src/trading)
 add_subdirectory(src/api)
 add_subdirectory(src/engine)
 add_subdirectory(src/main)

CONFIG.md

@@ -150,6 +150,7 @@ Refer to the hardcoded default json example as a model in case of doubt.
 | *query*     | **updateFrequency.depositWallet**  | Duration string (ex: `1min`)                                                     | Minimum duration between two consecutive requests of deposit information (including wallet)                                                                                                                                                                                                                                                                                                                              |
 | *query*     | **updateFrequency.currencyInfo**   | Duration string (ex: `4h`)                                                       | Minimum duration between two consecutive requests of dynamic currency info retrieval on Bithumb only (used for place order)                                                                                                                                                                                                                                                                                              |
 | *query*     | **placeSimulateRealOrder**         | Boolean (`true` or `false`)                                                      | If `true`, in trade simulation mode (with `--sim`) exchanges which do not support simulated mode in place order will actually place a real order, with the following characteristics: <ul><li>trade strategy forced to `maker`</li><li>price will be changed to a maximum for a sell, to a minimum for a buy</li></ul> This will allow place of a 'real' order that cannot be matched in practice (if it is, lucky you!) |
+| *query*     | **marketDataSerialization**        | Boolean (`true` or `false`)                                                      | If `true` and `coincenter` is compiled with **protobuf** support, some market data will automatically be exported in the `data/serialization` directory (`orderbook` and `last-trades`) for a long term storage                                                                                                                                                                                                          |
 | *query*     | **multiTradeAllowedByDefault**     | Boolean (`true` or `false`)                                                      | If `true`, [multi-trade](README.md#multi-trade) will be allowed by default for `trade`, `buy` and `sell`. It can be overridden at command line level with `--no-multi-trade` and `--multi-trade`.                                                                                                                                                                                                                        |
 | *query*     | **validateApiKey**                 | Boolean (`true` or `false`)                                                      | If `true`, each loaded private key will be tested at start of the program. In case of a failure, it will be removed from the list of private accounts loaded by `coincenter`, so that later queries do not consider it instead of raising a runtime exception. The downside is that it will make an additional check that will make startup slower.                                                                      |  |
 | *tradefees* | **maker**                          | String as decimal number representing a percentage (for instance, "0.15")        | Trade fees occurring when a maker order is matched                                                                                                                                                                                                                                                                                                                                                                       |

Dockerfile

@@ -4,7 +4,7 @@ FROM ubuntu:22.04 AS build
 # Install base & build dependencies, needed certificates for curl to work with https
 RUN apt update && \
     apt upgrade -y && \
-    apt install build-essential curl libcurl4-gnutls-dev libssl-dev cmake git ca-certificates gzip -y && \
+    apt install build-essential curl libcurl4-gnutls-dev libssl-dev cmake git ca-certificates gzip --no-install-recommends -y && \
     curl -L -o /usr/local/bin/ninja.gz https://github.com/ninja-build/ninja/releases/latest/download/ninja-linux.zip && \
     gunzip /usr/local/bin/ninja.gz && \
     chmod a+x /usr/local/bin/ninja
@@ -23,12 +23,14 @@ ARG BUILD_MODE=Release
 ARG BUILD_TEST=0
 ARG BUILD_ASAN=0
 ARG BUILD_WITH_PROMETHEUS=1
+ARG BUILD_WITH_PROTOBUF=1
 
 # Build and launch tests if any
 RUN cmake -DCMAKE_BUILD_TYPE=${BUILD_MODE} \
     -DCCT_ENABLE_TESTS=${BUILD_TEST} \
     -DCCT_ENABLE_ASAN=${BUILD_ASAN} \
     -DCCT_BUILD_PROMETHEUS_FROM_SRC=${BUILD_WITH_PROMETHEUS} \
+    -DCCT_ENABLE_PROTO=${BUILD_WITH_PROTOBUF} \
     -GNinja .. && \
     ninja && \
     if [ "$BUILD_TEST" = "1" -o "$BUILD_TEST" = "ON" ]; then \
@@ -56,4 +58,4 @@ COPY --from=build /app/bin/coincenter /app/coincenter
 # 'data' directory of host machine can be mounted when launching the container.
 # To do this, you can use --mount option:
 # docker run --mount type=bind,source=<path-to-data-dir-on-host>,target=/app/data sjanel/coincenter
-ENTRYPOINT [ "/app/coincenter" ]
+ENTRYPOINT [ "/app/coincenter" ]

alpine.Dockerfile

@@ -2,7 +2,7 @@
 FROM alpine:3.19.1 AS build
 
 # Install base & build dependencies, needed certificates for curl to work with https
-RUN apk add --update --upgrade --no-cache g++ libc-dev curl-dev cmake ninja git ca-certificates
+RUN apk add --update --upgrade --no-cache g++ linux-headers libc-dev curl-dev cmake ninja git ca-certificates
 
 # Set default directory for application
 WORKDIR /app
@@ -18,12 +18,14 @@ ARG BUILD_MODE=Release
 ARG BUILD_TEST=0
 ARG BUILD_ASAN=0
 ARG BUILD_WITH_PROMETHEUS=1
+ARG BUILD_WITH_PROTOBUF=1
 
 # Build and launch tests if any
 RUN cmake -DCMAKE_BUILD_TYPE=${BUILD_MODE} \
     -DCCT_ENABLE_TESTS=${BUILD_TEST} \
     -DCCT_ENABLE_ASAN=${BUILD_ASAN} \
     -DCCT_BUILD_PROMETHEUS_FROM_SRC=${BUILD_WITH_PROMETHEUS} \
+    -DCCT_ENABLE_PROTO=${BUILD_WITH_PROTOBUF} \
     -GNinja .. && \
     ninja && \
     if [ "$BUILD_TEST" = "1" -o "$BUILD_TEST" = "ON" ]; then \

src/api-objects/include/closed-order.hpp

@@ -0,0 +1,29 @@
+#pragma once
+
+#include <compare>
+#include <string_view>
+
+#include "cct_string.hpp"
+#include "cct_type_traits.hpp"
+#include "market.hpp"
+#include "monetaryamount.hpp"
+#include "order.hpp"
+#include "orderid.hpp"
+#include "timedef.hpp"
+#include "tradeside.hpp"
+
+namespace cct {
+
+class ClosedOrder : public Order {
+ public:
+  ClosedOrder(OrderId id, MonetaryAmount matchedVolume, MonetaryAmount price, TimePoint placedTime,
+              TimePoint matchedTime, TradeSide side);
+
+  TimePoint matchedTime() const { return _matchedTime; }
+
+  string matchedTimeStr() const;
+
+ private:
+  TimePoint _matchedTime;
+};
+}  // namespace cct

src/api-objects/include/exchangeprivateapitypes.hpp

@@ -2,8 +2,9 @@
 
 #include "cct_flatset.hpp"
 #include "cct_vector.hpp"
+#include "closed-order.hpp"
 #include "deposit.hpp"
-#include "order.hpp"
+#include "opened-order.hpp"
 #include "tradedamounts.hpp"
 #include "withdraw.hpp"
 
@@ -14,8 +15,10 @@ using DepositsSet = FlatSet<Deposit>;
 using Withdraws = vector<Withdraw>;
 using WithdrawsSet = FlatSet<Withdraw>;
 
-using Orders = vector<Order>;
-using OrdersSet = FlatSet<Order>;
+using OpenedOrderVector = vector<OpenedOrder>;
+using OpenedOrderSet = FlatSet<OpenedOrder>;
+
+using ClosedOrders = vector<ClosedOrder>;
 
 using TradedAmountsVector = vector<TradedAmounts>;
 

src/api-objects/include/exchangepublicapitypes.hpp

@@ -12,9 +12,12 @@
 #include "publictrade.hpp"
 
 namespace cct {
+
 using MarketSet = FlatSet<Market>;
 using MarketOrderBookMap = std::unordered_map<Market, MarketOrderBook>;
 using MarketPriceMap = std::unordered_map<Market, MonetaryAmount>;
 using MarketsPath = SmallVector<Market, 3>;
 using TradesVector = vector<PublicTrade>;
-}  // namespace cct
+using MarketOrderBookVector = vector<MarketOrderBook>;
+
+}  // namespace cct

src/api-objects/include/opened-order.hpp

@@ -0,0 +1,22 @@
+#pragma once
+
+#include "monetaryamount.hpp"
+#include "order.hpp"
+#include "orderid.hpp"
+#include "timedef.hpp"
+#include "tradeside.hpp"
+
+namespace cct {
+
+class OpenedOrder : public Order {
+ public:
+  OpenedOrder(OrderId id, MonetaryAmount matchedVolume, MonetaryAmount remainingVolume, MonetaryAmount price,
+              TimePoint placedTime, TradeSide side);
+
+  MonetaryAmount originalVolume() const { return matchedVolume() + _remainingVolume; }
+  MonetaryAmount remainingVolume() const { return _remainingVolume; }
+
+ private:
+  MonetaryAmount _remainingVolume;
+};
+}  // namespace cct

src/api-objects/include/order.hpp

@@ -15,25 +15,13 @@ namespace cct {
 
 class Order {
  public:
-  Order(const char *id, MonetaryAmount matchedVolume, MonetaryAmount remainingVolume, MonetaryAmount price,
-        TimePoint placedTime, TradeSide side)
-      : Order(OrderId(id), matchedVolume, remainingVolume, price, placedTime, side) {}
-
-  Order(std::string_view id, MonetaryAmount matchedVolume, MonetaryAmount remainingVolume, MonetaryAmount price,
-        TimePoint placedTime, TradeSide side);
-
-  Order(OrderId &&id, MonetaryAmount matchedVolume, MonetaryAmount remainingVolume, MonetaryAmount price,
-        TimePoint placedTime, TradeSide side);
-
-  MonetaryAmount originalVolume() const { return _matchedVolume + _remainingVolume; }
-  MonetaryAmount matchedVolume() const { return _matchedVolume; }
-  MonetaryAmount remainingVolume() const { return _remainingVolume; }
-  MonetaryAmount price() const { return _price; }
+  TimePoint placedTime() const { return _placedTime; }
 
   OrderId &id() { return _id; }
   const OrderId &id() const { return _id; }
 
-  TimePoint placedTime() const { return _placedTime; }
+  MonetaryAmount matchedVolume() const { return _matchedVolume; }
+  MonetaryAmount price() const { return _price; }
 
   TradeSide side() const { return _side; }
 
@@ -47,11 +35,13 @@ class Order {
 
   using trivially_relocatable = is_trivially_relocatable<OrderId>::type;
 
+ protected:
+  Order(OrderId id, MonetaryAmount matchedVolume, MonetaryAmount price, TimePoint placedTime, TradeSide side);
+
  private:
   TimePoint _placedTime;
   OrderId _id;  // exchange internal id, format specific to each exchange
   MonetaryAmount _matchedVolume;
-  MonetaryAmount _remainingVolume;
   MonetaryAmount _price;
   TradeSide _side;
 };

src/api-objects/src/closed-order.cpp

@@ -0,0 +1,19 @@
+#include "closed-order.hpp"
+
+#include <utility>
+
+#include "cct_string.hpp"
+#include "monetaryamount.hpp"
+#include "order.hpp"
+#include "timedef.hpp"
+#include "timestring.hpp"
+#include "tradeside.hpp"
+
+namespace cct {
+
+ClosedOrder::ClosedOrder(OrderId id, MonetaryAmount matchedVolume, MonetaryAmount price, TimePoint placedTime,
+                         TimePoint matchedTime, TradeSide side)
+    : Order(std::move(id), matchedVolume, price, placedTime, side), _matchedTime(matchedTime) {}
+
+string ClosedOrder::matchedTimeStr() const { return ToString(_matchedTime); }
+}  // namespace cct

src/api-objects/src/opened-order.cpp

@@ -0,0 +1,11 @@
+#include "opened-order.hpp"
+
+#include <utility>
+
+namespace cct {
+
+OpenedOrder::OpenedOrder(string id, MonetaryAmount matchedVolume, MonetaryAmount remainingVolume, MonetaryAmount price,
+                         TimePoint placedTime, TradeSide side)
+    : Order(std::move(id), matchedVolume, price, placedTime, side), _remainingVolume(remainingVolume) {}
+
+}  // namespace cct

src/api-objects/src/order.cpp

@@ -11,23 +11,8 @@
 
 namespace cct {
 
-Order::Order(std::string_view id, MonetaryAmount matchedVolume, MonetaryAmount remainingVolume, MonetaryAmount price,
-             TimePoint placedTime, TradeSide side)
-    : _placedTime(placedTime),
-      _id(id),
-      _matchedVolume(matchedVolume),
-      _remainingVolume(remainingVolume),
-      _price(price),
-      _side(side) {}
-
-Order::Order(string &&id, MonetaryAmount matchedVolume, MonetaryAmount remainingVolume, MonetaryAmount price,
-             TimePoint placedTime, TradeSide side)
-    : _placedTime(placedTime),
-      _id(std::move(id)),
-      _matchedVolume(matchedVolume),
-      _remainingVolume(remainingVolume),
-      _price(price),
-      _side(side) {}
+Order::Order(string id, MonetaryAmount matchedVolume, MonetaryAmount price, TimePoint placedTime, TradeSide side)
+    : _placedTime(placedTime), _id(std::move(id)), _matchedVolume(matchedVolume), _price(price), _side(side) {}
 
 std::string_view Order::sideStr() const { return SideStr(_side); }
 

src/api/common/include/exchangeprivateapi.hpp

@@ -61,7 +61,8 @@ class ExchangePrivate : public ExchangeBase {
   virtual bool canGenerateDepositAddress() const = 0;
 
   /// Get opened orders filtered according to given constraints
-  virtual Orders queryOpenedOrders(const OrdersConstraints &openedOrdersConstraints = OrdersConstraints()) = 0;
+  virtual OpenedOrderVector queryOpenedOrders(
+      const OrdersConstraints &openedOrdersConstraints = OrdersConstraints()) = 0;
 
   /// Cancel all opened orders on the exchange that matches given constraints
   /// @return number of opened orders cancelled