diff --git a/.github/workflows/ubuntu-monitoring.yml b/.github/workflows/ubuntu-special.yml
similarity index 57%
rename from .github/workflows/ubuntu-monitoring.yml
rename to .github/workflows/ubuntu-special.yml
index 1a269f50..1940abef 100644
--- a/.github/workflows/ubuntu-monitoring.yml
+++ b/.github/workflows/ubuntu-special.yml
@@ -1,4 +1,4 @@
-name: Monitoring
+name: Special
on:
push:
@@ -7,14 +7,15 @@ on:
pull_request:
jobs:
- ubuntu-monitoring-build:
- name: Build on Ubuntu with monitoring support
+ ubuntu-special-build:
+ name: Build on Ubuntu with monitoring / protobuf support
runs-on: ubuntu-latest
strategy:
matrix:
compiler: [g++-11]
buildmode: [Debug]
- build-prometheus-from-source: [0, 1]
+ build-special-from-source: [0, 1]
+ prometheus-options: ["-DBUILD_SHARED_LIBS=ON -DENABLE_PULL=OFF -DENABLE_PUSH=ON -DENABLE_COMPRESSION=OFF -DENABLE_TESTING=OFF"]
steps:
- name: Checkout repository code
@@ -35,19 +36,21 @@ jobs:
mkdir _build
cd _build
- cmake .. -DBUILD_SHARED_LIBS=ON -DENABLE_PULL=OFF -DENABLE_PUSH=ON -DENABLE_COMPRESSION=OFF -DENABLE_TESTING=OFF -DCMAKE_BUILD_TYPE=${{matrix.buildmode}} -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -GNinja
+ cmake .. ${{matrix.prometheus-options}} -GNinja
cmake --build .
sudo cmake --install .
- if: matrix.build-prometheus-from-source == 0
-
- - name: Create Build Environment
- run: cmake -E make_directory ${{github.workspace}}/build
+ if: matrix.build-special-from-source == 0
+ env:
+ CXX: ${{matrix.compiler}}
+ CMAKE_BUILD_TYPE: ${{matrix.buildmode}}
- name: Configure CMake
- working-directory: ${{github.workspace}}/build
shell: bash
- run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{matrix.buildmode}} -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCCT_BUILD_PROMETHEUS_FROM_SRC=${{matrix.build-prometheus-from-source}} -DCCT_ENABLE_ASAN=OFF -GNinja
+ run: cmake -S . -B build ${{matrix.prometheus-options}} -DCCT_BUILD_PROMETHEUS_FROM_SRC=${{matrix.build-special-from-source}} -DCCT_ENABLE_PROTO=${{matrix.build-special-from-source}} -DCCT_ENABLE_ASAN=OFF -GNinja
+ env:
+ CXX: ${{matrix.compiler}}
+ CMAKE_BUILD_TYPE: ${{matrix.buildmode}}
- name: Build
working-directory: ${{github.workspace}}/build
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
index 3de55c48..ba3d2578 100644
--- a/.github/workflows/windows.yml
+++ b/.github/workflows/windows.yml
@@ -19,7 +19,7 @@ jobs:
- name: Install dependencies
run: |
- vcpkg install curl
+ vcpkg install curl protobuf
- name: End vcpkg install
run: |
@@ -27,7 +27,10 @@ jobs:
- name: Configure CMake
run: |
- cmake -DCMAKE_TOOLCHAIN_FILE="C:/vcpkg/scripts/buildsystems/vcpkg.cmake" -DCMAKE_BUILD_TYPE=${{matrix.buildmode}} -S . -B build
+ cmake -S . -B build
+ env:
+ CMAKE_BUILD_TYPE: ${{matrix.buildmode}}
+ CMAKE_TOOLCHAIN_FILE: "C:/vcpkg/scripts/buildsystems/vcpkg.cmake"
- name: Build
working-directory: ${{github.workspace}}/build
diff --git a/.gitignore b/.gitignore
index 176b2be7..e8db4630 100644
--- a/.gitignore
+++ b/.gitignore
@@ -64,6 +64,7 @@ data/cache
data/log
data/secret
!data/secret/secret_test.json
+data/serialized
data/static/exchangeconfig.json
data/static/generalconfig.json
monitoring/data/grafana/*
diff --git a/CMakeLists.txt b/CMakeLists.txt
index c6f83236..3b735119 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -38,6 +38,7 @@ option(CCT_ENABLE_TESTS "Build the unit tests" ${MAIN_PROJECT})
option(CCT_BUILD_EXEC "Build an executable instead of a static library" ${MAIN_PROJECT})
option(CCT_ENABLE_ASAN "Compile with AddressSanitizer" ${CCT_ASAN_BUILD})
option(CCT_ENABLE_CLANG_TIDY "Compile with clang-tidy checks" OFF)
+option(CCT_ENABLE_PROTO "Compile with protobuf support (to export data to the outside world)" ON)
option(CCT_BUILD_PROMETHEUS_FROM_SRC "Fetch and build from prometheus-cpp sources" OFF)
set(CCT_DATA_DIR "${CMAKE_CURRENT_SOURCE_DIR}/data" CACHE PATH "Needed data directory for coincenter. Can also be overriden at runtime with this environment variable")
@@ -141,6 +142,31 @@ if(NOT spdlog_FOUND)
FetchContent_MakeAvailable(spdlog)
endif()
+# protobuf - serialization / deserialization library
+if(CCT_ENABLE_PROTO)
+ find_package(Protobuf CONFIG)
+ if(Protobuf_FOUND)
+ message(STATUS "Linking with protobuf ${protobuf_VERSION}")
+ else()
+ set(PROTOBUF_VERSION v26.1)
+
+ message(STATUS "Compiling protobuf ${PROTOBUF_VERSION} from sources")
+
+ set(protobuf_BUILD_TESTS OFF)
+ set(ABSL_PROPAGATE_CXX_STD ON)
+
+ FetchContent_Declare(
+ protobuf
+ GIT_REPOSITORY https://github.com/protocolbuffers/protobuf.git
+ GIT_TAG ${PROTOBUF_VERSION}
+ )
+ FetchContent_MakeAvailable(protobuf)
+
+ include(${protobuf_SOURCE_DIR}/cmake/protobuf-generate.cmake)
+
+ endif()
+endif()
+
# Unit Tests
include(cmake/AddUnitTest.cmake)
@@ -179,13 +205,20 @@ if(CCT_ENABLE_PROMETHEUS)
add_compile_definitions(CCT_ENABLE_PROMETHEUS)
endif()
+if(CCT_ENABLE_PROTO)
+ add_compile_definitions(CCT_ENABLE_PROTO)
+ add_compile_definitions("CCT_PROTOBUF_VERSION=\"${PROTOBUF_VERSION}\"")
+endif()
+
# Link to sub folders CMakeLists.txt, from the lowest level to the highest level for documentation
# (beware of cyclic dependencies)
add_subdirectory(src/tech)
add_subdirectory(src/monitoring)
add_subdirectory(src/http-request)
add_subdirectory(src/objects)
+add_subdirectory(src/serialization)
add_subdirectory(src/api-objects)
+add_subdirectory(src/trading)
add_subdirectory(src/api)
add_subdirectory(src/engine)
add_subdirectory(src/main)
diff --git a/CONFIG.md b/CONFIG.md
index f7302a28..02f1ca4b 100644
--- a/CONFIG.md
+++ b/CONFIG.md
@@ -48,17 +48,18 @@ Configures the logging, tracking activity of relevant commands, and console outp
#### General options description
-| Name | Value | Description |
-| ---------------------------------------------- | ---------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| **apiOutputType** | String among {`off`, `table`, `json`} | Configure the default output type of coincenter (can be overridden by command line)queries |
-| **fiatConversion.rate** | Duration string (ex: `8h`) | Minimum duration between two consecutive requests of the same fiat conversion |
-| **log.activityTracking.commandTypes** | Array of strings (ex: `["Buy", "Sell"]`) | Array of command types whose output will be stored to activity history files. |
-| **log.activityTracking.dateFileNameFormat** | String (ex: `%Y-%m` for month split) | Defines the date string format suffix used by activity history files. The string should be compatible with [std::strftime](https://en.cppreference.com/w/cpp/chrono/c/strftime). Old data will never be clean-up by `coincenter` (as it may contain important data). User should manage the clean-up / storage. |
-| **log.consoleLevel** | String | Defines the log level for standard output. Can be {'off', 'critical', 'error', 'warning', 'info', 'debug', 'trace'} |
-| **log.fileLevel** | String | Defines the log level in files. Can be {'off', 'critical', 'error', 'warning', 'info', 'debug', 'trace'} |
-| **log.maxFileSize** | String (ex: `5Mi` for 5 Megabytes) | Defines in bytes the maximum logging file size. A string representation of an integral, possibly with one suffix ending such as k, M, G, T (1k multipliers) or Ki, Mi, Gi, Ti (1024 multipliers) are supported. |
-| **log.maxNbFiles** | Integer | Number of maximum rotating files for log in files |
-| **requests.concurrency.nbMaxParallelRequests** | Integer | Size of the thread pool that makes exchange requests. |
+| Name | Value | Description |
+| -------------------------------------------------------- | ---------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| **apiOutputType** | String among {`off`, `table`, `json`} | Configure the default output type of coincenter (can be overridden by command line)queries |
+| **fiatConversion.rate** | Duration string (ex: `8h`) | Minimum duration between two consecutive requests of the same fiat conversion |
+| **log.activityTracking.commandTypes** | Array of strings (ex: `["Buy", "Sell"]`) | Array of command types whose output will be stored to activity history files. |
+| **log.activityTracking.dateFileNameFormat** | String (ex: `%Y-%m` for month split) | Defines the date string format suffix used by activity history files. The string should be compatible with [std::strftime](https://en.cppreference.com/w/cpp/chrono/c/strftime). Old data will never be clean-up by `coincenter` (as it may contain important data). User should manage the clean-up / storage. |
+| **log.consoleLevel** | String | Defines the log level for standard output. Can be {'off', 'critical', 'error', 'warning', 'info', 'debug', 'trace'} |
+| **log.fileLevel** | String | Defines the log level in files. Can be {'off', 'critical', 'error', 'warning', 'info', 'debug', 'trace'} |
+| **log.maxFileSize** | String (ex: `5Mi` for 5 Megabytes) | Defines in bytes the maximum logging file size. A string representation of an integral, possibly with one suffix ending such as k, M, G, T (1k multipliers) or Ki, Mi, Gi, Ti (1024 multipliers) are supported. |
+| **log.maxNbFiles** | Integer | Number of maximum rotating files for log in files |
+| **requests.concurrency.nbMaxParallelRequests** | Integer | Size of the thread pool that makes exchange requests. |
+| **trading.automation.deserialization.loadChunkDuration** | Duration string (ex: `1w`) | Time window duration of historic stored data loaded and replayed at once given to the trading engine |
### static/exchangeconfig.json
@@ -151,6 +152,7 @@ Refer to the hardcoded default json example as a model in case of doubt.
| *query* | **updateFrequency.depositWallet** | Duration string (ex: `1min`) | Minimum duration between two consecutive requests of deposit information (including wallet) |
| *query* | **updateFrequency.currencyInfo** | Duration string (ex: `4h`) | Minimum duration between two consecutive requests of dynamic currency info retrieval on Bithumb only (used for place order) |
| *query* | **placeSimulateRealOrder** | Boolean (`true` or `false`) | If `true`, in trade simulation mode (with `--sim`) exchanges which do not support simulated mode in place order will actually place a real order, with the following characteristics:
- trade strategy forced to `maker`
- price will be changed to a maximum for a sell, to a minimum for a buy
This will allow place of a 'real' order that cannot be matched in practice (if it is, lucky you!) |
+| *query* | **marketDataSerialization** | Boolean (`true` or `false`) | If `true` and `coincenter` is compiled with **protobuf** support, some market data will automatically be exported in the `data/serialization` directory (`orderbook` and `last-trades`) for a long term storage |
| *query* | **multiTradeAllowedByDefault** | Boolean (`true` or `false`) | If `true`, [multi-trade](README.md#multi-trade) will be allowed by default for `trade`, `buy` and `sell`. It can be overridden at command line level with `--no-multi-trade` and `--multi-trade`. |
| *query* | **validateApiKey** | Boolean (`true` or `false`) | If `true`, each loaded private key will be tested at start of the program. In case of a failure, it will be removed from the list of private accounts loaded by `coincenter`, so that later queries do not consider it instead of raising a runtime exception. The downside is that it will make an additional check that will make startup slower. | |
| *tradefees* | **maker** | String as decimal number representing a percentage (for instance, "0.15") | Trade fees occurring when a maker order is matched |
diff --git a/Dockerfile b/Dockerfile
index 91feb38d..dd043f0c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -5,6 +5,7 @@ ARG BUILD_MODE=Release
ARG BUILD_TEST=0
ARG BUILD_ASAN=0
ARG BUILD_WITH_PROMETHEUS=1
+ARG BUILD_WITH_PROTOBUF=1
# Install base & build dependencies, needed certificates for curl to work with https
RUN apt update && \
@@ -41,6 +42,7 @@ RUN cmake -DCMAKE_BUILD_TYPE=${BUILD_MODE} \
-DCCT_ENABLE_TESTS=${BUILD_TEST} \
-DCCT_ENABLE_ASAN=${BUILD_ASAN} \
-DCCT_BUILD_PROMETHEUS_FROM_SRC=${BUILD_WITH_PROMETHEUS} \
+ -DCCT_ENABLE_PROTO=${BUILD_WITH_PROTOBUF} \
-GNinja ..
# Build
diff --git a/INSTALL.md b/INSTALL.md
index a9bbd9d3..69b12b85 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -135,6 +135,7 @@ The minimum tested version is cmake `3.15`, but it's recommended that you use th
| `CCT_BUILD_EXEC` | `ON` if main project | Build an executable instead of a static library |
| `CCT_ENABLE_ASAN` | `ON` if Debug mode | Compile with AddressSanitizer |
| `CCT_ENABLE_CLANG_TIDY` | `ON` if Debug mode and `clang-tidy` is found in `PATH` | Compile with clang-tidy checks |
+| `CCT_ENABLE_PROTO` | `ON` | Compile with protobuf support |
Example on Linux: to compile it in `Release` mode and `ninja` generator
diff --git a/README.md b/README.md
index 9cce03c7..7aff305b 100644
--- a/README.md
+++ b/README.md
@@ -38,7 +38,16 @@ Main features:
- Cancel opened orders
- Withdraw (with check at destination that funds are well received)
- Dust sweeper
-
+
+## Market data storage & replay
+
+`coincenter` is able to store the following market data in serialized [binary protobuf](https://protobuf.dev/) format for offline trading algorithm replay:
+
+- Market order book
+- Public trades
+
+Refer to the dedicated [documentation page](TRADING.md) for more information.
+
## Supported exchanges
| Exchange | Link |
@@ -56,6 +65,7 @@ Main features:
- [coincenter](#coincenter)
- [Market Data](#market-data)
- [Account requests](#account-requests)
+ - [Market data storage \& replay](#market-data-storage--replay)
- [Supported exchanges](#supported-exchanges)
- [About](#about)
- [Installation](#installation)
@@ -1285,4 +1295,4 @@ Possible output:
| kucoin | 6090943.32410022531 SHIB |
| upbit | 6084383.631243834 SHIB |
+----------+------------------------------+
-```
\ No newline at end of file
+```
diff --git a/TRADING.md b/TRADING.md
new file mode 100644
index 00000000..017712ae
--- /dev/null
+++ b/TRADING.md
@@ -0,0 +1,108 @@
+# Trading
+
+`coincenter` is able to serialize market data and deserialize it later on for future usage.
+
+## Overview
+
+Currently, these two sources of data are serializable into [protobuf](https://protobuf.dev/) objects:
+
+- Market order book
+- Public trades
+
+## Configuration
+
+### Compilation
+
+By default, `coincenter` will be built with **protobuf** support (controlled by `cmake` flag `CCT_ENABLE_PROTO` that defaults to `ON`).
+
+It will try to link to a known installation of **protobuf** if found on current system (oldest tested version is v25, make sure to use this version at least), otherwise it will download and compile it from sources.
+
+### Serialization configuration
+
+To be able to serialize market data on disk, make sure that you set the **marketDataSerialization** variable to `true` in `exchangeconfig.json` for the exchanges you would like to interact with.
+See the [exchange configuration part](CONFIG.md#exchanges-options-description) for more information about how to configure it.
+
+## Serialization of market data
+
+The data will be organized by exchange, then market (asset pair, for instance `BTC-USD`), and finally dates (with directories from **year**, **month**, and finally **day** and files as **hours**).
+
+All will be stored in `coincenter` data directory, under `serialized` sub folder.
+
+Here is an example of the structure of files you will obtain:
+
+```bash
+data/serialized//
+├── binance
+│ ├── BTC-EUR
+│ │ └── 2024
+│ │ └── 01
+│ │ ├── 11
+│ │ │ └── 22:00:00_22:59:59.binpb
+│ │ ├── 12
+│ │ │ ├── 08:00:00_08:59:59.binpb
+│ │ │ ├── 09:00:00_09:59:59.binpb
+│ │ │ └── 11:00:00_11:59:59.binpb
+│ │ └── 14
+│ │ ├── 07:00:00_07:59:59.binpb
+│ │ └── 08:00:00_08:59:59.binpb
+│ ├── ETH-USDT
+│ │ └── 2024
+│ │ ├── 01
+│ │ │ ├── 09
+│ │ │ │ ├── 08:00:00_08:59:59.binpb
+│ │ │ │ ├── 09:00:00_09:59:59.binpb
+│ │ │ │ ├── 10:00:00_10:59:59.binpb
+│ │ │ │ ├── 11:00:00_11:59:59.binpb
+├── huobi
+│ ├── ADA-USDT
+│ │ └── 2024
+│ │ └── 02
+│ │ └── 10
+│ │ └── 16:00:00_16:59:59.binpb
+│ ├── BTC-EUR
+│ │ └── 2024
+│ │ └── 01
+│ │ ├── 11
+│ │ │ └── 22:00:00_22:59:59.binpb
+....
+```
+
+To retrieve market data, it's possible to either use multi-commands with both `orderbook` and `last-trades` commands stacked together, or you can use the more handy `market-data` option that is basically a combination of the two without the output by default (it has been created only for serialization purposes).
+
+For instance, to retrieve continuously data and serialize them indefinitely, you can use the following command:
+
+```bash
+coincenter -r --repeat-time 2s --log warning \
+ market-data btc-eur,binance \
+ market-data eth-usdt,kucoin \
+ market-data ada-usdt,huobi \
+ market-data btc-eur,kraken
+```
+
+Note the usage of the `-r` (repeat option) to keep querying the data as long as you leave `coincenter` up. It's a good idea to also limit the number of logs with setting console log level to `warning` but not mandatory.
+
+With this command running for an extended period of time, you should obtain a list of files like in the above example.
+
+Stacking `market-data` commands together with different exchanges (like in the above example) will allow `coincenter` to perform the queries in parallel, ensuring optimal frequency of data updates. This optimization may be implemented for other commands in the future, but it's currently supported only for `market-data`.
+
+### Graceful shutdown
+
+Data is flushed on the disk at regular intervals (around 10 minutes). If you wish to restart / shutdown `coincenter` with an infinite `repeat` command to store continuously market data, you can send `SIGINT` or `SIGTERM` so that `coincenter` can gracefully stop after current request and flush its remaining data on disk before shutdown.
+
+## Replaying historic market data
+
+Of course, serialization is useful only if we re-use the data one day. Being **protobuf**, not only `coincenter` could read them, but also other tools, but `coincenter` is also able to read this data.
+
+If you want to use a third-party tool to read this data, locate the `.proto` files in the `src` directory for your external program to be able to deserialize the `.binpb` files.
+
+We will focus here on `coincenter` features concerning this data.
+
+### BETA - Testing trading algorithms
+
+`coincenter` embeds a trading simulator engine that is able to be used for any custom trading algorithm that would derive from the interface.
+
+This trading simulator will read chunks of historic data stored in **protobuf** and inject them in trading algorithms.
+
+Locate the `AbstractMarketTrader` class and derive it - you need to return a `TraderCommand` for each market order book and a list of last public trades that occurred at this specific point of time.
+
+TODO: extend this documentation.
diff --git a/alpine.Dockerfile b/alpine.Dockerfile
index b4a159d4..4e6d9ce2 100644
--- a/alpine.Dockerfile
+++ b/alpine.Dockerfile
@@ -5,9 +5,10 @@ ARG BUILD_MODE=Release
ARG BUILD_TEST=0
ARG BUILD_ASAN=0
ARG BUILD_WITH_PROMETHEUS=1
+ARG BUILD_WITH_PROTOBUF=1
# Install base & build dependencies, needed certificates for curl to work with https
-RUN apk add --update --upgrade --no-cache g++ libc-dev openssl-dev curl-dev cmake ninja git ca-certificates
+RUN apk add --update --upgrade --no-cache linux-headers g++ libc-dev openssl-dev curl-dev cmake ninja git ca-certificates
# Copy source files
WORKDIR /app/src
@@ -39,6 +40,7 @@ RUN cmake -DCMAKE_BUILD_TYPE=${BUILD_MODE} \
-DCCT_ENABLE_TESTS=${BUILD_TEST} \
-DCCT_ENABLE_ASAN=${BUILD_ASAN} \
-DCCT_BUILD_PROMETHEUS_FROM_SRC=${BUILD_WITH_PROMETHEUS} \
+ -DCCT_ENABLE_PROTO=${BUILD_WITH_PROTOBUF} \
-GNinja ..
# Build
diff --git a/src/api/common/CMakeLists.txt b/src/api/common/CMakeLists.txt
index 9464b950..006b03a4 100644
--- a/src/api/common/CMakeLists.txt
+++ b/src/api/common/CMakeLists.txt
@@ -7,6 +7,7 @@ add_library(coincenter_api-common STATIC ${API_COMMON_SRC})
target_link_libraries(coincenter_api-common PUBLIC coincenter_api-objects)
target_link_libraries(coincenter_api-common PUBLIC coincenter_objects)
target_link_libraries(coincenter_api-common PUBLIC coincenter_http-request)
+target_link_libraries(coincenter_api-common PUBLIC coincenter_serialization)
target_link_libraries(coincenter_api-common PRIVATE OpenSSL::SSL)
target_include_directories(coincenter_api-common PUBLIC include)
diff --git a/src/api/common/include/exchangepublicapi.hpp b/src/api/common/include/exchangepublicapi.hpp
index 03c63351..9b2dc4e0 100644
--- a/src/api/common/include/exchangepublicapi.hpp
+++ b/src/api/common/include/exchangepublicapi.hpp
@@ -1,5 +1,6 @@
#pragma once
+#include
#include
#include
#include
@@ -9,15 +10,20 @@
#include "currencyexchangeflatset.hpp"
#include "exchangebase.hpp"
#include "exchangepublicapitypes.hpp"
+#include "market-order-book-vector.hpp"
+#include "market-timestamp-set.hpp"
#include "market.hpp"
#include "marketorderbook.hpp"
#include "monetaryamount.hpp"
#include "monetaryamountbycurrencyset.hpp"
#include "priceoptions.hpp"
#include "public-trade-vector.hpp"
+#include "time-window.hpp"
namespace cct {
+class AbstractMarketDataDeserializer;
+class AbstractMarketDataSerializer;
class CoincenterInfo;
class ExchangeConfig;
class FiatConverter;
@@ -41,7 +47,7 @@ class ExchangePublic : public ExchangeBase {
using Fiats = CommonAPI::Fiats;
- virtual ~ExchangePublic() = default;
+ virtual ~ExchangePublic();
/// Check if public exchange is responding to basic health check, return true in this case.
/// Exchange that implements the HealthCheck do not need to add a retry mechanism.
@@ -98,14 +104,14 @@ class ExchangePublic : public ExchangeBase {
/// Retrieve the order book of given market.
/// It should be more precise that previous version with possibility to go deeper.
- virtual MarketOrderBook queryOrderBook(Market mk, int depth = kDefaultDepth) = 0;
+ MarketOrderBook getOrderBook(Market mk, int depth = kDefaultDepth);
+
+ /// Retrieve an ordered vector of recent last trades
+ PublicTradeVector getLastTrades(Market mk, int nbTrades = kNbLastTradesDefault);
/// Retrieve the total volume exchange on given market in the last 24 hours.
virtual MonetaryAmount queryLast24hVolume(Market mk) = 0;
- /// Retrieve an ordered vector of recent last trades
- virtual PublicTradeVector queryLastTrades(Market mk, int nbTrades = kNbLastTradesDefault) = 0;
-
/// Retrieve the last price of given market.
virtual MonetaryAmount queryLastPrice(Market mk) = 0;
@@ -177,7 +183,22 @@ class ExchangePublic : public ExchangeBase {
/// If no data found, return a 0 MonetaryAmount on given currency.
MonetaryAmount queryWithdrawalFeeOrZero(CurrencyCode currencyCode);
+ MarketTimestampSet pullMarketOrderBooksMarkets(TimeWindow timeWindow);
+
+ MarketTimestampSet pullTradeMarkets(TimeWindow timeWindow);
+
+ PublicTradeVector pullTradesForReplay(Market market, TimeWindow timeWindow);
+
+ MarketOrderBookVector pullMarketOrderBooksForReplay(Market market, TimeWindow timeWindow);
+
protected:
+ /// Retrieve the order book of given market.
+ /// It should be more precise that previous version with possibility to go deeper.
+ virtual MarketOrderBook queryOrderBook(Market mk, int depth = kDefaultDepth) = 0;
+
+ /// Retrieve an ordered vector of recent last trades
+ virtual PublicTradeVector queryLastTrades(Market mk, int nbTrades = kNbLastTradesDefault) = 0;
+
friend class ExchangePrivate;
ExchangePublic(std::string_view name, FiatConverter &fiatConverter, CommonAPI &commonApi,
@@ -189,7 +210,12 @@ class ExchangePublic : public ExchangeBase {
CommonAPI &_commonApi;
const CoincenterInfo &_coincenterInfo;
const ExchangeConfig &_exchangeConfig;
+ std::unique_ptr _marketDataDeserializerPtr;
+ std::unique_ptr _marketDataSerializerPtr;
std::recursive_mutex _publicRequestsMutex;
+
+ private:
+ AbstractMarketDataSerializer &getMarketDataSerializer();
};
} // namespace api
} // namespace cct
diff --git a/src/api/common/src/exchangeprivateapi.cpp b/src/api/common/src/exchangeprivateapi.cpp
index e463acfb..8ff7cbc6 100644
--- a/src/api/common/src/exchangeprivateapi.cpp
+++ b/src/api/common/src/exchangeprivateapi.cpp
@@ -535,7 +535,7 @@ PlaceOrderInfo ExchangePrivate::placeOrderProcess(MonetaryAmount &from, Monetary
if (tradeInfo.options.isSimulation() && !isSimulatedOrderSupported()) {
if (exchangeConfig().placeSimulateRealOrder()) {
log::debug("Place simulate real order - price {} will be overriden", price);
- MarketOrderBook marketOrderbook = _exchangePublic.queryOrderBook(mk);
+ MarketOrderBook marketOrderbook = _exchangePublic.getOrderBook(mk);
price = isSell ? marketOrderbook.getHighestTheoreticalPrice() : marketOrderbook.getLowestTheoreticalPrice();
} else {
PlaceOrderInfo placeOrderInfo = computeSimulatedMatchedPlacedOrderInfo(volume, price, tradeInfo);
diff --git a/src/api/common/src/exchangepublicapi.cpp b/src/api/common/src/exchangepublicapi.cpp
index 4a345373..599560b2 100644
--- a/src/api/common/src/exchangepublicapi.cpp
+++ b/src/api/common/src/exchangepublicapi.cpp
@@ -24,21 +24,44 @@
#include "exchangeconfig.hpp"
#include "exchangepublicapitypes.hpp"
#include "fiatconverter.hpp"
+#include "market-timestamp-set.hpp"
#include "market.hpp"
#include "marketorderbook.hpp"
#include "monetaryamount.hpp"
#include "priceoptions.hpp"
#include "priceoptionsdef.hpp"
+#include "time-window.hpp"
+#include "timedef.hpp"
#include "unreachable.hpp"
+#ifdef CCT_ENABLE_PROTO
+#include "proto-market-data-deserializer.hpp"
+#include "proto-market-data-serializer.hpp"
+#else
+#include "dummy-market-data-deserializer.hpp"
+#include "dummy-market-data-serializer.hpp"
+#endif
+
namespace cct::api {
+
+#ifdef CCT_ENABLE_PROTO
+using MarketDataDeserializer = ProtoMarketDataDeserializer;
+using MarketDataSerializer = ProtoMarketDataSerializer;
+#else
+using MarketDataDeserializer = DummyMarketDataDeserializer;
+using MarketDataSerializer = DummyMarketDataSerializer;
+#endif
+
ExchangePublic::ExchangePublic(std::string_view name, FiatConverter &fiatConverter, CommonAPI &commonApi,
const CoincenterInfo &coincenterInfo)
: _name(name),
_fiatConverter(fiatConverter),
_commonApi(commonApi),
_coincenterInfo(coincenterInfo),
- _exchangeConfig(coincenterInfo.exchangeConfig(name)) {}
+ _exchangeConfig(coincenterInfo.exchangeConfig(name)),
+ _marketDataDeserializerPtr(new MarketDataDeserializer(coincenterInfo.dataDir(), name)) {}
+
+ExchangePublic::~ExchangePublic() = default;
std::optional ExchangePublic::convert(MonetaryAmount from, CurrencyCode toCurrency,
const MarketsPath &conversionPath, const Fiats &fiats,
@@ -278,7 +301,7 @@ ExchangePublic::CurrenciesPath ExchangePublic::findCurrenciesPath(CurrencyCode f
std::optional ExchangePublic::computeLimitOrderPrice(Market mk, CurrencyCode fromCurrencyCode,
const PriceOptions &priceOptions) {
const int depth = priceOptions.isRelativePrice() ? std::abs(priceOptions.relativePrice()) : 1;
- return queryOrderBook(mk, depth).computeLimitPrice(fromCurrencyCode, priceOptions);
+ return getOrderBook(mk, depth).computeLimitPrice(fromCurrencyCode, priceOptions);
}
std::optional ExchangePublic::computeAvgOrderPrice(Market mk, MonetaryAmount from,
@@ -292,7 +315,7 @@ std::optional ExchangePublic::computeAvgOrderPrice(Market mk, Mo
} else if (priceOptions.priceStrategy() == PriceStrategy::kTaker) {
depth = kDefaultDepth;
}
- return queryOrderBook(mk, depth).computeAvgPrice(from, priceOptions);
+ return getOrderBook(mk, depth).computeAvgPrice(from, priceOptions);
}
std::optional ExchangePublic::RetrieveMarket(CurrencyCode c1, CurrencyCode c2, const MarketSet &markets) {
@@ -433,4 +456,67 @@ MonetaryAmount ExchangePublic::queryWithdrawalFeeOrZero(CurrencyCode currencyCod
return withdrawFee;
}
+MarketOrderBook ExchangePublic::getOrderBook(Market mk, int depth) {
+ std::lock_guard guard(_publicRequestsMutex);
+ const auto marketOrderBook = queryOrderBook(mk, depth);
+
+ if (_exchangeConfig.withMarketDataSerialization()) {
+ getMarketDataSerializer().push(marketOrderBook);
+ }
+ return marketOrderBook;
+}
+
+/// Retrieve an ordered vector of recent last trades
+PublicTradeVector ExchangePublic::getLastTrades(Market mk, int nbTrades) {
+ std::lock_guard guard(_publicRequestsMutex);
+ const auto lastTrades = queryLastTrades(mk, nbTrades);
+ if (_exchangeConfig.withMarketDataSerialization()) {
+ getMarketDataSerializer().push(mk, lastTrades);
+ }
+ return lastTrades;
+}
+
+MarketTimestampSet ExchangePublic::pullMarketOrderBooksMarkets(TimeWindow timeWindow) {
+ if (_marketDataDeserializerPtr) {
+ return _marketDataDeserializerPtr->pullMarketOrderBooksMarkets(timeWindow);
+ }
+ throw exception("No market data deserializer available to retrieve market order books");
+}
+
+MarketTimestampSet ExchangePublic::pullTradeMarkets(TimeWindow timeWindow) {
+ if (_marketDataDeserializerPtr) {
+ return _marketDataDeserializerPtr->pullTradeMarkets(timeWindow);
+ }
+ throw exception("No market data deserializer available to retrieve trade markets");
+}
+
+PublicTradeVector ExchangePublic::pullTradesForReplay(Market market, TimeWindow timeWindow) {
+ return _marketDataDeserializerPtr->pullTrades(market, timeWindow);
+}
+
+MarketOrderBookVector ExchangePublic::pullMarketOrderBooksForReplay(Market market, TimeWindow timeWindow) {
+ return _marketDataDeserializerPtr->pullMarketOrderBooks(market, timeWindow);
+}
+
+AbstractMarketDataSerializer &ExchangePublic::getMarketDataSerializer() {
+ if (_marketDataSerializerPtr) {
+ return *_marketDataSerializerPtr;
+ }
+
+ auto nowTime = Clock::now();
+
+ // Heuristic: load up to 1 week of data to retrieve the youngest written timestamp.
+ // This will be used in order not to write duplicate objects at the start of a new program after that a previous
+ // program run was stopped.
+ TimeWindow largeTimeWindow{nowTime - std::chrono::weeks{1}, nowTime};
+
+ MarketTimestampSets marketTimestampSets{pullMarketOrderBooksMarkets(largeTimeWindow),
+ pullTradeMarkets(largeTimeWindow)};
+
+ _marketDataSerializerPtr = std::unique_ptr(
+ new MarketDataSerializer(_coincenterInfo.dataDir(), marketTimestampSets, name()));
+
+ return *_marketDataSerializerPtr;
+}
+
} // namespace cct::api
diff --git a/src/api/exchanges/src/binancepublicapi.cpp b/src/api/exchanges/src/binancepublicapi.cpp
index ba0c2015..87ddfff5 100644
--- a/src/api/exchanges/src/binancepublicapi.cpp
+++ b/src/api/exchanges/src/binancepublicapi.cpp
@@ -329,7 +329,7 @@ MonetaryAmount BinancePublic::sanitizePrice(Market mk, MonetaryAmount pri) {
MonetaryAmount BinancePublic::computePriceForNotional(Market mk, int avgPriceMins) {
if (avgPriceMins == 0) {
// price should be the last matched price
- PublicTradeVector lastTrades = queryLastTrades(mk, 1);
+ PublicTradeVector lastTrades = getLastTrades(mk, 1);
if (!lastTrades.empty()) {
return lastTrades.front().price();
}
diff --git a/src/api/exchanges/src/bithumbpublicapi.cpp b/src/api/exchanges/src/bithumbpublicapi.cpp
index 7c66fcb7..b5efe371 100644
--- a/src/api/exchanges/src/bithumbpublicapi.cpp
+++ b/src/api/exchanges/src/bithumbpublicapi.cpp
@@ -141,7 +141,7 @@ std::optional BithumbPublic::queryWithdrawalFee(CurrencyCode cur
MonetaryAmount BithumbPublic::queryLastPrice(Market mk) {
// Bithumb does not have a REST API endpoint for last price, let's compute it from the orderbook
- std::optional avgPrice = queryOrderBook(mk).averagePrice();
+ std::optional avgPrice = getOrderBook(mk).averagePrice();
if (!avgPrice) {
log::error("Empty order book for {} on {} cannot compute average price", mk, _name);
return MonetaryAmount(0, mk.quote());
diff --git a/src/api/exchanges/src/krakenprivateapi.cpp b/src/api/exchanges/src/krakenprivateapi.cpp
index a24c4f0d..03f61d5f 100644
--- a/src/api/exchanges/src/krakenprivateapi.cpp
+++ b/src/api/exchanges/src/krakenprivateapi.cpp
@@ -63,8 +63,8 @@ namespace {
enum class KrakenErrorEnum : int8_t { kExpiredOrder, kUnknownWithdrawKey, kUnknownError, kNoError };
template
-std::pair PrivateQuery(CurlHandle& curlHandle, const APIKey& apiKey, std::string_view method,
- CurlPostDataT&& curlPostData = CurlPostData()) {
+auto PrivateQuery(CurlHandle& curlHandle, const APIKey& apiKey, std::string_view method,
+ CurlPostDataT&& curlPostData = CurlPostData()) {
CurlOptions opts(HttpRequestType::kPost, std::forward(curlPostData));
opts.mutableHttpHeaders().emplace_back("API-Key", apiKey.key());
diff --git a/src/api/interface/include/exchange.hpp b/src/api/interface/include/exchange.hpp
index a542430f..f6a5b12e 100644
--- a/src/api/interface/include/exchange.hpp
+++ b/src/api/interface/include/exchange.hpp
@@ -94,12 +94,12 @@ class Exchange {
return apiPublic().queryAllApproximatedOrderBooks(depth);
}
- MarketOrderBook queryOrderBook(Market mk, int depth = ExchangePublic::kDefaultDepth);
+ MarketOrderBook getOrderBook(Market mk, int depth = ExchangePublic::kDefaultDepth);
MonetaryAmount queryLast24hVolume(Market mk) { return apiPublic().queryLast24hVolume(mk); }
/// Retrieve an ordered vector of recent last trades
- PublicTradeVector queryLastTrades(Market mk, int nbTrades = ExchangePublic::kNbLastTradesDefault);
+ PublicTradeVector getLastTrades(Market mk, int nbTrades = ExchangePublic::kNbLastTradesDefault);
/// Retrieve the last price of given market.
MonetaryAmount queryLastPrice(Market mk) { return apiPublic().queryLastPrice(mk); }
diff --git a/src/api/interface/src/exchange.cpp b/src/api/interface/src/exchange.cpp
index 1763ae7e..f33f7a4c 100644
--- a/src/api/interface/src/exchange.cpp
+++ b/src/api/interface/src/exchange.cpp
@@ -48,12 +48,10 @@ bool Exchange::canDeposit(CurrencyCode currencyCode, const CurrencyExchangeFlatS
return lb->canDeposit();
}
-MarketOrderBook Exchange::queryOrderBook(Market mk, int depth) { return apiPublic().queryOrderBook(mk, depth); }
+MarketOrderBook Exchange::getOrderBook(Market mk, int depth) { return apiPublic().getOrderBook(mk, depth); }
/// Retrieve an ordered vector of recent last trades
-PublicTradeVector Exchange::queryLastTrades(Market mk, int nbTrades) {
- return apiPublic().queryLastTrades(mk, nbTrades);
-}
+PublicTradeVector Exchange::getLastTrades(Market mk, int nbTrades) { return apiPublic().getLastTrades(mk, nbTrades); }
void Exchange::updateCacheFile() const {
apiPublic().updateCacheFile();
diff --git a/src/engine/CMakeLists.txt b/src/engine/CMakeLists.txt
index f6191775..791f7987 100644
--- a/src/engine/CMakeLists.txt
+++ b/src/engine/CMakeLists.txt
@@ -6,6 +6,8 @@ target_link_libraries(coincenter_engine PUBLIC coincenter_api-common)
target_link_libraries(coincenter_engine PUBLIC coincenter_api-exchange)
target_link_libraries(coincenter_engine PUBLIC coincenter_api-interface)
target_link_libraries(coincenter_engine PUBLIC coincenter_objects)
+target_link_libraries(coincenter_engine PUBLIC coincenter_trading-algorithms)
+target_link_libraries(coincenter_engine PUBLIC coincenter_trading-common)
target_include_directories(coincenter_engine PUBLIC include)
add_unit_test(
@@ -81,6 +83,13 @@ add_unit_test(
../api/common/test/include
)
+add_unit_test(
+ replay-algorithm-name-iterator_test
+ test/replay-algorithm-name-iterator_test.cpp
+ LIBRARIES
+ coincenter_engine
+)
+
add_unit_test(
stringoptionparser_test
test/stringoptionparser_test.cpp
diff --git a/src/engine/include/coincenter-commands-iterator.hpp b/src/engine/include/coincenter-commands-iterator.hpp
new file mode 100644
index 00000000..809037dd
--- /dev/null
+++ b/src/engine/include/coincenter-commands-iterator.hpp
@@ -0,0 +1,28 @@
+#pragma once
+
+#include
+
+#include "coincentercommand.hpp"
+
+namespace cct {
+
+class CoincenterCommandsIterator {
+ public:
+ using CoincenterCommandSpan = std::span;
+
+ /// Initializes a new iterator on all coincenter commands.
+ explicit CoincenterCommandsIterator(CoincenterCommandSpan commands = CoincenterCommandSpan()) noexcept;
+
+ /// Returns 'true' if this iterator has still some command groups.
+ bool hasNextCommandGroup() const;
+
+ /// Get next grouped commands and advance the iterator.
+ /// The grouped commands are guaranteed to have same type and make it possible to parallelize requests when possible.
+ CoincenterCommandSpan nextCommandGroup();
+
+ private:
+ CoincenterCommandSpan _commands;
+ CoincenterCommandSpan::size_type _pos;
+};
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/engine/include/coincenter.hpp b/src/engine/include/coincenter.hpp
index eecef086..3cc68998 100644
--- a/src/engine/include/coincenter.hpp
+++ b/src/engine/include/coincenter.hpp
@@ -4,6 +4,8 @@
#include
#include "apikeysprovider.hpp"
+#include "cct_const.hpp"
+#include "cct_fixedcapacityvector.hpp"
#include "coincenterinfo.hpp"
#include "commonapi.hpp"
#include "exchange-names.hpp"
@@ -11,14 +13,18 @@
#include "exchangepool.hpp"
#include "exchangesorchestrator.hpp"
#include "fiatconverter.hpp"
+#include "market-trader-engine.hpp"
+#include "market.hpp"
#include "metricsexporter.hpp"
#include "ordersconstraints.hpp"
#include "queryresultprinter.hpp"
#include "queryresulttypes.hpp"
+#include "replay-options.hpp"
#include "transferablecommandresult.hpp"
namespace cct {
+class AbstractMarketTraderFactory;
class CoincenterCommand;
class CoincenterCommands;
class TradeOptions;
@@ -30,6 +36,7 @@ class Coincenter {
Coincenter(const CoincenterInfo &coincenterInfo, const ExchangeSecretsInfo &exchangeSecretsInfo);
+ /// Launch given commands and return the number of processed commands.
int process(const CoincenterCommands &coincenterCommands);
ExchangeHealthCheckStatus healthCheck(ExchangeNameSpan exchangeNames);
@@ -50,6 +57,10 @@ class Coincenter {
CurrencyCode equiCurrencyCode,
std::optional depth = std::nullopt);
+ /// Query market data without returning it.
+ /// This method is especially useful for serialization and metric exports.
+ void queryMarketDataPerExchange(std::span marketPerPublicExchange);
+
/// Retrieve the last 24h traded volume for exchanges supporting given market.
MonetaryAmountPerExchange getLast24hTradedVolumePerExchange(Market mk, ExchangeNameSpan exchangeNames);
@@ -132,6 +143,16 @@ class Coincenter {
const ExchangeName &toPrivateExchangeName,
const WithdrawOptions &withdrawOptions);
+ /// Retrieves the markets available for replay for exchanges selection that has some data during the last
+ /// 'replayDuration' time (so within the time frame [now - replayDuration, now])
+ MarketTimestampSetsPerExchange getMarketsAvailableForReplay(const ReplayOptions &replayOptions,
+ ExchangeNameSpan exchangeNames);
+
+ /// Replay all markets for exchanges selection that has some data during the last
+ /// 'replayDuration' time (so within the time frame [now - replayDuration, now])
+ void replay(const AbstractMarketTraderFactory &marketTraderFactory, const ReplayOptions &replayOptions, Market market,
+ ExchangeNameSpan exchangeNames);
+
/// Dumps the content of all file caches in data directory to save cURL queries.
void updateFileCaches() const;
@@ -147,8 +168,23 @@ class Coincenter {
const FiatConverter &fiatConverter() const { return _fiatConverter; }
private:
- TransferableCommandResultVector processCommand(
- const CoincenterCommand &cmd, std::span previousTransferableResults);
+ TransferableCommandResultVector processGroupedCommands(
+ std::span groupedCommands,
+ std::span previousTransferableResults);
+
+ using MarketTraderEngineVector = FixedCapacityVector;
+
+ void replayAlgorithm(const AbstractMarketTraderFactory &marketTraderFactory, std::string_view algorithmName,
+ const ReplayOptions &replayOptions, std::span marketTraderEngines,
+ const PublicExchangeNameVector &exchangesWithThisMarketData);
+
+ // TODO: may be moved somewhere else?
+ MarketTraderEngineVector createMarketTraderEngines(const ReplayOptions &replayOptions, Market market,
+ PublicExchangeNameVector &exchangesWithThisMarketData);
+
+ MarketTradeRangeStatsPerExchange tradingProcess(const ReplayOptions &replayOptions,
+ std::span marketTraderEngines,
+ ExchangeNameSpan exchangesWithThisMarketData);
const CoincenterInfo &_coincenterInfo;
api::CommonAPI _commonAPI;
diff --git a/src/engine/include/coincentercommand.hpp b/src/engine/include/coincentercommand.hpp
index df683d1f..db0894d1 100644
--- a/src/engine/include/coincentercommand.hpp
+++ b/src/engine/include/coincentercommand.hpp
@@ -13,6 +13,7 @@
#include "market.hpp"
#include "monetaryamount.hpp"
#include "ordersconstraints.hpp"
+#include "replay-options.hpp"
#include "tradeoptions.hpp"
#include "withdrawoptions.hpp"
#include "withdrawsconstraints.hpp"
@@ -44,6 +45,8 @@ class CoincenterCommand {
CoincenterCommand& setCur1(CurrencyCode cur1);
CoincenterCommand& setCur2(CurrencyCode cur2);
+ CoincenterCommand& setReplayOptions(ReplayOptions replayOptions);
+
CoincenterCommand& setPercentageAmount(bool value = true);
CoincenterCommand& withBalanceInUse(bool value = true);
@@ -74,16 +77,19 @@ class CoincenterCommand {
bool isPercentageAmount() const { return _isPercentageAmount; }
bool withBalanceInUse() const { return _withBalanceInUse; }
+ const ReplayOptions& replayOptions() const { return std::get(_specialOptions); }
+
bool operator==(const CoincenterCommand&) const noexcept = default;
using trivially_relocatable =
std::bool_constant && is_trivially_relocatable_v &&
is_trivially_relocatable_v &&
- is_trivially_relocatable_v && is_trivially_relocatable_v>::type;
+ is_trivially_relocatable_v && is_trivially_relocatable_v &&
+ is_trivially_relocatable_v>::type;
private:
- using SpecialOptions =
- std::variant;
+ using SpecialOptions = std::variant;
ExchangeNames _exchangeNames;
SpecialOptions _specialOptions;
diff --git a/src/engine/include/coincenteroptions.hpp b/src/engine/include/coincenteroptions.hpp
index a0e19976..6011126d 100644
--- a/src/engine/include/coincenteroptions.hpp
+++ b/src/engine/include/coincenteroptions.hpp
@@ -9,7 +9,7 @@
#include "coincentercommandtype.hpp"
#include "coincenteroptionsdef.hpp"
#include "commandlineoption.hpp"
-#include "exchangepublicapi.hpp"
+#include "replay-options.hpp"
#include "timedef.hpp"
#include "tradedefinitions.hpp"
#include "tradeoptions.hpp"
@@ -30,6 +30,8 @@ class CoincenterCmdLineOptions {
TradeOptions computeTradeOptions() const;
WithdrawOptions computeWithdrawOptions() const;
+ ReplayOptions computeReplayOptions(Duration dur) const;
+
std::string_view getDataDir() const { return dataDir.empty() ? SelectDefaultDataDir() : dataDir; }
std::pair getTradeArgStr() const;
@@ -100,6 +102,13 @@ class CoincenterCmdLineOptions {
std::string_view lastTrades;
+ std::string_view marketData;
+
+ std::optional replay;
+ std::string_view algorithmNames;
+ std::string_view market;
+ std::optional replayMarkets;
+
CommandLineOptionalInt32 repeats;
int32_t monitoringPort = CoincenterCmdLineOptionsDefinitions::kDefaultMonitoringPort;
int32_t depth = kUndefinedDepth;
@@ -114,6 +123,8 @@ class CoincenterCmdLineOptions {
bool version = false;
bool useMonitoring = false;
bool withBalanceInUse = false;
+ bool validate = false;
+ bool validateOnly = false;
bool operator==(const CoincenterCmdLineOptions&) const noexcept = default;
diff --git a/src/engine/include/coincenteroptionsdef.hpp b/src/engine/include/coincenteroptionsdef.hpp
index 98478314..7bfabb3c 100644
--- a/src/engine/include/coincenteroptionsdef.hpp
+++ b/src/engine/include/coincenteroptionsdef.hpp
@@ -426,6 +426,50 @@ struct CoincenterAllowedOptions : private CoincenterCmdLineOptionsDefinitions {
"Prints withdraw fees for matching currency and exchanges.\n"
"Currency and exchanges are optional, if specified, output will be filtered to match them."},
&OptValueType::withdrawFees},
+ {{{"Automation", 8000},
+ "market-data",
+ "",
+ "Query last trades and order books of given market without printing the result on screen, for given exchanges "
+ "if specified.\n"
+ "This is the equivalent of calling last-trades and order-book but is useful combined with the repeat "
+ "command to store market data on disk."},
+ &OptValueType::marketData},
+ {{{"Automation", 8000},
+ "replay-markets",
+ "",
+ "Print markets available for replay, that is, markets that have some data within the time "
+ "window {now - duration, now}."},
+ &OptValueType::replayMarkets},
+ {{{"Automation", 8001},
+ "replay",
+ "",
+ "Replay algorithms on serialized, historical data."
+ "\nAll known algorithms will be replayed one by one, on all stored markets that have some data within the time "
+ "window {now - duration, now}. Use below flags to filter more precisely on which data to replay from."},
+ &OptValueType::replay},
+ {{{"Automation", 8002},
+ "--algorithms",
+ "",
+ "Pick specific algorithm names to replay with. Default will replay with all known ones."},
+ &OptValueType::algorithmNames},
+ {{{"Automation", 8003},
+ "--market",
+ "",
+ "Only replay for specific market. Default will replay all stored markets."},
+ &OptValueType::market},
+ {{{"Automation", 8003},
+ "--validate",
+ "",
+ "Filter invalid data during replay.\nThis is disabled by default, use this option when you suspect that "
+ "invalid data may be present in the replayed time window."},
+ &OptValueType::validate},
+ {{{"Automation", 8003},
+ "--validate-only",
+ "",
+ "Instead of launching replay algorithm, only validates serialized data."
+ "\nNominal replay will not validate input data to optimize performance, use this option to validate data once "
+ "and for all."},
+ &OptValueType::validateOnly},
{{{"Monitoring", 9000},
"--monitoring",
"",
diff --git a/src/engine/include/exchangesorchestrator.hpp b/src/engine/include/exchangesorchestrator.hpp
index a2c052dd..4498a709 100644
--- a/src/engine/include/exchangesorchestrator.hpp
+++ b/src/engine/include/exchangesorchestrator.hpp
@@ -6,14 +6,18 @@
#include "exchange-names.hpp"
#include "exchangename.hpp"
#include "exchangeretriever.hpp"
+#include "market-trader-engine.hpp"
#include "market.hpp"
#include "queryresulttypes.hpp"
#include "threadpool.hpp"
+#include "time-window.hpp"
#include "withdrawoptions.hpp"
namespace cct {
+class ReplayOptions;
class RequestsConfig;
+
class ExchangesOrchestrator {
public:
using UniquePublicSelectedExchanges = ExchangeRetriever::UniquePublicSelectedExchanges;
@@ -89,6 +93,19 @@ class ExchangesOrchestrator {
MonetaryAmountPerExchange getLastPricePerExchange(Market mk, ExchangeNameSpan exchangeNames);
+ MarketDataPerExchange getMarketDataPerExchange(std::span marketPerPublicExchange,
+ ExchangeNameSpan exchangeNames);
+
+ MarketTimestampSetsPerExchange pullAvailableMarketsForReplay(TimeWindow timeWindow, ExchangeNameSpan exchangeNames);
+
+ MarketTradeRangeStatsPerExchange traderConsumeRange(const ReplayOptions &replayOptions, TimeWindow subTimeWindow,
+ std::span marketTraderEngines,
+ ExchangeNameSpan exchangeNames);
+
+ MarketTradingGlobalResultPerExchange getMarketTraderResultPerExchange(
+ std::span marketTraderEngines, MarketTradeRangeStatsPerExchange &&tradeRangeStatsPerExchange,
+ ExchangeNameSpan exchangeNames);
+
private:
ExchangeRetriever _exchangeRetriever;
ThreadPool _threadPool;
diff --git a/src/engine/include/query-result-type-helpers.hpp b/src/engine/include/query-result-type-helpers.hpp
new file mode 100644
index 00000000..ade76b98
--- /dev/null
+++ b/src/engine/include/query-result-type-helpers.hpp
@@ -0,0 +1,15 @@
+#pragma once
+
+#include "exchangepublicapitypes.hpp"
+#include "market.hpp"
+#include "queryresulttypes.hpp"
+
+namespace cct {
+
+bool ContainsMarket(Market market, const MarketTimestampSet &marketTimestampSet);
+
+bool ContainsMarket(Market market, const MarketTimestampSets &marketTimestampSets);
+
+MarketSet ComputeAllMarkets(const MarketTimestampSetsPerExchange &marketTimestampSetsPerExchange);
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/engine/include/queryresultprinter.hpp b/src/engine/include/queryresultprinter.hpp
index 7e7edec7..904f16d7 100644
--- a/src/engine/include/queryresultprinter.hpp
+++ b/src/engine/include/queryresultprinter.hpp
@@ -15,6 +15,7 @@
#include "ordersconstraints.hpp"
#include "queryresulttypes.hpp"
#include "simpletable.hpp"
+#include "time-window.hpp"
#include "withdrawsconstraints.hpp"
namespace cct {
@@ -66,7 +67,7 @@ class QueryResultPrinter {
}
void printClosedOrders(const ClosedOrdersPerExchange &closedOrdersPerExchange,
- const OrdersConstraints &ordersConstraints) const;
+ const OrdersConstraints &ordersConstraints = OrdersConstraints{}) const;
void printOpenedOrders(const OpenedOrdersPerExchange &openedOrdersPerExchange,
const OrdersConstraints &ordersConstraints) const;
@@ -104,6 +105,13 @@ class QueryResultPrinter {
const TradedAmountsVectorWithFinalAmountPerExchange &tradedAmountsVectorWithFinalAmountPerExchange,
CurrencyCode currencyCode) const;
+ void printMarketsForReplay(TimeWindow timeWindow,
+ const MarketTimestampSetsPerExchange &marketTimestampSetsPerExchange);
+
+ void printMarketTradingResults(TimeWindow timeWindow,
+ const MarketTradingGlobalResultPerExchange &marketTradingResultPerExchange,
+ CoincenterCommandType commandType) const;
+
private:
void printTrades(const TradeResultPerExchange &tradeResultPerExchange, MonetaryAmount amount, bool isPercentageTrade,
CurrencyCode toCurrency, const TradeOptions &tradeOptions, CoincenterCommandType commandType) const;
diff --git a/src/engine/include/queryresulttypes.hpp b/src/engine/include/queryresulttypes.hpp
index 60f1028b..5af47573 100644
--- a/src/engine/include/queryresulttypes.hpp
+++ b/src/engine/include/queryresulttypes.hpp
@@ -13,10 +13,14 @@
#include "currencyexchangeflatset.hpp"
#include "exchangeprivateapitypes.hpp"
#include "exchangepublicapitypes.hpp"
+#include "market-timestamp-set.hpp"
+#include "market-trading-global-result.hpp"
+#include "market-trading-result.hpp"
#include "marketorderbook.hpp"
#include "monetaryamount.hpp"
#include "monetaryamountbycurrencyset.hpp"
#include "public-trade-vector.hpp"
+#include "trade-range-stats.hpp"
#include "traderesult.hpp"
#include "wallet.hpp"
#include "withdrawinfo.hpp"
@@ -31,6 +35,8 @@ using MarketOrderBookConversionRate = std::tuple;
+using MarketPerExchange = FixedCapacityVector, kNbSupportedExchanges>;
+
using MarketsPerExchange = FixedCapacityVector, kNbSupportedExchanges>;
using MonetaryAmountPerExchange = FixedCapacityVector, kNbSupportedExchanges>;
@@ -40,6 +46,9 @@ using MonetaryAmountByCurrencySetPerExchange =
using TradesPerExchange = FixedCapacityVector, kNbSupportedExchanges>;
+using MarketDataPerExchange =
+ FixedCapacityVector>, kNbSupportedExchanges>;
+
using TradeResultPerExchange = SmallVector, kTypicalNbPrivateAccounts>;
using TradedAmountsVectorWithFinalAmountPerExchange =
@@ -68,4 +77,14 @@ using DeliveredWithdrawInfoWithExchanges = std::pair, kTypicalNbPrivateAccounts>;
using ConversionPathPerExchange = FixedCapacityVector, kNbSupportedExchanges>;
+
+using MarketTimestampSetsPerExchange = FixedCapacityVector, kNbSupportedExchanges>;
+
+using MarketTradeRangeStatsPerExchange = FixedCapacityVector, kNbSupportedExchanges>;
+
+using MarketTradingResultPerExchange = FixedCapacityVector, kNbSupportedExchanges>;
+
+using MarketTradingGlobalResultPerExchange =
+ FixedCapacityVector, kNbSupportedExchanges>;
+
} // namespace cct
diff --git a/src/engine/include/replay-algorithm-name-iterator.hpp b/src/engine/include/replay-algorithm-name-iterator.hpp
new file mode 100644
index 00000000..21995286
--- /dev/null
+++ b/src/engine/include/replay-algorithm-name-iterator.hpp
@@ -0,0 +1,29 @@
+#pragma once
+
+#include
+#include
+#include
+
+namespace cct {
+
+/// Convenient class to iterate on the algorithm names, comma separated.
+/// If 'algorithmNames' is empty, it will loop on all available ones (given by 'allAlgorithms')
+class ReplayAlgorithmNameIterator {
+ public:
+ ReplayAlgorithmNameIterator(std::string_view algorithmNames, std::span allAlgorithms);
+
+ /// Returns true if and only if there is at least one additional algorithm name to iterate on.
+ bool hasNext() const;
+
+ /// Get next algorithm name and advance the iterator.
+ /// Undefined behavior if 'hasNext' is 'false'.
+ std::string_view next();
+
+ private:
+ std::span _allAlgorithms;
+ std::string_view _algorithmNames;
+ int32_t _begPos;
+ int32_t _endPos;
+};
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/engine/include/replay-options.hpp b/src/engine/include/replay-options.hpp
new file mode 100644
index 00000000..a490e95e
--- /dev/null
+++ b/src/engine/include/replay-options.hpp
@@ -0,0 +1,33 @@
+#pragma once
+
+#include
+#include
+
+#include "time-window.hpp"
+
+namespace cct {
+
+class ReplayOptions {
+ public:
+ enum class ReplayMode : int8_t { kValidateOnly, kCheckedLaunchAlgorithm, kUncheckedLaunchAlgorithm };
+
+ ReplayOptions() noexcept = default;
+
+ /// Algorithm names should be comma separated. Empty string will match all.
+ ReplayOptions(TimeWindow timeWindow, std::string_view algorithmsNames, ReplayMode replayMode);
+
+ TimeWindow timeWindow() const { return _timeWindow; }
+
+ std::string_view algorithmNames() const;
+
+ ReplayMode replayMode() const { return _replayMode; }
+
+ bool operator==(const ReplayOptions &) const noexcept = default;
+
+ private:
+ TimeWindow _timeWindow;
+ std::string_view _algorithmNames;
+ ReplayMode _replayMode;
+};
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/engine/include/stringoptionparser.hpp b/src/engine/include/stringoptionparser.hpp
index 58bed922..330d3aeb 100644
--- a/src/engine/include/stringoptionparser.hpp
+++ b/src/engine/include/stringoptionparser.hpp
@@ -10,6 +10,7 @@
#include "exchange-names.hpp"
#include "market.hpp"
#include "monetaryamount.hpp"
+#include "timedef.hpp"
namespace cct {
class StringOptionParser {
@@ -29,6 +30,10 @@ class StringOptionParser {
/// after the currency
CurrencyCode parseCurrency(FieldIs fieldIs = FieldIs::kMandatory, char delimiter = ',');
+ /// If FieldIs is kOptional and there is no duration, kUndefinedDuration duration will be returned.
+ /// otherwise exception invalid_argument will be raised
+ Duration parseDuration(FieldIs fieldIs = FieldIs::kMandatory);
+
/// If FieldIs is kOptional and there is no market, default market will be returned.
/// otherwise exception invalid_argument will be raised.
/// @param delimiter defines the expected character (could be not present, which means end of parsing)
@@ -55,4 +60,4 @@ class StringOptionParser {
std::string_view _opt;
std::string_view::size_type _pos{};
};
-} // namespace cct
\ No newline at end of file
+} // namespace cct
diff --git a/src/engine/src/coincenter-commands-iterator.cpp b/src/engine/src/coincenter-commands-iterator.cpp
new file mode 100644
index 00000000..f0f773bd
--- /dev/null
+++ b/src/engine/src/coincenter-commands-iterator.cpp
@@ -0,0 +1,74 @@
+#include "coincenter-commands-iterator.hpp"
+
+#include
+
+#include "cct_const.hpp"
+#include "coincentercommandtype.hpp"
+#include "exchangename.hpp"
+
+namespace cct {
+
+CoincenterCommandsIterator::CoincenterCommandsIterator(CoincenterCommandSpan commands) noexcept
+ : _commands(commands), _pos() {}
+
+namespace {
+using PublicExchangePresenceBitset = std::bitset;
+
+bool UpdateBitsetAreNewExchanges(const CoincenterCommand &command,
+ PublicExchangePresenceBitset &publicExchangePresence) {
+ if (command.exchangeNames().empty()) {
+ // All public exchanges used
+ const auto result = publicExchangePresence.none();
+ publicExchangePresence.set();
+ return result;
+ }
+ for (const ExchangeName &exchangeName : command.exchangeNames()) {
+ const auto exchangePos = exchangeName.publicExchangePos();
+ if (publicExchangePresence[exchangePos]) {
+ return false;
+ }
+ publicExchangePresence.set(exchangePos);
+ }
+ return true;
+}
+
+bool CommandTypeCanBeGrouped(CoincenterCommandType type) {
+ // Compatible command types need to be explicitly set
+ // For now, only market data is compatible
+ switch (type) {
+ case CoincenterCommandType::kMarketData:
+ return true;
+ default:
+ return false;
+ }
+}
+
+} // namespace
+
+bool CoincenterCommandsIterator::hasNextCommandGroup() const { return _pos < _commands.size(); }
+
+CoincenterCommandsIterator::CoincenterCommandSpan CoincenterCommandsIterator::nextCommandGroup() {
+ CoincenterCommandSpan groupedCommands(_commands.begin() + _pos, 1U);
+
+ if (CommandTypeCanBeGrouped(groupedCommands.front().type())) {
+ PublicExchangePresenceBitset publicExchangePresence;
+ UpdateBitsetAreNewExchanges(groupedCommands.front(), publicExchangePresence);
+
+ while (_pos + groupedCommands.size() < _commands.size()) {
+ const CoincenterCommand &nextCommand = _commands[_pos + groupedCommands.size()];
+ if (nextCommand.type() != groupedCommands.front().type()) {
+ break;
+ }
+ if (!UpdateBitsetAreNewExchanges(nextCommand, publicExchangePresence)) {
+ break;
+ }
+ // Add new command to group
+ groupedCommands = CoincenterCommandSpan(groupedCommands.data(), groupedCommands.size() + 1);
+ }
+ }
+
+ _pos += groupedCommands.size();
+ return groupedCommands;
+}
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/engine/src/coincenter.cpp b/src/engine/src/coincenter.cpp
index 1a08e76e..0ce5817d 100644
--- a/src/engine/src/coincenter.cpp
+++ b/src/engine/src/coincenter.cpp
@@ -13,6 +13,7 @@
#include "cct_exception.hpp"
#include "cct_invalid_argument_exception.hpp"
#include "cct_log.hpp"
+#include "coincenter-commands-iterator.hpp"
#include "coincentercommand.hpp"
#include "coincentercommands.hpp"
#include "coincentercommandtype.hpp"
@@ -25,11 +26,18 @@
#include "exchangepublicapi.hpp"
#include "exchangeretriever.hpp"
#include "exchangesecretsinfo.hpp"
+#include "market-timestamp-set.hpp"
+#include "market-trader-engine.hpp"
+#include "market-trader-factory.hpp"
#include "market.hpp"
#include "monetaryamount.hpp"
#include "ordersconstraints.hpp"
+#include "query-result-type-helpers.hpp"
#include "queryresultprinter.hpp"
#include "queryresulttypes.hpp"
+#include "replay-algorithm-name-iterator.hpp"
+#include "replay-options.hpp"
+#include "time-window.hpp"
#include "timedef.hpp"
#include "transferablecommandresult.hpp"
#include "withdrawsconstraints.hpp"
@@ -113,35 +121,40 @@ int Coincenter::process(const CoincenterCommands &coincenterCommands) {
}
}
TransferableCommandResultVector transferableResults;
- for (const auto &cmd : commands) {
- transferableResults = processCommand(cmd, transferableResults);
+ CoincenterCommandsIterator commandsIterator(commands);
+ while (commandsIterator.hasNextCommandGroup()) {
+ const auto groupedCommands = commandsIterator.nextCommandGroup();
+ transferableResults = processGroupedCommands(groupedCommands, transferableResults);
++nbCommandsProcessed;
}
}
return nbCommandsProcessed;
}
-TransferableCommandResultVector Coincenter::processCommand(
- const CoincenterCommand &cmd, std::span previousTransferableResults) {
+TransferableCommandResultVector Coincenter::processGroupedCommands(
+ std::span groupedCommands,
+ std::span previousTransferableResults) {
TransferableCommandResultVector transferableResults;
- switch (cmd.type()) {
+ const auto &firstCmd = groupedCommands.front();
+ // All grouped commands have same type - logic to handle multiple commands in a group should be handled per use case
+ switch (firstCmd.type()) {
case CoincenterCommandType::kHealthCheck: {
- const auto healthCheckStatus = healthCheck(cmd.exchangeNames());
+ const auto healthCheckStatus = healthCheck(firstCmd.exchangeNames());
_queryResultPrinter.printHealthCheck(healthCheckStatus);
break;
}
case CoincenterCommandType::kCurrencies: {
- const auto currenciesPerExchange = getCurrenciesPerExchange(cmd.exchangeNames());
+ const auto currenciesPerExchange = getCurrenciesPerExchange(firstCmd.exchangeNames());
_queryResultPrinter.printCurrencies(currenciesPerExchange);
break;
}
case CoincenterCommandType::kMarkets: {
- const auto marketsPerExchange = getMarketsPerExchange(cmd.cur1(), cmd.cur2(), cmd.exchangeNames());
- _queryResultPrinter.printMarkets(cmd.cur1(), cmd.cur2(), marketsPerExchange, cmd.type());
+ const auto marketsPerExchange = getMarketsPerExchange(firstCmd.cur1(), firstCmd.cur2(), firstCmd.exchangeNames());
+ _queryResultPrinter.printMarkets(firstCmd.cur1(), firstCmd.cur2(), marketsPerExchange, firstCmd.type());
break;
}
case CoincenterCommandType::kConversion: {
- if (cmd.amount().isDefault()) {
+ if (firstCmd.amount().isDefault()) {
std::array startAmountsPerExchangePos;
bool oneSet = false;
for (const auto &transferableResult : previousTransferableResults) {
@@ -158,90 +171,96 @@ TransferableCommandResultVector Coincenter::processCommand(
throw invalid_argument("Missing input amount to convert from");
}
- const auto conversionPerExchange = getConversion(startAmountsPerExchangePos, cmd.cur1(), cmd.exchangeNames());
- _queryResultPrinter.printConversion(startAmountsPerExchangePos, cmd.cur1(), conversionPerExchange);
+ const auto conversionPerExchange =
+ getConversion(startAmountsPerExchangePos, firstCmd.cur1(), firstCmd.exchangeNames());
+ _queryResultPrinter.printConversion(startAmountsPerExchangePos, firstCmd.cur1(), conversionPerExchange);
FillConversionTransferableCommandResults(conversionPerExchange, transferableResults);
} else {
- const auto conversionPerExchange = getConversion(cmd.amount(), cmd.cur1(), cmd.exchangeNames());
- _queryResultPrinter.printConversion(cmd.amount(), cmd.cur1(), conversionPerExchange);
+ const auto conversionPerExchange = getConversion(firstCmd.amount(), firstCmd.cur1(), firstCmd.exchangeNames());
+ _queryResultPrinter.printConversion(firstCmd.amount(), firstCmd.cur1(), conversionPerExchange);
FillConversionTransferableCommandResults(conversionPerExchange, transferableResults);
}
break;
}
case CoincenterCommandType::kConversionPath: {
- const auto conversionPathPerExchange = getConversionPaths(cmd.market(), cmd.exchangeNames());
- _queryResultPrinter.printConversionPath(cmd.market(), conversionPathPerExchange);
+ const auto conversionPathPerExchange = getConversionPaths(firstCmd.market(), firstCmd.exchangeNames());
+ _queryResultPrinter.printConversionPath(firstCmd.market(), conversionPathPerExchange);
break;
}
case CoincenterCommandType::kLastPrice: {
- const auto lastPricePerExchange = getLastPricePerExchange(cmd.market(), cmd.exchangeNames());
- _queryResultPrinter.printLastPrice(cmd.market(), lastPricePerExchange);
+ const auto lastPricePerExchange = getLastPricePerExchange(firstCmd.market(), firstCmd.exchangeNames());
+ _queryResultPrinter.printLastPrice(firstCmd.market(), lastPricePerExchange);
break;
}
case CoincenterCommandType::kTicker: {
- const auto exchangeTickerMaps = getTickerInformation(cmd.exchangeNames());
+ const auto exchangeTickerMaps = getTickerInformation(firstCmd.exchangeNames());
_queryResultPrinter.printTickerInformation(exchangeTickerMaps);
break;
}
case CoincenterCommandType::kOrderbook: {
const auto marketOrderBooksConversionRates =
- getMarketOrderBooks(cmd.market(), cmd.exchangeNames(), cmd.cur1(), cmd.optDepth());
- _queryResultPrinter.printMarketOrderBooks(cmd.market(), cmd.cur1(), cmd.optDepth(),
+ getMarketOrderBooks(firstCmd.market(), firstCmd.exchangeNames(), firstCmd.cur1(), firstCmd.optDepth());
+ _queryResultPrinter.printMarketOrderBooks(firstCmd.market(), firstCmd.cur1(), firstCmd.optDepth(),
marketOrderBooksConversionRates);
break;
}
case CoincenterCommandType::kLastTrades: {
- const auto lastTradesPerExchange = getLastTradesPerExchange(cmd.market(), cmd.exchangeNames(), cmd.optDepth());
- _queryResultPrinter.printLastTrades(cmd.market(), cmd.optDepth(), lastTradesPerExchange);
+ const auto lastTradesPerExchange =
+ getLastTradesPerExchange(firstCmd.market(), firstCmd.exchangeNames(), firstCmd.optDepth());
+ _queryResultPrinter.printLastTrades(firstCmd.market(), firstCmd.optDepth(), lastTradesPerExchange);
break;
}
case CoincenterCommandType::kLast24hTradedVolume: {
- const auto tradedVolumePerExchange = getLast24hTradedVolumePerExchange(cmd.market(), cmd.exchangeNames());
- _queryResultPrinter.printLast24hTradedVolume(cmd.market(), tradedVolumePerExchange);
+ const auto tradedVolumePerExchange =
+ getLast24hTradedVolumePerExchange(firstCmd.market(), firstCmd.exchangeNames());
+ _queryResultPrinter.printLast24hTradedVolume(firstCmd.market(), tradedVolumePerExchange);
break;
}
case CoincenterCommandType::kWithdrawFees: {
- const auto withdrawFeesPerExchange = getWithdrawFees(cmd.cur1(), cmd.exchangeNames());
- _queryResultPrinter.printWithdrawFees(withdrawFeesPerExchange, cmd.cur1());
+ const auto withdrawFeesPerExchange = getWithdrawFees(firstCmd.cur1(), firstCmd.exchangeNames());
+ _queryResultPrinter.printWithdrawFees(withdrawFeesPerExchange, firstCmd.cur1());
break;
}
case CoincenterCommandType::kBalance: {
- const auto amountIncludePolicy = cmd.withBalanceInUse() ? BalanceOptions::AmountIncludePolicy::kWithBalanceInUse
- : BalanceOptions::AmountIncludePolicy::kOnlyAvailable;
- const BalanceOptions balanceOptions(amountIncludePolicy, cmd.cur1());
- const auto balancePerExchange = getBalance(cmd.exchangeNames(), balanceOptions);
- _queryResultPrinter.printBalance(balancePerExchange, cmd.cur1());
+ const auto amountIncludePolicy = firstCmd.withBalanceInUse()
+ ? BalanceOptions::AmountIncludePolicy::kWithBalanceInUse
+ : BalanceOptions::AmountIncludePolicy::kOnlyAvailable;
+ const BalanceOptions balanceOptions(amountIncludePolicy, firstCmd.cur1());
+ const auto balancePerExchange = getBalance(firstCmd.exchangeNames(), balanceOptions);
+ _queryResultPrinter.printBalance(balancePerExchange, firstCmd.cur1());
break;
}
case CoincenterCommandType::kDepositInfo: {
- const auto walletPerExchange = getDepositInfo(cmd.exchangeNames(), cmd.cur1());
- _queryResultPrinter.printDepositInfo(cmd.cur1(), walletPerExchange);
+ const auto walletPerExchange = getDepositInfo(firstCmd.exchangeNames(), firstCmd.cur1());
+ _queryResultPrinter.printDepositInfo(firstCmd.cur1(), walletPerExchange);
break;
}
case CoincenterCommandType::kOrdersClosed: {
- const auto closedOrdersPerExchange = getClosedOrders(cmd.exchangeNames(), cmd.ordersConstraints());
- _queryResultPrinter.printClosedOrders(closedOrdersPerExchange, cmd.ordersConstraints());
+ const auto closedOrdersPerExchange = getClosedOrders(firstCmd.exchangeNames(), firstCmd.ordersConstraints());
+ _queryResultPrinter.printClosedOrders(closedOrdersPerExchange, firstCmd.ordersConstraints());
break;
}
case CoincenterCommandType::kOrdersOpened: {
- const auto openedOrdersPerExchange = getOpenedOrders(cmd.exchangeNames(), cmd.ordersConstraints());
- _queryResultPrinter.printOpenedOrders(openedOrdersPerExchange, cmd.ordersConstraints());
+ const auto openedOrdersPerExchange = getOpenedOrders(firstCmd.exchangeNames(), firstCmd.ordersConstraints());
+ _queryResultPrinter.printOpenedOrders(openedOrdersPerExchange, firstCmd.ordersConstraints());
break;
}
case CoincenterCommandType::kOrdersCancel: {
- const auto nbCancelledOrdersPerExchange = cancelOrders(cmd.exchangeNames(), cmd.ordersConstraints());
- _queryResultPrinter.printCancelledOrders(nbCancelledOrdersPerExchange, cmd.ordersConstraints());
+ const auto nbCancelledOrdersPerExchange = cancelOrders(firstCmd.exchangeNames(), firstCmd.ordersConstraints());
+ _queryResultPrinter.printCancelledOrders(nbCancelledOrdersPerExchange, firstCmd.ordersConstraints());
break;
}
case CoincenterCommandType::kRecentDeposits: {
- const auto depositsPerExchange = getRecentDeposits(cmd.exchangeNames(), cmd.withdrawsOrDepositsConstraints());
- _queryResultPrinter.printRecentDeposits(depositsPerExchange, cmd.withdrawsOrDepositsConstraints());
+ const auto depositsPerExchange =
+ getRecentDeposits(firstCmd.exchangeNames(), firstCmd.withdrawsOrDepositsConstraints());
+ _queryResultPrinter.printRecentDeposits(depositsPerExchange, firstCmd.withdrawsOrDepositsConstraints());
break;
}
case CoincenterCommandType::kRecentWithdraws: {
- const auto withdrawsPerExchange = getRecentWithdraws(cmd.exchangeNames(), cmd.withdrawsOrDepositsConstraints());
- _queryResultPrinter.printRecentWithdraws(withdrawsPerExchange, cmd.withdrawsOrDepositsConstraints());
+ const auto withdrawsPerExchange =
+ getRecentWithdraws(firstCmd.exchangeNames(), firstCmd.withdrawsOrDepositsConstraints());
+ _queryResultPrinter.printRecentWithdraws(withdrawsPerExchange, firstCmd.withdrawsOrDepositsConstraints());
break;
}
case CoincenterCommandType::kTrade: {
@@ -249,50 +268,83 @@ TransferableCommandResultVector Coincenter::processCommand(
// - standard full information with an amount to trade, a destination currency and an optional list of exchanges
// where to trade
// - a currency - the destination one, and start amount and exchange(s) should come from previous command result
- auto [startAmount, exchangeNames] = ComputeTradeAmountAndExchanges(cmd, previousTransferableResults);
+ auto [startAmount, exchangeNames] = ComputeTradeAmountAndExchanges(firstCmd, previousTransferableResults);
if (startAmount.isDefault()) {
break;
}
const auto tradeResultPerExchange =
- trade(startAmount, cmd.isPercentageAmount(), cmd.cur1(), exchangeNames, cmd.tradeOptions());
- _queryResultPrinter.printTrades(tradeResultPerExchange, startAmount, cmd.isPercentageAmount(), cmd.cur1(),
- cmd.tradeOptions());
+ trade(startAmount, firstCmd.isPercentageAmount(), firstCmd.cur1(), exchangeNames, firstCmd.tradeOptions());
+ _queryResultPrinter.printTrades(tradeResultPerExchange, startAmount, firstCmd.isPercentageAmount(),
+ firstCmd.cur1(), firstCmd.tradeOptions());
FillTradeTransferableCommandResults(tradeResultPerExchange, transferableResults);
break;
}
case CoincenterCommandType::kBuy: {
- const auto tradeResultPerExchange = smartBuy(cmd.amount(), cmd.exchangeNames(), cmd.tradeOptions());
- _queryResultPrinter.printBuyTrades(tradeResultPerExchange, cmd.amount(), cmd.tradeOptions());
+ const auto tradeResultPerExchange =
+ smartBuy(firstCmd.amount(), firstCmd.exchangeNames(), firstCmd.tradeOptions());
+ _queryResultPrinter.printBuyTrades(tradeResultPerExchange, firstCmd.amount(), firstCmd.tradeOptions());
FillTradeTransferableCommandResults(tradeResultPerExchange, transferableResults);
break;
}
case CoincenterCommandType::kSell: {
- auto [startAmount, exchangeNames] = ComputeTradeAmountAndExchanges(cmd, previousTransferableResults);
+ auto [startAmount, exchangeNames] = ComputeTradeAmountAndExchanges(firstCmd, previousTransferableResults);
if (startAmount.isDefault()) {
break;
}
const auto tradeResultPerExchange =
- smartSell(startAmount, cmd.isPercentageAmount(), exchangeNames, cmd.tradeOptions());
- _queryResultPrinter.printSellTrades(tradeResultPerExchange, cmd.amount(), cmd.isPercentageAmount(),
- cmd.tradeOptions());
+ smartSell(startAmount, firstCmd.isPercentageAmount(), exchangeNames, firstCmd.tradeOptions());
+ _queryResultPrinter.printSellTrades(tradeResultPerExchange, firstCmd.amount(), firstCmd.isPercentageAmount(),
+ firstCmd.tradeOptions());
FillTradeTransferableCommandResults(tradeResultPerExchange, transferableResults);
break;
}
case CoincenterCommandType::kWithdrawApply: {
- const auto [grossAmount, exchangeName] = ComputeWithdrawAmount(cmd, previousTransferableResults);
+ const auto [grossAmount, exchangeName] = ComputeWithdrawAmount(firstCmd, previousTransferableResults);
if (grossAmount.isDefault()) {
break;
}
- const auto deliveredWithdrawInfoWithExchanges = withdraw(grossAmount, cmd.isPercentageAmount(), exchangeName,
- cmd.exchangeNames().back(), cmd.withdrawOptions());
- _queryResultPrinter.printWithdraw(deliveredWithdrawInfoWithExchanges, cmd.isPercentageAmount(),
- cmd.withdrawOptions());
+ const auto deliveredWithdrawInfoWithExchanges =
+ withdraw(grossAmount, firstCmd.isPercentageAmount(), exchangeName, firstCmd.exchangeNames().back(),
+ firstCmd.withdrawOptions());
+ _queryResultPrinter.printWithdraw(deliveredWithdrawInfoWithExchanges, firstCmd.isPercentageAmount(),
+ firstCmd.withdrawOptions());
transferableResults.emplace_back(deliveredWithdrawInfoWithExchanges.first[1]->createExchangeName(),
deliveredWithdrawInfoWithExchanges.second.receivedAmount());
break;
}
case CoincenterCommandType::kDustSweeper: {
- _queryResultPrinter.printDustSweeper(dustSweeper(cmd.exchangeNames(), cmd.cur1()), cmd.cur1());
+ _queryResultPrinter.printDustSweeper(dustSweeper(firstCmd.exchangeNames(), firstCmd.cur1()), firstCmd.cur1());
+ break;
+ }
+ case CoincenterCommandType::kMarketData: {
+ std::array marketPerPublicExchange;
+ for (const auto &cmd : groupedCommands) {
+ if (cmd.exchangeNames().empty()) {
+ std::ranges::fill(marketPerPublicExchange, cmd.market());
+ } else {
+ for (const auto &exchangeName : cmd.exchangeNames()) {
+ marketPerPublicExchange[exchangeName.publicExchangePos()] = cmd.market();
+ }
+ }
+ }
+ // No return value here, this command is made only for storing purposes.
+ queryMarketDataPerExchange(marketPerPublicExchange);
+ break;
+ }
+ case CoincenterCommandType::kReplay: {
+ /// This implementation of AbstractMarketTraderFactory is only provided as an example.
+ /// You can extend coincenter library and:
+ /// - Provide your own algorithms by implementing your own MarketTraderFactory will all your algorithms.
+ /// - Create your own CommandType that will call coincenter.replay with the same parameters as below, with your
+ /// own MarketTraderFactory.
+ MarketTraderFactory marketTraderFactory;
+ replay(marketTraderFactory, firstCmd.replayOptions(), firstCmd.market(), firstCmd.exchangeNames());
+ break;
+ }
+ case CoincenterCommandType::kReplayMarkets: {
+ const auto marketTimestampSetsPerExchange =
+ getMarketsAvailableForReplay(firstCmd.replayOptions(), firstCmd.exchangeNames());
+ _queryResultPrinter.printMarketsForReplay(firstCmd.replayOptions().timeWindow(), marketTimestampSetsPerExchange);
break;
}
default:
@@ -327,6 +379,40 @@ MarketOrderBookConversionRates Coincenter::getMarketOrderBooks(Market mk, Exchan
return ret;
}
+void Coincenter::queryMarketDataPerExchange(std::span marketPerPublicExchange) {
+ ExchangeNames exchangeNames;
+
+ int exchangePos{};
+ for (Market market : marketPerPublicExchange) {
+ if (market.isDefined()) {
+ exchangeNames.emplace_back(kSupportedExchanges[exchangePos]);
+ }
+ ++exchangePos;
+ }
+
+ const auto marketDataPerExchange =
+ _exchangesOrchestrator.getMarketDataPerExchange(marketPerPublicExchange, exchangeNames);
+
+ // Transform data structures to export metrics input format
+ MarketOrderBookConversionRates marketOrderBookConversionRates(marketDataPerExchange.size());
+ TradesPerExchange lastTradesPerExchange(marketDataPerExchange.size());
+
+ std::ranges::transform(marketDataPerExchange, marketOrderBookConversionRates.begin(),
+ [](const auto &exchangeWithPairOrderBooksAndTrades) {
+ return std::make_tuple(exchangeWithPairOrderBooksAndTrades.first->name(),
+ exchangeWithPairOrderBooksAndTrades.second.first, std::nullopt);
+ });
+
+ std::ranges::transform(marketDataPerExchange, lastTradesPerExchange.begin(),
+ [](const auto &exchangeWithPairOrderBooksAndTrades) {
+ return std::make_pair(exchangeWithPairOrderBooksAndTrades.first,
+ exchangeWithPairOrderBooksAndTrades.second.second);
+ });
+
+ _metricsExporter.exportOrderbookMetrics(marketOrderBookConversionRates);
+ _metricsExporter.exportLastTradesMetrics(lastTradesPerExchange);
+}
+
BalancePerExchange Coincenter::getBalance(std::span privateExchangeNames,
const BalanceOptions &balanceOptions) {
CurrencyCode equiCurrency = balanceOptions.equiCurrency();
@@ -459,6 +545,198 @@ MonetaryAmountPerExchange Coincenter::getLastPricePerExchange(Market mk, Exchang
return _exchangesOrchestrator.getLastPricePerExchange(mk, exchangeNames);
}
+MarketTimestampSetsPerExchange Coincenter::getMarketsAvailableForReplay(const ReplayOptions &replayOptions,
+ ExchangeNameSpan exchangeNames) {
+ return _exchangesOrchestrator.pullAvailableMarketsForReplay(replayOptions.timeWindow(), exchangeNames);
+}
+
+namespace {
+auto CreateExchangeNameVector(Market market, const MarketTimestampSetsPerExchange &marketTimestampSetsPerExchange) {
+ PublicExchangeNameVector exchangesWithThisMarketData;
+ for (const auto &[exchange, marketTimestampSets] : marketTimestampSetsPerExchange) {
+ if (ContainsMarket(market, marketTimestampSets)) {
+ exchangesWithThisMarketData.emplace_back(exchange->name());
+ }
+ }
+ return exchangesWithThisMarketData;
+}
+
+void CreateAndRegisterTraderAlgorithms(const AbstractMarketTraderFactory &marketTraderFactory,
+ std::string_view algorithmName,
+ std::span marketTraderEngines) {
+ for (auto &marketTraderEngine : marketTraderEngines) {
+ const auto &marketTraderEngineState = marketTraderEngine.marketTraderEngineState();
+
+ marketTraderEngine.registerMarketTrader(marketTraderFactory.construct(algorithmName, marketTraderEngineState));
+ }
+}
+
+bool Filter(Market market, MarketTimestampSet &marketTimestampSet) {
+ auto it = std::partition_point(marketTimestampSet.begin(), marketTimestampSet.end(),
+ [market](const auto &marketTimestamp) { return marketTimestamp.market < market; });
+ if (it != marketTimestampSet.end() && it->market == market) {
+ auto marketTimestamp = *it;
+ marketTimestampSet.clear();
+ marketTimestampSet.insert(marketTimestamp);
+ return false;
+ }
+
+ marketTimestampSet.clear();
+ return true;
+}
+
+void Filter(Market market, MarketTimestampSetsPerExchange &marketTimestampSetsPerExchange) {
+ for (auto it = marketTimestampSetsPerExchange.begin(); it != marketTimestampSetsPerExchange.end();) {
+ const bool orderBooksEmpty = Filter(market, it->second.orderBooksMarkets);
+ const bool tradesEmpty = Filter(market, it->second.tradesMarkets);
+
+ if (orderBooksEmpty && tradesEmpty) {
+ // no more data, remove the exchange entry completely
+ it = marketTimestampSetsPerExchange.erase(it);
+ } else {
+ ++it;
+ }
+ }
+}
+
+} // namespace
+
+void Coincenter::replay(const AbstractMarketTraderFactory &marketTraderFactory, const ReplayOptions &replayOptions,
+ Market market, ExchangeNameSpan exchangeNames) {
+ const TimeWindow timeWindow = replayOptions.timeWindow();
+ auto marketTimestampSetsPerExchange = _exchangesOrchestrator.pullAvailableMarketsForReplay(timeWindow, exchangeNames);
+
+ if (market.isDefined()) {
+ Filter(market, marketTimestampSetsPerExchange);
+ }
+
+ MarketSet allMarkets = ComputeAllMarkets(marketTimestampSetsPerExchange);
+
+ ReplayAlgorithmNameIterator replayAlgorithmNameIterator(replayOptions.algorithmNames(),
+ marketTraderFactory.allSupportedAlgorithms());
+
+ while (replayAlgorithmNameIterator.hasNext()) {
+ std::string_view algorithmName = replayAlgorithmNameIterator.next();
+
+ for (const Market replayMarket : allMarkets) {
+ auto exchangesWithThisMarketData = CreateExchangeNameVector(replayMarket, marketTimestampSetsPerExchange);
+
+ // Create the MarketTraderEngines based on this market, filtering out exchanges without available amount to
+ // trade
+ MarketTraderEngineVector marketTraderEngines =
+ createMarketTraderEngines(replayOptions, replayMarket, exchangesWithThisMarketData);
+
+ replayAlgorithm(marketTraderFactory, algorithmName, replayOptions, marketTraderEngines,
+ exchangesWithThisMarketData);
+ }
+ }
+}
+
+void Coincenter::replayAlgorithm(const AbstractMarketTraderFactory &marketTraderFactory, std::string_view algorithmName,
+ const ReplayOptions &replayOptions, std::span marketTraderEngines,
+ const PublicExchangeNameVector &exchangesWithThisMarketData) {
+ CreateAndRegisterTraderAlgorithms(marketTraderFactory, algorithmName, marketTraderEngines);
+
+ MarketTradeRangeStatsPerExchange tradeRangeStatsPerExchange =
+ tradingProcess(replayOptions, marketTraderEngines, exchangesWithThisMarketData);
+
+ // Finally retrieve and print results for this market
+ MarketTradingGlobalResultPerExchange marketTradingResultPerExchange =
+ _exchangesOrchestrator.getMarketTraderResultPerExchange(
+ marketTraderEngines, std::move(tradeRangeStatsPerExchange), exchangesWithThisMarketData);
+
+ _queryResultPrinter.printMarketTradingResults(replayOptions.timeWindow(), marketTradingResultPerExchange,
+ CoincenterCommandType::kReplay);
+}
+
+Coincenter::MarketTraderEngineVector Coincenter::createMarketTraderEngines(
+ const ReplayOptions &replayOptions, Market market, PublicExchangeNameVector &exchangesWithThisMarketData) {
+ auto nbExchanges = exchangesWithThisMarketData.size();
+
+ const auto &automationConfig = _coincenterInfo.generalConfig().tradingConfig().automationConfig();
+ const auto startBaseAmountEquivalent = automationConfig.startBaseAmountEquivalent();
+ const auto startQuoteAmountEquivalent = automationConfig.startQuoteAmountEquivalent();
+ const bool isValidateOnly = replayOptions.replayMode() == ReplayOptions::ReplayMode::kValidateOnly;
+
+ auto convertedBaseAmountPerExchange =
+ isValidateOnly ? MonetaryAmountPerExchange{}
+ : getConversion(startBaseAmountEquivalent, market.base(), exchangesWithThisMarketData);
+ auto convertedQuoteAmountPerExchange =
+ isValidateOnly ? MonetaryAmountPerExchange{}
+ : getConversion(startQuoteAmountEquivalent, market.quote(), exchangesWithThisMarketData);
+
+ MarketTraderEngineVector marketTraderEngines;
+ for (decltype(nbExchanges) exchangePos = 0; exchangePos < nbExchanges; ++exchangePos) {
+ MonetaryAmount startBaseAmount =
+ isValidateOnly ? MonetaryAmount{0, market.base()} : convertedBaseAmountPerExchange[exchangePos].second;
+ MonetaryAmount startQuoteAmount =
+ isValidateOnly ? MonetaryAmount{0, market.quote()} : convertedQuoteAmountPerExchange[exchangePos].second;
+
+ if (startBaseAmount.currencyCode() != market.base()) {
+ // This is possible as conversion may use equivalent fiats and stable coins
+ log::info("Target converted currency is different from market one, replace with market currency {} -> {}",
+ startBaseAmount.currencyCode(), market.base());
+ startBaseAmount = MonetaryAmount(startBaseAmount.amount(), market.base(), startBaseAmount.nbDecimals());
+ }
+ if (startQuoteAmount.currencyCode() != market.quote()) {
+ // This is possible as conversion may use equivalent fiats and stable coins
+ log::info("Target converted currency is different from market one, replace with market currency {} -> {}",
+ startQuoteAmount.currencyCode(), market.quote());
+ startQuoteAmount = MonetaryAmount(startQuoteAmount.amount(), market.quote(), startQuoteAmount.nbDecimals());
+ }
+
+ if (!isValidateOnly && (startBaseAmount == 0 || startQuoteAmount == 0)) {
+ log::warn("Cannot convert to start base / quote amounts for {} ({} / {})",
+ exchangesWithThisMarketData[exchangePos], startBaseAmount, startQuoteAmount);
+ exchangesWithThisMarketData.erase(exchangesWithThisMarketData.begin() + exchangePos);
+ convertedBaseAmountPerExchange.erase(convertedBaseAmountPerExchange.begin() + exchangePos);
+ convertedQuoteAmountPerExchange.erase(convertedQuoteAmountPerExchange.begin() + exchangePos);
+ --exchangePos;
+ --nbExchanges;
+ continue;
+ }
+
+ const ExchangeConfig &exchangeConfig =
+ _coincenterInfo.exchangeConfig(exchangesWithThisMarketData[exchangePos].name());
+
+ marketTraderEngines.emplace_back(exchangeConfig, market, startBaseAmount, startQuoteAmount);
+ }
+ return marketTraderEngines;
+}
+
+MarketTradeRangeStatsPerExchange Coincenter::tradingProcess(const ReplayOptions &replayOptions,
+ std::span marketTraderEngines,
+ ExchangeNameSpan exchangesWithThisMarketData) {
+ const auto &automationConfig = _coincenterInfo.generalConfig().tradingConfig().automationConfig();
+ const auto loadChunkDuration = automationConfig.loadChunkDuration();
+ const auto timeWindow = replayOptions.timeWindow();
+
+ MarketTradeRangeStatsPerExchange tradeRangeResultsPerExchange;
+
+ // Main loop - parallelized by exchange, with time window chunks of loadChunkDuration
+
+ TimeWindow subTimeWindow(timeWindow.from(), loadChunkDuration);
+ while (subTimeWindow.overlaps(timeWindow)) {
+ auto subRangeResultsPerExchange = _exchangesOrchestrator.traderConsumeRange(
+ replayOptions, subTimeWindow, marketTraderEngines, exchangesWithThisMarketData);
+
+ if (tradeRangeResultsPerExchange.empty()) {
+ tradeRangeResultsPerExchange = std::move(subRangeResultsPerExchange);
+ } else {
+ int pos{};
+ for (auto &[exchange, result] : subRangeResultsPerExchange) {
+ tradeRangeResultsPerExchange[pos].second += result;
+ ++pos;
+ }
+ }
+
+ // Go to next sub time window
+ subTimeWindow = TimeWindow(subTimeWindow.to(), loadChunkDuration);
+ }
+
+ return tradeRangeResultsPerExchange;
+}
+
void Coincenter::updateFileCaches() const {
log::debug("Store all cache files");
diff --git a/src/engine/src/coincentercommand.cpp b/src/engine/src/coincentercommand.cpp
index aa80f1e2..1ac776b2 100644
--- a/src/engine/src/coincentercommand.cpp
+++ b/src/engine/src/coincentercommand.cpp
@@ -11,6 +11,7 @@
#include "market.hpp"
#include "monetaryamount.hpp"
#include "ordersconstraints.hpp"
+#include "timedef.hpp"
#include "tradeoptions.hpp"
#include "withdrawoptions.hpp"
#include "withdrawsconstraints.hpp"
@@ -124,4 +125,10 @@ CoincenterCommand& CoincenterCommand::withBalanceInUse(bool value) {
_withBalanceInUse = value;
return *this;
}
+
+CoincenterCommand& CoincenterCommand::setReplayOptions(ReplayOptions replayOptions) {
+ _specialOptions = std::move(replayOptions);
+ return *this;
+}
+
} // namespace cct
diff --git a/src/engine/src/coincentercommands.cpp b/src/engine/src/coincentercommands.cpp
index 6c6db1ed..f3bf85d7 100644
--- a/src/engine/src/coincentercommands.cpp
+++ b/src/engine/src/coincentercommands.cpp
@@ -11,7 +11,10 @@
#include "coincenteroptions.hpp"
#include "currencycode.hpp"
#include "depositsconstraints.hpp"
+#include "market.hpp"
+#include "replay-options.hpp"
#include "stringoptionparser.hpp"
+#include "time-window.hpp"
#include "timedef.hpp"
#include "withdrawsconstraints.hpp"
@@ -199,6 +202,46 @@ void CoincenterCommands::addOption(const CoincenterCmdLineOptions &cmdLineOption
.setExchangeNames(optionParser.parseExchanges());
}
+ if (!cmdLineOptions.marketData.empty()) {
+ optionParser = StringOptionParser(cmdLineOptions.marketData);
+
+ _commands.emplace_back(CoincenterCommandType::kMarketData)
+ .setMarket(optionParser.parseMarket())
+ .setExchangeNames(optionParser.parseExchanges());
+ }
+
+ if (cmdLineOptions.replay) {
+ optionParser = StringOptionParser(*cmdLineOptions.replay);
+
+ auto dur = optionParser.parseDuration(StringOptionParser::FieldIs::kOptional);
+
+ auto &cmd = _commands.emplace_back(CoincenterCommandType::kReplay)
+ .setReplayOptions(cmdLineOptions.computeReplayOptions(dur))
+ .setExchangeNames(optionParser.parseExchanges());
+
+ if (!cmdLineOptions.market.empty()) {
+ cmd.setMarket(Market(cmdLineOptions.market));
+ }
+ }
+
+ if (cmdLineOptions.replayMarkets) {
+ optionParser = StringOptionParser(*cmdLineOptions.replayMarkets);
+
+ TimeWindow timeWindow;
+ auto dur = optionParser.parseDuration(StringOptionParser::FieldIs::kOptional);
+ auto nowTime = Clock::now();
+ if (dur == kUndefinedDuration) {
+ timeWindow = TimeWindow(TimePoint{}, nowTime);
+ } else {
+ timeWindow = TimeWindow(nowTime - dur, nowTime);
+ }
+
+ _commands.emplace_back(CoincenterCommandType::kReplayMarkets)
+ .setReplayOptions(
+ ReplayOptions(timeWindow, cmdLineOptions.algorithmNames, ReplayOptions::ReplayMode::kValidateOnly))
+ .setExchangeNames(optionParser.parseExchanges());
+ }
+
optionParser.checkEndParsing(); // No more option part should be remaining
}
diff --git a/src/engine/src/coincenterinfo_create.cpp b/src/engine/src/coincenterinfo_create.cpp
index 061b6ca1..a3a88ba0 100644
--- a/src/engine/src/coincenterinfo_create.cpp
+++ b/src/engine/src/coincenterinfo_create.cpp
@@ -4,6 +4,7 @@
#include
#include "apioutputtype.hpp"
+#include "automation-config.hpp"
#include "cct_json.hpp"
#include "cct_string.hpp"
#include "coincenterinfo.hpp"
@@ -19,6 +20,7 @@
#include "runmodes.hpp"
#include "stringoptionparser.hpp"
#include "timedef.hpp"
+#include "trading-config.hpp"
namespace cct {
@@ -69,8 +71,21 @@ CoincenterInfo CoincenterInfo_Create(std::string_view programName, const Coincen
RequestsConfig requestsConfig(
generalConfigData.at("requests").at("concurrency").at("nbMaxParallelRequests").get());
- GeneralConfig generalConfig(std::move(loggingInfo), std::move(requestsConfig), fiatConversionQueryRate,
- apiOutputType);
+ const auto &automationJsonPart = generalConfigData.at("trading").at("automation");
+ const auto &deserializationJsonPart = automationJsonPart.at("deserialization");
+ const auto &startingContextJsonPart = automationJsonPart.at("startingContext");
+
+ Duration loadChunkDuration = ParseDuration(deserializationJsonPart.at("loadChunkDuration").get());
+ MonetaryAmount startBaseAmountEquivalent{
+ startingContextJsonPart.at("startBaseAmountEquivalent").get()};
+ MonetaryAmount startQuoteAmountEquivalent{
+ startingContextJsonPart.at("startQuoteAmountEquivalent").get()};
+
+ AutomationConfig automationConfig(loadChunkDuration, startBaseAmountEquivalent, startQuoteAmountEquivalent);
+ TradingConfig tradingConfig(std::move(automationConfig));
+
+ GeneralConfig generalConfig(std::move(loggingInfo), std::move(requestsConfig), std::move(tradingConfig),
+ fiatConversionQueryRate, apiOutputType);
const LoadConfiguration loadConfiguration(dataDir, LoadConfiguration::ExchangeConfigFileType::kProd);
diff --git a/src/engine/src/coincenteroptions.cpp b/src/engine/src/coincenteroptions.cpp
index 7871fee2..a82abc95 100644
--- a/src/engine/src/coincenteroptions.cpp
+++ b/src/engine/src/coincenteroptions.cpp
@@ -14,6 +14,7 @@
#include "priceoptions.hpp"
#include "priceoptionsdef.hpp"
#include "ssl_sha.hpp"
+#include "timedef.hpp"
#include "tradedefinitions.hpp"
#include "tradeoptions.hpp"
#include "withdrawoptions.hpp"
@@ -37,6 +38,10 @@ std::ostream& CoincenterCmdLineOptions::PrintVersion(std::string_view programNam
os << "compiled with " << CCT_COMPILER_VERSION << " on " << __DATE__ << " at " << __TIME__ << '\n';
os << " " << GetCurlVersionInfo() << '\n';
os << " " << ssl::GetOpenSSLVersion() << '\n';
+#ifdef CCT_PROTOBUF_VERSION
+ os << " "
+ << "protobuf " << CCT_PROTOBUF_VERSION << '\n';
+#endif
return os;
}
@@ -144,6 +149,31 @@ WithdrawOptions CoincenterCmdLineOptions::computeWithdrawOptions() const {
return {withdrawRefreshTime, withdrawSyncPolicy};
}
+ReplayOptions CoincenterCmdLineOptions::computeReplayOptions(Duration dur) const {
+ if (validate && validateOnly) {
+ throw invalid_argument("--validate and --validate-only cannot be specified simultaneously");
+ }
+
+ ReplayOptions::ReplayMode replayMode;
+ if (validateOnly) {
+ replayMode = ReplayOptions::ReplayMode::kValidateOnly;
+ } else if (validate) {
+ replayMode = ReplayOptions::ReplayMode::kCheckedLaunchAlgorithm;
+ } else {
+ replayMode = ReplayOptions::ReplayMode::kUncheckedLaunchAlgorithm;
+ }
+
+ TimeWindow timeWindow;
+ const auto nowTime = Clock::now();
+ if (dur == kUndefinedDuration) {
+ timeWindow = TimeWindow(TimePoint{}, nowTime);
+ } else {
+ timeWindow = TimeWindow(nowTime - dur, nowTime);
+ }
+
+ return ReplayOptions(timeWindow, algorithmNames, replayMode);
+}
+
std::pair CoincenterCmdLineOptions::getTradeArgStr() const {
if (!tradeStrategy.empty() && !tradePrice.empty()) {
throw invalid_argument("Trade price and trade strategy cannot be set together");
diff --git a/src/engine/src/exchangesorchestrator.cpp b/src/engine/src/exchangesorchestrator.cpp
index 05dd2ade..69af6f42 100644
--- a/src/engine/src/exchangesorchestrator.cpp
+++ b/src/engine/src/exchangesorchestrator.cpp
@@ -31,13 +31,16 @@
#include "exchangepublicapi.hpp"
#include "exchangepublicapitypes.hpp"
#include "exchangeretriever.hpp"
+#include "market-trader-engine.hpp"
#include "market.hpp"
#include "monetaryamount.hpp"
#include "monetaryamountbycurrencyset.hpp"
#include "ordersconstraints.hpp"
#include "queryresulttypes.hpp"
+#include "replay-options.hpp"
#include "requestsconfig.hpp"
#include "threadpool.hpp"
+#include "trade-range-stats.hpp"
#include "tradedamounts.hpp"
#include "tradeoptions.hpp"
#include "traderesult.hpp"
@@ -118,7 +121,7 @@ ExchangeRetriever::PublicExchangesVec SelectUniquePublicExchanges(ExchangeRetrie
ExchangesOrchestrator::ExchangesOrchestrator(const RequestsConfig &requestsConfig, std::span exchangesSpan)
: _exchangeRetriever(exchangesSpan),
_threadPool(requestsConfig.nbMaxParallelRequests(static_cast(exchangesSpan.size()))) {
- log::info("Created a thread pool with {} workers for exchange requests", _threadPool.nbWorkers());
+ log::debug("Created a thread pool with {} workers for exchange requests", _threadPool.nbWorkers());
}
ExchangeHealthCheckStatus ExchangesOrchestrator::healthCheck(ExchangeNameSpan exchangeNames) {
@@ -170,7 +173,7 @@ MarketOrderBookConversionRates ExchangesOrchestrator::getMarketOrderBooks(Market
if (!optConversionRate && !equiCurrencyCode.isNeutral()) {
log::warn("Unable to convert {} into {} on {}", mk.quote(), equiCurrencyCode, exchange->name());
}
- return std::make_tuple(exchange->name(), exchange->queryOrderBook(mk, actualDepth), optConversionRate);
+ return std::make_tuple(exchange->name(), exchange->getOrderBook(mk, actualDepth), optConversionRate);
};
_threadPool.parallelTransform(selectedExchanges.begin(), selectedExchanges.end(), ret.begin(), marketOrderBooksFunc);
return ret;
@@ -940,7 +943,7 @@ TradesPerExchange ExchangesOrchestrator::getLastTradesPerExchange(Market mk, Exc
TradesPerExchange ret(selectedExchanges.size());
_threadPool.parallelTransform(
selectedExchanges.begin(), selectedExchanges.end(), ret.begin(), [mk, nbLastTrades](Exchange *exchange) {
- return std::make_pair(static_cast(exchange), exchange->queryLastTrades(mk, nbLastTrades));
+ return std::make_pair(static_cast(exchange), exchange->getLastTrades(mk, nbLastTrades));
});
return ret;
@@ -957,4 +960,116 @@ MonetaryAmountPerExchange ExchangesOrchestrator::getLastPricePerExchange(Market
return lastPricePerExchange;
}
+MarketDataPerExchange ExchangesOrchestrator::getMarketDataPerExchange(std::span marketPerPublicExchange,
+ ExchangeNameSpan exchangeNames) {
+ UniquePublicSelectedExchanges selectedExchanges = _exchangeRetriever.selectOneAccount(exchangeNames);
+
+ std::array isMarketTradable;
+
+ _threadPool.parallelTransform(selectedExchanges.begin(), selectedExchanges.end(), isMarketTradable.begin(),
+ [&marketPerPublicExchange](Exchange *exchange) {
+ Market market = marketPerPublicExchange[exchange->publicExchangePos()];
+ return market.isDefined() && exchange->queryTradableMarkets().contains(market);
+ });
+
+ FilterVector(selectedExchanges, isMarketTradable);
+
+ MarketDataPerExchange ret(selectedExchanges.size());
+ _threadPool.parallelTransform(
+ selectedExchanges.begin(), selectedExchanges.end(), ret.begin(), [&marketPerPublicExchange](Exchange *exchange) {
+ if (!exchange->exchangeConfig().withMarketDataSerialization()) {
+ log::warn("Calling market-data on {} with data serialization disabled", exchange->name());
+ }
+ // Call order book and last trades sequentially for this exchange
+ Market market = marketPerPublicExchange[exchange->publicExchangePos()];
+ return std::make_pair(exchange,
+ std::make_pair(exchange->getOrderBook(market), exchange->getLastTrades(market)));
+ });
+ return ret;
+}
+
+MarketTimestampSetsPerExchange ExchangesOrchestrator::pullAvailableMarketsForReplay(TimeWindow timeWindow,
+ ExchangeNameSpan exchangeNames) {
+ log::info("Query available markets for replay from {} within {}", ConstructAccumulatedExchangeNames(exchangeNames),
+ timeWindow);
+ UniquePublicSelectedExchanges selectedExchanges = _exchangeRetriever.selectOneAccount(exchangeNames);
+ MarketTimestampSetsPerExchange marketTimestampSetsPerExchange(selectedExchanges.size());
+ _threadPool.parallelTransform(selectedExchanges.begin(), selectedExchanges.end(),
+ marketTimestampSetsPerExchange.begin(), [timeWindow](Exchange *exchange) {
+ return std::make_pair(
+ exchange,
+ MarketTimestampSets{exchange->apiPublic().pullMarketOrderBooksMarkets(timeWindow),
+ exchange->apiPublic().pullTradeMarkets(timeWindow)});
+ });
+ return marketTimestampSetsPerExchange;
+}
+
+MarketTradeRangeStatsPerExchange ExchangesOrchestrator::traderConsumeRange(
+ const ReplayOptions &replayOptions, TimeWindow subTimeWindow, std::span marketTraderEngines,
+ ExchangeNameSpan exchangeNames) {
+ UniquePublicSelectedExchanges selectedExchanges = _exchangeRetriever.selectOneAccount(exchangeNames);
+
+ MarketTradeRangeStatsPerExchange tradeRangeResultsPerExchange(selectedExchanges.size());
+
+ _threadPool.parallelTransform(
+ selectedExchanges.begin(), selectedExchanges.end(), marketTraderEngines.begin(),
+ tradeRangeResultsPerExchange.begin(),
+ [subTimeWindow, &replayOptions](Exchange *exchange, MarketTraderEngine &marketTraderEngine) {
+ Market market = marketTraderEngine.market();
+ auto &apiPublic = exchange->apiPublic();
+ auto marketOrderBooks = apiPublic.pullMarketOrderBooksForReplay(market, subTimeWindow);
+ auto publicTrades = apiPublic.pullTradesForReplay(market, subTimeWindow);
+
+ TradeRangeStats tradeRangeStats;
+
+ switch (replayOptions.replayMode()) {
+ case ReplayOptions::ReplayMode::kValidateOnly:
+ tradeRangeStats = marketTraderEngine.validateRange(std::move(marketOrderBooks), std::move(publicTrades));
+ break;
+ case ReplayOptions::ReplayMode::kCheckedLaunchAlgorithm:
+ tradeRangeStats = marketTraderEngine.validateRange(marketOrderBooks, publicTrades);
+ marketTraderEngine.tradeRange(std::move(marketOrderBooks), std::move(publicTrades));
+ break;
+ case ReplayOptions::ReplayMode::kUncheckedLaunchAlgorithm:
+ tradeRangeStats = marketTraderEngine.tradeRange(std::move(marketOrderBooks), std::move(publicTrades));
+ break;
+ default:
+ break;
+ }
+
+ return std::make_pair(exchange, std::move(tradeRangeStats));
+ });
+
+ return tradeRangeResultsPerExchange;
+}
+
+MarketTradingGlobalResultPerExchange ExchangesOrchestrator::getMarketTraderResultPerExchange(
+ std::span marketTraderEngines, MarketTradeRangeStatsPerExchange &&tradeRangeStatsPerExchange,
+ ExchangeNameSpan exchangeNames) {
+ UniquePublicSelectedExchanges selectedExchanges = _exchangeRetriever.selectOneAccount(exchangeNames);
+
+ if (selectedExchanges.size() != tradeRangeStatsPerExchange.size()) {
+ throw exception("Inconsistent selected exchange sizes");
+ }
+
+ MarketTradingResultPerExchange marketTradingResultPerExchange(selectedExchanges.size());
+
+ _threadPool.parallelTransform(selectedExchanges.begin(), selectedExchanges.end(), marketTraderEngines.begin(),
+ marketTradingResultPerExchange.begin(),
+ [](const Exchange *exchange, MarketTraderEngine &marketTraderEngine) {
+ return std::make_pair(exchange, marketTraderEngine.finalizeAndComputeResult());
+ });
+
+ MarketTradingGlobalResultPerExchange marketTradingGlobalResultPerExchange(selectedExchanges.size());
+ std::transform(marketTradingResultPerExchange.begin(), marketTradingResultPerExchange.end(),
+ tradeRangeStatsPerExchange.begin(), marketTradingGlobalResultPerExchange.begin(),
+ [](auto &exchangeMarketTradingResult, auto &exchangeTradeRangeStats) {
+ return std::make_pair(exchangeMarketTradingResult.first,
+ MarketTradingGlobalResult{std::move(exchangeMarketTradingResult.second),
+ std::move(exchangeTradeRangeStats.second)});
+ });
+
+ return marketTradingGlobalResultPerExchange;
+}
+
} // namespace cct
diff --git a/src/engine/src/query-result-type-helpers.cpp b/src/engine/src/query-result-type-helpers.cpp
new file mode 100644
index 00000000..a6f8dcf3
--- /dev/null
+++ b/src/engine/src/query-result-type-helpers.cpp
@@ -0,0 +1,31 @@
+#include "query-result-type-helpers.hpp"
+
+#include
+#include
+
+#include "market.hpp"
+
+namespace cct {
+
+bool ContainsMarket(Market market, const MarketTimestampSet &marketTimestampSet) {
+ auto it = std::ranges::partition_point(
+ marketTimestampSet, [market](const auto &marketTimestamp) { return marketTimestamp.market < market; });
+ return it != marketTimestampSet.end() && it->market == market;
+}
+
+bool ContainsMarket(Market market, const MarketTimestampSets &marketTimestampSets) {
+ return ContainsMarket(market, marketTimestampSets.orderBooksMarkets) ||
+ ContainsMarket(market, marketTimestampSets.tradesMarkets);
+}
+
+MarketSet ComputeAllMarkets(const MarketTimestampSetsPerExchange &marketTimestampSetsPerExchange) {
+ MarketSet allMarkets;
+ for (const auto &[_, marketTimestamps] : marketTimestampSetsPerExchange) {
+ std::ranges::transform(marketTimestamps.orderBooksMarkets, std::inserter(allMarkets, allMarkets.end()),
+ [](const auto &marketTimestamp) { return marketTimestamp.market; });
+ std::ranges::transform(marketTimestamps.tradesMarkets, std::inserter(allMarkets, allMarkets.end()),
+ [](const auto &marketTimestamp) { return marketTimestamp.market; });
+ }
+ return allMarkets;
+}
+} // namespace cct
\ No newline at end of file
diff --git a/src/engine/src/queryresultprinter.cpp b/src/engine/src/queryresultprinter.cpp
index ee88df19..40ad3557 100644
--- a/src/engine/src/queryresultprinter.cpp
+++ b/src/engine/src/queryresultprinter.cpp
@@ -27,6 +27,7 @@
#include "exchange.hpp"
#include "file.hpp"
#include "logginginfo.hpp"
+#include "market-timestamp.hpp"
#include "market.hpp"
#include "marketorderbook.hpp"
#include "monetaryamount.hpp"
@@ -36,9 +37,11 @@
#include "priceoptions.hpp"
#include "priceoptionsdef.hpp"
#include "publictrade.hpp"
+#include "query-result-type-helpers.hpp"
#include "queryresulttypes.hpp"
#include "simpletable.hpp"
#include "stringhelpers.hpp"
+#include "time-window.hpp"
#include "timestring.hpp"
#include "tradedamounts.hpp"
#include "tradedefinitions.hpp"
@@ -126,6 +129,47 @@ json MarketsJson(CurrencyCode cur1, CurrencyCode cur2, const MarketsPerExchange
return ToJson(CoincenterCommandType::kMarkets, std::move(in), std::move(out));
}
+json MarketsForReplayJson(TimeWindow timeWindow, const MarketTimestampSetsPerExchange &marketTimestampSetsPerExchange) {
+ json in;
+ json inOpt = json::object();
+ if (timeWindow != TimeWindow{}) {
+ inOpt.emplace("timeWindow", timeWindow.str());
+ }
+ in.emplace("opt", std::move(inOpt));
+
+ json out = json::object();
+ for (const auto &[e, marketTimestampSets] : marketTimestampSetsPerExchange) {
+ json orderBookMarketsPerExchange;
+ for (const MarketTimestamp &marketTimestamp : marketTimestampSets.orderBooksMarkets) {
+ json marketTimestampJson;
+
+ marketTimestampJson.emplace("market", marketTimestamp.market.str());
+ marketTimestampJson.emplace("lastTimestamp", ToString(marketTimestamp.timePoint));
+
+ orderBookMarketsPerExchange.emplace_back(std::move(marketTimestampJson));
+ }
+
+ json tradesMarketsPerExchange;
+ for (const MarketTimestamp &marketTimestamp : marketTimestampSets.tradesMarkets) {
+ json marketTimestampJson;
+
+ marketTimestampJson.emplace("market", marketTimestamp.market.str());
+ marketTimestampJson.emplace("lastTimestamp", ToString(marketTimestamp.timePoint));
+
+ tradesMarketsPerExchange.emplace_back(std::move(marketTimestampJson));
+ }
+
+ json exchangePart;
+
+ exchangePart.emplace("orderBooks", std::move(orderBookMarketsPerExchange));
+ exchangePart.emplace("trades", std::move(tradesMarketsPerExchange));
+
+ out.emplace(e->name(), std::move(exchangePart));
+ }
+
+ return ToJson(CoincenterCommandType::kReplayMarkets, std::move(in), std::move(out));
+}
+
json TickerInformationJson(const ExchangeTickerMaps &exchangeTickerMaps) {
json in;
json out = json::object();
@@ -720,6 +764,57 @@ json DustSweeperJson(const TradedAmountsVectorWithFinalAmountPerExchange &traded
return ToJson(CoincenterCommandType::kDustSweeper, std::move(in), std::move(out));
}
+json MarketTradingResultsJson(TimeWindow timeWindow,
+ const MarketTradingGlobalResultPerExchange &marketTradingResultPerExchange,
+ CoincenterCommandType commandType) {
+ json in;
+ json inOpt;
+ inOpt.emplace("time-window", timeWindow.str());
+ in.emplace("opt", std::move(inOpt));
+
+ json out = json::object();
+
+ for (const auto &[exchangePtr, marketGlobalTradingResult] : marketTradingResultPerExchange) {
+ const auto &marketTradingResult = marketGlobalTradingResult.result;
+ const auto &stats = marketGlobalTradingResult.stats;
+
+ json startAmounts;
+ startAmounts.emplace("base", marketTradingResult.startBaseAmount().str());
+ startAmounts.emplace("quote", marketTradingResult.startQuoteAmount().str());
+
+ json orderBookStats;
+ orderBookStats.emplace("nb-successful", stats.marketOrderBookStats.nbSuccessful);
+ orderBookStats.emplace("nb-error", stats.marketOrderBookStats.nbError);
+
+ json tradeStats;
+ tradeStats.emplace("nb-successful", stats.publicTradeStats.nbSuccessful);
+ tradeStats.emplace("nb-error", stats.publicTradeStats.nbError);
+
+ json jsonStats;
+ jsonStats.emplace("order-books", std::move(orderBookStats));
+ jsonStats.emplace("trades", std::move(tradeStats));
+
+ json marketTradingResultJson;
+ marketTradingResultJson.emplace("algorithm", marketTradingResult.algorithmName());
+ marketTradingResultJson.emplace("market", marketTradingResult.market().str());
+ marketTradingResultJson.emplace("start-amounts", std::move(startAmounts));
+ marketTradingResultJson.emplace("profit-and-loss", marketTradingResult.quoteAmountDelta().str());
+ marketTradingResultJson.emplace("stats", std::move(jsonStats));
+
+ json closedOrdersArray = json::array_t();
+
+ for (const ClosedOrder &closedOrder : marketTradingResult.matchedOrders()) {
+ closedOrdersArray.push_back(OrderJson(closedOrder));
+ }
+
+ marketTradingResultJson.emplace("matched-orders", std::move(closedOrdersArray));
+
+ out.emplace(exchangePtr->name(), std::move(marketTradingResultJson));
+ }
+
+ return ToJson(commandType, std::move(in), std::move(out));
+}
+
template
void RemoveDuplicates(VecType &vec) {
std::ranges::sort(vec);
@@ -1445,6 +1540,123 @@ void QueryResultPrinter::printDustSweeper(
logActivity(CoincenterCommandType::kDustSweeper, jsonData);
}
+void QueryResultPrinter::printMarketsForReplay(TimeWindow timeWindow,
+ const MarketTimestampSetsPerExchange &marketTimestampSetsPerExchange) {
+ json jsonData = MarketsForReplayJson(timeWindow, marketTimestampSetsPerExchange);
+ switch (_apiOutputType) {
+ case ApiOutputType::kFormattedTable: {
+ MarketSet allMarkets = ComputeAllMarkets(marketTimestampSetsPerExchange);
+
+ SimpleTable table;
+ table.reserve(allMarkets.size() + 1U);
+ table.emplace_back("Markets", "Last order books timestamp", "Last trades timestamp");
+
+ for (const Market market : allMarkets) {
+ table::Cell orderBookCell;
+ table::Cell tradesCell;
+ for (const auto &[e, marketTimestamps] : marketTimestampSetsPerExchange) {
+ const auto &orderBooksMarkets = marketTimestamps.orderBooksMarkets;
+ const auto &tradesMarkets = marketTimestamps.tradesMarkets;
+ const auto marketPartitionPred = [market](const auto &marketTimestamp) {
+ return marketTimestamp.market < market;
+ };
+ const auto orderBooksIt = std::ranges::partition_point(orderBooksMarkets, marketPartitionPred);
+ const auto tradesIt = std::ranges::partition_point(tradesMarkets, marketPartitionPred);
+
+ if (orderBooksIt != orderBooksMarkets.end() && orderBooksIt->market == market) {
+ string str = ToString(orderBooksIt->timePoint);
+ str.append(" @ ");
+ str.append(e->name());
+
+ orderBookCell.emplace_back(std::move(str));
+ }
+
+ if (tradesIt != tradesMarkets.end() && tradesIt->market == market) {
+ string str = ToString(tradesIt->timePoint);
+ str.append(" @ ");
+ str.append(e->name());
+
+ tradesCell.emplace_back(std::move(str));
+ }
+ }
+
+ table.emplace_back(market.str(), std::move(orderBookCell), std::move(tradesCell));
+ }
+ printTable(table);
+ break;
+ }
+ case ApiOutputType::kJson:
+ printJson(jsonData);
+ break;
+ case ApiOutputType::kNoPrint:
+ break;
+ }
+ logActivity(CoincenterCommandType::kReplayMarkets, jsonData);
+}
+
+void QueryResultPrinter::printMarketTradingResults(
+ TimeWindow timeWindow, const MarketTradingGlobalResultPerExchange &marketTradingResultPerExchange,
+ CoincenterCommandType commandType) const {
+ json jsonData = MarketTradingResultsJson(timeWindow, marketTradingResultPerExchange, commandType);
+ switch (_apiOutputType) {
+ case ApiOutputType::kFormattedTable: {
+ SimpleTable table;
+ table.reserve(1U + marketTradingResultPerExchange.size());
+ table.emplace_back("Exchange", "Time window", "Market", "Algorithm", "Start amounts", "Profit / Loss",
+ "Matched orders", "Stats");
+ for (const auto &[exchangePtr, marketGlobalTradingResults] : marketTradingResultPerExchange) {
+ const auto &marketTradingResults = marketGlobalTradingResults.result;
+ const auto &stats = marketGlobalTradingResults.stats;
+
+ table::Cell trades;
+ for (const ClosedOrder &closedOrder : marketTradingResults.matchedOrders()) {
+ string orderStr = closedOrder.placedTimeStr();
+ orderStr.append(" - ");
+ orderStr.append(closedOrder.sideStr());
+ orderStr.append(" - ");
+ orderStr.append(closedOrder.matchedVolume().str());
+ orderStr.append(" @ ");
+ orderStr.append(closedOrder.price().str());
+ trades.emplace_back(std::move(orderStr));
+ }
+
+ string orderBookStats("order books: ");
+ orderBookStats.append(ToString(stats.marketOrderBookStats.nbSuccessful));
+ orderBookStats.append(" OK");
+ if (stats.marketOrderBookStats.nbError != 0) {
+ orderBookStats.append(", ");
+ orderBookStats.append(ToString(stats.marketOrderBookStats.nbError));
+ orderBookStats.append(" KO");
+ }
+
+ string tradesStats("trades: ");
+ tradesStats.append(ToString(stats.publicTradeStats.nbSuccessful));
+ tradesStats.append(" OK");
+ if (stats.publicTradeStats.nbError != 0) {
+ tradesStats.append(", ");
+ tradesStats.append(ToString(stats.publicTradeStats.nbError));
+ tradesStats.append(" KO");
+ }
+
+ table.emplace_back(
+ exchangePtr->name(), table::Cell{ToString(timeWindow.from()), ToString(timeWindow.to())},
+ marketTradingResults.market().str(), marketTradingResults.algorithmName(),
+ table::Cell{marketTradingResults.startBaseAmount().str(), marketTradingResults.startQuoteAmount().str()},
+ marketTradingResults.quoteAmountDelta().str(), std::move(trades),
+ table::Cell{std::move(orderBookStats), std::move(tradesStats)});
+ }
+ printTable(table);
+ break;
+ }
+ case ApiOutputType::kJson:
+ printJson(jsonData);
+ break;
+ case ApiOutputType::kNoPrint:
+ break;
+ }
+ logActivity(commandType, jsonData);
+}
+
void QueryResultPrinter::printTable(const SimpleTable &table) const {
std::ostringstream ss;
std::ostream &os = _pOs != nullptr ? *_pOs : ss;
diff --git a/src/engine/src/replay-algorithm-name-iterator.cpp b/src/engine/src/replay-algorithm-name-iterator.cpp
new file mode 100644
index 00000000..3d5e30b8
--- /dev/null
+++ b/src/engine/src/replay-algorithm-name-iterator.cpp
@@ -0,0 +1,62 @@
+#include "replay-algorithm-name-iterator.hpp"
+
+#include
+#include
+
+#include "cct_exception.hpp"
+
+namespace cct {
+
+namespace {
+constexpr std::string_view kAlgorithmNameSeparator = ",";
+
+auto FindNextSeparatorPos(std::string_view str, std::string_view::size_type pos = 0) {
+ pos = str.find(kAlgorithmNameSeparator, pos);
+ if (pos == std::string_view::npos) {
+ pos = str.length();
+ }
+ return pos;
+}
+} // namespace
+
+ReplayAlgorithmNameIterator::ReplayAlgorithmNameIterator(std::string_view algorithmNames,
+ std::span allAlgorithms)
+ : _allAlgorithms(allAlgorithms),
+ _algorithmNames(algorithmNames),
+ _begPos(0),
+ _endPos(FindNextSeparatorPos(_algorithmNames)) {
+ if (std::ranges::any_of(allAlgorithms, [](const auto algName) {
+ return algName.find(kAlgorithmNameSeparator) != std::string_view::npos;
+ })) {
+ throw exception("Algorithm names cannot contain '{}' as it's used as a separator", kAlgorithmNameSeparator);
+ }
+}
+
+bool ReplayAlgorithmNameIterator::hasNext() const {
+ using PosT = decltype(_begPos);
+
+ if (_algorithmNames.empty()) {
+ return _begPos < static_cast(_allAlgorithms.size());
+ }
+
+ return _begPos != static_cast(_algorithmNames.length());
+}
+
+std::string_view ReplayAlgorithmNameIterator::next() {
+ if (_algorithmNames.empty()) {
+ return _allAlgorithms[_begPos++];
+ }
+
+ std::string_view nextAlgorithmName(_algorithmNames.begin() + _begPos, _algorithmNames.begin() + _endPos);
+
+ if (_endPos == static_cast(_algorithmNames.length())) {
+ _begPos = _endPos;
+ } else {
+ _begPos = _endPos + kAlgorithmNameSeparator.length();
+ _endPos = FindNextSeparatorPos(_algorithmNames, _begPos);
+ }
+
+ return nextAlgorithmName;
+}
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/engine/src/replay-options.cpp b/src/engine/src/replay-options.cpp
new file mode 100644
index 00000000..e0ab2694
--- /dev/null
+++ b/src/engine/src/replay-options.cpp
@@ -0,0 +1,19 @@
+#include "replay-options.hpp"
+
+#include
+
+#include "dummy-market-trader.hpp"
+#include "time-window.hpp"
+
+namespace cct {
+ReplayOptions::ReplayOptions(TimeWindow timeWindow, std::string_view algorithmNames, ReplayMode replayMode)
+ : _timeWindow(timeWindow), _algorithmNames(algorithmNames), _replayMode(replayMode) {}
+
+std::string_view ReplayOptions::algorithmNames() const {
+ if (_replayMode == ReplayMode::kValidateOnly) {
+ return DummyMarketTrader::kName;
+ }
+ return _algorithmNames;
+}
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/engine/src/stringoptionparser.cpp b/src/engine/src/stringoptionparser.cpp
index 11502459..6435837d 100644
--- a/src/engine/src/stringoptionparser.cpp
+++ b/src/engine/src/stringoptionparser.cpp
@@ -9,10 +9,12 @@
#include "cct_string.hpp"
#include "cct_vector.hpp"
#include "currencycode.hpp"
+#include "durationstring.hpp"
#include "exchange-names.hpp"
#include "exchangename.hpp"
#include "market.hpp"
#include "monetaryamount.hpp"
+#include "timedef.hpp"
namespace cct {
@@ -38,6 +40,23 @@ CurrencyCode StringOptionParser::parseCurrency(FieldIs fieldIs, char delimiter)
return {};
}
+Duration StringOptionParser::parseDuration(FieldIs fieldIs) {
+ auto dur = kUndefinedDuration;
+ const std::string_view currentToken(_opt.begin() + _pos, _opt.end());
+ const auto durationLen = DurationLen(currentToken);
+ if (durationLen > 0) {
+ const std::string_view durationStr(_opt.data() + _pos, static_cast(durationLen));
+
+ dur = ParseDuration(durationStr);
+ } else if (fieldIs == FieldIs::kMandatory) {
+ throw invalid_argument("Expected a valid duration in '{}'", currentToken);
+ }
+
+ _pos += durationLen;
+
+ return dur;
+}
+
// At the end of the market, either the end of the string or a comma is expected.
Market StringOptionParser::parseMarket(FieldIs fieldIs, char delimiter) {
const auto oldPos = _pos;
diff --git a/src/engine/test/queryresultprinter_public_test.cpp b/src/engine/test/queryresultprinter_public_test.cpp
index 57bbc12e..6ba960df 100644
--- a/src/engine/test/queryresultprinter_public_test.cpp
+++ b/src/engine/test/queryresultprinter_public_test.cpp
@@ -9,15 +9,21 @@
#include "currencycode.hpp"
#include "currencyexchange.hpp"
#include "currencyexchangeflatset.hpp"
+#include "exchangeprivateapitypes.hpp"
#include "exchangepublicapitypes.hpp"
+#include "market-trading-global-result.hpp"
+#include "market-trading-result.hpp"
#include "market.hpp"
#include "marketorderbook.hpp"
#include "monetaryamount.hpp"
#include "monetaryamountbycurrencyset.hpp"
+#include "public-trade-vector.hpp"
#include "publictrade.hpp"
#include "queryresultprinter.hpp"
#include "queryresultprinter_base_test.hpp"
#include "queryresulttypes.hpp"
+#include "time-window.hpp"
+#include "trade-range-stats.hpp"
#include "tradeside.hpp"
namespace cct {
@@ -243,7 +249,6 @@ TEST_F(QueryResultPrinterMarketsTest, FormattedTableNoCurrency) {
| huobi | XRP-EUR |
+----------+---------+
)";
-
expectStr(kExpected);
}
@@ -262,7 +267,6 @@ TEST_F(QueryResultPrinterMarketsTest, FormattedTableOneCurrency) {
| huobi | XRP-EUR |
+----------+------------------+
)";
-
expectStr(kExpected);
}
@@ -279,7 +283,6 @@ TEST_F(QueryResultPrinterMarketsTest, FormattedTableTwoCurrencies) {
| huobi | XRP-EUR |
+----------+----------------------+
)";
-
expectStr(kExpected);
}
@@ -1176,4 +1179,303 @@ TEST_F(QueryResultPrinterLastPriceTest, NoPrint) {
expectNoStr();
}
+class QueryResultPrinterReplayBaseTest : public QueryResultPrinterTest {
+ protected:
+ Market market1{"ETH", "KRW"};
+ Market market2{"BTC", "USD"};
+ Market market3{"SHIB", "USDT"};
+ Market market4{"SOL", "BTC"};
+ Market market5{"SOL", "ETH"};
+ Market market6{"ETH", "BTC"};
+ Market market7{"DOGE", "CAD"};
+
+ TimePoint tp1{milliseconds{std::numeric_limits::max() / 10000000}};
+ TimePoint tp2{milliseconds{std::numeric_limits::max() / 9900000}};
+ TimePoint tp3{milliseconds{std::numeric_limits::max() / 9800000}};
+ TimePoint tp4{milliseconds{std::numeric_limits::max() / 9600000}};
+ TimePoint tp5{milliseconds{std::numeric_limits::max() / 9500000}};
+
+ TimeWindow timeWindow{tp1, tp5};
+};
+
+class QueryResultPrinterReplayMarketsTest : public QueryResultPrinterReplayBaseTest {
+ protected:
+ MarketTimestampSetsPerExchange marketTimestampSetsPerExchange{
+ {&exchange1,
+ MarketTimestampSets{MarketTimestampSet{MarketTimestamp{market1, tp1}, MarketTimestamp{market2, tp2},
+ MarketTimestamp{market3, tp3}},
+ MarketTimestampSet{MarketTimestamp{market1, tp1}, MarketTimestamp{market2, tp1}}}},
+ {&exchange2, MarketTimestampSets{MarketTimestampSet{MarketTimestamp{market2, tp4}, MarketTimestamp{market4, tp5}},
+ MarketTimestampSet{MarketTimestamp{market6, tp1}}}},
+ {&exchange3, MarketTimestampSets{MarketTimestampSet{}, MarketTimestampSet{MarketTimestamp{market1, tp1},
+ MarketTimestamp{market7, tp4}}}}};
+};
+
+TEST_F(QueryResultPrinterReplayMarketsTest, FormattedTable) {
+ basicQueryResultPrinter(ApiOutputType::kFormattedTable)
+ .printMarketsForReplay(timeWindow, marketTimestampSetsPerExchange);
+ static constexpr std::string_view kExpected = R"(
++-----------+--------------------------------+--------------------------------+
+| Markets | Last order books timestamp | Last trades timestamp |
++-----------+--------------------------------+--------------------------------+
+| BTC-USD | 1999-07-11T00:42:21Z @ binance | 1999-03-25T04:46:43Z @ binance |
+| | 2000-06-11T23:58:40Z @ bithumb | |
+|~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|
+| DOGE-CAD | | 2000-06-11T23:58:40Z @ huobi |
+| ETH-BTC | | 1999-03-25T04:46:43Z @ bithumb |
+|~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|
+| ETH-KRW | 1999-03-25T04:46:43Z @ binance | 1999-03-25T04:46:43Z @ binance |
+| | | 1999-03-25T04:46:43Z @ huobi |
+|~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|
+| SHIB-USDT | 1999-10-29T01:26:51Z @ binance | |
+| SOL-BTC | 2000-10-07T01:14:27Z @ bithumb | |
++-----------+--------------------------------+--------------------------------+
+)";
+ expectStr(kExpected);
+}
+
+TEST_F(QueryResultPrinterReplayMarketsTest, EmptyJson) {
+ basicQueryResultPrinter(ApiOutputType::kJson).printMarketsForReplay(timeWindow, MarketTimestampSetsPerExchange{});
+ static constexpr std::string_view kExpected = R"json(
+{
+ "in": {
+ "opt": {
+ "timeWindow": "[1999-03-25 04:46:43 -> 2000-10-07 01:14:27)"
+ },
+ "req": "ReplayMarkets"
+ },
+ "out": {}
+})json";
+ expectJson(kExpected);
+}
+
+TEST_F(QueryResultPrinterReplayMarketsTest, Json) {
+ basicQueryResultPrinter(ApiOutputType::kJson).printMarketsForReplay(timeWindow, marketTimestampSetsPerExchange);
+ static constexpr std::string_view kExpected = R"json(
+{
+ "in": {
+ "opt": {
+ "timeWindow": "[1999-03-25 04:46:43 -> 2000-10-07 01:14:27)"
+ },
+ "req": "ReplayMarkets"
+ },
+ "out": {
+ "binance": {
+ "orderBooks": [
+ {
+ "lastTimestamp": "1999-07-11T00:42:21Z",
+ "market": "BTC-USD"
+ },
+ {
+ "lastTimestamp": "1999-03-25T04:46:43Z",
+ "market": "ETH-KRW"
+ },
+ {
+ "lastTimestamp": "1999-10-29T01:26:51Z",
+ "market": "SHIB-USDT"
+ }
+ ],
+ "trades": [
+ {
+ "lastTimestamp": "1999-03-25T04:46:43Z",
+ "market": "BTC-USD"
+ },
+ {
+ "lastTimestamp": "1999-03-25T04:46:43Z",
+ "market": "ETH-KRW"
+ }
+ ]
+ },
+ "bithumb": {
+ "orderBooks": [
+ {
+ "lastTimestamp": "2000-06-11T23:58:40Z",
+ "market": "BTC-USD"
+ },
+ {
+ "lastTimestamp": "2000-10-07T01:14:27Z",
+ "market": "SOL-BTC"
+ }
+ ],
+ "trades": [
+ {
+ "lastTimestamp": "1999-03-25T04:46:43Z",
+ "market": "ETH-BTC"
+ }
+ ]
+ },
+ "huobi": {
+ "orderBooks": null,
+ "trades": [
+ {
+ "lastTimestamp": "2000-06-11T23:58:40Z",
+ "market": "DOGE-CAD"
+ },
+ {
+ "lastTimestamp": "1999-03-25T04:46:43Z",
+ "market": "ETH-KRW"
+ }
+ ]
+ }
+ }
+})json";
+ expectJson(kExpected);
+}
+
+TEST_F(QueryResultPrinterReplayMarketsTest, NoPrint) {
+ basicQueryResultPrinter(ApiOutputType::kNoPrint).printMarketsForReplay(timeWindow, marketTimestampSetsPerExchange);
+ expectNoStr();
+}
+
+class QueryResultPrinterReplayTest : public QueryResultPrinterReplayBaseTest {
+ protected:
+ ClosedOrder closedOrder1{"1", MonetaryAmount(15, "BTC", 1), MonetaryAmount(35000, "USDT"), tp1, tp1, TradeSide::kBuy};
+ ClosedOrder closedOrder2{"2", MonetaryAmount(25, "BTC", 1), MonetaryAmount(45000, "USDT"), tp2, tp2, TradeSide::kBuy};
+ ClosedOrder closedOrder3{"3", MonetaryAmount(5, "BTC", 2), MonetaryAmount(35000, "USDT"), tp3, tp4, TradeSide::kSell};
+ ClosedOrder closedOrder4{
+ "4", MonetaryAmount(17, "BTC", 1), MonetaryAmount(50000, "USDT"), tp3, tp4, TradeSide::kSell};
+ ClosedOrder closedOrder5{
+ "5", MonetaryAmount(36, "BTC", 3), MonetaryAmount(47899, "USDT"), tp4, tp5, TradeSide::kSell};
+
+ std::string_view algorithmName = "test-algo";
+ MonetaryAmount startBaseAmount{1, "BTC"};
+ MonetaryAmount startQuoteAmount{1000, "EUR"};
+
+ MarketTradingResult marketTradingResult1{algorithmName, startBaseAmount, startQuoteAmount, MonetaryAmount{0, "EUR"},
+ ClosedOrderVector{}};
+ MarketTradingResult marketTradingResult3{algorithmName, startBaseAmount, startQuoteAmount, MonetaryAmount{500, "EUR"},
+ ClosedOrderVector{closedOrder1, closedOrder5}};
+ MarketTradingResult marketTradingResult4{algorithmName, startBaseAmount, startQuoteAmount, MonetaryAmount{780, "EUR"},
+ ClosedOrderVector{closedOrder2, closedOrder3, closedOrder4}};
+
+ TradeRangeStats tradeRangeStats1{TradeRangeResultsStats{42, 0}, TradeRangeResultsStats{3, 10}};
+ TradeRangeStats tradeRangeStats3{TradeRangeResultsStats{500000, 2}, TradeRangeResultsStats{0, 0}};
+ TradeRangeStats tradeRangeStats4{TradeRangeResultsStats{79009, 0}, TradeRangeResultsStats{1555555555, 45}};
+
+ MarketTradingGlobalResultPerExchange marketTradingResultPerExchange{
+ {&exchange1, MarketTradingGlobalResult{marketTradingResult1, tradeRangeStats1}},
+ {&exchange3, MarketTradingGlobalResult{marketTradingResult3, tradeRangeStats3}},
+ {&exchange4, MarketTradingGlobalResult{marketTradingResult4, tradeRangeStats4}}};
+ CoincenterCommandType commandType{CoincenterCommandType::kReplay};
+};
+
+TEST_F(QueryResultPrinterReplayTest, FormattedTable) {
+ basicQueryResultPrinter(ApiOutputType::kFormattedTable)
+ .printMarketTradingResults(timeWindow, marketTradingResultPerExchange, commandType);
+ static constexpr std::string_view kExpected = R"(
++----------+----------------------+---------+-----------+---------------+---------------+------------------------------------------------------+------------------------------+
+| Exchange | Time window | Market | Algorithm | Start amounts | Profit / Loss | Matched orders | Stats |
++----------+----------------------+---------+-----------+---------------+---------------+------------------------------------------------------+------------------------------+
+| binance | 1999-03-25T04:46:43Z | BTC-EUR | test-algo | 1 BTC | 0 EUR | | order books: 42 OK |
+| | 2000-10-07T01:14:27Z | | | 1000 EUR | | | trades: 3 OK, 10 KO |
+|~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~|~~~~~~~~~|~~~~~~~~~~~|~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|
+| huobi | 1999-03-25T04:46:43Z | BTC-EUR | test-algo | 1 BTC | 500 EUR | 1999-03-25T04:46:43Z - Buy - 1.5 BTC @ 35000 USDT | order books: 500000 OK, 2 KO |
+| | 2000-10-07T01:14:27Z | | | 1000 EUR | | 2000-06-11T23:58:40Z - Sell - 0.036 BTC @ 47899 USDT | trades: 0 OK |
+|~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~|~~~~~~~~~|~~~~~~~~~~~|~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|
+| huobi | 1999-03-25T04:46:43Z | BTC-EUR | test-algo | 1 BTC | 780 EUR | 1999-07-11T00:42:21Z - Buy - 2.5 BTC @ 45000 USDT | order books: 79009 OK |
+| | 2000-10-07T01:14:27Z | | | 1000 EUR | | 1999-10-29T01:26:51Z - Sell - 0.05 BTC @ 35000 USDT | trades: 1555555555 OK, 45 KO |
+| | | | | | | 1999-10-29T01:26:51Z - Sell - 1.7 BTC @ 50000 USDT | |
++----------+----------------------+---------+-----------+---------------+---------------+------------------------------------------------------+------------------------------+
+)";
+ expectStr(kExpected);
+}
+
+TEST_F(QueryResultPrinterReplayTest, EmptyJson) {
+ basicQueryResultPrinter(ApiOutputType::kJson)
+ .printMarketTradingResults(timeWindow, MarketTradingGlobalResultPerExchange{}, commandType);
+ static constexpr std::string_view kExpected = R"json(
+{
+ "in": {
+ "opt": {
+ "time-window": "[1999-03-25 04:46:43 -> 2000-10-07 01:14:27)"
+ },
+ "req": "Replay"
+ },
+ "out": {}
+})json";
+ expectJson(kExpected);
+}
+
+TEST_F(QueryResultPrinterReplayTest, Json) {
+ basicQueryResultPrinter(ApiOutputType::kJson)
+ .printMarketTradingResults(timeWindow, marketTradingResultPerExchange, commandType);
+ static constexpr std::string_view kExpected = R"json(
+{
+ "in": {
+ "opt": {
+ "time-window": "[1999-03-25 04:46:43 -> 2000-10-07 01:14:27)"
+ },
+ "req": "Replay"
+ },
+ "out": {
+ "binance": {
+ "algorithm": "test-algo",
+ "market": "BTC-EUR",
+ "matched-orders": [],
+ "profit-and-loss": "0 EUR",
+ "start-amounts": {
+ "base": "1 BTC",
+ "quote": "1000 EUR"
+ },
+ "stats": {
+ "order-books": {
+ "nb-error": 0,
+ "nb-successful": 42
+ },
+ "trades": {
+ "nb-error": 10,
+ "nb-successful": 3
+ }
+ }
+ },
+ "huobi": {
+ "algorithm": "test-algo",
+ "market": "BTC-EUR",
+ "matched-orders": [
+ {
+ "id": "1",
+ "matched": "1.5",
+ "matchedTime": "1999-03-25T04:46:43Z",
+ "pair": "BTC-USDT",
+ "placedTime": "1999-03-25T04:46:43Z",
+ "price": "35000",
+ "side": "Buy"
+ },
+ {
+ "id": "5",
+ "matched": "0.036",
+ "matchedTime": "2000-10-07T01:14:27Z",
+ "pair": "BTC-USDT",
+ "placedTime": "2000-06-11T23:58:40Z",
+ "price": "47899",
+ "side": "Sell"
+ }
+ ],
+ "profit-and-loss": "500 EUR",
+ "start-amounts": {
+ "base": "1 BTC",
+ "quote": "1000 EUR"
+ },
+ "stats": {
+ "order-books": {
+ "nb-error": 2,
+ "nb-successful": 500000
+ },
+ "trades": {
+ "nb-error": 0,
+ "nb-successful": 0
+ }
+ }
+ }
+ }
+})json";
+ expectJson(kExpected);
+}
+
+TEST_F(QueryResultPrinterReplayTest, NoPrint) {
+ basicQueryResultPrinter(ApiOutputType::kNoPrint)
+ .printMarketTradingResults(timeWindow, marketTradingResultPerExchange, commandType);
+ expectNoStr();
+}
+
} // namespace cct
diff --git a/src/engine/test/replay-algorithm-name-iterator_test.cpp b/src/engine/test/replay-algorithm-name-iterator_test.cpp
new file mode 100644
index 00000000..ee930a0b
--- /dev/null
+++ b/src/engine/test/replay-algorithm-name-iterator_test.cpp
@@ -0,0 +1,106 @@
+#include "replay-algorithm-name-iterator.hpp"
+
+#include
+
+#include
+
+#include "cct_exception.hpp"
+
+namespace cct {
+class ReplayAlgorithmNameIteratorTest : public ::testing::Test {
+ protected:
+ static constexpr std::string_view kInvalidAlgorithmNames[] = {"any", "so-what,"};
+ static constexpr std::string_view kAlgorithmNames[] = {"any", "so-what", "angry",
+ "bird", "Jack", "a-more-complex algorithm Name"};
+};
+
+TEST_F(ReplayAlgorithmNameIteratorTest, AlgorithmNamesValidity) {
+ EXPECT_THROW(ReplayAlgorithmNameIterator("", kInvalidAlgorithmNames), exception);
+ EXPECT_NO_THROW(ReplayAlgorithmNameIterator("", kAlgorithmNames));
+}
+
+TEST_F(ReplayAlgorithmNameIteratorTest, IteratorWithAll) {
+ ReplayAlgorithmNameIterator it("", kAlgorithmNames);
+
+ int algorithmPos = 0;
+ while (it.hasNext()) {
+ auto next = it.next();
+
+ switch (algorithmPos) {
+ case 0:
+ [[fallthrough]];
+ case 1:
+ [[fallthrough]];
+ case 2:
+ [[fallthrough]];
+ case 3:
+ [[fallthrough]];
+ case 4:
+ [[fallthrough]];
+ case 5:
+ EXPECT_EQ(next, kAlgorithmNames[algorithmPos]);
+ break;
+ default:
+ throw exception("Unexpected number of algorithm names");
+ }
+
+ ++algorithmPos;
+ }
+
+ EXPECT_EQ(algorithmPos, 6);
+}
+
+TEST_F(ReplayAlgorithmNameIteratorTest, IteratorWithUniqueAlgorithmSpecified) {
+ ReplayAlgorithmNameIterator it("so-What", kAlgorithmNames);
+
+ int algorithmPos = 0;
+ while (it.hasNext()) {
+ auto next = it.next();
+
+ switch (algorithmPos) {
+ case 0:
+ EXPECT_EQ(next, "so-What");
+ break;
+ default:
+ throw exception("Unexpected number of algorithm names");
+ }
+
+ ++algorithmPos;
+ }
+
+ EXPECT_EQ(algorithmPos, 1);
+}
+
+TEST_F(ReplayAlgorithmNameIteratorTest, IteratorWithSpecifiedList) {
+ ReplayAlgorithmNameIterator it("Jack,whatever,so-what,some-algorithmNameThatIsNotInAll,with spaces", kAlgorithmNames);
+
+ int algorithmPos = 0;
+ while (it.hasNext()) {
+ auto next = it.next();
+
+ switch (algorithmPos) {
+ case 0:
+ EXPECT_EQ(next, "Jack");
+ break;
+ case 1:
+ EXPECT_EQ(next, "whatever");
+ break;
+ case 2:
+ EXPECT_EQ(next, "so-what");
+ break;
+ case 3:
+ EXPECT_EQ(next, "some-algorithmNameThatIsNotInAll");
+ break;
+ case 4:
+ EXPECT_EQ(next, "with spaces");
+ break;
+ default:
+ throw exception("Unexpected number of algorithm names");
+ }
+
+ ++algorithmPos;
+ }
+
+ EXPECT_EQ(algorithmPos, 5);
+}
+} // namespace cct
\ No newline at end of file
diff --git a/src/engine/test/stringoptionparser_test.cpp b/src/engine/test/stringoptionparser_test.cpp
index e8869af4..4059145b 100644
--- a/src/engine/test/stringoptionparser_test.cpp
+++ b/src/engine/test/stringoptionparser_test.cpp
@@ -2,6 +2,7 @@
#include
+#include
#include
#include "cct_invalid_argument_exception.hpp"
@@ -12,6 +13,7 @@
#include "exchangename.hpp"
#include "market.hpp"
#include "monetaryamount.hpp"
+#include "timedef.hpp"
namespace cct {
namespace {
@@ -211,4 +213,26 @@ TEST(StringOptionParserTest, ExchangesNotLast) {
EXPECT_NO_THROW(parser.checkEndParsing());
}
+TEST(StringOptionParserTest, ParseDurationMandatory) {
+ StringOptionParser parser(" 45min83s,kraken,upbit");
+
+ EXPECT_EQ(parser.parseDuration(StringOptionParser::FieldIs::kMandatory),
+ std::chrono::minutes{45} + std::chrono::seconds{83});
+ EXPECT_EQ(parser.parseExchanges(',', '\0'), ExchangeNames({ExchangeName("kraken"), ExchangeName("upbit")}));
+
+ EXPECT_NO_THROW(parser.checkEndParsing());
+}
+
+TEST(StringOptionParserTest, ParseDurationOptional) {
+ StringOptionParser parser("binance,huobi_user1,34h 4500ms");
+
+ EXPECT_EQ(parser.parseDuration(StringOptionParser::FieldIs::kOptional), kUndefinedDuration);
+ EXPECT_EQ(parser.parseExchanges(',', '\0'), ExchangeNames({ExchangeName("binance"), ExchangeName("huobi", "user1")}));
+
+ EXPECT_EQ(parser.parseDuration(StringOptionParser::FieldIs::kOptional),
+ std::chrono::hours{34} + std::chrono::milliseconds{4500});
+
+ EXPECT_NO_THROW(parser.checkEndParsing());
+}
+
} // namespace cct
\ No newline at end of file
diff --git a/src/http-request/include/request-retry.hpp b/src/http-request/include/request-retry.hpp
index 0b3598f3..3055b6cf 100644
--- a/src/http-request/include/request-retry.hpp
+++ b/src/http-request/include/request-retry.hpp
@@ -1,8 +1,8 @@
#pragma once
#include
-#include
#include
+#include
#include "cct_exception.hpp"
#include "cct_json.hpp"
@@ -10,7 +10,6 @@
#include "cct_type_traits.hpp"
#include "curlhandle.hpp"
#include "curloptions.hpp"
-#include "curlpostdata.hpp"
#include "durationstring.hpp"
#include "query-retry-policy.hpp"
#include "timedef.hpp"
diff --git a/src/main/CMakeLists.txt b/src/main/CMakeLists.txt
index 0bf15a3e..b2621218 100644
--- a/src/main/CMakeLists.txt
+++ b/src/main/CMakeLists.txt
@@ -24,6 +24,10 @@ endif()
target_link_libraries(coincenter PUBLIC coincenter_engine)
+if(CCT_ENABLE_PROTO)
+ target_link_libraries(coincenter PUBLIC protobuf::libprotobuf)
+endif()
+
set_target_properties(coincenter PROPERTIES
VERSION ${PROJECT_VERSION}
COMPILE_DEFINITIONS_DEBUG "JSON_DEBUG;JSON_SAFE;JSON_ISO_STRICT"
diff --git a/src/objects/CMakeLists.txt b/src/objects/CMakeLists.txt
index 7bacf80b..385bbe35 100644
--- a/src/objects/CMakeLists.txt
+++ b/src/objects/CMakeLists.txt
@@ -99,6 +99,15 @@ add_unit_test(
CCT_DISABLE_SPDLOG
)
+add_unit_test(
+ time-window_test
+ test/time-window_test.cpp
+ LIBRARIES
+ coincenter_objects
+ DEFINITIONS
+ CCT_DISABLE_SPDLOG
+)
+
add_unit_test(
wallet_test
test/wallet_test.cpp
diff --git a/src/objects/include/automation-config.hpp b/src/objects/include/automation-config.hpp
new file mode 100644
index 00000000..669bb8dc
--- /dev/null
+++ b/src/objects/include/automation-config.hpp
@@ -0,0 +1,30 @@
+#pragma once
+
+#include
+
+#include "monetaryamount.hpp"
+#include "timedef.hpp"
+
+namespace cct {
+class AutomationConfig {
+ public:
+ AutomationConfig() noexcept = default;
+
+ AutomationConfig(Duration loadChunkDuration, MonetaryAmount startBaseAmountEquivalent,
+ MonetaryAmount startQuoteAmountEquivalent)
+ : _loadChunkDuration(loadChunkDuration),
+ _startBaseAmountEquivalent(startBaseAmountEquivalent),
+ _startQuoteAmountEquivalent(startQuoteAmountEquivalent) {}
+
+ Duration loadChunkDuration() const { return _loadChunkDuration; }
+
+ MonetaryAmount startBaseAmountEquivalent() const { return _startBaseAmountEquivalent; }
+
+ MonetaryAmount startQuoteAmountEquivalent() const { return _startQuoteAmountEquivalent; }
+
+ private:
+ Duration _loadChunkDuration = std::chrono::weeks(1);
+ MonetaryAmount _startBaseAmountEquivalent;
+ MonetaryAmount _startQuoteAmountEquivalent;
+};
+} // namespace cct
\ No newline at end of file
diff --git a/src/objects/include/coincentercommandtype.hpp b/src/objects/include/coincentercommandtype.hpp
index cc1002dc..49967dc6 100644
--- a/src/objects/include/coincentercommandtype.hpp
+++ b/src/objects/include/coincentercommandtype.hpp
@@ -31,6 +31,10 @@ enum class CoincenterCommandType : int8_t {
kWithdrawApply,
kDustSweeper,
+ kMarketData,
+ kReplay,
+ kReplayMarkets,
+
kLast
};
@@ -39,4 +43,4 @@ std::string_view CoincenterCommandTypeToString(CoincenterCommandType type);
CoincenterCommandType CoincenterCommandTypeFromString(std::string_view str);
bool IsAnyTrade(CoincenterCommandType type);
-} // namespace cct
\ No newline at end of file
+} // namespace cct
diff --git a/src/objects/include/exchange-names.hpp b/src/objects/include/exchange-names.hpp
index 339a5e18..b9850579 100644
--- a/src/objects/include/exchange-names.hpp
+++ b/src/objects/include/exchange-names.hpp
@@ -2,6 +2,8 @@
#include
+#include "cct_const.hpp"
+#include "cct_fixedcapacityvector.hpp"
#include "cct_smallvector.hpp"
#include "cct_string.hpp"
#include "exchangename.hpp"
@@ -11,6 +13,8 @@ namespace cct {
using ExchangeNameSpan = std::span;
using ExchangeNames = SmallVector;
+using PublicExchangeNameVector = FixedCapacityVector;
+
string ConstructAccumulatedExchangeNames(ExchangeNameSpan exchangeNames);
} // namespace cct
\ No newline at end of file
diff --git a/src/objects/include/exchangeconfig.hpp b/src/objects/include/exchangeconfig.hpp
index cf97aca7..008e77cf 100644
--- a/src/objects/include/exchangeconfig.hpp
+++ b/src/objects/include/exchangeconfig.hpp
@@ -1,5 +1,6 @@
#pragma once
+#include
#include
#include "apiquerytypeenum.hpp"
@@ -18,7 +19,8 @@
namespace cct {
class ExchangeConfig {
public:
- enum struct FeeType { kMaker, kTaker };
+ enum class FeeType : int8_t { kMaker, kTaker };
+ enum class MarketDataSerialization : int8_t { kYes, kNo };
struct APIUpdateFrequencies {
Duration freq[api::kQueryTypeMax];
@@ -31,7 +33,8 @@ class ExchangeConfig {
std::string_view acceptEncoding, int dustSweeperMaxNbTrades,
log::level::level_enum requestsCallLogLevel, log::level::level_enum requestsAnswerLogLevel,
bool multiTradeAllowedByDefault, bool validateDepositAddressesInFile, bool placeSimulateRealOrder,
- bool validateApiKey, TradeConfig tradeConfig, HttpConfig httpConfig);
+ bool validateApiKey, TradeConfig tradeConfig, HttpConfig httpConfig,
+ MarketDataSerialization marketDataSerialization);
/// Get a reference to the list of statically excluded currency codes to consider for the exchange,
/// In both trading and withdrawal.
@@ -106,6 +109,8 @@ class ExchangeConfig {
PermanentCurlOptions::Builder curlOptionsBuilderBase(Api api) const;
+ bool withMarketDataSerialization() const { return _withMarketSerialization; }
+
private:
CurrencyCodeSet _excludedCurrenciesAll; // Currencies will be completely ignored by the exchange
CurrencyCodeSet _excludedCurrenciesWithdrawal; // Currencies unavailable for withdrawals
@@ -127,5 +132,6 @@ class ExchangeConfig {
bool _validateDepositAddressesInFile;
bool _placeSimulateRealOrder;
bool _validateApiKey;
+ bool _withMarketSerialization;
};
} // namespace cct
diff --git a/src/objects/include/generalconfig.hpp b/src/objects/include/generalconfig.hpp
index 0fd87678..1a30b897 100644
--- a/src/objects/include/generalconfig.hpp
+++ b/src/objects/include/generalconfig.hpp
@@ -7,6 +7,7 @@
#include "logginginfo.hpp"
#include "requestsconfig.hpp"
#include "timedef.hpp"
+#include "trading-config.hpp"
namespace cct {
@@ -18,13 +19,15 @@ class GeneralConfig {
GeneralConfig() = default;
- GeneralConfig(LoggingInfo &&loggingInfo, RequestsConfig &&requestsConfig, Duration fiatConversionQueryRate,
- ApiOutputType apiOutputType);
+ GeneralConfig(LoggingInfo &&loggingInfo, RequestsConfig &&requestsConfig, TradingConfig &&tradingConfig,
+ Duration fiatConversionQueryRate, ApiOutputType apiOutputType);
const LoggingInfo &loggingInfo() const { return _loggingInfo; }
const RequestsConfig &requestsConfig() const { return _requestsConfig; }
+ const TradingConfig &tradingConfig() const { return _tradingConfig; }
+
ApiOutputType apiOutputType() const { return _apiOutputType; }
Duration fiatConversionQueryRate() const { return _fiatConversionQueryRate; }
@@ -32,6 +35,7 @@ class GeneralConfig {
private:
LoggingInfo _loggingInfo{LoggingInfo::WithLoggersCreation::kYes};
RequestsConfig _requestsConfig;
+ TradingConfig _tradingConfig;
Duration _fiatConversionQueryRate = std::chrono::hours(8);
ApiOutputType _apiOutputType = ApiOutputType::kFormattedTable;
};
diff --git a/src/objects/include/generalconfigdefault.hpp b/src/objects/include/generalconfigdefault.hpp
index f52cd5e7..448fe2f9 100644
--- a/src/objects/include/generalconfigdefault.hpp
+++ b/src/objects/include/generalconfigdefault.hpp
@@ -34,6 +34,17 @@ struct GeneralConfigDefault {
"concurrency": {
"nbMaxParallelRequests": 1
}
+ },
+ "trading": {
+ "automation": {
+ "deserialization": {
+ "loadChunkDuration": "1w"
+ },
+ "startingContext": {
+ "startBaseAmountEquivalent": "1000 EUR",
+ "startQuoteAmountEquivalent": "1000 EUR"
+ }
+ }
}
}
)"_json;
diff --git a/src/objects/include/marketorderbook.hpp b/src/objects/include/marketorderbook.hpp
index 0fbabfda..0a1c5e4a 100644
--- a/src/objects/include/marketorderbook.hpp
+++ b/src/objects/include/marketorderbook.hpp
@@ -43,8 +43,14 @@ class MarketOrderBook {
Market market() const { return _market; }
bool empty() const { return _orders.empty(); }
+
int size() const { return _orders.size(); }
+ /// Check if data stored in this MarketOrderBook is valid.
+ /// This is especially useful for optional check of data after deserialization,
+ /// as for the standard case the market order book should be valid by design.
+ bool isValid() const;
+
bool isArtificiallyExtended() const { return _isArtificiallyExtended; }
/// Get the highest bid price that a buyer is willing to pay
@@ -185,6 +191,12 @@ class MarketOrderBook {
/// 0.35 20
/// 0.34 23
+ // To allow faster MarketOrderBook constructs
+ friend class MarketOrderBookConverter;
+
+ MarketOrderBook(TimePoint timeStamp, Market market, AmountPriceVector&& orders, int32_t highestBidPricePos,
+ int32_t lowestAskPricePos, VolAndPriNbDecimals volAndPriNbDecimals);
+
MonetaryAmount amountAt(int pos) const {
return MonetaryAmount(_orders[pos].amount, _market.base(), _volAndPriNbDecimals.volNbDecimals);
}
diff --git a/src/objects/include/publictrade.hpp b/src/objects/include/publictrade.hpp
index 236e0e50..1929bc38 100644
--- a/src/objects/include/publictrade.hpp
+++ b/src/objects/include/publictrade.hpp
@@ -30,8 +30,7 @@ class PublicTrade {
bool isValid() const;
- /// 3 way operator - make compiler generate all 6 operators (including == and !=)
- /// we order by time first, then amount, price, etc. Do not change the fields order!
+ /// We order by time first, then amount, price, etc. Do not change the fields order!
std::strong_ordering operator<=>(const PublicTrade&) const noexcept = default;
private:
@@ -40,4 +39,5 @@ class PublicTrade {
MonetaryAmount _price;
TradeSide _side;
};
+
} // namespace cct
diff --git a/src/objects/include/time-window.hpp b/src/objects/include/time-window.hpp
new file mode 100644
index 00000000..4d641d90
--- /dev/null
+++ b/src/objects/include/time-window.hpp
@@ -0,0 +1,70 @@
+#pragma once
+
+#include
+#include
+
+#include "cct_format.hpp"
+#include "cct_invalid_argument_exception.hpp"
+#include "timedef.hpp"
+
+namespace cct {
+
+/// Simple utility class representing a time window with a beginning time, and an end time.
+/// The beginning time includes the corresponding time point, but the end time excludes it.
+class TimeWindow {
+ public:
+ /// Create a zero duration time window starting from the zero-initialized time point.
+ TimeWindow() noexcept = default;
+
+ /// Create a time window spanning from 'from' (included) to 'to' (excluded) time points.
+ TimeWindow(TimePoint from, TimePoint to) : _from(from), _to(to) {
+ if (_to < _from) {
+ throw invalid_argument("Invalid time window - 'from' should not be larger than 'to'");
+ }
+ }
+
+ /// Create a time window starting at 'from' with 'dur' duration.
+ TimeWindow(TimePoint from, Duration dur) : TimeWindow(from, from + dur) {}
+
+ TimePoint from() const { return _from; }
+
+ TimePoint to() const { return _to; }
+
+ Duration duration() const { return _to - _from; }
+
+ bool contains(TimePoint tp) const { return _from <= tp && tp < _to; }
+
+ bool contains(int64_t unixTimestampInMs) const { return contains(TimePoint(milliseconds{unixTimestampInMs})); }
+
+ bool contains(TimeWindow rhs) const { return _from <= rhs._from && rhs._to <= _to; }
+
+ bool overlaps(TimeWindow rhs) const { return _from < rhs._to && rhs._from < _to; }
+
+ string str() const;
+
+ bool operator==(const TimeWindow&) const noexcept = default;
+
+ private:
+ TimePoint _from;
+ TimePoint _to;
+};
+} // namespace cct
+
+#ifndef CCT_DISABLE_SPDLOG
+template <>
+struct fmt::formatter {
+ constexpr auto parse(format_parse_context& ctx) -> decltype(ctx.begin()) {
+ const auto it = ctx.begin();
+ const auto end = ctx.end();
+ if (it != end && *it != '}') {
+ throw format_error("invalid format");
+ }
+ return it;
+ }
+
+ template
+ auto format(const cct::TimeWindow& timeWindow, FormatContext& ctx) const -> decltype(ctx.out()) {
+ return fmt::format_to(ctx.out(), "{}", timeWindow.str());
+ }
+};
+#endif
diff --git a/src/objects/include/trading-config.hpp b/src/objects/include/trading-config.hpp
new file mode 100644
index 00000000..b265e9dc
--- /dev/null
+++ b/src/objects/include/trading-config.hpp
@@ -0,0 +1,19 @@
+#pragma once
+
+#include
+
+#include "automation-config.hpp"
+
+namespace cct {
+class TradingConfig {
+ public:
+ TradingConfig() noexcept = default;
+
+ TradingConfig(AutomationConfig automationConfig) : _automationConfig(std::move(automationConfig)) {}
+
+ const AutomationConfig &automationConfig() const { return _automationConfig; }
+
+ private:
+ AutomationConfig _automationConfig;
+};
+} // namespace cct
\ No newline at end of file
diff --git a/src/objects/src/coincentercommandtype.cpp b/src/objects/src/coincentercommandtype.cpp
index 8ec46ee1..bbb481ef 100644
--- a/src/objects/src/coincentercommandtype.cpp
+++ b/src/objects/src/coincentercommandtype.cpp
@@ -16,8 +16,7 @@ constexpr std::string_view kCommandTypeNames[] = {
"Balance", "DepositInfo", "OrdersClosed", "OrdersOpened", "OrdersCancel",
"RecentDeposits", "RecentWithdraws", "Trade", "Buy", "Sell",
- "Withdraw", "DustSweeper",
-};
+ "Withdraw", "DustSweeper", "MarketData", "Replay", "ReplayMarkets"};
static_assert(std::size(kCommandTypeNames) == static_cast(CoincenterCommandType::kLast));
} // namespace
@@ -51,4 +50,4 @@ bool IsAnyTrade(CoincenterCommandType type) {
return false;
}
}
-} // namespace cct
\ No newline at end of file
+} // namespace cct
diff --git a/src/objects/src/exchangeconfig.cpp b/src/objects/src/exchangeconfig.cpp
index 8e01941b..fe655229 100644
--- a/src/objects/src/exchangeconfig.cpp
+++ b/src/objects/src/exchangeconfig.cpp
@@ -66,7 +66,8 @@ ExchangeConfig::ExchangeConfig(
const APIUpdateFrequencies &apiUpdateFrequencies, Duration publicAPIRate, Duration privateAPIRate,
std::string_view acceptEncoding, int dustSweeperMaxNbTrades, log::level::level_enum requestsCallLogLevel,
log::level::level_enum requestsAnswerLogLevel, bool multiTradeAllowedByDefault, bool validateDepositAddressesInFile,
- bool placeSimulateRealOrder, bool validateApiKey, TradeConfig tradeConfig, HttpConfig httpConfig)
+ bool placeSimulateRealOrder, bool validateApiKey, TradeConfig tradeConfig, HttpConfig httpConfig,
+ MarketDataSerialization marketDataSerialization)
: _excludedCurrenciesAll(std::move(excludedAllCurrencies)),
_excludedCurrenciesWithdrawal(std::move(excludedCurrenciesWithdraw)),
_preferredPaymentCurrencies(std::move(preferredPaymentCurrencies)),
@@ -85,7 +86,8 @@ ExchangeConfig::ExchangeConfig(
_multiTradeAllowedByDefault(multiTradeAllowedByDefault),
_validateDepositAddressesInFile(validateDepositAddressesInFile),
_placeSimulateRealOrder(placeSimulateRealOrder),
- _validateApiKey(validateApiKey) {
+ _validateApiKey(validateApiKey),
+ _withMarketSerialization(marketDataSerialization == MarketDataSerialization::kYes) {
if (dustSweeperMaxNbTrades > std::numeric_limits::max() || dustSweeperMaxNbTrades < 0) {
throw exception("Invalid number of dust sweeper max trades '{}', should be in [0, {}]", dustSweeperMaxNbTrades,
std::numeric_limits::max());
@@ -112,6 +114,7 @@ ExchangeConfig::ExchangeConfig(
_validateDepositAddressesInFile ? kDepositAddressesFileName : "");
log::trace(" - Order placing in simulation : {}", _placeSimulateRealOrder ? "real, unmatchable" : "none");
log::trace(" - Validate API Key : {}", _validateApiKey ? "yes" : "no");
+ log::trace(" - Market data serialization : {}", _withMarketSerialization ? "yes" : "no");
}
if (_preferredPaymentCurrencies.empty()) {
log::warn("{} list of preferred currencies is empty, buy and sell commands cannot perform trades", exchangeNameStr);
diff --git a/src/objects/src/exchangeconfigdefault.hpp b/src/objects/src/exchangeconfigdefault.hpp
index 3199cbd1..93e48258 100644
--- a/src/objects/src/exchangeconfigdefault.hpp
+++ b/src/objects/src/exchangeconfigdefault.hpp
@@ -59,6 +59,7 @@ struct ExchangeConfigDefault {
"requestsCall": "info",
"requestsAnswer": "trace"
},
+ "marketDataSerialization": true,
"multiTradeAllowedByDefault": false,
"placeSimulateRealOrder": false,
"trade": {
@@ -186,6 +187,7 @@ struct ExchangeConfigDefault {
"requestsCall": "info",
"requestsAnswer": "trace"
},
+ "marketDataSerialization": false,
"multiTradeAllowedByDefault": true,
"privateAPIRate": "1055ms",
"publicAPIRate": "1236ms",
diff --git a/src/objects/src/exchangeconfigmap.cpp b/src/objects/src/exchangeconfigmap.cpp
index d9b5a472..16f9e6f6 100644
--- a/src/objects/src/exchangeconfigmap.cpp
+++ b/src/objects/src/exchangeconfigmap.cpp
@@ -58,6 +58,10 @@ ExchangeConfigMap ComputeExchangeConfigMap(std::string_view fileName, const json
withdrawTopLevelOption.getBool(exchangeName, "validateDepositAddressesInFile");
const bool placeSimulatedRealOrder = queryTopLevelOption.getBool(exchangeName, "placeSimulateRealOrder");
const bool validateApiKey = queryTopLevelOption.getBool(exchangeName, "validateApiKey");
+ const ExchangeConfig::MarketDataSerialization marketDataSerialization =
+ queryTopLevelOption.getBool(exchangeName, "marketDataSerialization")
+ ? ExchangeConfig::MarketDataSerialization::kYes
+ : ExchangeConfig::MarketDataSerialization::kNo;
MonetaryAmountByCurrencySet dustAmountsThresholds(
queryTopLevelOption.getMonetaryAmountsArray(exchangeName, "dustAmountsThreshold"));
@@ -90,7 +94,7 @@ ExchangeConfigMap ComputeExchangeConfigMap(std::string_view fileName, const json
std::move(dustAmountsThresholds), std::move(apiUpdateFrequencies), publicAPIRate, privateAPIRate,
acceptEncoding, dustSweeperMaxNbTrades, requestsCallLogLevel, requestsAnswerLogLevel,
multiTradeAllowedByDefault, validateDepositAddressesInFile, placeSimulatedRealOrder,
- validateApiKey, std::move(tradeConfig), std::move(httpConfig)));
+ validateApiKey, std::move(tradeConfig), std::move(httpConfig), marketDataSerialization));
} // namespace cct
// Print json unused values
@@ -120,4 +124,4 @@ ExchangeConfigMap ComputeExchangeConfigMap(std::string_view fileName, const json
return map;
}
-} // namespace cct
\ No newline at end of file
+} // namespace cct
diff --git a/src/objects/src/generalconfig.cpp b/src/objects/src/generalconfig.cpp
index c08c3270..96494ec1 100644
--- a/src/objects/src/generalconfig.cpp
+++ b/src/objects/src/generalconfig.cpp
@@ -11,13 +11,15 @@
#include "logginginfo.hpp"
#include "requestsconfig.hpp"
#include "timedef.hpp"
+#include "trading-config.hpp"
namespace cct {
-GeneralConfig::GeneralConfig(LoggingInfo &&loggingInfo, RequestsConfig &&requestsConfig,
+GeneralConfig::GeneralConfig(LoggingInfo &&loggingInfo, RequestsConfig &&requestsConfig, TradingConfig &&tradingConfig,
Duration fiatConversionQueryRate, ApiOutputType apiOutputType)
: _loggingInfo(std::move(loggingInfo)),
_requestsConfig(std::move(requestsConfig)),
+ _tradingConfig(std::move(tradingConfig)),
_fiatConversionQueryRate(fiatConversionQueryRate),
_apiOutputType(apiOutputType) {}
diff --git a/src/objects/src/marketorderbook.cpp b/src/objects/src/marketorderbook.cpp
index 10e8be56..2d9b5339 100644
--- a/src/objects/src/marketorderbook.cpp
+++ b/src/objects/src/marketorderbook.cpp
@@ -207,6 +207,36 @@ MarketOrderBook::MarketOrderBook(TimePoint timeStamp, MonetaryAmount askPrice, M
}
}
+MarketOrderBook::MarketOrderBook(TimePoint timeStamp, Market market, AmountPriceVector&& orders,
+ int32_t highestBidPricePos, int32_t lowestAskPricePos,
+ VolAndPriNbDecimals volAndPriNbDecimals)
+ : _time(timeStamp),
+ _market(market),
+ _orders(std::move(orders)),
+ _highestBidPricePos(highestBidPricePos),
+ _lowestAskPricePos(lowestAskPricePos),
+ _volAndPriNbDecimals(volAndPriNbDecimals) {}
+
+bool MarketOrderBook::isValid() const {
+ if (_orders.size() < 2U) {
+ log::error("Market order book is invalid as size is {}", _orders.size());
+ return false;
+ }
+ if (!std::ranges::is_sorted(_orders, [](auto lhs, auto rhs) { return lhs.price < rhs.price; })) {
+ log::error("Market order book is invalid because orders are not sorted by price");
+ return false;
+ }
+ if (std::ranges::adjacent_find(_orders, [](auto lhs, auto rhs) { return lhs.price == rhs.price; }) != _orders.end()) {
+ log::error("Market order book is invalid because of duplicate prices");
+ return false;
+ }
+ if (!std::ranges::is_partitioned(_orders, [](auto amountPrice) { return amountPrice.amount > 0; })) {
+ log::error("Market order book is invalid because lines are not partitioned by asks / bids");
+ return false;
+ }
+ return true;
+}
+
std::optional MarketOrderBook::averagePrice() const {
switch (_orders.size()) {
case 0U:
diff --git a/src/objects/src/publictrade.cpp b/src/objects/src/publictrade.cpp
index f701d5c7..5fde8d63 100644
--- a/src/objects/src/publictrade.cpp
+++ b/src/objects/src/publictrade.cpp
@@ -1,5 +1,6 @@
#include "publictrade.hpp"
+#include "cct_log.hpp"
#include "cct_string.hpp"
#include "timedef.hpp"
#include "timestring.hpp"
@@ -11,18 +12,23 @@ string PublicTrade::timeStr() const { return ToString(_time); }
bool PublicTrade::isValid() const {
if (time() == TimePoint{}) {
+ log::error("Public trade is invalid as no timestamp");
return false;
}
if (amount() <= 0 || amount().hasNeutralCurrency()) {
+ log::error("Public trade has an invalid amount");
return false;
}
if (price() <= 0 || price().hasNeutralCurrency()) {
+ log::error("Public trade has an invalid price");
return false;
}
if (amount().currencyCode() == price().currencyCode()) {
+ log::error("Public trade has an invalid market");
return false;
}
if (side() != TradeSide::kBuy && side() != TradeSide::kSell) {
+ log::error("Public trade has an invalid trade side");
return false;
}
return true;
diff --git a/src/objects/src/time-window.cpp b/src/objects/src/time-window.cpp
new file mode 100644
index 00000000..088281f6
--- /dev/null
+++ b/src/objects/src/time-window.cpp
@@ -0,0 +1,15 @@
+#include "time-window.hpp"
+
+#include "timestring.hpp"
+
+namespace cct {
+string TimeWindow::str() const {
+ string ret;
+ ret.push_back('[');
+ ret.append(ToString(from(), kTimeYearToSecondSpaceSeparatedFormat));
+ ret.append(" -> ");
+ ret.append(ToString(to(), kTimeYearToSecondSpaceSeparatedFormat));
+ ret.push_back(')');
+ return ret;
+}
+} // namespace cct
\ No newline at end of file
diff --git a/src/objects/test/marketorderbook_test.cpp b/src/objects/test/marketorderbook_test.cpp
index 6997c26d..ba64db64 100644
--- a/src/objects/test/marketorderbook_test.cpp
+++ b/src/objects/test/marketorderbook_test.cpp
@@ -37,6 +37,12 @@ constexpr bool operator==(const AmountPrice &lhs, const AmountPrice &rhs) {
return lhs.amount == rhs.amount && lhs.price == rhs.price;
}
+TEST(MarketOrderBookTest, DefaultConstructor) {
+ MarketOrderBook marketOrderBook;
+
+ EXPECT_FALSE(marketOrderBook.isValid());
+}
+
TEST(MarketOrderBookTest, Basic) { EXPECT_TRUE(MarketOrderBook(Clock::now(), Market("ETH", "EUR"), {}).empty()); }
class MarketOrderBookTestCase1 : public ::testing::Test {
@@ -53,6 +59,8 @@ class MarketOrderBookTestCase1 : public ::testing::Test {
OrderBookLine::Type::kAsk)})};
};
+TEST_F(MarketOrderBookTestCase1, IsValid) { EXPECT_TRUE(marketOrderBook.isValid()); }
+
TEST_F(MarketOrderBookTestCase1, NumberOfElements) {
EXPECT_EQ(marketOrderBook.size(), 5);
EXPECT_EQ(marketOrderBook.nbAskPrices(), 3);
@@ -176,6 +184,8 @@ class MarketOrderBookTestDuplicatedLines : public ::testing::Test {
OrderBookLine::Type::kAsk)})};
};
+TEST_F(MarketOrderBookTestDuplicatedLines, IsValid) { EXPECT_TRUE(marketOrderBook.isValid()); }
+
TEST_F(MarketOrderBookTestDuplicatedLines, NumberOfElements) {
EXPECT_EQ(marketOrderBook.size(), 5);
EXPECT_EQ(marketOrderBook.nbAskPrices(), 3);
@@ -211,6 +221,8 @@ class MarketOrderBookTestCase2 : public ::testing::Test {
OrderBookLine::Type::kBid)})};
};
+TEST_F(MarketOrderBookTestCase2, IsValid) { EXPECT_TRUE(marketOrderBook.isValid()); }
+
TEST_F(MarketOrderBookTestCase2, NbDecimals) {
const auto [volNbDecimals, priNbDecimals] = marketOrderBook.volAndPriNbDecimals();
@@ -288,6 +300,8 @@ class MarketOrderBookTestCase3 : public ::testing::Test {
MonetaryAmount("0.000007080", "BTC"), OrderBookLine::Type::kBid)})};
};
+TEST_F(MarketOrderBookTestCase3, IsValid) { EXPECT_TRUE(marketOrderBook.isValid()); }
+
TEST_F(MarketOrderBookTestCase3, Convert) {
EXPECT_EQ(marketOrderBook.convert(MonetaryAmount("600000", "XLM")), std::nullopt);
EXPECT_EQ(marketOrderBook.convert(MonetaryAmount(3, "BTC")), std::nullopt);
@@ -324,6 +338,8 @@ class MarketOrderBookTestCaseExtended1 : public ::testing::Test {
50};
};
+TEST_F(MarketOrderBookTestCaseExtended1, IsValid) { EXPECT_TRUE(marketOrderBook.isValid()); }
+
TEST_F(MarketOrderBookTestCaseExtended1, LimitPrice) {
EXPECT_EQ(marketOrderBook.highestBidPrice(), MonetaryAmount("2300.4 EUR"));
EXPECT_EQ(marketOrderBook.lowestAskPrice(), MonetaryAmount("2300.45 EUR"));
@@ -339,6 +355,8 @@ TEST(MarketOrderBookExtendedTest, ComputeVolAndPriNbDecimalsFromTickerInfo) {
MonetaryAmount("193.0900000000078 ADA"), MonetaryAmount("12355.00002486 XLM"),
MonetaryAmount("504787104.7801 ADA"), {4, 8}, 10);
+ ASSERT_TRUE(marketOrderBook.isValid());
+
EXPECT_EQ(marketOrderBook.highestBidPrice(), MonetaryAmount("12355.00002486 XLM"));
EXPECT_EQ(marketOrderBook.lowestAskPrice(), MonetaryAmount("12355.00002487 XLM"));
}
diff --git a/src/objects/test/time-window_test.cpp b/src/objects/test/time-window_test.cpp
new file mode 100644
index 00000000..42f6b04a
--- /dev/null
+++ b/src/objects/test/time-window_test.cpp
@@ -0,0 +1,140 @@
+#include "time-window.hpp"
+
+#include
+
+#include
+
+#include "cct_invalid_argument_exception.hpp"
+#include "timedef.hpp"
+
+namespace cct {
+class TimeWindowTest : public ::testing::Test {
+ protected:
+ TimePoint tp1{milliseconds{std::numeric_limits::max() / 10000000}};
+ TimePoint tp2{milliseconds{std::numeric_limits::max() / 9900000}};
+ TimePoint tp3{milliseconds{std::numeric_limits::max() / 9800000}};
+ TimePoint tp4{milliseconds{std::numeric_limits::max() / 9500000}};
+ TimePoint tp5{milliseconds{std::numeric_limits::max() / 9000000}};
+
+ Duration dur1{seconds{100}};
+ Duration dur2{seconds{1000}};
+ Duration dur3{seconds{10000}};
+};
+
+TEST_F(TimeWindowTest, DefaultConstructor) {
+ TimeWindow tw;
+
+ EXPECT_EQ(tw.from(), TimePoint{});
+ EXPECT_EQ(tw.to(), TimePoint{});
+ EXPECT_EQ(tw.duration(), milliseconds{});
+ EXPECT_FALSE(tw.contains(TimePoint{}));
+ EXPECT_FALSE(tw.contains(0));
+ EXPECT_TRUE(tw.contains(tw));
+}
+
+TEST_F(TimeWindowTest, InvalidTimeWindowFromTime) { EXPECT_THROW(TimeWindow(tp2, tp1), invalid_argument); }
+TEST_F(TimeWindowTest, InvalidTimeWindowFromDuration) { EXPECT_THROW(TimeWindow(tp1, tp1 - tp2), invalid_argument); }
+
+TEST_F(TimeWindowTest, DurationConstructor) {
+ TimeWindow tw(tp1, tp2 - tp1);
+
+ EXPECT_EQ(tw, TimeWindow(tp1, tp2));
+}
+
+TEST_F(TimeWindowTest, Duration) {
+ TimeWindow tw(tp1, tp2);
+
+ EXPECT_EQ(tw.duration(), tp2 - tp1);
+}
+
+TEST_F(TimeWindowTest, ContainsTimePoint) {
+ TimeWindow tw1(tp1, tp2);
+
+ EXPECT_TRUE(tw1.contains(tp1));
+ EXPECT_TRUE(tw1.contains(tp1 + dur1));
+ EXPECT_FALSE(tw1.contains(tp2));
+ EXPECT_FALSE(tw1.contains(tp3));
+}
+
+TEST_F(TimeWindowTest, ContainsTimeWindow) {
+ // [ ]
+ // [ ]
+ TimeWindow tw1(tp1, tp4);
+ TimeWindow tw2(tp2, tp3);
+
+ EXPECT_TRUE(tw1.contains(tw1));
+ EXPECT_TRUE(tw1.overlaps(tw1));
+
+ EXPECT_TRUE(tw1.overlaps(tw2));
+ EXPECT_TRUE(tw1.contains(tw2));
+
+ EXPECT_TRUE(tw2.overlaps(tw1));
+ EXPECT_FALSE(tw2.contains(tw1));
+}
+
+TEST_F(TimeWindowTest, OverlapNominal) {
+ // [ ]
+ // [ ]
+ TimeWindow tw1(tp2, tp4);
+ TimeWindow tw2(tp1, tp3);
+
+ EXPECT_TRUE(tw1.overlaps(tw2));
+ EXPECT_FALSE(tw1.contains(tw2));
+
+ EXPECT_TRUE(tw2.overlaps(tw1));
+ EXPECT_FALSE(tw2.contains(tw1));
+}
+
+TEST_F(TimeWindowTest, OverlapEqualTo) {
+ // [ ]
+ // [ ]
+ TimeWindow tw1(tp1, tp3);
+ TimeWindow tw2(tp2, tp3);
+
+ EXPECT_TRUE(tw1.overlaps(tw2));
+ EXPECT_TRUE(tw1.contains(tw2));
+
+ EXPECT_TRUE(tw2.overlaps(tw1));
+ EXPECT_FALSE(tw2.contains(tw1));
+}
+
+TEST_F(TimeWindowTest, OverlapEqualFrom) {
+ // [ ]
+ // [ ]
+ TimeWindow tw1(tp1, tp3);
+ TimeWindow tw2(tp1, tp2);
+
+ EXPECT_TRUE(tw1.overlaps(tw2));
+ EXPECT_TRUE(tw1.contains(tw2));
+
+ EXPECT_TRUE(tw2.overlaps(tw1));
+ EXPECT_FALSE(tw2.contains(tw1));
+}
+
+TEST_F(TimeWindowTest, NoOverlapNominal) {
+ // [ ]
+ // [ ]
+ TimeWindow tw1(tp1, tp2);
+ TimeWindow tw2(tp3, tp4);
+
+ EXPECT_FALSE(tw1.overlaps(tw2));
+ EXPECT_FALSE(tw1.contains(tw2));
+
+ EXPECT_FALSE(tw2.overlaps(tw1));
+ EXPECT_FALSE(tw2.contains(tw1));
+}
+
+TEST_F(TimeWindowTest, NoOverlapEqual) {
+ // [ ]
+ // [ ]
+ TimeWindow tw1(tp1, tp3);
+ TimeWindow tw2(tp3, tp4);
+
+ EXPECT_FALSE(tw1.overlaps(tw2));
+ EXPECT_FALSE(tw1.contains(tw2));
+
+ EXPECT_FALSE(tw2.overlaps(tw1));
+ EXPECT_FALSE(tw2.contains(tw1));
+}
+
+} // namespace cct
diff --git a/src/serialization/CMakeLists.txt b/src/serialization/CMakeLists.txt
new file mode 100644
index 00000000..4054e6dc
--- /dev/null
+++ b/src/serialization/CMakeLists.txt
@@ -0,0 +1,70 @@
+if(CCT_ENABLE_PROTO)
+ aux_source_directory(src SERIALIZATION_SRC)
+
+ list(APPEND SERIALIZATION_SRC "${CMAKE_CURRENT_LIST_DIR}/proto/market-order-book-timed-data.proto")
+ list(APPEND SERIALIZATION_SRC "${CMAKE_CURRENT_LIST_DIR}/proto/trade-data.proto")
+else()
+ set(SERIALIZATION_SRC "src/dummy-market-data-serializer.cpp" "src/dummy-market-data-deserializer.cpp")
+endif()
+
+add_library(coincenter_serialization OBJECT ${SERIALIZATION_SRC})
+
+target_include_directories(coincenter_serialization PUBLIC include)
+target_link_libraries(coincenter_serialization PUBLIC coincenter_objects)
+
+if(CCT_ENABLE_PROTO)
+ set(PROTO_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/generated")
+
+ target_include_directories(coincenter_serialization PUBLIC "$")
+
+ target_link_libraries(coincenter_serialization PUBLIC protobuf::libprotobuf)
+
+ protobuf_generate(
+ TARGET coincenter_serialization
+ IMPORT_DIRS "${CMAKE_CURRENT_LIST_DIR}/proto"
+ PROTOC_OUT_DIR "${PROTO_BINARY_DIR}"
+ )
+
+ add_unit_test(
+ continuous-iterator_test
+ test/continuous-iterator_test.cpp
+ LIBRARIES
+ coincenter_serialization
+ )
+
+ add_unit_test(
+ proto-market-order-book_test
+ test/proto-market-order-book_test.cpp
+ LIBRARIES
+ coincenter_serialization
+ )
+
+ add_unit_test(
+ proto-multiple-messages-handler_test
+ test/proto-multiple-messages-handler_test.cpp
+ LIBRARIES
+ coincenter_serialization
+ )
+
+ add_unit_test(
+ proto-public-trade_test
+ test/proto-public-trade_test.cpp
+ LIBRARIES
+ coincenter_serialization
+ )
+
+ add_unit_test(
+ proto-serialization-and-deserialization_test
+ test/proto-serialization-and-deserialization_test.cpp
+ LIBRARIES
+ coincenter_serialization
+ )
+
+ add_unit_test(
+ serialization-tools_test
+ test/serialization-tools_test.cpp
+ LIBRARIES
+ coincenter_serialization
+ )
+
+endif()
\ No newline at end of file
diff --git a/src/serialization/include/abstract-market-data-deserializer.hpp b/src/serialization/include/abstract-market-data-deserializer.hpp
new file mode 100644
index 00000000..fd66d9d1
--- /dev/null
+++ b/src/serialization/include/abstract-market-data-deserializer.hpp
@@ -0,0 +1,24 @@
+#pragma once
+
+#include "market-order-book-vector.hpp"
+#include "market-timestamp-set.hpp"
+#include "market.hpp"
+#include "public-trade-vector.hpp"
+#include "time-window.hpp"
+
+namespace cct {
+
+class AbstractMarketDataDeserializer {
+ public:
+ virtual ~AbstractMarketDataDeserializer() = default;
+
+ virtual MarketTimestampSet pullMarketOrderBooksMarkets(TimeWindow timeWindow) = 0;
+
+ virtual MarketTimestampSet pullTradeMarkets(TimeWindow timeWindow) = 0;
+
+ virtual MarketOrderBookVector pullMarketOrderBooks(Market market, TimeWindow timeWindow) = 0;
+
+ virtual PublicTradeVector pullTrades(Market market, TimeWindow timeWindow) = 0;
+};
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/serialization/include/abstract-market-data-serializer.hpp b/src/serialization/include/abstract-market-data-serializer.hpp
new file mode 100644
index 00000000..65cc1b79
--- /dev/null
+++ b/src/serialization/include/abstract-market-data-serializer.hpp
@@ -0,0 +1,24 @@
+#pragma once
+
+#include
+
+#include "market.hpp"
+#include "publictrade.hpp"
+
+namespace cct {
+
+class MarketOrderBook;
+
+class AbstractMarketDataSerializer {
+ public:
+ virtual ~AbstractMarketDataSerializer() = default;
+
+ /// Push market order book in the MarketDataSerializer.
+ virtual void push(const MarketOrderBook &marketOrderBook) = 0;
+
+ /// Push public trades in the MarketDataSerializer.
+ /// They should come from the same market.
+ virtual void push(Market market, std::span publicTrades) = 0;
+};
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/serialization/include/continuous-iterator.hpp b/src/serialization/include/continuous-iterator.hpp
new file mode 100644
index 00000000..91b71cd1
--- /dev/null
+++ b/src/serialization/include/continuous-iterator.hpp
@@ -0,0 +1,23 @@
+#pragma once
+
+namespace cct {
+
+/// Simple utility class that may iterate in both directions.
+class ContinuousIterator {
+ public:
+ ContinuousIterator(int from, int to) : _to(to), _curr(from), _incr(to < from ? -1 : 1) {}
+
+ bool hasNext() const { return _curr != _to + _incr; }
+
+ auto next() {
+ _curr += _incr;
+ return _curr - _incr;
+ }
+
+ private:
+ int _to;
+ int _curr;
+ int _incr;
+};
+
+} // namespace cct
diff --git a/src/serialization/include/dummy-market-data-deserializer.hpp b/src/serialization/include/dummy-market-data-deserializer.hpp
new file mode 100644
index 00000000..e3b0227c
--- /dev/null
+++ b/src/serialization/include/dummy-market-data-deserializer.hpp
@@ -0,0 +1,29 @@
+#pragma once
+
+#include
+
+#include "abstract-market-data-deserializer.hpp"
+#include "market-order-book-vector.hpp"
+#include "market-timestamp-set.hpp"
+#include "market.hpp"
+#include "public-trade-vector.hpp"
+#include "time-window.hpp"
+
+namespace cct {
+
+class DummyMarketDataDeserializer : public AbstractMarketDataDeserializer {
+ public:
+ DummyMarketDataDeserializer([[maybe_unused]] std::string_view dataDir,
+ [[maybe_unused]] std::string_view exchangeName);
+
+ MarketTimestampSet pullMarketOrderBooksMarkets([[maybe_unused]] TimeWindow timeWindow) override;
+
+ MarketTimestampSet pullTradeMarkets([[maybe_unused]] TimeWindow timeWindow) override;
+
+ MarketOrderBookVector pullMarketOrderBooks([[maybe_unused]] Market market,
+ [[maybe_unused]] TimeWindow timeWindow) override;
+
+ PublicTradeVector pullTrades([[maybe_unused]] Market market, [[maybe_unused]] TimeWindow timeWindow) override;
+};
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/serialization/include/dummy-market-data-serializer.hpp b/src/serialization/include/dummy-market-data-serializer.hpp
new file mode 100644
index 00000000..2e42e287
--- /dev/null
+++ b/src/serialization/include/dummy-market-data-serializer.hpp
@@ -0,0 +1,26 @@
+#pragma once
+
+#include
+#include
+
+#include "abstract-market-data-serializer.hpp"
+#include "market-timestamp-set.hpp"
+#include "publictrade.hpp"
+
+namespace cct {
+
+class MarketOrderBook;
+
+/// Implementation of a market data serializer that does nothing.
+/// Useful if coincenter is not compiled with protobuf support.
+class DummyMarketDataSerializer : public AbstractMarketDataSerializer {
+ public:
+ DummyMarketDataSerializer(std::string_view dataDir, const MarketTimestampSets &lastWrittenObjectsMarketTimestamp,
+ std::string_view exchangeName);
+
+ void push(const MarketOrderBook &marketOrderBook) override;
+
+ void push(Market market, std::span publicTrades) override;
+};
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/serialization/include/market-timestamp-set.hpp b/src/serialization/include/market-timestamp-set.hpp
new file mode 100644
index 00000000..afda7143
--- /dev/null
+++ b/src/serialization/include/market-timestamp-set.hpp
@@ -0,0 +1,15 @@
+#pragma once
+
+#include "cct_flatset.hpp"
+#include "market-timestamp.hpp"
+
+namespace cct {
+
+using MarketTimestampSet = FlatSet;
+
+struct MarketTimestampSets {
+ MarketTimestampSet orderBooksMarkets;
+ MarketTimestampSet tradesMarkets;
+};
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/serialization/include/market-timestamp.hpp b/src/serialization/include/market-timestamp.hpp
new file mode 100644
index 00000000..67317c69
--- /dev/null
+++ b/src/serialization/include/market-timestamp.hpp
@@ -0,0 +1,17 @@
+#pragma once
+
+#include
+
+#include "market.hpp"
+#include "timedef.hpp"
+
+namespace cct {
+
+struct MarketTimestamp {
+ Market market;
+ TimePoint timePoint;
+
+ std::strong_ordering operator<=>(const MarketTimestamp &) const noexcept = default;
+};
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/serialization/include/proto-constants.hpp b/src/serialization/include/proto-constants.hpp
new file mode 100644
index 00000000..e7a59523
--- /dev/null
+++ b/src/serialization/include/proto-constants.hpp
@@ -0,0 +1,15 @@
+#pragma once
+
+#include
+#include
+
+namespace cct {
+
+enum class ProtobufObject : int8_t { kMarketOrderBook, kTrade };
+
+static constexpr std::string_view kBinProtobufExtension = ".binpb";
+
+static constexpr std::string_view kSubPathMarketOrderBook = "market-order-book";
+static constexpr std::string_view kSubPathTrades = "trades";
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/serialization/include/proto-deserializer.hpp b/src/serialization/include/proto-deserializer.hpp
new file mode 100644
index 00000000..cd61d9d0
--- /dev/null
+++ b/src/serialization/include/proto-deserializer.hpp
@@ -0,0 +1,184 @@
+#pragma once
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "cct_format.hpp"
+#include "cct_log.hpp"
+#include "cct_vector.hpp"
+#include "continuous-iterator.hpp"
+#include "market-timestamp-set.hpp"
+#include "market-timestamp.hpp"
+#include "market.hpp"
+#include "proto-multiple-messages-handler.hpp"
+#include "serialization-tools.hpp"
+#include "time-window.hpp"
+#include "timedef.hpp"
+
+namespace cct {
+
+template
+class ProtobufObjectsDeserializer {
+ public:
+ using CoincenterObjectType = std::invoke_result_t;
+
+ explicit ProtobufObjectsDeserializer(std::string exchangeSerializedDataPath) noexcept
+ : _exchangeSerializedDataPath(std::move(exchangeSerializedDataPath)) {}
+
+ /// Load all markets found on disk which has some data in the given time window
+ MarketTimestampSet listMarkets(TimeWindow timeWindow) {
+ vector marketTimestamps;
+
+ if (std::filesystem::is_directory(_exchangeSerializedDataPath)) {
+ for (const auto& marketDirectory : std::filesystem::directory_iterator(_exchangeSerializedDataPath)) {
+ auto ts = loadMarket(marketDirectory, timeWindow, ActionType::kCheckPresence).second;
+ if (ts != TimePoint{}) {
+ const auto& marketPath = marketDirectory.path();
+ const auto marketStr = marketPath.filename().string();
+
+ marketTimestamps.emplace_back(Market{marketStr}, ts);
+ }
+ }
+ }
+
+ return MarketTimestampSet(std::move(marketTimestamps));
+ }
+
+ /// Load all data found on disk for given market for the time window
+ vector loadMarket(Market market, TimeWindow timeWindow) {
+ std::string marketPathStr(_exchangeSerializedDataPath);
+ marketPathStr.push_back('/');
+ marketPathStr.append(market.str());
+
+ std::filesystem::path marketPath(marketPathStr);
+
+ return loadMarket(std::filesystem::directory_entry(marketPath), timeWindow);
+ }
+
+ /// Load all data found on disk for given market for the time window
+ vector loadMarket(const std::filesystem::directory_entry& marketDirectory,
+ TimeWindow timeWindow) {
+ return loadMarket(marketDirectory, timeWindow, ActionType::kLoad).first;
+ }
+
+ private:
+ static bool ValidateTimestamp(const ProtobufObjType& msg, TimeWindow timeWindow) {
+ if (!msg.has_unixtimestampinms()) {
+ log::error("Invalid data loaded for protobuf object, no unix timestamp set");
+ return false;
+ }
+ return timeWindow.contains(msg.unixtimestampinms());
+ }
+
+ enum class ActionType : int8_t { kLoad, kCheckPresence };
+
+ static ContinuousIterator CreateIt(int from, int to, ActionType actionType) {
+ if (actionType == ActionType::kCheckPresence) {
+ std::swap(from, to);
+ }
+ return {from, to};
+ }
+
+ /// Load all data found on disk for given market for the time window
+ auto loadMarket(const std::filesystem::directory_entry& marketDirectory, TimeWindow timeWindow,
+ ActionType actionType) {
+ std::pair, TimePoint> ret;
+ if (!marketDirectory.is_directory()) {
+ return ret;
+ }
+ const auto fromDays = std::chrono::floor(timeWindow.from());
+ const std::chrono::year_month_day fromYmd{fromDays};
+ const std::chrono::hh_mm_ss fromTime{std::chrono::floor(timeWindow.from() - fromDays)};
+
+ const auto toDays = std::chrono::floor(timeWindow.to());
+ const std::chrono::year_month_day toYmd{toDays};
+ const std::chrono::hh_mm_ss toTime{std::chrono::floor(timeWindow.to() - toDays)};
+
+ const auto& marketPath = marketDirectory.path();
+ const auto marketFilename = marketPath.filename();
+ const Market market(marketFilename.string());
+
+ ProtoToCoincenterObjectsFunc converter(market);
+
+ const int fromYear = static_cast(fromYmd.year());
+ const int toYear = static_cast(toYmd.year());
+
+ for (ContinuousIterator yearIt = CreateIt(fromYear, toYear, actionType); yearIt.hasNext();) {
+ const auto year = yearIt.next();
+ const auto yearPath = marketPath / format("{:04}", year);
+ if (!std::filesystem::is_directory(yearPath)) {
+ continue;
+ }
+ const bool isYearFromExtremity = year == fromYear;
+ const bool isYearToExtremity = year == toYear;
+ const auto fromMonth = isYearFromExtremity ? static_cast(static_cast(fromYmd.month())) : 1;
+ const auto toMonth = isYearToExtremity ? static_cast(static_cast(toYmd.month())) : 12;
+
+ for (ContinuousIterator monthIt = CreateIt(fromMonth, toMonth, actionType); monthIt.hasNext();) {
+ const auto month = monthIt.next();
+ const auto monthPath = yearPath / format("{:02}", month);
+ if (!std::filesystem::is_directory(monthPath)) {
+ continue;
+ }
+ const bool isMonthFromExtremity = isYearFromExtremity && month == fromMonth;
+ const bool isMonthToExtremity = isYearToExtremity && month == toMonth;
+ const auto fromDay = isMonthFromExtremity ? static_cast(static_cast(fromYmd.day())) : 1;
+ const auto toDay = isMonthToExtremity ? static_cast(static_cast(toYmd.day())) : 31;
+
+ for (ContinuousIterator dayIt = CreateIt(fromDay, toDay, actionType); dayIt.hasNext();) {
+ const auto day = dayIt.next();
+ const auto dayPath = monthPath / format("{:02}", day);
+ if (!std::filesystem::is_directory(dayPath)) {
+ continue;
+ }
+
+ const bool isDayFromExtremity = isMonthFromExtremity && day == fromDay;
+ const bool isDayToExtremity = isMonthToExtremity && day == toDay;
+ const auto fromHour = isDayFromExtremity ? static_cast(fromTime.hours().count()) : 0;
+ const auto toHour = isDayToExtremity ? static_cast(toTime.hours().count()) : 23;
+
+ for (ContinuousIterator hourOfDayIt = CreateIt(fromHour, toHour, actionType); hourOfDayIt.hasNext();) {
+ const auto hourOfDay = hourOfDayIt.next();
+ const auto hourPath = dayPath / ComputeProtoFileName(hourOfDay);
+ if (!std::filesystem::exists(hourPath)) {
+ continue;
+ }
+
+ decltype(std::declval().unixtimestampinms()) lastTs = 0;
+
+ std::ifstream ifs(hourPath, std::ios::in | std::ios::binary);
+ for (ProtobufMessagesReader protobufMessagesReader(ifs); protobufMessagesReader.hasNext();) {
+ auto msg = protobufMessagesReader.next();
+ if (!ValidateTimestamp(msg, timeWindow)) {
+ continue;
+ }
+
+ // In Check presence mode, we read all the file to retrieve the latest timestamp.
+ // There's no other way to do it.
+ lastTs = msg.unixtimestampinms();
+
+ if (actionType == ActionType::kLoad) {
+ ret.first.push_back(converter(std::move(msg)));
+ }
+ }
+
+ ret.second = TimePoint{milliseconds{static_cast(lastTs)}};
+ if (actionType == ActionType::kCheckPresence) {
+ return ret;
+ }
+ }
+ }
+ }
+ }
+ return ret;
+ }
+
+ std::string _exchangeSerializedDataPath;
+};
+
+} // namespace cct
diff --git a/src/serialization/include/proto-market-data-deserializer.hpp b/src/serialization/include/proto-market-data-deserializer.hpp
new file mode 100644
index 00000000..cea45845
--- /dev/null
+++ b/src/serialization/include/proto-market-data-deserializer.hpp
@@ -0,0 +1,36 @@
+#pragma once
+
+#include
+
+#include "abstract-market-data-deserializer.hpp"
+#include "market-order-book-timed-data.pb.h"
+#include "market-order-book-vector.hpp"
+#include "market-timestamp-set.hpp"
+#include "market.hpp"
+#include "proto-deserializer.hpp"
+#include "proto-market-order-book.hpp"
+#include "proto-public-trade.hpp"
+#include "public-trade-vector.hpp"
+#include "time-window.hpp"
+#include "trade-data.pb.h"
+
+namespace cct {
+
+class ProtoMarketDataDeserializer : public AbstractMarketDataDeserializer {
+ public:
+ ProtoMarketDataDeserializer(std::string_view dataDir, std::string_view exchangeName);
+
+ MarketTimestampSet pullMarketOrderBooksMarkets(TimeWindow timeWindow) override;
+
+ MarketTimestampSet pullTradeMarkets(TimeWindow timeWindow) override;
+
+ MarketOrderBookVector pullMarketOrderBooks(Market market, TimeWindow timeWindow) override;
+
+ PublicTradeVector pullTrades(Market market, TimeWindow timeWindow) override;
+
+ private:
+ ProtobufObjectsDeserializer<::objects::MarketOrderBookTimedData, MarketOrderBookConverter>
+ _marketOrderBookDeserializer;
+ ProtobufObjectsDeserializer<::objects::TradeData, TradeDataToPublicTradeConverter> _tradeDataDeserializer;
+};
+} // namespace cct
\ No newline at end of file
diff --git a/src/serialization/include/proto-market-data-serializer.hpp b/src/serialization/include/proto-market-data-serializer.hpp
new file mode 100644
index 00000000..26ed3d38
--- /dev/null
+++ b/src/serialization/include/proto-market-data-serializer.hpp
@@ -0,0 +1,34 @@
+#pragma once
+
+#include
+#include
+
+#include "abstract-market-data-serializer.hpp"
+#include "market-order-book-timed-data.pb.h"
+#include "market-timestamp-set.hpp"
+#include "market.hpp"
+#include "marketorderbook.hpp"
+#include "proto-serializer.hpp"
+#include "proto-trade-data-compare.hpp"
+#include "publictrade.hpp"
+#include "trade-data.pb.h"
+
+namespace cct {
+
+/// This class is responsible of managing the periodic writes to disk of timed market data, for a given exchange.
+/// This class is not thread safe
+class ProtoMarketDataSerializer : public AbstractMarketDataSerializer {
+ public:
+ ProtoMarketDataSerializer(std::string_view dataDir, const MarketTimestampSets &lastWrittenObjectsMarketTimestamp,
+ std::string_view exchangeName);
+
+ void push(const MarketOrderBook &marketOrderBook) override;
+
+ void push(Market market, std::span publicTrades) override;
+
+ private:
+ ProtobufObjectsSerializer<::objects::MarketOrderBookTimedData> _marketOrderBookSerializer;
+ ProtobufObjectsSerializer<::objects::TradeData, TradeDataComp, TradeDataEqual> _tradesSerializer;
+};
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/serialization/include/proto-market-order-book.hpp b/src/serialization/include/proto-market-order-book.hpp
new file mode 100644
index 00000000..65132ec9
--- /dev/null
+++ b/src/serialization/include/proto-market-order-book.hpp
@@ -0,0 +1,21 @@
+#pragma once
+
+#include "market-order-book-timed-data.pb.h"
+#include "market.hpp"
+#include "marketorderbook.hpp"
+
+namespace cct {
+
+::objects::MarketOrderBookTimedData CreateMarketOrderBookTimedData(const MarketOrderBook &marketOrderBook);
+
+class MarketOrderBookConverter {
+ public:
+ explicit MarketOrderBookConverter(Market market) : _market(market) {}
+
+ MarketOrderBook operator()(const ::objects::MarketOrderBookTimedData &marketOrderBookTimedData);
+
+ private:
+ Market _market;
+};
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/serialization/include/proto-multiple-messages-handler.hpp b/src/serialization/include/proto-multiple-messages-handler.hpp
new file mode 100644
index 00000000..d32c550e
--- /dev/null
+++ b/src/serialization/include/proto-multiple-messages-handler.hpp
@@ -0,0 +1,79 @@
+#pragma once
+
+#include
+
+#include
+#include
+#include
+#include
+
+#include "cct_exception.hpp"
+#include "cct_log.hpp"
+
+namespace cct {
+
+class ProtobufMessagesReader {
+ public:
+ explicit ProtobufMessagesReader(std::istream& is) : _is(is), _iis(&_is), _cis(&_iis) {}
+
+ bool hasNext() { return _cis.ReadVarint64(&_nextSize); }
+
+ template
+ MsgT next() {
+ MsgT msg;
+ auto msgLimit = _cis.PushLimit(_nextSize);
+ if (!msg.ParseFromCodedStream(&_cis)) {
+ log::error("Error reading single protobuf message of size {}", _nextSize);
+ }
+ _cis.PopLimit(msgLimit);
+ return msg;
+ }
+
+ private:
+ std::istream& _is;
+ ::google::protobuf::io::IstreamInputStream _iis;
+ ::google::protobuf::io::CodedInputStream _cis;
+ uint64_t _nextSize{};
+};
+
+template
+class ProtobufMessagesWriter {
+ public:
+ void open(OStreamType&& newOs) {
+ // reverse destroy streams to flush latest data. Recreate the streams after creation of new ofstream
+ _cos.reset();
+ _oos.reset();
+ _os = std::move(newOs);
+ _oos = std::make_unique<::google::protobuf::io::OstreamOutputStream>(&_os);
+ _cos = std::make_unique<::google::protobuf::io::CodedOutputStream>(_oos.get());
+ }
+
+ template
+ void write(const MsgT& msg) {
+ if (!_cos) {
+ throw exception("ProtobufMessagesWriter::open should have been called first");
+ }
+
+ _cos->WriteVarint64(msg.ByteSizeLong());
+
+ if (!msg.SerializeToCodedStream(_cos.get())) {
+ log::error("Failed to serialize to coded stream");
+ }
+ }
+
+ OStreamType flush() {
+ _cos.reset();
+ _oos.reset();
+
+ OStreamType ret(std::move(_os));
+ _os = OStreamType();
+ return ret;
+ }
+
+ private:
+ OStreamType _os;
+ std::unique_ptr<::google::protobuf::io::OstreamOutputStream> _oos;
+ std::unique_ptr<::google::protobuf::io::CodedOutputStream> _cos;
+};
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/serialization/include/proto-public-trade.hpp b/src/serialization/include/proto-public-trade.hpp
new file mode 100644
index 00000000..3a3af689
--- /dev/null
+++ b/src/serialization/include/proto-public-trade.hpp
@@ -0,0 +1,21 @@
+#pragma once
+
+#include "market.hpp"
+#include "publictrade.hpp"
+#include "trade-data.pb.h"
+
+namespace cct {
+
+::objects::TradeData ConvertPublicTradeToTradeData(const PublicTrade &publicTrade);
+
+class TradeDataToPublicTradeConverter {
+ public:
+ explicit TradeDataToPublicTradeConverter(Market market) : _market(market) {}
+
+ PublicTrade operator()(const ::objects::TradeData &tradeData) const;
+
+ private:
+ Market _market;
+};
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/serialization/include/proto-serializer.hpp b/src/serialization/include/proto-serializer.hpp
new file mode 100644
index 00000000..53ae6549
--- /dev/null
+++ b/src/serialization/include/proto-serializer.hpp
@@ -0,0 +1,257 @@
+#pragma once
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "cct_log.hpp"
+#include "cct_vector.hpp"
+#include "durationstring.hpp"
+#include "market-timestamp-set.hpp"
+#include "market.hpp"
+#include "proto-multiple-messages-handler.hpp"
+#include "serialization-tools.hpp"
+#include "timedef.hpp"
+
+namespace cct {
+
+/// Class responsible to accumulate protobuf objects in memory and perform regular flushes of its data to the disk.
+/// Data is accumulated by Market and will write to following files (from subPath):
+/// 'BASECUR-QUOTECUR/YYYY/MM/DD/HH:00:00_HH:59:59.binpb'
+///
+/// If you may push duplicated objects, you have to provide Comp and Equal types.
+/// In this case, Equal must be consistent with Comp, and the first criteria of the comparison should be the timestamp
+/// (ordered from oldest to youngest).
+///
+/// You may not provide any Comp and Equal if by design you will not push duplicated data.
+template
+class ProtobufObjectsSerializer {
+ public:
+ /// Creates a new ProtobufObjectsSerializer.
+ /// @param marketTimestampSet the latest written timestamp for all markets to avoid writing duplicate entries between
+ /// coincenter restarts.
+ ProtobufObjectsSerializer(std::string subPath, const MarketTimestampSet &marketTimestampSet,
+ int32_t nbObjectsPerMarketInMemory) noexcept
+ : _subPath(std::move(subPath)), _nbObjectsPerMarketInMemory(nbObjectsPerMarketInMemory) {
+ for (const auto &[market, timestamp] : marketTimestampSet) {
+ auto &lastWrittenObjectTimestamp = _marketDataMap[market].lastWrittenObjectTimestamp;
+
+ lastWrittenObjectTimestamp = timestamp;
+
+ // When program starts, we want to exclude equal timestamps to avoid writing objects that may have been wrote
+ // already from a previous run (the SortUnique will not protect us here)
+ ++lastWrittenObjectTimestamp;
+ }
+ }
+
+ ProtobufObjectsSerializer(const ProtobufObjectsSerializer &) = delete;
+ ProtobufObjectsSerializer &operator=(const ProtobufObjectsSerializer &) = delete;
+
+ ProtobufObjectsSerializer(ProtobufObjectsSerializer &&other) noexcept { swap(other); }
+
+ ProtobufObjectsSerializer &operator=(ProtobufObjectsSerializer &&other) noexcept {
+ if (&other != this) {
+ swap(other);
+ }
+ return *this;
+ }
+
+ ~ProtobufObjectsSerializer() {
+ try {
+ for (auto &[market, marketData] : _marketDataMap) {
+ writeOnDisk(market, marketData);
+ }
+ } catch (const std::exception &e) {
+ log::error("exception caught in writeOnDisk: {}", e.what());
+ }
+ }
+
+ /// Pushes a new object into the serializer.
+ /// The new object is guaranteed to be written upon destruction of this serializer at the latest unless:
+ /// - its timestamp is older than the latest written timestamp of this market
+ /// - it has invalid data
+ template
+ void push(Market market, ProtobufObjectTypeU &&protoObj) {
+ auto &marketData = _marketDataMap[market];
+
+ if (!isValid(marketData, protoObj)) {
+ return;
+ }
+
+ marketData.dataVector.push_back(std::forward(protoObj));
+
+ checkWriteOnDisk(market, marketData);
+ }
+
+ void swap(ProtobufObjectsSerializer &rhs) noexcept {
+ _marketDataMap.swap(rhs._marketDataMap);
+ _subPath.swap(rhs._subPath);
+ std::swap(_nbObjectsPerMarketInMemory, rhs._nbObjectsPerMarketInMemory);
+ std::swap(_flushCounter, rhs._flushCounter);
+ }
+
+ private:
+ using ProtobufObjectTypeVector = vector;
+
+ struct MarketData {
+ ProtobufObjectTypeVector dataVector;
+ TimePoint lastWrittenObjectTimestamp;
+ };
+
+ void checkWriteOnDisk(Market market, MarketData &marketData) {
+ auto &dataVector = marketData.dataVector;
+ if (dataVector.size() == static_cast(_nbObjectsPerMarketInMemory)) {
+ writeOnDisk(market, marketData);
+
+ // shrink_to_fit as vector will never grow-up larger than its current size
+ dataVector.shrink_to_fit();
+ dataVector.clear();
+
+ checkPeriodicFlush();
+ }
+ }
+
+ void writeOnDisk(Market market, MarketData &marketData) {
+ auto &dataVector = marketData.dataVector;
+ if (dataVector.empty()) {
+ return;
+ }
+
+ const auto nowTime = std::chrono::steady_clock::now();
+
+ SortUnique(dataVector);
+
+ std::string pathStr = _subPath;
+
+ std::chrono::hours prevHourOfDay{-1};
+
+ ProtobufMessagesWriter protobufMessagesWriter;
+ for (const auto &protobufObject : dataVector) {
+ checkOpenFile(market, protobufObject, prevHourOfDay, pathStr, protobufMessagesWriter);
+
+ protobufMessagesWriter.write(protobufObject);
+ }
+
+ marketData.lastWrittenObjectTimestamp = TimePoint{milliseconds{dataVector.back().unixtimestampinms()}};
+
+ const auto nbElemsWritten = dataVector.size();
+
+ const auto steadyClockDuration = std::chrono::steady_clock::now() - nowTime;
+ const auto dur = std::chrono::duration_cast(steadyClockDuration);
+
+ log::info("Serialized {} object(s) for {} data in {}, last in {}", nbElemsWritten, market, DurationToString(dur),
+ pathStr);
+ }
+
+ // Periodic memory release to avoid possible leaks for long time running (if market data unused anymore for instance)
+ void checkPeriodicFlush() {
+ if (++_flushCounter != RehashThreshold) {
+ return;
+ }
+
+ _flushCounter = 0;
+
+ auto nowTime = Clock::now();
+
+ for (auto it = _marketDataMap.begin(); it != _marketDataMap.end();) {
+ if (it->second.lastWrittenObjectTimestamp + DurationType{static_cast(DurationValue)} < nowTime) {
+ // Unchanged data since a long time - write data if any, and clears the entry in the map
+ const Market market = it->first;
+ MarketData &marketData = it->second;
+
+ writeOnDisk(market, marketData);
+
+ log::info("Released {} protobuf objects for {}", marketData.dataVector.capacity(), market);
+
+ it = _marketDataMap.erase(it);
+ } else {
+ ++it;
+ }
+ }
+
+ _marketDataMap.rehash(_marketDataMap.size());
+ }
+
+ static void SortUnique(ProtobufObjectTypeVector &dataVector) {
+ static_assert((std::is_void_v && std::is_void_v) || (!std::is_void_v && !std::is_void_v));
+
+ if constexpr (std::is_void_v) {
+ // Sort by timestamp (required by 'writeOnDisk' algorithm)
+ std::ranges::sort(dataVector, [](const auto &lhs, const auto &rhs) {
+ return lhs.unixtimestampinms() < rhs.unixtimestampinms();
+ });
+ } else {
+ // We assume that timestamp is the first sorting criteria
+ std::ranges::sort(dataVector, Comp{});
+ }
+
+ // If duplicate elements are possible, remove them
+ if constexpr (!std::is_void_v) {
+ const auto [eraseIt1, eraseIt2] = std::ranges::unique(dataVector, Equal{});
+ dataVector.erase(eraseIt1, eraseIt2);
+ }
+ }
+
+ void checkOpenFile(Market market, const ProtobufObjectType &protobufObject, std::chrono::hours &prevHourOfDay,
+ std::string &pathStr, ProtobufMessagesWriter &protobufMessagesWriter) {
+ const TimePoint tp{milliseconds{protobufObject.unixtimestampinms()}};
+ const auto hourOfDay = GetHourOfDay(tp);
+
+ if (prevHourOfDay != hourOfDay) {
+ // open new outfile
+ setDirectory(market, tp, pathStr);
+ std::filesystem::create_directories(std::filesystem::path(pathStr));
+
+ pathStr.append(ComputeProtoFileName(std::chrono::duration_cast(hourOfDay).count()));
+
+ std::filesystem::path filePath(pathStr);
+
+ protobufMessagesWriter.open(std::ofstream(filePath, std::ios_base::app));
+ prevHourOfDay = hourOfDay;
+ }
+ }
+
+ static std::chrono::hours GetHourOfDay(TimePoint tp) {
+ const auto dp = std::chrono::floor(tp);
+
+ return std::chrono::floor(tp - dp);
+ }
+
+ bool isValid(const MarketData &marketData, const ProtobufObjectType &protoObj) const {
+ if (!protoObj.has_unixtimestampinms()) {
+ throw exception("Attempt to push proto object without any timestamp");
+ }
+ if (TimePoint{milliseconds{protoObj.unixtimestampinms()}} < marketData.lastWrittenObjectTimestamp) {
+ // do not push an object that has an older timestamp of the last written object
+ return false;
+ }
+ return true;
+ }
+
+ void setDirectory(Market market, TimePoint tp, std::string &pathStr) const {
+ // Note: below code could be simplified once compilers fully implement std::format and chrono C++20
+ // libraries.
+ const auto dp = std::chrono::floor(tp);
+ const std::chrono::year_month_day ymd{dp};
+
+ pathStr.replace(pathStr.begin() + _subPath.size(), pathStr.end(),
+ format("/{}/{:04}/{:02}/{:02}/", market, static_cast(ymd.year()),
+ static_cast(ymd.month()), static_cast(ymd.day())));
+ }
+
+ using MarketDataMap = std::unordered_map;
+
+ MarketDataMap _marketDataMap;
+ std::string _subPath;
+ int32_t _nbObjectsPerMarketInMemory;
+ int32_t _flushCounter{};
+};
+
+} // namespace cct
diff --git a/src/serialization/include/proto-trade-data-compare.hpp b/src/serialization/include/proto-trade-data-compare.hpp
new file mode 100644
index 00000000..d1fb640a
--- /dev/null
+++ b/src/serialization/include/proto-trade-data-compare.hpp
@@ -0,0 +1,15 @@
+#pragma once
+
+#include "trade-data.pb.h"
+
+namespace cct {
+
+struct TradeDataComp {
+ bool operator()(const ::objects::TradeData &lhs, const ::objects::TradeData &rhs) const;
+};
+
+struct TradeDataEqual {
+ bool operator()(const ::objects::TradeData &lhs, const ::objects::TradeData &rhs) const;
+};
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/serialization/include/serialization-tools.hpp b/src/serialization/include/serialization-tools.hpp
new file mode 100644
index 00000000..2a6e3b86
--- /dev/null
+++ b/src/serialization/include/serialization-tools.hpp
@@ -0,0 +1,16 @@
+#pragma once
+
+#include
+#include
+
+namespace cct {
+
+std::string ComputeProtoSubPath(std::string_view dataDir, std::string_view exchangeName,
+ std::string_view protobufObjectName);
+
+/// From an hour of day in [0, 23], return the file name for a protobuf binary serialization file.
+/// Example:
+/// ComputeProtoFileName(4) -> "04:00:00_04:59:59.binpb"
+std::string_view ComputeProtoFileName(int hourOfDay);
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/serialization/proto/market-order-book-timed-data.proto b/src/serialization/proto/market-order-book-timed-data.proto
new file mode 100644
index 00000000..286fdf0b
--- /dev/null
+++ b/src/serialization/proto/market-order-book-timed-data.proto
@@ -0,0 +1,21 @@
+syntax = "proto3";
+
+package objects;
+
+message MarketOrderBookTimedData {
+ optional int64 unixTimestampInMs = 1;
+ optional int32 volumeNbDecimals = 2;
+ optional int32 priceNbDecimals = 3;
+
+ message PricedVolume {
+ optional int64 price = 1;
+ optional int64 volume = 2;
+ }
+
+ message OrderBook {
+ repeated PricedVolume asks = 1;
+ repeated PricedVolume bids = 2;
+ }
+
+ optional OrderBook orderBook = 4;
+}
\ No newline at end of file
diff --git a/src/serialization/proto/trade-data.proto b/src/serialization/proto/trade-data.proto
new file mode 100644
index 00000000..f1399429
--- /dev/null
+++ b/src/serialization/proto/trade-data.proto
@@ -0,0 +1,18 @@
+syntax = "proto3";
+
+package objects;
+
+enum TradeSide {
+ TRADE_UNSPECIFIED = 0;
+ TRADE_BUY = 1;
+ TRADE_SELL = 2;
+}
+
+message TradeData {
+ optional int64 unixTimestampInMs = 1;
+ optional int64 priceAmount = 2;
+ optional int64 volumeAmount = 3;
+ optional int32 priceNbDecimals = 4;
+ optional int32 volumeNbDecimals = 5;
+ TradeSide tradeSide = 6;
+}
\ No newline at end of file
diff --git a/src/serialization/src/dummy-market-data-deserializer.cpp b/src/serialization/src/dummy-market-data-deserializer.cpp
new file mode 100644
index 00000000..ab9c1643
--- /dev/null
+++ b/src/serialization/src/dummy-market-data-deserializer.cpp
@@ -0,0 +1,29 @@
+#include "dummy-market-data-deserializer.hpp"
+
+#include "market-order-book-vector.hpp"
+#include "market-timestamp-set.hpp"
+#include "market.hpp"
+#include "public-trade-vector.hpp"
+#include "time-window.hpp"
+
+namespace cct {
+
+DummyMarketDataDeserializer::DummyMarketDataDeserializer([[maybe_unused]] std::string_view dataDir,
+ [[maybe_unused]] std::string_view exchangeName) {}
+
+MarketTimestampSet DummyMarketDataDeserializer::pullMarketOrderBooksMarkets([[maybe_unused]] TimeWindow timeWindow) {
+ return {};
+}
+
+MarketTimestampSet DummyMarketDataDeserializer::pullTradeMarkets([[maybe_unused]] TimeWindow timeWindow) { return {}; }
+
+MarketOrderBookVector DummyMarketDataDeserializer::pullMarketOrderBooks([[maybe_unused]] Market market,
+ [[maybe_unused]] TimeWindow timeWindow) {
+ return {};
+}
+
+PublicTradeVector DummyMarketDataDeserializer::pullTrades([[maybe_unused]] Market market,
+ [[maybe_unused]] TimeWindow timeWindow) {
+ return {};
+}
+} // namespace cct
\ No newline at end of file
diff --git a/src/serialization/src/dummy-market-data-serializer.cpp b/src/serialization/src/dummy-market-data-serializer.cpp
new file mode 100644
index 00000000..9642330e
--- /dev/null
+++ b/src/serialization/src/dummy-market-data-serializer.cpp
@@ -0,0 +1,23 @@
+#include "dummy-market-data-serializer.hpp"
+
+#include
+#include
+
+#include "market-timestamp-set.hpp"
+#include "market.hpp"
+#include "marketorderbook.hpp"
+#include "publictrade.hpp"
+
+namespace cct {
+
+DummyMarketDataSerializer::DummyMarketDataSerializer(
+ [[maybe_unused]] std::string_view dataDir,
+ [[maybe_unused]] const MarketTimestampSets &lastWrittenObjectsMarketTimestamp,
+ [[maybe_unused]] std::string_view exchangeName) {}
+
+void DummyMarketDataSerializer::push([[maybe_unused]] const MarketOrderBook &marketOrderBook) {}
+
+void DummyMarketDataSerializer::push([[maybe_unused]] Market market,
+ [[maybe_unused]] std::span publicTrades) {}
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/serialization/src/proto-market-data-deserializer.cpp b/src/serialization/src/proto-market-data-deserializer.cpp
new file mode 100644
index 00000000..9176c209
--- /dev/null
+++ b/src/serialization/src/proto-market-data-deserializer.cpp
@@ -0,0 +1,32 @@
+#include "proto-market-data-deserializer.hpp"
+
+#include
+
+#include "market-order-book-vector.hpp"
+#include "market-timestamp-set.hpp"
+#include "proto-constants.hpp"
+#include "public-trade-vector.hpp"
+#include "serialization-tools.hpp"
+#include "time-window.hpp"
+
+namespace cct {
+ProtoMarketDataDeserializer::ProtoMarketDataDeserializer(std::string_view dataDir, std::string_view exchangeName)
+ : _marketOrderBookDeserializer(ComputeProtoSubPath(dataDir, exchangeName, kSubPathMarketOrderBook)),
+ _tradeDataDeserializer(ComputeProtoSubPath(dataDir, exchangeName, kSubPathTrades)) {}
+
+MarketTimestampSet ProtoMarketDataDeserializer::pullMarketOrderBooksMarkets(TimeWindow timeWindow) {
+ return _tradeDataDeserializer.listMarkets(timeWindow);
+}
+
+MarketTimestampSet ProtoMarketDataDeserializer::pullTradeMarkets(TimeWindow timeWindow) {
+ return _marketOrderBookDeserializer.listMarkets(timeWindow);
+}
+
+MarketOrderBookVector ProtoMarketDataDeserializer::pullMarketOrderBooks(Market market, TimeWindow timeWindow) {
+ return _marketOrderBookDeserializer.loadMarket(market, timeWindow);
+}
+
+PublicTradeVector ProtoMarketDataDeserializer::pullTrades(Market market, TimeWindow timeWindow) {
+ return _tradeDataDeserializer.loadMarket(market, timeWindow);
+}
+} // namespace cct
\ No newline at end of file
diff --git a/src/serialization/src/proto-market-data-serializer.cpp b/src/serialization/src/proto-market-data-serializer.cpp
new file mode 100644
index 00000000..f672e828
--- /dev/null
+++ b/src/serialization/src/proto-market-data-serializer.cpp
@@ -0,0 +1,42 @@
+#include "proto-market-data-serializer.hpp"
+
+#include
+#include
+
+#include "market-timestamp-set.hpp"
+#include "marketorderbook.hpp"
+#include "proto-constants.hpp"
+#include "proto-market-order-book.hpp"
+#include "proto-public-trade.hpp"
+#include "publictrade.hpp"
+#include "serialization-tools.hpp"
+
+namespace cct {
+
+ProtoMarketDataSerializer::ProtoMarketDataSerializer(std::string_view dataDir,
+ const MarketTimestampSets& lastWrittenObjectsMarketTimestamp,
+ std::string_view exchangeName)
+ : _marketOrderBookSerializer(ComputeProtoSubPath(dataDir, exchangeName, kSubPathMarketOrderBook),
+ lastWrittenObjectsMarketTimestamp.orderBooksMarkets, 1000),
+ _tradesSerializer(ComputeProtoSubPath(dataDir, exchangeName, kSubPathTrades),
+ lastWrittenObjectsMarketTimestamp.tradesMarkets, 25000) {}
+
+void ProtoMarketDataSerializer::push(const MarketOrderBook& marketOrderBook) {
+ if (!marketOrderBook.isValid()) {
+ log::error("Do not serialize invalid market order book");
+ return;
+ }
+ _marketOrderBookSerializer.push(marketOrderBook.market(), CreateMarketOrderBookTimedData(marketOrderBook));
+}
+
+void ProtoMarketDataSerializer::push(Market market, std::span publicTrades) {
+ for (const auto& publicTrade : publicTrades) {
+ if (!publicTrade.isValid()) {
+ log::error("Do not serialize invalid public trade");
+ continue;
+ }
+ _tradesSerializer.push(market, ConvertPublicTradeToTradeData(publicTrade));
+ }
+}
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/serialization/src/proto-market-order-book.cpp b/src/serialization/src/proto-market-order-book.cpp
new file mode 100644
index 00000000..7f278275
--- /dev/null
+++ b/src/serialization/src/proto-market-order-book.cpp
@@ -0,0 +1,77 @@
+#include "proto-market-order-book.hpp"
+
+#include
+#include
+
+#include "market-order-book-timed-data.pb.h"
+#include "marketorderbook.hpp"
+#include "monetaryamount.hpp"
+#include "timedef.hpp"
+#include "volumeandpricenbdecimals.hpp"
+
+namespace cct {
+::objects::MarketOrderBookTimedData CreateMarketOrderBookTimedData(const MarketOrderBook& marketOrderBook) {
+ ::objects::MarketOrderBookTimedData protoObj;
+
+ const auto [volNbDecimals, priNbDecimals] = marketOrderBook.volAndPriNbDecimals();
+ const auto unixTimestampInMs = TimestampToMillisecondsSinceEpoch(marketOrderBook.time());
+
+ protoObj.set_unixtimestampinms(unixTimestampInMs);
+ protoObj.set_volumenbdecimals(volNbDecimals);
+ protoObj.set_pricenbdecimals(priNbDecimals);
+
+ auto& orderBook = *protoObj.mutable_orderbook();
+
+ const int nbBids = marketOrderBook.nbBidPrices();
+ for (int bidPos = 1; bidPos <= nbBids; ++bidPos) {
+ const auto [volume, price] = marketOrderBook[-bidPos];
+ auto& pricedVolume = *orderBook.add_bids();
+
+ pricedVolume.set_volume(volume.amount(volNbDecimals).value());
+ pricedVolume.set_price(price.amount(priNbDecimals).value());
+ }
+
+ const int nbAsks = marketOrderBook.nbAskPrices();
+ for (int askPos = 1; askPos <= nbAsks; ++askPos) {
+ const auto [volume, price] = marketOrderBook[askPos];
+ auto& pricedVolume = *orderBook.add_asks();
+
+ pricedVolume.set_volume(volume.amount(volNbDecimals).value());
+ pricedVolume.set_price(price.amount(priNbDecimals).value());
+ }
+
+ return protoObj;
+}
+
+MarketOrderBook MarketOrderBookConverter::operator()(
+ const ::objects::MarketOrderBookTimedData& marketOrderBookTimedData) {
+ const TimePoint timeStamp(milliseconds(marketOrderBookTimedData.unixtimestampinms()));
+ const VolAndPriNbDecimals volAndPriNbDecimals(marketOrderBookTimedData.volumenbdecimals(),
+ marketOrderBookTimedData.pricenbdecimals());
+
+ const auto& bids = marketOrderBookTimedData.orderbook().bids();
+ const auto& asks = marketOrderBookTimedData.orderbook().asks();
+ const int32_t lowestAskPricePos = static_cast(bids.size());
+ const int32_t highestBidPricePos = lowestAskPricePos - 1;
+
+ // We directly construct the MarketOrderBook here - we trust the protobuf data (it should have been written from a
+ // valid MarketOrderBook at the source)
+ // Possible optimization - allocate in a reusable arena of memory instead of allocating a new buffer for each new
+ // object.
+ MarketOrderBook::AmountPriceVector orders;
+
+ orders.reserve(bids.size() + asks.size());
+
+ for (const auto& bid : std::ranges::reverse_view(bids)) {
+ orders.emplace_back(bid.volume(), bid.price());
+ }
+
+ for (const auto& ask : asks) {
+ orders.emplace_back(-ask.volume(), ask.price());
+ }
+
+ return MarketOrderBook{timeStamp, _market, std::move(orders),
+ highestBidPricePos, lowestAskPricePos, volAndPriNbDecimals};
+}
+
+} // namespace cct
diff --git a/src/serialization/src/proto-public-trade.cpp b/src/serialization/src/proto-public-trade.cpp
new file mode 100644
index 00000000..bfcd174e
--- /dev/null
+++ b/src/serialization/src/proto-public-trade.cpp
@@ -0,0 +1,62 @@
+#include "proto-public-trade.hpp"
+
+#include "monetaryamount.hpp"
+#include "publictrade.hpp"
+#include "timedef.hpp"
+#include "trade-data.pb.h"
+#include "tradeside.hpp"
+#include "unreachable.hpp"
+
+namespace cct {
+namespace {
+::objects::TradeSide ConvertTradeSide(TradeSide tradeSide) {
+ switch (tradeSide) {
+ case TradeSide::kBuy:
+ return ::objects::TRADE_BUY;
+ case TradeSide::kSell:
+ return ::objects::TRADE_SELL;
+ default:
+ unreachable();
+ }
+}
+
+TradeSide ConvertTradeSide(::objects::TradeSide tradeSide) {
+ switch (tradeSide) {
+ case ::objects::TRADE_BUY:
+ return TradeSide::kBuy;
+ case ::objects::TRADE_SELL:
+ return TradeSide::kSell;
+ default:
+ unreachable();
+ }
+}
+
+} // namespace
+
+::objects::TradeData ConvertPublicTradeToTradeData(const PublicTrade &publicTrade) {
+ ::objects::TradeData protoObj;
+
+ protoObj.set_unixtimestampinms(TimestampToMillisecondsSinceEpoch(publicTrade.time()));
+
+ const auto price = publicTrade.price();
+ protoObj.set_priceamount(price.amount());
+ protoObj.set_pricenbdecimals(price.nbDecimals());
+
+ const auto volume = publicTrade.amount();
+ protoObj.set_volumeamount(volume.amount());
+ protoObj.set_volumenbdecimals(volume.nbDecimals());
+
+ protoObj.set_tradeside(ConvertTradeSide(publicTrade.side()));
+
+ return protoObj;
+}
+
+PublicTrade TradeDataToPublicTradeConverter::operator()(const ::objects::TradeData &tradeData) const {
+ const MonetaryAmount amount(tradeData.volumeamount(), _market.base(), tradeData.volumenbdecimals());
+ const MonetaryAmount price(tradeData.priceamount(), _market.quote(), tradeData.pricenbdecimals());
+ const TimePoint timeStamp(milliseconds(tradeData.unixtimestampinms()));
+
+ return {ConvertTradeSide(tradeData.tradeside()), amount, price, timeStamp};
+}
+
+} // namespace cct
diff --git a/src/serialization/src/proto-trade-data-compare.cpp b/src/serialization/src/proto-trade-data-compare.cpp
new file mode 100644
index 00000000..d0bd28e8
--- /dev/null
+++ b/src/serialization/src/proto-trade-data-compare.cpp
@@ -0,0 +1,43 @@
+#include "proto-trade-data-compare.hpp"
+
+#include "currencycode.hpp"
+#include "monetaryamount.hpp"
+
+namespace cct {
+
+namespace {
+MonetaryAmount PriceMonetaryAmount(const ::objects::TradeData& obj) {
+ return MonetaryAmount(obj.priceamount(), CurrencyCode{}, obj.pricenbdecimals());
+}
+
+MonetaryAmount VolumeMonetaryAmount(const ::objects::TradeData& obj) {
+ return MonetaryAmount(obj.volumeamount(), CurrencyCode{}, obj.volumenbdecimals());
+}
+} // namespace
+
+bool TradeDataComp::operator()(const ::objects::TradeData& lhs, const ::objects::TradeData& rhs) const {
+ if (lhs.unixtimestampinms() != rhs.unixtimestampinms()) {
+ return lhs.unixtimestampinms() < rhs.unixtimestampinms();
+ }
+ MonetaryAmount lhsAmount = VolumeMonetaryAmount(lhs);
+ MonetaryAmount rhsAmount = VolumeMonetaryAmount(rhs);
+ if (lhsAmount != rhsAmount) {
+ return lhsAmount < rhsAmount;
+ }
+ MonetaryAmount lhsPrice = PriceMonetaryAmount(lhs);
+ MonetaryAmount rhsPrice = PriceMonetaryAmount(rhs);
+ if (lhsPrice != rhsPrice) {
+ return lhsPrice < rhsPrice;
+ }
+ if (lhs.tradeside() != rhs.tradeside()) {
+ return lhs.tradeside() < rhs.tradeside();
+ }
+ return false;
+}
+
+bool TradeDataEqual::operator()(const ::objects::TradeData& lhs, const ::objects::TradeData& rhs) const {
+ return lhs.unixtimestampinms() == rhs.unixtimestampinms() && VolumeMonetaryAmount(lhs) == VolumeMonetaryAmount(rhs) &&
+ PriceMonetaryAmount(lhs) == PriceMonetaryAmount(rhs) && lhs.tradeside() == rhs.tradeside();
+}
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/serialization/src/serialization-tools.cpp b/src/serialization/src/serialization-tools.cpp
new file mode 100644
index 00000000..ef5595f2
--- /dev/null
+++ b/src/serialization/src/serialization-tools.cpp
@@ -0,0 +1,67 @@
+#include "serialization-tools.hpp"
+
+#include
+#include
+#include
+#include
+
+#include "proto-constants.hpp"
+
+namespace cct {
+
+std::string ComputeProtoSubPath(std::string_view dataDir, std::string_view exchangeName,
+ std::string_view protobufObjectName) {
+ std::string ret;
+
+ static constexpr std::string_view kSerializedDataSubPath = "/serialized/";
+
+ ret.reserve(dataDir.size() + kSerializedDataSubPath.size() + protobufObjectName.size() + exchangeName.size() + 1U);
+
+ ret.append(dataDir);
+ ret.append(kSerializedDataSubPath);
+ ret.append(protobufObjectName);
+ ret.push_back('/');
+ ret.append(exchangeName);
+ return ret;
+}
+
+namespace {
+
+consteval auto BuildBinProtoFileNames() {
+ constexpr std::string_view kBinProtoFilePart2 = ":00:00_";
+ constexpr std::string_view kBinProtoFilePart3 = ":59:59";
+
+ using ProtoFileNameBuffer =
+ std::array(2 * 2)>;
+
+ constexpr auto kNbHourInDay = 24;
+
+ std::array ret;
+
+ for (auto hourOfDay = 0; hourOfDay < kNbHourInDay; ++hourOfDay) {
+ std::array hourStr = {static_cast