From 8395c5c30af90247885734f0478ed3967eec500e Mon Sep 17 00:00:00 2001 From: Stephane Janel Date: Sun, 24 Dec 2023 17:21:10 +0100 Subject: [PATCH] [Feature] Serialization & Deserialization of market data with protobuf --- ...untu-monitoring.yml => ubuntu-special.yml} | 25 +- .github/workflows/windows.yml | 7 +- .gitignore | 1 + CMakeLists.txt | 33 + CONFIG.md | 24 +- Dockerfile | 2 + INSTALL.md | 1 + README.md | 14 +- TRADING.md | 108 ++++ alpine.Dockerfile | 4 +- src/api/common/CMakeLists.txt | 1 + src/api/common/include/exchangepublicapi.hpp | 36 +- src/api/common/src/exchangeprivateapi.cpp | 2 +- src/api/common/src/exchangepublicapi.cpp | 92 ++- src/api/exchanges/src/binancepublicapi.cpp | 2 +- src/api/exchanges/src/bithumbpublicapi.cpp | 2 +- src/api/exchanges/src/krakenprivateapi.cpp | 4 +- src/api/interface/include/exchange.hpp | 4 +- src/api/interface/src/exchange.cpp | 6 +- src/engine/CMakeLists.txt | 9 + .../include/coincenter-commands-iterator.hpp | 28 + src/engine/include/coincenter.hpp | 40 +- src/engine/include/coincentercommand.hpp | 12 +- src/engine/include/coincenteroptions.hpp | 13 +- src/engine/include/coincenteroptionsdef.hpp | 44 ++ src/engine/include/exchangesorchestrator.hpp | 17 + .../include/query-result-type-helpers.hpp | 15 + src/engine/include/queryresultprinter.hpp | 10 +- src/engine/include/queryresulttypes.hpp | 19 + .../replay-algorithm-name-iterator.hpp | 29 + src/engine/include/replay-options.hpp | 33 + src/engine/include/stringoptionparser.hpp | 7 +- .../src/coincenter-commands-iterator.cpp | 74 +++ src/engine/src/coincenter.cpp | 398 ++++++++++-- src/engine/src/coincentercommand.cpp | 7 + src/engine/src/coincentercommands.cpp | 43 ++ src/engine/src/coincenterinfo_create.cpp | 19 +- src/engine/src/coincenteroptions.cpp | 30 + src/engine/src/exchangesorchestrator.cpp | 121 +++- src/engine/src/query-result-type-helpers.cpp | 31 + src/engine/src/queryresultprinter.cpp | 212 +++++++ .../src/replay-algorithm-name-iterator.cpp | 62 ++ src/engine/src/replay-options.cpp | 19 + src/engine/src/stringoptionparser.cpp | 19 + .../test/queryresultprinter_public_test.cpp | 308 ++++++++- .../replay-algorithm-name-iterator_test.cpp | 106 ++++ src/engine/test/stringoptionparser_test.cpp | 24 + src/http-request/include/request-retry.hpp | 3 +- src/main/CMakeLists.txt | 4 + src/objects/CMakeLists.txt | 9 + src/objects/include/automation-config.hpp | 30 + src/objects/include/coincentercommandtype.hpp | 6 +- src/objects/include/exchange-names.hpp | 4 + src/objects/include/exchangeconfig.hpp | 10 +- src/objects/include/generalconfig.hpp | 8 +- src/objects/include/generalconfigdefault.hpp | 11 + src/objects/include/marketorderbook.hpp | 12 + src/objects/include/publictrade.hpp | 4 +- src/objects/include/time-window.hpp | 70 +++ src/objects/include/trading-config.hpp | 19 + src/objects/src/coincentercommandtype.cpp | 5 +- src/objects/src/exchangeconfig.cpp | 7 +- src/objects/src/exchangeconfigdefault.hpp | 2 + src/objects/src/exchangeconfigmap.cpp | 8 +- src/objects/src/generalconfig.cpp | 4 +- src/objects/src/marketorderbook.cpp | 30 + src/objects/src/publictrade.cpp | 6 + src/objects/src/time-window.cpp | 15 + src/objects/test/marketorderbook_test.cpp | 18 + src/objects/test/time-window_test.cpp | 140 +++++ src/serialization/CMakeLists.txt | 70 +++ .../abstract-market-data-deserializer.hpp | 24 + .../abstract-market-data-serializer.hpp | 24 + .../include/continuous-iterator.hpp | 23 + .../dummy-market-data-deserializer.hpp | 29 + .../include/dummy-market-data-serializer.hpp | 26 + .../include/market-timestamp-set.hpp | 15 + .../include/market-timestamp.hpp | 17 + src/serialization/include/proto-constants.hpp | 15 + .../include/proto-deserializer.hpp | 184 ++++++ .../proto-market-data-deserializer.hpp | 36 ++ .../include/proto-market-data-serializer.hpp | 34 + .../include/proto-market-order-book.hpp | 21 + .../proto-multiple-messages-handler.hpp | 79 +++ .../include/proto-public-trade.hpp | 21 + .../include/proto-serializer.hpp | 257 ++++++++ .../include/proto-trade-data-compare.hpp | 15 + .../include/serialization-tools.hpp | 16 + .../proto/market-order-book-timed-data.proto | 21 + src/serialization/proto/trade-data.proto | 18 + .../src/dummy-market-data-deserializer.cpp | 29 + .../src/dummy-market-data-serializer.cpp | 23 + .../src/proto-market-data-deserializer.cpp | 32 + .../src/proto-market-data-serializer.cpp | 42 ++ .../src/proto-market-order-book.cpp | 77 +++ src/serialization/src/proto-public-trade.cpp | 62 ++ .../src/proto-trade-data-compare.cpp | 43 ++ src/serialization/src/serialization-tools.cpp | 67 ++ .../test/continuous-iterator_test.cpp | 41 ++ .../test/proto-market-order-book_test.cpp | 115 ++++ .../proto-multiple-messages-handler_test.cpp | 109 ++++ .../test/proto-public-trade_test.cpp | 25 + ...serialization-and-deserialization_test.cpp | 591 ++++++++++++++++++ src/serialization/test/proto-test-data.hpp | 66 ++ .../test/serialization-tools_test.cpp | 24 + src/tech/include/durationstring.hpp | 5 + src/tech/include/unitsparser.hpp | 2 + src/tech/src/durationstring.cpp | 36 ++ src/tech/test/durationstring_test.cpp | 14 + src/tech/test/simpletable_test.cpp | 1 - src/trading/CMakeLists.txt | 2 + src/trading/algorithms/CMakeLists.txt | 10 + .../include/dummy-market-trader.hpp | 20 + .../include/example-market-trader.hpp | 20 + .../include/market-trader-factory.hpp | 22 + .../algorithms/src/dummy-market-trader.cpp | 17 + .../algorithms/src/example-market-trader.cpp | 17 + .../algorithms/src/market-trader-factory.cpp | 33 + src/trading/common/CMakeLists.txt | 7 + .../abstract-market-trader-factory.hpp | 24 + .../common/include/abstract-market-trader.hpp | 36 ++ .../common/include/market-data-view.hpp | 46 ++ .../include/market-trader-engine-state.hpp | 69 ++ .../common/include/market-trader-engine.hpp | 64 ++ .../include/market-trading-global-result.hpp | 13 + .../common/include/market-trading-result.hpp | 43 ++ .../common/include/trade-range-stats.hpp | 29 + src/trading/common/include/trader-command.hpp | 56 ++ .../common/src/abstract-market-trader.cpp | 13 + src/trading/common/src/market-data-view.cpp | 27 + .../common/src/market-trader-engine-state.cpp | 136 ++++ .../common/src/market-trader-engine.cpp | 329 ++++++++++ .../common/src/market-trading-result.cpp | 20 + src/trading/common/src/trader-command.cpp | 62 ++ 134 files changed, 5866 insertions(+), 140 deletions(-) rename .github/workflows/{ubuntu-monitoring.yml => ubuntu-special.yml} (57%) create mode 100644 TRADING.md create mode 100644 src/engine/include/coincenter-commands-iterator.hpp create mode 100644 src/engine/include/query-result-type-helpers.hpp create mode 100644 src/engine/include/replay-algorithm-name-iterator.hpp create mode 100644 src/engine/include/replay-options.hpp create mode 100644 src/engine/src/coincenter-commands-iterator.cpp create mode 100644 src/engine/src/query-result-type-helpers.cpp create mode 100644 src/engine/src/replay-algorithm-name-iterator.cpp create mode 100644 src/engine/src/replay-options.cpp create mode 100644 src/engine/test/replay-algorithm-name-iterator_test.cpp create mode 100644 src/objects/include/automation-config.hpp create mode 100644 src/objects/include/time-window.hpp create mode 100644 src/objects/include/trading-config.hpp create mode 100644 src/objects/src/time-window.cpp create mode 100644 src/objects/test/time-window_test.cpp create mode 100644 src/serialization/CMakeLists.txt create mode 100644 src/serialization/include/abstract-market-data-deserializer.hpp create mode 100644 src/serialization/include/abstract-market-data-serializer.hpp create mode 100644 src/serialization/include/continuous-iterator.hpp create mode 100644 src/serialization/include/dummy-market-data-deserializer.hpp create mode 100644 src/serialization/include/dummy-market-data-serializer.hpp create mode 100644 src/serialization/include/market-timestamp-set.hpp create mode 100644 src/serialization/include/market-timestamp.hpp create mode 100644 src/serialization/include/proto-constants.hpp create mode 100644 src/serialization/include/proto-deserializer.hpp create mode 100644 src/serialization/include/proto-market-data-deserializer.hpp create mode 100644 src/serialization/include/proto-market-data-serializer.hpp create mode 100644 src/serialization/include/proto-market-order-book.hpp create mode 100644 src/serialization/include/proto-multiple-messages-handler.hpp create mode 100644 src/serialization/include/proto-public-trade.hpp create mode 100644 src/serialization/include/proto-serializer.hpp create mode 100644 src/serialization/include/proto-trade-data-compare.hpp create mode 100644 src/serialization/include/serialization-tools.hpp create mode 100644 src/serialization/proto/market-order-book-timed-data.proto create mode 100644 src/serialization/proto/trade-data.proto create mode 100644 src/serialization/src/dummy-market-data-deserializer.cpp create mode 100644 src/serialization/src/dummy-market-data-serializer.cpp create mode 100644 src/serialization/src/proto-market-data-deserializer.cpp create mode 100644 src/serialization/src/proto-market-data-serializer.cpp create mode 100644 src/serialization/src/proto-market-order-book.cpp create mode 100644 src/serialization/src/proto-public-trade.cpp create mode 100644 src/serialization/src/proto-trade-data-compare.cpp create mode 100644 src/serialization/src/serialization-tools.cpp create mode 100644 src/serialization/test/continuous-iterator_test.cpp create mode 100644 src/serialization/test/proto-market-order-book_test.cpp create mode 100644 src/serialization/test/proto-multiple-messages-handler_test.cpp create mode 100644 src/serialization/test/proto-public-trade_test.cpp create mode 100644 src/serialization/test/proto-serialization-and-deserialization_test.cpp create mode 100644 src/serialization/test/proto-test-data.hpp create mode 100644 src/serialization/test/serialization-tools_test.cpp create mode 100644 src/trading/CMakeLists.txt create mode 100644 src/trading/algorithms/CMakeLists.txt create mode 100644 src/trading/algorithms/include/dummy-market-trader.hpp create mode 100644 src/trading/algorithms/include/example-market-trader.hpp create mode 100644 src/trading/algorithms/include/market-trader-factory.hpp create mode 100644 src/trading/algorithms/src/dummy-market-trader.cpp create mode 100644 src/trading/algorithms/src/example-market-trader.cpp create mode 100644 src/trading/algorithms/src/market-trader-factory.cpp create mode 100644 src/trading/common/CMakeLists.txt create mode 100644 src/trading/common/include/abstract-market-trader-factory.hpp create mode 100644 src/trading/common/include/abstract-market-trader.hpp create mode 100644 src/trading/common/include/market-data-view.hpp create mode 100644 src/trading/common/include/market-trader-engine-state.hpp create mode 100644 src/trading/common/include/market-trader-engine.hpp create mode 100644 src/trading/common/include/market-trading-global-result.hpp create mode 100644 src/trading/common/include/market-trading-result.hpp create mode 100644 src/trading/common/include/trade-range-stats.hpp create mode 100644 src/trading/common/include/trader-command.hpp create mode 100644 src/trading/common/src/abstract-market-trader.cpp create mode 100644 src/trading/common/src/market-data-view.cpp create mode 100644 src/trading/common/src/market-trader-engine-state.cpp create mode 100644 src/trading/common/src/market-trader-engine.cpp create mode 100644 src/trading/common/src/market-trading-result.cpp create mode 100644 src/trading/common/src/trader-command.cpp diff --git a/.github/workflows/ubuntu-monitoring.yml b/.github/workflows/ubuntu-special.yml similarity index 57% rename from .github/workflows/ubuntu-monitoring.yml rename to .github/workflows/ubuntu-special.yml index 1a269f50..1940abef 100644 --- a/.github/workflows/ubuntu-monitoring.yml +++ b/.github/workflows/ubuntu-special.yml @@ -1,4 +1,4 @@ -name: Monitoring +name: Special on: push: @@ -7,14 +7,15 @@ on: pull_request: jobs: - ubuntu-monitoring-build: - name: Build on Ubuntu with monitoring support + ubuntu-special-build: + name: Build on Ubuntu with monitoring / protobuf support runs-on: ubuntu-latest strategy: matrix: compiler: [g++-11] buildmode: [Debug] - build-prometheus-from-source: [0, 1] + build-special-from-source: [0, 1] + prometheus-options: ["-DBUILD_SHARED_LIBS=ON -DENABLE_PULL=OFF -DENABLE_PUSH=ON -DENABLE_COMPRESSION=OFF -DENABLE_TESTING=OFF"] steps: - name: Checkout repository code @@ -35,19 +36,21 @@ jobs: mkdir _build cd _build - cmake .. -DBUILD_SHARED_LIBS=ON -DENABLE_PULL=OFF -DENABLE_PUSH=ON -DENABLE_COMPRESSION=OFF -DENABLE_TESTING=OFF -DCMAKE_BUILD_TYPE=${{matrix.buildmode}} -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -GNinja + cmake .. ${{matrix.prometheus-options}} -GNinja cmake --build . sudo cmake --install . - if: matrix.build-prometheus-from-source == 0 - - - name: Create Build Environment - run: cmake -E make_directory ${{github.workspace}}/build + if: matrix.build-special-from-source == 0 + env: + CXX: ${{matrix.compiler}} + CMAKE_BUILD_TYPE: ${{matrix.buildmode}} - name: Configure CMake - working-directory: ${{github.workspace}}/build shell: bash - run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{matrix.buildmode}} -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCCT_BUILD_PROMETHEUS_FROM_SRC=${{matrix.build-prometheus-from-source}} -DCCT_ENABLE_ASAN=OFF -GNinja + run: cmake -S . -B build ${{matrix.prometheus-options}} -DCCT_BUILD_PROMETHEUS_FROM_SRC=${{matrix.build-special-from-source}} -DCCT_ENABLE_PROTO=${{matrix.build-special-from-source}} -DCCT_ENABLE_ASAN=OFF -GNinja + env: + CXX: ${{matrix.compiler}} + CMAKE_BUILD_TYPE: ${{matrix.buildmode}} - name: Build working-directory: ${{github.workspace}}/build diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 3de55c48..ba3d2578 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -19,7 +19,7 @@ jobs: - name: Install dependencies run: | - vcpkg install curl + vcpkg install curl protobuf - name: End vcpkg install run: | @@ -27,7 +27,10 @@ jobs: - name: Configure CMake run: | - cmake -DCMAKE_TOOLCHAIN_FILE="C:/vcpkg/scripts/buildsystems/vcpkg.cmake" -DCMAKE_BUILD_TYPE=${{matrix.buildmode}} -S . -B build + cmake -S . -B build + env: + CMAKE_BUILD_TYPE: ${{matrix.buildmode}} + CMAKE_TOOLCHAIN_FILE: "C:/vcpkg/scripts/buildsystems/vcpkg.cmake" - name: Build working-directory: ${{github.workspace}}/build diff --git a/.gitignore b/.gitignore index 176b2be7..e8db4630 100644 --- a/.gitignore +++ b/.gitignore @@ -64,6 +64,7 @@ data/cache data/log data/secret !data/secret/secret_test.json +data/serialized data/static/exchangeconfig.json data/static/generalconfig.json monitoring/data/grafana/* diff --git a/CMakeLists.txt b/CMakeLists.txt index c6f83236..3b735119 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,6 +38,7 @@ option(CCT_ENABLE_TESTS "Build the unit tests" ${MAIN_PROJECT}) option(CCT_BUILD_EXEC "Build an executable instead of a static library" ${MAIN_PROJECT}) option(CCT_ENABLE_ASAN "Compile with AddressSanitizer" ${CCT_ASAN_BUILD}) option(CCT_ENABLE_CLANG_TIDY "Compile with clang-tidy checks" OFF) +option(CCT_ENABLE_PROTO "Compile with protobuf support (to export data to the outside world)" ON) option(CCT_BUILD_PROMETHEUS_FROM_SRC "Fetch and build from prometheus-cpp sources" OFF) set(CCT_DATA_DIR "${CMAKE_CURRENT_SOURCE_DIR}/data" CACHE PATH "Needed data directory for coincenter. Can also be overriden at runtime with this environment variable") @@ -141,6 +142,31 @@ if(NOT spdlog_FOUND) FetchContent_MakeAvailable(spdlog) endif() +# protobuf - serialization / deserialization library +if(CCT_ENABLE_PROTO) + find_package(Protobuf CONFIG) + if(Protobuf_FOUND) + message(STATUS "Linking with protobuf ${protobuf_VERSION}") + else() + set(PROTOBUF_VERSION v26.1) + + message(STATUS "Compiling protobuf ${PROTOBUF_VERSION} from sources") + + set(protobuf_BUILD_TESTS OFF) + set(ABSL_PROPAGATE_CXX_STD ON) + + FetchContent_Declare( + protobuf + GIT_REPOSITORY https://github.com/protocolbuffers/protobuf.git + GIT_TAG ${PROTOBUF_VERSION} + ) + FetchContent_MakeAvailable(protobuf) + + include(${protobuf_SOURCE_DIR}/cmake/protobuf-generate.cmake) + + endif() +endif() + # Unit Tests include(cmake/AddUnitTest.cmake) @@ -179,13 +205,20 @@ if(CCT_ENABLE_PROMETHEUS) add_compile_definitions(CCT_ENABLE_PROMETHEUS) endif() +if(CCT_ENABLE_PROTO) + add_compile_definitions(CCT_ENABLE_PROTO) + add_compile_definitions("CCT_PROTOBUF_VERSION=\"${PROTOBUF_VERSION}\"") +endif() + # Link to sub folders CMakeLists.txt, from the lowest level to the highest level for documentation # (beware of cyclic dependencies) add_subdirectory(src/tech) add_subdirectory(src/monitoring) add_subdirectory(src/http-request) add_subdirectory(src/objects) +add_subdirectory(src/serialization) add_subdirectory(src/api-objects) +add_subdirectory(src/trading) add_subdirectory(src/api) add_subdirectory(src/engine) add_subdirectory(src/main) diff --git a/CONFIG.md b/CONFIG.md index f7302a28..02f1ca4b 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -48,17 +48,18 @@ Configures the logging, tracking activity of relevant commands, and console outp #### General options description -| Name | Value | Description | -| ---------------------------------------------- | ---------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **apiOutputType** | String among {`off`, `table`, `json`} | Configure the default output type of coincenter (can be overridden by command line)queries | -| **fiatConversion.rate** | Duration string (ex: `8h`) | Minimum duration between two consecutive requests of the same fiat conversion | -| **log.activityTracking.commandTypes** | Array of strings (ex: `["Buy", "Sell"]`) | Array of command types whose output will be stored to activity history files. | -| **log.activityTracking.dateFileNameFormat** | String (ex: `%Y-%m` for month split) | Defines the date string format suffix used by activity history files. The string should be compatible with [std::strftime](https://en.cppreference.com/w/cpp/chrono/c/strftime). Old data will never be clean-up by `coincenter` (as it may contain important data). User should manage the clean-up / storage. | -| **log.consoleLevel** | String | Defines the log level for standard output. Can be {'off', 'critical', 'error', 'warning', 'info', 'debug', 'trace'} | -| **log.fileLevel** | String | Defines the log level in files. Can be {'off', 'critical', 'error', 'warning', 'info', 'debug', 'trace'} | -| **log.maxFileSize** | String (ex: `5Mi` for 5 Megabytes) | Defines in bytes the maximum logging file size. A string representation of an integral, possibly with one suffix ending such as k, M, G, T (1k multipliers) or Ki, Mi, Gi, Ti (1024 multipliers) are supported. | -| **log.maxNbFiles** | Integer | Number of maximum rotating files for log in files | -| **requests.concurrency.nbMaxParallelRequests** | Integer | Size of the thread pool that makes exchange requests. | +| Name | Value | Description | +| -------------------------------------------------------- | ---------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **apiOutputType** | String among {`off`, `table`, `json`} | Configure the default output type of coincenter (can be overridden by command line)queries | +| **fiatConversion.rate** | Duration string (ex: `8h`) | Minimum duration between two consecutive requests of the same fiat conversion | +| **log.activityTracking.commandTypes** | Array of strings (ex: `["Buy", "Sell"]`) | Array of command types whose output will be stored to activity history files. | +| **log.activityTracking.dateFileNameFormat** | String (ex: `%Y-%m` for month split) | Defines the date string format suffix used by activity history files. The string should be compatible with [std::strftime](https://en.cppreference.com/w/cpp/chrono/c/strftime). Old data will never be clean-up by `coincenter` (as it may contain important data). User should manage the clean-up / storage. | +| **log.consoleLevel** | String | Defines the log level for standard output. Can be {'off', 'critical', 'error', 'warning', 'info', 'debug', 'trace'} | +| **log.fileLevel** | String | Defines the log level in files. Can be {'off', 'critical', 'error', 'warning', 'info', 'debug', 'trace'} | +| **log.maxFileSize** | String (ex: `5Mi` for 5 Megabytes) | Defines in bytes the maximum logging file size. A string representation of an integral, possibly with one suffix ending such as k, M, G, T (1k multipliers) or Ki, Mi, Gi, Ti (1024 multipliers) are supported. | +| **log.maxNbFiles** | Integer | Number of maximum rotating files for log in files | +| **requests.concurrency.nbMaxParallelRequests** | Integer | Size of the thread pool that makes exchange requests. | +| **trading.automation.deserialization.loadChunkDuration** | Duration string (ex: `1w`) | Time window duration of historic stored data loaded and replayed at once given to the trading engine | ### static/exchangeconfig.json @@ -151,6 +152,7 @@ Refer to the hardcoded default json example as a model in case of doubt. | *query* | **updateFrequency.depositWallet** | Duration string (ex: `1min`) | Minimum duration between two consecutive requests of deposit information (including wallet) | | *query* | **updateFrequency.currencyInfo** | Duration string (ex: `4h`) | Minimum duration between two consecutive requests of dynamic currency info retrieval on Bithumb only (used for place order) | | *query* | **placeSimulateRealOrder** | Boolean (`true` or `false`) | If `true`, in trade simulation mode (with `--sim`) exchanges which do not support simulated mode in place order will actually place a real order, with the following characteristics: This will allow place of a 'real' order that cannot be matched in practice (if it is, lucky you!) | +| *query* | **marketDataSerialization** | Boolean (`true` or `false`) | If `true` and `coincenter` is compiled with **protobuf** support, some market data will automatically be exported in the `data/serialization` directory (`orderbook` and `last-trades`) for a long term storage | | *query* | **multiTradeAllowedByDefault** | Boolean (`true` or `false`) | If `true`, [multi-trade](README.md#multi-trade) will be allowed by default for `trade`, `buy` and `sell`. It can be overridden at command line level with `--no-multi-trade` and `--multi-trade`. | | *query* | **validateApiKey** | Boolean (`true` or `false`) | If `true`, each loaded private key will be tested at start of the program. In case of a failure, it will be removed from the list of private accounts loaded by `coincenter`, so that later queries do not consider it instead of raising a runtime exception. The downside is that it will make an additional check that will make startup slower. | | | *tradefees* | **maker** | String as decimal number representing a percentage (for instance, "0.15") | Trade fees occurring when a maker order is matched | diff --git a/Dockerfile b/Dockerfile index 91feb38d..dd043f0c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,7 @@ ARG BUILD_MODE=Release ARG BUILD_TEST=0 ARG BUILD_ASAN=0 ARG BUILD_WITH_PROMETHEUS=1 +ARG BUILD_WITH_PROTOBUF=1 # Install base & build dependencies, needed certificates for curl to work with https RUN apt update && \ @@ -41,6 +42,7 @@ RUN cmake -DCMAKE_BUILD_TYPE=${BUILD_MODE} \ -DCCT_ENABLE_TESTS=${BUILD_TEST} \ -DCCT_ENABLE_ASAN=${BUILD_ASAN} \ -DCCT_BUILD_PROMETHEUS_FROM_SRC=${BUILD_WITH_PROMETHEUS} \ + -DCCT_ENABLE_PROTO=${BUILD_WITH_PROTOBUF} \ -GNinja .. # Build diff --git a/INSTALL.md b/INSTALL.md index a9bbd9d3..69b12b85 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -135,6 +135,7 @@ The minimum tested version is cmake `3.15`, but it's recommended that you use th | `CCT_BUILD_EXEC` | `ON` if main project | Build an executable instead of a static library | | `CCT_ENABLE_ASAN` | `ON` if Debug mode | Compile with AddressSanitizer | | `CCT_ENABLE_CLANG_TIDY` | `ON` if Debug mode and `clang-tidy` is found in `PATH` | Compile with clang-tidy checks | +| `CCT_ENABLE_PROTO` | `ON` | Compile with protobuf support | Example on Linux: to compile it in `Release` mode and `ninja` generator diff --git a/README.md b/README.md index 9cce03c7..7aff305b 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,16 @@ Main features: - Cancel opened orders - Withdraw (with check at destination that funds are well received) - Dust sweeper - + +## Market data storage & replay + +`coincenter` is able to store the following market data in serialized [binary protobuf](https://protobuf.dev/) format for offline trading algorithm replay: + +- Market order book +- Public trades + +Refer to the dedicated [documentation page](TRADING.md) for more information. + ## Supported exchanges | Exchange | Link | @@ -56,6 +65,7 @@ Main features: - [coincenter](#coincenter) - [Market Data](#market-data) - [Account requests](#account-requests) + - [Market data storage \& replay](#market-data-storage--replay) - [Supported exchanges](#supported-exchanges) - [About](#about) - [Installation](#installation) @@ -1285,4 +1295,4 @@ Possible output: | kucoin | 6090943.32410022531 SHIB | | upbit | 6084383.631243834 SHIB | +----------+------------------------------+ -``` \ No newline at end of file +``` diff --git a/TRADING.md b/TRADING.md new file mode 100644 index 00000000..017712ae --- /dev/null +++ b/TRADING.md @@ -0,0 +1,108 @@ +# Trading + +`coincenter` is able to serialize market data and deserialize it later on for future usage. + +## Overview + +Currently, these two sources of data are serializable into [protobuf](https://protobuf.dev/) objects: + +- Market order book +- Public trades + +## Configuration + +### Compilation + +By default, `coincenter` will be built with **protobuf** support (controlled by `cmake` flag `CCT_ENABLE_PROTO` that defaults to `ON`). + +It will try to link to a known installation of **protobuf** if found on current system (oldest tested version is v25, make sure to use this version at least), otherwise it will download and compile it from sources. + +### Serialization configuration + +To be able to serialize market data on disk, make sure that you set the **marketDataSerialization** variable to `true` in `exchangeconfig.json` for the exchanges you would like to interact with. +See the [exchange configuration part](CONFIG.md#exchanges-options-description) for more information about how to configure it. + +## Serialization of market data + +The data will be organized by exchange, then market (asset pair, for instance `BTC-USD`), and finally dates (with directories from **year**, **month**, and finally **day** and files as **hours**). + +All will be stored in `coincenter` data directory, under `serialized` sub folder. + +Here is an example of the structure of files you will obtain: + +```bash +data/serialized// +├── binance +│   ├── BTC-EUR +│   │   └── 2024 +│   │   └── 01 +│   │   ├── 11 +│   │   │   └── 22:00:00_22:59:59.binpb +│   │   ├── 12 +│   │   │   ├── 08:00:00_08:59:59.binpb +│   │   │   ├── 09:00:00_09:59:59.binpb +│   │   │   └── 11:00:00_11:59:59.binpb +│   │   └── 14 +│   │   ├── 07:00:00_07:59:59.binpb +│   │   └── 08:00:00_08:59:59.binpb +│   ├── ETH-USDT +│   │   └── 2024 +│   │   ├── 01 +│   │   │   ├── 09 +│   │   │   │   ├── 08:00:00_08:59:59.binpb +│   │   │   │   ├── 09:00:00_09:59:59.binpb +│   │   │   │   ├── 10:00:00_10:59:59.binpb +│   │   │   │   ├── 11:00:00_11:59:59.binpb +├── huobi +│   ├── ADA-USDT +│   │   └── 2024 +│   │   └── 02 +│   │   └── 10 +│   │   └── 16:00:00_16:59:59.binpb +│   ├── BTC-EUR +│   │   └── 2024 +│   │   └── 01 +│   │   ├── 11 +│   │   │   └── 22:00:00_22:59:59.binpb +.... +``` + +To retrieve market data, it's possible to either use multi-commands with both `orderbook` and `last-trades` commands stacked together, or you can use the more handy `market-data` option that is basically a combination of the two without the output by default (it has been created only for serialization purposes). + +For instance, to retrieve continuously data and serialize them indefinitely, you can use the following command: + +```bash +coincenter -r --repeat-time 2s --log warning \ + market-data btc-eur,binance \ + market-data eth-usdt,kucoin \ + market-data ada-usdt,huobi \ + market-data btc-eur,kraken +``` + +Note the usage of the `-r` (repeat option) to keep querying the data as long as you leave `coincenter` up. It's a good idea to also limit the number of logs with setting console log level to `warning` but not mandatory. + +With this command running for an extended period of time, you should obtain a list of files like in the above example. + +Stacking `market-data` commands together with different exchanges (like in the above example) will allow `coincenter` to perform the queries in parallel, ensuring optimal frequency of data updates. This optimization may be implemented for other commands in the future, but it's currently supported only for `market-data`. + +### Graceful shutdown + +Data is flushed on the disk at regular intervals (around 10 minutes). If you wish to restart / shutdown `coincenter` with an infinite `repeat` command to store continuously market data, you can send `SIGINT` or `SIGTERM` so that `coincenter` can gracefully stop after current request and flush its remaining data on disk before shutdown. + +## Replaying historic market data + +Of course, serialization is useful only if we re-use the data one day. Being **protobuf**, not only `coincenter` could read them, but also other tools, but `coincenter` is also able to read this data. + +If you want to use a third-party tool to read this data, locate the `.proto` files in the `src` directory for your external program to be able to deserialize the `.binpb` files. + +We will focus here on `coincenter` features concerning this data. + +### BETA - Testing trading algorithms + +`coincenter` embeds a trading simulator engine that is able to be used for any custom trading algorithm that would derive from the interface. + +This trading simulator will read chunks of historic data stored in **protobuf** and inject them in trading algorithms. + +Locate the `AbstractMarketTrader` class and derive it - you need to return a `TraderCommand` for each market order book and a list of last public trades that occurred at this specific point of time. + +TODO: extend this documentation. diff --git a/alpine.Dockerfile b/alpine.Dockerfile index b4a159d4..4e6d9ce2 100644 --- a/alpine.Dockerfile +++ b/alpine.Dockerfile @@ -5,9 +5,10 @@ ARG BUILD_MODE=Release ARG BUILD_TEST=0 ARG BUILD_ASAN=0 ARG BUILD_WITH_PROMETHEUS=1 +ARG BUILD_WITH_PROTOBUF=1 # Install base & build dependencies, needed certificates for curl to work with https -RUN apk add --update --upgrade --no-cache g++ libc-dev openssl-dev curl-dev cmake ninja git ca-certificates +RUN apk add --update --upgrade --no-cache linux-headers g++ libc-dev openssl-dev curl-dev cmake ninja git ca-certificates # Copy source files WORKDIR /app/src @@ -39,6 +40,7 @@ RUN cmake -DCMAKE_BUILD_TYPE=${BUILD_MODE} \ -DCCT_ENABLE_TESTS=${BUILD_TEST} \ -DCCT_ENABLE_ASAN=${BUILD_ASAN} \ -DCCT_BUILD_PROMETHEUS_FROM_SRC=${BUILD_WITH_PROMETHEUS} \ + -DCCT_ENABLE_PROTO=${BUILD_WITH_PROTOBUF} \ -GNinja .. # Build diff --git a/src/api/common/CMakeLists.txt b/src/api/common/CMakeLists.txt index 9464b950..006b03a4 100644 --- a/src/api/common/CMakeLists.txt +++ b/src/api/common/CMakeLists.txt @@ -7,6 +7,7 @@ add_library(coincenter_api-common STATIC ${API_COMMON_SRC}) target_link_libraries(coincenter_api-common PUBLIC coincenter_api-objects) target_link_libraries(coincenter_api-common PUBLIC coincenter_objects) target_link_libraries(coincenter_api-common PUBLIC coincenter_http-request) +target_link_libraries(coincenter_api-common PUBLIC coincenter_serialization) target_link_libraries(coincenter_api-common PRIVATE OpenSSL::SSL) target_include_directories(coincenter_api-common PUBLIC include) diff --git a/src/api/common/include/exchangepublicapi.hpp b/src/api/common/include/exchangepublicapi.hpp index 03c63351..9b2dc4e0 100644 --- a/src/api/common/include/exchangepublicapi.hpp +++ b/src/api/common/include/exchangepublicapi.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -9,15 +10,20 @@ #include "currencyexchangeflatset.hpp" #include "exchangebase.hpp" #include "exchangepublicapitypes.hpp" +#include "market-order-book-vector.hpp" +#include "market-timestamp-set.hpp" #include "market.hpp" #include "marketorderbook.hpp" #include "monetaryamount.hpp" #include "monetaryamountbycurrencyset.hpp" #include "priceoptions.hpp" #include "public-trade-vector.hpp" +#include "time-window.hpp" namespace cct { +class AbstractMarketDataDeserializer; +class AbstractMarketDataSerializer; class CoincenterInfo; class ExchangeConfig; class FiatConverter; @@ -41,7 +47,7 @@ class ExchangePublic : public ExchangeBase { using Fiats = CommonAPI::Fiats; - virtual ~ExchangePublic() = default; + virtual ~ExchangePublic(); /// Check if public exchange is responding to basic health check, return true in this case. /// Exchange that implements the HealthCheck do not need to add a retry mechanism. @@ -98,14 +104,14 @@ class ExchangePublic : public ExchangeBase { /// Retrieve the order book of given market. /// It should be more precise that previous version with possibility to go deeper. - virtual MarketOrderBook queryOrderBook(Market mk, int depth = kDefaultDepth) = 0; + MarketOrderBook getOrderBook(Market mk, int depth = kDefaultDepth); + + /// Retrieve an ordered vector of recent last trades + PublicTradeVector getLastTrades(Market mk, int nbTrades = kNbLastTradesDefault); /// Retrieve the total volume exchange on given market in the last 24 hours. virtual MonetaryAmount queryLast24hVolume(Market mk) = 0; - /// Retrieve an ordered vector of recent last trades - virtual PublicTradeVector queryLastTrades(Market mk, int nbTrades = kNbLastTradesDefault) = 0; - /// Retrieve the last price of given market. virtual MonetaryAmount queryLastPrice(Market mk) = 0; @@ -177,7 +183,22 @@ class ExchangePublic : public ExchangeBase { /// If no data found, return a 0 MonetaryAmount on given currency. MonetaryAmount queryWithdrawalFeeOrZero(CurrencyCode currencyCode); + MarketTimestampSet pullMarketOrderBooksMarkets(TimeWindow timeWindow); + + MarketTimestampSet pullTradeMarkets(TimeWindow timeWindow); + + PublicTradeVector pullTradesForReplay(Market market, TimeWindow timeWindow); + + MarketOrderBookVector pullMarketOrderBooksForReplay(Market market, TimeWindow timeWindow); + protected: + /// Retrieve the order book of given market. + /// It should be more precise that previous version with possibility to go deeper. + virtual MarketOrderBook queryOrderBook(Market mk, int depth = kDefaultDepth) = 0; + + /// Retrieve an ordered vector of recent last trades + virtual PublicTradeVector queryLastTrades(Market mk, int nbTrades = kNbLastTradesDefault) = 0; + friend class ExchangePrivate; ExchangePublic(std::string_view name, FiatConverter &fiatConverter, CommonAPI &commonApi, @@ -189,7 +210,12 @@ class ExchangePublic : public ExchangeBase { CommonAPI &_commonApi; const CoincenterInfo &_coincenterInfo; const ExchangeConfig &_exchangeConfig; + std::unique_ptr _marketDataDeserializerPtr; + std::unique_ptr _marketDataSerializerPtr; std::recursive_mutex _publicRequestsMutex; + + private: + AbstractMarketDataSerializer &getMarketDataSerializer(); }; } // namespace api } // namespace cct diff --git a/src/api/common/src/exchangeprivateapi.cpp b/src/api/common/src/exchangeprivateapi.cpp index e463acfb..8ff7cbc6 100644 --- a/src/api/common/src/exchangeprivateapi.cpp +++ b/src/api/common/src/exchangeprivateapi.cpp @@ -535,7 +535,7 @@ PlaceOrderInfo ExchangePrivate::placeOrderProcess(MonetaryAmount &from, Monetary if (tradeInfo.options.isSimulation() && !isSimulatedOrderSupported()) { if (exchangeConfig().placeSimulateRealOrder()) { log::debug("Place simulate real order - price {} will be overriden", price); - MarketOrderBook marketOrderbook = _exchangePublic.queryOrderBook(mk); + MarketOrderBook marketOrderbook = _exchangePublic.getOrderBook(mk); price = isSell ? marketOrderbook.getHighestTheoreticalPrice() : marketOrderbook.getLowestTheoreticalPrice(); } else { PlaceOrderInfo placeOrderInfo = computeSimulatedMatchedPlacedOrderInfo(volume, price, tradeInfo); diff --git a/src/api/common/src/exchangepublicapi.cpp b/src/api/common/src/exchangepublicapi.cpp index 4a345373..599560b2 100644 --- a/src/api/common/src/exchangepublicapi.cpp +++ b/src/api/common/src/exchangepublicapi.cpp @@ -24,21 +24,44 @@ #include "exchangeconfig.hpp" #include "exchangepublicapitypes.hpp" #include "fiatconverter.hpp" +#include "market-timestamp-set.hpp" #include "market.hpp" #include "marketorderbook.hpp" #include "monetaryamount.hpp" #include "priceoptions.hpp" #include "priceoptionsdef.hpp" +#include "time-window.hpp" +#include "timedef.hpp" #include "unreachable.hpp" +#ifdef CCT_ENABLE_PROTO +#include "proto-market-data-deserializer.hpp" +#include "proto-market-data-serializer.hpp" +#else +#include "dummy-market-data-deserializer.hpp" +#include "dummy-market-data-serializer.hpp" +#endif + namespace cct::api { + +#ifdef CCT_ENABLE_PROTO +using MarketDataDeserializer = ProtoMarketDataDeserializer; +using MarketDataSerializer = ProtoMarketDataSerializer; +#else +using MarketDataDeserializer = DummyMarketDataDeserializer; +using MarketDataSerializer = DummyMarketDataSerializer; +#endif + ExchangePublic::ExchangePublic(std::string_view name, FiatConverter &fiatConverter, CommonAPI &commonApi, const CoincenterInfo &coincenterInfo) : _name(name), _fiatConverter(fiatConverter), _commonApi(commonApi), _coincenterInfo(coincenterInfo), - _exchangeConfig(coincenterInfo.exchangeConfig(name)) {} + _exchangeConfig(coincenterInfo.exchangeConfig(name)), + _marketDataDeserializerPtr(new MarketDataDeserializer(coincenterInfo.dataDir(), name)) {} + +ExchangePublic::~ExchangePublic() = default; std::optional ExchangePublic::convert(MonetaryAmount from, CurrencyCode toCurrency, const MarketsPath &conversionPath, const Fiats &fiats, @@ -278,7 +301,7 @@ ExchangePublic::CurrenciesPath ExchangePublic::findCurrenciesPath(CurrencyCode f std::optional ExchangePublic::computeLimitOrderPrice(Market mk, CurrencyCode fromCurrencyCode, const PriceOptions &priceOptions) { const int depth = priceOptions.isRelativePrice() ? std::abs(priceOptions.relativePrice()) : 1; - return queryOrderBook(mk, depth).computeLimitPrice(fromCurrencyCode, priceOptions); + return getOrderBook(mk, depth).computeLimitPrice(fromCurrencyCode, priceOptions); } std::optional ExchangePublic::computeAvgOrderPrice(Market mk, MonetaryAmount from, @@ -292,7 +315,7 @@ std::optional ExchangePublic::computeAvgOrderPrice(Market mk, Mo } else if (priceOptions.priceStrategy() == PriceStrategy::kTaker) { depth = kDefaultDepth; } - return queryOrderBook(mk, depth).computeAvgPrice(from, priceOptions); + return getOrderBook(mk, depth).computeAvgPrice(from, priceOptions); } std::optional ExchangePublic::RetrieveMarket(CurrencyCode c1, CurrencyCode c2, const MarketSet &markets) { @@ -433,4 +456,67 @@ MonetaryAmount ExchangePublic::queryWithdrawalFeeOrZero(CurrencyCode currencyCod return withdrawFee; } +MarketOrderBook ExchangePublic::getOrderBook(Market mk, int depth) { + std::lock_guard guard(_publicRequestsMutex); + const auto marketOrderBook = queryOrderBook(mk, depth); + + if (_exchangeConfig.withMarketDataSerialization()) { + getMarketDataSerializer().push(marketOrderBook); + } + return marketOrderBook; +} + +/// Retrieve an ordered vector of recent last trades +PublicTradeVector ExchangePublic::getLastTrades(Market mk, int nbTrades) { + std::lock_guard guard(_publicRequestsMutex); + const auto lastTrades = queryLastTrades(mk, nbTrades); + if (_exchangeConfig.withMarketDataSerialization()) { + getMarketDataSerializer().push(mk, lastTrades); + } + return lastTrades; +} + +MarketTimestampSet ExchangePublic::pullMarketOrderBooksMarkets(TimeWindow timeWindow) { + if (_marketDataDeserializerPtr) { + return _marketDataDeserializerPtr->pullMarketOrderBooksMarkets(timeWindow); + } + throw exception("No market data deserializer available to retrieve market order books"); +} + +MarketTimestampSet ExchangePublic::pullTradeMarkets(TimeWindow timeWindow) { + if (_marketDataDeserializerPtr) { + return _marketDataDeserializerPtr->pullTradeMarkets(timeWindow); + } + throw exception("No market data deserializer available to retrieve trade markets"); +} + +PublicTradeVector ExchangePublic::pullTradesForReplay(Market market, TimeWindow timeWindow) { + return _marketDataDeserializerPtr->pullTrades(market, timeWindow); +} + +MarketOrderBookVector ExchangePublic::pullMarketOrderBooksForReplay(Market market, TimeWindow timeWindow) { + return _marketDataDeserializerPtr->pullMarketOrderBooks(market, timeWindow); +} + +AbstractMarketDataSerializer &ExchangePublic::getMarketDataSerializer() { + if (_marketDataSerializerPtr) { + return *_marketDataSerializerPtr; + } + + auto nowTime = Clock::now(); + + // Heuristic: load up to 1 week of data to retrieve the youngest written timestamp. + // This will be used in order not to write duplicate objects at the start of a new program after that a previous + // program run was stopped. + TimeWindow largeTimeWindow{nowTime - std::chrono::weeks{1}, nowTime}; + + MarketTimestampSets marketTimestampSets{pullMarketOrderBooksMarkets(largeTimeWindow), + pullTradeMarkets(largeTimeWindow)}; + + _marketDataSerializerPtr = std::unique_ptr( + new MarketDataSerializer(_coincenterInfo.dataDir(), marketTimestampSets, name())); + + return *_marketDataSerializerPtr; +} + } // namespace cct::api diff --git a/src/api/exchanges/src/binancepublicapi.cpp b/src/api/exchanges/src/binancepublicapi.cpp index ba0c2015..87ddfff5 100644 --- a/src/api/exchanges/src/binancepublicapi.cpp +++ b/src/api/exchanges/src/binancepublicapi.cpp @@ -329,7 +329,7 @@ MonetaryAmount BinancePublic::sanitizePrice(Market mk, MonetaryAmount pri) { MonetaryAmount BinancePublic::computePriceForNotional(Market mk, int avgPriceMins) { if (avgPriceMins == 0) { // price should be the last matched price - PublicTradeVector lastTrades = queryLastTrades(mk, 1); + PublicTradeVector lastTrades = getLastTrades(mk, 1); if (!lastTrades.empty()) { return lastTrades.front().price(); } diff --git a/src/api/exchanges/src/bithumbpublicapi.cpp b/src/api/exchanges/src/bithumbpublicapi.cpp index 7c66fcb7..b5efe371 100644 --- a/src/api/exchanges/src/bithumbpublicapi.cpp +++ b/src/api/exchanges/src/bithumbpublicapi.cpp @@ -141,7 +141,7 @@ std::optional BithumbPublic::queryWithdrawalFee(CurrencyCode cur MonetaryAmount BithumbPublic::queryLastPrice(Market mk) { // Bithumb does not have a REST API endpoint for last price, let's compute it from the orderbook - std::optional avgPrice = queryOrderBook(mk).averagePrice(); + std::optional avgPrice = getOrderBook(mk).averagePrice(); if (!avgPrice) { log::error("Empty order book for {} on {} cannot compute average price", mk, _name); return MonetaryAmount(0, mk.quote()); diff --git a/src/api/exchanges/src/krakenprivateapi.cpp b/src/api/exchanges/src/krakenprivateapi.cpp index a24c4f0d..03f61d5f 100644 --- a/src/api/exchanges/src/krakenprivateapi.cpp +++ b/src/api/exchanges/src/krakenprivateapi.cpp @@ -63,8 +63,8 @@ namespace { enum class KrakenErrorEnum : int8_t { kExpiredOrder, kUnknownWithdrawKey, kUnknownError, kNoError }; template -std::pair PrivateQuery(CurlHandle& curlHandle, const APIKey& apiKey, std::string_view method, - CurlPostDataT&& curlPostData = CurlPostData()) { +auto PrivateQuery(CurlHandle& curlHandle, const APIKey& apiKey, std::string_view method, + CurlPostDataT&& curlPostData = CurlPostData()) { CurlOptions opts(HttpRequestType::kPost, std::forward(curlPostData)); opts.mutableHttpHeaders().emplace_back("API-Key", apiKey.key()); diff --git a/src/api/interface/include/exchange.hpp b/src/api/interface/include/exchange.hpp index a542430f..f6a5b12e 100644 --- a/src/api/interface/include/exchange.hpp +++ b/src/api/interface/include/exchange.hpp @@ -94,12 +94,12 @@ class Exchange { return apiPublic().queryAllApproximatedOrderBooks(depth); } - MarketOrderBook queryOrderBook(Market mk, int depth = ExchangePublic::kDefaultDepth); + MarketOrderBook getOrderBook(Market mk, int depth = ExchangePublic::kDefaultDepth); MonetaryAmount queryLast24hVolume(Market mk) { return apiPublic().queryLast24hVolume(mk); } /// Retrieve an ordered vector of recent last trades - PublicTradeVector queryLastTrades(Market mk, int nbTrades = ExchangePublic::kNbLastTradesDefault); + PublicTradeVector getLastTrades(Market mk, int nbTrades = ExchangePublic::kNbLastTradesDefault); /// Retrieve the last price of given market. MonetaryAmount queryLastPrice(Market mk) { return apiPublic().queryLastPrice(mk); } diff --git a/src/api/interface/src/exchange.cpp b/src/api/interface/src/exchange.cpp index 1763ae7e..f33f7a4c 100644 --- a/src/api/interface/src/exchange.cpp +++ b/src/api/interface/src/exchange.cpp @@ -48,12 +48,10 @@ bool Exchange::canDeposit(CurrencyCode currencyCode, const CurrencyExchangeFlatS return lb->canDeposit(); } -MarketOrderBook Exchange::queryOrderBook(Market mk, int depth) { return apiPublic().queryOrderBook(mk, depth); } +MarketOrderBook Exchange::getOrderBook(Market mk, int depth) { return apiPublic().getOrderBook(mk, depth); } /// Retrieve an ordered vector of recent last trades -PublicTradeVector Exchange::queryLastTrades(Market mk, int nbTrades) { - return apiPublic().queryLastTrades(mk, nbTrades); -} +PublicTradeVector Exchange::getLastTrades(Market mk, int nbTrades) { return apiPublic().getLastTrades(mk, nbTrades); } void Exchange::updateCacheFile() const { apiPublic().updateCacheFile(); diff --git a/src/engine/CMakeLists.txt b/src/engine/CMakeLists.txt index f6191775..791f7987 100644 --- a/src/engine/CMakeLists.txt +++ b/src/engine/CMakeLists.txt @@ -6,6 +6,8 @@ target_link_libraries(coincenter_engine PUBLIC coincenter_api-common) target_link_libraries(coincenter_engine PUBLIC coincenter_api-exchange) target_link_libraries(coincenter_engine PUBLIC coincenter_api-interface) target_link_libraries(coincenter_engine PUBLIC coincenter_objects) +target_link_libraries(coincenter_engine PUBLIC coincenter_trading-algorithms) +target_link_libraries(coincenter_engine PUBLIC coincenter_trading-common) target_include_directories(coincenter_engine PUBLIC include) add_unit_test( @@ -81,6 +83,13 @@ add_unit_test( ../api/common/test/include ) +add_unit_test( + replay-algorithm-name-iterator_test + test/replay-algorithm-name-iterator_test.cpp + LIBRARIES + coincenter_engine +) + add_unit_test( stringoptionparser_test test/stringoptionparser_test.cpp diff --git a/src/engine/include/coincenter-commands-iterator.hpp b/src/engine/include/coincenter-commands-iterator.hpp new file mode 100644 index 00000000..809037dd --- /dev/null +++ b/src/engine/include/coincenter-commands-iterator.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include + +#include "coincentercommand.hpp" + +namespace cct { + +class CoincenterCommandsIterator { + public: + using CoincenterCommandSpan = std::span; + + /// Initializes a new iterator on all coincenter commands. + explicit CoincenterCommandsIterator(CoincenterCommandSpan commands = CoincenterCommandSpan()) noexcept; + + /// Returns 'true' if this iterator has still some command groups. + bool hasNextCommandGroup() const; + + /// Get next grouped commands and advance the iterator. + /// The grouped commands are guaranteed to have same type and make it possible to parallelize requests when possible. + CoincenterCommandSpan nextCommandGroup(); + + private: + CoincenterCommandSpan _commands; + CoincenterCommandSpan::size_type _pos; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/engine/include/coincenter.hpp b/src/engine/include/coincenter.hpp index eecef086..3cc68998 100644 --- a/src/engine/include/coincenter.hpp +++ b/src/engine/include/coincenter.hpp @@ -4,6 +4,8 @@ #include #include "apikeysprovider.hpp" +#include "cct_const.hpp" +#include "cct_fixedcapacityvector.hpp" #include "coincenterinfo.hpp" #include "commonapi.hpp" #include "exchange-names.hpp" @@ -11,14 +13,18 @@ #include "exchangepool.hpp" #include "exchangesorchestrator.hpp" #include "fiatconverter.hpp" +#include "market-trader-engine.hpp" +#include "market.hpp" #include "metricsexporter.hpp" #include "ordersconstraints.hpp" #include "queryresultprinter.hpp" #include "queryresulttypes.hpp" +#include "replay-options.hpp" #include "transferablecommandresult.hpp" namespace cct { +class AbstractMarketTraderFactory; class CoincenterCommand; class CoincenterCommands; class TradeOptions; @@ -30,6 +36,7 @@ class Coincenter { Coincenter(const CoincenterInfo &coincenterInfo, const ExchangeSecretsInfo &exchangeSecretsInfo); + /// Launch given commands and return the number of processed commands. int process(const CoincenterCommands &coincenterCommands); ExchangeHealthCheckStatus healthCheck(ExchangeNameSpan exchangeNames); @@ -50,6 +57,10 @@ class Coincenter { CurrencyCode equiCurrencyCode, std::optional depth = std::nullopt); + /// Query market data without returning it. + /// This method is especially useful for serialization and metric exports. + void queryMarketDataPerExchange(std::span marketPerPublicExchange); + /// Retrieve the last 24h traded volume for exchanges supporting given market. MonetaryAmountPerExchange getLast24hTradedVolumePerExchange(Market mk, ExchangeNameSpan exchangeNames); @@ -132,6 +143,16 @@ class Coincenter { const ExchangeName &toPrivateExchangeName, const WithdrawOptions &withdrawOptions); + /// Retrieves the markets available for replay for exchanges selection that has some data during the last + /// 'replayDuration' time (so within the time frame [now - replayDuration, now]) + MarketTimestampSetsPerExchange getMarketsAvailableForReplay(const ReplayOptions &replayOptions, + ExchangeNameSpan exchangeNames); + + /// Replay all markets for exchanges selection that has some data during the last + /// 'replayDuration' time (so within the time frame [now - replayDuration, now]) + void replay(const AbstractMarketTraderFactory &marketTraderFactory, const ReplayOptions &replayOptions, Market market, + ExchangeNameSpan exchangeNames); + /// Dumps the content of all file caches in data directory to save cURL queries. void updateFileCaches() const; @@ -147,8 +168,23 @@ class Coincenter { const FiatConverter &fiatConverter() const { return _fiatConverter; } private: - TransferableCommandResultVector processCommand( - const CoincenterCommand &cmd, std::span previousTransferableResults); + TransferableCommandResultVector processGroupedCommands( + std::span groupedCommands, + std::span previousTransferableResults); + + using MarketTraderEngineVector = FixedCapacityVector; + + void replayAlgorithm(const AbstractMarketTraderFactory &marketTraderFactory, std::string_view algorithmName, + const ReplayOptions &replayOptions, std::span marketTraderEngines, + const PublicExchangeNameVector &exchangesWithThisMarketData); + + // TODO: may be moved somewhere else? + MarketTraderEngineVector createMarketTraderEngines(const ReplayOptions &replayOptions, Market market, + PublicExchangeNameVector &exchangesWithThisMarketData); + + MarketTradeRangeStatsPerExchange tradingProcess(const ReplayOptions &replayOptions, + std::span marketTraderEngines, + ExchangeNameSpan exchangesWithThisMarketData); const CoincenterInfo &_coincenterInfo; api::CommonAPI _commonAPI; diff --git a/src/engine/include/coincentercommand.hpp b/src/engine/include/coincentercommand.hpp index df683d1f..db0894d1 100644 --- a/src/engine/include/coincentercommand.hpp +++ b/src/engine/include/coincentercommand.hpp @@ -13,6 +13,7 @@ #include "market.hpp" #include "monetaryamount.hpp" #include "ordersconstraints.hpp" +#include "replay-options.hpp" #include "tradeoptions.hpp" #include "withdrawoptions.hpp" #include "withdrawsconstraints.hpp" @@ -44,6 +45,8 @@ class CoincenterCommand { CoincenterCommand& setCur1(CurrencyCode cur1); CoincenterCommand& setCur2(CurrencyCode cur2); + CoincenterCommand& setReplayOptions(ReplayOptions replayOptions); + CoincenterCommand& setPercentageAmount(bool value = true); CoincenterCommand& withBalanceInUse(bool value = true); @@ -74,16 +77,19 @@ class CoincenterCommand { bool isPercentageAmount() const { return _isPercentageAmount; } bool withBalanceInUse() const { return _withBalanceInUse; } + const ReplayOptions& replayOptions() const { return std::get(_specialOptions); } + bool operator==(const CoincenterCommand&) const noexcept = default; using trivially_relocatable = std::bool_constant && is_trivially_relocatable_v && is_trivially_relocatable_v && - is_trivially_relocatable_v && is_trivially_relocatable_v>::type; + is_trivially_relocatable_v && is_trivially_relocatable_v && + is_trivially_relocatable_v>::type; private: - using SpecialOptions = - std::variant; + using SpecialOptions = std::variant; ExchangeNames _exchangeNames; SpecialOptions _specialOptions; diff --git a/src/engine/include/coincenteroptions.hpp b/src/engine/include/coincenteroptions.hpp index a0e19976..6011126d 100644 --- a/src/engine/include/coincenteroptions.hpp +++ b/src/engine/include/coincenteroptions.hpp @@ -9,7 +9,7 @@ #include "coincentercommandtype.hpp" #include "coincenteroptionsdef.hpp" #include "commandlineoption.hpp" -#include "exchangepublicapi.hpp" +#include "replay-options.hpp" #include "timedef.hpp" #include "tradedefinitions.hpp" #include "tradeoptions.hpp" @@ -30,6 +30,8 @@ class CoincenterCmdLineOptions { TradeOptions computeTradeOptions() const; WithdrawOptions computeWithdrawOptions() const; + ReplayOptions computeReplayOptions(Duration dur) const; + std::string_view getDataDir() const { return dataDir.empty() ? SelectDefaultDataDir() : dataDir; } std::pair getTradeArgStr() const; @@ -100,6 +102,13 @@ class CoincenterCmdLineOptions { std::string_view lastTrades; + std::string_view marketData; + + std::optional replay; + std::string_view algorithmNames; + std::string_view market; + std::optional replayMarkets; + CommandLineOptionalInt32 repeats; int32_t monitoringPort = CoincenterCmdLineOptionsDefinitions::kDefaultMonitoringPort; int32_t depth = kUndefinedDepth; @@ -114,6 +123,8 @@ class CoincenterCmdLineOptions { bool version = false; bool useMonitoring = false; bool withBalanceInUse = false; + bool validate = false; + bool validateOnly = false; bool operator==(const CoincenterCmdLineOptions&) const noexcept = default; diff --git a/src/engine/include/coincenteroptionsdef.hpp b/src/engine/include/coincenteroptionsdef.hpp index 98478314..7bfabb3c 100644 --- a/src/engine/include/coincenteroptionsdef.hpp +++ b/src/engine/include/coincenteroptionsdef.hpp @@ -426,6 +426,50 @@ struct CoincenterAllowedOptions : private CoincenterCmdLineOptionsDefinitions { "Prints withdraw fees for matching currency and exchanges.\n" "Currency and exchanges are optional, if specified, output will be filtered to match them."}, &OptValueType::withdrawFees}, + {{{"Automation", 8000}, + "market-data", + "", + "Query last trades and order books of given market without printing the result on screen, for given exchanges " + "if specified.\n" + "This is the equivalent of calling last-trades and order-book but is useful combined with the repeat " + "command to store market data on disk."}, + &OptValueType::marketData}, + {{{"Automation", 8000}, + "replay-markets", + "", + "Print markets available for replay, that is, markets that have some data within the time " + "window {now - duration, now}."}, + &OptValueType::replayMarkets}, + {{{"Automation", 8001}, + "replay", + "", + "Replay algorithms on serialized, historical data." + "\nAll known algorithms will be replayed one by one, on all stored markets that have some data within the time " + "window {now - duration, now}. Use below flags to filter more precisely on which data to replay from."}, + &OptValueType::replay}, + {{{"Automation", 8002}, + "--algorithms", + "", + "Pick specific algorithm names to replay with. Default will replay with all known ones."}, + &OptValueType::algorithmNames}, + {{{"Automation", 8003}, + "--market", + "", + "Only replay for specific market. Default will replay all stored markets."}, + &OptValueType::market}, + {{{"Automation", 8003}, + "--validate", + "", + "Filter invalid data during replay.\nThis is disabled by default, use this option when you suspect that " + "invalid data may be present in the replayed time window."}, + &OptValueType::validate}, + {{{"Automation", 8003}, + "--validate-only", + "", + "Instead of launching replay algorithm, only validates serialized data." + "\nNominal replay will not validate input data to optimize performance, use this option to validate data once " + "and for all."}, + &OptValueType::validateOnly}, {{{"Monitoring", 9000}, "--monitoring", "", diff --git a/src/engine/include/exchangesorchestrator.hpp b/src/engine/include/exchangesorchestrator.hpp index a2c052dd..4498a709 100644 --- a/src/engine/include/exchangesorchestrator.hpp +++ b/src/engine/include/exchangesorchestrator.hpp @@ -6,14 +6,18 @@ #include "exchange-names.hpp" #include "exchangename.hpp" #include "exchangeretriever.hpp" +#include "market-trader-engine.hpp" #include "market.hpp" #include "queryresulttypes.hpp" #include "threadpool.hpp" +#include "time-window.hpp" #include "withdrawoptions.hpp" namespace cct { +class ReplayOptions; class RequestsConfig; + class ExchangesOrchestrator { public: using UniquePublicSelectedExchanges = ExchangeRetriever::UniquePublicSelectedExchanges; @@ -89,6 +93,19 @@ class ExchangesOrchestrator { MonetaryAmountPerExchange getLastPricePerExchange(Market mk, ExchangeNameSpan exchangeNames); + MarketDataPerExchange getMarketDataPerExchange(std::span marketPerPublicExchange, + ExchangeNameSpan exchangeNames); + + MarketTimestampSetsPerExchange pullAvailableMarketsForReplay(TimeWindow timeWindow, ExchangeNameSpan exchangeNames); + + MarketTradeRangeStatsPerExchange traderConsumeRange(const ReplayOptions &replayOptions, TimeWindow subTimeWindow, + std::span marketTraderEngines, + ExchangeNameSpan exchangeNames); + + MarketTradingGlobalResultPerExchange getMarketTraderResultPerExchange( + std::span marketTraderEngines, MarketTradeRangeStatsPerExchange &&tradeRangeStatsPerExchange, + ExchangeNameSpan exchangeNames); + private: ExchangeRetriever _exchangeRetriever; ThreadPool _threadPool; diff --git a/src/engine/include/query-result-type-helpers.hpp b/src/engine/include/query-result-type-helpers.hpp new file mode 100644 index 00000000..ade76b98 --- /dev/null +++ b/src/engine/include/query-result-type-helpers.hpp @@ -0,0 +1,15 @@ +#pragma once + +#include "exchangepublicapitypes.hpp" +#include "market.hpp" +#include "queryresulttypes.hpp" + +namespace cct { + +bool ContainsMarket(Market market, const MarketTimestampSet &marketTimestampSet); + +bool ContainsMarket(Market market, const MarketTimestampSets &marketTimestampSets); + +MarketSet ComputeAllMarkets(const MarketTimestampSetsPerExchange &marketTimestampSetsPerExchange); + +} // namespace cct \ No newline at end of file diff --git a/src/engine/include/queryresultprinter.hpp b/src/engine/include/queryresultprinter.hpp index 7e7edec7..904f16d7 100644 --- a/src/engine/include/queryresultprinter.hpp +++ b/src/engine/include/queryresultprinter.hpp @@ -15,6 +15,7 @@ #include "ordersconstraints.hpp" #include "queryresulttypes.hpp" #include "simpletable.hpp" +#include "time-window.hpp" #include "withdrawsconstraints.hpp" namespace cct { @@ -66,7 +67,7 @@ class QueryResultPrinter { } void printClosedOrders(const ClosedOrdersPerExchange &closedOrdersPerExchange, - const OrdersConstraints &ordersConstraints) const; + const OrdersConstraints &ordersConstraints = OrdersConstraints{}) const; void printOpenedOrders(const OpenedOrdersPerExchange &openedOrdersPerExchange, const OrdersConstraints &ordersConstraints) const; @@ -104,6 +105,13 @@ class QueryResultPrinter { const TradedAmountsVectorWithFinalAmountPerExchange &tradedAmountsVectorWithFinalAmountPerExchange, CurrencyCode currencyCode) const; + void printMarketsForReplay(TimeWindow timeWindow, + const MarketTimestampSetsPerExchange &marketTimestampSetsPerExchange); + + void printMarketTradingResults(TimeWindow timeWindow, + const MarketTradingGlobalResultPerExchange &marketTradingResultPerExchange, + CoincenterCommandType commandType) const; + private: void printTrades(const TradeResultPerExchange &tradeResultPerExchange, MonetaryAmount amount, bool isPercentageTrade, CurrencyCode toCurrency, const TradeOptions &tradeOptions, CoincenterCommandType commandType) const; diff --git a/src/engine/include/queryresulttypes.hpp b/src/engine/include/queryresulttypes.hpp index 60f1028b..5af47573 100644 --- a/src/engine/include/queryresulttypes.hpp +++ b/src/engine/include/queryresulttypes.hpp @@ -13,10 +13,14 @@ #include "currencyexchangeflatset.hpp" #include "exchangeprivateapitypes.hpp" #include "exchangepublicapitypes.hpp" +#include "market-timestamp-set.hpp" +#include "market-trading-global-result.hpp" +#include "market-trading-result.hpp" #include "marketorderbook.hpp" #include "monetaryamount.hpp" #include "monetaryamountbycurrencyset.hpp" #include "public-trade-vector.hpp" +#include "trade-range-stats.hpp" #include "traderesult.hpp" #include "wallet.hpp" #include "withdrawinfo.hpp" @@ -31,6 +35,8 @@ using MarketOrderBookConversionRate = std::tuple; +using MarketPerExchange = FixedCapacityVector, kNbSupportedExchanges>; + using MarketsPerExchange = FixedCapacityVector, kNbSupportedExchanges>; using MonetaryAmountPerExchange = FixedCapacityVector, kNbSupportedExchanges>; @@ -40,6 +46,9 @@ using MonetaryAmountByCurrencySetPerExchange = using TradesPerExchange = FixedCapacityVector, kNbSupportedExchanges>; +using MarketDataPerExchange = + FixedCapacityVector>, kNbSupportedExchanges>; + using TradeResultPerExchange = SmallVector, kTypicalNbPrivateAccounts>; using TradedAmountsVectorWithFinalAmountPerExchange = @@ -68,4 +77,14 @@ using DeliveredWithdrawInfoWithExchanges = std::pair, kTypicalNbPrivateAccounts>; using ConversionPathPerExchange = FixedCapacityVector, kNbSupportedExchanges>; + +using MarketTimestampSetsPerExchange = FixedCapacityVector, kNbSupportedExchanges>; + +using MarketTradeRangeStatsPerExchange = FixedCapacityVector, kNbSupportedExchanges>; + +using MarketTradingResultPerExchange = FixedCapacityVector, kNbSupportedExchanges>; + +using MarketTradingGlobalResultPerExchange = + FixedCapacityVector, kNbSupportedExchanges>; + } // namespace cct diff --git a/src/engine/include/replay-algorithm-name-iterator.hpp b/src/engine/include/replay-algorithm-name-iterator.hpp new file mode 100644 index 00000000..21995286 --- /dev/null +++ b/src/engine/include/replay-algorithm-name-iterator.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include +#include +#include + +namespace cct { + +/// Convenient class to iterate on the algorithm names, comma separated. +/// If 'algorithmNames' is empty, it will loop on all available ones (given by 'allAlgorithms') +class ReplayAlgorithmNameIterator { + public: + ReplayAlgorithmNameIterator(std::string_view algorithmNames, std::span allAlgorithms); + + /// Returns true if and only if there is at least one additional algorithm name to iterate on. + bool hasNext() const; + + /// Get next algorithm name and advance the iterator. + /// Undefined behavior if 'hasNext' is 'false'. + std::string_view next(); + + private: + std::span _allAlgorithms; + std::string_view _algorithmNames; + int32_t _begPos; + int32_t _endPos; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/engine/include/replay-options.hpp b/src/engine/include/replay-options.hpp new file mode 100644 index 00000000..a490e95e --- /dev/null +++ b/src/engine/include/replay-options.hpp @@ -0,0 +1,33 @@ +#pragma once + +#include +#include + +#include "time-window.hpp" + +namespace cct { + +class ReplayOptions { + public: + enum class ReplayMode : int8_t { kValidateOnly, kCheckedLaunchAlgorithm, kUncheckedLaunchAlgorithm }; + + ReplayOptions() noexcept = default; + + /// Algorithm names should be comma separated. Empty string will match all. + ReplayOptions(TimeWindow timeWindow, std::string_view algorithmsNames, ReplayMode replayMode); + + TimeWindow timeWindow() const { return _timeWindow; } + + std::string_view algorithmNames() const; + + ReplayMode replayMode() const { return _replayMode; } + + bool operator==(const ReplayOptions &) const noexcept = default; + + private: + TimeWindow _timeWindow; + std::string_view _algorithmNames; + ReplayMode _replayMode; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/engine/include/stringoptionparser.hpp b/src/engine/include/stringoptionparser.hpp index 58bed922..330d3aeb 100644 --- a/src/engine/include/stringoptionparser.hpp +++ b/src/engine/include/stringoptionparser.hpp @@ -10,6 +10,7 @@ #include "exchange-names.hpp" #include "market.hpp" #include "monetaryamount.hpp" +#include "timedef.hpp" namespace cct { class StringOptionParser { @@ -29,6 +30,10 @@ class StringOptionParser { /// after the currency CurrencyCode parseCurrency(FieldIs fieldIs = FieldIs::kMandatory, char delimiter = ','); + /// If FieldIs is kOptional and there is no duration, kUndefinedDuration duration will be returned. + /// otherwise exception invalid_argument will be raised + Duration parseDuration(FieldIs fieldIs = FieldIs::kMandatory); + /// If FieldIs is kOptional and there is no market, default market will be returned. /// otherwise exception invalid_argument will be raised. /// @param delimiter defines the expected character (could be not present, which means end of parsing) @@ -55,4 +60,4 @@ class StringOptionParser { std::string_view _opt; std::string_view::size_type _pos{}; }; -} // namespace cct \ No newline at end of file +} // namespace cct diff --git a/src/engine/src/coincenter-commands-iterator.cpp b/src/engine/src/coincenter-commands-iterator.cpp new file mode 100644 index 00000000..f0f773bd --- /dev/null +++ b/src/engine/src/coincenter-commands-iterator.cpp @@ -0,0 +1,74 @@ +#include "coincenter-commands-iterator.hpp" + +#include + +#include "cct_const.hpp" +#include "coincentercommandtype.hpp" +#include "exchangename.hpp" + +namespace cct { + +CoincenterCommandsIterator::CoincenterCommandsIterator(CoincenterCommandSpan commands) noexcept + : _commands(commands), _pos() {} + +namespace { +using PublicExchangePresenceBitset = std::bitset; + +bool UpdateBitsetAreNewExchanges(const CoincenterCommand &command, + PublicExchangePresenceBitset &publicExchangePresence) { + if (command.exchangeNames().empty()) { + // All public exchanges used + const auto result = publicExchangePresence.none(); + publicExchangePresence.set(); + return result; + } + for (const ExchangeName &exchangeName : command.exchangeNames()) { + const auto exchangePos = exchangeName.publicExchangePos(); + if (publicExchangePresence[exchangePos]) { + return false; + } + publicExchangePresence.set(exchangePos); + } + return true; +} + +bool CommandTypeCanBeGrouped(CoincenterCommandType type) { + // Compatible command types need to be explicitly set + // For now, only market data is compatible + switch (type) { + case CoincenterCommandType::kMarketData: + return true; + default: + return false; + } +} + +} // namespace + +bool CoincenterCommandsIterator::hasNextCommandGroup() const { return _pos < _commands.size(); } + +CoincenterCommandsIterator::CoincenterCommandSpan CoincenterCommandsIterator::nextCommandGroup() { + CoincenterCommandSpan groupedCommands(_commands.begin() + _pos, 1U); + + if (CommandTypeCanBeGrouped(groupedCommands.front().type())) { + PublicExchangePresenceBitset publicExchangePresence; + UpdateBitsetAreNewExchanges(groupedCommands.front(), publicExchangePresence); + + while (_pos + groupedCommands.size() < _commands.size()) { + const CoincenterCommand &nextCommand = _commands[_pos + groupedCommands.size()]; + if (nextCommand.type() != groupedCommands.front().type()) { + break; + } + if (!UpdateBitsetAreNewExchanges(nextCommand, publicExchangePresence)) { + break; + } + // Add new command to group + groupedCommands = CoincenterCommandSpan(groupedCommands.data(), groupedCommands.size() + 1); + } + } + + _pos += groupedCommands.size(); + return groupedCommands; +} + +} // namespace cct \ No newline at end of file diff --git a/src/engine/src/coincenter.cpp b/src/engine/src/coincenter.cpp index 1a08e76e..0ce5817d 100644 --- a/src/engine/src/coincenter.cpp +++ b/src/engine/src/coincenter.cpp @@ -13,6 +13,7 @@ #include "cct_exception.hpp" #include "cct_invalid_argument_exception.hpp" #include "cct_log.hpp" +#include "coincenter-commands-iterator.hpp" #include "coincentercommand.hpp" #include "coincentercommands.hpp" #include "coincentercommandtype.hpp" @@ -25,11 +26,18 @@ #include "exchangepublicapi.hpp" #include "exchangeretriever.hpp" #include "exchangesecretsinfo.hpp" +#include "market-timestamp-set.hpp" +#include "market-trader-engine.hpp" +#include "market-trader-factory.hpp" #include "market.hpp" #include "monetaryamount.hpp" #include "ordersconstraints.hpp" +#include "query-result-type-helpers.hpp" #include "queryresultprinter.hpp" #include "queryresulttypes.hpp" +#include "replay-algorithm-name-iterator.hpp" +#include "replay-options.hpp" +#include "time-window.hpp" #include "timedef.hpp" #include "transferablecommandresult.hpp" #include "withdrawsconstraints.hpp" @@ -113,35 +121,40 @@ int Coincenter::process(const CoincenterCommands &coincenterCommands) { } } TransferableCommandResultVector transferableResults; - for (const auto &cmd : commands) { - transferableResults = processCommand(cmd, transferableResults); + CoincenterCommandsIterator commandsIterator(commands); + while (commandsIterator.hasNextCommandGroup()) { + const auto groupedCommands = commandsIterator.nextCommandGroup(); + transferableResults = processGroupedCommands(groupedCommands, transferableResults); ++nbCommandsProcessed; } } return nbCommandsProcessed; } -TransferableCommandResultVector Coincenter::processCommand( - const CoincenterCommand &cmd, std::span previousTransferableResults) { +TransferableCommandResultVector Coincenter::processGroupedCommands( + std::span groupedCommands, + std::span previousTransferableResults) { TransferableCommandResultVector transferableResults; - switch (cmd.type()) { + const auto &firstCmd = groupedCommands.front(); + // All grouped commands have same type - logic to handle multiple commands in a group should be handled per use case + switch (firstCmd.type()) { case CoincenterCommandType::kHealthCheck: { - const auto healthCheckStatus = healthCheck(cmd.exchangeNames()); + const auto healthCheckStatus = healthCheck(firstCmd.exchangeNames()); _queryResultPrinter.printHealthCheck(healthCheckStatus); break; } case CoincenterCommandType::kCurrencies: { - const auto currenciesPerExchange = getCurrenciesPerExchange(cmd.exchangeNames()); + const auto currenciesPerExchange = getCurrenciesPerExchange(firstCmd.exchangeNames()); _queryResultPrinter.printCurrencies(currenciesPerExchange); break; } case CoincenterCommandType::kMarkets: { - const auto marketsPerExchange = getMarketsPerExchange(cmd.cur1(), cmd.cur2(), cmd.exchangeNames()); - _queryResultPrinter.printMarkets(cmd.cur1(), cmd.cur2(), marketsPerExchange, cmd.type()); + const auto marketsPerExchange = getMarketsPerExchange(firstCmd.cur1(), firstCmd.cur2(), firstCmd.exchangeNames()); + _queryResultPrinter.printMarkets(firstCmd.cur1(), firstCmd.cur2(), marketsPerExchange, firstCmd.type()); break; } case CoincenterCommandType::kConversion: { - if (cmd.amount().isDefault()) { + if (firstCmd.amount().isDefault()) { std::array startAmountsPerExchangePos; bool oneSet = false; for (const auto &transferableResult : previousTransferableResults) { @@ -158,90 +171,96 @@ TransferableCommandResultVector Coincenter::processCommand( throw invalid_argument("Missing input amount to convert from"); } - const auto conversionPerExchange = getConversion(startAmountsPerExchangePos, cmd.cur1(), cmd.exchangeNames()); - _queryResultPrinter.printConversion(startAmountsPerExchangePos, cmd.cur1(), conversionPerExchange); + const auto conversionPerExchange = + getConversion(startAmountsPerExchangePos, firstCmd.cur1(), firstCmd.exchangeNames()); + _queryResultPrinter.printConversion(startAmountsPerExchangePos, firstCmd.cur1(), conversionPerExchange); FillConversionTransferableCommandResults(conversionPerExchange, transferableResults); } else { - const auto conversionPerExchange = getConversion(cmd.amount(), cmd.cur1(), cmd.exchangeNames()); - _queryResultPrinter.printConversion(cmd.amount(), cmd.cur1(), conversionPerExchange); + const auto conversionPerExchange = getConversion(firstCmd.amount(), firstCmd.cur1(), firstCmd.exchangeNames()); + _queryResultPrinter.printConversion(firstCmd.amount(), firstCmd.cur1(), conversionPerExchange); FillConversionTransferableCommandResults(conversionPerExchange, transferableResults); } break; } case CoincenterCommandType::kConversionPath: { - const auto conversionPathPerExchange = getConversionPaths(cmd.market(), cmd.exchangeNames()); - _queryResultPrinter.printConversionPath(cmd.market(), conversionPathPerExchange); + const auto conversionPathPerExchange = getConversionPaths(firstCmd.market(), firstCmd.exchangeNames()); + _queryResultPrinter.printConversionPath(firstCmd.market(), conversionPathPerExchange); break; } case CoincenterCommandType::kLastPrice: { - const auto lastPricePerExchange = getLastPricePerExchange(cmd.market(), cmd.exchangeNames()); - _queryResultPrinter.printLastPrice(cmd.market(), lastPricePerExchange); + const auto lastPricePerExchange = getLastPricePerExchange(firstCmd.market(), firstCmd.exchangeNames()); + _queryResultPrinter.printLastPrice(firstCmd.market(), lastPricePerExchange); break; } case CoincenterCommandType::kTicker: { - const auto exchangeTickerMaps = getTickerInformation(cmd.exchangeNames()); + const auto exchangeTickerMaps = getTickerInformation(firstCmd.exchangeNames()); _queryResultPrinter.printTickerInformation(exchangeTickerMaps); break; } case CoincenterCommandType::kOrderbook: { const auto marketOrderBooksConversionRates = - getMarketOrderBooks(cmd.market(), cmd.exchangeNames(), cmd.cur1(), cmd.optDepth()); - _queryResultPrinter.printMarketOrderBooks(cmd.market(), cmd.cur1(), cmd.optDepth(), + getMarketOrderBooks(firstCmd.market(), firstCmd.exchangeNames(), firstCmd.cur1(), firstCmd.optDepth()); + _queryResultPrinter.printMarketOrderBooks(firstCmd.market(), firstCmd.cur1(), firstCmd.optDepth(), marketOrderBooksConversionRates); break; } case CoincenterCommandType::kLastTrades: { - const auto lastTradesPerExchange = getLastTradesPerExchange(cmd.market(), cmd.exchangeNames(), cmd.optDepth()); - _queryResultPrinter.printLastTrades(cmd.market(), cmd.optDepth(), lastTradesPerExchange); + const auto lastTradesPerExchange = + getLastTradesPerExchange(firstCmd.market(), firstCmd.exchangeNames(), firstCmd.optDepth()); + _queryResultPrinter.printLastTrades(firstCmd.market(), firstCmd.optDepth(), lastTradesPerExchange); break; } case CoincenterCommandType::kLast24hTradedVolume: { - const auto tradedVolumePerExchange = getLast24hTradedVolumePerExchange(cmd.market(), cmd.exchangeNames()); - _queryResultPrinter.printLast24hTradedVolume(cmd.market(), tradedVolumePerExchange); + const auto tradedVolumePerExchange = + getLast24hTradedVolumePerExchange(firstCmd.market(), firstCmd.exchangeNames()); + _queryResultPrinter.printLast24hTradedVolume(firstCmd.market(), tradedVolumePerExchange); break; } case CoincenterCommandType::kWithdrawFees: { - const auto withdrawFeesPerExchange = getWithdrawFees(cmd.cur1(), cmd.exchangeNames()); - _queryResultPrinter.printWithdrawFees(withdrawFeesPerExchange, cmd.cur1()); + const auto withdrawFeesPerExchange = getWithdrawFees(firstCmd.cur1(), firstCmd.exchangeNames()); + _queryResultPrinter.printWithdrawFees(withdrawFeesPerExchange, firstCmd.cur1()); break; } case CoincenterCommandType::kBalance: { - const auto amountIncludePolicy = cmd.withBalanceInUse() ? BalanceOptions::AmountIncludePolicy::kWithBalanceInUse - : BalanceOptions::AmountIncludePolicy::kOnlyAvailable; - const BalanceOptions balanceOptions(amountIncludePolicy, cmd.cur1()); - const auto balancePerExchange = getBalance(cmd.exchangeNames(), balanceOptions); - _queryResultPrinter.printBalance(balancePerExchange, cmd.cur1()); + const auto amountIncludePolicy = firstCmd.withBalanceInUse() + ? BalanceOptions::AmountIncludePolicy::kWithBalanceInUse + : BalanceOptions::AmountIncludePolicy::kOnlyAvailable; + const BalanceOptions balanceOptions(amountIncludePolicy, firstCmd.cur1()); + const auto balancePerExchange = getBalance(firstCmd.exchangeNames(), balanceOptions); + _queryResultPrinter.printBalance(balancePerExchange, firstCmd.cur1()); break; } case CoincenterCommandType::kDepositInfo: { - const auto walletPerExchange = getDepositInfo(cmd.exchangeNames(), cmd.cur1()); - _queryResultPrinter.printDepositInfo(cmd.cur1(), walletPerExchange); + const auto walletPerExchange = getDepositInfo(firstCmd.exchangeNames(), firstCmd.cur1()); + _queryResultPrinter.printDepositInfo(firstCmd.cur1(), walletPerExchange); break; } case CoincenterCommandType::kOrdersClosed: { - const auto closedOrdersPerExchange = getClosedOrders(cmd.exchangeNames(), cmd.ordersConstraints()); - _queryResultPrinter.printClosedOrders(closedOrdersPerExchange, cmd.ordersConstraints()); + const auto closedOrdersPerExchange = getClosedOrders(firstCmd.exchangeNames(), firstCmd.ordersConstraints()); + _queryResultPrinter.printClosedOrders(closedOrdersPerExchange, firstCmd.ordersConstraints()); break; } case CoincenterCommandType::kOrdersOpened: { - const auto openedOrdersPerExchange = getOpenedOrders(cmd.exchangeNames(), cmd.ordersConstraints()); - _queryResultPrinter.printOpenedOrders(openedOrdersPerExchange, cmd.ordersConstraints()); + const auto openedOrdersPerExchange = getOpenedOrders(firstCmd.exchangeNames(), firstCmd.ordersConstraints()); + _queryResultPrinter.printOpenedOrders(openedOrdersPerExchange, firstCmd.ordersConstraints()); break; } case CoincenterCommandType::kOrdersCancel: { - const auto nbCancelledOrdersPerExchange = cancelOrders(cmd.exchangeNames(), cmd.ordersConstraints()); - _queryResultPrinter.printCancelledOrders(nbCancelledOrdersPerExchange, cmd.ordersConstraints()); + const auto nbCancelledOrdersPerExchange = cancelOrders(firstCmd.exchangeNames(), firstCmd.ordersConstraints()); + _queryResultPrinter.printCancelledOrders(nbCancelledOrdersPerExchange, firstCmd.ordersConstraints()); break; } case CoincenterCommandType::kRecentDeposits: { - const auto depositsPerExchange = getRecentDeposits(cmd.exchangeNames(), cmd.withdrawsOrDepositsConstraints()); - _queryResultPrinter.printRecentDeposits(depositsPerExchange, cmd.withdrawsOrDepositsConstraints()); + const auto depositsPerExchange = + getRecentDeposits(firstCmd.exchangeNames(), firstCmd.withdrawsOrDepositsConstraints()); + _queryResultPrinter.printRecentDeposits(depositsPerExchange, firstCmd.withdrawsOrDepositsConstraints()); break; } case CoincenterCommandType::kRecentWithdraws: { - const auto withdrawsPerExchange = getRecentWithdraws(cmd.exchangeNames(), cmd.withdrawsOrDepositsConstraints()); - _queryResultPrinter.printRecentWithdraws(withdrawsPerExchange, cmd.withdrawsOrDepositsConstraints()); + const auto withdrawsPerExchange = + getRecentWithdraws(firstCmd.exchangeNames(), firstCmd.withdrawsOrDepositsConstraints()); + _queryResultPrinter.printRecentWithdraws(withdrawsPerExchange, firstCmd.withdrawsOrDepositsConstraints()); break; } case CoincenterCommandType::kTrade: { @@ -249,50 +268,83 @@ TransferableCommandResultVector Coincenter::processCommand( // - standard full information with an amount to trade, a destination currency and an optional list of exchanges // where to trade // - a currency - the destination one, and start amount and exchange(s) should come from previous command result - auto [startAmount, exchangeNames] = ComputeTradeAmountAndExchanges(cmd, previousTransferableResults); + auto [startAmount, exchangeNames] = ComputeTradeAmountAndExchanges(firstCmd, previousTransferableResults); if (startAmount.isDefault()) { break; } const auto tradeResultPerExchange = - trade(startAmount, cmd.isPercentageAmount(), cmd.cur1(), exchangeNames, cmd.tradeOptions()); - _queryResultPrinter.printTrades(tradeResultPerExchange, startAmount, cmd.isPercentageAmount(), cmd.cur1(), - cmd.tradeOptions()); + trade(startAmount, firstCmd.isPercentageAmount(), firstCmd.cur1(), exchangeNames, firstCmd.tradeOptions()); + _queryResultPrinter.printTrades(tradeResultPerExchange, startAmount, firstCmd.isPercentageAmount(), + firstCmd.cur1(), firstCmd.tradeOptions()); FillTradeTransferableCommandResults(tradeResultPerExchange, transferableResults); break; } case CoincenterCommandType::kBuy: { - const auto tradeResultPerExchange = smartBuy(cmd.amount(), cmd.exchangeNames(), cmd.tradeOptions()); - _queryResultPrinter.printBuyTrades(tradeResultPerExchange, cmd.amount(), cmd.tradeOptions()); + const auto tradeResultPerExchange = + smartBuy(firstCmd.amount(), firstCmd.exchangeNames(), firstCmd.tradeOptions()); + _queryResultPrinter.printBuyTrades(tradeResultPerExchange, firstCmd.amount(), firstCmd.tradeOptions()); FillTradeTransferableCommandResults(tradeResultPerExchange, transferableResults); break; } case CoincenterCommandType::kSell: { - auto [startAmount, exchangeNames] = ComputeTradeAmountAndExchanges(cmd, previousTransferableResults); + auto [startAmount, exchangeNames] = ComputeTradeAmountAndExchanges(firstCmd, previousTransferableResults); if (startAmount.isDefault()) { break; } const auto tradeResultPerExchange = - smartSell(startAmount, cmd.isPercentageAmount(), exchangeNames, cmd.tradeOptions()); - _queryResultPrinter.printSellTrades(tradeResultPerExchange, cmd.amount(), cmd.isPercentageAmount(), - cmd.tradeOptions()); + smartSell(startAmount, firstCmd.isPercentageAmount(), exchangeNames, firstCmd.tradeOptions()); + _queryResultPrinter.printSellTrades(tradeResultPerExchange, firstCmd.amount(), firstCmd.isPercentageAmount(), + firstCmd.tradeOptions()); FillTradeTransferableCommandResults(tradeResultPerExchange, transferableResults); break; } case CoincenterCommandType::kWithdrawApply: { - const auto [grossAmount, exchangeName] = ComputeWithdrawAmount(cmd, previousTransferableResults); + const auto [grossAmount, exchangeName] = ComputeWithdrawAmount(firstCmd, previousTransferableResults); if (grossAmount.isDefault()) { break; } - const auto deliveredWithdrawInfoWithExchanges = withdraw(grossAmount, cmd.isPercentageAmount(), exchangeName, - cmd.exchangeNames().back(), cmd.withdrawOptions()); - _queryResultPrinter.printWithdraw(deliveredWithdrawInfoWithExchanges, cmd.isPercentageAmount(), - cmd.withdrawOptions()); + const auto deliveredWithdrawInfoWithExchanges = + withdraw(grossAmount, firstCmd.isPercentageAmount(), exchangeName, firstCmd.exchangeNames().back(), + firstCmd.withdrawOptions()); + _queryResultPrinter.printWithdraw(deliveredWithdrawInfoWithExchanges, firstCmd.isPercentageAmount(), + firstCmd.withdrawOptions()); transferableResults.emplace_back(deliveredWithdrawInfoWithExchanges.first[1]->createExchangeName(), deliveredWithdrawInfoWithExchanges.second.receivedAmount()); break; } case CoincenterCommandType::kDustSweeper: { - _queryResultPrinter.printDustSweeper(dustSweeper(cmd.exchangeNames(), cmd.cur1()), cmd.cur1()); + _queryResultPrinter.printDustSweeper(dustSweeper(firstCmd.exchangeNames(), firstCmd.cur1()), firstCmd.cur1()); + break; + } + case CoincenterCommandType::kMarketData: { + std::array marketPerPublicExchange; + for (const auto &cmd : groupedCommands) { + if (cmd.exchangeNames().empty()) { + std::ranges::fill(marketPerPublicExchange, cmd.market()); + } else { + for (const auto &exchangeName : cmd.exchangeNames()) { + marketPerPublicExchange[exchangeName.publicExchangePos()] = cmd.market(); + } + } + } + // No return value here, this command is made only for storing purposes. + queryMarketDataPerExchange(marketPerPublicExchange); + break; + } + case CoincenterCommandType::kReplay: { + /// This implementation of AbstractMarketTraderFactory is only provided as an example. + /// You can extend coincenter library and: + /// - Provide your own algorithms by implementing your own MarketTraderFactory will all your algorithms. + /// - Create your own CommandType that will call coincenter.replay with the same parameters as below, with your + /// own MarketTraderFactory. + MarketTraderFactory marketTraderFactory; + replay(marketTraderFactory, firstCmd.replayOptions(), firstCmd.market(), firstCmd.exchangeNames()); + break; + } + case CoincenterCommandType::kReplayMarkets: { + const auto marketTimestampSetsPerExchange = + getMarketsAvailableForReplay(firstCmd.replayOptions(), firstCmd.exchangeNames()); + _queryResultPrinter.printMarketsForReplay(firstCmd.replayOptions().timeWindow(), marketTimestampSetsPerExchange); break; } default: @@ -327,6 +379,40 @@ MarketOrderBookConversionRates Coincenter::getMarketOrderBooks(Market mk, Exchan return ret; } +void Coincenter::queryMarketDataPerExchange(std::span marketPerPublicExchange) { + ExchangeNames exchangeNames; + + int exchangePos{}; + for (Market market : marketPerPublicExchange) { + if (market.isDefined()) { + exchangeNames.emplace_back(kSupportedExchanges[exchangePos]); + } + ++exchangePos; + } + + const auto marketDataPerExchange = + _exchangesOrchestrator.getMarketDataPerExchange(marketPerPublicExchange, exchangeNames); + + // Transform data structures to export metrics input format + MarketOrderBookConversionRates marketOrderBookConversionRates(marketDataPerExchange.size()); + TradesPerExchange lastTradesPerExchange(marketDataPerExchange.size()); + + std::ranges::transform(marketDataPerExchange, marketOrderBookConversionRates.begin(), + [](const auto &exchangeWithPairOrderBooksAndTrades) { + return std::make_tuple(exchangeWithPairOrderBooksAndTrades.first->name(), + exchangeWithPairOrderBooksAndTrades.second.first, std::nullopt); + }); + + std::ranges::transform(marketDataPerExchange, lastTradesPerExchange.begin(), + [](const auto &exchangeWithPairOrderBooksAndTrades) { + return std::make_pair(exchangeWithPairOrderBooksAndTrades.first, + exchangeWithPairOrderBooksAndTrades.second.second); + }); + + _metricsExporter.exportOrderbookMetrics(marketOrderBookConversionRates); + _metricsExporter.exportLastTradesMetrics(lastTradesPerExchange); +} + BalancePerExchange Coincenter::getBalance(std::span privateExchangeNames, const BalanceOptions &balanceOptions) { CurrencyCode equiCurrency = balanceOptions.equiCurrency(); @@ -459,6 +545,198 @@ MonetaryAmountPerExchange Coincenter::getLastPricePerExchange(Market mk, Exchang return _exchangesOrchestrator.getLastPricePerExchange(mk, exchangeNames); } +MarketTimestampSetsPerExchange Coincenter::getMarketsAvailableForReplay(const ReplayOptions &replayOptions, + ExchangeNameSpan exchangeNames) { + return _exchangesOrchestrator.pullAvailableMarketsForReplay(replayOptions.timeWindow(), exchangeNames); +} + +namespace { +auto CreateExchangeNameVector(Market market, const MarketTimestampSetsPerExchange &marketTimestampSetsPerExchange) { + PublicExchangeNameVector exchangesWithThisMarketData; + for (const auto &[exchange, marketTimestampSets] : marketTimestampSetsPerExchange) { + if (ContainsMarket(market, marketTimestampSets)) { + exchangesWithThisMarketData.emplace_back(exchange->name()); + } + } + return exchangesWithThisMarketData; +} + +void CreateAndRegisterTraderAlgorithms(const AbstractMarketTraderFactory &marketTraderFactory, + std::string_view algorithmName, + std::span marketTraderEngines) { + for (auto &marketTraderEngine : marketTraderEngines) { + const auto &marketTraderEngineState = marketTraderEngine.marketTraderEngineState(); + + marketTraderEngine.registerMarketTrader(marketTraderFactory.construct(algorithmName, marketTraderEngineState)); + } +} + +bool Filter(Market market, MarketTimestampSet &marketTimestampSet) { + auto it = std::partition_point(marketTimestampSet.begin(), marketTimestampSet.end(), + [market](const auto &marketTimestamp) { return marketTimestamp.market < market; }); + if (it != marketTimestampSet.end() && it->market == market) { + auto marketTimestamp = *it; + marketTimestampSet.clear(); + marketTimestampSet.insert(marketTimestamp); + return false; + } + + marketTimestampSet.clear(); + return true; +} + +void Filter(Market market, MarketTimestampSetsPerExchange &marketTimestampSetsPerExchange) { + for (auto it = marketTimestampSetsPerExchange.begin(); it != marketTimestampSetsPerExchange.end();) { + const bool orderBooksEmpty = Filter(market, it->second.orderBooksMarkets); + const bool tradesEmpty = Filter(market, it->second.tradesMarkets); + + if (orderBooksEmpty && tradesEmpty) { + // no more data, remove the exchange entry completely + it = marketTimestampSetsPerExchange.erase(it); + } else { + ++it; + } + } +} + +} // namespace + +void Coincenter::replay(const AbstractMarketTraderFactory &marketTraderFactory, const ReplayOptions &replayOptions, + Market market, ExchangeNameSpan exchangeNames) { + const TimeWindow timeWindow = replayOptions.timeWindow(); + auto marketTimestampSetsPerExchange = _exchangesOrchestrator.pullAvailableMarketsForReplay(timeWindow, exchangeNames); + + if (market.isDefined()) { + Filter(market, marketTimestampSetsPerExchange); + } + + MarketSet allMarkets = ComputeAllMarkets(marketTimestampSetsPerExchange); + + ReplayAlgorithmNameIterator replayAlgorithmNameIterator(replayOptions.algorithmNames(), + marketTraderFactory.allSupportedAlgorithms()); + + while (replayAlgorithmNameIterator.hasNext()) { + std::string_view algorithmName = replayAlgorithmNameIterator.next(); + + for (const Market replayMarket : allMarkets) { + auto exchangesWithThisMarketData = CreateExchangeNameVector(replayMarket, marketTimestampSetsPerExchange); + + // Create the MarketTraderEngines based on this market, filtering out exchanges without available amount to + // trade + MarketTraderEngineVector marketTraderEngines = + createMarketTraderEngines(replayOptions, replayMarket, exchangesWithThisMarketData); + + replayAlgorithm(marketTraderFactory, algorithmName, replayOptions, marketTraderEngines, + exchangesWithThisMarketData); + } + } +} + +void Coincenter::replayAlgorithm(const AbstractMarketTraderFactory &marketTraderFactory, std::string_view algorithmName, + const ReplayOptions &replayOptions, std::span marketTraderEngines, + const PublicExchangeNameVector &exchangesWithThisMarketData) { + CreateAndRegisterTraderAlgorithms(marketTraderFactory, algorithmName, marketTraderEngines); + + MarketTradeRangeStatsPerExchange tradeRangeStatsPerExchange = + tradingProcess(replayOptions, marketTraderEngines, exchangesWithThisMarketData); + + // Finally retrieve and print results for this market + MarketTradingGlobalResultPerExchange marketTradingResultPerExchange = + _exchangesOrchestrator.getMarketTraderResultPerExchange( + marketTraderEngines, std::move(tradeRangeStatsPerExchange), exchangesWithThisMarketData); + + _queryResultPrinter.printMarketTradingResults(replayOptions.timeWindow(), marketTradingResultPerExchange, + CoincenterCommandType::kReplay); +} + +Coincenter::MarketTraderEngineVector Coincenter::createMarketTraderEngines( + const ReplayOptions &replayOptions, Market market, PublicExchangeNameVector &exchangesWithThisMarketData) { + auto nbExchanges = exchangesWithThisMarketData.size(); + + const auto &automationConfig = _coincenterInfo.generalConfig().tradingConfig().automationConfig(); + const auto startBaseAmountEquivalent = automationConfig.startBaseAmountEquivalent(); + const auto startQuoteAmountEquivalent = automationConfig.startQuoteAmountEquivalent(); + const bool isValidateOnly = replayOptions.replayMode() == ReplayOptions::ReplayMode::kValidateOnly; + + auto convertedBaseAmountPerExchange = + isValidateOnly ? MonetaryAmountPerExchange{} + : getConversion(startBaseAmountEquivalent, market.base(), exchangesWithThisMarketData); + auto convertedQuoteAmountPerExchange = + isValidateOnly ? MonetaryAmountPerExchange{} + : getConversion(startQuoteAmountEquivalent, market.quote(), exchangesWithThisMarketData); + + MarketTraderEngineVector marketTraderEngines; + for (decltype(nbExchanges) exchangePos = 0; exchangePos < nbExchanges; ++exchangePos) { + MonetaryAmount startBaseAmount = + isValidateOnly ? MonetaryAmount{0, market.base()} : convertedBaseAmountPerExchange[exchangePos].second; + MonetaryAmount startQuoteAmount = + isValidateOnly ? MonetaryAmount{0, market.quote()} : convertedQuoteAmountPerExchange[exchangePos].second; + + if (startBaseAmount.currencyCode() != market.base()) { + // This is possible as conversion may use equivalent fiats and stable coins + log::info("Target converted currency is different from market one, replace with market currency {} -> {}", + startBaseAmount.currencyCode(), market.base()); + startBaseAmount = MonetaryAmount(startBaseAmount.amount(), market.base(), startBaseAmount.nbDecimals()); + } + if (startQuoteAmount.currencyCode() != market.quote()) { + // This is possible as conversion may use equivalent fiats and stable coins + log::info("Target converted currency is different from market one, replace with market currency {} -> {}", + startQuoteAmount.currencyCode(), market.quote()); + startQuoteAmount = MonetaryAmount(startQuoteAmount.amount(), market.quote(), startQuoteAmount.nbDecimals()); + } + + if (!isValidateOnly && (startBaseAmount == 0 || startQuoteAmount == 0)) { + log::warn("Cannot convert to start base / quote amounts for {} ({} / {})", + exchangesWithThisMarketData[exchangePos], startBaseAmount, startQuoteAmount); + exchangesWithThisMarketData.erase(exchangesWithThisMarketData.begin() + exchangePos); + convertedBaseAmountPerExchange.erase(convertedBaseAmountPerExchange.begin() + exchangePos); + convertedQuoteAmountPerExchange.erase(convertedQuoteAmountPerExchange.begin() + exchangePos); + --exchangePos; + --nbExchanges; + continue; + } + + const ExchangeConfig &exchangeConfig = + _coincenterInfo.exchangeConfig(exchangesWithThisMarketData[exchangePos].name()); + + marketTraderEngines.emplace_back(exchangeConfig, market, startBaseAmount, startQuoteAmount); + } + return marketTraderEngines; +} + +MarketTradeRangeStatsPerExchange Coincenter::tradingProcess(const ReplayOptions &replayOptions, + std::span marketTraderEngines, + ExchangeNameSpan exchangesWithThisMarketData) { + const auto &automationConfig = _coincenterInfo.generalConfig().tradingConfig().automationConfig(); + const auto loadChunkDuration = automationConfig.loadChunkDuration(); + const auto timeWindow = replayOptions.timeWindow(); + + MarketTradeRangeStatsPerExchange tradeRangeResultsPerExchange; + + // Main loop - parallelized by exchange, with time window chunks of loadChunkDuration + + TimeWindow subTimeWindow(timeWindow.from(), loadChunkDuration); + while (subTimeWindow.overlaps(timeWindow)) { + auto subRangeResultsPerExchange = _exchangesOrchestrator.traderConsumeRange( + replayOptions, subTimeWindow, marketTraderEngines, exchangesWithThisMarketData); + + if (tradeRangeResultsPerExchange.empty()) { + tradeRangeResultsPerExchange = std::move(subRangeResultsPerExchange); + } else { + int pos{}; + for (auto &[exchange, result] : subRangeResultsPerExchange) { + tradeRangeResultsPerExchange[pos].second += result; + ++pos; + } + } + + // Go to next sub time window + subTimeWindow = TimeWindow(subTimeWindow.to(), loadChunkDuration); + } + + return tradeRangeResultsPerExchange; +} + void Coincenter::updateFileCaches() const { log::debug("Store all cache files"); diff --git a/src/engine/src/coincentercommand.cpp b/src/engine/src/coincentercommand.cpp index aa80f1e2..1ac776b2 100644 --- a/src/engine/src/coincentercommand.cpp +++ b/src/engine/src/coincentercommand.cpp @@ -11,6 +11,7 @@ #include "market.hpp" #include "monetaryamount.hpp" #include "ordersconstraints.hpp" +#include "timedef.hpp" #include "tradeoptions.hpp" #include "withdrawoptions.hpp" #include "withdrawsconstraints.hpp" @@ -124,4 +125,10 @@ CoincenterCommand& CoincenterCommand::withBalanceInUse(bool value) { _withBalanceInUse = value; return *this; } + +CoincenterCommand& CoincenterCommand::setReplayOptions(ReplayOptions replayOptions) { + _specialOptions = std::move(replayOptions); + return *this; +} + } // namespace cct diff --git a/src/engine/src/coincentercommands.cpp b/src/engine/src/coincentercommands.cpp index 6c6db1ed..f3bf85d7 100644 --- a/src/engine/src/coincentercommands.cpp +++ b/src/engine/src/coincentercommands.cpp @@ -11,7 +11,10 @@ #include "coincenteroptions.hpp" #include "currencycode.hpp" #include "depositsconstraints.hpp" +#include "market.hpp" +#include "replay-options.hpp" #include "stringoptionparser.hpp" +#include "time-window.hpp" #include "timedef.hpp" #include "withdrawsconstraints.hpp" @@ -199,6 +202,46 @@ void CoincenterCommands::addOption(const CoincenterCmdLineOptions &cmdLineOption .setExchangeNames(optionParser.parseExchanges()); } + if (!cmdLineOptions.marketData.empty()) { + optionParser = StringOptionParser(cmdLineOptions.marketData); + + _commands.emplace_back(CoincenterCommandType::kMarketData) + .setMarket(optionParser.parseMarket()) + .setExchangeNames(optionParser.parseExchanges()); + } + + if (cmdLineOptions.replay) { + optionParser = StringOptionParser(*cmdLineOptions.replay); + + auto dur = optionParser.parseDuration(StringOptionParser::FieldIs::kOptional); + + auto &cmd = _commands.emplace_back(CoincenterCommandType::kReplay) + .setReplayOptions(cmdLineOptions.computeReplayOptions(dur)) + .setExchangeNames(optionParser.parseExchanges()); + + if (!cmdLineOptions.market.empty()) { + cmd.setMarket(Market(cmdLineOptions.market)); + } + } + + if (cmdLineOptions.replayMarkets) { + optionParser = StringOptionParser(*cmdLineOptions.replayMarkets); + + TimeWindow timeWindow; + auto dur = optionParser.parseDuration(StringOptionParser::FieldIs::kOptional); + auto nowTime = Clock::now(); + if (dur == kUndefinedDuration) { + timeWindow = TimeWindow(TimePoint{}, nowTime); + } else { + timeWindow = TimeWindow(nowTime - dur, nowTime); + } + + _commands.emplace_back(CoincenterCommandType::kReplayMarkets) + .setReplayOptions( + ReplayOptions(timeWindow, cmdLineOptions.algorithmNames, ReplayOptions::ReplayMode::kValidateOnly)) + .setExchangeNames(optionParser.parseExchanges()); + } + optionParser.checkEndParsing(); // No more option part should be remaining } diff --git a/src/engine/src/coincenterinfo_create.cpp b/src/engine/src/coincenterinfo_create.cpp index 061b6ca1..a3a88ba0 100644 --- a/src/engine/src/coincenterinfo_create.cpp +++ b/src/engine/src/coincenterinfo_create.cpp @@ -4,6 +4,7 @@ #include #include "apioutputtype.hpp" +#include "automation-config.hpp" #include "cct_json.hpp" #include "cct_string.hpp" #include "coincenterinfo.hpp" @@ -19,6 +20,7 @@ #include "runmodes.hpp" #include "stringoptionparser.hpp" #include "timedef.hpp" +#include "trading-config.hpp" namespace cct { @@ -69,8 +71,21 @@ CoincenterInfo CoincenterInfo_Create(std::string_view programName, const Coincen RequestsConfig requestsConfig( generalConfigData.at("requests").at("concurrency").at("nbMaxParallelRequests").get()); - GeneralConfig generalConfig(std::move(loggingInfo), std::move(requestsConfig), fiatConversionQueryRate, - apiOutputType); + const auto &automationJsonPart = generalConfigData.at("trading").at("automation"); + const auto &deserializationJsonPart = automationJsonPart.at("deserialization"); + const auto &startingContextJsonPart = automationJsonPart.at("startingContext"); + + Duration loadChunkDuration = ParseDuration(deserializationJsonPart.at("loadChunkDuration").get()); + MonetaryAmount startBaseAmountEquivalent{ + startingContextJsonPart.at("startBaseAmountEquivalent").get()}; + MonetaryAmount startQuoteAmountEquivalent{ + startingContextJsonPart.at("startQuoteAmountEquivalent").get()}; + + AutomationConfig automationConfig(loadChunkDuration, startBaseAmountEquivalent, startQuoteAmountEquivalent); + TradingConfig tradingConfig(std::move(automationConfig)); + + GeneralConfig generalConfig(std::move(loggingInfo), std::move(requestsConfig), std::move(tradingConfig), + fiatConversionQueryRate, apiOutputType); const LoadConfiguration loadConfiguration(dataDir, LoadConfiguration::ExchangeConfigFileType::kProd); diff --git a/src/engine/src/coincenteroptions.cpp b/src/engine/src/coincenteroptions.cpp index 7871fee2..a82abc95 100644 --- a/src/engine/src/coincenteroptions.cpp +++ b/src/engine/src/coincenteroptions.cpp @@ -14,6 +14,7 @@ #include "priceoptions.hpp" #include "priceoptionsdef.hpp" #include "ssl_sha.hpp" +#include "timedef.hpp" #include "tradedefinitions.hpp" #include "tradeoptions.hpp" #include "withdrawoptions.hpp" @@ -37,6 +38,10 @@ std::ostream& CoincenterCmdLineOptions::PrintVersion(std::string_view programNam os << "compiled with " << CCT_COMPILER_VERSION << " on " << __DATE__ << " at " << __TIME__ << '\n'; os << " " << GetCurlVersionInfo() << '\n'; os << " " << ssl::GetOpenSSLVersion() << '\n'; +#ifdef CCT_PROTOBUF_VERSION + os << " " + << "protobuf " << CCT_PROTOBUF_VERSION << '\n'; +#endif return os; } @@ -144,6 +149,31 @@ WithdrawOptions CoincenterCmdLineOptions::computeWithdrawOptions() const { return {withdrawRefreshTime, withdrawSyncPolicy}; } +ReplayOptions CoincenterCmdLineOptions::computeReplayOptions(Duration dur) const { + if (validate && validateOnly) { + throw invalid_argument("--validate and --validate-only cannot be specified simultaneously"); + } + + ReplayOptions::ReplayMode replayMode; + if (validateOnly) { + replayMode = ReplayOptions::ReplayMode::kValidateOnly; + } else if (validate) { + replayMode = ReplayOptions::ReplayMode::kCheckedLaunchAlgorithm; + } else { + replayMode = ReplayOptions::ReplayMode::kUncheckedLaunchAlgorithm; + } + + TimeWindow timeWindow; + const auto nowTime = Clock::now(); + if (dur == kUndefinedDuration) { + timeWindow = TimeWindow(TimePoint{}, nowTime); + } else { + timeWindow = TimeWindow(nowTime - dur, nowTime); + } + + return ReplayOptions(timeWindow, algorithmNames, replayMode); +} + std::pair CoincenterCmdLineOptions::getTradeArgStr() const { if (!tradeStrategy.empty() && !tradePrice.empty()) { throw invalid_argument("Trade price and trade strategy cannot be set together"); diff --git a/src/engine/src/exchangesorchestrator.cpp b/src/engine/src/exchangesorchestrator.cpp index 05dd2ade..69af6f42 100644 --- a/src/engine/src/exchangesorchestrator.cpp +++ b/src/engine/src/exchangesorchestrator.cpp @@ -31,13 +31,16 @@ #include "exchangepublicapi.hpp" #include "exchangepublicapitypes.hpp" #include "exchangeretriever.hpp" +#include "market-trader-engine.hpp" #include "market.hpp" #include "monetaryamount.hpp" #include "monetaryamountbycurrencyset.hpp" #include "ordersconstraints.hpp" #include "queryresulttypes.hpp" +#include "replay-options.hpp" #include "requestsconfig.hpp" #include "threadpool.hpp" +#include "trade-range-stats.hpp" #include "tradedamounts.hpp" #include "tradeoptions.hpp" #include "traderesult.hpp" @@ -118,7 +121,7 @@ ExchangeRetriever::PublicExchangesVec SelectUniquePublicExchanges(ExchangeRetrie ExchangesOrchestrator::ExchangesOrchestrator(const RequestsConfig &requestsConfig, std::span exchangesSpan) : _exchangeRetriever(exchangesSpan), _threadPool(requestsConfig.nbMaxParallelRequests(static_cast(exchangesSpan.size()))) { - log::info("Created a thread pool with {} workers for exchange requests", _threadPool.nbWorkers()); + log::debug("Created a thread pool with {} workers for exchange requests", _threadPool.nbWorkers()); } ExchangeHealthCheckStatus ExchangesOrchestrator::healthCheck(ExchangeNameSpan exchangeNames) { @@ -170,7 +173,7 @@ MarketOrderBookConversionRates ExchangesOrchestrator::getMarketOrderBooks(Market if (!optConversionRate && !equiCurrencyCode.isNeutral()) { log::warn("Unable to convert {} into {} on {}", mk.quote(), equiCurrencyCode, exchange->name()); } - return std::make_tuple(exchange->name(), exchange->queryOrderBook(mk, actualDepth), optConversionRate); + return std::make_tuple(exchange->name(), exchange->getOrderBook(mk, actualDepth), optConversionRate); }; _threadPool.parallelTransform(selectedExchanges.begin(), selectedExchanges.end(), ret.begin(), marketOrderBooksFunc); return ret; @@ -940,7 +943,7 @@ TradesPerExchange ExchangesOrchestrator::getLastTradesPerExchange(Market mk, Exc TradesPerExchange ret(selectedExchanges.size()); _threadPool.parallelTransform( selectedExchanges.begin(), selectedExchanges.end(), ret.begin(), [mk, nbLastTrades](Exchange *exchange) { - return std::make_pair(static_cast(exchange), exchange->queryLastTrades(mk, nbLastTrades)); + return std::make_pair(static_cast(exchange), exchange->getLastTrades(mk, nbLastTrades)); }); return ret; @@ -957,4 +960,116 @@ MonetaryAmountPerExchange ExchangesOrchestrator::getLastPricePerExchange(Market return lastPricePerExchange; } +MarketDataPerExchange ExchangesOrchestrator::getMarketDataPerExchange(std::span marketPerPublicExchange, + ExchangeNameSpan exchangeNames) { + UniquePublicSelectedExchanges selectedExchanges = _exchangeRetriever.selectOneAccount(exchangeNames); + + std::array isMarketTradable; + + _threadPool.parallelTransform(selectedExchanges.begin(), selectedExchanges.end(), isMarketTradable.begin(), + [&marketPerPublicExchange](Exchange *exchange) { + Market market = marketPerPublicExchange[exchange->publicExchangePos()]; + return market.isDefined() && exchange->queryTradableMarkets().contains(market); + }); + + FilterVector(selectedExchanges, isMarketTradable); + + MarketDataPerExchange ret(selectedExchanges.size()); + _threadPool.parallelTransform( + selectedExchanges.begin(), selectedExchanges.end(), ret.begin(), [&marketPerPublicExchange](Exchange *exchange) { + if (!exchange->exchangeConfig().withMarketDataSerialization()) { + log::warn("Calling market-data on {} with data serialization disabled", exchange->name()); + } + // Call order book and last trades sequentially for this exchange + Market market = marketPerPublicExchange[exchange->publicExchangePos()]; + return std::make_pair(exchange, + std::make_pair(exchange->getOrderBook(market), exchange->getLastTrades(market))); + }); + return ret; +} + +MarketTimestampSetsPerExchange ExchangesOrchestrator::pullAvailableMarketsForReplay(TimeWindow timeWindow, + ExchangeNameSpan exchangeNames) { + log::info("Query available markets for replay from {} within {}", ConstructAccumulatedExchangeNames(exchangeNames), + timeWindow); + UniquePublicSelectedExchanges selectedExchanges = _exchangeRetriever.selectOneAccount(exchangeNames); + MarketTimestampSetsPerExchange marketTimestampSetsPerExchange(selectedExchanges.size()); + _threadPool.parallelTransform(selectedExchanges.begin(), selectedExchanges.end(), + marketTimestampSetsPerExchange.begin(), [timeWindow](Exchange *exchange) { + return std::make_pair( + exchange, + MarketTimestampSets{exchange->apiPublic().pullMarketOrderBooksMarkets(timeWindow), + exchange->apiPublic().pullTradeMarkets(timeWindow)}); + }); + return marketTimestampSetsPerExchange; +} + +MarketTradeRangeStatsPerExchange ExchangesOrchestrator::traderConsumeRange( + const ReplayOptions &replayOptions, TimeWindow subTimeWindow, std::span marketTraderEngines, + ExchangeNameSpan exchangeNames) { + UniquePublicSelectedExchanges selectedExchanges = _exchangeRetriever.selectOneAccount(exchangeNames); + + MarketTradeRangeStatsPerExchange tradeRangeResultsPerExchange(selectedExchanges.size()); + + _threadPool.parallelTransform( + selectedExchanges.begin(), selectedExchanges.end(), marketTraderEngines.begin(), + tradeRangeResultsPerExchange.begin(), + [subTimeWindow, &replayOptions](Exchange *exchange, MarketTraderEngine &marketTraderEngine) { + Market market = marketTraderEngine.market(); + auto &apiPublic = exchange->apiPublic(); + auto marketOrderBooks = apiPublic.pullMarketOrderBooksForReplay(market, subTimeWindow); + auto publicTrades = apiPublic.pullTradesForReplay(market, subTimeWindow); + + TradeRangeStats tradeRangeStats; + + switch (replayOptions.replayMode()) { + case ReplayOptions::ReplayMode::kValidateOnly: + tradeRangeStats = marketTraderEngine.validateRange(std::move(marketOrderBooks), std::move(publicTrades)); + break; + case ReplayOptions::ReplayMode::kCheckedLaunchAlgorithm: + tradeRangeStats = marketTraderEngine.validateRange(marketOrderBooks, publicTrades); + marketTraderEngine.tradeRange(std::move(marketOrderBooks), std::move(publicTrades)); + break; + case ReplayOptions::ReplayMode::kUncheckedLaunchAlgorithm: + tradeRangeStats = marketTraderEngine.tradeRange(std::move(marketOrderBooks), std::move(publicTrades)); + break; + default: + break; + } + + return std::make_pair(exchange, std::move(tradeRangeStats)); + }); + + return tradeRangeResultsPerExchange; +} + +MarketTradingGlobalResultPerExchange ExchangesOrchestrator::getMarketTraderResultPerExchange( + std::span marketTraderEngines, MarketTradeRangeStatsPerExchange &&tradeRangeStatsPerExchange, + ExchangeNameSpan exchangeNames) { + UniquePublicSelectedExchanges selectedExchanges = _exchangeRetriever.selectOneAccount(exchangeNames); + + if (selectedExchanges.size() != tradeRangeStatsPerExchange.size()) { + throw exception("Inconsistent selected exchange sizes"); + } + + MarketTradingResultPerExchange marketTradingResultPerExchange(selectedExchanges.size()); + + _threadPool.parallelTransform(selectedExchanges.begin(), selectedExchanges.end(), marketTraderEngines.begin(), + marketTradingResultPerExchange.begin(), + [](const Exchange *exchange, MarketTraderEngine &marketTraderEngine) { + return std::make_pair(exchange, marketTraderEngine.finalizeAndComputeResult()); + }); + + MarketTradingGlobalResultPerExchange marketTradingGlobalResultPerExchange(selectedExchanges.size()); + std::transform(marketTradingResultPerExchange.begin(), marketTradingResultPerExchange.end(), + tradeRangeStatsPerExchange.begin(), marketTradingGlobalResultPerExchange.begin(), + [](auto &exchangeMarketTradingResult, auto &exchangeTradeRangeStats) { + return std::make_pair(exchangeMarketTradingResult.first, + MarketTradingGlobalResult{std::move(exchangeMarketTradingResult.second), + std::move(exchangeTradeRangeStats.second)}); + }); + + return marketTradingGlobalResultPerExchange; +} + } // namespace cct diff --git a/src/engine/src/query-result-type-helpers.cpp b/src/engine/src/query-result-type-helpers.cpp new file mode 100644 index 00000000..a6f8dcf3 --- /dev/null +++ b/src/engine/src/query-result-type-helpers.cpp @@ -0,0 +1,31 @@ +#include "query-result-type-helpers.hpp" + +#include +#include + +#include "market.hpp" + +namespace cct { + +bool ContainsMarket(Market market, const MarketTimestampSet &marketTimestampSet) { + auto it = std::ranges::partition_point( + marketTimestampSet, [market](const auto &marketTimestamp) { return marketTimestamp.market < market; }); + return it != marketTimestampSet.end() && it->market == market; +} + +bool ContainsMarket(Market market, const MarketTimestampSets &marketTimestampSets) { + return ContainsMarket(market, marketTimestampSets.orderBooksMarkets) || + ContainsMarket(market, marketTimestampSets.tradesMarkets); +} + +MarketSet ComputeAllMarkets(const MarketTimestampSetsPerExchange &marketTimestampSetsPerExchange) { + MarketSet allMarkets; + for (const auto &[_, marketTimestamps] : marketTimestampSetsPerExchange) { + std::ranges::transform(marketTimestamps.orderBooksMarkets, std::inserter(allMarkets, allMarkets.end()), + [](const auto &marketTimestamp) { return marketTimestamp.market; }); + std::ranges::transform(marketTimestamps.tradesMarkets, std::inserter(allMarkets, allMarkets.end()), + [](const auto &marketTimestamp) { return marketTimestamp.market; }); + } + return allMarkets; +} +} // namespace cct \ No newline at end of file diff --git a/src/engine/src/queryresultprinter.cpp b/src/engine/src/queryresultprinter.cpp index ee88df19..40ad3557 100644 --- a/src/engine/src/queryresultprinter.cpp +++ b/src/engine/src/queryresultprinter.cpp @@ -27,6 +27,7 @@ #include "exchange.hpp" #include "file.hpp" #include "logginginfo.hpp" +#include "market-timestamp.hpp" #include "market.hpp" #include "marketorderbook.hpp" #include "monetaryamount.hpp" @@ -36,9 +37,11 @@ #include "priceoptions.hpp" #include "priceoptionsdef.hpp" #include "publictrade.hpp" +#include "query-result-type-helpers.hpp" #include "queryresulttypes.hpp" #include "simpletable.hpp" #include "stringhelpers.hpp" +#include "time-window.hpp" #include "timestring.hpp" #include "tradedamounts.hpp" #include "tradedefinitions.hpp" @@ -126,6 +129,47 @@ json MarketsJson(CurrencyCode cur1, CurrencyCode cur2, const MarketsPerExchange return ToJson(CoincenterCommandType::kMarkets, std::move(in), std::move(out)); } +json MarketsForReplayJson(TimeWindow timeWindow, const MarketTimestampSetsPerExchange &marketTimestampSetsPerExchange) { + json in; + json inOpt = json::object(); + if (timeWindow != TimeWindow{}) { + inOpt.emplace("timeWindow", timeWindow.str()); + } + in.emplace("opt", std::move(inOpt)); + + json out = json::object(); + for (const auto &[e, marketTimestampSets] : marketTimestampSetsPerExchange) { + json orderBookMarketsPerExchange; + for (const MarketTimestamp &marketTimestamp : marketTimestampSets.orderBooksMarkets) { + json marketTimestampJson; + + marketTimestampJson.emplace("market", marketTimestamp.market.str()); + marketTimestampJson.emplace("lastTimestamp", ToString(marketTimestamp.timePoint)); + + orderBookMarketsPerExchange.emplace_back(std::move(marketTimestampJson)); + } + + json tradesMarketsPerExchange; + for (const MarketTimestamp &marketTimestamp : marketTimestampSets.tradesMarkets) { + json marketTimestampJson; + + marketTimestampJson.emplace("market", marketTimestamp.market.str()); + marketTimestampJson.emplace("lastTimestamp", ToString(marketTimestamp.timePoint)); + + tradesMarketsPerExchange.emplace_back(std::move(marketTimestampJson)); + } + + json exchangePart; + + exchangePart.emplace("orderBooks", std::move(orderBookMarketsPerExchange)); + exchangePart.emplace("trades", std::move(tradesMarketsPerExchange)); + + out.emplace(e->name(), std::move(exchangePart)); + } + + return ToJson(CoincenterCommandType::kReplayMarkets, std::move(in), std::move(out)); +} + json TickerInformationJson(const ExchangeTickerMaps &exchangeTickerMaps) { json in; json out = json::object(); @@ -720,6 +764,57 @@ json DustSweeperJson(const TradedAmountsVectorWithFinalAmountPerExchange &traded return ToJson(CoincenterCommandType::kDustSweeper, std::move(in), std::move(out)); } +json MarketTradingResultsJson(TimeWindow timeWindow, + const MarketTradingGlobalResultPerExchange &marketTradingResultPerExchange, + CoincenterCommandType commandType) { + json in; + json inOpt; + inOpt.emplace("time-window", timeWindow.str()); + in.emplace("opt", std::move(inOpt)); + + json out = json::object(); + + for (const auto &[exchangePtr, marketGlobalTradingResult] : marketTradingResultPerExchange) { + const auto &marketTradingResult = marketGlobalTradingResult.result; + const auto &stats = marketGlobalTradingResult.stats; + + json startAmounts; + startAmounts.emplace("base", marketTradingResult.startBaseAmount().str()); + startAmounts.emplace("quote", marketTradingResult.startQuoteAmount().str()); + + json orderBookStats; + orderBookStats.emplace("nb-successful", stats.marketOrderBookStats.nbSuccessful); + orderBookStats.emplace("nb-error", stats.marketOrderBookStats.nbError); + + json tradeStats; + tradeStats.emplace("nb-successful", stats.publicTradeStats.nbSuccessful); + tradeStats.emplace("nb-error", stats.publicTradeStats.nbError); + + json jsonStats; + jsonStats.emplace("order-books", std::move(orderBookStats)); + jsonStats.emplace("trades", std::move(tradeStats)); + + json marketTradingResultJson; + marketTradingResultJson.emplace("algorithm", marketTradingResult.algorithmName()); + marketTradingResultJson.emplace("market", marketTradingResult.market().str()); + marketTradingResultJson.emplace("start-amounts", std::move(startAmounts)); + marketTradingResultJson.emplace("profit-and-loss", marketTradingResult.quoteAmountDelta().str()); + marketTradingResultJson.emplace("stats", std::move(jsonStats)); + + json closedOrdersArray = json::array_t(); + + for (const ClosedOrder &closedOrder : marketTradingResult.matchedOrders()) { + closedOrdersArray.push_back(OrderJson(closedOrder)); + } + + marketTradingResultJson.emplace("matched-orders", std::move(closedOrdersArray)); + + out.emplace(exchangePtr->name(), std::move(marketTradingResultJson)); + } + + return ToJson(commandType, std::move(in), std::move(out)); +} + template void RemoveDuplicates(VecType &vec) { std::ranges::sort(vec); @@ -1445,6 +1540,123 @@ void QueryResultPrinter::printDustSweeper( logActivity(CoincenterCommandType::kDustSweeper, jsonData); } +void QueryResultPrinter::printMarketsForReplay(TimeWindow timeWindow, + const MarketTimestampSetsPerExchange &marketTimestampSetsPerExchange) { + json jsonData = MarketsForReplayJson(timeWindow, marketTimestampSetsPerExchange); + switch (_apiOutputType) { + case ApiOutputType::kFormattedTable: { + MarketSet allMarkets = ComputeAllMarkets(marketTimestampSetsPerExchange); + + SimpleTable table; + table.reserve(allMarkets.size() + 1U); + table.emplace_back("Markets", "Last order books timestamp", "Last trades timestamp"); + + for (const Market market : allMarkets) { + table::Cell orderBookCell; + table::Cell tradesCell; + for (const auto &[e, marketTimestamps] : marketTimestampSetsPerExchange) { + const auto &orderBooksMarkets = marketTimestamps.orderBooksMarkets; + const auto &tradesMarkets = marketTimestamps.tradesMarkets; + const auto marketPartitionPred = [market](const auto &marketTimestamp) { + return marketTimestamp.market < market; + }; + const auto orderBooksIt = std::ranges::partition_point(orderBooksMarkets, marketPartitionPred); + const auto tradesIt = std::ranges::partition_point(tradesMarkets, marketPartitionPred); + + if (orderBooksIt != orderBooksMarkets.end() && orderBooksIt->market == market) { + string str = ToString(orderBooksIt->timePoint); + str.append(" @ "); + str.append(e->name()); + + orderBookCell.emplace_back(std::move(str)); + } + + if (tradesIt != tradesMarkets.end() && tradesIt->market == market) { + string str = ToString(tradesIt->timePoint); + str.append(" @ "); + str.append(e->name()); + + tradesCell.emplace_back(std::move(str)); + } + } + + table.emplace_back(market.str(), std::move(orderBookCell), std::move(tradesCell)); + } + printTable(table); + break; + } + case ApiOutputType::kJson: + printJson(jsonData); + break; + case ApiOutputType::kNoPrint: + break; + } + logActivity(CoincenterCommandType::kReplayMarkets, jsonData); +} + +void QueryResultPrinter::printMarketTradingResults( + TimeWindow timeWindow, const MarketTradingGlobalResultPerExchange &marketTradingResultPerExchange, + CoincenterCommandType commandType) const { + json jsonData = MarketTradingResultsJson(timeWindow, marketTradingResultPerExchange, commandType); + switch (_apiOutputType) { + case ApiOutputType::kFormattedTable: { + SimpleTable table; + table.reserve(1U + marketTradingResultPerExchange.size()); + table.emplace_back("Exchange", "Time window", "Market", "Algorithm", "Start amounts", "Profit / Loss", + "Matched orders", "Stats"); + for (const auto &[exchangePtr, marketGlobalTradingResults] : marketTradingResultPerExchange) { + const auto &marketTradingResults = marketGlobalTradingResults.result; + const auto &stats = marketGlobalTradingResults.stats; + + table::Cell trades; + for (const ClosedOrder &closedOrder : marketTradingResults.matchedOrders()) { + string orderStr = closedOrder.placedTimeStr(); + orderStr.append(" - "); + orderStr.append(closedOrder.sideStr()); + orderStr.append(" - "); + orderStr.append(closedOrder.matchedVolume().str()); + orderStr.append(" @ "); + orderStr.append(closedOrder.price().str()); + trades.emplace_back(std::move(orderStr)); + } + + string orderBookStats("order books: "); + orderBookStats.append(ToString(stats.marketOrderBookStats.nbSuccessful)); + orderBookStats.append(" OK"); + if (stats.marketOrderBookStats.nbError != 0) { + orderBookStats.append(", "); + orderBookStats.append(ToString(stats.marketOrderBookStats.nbError)); + orderBookStats.append(" KO"); + } + + string tradesStats("trades: "); + tradesStats.append(ToString(stats.publicTradeStats.nbSuccessful)); + tradesStats.append(" OK"); + if (stats.publicTradeStats.nbError != 0) { + tradesStats.append(", "); + tradesStats.append(ToString(stats.publicTradeStats.nbError)); + tradesStats.append(" KO"); + } + + table.emplace_back( + exchangePtr->name(), table::Cell{ToString(timeWindow.from()), ToString(timeWindow.to())}, + marketTradingResults.market().str(), marketTradingResults.algorithmName(), + table::Cell{marketTradingResults.startBaseAmount().str(), marketTradingResults.startQuoteAmount().str()}, + marketTradingResults.quoteAmountDelta().str(), std::move(trades), + table::Cell{std::move(orderBookStats), std::move(tradesStats)}); + } + printTable(table); + break; + } + case ApiOutputType::kJson: + printJson(jsonData); + break; + case ApiOutputType::kNoPrint: + break; + } + logActivity(commandType, jsonData); +} + void QueryResultPrinter::printTable(const SimpleTable &table) const { std::ostringstream ss; std::ostream &os = _pOs != nullptr ? *_pOs : ss; diff --git a/src/engine/src/replay-algorithm-name-iterator.cpp b/src/engine/src/replay-algorithm-name-iterator.cpp new file mode 100644 index 00000000..3d5e30b8 --- /dev/null +++ b/src/engine/src/replay-algorithm-name-iterator.cpp @@ -0,0 +1,62 @@ +#include "replay-algorithm-name-iterator.hpp" + +#include +#include + +#include "cct_exception.hpp" + +namespace cct { + +namespace { +constexpr std::string_view kAlgorithmNameSeparator = ","; + +auto FindNextSeparatorPos(std::string_view str, std::string_view::size_type pos = 0) { + pos = str.find(kAlgorithmNameSeparator, pos); + if (pos == std::string_view::npos) { + pos = str.length(); + } + return pos; +} +} // namespace + +ReplayAlgorithmNameIterator::ReplayAlgorithmNameIterator(std::string_view algorithmNames, + std::span allAlgorithms) + : _allAlgorithms(allAlgorithms), + _algorithmNames(algorithmNames), + _begPos(0), + _endPos(FindNextSeparatorPos(_algorithmNames)) { + if (std::ranges::any_of(allAlgorithms, [](const auto algName) { + return algName.find(kAlgorithmNameSeparator) != std::string_view::npos; + })) { + throw exception("Algorithm names cannot contain '{}' as it's used as a separator", kAlgorithmNameSeparator); + } +} + +bool ReplayAlgorithmNameIterator::hasNext() const { + using PosT = decltype(_begPos); + + if (_algorithmNames.empty()) { + return _begPos < static_cast(_allAlgorithms.size()); + } + + return _begPos != static_cast(_algorithmNames.length()); +} + +std::string_view ReplayAlgorithmNameIterator::next() { + if (_algorithmNames.empty()) { + return _allAlgorithms[_begPos++]; + } + + std::string_view nextAlgorithmName(_algorithmNames.begin() + _begPos, _algorithmNames.begin() + _endPos); + + if (_endPos == static_cast(_algorithmNames.length())) { + _begPos = _endPos; + } else { + _begPos = _endPos + kAlgorithmNameSeparator.length(); + _endPos = FindNextSeparatorPos(_algorithmNames, _begPos); + } + + return nextAlgorithmName; +} + +} // namespace cct \ No newline at end of file diff --git a/src/engine/src/replay-options.cpp b/src/engine/src/replay-options.cpp new file mode 100644 index 00000000..e0ab2694 --- /dev/null +++ b/src/engine/src/replay-options.cpp @@ -0,0 +1,19 @@ +#include "replay-options.hpp" + +#include + +#include "dummy-market-trader.hpp" +#include "time-window.hpp" + +namespace cct { +ReplayOptions::ReplayOptions(TimeWindow timeWindow, std::string_view algorithmNames, ReplayMode replayMode) + : _timeWindow(timeWindow), _algorithmNames(algorithmNames), _replayMode(replayMode) {} + +std::string_view ReplayOptions::algorithmNames() const { + if (_replayMode == ReplayMode::kValidateOnly) { + return DummyMarketTrader::kName; + } + return _algorithmNames; +} + +} // namespace cct \ No newline at end of file diff --git a/src/engine/src/stringoptionparser.cpp b/src/engine/src/stringoptionparser.cpp index 11502459..6435837d 100644 --- a/src/engine/src/stringoptionparser.cpp +++ b/src/engine/src/stringoptionparser.cpp @@ -9,10 +9,12 @@ #include "cct_string.hpp" #include "cct_vector.hpp" #include "currencycode.hpp" +#include "durationstring.hpp" #include "exchange-names.hpp" #include "exchangename.hpp" #include "market.hpp" #include "monetaryamount.hpp" +#include "timedef.hpp" namespace cct { @@ -38,6 +40,23 @@ CurrencyCode StringOptionParser::parseCurrency(FieldIs fieldIs, char delimiter) return {}; } +Duration StringOptionParser::parseDuration(FieldIs fieldIs) { + auto dur = kUndefinedDuration; + const std::string_view currentToken(_opt.begin() + _pos, _opt.end()); + const auto durationLen = DurationLen(currentToken); + if (durationLen > 0) { + const std::string_view durationStr(_opt.data() + _pos, static_cast(durationLen)); + + dur = ParseDuration(durationStr); + } else if (fieldIs == FieldIs::kMandatory) { + throw invalid_argument("Expected a valid duration in '{}'", currentToken); + } + + _pos += durationLen; + + return dur; +} + // At the end of the market, either the end of the string or a comma is expected. Market StringOptionParser::parseMarket(FieldIs fieldIs, char delimiter) { const auto oldPos = _pos; diff --git a/src/engine/test/queryresultprinter_public_test.cpp b/src/engine/test/queryresultprinter_public_test.cpp index 57bbc12e..6ba960df 100644 --- a/src/engine/test/queryresultprinter_public_test.cpp +++ b/src/engine/test/queryresultprinter_public_test.cpp @@ -9,15 +9,21 @@ #include "currencycode.hpp" #include "currencyexchange.hpp" #include "currencyexchangeflatset.hpp" +#include "exchangeprivateapitypes.hpp" #include "exchangepublicapitypes.hpp" +#include "market-trading-global-result.hpp" +#include "market-trading-result.hpp" #include "market.hpp" #include "marketorderbook.hpp" #include "monetaryamount.hpp" #include "monetaryamountbycurrencyset.hpp" +#include "public-trade-vector.hpp" #include "publictrade.hpp" #include "queryresultprinter.hpp" #include "queryresultprinter_base_test.hpp" #include "queryresulttypes.hpp" +#include "time-window.hpp" +#include "trade-range-stats.hpp" #include "tradeside.hpp" namespace cct { @@ -243,7 +249,6 @@ TEST_F(QueryResultPrinterMarketsTest, FormattedTableNoCurrency) { | huobi | XRP-EUR | +----------+---------+ )"; - expectStr(kExpected); } @@ -262,7 +267,6 @@ TEST_F(QueryResultPrinterMarketsTest, FormattedTableOneCurrency) { | huobi | XRP-EUR | +----------+------------------+ )"; - expectStr(kExpected); } @@ -279,7 +283,6 @@ TEST_F(QueryResultPrinterMarketsTest, FormattedTableTwoCurrencies) { | huobi | XRP-EUR | +----------+----------------------+ )"; - expectStr(kExpected); } @@ -1176,4 +1179,303 @@ TEST_F(QueryResultPrinterLastPriceTest, NoPrint) { expectNoStr(); } +class QueryResultPrinterReplayBaseTest : public QueryResultPrinterTest { + protected: + Market market1{"ETH", "KRW"}; + Market market2{"BTC", "USD"}; + Market market3{"SHIB", "USDT"}; + Market market4{"SOL", "BTC"}; + Market market5{"SOL", "ETH"}; + Market market6{"ETH", "BTC"}; + Market market7{"DOGE", "CAD"}; + + TimePoint tp1{milliseconds{std::numeric_limits::max() / 10000000}}; + TimePoint tp2{milliseconds{std::numeric_limits::max() / 9900000}}; + TimePoint tp3{milliseconds{std::numeric_limits::max() / 9800000}}; + TimePoint tp4{milliseconds{std::numeric_limits::max() / 9600000}}; + TimePoint tp5{milliseconds{std::numeric_limits::max() / 9500000}}; + + TimeWindow timeWindow{tp1, tp5}; +}; + +class QueryResultPrinterReplayMarketsTest : public QueryResultPrinterReplayBaseTest { + protected: + MarketTimestampSetsPerExchange marketTimestampSetsPerExchange{ + {&exchange1, + MarketTimestampSets{MarketTimestampSet{MarketTimestamp{market1, tp1}, MarketTimestamp{market2, tp2}, + MarketTimestamp{market3, tp3}}, + MarketTimestampSet{MarketTimestamp{market1, tp1}, MarketTimestamp{market2, tp1}}}}, + {&exchange2, MarketTimestampSets{MarketTimestampSet{MarketTimestamp{market2, tp4}, MarketTimestamp{market4, tp5}}, + MarketTimestampSet{MarketTimestamp{market6, tp1}}}}, + {&exchange3, MarketTimestampSets{MarketTimestampSet{}, MarketTimestampSet{MarketTimestamp{market1, tp1}, + MarketTimestamp{market7, tp4}}}}}; +}; + +TEST_F(QueryResultPrinterReplayMarketsTest, FormattedTable) { + basicQueryResultPrinter(ApiOutputType::kFormattedTable) + .printMarketsForReplay(timeWindow, marketTimestampSetsPerExchange); + static constexpr std::string_view kExpected = R"( ++-----------+--------------------------------+--------------------------------+ +| Markets | Last order books timestamp | Last trades timestamp | ++-----------+--------------------------------+--------------------------------+ +| BTC-USD | 1999-07-11T00:42:21Z @ binance | 1999-03-25T04:46:43Z @ binance | +| | 2000-06-11T23:58:40Z @ bithumb | | +|~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~| +| DOGE-CAD | | 2000-06-11T23:58:40Z @ huobi | +| ETH-BTC | | 1999-03-25T04:46:43Z @ bithumb | +|~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~| +| ETH-KRW | 1999-03-25T04:46:43Z @ binance | 1999-03-25T04:46:43Z @ binance | +| | | 1999-03-25T04:46:43Z @ huobi | +|~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~| +| SHIB-USDT | 1999-10-29T01:26:51Z @ binance | | +| SOL-BTC | 2000-10-07T01:14:27Z @ bithumb | | ++-----------+--------------------------------+--------------------------------+ +)"; + expectStr(kExpected); +} + +TEST_F(QueryResultPrinterReplayMarketsTest, EmptyJson) { + basicQueryResultPrinter(ApiOutputType::kJson).printMarketsForReplay(timeWindow, MarketTimestampSetsPerExchange{}); + static constexpr std::string_view kExpected = R"json( +{ + "in": { + "opt": { + "timeWindow": "[1999-03-25 04:46:43 -> 2000-10-07 01:14:27)" + }, + "req": "ReplayMarkets" + }, + "out": {} +})json"; + expectJson(kExpected); +} + +TEST_F(QueryResultPrinterReplayMarketsTest, Json) { + basicQueryResultPrinter(ApiOutputType::kJson).printMarketsForReplay(timeWindow, marketTimestampSetsPerExchange); + static constexpr std::string_view kExpected = R"json( +{ + "in": { + "opt": { + "timeWindow": "[1999-03-25 04:46:43 -> 2000-10-07 01:14:27)" + }, + "req": "ReplayMarkets" + }, + "out": { + "binance": { + "orderBooks": [ + { + "lastTimestamp": "1999-07-11T00:42:21Z", + "market": "BTC-USD" + }, + { + "lastTimestamp": "1999-03-25T04:46:43Z", + "market": "ETH-KRW" + }, + { + "lastTimestamp": "1999-10-29T01:26:51Z", + "market": "SHIB-USDT" + } + ], + "trades": [ + { + "lastTimestamp": "1999-03-25T04:46:43Z", + "market": "BTC-USD" + }, + { + "lastTimestamp": "1999-03-25T04:46:43Z", + "market": "ETH-KRW" + } + ] + }, + "bithumb": { + "orderBooks": [ + { + "lastTimestamp": "2000-06-11T23:58:40Z", + "market": "BTC-USD" + }, + { + "lastTimestamp": "2000-10-07T01:14:27Z", + "market": "SOL-BTC" + } + ], + "trades": [ + { + "lastTimestamp": "1999-03-25T04:46:43Z", + "market": "ETH-BTC" + } + ] + }, + "huobi": { + "orderBooks": null, + "trades": [ + { + "lastTimestamp": "2000-06-11T23:58:40Z", + "market": "DOGE-CAD" + }, + { + "lastTimestamp": "1999-03-25T04:46:43Z", + "market": "ETH-KRW" + } + ] + } + } +})json"; + expectJson(kExpected); +} + +TEST_F(QueryResultPrinterReplayMarketsTest, NoPrint) { + basicQueryResultPrinter(ApiOutputType::kNoPrint).printMarketsForReplay(timeWindow, marketTimestampSetsPerExchange); + expectNoStr(); +} + +class QueryResultPrinterReplayTest : public QueryResultPrinterReplayBaseTest { + protected: + ClosedOrder closedOrder1{"1", MonetaryAmount(15, "BTC", 1), MonetaryAmount(35000, "USDT"), tp1, tp1, TradeSide::kBuy}; + ClosedOrder closedOrder2{"2", MonetaryAmount(25, "BTC", 1), MonetaryAmount(45000, "USDT"), tp2, tp2, TradeSide::kBuy}; + ClosedOrder closedOrder3{"3", MonetaryAmount(5, "BTC", 2), MonetaryAmount(35000, "USDT"), tp3, tp4, TradeSide::kSell}; + ClosedOrder closedOrder4{ + "4", MonetaryAmount(17, "BTC", 1), MonetaryAmount(50000, "USDT"), tp3, tp4, TradeSide::kSell}; + ClosedOrder closedOrder5{ + "5", MonetaryAmount(36, "BTC", 3), MonetaryAmount(47899, "USDT"), tp4, tp5, TradeSide::kSell}; + + std::string_view algorithmName = "test-algo"; + MonetaryAmount startBaseAmount{1, "BTC"}; + MonetaryAmount startQuoteAmount{1000, "EUR"}; + + MarketTradingResult marketTradingResult1{algorithmName, startBaseAmount, startQuoteAmount, MonetaryAmount{0, "EUR"}, + ClosedOrderVector{}}; + MarketTradingResult marketTradingResult3{algorithmName, startBaseAmount, startQuoteAmount, MonetaryAmount{500, "EUR"}, + ClosedOrderVector{closedOrder1, closedOrder5}}; + MarketTradingResult marketTradingResult4{algorithmName, startBaseAmount, startQuoteAmount, MonetaryAmount{780, "EUR"}, + ClosedOrderVector{closedOrder2, closedOrder3, closedOrder4}}; + + TradeRangeStats tradeRangeStats1{TradeRangeResultsStats{42, 0}, TradeRangeResultsStats{3, 10}}; + TradeRangeStats tradeRangeStats3{TradeRangeResultsStats{500000, 2}, TradeRangeResultsStats{0, 0}}; + TradeRangeStats tradeRangeStats4{TradeRangeResultsStats{79009, 0}, TradeRangeResultsStats{1555555555, 45}}; + + MarketTradingGlobalResultPerExchange marketTradingResultPerExchange{ + {&exchange1, MarketTradingGlobalResult{marketTradingResult1, tradeRangeStats1}}, + {&exchange3, MarketTradingGlobalResult{marketTradingResult3, tradeRangeStats3}}, + {&exchange4, MarketTradingGlobalResult{marketTradingResult4, tradeRangeStats4}}}; + CoincenterCommandType commandType{CoincenterCommandType::kReplay}; +}; + +TEST_F(QueryResultPrinterReplayTest, FormattedTable) { + basicQueryResultPrinter(ApiOutputType::kFormattedTable) + .printMarketTradingResults(timeWindow, marketTradingResultPerExchange, commandType); + static constexpr std::string_view kExpected = R"( ++----------+----------------------+---------+-----------+---------------+---------------+------------------------------------------------------+------------------------------+ +| Exchange | Time window | Market | Algorithm | Start amounts | Profit / Loss | Matched orders | Stats | ++----------+----------------------+---------+-----------+---------------+---------------+------------------------------------------------------+------------------------------+ +| binance | 1999-03-25T04:46:43Z | BTC-EUR | test-algo | 1 BTC | 0 EUR | | order books: 42 OK | +| | 2000-10-07T01:14:27Z | | | 1000 EUR | | | trades: 3 OK, 10 KO | +|~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~|~~~~~~~~~|~~~~~~~~~~~|~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~| +| huobi | 1999-03-25T04:46:43Z | BTC-EUR | test-algo | 1 BTC | 500 EUR | 1999-03-25T04:46:43Z - Buy - 1.5 BTC @ 35000 USDT | order books: 500000 OK, 2 KO | +| | 2000-10-07T01:14:27Z | | | 1000 EUR | | 2000-06-11T23:58:40Z - Sell - 0.036 BTC @ 47899 USDT | trades: 0 OK | +|~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~|~~~~~~~~~|~~~~~~~~~~~|~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~| +| huobi | 1999-03-25T04:46:43Z | BTC-EUR | test-algo | 1 BTC | 780 EUR | 1999-07-11T00:42:21Z - Buy - 2.5 BTC @ 45000 USDT | order books: 79009 OK | +| | 2000-10-07T01:14:27Z | | | 1000 EUR | | 1999-10-29T01:26:51Z - Sell - 0.05 BTC @ 35000 USDT | trades: 1555555555 OK, 45 KO | +| | | | | | | 1999-10-29T01:26:51Z - Sell - 1.7 BTC @ 50000 USDT | | ++----------+----------------------+---------+-----------+---------------+---------------+------------------------------------------------------+------------------------------+ +)"; + expectStr(kExpected); +} + +TEST_F(QueryResultPrinterReplayTest, EmptyJson) { + basicQueryResultPrinter(ApiOutputType::kJson) + .printMarketTradingResults(timeWindow, MarketTradingGlobalResultPerExchange{}, commandType); + static constexpr std::string_view kExpected = R"json( +{ + "in": { + "opt": { + "time-window": "[1999-03-25 04:46:43 -> 2000-10-07 01:14:27)" + }, + "req": "Replay" + }, + "out": {} +})json"; + expectJson(kExpected); +} + +TEST_F(QueryResultPrinterReplayTest, Json) { + basicQueryResultPrinter(ApiOutputType::kJson) + .printMarketTradingResults(timeWindow, marketTradingResultPerExchange, commandType); + static constexpr std::string_view kExpected = R"json( +{ + "in": { + "opt": { + "time-window": "[1999-03-25 04:46:43 -> 2000-10-07 01:14:27)" + }, + "req": "Replay" + }, + "out": { + "binance": { + "algorithm": "test-algo", + "market": "BTC-EUR", + "matched-orders": [], + "profit-and-loss": "0 EUR", + "start-amounts": { + "base": "1 BTC", + "quote": "1000 EUR" + }, + "stats": { + "order-books": { + "nb-error": 0, + "nb-successful": 42 + }, + "trades": { + "nb-error": 10, + "nb-successful": 3 + } + } + }, + "huobi": { + "algorithm": "test-algo", + "market": "BTC-EUR", + "matched-orders": [ + { + "id": "1", + "matched": "1.5", + "matchedTime": "1999-03-25T04:46:43Z", + "pair": "BTC-USDT", + "placedTime": "1999-03-25T04:46:43Z", + "price": "35000", + "side": "Buy" + }, + { + "id": "5", + "matched": "0.036", + "matchedTime": "2000-10-07T01:14:27Z", + "pair": "BTC-USDT", + "placedTime": "2000-06-11T23:58:40Z", + "price": "47899", + "side": "Sell" + } + ], + "profit-and-loss": "500 EUR", + "start-amounts": { + "base": "1 BTC", + "quote": "1000 EUR" + }, + "stats": { + "order-books": { + "nb-error": 2, + "nb-successful": 500000 + }, + "trades": { + "nb-error": 0, + "nb-successful": 0 + } + } + } + } +})json"; + expectJson(kExpected); +} + +TEST_F(QueryResultPrinterReplayTest, NoPrint) { + basicQueryResultPrinter(ApiOutputType::kNoPrint) + .printMarketTradingResults(timeWindow, marketTradingResultPerExchange, commandType); + expectNoStr(); +} + } // namespace cct diff --git a/src/engine/test/replay-algorithm-name-iterator_test.cpp b/src/engine/test/replay-algorithm-name-iterator_test.cpp new file mode 100644 index 00000000..ee930a0b --- /dev/null +++ b/src/engine/test/replay-algorithm-name-iterator_test.cpp @@ -0,0 +1,106 @@ +#include "replay-algorithm-name-iterator.hpp" + +#include + +#include + +#include "cct_exception.hpp" + +namespace cct { +class ReplayAlgorithmNameIteratorTest : public ::testing::Test { + protected: + static constexpr std::string_view kInvalidAlgorithmNames[] = {"any", "so-what,"}; + static constexpr std::string_view kAlgorithmNames[] = {"any", "so-what", "angry", + "bird", "Jack", "a-more-complex algorithm Name"}; +}; + +TEST_F(ReplayAlgorithmNameIteratorTest, AlgorithmNamesValidity) { + EXPECT_THROW(ReplayAlgorithmNameIterator("", kInvalidAlgorithmNames), exception); + EXPECT_NO_THROW(ReplayAlgorithmNameIterator("", kAlgorithmNames)); +} + +TEST_F(ReplayAlgorithmNameIteratorTest, IteratorWithAll) { + ReplayAlgorithmNameIterator it("", kAlgorithmNames); + + int algorithmPos = 0; + while (it.hasNext()) { + auto next = it.next(); + + switch (algorithmPos) { + case 0: + [[fallthrough]]; + case 1: + [[fallthrough]]; + case 2: + [[fallthrough]]; + case 3: + [[fallthrough]]; + case 4: + [[fallthrough]]; + case 5: + EXPECT_EQ(next, kAlgorithmNames[algorithmPos]); + break; + default: + throw exception("Unexpected number of algorithm names"); + } + + ++algorithmPos; + } + + EXPECT_EQ(algorithmPos, 6); +} + +TEST_F(ReplayAlgorithmNameIteratorTest, IteratorWithUniqueAlgorithmSpecified) { + ReplayAlgorithmNameIterator it("so-What", kAlgorithmNames); + + int algorithmPos = 0; + while (it.hasNext()) { + auto next = it.next(); + + switch (algorithmPos) { + case 0: + EXPECT_EQ(next, "so-What"); + break; + default: + throw exception("Unexpected number of algorithm names"); + } + + ++algorithmPos; + } + + EXPECT_EQ(algorithmPos, 1); +} + +TEST_F(ReplayAlgorithmNameIteratorTest, IteratorWithSpecifiedList) { + ReplayAlgorithmNameIterator it("Jack,whatever,so-what,some-algorithmNameThatIsNotInAll,with spaces", kAlgorithmNames); + + int algorithmPos = 0; + while (it.hasNext()) { + auto next = it.next(); + + switch (algorithmPos) { + case 0: + EXPECT_EQ(next, "Jack"); + break; + case 1: + EXPECT_EQ(next, "whatever"); + break; + case 2: + EXPECT_EQ(next, "so-what"); + break; + case 3: + EXPECT_EQ(next, "some-algorithmNameThatIsNotInAll"); + break; + case 4: + EXPECT_EQ(next, "with spaces"); + break; + default: + throw exception("Unexpected number of algorithm names"); + } + + ++algorithmPos; + } + + EXPECT_EQ(algorithmPos, 5); +} +} // namespace cct \ No newline at end of file diff --git a/src/engine/test/stringoptionparser_test.cpp b/src/engine/test/stringoptionparser_test.cpp index e8869af4..4059145b 100644 --- a/src/engine/test/stringoptionparser_test.cpp +++ b/src/engine/test/stringoptionparser_test.cpp @@ -2,6 +2,7 @@ #include +#include #include #include "cct_invalid_argument_exception.hpp" @@ -12,6 +13,7 @@ #include "exchangename.hpp" #include "market.hpp" #include "monetaryamount.hpp" +#include "timedef.hpp" namespace cct { namespace { @@ -211,4 +213,26 @@ TEST(StringOptionParserTest, ExchangesNotLast) { EXPECT_NO_THROW(parser.checkEndParsing()); } +TEST(StringOptionParserTest, ParseDurationMandatory) { + StringOptionParser parser(" 45min83s,kraken,upbit"); + + EXPECT_EQ(parser.parseDuration(StringOptionParser::FieldIs::kMandatory), + std::chrono::minutes{45} + std::chrono::seconds{83}); + EXPECT_EQ(parser.parseExchanges(',', '\0'), ExchangeNames({ExchangeName("kraken"), ExchangeName("upbit")})); + + EXPECT_NO_THROW(parser.checkEndParsing()); +} + +TEST(StringOptionParserTest, ParseDurationOptional) { + StringOptionParser parser("binance,huobi_user1,34h 4500ms"); + + EXPECT_EQ(parser.parseDuration(StringOptionParser::FieldIs::kOptional), kUndefinedDuration); + EXPECT_EQ(parser.parseExchanges(',', '\0'), ExchangeNames({ExchangeName("binance"), ExchangeName("huobi", "user1")})); + + EXPECT_EQ(parser.parseDuration(StringOptionParser::FieldIs::kOptional), + std::chrono::hours{34} + std::chrono::milliseconds{4500}); + + EXPECT_NO_THROW(parser.checkEndParsing()); +} + } // namespace cct \ No newline at end of file diff --git a/src/http-request/include/request-retry.hpp b/src/http-request/include/request-retry.hpp index 0b3598f3..3055b6cf 100644 --- a/src/http-request/include/request-retry.hpp +++ b/src/http-request/include/request-retry.hpp @@ -1,8 +1,8 @@ #pragma once #include -#include #include +#include #include "cct_exception.hpp" #include "cct_json.hpp" @@ -10,7 +10,6 @@ #include "cct_type_traits.hpp" #include "curlhandle.hpp" #include "curloptions.hpp" -#include "curlpostdata.hpp" #include "durationstring.hpp" #include "query-retry-policy.hpp" #include "timedef.hpp" diff --git a/src/main/CMakeLists.txt b/src/main/CMakeLists.txt index 0bf15a3e..b2621218 100644 --- a/src/main/CMakeLists.txt +++ b/src/main/CMakeLists.txt @@ -24,6 +24,10 @@ endif() target_link_libraries(coincenter PUBLIC coincenter_engine) +if(CCT_ENABLE_PROTO) + target_link_libraries(coincenter PUBLIC protobuf::libprotobuf) +endif() + set_target_properties(coincenter PROPERTIES VERSION ${PROJECT_VERSION} COMPILE_DEFINITIONS_DEBUG "JSON_DEBUG;JSON_SAFE;JSON_ISO_STRICT" diff --git a/src/objects/CMakeLists.txt b/src/objects/CMakeLists.txt index 7bacf80b..385bbe35 100644 --- a/src/objects/CMakeLists.txt +++ b/src/objects/CMakeLists.txt @@ -99,6 +99,15 @@ add_unit_test( CCT_DISABLE_SPDLOG ) +add_unit_test( + time-window_test + test/time-window_test.cpp + LIBRARIES + coincenter_objects + DEFINITIONS + CCT_DISABLE_SPDLOG +) + add_unit_test( wallet_test test/wallet_test.cpp diff --git a/src/objects/include/automation-config.hpp b/src/objects/include/automation-config.hpp new file mode 100644 index 00000000..669bb8dc --- /dev/null +++ b/src/objects/include/automation-config.hpp @@ -0,0 +1,30 @@ +#pragma once + +#include + +#include "monetaryamount.hpp" +#include "timedef.hpp" + +namespace cct { +class AutomationConfig { + public: + AutomationConfig() noexcept = default; + + AutomationConfig(Duration loadChunkDuration, MonetaryAmount startBaseAmountEquivalent, + MonetaryAmount startQuoteAmountEquivalent) + : _loadChunkDuration(loadChunkDuration), + _startBaseAmountEquivalent(startBaseAmountEquivalent), + _startQuoteAmountEquivalent(startQuoteAmountEquivalent) {} + + Duration loadChunkDuration() const { return _loadChunkDuration; } + + MonetaryAmount startBaseAmountEquivalent() const { return _startBaseAmountEquivalent; } + + MonetaryAmount startQuoteAmountEquivalent() const { return _startQuoteAmountEquivalent; } + + private: + Duration _loadChunkDuration = std::chrono::weeks(1); + MonetaryAmount _startBaseAmountEquivalent; + MonetaryAmount _startQuoteAmountEquivalent; +}; +} // namespace cct \ No newline at end of file diff --git a/src/objects/include/coincentercommandtype.hpp b/src/objects/include/coincentercommandtype.hpp index cc1002dc..49967dc6 100644 --- a/src/objects/include/coincentercommandtype.hpp +++ b/src/objects/include/coincentercommandtype.hpp @@ -31,6 +31,10 @@ enum class CoincenterCommandType : int8_t { kWithdrawApply, kDustSweeper, + kMarketData, + kReplay, + kReplayMarkets, + kLast }; @@ -39,4 +43,4 @@ std::string_view CoincenterCommandTypeToString(CoincenterCommandType type); CoincenterCommandType CoincenterCommandTypeFromString(std::string_view str); bool IsAnyTrade(CoincenterCommandType type); -} // namespace cct \ No newline at end of file +} // namespace cct diff --git a/src/objects/include/exchange-names.hpp b/src/objects/include/exchange-names.hpp index 339a5e18..b9850579 100644 --- a/src/objects/include/exchange-names.hpp +++ b/src/objects/include/exchange-names.hpp @@ -2,6 +2,8 @@ #include +#include "cct_const.hpp" +#include "cct_fixedcapacityvector.hpp" #include "cct_smallvector.hpp" #include "cct_string.hpp" #include "exchangename.hpp" @@ -11,6 +13,8 @@ namespace cct { using ExchangeNameSpan = std::span; using ExchangeNames = SmallVector; +using PublicExchangeNameVector = FixedCapacityVector; + string ConstructAccumulatedExchangeNames(ExchangeNameSpan exchangeNames); } // namespace cct \ No newline at end of file diff --git a/src/objects/include/exchangeconfig.hpp b/src/objects/include/exchangeconfig.hpp index cf97aca7..008e77cf 100644 --- a/src/objects/include/exchangeconfig.hpp +++ b/src/objects/include/exchangeconfig.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include "apiquerytypeenum.hpp" @@ -18,7 +19,8 @@ namespace cct { class ExchangeConfig { public: - enum struct FeeType { kMaker, kTaker }; + enum class FeeType : int8_t { kMaker, kTaker }; + enum class MarketDataSerialization : int8_t { kYes, kNo }; struct APIUpdateFrequencies { Duration freq[api::kQueryTypeMax]; @@ -31,7 +33,8 @@ class ExchangeConfig { std::string_view acceptEncoding, int dustSweeperMaxNbTrades, log::level::level_enum requestsCallLogLevel, log::level::level_enum requestsAnswerLogLevel, bool multiTradeAllowedByDefault, bool validateDepositAddressesInFile, bool placeSimulateRealOrder, - bool validateApiKey, TradeConfig tradeConfig, HttpConfig httpConfig); + bool validateApiKey, TradeConfig tradeConfig, HttpConfig httpConfig, + MarketDataSerialization marketDataSerialization); /// Get a reference to the list of statically excluded currency codes to consider for the exchange, /// In both trading and withdrawal. @@ -106,6 +109,8 @@ class ExchangeConfig { PermanentCurlOptions::Builder curlOptionsBuilderBase(Api api) const; + bool withMarketDataSerialization() const { return _withMarketSerialization; } + private: CurrencyCodeSet _excludedCurrenciesAll; // Currencies will be completely ignored by the exchange CurrencyCodeSet _excludedCurrenciesWithdrawal; // Currencies unavailable for withdrawals @@ -127,5 +132,6 @@ class ExchangeConfig { bool _validateDepositAddressesInFile; bool _placeSimulateRealOrder; bool _validateApiKey; + bool _withMarketSerialization; }; } // namespace cct diff --git a/src/objects/include/generalconfig.hpp b/src/objects/include/generalconfig.hpp index 0fd87678..1a30b897 100644 --- a/src/objects/include/generalconfig.hpp +++ b/src/objects/include/generalconfig.hpp @@ -7,6 +7,7 @@ #include "logginginfo.hpp" #include "requestsconfig.hpp" #include "timedef.hpp" +#include "trading-config.hpp" namespace cct { @@ -18,13 +19,15 @@ class GeneralConfig { GeneralConfig() = default; - GeneralConfig(LoggingInfo &&loggingInfo, RequestsConfig &&requestsConfig, Duration fiatConversionQueryRate, - ApiOutputType apiOutputType); + GeneralConfig(LoggingInfo &&loggingInfo, RequestsConfig &&requestsConfig, TradingConfig &&tradingConfig, + Duration fiatConversionQueryRate, ApiOutputType apiOutputType); const LoggingInfo &loggingInfo() const { return _loggingInfo; } const RequestsConfig &requestsConfig() const { return _requestsConfig; } + const TradingConfig &tradingConfig() const { return _tradingConfig; } + ApiOutputType apiOutputType() const { return _apiOutputType; } Duration fiatConversionQueryRate() const { return _fiatConversionQueryRate; } @@ -32,6 +35,7 @@ class GeneralConfig { private: LoggingInfo _loggingInfo{LoggingInfo::WithLoggersCreation::kYes}; RequestsConfig _requestsConfig; + TradingConfig _tradingConfig; Duration _fiatConversionQueryRate = std::chrono::hours(8); ApiOutputType _apiOutputType = ApiOutputType::kFormattedTable; }; diff --git a/src/objects/include/generalconfigdefault.hpp b/src/objects/include/generalconfigdefault.hpp index f52cd5e7..448fe2f9 100644 --- a/src/objects/include/generalconfigdefault.hpp +++ b/src/objects/include/generalconfigdefault.hpp @@ -34,6 +34,17 @@ struct GeneralConfigDefault { "concurrency": { "nbMaxParallelRequests": 1 } + }, + "trading": { + "automation": { + "deserialization": { + "loadChunkDuration": "1w" + }, + "startingContext": { + "startBaseAmountEquivalent": "1000 EUR", + "startQuoteAmountEquivalent": "1000 EUR" + } + } } } )"_json; diff --git a/src/objects/include/marketorderbook.hpp b/src/objects/include/marketorderbook.hpp index 0fbabfda..0a1c5e4a 100644 --- a/src/objects/include/marketorderbook.hpp +++ b/src/objects/include/marketorderbook.hpp @@ -43,8 +43,14 @@ class MarketOrderBook { Market market() const { return _market; } bool empty() const { return _orders.empty(); } + int size() const { return _orders.size(); } + /// Check if data stored in this MarketOrderBook is valid. + /// This is especially useful for optional check of data after deserialization, + /// as for the standard case the market order book should be valid by design. + bool isValid() const; + bool isArtificiallyExtended() const { return _isArtificiallyExtended; } /// Get the highest bid price that a buyer is willing to pay @@ -185,6 +191,12 @@ class MarketOrderBook { /// 0.35 20 /// 0.34 23 + // To allow faster MarketOrderBook constructs + friend class MarketOrderBookConverter; + + MarketOrderBook(TimePoint timeStamp, Market market, AmountPriceVector&& orders, int32_t highestBidPricePos, + int32_t lowestAskPricePos, VolAndPriNbDecimals volAndPriNbDecimals); + MonetaryAmount amountAt(int pos) const { return MonetaryAmount(_orders[pos].amount, _market.base(), _volAndPriNbDecimals.volNbDecimals); } diff --git a/src/objects/include/publictrade.hpp b/src/objects/include/publictrade.hpp index 236e0e50..1929bc38 100644 --- a/src/objects/include/publictrade.hpp +++ b/src/objects/include/publictrade.hpp @@ -30,8 +30,7 @@ class PublicTrade { bool isValid() const; - /// 3 way operator - make compiler generate all 6 operators (including == and !=) - /// we order by time first, then amount, price, etc. Do not change the fields order! + /// We order by time first, then amount, price, etc. Do not change the fields order! std::strong_ordering operator<=>(const PublicTrade&) const noexcept = default; private: @@ -40,4 +39,5 @@ class PublicTrade { MonetaryAmount _price; TradeSide _side; }; + } // namespace cct diff --git a/src/objects/include/time-window.hpp b/src/objects/include/time-window.hpp new file mode 100644 index 00000000..4d641d90 --- /dev/null +++ b/src/objects/include/time-window.hpp @@ -0,0 +1,70 @@ +#pragma once + +#include +#include + +#include "cct_format.hpp" +#include "cct_invalid_argument_exception.hpp" +#include "timedef.hpp" + +namespace cct { + +/// Simple utility class representing a time window with a beginning time, and an end time. +/// The beginning time includes the corresponding time point, but the end time excludes it. +class TimeWindow { + public: + /// Create a zero duration time window starting from the zero-initialized time point. + TimeWindow() noexcept = default; + + /// Create a time window spanning from 'from' (included) to 'to' (excluded) time points. + TimeWindow(TimePoint from, TimePoint to) : _from(from), _to(to) { + if (_to < _from) { + throw invalid_argument("Invalid time window - 'from' should not be larger than 'to'"); + } + } + + /// Create a time window starting at 'from' with 'dur' duration. + TimeWindow(TimePoint from, Duration dur) : TimeWindow(from, from + dur) {} + + TimePoint from() const { return _from; } + + TimePoint to() const { return _to; } + + Duration duration() const { return _to - _from; } + + bool contains(TimePoint tp) const { return _from <= tp && tp < _to; } + + bool contains(int64_t unixTimestampInMs) const { return contains(TimePoint(milliseconds{unixTimestampInMs})); } + + bool contains(TimeWindow rhs) const { return _from <= rhs._from && rhs._to <= _to; } + + bool overlaps(TimeWindow rhs) const { return _from < rhs._to && rhs._from < _to; } + + string str() const; + + bool operator==(const TimeWindow&) const noexcept = default; + + private: + TimePoint _from; + TimePoint _to; +}; +} // namespace cct + +#ifndef CCT_DISABLE_SPDLOG +template <> +struct fmt::formatter { + constexpr auto parse(format_parse_context& ctx) -> decltype(ctx.begin()) { + const auto it = ctx.begin(); + const auto end = ctx.end(); + if (it != end && *it != '}') { + throw format_error("invalid format"); + } + return it; + } + + template + auto format(const cct::TimeWindow& timeWindow, FormatContext& ctx) const -> decltype(ctx.out()) { + return fmt::format_to(ctx.out(), "{}", timeWindow.str()); + } +}; +#endif diff --git a/src/objects/include/trading-config.hpp b/src/objects/include/trading-config.hpp new file mode 100644 index 00000000..b265e9dc --- /dev/null +++ b/src/objects/include/trading-config.hpp @@ -0,0 +1,19 @@ +#pragma once + +#include + +#include "automation-config.hpp" + +namespace cct { +class TradingConfig { + public: + TradingConfig() noexcept = default; + + TradingConfig(AutomationConfig automationConfig) : _automationConfig(std::move(automationConfig)) {} + + const AutomationConfig &automationConfig() const { return _automationConfig; } + + private: + AutomationConfig _automationConfig; +}; +} // namespace cct \ No newline at end of file diff --git a/src/objects/src/coincentercommandtype.cpp b/src/objects/src/coincentercommandtype.cpp index 8ec46ee1..bbb481ef 100644 --- a/src/objects/src/coincentercommandtype.cpp +++ b/src/objects/src/coincentercommandtype.cpp @@ -16,8 +16,7 @@ constexpr std::string_view kCommandTypeNames[] = { "Balance", "DepositInfo", "OrdersClosed", "OrdersOpened", "OrdersCancel", "RecentDeposits", "RecentWithdraws", "Trade", "Buy", "Sell", - "Withdraw", "DustSweeper", -}; + "Withdraw", "DustSweeper", "MarketData", "Replay", "ReplayMarkets"}; static_assert(std::size(kCommandTypeNames) == static_cast(CoincenterCommandType::kLast)); } // namespace @@ -51,4 +50,4 @@ bool IsAnyTrade(CoincenterCommandType type) { return false; } } -} // namespace cct \ No newline at end of file +} // namespace cct diff --git a/src/objects/src/exchangeconfig.cpp b/src/objects/src/exchangeconfig.cpp index 8e01941b..fe655229 100644 --- a/src/objects/src/exchangeconfig.cpp +++ b/src/objects/src/exchangeconfig.cpp @@ -66,7 +66,8 @@ ExchangeConfig::ExchangeConfig( const APIUpdateFrequencies &apiUpdateFrequencies, Duration publicAPIRate, Duration privateAPIRate, std::string_view acceptEncoding, int dustSweeperMaxNbTrades, log::level::level_enum requestsCallLogLevel, log::level::level_enum requestsAnswerLogLevel, bool multiTradeAllowedByDefault, bool validateDepositAddressesInFile, - bool placeSimulateRealOrder, bool validateApiKey, TradeConfig tradeConfig, HttpConfig httpConfig) + bool placeSimulateRealOrder, bool validateApiKey, TradeConfig tradeConfig, HttpConfig httpConfig, + MarketDataSerialization marketDataSerialization) : _excludedCurrenciesAll(std::move(excludedAllCurrencies)), _excludedCurrenciesWithdrawal(std::move(excludedCurrenciesWithdraw)), _preferredPaymentCurrencies(std::move(preferredPaymentCurrencies)), @@ -85,7 +86,8 @@ ExchangeConfig::ExchangeConfig( _multiTradeAllowedByDefault(multiTradeAllowedByDefault), _validateDepositAddressesInFile(validateDepositAddressesInFile), _placeSimulateRealOrder(placeSimulateRealOrder), - _validateApiKey(validateApiKey) { + _validateApiKey(validateApiKey), + _withMarketSerialization(marketDataSerialization == MarketDataSerialization::kYes) { if (dustSweeperMaxNbTrades > std::numeric_limits::max() || dustSweeperMaxNbTrades < 0) { throw exception("Invalid number of dust sweeper max trades '{}', should be in [0, {}]", dustSweeperMaxNbTrades, std::numeric_limits::max()); @@ -112,6 +114,7 @@ ExchangeConfig::ExchangeConfig( _validateDepositAddressesInFile ? kDepositAddressesFileName : ""); log::trace(" - Order placing in simulation : {}", _placeSimulateRealOrder ? "real, unmatchable" : "none"); log::trace(" - Validate API Key : {}", _validateApiKey ? "yes" : "no"); + log::trace(" - Market data serialization : {}", _withMarketSerialization ? "yes" : "no"); } if (_preferredPaymentCurrencies.empty()) { log::warn("{} list of preferred currencies is empty, buy and sell commands cannot perform trades", exchangeNameStr); diff --git a/src/objects/src/exchangeconfigdefault.hpp b/src/objects/src/exchangeconfigdefault.hpp index 3199cbd1..93e48258 100644 --- a/src/objects/src/exchangeconfigdefault.hpp +++ b/src/objects/src/exchangeconfigdefault.hpp @@ -59,6 +59,7 @@ struct ExchangeConfigDefault { "requestsCall": "info", "requestsAnswer": "trace" }, + "marketDataSerialization": true, "multiTradeAllowedByDefault": false, "placeSimulateRealOrder": false, "trade": { @@ -186,6 +187,7 @@ struct ExchangeConfigDefault { "requestsCall": "info", "requestsAnswer": "trace" }, + "marketDataSerialization": false, "multiTradeAllowedByDefault": true, "privateAPIRate": "1055ms", "publicAPIRate": "1236ms", diff --git a/src/objects/src/exchangeconfigmap.cpp b/src/objects/src/exchangeconfigmap.cpp index d9b5a472..16f9e6f6 100644 --- a/src/objects/src/exchangeconfigmap.cpp +++ b/src/objects/src/exchangeconfigmap.cpp @@ -58,6 +58,10 @@ ExchangeConfigMap ComputeExchangeConfigMap(std::string_view fileName, const json withdrawTopLevelOption.getBool(exchangeName, "validateDepositAddressesInFile"); const bool placeSimulatedRealOrder = queryTopLevelOption.getBool(exchangeName, "placeSimulateRealOrder"); const bool validateApiKey = queryTopLevelOption.getBool(exchangeName, "validateApiKey"); + const ExchangeConfig::MarketDataSerialization marketDataSerialization = + queryTopLevelOption.getBool(exchangeName, "marketDataSerialization") + ? ExchangeConfig::MarketDataSerialization::kYes + : ExchangeConfig::MarketDataSerialization::kNo; MonetaryAmountByCurrencySet dustAmountsThresholds( queryTopLevelOption.getMonetaryAmountsArray(exchangeName, "dustAmountsThreshold")); @@ -90,7 +94,7 @@ ExchangeConfigMap ComputeExchangeConfigMap(std::string_view fileName, const json std::move(dustAmountsThresholds), std::move(apiUpdateFrequencies), publicAPIRate, privateAPIRate, acceptEncoding, dustSweeperMaxNbTrades, requestsCallLogLevel, requestsAnswerLogLevel, multiTradeAllowedByDefault, validateDepositAddressesInFile, placeSimulatedRealOrder, - validateApiKey, std::move(tradeConfig), std::move(httpConfig))); + validateApiKey, std::move(tradeConfig), std::move(httpConfig), marketDataSerialization)); } // namespace cct // Print json unused values @@ -120,4 +124,4 @@ ExchangeConfigMap ComputeExchangeConfigMap(std::string_view fileName, const json return map; } -} // namespace cct \ No newline at end of file +} // namespace cct diff --git a/src/objects/src/generalconfig.cpp b/src/objects/src/generalconfig.cpp index c08c3270..96494ec1 100644 --- a/src/objects/src/generalconfig.cpp +++ b/src/objects/src/generalconfig.cpp @@ -11,13 +11,15 @@ #include "logginginfo.hpp" #include "requestsconfig.hpp" #include "timedef.hpp" +#include "trading-config.hpp" namespace cct { -GeneralConfig::GeneralConfig(LoggingInfo &&loggingInfo, RequestsConfig &&requestsConfig, +GeneralConfig::GeneralConfig(LoggingInfo &&loggingInfo, RequestsConfig &&requestsConfig, TradingConfig &&tradingConfig, Duration fiatConversionQueryRate, ApiOutputType apiOutputType) : _loggingInfo(std::move(loggingInfo)), _requestsConfig(std::move(requestsConfig)), + _tradingConfig(std::move(tradingConfig)), _fiatConversionQueryRate(fiatConversionQueryRate), _apiOutputType(apiOutputType) {} diff --git a/src/objects/src/marketorderbook.cpp b/src/objects/src/marketorderbook.cpp index 10e8be56..2d9b5339 100644 --- a/src/objects/src/marketorderbook.cpp +++ b/src/objects/src/marketorderbook.cpp @@ -207,6 +207,36 @@ MarketOrderBook::MarketOrderBook(TimePoint timeStamp, MonetaryAmount askPrice, M } } +MarketOrderBook::MarketOrderBook(TimePoint timeStamp, Market market, AmountPriceVector&& orders, + int32_t highestBidPricePos, int32_t lowestAskPricePos, + VolAndPriNbDecimals volAndPriNbDecimals) + : _time(timeStamp), + _market(market), + _orders(std::move(orders)), + _highestBidPricePos(highestBidPricePos), + _lowestAskPricePos(lowestAskPricePos), + _volAndPriNbDecimals(volAndPriNbDecimals) {} + +bool MarketOrderBook::isValid() const { + if (_orders.size() < 2U) { + log::error("Market order book is invalid as size is {}", _orders.size()); + return false; + } + if (!std::ranges::is_sorted(_orders, [](auto lhs, auto rhs) { return lhs.price < rhs.price; })) { + log::error("Market order book is invalid because orders are not sorted by price"); + return false; + } + if (std::ranges::adjacent_find(_orders, [](auto lhs, auto rhs) { return lhs.price == rhs.price; }) != _orders.end()) { + log::error("Market order book is invalid because of duplicate prices"); + return false; + } + if (!std::ranges::is_partitioned(_orders, [](auto amountPrice) { return amountPrice.amount > 0; })) { + log::error("Market order book is invalid because lines are not partitioned by asks / bids"); + return false; + } + return true; +} + std::optional MarketOrderBook::averagePrice() const { switch (_orders.size()) { case 0U: diff --git a/src/objects/src/publictrade.cpp b/src/objects/src/publictrade.cpp index f701d5c7..5fde8d63 100644 --- a/src/objects/src/publictrade.cpp +++ b/src/objects/src/publictrade.cpp @@ -1,5 +1,6 @@ #include "publictrade.hpp" +#include "cct_log.hpp" #include "cct_string.hpp" #include "timedef.hpp" #include "timestring.hpp" @@ -11,18 +12,23 @@ string PublicTrade::timeStr() const { return ToString(_time); } bool PublicTrade::isValid() const { if (time() == TimePoint{}) { + log::error("Public trade is invalid as no timestamp"); return false; } if (amount() <= 0 || amount().hasNeutralCurrency()) { + log::error("Public trade has an invalid amount"); return false; } if (price() <= 0 || price().hasNeutralCurrency()) { + log::error("Public trade has an invalid price"); return false; } if (amount().currencyCode() == price().currencyCode()) { + log::error("Public trade has an invalid market"); return false; } if (side() != TradeSide::kBuy && side() != TradeSide::kSell) { + log::error("Public trade has an invalid trade side"); return false; } return true; diff --git a/src/objects/src/time-window.cpp b/src/objects/src/time-window.cpp new file mode 100644 index 00000000..088281f6 --- /dev/null +++ b/src/objects/src/time-window.cpp @@ -0,0 +1,15 @@ +#include "time-window.hpp" + +#include "timestring.hpp" + +namespace cct { +string TimeWindow::str() const { + string ret; + ret.push_back('['); + ret.append(ToString(from(), kTimeYearToSecondSpaceSeparatedFormat)); + ret.append(" -> "); + ret.append(ToString(to(), kTimeYearToSecondSpaceSeparatedFormat)); + ret.push_back(')'); + return ret; +} +} // namespace cct \ No newline at end of file diff --git a/src/objects/test/marketorderbook_test.cpp b/src/objects/test/marketorderbook_test.cpp index 6997c26d..ba64db64 100644 --- a/src/objects/test/marketorderbook_test.cpp +++ b/src/objects/test/marketorderbook_test.cpp @@ -37,6 +37,12 @@ constexpr bool operator==(const AmountPrice &lhs, const AmountPrice &rhs) { return lhs.amount == rhs.amount && lhs.price == rhs.price; } +TEST(MarketOrderBookTest, DefaultConstructor) { + MarketOrderBook marketOrderBook; + + EXPECT_FALSE(marketOrderBook.isValid()); +} + TEST(MarketOrderBookTest, Basic) { EXPECT_TRUE(MarketOrderBook(Clock::now(), Market("ETH", "EUR"), {}).empty()); } class MarketOrderBookTestCase1 : public ::testing::Test { @@ -53,6 +59,8 @@ class MarketOrderBookTestCase1 : public ::testing::Test { OrderBookLine::Type::kAsk)})}; }; +TEST_F(MarketOrderBookTestCase1, IsValid) { EXPECT_TRUE(marketOrderBook.isValid()); } + TEST_F(MarketOrderBookTestCase1, NumberOfElements) { EXPECT_EQ(marketOrderBook.size(), 5); EXPECT_EQ(marketOrderBook.nbAskPrices(), 3); @@ -176,6 +184,8 @@ class MarketOrderBookTestDuplicatedLines : public ::testing::Test { OrderBookLine::Type::kAsk)})}; }; +TEST_F(MarketOrderBookTestDuplicatedLines, IsValid) { EXPECT_TRUE(marketOrderBook.isValid()); } + TEST_F(MarketOrderBookTestDuplicatedLines, NumberOfElements) { EXPECT_EQ(marketOrderBook.size(), 5); EXPECT_EQ(marketOrderBook.nbAskPrices(), 3); @@ -211,6 +221,8 @@ class MarketOrderBookTestCase2 : public ::testing::Test { OrderBookLine::Type::kBid)})}; }; +TEST_F(MarketOrderBookTestCase2, IsValid) { EXPECT_TRUE(marketOrderBook.isValid()); } + TEST_F(MarketOrderBookTestCase2, NbDecimals) { const auto [volNbDecimals, priNbDecimals] = marketOrderBook.volAndPriNbDecimals(); @@ -288,6 +300,8 @@ class MarketOrderBookTestCase3 : public ::testing::Test { MonetaryAmount("0.000007080", "BTC"), OrderBookLine::Type::kBid)})}; }; +TEST_F(MarketOrderBookTestCase3, IsValid) { EXPECT_TRUE(marketOrderBook.isValid()); } + TEST_F(MarketOrderBookTestCase3, Convert) { EXPECT_EQ(marketOrderBook.convert(MonetaryAmount("600000", "XLM")), std::nullopt); EXPECT_EQ(marketOrderBook.convert(MonetaryAmount(3, "BTC")), std::nullopt); @@ -324,6 +338,8 @@ class MarketOrderBookTestCaseExtended1 : public ::testing::Test { 50}; }; +TEST_F(MarketOrderBookTestCaseExtended1, IsValid) { EXPECT_TRUE(marketOrderBook.isValid()); } + TEST_F(MarketOrderBookTestCaseExtended1, LimitPrice) { EXPECT_EQ(marketOrderBook.highestBidPrice(), MonetaryAmount("2300.4 EUR")); EXPECT_EQ(marketOrderBook.lowestAskPrice(), MonetaryAmount("2300.45 EUR")); @@ -339,6 +355,8 @@ TEST(MarketOrderBookExtendedTest, ComputeVolAndPriNbDecimalsFromTickerInfo) { MonetaryAmount("193.0900000000078 ADA"), MonetaryAmount("12355.00002486 XLM"), MonetaryAmount("504787104.7801 ADA"), {4, 8}, 10); + ASSERT_TRUE(marketOrderBook.isValid()); + EXPECT_EQ(marketOrderBook.highestBidPrice(), MonetaryAmount("12355.00002486 XLM")); EXPECT_EQ(marketOrderBook.lowestAskPrice(), MonetaryAmount("12355.00002487 XLM")); } diff --git a/src/objects/test/time-window_test.cpp b/src/objects/test/time-window_test.cpp new file mode 100644 index 00000000..42f6b04a --- /dev/null +++ b/src/objects/test/time-window_test.cpp @@ -0,0 +1,140 @@ +#include "time-window.hpp" + +#include + +#include + +#include "cct_invalid_argument_exception.hpp" +#include "timedef.hpp" + +namespace cct { +class TimeWindowTest : public ::testing::Test { + protected: + TimePoint tp1{milliseconds{std::numeric_limits::max() / 10000000}}; + TimePoint tp2{milliseconds{std::numeric_limits::max() / 9900000}}; + TimePoint tp3{milliseconds{std::numeric_limits::max() / 9800000}}; + TimePoint tp4{milliseconds{std::numeric_limits::max() / 9500000}}; + TimePoint tp5{milliseconds{std::numeric_limits::max() / 9000000}}; + + Duration dur1{seconds{100}}; + Duration dur2{seconds{1000}}; + Duration dur3{seconds{10000}}; +}; + +TEST_F(TimeWindowTest, DefaultConstructor) { + TimeWindow tw; + + EXPECT_EQ(tw.from(), TimePoint{}); + EXPECT_EQ(tw.to(), TimePoint{}); + EXPECT_EQ(tw.duration(), milliseconds{}); + EXPECT_FALSE(tw.contains(TimePoint{})); + EXPECT_FALSE(tw.contains(0)); + EXPECT_TRUE(tw.contains(tw)); +} + +TEST_F(TimeWindowTest, InvalidTimeWindowFromTime) { EXPECT_THROW(TimeWindow(tp2, tp1), invalid_argument); } +TEST_F(TimeWindowTest, InvalidTimeWindowFromDuration) { EXPECT_THROW(TimeWindow(tp1, tp1 - tp2), invalid_argument); } + +TEST_F(TimeWindowTest, DurationConstructor) { + TimeWindow tw(tp1, tp2 - tp1); + + EXPECT_EQ(tw, TimeWindow(tp1, tp2)); +} + +TEST_F(TimeWindowTest, Duration) { + TimeWindow tw(tp1, tp2); + + EXPECT_EQ(tw.duration(), tp2 - tp1); +} + +TEST_F(TimeWindowTest, ContainsTimePoint) { + TimeWindow tw1(tp1, tp2); + + EXPECT_TRUE(tw1.contains(tp1)); + EXPECT_TRUE(tw1.contains(tp1 + dur1)); + EXPECT_FALSE(tw1.contains(tp2)); + EXPECT_FALSE(tw1.contains(tp3)); +} + +TEST_F(TimeWindowTest, ContainsTimeWindow) { + // [ ] + // [ ] + TimeWindow tw1(tp1, tp4); + TimeWindow tw2(tp2, tp3); + + EXPECT_TRUE(tw1.contains(tw1)); + EXPECT_TRUE(tw1.overlaps(tw1)); + + EXPECT_TRUE(tw1.overlaps(tw2)); + EXPECT_TRUE(tw1.contains(tw2)); + + EXPECT_TRUE(tw2.overlaps(tw1)); + EXPECT_FALSE(tw2.contains(tw1)); +} + +TEST_F(TimeWindowTest, OverlapNominal) { + // [ ] + // [ ] + TimeWindow tw1(tp2, tp4); + TimeWindow tw2(tp1, tp3); + + EXPECT_TRUE(tw1.overlaps(tw2)); + EXPECT_FALSE(tw1.contains(tw2)); + + EXPECT_TRUE(tw2.overlaps(tw1)); + EXPECT_FALSE(tw2.contains(tw1)); +} + +TEST_F(TimeWindowTest, OverlapEqualTo) { + // [ ] + // [ ] + TimeWindow tw1(tp1, tp3); + TimeWindow tw2(tp2, tp3); + + EXPECT_TRUE(tw1.overlaps(tw2)); + EXPECT_TRUE(tw1.contains(tw2)); + + EXPECT_TRUE(tw2.overlaps(tw1)); + EXPECT_FALSE(tw2.contains(tw1)); +} + +TEST_F(TimeWindowTest, OverlapEqualFrom) { + // [ ] + // [ ] + TimeWindow tw1(tp1, tp3); + TimeWindow tw2(tp1, tp2); + + EXPECT_TRUE(tw1.overlaps(tw2)); + EXPECT_TRUE(tw1.contains(tw2)); + + EXPECT_TRUE(tw2.overlaps(tw1)); + EXPECT_FALSE(tw2.contains(tw1)); +} + +TEST_F(TimeWindowTest, NoOverlapNominal) { + // [ ] + // [ ] + TimeWindow tw1(tp1, tp2); + TimeWindow tw2(tp3, tp4); + + EXPECT_FALSE(tw1.overlaps(tw2)); + EXPECT_FALSE(tw1.contains(tw2)); + + EXPECT_FALSE(tw2.overlaps(tw1)); + EXPECT_FALSE(tw2.contains(tw1)); +} + +TEST_F(TimeWindowTest, NoOverlapEqual) { + // [ ] + // [ ] + TimeWindow tw1(tp1, tp3); + TimeWindow tw2(tp3, tp4); + + EXPECT_FALSE(tw1.overlaps(tw2)); + EXPECT_FALSE(tw1.contains(tw2)); + + EXPECT_FALSE(tw2.overlaps(tw1)); + EXPECT_FALSE(tw2.contains(tw1)); +} + +} // namespace cct diff --git a/src/serialization/CMakeLists.txt b/src/serialization/CMakeLists.txt new file mode 100644 index 00000000..4054e6dc --- /dev/null +++ b/src/serialization/CMakeLists.txt @@ -0,0 +1,70 @@ +if(CCT_ENABLE_PROTO) + aux_source_directory(src SERIALIZATION_SRC) + + list(APPEND SERIALIZATION_SRC "${CMAKE_CURRENT_LIST_DIR}/proto/market-order-book-timed-data.proto") + list(APPEND SERIALIZATION_SRC "${CMAKE_CURRENT_LIST_DIR}/proto/trade-data.proto") +else() + set(SERIALIZATION_SRC "src/dummy-market-data-serializer.cpp" "src/dummy-market-data-deserializer.cpp") +endif() + +add_library(coincenter_serialization OBJECT ${SERIALIZATION_SRC}) + +target_include_directories(coincenter_serialization PUBLIC include) +target_link_libraries(coincenter_serialization PUBLIC coincenter_objects) + +if(CCT_ENABLE_PROTO) + set(PROTO_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/generated") + + target_include_directories(coincenter_serialization PUBLIC "$") + + target_link_libraries(coincenter_serialization PUBLIC protobuf::libprotobuf) + + protobuf_generate( + TARGET coincenter_serialization + IMPORT_DIRS "${CMAKE_CURRENT_LIST_DIR}/proto" + PROTOC_OUT_DIR "${PROTO_BINARY_DIR}" + ) + + add_unit_test( + continuous-iterator_test + test/continuous-iterator_test.cpp + LIBRARIES + coincenter_serialization + ) + + add_unit_test( + proto-market-order-book_test + test/proto-market-order-book_test.cpp + LIBRARIES + coincenter_serialization + ) + + add_unit_test( + proto-multiple-messages-handler_test + test/proto-multiple-messages-handler_test.cpp + LIBRARIES + coincenter_serialization + ) + + add_unit_test( + proto-public-trade_test + test/proto-public-trade_test.cpp + LIBRARIES + coincenter_serialization + ) + + add_unit_test( + proto-serialization-and-deserialization_test + test/proto-serialization-and-deserialization_test.cpp + LIBRARIES + coincenter_serialization + ) + + add_unit_test( + serialization-tools_test + test/serialization-tools_test.cpp + LIBRARIES + coincenter_serialization + ) + +endif() \ No newline at end of file diff --git a/src/serialization/include/abstract-market-data-deserializer.hpp b/src/serialization/include/abstract-market-data-deserializer.hpp new file mode 100644 index 00000000..fd66d9d1 --- /dev/null +++ b/src/serialization/include/abstract-market-data-deserializer.hpp @@ -0,0 +1,24 @@ +#pragma once + +#include "market-order-book-vector.hpp" +#include "market-timestamp-set.hpp" +#include "market.hpp" +#include "public-trade-vector.hpp" +#include "time-window.hpp" + +namespace cct { + +class AbstractMarketDataDeserializer { + public: + virtual ~AbstractMarketDataDeserializer() = default; + + virtual MarketTimestampSet pullMarketOrderBooksMarkets(TimeWindow timeWindow) = 0; + + virtual MarketTimestampSet pullTradeMarkets(TimeWindow timeWindow) = 0; + + virtual MarketOrderBookVector pullMarketOrderBooks(Market market, TimeWindow timeWindow) = 0; + + virtual PublicTradeVector pullTrades(Market market, TimeWindow timeWindow) = 0; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/include/abstract-market-data-serializer.hpp b/src/serialization/include/abstract-market-data-serializer.hpp new file mode 100644 index 00000000..65cc1b79 --- /dev/null +++ b/src/serialization/include/abstract-market-data-serializer.hpp @@ -0,0 +1,24 @@ +#pragma once + +#include + +#include "market.hpp" +#include "publictrade.hpp" + +namespace cct { + +class MarketOrderBook; + +class AbstractMarketDataSerializer { + public: + virtual ~AbstractMarketDataSerializer() = default; + + /// Push market order book in the MarketDataSerializer. + virtual void push(const MarketOrderBook &marketOrderBook) = 0; + + /// Push public trades in the MarketDataSerializer. + /// They should come from the same market. + virtual void push(Market market, std::span publicTrades) = 0; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/include/continuous-iterator.hpp b/src/serialization/include/continuous-iterator.hpp new file mode 100644 index 00000000..91b71cd1 --- /dev/null +++ b/src/serialization/include/continuous-iterator.hpp @@ -0,0 +1,23 @@ +#pragma once + +namespace cct { + +/// Simple utility class that may iterate in both directions. +class ContinuousIterator { + public: + ContinuousIterator(int from, int to) : _to(to), _curr(from), _incr(to < from ? -1 : 1) {} + + bool hasNext() const { return _curr != _to + _incr; } + + auto next() { + _curr += _incr; + return _curr - _incr; + } + + private: + int _to; + int _curr; + int _incr; +}; + +} // namespace cct diff --git a/src/serialization/include/dummy-market-data-deserializer.hpp b/src/serialization/include/dummy-market-data-deserializer.hpp new file mode 100644 index 00000000..e3b0227c --- /dev/null +++ b/src/serialization/include/dummy-market-data-deserializer.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include + +#include "abstract-market-data-deserializer.hpp" +#include "market-order-book-vector.hpp" +#include "market-timestamp-set.hpp" +#include "market.hpp" +#include "public-trade-vector.hpp" +#include "time-window.hpp" + +namespace cct { + +class DummyMarketDataDeserializer : public AbstractMarketDataDeserializer { + public: + DummyMarketDataDeserializer([[maybe_unused]] std::string_view dataDir, + [[maybe_unused]] std::string_view exchangeName); + + MarketTimestampSet pullMarketOrderBooksMarkets([[maybe_unused]] TimeWindow timeWindow) override; + + MarketTimestampSet pullTradeMarkets([[maybe_unused]] TimeWindow timeWindow) override; + + MarketOrderBookVector pullMarketOrderBooks([[maybe_unused]] Market market, + [[maybe_unused]] TimeWindow timeWindow) override; + + PublicTradeVector pullTrades([[maybe_unused]] Market market, [[maybe_unused]] TimeWindow timeWindow) override; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/include/dummy-market-data-serializer.hpp b/src/serialization/include/dummy-market-data-serializer.hpp new file mode 100644 index 00000000..2e42e287 --- /dev/null +++ b/src/serialization/include/dummy-market-data-serializer.hpp @@ -0,0 +1,26 @@ +#pragma once + +#include +#include + +#include "abstract-market-data-serializer.hpp" +#include "market-timestamp-set.hpp" +#include "publictrade.hpp" + +namespace cct { + +class MarketOrderBook; + +/// Implementation of a market data serializer that does nothing. +/// Useful if coincenter is not compiled with protobuf support. +class DummyMarketDataSerializer : public AbstractMarketDataSerializer { + public: + DummyMarketDataSerializer(std::string_view dataDir, const MarketTimestampSets &lastWrittenObjectsMarketTimestamp, + std::string_view exchangeName); + + void push(const MarketOrderBook &marketOrderBook) override; + + void push(Market market, std::span publicTrades) override; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/include/market-timestamp-set.hpp b/src/serialization/include/market-timestamp-set.hpp new file mode 100644 index 00000000..afda7143 --- /dev/null +++ b/src/serialization/include/market-timestamp-set.hpp @@ -0,0 +1,15 @@ +#pragma once + +#include "cct_flatset.hpp" +#include "market-timestamp.hpp" + +namespace cct { + +using MarketTimestampSet = FlatSet; + +struct MarketTimestampSets { + MarketTimestampSet orderBooksMarkets; + MarketTimestampSet tradesMarkets; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/include/market-timestamp.hpp b/src/serialization/include/market-timestamp.hpp new file mode 100644 index 00000000..67317c69 --- /dev/null +++ b/src/serialization/include/market-timestamp.hpp @@ -0,0 +1,17 @@ +#pragma once + +#include + +#include "market.hpp" +#include "timedef.hpp" + +namespace cct { + +struct MarketTimestamp { + Market market; + TimePoint timePoint; + + std::strong_ordering operator<=>(const MarketTimestamp &) const noexcept = default; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/include/proto-constants.hpp b/src/serialization/include/proto-constants.hpp new file mode 100644 index 00000000..e7a59523 --- /dev/null +++ b/src/serialization/include/proto-constants.hpp @@ -0,0 +1,15 @@ +#pragma once + +#include +#include + +namespace cct { + +enum class ProtobufObject : int8_t { kMarketOrderBook, kTrade }; + +static constexpr std::string_view kBinProtobufExtension = ".binpb"; + +static constexpr std::string_view kSubPathMarketOrderBook = "market-order-book"; +static constexpr std::string_view kSubPathTrades = "trades"; + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/include/proto-deserializer.hpp b/src/serialization/include/proto-deserializer.hpp new file mode 100644 index 00000000..cd61d9d0 --- /dev/null +++ b/src/serialization/include/proto-deserializer.hpp @@ -0,0 +1,184 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "cct_format.hpp" +#include "cct_log.hpp" +#include "cct_vector.hpp" +#include "continuous-iterator.hpp" +#include "market-timestamp-set.hpp" +#include "market-timestamp.hpp" +#include "market.hpp" +#include "proto-multiple-messages-handler.hpp" +#include "serialization-tools.hpp" +#include "time-window.hpp" +#include "timedef.hpp" + +namespace cct { + +template +class ProtobufObjectsDeserializer { + public: + using CoincenterObjectType = std::invoke_result_t; + + explicit ProtobufObjectsDeserializer(std::string exchangeSerializedDataPath) noexcept + : _exchangeSerializedDataPath(std::move(exchangeSerializedDataPath)) {} + + /// Load all markets found on disk which has some data in the given time window + MarketTimestampSet listMarkets(TimeWindow timeWindow) { + vector marketTimestamps; + + if (std::filesystem::is_directory(_exchangeSerializedDataPath)) { + for (const auto& marketDirectory : std::filesystem::directory_iterator(_exchangeSerializedDataPath)) { + auto ts = loadMarket(marketDirectory, timeWindow, ActionType::kCheckPresence).second; + if (ts != TimePoint{}) { + const auto& marketPath = marketDirectory.path(); + const auto marketStr = marketPath.filename().string(); + + marketTimestamps.emplace_back(Market{marketStr}, ts); + } + } + } + + return MarketTimestampSet(std::move(marketTimestamps)); + } + + /// Load all data found on disk for given market for the time window + vector loadMarket(Market market, TimeWindow timeWindow) { + std::string marketPathStr(_exchangeSerializedDataPath); + marketPathStr.push_back('/'); + marketPathStr.append(market.str()); + + std::filesystem::path marketPath(marketPathStr); + + return loadMarket(std::filesystem::directory_entry(marketPath), timeWindow); + } + + /// Load all data found on disk for given market for the time window + vector loadMarket(const std::filesystem::directory_entry& marketDirectory, + TimeWindow timeWindow) { + return loadMarket(marketDirectory, timeWindow, ActionType::kLoad).first; + } + + private: + static bool ValidateTimestamp(const ProtobufObjType& msg, TimeWindow timeWindow) { + if (!msg.has_unixtimestampinms()) { + log::error("Invalid data loaded for protobuf object, no unix timestamp set"); + return false; + } + return timeWindow.contains(msg.unixtimestampinms()); + } + + enum class ActionType : int8_t { kLoad, kCheckPresence }; + + static ContinuousIterator CreateIt(int from, int to, ActionType actionType) { + if (actionType == ActionType::kCheckPresence) { + std::swap(from, to); + } + return {from, to}; + } + + /// Load all data found on disk for given market for the time window + auto loadMarket(const std::filesystem::directory_entry& marketDirectory, TimeWindow timeWindow, + ActionType actionType) { + std::pair, TimePoint> ret; + if (!marketDirectory.is_directory()) { + return ret; + } + const auto fromDays = std::chrono::floor(timeWindow.from()); + const std::chrono::year_month_day fromYmd{fromDays}; + const std::chrono::hh_mm_ss fromTime{std::chrono::floor(timeWindow.from() - fromDays)}; + + const auto toDays = std::chrono::floor(timeWindow.to()); + const std::chrono::year_month_day toYmd{toDays}; + const std::chrono::hh_mm_ss toTime{std::chrono::floor(timeWindow.to() - toDays)}; + + const auto& marketPath = marketDirectory.path(); + const auto marketFilename = marketPath.filename(); + const Market market(marketFilename.string()); + + ProtoToCoincenterObjectsFunc converter(market); + + const int fromYear = static_cast(fromYmd.year()); + const int toYear = static_cast(toYmd.year()); + + for (ContinuousIterator yearIt = CreateIt(fromYear, toYear, actionType); yearIt.hasNext();) { + const auto year = yearIt.next(); + const auto yearPath = marketPath / format("{:04}", year); + if (!std::filesystem::is_directory(yearPath)) { + continue; + } + const bool isYearFromExtremity = year == fromYear; + const bool isYearToExtremity = year == toYear; + const auto fromMonth = isYearFromExtremity ? static_cast(static_cast(fromYmd.month())) : 1; + const auto toMonth = isYearToExtremity ? static_cast(static_cast(toYmd.month())) : 12; + + for (ContinuousIterator monthIt = CreateIt(fromMonth, toMonth, actionType); monthIt.hasNext();) { + const auto month = monthIt.next(); + const auto monthPath = yearPath / format("{:02}", month); + if (!std::filesystem::is_directory(monthPath)) { + continue; + } + const bool isMonthFromExtremity = isYearFromExtremity && month == fromMonth; + const bool isMonthToExtremity = isYearToExtremity && month == toMonth; + const auto fromDay = isMonthFromExtremity ? static_cast(static_cast(fromYmd.day())) : 1; + const auto toDay = isMonthToExtremity ? static_cast(static_cast(toYmd.day())) : 31; + + for (ContinuousIterator dayIt = CreateIt(fromDay, toDay, actionType); dayIt.hasNext();) { + const auto day = dayIt.next(); + const auto dayPath = monthPath / format("{:02}", day); + if (!std::filesystem::is_directory(dayPath)) { + continue; + } + + const bool isDayFromExtremity = isMonthFromExtremity && day == fromDay; + const bool isDayToExtremity = isMonthToExtremity && day == toDay; + const auto fromHour = isDayFromExtremity ? static_cast(fromTime.hours().count()) : 0; + const auto toHour = isDayToExtremity ? static_cast(toTime.hours().count()) : 23; + + for (ContinuousIterator hourOfDayIt = CreateIt(fromHour, toHour, actionType); hourOfDayIt.hasNext();) { + const auto hourOfDay = hourOfDayIt.next(); + const auto hourPath = dayPath / ComputeProtoFileName(hourOfDay); + if (!std::filesystem::exists(hourPath)) { + continue; + } + + decltype(std::declval().unixtimestampinms()) lastTs = 0; + + std::ifstream ifs(hourPath, std::ios::in | std::ios::binary); + for (ProtobufMessagesReader protobufMessagesReader(ifs); protobufMessagesReader.hasNext();) { + auto msg = protobufMessagesReader.next(); + if (!ValidateTimestamp(msg, timeWindow)) { + continue; + } + + // In Check presence mode, we read all the file to retrieve the latest timestamp. + // There's no other way to do it. + lastTs = msg.unixtimestampinms(); + + if (actionType == ActionType::kLoad) { + ret.first.push_back(converter(std::move(msg))); + } + } + + ret.second = TimePoint{milliseconds{static_cast(lastTs)}}; + if (actionType == ActionType::kCheckPresence) { + return ret; + } + } + } + } + } + return ret; + } + + std::string _exchangeSerializedDataPath; +}; + +} // namespace cct diff --git a/src/serialization/include/proto-market-data-deserializer.hpp b/src/serialization/include/proto-market-data-deserializer.hpp new file mode 100644 index 00000000..cea45845 --- /dev/null +++ b/src/serialization/include/proto-market-data-deserializer.hpp @@ -0,0 +1,36 @@ +#pragma once + +#include + +#include "abstract-market-data-deserializer.hpp" +#include "market-order-book-timed-data.pb.h" +#include "market-order-book-vector.hpp" +#include "market-timestamp-set.hpp" +#include "market.hpp" +#include "proto-deserializer.hpp" +#include "proto-market-order-book.hpp" +#include "proto-public-trade.hpp" +#include "public-trade-vector.hpp" +#include "time-window.hpp" +#include "trade-data.pb.h" + +namespace cct { + +class ProtoMarketDataDeserializer : public AbstractMarketDataDeserializer { + public: + ProtoMarketDataDeserializer(std::string_view dataDir, std::string_view exchangeName); + + MarketTimestampSet pullMarketOrderBooksMarkets(TimeWindow timeWindow) override; + + MarketTimestampSet pullTradeMarkets(TimeWindow timeWindow) override; + + MarketOrderBookVector pullMarketOrderBooks(Market market, TimeWindow timeWindow) override; + + PublicTradeVector pullTrades(Market market, TimeWindow timeWindow) override; + + private: + ProtobufObjectsDeserializer<::objects::MarketOrderBookTimedData, MarketOrderBookConverter> + _marketOrderBookDeserializer; + ProtobufObjectsDeserializer<::objects::TradeData, TradeDataToPublicTradeConverter> _tradeDataDeserializer; +}; +} // namespace cct \ No newline at end of file diff --git a/src/serialization/include/proto-market-data-serializer.hpp b/src/serialization/include/proto-market-data-serializer.hpp new file mode 100644 index 00000000..26ed3d38 --- /dev/null +++ b/src/serialization/include/proto-market-data-serializer.hpp @@ -0,0 +1,34 @@ +#pragma once + +#include +#include + +#include "abstract-market-data-serializer.hpp" +#include "market-order-book-timed-data.pb.h" +#include "market-timestamp-set.hpp" +#include "market.hpp" +#include "marketorderbook.hpp" +#include "proto-serializer.hpp" +#include "proto-trade-data-compare.hpp" +#include "publictrade.hpp" +#include "trade-data.pb.h" + +namespace cct { + +/// This class is responsible of managing the periodic writes to disk of timed market data, for a given exchange. +/// This class is not thread safe +class ProtoMarketDataSerializer : public AbstractMarketDataSerializer { + public: + ProtoMarketDataSerializer(std::string_view dataDir, const MarketTimestampSets &lastWrittenObjectsMarketTimestamp, + std::string_view exchangeName); + + void push(const MarketOrderBook &marketOrderBook) override; + + void push(Market market, std::span publicTrades) override; + + private: + ProtobufObjectsSerializer<::objects::MarketOrderBookTimedData> _marketOrderBookSerializer; + ProtobufObjectsSerializer<::objects::TradeData, TradeDataComp, TradeDataEqual> _tradesSerializer; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/include/proto-market-order-book.hpp b/src/serialization/include/proto-market-order-book.hpp new file mode 100644 index 00000000..65132ec9 --- /dev/null +++ b/src/serialization/include/proto-market-order-book.hpp @@ -0,0 +1,21 @@ +#pragma once + +#include "market-order-book-timed-data.pb.h" +#include "market.hpp" +#include "marketorderbook.hpp" + +namespace cct { + +::objects::MarketOrderBookTimedData CreateMarketOrderBookTimedData(const MarketOrderBook &marketOrderBook); + +class MarketOrderBookConverter { + public: + explicit MarketOrderBookConverter(Market market) : _market(market) {} + + MarketOrderBook operator()(const ::objects::MarketOrderBookTimedData &marketOrderBookTimedData); + + private: + Market _market; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/include/proto-multiple-messages-handler.hpp b/src/serialization/include/proto-multiple-messages-handler.hpp new file mode 100644 index 00000000..d32c550e --- /dev/null +++ b/src/serialization/include/proto-multiple-messages-handler.hpp @@ -0,0 +1,79 @@ +#pragma once + +#include + +#include +#include +#include +#include + +#include "cct_exception.hpp" +#include "cct_log.hpp" + +namespace cct { + +class ProtobufMessagesReader { + public: + explicit ProtobufMessagesReader(std::istream& is) : _is(is), _iis(&_is), _cis(&_iis) {} + + bool hasNext() { return _cis.ReadVarint64(&_nextSize); } + + template + MsgT next() { + MsgT msg; + auto msgLimit = _cis.PushLimit(_nextSize); + if (!msg.ParseFromCodedStream(&_cis)) { + log::error("Error reading single protobuf message of size {}", _nextSize); + } + _cis.PopLimit(msgLimit); + return msg; + } + + private: + std::istream& _is; + ::google::protobuf::io::IstreamInputStream _iis; + ::google::protobuf::io::CodedInputStream _cis; + uint64_t _nextSize{}; +}; + +template +class ProtobufMessagesWriter { + public: + void open(OStreamType&& newOs) { + // reverse destroy streams to flush latest data. Recreate the streams after creation of new ofstream + _cos.reset(); + _oos.reset(); + _os = std::move(newOs); + _oos = std::make_unique<::google::protobuf::io::OstreamOutputStream>(&_os); + _cos = std::make_unique<::google::protobuf::io::CodedOutputStream>(_oos.get()); + } + + template + void write(const MsgT& msg) { + if (!_cos) { + throw exception("ProtobufMessagesWriter::open should have been called first"); + } + + _cos->WriteVarint64(msg.ByteSizeLong()); + + if (!msg.SerializeToCodedStream(_cos.get())) { + log::error("Failed to serialize to coded stream"); + } + } + + OStreamType flush() { + _cos.reset(); + _oos.reset(); + + OStreamType ret(std::move(_os)); + _os = OStreamType(); + return ret; + } + + private: + OStreamType _os; + std::unique_ptr<::google::protobuf::io::OstreamOutputStream> _oos; + std::unique_ptr<::google::protobuf::io::CodedOutputStream> _cos; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/include/proto-public-trade.hpp b/src/serialization/include/proto-public-trade.hpp new file mode 100644 index 00000000..3a3af689 --- /dev/null +++ b/src/serialization/include/proto-public-trade.hpp @@ -0,0 +1,21 @@ +#pragma once + +#include "market.hpp" +#include "publictrade.hpp" +#include "trade-data.pb.h" + +namespace cct { + +::objects::TradeData ConvertPublicTradeToTradeData(const PublicTrade &publicTrade); + +class TradeDataToPublicTradeConverter { + public: + explicit TradeDataToPublicTradeConverter(Market market) : _market(market) {} + + PublicTrade operator()(const ::objects::TradeData &tradeData) const; + + private: + Market _market; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/include/proto-serializer.hpp b/src/serialization/include/proto-serializer.hpp new file mode 100644 index 00000000..53ae6549 --- /dev/null +++ b/src/serialization/include/proto-serializer.hpp @@ -0,0 +1,257 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cct_log.hpp" +#include "cct_vector.hpp" +#include "durationstring.hpp" +#include "market-timestamp-set.hpp" +#include "market.hpp" +#include "proto-multiple-messages-handler.hpp" +#include "serialization-tools.hpp" +#include "timedef.hpp" + +namespace cct { + +/// Class responsible to accumulate protobuf objects in memory and perform regular flushes of its data to the disk. +/// Data is accumulated by Market and will write to following files (from subPath): +/// 'BASECUR-QUOTECUR/YYYY/MM/DD/HH:00:00_HH:59:59.binpb' +/// +/// If you may push duplicated objects, you have to provide Comp and Equal types. +/// In this case, Equal must be consistent with Comp, and the first criteria of the comparison should be the timestamp +/// (ordered from oldest to youngest). +/// +/// You may not provide any Comp and Equal if by design you will not push duplicated data. +template +class ProtobufObjectsSerializer { + public: + /// Creates a new ProtobufObjectsSerializer. + /// @param marketTimestampSet the latest written timestamp for all markets to avoid writing duplicate entries between + /// coincenter restarts. + ProtobufObjectsSerializer(std::string subPath, const MarketTimestampSet &marketTimestampSet, + int32_t nbObjectsPerMarketInMemory) noexcept + : _subPath(std::move(subPath)), _nbObjectsPerMarketInMemory(nbObjectsPerMarketInMemory) { + for (const auto &[market, timestamp] : marketTimestampSet) { + auto &lastWrittenObjectTimestamp = _marketDataMap[market].lastWrittenObjectTimestamp; + + lastWrittenObjectTimestamp = timestamp; + + // When program starts, we want to exclude equal timestamps to avoid writing objects that may have been wrote + // already from a previous run (the SortUnique will not protect us here) + ++lastWrittenObjectTimestamp; + } + } + + ProtobufObjectsSerializer(const ProtobufObjectsSerializer &) = delete; + ProtobufObjectsSerializer &operator=(const ProtobufObjectsSerializer &) = delete; + + ProtobufObjectsSerializer(ProtobufObjectsSerializer &&other) noexcept { swap(other); } + + ProtobufObjectsSerializer &operator=(ProtobufObjectsSerializer &&other) noexcept { + if (&other != this) { + swap(other); + } + return *this; + } + + ~ProtobufObjectsSerializer() { + try { + for (auto &[market, marketData] : _marketDataMap) { + writeOnDisk(market, marketData); + } + } catch (const std::exception &e) { + log::error("exception caught in writeOnDisk: {}", e.what()); + } + } + + /// Pushes a new object into the serializer. + /// The new object is guaranteed to be written upon destruction of this serializer at the latest unless: + /// - its timestamp is older than the latest written timestamp of this market + /// - it has invalid data + template + void push(Market market, ProtobufObjectTypeU &&protoObj) { + auto &marketData = _marketDataMap[market]; + + if (!isValid(marketData, protoObj)) { + return; + } + + marketData.dataVector.push_back(std::forward(protoObj)); + + checkWriteOnDisk(market, marketData); + } + + void swap(ProtobufObjectsSerializer &rhs) noexcept { + _marketDataMap.swap(rhs._marketDataMap); + _subPath.swap(rhs._subPath); + std::swap(_nbObjectsPerMarketInMemory, rhs._nbObjectsPerMarketInMemory); + std::swap(_flushCounter, rhs._flushCounter); + } + + private: + using ProtobufObjectTypeVector = vector; + + struct MarketData { + ProtobufObjectTypeVector dataVector; + TimePoint lastWrittenObjectTimestamp; + }; + + void checkWriteOnDisk(Market market, MarketData &marketData) { + auto &dataVector = marketData.dataVector; + if (dataVector.size() == static_cast(_nbObjectsPerMarketInMemory)) { + writeOnDisk(market, marketData); + + // shrink_to_fit as vector will never grow-up larger than its current size + dataVector.shrink_to_fit(); + dataVector.clear(); + + checkPeriodicFlush(); + } + } + + void writeOnDisk(Market market, MarketData &marketData) { + auto &dataVector = marketData.dataVector; + if (dataVector.empty()) { + return; + } + + const auto nowTime = std::chrono::steady_clock::now(); + + SortUnique(dataVector); + + std::string pathStr = _subPath; + + std::chrono::hours prevHourOfDay{-1}; + + ProtobufMessagesWriter protobufMessagesWriter; + for (const auto &protobufObject : dataVector) { + checkOpenFile(market, protobufObject, prevHourOfDay, pathStr, protobufMessagesWriter); + + protobufMessagesWriter.write(protobufObject); + } + + marketData.lastWrittenObjectTimestamp = TimePoint{milliseconds{dataVector.back().unixtimestampinms()}}; + + const auto nbElemsWritten = dataVector.size(); + + const auto steadyClockDuration = std::chrono::steady_clock::now() - nowTime; + const auto dur = std::chrono::duration_cast(steadyClockDuration); + + log::info("Serialized {} object(s) for {} data in {}, last in {}", nbElemsWritten, market, DurationToString(dur), + pathStr); + } + + // Periodic memory release to avoid possible leaks for long time running (if market data unused anymore for instance) + void checkPeriodicFlush() { + if (++_flushCounter != RehashThreshold) { + return; + } + + _flushCounter = 0; + + auto nowTime = Clock::now(); + + for (auto it = _marketDataMap.begin(); it != _marketDataMap.end();) { + if (it->second.lastWrittenObjectTimestamp + DurationType{static_cast(DurationValue)} < nowTime) { + // Unchanged data since a long time - write data if any, and clears the entry in the map + const Market market = it->first; + MarketData &marketData = it->second; + + writeOnDisk(market, marketData); + + log::info("Released {} protobuf objects for {}", marketData.dataVector.capacity(), market); + + it = _marketDataMap.erase(it); + } else { + ++it; + } + } + + _marketDataMap.rehash(_marketDataMap.size()); + } + + static void SortUnique(ProtobufObjectTypeVector &dataVector) { + static_assert((std::is_void_v && std::is_void_v) || (!std::is_void_v && !std::is_void_v)); + + if constexpr (std::is_void_v) { + // Sort by timestamp (required by 'writeOnDisk' algorithm) + std::ranges::sort(dataVector, [](const auto &lhs, const auto &rhs) { + return lhs.unixtimestampinms() < rhs.unixtimestampinms(); + }); + } else { + // We assume that timestamp is the first sorting criteria + std::ranges::sort(dataVector, Comp{}); + } + + // If duplicate elements are possible, remove them + if constexpr (!std::is_void_v) { + const auto [eraseIt1, eraseIt2] = std::ranges::unique(dataVector, Equal{}); + dataVector.erase(eraseIt1, eraseIt2); + } + } + + void checkOpenFile(Market market, const ProtobufObjectType &protobufObject, std::chrono::hours &prevHourOfDay, + std::string &pathStr, ProtobufMessagesWriter &protobufMessagesWriter) { + const TimePoint tp{milliseconds{protobufObject.unixtimestampinms()}}; + const auto hourOfDay = GetHourOfDay(tp); + + if (prevHourOfDay != hourOfDay) { + // open new outfile + setDirectory(market, tp, pathStr); + std::filesystem::create_directories(std::filesystem::path(pathStr)); + + pathStr.append(ComputeProtoFileName(std::chrono::duration_cast(hourOfDay).count())); + + std::filesystem::path filePath(pathStr); + + protobufMessagesWriter.open(std::ofstream(filePath, std::ios_base::app)); + prevHourOfDay = hourOfDay; + } + } + + static std::chrono::hours GetHourOfDay(TimePoint tp) { + const auto dp = std::chrono::floor(tp); + + return std::chrono::floor(tp - dp); + } + + bool isValid(const MarketData &marketData, const ProtobufObjectType &protoObj) const { + if (!protoObj.has_unixtimestampinms()) { + throw exception("Attempt to push proto object without any timestamp"); + } + if (TimePoint{milliseconds{protoObj.unixtimestampinms()}} < marketData.lastWrittenObjectTimestamp) { + // do not push an object that has an older timestamp of the last written object + return false; + } + return true; + } + + void setDirectory(Market market, TimePoint tp, std::string &pathStr) const { + // Note: below code could be simplified once compilers fully implement std::format and chrono C++20 + // libraries. + const auto dp = std::chrono::floor(tp); + const std::chrono::year_month_day ymd{dp}; + + pathStr.replace(pathStr.begin() + _subPath.size(), pathStr.end(), + format("/{}/{:04}/{:02}/{:02}/", market, static_cast(ymd.year()), + static_cast(ymd.month()), static_cast(ymd.day()))); + } + + using MarketDataMap = std::unordered_map; + + MarketDataMap _marketDataMap; + std::string _subPath; + int32_t _nbObjectsPerMarketInMemory; + int32_t _flushCounter{}; +}; + +} // namespace cct diff --git a/src/serialization/include/proto-trade-data-compare.hpp b/src/serialization/include/proto-trade-data-compare.hpp new file mode 100644 index 00000000..d1fb640a --- /dev/null +++ b/src/serialization/include/proto-trade-data-compare.hpp @@ -0,0 +1,15 @@ +#pragma once + +#include "trade-data.pb.h" + +namespace cct { + +struct TradeDataComp { + bool operator()(const ::objects::TradeData &lhs, const ::objects::TradeData &rhs) const; +}; + +struct TradeDataEqual { + bool operator()(const ::objects::TradeData &lhs, const ::objects::TradeData &rhs) const; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/include/serialization-tools.hpp b/src/serialization/include/serialization-tools.hpp new file mode 100644 index 00000000..2a6e3b86 --- /dev/null +++ b/src/serialization/include/serialization-tools.hpp @@ -0,0 +1,16 @@ +#pragma once + +#include +#include + +namespace cct { + +std::string ComputeProtoSubPath(std::string_view dataDir, std::string_view exchangeName, + std::string_view protobufObjectName); + +/// From an hour of day in [0, 23], return the file name for a protobuf binary serialization file. +/// Example: +/// ComputeProtoFileName(4) -> "04:00:00_04:59:59.binpb" +std::string_view ComputeProtoFileName(int hourOfDay); + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/proto/market-order-book-timed-data.proto b/src/serialization/proto/market-order-book-timed-data.proto new file mode 100644 index 00000000..286fdf0b --- /dev/null +++ b/src/serialization/proto/market-order-book-timed-data.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +package objects; + +message MarketOrderBookTimedData { + optional int64 unixTimestampInMs = 1; + optional int32 volumeNbDecimals = 2; + optional int32 priceNbDecimals = 3; + + message PricedVolume { + optional int64 price = 1; + optional int64 volume = 2; + } + + message OrderBook { + repeated PricedVolume asks = 1; + repeated PricedVolume bids = 2; + } + + optional OrderBook orderBook = 4; +} \ No newline at end of file diff --git a/src/serialization/proto/trade-data.proto b/src/serialization/proto/trade-data.proto new file mode 100644 index 00000000..f1399429 --- /dev/null +++ b/src/serialization/proto/trade-data.proto @@ -0,0 +1,18 @@ +syntax = "proto3"; + +package objects; + +enum TradeSide { + TRADE_UNSPECIFIED = 0; + TRADE_BUY = 1; + TRADE_SELL = 2; +} + +message TradeData { + optional int64 unixTimestampInMs = 1; + optional int64 priceAmount = 2; + optional int64 volumeAmount = 3; + optional int32 priceNbDecimals = 4; + optional int32 volumeNbDecimals = 5; + TradeSide tradeSide = 6; +} \ No newline at end of file diff --git a/src/serialization/src/dummy-market-data-deserializer.cpp b/src/serialization/src/dummy-market-data-deserializer.cpp new file mode 100644 index 00000000..ab9c1643 --- /dev/null +++ b/src/serialization/src/dummy-market-data-deserializer.cpp @@ -0,0 +1,29 @@ +#include "dummy-market-data-deserializer.hpp" + +#include "market-order-book-vector.hpp" +#include "market-timestamp-set.hpp" +#include "market.hpp" +#include "public-trade-vector.hpp" +#include "time-window.hpp" + +namespace cct { + +DummyMarketDataDeserializer::DummyMarketDataDeserializer([[maybe_unused]] std::string_view dataDir, + [[maybe_unused]] std::string_view exchangeName) {} + +MarketTimestampSet DummyMarketDataDeserializer::pullMarketOrderBooksMarkets([[maybe_unused]] TimeWindow timeWindow) { + return {}; +} + +MarketTimestampSet DummyMarketDataDeserializer::pullTradeMarkets([[maybe_unused]] TimeWindow timeWindow) { return {}; } + +MarketOrderBookVector DummyMarketDataDeserializer::pullMarketOrderBooks([[maybe_unused]] Market market, + [[maybe_unused]] TimeWindow timeWindow) { + return {}; +} + +PublicTradeVector DummyMarketDataDeserializer::pullTrades([[maybe_unused]] Market market, + [[maybe_unused]] TimeWindow timeWindow) { + return {}; +} +} // namespace cct \ No newline at end of file diff --git a/src/serialization/src/dummy-market-data-serializer.cpp b/src/serialization/src/dummy-market-data-serializer.cpp new file mode 100644 index 00000000..9642330e --- /dev/null +++ b/src/serialization/src/dummy-market-data-serializer.cpp @@ -0,0 +1,23 @@ +#include "dummy-market-data-serializer.hpp" + +#include +#include + +#include "market-timestamp-set.hpp" +#include "market.hpp" +#include "marketorderbook.hpp" +#include "publictrade.hpp" + +namespace cct { + +DummyMarketDataSerializer::DummyMarketDataSerializer( + [[maybe_unused]] std::string_view dataDir, + [[maybe_unused]] const MarketTimestampSets &lastWrittenObjectsMarketTimestamp, + [[maybe_unused]] std::string_view exchangeName) {} + +void DummyMarketDataSerializer::push([[maybe_unused]] const MarketOrderBook &marketOrderBook) {} + +void DummyMarketDataSerializer::push([[maybe_unused]] Market market, + [[maybe_unused]] std::span publicTrades) {} + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/src/proto-market-data-deserializer.cpp b/src/serialization/src/proto-market-data-deserializer.cpp new file mode 100644 index 00000000..9176c209 --- /dev/null +++ b/src/serialization/src/proto-market-data-deserializer.cpp @@ -0,0 +1,32 @@ +#include "proto-market-data-deserializer.hpp" + +#include + +#include "market-order-book-vector.hpp" +#include "market-timestamp-set.hpp" +#include "proto-constants.hpp" +#include "public-trade-vector.hpp" +#include "serialization-tools.hpp" +#include "time-window.hpp" + +namespace cct { +ProtoMarketDataDeserializer::ProtoMarketDataDeserializer(std::string_view dataDir, std::string_view exchangeName) + : _marketOrderBookDeserializer(ComputeProtoSubPath(dataDir, exchangeName, kSubPathMarketOrderBook)), + _tradeDataDeserializer(ComputeProtoSubPath(dataDir, exchangeName, kSubPathTrades)) {} + +MarketTimestampSet ProtoMarketDataDeserializer::pullMarketOrderBooksMarkets(TimeWindow timeWindow) { + return _tradeDataDeserializer.listMarkets(timeWindow); +} + +MarketTimestampSet ProtoMarketDataDeserializer::pullTradeMarkets(TimeWindow timeWindow) { + return _marketOrderBookDeserializer.listMarkets(timeWindow); +} + +MarketOrderBookVector ProtoMarketDataDeserializer::pullMarketOrderBooks(Market market, TimeWindow timeWindow) { + return _marketOrderBookDeserializer.loadMarket(market, timeWindow); +} + +PublicTradeVector ProtoMarketDataDeserializer::pullTrades(Market market, TimeWindow timeWindow) { + return _tradeDataDeserializer.loadMarket(market, timeWindow); +} +} // namespace cct \ No newline at end of file diff --git a/src/serialization/src/proto-market-data-serializer.cpp b/src/serialization/src/proto-market-data-serializer.cpp new file mode 100644 index 00000000..f672e828 --- /dev/null +++ b/src/serialization/src/proto-market-data-serializer.cpp @@ -0,0 +1,42 @@ +#include "proto-market-data-serializer.hpp" + +#include +#include + +#include "market-timestamp-set.hpp" +#include "marketorderbook.hpp" +#include "proto-constants.hpp" +#include "proto-market-order-book.hpp" +#include "proto-public-trade.hpp" +#include "publictrade.hpp" +#include "serialization-tools.hpp" + +namespace cct { + +ProtoMarketDataSerializer::ProtoMarketDataSerializer(std::string_view dataDir, + const MarketTimestampSets& lastWrittenObjectsMarketTimestamp, + std::string_view exchangeName) + : _marketOrderBookSerializer(ComputeProtoSubPath(dataDir, exchangeName, kSubPathMarketOrderBook), + lastWrittenObjectsMarketTimestamp.orderBooksMarkets, 1000), + _tradesSerializer(ComputeProtoSubPath(dataDir, exchangeName, kSubPathTrades), + lastWrittenObjectsMarketTimestamp.tradesMarkets, 25000) {} + +void ProtoMarketDataSerializer::push(const MarketOrderBook& marketOrderBook) { + if (!marketOrderBook.isValid()) { + log::error("Do not serialize invalid market order book"); + return; + } + _marketOrderBookSerializer.push(marketOrderBook.market(), CreateMarketOrderBookTimedData(marketOrderBook)); +} + +void ProtoMarketDataSerializer::push(Market market, std::span publicTrades) { + for (const auto& publicTrade : publicTrades) { + if (!publicTrade.isValid()) { + log::error("Do not serialize invalid public trade"); + continue; + } + _tradesSerializer.push(market, ConvertPublicTradeToTradeData(publicTrade)); + } +} + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/src/proto-market-order-book.cpp b/src/serialization/src/proto-market-order-book.cpp new file mode 100644 index 00000000..7f278275 --- /dev/null +++ b/src/serialization/src/proto-market-order-book.cpp @@ -0,0 +1,77 @@ +#include "proto-market-order-book.hpp" + +#include +#include + +#include "market-order-book-timed-data.pb.h" +#include "marketorderbook.hpp" +#include "monetaryamount.hpp" +#include "timedef.hpp" +#include "volumeandpricenbdecimals.hpp" + +namespace cct { +::objects::MarketOrderBookTimedData CreateMarketOrderBookTimedData(const MarketOrderBook& marketOrderBook) { + ::objects::MarketOrderBookTimedData protoObj; + + const auto [volNbDecimals, priNbDecimals] = marketOrderBook.volAndPriNbDecimals(); + const auto unixTimestampInMs = TimestampToMillisecondsSinceEpoch(marketOrderBook.time()); + + protoObj.set_unixtimestampinms(unixTimestampInMs); + protoObj.set_volumenbdecimals(volNbDecimals); + protoObj.set_pricenbdecimals(priNbDecimals); + + auto& orderBook = *protoObj.mutable_orderbook(); + + const int nbBids = marketOrderBook.nbBidPrices(); + for (int bidPos = 1; bidPos <= nbBids; ++bidPos) { + const auto [volume, price] = marketOrderBook[-bidPos]; + auto& pricedVolume = *orderBook.add_bids(); + + pricedVolume.set_volume(volume.amount(volNbDecimals).value()); + pricedVolume.set_price(price.amount(priNbDecimals).value()); + } + + const int nbAsks = marketOrderBook.nbAskPrices(); + for (int askPos = 1; askPos <= nbAsks; ++askPos) { + const auto [volume, price] = marketOrderBook[askPos]; + auto& pricedVolume = *orderBook.add_asks(); + + pricedVolume.set_volume(volume.amount(volNbDecimals).value()); + pricedVolume.set_price(price.amount(priNbDecimals).value()); + } + + return protoObj; +} + +MarketOrderBook MarketOrderBookConverter::operator()( + const ::objects::MarketOrderBookTimedData& marketOrderBookTimedData) { + const TimePoint timeStamp(milliseconds(marketOrderBookTimedData.unixtimestampinms())); + const VolAndPriNbDecimals volAndPriNbDecimals(marketOrderBookTimedData.volumenbdecimals(), + marketOrderBookTimedData.pricenbdecimals()); + + const auto& bids = marketOrderBookTimedData.orderbook().bids(); + const auto& asks = marketOrderBookTimedData.orderbook().asks(); + const int32_t lowestAskPricePos = static_cast(bids.size()); + const int32_t highestBidPricePos = lowestAskPricePos - 1; + + // We directly construct the MarketOrderBook here - we trust the protobuf data (it should have been written from a + // valid MarketOrderBook at the source) + // Possible optimization - allocate in a reusable arena of memory instead of allocating a new buffer for each new + // object. + MarketOrderBook::AmountPriceVector orders; + + orders.reserve(bids.size() + asks.size()); + + for (const auto& bid : std::ranges::reverse_view(bids)) { + orders.emplace_back(bid.volume(), bid.price()); + } + + for (const auto& ask : asks) { + orders.emplace_back(-ask.volume(), ask.price()); + } + + return MarketOrderBook{timeStamp, _market, std::move(orders), + highestBidPricePos, lowestAskPricePos, volAndPriNbDecimals}; +} + +} // namespace cct diff --git a/src/serialization/src/proto-public-trade.cpp b/src/serialization/src/proto-public-trade.cpp new file mode 100644 index 00000000..bfcd174e --- /dev/null +++ b/src/serialization/src/proto-public-trade.cpp @@ -0,0 +1,62 @@ +#include "proto-public-trade.hpp" + +#include "monetaryamount.hpp" +#include "publictrade.hpp" +#include "timedef.hpp" +#include "trade-data.pb.h" +#include "tradeside.hpp" +#include "unreachable.hpp" + +namespace cct { +namespace { +::objects::TradeSide ConvertTradeSide(TradeSide tradeSide) { + switch (tradeSide) { + case TradeSide::kBuy: + return ::objects::TRADE_BUY; + case TradeSide::kSell: + return ::objects::TRADE_SELL; + default: + unreachable(); + } +} + +TradeSide ConvertTradeSide(::objects::TradeSide tradeSide) { + switch (tradeSide) { + case ::objects::TRADE_BUY: + return TradeSide::kBuy; + case ::objects::TRADE_SELL: + return TradeSide::kSell; + default: + unreachable(); + } +} + +} // namespace + +::objects::TradeData ConvertPublicTradeToTradeData(const PublicTrade &publicTrade) { + ::objects::TradeData protoObj; + + protoObj.set_unixtimestampinms(TimestampToMillisecondsSinceEpoch(publicTrade.time())); + + const auto price = publicTrade.price(); + protoObj.set_priceamount(price.amount()); + protoObj.set_pricenbdecimals(price.nbDecimals()); + + const auto volume = publicTrade.amount(); + protoObj.set_volumeamount(volume.amount()); + protoObj.set_volumenbdecimals(volume.nbDecimals()); + + protoObj.set_tradeside(ConvertTradeSide(publicTrade.side())); + + return protoObj; +} + +PublicTrade TradeDataToPublicTradeConverter::operator()(const ::objects::TradeData &tradeData) const { + const MonetaryAmount amount(tradeData.volumeamount(), _market.base(), tradeData.volumenbdecimals()); + const MonetaryAmount price(tradeData.priceamount(), _market.quote(), tradeData.pricenbdecimals()); + const TimePoint timeStamp(milliseconds(tradeData.unixtimestampinms())); + + return {ConvertTradeSide(tradeData.tradeside()), amount, price, timeStamp}; +} + +} // namespace cct diff --git a/src/serialization/src/proto-trade-data-compare.cpp b/src/serialization/src/proto-trade-data-compare.cpp new file mode 100644 index 00000000..d0bd28e8 --- /dev/null +++ b/src/serialization/src/proto-trade-data-compare.cpp @@ -0,0 +1,43 @@ +#include "proto-trade-data-compare.hpp" + +#include "currencycode.hpp" +#include "monetaryamount.hpp" + +namespace cct { + +namespace { +MonetaryAmount PriceMonetaryAmount(const ::objects::TradeData& obj) { + return MonetaryAmount(obj.priceamount(), CurrencyCode{}, obj.pricenbdecimals()); +} + +MonetaryAmount VolumeMonetaryAmount(const ::objects::TradeData& obj) { + return MonetaryAmount(obj.volumeamount(), CurrencyCode{}, obj.volumenbdecimals()); +} +} // namespace + +bool TradeDataComp::operator()(const ::objects::TradeData& lhs, const ::objects::TradeData& rhs) const { + if (lhs.unixtimestampinms() != rhs.unixtimestampinms()) { + return lhs.unixtimestampinms() < rhs.unixtimestampinms(); + } + MonetaryAmount lhsAmount = VolumeMonetaryAmount(lhs); + MonetaryAmount rhsAmount = VolumeMonetaryAmount(rhs); + if (lhsAmount != rhsAmount) { + return lhsAmount < rhsAmount; + } + MonetaryAmount lhsPrice = PriceMonetaryAmount(lhs); + MonetaryAmount rhsPrice = PriceMonetaryAmount(rhs); + if (lhsPrice != rhsPrice) { + return lhsPrice < rhsPrice; + } + if (lhs.tradeside() != rhs.tradeside()) { + return lhs.tradeside() < rhs.tradeside(); + } + return false; +} + +bool TradeDataEqual::operator()(const ::objects::TradeData& lhs, const ::objects::TradeData& rhs) const { + return lhs.unixtimestampinms() == rhs.unixtimestampinms() && VolumeMonetaryAmount(lhs) == VolumeMonetaryAmount(rhs) && + PriceMonetaryAmount(lhs) == PriceMonetaryAmount(rhs) && lhs.tradeside() == rhs.tradeside(); +} + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/src/serialization-tools.cpp b/src/serialization/src/serialization-tools.cpp new file mode 100644 index 00000000..ef5595f2 --- /dev/null +++ b/src/serialization/src/serialization-tools.cpp @@ -0,0 +1,67 @@ +#include "serialization-tools.hpp" + +#include +#include +#include +#include + +#include "proto-constants.hpp" + +namespace cct { + +std::string ComputeProtoSubPath(std::string_view dataDir, std::string_view exchangeName, + std::string_view protobufObjectName) { + std::string ret; + + static constexpr std::string_view kSerializedDataSubPath = "/serialized/"; + + ret.reserve(dataDir.size() + kSerializedDataSubPath.size() + protobufObjectName.size() + exchangeName.size() + 1U); + + ret.append(dataDir); + ret.append(kSerializedDataSubPath); + ret.append(protobufObjectName); + ret.push_back('/'); + ret.append(exchangeName); + return ret; +} + +namespace { + +consteval auto BuildBinProtoFileNames() { + constexpr std::string_view kBinProtoFilePart2 = ":00:00_"; + constexpr std::string_view kBinProtoFilePart3 = ":59:59"; + + using ProtoFileNameBuffer = + std::array(2 * 2)>; + + constexpr auto kNbHourInDay = 24; + + std::array ret; + + for (auto hourOfDay = 0; hourOfDay < kNbHourInDay; ++hourOfDay) { + std::array hourStr = {static_cast((hourOfDay / 10) + '0'), static_cast((hourOfDay % 10) + '0')}; + + auto it = ret[hourOfDay].begin(); + + it = std::ranges::copy(hourStr, it).out; + it = std::ranges::copy(kBinProtoFilePart2, it).out; + it = std::ranges::copy(hourStr, it).out; + it = std::ranges::copy(kBinProtoFilePart3, it).out; + it = std::ranges::copy(kBinProtobufExtension, it).out; + } + + return ret; +} + +} // namespace + +std::string_view ComputeProtoFileName(int hourOfDay) { + static constexpr auto kBinProtoFileNames = BuildBinProtoFileNames(); + + const auto &protoFileName = kBinProtoFileNames[hourOfDay]; + + return {protoFileName.data(), protoFileName.size()}; +} + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/test/continuous-iterator_test.cpp b/src/serialization/test/continuous-iterator_test.cpp new file mode 100644 index 00000000..b6eaa089 --- /dev/null +++ b/src/serialization/test/continuous-iterator_test.cpp @@ -0,0 +1,41 @@ +#include "continuous-iterator.hpp" + +#include + +namespace cct { + +TEST(ContinuousIterator, UniqueElement) { + ContinuousIterator it(1, 1); + + EXPECT_TRUE(it.hasNext()); + EXPECT_EQ(it.next(), 1); + EXPECT_FALSE(it.hasNext()); +} + +TEST(ContinuousIterator, SeveralElements) { + ContinuousIterator it(1, 3); + + EXPECT_TRUE(it.hasNext()); + EXPECT_EQ(it.next(), 1); + EXPECT_TRUE(it.hasNext()); + EXPECT_EQ(it.next(), 2); + EXPECT_TRUE(it.hasNext()); + EXPECT_EQ(it.next(), 3); + EXPECT_FALSE(it.hasNext()); +} + +TEST(ContinuousIterator, Reverse) { + ContinuousIterator it(1, -2); + + EXPECT_TRUE(it.hasNext()); + EXPECT_EQ(it.next(), 1); + EXPECT_TRUE(it.hasNext()); + EXPECT_EQ(it.next(), 0); + EXPECT_TRUE(it.hasNext()); + EXPECT_EQ(it.next(), -1); + EXPECT_TRUE(it.hasNext()); + EXPECT_EQ(it.next(), -2); + EXPECT_FALSE(it.hasNext()); +} + +} // namespace cct diff --git a/src/serialization/test/proto-market-order-book_test.cpp b/src/serialization/test/proto-market-order-book_test.cpp new file mode 100644 index 00000000..aba9c161 --- /dev/null +++ b/src/serialization/test/proto-market-order-book_test.cpp @@ -0,0 +1,115 @@ +#include "proto-market-order-book.hpp" + +#include + +#include + +#include "amount-price.hpp" +#include "marketorderbook.hpp" +#include "monetaryamount.hpp" +#include "order-book-line.hpp" + +namespace cct { + +// TODO: factorize duplicated code from marketorderbook_test +namespace { +using AmountAtPriceVec = MarketOrderBook::AmountPerPriceVec; +} // namespace + +MarketOrderBookLines CreateMarketOrderBookLines(std::initializer_list init) { + MarketOrderBookLines marketOrderBookLines; + marketOrderBookLines.reserve(init.size()); + + for (const auto &orderBookLine : init) { + if (orderBookLine.amount() < 0) { + marketOrderBookLines.pushAsk(-orderBookLine.amount(), orderBookLine.price()); + } else { + marketOrderBookLines.pushBid(orderBookLine.amount(), orderBookLine.price()); + } + } + + return marketOrderBookLines; +} + +constexpr bool operator==(const AmountPrice &lhs, const AmountPrice &rhs) { + return lhs.amount == rhs.amount && lhs.price == rhs.price; +} + +class ProtoMarketOrderBookTest : public ::testing::Test { + protected: + TimePoint time; + Market market{"APM", "KRW"}; + MarketOrderBook marketOrderBook{ + time, market, + CreateMarketOrderBookLines( + {OrderBookLine(MonetaryAmount("1991.3922", "APM"), MonetaryAmount("57.8", "KRW"), OrderBookLine::Type::kAsk), + OrderBookLine(MonetaryAmount("90184.3951", "APM"), MonetaryAmount("57.81", "KRW"), + OrderBookLine::Type::kAsk), + OrderBookLine(MonetaryAmount("91.1713", "APM"), MonetaryAmount("57.84", "KRW"), OrderBookLine::Type::kAsk), + OrderBookLine(MonetaryAmount("41.0131", "APM"), MonetaryAmount("57.9", "KRW"), OrderBookLine::Type::kAsk), + OrderBookLine(MonetaryAmount("33.5081914157147802", "APM"), MonetaryAmount("57.78", "KRW"), + OrderBookLine::Type::kAsk), + OrderBookLine(MonetaryAmount("3890.879", "APM"), MonetaryAmount("57.19", "KRW"), OrderBookLine::Type::kBid), + OrderBookLine(MonetaryAmount("14", "APM"), MonetaryAmount("57.18", "KRW"), OrderBookLine::Type::kBid), + OrderBookLine(MonetaryAmount("14", "APM"), MonetaryAmount("57.17", "KRW"), OrderBookLine::Type::kBid), + OrderBookLine(MonetaryAmount("3848.8453", "APM"), MonetaryAmount("57.16", "KRW"), + OrderBookLine::Type::kBid)})}; + + MarketOrderBookConverter marketOrderBookConverter{market}; +}; + +TEST_F(ProtoMarketOrderBookTest, Serialization) { + const auto protoObj = CreateMarketOrderBookTimedData(marketOrderBook); + const auto [volNbDecimals, priNbDecimals] = marketOrderBook.volAndPriNbDecimals(); + + EXPECT_EQ(TimePoint{milliseconds{protoObj.unixtimestampinms()}}, marketOrderBook.time()); + EXPECT_EQ(protoObj.volumenbdecimals(), volNbDecimals); + EXPECT_EQ(protoObj.pricenbdecimals(), priNbDecimals); + + ASSERT_TRUE(protoObj.has_orderbook()); + ASSERT_EQ(protoObj.orderbook().asks_size(), 5U); + + const auto &asks = protoObj.orderbook().asks(); + + EXPECT_EQ(asks[0].volume(), 335081914157147); + EXPECT_EQ(asks[0].price(), 577800000000000000); + + EXPECT_EQ(asks[1].volume(), 19913922000000000); + EXPECT_EQ(asks[1].price(), 578000000000000000); + + EXPECT_EQ(asks[2].volume(), 901843951000000000); + EXPECT_EQ(asks[2].price(), 578100000000000000); + + EXPECT_EQ(asks[3].volume(), 911713000000000); + EXPECT_EQ(asks[3].price(), 578400000000000000); + + EXPECT_EQ(asks[4].volume(), 410131000000000); + EXPECT_EQ(asks[4].price(), 579000000000000000); + + ASSERT_EQ(protoObj.orderbook().bids_size(), 4U); + + const auto &bids = protoObj.orderbook().bids(); + + EXPECT_EQ(bids[0].volume(), 38908790000000000); + EXPECT_EQ(bids[0].price(), 571900000000000000); + + EXPECT_EQ(bids[1].volume(), 140000000000000); + EXPECT_EQ(bids[1].price(), 571800000000000000); + + EXPECT_EQ(bids[2].volume(), 140000000000000); + EXPECT_EQ(bids[2].price(), 571700000000000000); + + EXPECT_EQ(bids[3].volume(), 38488453000000000); + EXPECT_EQ(bids[3].price(), 571600000000000000); +} + +TEST_F(ProtoMarketOrderBookTest, SerializeThenDeserializeShouldGiveSameObject) { + const auto protoObj = CreateMarketOrderBookTimedData(marketOrderBook); + + const auto marketOrderBookConvertedBack = marketOrderBookConverter(protoObj); + + EXPECT_TRUE(marketOrderBookConvertedBack.isValid()); + + EXPECT_EQ(marketOrderBook, marketOrderBookConvertedBack); +} +} // namespace cct \ No newline at end of file diff --git a/src/serialization/test/proto-multiple-messages-handler_test.cpp b/src/serialization/test/proto-multiple-messages-handler_test.cpp new file mode 100644 index 00000000..fd131a85 --- /dev/null +++ b/src/serialization/test/proto-multiple-messages-handler_test.cpp @@ -0,0 +1,109 @@ +#include "proto-multiple-messages-handler.hpp" + +#include + +#include + +#include "proto-public-trade.hpp" +#include "proto-test-data.hpp" +#include "publictrade.hpp" +#include "trade-data.pb.h" + +namespace cct { + +class ProtobufMessagesTest : public ProtobufBaseDataTest { + protected: + TradeDataToPublicTradeConverter protoTradeDataConverter{mk1}; + + ProtobufMessagesWriter writer; +}; + +TEST_F(ProtobufMessagesTest, WriteReadSingle) { + writer.open(std::stringstream{}); + writer.write(td1); + + std::stringstream ss = writer.flush(); + + ProtobufMessagesReader reader{ss}; + + int nbObjectsRead = 0; + + while (reader.hasNext()) { + auto nextObj = reader.next<::objects::TradeData>(); + PublicTrade pt = protoTradeDataConverter(nextObj); + + EXPECT_EQ(pt, pt1); + ++nbObjectsRead; + } + EXPECT_EQ(nbObjectsRead, 1); +} + +TEST_F(ProtobufMessagesTest, WriteRead2Flushes) { + writer.open(std::stringstream{}); + writer.write(td1); + std::stringstream ss1 = writer.flush(); + + writer.open(std::stringstream{}); + writer.write(td2); + std::stringstream ss2 = writer.flush(); + + ProtobufMessagesReader reader1{ss1}; + + int nbObjectsRead = 0; + + while (reader1.hasNext()) { + auto nextObj = reader1.next<::objects::TradeData>(); + PublicTrade pt = protoTradeDataConverter(nextObj); + + EXPECT_EQ(pt, pt1); + ++nbObjectsRead; + } + EXPECT_EQ(nbObjectsRead, 1); + + ProtobufMessagesReader reader2{ss2}; + + while (reader2.hasNext()) { + auto nextObj = reader2.next<::objects::TradeData>(); + PublicTrade pt = protoTradeDataConverter(nextObj); + + EXPECT_EQ(pt, pt2); + ++nbObjectsRead; + } + EXPECT_EQ(nbObjectsRead, 2); +} + +TEST_F(ProtobufMessagesTest, WriteReadSeveral) { + writer.open(std::stringstream{}); + writer.write(td1); + writer.write(td2); + writer.write(td3); + + std::stringstream ss = writer.flush(); + + ProtobufMessagesReader reader{ss}; + + int nbObjectsRead = 0; + + while (reader.hasNext()) { + auto nextObj = reader.next<::objects::TradeData>(); + PublicTrade pt = protoTradeDataConverter(nextObj); + + switch (nbObjectsRead) { + case 0: + EXPECT_EQ(pt, pt1); + break; + case 1: + EXPECT_EQ(pt, pt2); + break; + case 2: + EXPECT_EQ(pt, pt3); + break; + default: + break; + } + + ++nbObjectsRead; + } + EXPECT_EQ(nbObjectsRead, 3); +} +} // namespace cct diff --git a/src/serialization/test/proto-public-trade_test.cpp b/src/serialization/test/proto-public-trade_test.cpp new file mode 100644 index 00000000..36e1e11b --- /dev/null +++ b/src/serialization/test/proto-public-trade_test.cpp @@ -0,0 +1,25 @@ +#include "proto-public-trade.hpp" + +#include + +#include "monetaryamount.hpp" +#include "publictrade.hpp" + +namespace cct { + +class ProtoPublicTradeTest : public ::testing::Test { + protected: + TimePoint tp{milliseconds{std::numeric_limits::max() / 10000000}}; + Market market{"ETH", "USDT"}; + PublicTrade pt{TradeSide::kBuy, MonetaryAmount{"0.13", "ETH"}, MonetaryAmount{"1500.5", "USDT"}, tp}; + TradeDataToPublicTradeConverter publicTradeConverter{market}; +}; + +TEST_F(ProtoPublicTradeTest, SerializeThenDeserializeShouldGiveSameObject) { + const auto protoObj = ConvertPublicTradeToTradeData(pt); + + const auto objBack = publicTradeConverter(protoObj); + + EXPECT_EQ(pt, objBack); +} +} // namespace cct \ No newline at end of file diff --git a/src/serialization/test/proto-serialization-and-deserialization_test.cpp b/src/serialization/test/proto-serialization-and-deserialization_test.cpp new file mode 100644 index 00000000..e46a4c24 --- /dev/null +++ b/src/serialization/test/proto-serialization-and-deserialization_test.cpp @@ -0,0 +1,591 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include "market-timestamp-set.hpp" +#include "market-timestamp.hpp" +#include "monetaryamount.hpp" +#include "proto-deserializer.hpp" +#include "proto-serializer.hpp" +#include "proto-test-data.hpp" +#include "proto-trade-data-compare.hpp" +#include "public-trade-vector.hpp" +#include "publictrade.hpp" +#include "time-window.hpp" +#include "timedef.hpp" +#include "trade-data.pb.h" + +namespace cct { + +class ProtobufSerializerDeserializerTest : public ProtobufBaseDataTest { + protected: + void SetUp() override { + // Clean-up at set-up instead of tear down to let objects on disk in case of manual debugging and check + std::filesystem::remove_all(subPath1); + std::filesystem::remove_all(subPath2); + } + + static constexpr int32_t kSmallRehashThreshold = 3; + + using DurationTypeFlush = milliseconds; + static constexpr int32_t kFlushNbMillis = 1; + + std::string subPath1{"test-proto-data-delete-me"}; + std::string subPath2{"another-sub-path"}; + + int32_t nbTradesPerMarketInMemory{100}; + + using Serializer = ProtobufObjectsSerializer<::objects::TradeData, TradeDataComp, TradeDataEqual, + kSmallRehashThreshold, DurationTypeFlush, kFlushNbMillis>; + using Deserializer = ProtobufObjectsDeserializer<::objects::TradeData, TradeDataToPublicTradeConverter>; + + Serializer createSerializer(const MarketTimestampSet &marketTimestampSet = MarketTimestampSet{}) { + return Serializer{subPath1, marketTimestampSet, nbTradesPerMarketInMemory}; + } + Deserializer createDeserializer() { return Deserializer{subPath1}; } + + void serializeSomeObjects(Serializer &serializer) { + // push two times same object (should not be duplicated during writing) + serializer.push(mk1, td2); + serializer.push(mk1, td2); + + // Even if older object, should be pushed as well as serializer has not written any objects yet (they should be + // ordered internally before writes) + serializer.push(mk1, td1); + + serializer.push(mk1, td3); + serializer.push(mk3, td4); + serializer.push(mk4, td5); + + serializer.push(mk1, td9); + + std::this_thread::sleep_for(milliseconds{2}); + + serializer.push(mk7, td7); + + Serializer anotherSerializer{subPath2, MarketTimestampSet{}, nbTradesPerMarketInMemory}; + + serializer.swap(anotherSerializer); + + // Should not be pushed to 'subPath' + serializer.push(mk6, td6); + } +}; + +TEST_F(ProtobufSerializerDeserializerTest, SerializeThenDeserializeSomeObjects) { + { + auto serializer = createSerializer(); + serializeSomeObjects(serializer); + } + + // make sure serializer writes all at destruction + + static constexpr std::string_view kExpectedFiles[] = { + "another-sub-path/ETH-BTC/2012/12/24/19:00:00_19:59:59.binpb", + "test-proto-data-delete-me/DOGE-CAD/2014/04/14/13:00:00_13:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2006/07/14/23:00:00_23:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/23/07:00:00_07:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/25/04:00:00_04:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2012/05/11/07:00:00_07:59:59.binpb", + "test-proto-data-delete-me/SHIB-USDT/1999/03/25/04:00:00_04:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2013/08/16/03:00:00_03:59:59.binpb"}; + + EXPECT_TRUE( + std::ranges::all_of(kExpectedFiles, [](const auto &fileName) { return std::filesystem::exists(fileName); })); + + auto deserializer = createDeserializer(); + auto marketTimestampSet = deserializer.listMarkets(timeWindowAll); + + EXPECT_EQ(marketTimestampSet, MarketTimestampSet({MarketTimestamp(mk7, tp8), MarketTimestamp(mk1, tp5), + MarketTimestamp(mk3, tp1), MarketTimestamp(mk4, tp7)})); + + // Should not serialize again as timestamps are set to last written + { + auto serializer = createSerializer(marketTimestampSet); + serializeSomeObjects(serializer); + } + + EXPECT_TRUE(deserializer.loadMarket(mk1, timeWindow79).empty()); + EXPECT_EQ(deserializer.loadMarket(mk1, timeWindowAll), vector({pt1, pt2, pt3, pt9})); + EXPECT_EQ(deserializer.loadMarket(mk4, timeWindowAll), vector({pt5})); + EXPECT_TRUE(deserializer.loadMarket(mk4, timeWindow14).empty()); + EXPECT_EQ(deserializer.loadMarket(mk7, timeWindowAll), vector({pt7})); + EXPECT_EQ(deserializer.loadMarket(mk7, timeWindow79), vector({pt7})); + EXPECT_TRUE(deserializer.loadMarket(Market{"UNK", "OTH"}, timeWindowAll).empty()); +} + +TEST_F(ProtobufSerializerDeserializerTest, ManySerializationsDifferentHoursOfDay) { + static const TimePoint kTimePoints[] = {tp1, tp2}; + static const Market kMarkets[] = {mk1, mk4}; + static constexpr Duration kDurationRange = std::chrono::weeks(2); + static constexpr Duration kDurationStep = std::chrono::minutes(199); + + std::map pushedPublicTrades; + + { + auto serializer = createSerializer(); + + for (Market market : kMarkets) { + for (TimePoint tp : kTimePoints) { + for (auto ts = tp; ts < tp + kDurationRange; ts += kDurationStep) { + TradeSide side = tp == tp1 ? TradeSide::kBuy : TradeSide::kSell; + MonetaryAmount amount{"0.13", market.base()}; + MonetaryAmount price{"1500.5", market.quote()}; + PublicTrade pt{side, amount, price, ts}; + + pushedPublicTrades[market].push_back(pt); + serializer.push(market, ConvertPublicTradeToTradeData(pt)); + } + + std::this_thread::sleep_for(milliseconds{2}); + } + } + } + + static constexpr std::string_view kExpectedFiles[] = { + "test-proto-data-delete-me/ETH-USDT/1999/03/25/04:00:00_04:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/25/08:00:00_08:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/25/11:00:00_11:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/25/14:00:00_14:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/25/18:00:00_18:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/25/21:00:00_21:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/26/00:00:00_00:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/26/03:00:00_03:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/26/07:00:00_07:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/26/10:00:00_10:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/26/13:00:00_13:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/26/17:00:00_17:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/26/20:00:00_20:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/26/23:00:00_23:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/27/03:00:00_03:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/27/06:00:00_06:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/27/09:00:00_09:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/27/13:00:00_13:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/27/16:00:00_16:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/27/19:00:00_19:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/27/23:00:00_23:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/28/02:00:00_02:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/28/05:00:00_05:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/28/09:00:00_09:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/28/12:00:00_12:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/28/15:00:00_15:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/28/19:00:00_19:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/28/22:00:00_22:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/29/01:00:00_01:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/29/04:00:00_04:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/29/08:00:00_08:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/29/11:00:00_11:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/29/14:00:00_14:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/29/18:00:00_18:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/29/21:00:00_21:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/30/00:00:00_00:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/30/04:00:00_04:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/30/07:00:00_07:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/30/10:00:00_10:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/30/14:00:00_14:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/30/17:00:00_17:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/30/20:00:00_20:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/31/00:00:00_00:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/31/03:00:00_03:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/31/06:00:00_06:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/31/10:00:00_10:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/31/13:00:00_13:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/31/16:00:00_16:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/31/19:00:00_19:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/03/31/23:00:00_23:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/01/02:00:00_02:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/01/05:00:00_05:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/01/09:00:00_09:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/01/12:00:00_12:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/01/15:00:00_15:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/01/19:00:00_19:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/01/22:00:00_22:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/02/01:00:00_01:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/02/05:00:00_05:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/02/08:00:00_08:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/02/11:00:00_11:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/02/15:00:00_15:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/02/18:00:00_18:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/02/21:00:00_21:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/03/01:00:00_01:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/03/04:00:00_04:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/03/07:00:00_07:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/03/10:00:00_10:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/03/14:00:00_14:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/03/17:00:00_17:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/03/20:00:00_20:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/04/00:00:00_00:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/04/03:00:00_03:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/04/06:00:00_06:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/04/10:00:00_10:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/04/13:00:00_13:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/04/16:00:00_16:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/04/20:00:00_20:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/04/23:00:00_23:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/05/02:00:00_02:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/05/06:00:00_06:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/05/09:00:00_09:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/05/12:00:00_12:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/05/16:00:00_16:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/05/19:00:00_19:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/05/22:00:00_22:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/06/02:00:00_02:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/06/05:00:00_05:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/06/08:00:00_08:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/06/11:00:00_11:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/06/15:00:00_15:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/06/18:00:00_18:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/06/21:00:00_21:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/07/01:00:00_01:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/07/04:00:00_04:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/07/07:00:00_07:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/07/11:00:00_11:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/07/14:00:00_14:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/07/17:00:00_17:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/07/21:00:00_21:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/08/00:00:00_00:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/1999/04/08/03:00:00_03:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/23/07:00:00_07:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/23/11:00:00_11:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/23/14:00:00_14:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/23/17:00:00_17:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/23/21:00:00_21:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/24/00:00:00_00:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/24/03:00:00_03:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/24/07:00:00_07:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/24/10:00:00_10:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/24/13:00:00_13:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/24/17:00:00_17:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/24/20:00:00_20:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/24/23:00:00_23:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/25/03:00:00_03:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/25/06:00:00_06:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/25/09:00:00_09:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/25/13:00:00_13:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/25/16:00:00_16:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/25/19:00:00_19:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/25/22:00:00_22:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/26/02:00:00_02:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/26/05:00:00_05:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/26/08:00:00_08:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/26/12:00:00_12:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/26/15:00:00_15:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/26/18:00:00_18:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/26/22:00:00_22:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/27/01:00:00_01:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/27/04:00:00_04:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/27/08:00:00_08:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/27/11:00:00_11:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/27/14:00:00_14:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/27/18:00:00_18:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/27/21:00:00_21:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/28/00:00:00_00:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/28/04:00:00_04:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/28/07:00:00_07:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/28/10:00:00_10:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/28/14:00:00_14:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/28/17:00:00_17:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/28/20:00:00_20:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/28/23:00:00_23:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/29/03:00:00_03:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/29/06:00:00_06:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/29/09:00:00_09:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/29/13:00:00_13:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/29/16:00:00_16:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/29/19:00:00_19:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/29/23:00:00_23:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/30/02:00:00_02:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/30/05:00:00_05:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/30/09:00:00_09:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/30/12:00:00_12:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/30/15:00:00_15:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/30/19:00:00_19:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/06/30/22:00:00_22:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/01/01:00:00_01:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/01/05:00:00_05:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/01/08:00:00_08:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/01/11:00:00_11:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/01/14:00:00_14:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/01/18:00:00_18:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/01/21:00:00_21:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/02/00:00:00_00:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/02/04:00:00_04:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/02/07:00:00_07:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/02/10:00:00_10:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/02/14:00:00_14:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/02/17:00:00_17:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/02/20:00:00_20:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/03/00:00:00_00:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/03/03:00:00_03:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/03/06:00:00_06:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/03/10:00:00_10:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/03/13:00:00_13:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/03/16:00:00_16:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/03/20:00:00_20:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/03/23:00:00_23:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/04/02:00:00_02:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/04/05:00:00_05:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/04/09:00:00_09:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/04/12:00:00_12:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/04/15:00:00_15:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/04/19:00:00_19:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/04/22:00:00_22:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/05/01:00:00_01:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/05/05:00:00_05:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/05/08:00:00_08:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/05/11:00:00_11:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/05/15:00:00_15:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/05/18:00:00_18:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/05/21:00:00_21:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/06/01:00:00_01:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/06/04:00:00_04:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/06/07:00:00_07:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/06/11:00:00_11:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/06/14:00:00_14:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/06/17:00:00_17:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/06/21:00:00_21:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/07/00:00:00_00:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/07/03:00:00_03:59:59.binpb", + "test-proto-data-delete-me/ETH-USDT/2002/07/07/06:00:00_06:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/25/04:00:00_04:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/25/08:00:00_08:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/25/11:00:00_11:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/25/14:00:00_14:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/25/18:00:00_18:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/25/21:00:00_21:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/26/00:00:00_00:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/26/03:00:00_03:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/26/07:00:00_07:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/26/10:00:00_10:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/26/13:00:00_13:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/26/17:00:00_17:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/26/20:00:00_20:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/26/23:00:00_23:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/27/03:00:00_03:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/27/06:00:00_06:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/27/09:00:00_09:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/27/13:00:00_13:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/27/16:00:00_16:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/27/19:00:00_19:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/27/23:00:00_23:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/28/02:00:00_02:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/28/05:00:00_05:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/28/09:00:00_09:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/28/12:00:00_12:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/28/15:00:00_15:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/28/19:00:00_19:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/28/22:00:00_22:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/29/01:00:00_01:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/29/04:00:00_04:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/29/08:00:00_08:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/29/11:00:00_11:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/29/14:00:00_14:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/29/18:00:00_18:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/29/21:00:00_21:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/30/00:00:00_00:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/30/04:00:00_04:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/30/07:00:00_07:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/30/10:00:00_10:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/30/14:00:00_14:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/30/17:00:00_17:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/30/20:00:00_20:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/31/00:00:00_00:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/31/03:00:00_03:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/31/06:00:00_06:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/31/10:00:00_10:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/31/13:00:00_13:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/31/16:00:00_16:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/31/19:00:00_19:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/03/31/23:00:00_23:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/01/02:00:00_02:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/01/05:00:00_05:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/01/09:00:00_09:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/01/12:00:00_12:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/01/15:00:00_15:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/01/19:00:00_19:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/01/22:00:00_22:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/02/01:00:00_01:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/02/05:00:00_05:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/02/08:00:00_08:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/02/11:00:00_11:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/02/15:00:00_15:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/02/18:00:00_18:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/02/21:00:00_21:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/03/01:00:00_01:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/03/04:00:00_04:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/03/07:00:00_07:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/03/10:00:00_10:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/03/14:00:00_14:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/03/17:00:00_17:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/03/20:00:00_20:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/04/00:00:00_00:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/04/03:00:00_03:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/04/06:00:00_06:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/04/10:00:00_10:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/04/13:00:00_13:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/04/16:00:00_16:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/04/20:00:00_20:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/04/23:00:00_23:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/05/02:00:00_02:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/05/06:00:00_06:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/05/09:00:00_09:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/05/12:00:00_12:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/05/16:00:00_16:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/05/19:00:00_19:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/05/22:00:00_22:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/06/02:00:00_02:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/06/05:00:00_05:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/06/08:00:00_08:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/06/11:00:00_11:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/06/15:00:00_15:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/06/18:00:00_18:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/06/21:00:00_21:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/07/01:00:00_01:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/07/04:00:00_04:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/07/07:00:00_07:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/07/11:00:00_11:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/07/14:00:00_14:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/07/17:00:00_17:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/07/21:00:00_21:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/08/00:00:00_00:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/1999/04/08/03:00:00_03:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/23/07:00:00_07:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/23/11:00:00_11:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/23/14:00:00_14:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/23/17:00:00_17:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/23/21:00:00_21:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/24/00:00:00_00:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/24/03:00:00_03:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/24/07:00:00_07:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/24/10:00:00_10:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/24/13:00:00_13:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/24/17:00:00_17:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/24/20:00:00_20:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/24/23:00:00_23:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/25/03:00:00_03:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/25/06:00:00_06:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/25/09:00:00_09:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/25/13:00:00_13:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/25/16:00:00_16:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/25/19:00:00_19:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/25/22:00:00_22:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/26/02:00:00_02:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/26/05:00:00_05:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/26/08:00:00_08:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/26/12:00:00_12:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/26/15:00:00_15:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/26/18:00:00_18:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/26/22:00:00_22:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/27/01:00:00_01:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/27/04:00:00_04:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/27/08:00:00_08:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/27/11:00:00_11:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/27/14:00:00_14:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/27/18:00:00_18:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/27/21:00:00_21:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/28/00:00:00_00:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/28/04:00:00_04:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/28/07:00:00_07:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/28/10:00:00_10:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/28/14:00:00_14:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/28/17:00:00_17:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/28/20:00:00_20:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/28/23:00:00_23:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/29/03:00:00_03:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/29/06:00:00_06:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/29/09:00:00_09:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/29/13:00:00_13:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/29/16:00:00_16:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/29/19:00:00_19:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/29/23:00:00_23:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/30/02:00:00_02:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/30/05:00:00_05:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/30/09:00:00_09:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/30/12:00:00_12:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/30/15:00:00_15:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/30/19:00:00_19:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/06/30/22:00:00_22:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/01/01:00:00_01:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/01/05:00:00_05:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/01/08:00:00_08:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/01/11:00:00_11:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/01/14:00:00_14:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/01/18:00:00_18:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/01/21:00:00_21:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/02/00:00:00_00:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/02/04:00:00_04:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/02/07:00:00_07:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/02/10:00:00_10:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/02/14:00:00_14:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/02/17:00:00_17:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/02/20:00:00_20:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/03/00:00:00_00:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/03/03:00:00_03:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/03/06:00:00_06:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/03/10:00:00_10:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/03/13:00:00_13:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/03/16:00:00_16:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/03/20:00:00_20:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/03/23:00:00_23:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/04/02:00:00_02:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/04/05:00:00_05:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/04/09:00:00_09:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/04/12:00:00_12:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/04/15:00:00_15:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/04/19:00:00_19:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/04/22:00:00_22:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/05/01:00:00_01:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/05/05:00:00_05:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/05/08:00:00_08:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/05/11:00:00_11:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/05/15:00:00_15:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/05/18:00:00_18:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/05/21:00:00_21:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/06/01:00:00_01:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/06/04:00:00_04:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/06/07:00:00_07:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/06/11:00:00_11:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/06/14:00:00_14:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/06/17:00:00_17:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/06/21:00:00_21:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/07/00:00:00_00:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/07/03:00:00_03:59:59.binpb", + "test-proto-data-delete-me/SOL-BTC/2002/07/07/06:00:00_06:59:59.binpb", + }; + + EXPECT_TRUE( + std::ranges::all_of(kExpectedFiles, [](const auto &fileName) { return std::filesystem::exists(fileName); })); + + auto deserializer = createDeserializer(); + auto marketTimestampSet = deserializer.listMarkets(timeWindowAll); + + ASSERT_EQ(marketTimestampSet.size(), 2U); + + const auto &marketTimestamp = marketTimestampSet.front(); + + const auto lastTp = *std::next(std::end(kTimePoints), -1); + + EXPECT_EQ(marketTimestamp.market, mk1); + EXPECT_GT(marketTimestamp.timePoint + kDurationStep, lastTp + kDurationRange); + + for (Market market : kMarkets) { + auto allData = deserializer.loadMarket(market, timeWindowAll); + + EXPECT_EQ(pushedPublicTrades[market], allData); + + auto partialData = + deserializer.loadMarket(market, TimeWindow{kTimePoints[0], kTimePoints[0] + std::chrono::days(1)}); + + EXPECT_LT(partialData.size(), allData.size()); + EXPECT_EQ(partialData.size(), 8U); + EXPECT_TRUE( + std::equal(partialData.begin(), partialData.end(), allData.begin(), allData.begin() + partialData.size())); + } +} + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/test/proto-test-data.hpp b/src/serialization/test/proto-test-data.hpp new file mode 100644 index 00000000..1f027043 --- /dev/null +++ b/src/serialization/test/proto-test-data.hpp @@ -0,0 +1,66 @@ +#pragma once + +#include + +#include +#include + +#include "market.hpp" +#include "monetaryamount.hpp" +#include "proto-public-trade.hpp" +#include "publictrade.hpp" +#include "time-window.hpp" +#include "timedef.hpp" +#include "trade-data.pb.h" +#include "tradeside.hpp" + +namespace cct { + +class ProtobufBaseDataTest : public ::testing::Test { + protected: + TimePoint tp1{milliseconds{std::numeric_limits::max() / 10000000}}; + TimePoint tp2{milliseconds{std::numeric_limits::max() / 9000000}}; + TimePoint tp3{milliseconds{std::numeric_limits::max() / 8000000}}; + TimePoint tp4{milliseconds{std::numeric_limits::max() / 7000000}}; + TimePoint tp5{milliseconds{std::numeric_limits::max() / 6900000}}; + TimePoint tp6{milliseconds{std::numeric_limits::max() / 6800000}}; + TimePoint tp7{milliseconds{std::numeric_limits::max() / 6700000}}; + TimePoint tp8{milliseconds{std::numeric_limits::max() / 6600000}}; + TimePoint tp9{milliseconds{std::numeric_limits::max() / 6500000}}; + + TimeWindow timeWindowAll{tp1, tp9 + milliseconds{1}}; + TimeWindow timeWindow14{tp1, tp5}; + TimeWindow timeWindow79{tp7, tp9 + milliseconds{1}}; + + Market mk1{"ETH", "USDT"}; + Market mk2{"BTC", "USD"}; + Market mk3{"SHIB", "USDT"}; + Market mk4{"SOL", "BTC"}; + Market mk5{"SOL", "ETH"}; + Market mk6{"ETH", "BTC"}; + Market mk7{"DOGE", "CAD"}; + + PublicTrade pt1{TradeSide::kBuy, MonetaryAmount{"0.13", mk1.base()}, MonetaryAmount{"1500.5", mk1.quote()}, tp1}; + PublicTrade pt2{TradeSide::kSell, MonetaryAmount{"3.7", mk1.base()}, MonetaryAmount{"1500.5", mk1.quote()}, tp2}; + PublicTrade pt3{TradeSide::kBuy, MonetaryAmount{"0.004", mk1.base()}, MonetaryAmount{1501, mk1.quote()}, tp3}; + PublicTrade pt4{TradeSide::kBuy, MonetaryAmount{"44473434", mk3.base()}, MonetaryAmount{"0.00045", mk3.quote()}, tp1}; + PublicTrade pt5{TradeSide::kBuy, MonetaryAmount{"45.0986", mk4.base()}, MonetaryAmount{"0.00045", mk4.quote()}, tp7}; + PublicTrade pt6{TradeSide::kSell, MonetaryAmount{"0.81153", mk6.base()}, MonetaryAmount{"0.0834", mk6.quote()}, tp6}; + PublicTrade pt7{TradeSide::kSell, MonetaryAmount{694873, mk7.base()}, MonetaryAmount{"0.045", mk7.quote()}, tp8}; + PublicTrade pt8{TradeSide::kSell, MonetaryAmount{"0.1", mk2.base()}, MonetaryAmount{50000, mk2.quote()}, tp4}; + PublicTrade pt9{TradeSide::kSell, MonetaryAmount{"56", mk1.base()}, MonetaryAmount{1300, mk1.quote()}, tp5}; + PublicTrade pt10{TradeSide::kBuy, MonetaryAmount{"37.8", mk5.base()}, MonetaryAmount{"0.032", mk5.quote()}, tp9}; + + ::objects::TradeData td1{ConvertPublicTradeToTradeData(pt1)}; + ::objects::TradeData td2{ConvertPublicTradeToTradeData(pt2)}; + ::objects::TradeData td3{ConvertPublicTradeToTradeData(pt3)}; + ::objects::TradeData td4{ConvertPublicTradeToTradeData(pt4)}; + ::objects::TradeData td5{ConvertPublicTradeToTradeData(pt5)}; + ::objects::TradeData td6{ConvertPublicTradeToTradeData(pt6)}; + ::objects::TradeData td7{ConvertPublicTradeToTradeData(pt7)}; + ::objects::TradeData td8{ConvertPublicTradeToTradeData(pt8)}; + ::objects::TradeData td9{ConvertPublicTradeToTradeData(pt9)}; + ::objects::TradeData td10{ConvertPublicTradeToTradeData(pt10)}; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/test/serialization-tools_test.cpp b/src/serialization/test/serialization-tools_test.cpp new file mode 100644 index 00000000..e87571b3 --- /dev/null +++ b/src/serialization/test/serialization-tools_test.cpp @@ -0,0 +1,24 @@ +#include "serialization-tools.hpp" + +#include + +namespace cct { + +TEST(SerializationTools, ComputeProtoSubPath) { + EXPECT_EQ(ComputeProtoSubPath("/path/to/data", "upbit", "market-order-book"), + "/path/to/data/serialized/market-order-book/upbit"); + EXPECT_EQ(ComputeProtoSubPath("another-path", "bithumb", "trades"), "another-path/serialized/trades/bithumb"); + EXPECT_EQ(ComputeProtoSubPath(".", "kraken", "trades"), "./serialized/trades/kraken"); + EXPECT_EQ(ComputeProtoSubPath("/path/to/data", "binance", "market-order-book"), + "/path/to/data/serialized/market-order-book/binance"); + EXPECT_EQ(ComputeProtoSubPath("/path/to/data", "kucoin", "trades"), "/path/to/data/serialized/trades/kucoin"); +} + +TEST(SerializationTools, ComputeProtoFileName) { + EXPECT_EQ(ComputeProtoFileName(0), "00:00:00_00:59:59.binpb"); + EXPECT_EQ(ComputeProtoFileName(4), "04:00:00_04:59:59.binpb"); + EXPECT_EQ(ComputeProtoFileName(17), "17:00:00_17:59:59.binpb"); + EXPECT_EQ(ComputeProtoFileName(23), "23:00:00_23:59:59.binpb"); +} + +} // namespace cct \ No newline at end of file diff --git a/src/tech/include/durationstring.hpp b/src/tech/include/durationstring.hpp index 0c1b7a0d..04133d0c 100644 --- a/src/tech/include/durationstring.hpp +++ b/src/tech/include/durationstring.hpp @@ -6,6 +6,11 @@ #include "timedef.hpp" namespace cct { + +/// Check if 'str' starts with a Duration. +/// Returns the duration string length (0 if no duration detected) +int DurationLen(std::string_view str); + /// Parse given string representation of a duration and return the duration. /// Amounts and units may be separated by spaces. For example: /// "1h45min" is allowed, as well as "1h 45min" and "1 h 45 min " diff --git a/src/tech/include/unitsparser.hpp b/src/tech/include/unitsparser.hpp index 9a2a95ed..e7cfd975 100644 --- a/src/tech/include/unitsparser.hpp +++ b/src/tech/include/unitsparser.hpp @@ -4,6 +4,7 @@ #include namespace cct { + /// Parses a string representation of a number of bytes. /// string should contain an integral number (decimal not supported) possibly followed by one of these units: /// - T, G, M, k for multiples of 1000 @@ -11,4 +12,5 @@ namespace cct { /// Note: it is a simplified version of the syntax used by Kubernetes: /// https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ int64_t ParseNumberOfBytes(std::string_view sizeStr); + } // namespace cct \ No newline at end of file diff --git a/src/tech/src/durationstring.cpp b/src/tech/src/durationstring.cpp index a46aad75..e8276bfc 100644 --- a/src/tech/src/durationstring.cpp +++ b/src/tech/src/durationstring.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include "cct_cctype.hpp" @@ -23,6 +24,41 @@ constexpr std::pair kDurationUnits[] = { }; } +int DurationLen(std::string_view str) { + const std::size_t sz = str.size(); + if (sz == 0) { + return 0; + } + std::size_t charPos = 0; + while (charPos < sz && isspace(str[charPos])) { + ++charPos; + } + int value{}; + const auto [ptr, err] = std::from_chars(str.data() + charPos, str.data() + str.size(), value); + if (err != std::errc() || value <= 0) { + return 0; + } + charPos = ptr - str.data(); + + while (charPos < sz && isspace(str[charPos])) { + ++charPos; + } + const std::size_t first = charPos; + while (charPos < sz && islower(str[charPos])) { + ++charPos; + } + const std::string_view timeUnitStr(str.begin() + first, str.begin() + charPos); + + const auto it = std::ranges::find_if(kDurationUnits, [timeUnitStr](const auto &durationUnitWithDuration) { + return durationUnitWithDuration.first == timeUnitStr; + }); + if (it == std::end(kDurationUnits)) { + return 0; + } + // There is a substring with size 'charPos' that represents a duration + return charPos + DurationLen(str.substr(charPos)); +} + Duration ParseDuration(std::string_view durationStr) { while (!durationStr.empty() && isspace(durationStr.front())) { durationStr.remove_prefix(1); diff --git a/src/tech/test/durationstring_test.cpp b/src/tech/test/durationstring_test.cpp index 96cb74ae..a524f133 100644 --- a/src/tech/test/durationstring_test.cpp +++ b/src/tech/test/durationstring_test.cpp @@ -9,6 +9,20 @@ namespace cct { +TEST(DurationLen, Basic) { EXPECT_EQ(DurationLen("99min"), 5); } + +TEST(DurationLen, BasicComplex) { EXPECT_EQ(DurationLen("34d45min"), 8); } + +TEST(DurationLen, BasicWithComma) { EXPECT_EQ(DurationLen("23s,bithumb"), 3); } + +TEST(DurationLen, ComplexWithSpaces) { EXPECT_EQ(DurationLen(" 1 d 52 h,kraken"), 9); } + +TEST(DurationLen, NegativeValue) { EXPECT_EQ(DurationLen("-3sec"), 0); } + +TEST(DurationLen, InvalidTimeUnit) { EXPECT_EQ(DurationLen("63po"), 0); } + +TEST(DurationLen, DoesNotStartWithNumber) { EXPECT_EQ(DurationLen("us"), 0); } + TEST(ParseDuration, EmptyDurationNotAllowed) { EXPECT_THROW(ParseDuration(""), invalid_argument); } TEST(ParseDuration, DurationDays) { EXPECT_EQ(ParseDuration("37d"), std::chrono::days(37)); } diff --git a/src/tech/test/simpletable_test.cpp b/src/tech/test/simpletable_test.cpp index 9000329e..65d62868 100644 --- a/src/tech/test/simpletable_test.cpp +++ b/src/tech/test/simpletable_test.cpp @@ -157,7 +157,6 @@ TEST_F(SimpleTableTest, EmptyCellShouldBePossible) { | 12 | | Nothing here | | -4 | | | +---------------+----------+-----------------------+)"; - EXPECT_EQ(ss.view(), kExpected); } diff --git a/src/trading/CMakeLists.txt b/src/trading/CMakeLists.txt new file mode 100644 index 00000000..5df5ce59 --- /dev/null +++ b/src/trading/CMakeLists.txt @@ -0,0 +1,2 @@ +add_subdirectory(algorithms) +add_subdirectory(common) \ No newline at end of file diff --git a/src/trading/algorithms/CMakeLists.txt b/src/trading/algorithms/CMakeLists.txt new file mode 100644 index 00000000..51e64005 --- /dev/null +++ b/src/trading/algorithms/CMakeLists.txt @@ -0,0 +1,10 @@ +aux_source_directory(src TRADING-ALGORITHMS_SRC) + +add_library(coincenter_trading-algorithms STATIC ${TRADING-ALGORITHMS_SRC}) + +target_link_libraries(coincenter_trading-algorithms PUBLIC coincenter_api-objects) +target_link_libraries(coincenter_trading-algorithms PUBLIC coincenter_objects) +target_link_libraries(coincenter_trading-algorithms PUBLIC coincenter_tech) +target_link_libraries(coincenter_trading-algorithms PUBLIC coincenter_trading-common) + +target_include_directories(coincenter_trading-algorithms PUBLIC include) \ No newline at end of file diff --git a/src/trading/algorithms/include/dummy-market-trader.hpp b/src/trading/algorithms/include/dummy-market-trader.hpp new file mode 100644 index 00000000..6d64c817 --- /dev/null +++ b/src/trading/algorithms/include/dummy-market-trader.hpp @@ -0,0 +1,20 @@ +#pragma once + +#include "abstract-market-trader.hpp" +#include "trader-command.hpp" + +namespace cct { + +class MarketDataView; +class MarketTraderEngineState; + +class DummyMarketTrader : public AbstractMarketTrader { + public: + static constexpr std::string_view kName = "dummy-trader"; + + DummyMarketTrader(const MarketTraderEngineState &marketTraderEngineState) noexcept; + + TraderCommand trade([[maybe_unused]] const MarketDataView &marketDataView) override; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/trading/algorithms/include/example-market-trader.hpp b/src/trading/algorithms/include/example-market-trader.hpp new file mode 100644 index 00000000..90917d4a --- /dev/null +++ b/src/trading/algorithms/include/example-market-trader.hpp @@ -0,0 +1,20 @@ +#pragma once + +#include "abstract-market-trader.hpp" +#include "trader-command.hpp" + +namespace cct { + +class MarketDataView; +class MarketTraderEngineState; + +class ExampleMarketTrader : public AbstractMarketTrader { + public: + static constexpr std::string_view kName = "example-trader"; + + ExampleMarketTrader(const MarketTraderEngineState &marketTraderEngineState) noexcept; + + TraderCommand trade([[maybe_unused]] const MarketDataView &marketDataView) override; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/trading/algorithms/include/market-trader-factory.hpp b/src/trading/algorithms/include/market-trader-factory.hpp new file mode 100644 index 00000000..a4cfb3d6 --- /dev/null +++ b/src/trading/algorithms/include/market-trader-factory.hpp @@ -0,0 +1,22 @@ +#pragma once + +#include +#include + +#include "abstract-market-trader-factory.hpp" +#include "abstract-market-trader.hpp" + +namespace cct { + +class MarketTraderEngineState; + +class MarketTraderFactory : public AbstractMarketTraderFactory { + public: + std::span allSupportedAlgorithms() const override; + + /// Creates a new MarketTrader from the underlying type of the algorithm name. + /// For instance, create("dummy-trader") will return a DummyMarketTrader. + std::unique_ptr construct( + std::string_view algorithmName, const MarketTraderEngineState& marketTraderEngineState) const override; +}; +} // namespace cct \ No newline at end of file diff --git a/src/trading/algorithms/src/dummy-market-trader.cpp b/src/trading/algorithms/src/dummy-market-trader.cpp new file mode 100644 index 00000000..2b034b03 --- /dev/null +++ b/src/trading/algorithms/src/dummy-market-trader.cpp @@ -0,0 +1,17 @@ +#include "dummy-market-trader.hpp" + +#include "abstract-market-trader.hpp" +#include "market-data-view.hpp" +#include "market-trader-engine-state.hpp" +#include "trader-command.hpp" + +namespace cct { + +DummyMarketTrader::DummyMarketTrader(const MarketTraderEngineState &marketTraderEngineState) noexcept + : AbstractMarketTrader(kName, marketTraderEngineState) {} + +TraderCommand DummyMarketTrader::trade([[maybe_unused]] const MarketDataView &marketDataView) { + return TraderCommand::Wait(); +} + +} // namespace cct \ No newline at end of file diff --git a/src/trading/algorithms/src/example-market-trader.cpp b/src/trading/algorithms/src/example-market-trader.cpp new file mode 100644 index 00000000..f763859b --- /dev/null +++ b/src/trading/algorithms/src/example-market-trader.cpp @@ -0,0 +1,17 @@ +#include "example-market-trader.hpp" + +#include "abstract-market-trader.hpp" +#include "market-data-view.hpp" +#include "market-trader-engine-state.hpp" +#include "trader-command.hpp" + +namespace cct { + +ExampleMarketTrader::ExampleMarketTrader(const MarketTraderEngineState &marketTraderEngineState) noexcept + : AbstractMarketTrader(kName, marketTraderEngineState) {} + +TraderCommand ExampleMarketTrader::trade([[maybe_unused]] const MarketDataView &marketDataView) { + return TraderCommand::Place(TradeSide::kSell); +} + +} // namespace cct \ No newline at end of file diff --git a/src/trading/algorithms/src/market-trader-factory.cpp b/src/trading/algorithms/src/market-trader-factory.cpp new file mode 100644 index 00000000..a6a01138 --- /dev/null +++ b/src/trading/algorithms/src/market-trader-factory.cpp @@ -0,0 +1,33 @@ +#include "market-trader-factory.hpp" + +#include +#include +#include + +#include "cct_invalid_argument_exception.hpp" +#include "dummy-market-trader.hpp" +#include "example-market-trader.hpp" + +namespace cct { + +class MarketTraderEngineState; + +std::span MarketTraderFactory::allSupportedAlgorithms() const { + static constexpr std::string_view kAllAlgorithms[] = {DummyMarketTrader::kName, ExampleMarketTrader::kName}; + return kAllAlgorithms; +} + +std::unique_ptr MarketTraderFactory::construct( + std::string_view algorithmName, const MarketTraderEngineState &marketTraderEngineState) const { + if (algorithmName == DummyMarketTrader::kName) { + return std::make_unique(marketTraderEngineState); + } + + if (algorithmName == ExampleMarketTrader::kName) { + return std::make_unique(marketTraderEngineState); + } + + throw invalid_argument("Unknown trader algorithm '{}'", algorithmName); +} + +} // namespace cct \ No newline at end of file diff --git a/src/trading/common/CMakeLists.txt b/src/trading/common/CMakeLists.txt new file mode 100644 index 00000000..cd1bb22f --- /dev/null +++ b/src/trading/common/CMakeLists.txt @@ -0,0 +1,7 @@ +aux_source_directory(src TRADING-COMMON_SRC) + +add_library(coincenter_trading-common STATIC ${TRADING-COMMON_SRC}) +target_link_libraries(coincenter_trading-common PUBLIC coincenter_api-objects) +target_link_libraries(coincenter_trading-common PUBLIC coincenter_objects) +target_link_libraries(coincenter_trading-common PUBLIC coincenter_tech) +target_include_directories(coincenter_trading-common PUBLIC include) \ No newline at end of file diff --git a/src/trading/common/include/abstract-market-trader-factory.hpp b/src/trading/common/include/abstract-market-trader-factory.hpp new file mode 100644 index 00000000..14bbbaa8 --- /dev/null +++ b/src/trading/common/include/abstract-market-trader-factory.hpp @@ -0,0 +1,24 @@ +#pragma once + +#include +#include +#include + +namespace cct { + +class AbstractMarketTrader; +class MarketTraderEngineState; + +/// Interface that you need to derive to provide your own algorithms to coincenter. +class AbstractMarketTraderFactory { + public: + /// Returns a span of all supported algorithms of this market trader factory. + virtual std::span allSupportedAlgorithms() const = 0; + + /// Creates a new MarketTrader from the underlying type of the algorithm name. + /// For instance, create("dummy-trader") will return a DummyMarketTrader. + virtual std::unique_ptr construct( + std::string_view algorithmName, const MarketTraderEngineState& marketTraderEngineState) const = 0; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/trading/common/include/abstract-market-trader.hpp b/src/trading/common/include/abstract-market-trader.hpp new file mode 100644 index 00000000..db7a41ae --- /dev/null +++ b/src/trading/common/include/abstract-market-trader.hpp @@ -0,0 +1,36 @@ +#pragma once + +#include + +#include "trader-command.hpp" + +namespace cct { + +class MarketDataView; +class MarketTraderEngineState; + +/// Base class for a trading algorithm. +/// It can be derived and only need to implement the trade method to be used in the MarketTraderEngine. +/// the Market Trader Engine state is also provided as a const reference to have the data of the context (orders, +/// available amounts). +class AbstractMarketTrader { + public: + virtual ~AbstractMarketTrader() = default; + + virtual TraderCommand trade(const MarketDataView &marketDataView) = 0; + + std::string_view name() const { return _name; } + + const MarketTraderEngineState &marketTraderEngineState() const { return _marketTraderEngineState; } + + protected: + /// Constructs a new AbstractMarketTrader. + /// @param name should be a view to a constant string as only a std::string_view will be stored in this object. + AbstractMarketTrader(std::string_view name, const MarketTraderEngineState &marketTraderEngineState) noexcept; + + private: + std::string_view _name; + const MarketTraderEngineState &_marketTraderEngineState; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/trading/common/include/market-data-view.hpp b/src/trading/common/include/market-data-view.hpp new file mode 100644 index 00000000..8e08b83a --- /dev/null +++ b/src/trading/common/include/market-data-view.hpp @@ -0,0 +1,46 @@ +#pragma once + +#include +#include + +#include "marketorderbook.hpp" +#include "publictrade.hpp" +#include "timedef.hpp" + +namespace cct { + +/// A class providing a view to current and historical market data for the market trader. +class MarketDataView { + public: + /// Get a reference to last (current for this turn) market order book + const MarketOrderBook ¤tMarketOrderBook() const { return _pOrderBooks[_currentOrderBookEndPos - 1U]; } + + /// Get a span of all historical market order books since the start of the market trader engine (including current / + /// last one) + std::span pastMarketOrderBooks() const { return {_pOrderBooks, _currentOrderBookEndPos}; } + + /// Get a span of all new public trades that occurred before last (current for this turn) market order book that have + /// not been seen before. + std::span currentPublicTrades() const { return {_pCurrentTradesBeg, _pCurrentTradesEnd}; } + + /// Get a span of all public trades since the start of the market trader engine (including current / last ones). + std::span pastPublicTrades() const { return {_pPublicTradesBeg, _pCurrentTradesEnd}; } + + private: + friend class MarketTraderEngine; + + MarketDataView(const MarketOrderBook *pOrderBooks, const PublicTrade *pPublicTradesBeg, + const PublicTrade *pPublicTradesEnd) noexcept; + + void advanceUntil(TimePoint marketOrderBookTs); + + const MarketOrderBook *_pOrderBooks; + const PublicTrade *_pPublicTradesBeg; + const PublicTrade *_pPublicTradesEnd; + + const PublicTrade *_pCurrentTradesBeg; + const PublicTrade *_pCurrentTradesEnd; + std::size_t _currentOrderBookEndPos{}; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/trading/common/include/market-trader-engine-state.hpp b/src/trading/common/include/market-trader-engine-state.hpp new file mode 100644 index 00000000..a2951519 --- /dev/null +++ b/src/trading/common/include/market-trader-engine-state.hpp @@ -0,0 +1,69 @@ +#pragma once + +#include +#include +#include + +#include "cct_type_traits.hpp" +#include "closed-order.hpp" +#include "exchangeconfig.hpp" +#include "exchangeprivateapitypes.hpp" +#include "monetaryamount.hpp" +#include "opened-order.hpp" +#include "stringhelpers.hpp" +#include "trader-command.hpp" + +namespace cct { + +/// Contains the mutable state of the market trader engine. +class MarketTraderEngineState { + public: + MarketTraderEngineState(MonetaryAmount startAmountBase, MonetaryAmount startAmountQuote); + + MonetaryAmount availableBaseAmount() const { return _availableBaseAmount; } + MonetaryAmount availableQuoteAmount() const { return _availableQuoteAmount; } + + std::span openedOrders() const { return _openedOrders; } + std::span closedOrders() const { return _closedOrders; } + + using trivially_relocatable = std::bool_constant && + is_trivially_relocatable_v>::type; + + private: + friend class MarketTraderEngine; + + MonetaryAmount computeBuyFrom(TraderCommand traderCommand) const; + + MonetaryAmount computeSellVolume(TraderCommand traderCommand) const; + + void placeBuyOrder(const ExchangeConfig &exchangeConfig, TimePoint placedTime, MonetaryAmount remainingVolume, + MonetaryAmount price, MonetaryAmount matchedVolume, MonetaryAmount from, + ExchangeConfig::FeeType feeType); + + void placeSellOrder(const ExchangeConfig &exchangeConfig, TimePoint placedTime, MonetaryAmount remainingVolume, + MonetaryAmount price, MonetaryAmount matchedVolume, ExchangeConfig::FeeType feeType); + + auto nextOrderId() { return ToString(++_nextOrderId); } + + void adjustOpenedOrderRemainingVolume(const OpenedOrder &matchedOrder, MonetaryAmount newMatchedVolume); + + void countMatchedPart(const ExchangeConfig &exchangeConfig, const OpenedOrder &matchedOrder, MonetaryAmount price, + MonetaryAmount newMatchedVolume, TimePoint matchedTime); + + void cancelOpenedOrder(int32_t orderId); + + OpenedOrderVector::const_iterator findOpenedOrder(int32_t orderId); + + void cancelAllOpenedOrders(); + + void eraseClosedOpenedOrders(std::span closedOpenedOrders); + + void adjustAvailableAmountsCancel(const OpenedOrder &openedOrder); + + MonetaryAmount _availableBaseAmount; + MonetaryAmount _availableQuoteAmount; + OpenedOrderVector _openedOrders; + ClosedOrderVector _closedOrders; + int _nextOrderId{}; +}; +} // namespace cct \ No newline at end of file diff --git a/src/trading/common/include/market-trader-engine.hpp b/src/trading/common/include/market-trader-engine.hpp new file mode 100644 index 00000000..9fb686e7 --- /dev/null +++ b/src/trading/common/include/market-trader-engine.hpp @@ -0,0 +1,64 @@ +#pragma once + +#include +#include + +#include "abstract-market-trader.hpp" +#include "cct_type_traits.hpp" +#include "exchangeconfig.hpp" +#include "exchangeprivateapitypes.hpp" +#include "market-order-book-vector.hpp" +#include "market-trader-engine-state.hpp" +#include "market-trading-result.hpp" +#include "market.hpp" +#include "marketorderbook.hpp" +#include "monetaryamount.hpp" +#include "public-trade-vector.hpp" +#include "trade-range-stats.hpp" +#include "trader-command.hpp" + +namespace cct { + +class MarketTraderEngine { + public: + MarketTraderEngine(const ExchangeConfig &exchangeConfig, Market market, MonetaryAmount startAmountBase, + MonetaryAmount startAmountQuote); + + Market market() const { return {_startAmountBase.currencyCode(), _startAmountQuote.currencyCode()}; } + + void registerMarketTrader(std::unique_ptr marketTrader); + + TradeRangeStats validateRange(MarketOrderBookVector &marketOrderBooks, PublicTradeVector &publicTrades); + + TradeRangeStats validateRange(MarketOrderBookVector &&marketOrderBooks, PublicTradeVector &&publicTrades); + + TradeRangeStats tradeRange(MarketOrderBookVector &&marketOrderBooks, PublicTradeVector &&publicTrades); + + const MarketTraderEngineState &marketTraderEngineState() const { return _marketTraderEngineState; } + + MarketTradingResult finalizeAndComputeResult(); + + using trivially_relocatable = + std::bool_constant && is_trivially_relocatable_v && + is_trivially_relocatable_v>::type; + + private: + void buy(const MarketOrderBook &marketOrderBook, MonetaryAmount from, PriceStrategy priceStrategy); + void sell(const MarketOrderBook &marketOrderBook, MonetaryAmount volume, PriceStrategy priceStrategy); + + void updatePrice(const MarketOrderBook &marketOrderBook, TraderCommand traderCommand); + + void cancelCommand(int32_t orderId); + + void checkOpenedOrdersMatching(const MarketOrderBook &marketOrderBook); + + MonetaryAmount _startAmountBase; + MonetaryAmount _startAmountQuote; + const ExchangeConfig &_exchangeConfig; + std::unique_ptr _marketTrader; + Market _market; + MarketTraderEngineState _marketTraderEngineState; + OpenedOrderVector _newlyClosedOrders; + MarketOrderBook _lastMarketOrderBook; +}; +} // namespace cct \ No newline at end of file diff --git a/src/trading/common/include/market-trading-global-result.hpp b/src/trading/common/include/market-trading-global-result.hpp new file mode 100644 index 00000000..8d404460 --- /dev/null +++ b/src/trading/common/include/market-trading-global-result.hpp @@ -0,0 +1,13 @@ +#pragma once + +#include "market-trading-result.hpp" +#include "trade-range-stats.hpp" + +namespace cct { + +struct MarketTradingGlobalResult { + MarketTradingResult result; + TradeRangeStats stats; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/trading/common/include/market-trading-result.hpp b/src/trading/common/include/market-trading-result.hpp new file mode 100644 index 00000000..7c462393 --- /dev/null +++ b/src/trading/common/include/market-trading-result.hpp @@ -0,0 +1,43 @@ +#pragma once + +#include +#include + +#include "cct_type_traits.hpp" +#include "closed-order.hpp" +#include "exchangeprivateapitypes.hpp" +#include "market.hpp" +#include "monetaryamount.hpp" + +namespace cct { + +class MarketTradingResult { + public: + MarketTradingResult() noexcept = default; + + MarketTradingResult(std::string_view algorithmName, MonetaryAmount startBaseAmount, MonetaryAmount startQuoteAmount, + MonetaryAmount quoteAmountDelta, ClosedOrderVector matchedOrders); + + std::string_view algorithmName() const { return _algorithmName; } + + Market market() const { return {_startBaseAmount.currencyCode(), _startQuoteAmount.currencyCode()}; } + + MonetaryAmount startBaseAmount() const { return _startBaseAmount; } + + MonetaryAmount startQuoteAmount() const { return _startQuoteAmount; } + + MonetaryAmount quoteAmountDelta() const { return _quoteAmountDelta; } + + std::span matchedOrders() const { return _matchedOrders; } + + using trivially_relocatable = is_trivially_relocatable::type; + + private: + std::string_view _algorithmName; + MonetaryAmount _startBaseAmount; + MonetaryAmount _startQuoteAmount; + MonetaryAmount _quoteAmountDelta; + ClosedOrderVector _matchedOrders; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/trading/common/include/trade-range-stats.hpp b/src/trading/common/include/trade-range-stats.hpp new file mode 100644 index 00000000..a5499d54 --- /dev/null +++ b/src/trading/common/include/trade-range-stats.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include + +namespace cct { + +struct TradeRangeResultsStats { + int32_t nbSuccessful{}; + int32_t nbError{}; + + TradeRangeResultsStats operator+(const TradeRangeResultsStats &rhs) const { + return TradeRangeResultsStats{nbSuccessful + rhs.nbSuccessful, nbError + rhs.nbError}; + } + + TradeRangeResultsStats &operator+=(const TradeRangeResultsStats &rhs) { return *this = *this + rhs; } +}; + +struct TradeRangeStats { + TradeRangeResultsStats marketOrderBookStats; + TradeRangeResultsStats publicTradeStats; + + TradeRangeStats operator+(const TradeRangeStats &rhs) const { + return TradeRangeStats{marketOrderBookStats + rhs.marketOrderBookStats, publicTradeStats + rhs.publicTradeStats}; + } + + TradeRangeStats &operator+=(const TradeRangeStats &rhs) { return *this = *this + rhs; } +}; + +} // namespace cct \ No newline at end of file diff --git a/src/trading/common/include/trader-command.hpp b/src/trading/common/include/trader-command.hpp new file mode 100644 index 00000000..ced95e0d --- /dev/null +++ b/src/trading/common/include/trader-command.hpp @@ -0,0 +1,56 @@ +#pragma once + +#include +#include + +#include "orderid.hpp" +#include "priceoptionsdef.hpp" +#include "tradeside.hpp" + +namespace cct { + +class TraderCommand { + public: + enum class Type : int8_t { kWait, kBuy, kSell, kUpdatePrice, kCancel }; + + static constexpr int32_t kAllOrdersId = 0; + + /// Creates a wait command. + static TraderCommand Wait(); + + /// Creates a Place command with given intensity, side and strategy. It should be in the range [0, 100]. + static TraderCommand Place(TradeSide tradeSide, int8_t amountIntensityPercentage = 100, + PriceStrategy priceStrategy = PriceStrategy::kMaker); + + /// Creates a Cancel command with optional orderId. + /// If orderId is not specified (or empty string), will cancel all opened orders. + static TraderCommand Cancel(OrderIdView orderId = std::string_view()); + + /// Creates an Update command for specified orderId. + /// Equivalent to a Cancel and a Place at new price for remaining unmatched amount at the same turn. + static TraderCommand UpdatePrice(OrderIdView orderId, PriceStrategy priceStrategy = PriceStrategy::kMaker); + + int32_t orderId() const { return _orderId; } + + /// If this is a Place command, return the amount intensity percentage in [0, 100] + int8_t amountIntensityPercentage() const { return _amountIntensityPercentage; } + + Type type() const { return _type; } + + TradeSide tradeSide() const; + + PriceStrategy priceStrategy() const { return _priceStrategy; } + + private: + static constexpr int8_t kWaitValue = 0; + static constexpr int8_t kCancelValue = std::numeric_limits::min(); + + TraderCommand(Type type, int32_t orderId, int8_t amountIntensityPercentage, PriceStrategy priceStrategy); + + int32_t _orderId; + Type _type; + int8_t _amountIntensityPercentage; + PriceStrategy _priceStrategy; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/trading/common/src/abstract-market-trader.cpp b/src/trading/common/src/abstract-market-trader.cpp new file mode 100644 index 00000000..3bd381c1 --- /dev/null +++ b/src/trading/common/src/abstract-market-trader.cpp @@ -0,0 +1,13 @@ +#include "abstract-market-trader.hpp" + +#include + +#include "market-trader-engine-state.hpp" + +namespace cct { + +AbstractMarketTrader::AbstractMarketTrader(std::string_view name, + const MarketTraderEngineState& marketTraderEngineState) noexcept + : _name(name), _marketTraderEngineState(marketTraderEngineState) {} + +} // namespace cct \ No newline at end of file diff --git a/src/trading/common/src/market-data-view.cpp b/src/trading/common/src/market-data-view.cpp new file mode 100644 index 00000000..14d244a2 --- /dev/null +++ b/src/trading/common/src/market-data-view.cpp @@ -0,0 +1,27 @@ +#include "market-data-view.hpp" + +#include + +#include "publictrade.hpp" + +namespace cct { + +MarketDataView::MarketDataView(const MarketOrderBook *pOrderBooks, const PublicTrade *pPublicTradesBeg, + const PublicTrade *pPublicTradesEnd) noexcept + : _pOrderBooks(pOrderBooks), + _pPublicTradesBeg(pPublicTradesBeg), + _pPublicTradesEnd(pPublicTradesEnd), + _pCurrentTradesBeg(pPublicTradesBeg), + _pCurrentTradesEnd(pPublicTradesEnd) {} + +void MarketDataView::advanceUntil(TimePoint marketOrderBookTs) { + // Advance the public trades iterator until we reach one that occurred after our current market order book + _pCurrentTradesBeg = _pCurrentTradesEnd; + _pCurrentTradesEnd = std::partition_point( + _pCurrentTradesBeg, _pPublicTradesEnd, + [marketOrderBookTs](const auto &publicTrade) { return publicTrade.time() < marketOrderBookTs; }); + + ++_currentOrderBookEndPos; +} + +} // namespace cct \ No newline at end of file diff --git a/src/trading/common/src/market-trader-engine-state.cpp b/src/trading/common/src/market-trader-engine-state.cpp new file mode 100644 index 00000000..45cb2fbe --- /dev/null +++ b/src/trading/common/src/market-trader-engine-state.cpp @@ -0,0 +1,136 @@ +#include "market-trader-engine-state.hpp" + +#include +#include + +#include "cct_exception.hpp" +#include "closed-order.hpp" +#include "exchangeconfig.hpp" +#include "monetaryamount.hpp" +#include "opened-order.hpp" +#include "timedef.hpp" +#include "trader-command.hpp" +#include "tradeside.hpp" + +namespace cct { +MarketTraderEngineState::MarketTraderEngineState(MonetaryAmount startAmountBase, MonetaryAmount startAmountQuote) + : _availableBaseAmount(startAmountBase), _availableQuoteAmount(startAmountQuote) {} + +MonetaryAmount MarketTraderEngineState::computeBuyFrom(TraderCommand traderCommand) const { + return (_availableQuoteAmount * traderCommand.amountIntensityPercentage()) / 100; +} + +MonetaryAmount MarketTraderEngineState::computeSellVolume(TraderCommand traderCommand) const { + return (_availableBaseAmount * traderCommand.amountIntensityPercentage()) / 100; +} + +void MarketTraderEngineState::placeBuyOrder(const ExchangeConfig &exchangeConfig, TimePoint placedTime, + MonetaryAmount remainingVolume, MonetaryAmount price, + MonetaryAmount matchedVolume, MonetaryAmount from, + ExchangeConfig::FeeType feeType) { + _availableBaseAmount += exchangeConfig.applyFee(matchedVolume, feeType); + _availableQuoteAmount -= from; + + if (remainingVolume == 0) { + _closedOrders.emplace_back(nextOrderId(), matchedVolume, price, placedTime, placedTime, TradeSide::kBuy); + } else { + _openedOrders.emplace_back(nextOrderId(), matchedVolume, remainingVolume, price, placedTime, TradeSide::kBuy); + } +} + +void MarketTraderEngineState::placeSellOrder(const ExchangeConfig &exchangeConfig, TimePoint placedTime, + MonetaryAmount remainingVolume, MonetaryAmount price, + MonetaryAmount matchedVolume, ExchangeConfig::FeeType feeType) { + _availableBaseAmount -= (remainingVolume + matchedVolume); + _availableQuoteAmount += exchangeConfig.applyFee(matchedVolume.toNeutral() * price, feeType); + + if (remainingVolume == 0) { + _closedOrders.emplace_back(nextOrderId(), matchedVolume, price, placedTime, placedTime, TradeSide::kSell); + } else { + _openedOrders.emplace_back(nextOrderId(), matchedVolume, remainingVolume, price, placedTime, TradeSide::kSell); + } +} + +void MarketTraderEngineState::adjustOpenedOrderRemainingVolume(const OpenedOrder &matchedOrder, + MonetaryAmount newMatchedVolume) { + auto openedOrderIt = std::ranges::find_if( + _openedOrders, [&matchedOrder](const auto &openedOrder) { return matchedOrder.id() == openedOrder.id(); }); + + *openedOrderIt = OpenedOrder(matchedOrder.id(), matchedOrder.matchedVolume() + newMatchedVolume, + matchedOrder.remainingVolume() - newMatchedVolume, matchedOrder.price(), + matchedOrder.placedTime(), matchedOrder.side()); +} + +void MarketTraderEngineState::countMatchedPart(const ExchangeConfig &exchangeConfig, const OpenedOrder &matchedOrder, + MonetaryAmount price, MonetaryAmount newMatchedVolume, + TimePoint matchedTime) { + switch (matchedOrder.side()) { + case TradeSide::kBuy: + _availableBaseAmount += exchangeConfig.applyFee(newMatchedVolume, ExchangeConfig::FeeType::kMaker); + break; + case TradeSide::kSell: + _availableQuoteAmount += + exchangeConfig.applyFee(newMatchedVolume.toNeutral() * price, ExchangeConfig::FeeType::kMaker); + break; + default: + throw exception("Unknown trade side {}", static_cast(matchedOrder.side())); + } + + ClosedOrder newClosedOrder(matchedOrder.id(), newMatchedVolume, price, matchedOrder.placedTime(), matchedTime, + matchedOrder.side()); + + auto closedOrderIt = + std::ranges::find_if(_closedOrders.rbegin(), _closedOrders.rend(), + [&matchedOrder](const auto &closedOrder) { return closedOrder.id() == matchedOrder.id(); }); + if (closedOrderIt != _closedOrders.rend()) { + *closedOrderIt = closedOrderIt->mergeWith(newClosedOrder); + } else { + _closedOrders.push_back(std::move(newClosedOrder)); + } +} + +void MarketTraderEngineState::cancelOpenedOrder(int32_t orderId) { + const auto orderIdIt = findOpenedOrder(orderId); + adjustAvailableAmountsCancel(*orderIdIt); + _openedOrders.erase(orderIdIt); +} + +OpenedOrderVector::const_iterator MarketTraderEngineState::findOpenedOrder(int32_t orderId) { + const auto orderIdIt = std::ranges::find_if(_openedOrders, [orderId](const OpenedOrder &openedOrder) { + return FromString(openedOrder.id()) == orderId; + }); + if (orderIdIt == _openedOrders.end()) { + throw exception("Unable to find opened order id {}", orderId); + } + return orderIdIt; +} + +void MarketTraderEngineState::cancelAllOpenedOrders() { + std::ranges::for_each(_openedOrders, + [this](const OpenedOrder &openedOrder) { this->adjustAvailableAmountsCancel(openedOrder); }); + _openedOrders.clear(); +} + +void MarketTraderEngineState::adjustAvailableAmountsCancel(const OpenedOrder &openedOrder) { + switch (openedOrder.side()) { + case TradeSide::kBuy: + _availableQuoteAmount += openedOrder.remainingVolume().toNeutral() * openedOrder.price(); + break; + case TradeSide::kSell: + _availableBaseAmount += openedOrder.remainingVolume(); + break; + default: + throw exception("Unknown trade side {}", static_cast(openedOrder.side())); + } +} + +void MarketTraderEngineState::eraseClosedOpenedOrders(std::span closedOpenedOrders) { + const auto [first, last] = std::ranges::remove_if(_openedOrders, [closedOpenedOrders](const auto &openedOrder) { + return std::ranges::any_of(closedOpenedOrders, [&openedOrder](const auto &closedOpenedOrder) { + return openedOrder.id() == closedOpenedOrder.id(); + }); + }); + _openedOrders.erase(first, last); +} + +} // namespace cct \ No newline at end of file diff --git a/src/trading/common/src/market-trader-engine.cpp b/src/trading/common/src/market-trader-engine.cpp new file mode 100644 index 00000000..cee6aa97 --- /dev/null +++ b/src/trading/common/src/market-trader-engine.cpp @@ -0,0 +1,329 @@ +#include "market-trader-engine.hpp" + +#include +#include +#include + +#include "abstract-market-trader.hpp" +#include "cct_exception.hpp" +#include "cct_log.hpp" +#include "market-data-view.hpp" +#include "marketorderbook.hpp" +#include "monetaryamount.hpp" +#include "priceoptionsdef.hpp" +#include "public-trade-vector.hpp" +#include "publictrade.hpp" +#include "timestring.hpp" +#include "trade-range-stats.hpp" +#include "trader-command.hpp" + +namespace cct { + +MarketTraderEngine::MarketTraderEngine(const ExchangeConfig &exchangeConfig, Market market, + MonetaryAmount startAmountBase, MonetaryAmount startAmountQuote) + : _startAmountBase(startAmountBase), + _startAmountQuote(startAmountQuote), + _exchangeConfig(exchangeConfig), + _market(market), + _marketTraderEngineState(startAmountBase, startAmountQuote) { + if (market != this->market()) { + throw exception("Inconsistent market {} and start amounts {} & {} for MarketTraderEngine", market, startAmountBase, + startAmountQuote); + } +} + +void MarketTraderEngine::registerMarketTrader(std::unique_ptr marketTrader) { + if (_marketTrader) { + throw exception("Cannot register twice a market trader to this MarketTraderEngine"); + } + _marketTrader.swap(marketTrader); +} + +namespace { + +template +TradeRangeResultsStats ValidateRange(VectorType &vec, TimePoint earliestPossibleTime) { + using std::erase_if; + + using ObjType = std::remove_cvref_t().begin())>; + + static_assert(std::is_same_v || std::is_same_v); + + static constexpr std::string_view kObjName = std::is_same_v ? "order book" : "trade"; + + TradeRangeResultsStats stats; + + stats.nbSuccessful = static_cast(vec.size()); + + const auto nbInvalidObjects = erase_if(vec, [](const auto &obj) { return !obj.isValid(); }); + if (nbInvalidObjects != 0) { + log::error("{} {}(s) with invalid data detected", nbInvalidObjects, kObjName); + } + + const auto nbUnsortedObjectsRemoved = erase_if(vec, [&earliestPossibleTime](const auto &obj) { + if (obj.time() < earliestPossibleTime) { + return true; + } + earliestPossibleTime = obj.time(); + return false; + }); + if (nbUnsortedObjectsRemoved != 0) { + log::error("{} {}(s) are not in chronological order", nbUnsortedObjectsRemoved, kObjName); + } + + stats.nbError = nbInvalidObjects + nbUnsortedObjectsRemoved; + stats.nbSuccessful -= stats.nbError; + + return stats; +} + +} // namespace + +TradeRangeStats MarketTraderEngine::validateRange(MarketOrderBookVector &marketOrderBooks, + PublicTradeVector &publicTrades) { + TimePoint earliestPossibleTime; + if (_lastMarketOrderBook.market().isDefined()) { + earliestPossibleTime = _lastMarketOrderBook.time(); + } + + TradeRangeStats tradeRangeStats; + tradeRangeStats.marketOrderBookStats = ValidateRange(marketOrderBooks, earliestPossibleTime); + tradeRangeStats.publicTradeStats = ValidateRange(publicTrades, earliestPossibleTime); + + return tradeRangeStats; +} + +TradeRangeStats MarketTraderEngine::validateRange(MarketOrderBookVector &&marketOrderBooks, + PublicTradeVector &&publicTrades) { + const TradeRangeStats tradeRangeStats = validateRange(marketOrderBooks, publicTrades); + + if (!marketOrderBooks.empty()) { + _lastMarketOrderBook = std::move(marketOrderBooks.back()); + } + + return tradeRangeStats; +} + +TradeRangeStats MarketTraderEngine::tradeRange(MarketOrderBookVector &&marketOrderBooks, + PublicTradeVector &&publicTrades) { + if (!_marketTrader) { + throw exception("registerMarketTrader should have been called before launching the trade engine"); + } + + TradeRangeStats tradeRangeStats{{TradeRangeResultsStats{static_cast(marketOrderBooks.size()), 0}}, + TradeRangeResultsStats{static_cast(publicTrades.size()), 0}}; + + if (marketOrderBooks.empty()) { + return tradeRangeStats; + } + + log::info("[{}] at {} on {} replaying {} order books and {} trades", _marketTrader->name(), + ToString(marketOrderBooks.front().time()), _market, marketOrderBooks.size(), publicTrades.size()); + + // Rolling window of data provided to underlying market trader with data up to latest market order book. + MarketDataView marketDataView(marketOrderBooks.data(), publicTrades.data(), + publicTrades.data() + publicTrades.size()); + + for (const MarketOrderBook &marketOrderBook : marketOrderBooks) { + // First check opened orders status with new market order book data that may match some + checkOpenedOrdersMatching(marketOrderBook); + + // We expect market data (order books and trades) to be sorted by time. + // Advance the market data view iterator until including all data until last market order book time stamp. + marketDataView.advanceUntil(marketOrderBook.time()); + + // Call the user algorithm trading engine and retrieve its decision for next move + const TraderCommand traderCommand = _marketTrader->trade(marketDataView); + + switch (traderCommand.type()) { + case TraderCommand::Type::kWait: + break; + case TraderCommand::Type::kBuy: { + const MonetaryAmount from = _marketTraderEngineState.computeBuyFrom(traderCommand); + + if (from != 0) { + // Attempt to place an order without any available amount, do nothing instead + buy(marketOrderBook, from, traderCommand.priceStrategy()); + } + break; + } + case TraderCommand::Type::kSell: { + const MonetaryAmount volume = _marketTraderEngineState.computeSellVolume(traderCommand); + + if (volume != 0) { + // Attempt to place an order without any available amount, do nothing instead + sell(marketOrderBook, volume, traderCommand.priceStrategy()); + } + break; + } + case TraderCommand::Type::kUpdatePrice: + updatePrice(marketOrderBook, traderCommand); + break; + case TraderCommand::Type::kCancel: + cancelCommand(traderCommand.orderId()); + break; + default: + throw exception("Unsupported trader command {}", static_cast(traderCommand.type())); + } + } + + _lastMarketOrderBook = std::move(marketOrderBooks.back()); + + return tradeRangeStats; +} + +MarketTradingResult MarketTraderEngine::finalizeAndComputeResult() { + if (!_marketTrader) { + throw exception("registerMarketTrader should have been called before computing results"); + } + + _marketTraderEngineState.cancelAllOpenedOrders(); + + // How to compute gain / losses ? + // Let's say we have {x1 XXX + y1 YYY} at the beginning, XXX-YYY being the market, + // and {x2 XXX + y2 YYY} at the end. + // The idea is that we speculate on the YYY currency on this market (we want to increase our YYY amount). + // The formula used to compute gains / losses is the following: + // (y2 - y1) YYY + conversion((x2 - x1) XXX)->YYY + at market price of the last market order book. + + MonetaryAmount quoteAmountDelta = _marketTraderEngineState.availableQuoteAmount() - _startAmountQuote; + MonetaryAmount baseAmountDelta = _marketTraderEngineState.availableBaseAmount() - _startAmountBase; + + if (_lastMarketOrderBook.market().isNeutral()) { + log::debug("Calling finalize on a market trader engine that has not been run"); + } else { + auto [_, avgPrice] = _lastMarketOrderBook.avgPriceAndMatchedAmountTaker(baseAmountDelta.abs()); + + quoteAmountDelta += baseAmountDelta.toNeutral() * avgPrice; + } + + const auto closedOrdersSpan = _marketTraderEngineState.closedOrders(); + + return MarketTradingResult(_marketTrader->name(), _startAmountBase, _startAmountQuote, quoteAmountDelta, + ClosedOrderVector(closedOrdersSpan.begin(), closedOrdersSpan.end())); +} + +void MarketTraderEngine::buy(const MarketOrderBook &marketOrderBook, MonetaryAmount from, PriceStrategy priceStrategy) { + const auto ts = marketOrderBook.time(); + + switch (priceStrategy) { + case PriceStrategy::kMaker: { + const MonetaryAmount price = marketOrderBook.highestBidPrice(); + const MonetaryAmount remainingVolume(from / price, _market.base()); + constexpr MonetaryAmount matchedVolume; + + _marketTraderEngineState.placeBuyOrder(_exchangeConfig, ts, remainingVolume, price, matchedVolume, from, + ExchangeConfig::FeeType::kMaker); + break; + } + case PriceStrategy::kNibble: { + const MonetaryAmount price = marketOrderBook.lowestAskPrice(); + const MonetaryAmount volume(from / price, _market.base()); + const MonetaryAmount matchedVolume = std::min(marketOrderBook.amountAtAskPrice(), volume); + const MonetaryAmount remainingVolume = volume - matchedVolume; + + _marketTraderEngineState.placeBuyOrder(_exchangeConfig, ts, remainingVolume, price, matchedVolume, from, + ExchangeConfig::FeeType::kTaker); + break; + } + case PriceStrategy::kTaker: { + const auto [totalMatchedAmount, avgPrice] = marketOrderBook.avgPriceAndMatchedAmountTaker(from); + if (totalMatchedAmount != 0) { + constexpr MonetaryAmount remainingVolume; + + _marketTraderEngineState.placeBuyOrder(_exchangeConfig, ts, remainingVolume, avgPrice, totalMatchedAmount, from, + ExchangeConfig::FeeType::kTaker); + } + break; + } + default: + throw exception("Unsupported price strategy {}", static_cast(priceStrategy)); + } +} + +void MarketTraderEngine::sell(const MarketOrderBook &marketOrderBook, MonetaryAmount volume, + PriceStrategy priceStrategy) { + switch (priceStrategy) { + case PriceStrategy::kMaker: { + const MonetaryAmount price = marketOrderBook.lowestAskPrice(); + constexpr MonetaryAmount matchedVolume; + + _marketTraderEngineState.placeSellOrder(_exchangeConfig, marketOrderBook.time(), volume, price, matchedVolume, + ExchangeConfig::FeeType::kMaker); + break; + } + case PriceStrategy::kNibble: { + const MonetaryAmount price = marketOrderBook.highestBidPrice(); + const MonetaryAmount matchedVolume = std::min(marketOrderBook.amountAtBidPrice(), volume); + + _marketTraderEngineState.placeSellOrder(_exchangeConfig, marketOrderBook.time(), volume - matchedVolume, price, + matchedVolume, ExchangeConfig::FeeType::kTaker); + break; + } + case PriceStrategy::kTaker: { + const auto [totalMatchedAmount, avgPrice] = marketOrderBook.avgPriceAndMatchedAmountTaker(volume); + + if (totalMatchedAmount != 0) { + constexpr MonetaryAmount remainingVolume; + + _marketTraderEngineState.placeSellOrder(_exchangeConfig, marketOrderBook.time(), remainingVolume, avgPrice, + totalMatchedAmount, ExchangeConfig::FeeType::kTaker); + } + break; + } + default: + throw exception("Unsupported price strategy {}", static_cast(priceStrategy)); + } +} + +void MarketTraderEngine::updatePrice(const MarketOrderBook &marketOrderBook, TraderCommand traderCommand) { + const auto orderIdIt = _marketTraderEngineState.findOpenedOrder(traderCommand.orderId()); + MonetaryAmount remainingAmount = orderIdIt->remainingVolume(); + TradeSide tradeSide = orderIdIt->side(); + MonetaryAmount price = orderIdIt->price(); + + _marketTraderEngineState.cancelOpenedOrder(traderCommand.orderId()); + + switch (tradeSide) { + case TradeSide::kBuy: + buy(marketOrderBook, remainingAmount.toNeutral() * price, traderCommand.priceStrategy()); + break; + case TradeSide::kSell: + sell(marketOrderBook, remainingAmount, traderCommand.priceStrategy()); + break; + default: + throw exception("Unsupported trade side"); + } +} + +void MarketTraderEngine::cancelCommand(int32_t orderId) { + if (orderId == TraderCommand::kAllOrdersId) { + _marketTraderEngineState.cancelAllOpenedOrders(); + } else { + _marketTraderEngineState.cancelOpenedOrder(orderId); + } +} + +void MarketTraderEngine::checkOpenedOrdersMatching(const MarketOrderBook &marketOrderBook) { + _newlyClosedOrders.clear(); + for (const OpenedOrder &openedOrder : _marketTraderEngineState.openedOrders()) { + const auto [newMatchedVolume, avgPrice] = marketOrderBook.avgPriceAndMatchedVolume( + openedOrder.side(), openedOrder.remainingVolume(), openedOrder.price()); + if (newMatchedVolume == 0) { + continue; + } + + _marketTraderEngineState.countMatchedPart(_exchangeConfig, openedOrder, avgPrice, newMatchedVolume, + marketOrderBook.time()); + + if (newMatchedVolume == openedOrder.remainingVolume()) { + _newlyClosedOrders.push_back(openedOrder); + } else { + _marketTraderEngineState.adjustOpenedOrderRemainingVolume(openedOrder, newMatchedVolume); + } + } + + _marketTraderEngineState.eraseClosedOpenedOrders(_newlyClosedOrders); +} + +} // namespace cct diff --git a/src/trading/common/src/market-trading-result.cpp b/src/trading/common/src/market-trading-result.cpp new file mode 100644 index 00000000..ad7e2f59 --- /dev/null +++ b/src/trading/common/src/market-trading-result.cpp @@ -0,0 +1,20 @@ +#include "market-trading-result.hpp" + +#include +#include + +#include "exchangeprivateapitypes.hpp" +#include "monetaryamount.hpp" + +namespace cct { + +MarketTradingResult::MarketTradingResult(std::string_view algorithmName, MonetaryAmount startBaseAmount, + MonetaryAmount startQuoteAmount, MonetaryAmount quoteAmountDelta, + ClosedOrderVector matchedOrders) + : _algorithmName(algorithmName), + _startBaseAmount(startBaseAmount), + _startQuoteAmount(startQuoteAmount), + _quoteAmountDelta(quoteAmountDelta), + _matchedOrders(std::move(matchedOrders)) {} + +} // namespace cct \ No newline at end of file diff --git a/src/trading/common/src/trader-command.cpp b/src/trading/common/src/trader-command.cpp new file mode 100644 index 00000000..e7ddbf63 --- /dev/null +++ b/src/trading/common/src/trader-command.cpp @@ -0,0 +1,62 @@ +#include "trader-command.hpp" + +#include + +#include "cct_exception.hpp" +#include "priceoptionsdef.hpp" +#include "stringhelpers.hpp" +#include "tradeside.hpp" + +namespace cct { +TraderCommand::TraderCommand(Type type, int32_t orderId, int8_t amountIntensityPercentage, PriceStrategy priceStrategy) + : _orderId(orderId), + _type(type), + _amountIntensityPercentage(amountIntensityPercentage), + _priceStrategy(priceStrategy) {} + +TraderCommand TraderCommand::Wait() { return TraderCommand(Type::kWait, kAllOrdersId, 0, PriceStrategy::kMaker); } + +TraderCommand TraderCommand::Place(TradeSide tradeSide, int8_t amountIntensityPercentage, PriceStrategy priceStrategy) { + if (amountIntensityPercentage > 100 || amountIntensityPercentage <= 0) { + throw exception("Invalid amountIntensityPercentage {}", amountIntensityPercentage); + } + Type type; + switch (tradeSide) { + case TradeSide::kBuy: + type = Type::kBuy; + break; + case TradeSide::kSell: + type = Type::kSell; + break; + default: + throw exception("Unexpected trade side"); + } + return TraderCommand(type, kAllOrdersId, amountIntensityPercentage, priceStrategy); +} + +TraderCommand TraderCommand::Cancel(OrderIdView orderId) { + int32_t orderIdInt; + if (!orderId.empty()) { + orderIdInt = FromString(orderId); + } else { + orderIdInt = kAllOrdersId; + } + return TraderCommand(Type::kCancel, orderIdInt, 0, PriceStrategy::kMaker); +} + +TraderCommand TraderCommand::UpdatePrice(OrderIdView orderId, PriceStrategy priceStrategy) { + return TraderCommand(Type::kUpdatePrice, FromString(orderId), 100, priceStrategy); +} + +TradeSide TraderCommand::tradeSide() const { + switch (_type) { + case Type::kBuy: + return TradeSide::kBuy; + case Type::kSell: + return TradeSide::kSell; + default: + throw exception("Unexpected trade command type for trade side"); + } +} + +} // namespace cct \ No newline at end of file