diff --git a/.dockerignore b/.dockerignore index d07c51f7..d8d4c498 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,6 +9,7 @@ LICENSE **/*secret.json data/log data/secret +data/serialized !data/secret/secret_test.json monitoring resources diff --git a/.github/workflows/ubuntu-monitoring.yml b/.github/workflows/ubuntu-special.yml similarity index 85% rename from .github/workflows/ubuntu-monitoring.yml rename to .github/workflows/ubuntu-special.yml index a0dc9091..d7554cff 100644 --- a/.github/workflows/ubuntu-monitoring.yml +++ b/.github/workflows/ubuntu-special.yml @@ -1,4 +1,4 @@ -name: Monitoring +name: Special on: push: @@ -7,14 +7,14 @@ on: pull_request: jobs: - ubuntu-monitoring-build: - name: Build on Ubuntu with monitoring support + ubuntu-special-build: + name: Build on Ubuntu with monitoring / protobuf support runs-on: ubuntu-latest strategy: matrix: compiler: [g++-11] buildmode: [Debug] - build-prometheus-from-source: [0, 1] + build-special-from-source: [0, 1] steps: - name: Checkout repository code @@ -38,7 +38,7 @@ jobs: ninja sudo cmake --install . - if: matrix.build-prometheus-from-source == 0 + if: matrix.build-special-from-source == 0 - name: Create Build Environment run: cmake -E make_directory ${{github.workspace}}/build @@ -46,7 +46,7 @@ jobs: - name: Configure CMake working-directory: ${{github.workspace}}/build shell: bash - run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{matrix.buildmode}} -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCCT_BUILD_PROMETHEUS_FROM_SRC=${{matrix.build-prometheus-from-source}} -GNinja + run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{matrix.buildmode}} -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCCT_BUILD_PROMETHEUS_FROM_SRC=${{matrix.build-special-from-source}} -DCCT_ENABLE_PROTO=${{matrix.build-special-from-source}} -GNinja - name: Build working-directory: ${{github.workspace}}/build diff --git a/.gitignore b/.gitignore index 8c01a78d..0e8592b9 100644 --- a/.gitignore +++ b/.gitignore @@ -66,6 +66,7 @@ cscope* data/cache data/log data/secret +data/serialized data/static/exchangeconfig.json data/static/generalconfig.json monitoring/data/grafana/* diff --git a/CMakeLists.txt b/CMakeLists.txt index a8d536a0..45e25a80 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,6 +36,7 @@ option(CCT_ENABLE_TESTS "Build the unit tests" ${MAIN_PROJECT}) option(CCT_BUILD_EXEC "Build an executable instead of a static library" ${MAIN_PROJECT}) option(CCT_ENABLE_ASAN "Compile with AddressSanitizer" ${CCT_ASAN_BUILD}) option(CCT_ENABLE_CLANG_TIDY "Compile with clang-tidy checks" OFF) +option(CCT_ENABLE_PROTO "Compile with protobuf support (to export data to the outside world)" ON) option(CCT_BUILD_PROMETHEUS_FROM_SRC "Fetch and build from prometheus-cpp sources" OFF) set(CCT_DATA_DIR "${CMAKE_CURRENT_SOURCE_DIR}/data" CACHE PATH "Needed data directory for coincenter. Can also be overriden at runtime with this environment variable") @@ -83,6 +84,7 @@ if(CCT_ENABLE_TESTS) enable_testing() endif() +# nlohmann_json - coincenter json library find_package(nlohmann_json CONFIG) if(NOT nlohmann_json_FOUND) FetchContent_Declare( @@ -94,6 +96,7 @@ if(NOT nlohmann_json_FOUND) FetchContent_MakeAvailable(nlohmann_json) endif() +# spdlog - coincenter logging library find_package(spdlog CONFIG) if(NOT spdlog_FOUND) FetchContent_Declare( @@ -133,6 +136,34 @@ else() endif() endif() +if(CCT_ENABLE_PROTO) + find_package(Protobuf CONFIG) + if(protobuf_FOUND) + message(STATUS "Linking with protobuf ${protobuf_VERSION}") + else() + set(PROTOBUF_VERSION v25.2) + if (MSVC) + # protobuf v25.0 does not compile yet with MSVC: https://github.com/protocolbuffers/protobuf/issues/14602 + set(PROTOBUF_VERSION v24.4) + endif() + + message(STATUS "Compiling protobuf ${PROTOBUF_VERSION} from sources") + + set(protobuf_BUILD_TESTS OFF) + set(ABSL_PROPAGATE_CXX_STD ON) + + FetchContent_Declare( + protobuf + GIT_REPOSITORY https://github.com/protocolbuffers/protobuf.git + GIT_TAG ${PROTOBUF_VERSION} + ) + FetchContent_MakeAvailable(protobuf) + + include(${protobuf_SOURCE_DIR}/cmake/protobuf-generate.cmake) + + endif() +endif() + # Unit Tests #[[ Create an executable @@ -248,12 +279,18 @@ if(CCT_ENABLE_PROMETHEUS) add_compile_definitions(CCT_ENABLE_PROMETHEUS) endif() +if(CCT_ENABLE_PROTO) + add_compile_definitions(CCT_ENABLE_PROTO) + add_compile_definitions("CCT_PROTOBUF_VERSION=\"${PROTOBUF_VERSION}\"") +endif() + # Link to sub folders CMakeLists.txt, from the lowest level to the highest level for documentation # (beware of cyclic dependencies) add_subdirectory(src/tech) add_subdirectory(src/monitoring) add_subdirectory(src/http-request) add_subdirectory(src/objects) +add_subdirectory(src/serialization) add_subdirectory(src/api-objects) add_subdirectory(src/api) add_subdirectory(src/engine) diff --git a/CONFIG.md b/CONFIG.md index efba43e0..b2c031c0 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -150,6 +150,7 @@ Refer to the hardcoded default json example as a model in case of doubt. | *query* | **updateFrequency.depositWallet** | Duration string (ex: `1min`) | Minimum duration between two consecutive requests of deposit information (including wallet) | | *query* | **updateFrequency.currencyInfo** | Duration string (ex: `4h`) | Minimum duration between two consecutive requests of dynamic currency info retrieval on Bithumb only (used for place order) | | *query* | **placeSimulateRealOrder** | Boolean (`true` or `false`) | If `true`, in trade simulation mode (with `--sim`) exchanges which do not support simulated mode in place order will actually place a real order, with the following characteristics: This will allow place of a 'real' order that cannot be matched in practice (if it is, lucky you!) | +| *query* | **marketDataSerialization** | Boolean (`true` or `false`) | If `true` and `coincenter` is compiled with **protobuf** support, some market data will automatically be exported in the `data/serialization` directory (`orderbook` and `last-trades`) for a long term storage | | *query* | **multiTradeAllowedByDefault** | Boolean (`true` or `false`) | If `true`, [multi-trade](README.md#multi-trade) will be allowed by default for `trade`, `buy` and `sell`. It can be overridden at command line level with `--no-multi-trade` and `--multi-trade`. | | *query* | **validateApiKey** | Boolean (`true` or `false`) | If `true`, each loaded private key will be tested at start of the program. In case of a failure, it will be removed from the list of private accounts loaded by `coincenter`, so that later queries do not consider it instead of raising a runtime exception. The downside is that it will make an additional check that will make startup slower. | | | *tradefees* | **maker** | String as decimal number representing a percentage (for instance, "0.15") | Trade fees occurring when a maker order is matched | diff --git a/Dockerfile b/Dockerfile index 73202ad0..32dd1a6c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,12 +23,14 @@ ARG BUILD_MODE=Release ARG BUILD_TEST=0 ARG BUILD_ASAN=0 ARG BUILD_WITH_PROMETHEUS=1 +ARG BUILD_WITH_PROTOBUF=1 # Build and launch tests if any RUN cmake -DCMAKE_BUILD_TYPE=${BUILD_MODE} \ -DCCT_ENABLE_TESTS=${BUILD_TEST} \ -DCCT_ENABLE_ASAN=${BUILD_ASAN} \ -DCCT_BUILD_PROMETHEUS_FROM_SRC=${BUILD_WITH_PROMETHEUS} \ + -DCCT_ENABLE_PROTO=${BUILD_WITH_PROTOBUF} \ -GNinja .. && \ ninja && \ if [ "$BUILD_TEST" = "1" -o "$BUILD_TEST" = "ON" ]; then \ diff --git a/alpine.Dockerfile b/alpine.Dockerfile index f862cc90..86e83c6e 100644 --- a/alpine.Dockerfile +++ b/alpine.Dockerfile @@ -2,7 +2,7 @@ FROM alpine:3.19.0 AS build # Install base & build dependencies, needed certificates for curl to work with https -RUN apk add --update --upgrade --no-cache g++ libc-dev curl-dev cmake ninja git ca-certificates +RUN apk add --update --upgrade --no-cache g++ linux-headers libc-dev curl-dev cmake ninja git ca-certificates # Set default directory for application WORKDIR /app @@ -18,12 +18,14 @@ ARG BUILD_MODE=Release ARG BUILD_TEST=0 ARG BUILD_ASAN=0 ARG BUILD_WITH_PROMETHEUS=1 +ARG BUILD_WITH_PROTOBUF=1 # Build and launch tests if any RUN cmake -DCMAKE_BUILD_TYPE=${BUILD_MODE} \ -DCCT_ENABLE_TESTS=${BUILD_TEST} \ -DCCT_ENABLE_ASAN=${BUILD_ASAN} \ -DCCT_BUILD_PROMETHEUS_FROM_SRC=${BUILD_WITH_PROMETHEUS} \ + -DCCT_ENABLE_PROTO=${BUILD_WITH_PROTOBUF} \ -GNinja .. && \ ninja && \ if [ "$BUILD_TEST" = "1" -o "$BUILD_TEST" = "ON" ]; then \ diff --git a/src/api-objects/include/order.hpp b/src/api-objects/include/order.hpp index d7c77af0..6dc689d2 100644 --- a/src/api-objects/include/order.hpp +++ b/src/api-objects/include/order.hpp @@ -44,7 +44,7 @@ class Order { Market market() const { return Market(_matchedVolume.currencyCode(), _price.currencyCode()); } /// default ordering by place time first, then matched volume, etc - auto operator<=>(const Order &) const = default; + std::strong_ordering operator<=>(const Order &) const noexcept = default; using trivially_relocatable = is_trivially_relocatable::type; diff --git a/src/api/interface/CMakeLists.txt b/src/api/interface/CMakeLists.txt index a17d1968..df998a70 100644 --- a/src/api/interface/CMakeLists.txt +++ b/src/api/interface/CMakeLists.txt @@ -3,6 +3,7 @@ aux_source_directory(src API_INTERFACE_SRC) add_library(coincenter_api-interface STATIC ${API_INTERFACE_SRC}) target_link_libraries(coincenter_api-interface PUBLIC coincenter_api-exchange) +target_link_libraries(coincenter_api-interface PUBLIC coincenter_serialization) target_link_libraries(coincenter_api-interface PRIVATE coincenter_monitoring) target_include_directories(coincenter_api-interface PUBLIC include) diff --git a/src/api/interface/include/exchange.hpp b/src/api/interface/include/exchange.hpp index 9d561f78..6a56ff8b 100644 --- a/src/api/interface/include/exchange.hpp +++ b/src/api/interface/include/exchange.hpp @@ -1,7 +1,9 @@ #pragma once +#include #include #include +#include #include "cct_exception.hpp" #include "currencycode.hpp" @@ -17,17 +19,20 @@ #include "monetaryamountbycurrencyset.hpp" namespace cct { +class AbstractMarketDataSerializer; class Exchange { public: using ExchangePublic = api::ExchangePublic; /// Builds a Exchange without private exchange. All private requests will be forbidden. - Exchange(const ExchangeConfig &exchangeConfig, api::ExchangePublic &exchangePublic); + Exchange(std::string_view dataDir, const ExchangeConfig &exchangeConfig, api::ExchangePublic &exchangePublic); /// Build a Exchange with both private and public exchanges - Exchange(const ExchangeConfig &exchangeConfig, api::ExchangePublic &exchangePublic, + Exchange(std::string_view dataDir, const ExchangeConfig &exchangeConfig, api::ExchangePublic &exchangePublic, api::ExchangePrivate &exchangePrivate); + ~Exchange(); + std::string_view name() const { return _exchangePublic.name(); } std::string_view keyName() const { return apiPrivate().keyName(); } @@ -86,16 +91,12 @@ class Exchange { return _exchangePublic.queryAllApproximatedOrderBooks(depth); } - MarketOrderBook queryOrderBook(Market mk, int depth = ExchangePublic::kDefaultDepth) { - return _exchangePublic.queryOrderBook(mk, depth); - } + MarketOrderBook queryOrderBook(Market mk, int depth = ExchangePublic::kDefaultDepth); MonetaryAmount queryLast24hVolume(Market mk) { return _exchangePublic.queryLast24hVolume(mk); } /// Retrieve an ordered vector of recent last trades - LastTradesVector queryLastTrades(Market mk, int nbTrades = ExchangePublic::kNbLastTradesDefault) { - return _exchangePublic.queryLastTrades(mk, nbTrades); - } + LastTradesVector queryLastTrades(Market mk, int nbTrades = ExchangePublic::kNbLastTradesDefault); /// Retrieve the last price of given market. MonetaryAmount queryLastPrice(Market mk) { return _exchangePublic.queryLastPrice(mk); } @@ -110,9 +111,15 @@ class Exchange { void updateCacheFile() const; + using trivially_relocatable = std::true_type; + private: + Exchange(std::string_view dataDir, const ExchangeConfig &exchangeConfig, api::ExchangePublic &exchangePublic, + api::ExchangePrivate *pExchangePrivate); + api::ExchangePublic &_exchangePublic; api::ExchangePrivate *_pExchangePrivate = nullptr; const ExchangeConfig &_exchangeConfig; + std::unique_ptr _marketDataSerializerPtr; }; } // namespace cct diff --git a/src/api/interface/src/exchange.cpp b/src/api/interface/src/exchange.cpp index b6ed7dd8..9631dd14 100644 --- a/src/api/interface/src/exchange.cpp +++ b/src/api/interface/src/exchange.cpp @@ -1,6 +1,7 @@ #include "exchange.hpp" #include +#include #include "cct_log.hpp" #include "currencycode.hpp" @@ -8,17 +9,39 @@ #include "exchangeconfig.hpp" #include "exchangeprivateapi.hpp" #include "exchangepublicapi.hpp" +#include "timedef.hpp" + +#ifdef CCT_ENABLE_PROTO +#include "proto-market-data-serializer.hpp" +#else +#include "dummy-market-data-serializer.hpp" +#endif namespace cct { -Exchange::Exchange(const ExchangeConfig &exchangeConfig, api::ExchangePublic &exchangePublic, +#ifdef CCT_ENABLE_PROTO +using MarketDataSerializer = ProtobufMarketDataSerializer; +#else +using MarketDataSerializer = DummyMarketDataSerializer; +#endif + +Exchange::Exchange(std::string_view dataDir, const ExchangeConfig &exchangeConfig, api::ExchangePublic &exchangePublic, api::ExchangePrivate &exchangePrivate) + : Exchange(dataDir, exchangeConfig, exchangePublic, std::addressof(exchangePrivate)) {} + +Exchange::Exchange(std::string_view dataDir, const ExchangeConfig &exchangeConfig, api::ExchangePublic &exchangePublic) + : Exchange(dataDir, exchangeConfig, exchangePublic, nullptr) {} + +Exchange::Exchange(std::string_view dataDir, const ExchangeConfig &exchangeConfig, api::ExchangePublic &exchangePublic, + api::ExchangePrivate *pExchangePrivate) : _exchangePublic(exchangePublic), - _pExchangePrivate(std::addressof(exchangePrivate)), - _exchangeConfig(exchangeConfig) {} + _pExchangePrivate(pExchangePrivate), + _exchangeConfig(exchangeConfig), + _marketDataSerializerPtr(_exchangeConfig.withMarketDataSerialization() + ? new MarketDataSerializer(dataDir, exchangePublic.name()) + : nullptr) {} -Exchange::Exchange(const ExchangeConfig &exchangeConfig, api::ExchangePublic &exchangePublic) - : _exchangePublic(exchangePublic), _exchangeConfig(exchangeConfig) {} +Exchange::~Exchange() = default; // declared here to have definition of ~MarketDataSerializer bool Exchange::canWithdraw(CurrencyCode currencyCode, const CurrencyExchangeFlatSet ¤cyExchangeSet) const { if (_exchangeConfig.excludedCurrenciesWithdrawal().contains(currencyCode)) { @@ -41,6 +64,23 @@ bool Exchange::canDeposit(CurrencyCode currencyCode, const CurrencyExchangeFlatS return lb->canDeposit(); } +MarketOrderBook Exchange::queryOrderBook(Market mk, int depth) { + auto marketOrderBook = _exchangePublic.queryOrderBook(mk, depth); + if (_marketDataSerializerPtr) { + _marketDataSerializerPtr->push(marketOrderBook); + } + return marketOrderBook; +} + +/// Retrieve an ordered vector of recent last trades +LastTradesVector Exchange::queryLastTrades(Market mk, int nbTrades) { + auto lastTrades = _exchangePublic.queryLastTrades(mk, nbTrades); + if (_marketDataSerializerPtr) { + _marketDataSerializerPtr->push(lastTrades); + } + return lastTrades; +} + void Exchange::updateCacheFile() const { _exchangePublic.updateCacheFile(); if (_pExchangePrivate != nullptr) { diff --git a/src/api/interface/src/exchangepool.cpp b/src/api/interface/src/exchangepool.cpp index 1d04d5ef..6886f76f 100644 --- a/src/api/interface/src/exchangepool.cpp +++ b/src/api/interface/src/exchangepool.cpp @@ -31,6 +31,7 @@ ExchangePool::ExchangePool(const CoincenterInfo& coincenterInfo, FiatConverter& _krakenPublic(_coincenterInfo, _fiatConverter, _commonAPI), _kucoinPublic(_coincenterInfo, _fiatConverter, _commonAPI), _upbitPublic(_coincenterInfo, _fiatConverter, _commonAPI) { + const auto dataDir = coincenterInfo.dataDir(); for (std::string_view exchangeStr : kSupportedExchanges) { api::ExchangePublic* exchangePublic; if (exchangeStr == "binance") { @@ -81,10 +82,10 @@ ExchangePool::ExchangePool(const CoincenterInfo& coincenterInfo, FiatConverter& } } - _exchanges.emplace_back(exchangeConfig, *exchangePublic, *exchangePrivate); + _exchanges.emplace_back(dataDir, exchangeConfig, *exchangePublic, *exchangePrivate); } } else { - _exchanges.emplace_back(exchangeConfig, *exchangePublic); + _exchanges.emplace_back(dataDir, exchangeConfig, *exchangePublic); } } _exchanges.shrink_to_fit(); diff --git a/src/engine/include/coincenteroptions.hpp b/src/engine/include/coincenteroptions.hpp index 7a11cf91..a17c2ee9 100644 --- a/src/engine/include/coincenteroptions.hpp +++ b/src/engine/include/coincenteroptions.hpp @@ -95,6 +95,8 @@ class CoincenterCmdLineOptions { std::string_view lastTrades; + std::optional replay; + CommandLineOptionalInt repeats; int nbLastTrades = api::ExchangePublic::kNbLastTradesDefault; int monitoringPort = CoincenterCmdLineOptionsDefinitions::kDefaultMonitoringPort; diff --git a/src/engine/include/coincenteroptionsdef.hpp b/src/engine/include/coincenteroptionsdef.hpp index 7aeaafcc..e0eec2d5 100644 --- a/src/engine/include/coincenteroptionsdef.hpp +++ b/src/engine/include/coincenteroptionsdef.hpp @@ -309,14 +309,14 @@ struct CoincenterAllowedOptions : private CoincenterCmdLineOptionsDefinitions { &OptValueType::minAge}, {{{"Private queries", 3902}, "--max-age", "