diff --git a/.dockerignore b/.dockerignore
index d07c51f7..d8d4c498 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -9,6 +9,7 @@ LICENSE
**/*secret.json
data/log
data/secret
+data/serialized
!data/secret/secret_test.json
monitoring
resources
diff --git a/.github/workflows/ubuntu-monitoring.yml b/.github/workflows/ubuntu-special.yml
similarity index 85%
rename from .github/workflows/ubuntu-monitoring.yml
rename to .github/workflows/ubuntu-special.yml
index a0dc9091..d7554cff 100644
--- a/.github/workflows/ubuntu-monitoring.yml
+++ b/.github/workflows/ubuntu-special.yml
@@ -1,4 +1,4 @@
-name: Monitoring
+name: Special
on:
push:
@@ -7,14 +7,14 @@ on:
pull_request:
jobs:
- ubuntu-monitoring-build:
- name: Build on Ubuntu with monitoring support
+ ubuntu-special-build:
+ name: Build on Ubuntu with monitoring / protobuf support
runs-on: ubuntu-latest
strategy:
matrix:
compiler: [g++-11]
buildmode: [Debug]
- build-prometheus-from-source: [0, 1]
+ build-special-from-source: [0, 1]
steps:
- name: Checkout repository code
@@ -38,7 +38,7 @@ jobs:
ninja
sudo cmake --install .
- if: matrix.build-prometheus-from-source == 0
+ if: matrix.build-special-from-source == 0
- name: Create Build Environment
run: cmake -E make_directory ${{github.workspace}}/build
@@ -46,7 +46,7 @@ jobs:
- name: Configure CMake
working-directory: ${{github.workspace}}/build
shell: bash
- run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{matrix.buildmode}} -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCCT_BUILD_PROMETHEUS_FROM_SRC=${{matrix.build-prometheus-from-source}} -GNinja
+ run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{matrix.buildmode}} -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCCT_BUILD_PROMETHEUS_FROM_SRC=${{matrix.build-special-from-source}} -DCCT_ENABLE_PROTO=${{matrix.build-special-from-source}} -GNinja
- name: Build
working-directory: ${{github.workspace}}/build
diff --git a/.gitignore b/.gitignore
index 8c01a78d..0e8592b9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -66,6 +66,7 @@ cscope*
data/cache
data/log
data/secret
+data/serialized
data/static/exchangeconfig.json
data/static/generalconfig.json
monitoring/data/grafana/*
diff --git a/CMakeLists.txt b/CMakeLists.txt
index a8d536a0..45e25a80 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -36,6 +36,7 @@ option(CCT_ENABLE_TESTS "Build the unit tests" ${MAIN_PROJECT})
option(CCT_BUILD_EXEC "Build an executable instead of a static library" ${MAIN_PROJECT})
option(CCT_ENABLE_ASAN "Compile with AddressSanitizer" ${CCT_ASAN_BUILD})
option(CCT_ENABLE_CLANG_TIDY "Compile with clang-tidy checks" OFF)
+option(CCT_ENABLE_PROTO "Compile with protobuf support (to export data to the outside world)" ON)
option(CCT_BUILD_PROMETHEUS_FROM_SRC "Fetch and build from prometheus-cpp sources" OFF)
set(CCT_DATA_DIR "${CMAKE_CURRENT_SOURCE_DIR}/data" CACHE PATH "Needed data directory for coincenter. Can also be overriden at runtime with this environment variable")
@@ -83,6 +84,7 @@ if(CCT_ENABLE_TESTS)
enable_testing()
endif()
+# nlohmann_json - coincenter json library
find_package(nlohmann_json CONFIG)
if(NOT nlohmann_json_FOUND)
FetchContent_Declare(
@@ -94,6 +96,7 @@ if(NOT nlohmann_json_FOUND)
FetchContent_MakeAvailable(nlohmann_json)
endif()
+# spdlog - coincenter logging library
find_package(spdlog CONFIG)
if(NOT spdlog_FOUND)
FetchContent_Declare(
@@ -133,6 +136,34 @@ else()
endif()
endif()
+if(CCT_ENABLE_PROTO)
+ find_package(Protobuf CONFIG)
+ if(protobuf_FOUND)
+ message(STATUS "Linking with protobuf ${protobuf_VERSION}")
+ else()
+ set(PROTOBUF_VERSION v25.2)
+ if (MSVC)
+ # protobuf v25.0 does not compile yet with MSVC: https://github.com/protocolbuffers/protobuf/issues/14602
+ set(PROTOBUF_VERSION v24.4)
+ endif()
+
+ message(STATUS "Compiling protobuf ${PROTOBUF_VERSION} from sources")
+
+ set(protobuf_BUILD_TESTS OFF)
+ set(ABSL_PROPAGATE_CXX_STD ON)
+
+ FetchContent_Declare(
+ protobuf
+ GIT_REPOSITORY https://github.com/protocolbuffers/protobuf.git
+ GIT_TAG ${PROTOBUF_VERSION}
+ )
+ FetchContent_MakeAvailable(protobuf)
+
+ include(${protobuf_SOURCE_DIR}/cmake/protobuf-generate.cmake)
+
+ endif()
+endif()
+
# Unit Tests
#[[ Create an executable
@@ -248,12 +279,18 @@ if(CCT_ENABLE_PROMETHEUS)
add_compile_definitions(CCT_ENABLE_PROMETHEUS)
endif()
+if(CCT_ENABLE_PROTO)
+ add_compile_definitions(CCT_ENABLE_PROTO)
+ add_compile_definitions("CCT_PROTOBUF_VERSION=\"${PROTOBUF_VERSION}\"")
+endif()
+
# Link to sub folders CMakeLists.txt, from the lowest level to the highest level for documentation
# (beware of cyclic dependencies)
add_subdirectory(src/tech)
add_subdirectory(src/monitoring)
add_subdirectory(src/http-request)
add_subdirectory(src/objects)
+add_subdirectory(src/serialization)
add_subdirectory(src/api-objects)
add_subdirectory(src/api)
add_subdirectory(src/engine)
diff --git a/CONFIG.md b/CONFIG.md
index efba43e0..b2c031c0 100644
--- a/CONFIG.md
+++ b/CONFIG.md
@@ -150,6 +150,7 @@ Refer to the hardcoded default json example as a model in case of doubt.
| *query* | **updateFrequency.depositWallet** | Duration string (ex: `1min`) | Minimum duration between two consecutive requests of deposit information (including wallet) |
| *query* | **updateFrequency.currencyInfo** | Duration string (ex: `4h`) | Minimum duration between two consecutive requests of dynamic currency info retrieval on Bithumb only (used for place order) |
| *query* | **placeSimulateRealOrder** | Boolean (`true` or `false`) | If `true`, in trade simulation mode (with `--sim`) exchanges which do not support simulated mode in place order will actually place a real order, with the following characteristics:
- trade strategy forced to `maker`
- price will be changed to a maximum for a sell, to a minimum for a buy
This will allow place of a 'real' order that cannot be matched in practice (if it is, lucky you!) |
+| *query* | **marketDataSerialization** | Boolean (`true` or `false`) | If `true` and `coincenter` is compiled with **protobuf** support, some market data will automatically be exported in the `data/serialization` directory (`orderbook` and `last-trades`) for a long term storage |
| *query* | **multiTradeAllowedByDefault** | Boolean (`true` or `false`) | If `true`, [multi-trade](README.md#multi-trade) will be allowed by default for `trade`, `buy` and `sell`. It can be overridden at command line level with `--no-multi-trade` and `--multi-trade`. |
| *query* | **validateApiKey** | Boolean (`true` or `false`) | If `true`, each loaded private key will be tested at start of the program. In case of a failure, it will be removed from the list of private accounts loaded by `coincenter`, so that later queries do not consider it instead of raising a runtime exception. The downside is that it will make an additional check that will make startup slower. | |
| *tradefees* | **maker** | String as decimal number representing a percentage (for instance, "0.15") | Trade fees occurring when a maker order is matched |
diff --git a/Dockerfile b/Dockerfile
index 73202ad0..32dd1a6c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -23,12 +23,14 @@ ARG BUILD_MODE=Release
ARG BUILD_TEST=0
ARG BUILD_ASAN=0
ARG BUILD_WITH_PROMETHEUS=1
+ARG BUILD_WITH_PROTOBUF=1
# Build and launch tests if any
RUN cmake -DCMAKE_BUILD_TYPE=${BUILD_MODE} \
-DCCT_ENABLE_TESTS=${BUILD_TEST} \
-DCCT_ENABLE_ASAN=${BUILD_ASAN} \
-DCCT_BUILD_PROMETHEUS_FROM_SRC=${BUILD_WITH_PROMETHEUS} \
+ -DCCT_ENABLE_PROTO=${BUILD_WITH_PROTOBUF} \
-GNinja .. && \
ninja && \
if [ "$BUILD_TEST" = "1" -o "$BUILD_TEST" = "ON" ]; then \
diff --git a/alpine.Dockerfile b/alpine.Dockerfile
index f862cc90..86e83c6e 100644
--- a/alpine.Dockerfile
+++ b/alpine.Dockerfile
@@ -2,7 +2,7 @@
FROM alpine:3.19.0 AS build
# Install base & build dependencies, needed certificates for curl to work with https
-RUN apk add --update --upgrade --no-cache g++ libc-dev curl-dev cmake ninja git ca-certificates
+RUN apk add --update --upgrade --no-cache g++ linux-headers libc-dev curl-dev cmake ninja git ca-certificates
# Set default directory for application
WORKDIR /app
@@ -18,12 +18,14 @@ ARG BUILD_MODE=Release
ARG BUILD_TEST=0
ARG BUILD_ASAN=0
ARG BUILD_WITH_PROMETHEUS=1
+ARG BUILD_WITH_PROTOBUF=1
# Build and launch tests if any
RUN cmake -DCMAKE_BUILD_TYPE=${BUILD_MODE} \
-DCCT_ENABLE_TESTS=${BUILD_TEST} \
-DCCT_ENABLE_ASAN=${BUILD_ASAN} \
-DCCT_BUILD_PROMETHEUS_FROM_SRC=${BUILD_WITH_PROMETHEUS} \
+ -DCCT_ENABLE_PROTO=${BUILD_WITH_PROTOBUF} \
-GNinja .. && \
ninja && \
if [ "$BUILD_TEST" = "1" -o "$BUILD_TEST" = "ON" ]; then \
diff --git a/src/api-objects/include/order.hpp b/src/api-objects/include/order.hpp
index d7c77af0..6dc689d2 100644
--- a/src/api-objects/include/order.hpp
+++ b/src/api-objects/include/order.hpp
@@ -44,7 +44,7 @@ class Order {
Market market() const { return Market(_matchedVolume.currencyCode(), _price.currencyCode()); }
/// default ordering by place time first, then matched volume, etc
- auto operator<=>(const Order &) const = default;
+ std::strong_ordering operator<=>(const Order &) const noexcept = default;
using trivially_relocatable = is_trivially_relocatable::type;
diff --git a/src/api/interface/CMakeLists.txt b/src/api/interface/CMakeLists.txt
index a17d1968..df998a70 100644
--- a/src/api/interface/CMakeLists.txt
+++ b/src/api/interface/CMakeLists.txt
@@ -3,6 +3,7 @@ aux_source_directory(src API_INTERFACE_SRC)
add_library(coincenter_api-interface STATIC ${API_INTERFACE_SRC})
target_link_libraries(coincenter_api-interface PUBLIC coincenter_api-exchange)
+target_link_libraries(coincenter_api-interface PUBLIC coincenter_serialization)
target_link_libraries(coincenter_api-interface PRIVATE coincenter_monitoring)
target_include_directories(coincenter_api-interface PUBLIC include)
diff --git a/src/api/interface/include/exchange.hpp b/src/api/interface/include/exchange.hpp
index 9d561f78..6a56ff8b 100644
--- a/src/api/interface/include/exchange.hpp
+++ b/src/api/interface/include/exchange.hpp
@@ -1,7 +1,9 @@
#pragma once
+#include
#include
#include
+#include
#include "cct_exception.hpp"
#include "currencycode.hpp"
@@ -17,17 +19,20 @@
#include "monetaryamountbycurrencyset.hpp"
namespace cct {
+class AbstractMarketDataSerializer;
class Exchange {
public:
using ExchangePublic = api::ExchangePublic;
/// Builds a Exchange without private exchange. All private requests will be forbidden.
- Exchange(const ExchangeConfig &exchangeConfig, api::ExchangePublic &exchangePublic);
+ Exchange(std::string_view dataDir, const ExchangeConfig &exchangeConfig, api::ExchangePublic &exchangePublic);
/// Build a Exchange with both private and public exchanges
- Exchange(const ExchangeConfig &exchangeConfig, api::ExchangePublic &exchangePublic,
+ Exchange(std::string_view dataDir, const ExchangeConfig &exchangeConfig, api::ExchangePublic &exchangePublic,
api::ExchangePrivate &exchangePrivate);
+ ~Exchange();
+
std::string_view name() const { return _exchangePublic.name(); }
std::string_view keyName() const { return apiPrivate().keyName(); }
@@ -86,16 +91,12 @@ class Exchange {
return _exchangePublic.queryAllApproximatedOrderBooks(depth);
}
- MarketOrderBook queryOrderBook(Market mk, int depth = ExchangePublic::kDefaultDepth) {
- return _exchangePublic.queryOrderBook(mk, depth);
- }
+ MarketOrderBook queryOrderBook(Market mk, int depth = ExchangePublic::kDefaultDepth);
MonetaryAmount queryLast24hVolume(Market mk) { return _exchangePublic.queryLast24hVolume(mk); }
/// Retrieve an ordered vector of recent last trades
- LastTradesVector queryLastTrades(Market mk, int nbTrades = ExchangePublic::kNbLastTradesDefault) {
- return _exchangePublic.queryLastTrades(mk, nbTrades);
- }
+ LastTradesVector queryLastTrades(Market mk, int nbTrades = ExchangePublic::kNbLastTradesDefault);
/// Retrieve the last price of given market.
MonetaryAmount queryLastPrice(Market mk) { return _exchangePublic.queryLastPrice(mk); }
@@ -110,9 +111,15 @@ class Exchange {
void updateCacheFile() const;
+ using trivially_relocatable = std::true_type;
+
private:
+ Exchange(std::string_view dataDir, const ExchangeConfig &exchangeConfig, api::ExchangePublic &exchangePublic,
+ api::ExchangePrivate *pExchangePrivate);
+
api::ExchangePublic &_exchangePublic;
api::ExchangePrivate *_pExchangePrivate = nullptr;
const ExchangeConfig &_exchangeConfig;
+ std::unique_ptr _marketDataSerializerPtr;
};
} // namespace cct
diff --git a/src/api/interface/src/exchange.cpp b/src/api/interface/src/exchange.cpp
index b6ed7dd8..9631dd14 100644
--- a/src/api/interface/src/exchange.cpp
+++ b/src/api/interface/src/exchange.cpp
@@ -1,6 +1,7 @@
#include "exchange.hpp"
#include
+#include
#include "cct_log.hpp"
#include "currencycode.hpp"
@@ -8,17 +9,39 @@
#include "exchangeconfig.hpp"
#include "exchangeprivateapi.hpp"
#include "exchangepublicapi.hpp"
+#include "timedef.hpp"
+
+#ifdef CCT_ENABLE_PROTO
+#include "proto-market-data-serializer.hpp"
+#else
+#include "dummy-market-data-serializer.hpp"
+#endif
namespace cct {
-Exchange::Exchange(const ExchangeConfig &exchangeConfig, api::ExchangePublic &exchangePublic,
+#ifdef CCT_ENABLE_PROTO
+using MarketDataSerializer = ProtobufMarketDataSerializer;
+#else
+using MarketDataSerializer = DummyMarketDataSerializer;
+#endif
+
+Exchange::Exchange(std::string_view dataDir, const ExchangeConfig &exchangeConfig, api::ExchangePublic &exchangePublic,
api::ExchangePrivate &exchangePrivate)
+ : Exchange(dataDir, exchangeConfig, exchangePublic, std::addressof(exchangePrivate)) {}
+
+Exchange::Exchange(std::string_view dataDir, const ExchangeConfig &exchangeConfig, api::ExchangePublic &exchangePublic)
+ : Exchange(dataDir, exchangeConfig, exchangePublic, nullptr) {}
+
+Exchange::Exchange(std::string_view dataDir, const ExchangeConfig &exchangeConfig, api::ExchangePublic &exchangePublic,
+ api::ExchangePrivate *pExchangePrivate)
: _exchangePublic(exchangePublic),
- _pExchangePrivate(std::addressof(exchangePrivate)),
- _exchangeConfig(exchangeConfig) {}
+ _pExchangePrivate(pExchangePrivate),
+ _exchangeConfig(exchangeConfig),
+ _marketDataSerializerPtr(_exchangeConfig.withMarketDataSerialization()
+ ? new MarketDataSerializer(dataDir, exchangePublic.name())
+ : nullptr) {}
-Exchange::Exchange(const ExchangeConfig &exchangeConfig, api::ExchangePublic &exchangePublic)
- : _exchangePublic(exchangePublic), _exchangeConfig(exchangeConfig) {}
+Exchange::~Exchange() = default; // declared here to have definition of ~MarketDataSerializer
bool Exchange::canWithdraw(CurrencyCode currencyCode, const CurrencyExchangeFlatSet ¤cyExchangeSet) const {
if (_exchangeConfig.excludedCurrenciesWithdrawal().contains(currencyCode)) {
@@ -41,6 +64,23 @@ bool Exchange::canDeposit(CurrencyCode currencyCode, const CurrencyExchangeFlatS
return lb->canDeposit();
}
+MarketOrderBook Exchange::queryOrderBook(Market mk, int depth) {
+ auto marketOrderBook = _exchangePublic.queryOrderBook(mk, depth);
+ if (_marketDataSerializerPtr) {
+ _marketDataSerializerPtr->push(marketOrderBook);
+ }
+ return marketOrderBook;
+}
+
+/// Retrieve an ordered vector of recent last trades
+LastTradesVector Exchange::queryLastTrades(Market mk, int nbTrades) {
+ auto lastTrades = _exchangePublic.queryLastTrades(mk, nbTrades);
+ if (_marketDataSerializerPtr) {
+ _marketDataSerializerPtr->push(lastTrades);
+ }
+ return lastTrades;
+}
+
void Exchange::updateCacheFile() const {
_exchangePublic.updateCacheFile();
if (_pExchangePrivate != nullptr) {
diff --git a/src/api/interface/src/exchangepool.cpp b/src/api/interface/src/exchangepool.cpp
index 1d04d5ef..6886f76f 100644
--- a/src/api/interface/src/exchangepool.cpp
+++ b/src/api/interface/src/exchangepool.cpp
@@ -31,6 +31,7 @@ ExchangePool::ExchangePool(const CoincenterInfo& coincenterInfo, FiatConverter&
_krakenPublic(_coincenterInfo, _fiatConverter, _commonAPI),
_kucoinPublic(_coincenterInfo, _fiatConverter, _commonAPI),
_upbitPublic(_coincenterInfo, _fiatConverter, _commonAPI) {
+ const auto dataDir = coincenterInfo.dataDir();
for (std::string_view exchangeStr : kSupportedExchanges) {
api::ExchangePublic* exchangePublic;
if (exchangeStr == "binance") {
@@ -81,10 +82,10 @@ ExchangePool::ExchangePool(const CoincenterInfo& coincenterInfo, FiatConverter&
}
}
- _exchanges.emplace_back(exchangeConfig, *exchangePublic, *exchangePrivate);
+ _exchanges.emplace_back(dataDir, exchangeConfig, *exchangePublic, *exchangePrivate);
}
} else {
- _exchanges.emplace_back(exchangeConfig, *exchangePublic);
+ _exchanges.emplace_back(dataDir, exchangeConfig, *exchangePublic);
}
}
_exchanges.shrink_to_fit();
diff --git a/src/engine/include/coincenteroptions.hpp b/src/engine/include/coincenteroptions.hpp
index 7a11cf91..a17c2ee9 100644
--- a/src/engine/include/coincenteroptions.hpp
+++ b/src/engine/include/coincenteroptions.hpp
@@ -95,6 +95,8 @@ class CoincenterCmdLineOptions {
std::string_view lastTrades;
+ std::optional replay;
+
CommandLineOptionalInt repeats;
int nbLastTrades = api::ExchangePublic::kNbLastTradesDefault;
int monitoringPort = CoincenterCmdLineOptionsDefinitions::kDefaultMonitoringPort;
diff --git a/src/engine/include/coincenteroptionsdef.hpp b/src/engine/include/coincenteroptionsdef.hpp
index 7aeaafcc..e0eec2d5 100644
--- a/src/engine/include/coincenteroptionsdef.hpp
+++ b/src/engine/include/coincenteroptionsdef.hpp
@@ -309,14 +309,14 @@ struct CoincenterAllowedOptions : private CoincenterCmdLineOptionsDefinitions {
&OptValueType::minAge},
{{{"Private queries", 3902}, "--max-age", "