Skip to content

Commit

Permalink
[Feature] Serialization & Deserialization of market data with protobuf
Browse files Browse the repository at this point in the history
  • Loading branch information
@sjanel
sjanel committed Apr 3, 2024
1 parent 1234a5f commit 8395c5c
Show file tree
Hide file tree
Showing 134 changed files with 5,866 additions and 140 deletions.
25 changes: 14 additions & 11 deletions .github/workflows/ubuntu-monitoring.yml → .github/workflows/ubuntu-special.yml
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Monitoring
name: Special

on:
push:
Expand All @@ -7,14 +7,15 @@ on:
pull_request:

jobs:
ubuntu-monitoring-build:
name: Build on Ubuntu with monitoring support
ubuntu-special-build:
name: Build on Ubuntu with monitoring / protobuf support
runs-on: ubuntu-latest
strategy:
matrix:
compiler: [g++-11]
buildmode: [Debug]
build-prometheus-from-source: [0, 1]
build-special-from-source: [0, 1]
prometheus-options: ["-DBUILD_SHARED_LIBS=ON -DENABLE_PULL=OFF -DENABLE_PUSH=ON -DENABLE_COMPRESSION=OFF -DENABLE_TESTING=OFF"]

steps:
- name: Checkout repository code
Expand All @@ -35,19 +36,21 @@ jobs:
mkdir _build
cd _build
cmake .. -DBUILD_SHARED_LIBS=ON -DENABLE_PULL=OFF -DENABLE_PUSH=ON -DENABLE_COMPRESSION=OFF -DENABLE_TESTING=OFF -DCMAKE_BUILD_TYPE=${{matrix.buildmode}} -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -GNinja
cmake .. ${{matrix.prometheus-options}} -GNinja
cmake --build .
sudo cmake --install .
if: matrix.build-prometheus-from-source == 0

- name: Create Build Environment
run: cmake -E make_directory ${{github.workspace}}/build
if: matrix.build-special-from-source == 0
env:
CXX: ${{matrix.compiler}}
CMAKE_BUILD_TYPE: ${{matrix.buildmode}}

- name: Configure CMake
working-directory: ${{github.workspace}}/build
shell: bash
run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{matrix.buildmode}} -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCCT_BUILD_PROMETHEUS_FROM_SRC=${{matrix.build-prometheus-from-source}} -DCCT_ENABLE_ASAN=OFF -GNinja
run: cmake -S . -B build ${{matrix.prometheus-options}} -DCCT_BUILD_PROMETHEUS_FROM_SRC=${{matrix.build-special-from-source}} -DCCT_ENABLE_PROTO=${{matrix.build-special-from-source}} -DCCT_ENABLE_ASAN=OFF -GNinja
env:
CXX: ${{matrix.compiler}}
CMAKE_BUILD_TYPE: ${{matrix.buildmode}}

- name: Build
working-directory: ${{github.workspace}}/build
Expand Down
7 changes: 5 additions & 2 deletions .github/workflows/windows.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,18 @@ jobs:

- name: Install dependencies
run: |
vcpkg install curl
vcpkg install curl protobuf
- name: End vcpkg install
run: |
vcpkg integrate install
- name: Configure CMake
run: |
cmake -DCMAKE_TOOLCHAIN_FILE="C:/vcpkg/scripts/buildsystems/vcpkg.cmake" -DCMAKE_BUILD_TYPE=${{matrix.buildmode}} -S . -B build
cmake -S . -B build
env:
CMAKE_BUILD_TYPE: ${{matrix.buildmode}}
CMAKE_TOOLCHAIN_FILE: "C:/vcpkg/scripts/buildsystems/vcpkg.cmake"

- name: Build
working-directory: ${{github.workspace}}/build
Expand Down
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ data/cache
data/log
data/secret
!data/secret/secret_test.json
data/serialized
data/static/exchangeconfig.json
data/static/generalconfig.json
monitoring/data/grafana/*
33 changes: 33 additions & 0 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ option(CCT_ENABLE_TESTS "Build the unit tests" ${MAIN_PROJECT})
option(CCT_BUILD_EXEC "Build an executable instead of a static library" ${MAIN_PROJECT})
option(CCT_ENABLE_ASAN "Compile with AddressSanitizer" ${CCT_ASAN_BUILD})
option(CCT_ENABLE_CLANG_TIDY "Compile with clang-tidy checks" OFF)
option(CCT_ENABLE_PROTO "Compile with protobuf support (to export data to the outside world)" ON)
option(CCT_BUILD_PROMETHEUS_FROM_SRC "Fetch and build from prometheus-cpp sources" OFF)

set(CCT_DATA_DIR "${CMAKE_CURRENT_SOURCE_DIR}/data" CACHE PATH "Needed data directory for coincenter. Can also be overriden at runtime with this environment variable")
Expand Down Expand Up @@ -141,6 +142,31 @@ if(NOT spdlog_FOUND)
FetchContent_MakeAvailable(spdlog)
endif()

# protobuf - serialization / deserialization library
if(CCT_ENABLE_PROTO)
find_package(Protobuf CONFIG)
if(Protobuf_FOUND)
message(STATUS "Linking with protobuf ${protobuf_VERSION}")
else()
set(PROTOBUF_VERSION v26.1)

message(STATUS "Compiling protobuf ${PROTOBUF_VERSION} from sources")

set(protobuf_BUILD_TESTS OFF)
set(ABSL_PROPAGATE_CXX_STD ON)

FetchContent_Declare(
protobuf
GIT_REPOSITORY https://github.com/protocolbuffers/protobuf.git
GIT_TAG ${PROTOBUF_VERSION}
)
FetchContent_MakeAvailable(protobuf)

include(${protobuf_SOURCE_DIR}/cmake/protobuf-generate.cmake)

endif()
endif()

# Unit Tests
include(cmake/AddUnitTest.cmake)

Expand Down Expand Up @@ -179,13 +205,20 @@ if(CCT_ENABLE_PROMETHEUS)
add_compile_definitions(CCT_ENABLE_PROMETHEUS)
endif()

if(CCT_ENABLE_PROTO)
add_compile_definitions(CCT_ENABLE_PROTO)
add_compile_definitions("CCT_PROTOBUF_VERSION=\"${PROTOBUF_VERSION}\"")
endif()

# Link to sub folders CMakeLists.txt, from the lowest level to the highest level for documentation
# (beware of cyclic dependencies)
add_subdirectory(src/tech)
add_subdirectory(src/monitoring)
add_subdirectory(src/http-request)
add_subdirectory(src/objects)
add_subdirectory(src/serialization)
add_subdirectory(src/api-objects)
add_subdirectory(src/trading)
add_subdirectory(src/api)
add_subdirectory(src/engine)
add_subdirectory(src/main)
24 changes: 13 additions & 11 deletions CONFIG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,17 +48,18 @@ Configures the logging, tracking activity of relevant commands, and console outp

#### General options description

| Name | Value | Description |
| ---------------------------------------------- | ---------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| **apiOutputType** | String among {`off`, `table`, `json`} | Configure the default output type of coincenter (can be overridden by command line)queries |
| **fiatConversion.rate** | Duration string (ex: `8h`) | Minimum duration between two consecutive requests of the same fiat conversion |
| **log.activityTracking.commandTypes** | Array of strings (ex: `["Buy", "Sell"]`) | Array of command types whose output will be stored to activity history files. |
| **log.activityTracking.dateFileNameFormat** | String (ex: `%Y-%m` for month split) | Defines the date string format suffix used by activity history files. The string should be compatible with [std::strftime](https://en.cppreference.com/w/cpp/chrono/c/strftime). Old data will never be clean-up by `coincenter` (as it may contain important data). User should manage the clean-up / storage. |
| **log.consoleLevel** | String | Defines the log level for standard output. Can be {'off', 'critical', 'error', 'warning', 'info', 'debug', 'trace'} |
| **log.fileLevel** | String | Defines the log level in files. Can be {'off', 'critical', 'error', 'warning', 'info', 'debug', 'trace'} |
| **log.maxFileSize** | String (ex: `5Mi` for 5 Megabytes) | Defines in bytes the maximum logging file size. A string representation of an integral, possibly with one suffix ending such as k, M, G, T (1k multipliers) or Ki, Mi, Gi, Ti (1024 multipliers) are supported. |
| **log.maxNbFiles** | Integer | Number of maximum rotating files for log in files |
| **requests.concurrency.nbMaxParallelRequests** | Integer | Size of the thread pool that makes exchange requests. |
| Name | Value | Description |
| -------------------------------------------------------- | ---------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| **apiOutputType** | String among {`off`, `table`, `json`} | Configure the default output type of coincenter (can be overridden by command line)queries |
| **fiatConversion.rate** | Duration string (ex: `8h`) | Minimum duration between two consecutive requests of the same fiat conversion |
| **log.activityTracking.commandTypes** | Array of strings (ex: `["Buy", "Sell"]`) | Array of command types whose output will be stored to activity history files. |
| **log.activityTracking.dateFileNameFormat** | String (ex: `%Y-%m` for month split) | Defines the date string format suffix used by activity history files. The string should be compatible with [std::strftime](https://en.cppreference.com/w/cpp/chrono/c/strftime). Old data will never be clean-up by `coincenter` (as it may contain important data). User should manage the clean-up / storage. |
| **log.consoleLevel** | String | Defines the log level for standard output. Can be {'off', 'critical', 'error', 'warning', 'info', 'debug', 'trace'} |
| **log.fileLevel** | String | Defines the log level in files. Can be {'off', 'critical', 'error', 'warning', 'info', 'debug', 'trace'} |
| **log.maxFileSize** | String (ex: `5Mi` for 5 Megabytes) | Defines in bytes the maximum logging file size. A string representation of an integral, possibly with one suffix ending such as k, M, G, T (1k multipliers) or Ki, Mi, Gi, Ti (1024 multipliers) are supported. |
| **log.maxNbFiles** | Integer | Number of maximum rotating files for log in files |
| **requests.concurrency.nbMaxParallelRequests** | Integer | Size of the thread pool that makes exchange requests. |
| **trading.automation.deserialization.loadChunkDuration** | Duration string (ex: `1w`) | Time window duration of historic stored data loaded and replayed at once given to the trading engine |

### static/exchangeconfig.json

Expand Down Expand Up @@ -151,6 +152,7 @@ Refer to the hardcoded default json example as a model in case of doubt.
| *query* | **updateFrequency.depositWallet** | Duration string (ex: `1min`) | Minimum duration between two consecutive requests of deposit information (including wallet) |
| *query* | **updateFrequency.currencyInfo** | Duration string (ex: `4h`) | Minimum duration between two consecutive requests of dynamic currency info retrieval on Bithumb only (used for place order) |
| *query* | **placeSimulateRealOrder** | Boolean (`true` or `false`) | If `true`, in trade simulation mode (with `--sim`) exchanges which do not support simulated mode in place order will actually place a real order, with the following characteristics: <ul><li>trade strategy forced to `maker`</li><li>price will be changed to a maximum for a sell, to a minimum for a buy</li></ul> This will allow place of a 'real' order that cannot be matched in practice (if it is, lucky you!) |
| *query* | **marketDataSerialization** | Boolean (`true` or `false`) | If `true` and `coincenter` is compiled with **protobuf** support, some market data will automatically be exported in the `data/serialization` directory (`orderbook` and `last-trades`) for a long term storage |
| *query* | **multiTradeAllowedByDefault** | Boolean (`true` or `false`) | If `true`, [multi-trade](README.md#multi-trade) will be allowed by default for `trade`, `buy` and `sell`. It can be overridden at command line level with `--no-multi-trade` and `--multi-trade`. |
| *query* | **validateApiKey** | Boolean (`true` or `false`) | If `true`, each loaded private key will be tested at start of the program. In case of a failure, it will be removed from the list of private accounts loaded by `coincenter`, so that later queries do not consider it instead of raising a runtime exception. The downside is that it will make an additional check that will make startup slower. | |
| *tradefees* | **maker** | String as decimal number representing a percentage (for instance, "0.15") | Trade fees occurring when a maker order is matched |
Expand Down
2 changes: 2 additions & 0 deletions Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ ARG BUILD_MODE=Release
ARG BUILD_TEST=0
ARG BUILD_ASAN=0
ARG BUILD_WITH_PROMETHEUS=1
ARG BUILD_WITH_PROTOBUF=1

# Install base & build dependencies, needed certificates for curl to work with https
RUN apt update && \
Expand Down Expand Up @@ -41,6 +42,7 @@ RUN cmake -DCMAKE_BUILD_TYPE=${BUILD_MODE} \
-DCCT_ENABLE_TESTS=${BUILD_TEST} \
-DCCT_ENABLE_ASAN=${BUILD_ASAN} \
-DCCT_BUILD_PROMETHEUS_FROM_SRC=${BUILD_WITH_PROMETHEUS} \
-DCCT_ENABLE_PROTO=${BUILD_WITH_PROTOBUF} \
-GNinja ..

# Build
Expand Down
1 change: 1 addition & 0 deletions INSTALL.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,7 @@ The minimum tested version is cmake `3.15`, but it's recommended that you use th
| `CCT_BUILD_EXEC` | `ON` if main project | Build an executable instead of a static library |
| `CCT_ENABLE_ASAN` | `ON` if Debug mode | Compile with AddressSanitizer |
| `CCT_ENABLE_CLANG_TIDY` | `ON` if Debug mode and `clang-tidy` is found in `PATH` | Compile with clang-tidy checks |
| `CCT_ENABLE_PROTO` | `ON` | Compile with protobuf support |

Example on Linux: to compile it in `Release` mode and `ninja` generator

Expand Down
Loading

0 comments on commit 8395c5c

Please sign in to comment.