Clean-up and make MarketOrderBook creation more user friendly

.github/workflows/codeql-analysis.yml

@@ -53,6 +53,7 @@ jobs:
       - name: Install dependencies
         run: |
           sudo apt update
+          sudo apt upgrade -y
           sudo apt install build-essential ninja-build libssl-dev libcurl4-gnutls-dev cmake git ca-certificates gzip -y --no-install-recommends
 
       - name: Create Build Environment

.github/workflows/ubuntu-clang-tidy.yml

@@ -22,6 +22,7 @@ jobs:
       - name: Install dependencies
         run: |
           sudo apt update
+          sudo apt upgrade -y
           sudo apt install cmake libssl-dev libcurl4-gnutls-dev ninja-build -y --no-install-recommends
 
       - name: Install clang (with clang-tidy)

.github/workflows/ubuntu-monitoring.yml

@@ -23,6 +23,7 @@ jobs:
       - name: Prerequisites
         run: |
           sudo apt update
+          sudo apt upgrade -y
           sudo apt install cmake libssl-dev git libcurl4-gnutls-dev ninja-build -y --no-install-recommends
 
       - name: Install prometheus-cpp
@@ -47,7 +48,7 @@ jobs:
       - name: Configure CMake
         working-directory: ${{github.workspace}}/build
         shell: bash
-        run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{matrix.buildmode}} -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCCT_BUILD_PROMETHEUS_FROM_SRC=${{matrix.build-prometheus-from-source}} -GNinja
+        run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{matrix.buildmode}} -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCCT_BUILD_PROMETHEUS_FROM_SRC=${{matrix.build-prometheus-from-source}} -DCCT_ENABLE_ASAN=OFF -GNinja
 
       - name: Build
         working-directory: ${{github.workspace}}/build

.github/workflows/ubuntu.yml

@@ -22,11 +22,15 @@ jobs:
       - name: Install dependencies
         run: |
           sudo apt update
+          sudo apt upgrade -y
           sudo apt install cmake libssl-dev libcurl4-gnutls-dev ninja-build -y --no-install-recommends
 
       - name: Install gcc
         run: |
           sudo apt install ${{matrix.compiler}} -y --no-install-recommends
+
+          # Temporary workaround for libasan bug stated here: https://github.com/google/sanitizers/issues/1716
+          sudo sysctl vm.mmap_rnd_bits=28
         if: startsWith(matrix.compiler, 'g++')
 
       - name: Install clang

INSTALL.md

@@ -71,7 +71,8 @@ Provided that your distribution is sufficiently recent, meta package `build-esse
 Otherwise you can still force it:
 
 ```bash
-sudo apt update && sudo apt install build-essential ninja-build libcurl4-gnutls-dev libssl-dev cmake git ca-certificates gzip
+sudo apt update
+sudo apt install build-essential ninja-build libcurl4-gnutls-dev libssl-dev cmake git ca-certificates gzip
 ```
 
 You can refer to the provided [Dockerfile](Dockerfile) for more information.

src/api-objects/src/closed-order.cpp

@@ -5,6 +5,7 @@
 #include "cct_string.hpp"
 #include "monetaryamount.hpp"
 #include "order.hpp"
+#include "orderid.hpp"
 #include "timedef.hpp"
 #include "timestring.hpp"
 #include "tradeside.hpp"
@@ -33,6 +34,6 @@ ClosedOrder ClosedOrder::mergeWith(const ClosedOrder &closedOrder) const {
         ((matchedVolume().toNeutral() * price()) + (closedOrder.matchedVolume().toNeutral() * closedOrder.price())) /
         totalMatchedVolume.toNeutral();
   }
-  return ClosedOrder(id(), totalMatchedVolume, avgPrice, placedTime(), avgMatchedTime, side());
+  return {id(), totalMatchedVolume, avgPrice, placedTime(), avgMatchedTime, side()};
 }
 }  // namespace cct

src/api-objects/test/closed-order_test.cpp

@@ -2,6 +2,7 @@
 
 #include <gtest/gtest.h>
 
+#include <cstdint>
 #include <limits>
 
 #include "monetaryamount.hpp"

src/api/common/src/commonapi.cpp

@@ -96,13 +96,13 @@ vector<CurrencyCode> CommonAPI::FiatsFunc::retrieveFiatsSource1() {
 
   std::string_view data = _curlHandle1.query("", CurlOptions(HttpRequestType::kGet));
   if (data.empty()) {
-    log::error("Error parsing currency codes, no fiats found from first source");
+    log::warn("Error parsing currency codes, no fiats found from first source");
     return fiatsVec;
   }
   static constexpr bool kAllowExceptions = false;
   json dataCSV = json::parse(data, nullptr, kAllowExceptions);
   if (dataCSV.is_discarded()) {
-    log::error("Error parsing json data of currency codes from source 1");
+    log::warn("Error parsing json data of currency codes from source 1");
     return fiatsVec;
   }
   for (const json& fiatData : dataCSV) {

src/api/common/src/withdrawalfees-crawler.cpp

@@ -1,5 +1,6 @@
 #include "withdrawalfees-crawler.hpp"
 
+#include <algorithm>
 #include <cstddef>
 #include <cstdint>
 #include <string_view>

src/api/common/test/exchangeprivateapi_test.cpp

@@ -99,7 +99,7 @@ class ExchangePrivateTest : public ::testing::Test {
 
   VolAndPriNbDecimals volAndPriDec{2, 2};
   int depth = 15;
-  TimePoint time{};
+  TimePoint time;
 
   MonetaryAmount askPrice1{"2300.45 EUR"};
   MonetaryAmount bidPrice1{"2300.4 EUR"};

src/api/common/test/exchangepublicapi_test.cpp

@@ -107,8 +107,8 @@ TEST_F(ExchangePublicTest, FindCurrenciesPath) {
 TEST_F(ExchangePublicTest, RetrieveMarket) {
   EXPECT_CALL(exchangePublic, queryTradableMarkets()).WillOnce(::testing::Return(markets));
 
-  EXPECT_EQ(exchangePublic.retrieveMarket("BTC", "KRW").value(), Market("BTC", "KRW"));
-  EXPECT_EQ(ExchangePublic::RetrieveMarket("KRW", "BTC", markets).value(), Market("BTC", "KRW"));
+  EXPECT_EQ(exchangePublic.retrieveMarket("BTC", "KRW").value_or(Market()), Market("BTC", "KRW"));
+  EXPECT_EQ(ExchangePublic::RetrieveMarket("KRW", "BTC", markets).value_or(Market()), Market("BTC", "KRW"));
   EXPECT_FALSE(ExchangePublic::RetrieveMarket("EUR", "EOS", markets).has_value());
 }
 
@@ -118,7 +118,7 @@ class ExchangePublicConvertTest : public ExchangePublicTest {
 
   VolAndPriNbDecimals volAndPriDec1{2, 6};
   int depth = 10;
-  TimePoint time{};
+  TimePoint time;
 
   MonetaryAmount askPrice1{"0.000017 BTC"};
   MonetaryAmount bidPrice1{"0.000016 BTC"};
@@ -189,8 +189,7 @@ TEST_F(ExchangePublicConvertTest, ConvertDouble) {
       marketOrderBook1.convert(from, priceOptions).value_or(MonetaryAmount{-1}), ExchangeConfig::FeeType::kMaker);
   res = exchangePublic.exchangeConfig().applyFee(
       marketOrderBook2.convert(res, priceOptions).value_or(MonetaryAmount{-1}), ExchangeConfig::FeeType::kMaker);
-
-  EXPECT_EQ(ret.value(), res);
+  EXPECT_EQ(ret.value_or(MonetaryAmount{}), res);
 }
 
 TEST_F(ExchangePublicConvertTest, ConvertWithFiatAtBeginning) {
@@ -208,10 +207,10 @@ TEST_F(ExchangePublicConvertTest, ConvertWithFiatAtBeginning) {
   ASSERT_TRUE(optRes.has_value());
 
   MonetaryAmount res = exchangePublic.exchangeConfig().applyFee(
-      marketOrderBook3.convert(optRes.value(), priceOptions).value_or(MonetaryAmount{-1}),
+      marketOrderBook3.convert(optRes.value_or(MonetaryAmount{}), priceOptions).value_or(MonetaryAmount{-1}),
       ExchangeConfig::FeeType::kMaker);
 
-  EXPECT_EQ(ret.value(), res);
+  EXPECT_EQ(ret.value_or(MonetaryAmount{}), res);
 }
 
 TEST_F(ExchangePublicConvertTest, ConvertWithFiatAtEnd) {

src/api/common/test/fiatconverter_test.cpp

@@ -102,20 +102,20 @@ class FiatConverterTest : public ::testing::Test {
 TEST_F(FiatConverterTest, DirectConversion) {
   constexpr double amount = 10;
 
-  AreDoubleEqual(converter.convert(amount, "KRW", "KRW").value(), amount);
-  AreDoubleEqual(converter.convert(amount, "EUR", "KRW").value(), amount * kKRW);
-  AreDoubleEqual(converter.convert(amount, "EUR", "USD").value(), amount * kUSD);
-  AreDoubleEqual(converter.convert(amount, "EUR", "GBP").value(), amount * kGBP);
+  AreDoubleEqual(converter.convert(amount, "KRW", "KRW").value_or(0), amount);
+  AreDoubleEqual(converter.convert(amount, "EUR", "KRW").value_or(0), amount * kKRW);
+  AreDoubleEqual(converter.convert(amount, "EUR", "USD").value_or(0), amount * kUSD);
+  AreDoubleEqual(converter.convert(amount, "EUR", "GBP").value_or(0), amount * kGBP);
 
   EXPECT_EQ(converter.convert(amount, "EUR", "SUSHI"), 367.8);
 }
 
 TEST_F(FiatConverterTest, DoubleConversion) {
   constexpr double amount = 20'000'000;
 
-  AreDoubleEqual(converter.convert(amount, "KRW", "EUR").value(), amount / kKRW);
-  AreDoubleEqual(converter.convert(amount, "KRW", "USD").value(), (amount / kKRW) * kUSD);
-  AreDoubleEqual(converter.convert(amount, "GBP", "USD").value(), (amount / kGBP) * kUSD);
+  AreDoubleEqual(converter.convert(amount, "KRW", "EUR").value_or(0), amount / kKRW);
+  AreDoubleEqual(converter.convert(amount, "KRW", "USD").value_or(0), (amount / kKRW) * kUSD);
+  AreDoubleEqual(converter.convert(amount, "GBP", "USD").value_or(0), (amount / kGBP) * kUSD);
 
   EXPECT_EQ(converter.convert(amount, "SUSHI", "KRW"), 729679173.46383917);
 }

src/api/exchanges/src/binancepublicapi.cpp

@@ -37,6 +37,7 @@
 #include "marketorderbook.hpp"
 #include "monetaryamount.hpp"
 #include "monetaryamountbycurrencyset.hpp"
+#include "order-book-line.hpp"
 #include "permanentcurloptions.hpp"
 #include "runmodes.hpp"
 #include "timedef.hpp"
@@ -485,28 +486,29 @@ MarketOrderBook BinancePublic::OrderBookFunc::operator()(Market mk, int depth) {
     lb = std::next(kAuthorizedDepths.end(), -1);
     log::error("Invalid depth {}, default to {}", depth, *lb);
   }
-  using OrderBookVec = vector<OrderBookLine>;
-  OrderBookVec orderBookLines;
 
-  CurlPostData postData{{"symbol", mk.assetsPairStrUpper()}, {"limit", *lb}};
-  json asksAndBids = PublicQuery(_commonInfo._curlHandle, "/api/v3/depth", postData);
+  MarketOrderBookLines orderBookLines;
+
+  const CurlPostData postData{{"symbol", mk.assetsPairStrUpper()}, {"limit", *lb}};
+  const json asksAndBids = PublicQuery(_commonInfo._curlHandle, "/api/v3/depth", postData);
+  const auto nowTime = Clock::now();
   const auto asksIt = asksAndBids.find("asks");
   const auto bidsIt = asksAndBids.find("bids");
 
   if (asksIt != asksAndBids.end() && bidsIt != asksAndBids.end()) {
-    orderBookLines.reserve(static_cast<OrderBookVec::size_type>(asksIt->size() + bidsIt->size()));
+    orderBookLines.reserve(asksIt->size() + bidsIt->size());
     for (const auto& asksOrBids : {asksIt, bidsIt}) {
       const auto type = asksOrBids == asksIt ? OrderBookLine::Type::kAsk : OrderBookLine::Type::kBid;
       for (const auto& priceQuantityPair : *asksOrBids) {
         MonetaryAmount amount(priceQuantityPair.back().get<std::string_view>(), mk.base());
         MonetaryAmount price(priceQuantityPair.front().get<std::string_view>(), mk.quote());
 
-        orderBookLines.emplace_back(amount, price, type);
+        orderBookLines.push(amount, price, type);
       }
     }
   }
 
-  return MarketOrderBook(Clock::now(), mk, orderBookLines);
+  return MarketOrderBook(nowTime, mk, orderBookLines);
 }
 
 MonetaryAmount BinancePublic::TradedVolumeFunc::operator()(Market mk) {

src/api/exchanges/src/bithumbprivateapi.cpp

@@ -8,6 +8,7 @@
 #include <iterator>
 #include <string_view>
 #include <thread>
+#include <type_traits>
 #include <utility>
 
 #include "accountowner.hpp"

src/api/exchanges/src/bithumbpublicapi.cpp

@@ -18,7 +18,6 @@
 #include "cct_json.hpp"
 #include "cct_log.hpp"
 #include "cct_string.hpp"
-#include "cct_vector.hpp"
 #include "coincenterinfo.hpp"
 #include "commonapi.hpp"
 #include "curlhandle.hpp"
@@ -35,6 +34,7 @@
 #include "market.hpp"
 #include "marketorderbook.hpp"
 #include "monetaryamount.hpp"
+#include "order-book-line.hpp"
 #include "permanentcurloptions.hpp"
 #include "stringhelpers.hpp"
 #include "timedef.hpp"
@@ -173,7 +173,7 @@ CurrencyExchangeFlatSet BithumbPublic::TradableCurrenciesFunc::operator()() {
 }
 
 namespace {
-MarketOrderBookMap GetOrderbooks(CurlHandle& curlHandle, const CoincenterInfo& config,
+MarketOrderBookMap GetOrderBooks(CurlHandle& curlHandle, const CoincenterInfo& config,
                                  const ExchangeConfig& exchangeConfig, std::optional<Market> optM = std::nullopt,
                                  std::optional<int> optDepth = std::nullopt) {
   MarketOrderBookMap ret;
@@ -195,11 +195,11 @@ MarketOrderBookMap GetOrderbooks(CurlHandle& curlHandle, const CoincenterInfo& c
   if (!result.empty()) {
     //  Note: as of 2021-02-24, Bithumb payment currency is always KRW. Format of json may change once it's not the case
     //  anymore
+    const auto nowTime = Clock::now();
     std::string_view quoteCurrency = result["payment_currency"].get<std::string_view>();
     if (quoteCurrency != "KRW") {
       log::error("Unexpected Bithumb reply for orderbook. May require code api update");
     }
-    const auto time = Clock::now();
     CurrencyCode quoteCurrencyCode(config.standardizeCurrencyCode(quoteCurrency));
     const CurrencyCodeSet& excludedCurrencies = exchangeConfig.excludedCurrenciesAll();
     for (const auto& [baseOrSpecial, asksAndBids] : result.items()) {
@@ -231,37 +231,36 @@ MarketOrderBookMap GetOrderbooks(CurlHandle& curlHandle, const CoincenterInfo& c
           "asks": [{"quantity" : "2.67575", "price" : "506000"},
                    {"quantity" : "3.54343","price" : "507000"}]
         */
-        using OrderBookVec = vector<OrderBookLine>;
-        OrderBookVec orderBookLines;
-        orderBookLines.reserve(static_cast<OrderBookVec::size_type>(asksBids[0]->size() + asksBids[1]->size()));
+        MarketOrderBookLines orderBookLines;
+        orderBookLines.reserve(asksBids[0]->size() + asksBids[1]->size());
         for (const json* asksOrBids : asksBids) {
           const auto type = asksOrBids == asksBids[0] ? OrderBookLine::Type::kAsk : OrderBookLine::Type::kBid;
           for (const json& priceQuantityPair : *asksOrBids) {
             MonetaryAmount amount(priceQuantityPair["quantity"].get<std::string_view>(), baseCurrencyCode);
             MonetaryAmount price(priceQuantityPair["price"].get<std::string_view>(), quoteCurrencyCode);
 
-            orderBookLines.emplace_back(amount, price, type);
+            orderBookLines.push(amount, price, type);
           }
         }
         Market market(baseCurrencyCode, quoteCurrencyCode);
-        ret.insert_or_assign(market, MarketOrderBook(time, market, orderBookLines));
+        ret.insert_or_assign(market, MarketOrderBook(nowTime, market, orderBookLines));
         if (singleMarketQuote) {
           break;
         }
       }
     }
   }
-  log::info("Retrieved {} markets (+ orderbooks) from Bithumb", ret.size());
+  log::info("Retrieved {} markets (+ order books) from Bithumb", ret.size());
   return ret;
 }
 }  // namespace
 
 MarketOrderBookMap BithumbPublic::AllOrderBooksFunc::operator()() {
-  return GetOrderbooks(_curlHandle, _coincenterInfo, _exchangeConfig);
+  return GetOrderBooks(_curlHandle, _coincenterInfo, _exchangeConfig);
 }
 
 MarketOrderBook BithumbPublic::OrderBookFunc::operator()(Market mk, int depth) {
-  MarketOrderBookMap marketOrderBookMap = GetOrderbooks(_curlHandle, _coincenterInfo, _exchangeConfig, mk, depth);
+  MarketOrderBookMap marketOrderBookMap = GetOrderBooks(_curlHandle, _coincenterInfo, _exchangeConfig, mk, depth);
   auto it = marketOrderBookMap.find(mk);
   if (it == marketOrderBookMap.end()) {
     throw exception("Cannot find {} in market order book map", mk);

src/api/exchanges/src/huobipublicapi.cpp

@@ -33,6 +33,7 @@
 #include "marketorderbook.hpp"
 #include "monetaryamount.hpp"
 #include "monetaryamountbycurrencyset.hpp"
+#include "order-book-line.hpp"
 #include "permanentcurloptions.hpp"
 #include "timedef.hpp"
 #include "toupperlower-string.hpp"
@@ -378,10 +379,10 @@ MarketOrderBook HuobiPublic::OrderBookFunc::operator()(Market mk, int depth) {
       postData.append("depth", *lb);
     }
   }
-  using OrderBookVec = vector<OrderBookLine>;
-  OrderBookVec orderBookLines;
+  MarketOrderBookLines orderBookLines;
 
   const json asksAndBids = PublicQuery(_curlHandle, "/market/depth", postData);
+  const auto nowTime = Clock::now();
   const auto asksIt = asksAndBids.find("asks");
   const auto bidsIt = asksAndBids.find("bids");
   if (asksIt != asksAndBids.end() && bidsIt != asksAndBids.end()) {
@@ -393,7 +394,7 @@ MarketOrderBook HuobiPublic::OrderBookFunc::operator()(Market mk, int depth) {
         MonetaryAmount amount(priceQuantityPair.back().get<double>(), mk.base());
         MonetaryAmount price(priceQuantityPair.front().get<double>(), mk.quote());
 
-        orderBookLines.emplace_back(amount, price, type);
+        orderBookLines.push(amount, price, type);
         if (++currentDepth == depth) {
           if (depth < static_cast<int>(asksOrBids->size())) {
             log::debug("Truncate number of {} prices in order book to {}",
@@ -404,7 +405,7 @@ MarketOrderBook HuobiPublic::OrderBookFunc::operator()(Market mk, int depth) {
       }
     }
   }
-  return MarketOrderBook(Clock::now(), mk, orderBookLines);
+  return MarketOrderBook(nowTime, mk, orderBookLines);
 }
 
 MonetaryAmount HuobiPublic::sanitizePrice(Market mk, MonetaryAmount pri) {