From ed48ba54b22c1af65123ef02e4f90607c3cfdbac Mon Sep 17 00:00:00 2001
From: Stephane Janel <stephane.janel@amadeus.com>
Date: Fri, 1 Mar 2024 19:46:32 +0100
Subject: [PATCH] [Feature] - New CLI option market-data

---
 src/api/interface/src/exchange.cpp            |  6 ++--
 src/engine/include/coincenter.hpp             |  4 +++
 src/engine/include/coincentercommand.hpp      |  3 +-
 src/engine/include/coincenteroptions.hpp      |  5 +--
 src/engine/include/coincenteroptionsdef.hpp   | 11 +++---
 src/engine/include/exchangesorchestrator.hpp  |  2 ++
 src/engine/include/queryresulttypes.hpp       |  3 +-
 src/engine/src/coincenter.cpp                 | 34 +++++++++++++++++--
 src/engine/src/coincentercommands.cpp         | 11 ++++--
 src/engine/src/exchangesorchestrator.cpp      | 28 ++++++++++++---
 src/engine/test/coincenteroptions_test.cpp    |  2 +-
 src/objects/include/coincentercommandtype.hpp |  1 +
 src/objects/src/coincentercommandtype.cpp     |  2 +-
 13 files changed, 87 insertions(+), 25 deletions(-)

diff --git a/src/api/interface/src/exchange.cpp b/src/api/interface/src/exchange.cpp
index 2f8ffc5b..5a3bddf4 100644
--- a/src/api/interface/src/exchange.cpp
+++ b/src/api/interface/src/exchange.cpp
@@ -63,7 +63,7 @@ bool Exchange::canWithdraw(CurrencyCode currencyCode, const CurrencyExchangeFlat
 }
 
 bool Exchange::canDeposit(CurrencyCode currencyCode, const CurrencyExchangeFlatSet &currencyExchangeSet) const {
-  auto lb = currencyExchangeSet.find(currencyCode);
+  const auto lb = currencyExchangeSet.find(currencyCode);
   if (lb == currencyExchangeSet.end()) {
     log::trace("{} cannot be deposited on {}", currencyCode, name());
     return false;
@@ -72,7 +72,7 @@ bool Exchange::canDeposit(CurrencyCode currencyCode, const CurrencyExchangeFlatS
 }
 
 MarketOrderBook Exchange::queryOrderBook(Market mk, int depth) {
-  auto marketOrderBook = _exchangePublic.queryOrderBook(mk, depth);
+  const auto marketOrderBook = _exchangePublic.queryOrderBook(mk, depth);
   if (_marketDataSerializerPtr) {
     _marketDataSerializerPtr->push(marketOrderBook);
   }
@@ -81,7 +81,7 @@ MarketOrderBook Exchange::queryOrderBook(Market mk, int depth) {
 
 /// Retrieve an ordered vector of recent last trades
 TradesVector Exchange::queryLastTrades(Market mk, int nbTrades) {
-  auto lastTrades = _exchangePublic.queryLastTrades(mk, nbTrades);
+  const auto lastTrades = _exchangePublic.queryLastTrades(mk, nbTrades);
   if (_marketDataSerializerPtr) {
     _marketDataSerializerPtr->push(lastTrades);
   }
diff --git a/src/engine/include/coincenter.hpp b/src/engine/include/coincenter.hpp
index 05dd5dc8..ce0b8492 100644
--- a/src/engine/include/coincenter.hpp
+++ b/src/engine/include/coincenter.hpp
@@ -49,6 +49,10 @@ class Coincenter {
                                                      CurrencyCode equiCurrencyCode,
                                                      std::optional<int> depth = std::nullopt);
 
+  /// Query market data without returning it.
+  /// This method is especially useful for serialization and metric exports.
+  void queryMarketDataPerExchange(Market mk, ExchangeNameSpan exchangeNames);
+
   /// Retrieve the last 24h traded volume for exchanges supporting given market.
   MonetaryAmountPerExchange getLast24hTradedVolumePerExchange(Market mk, ExchangeNameSpan exchangeNames);
 
diff --git a/src/engine/include/coincentercommand.hpp b/src/engine/include/coincentercommand.hpp
index 6e0dad96..60cd2668 100644
--- a/src/engine/include/coincentercommand.hpp
+++ b/src/engine/include/coincentercommand.hpp
@@ -44,7 +44,6 @@ class CoincenterCommand {
   CoincenterCommand& setAmount(MonetaryAmount amount);
 
   CoincenterCommand& setDepth(int depth);
-  CoincenterCommand& setNbLastTrades(int nbTrades) { return setDepth(nbTrades); }
 
   CoincenterCommand& setMarket(Market market);
 
@@ -70,7 +69,7 @@ class CoincenterCommand {
 
   MonetaryAmount amount() const { return _amount; }
 
-  int nbLastTrades() const { return _n; }
+  int depth() const { return _n; }
   std::optional<int> optDepth() const { return _n == -1 ? std::nullopt : std::optional<int>(_n); }
 
   Market market() const { return _market; }
diff --git a/src/engine/include/coincenteroptions.hpp b/src/engine/include/coincenteroptions.hpp
index 6b16a8c7..1ffe7d6d 100644
--- a/src/engine/include/coincenteroptions.hpp
+++ b/src/engine/include/coincenteroptions.hpp
@@ -97,13 +97,14 @@ class CoincenterCmdLineOptions {
 
   std::string_view lastTrades;
 
+  std::string_view marketData;
+
   std::optional<std::string_view> replay;
   std::optional<std::string_view> replayMarkets;
 
   CommandLineOptionalInt repeats;
-  int nbLastTrades = api::ExchangePublic::kNbLastTradesDefault;
   int monitoringPort = CoincenterCmdLineOptionsDefinitions::kDefaultMonitoringPort;
-  int orderbookDepth = 0;
+  int depth = 0;
 
   bool forceMultiTrade = false;
   bool forceSingleTrade = false;
diff --git a/src/engine/include/coincenteroptionsdef.hpp b/src/engine/include/coincenteroptionsdef.hpp
index 5ec85a52..0b0d52f5 100644
--- a/src/engine/include/coincenteroptionsdef.hpp
+++ b/src/engine/include/coincenteroptionsdef.hpp
@@ -208,8 +208,7 @@ struct CoincenterAllowedOptions : private CoincenterCmdLineOptionsDefinitions {
         "If conversion of cur2 into cur is possible (for each exchange), "
         "prints additional column converted to given asset"},
        &OptValueType::orderbookCur},
-      {{{"Public queries", 2300}, "--depth", "<n>", "Override default depth of order book"},
-       &OptValueType::orderbookDepth},
+      {{{"Public queries", 2300}, "--depth", "<n>", "Override default depth of order book"}, &OptValueType::depth},
       {{{"Public queries", 2400},
         "ticker",
         "<[exch1,...]>",
@@ -239,7 +238,7 @@ struct CoincenterAllowedOptions : private CoincenterCmdLineOptionsDefinitions {
         "Print last trades for market 'cur1'-'cur2' "
         "for all exchanges (or specified one)"},
        &OptValueType::lastTrades},
-      {{{"Public queries", 2800}, "--n", "<n>", kLastTradesN}, &OptValueType::nbLastTrades},
+      {{{"Public queries", 2800}, "--n", "<n>", kLastTradesN}, &OptValueType::depth},
       {{{"Public queries", 2900},
         "price",
         "<cur1-cur2[,exch1,...]>",
@@ -406,15 +405,15 @@ struct CoincenterAllowedOptions : private CoincenterCmdLineOptionsDefinitions {
        &OptValueType::async},
       {{{"Withdraw and deposit", 6010}, "--refresh-time", "<time>", kWithdrawRefreshTime},
        &OptValueType::withdrawRefreshTime},
-      {{{"Withdraw and deposit", 7000},
+      {{{"Withdraw and deposit", 6020},
         "withdraw-apply-all",
         "<cur,from-to>",
         "Withdraw all available amount instead of a specified amount."},
        &OptValueType::withdrawApplyAll},
-      {{{"Withdraw and deposit", 8000},
+      {{{"Withdraw and deposit", 6030},
         "withdraw-fees",
         "<[cur][,exch1,...]>",
-        "Prints withdraw fees for matching currency and exchanges."
+        "Prints withdraw fees for matching currency and exchanges.\n"
         "Currency and exchanges are optional, if specified, output will be filtered to match them."},
        &OptValueType::withdrawFees},
       {{{"Automation", 8000},
diff --git a/src/engine/include/exchangesorchestrator.hpp b/src/engine/include/exchangesorchestrator.hpp
index aba84452..eb9e6672 100644
--- a/src/engine/include/exchangesorchestrator.hpp
+++ b/src/engine/include/exchangesorchestrator.hpp
@@ -88,6 +88,8 @@ class ExchangesOrchestrator {
 
   MonetaryAmountPerExchange getLastPricePerExchange(Market mk, ExchangeNameSpan exchangeNames);
 
+  MarketDataPerExchange getMarketDataPerExchange(Market mk, ExchangeNameSpan exchangeNames);
+
   MarketsPerExchange pullAvailableMarketsForReplay(TimeWindow timeWindow, ExchangeNameSpan exchangeNames);
 
   void traderConsume(Market market, TimeWindow timeWindow, std::span<MarketTraderEngine> marketTraderEngines,
diff --git a/src/engine/include/queryresulttypes.hpp b/src/engine/include/queryresulttypes.hpp
index 9d22ae19..9404bac5 100644
--- a/src/engine/include/queryresulttypes.hpp
+++ b/src/engine/include/queryresulttypes.hpp
@@ -39,7 +39,8 @@ using MonetaryAmountByCurrencySetPerExchange =
 
 using TradesPerExchange = FixedCapacityVector<ExchangeWith<TradesVector>, kNbSupportedExchanges>;
 
-using MarketOrderBooksPerExchange = FixedCapacityVector<ExchangeWith<MarketOrderBookVector>, kNbSupportedExchanges>;
+using MarketDataPerExchange =
+    FixedCapacityVector<ExchangeWith<std::pair<MarketOrderBook, TradesVector>>, kNbSupportedExchanges>;
 
 using TradeResultPerExchange = SmallVector<ExchangeWith<TradeResult>, kTypicalNbPrivateAccounts>;
 
diff --git a/src/engine/src/coincenter.cpp b/src/engine/src/coincenter.cpp
index 902cdf29..5811370d 100644
--- a/src/engine/src/coincenter.cpp
+++ b/src/engine/src/coincenter.cpp
@@ -143,9 +143,9 @@ TransferableCommandResultVector Coincenter::processCommand(
       break;
     }
     case CoincenterCommandType::kLastTrades: {
-      const auto lastTradesPerExchange =
-          getLastTradesPerExchange(cmd.market(), cmd.exchangeNames(), cmd.nbLastTrades());
-      _queryResultPrinter.printLastTrades(cmd.market(), cmd.nbLastTrades(), lastTradesPerExchange);
+      const int depth = cmd.depth() == 0 ? api::ExchangePublic::kNbLastTradesDefault : cmd.depth();
+      const auto lastTradesPerExchange = getLastTradesPerExchange(cmd.market(), cmd.exchangeNames(), depth);
+      _queryResultPrinter.printLastTrades(cmd.market(), depth, lastTradesPerExchange);
       break;
     }
     case CoincenterCommandType::kLast24hTradedVolume: {
@@ -248,6 +248,11 @@ TransferableCommandResultVector Coincenter::processCommand(
       _queryResultPrinter.printDustSweeper(dustSweeper(cmd.exchangeNames(), cmd.cur1()), cmd.cur1());
       break;
     }
+    case CoincenterCommandType::kMarketData: {
+      // No return value here, this command is made only for storing purposes.
+      queryMarketDataPerExchange(cmd.market(), cmd.exchangeNames());
+      break;
+    }
     case CoincenterCommandType::kReplay: {
       replay(cmd.duration(), cmd.exchangeNames());
       break;
@@ -289,6 +294,29 @@ MarketOrderBookConversionRates Coincenter::getMarketOrderBooks(Market mk, Exchan
   return ret;
 }
 
+void Coincenter::queryMarketDataPerExchange(Market mk, ExchangeNameSpan exchangeNames) {
+  const auto marketDataPerExchange = _exchangesOrchestrator.getMarketDataPerExchange(mk, exchangeNames);
+
+  // Transform data structures to export metrics input format
+  MarketOrderBookConversionRates marketOrderBookConversionRates(marketDataPerExchange.size());
+  TradesPerExchange lastTradesPerExchange(marketDataPerExchange.size());
+
+  std::ranges::transform(marketDataPerExchange, marketOrderBookConversionRates.begin(),
+                         [](const auto &exchangeWithPairOrderBooksAndTrades) {
+                           return std::make_tuple(exchangeWithPairOrderBooksAndTrades.first->name(),
+                                                  exchangeWithPairOrderBooksAndTrades.second.first, std::nullopt);
+                         });
+
+  std::ranges::transform(marketDataPerExchange, lastTradesPerExchange.begin(),
+                         [](const auto &exchangeWithPairOrderBooksAndTrades) {
+                           return std::make_pair(exchangeWithPairOrderBooksAndTrades.first,
+                                                 exchangeWithPairOrderBooksAndTrades.second.second);
+                         });
+
+  _metricsExporter.exportOrderbookMetrics(mk, marketOrderBookConversionRates);
+  _metricsExporter.exportLastTradesMetrics(mk, lastTradesPerExchange);
+}
+
 BalancePerExchange Coincenter::getBalance(std::span<const ExchangeName> privateExchangeNames,
                                           const BalanceOptions &balanceOptions) {
   CurrencyCode equiCurrency = balanceOptions.equiCurrency();
diff --git a/src/engine/src/coincentercommands.cpp b/src/engine/src/coincentercommands.cpp
index 5bd616ab..a0d1fafe 100644
--- a/src/engine/src/coincentercommands.cpp
+++ b/src/engine/src/coincentercommands.cpp
@@ -66,7 +66,7 @@ void CoincenterCommands::addOption(const CoincenterCmdLineOptions &cmdLineOption
     _commands.emplace_back(CoincenterCommandType::kOrderbook)
         .setMarket(optionParser.parseMarket())
         .setExchangeNames(optionParser.parseExchanges())
-        .setDepth(cmdLineOptions.orderbookDepth)
+        .setDepth(cmdLineOptions.depth)
         .setCur1(cmdLineOptions.orderbookCur);
   }
 
@@ -184,7 +184,7 @@ void CoincenterCommands::addOption(const CoincenterCmdLineOptions &cmdLineOption
     optionParser = StringOptionParser(cmdLineOptions.lastTrades);
     _commands.emplace_back(CoincenterCommandType::kLastTrades)
         .setMarket(optionParser.parseMarket())
-        .setNbLastTrades(cmdLineOptions.nbLastTrades)
+        .setDepth(cmdLineOptions.depth)
         .setExchangeNames(optionParser.parseExchanges());
   }
 
@@ -195,6 +195,13 @@ void CoincenterCommands::addOption(const CoincenterCmdLineOptions &cmdLineOption
         .setExchangeNames(optionParser.parseExchanges());
   }
 
+  if (!cmdLineOptions.marketData.empty()) {
+    optionParser = StringOptionParser(cmdLineOptions.marketData);
+    _commands.emplace_back(CoincenterCommandType::kMarketData)
+        .setMarket(optionParser.parseMarket())
+        .setExchangeNames(optionParser.parseExchanges());
+  }
+
   if (cmdLineOptions.replay) {
     optionParser = StringOptionParser(*cmdLineOptions.replay);
 
diff --git a/src/engine/src/exchangesorchestrator.cpp b/src/engine/src/exchangesorchestrator.cpp
index 4c9204b3..9c7d852d 100644
--- a/src/engine/src/exchangesorchestrator.cpp
+++ b/src/engine/src/exchangesorchestrator.cpp
@@ -139,12 +139,12 @@ MarketOrderBookConversionRates ExchangesOrchestrator::getMarketOrderBooks(Market
         equiCurrencyCode.isNeutral()
             ? std::nullopt
             : exchange->apiPublic().estimatedConvert(MonetaryAmount(1, mk.quote()), equiCurrencyCode);
-    MarketOrderBook marketOrderBook(depth ? exchange->queryOrderBook(mk, *depth) : exchange->queryOrderBook(mk));
     if (!optConversionRate && !equiCurrencyCode.isNeutral()) {
-      log::warn("Unable to convert {} into {} on {}", marketOrderBook.market().quote(), equiCurrencyCode,
-                exchange->name());
+      log::warn("Unable to convert {} into {} on {}", mk.quote(), equiCurrencyCode, exchange->name());
     }
-    return std::make_tuple(exchange->name(), std::move(marketOrderBook), optConversionRate);
+    return std::make_tuple(exchange->name(),
+                           depth ? exchange->queryOrderBook(mk, *depth) : exchange->queryOrderBook(mk),
+                           optConversionRate);
   };
   _threadPool.parallelTransform(selectedExchanges.begin(), selectedExchanges.end(), ret.begin(), marketOrderBooksFunc);
   return ret;
@@ -902,6 +902,26 @@ MonetaryAmountPerExchange ExchangesOrchestrator::getLastPricePerExchange(Market
   return lastPricePerExchange;
 }
 
+MarketDataPerExchange ExchangesOrchestrator::getMarketDataPerExchange(Market mk, ExchangeNameSpan exchangeNames) {
+  log::info("Query market data for {} on {}", mk, ConstructAccumulatedExchangeNames(exchangeNames));
+
+  UniquePublicSelectedExchanges selectedExchanges = _exchangeRetriever.selectOneAccount(exchangeNames);
+  std::array<bool, kNbSupportedExchanges> isMarketTradable;
+
+  _threadPool.parallelTransform(selectedExchanges.begin(), selectedExchanges.end(), isMarketTradable.begin(),
+                                [mk](Exchange *exchange) { return exchange->queryTradableMarkets().contains(mk); });
+
+  FilterVector(selectedExchanges, isMarketTradable);
+
+  MarketDataPerExchange ret(selectedExchanges.size());
+  _threadPool.parallelTransform(
+      selectedExchanges.begin(), selectedExchanges.end(), ret.begin(), [mk](Exchange *exchange) {
+        // Call order book and last trades sequentially for this exchange
+        return std::make_pair(exchange, std::make_pair(exchange->queryOrderBook(mk), exchange->queryLastTrades(mk)));
+      });
+  return ret;
+}
+
 MarketsPerExchange ExchangesOrchestrator::pullAvailableMarketsForReplay(TimeWindow timeWindow,
                                                                         ExchangeNameSpan exchangeNames) {
   log::info("Query available markets for replay from {} within {}", ConstructAccumulatedExchangeNames(exchangeNames),
diff --git a/src/engine/test/coincenteroptions_test.cpp b/src/engine/test/coincenteroptions_test.cpp
index 81fae8ee..98669651 100644
--- a/src/engine/test/coincenteroptions_test.cpp
+++ b/src/engine/test/coincenteroptions_test.cpp
@@ -59,7 +59,7 @@ TEST_F(CoincenterCmdLineOptionsTest, MergeGlobal) {
 
   CoincenterCmdLineOptions rhs;
   rhs.trade = "some value";
-  rhs.nbLastTrades = 42;
+  rhs.depth = 42;
   rhs.repeatTime = std::chrono::minutes(45);
   rhs.monitoringPort = 999;
 
diff --git a/src/objects/include/coincentercommandtype.hpp b/src/objects/include/coincentercommandtype.hpp
index 5547243b..4a34416b 100644
--- a/src/objects/include/coincentercommandtype.hpp
+++ b/src/objects/include/coincentercommandtype.hpp
@@ -30,6 +30,7 @@ enum class CoincenterCommandType : int8_t {
   kWithdrawApply,
   kDustSweeper,
 
+  kMarketData,
   kReplay,
   kReplayMarkets,
 
diff --git a/src/objects/src/coincentercommandtype.cpp b/src/objects/src/coincentercommandtype.cpp
index 96a262ec..4786637f 100644
--- a/src/objects/src/coincentercommandtype.cpp
+++ b/src/objects/src/coincentercommandtype.cpp
@@ -15,7 +15,7 @@ constexpr std::string_view kCommandTypeNames[] = {
 
     "Balance",        "DepositInfo",     "OrdersClosed", "OrdersOpened", "OrdersCancel",
     "RecentDeposits", "RecentWithdraws", "Trade",        "Buy",          "Sell",
-    "Withdraw",       "DustSweeper",     "Replay",       "ReplayMarkets"};
+    "Withdraw",       "DustSweeper",     "MarketData",   "Replay",       "ReplayMarkets"};
 
 static_assert(std::size(kCommandTypeNames) == static_cast<std::size_t>(CoincenterCommandType::kLast));
 }  // namespace