Fix Bithumb withdrawal fees parsing, factorize crawlers code

src/api-objects/include/ordersconstraints.hpp

@@ -120,7 +120,8 @@ class OrdersConstraints {
 template <>
 struct fmt::formatter<cct::OrdersConstraints> {
   constexpr auto parse(format_parse_context &ctx) -> decltype(ctx.begin()) {
-    auto it = ctx.begin(), end = ctx.end();
+    const auto it = ctx.begin();
+    const auto end = ctx.end();
     if (it != end && *it != '}') {
       throw format_error("invalid format");
     }

src/api-objects/include/publictrade.hpp

@@ -26,12 +26,12 @@ class PublicTrade {
 
   /// 3 way operator - make compiler generate all 6 operators (including == and !=)
   /// we order by time first, then amount, price, etc. Do not change the fields order!
-  auto operator<=>(const PublicTrade &) const = default;
+  std::strong_ordering operator<=>(const PublicTrade &) const = default;
 
  private:
   TimePoint _time;
   MonetaryAmount _amount;
   MonetaryAmount _price;
   TradeSide _side;
 };
-}  // namespace cct
+}  // namespace cct

src/api-objects/include/recentdeposit.hpp

@@ -49,7 +49,8 @@ class ClosestRecentDepositPicker {
 template <>
 struct fmt::formatter<cct::api::RecentDeposit> {
   constexpr auto parse(format_parse_context &ctx) -> decltype(ctx.begin()) {
-    auto it = ctx.begin(), end = ctx.end();
+    const auto it = ctx.begin();
+    const auto end = ctx.end();
     if (it != end && *it != '}') {
       throw format_error("invalid format");
     }

src/api-objects/include/withdrawsordepositsconstraints.hpp

@@ -66,7 +66,8 @@ class WithdrawsOrDepositsConstraints {
 template <>
 struct fmt::formatter<cct::WithdrawsOrDepositsConstraints> {
   constexpr auto parse(format_parse_context &ctx) -> decltype(ctx.begin()) {
-    auto it = ctx.begin(), end = ctx.end();
+    const auto it = ctx.begin();
+    const auto end = ctx.end();
     if (it != end && *it != '}') {
       throw format_error("invalid format");
     }

src/api/common/CMakeLists.txt

@@ -4,8 +4,8 @@ aux_source_directory(src API_COMMON_SRC)
 include(FindOpenSSL)
 
 add_library(coincenter_api-common STATIC ${API_COMMON_SRC})
-target_link_libraries(coincenter_api-common PUBLIC coincenter_objects)
 target_link_libraries(coincenter_api-common PUBLIC coincenter_api-objects)
+target_link_libraries(coincenter_api-common PUBLIC coincenter_objects)
 target_link_libraries(coincenter_api-common PUBLIC coincenter_http-request)
 target_link_libraries(coincenter_api-common PRIVATE OpenSSL::SSL)
 
@@ -19,11 +19,8 @@ function(add_common_test name)
    add_unit_test(
      ${name} 
      ${MY_UNPARSED_ARGUMENTS}
-     src/commonapi.cpp
      LIBRARIES
-     coincenter_api-objects
-     coincenter_http-request
-     coincenter_objects
+     coincenter_api-common
    )
 endfunction()
 

src/api/common/include/commonapi.hpp

@@ -3,6 +3,7 @@
 #include <chrono>
 #include <cstdint>
 #include <mutex>
+#include <string_view>
 
 #include "cachedresult.hpp"
 #include "cct_flatset.hpp"
@@ -11,6 +12,7 @@
 #include "currencycode.hpp"
 #include "exchangebase.hpp"
 #include "timedef.hpp"
+#include "withdrawalfees-crawler.hpp"
 
 namespace cct {
 class CoincenterInfo;
@@ -22,22 +24,20 @@ class CommonAPI : public ExchangeBase {
 
   enum class AtInit : int8_t { kLoadFromFileCache, kNoLoadFromFileCache };
 
-  CommonAPI(const CoincenterInfo &config, Duration fiatsUpdateFrequency = std::chrono::hours(96),
-            AtInit atInit = AtInit::kLoadFromFileCache);
+  explicit CommonAPI(const CoincenterInfo &coincenterInfo, Duration fiatsUpdateFrequency = std::chrono::hours(96),
+                     Duration withdrawalFeesUpdateFrequency = std::chrono::hours(96),
+                     AtInit atInit = AtInit::kLoadFromFileCache);
 
   /// Returns a new set of fiat currencies.
-  Fiats queryFiats() {
-    std::lock_guard<std::mutex> guard(_fiatsMutex);
-    return _fiatsCache.get();
-  }
+  Fiats queryFiats();
 
   /// Tells whether given currency code is a fiat currency or not.
   /// Fiat currencies are traditional currencies, such as EUR, USD, GBP, KRW, etc.
   /// Information here: https://en.wikipedia.org/wiki/Fiat_money
-  bool queryIsCurrencyCodeFiat(CurrencyCode currencyCode) {
-    std::lock_guard<std::mutex> guard(_fiatsMutex);
-    return _fiatsCache.get().contains(currencyCode);
-  }
+  bool queryIsCurrencyCodeFiat(CurrencyCode currencyCode);
+
+  /// Query withdrawal fees from crawler sources. It's not guaranteed to work though.
+  WithdrawalFeesCrawler::WithdrawalInfoMaps queryWithdrawalFees(std::string_view exchangeName);
 
   void updateCacheFile() const override;
 
@@ -58,8 +58,10 @@ class CommonAPI : public ExchangeBase {
 
   CachedResultVault _cachedResultVault;
   const CoincenterInfo &_coincenterInfo;
-  std::mutex _fiatsMutex;
+  // A single mutex is needed as the cached result vault is shared
+  std::mutex _globalMutex;
   CachedResult<FiatsFunc> _fiatsCache;
+  WithdrawalFeesCrawler _withdrawalFeesCrawler;
 };
 }  // namespace api
 }  // namespace cct

src/api/common/include/withdrawalfees-crawler.hpp

@@ -0,0 +1,51 @@
+#pragma once
+
+#include <string_view>
+#include <unordered_map>
+#include <utility>
+
+#include "cachedresult.hpp"
+#include "cachedresultvault.hpp"
+#include "coincenterinfo.hpp"
+#include "curlhandle.hpp"
+#include "currencycode.hpp"
+#include "monetaryamount.hpp"
+#include "monetaryamountbycurrencyset.hpp"
+#include "timedef.hpp"
+
+namespace cct {
+
+/// This class is able to crawl some public withdrawal fees web pages in order to retrieve them from unofficial sources,
+/// which is better than nothing. This class is non thread-safe.
+class WithdrawalFeesCrawler {
+ public:
+  WithdrawalFeesCrawler(const CoincenterInfo& coincenterInfo, Duration minDurationBetweenQueries,
+                        CachedResultVault& cachedResultVault);
+
+  using WithdrawalMinMap = std::unordered_map<CurrencyCode, MonetaryAmount>;
+  using WithdrawalInfoMaps = std::pair<MonetaryAmountByCurrencySet, WithdrawalMinMap>;
+
+  WithdrawalInfoMaps get(std::string_view exchangeName) { return _withdrawalFeesCache.get(exchangeName); }
+
+  void updateCacheFile() const;
+
+ private:
+  class WithdrawalFeesFunc {
+   public:
+    explicit WithdrawalFeesFunc(const CoincenterInfo& coincenterInfo);
+
+    WithdrawalInfoMaps operator()(std::string_view exchangeName);
+
+   private:
+    WithdrawalInfoMaps get1(std::string_view exchangeName);
+    WithdrawalInfoMaps get2(std::string_view exchangeName);
+
+    CurlHandle _curlHandle1;
+    CurlHandle _curlHandle2;
+  };
+
+  const CoincenterInfo& _coincenterInfo;
+  CachedResult<WithdrawalFeesFunc, std::string_view> _withdrawalFeesCache;
+};
+
+}  // namespace cct

src/api/common/src/commonapi.cpp

@@ -26,8 +26,11 @@ File GetFiatCacheFile(std::string_view dataDir) {
 
 }  // namespace
 
-CommonAPI::CommonAPI(const CoincenterInfo& config, Duration fiatsUpdateFrequency, AtInit atInit)
-    : _coincenterInfo(config), _fiatsCache(CachedResultOptions(fiatsUpdateFrequency, _cachedResultVault)) {
+CommonAPI::CommonAPI(const CoincenterInfo& coincenterInfo, Duration fiatsUpdateFrequency,
+                     Duration withdrawalFeesUpdateFrequency, AtInit atInit)
+    : _coincenterInfo(coincenterInfo),
+      _fiatsCache(CachedResultOptions(fiatsUpdateFrequency, _cachedResultVault)),
+      _withdrawalFeesCrawler(coincenterInfo, withdrawalFeesUpdateFrequency, _cachedResultVault) {
   if (atInit == AtInit::kLoadFromFileCache) {
     json data = GetFiatCacheFile(_coincenterInfo.dataDir()).readAllJson();
     if (!data.empty()) {
@@ -45,6 +48,18 @@ CommonAPI::CommonAPI(const CoincenterInfo& config, Duration fiatsUpdateFrequency
   }
 }
 
+CommonAPI::Fiats CommonAPI::queryFiats() {
+  std::lock_guard<std::mutex> guard(_globalMutex);
+  return _fiatsCache.get();
+}
+
+bool CommonAPI::queryIsCurrencyCodeFiat(CurrencyCode currencyCode) { return queryFiats().contains(currencyCode); }
+
+WithdrawalFeesCrawler::WithdrawalInfoMaps CommonAPI::queryWithdrawalFees(std::string_view exchangeName) {
+  std::lock_guard<std::mutex> guard(_globalMutex);
+  return _withdrawalFeesCrawler.get(exchangeName);
+}
+
 namespace {
 constexpr std::string_view kFiatsUrlSource1 = "https://datahub.io/core/currency-codes/r/codes-all.json";
 constexpr std::string_view kFiatsUrlSource2 = "https://www.iban.com/currency-codes";
@@ -157,5 +172,7 @@ void CommonAPI::updateCacheFile() const {
     data["timeepoch"] = TimestampToS(fiatsPtrLastUpdatedTimePair.second);
     fiatsCacheFile.write(data);
   }
+
+  _withdrawalFeesCrawler.updateCacheFile();
 }
 }  // namespace cct::api