diff --git a/src/api/common/include/commonapi.hpp b/src/api/common/include/commonapi.hpp index 35e6318f..351506a7 100644 --- a/src/api/common/include/commonapi.hpp +++ b/src/api/common/include/commonapi.hpp @@ -6,6 +6,7 @@ #include "cachedresult.hpp" #include "cct_flatset.hpp" +#include "cct_vector.hpp" #include "curlhandle.hpp" #include "currencycode.hpp" #include "exchangebase.hpp" @@ -41,12 +42,18 @@ class CommonAPI : public ExchangeBase { void updateCacheFile() const override; private: - struct FiatsFunc { + class FiatsFunc { + public: FiatsFunc(); Fiats operator()(); - CurlHandle _curlHandle; + vector retrieveFiatsSource1(); + vector retrieveFiatsSource2(); + + private: + CurlHandle _curlHandle1; + CurlHandle _curlHandle2; }; CachedResultVault _cachedResultVault; diff --git a/src/api/common/src/commonapi.cpp b/src/api/common/src/commonapi.cpp index 28a661c3..1c360e3b 100644 --- a/src/api/common/src/commonapi.cpp +++ b/src/api/common/src/commonapi.cpp @@ -46,15 +46,43 @@ CommonAPI::CommonAPI(const CoincenterInfo& config, Duration fiatsUpdateFrequency } namespace { -constexpr std::string_view kFiatsUrl = "https://datahub.io/core/currency-codes/r/codes-all.json"; -} +constexpr std::string_view kFiatsUrlSource1 = "https://datahub.io/core/currency-codes/r/codes-all.json"; +constexpr std::string_view kFiatsUrlSource2 = "https://www.iban.com/currency-codes"; +} // namespace CommonAPI::FiatsFunc::FiatsFunc() - : _curlHandle(kFiatsUrl, nullptr, PermanentCurlOptions::Builder().setFollowLocation().build()) {} + : _curlHandle1(kFiatsUrlSource1, nullptr, + PermanentCurlOptions::Builder() + .setFollowLocation() + .setTooManyErrorsPolicy(PermanentCurlOptions::TooManyErrorsPolicy::kReturnEmptyResponse) + .build()), + _curlHandle2(kFiatsUrlSource2, nullptr, + PermanentCurlOptions::Builder() + .setTooManyErrorsPolicy(PermanentCurlOptions::TooManyErrorsPolicy::kReturnEmptyResponse) + .build()) {} CommonAPI::Fiats CommonAPI::FiatsFunc::operator()() { - json dataCSV = json::parse(_curlHandle.query("", CurlOptions(HttpRequestType::kGet))); + vector fiatsVec = retrieveFiatsSource1(); Fiats fiats; + if (!fiats.empty()) { + fiats = Fiats(std::move(fiatsVec)); + log::info("Stored {} fiats from first source", fiats.size()); + } else { + fiats = Fiats(retrieveFiatsSource2()); + log::info("Stored {} fiats from second source", fiats.size()); + } + return fiats; +} + +vector CommonAPI::FiatsFunc::retrieveFiatsSource1() { + vector fiatsVec; + + std::string_view data = _curlHandle1.query("", CurlOptions(HttpRequestType::kGet)); + if (data.empty()) { + log::error("Error parsing currency codes, no fiats found from first source"); + return fiatsVec; + } + json dataCSV = json::parse(data); for (const json& fiatData : dataCSV) { static constexpr std::string_view kCodeKey = "AlphabeticCode"; static constexpr std::string_view kWithdrawalDateKey = "WithdrawalDate"; @@ -62,16 +90,52 @@ CommonAPI::Fiats CommonAPI::FiatsFunc::operator()() { auto withdrawalDateIt = fiatData.find(kWithdrawalDateKey); if (codeIt != fiatData.end() && !codeIt->is_null() && withdrawalDateIt != fiatData.end() && withdrawalDateIt->is_null()) { - fiats.insert(CurrencyCode(codeIt->get())); + fiatsVec.emplace_back(codeIt->get()); log::debug("Stored {} fiat", codeIt->get()); } } - if (fiats.empty()) { - throw exception("Error parsing currency codes, no fiats found in {}", dataCSV.dump()); + + return fiatsVec; +} + +vector CommonAPI::FiatsFunc::retrieveFiatsSource2() { + vector fiatsVec; + std::string_view data = _curlHandle2.query("", CurlOptions(HttpRequestType::kGet)); + if (data.empty()) { + log::error("Error parsing currency codes, no fiats found from second source"); + return fiatsVec; } - log::info("Stored {} fiats", fiats.size()); - return fiats; + static constexpr std::string_view kTheadColumnStart = ""; + static constexpr std::string_view kCodeStrName = "Code"; + auto pos = data.find(""); + int codePos = 0; + for (pos = data.find(kTheadColumnStart, pos + 1U); pos != std::string_view::npos; + pos = data.find(kTheadColumnStart, pos + 1U), ++codePos) { + auto endPos = data.find("", pos + 1); + std::string_view colName(data.begin() + pos + kTheadColumnStart.size(), data.begin() + endPos); + if (colName == kCodeStrName) { + ++codePos; + break; + } + pos = endPos; + } + + pos = data.find(""); + for (pos = data.find("", pos + 1U); pos != std::string_view::npos; pos = data.find("", pos + 1U)) { + static constexpr std::string_view kColStart = ""; + for (int col = 0; col < codePos; ++col) { + pos = data.find(kColStart, pos + 1); + } + auto endPos = data.find("", pos + 1); + std::string_view curStr(data.begin() + pos + kColStart.size(), data.begin() + endPos); + if (!curStr.empty()) { + // Fiat data is sometimes empty + fiatsVec.emplace_back(curStr); + } + } + + return fiatsVec; } void CommonAPI::updateCacheFile() const { diff --git a/src/api/common/src/exchangepublicapi.cpp b/src/api/common/src/exchangepublicapi.cpp index 3862c65f..faa61e6b 100644 --- a/src/api/common/src/exchangepublicapi.cpp +++ b/src/api/common/src/exchangepublicapi.cpp @@ -393,4 +393,4 @@ MonetaryAmount ExchangePublic::queryWithdrawalFeeOrZero(CurrencyCode currencyCod return withdrawFee; } -} // namespace cct::api \ No newline at end of file +} // namespace cct::api diff --git a/src/api/common/test/commonapi_test.cpp b/src/api/common/test/commonapi_test.cpp index 54de6ddb..44d0b606 100644 --- a/src/api/common/test/commonapi_test.cpp +++ b/src/api/common/test/commonapi_test.cpp @@ -12,7 +12,7 @@ class CommonAPITest : public ::testing::Test { protected: settings::RunMode runMode = settings::RunMode::kTestKeys; CoincenterInfo config{runMode}; - CommonAPI commonAPI{config}; + CommonAPI commonAPI{config, Duration::max(), CommonAPI::AtInit::kNoLoadFromFileCache}; }; TEST_F(CommonAPITest, IsFiatService) { diff --git a/src/api/exchanges/include/binancepublicapi.hpp b/src/api/exchanges/include/binancepublicapi.hpp index 00cf2341..013cd804 100644 --- a/src/api/exchanges/include/binancepublicapi.hpp +++ b/src/api/exchanges/include/binancepublicapi.hpp @@ -147,4 +147,4 @@ class BinancePublic : public ExchangePublic { }; } // namespace api -} // namespace cct \ No newline at end of file +} // namespace cct diff --git a/src/api/exchanges/include/bithumbpublicapi.hpp b/src/api/exchanges/include/bithumbpublicapi.hpp index 19c6f5bf..243197f0 100644 --- a/src/api/exchanges/include/bithumbpublicapi.hpp +++ b/src/api/exchanges/include/bithumbpublicapi.hpp @@ -111,4 +111,4 @@ class BithumbPublic : public ExchangePublic { }; } // namespace api -} // namespace cct \ No newline at end of file +} // namespace cct diff --git a/src/api/exchanges/include/upbitprivateapi.hpp b/src/api/exchanges/include/upbitprivateapi.hpp index ae013feb..fb89d1f0 100644 --- a/src/api/exchanges/include/upbitprivateapi.hpp +++ b/src/api/exchanges/include/upbitprivateapi.hpp @@ -101,4 +101,4 @@ class UpbitPrivate : public ExchangePrivate { CachedResult _withdrawalFeesCache; }; } // namespace api -} // namespace cct \ No newline at end of file +} // namespace cct diff --git a/src/api/exchanges/include/upbitpublicapi.hpp b/src/api/exchanges/include/upbitpublicapi.hpp index afd9cc4a..605d7f85 100644 --- a/src/api/exchanges/include/upbitpublicapi.hpp +++ b/src/api/exchanges/include/upbitpublicapi.hpp @@ -124,4 +124,4 @@ class UpbitPublic : public ExchangePublic { }; } // namespace api -} // namespace cct \ No newline at end of file +} // namespace cct diff --git a/src/api/interface/src/exchange.cpp b/src/api/interface/src/exchange.cpp index 1c7a031d..da7a8787 100644 --- a/src/api/interface/src/exchange.cpp +++ b/src/api/interface/src/exchange.cpp @@ -47,4 +47,4 @@ void Exchange::updateCacheFile() const { _pExchangePrivate->updateCacheFile(); } } -} // namespace cct \ No newline at end of file +} // namespace cct diff --git a/src/http-request/include/besturlpicker.hpp b/src/http-request/include/besturlpicker.hpp index 56c975e5..55697ee1 100644 --- a/src/http-request/include/besturlpicker.hpp +++ b/src/http-request/include/besturlpicker.hpp @@ -21,6 +21,8 @@ class BestURLPicker { static constexpr int kNbMaxBaseUrl = 4; public: + BestURLPicker() noexcept = default; + /// Builds a BestURLPicker that will work with several base URLs. /// Warning: given base URL should come from static storage template 0) && N <= kNbMaxBaseUrl, bool> = true> @@ -67,7 +69,7 @@ class BestURLPicker { using ResponseTimeStatsPerBaseUrl = FixedCapacityVector; // Non-owning pointer, should come from static storage (default special operations are fine) - const std::string_view *_pBaseUrls; + const std::string_view *_pBaseUrls{}; ResponseTimeStatsPerBaseUrl _responseTimeStatsPerBaseUrl; }; } // namespace cct \ No newline at end of file diff --git a/src/http-request/include/curlhandle.hpp b/src/http-request/include/curlhandle.hpp index 924a07ba..894bd093 100644 --- a/src/http-request/include/curlhandle.hpp +++ b/src/http-request/include/curlhandle.hpp @@ -26,6 +26,8 @@ string GetCurlVersionInfo(); /// CurlHandle for faster similar queries. class CurlHandle { public: + CurlHandle() noexcept = default; + /// Constructs a new CurlHandle. /// @param bestURLPicker object managing which URL to pick at each query based on response time stats /// @param pMetricGateway if not null, queries will export some metrics @@ -38,9 +40,8 @@ class CurlHandle { CurlHandle(const CurlHandle &) = delete; CurlHandle &operator=(const CurlHandle &) = delete; - // Move operations are deleted but could be implemented if needed. It's just to avoid useless code. - CurlHandle(CurlHandle &&) = delete; - CurlHandle &operator=(CurlHandle &&) = delete; + CurlHandle(CurlHandle &&rhs) noexcept; + CurlHandle &operator=(CurlHandle &&rhs) noexcept; ~CurlHandle(); @@ -61,6 +62,8 @@ class CurlHandle { /// complexity in a flat key value string. void setOverridenQueryResponses(const std::map &queryResponsesMap); + void swap(CurlHandle &rhs) noexcept; + using trivially_relocatable = is_trivially_relocatable::type; private: @@ -68,14 +71,17 @@ class CurlHandle { // void pointer instead of CURL to avoid having to forward declare (we don't know about the underlying definition) // and to avoid clients to pull unnecessary curl dependencies by just including the header - void *_handle; - AbstractMetricGateway *_pMetricGateway; // non-owning pointer - Duration _minDurationBetweenQueries; + void *_handle = nullptr; + AbstractMetricGateway *_pMetricGateway = nullptr; // non-owning pointer + Duration _minDurationBetweenQueries{}; TimePoint _lastQueryTime{}; BestURLPicker _bestUrlPicker; string _queryData; - log::level::level_enum _requestCallLogLevel; - log::level::level_enum _requestAnswerLogLevel; + log::level::level_enum _requestCallLogLevel = log::level::level_enum::off; + log::level::level_enum _requestAnswerLogLevel = log::level::level_enum::off; + int _nbMaxRetries = PermanentCurlOptions::kDefaultNbMaxRetries; + PermanentCurlOptions::TooManyErrorsPolicy _tooManyErrorsPolicy = + PermanentCurlOptions::TooManyErrorsPolicy::kReturnEmptyResponse; }; // Simple RAII class managing global init and clean up of Curl library. diff --git a/src/http-request/include/permanentcurloptions.hpp b/src/http-request/include/permanentcurloptions.hpp index 6d0cd246..c57d9393 100644 --- a/src/http-request/include/permanentcurloptions.hpp +++ b/src/http-request/include/permanentcurloptions.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include "cct_log.hpp" @@ -10,18 +11,26 @@ namespace cct { class PermanentCurlOptions { public: + static constexpr auto kDefaultNbMaxRetries = 5; + + enum class TooManyErrorsPolicy : int8_t { kThrow, kReturnEmptyResponse }; + PermanentCurlOptions() noexcept = default; - const string &getUserAgent() const { return _userAgent; } + const auto &getUserAgent() const { return _userAgent; } + + const auto &getAcceptedEncoding() const { return _acceptedEncoding; } - const string &getAcceptedEncoding() const { return _acceptedEncoding; } + auto minDurationBetweenQueries() const { return _minDurationBetweenQueries; } - Duration minDurationBetweenQueries() const { return _minDurationBetweenQueries; } + auto followLocation() const { return _followLocation; } - bool followLocation() const { return _followLocation; } + auto requestCallLogLevel() const { return _requestCallLogLevel; } + auto requestAnswerLogLevel() const { return _requestAnswerLogLevel; } - log::level::level_enum requestCallLogLevel() const { return _requestCallLogLevel; } - log::level::level_enum requestAnswerLogLevel() const { return _requestAnswerLogLevel; } + auto tooManyErrorsPolicy() const { return _tooManyErrorsPolicy; } + + auto nbMaxRetries() const { return _nbMaxRetries; } class Builder { public: @@ -72,14 +81,25 @@ class PermanentCurlOptions { return *this; } + Builder &setNbMaxRetries(int nbMaxRetries) { + _nbMaxRetries = nbMaxRetries; + return *this; + } + Builder &setFollowLocation() { _followLocation = true; return *this; } + Builder &setTooManyErrorsPolicy(TooManyErrorsPolicy tooManyErrorsPolicy) { + _tooManyErrorsPolicy = tooManyErrorsPolicy; + return *this; + } + PermanentCurlOptions build() { return {std::move(_userAgent), std::move(_acceptedEncoding), _minDurationBetweenQueries, - _requestCallLogLevel, _requestAnswerLogLevel, _followLocation}; + _requestCallLogLevel, _requestAnswerLogLevel, _nbMaxRetries, + _followLocation, _tooManyErrorsPolicy}; } private: @@ -88,26 +108,32 @@ class PermanentCurlOptions { Duration _minDurationBetweenQueries{}; log::level::level_enum _requestCallLogLevel = log::level::level_enum::info; log::level::level_enum _requestAnswerLogLevel = log::level::level_enum::trace; + int _nbMaxRetries = kDefaultNbMaxRetries; bool _followLocation = false; + TooManyErrorsPolicy _tooManyErrorsPolicy = TooManyErrorsPolicy::kThrow; }; private: PermanentCurlOptions(string userAgent, string acceptedEncoding, Duration minDurationBetweenQueries, log::level::level_enum requestCallLogLevel, log::level::level_enum requestAnswerLogLevel, - bool followLocation) + int nbMaxRetries, bool followLocation, TooManyErrorsPolicy tooManyErrorsPolicy) : _userAgent(std::move(userAgent)), _acceptedEncoding(std::move(acceptedEncoding)), _minDurationBetweenQueries(minDurationBetweenQueries), _requestCallLogLevel(requestCallLogLevel), _requestAnswerLogLevel(requestAnswerLogLevel), - _followLocation(followLocation) {} + _nbMaxRetries(nbMaxRetries), + _followLocation(followLocation), + _tooManyErrorsPolicy(tooManyErrorsPolicy) {} string _userAgent; string _acceptedEncoding; Duration _minDurationBetweenQueries; log::level::level_enum _requestCallLogLevel; log::level::level_enum _requestAnswerLogLevel; + int _nbMaxRetries; bool _followLocation; + TooManyErrorsPolicy _tooManyErrorsPolicy; }; } // namespace cct \ No newline at end of file diff --git a/src/http-request/src/curlhandle.cpp b/src/http-request/src/curlhandle.cpp index ca21e653..1309ecb7 100644 --- a/src/http-request/src/curlhandle.cpp +++ b/src/http-request/src/curlhandle.cpp @@ -31,6 +31,7 @@ #include "proxy.hpp" #include "runmodes.hpp" #include "timedef.hpp" +#include "unreachable.hpp" namespace cct { @@ -81,7 +82,9 @@ CurlHandle::CurlHandle(const BestURLPicker &bestURLPicker, AbstractMetricGateway _minDurationBetweenQueries(permanentCurlOptions.minDurationBetweenQueries()), _bestUrlPicker(bestURLPicker), _requestCallLogLevel(permanentCurlOptions.requestCallLogLevel()), - _requestAnswerLogLevel(permanentCurlOptions.requestAnswerLogLevel()) { + _requestAnswerLogLevel(permanentCurlOptions.requestAnswerLogLevel()), + _nbMaxRetries(permanentCurlOptions.nbMaxRetries()), + _tooManyErrorsPolicy(permanentCurlOptions.tooManyErrorsPolicy()) { if (settings::AreQueryResponsesOverriden(runMode)) { _handle = nullptr; } else { @@ -237,7 +240,6 @@ std::string_view CurlHandle::query(std::string_view endpoint, const CurlOptions optsStr); // Actually make the query, with a fast retry mechanism - static constexpr int kNbMaxRetries = 5; Duration sleepingTime = std::chrono::milliseconds(100); int retryPos = 0; CURLcode res; @@ -247,7 +249,7 @@ std::string_view CurlHandle::query(std::string_view endpoint, const CurlOptions _pMetricGateway->add(MetricType::kCounter, MetricOperation::kIncrement, CurlMetrics::kNbRequestErrorKeys.find(opts.requestType())->second); } - log::error("Got curl error ({}), retry {}/{} after {}", static_cast(res), retryPos, kNbMaxRetries, + log::error("Got curl error ({}), retry {}/{} after {}", static_cast(res), retryPos, _nbMaxRetries, DurationToString(sleepingTime)); std::this_thread::sleep_for(sleepingTime); sleepingTime *= 2; @@ -275,9 +277,18 @@ std::string_view CurlHandle::query(std::string_view endpoint, const CurlOptions _queryData.shrink_to_fit(); } - } while (res != CURLE_OK && ++retryPos <= kNbMaxRetries); - if (retryPos > kNbMaxRetries) { - throw exception("Too many errors from curl, last ({})", static_cast(res)); + } while (res != CURLE_OK && ++retryPos <= _nbMaxRetries); + if (retryPos > _nbMaxRetries) { + switch (_tooManyErrorsPolicy) { + case PermanentCurlOptions::TooManyErrorsPolicy::kReturnEmptyResponse: + log::error("Too many errors from curl, return empty response"); + _queryData.clear(); + break; + case PermanentCurlOptions::TooManyErrorsPolicy::kThrow: + throw exception("Too many errors from curl, last ({})", static_cast(res)); + default: + unreachable(); + } } // Avoid polluting the logs for large response which are more likely to be HTML @@ -306,6 +317,27 @@ void CurlHandle::setOverridenQueryResponses(const std::map &quer _queryData = string(flatQueryResponses.str()); } +void CurlHandle::swap(CurlHandle &rhs) noexcept { + using std::swap; + + swap(_handle, rhs._handle); + swap(_pMetricGateway, rhs._pMetricGateway); + swap(_minDurationBetweenQueries, rhs._minDurationBetweenQueries); + swap(_lastQueryTime, rhs._lastQueryTime); + swap(_bestUrlPicker, rhs._bestUrlPicker); + _queryData.swap(rhs._queryData); + swap(_requestCallLogLevel, rhs._requestCallLogLevel); + swap(_requestAnswerLogLevel, rhs._requestAnswerLogLevel); + swap(_tooManyErrorsPolicy, rhs._tooManyErrorsPolicy); +} + +CurlHandle::CurlHandle(CurlHandle &&rhs) noexcept { swap(rhs); } + +CurlHandle &CurlHandle::operator=(CurlHandle &&rhs) noexcept { + swap(rhs); + return *this; +} + CurlHandle::~CurlHandle() { if (_handle != nullptr) { curl_easy_cleanup(reinterpret_cast(_handle)); diff --git a/src/objects/include/loadconfiguration.hpp b/src/objects/include/loadconfiguration.hpp index ce534a4d..b4948ec0 100644 --- a/src/objects/include/loadconfiguration.hpp +++ b/src/objects/include/loadconfiguration.hpp @@ -18,12 +18,12 @@ class LoadConfiguration { std::string_view exchangeConfigFileName() const; - ExchangeConfigFileType exchangeConfigFileType() const { return _exchangeConfigFileType; } + ExchangeConfigFileType exchangeConfigFileType() const { return _exchangeInfoFileType; } private: static constexpr std::string_view kTestDefaultExchangeConfigFile = "exchangeconfig_test.json"; std::string_view _dataDir; - ExchangeConfigFileType _exchangeConfigFileType; + ExchangeConfigFileType _exchangeInfoFileType; }; } // namespace cct \ No newline at end of file diff --git a/src/objects/src/loadconfiguration.cpp b/src/objects/src/loadconfiguration.cpp index 405da51c..8939ab9c 100644 --- a/src/objects/src/loadconfiguration.cpp +++ b/src/objects/src/loadconfiguration.cpp @@ -6,13 +6,13 @@ namespace cct { LoadConfiguration::LoadConfiguration() noexcept - : _dataDir(kDefaultDataDir), _exchangeConfigFileType(ExchangeConfigFileType::kProd) {} + : _dataDir(kDefaultDataDir), _exchangeInfoFileType(ExchangeConfigFileType::kProd) {} LoadConfiguration::LoadConfiguration(std::string_view dataDir, ExchangeConfigFileType exchangeConfigFileType) - : _dataDir(dataDir), _exchangeConfigFileType(exchangeConfigFileType) {} + : _dataDir(dataDir), _exchangeInfoFileType(exchangeConfigFileType) {} std::string_view LoadConfiguration::exchangeConfigFileName() const { - return _exchangeConfigFileType == ExchangeConfigFileType::kProd ? kProdDefaultExchangeConfigFile - : kTestDefaultExchangeConfigFile; + return _exchangeInfoFileType == ExchangeConfigFileType::kProd ? kProdDefaultExchangeConfigFile + : kTestDefaultExchangeConfigFile; } } // namespace cct \ No newline at end of file