Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add retry mechanism to requests #496

Merged
merged 1 commit into from
Jan 12, 2024
Merged

Add retry mechanism to requests #496

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions src/api/common/src/fiatconverter.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,11 +80,12 @@ void FiatConverter::updateCacheFile() const {
std::optional<double> FiatConverter::queryCurrencyRate(Market mk) {
string qStr(mk.assetsPairStrUpper('_'));
CurlOptions opts(HttpRequestType::kGet, {{"q", qStr}, {"apiKey", _apiKey}});
std::string_view dataStr = _curlHandle.query("/api/v7/convert", opts);
auto dataStr = _curlHandle.query("/api/v7/convert", opts);
static constexpr bool kAllowExceptions = false;
json data = json::parse(dataStr, nullptr, kAllowExceptions);
auto data = json::parse(dataStr, nullptr, kAllowExceptions);
//{"query":{"count":1},"results":{"EUR_KRW":{"id":"EUR_KRW","val":1329.475323,"to":"KRW","fr":"EUR"}}}
if (data == json::value_t::discarded || !data.contains("results") || !data["results"].contains(qStr)) {
auto resultsIt = data.find("results");
if (data == json::value_t::discarded || resultsIt == data.end() || !resultsIt->contains(qStr)) {
log::error("No JSON data received from fiat currency converter service for pair '{}'", mk);
auto it = _pricesMap.find(mk);
if (it != _pricesMap.end()) {
Expand All @@ -95,8 +96,7 @@ std::optional<double> FiatConverter::queryCurrencyRate(Market mk) {
}
return std::nullopt;
}
const json& res = data["results"];
const json& rates = res[qStr];
const auto& rates = (*resultsIt)[qStr];
double rate = rates["val"];
log::debug("Stored rate {} for market {}", rate, qStr);
TimePoint nowTime = Clock::now();
Expand Down
2 changes: 1 addition & 1 deletion src/api/common/test/fiatconverter_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ std::string_view CurlHandle::query([[maybe_unused]] std::string_view endpoint, c
json jsonData;

// Rates
std::string_view marketStr = opts.getPostData().get("q");
std::string_view marketStr = opts.postData().get("q");
std::string_view fromCurrency = marketStr.substr(0, 3);
std::string_view targetCurrency = marketStr.substr(4);
double rate = 0;
Expand Down
2 changes: 1 addition & 1 deletion src/api/exchanges/src/binanceprivateapi.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ json PrivateQuery(CurlHandle& curlHandle, const APIKey& apiKey, HttpRequestType
std::this_thread::sleep_for(sleepingTime);
sleepingTime = (3 * sleepingTime) / 2;
}
SetNonceAndSignature(apiKey, opts.getPostData(), queryDelay);
SetNonceAndSignature(apiKey, opts.mutablePostData(), queryDelay);
ret = json::parse(curlHandle.query(endpoint, opts));

auto codeIt = ret.find("code");
Expand Down
79 changes: 49 additions & 30 deletions src/api/exchanges/src/binancepublicapi.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
#include "exchangepublicapitypes.hpp"
#include "fiatconverter.hpp"
#include "httprequesttype.hpp"
#include "invariant-request-retry.hpp"
#include "market.hpp"
#include "marketorderbook.hpp"
#include "monetaryamount.hpp"
Expand All @@ -52,15 +53,18 @@ json PublicQuery(CurlHandle& curlHandle, std::string_view method, const CurlPost
endpoint.push_back('?');
endpoint.append(curlPostData.str());
}
json ret = json::parse(curlHandle.query(endpoint, CurlOptions(HttpRequestType::kGet)));
auto foundErrorIt = ret.find("code");
auto foundMsgIt = ret.find("msg");
if (foundErrorIt != ret.end() && foundMsgIt != ret.end()) {
const int statusCode = foundErrorIt->get<int>(); // "1100" for instance
log::error("Full Binance json error: '{}'", ret.dump());
throw exception("Error: {}, msg: ", MonetaryAmount(statusCode), foundMsgIt->get<std::string_view>());
}
return ret;
InvariantRequestRetry requestRetry(curlHandle, endpoint, CurlOptions(HttpRequestType::kGet));

return requestRetry.queryJson([](const json& jsonResponse) {
const auto foundErrorIt = jsonResponse.find("code");
const auto foundMsgIt = jsonResponse.find("msg");
if (foundErrorIt != jsonResponse.end() && foundMsgIt != jsonResponse.end()) {
const int statusCode = foundErrorIt->get<int>(); // "1100" for instance
log::warn("Binance error ({}), full json: '{}'", statusCode, jsonResponse.dump());
return InvariantRequestRetry::Status::kResponseError;
}
return InvariantRequestRetry::Status::kResponseOK;
});
}

template <class ExchangeInfoDataByMarket>
Expand Down Expand Up @@ -186,9 +190,12 @@ MarketSet BinancePublic::MarketsFunc::operator()() {
BinancePublic::ExchangeInfoFunc::ExchangeInfoDataByMarket BinancePublic::ExchangeInfoFunc::operator()() {
ExchangeInfoDataByMarket ret;
json exchangeInfoData = PublicQuery(_commonInfo._curlHandle, "/api/v3/exchangeInfo");
json& symbols = exchangeInfoData["symbols"];
for (auto it = std::make_move_iterator(symbols.begin()), endIt = std::make_move_iterator(symbols.end()); it != endIt;
++it) {
auto symbolsIt = exchangeInfoData.find("symbols");
if (symbolsIt == exchangeInfoData.end()) {
return ret;
}
for (auto it = std::make_move_iterator(symbolsIt->begin()), endIt = std::make_move_iterator(symbolsIt->end());
it != endIt; ++it) {
std::string_view baseAsset = (*it)["baseAsset"].get<std::string_view>();
std::string_view quoteAsset = (*it)["quoteAsset"].get<std::string_view>();
if ((*it)["status"].get<std::string_view>() != "TRADING") {
Expand Down Expand Up @@ -318,8 +325,11 @@ MonetaryAmount BinancePublic::computePriceForNotional(Market mk, int avgPriceMin
log::error("Unable to retrieve last trades from {}, use average price instead for notional", mk);
}

json result = PublicQuery(_commonInfo._curlHandle, "/api/v3/avgPrice", {{"symbol", mk.assetsPairStrUpper()}});
return {result["price"].get<std::string_view>(), mk.quote()};
const json result = PublicQuery(_commonInfo._curlHandle, "/api/v3/avgPrice", {{"symbol", mk.assetsPairStrUpper()}});
const auto priceIt = result.find("price");
const std::string_view priceStr = priceIt == result.end() ? std::string_view() : priceIt->get<std::string_view>();

return {priceStr, mk.quote()};
}

MonetaryAmount BinancePublic::sanitizeVolume(Market mk, MonetaryAmount vol, MonetaryAmount priceForNotional,
Expand Down Expand Up @@ -472,28 +482,36 @@ MarketOrderBook BinancePublic::OrderBookFunc::operator()(Market mk, int depth) {
lb = std::next(kAuthorizedDepths.end(), -1);
log::error("Invalid depth {}, default to {}", depth, *lb);
}
CurlPostData postData{{"symbol", mk.assetsPairStrUpper()}, {"limit", *lb}};
json asksAndBids = PublicQuery(_commonInfo._curlHandle, "/api/v3/depth", postData);
const json& asks = asksAndBids["asks"];
const json& bids = asksAndBids["bids"];
using OrderBookVec = vector<OrderBookLine>;
OrderBookVec orderBookLines;
orderBookLines.reserve(static_cast<OrderBookVec::size_type>(asks.size() + bids.size()));
for (auto asksOrBids : {std::addressof(asks), std::addressof(bids)}) {
const bool isAsk = asksOrBids == std::addressof(asks);
for (const auto& priceQuantityPair : *asksOrBids) {
MonetaryAmount amount(priceQuantityPair.back().get<std::string_view>(), mk.base());
MonetaryAmount price(priceQuantityPair.front().get<std::string_view>(), mk.quote());

orderBookLines.emplace_back(amount, price, isAsk);

CurlPostData postData{{"symbol", mk.assetsPairStrUpper()}, {"limit", *lb}};
json asksAndBids = PublicQuery(_commonInfo._curlHandle, "/api/v3/depth", postData);
const auto asksIt = asksAndBids.find("asks");
const auto bidsIt = asksAndBids.find("bids");

if (asksIt != asksAndBids.end() && bidsIt != asksAndBids.end()) {
orderBookLines.reserve(static_cast<OrderBookVec::size_type>(asksIt->size() + bidsIt->size()));
for (const auto& asksOrBids : {asksIt, bidsIt}) {
const bool isAsk = asksOrBids == asksIt;
for (const auto& priceQuantityPair : *asksOrBids) {
MonetaryAmount amount(priceQuantityPair.back().get<std::string_view>(), mk.base());
MonetaryAmount price(priceQuantityPair.front().get<std::string_view>(), mk.quote());

orderBookLines.emplace_back(amount, price, isAsk);
}
}
}

return MarketOrderBook(mk, orderBookLines);
}

MonetaryAmount BinancePublic::TradedVolumeFunc::operator()(Market mk) {
json result = PublicQuery(_commonInfo._curlHandle, "/api/v3/ticker/24hr", {{"symbol", mk.assetsPairStrUpper()}});
std::string_view last24hVol = result["volume"].get<std::string_view>();
const json result =
PublicQuery(_commonInfo._curlHandle, "/api/v3/ticker/24hr", {{"symbol", mk.assetsPairStrUpper()}});
const auto volumeIt = result.find("volume");
const std::string_view last24hVol = volumeIt == result.end() ? std::string_view() : volumeIt->get<std::string_view>();

return {last24hVol, mk.base()};
}

Expand All @@ -520,8 +538,9 @@ LastTradesVector BinancePublic::queryLastTrades(Market mk, int nbTrades) {
}

MonetaryAmount BinancePublic::TickerFunc::operator()(Market mk) {
json result = PublicQuery(_commonInfo._curlHandle, "/api/v3/ticker/price", {{"symbol", mk.assetsPairStrUpper()}});
std::string_view lastPrice = result["price"].get<std::string_view>();
const json data = PublicQuery(_commonInfo._curlHandle, "/api/v3/ticker/price", {{"symbol", mk.assetsPairStrUpper()}});
const auto priceIt = data.find("price");
const std::string_view lastPrice = priceIt == data.end() ? std::string_view() : priceIt->get<std::string_view>();
return {lastPrice, mk.quote()};
}

Expand Down
2 changes: 1 addition & 1 deletion src/api/exchanges/src/bithumbprivateapi.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ void SetHttpHeaders(CurlOptions& opts, const APIKey& apiKey, std::string_view si
}

json PrivateQueryProcess(CurlHandle& curlHandle, const APIKey& apiKey, std::string_view endpoint, CurlOptions& opts) {
auto strDataAndNoncePair = GetStrData(endpoint, opts.getPostData().str());
auto strDataAndNoncePair = GetStrData(endpoint, opts.postData().str());

string signature = B64Encode(ssl::ShaHex(ssl::ShaType::kSha512, strDataAndNoncePair.first, apiKey.privateKey()));

Expand Down
161 changes: 88 additions & 73 deletions src/api/exchanges/src/bithumbpublicapi.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
#include "exchangepublicapitypes.hpp"
#include "fiatconverter.hpp"
#include "httprequesttype.hpp"
#include "invariant-request-retry.hpp"
#include "market.hpp"
#include "marketorderbook.hpp"
#include "monetaryamount.hpp"
Expand All @@ -46,7 +47,6 @@ namespace {
json PublicQuery(CurlHandle& curlHandle, std::string_view endpoint, CurrencyCode base,
CurrencyCode quote = CurrencyCode(), std::string_view urlOpts = "") {
string methodUrl(endpoint);
methodUrl.push_back('/');
base.appendStrTo(methodUrl);
if (!quote.isNeutral()) {
methodUrl.push_back('_');
Expand All @@ -57,18 +57,26 @@ json PublicQuery(CurlHandle& curlHandle, std::string_view endpoint, CurrencyCode
methodUrl.append(urlOpts);
}

json ret = json::parse(curlHandle.query(methodUrl, CurlOptions(HttpRequestType::kGet)));
auto errorIt = ret.find("status");
if (errorIt != ret.end()) {
std::string_view statusCode = errorIt->get<std::string_view>(); // "5300" for instance
if (statusCode != BithumbPublic::kStatusOKStr) { // "0000" stands for: request OK
log::error("Full Bithumb json error: '{}'", ret.dump());
auto msgIt = ret.find("message");
throw exception("Bithumb error: {}, msg: {}", statusCode,
msgIt != ret.end() ? msgIt->get<std::string_view>() : "null");
InvariantRequestRetry requestRetry(curlHandle, methodUrl, CurlOptions(HttpRequestType::kGet));

json jsonResponse = requestRetry.queryJson([](const json& jsonResponse) {
const auto errorIt = jsonResponse.find("status");
if (errorIt != jsonResponse.end()) {
const std::string_view statusCode = errorIt->get<std::string_view>(); // "5300" for instance
if (statusCode != BithumbPublic::kStatusOKStr) { // "0000" stands for: request OK
log::warn("Full Bithumb json error ({}): '{}'", statusCode, jsonResponse.dump());
return InvariantRequestRetry::Status::kResponseError;
}
}
return InvariantRequestRetry::Status::kResponseOK;
});

const auto dataIt = jsonResponse.find("data");
json ret;
if (dataIt != jsonResponse.end()) {
ret.swap(*dataIt);
}
return ret["data"];
return ret;
}

} // namespace
Expand Down Expand Up @@ -193,7 +201,7 @@ MonetaryAmountByCurrencySet BithumbPublic::WithdrawalFeesFunc::operator()() {
}

CurrencyExchangeFlatSet BithumbPublic::TradableCurrenciesFunc::operator()() {
json result = PublicQuery(_curlHandle, "/public/assetsstatus", "all");
json result = PublicQuery(_curlHandle, "/public/assetsstatus/", "all");
CurrencyExchangeVector currencies;
currencies.reserve(static_cast<CurrencyExchangeVector::size_type>(result.size() + 1));
for (const auto& [asset, withdrawalDeposit] : result.items()) {
Expand Down Expand Up @@ -236,61 +244,62 @@ MarketOrderBookMap GetOrderbooks(CurlHandle& curlHandle, const CoincenterInfo& c
AppendString(urlOpts, *optDepth);
}

json result = PublicQuery(curlHandle, "/public/orderbook", base, quote, urlOpts);

// Note: as of 2021-02-24, Bithumb payment currency is always KRW. Format of json may change once it's not the case
// anymore
std::string_view quoteCurrency = result["payment_currency"].get<std::string_view>();
if (quoteCurrency != "KRW") {
log::error("Unexpected Bithumb reply for orderbook. May require code api update");
}
CurrencyCode quoteCurrencyCode(config.standardizeCurrencyCode(quoteCurrency));
const CurrencyCodeSet& excludedCurrencies = exchangeInfo.excludedCurrenciesAll();
for (const auto& [baseOrSpecial, asksAndBids] : result.items()) {
if (baseOrSpecial != "payment_currency" && baseOrSpecial != "timestamp") {
const json* asksBids[2];
CurrencyCode baseCurrencyCode;
if (singleMarketQuote && baseOrSpecial == "order_currency") {
// single market quote
baseCurrencyCode = base;
asksBids[0] = std::addressof(result["asks"]);
asksBids[1] = std::addressof(result["bids"]);
} else if (!singleMarketQuote) {
// then it's a base currency
baseCurrencyCode = config.standardizeCurrencyCode(baseOrSpecial);
if (excludedCurrencies.contains(baseCurrencyCode)) {
// Forbidden currency, do not consider its market
log::trace("Discard {} excluded by config", baseCurrencyCode);
json result = PublicQuery(curlHandle, "/public/orderbook/", base, quote, urlOpts);
if (!result.empty()) {
// Note: as of 2021-02-24, Bithumb payment currency is always KRW. Format of json may change once it's not the case
// anymore
std::string_view quoteCurrency = result["payment_currency"].get<std::string_view>();
if (quoteCurrency != "KRW") {
log::error("Unexpected Bithumb reply for orderbook. May require code api update");
}
CurrencyCode quoteCurrencyCode(config.standardizeCurrencyCode(quoteCurrency));
const CurrencyCodeSet& excludedCurrencies = exchangeInfo.excludedCurrenciesAll();
for (const auto& [baseOrSpecial, asksAndBids] : result.items()) {
if (baseOrSpecial != "payment_currency" && baseOrSpecial != "timestamp") {
const json* asksBids[2];
CurrencyCode baseCurrencyCode;
if (singleMarketQuote && baseOrSpecial == "order_currency") {
// single market quote
baseCurrencyCode = base;
asksBids[0] = std::addressof(result["asks"]);
asksBids[1] = std::addressof(result["bids"]);
} else if (!singleMarketQuote) {
// then it's a base currency
baseCurrencyCode = config.standardizeCurrencyCode(baseOrSpecial);
if (excludedCurrencies.contains(baseCurrencyCode)) {
// Forbidden currency, do not consider its market
log::trace("Discard {} excluded by config", baseCurrencyCode);
continue;
}
asksBids[0] = std::addressof(asksAndBids["asks"]);
asksBids[1] = std::addressof(asksAndBids["bids"]);
} else {
continue;
}
asksBids[0] = std::addressof(asksAndBids["asks"]);
asksBids[1] = std::addressof(asksAndBids["bids"]);
} else {
continue;
}

/*
"bids": [{"quantity" : "6.1189306","price" : "504000"},
{"quantity" : "10.35117828","price" : "503000"}],
"asks": [{"quantity" : "2.67575", "price" : "506000"},
{"quantity" : "3.54343","price" : "507000"}]
*/
using OrderBookVec = vector<OrderBookLine>;
OrderBookVec orderBookLines;
orderBookLines.reserve(static_cast<OrderBookVec::size_type>(asksBids[0]->size() + asksBids[1]->size()));
for (const json* asksOrBids : asksBids) {
const bool isAsk = asksOrBids == asksBids[0];
for (const json& priceQuantityPair : *asksOrBids) {
MonetaryAmount amount(priceQuantityPair["quantity"].get<std::string_view>(), baseCurrencyCode);
MonetaryAmount price(priceQuantityPair["price"].get<std::string_view>(), quoteCurrencyCode);
/*
"bids": [{"quantity" : "6.1189306","price" : "504000"},
{"quantity" : "10.35117828","price" : "503000"}],
"asks": [{"quantity" : "2.67575", "price" : "506000"},
{"quantity" : "3.54343","price" : "507000"}]
*/
using OrderBookVec = vector<OrderBookLine>;
OrderBookVec orderBookLines;
orderBookLines.reserve(static_cast<OrderBookVec::size_type>(asksBids[0]->size() + asksBids[1]->size()));
for (const json* asksOrBids : asksBids) {
const bool isAsk = asksOrBids == asksBids[0];
for (const json& priceQuantityPair : *asksOrBids) {
MonetaryAmount amount(priceQuantityPair["quantity"].get<std::string_view>(), baseCurrencyCode);
MonetaryAmount price(priceQuantityPair["price"].get<std::string_view>(), quoteCurrencyCode);

orderBookLines.emplace_back(amount, price, isAsk);
orderBookLines.emplace_back(amount, price, isAsk);
}
}
Market market(baseCurrencyCode, quoteCurrencyCode);
ret.insert_or_assign(market, MarketOrderBook(market, orderBookLines));
if (singleMarketQuote) {
break;
}
}
Market market(baseCurrencyCode, quoteCurrencyCode);
ret.insert_or_assign(market, MarketOrderBook(market, orderBookLines));
if (singleMarketQuote) {
break;
}
}
}
Expand All @@ -314,16 +323,21 @@ MarketOrderBook BithumbPublic::OrderBookFunc::operator()(Market mk, int depth) {

MonetaryAmount BithumbPublic::TradedVolumeFunc::operator()(Market mk) {
TimePoint t1 = Clock::now();
json result = PublicQuery(_curlHandle, "/public/ticker", mk.base(), mk.quote());
std::string_view last24hVol = result["units_traded_24H"].get<std::string_view>();
std::string_view bithumbTimestamp = result["date"].get<std::string_view>();
int64_t bithumbTimeMs = FromString<int64_t>(bithumbTimestamp);
int64_t t1Ms = TimestampToMs(t1);
int64_t t2Ms = TimestampToMs(Clock::now());
if (t1Ms < bithumbTimeMs && bithumbTimeMs < t2Ms) {
log::debug("Bithumb time is synchronized with us");
} else {
log::error("Bithumb time is not synchronized with us (Bithumb: {}, us: [{} - {}])", bithumbTimestamp, t1Ms, t2Ms);
json result = PublicQuery(_curlHandle, "/public/ticker/", mk.base(), mk.quote());
std::string_view last24hVol;
const auto dateIt = result.find("date");
if (dateIt != result.end()) {
std::string_view bithumbTimestamp = dateIt->get<std::string_view>();

last24hVol = result["units_traded_24H"].get<std::string_view>();
int64_t bithumbTimeMs = FromString<int64_t>(bithumbTimestamp);
int64_t t1Ms = TimestampToMs(t1);
int64_t t2Ms = TimestampToMs(Clock::now());
if (t1Ms < bithumbTimeMs && bithumbTimeMs < t2Ms) {
log::debug("Bithumb time is synchronized with us");
} else {
log::error("Bithumb time is not synchronized with us (Bithumb: {}, us: [{} - {}])", bithumbTimestamp, t1Ms, t2Ms);
}
}

return {last24hVol, mk.base()};
Expand All @@ -340,8 +354,9 @@ TimePoint EpochTime(std::string&& dateStr) {
} // namespace

LastTradesVector BithumbPublic::queryLastTrades(Market mk, [[maybe_unused]] int nbTrades) {
json result = PublicQuery(_curlHandle, "/public/transaction_history", mk.base(), mk.quote());
json result = PublicQuery(_curlHandle, "/public/transaction_history/", mk.base(), mk.quote());
LastTradesVector ret;
ret.reserve(result.size());
for (const json& detail : result) {
MonetaryAmount amount(detail["units_traded"].get<std::string_view>(), mk.base());
MonetaryAmount price(detail["price"].get<std::string_view>(), mk.quote());
Expand Down
Loading
Loading