Skip to content

Commit

Permalink
Add retry mechanism to requests
Browse files Browse the repository at this point in the history
  • Loading branch information
@sjanel
sjanel committed Jan 12, 2024
1 parent c8b734c commit 292c2e9
Show file tree
Hide file tree
Showing 24 changed files with 480 additions and 290 deletions.
10 changes: 5 additions & 5 deletions src/api/common/src/fiatconverter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,11 +80,12 @@ void FiatConverter::updateCacheFile() const {
std::optional<double> FiatConverter::queryCurrencyRate(Market mk) {
string qStr(mk.assetsPairStrUpper('_'));
CurlOptions opts(HttpRequestType::kGet, {{"q", qStr}, {"apiKey", _apiKey}});
std::string_view dataStr = _curlHandle.query("/api/v7/convert", opts);
auto dataStr = _curlHandle.query("/api/v7/convert", opts);
static constexpr bool kAllowExceptions = false;
json data = json::parse(dataStr, nullptr, kAllowExceptions);
auto data = json::parse(dataStr, nullptr, kAllowExceptions);
//{"query":{"count":1},"results":{"EUR_KRW":{"id":"EUR_KRW","val":1329.475323,"to":"KRW","fr":"EUR"}}}
if (data == json::value_t::discarded || !data.contains("results") || !data["results"].contains(qStr)) {
auto resultsIt = data.find("results");
if (data == json::value_t::discarded || resultsIt == data.end() || !resultsIt->contains(qStr)) {
log::error("No JSON data received from fiat currency converter service for pair '{}'", mk);
auto it = _pricesMap.find(mk);
if (it != _pricesMap.end()) {
Expand All @@ -95,8 +96,7 @@ std::optional<double> FiatConverter::queryCurrencyRate(Market mk) {
}
return std::nullopt;
}
const json& res = data["results"];
const json& rates = res[qStr];
const auto& rates = (*resultsIt)[qStr];
double rate = rates["val"];
log::debug("Stored rate {} for market {}", rate, qStr);
TimePoint nowTime = Clock::now();
Expand Down
2 changes: 1 addition & 1 deletion src/api/common/test/fiatconverter_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ std::string_view CurlHandle::query([[maybe_unused]] std::string_view endpoint, c
json jsonData;

// Rates
std::string_view marketStr = opts.getPostData().get("q");
std::string_view marketStr = opts.postData().get("q");
std::string_view fromCurrency = marketStr.substr(0, 3);
std::string_view targetCurrency = marketStr.substr(4);
double rate = 0;
Expand Down
2 changes: 1 addition & 1 deletion src/api/exchanges/src/binanceprivateapi.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ json PrivateQuery(CurlHandle& curlHandle, const APIKey& apiKey, HttpRequestType
std::this_thread::sleep_for(sleepingTime);
sleepingTime = (3 * sleepingTime) / 2;
}
SetNonceAndSignature(apiKey, opts.getPostData(), queryDelay);
SetNonceAndSignature(apiKey, opts.mutablePostData(), queryDelay);
ret = json::parse(curlHandle.query(endpoint, opts));

auto codeIt = ret.find("code");
Expand Down
79 changes: 49 additions & 30 deletions src/api/exchanges/src/binancepublicapi.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
#include "exchangepublicapitypes.hpp"
#include "fiatconverter.hpp"
#include "httprequesttype.hpp"
#include "invariant-request-retry.hpp"
#include "market.hpp"
#include "marketorderbook.hpp"
#include "monetaryamount.hpp"
Expand All @@ -52,15 +53,18 @@ json PublicQuery(CurlHandle& curlHandle, std::string_view method, const CurlPost
endpoint.push_back('?');
endpoint.append(curlPostData.str());
}
json ret = json::parse(curlHandle.query(endpoint, CurlOptions(HttpRequestType::kGet)));
auto foundErrorIt = ret.find("code");
auto foundMsgIt = ret.find("msg");
if (foundErrorIt != ret.end() && foundMsgIt != ret.end()) {
const int statusCode = foundErrorIt->get<int>(); // "1100" for instance
log::error("Full Binance json error: '{}'", ret.dump());
throw exception("Error: {}, msg: ", MonetaryAmount(statusCode), foundMsgIt->get<std::string_view>());
}
return ret;
InvariantRequestRetry requestRetry(curlHandle, endpoint, CurlOptions(HttpRequestType::kGet));

return requestRetry.queryJson([](const json& jsonResponse) {
const auto foundErrorIt = jsonResponse.find("code");
const auto foundMsgIt = jsonResponse.find("msg");
if (foundErrorIt != jsonResponse.end() && foundMsgIt != jsonResponse.end()) {
const int statusCode = foundErrorIt->get<int>(); // "1100" for instance
log::warn("Binance error ({}), full json: '{}'", statusCode, jsonResponse.dump());
return InvariantRequestRetry::Status::kResponseError;
}
return InvariantRequestRetry::Status::kResponseOK;
});
}

template <class ExchangeInfoDataByMarket>
Expand Down Expand Up @@ -186,9 +190,12 @@ MarketSet BinancePublic::MarketsFunc::operator()() {
BinancePublic::ExchangeInfoFunc::ExchangeInfoDataByMarket BinancePublic::ExchangeInfoFunc::operator()() {
ExchangeInfoDataByMarket ret;
json exchangeInfoData = PublicQuery(_commonInfo._curlHandle, "/api/v3/exchangeInfo");
json& symbols = exchangeInfoData["symbols"];
for (auto it = std::make_move_iterator(symbols.begin()), endIt = std::make_move_iterator(symbols.end()); it != endIt;
++it) {
auto symbolsIt = exchangeInfoData.find("symbols");
if (symbolsIt == exchangeInfoData.end()) {
return ret;
}
for (auto it = std::make_move_iterator(symbolsIt->begin()), endIt = std::make_move_iterator(symbolsIt->end());
it != endIt; ++it) {
std::string_view baseAsset = (*it)["baseAsset"].get<std::string_view>();
std::string_view quoteAsset = (*it)["quoteAsset"].get<std::string_view>();
if ((*it)["status"].get<std::string_view>() != "TRADING") {
Expand Down Expand Up @@ -318,8 +325,11 @@ MonetaryAmount BinancePublic::computePriceForNotional(Market mk, int avgPriceMin
log::error("Unable to retrieve last trades from {}, use average price instead for notional", mk);
}

json result = PublicQuery(_commonInfo._curlHandle, "/api/v3/avgPrice", {{"symbol", mk.assetsPairStrUpper()}});
return {result["price"].get<std::string_view>(), mk.quote()};
const json result = PublicQuery(_commonInfo._curlHandle, "/api/v3/avgPrice", {{"symbol", mk.assetsPairStrUpper()}});
const auto priceIt = result.find("price");
const std::string_view priceStr = priceIt == result.end() ? std::string_view() : priceIt->get<std::string_view>();

return {priceStr, mk.quote()};
}

MonetaryAmount BinancePublic::sanitizeVolume(Market mk, MonetaryAmount vol, MonetaryAmount priceForNotional,
Expand Down Expand Up @@ -472,28 +482,36 @@ MarketOrderBook BinancePublic::OrderBookFunc::operator()(Market mk, int depth) {
lb = std::next(kAuthorizedDepths.end(), -1);
log::error("Invalid depth {}, default to {}", depth, *lb);
}
CurlPostData postData{{"symbol", mk.assetsPairStrUpper()}, {"limit", *lb}};
json asksAndBids = PublicQuery(_commonInfo._curlHandle, "/api/v3/depth", postData);
const json& asks = asksAndBids["asks"];
const json& bids = asksAndBids["bids"];
using OrderBookVec = vector<OrderBookLine>;
OrderBookVec orderBookLines;
orderBookLines.reserve(static_cast<OrderBookVec::size_type>(asks.size() + bids.size()));
for (auto asksOrBids : {std::addressof(asks), std::addressof(bids)}) {
const bool isAsk = asksOrBids == std::addressof(asks);
for (const auto& priceQuantityPair : *asksOrBids) {
MonetaryAmount amount(priceQuantityPair.back().get<std::string_view>(), mk.base());
MonetaryAmount price(priceQuantityPair.front().get<std::string_view>(), mk.quote());

orderBookLines.emplace_back(amount, price, isAsk);

CurlPostData postData{{"symbol", mk.assetsPairStrUpper()}, {"limit", *lb}};
json asksAndBids = PublicQuery(_commonInfo._curlHandle, "/api/v3/depth", postData);
const auto asksIt = asksAndBids.find("asks");
const auto bidsIt = asksAndBids.find("bids");

if (asksIt != asksAndBids.end() && bidsIt != asksAndBids.end()) {
orderBookLines.reserve(static_cast<OrderBookVec::size_type>(asksIt->size() + bidsIt->size()));
for (const auto& asksOrBids : {asksIt, bidsIt}) {
const bool isAsk = asksOrBids == asksIt;
for (const auto& priceQuantityPair : *asksOrBids) {
MonetaryAmount amount(priceQuantityPair.back().get<std::string_view>(), mk.base());
MonetaryAmount price(priceQuantityPair.front().get<std::string_view>(), mk.quote());

orderBookLines.emplace_back(amount, price, isAsk);
}
}
}

return MarketOrderBook(mk, orderBookLines);
}

MonetaryAmount BinancePublic::TradedVolumeFunc::operator()(Market mk) {
json result = PublicQuery(_commonInfo._curlHandle, "/api/v3/ticker/24hr", {{"symbol", mk.assetsPairStrUpper()}});
std::string_view last24hVol = result["volume"].get<std::string_view>();
const json result =
PublicQuery(_commonInfo._curlHandle, "/api/v3/ticker/24hr", {{"symbol", mk.assetsPairStrUpper()}});
const auto volumeIt = result.find("volume");
const std::string_view last24hVol = volumeIt == result.end() ? std::string_view() : volumeIt->get<std::string_view>();

return {last24hVol, mk.base()};
}

Expand All @@ -520,8 +538,9 @@ LastTradesVector BinancePublic::queryLastTrades(Market mk, int nbTrades) {
}

MonetaryAmount BinancePublic::TickerFunc::operator()(Market mk) {
json result = PublicQuery(_commonInfo._curlHandle, "/api/v3/ticker/price", {{"symbol", mk.assetsPairStrUpper()}});
std::string_view lastPrice = result["price"].get<std::string_view>();
const json data = PublicQuery(_commonInfo._curlHandle, "/api/v3/ticker/price", {{"symbol", mk.assetsPairStrUpper()}});
const auto priceIt = data.find("price");
const std::string_view lastPrice = priceIt == data.end() ? std::string_view() : priceIt->get<std::string_view>();
return {lastPrice, mk.quote()};
}

Expand Down
2 changes: 1 addition & 1 deletion src/api/exchanges/src/bithumbprivateapi.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ void SetHttpHeaders(CurlOptions& opts, const APIKey& apiKey, std::string_view si
}

json PrivateQueryProcess(CurlHandle& curlHandle, const APIKey& apiKey, std::string_view endpoint, CurlOptions& opts) {
auto strDataAndNoncePair = GetStrData(endpoint, opts.getPostData().str());
auto strDataAndNoncePair = GetStrData(endpoint, opts.postData().str());

string signature = B64Encode(ssl::ShaHex(ssl::ShaType::kSha512, strDataAndNoncePair.first, apiKey.privateKey()));

Expand Down
161 changes: 88 additions & 73 deletions src/api/exchanges/src/bithumbpublicapi.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
#include "exchangepublicapitypes.hpp"
#include "fiatconverter.hpp"
#include "httprequesttype.hpp"
#include "invariant-request-retry.hpp"
#include "market.hpp"
#include "marketorderbook.hpp"
#include "monetaryamount.hpp"
Expand All @@ -46,7 +47,6 @@ namespace {
json PublicQuery(CurlHandle& curlHandle, std::string_view endpoint, CurrencyCode base,
CurrencyCode quote = CurrencyCode(), std::string_view urlOpts = "") {
string methodUrl(endpoint);
methodUrl.push_back('/');
base.appendStrTo(methodUrl);
if (!quote.isNeutral()) {
methodUrl.push_back('_');
Expand All @@ -57,18 +57,26 @@ json PublicQuery(CurlHandle& curlHandle, std::string_view endpoint, CurrencyCode
methodUrl.append(urlOpts);
}

json ret = json::parse(curlHandle.query(methodUrl, CurlOptions(HttpRequestType::kGet)));
auto errorIt = ret.find("status");
if (errorIt != ret.end()) {
std::string_view statusCode = errorIt->get<std::string_view>(); // "5300" for instance
if (statusCode != BithumbPublic::kStatusOKStr) { // "0000" stands for: request OK
log::error("Full Bithumb json error: '{}'", ret.dump());
auto msgIt = ret.find("message");
throw exception("Bithumb error: {}, msg: {}", statusCode,
msgIt != ret.end() ? msgIt->get<std::string_view>() : "null");
InvariantRequestRetry requestRetry(curlHandle, methodUrl, CurlOptions(HttpRequestType::kGet));

json jsonResponse = requestRetry.queryJson([](const json& jsonResponse) {
const auto errorIt = jsonResponse.find("status");
if (errorIt != jsonResponse.end()) {
const std::string_view statusCode = errorIt->get<std::string_view>(); // "5300" for instance
if (statusCode != BithumbPublic::kStatusOKStr) { // "0000" stands for: request OK
log::warn("Full Bithumb json error ({}): '{}'", statusCode, jsonResponse.dump());
return InvariantRequestRetry::Status::kResponseError;
}
}
return InvariantRequestRetry::Status::kResponseOK;
});

const auto dataIt = jsonResponse.find("data");
json ret;
if (dataIt != jsonResponse.end()) {
ret.swap(*dataIt);
}
return ret["data"];
return ret;
}

} // namespace
Expand Down Expand Up @@ -193,7 +201,7 @@ MonetaryAmountByCurrencySet BithumbPublic::WithdrawalFeesFunc::operator()() {
}

CurrencyExchangeFlatSet BithumbPublic::TradableCurrenciesFunc::operator()() {
json result = PublicQuery(_curlHandle, "/public/assetsstatus", "all");
json result = PublicQuery(_curlHandle, "/public/assetsstatus/", "all");
CurrencyExchangeVector currencies;
currencies.reserve(static_cast<CurrencyExchangeVector::size_type>(result.size() + 1));
for (const auto& [asset, withdrawalDeposit] : result.items()) {
Expand Down Expand Up @@ -236,61 +244,62 @@ MarketOrderBookMap GetOrderbooks(CurlHandle& curlHandle, const CoincenterInfo& c
AppendString(urlOpts, *optDepth);
}

json result = PublicQuery(curlHandle, "/public/orderbook", base, quote, urlOpts);

// Note: as of 2021-02-24, Bithumb payment currency is always KRW. Format of json may change once it's not the case
// anymore
std::string_view quoteCurrency = result["payment_currency"].get<std::string_view>();
if (quoteCurrency != "KRW") {
log::error("Unexpected Bithumb reply for orderbook. May require code api update");
}
CurrencyCode quoteCurrencyCode(config.standardizeCurrencyCode(quoteCurrency));
const CurrencyCodeSet& excludedCurrencies = exchangeInfo.excludedCurrenciesAll();
for (const auto& [baseOrSpecial, asksAndBids] : result.items()) {
if (baseOrSpecial != "payment_currency" && baseOrSpecial != "timestamp") {
const json* asksBids[2];
CurrencyCode baseCurrencyCode;
if (singleMarketQuote && baseOrSpecial == "order_currency") {
// single market quote
baseCurrencyCode = base;
asksBids[0] = std::addressof(result["asks"]);
asksBids[1] = std::addressof(result["bids"]);
} else if (!singleMarketQuote) {
// then it's a base currency
baseCurrencyCode = config.standardizeCurrencyCode(baseOrSpecial);
if (excludedCurrencies.contains(baseCurrencyCode)) {
// Forbidden currency, do not consider its market
log::trace("Discard {} excluded by config", baseCurrencyCode);
json result = PublicQuery(curlHandle, "/public/orderbook/", base, quote, urlOpts);
if (!result.empty()) {
// Note: as of 2021-02-24, Bithumb payment currency is always KRW. Format of json may change once it's not the case
// anymore
std::string_view quoteCurrency = result["payment_currency"].get<std::string_view>();
if (quoteCurrency != "KRW") {
log::error("Unexpected Bithumb reply for orderbook. May require code api update");
}
CurrencyCode quoteCurrencyCode(config.standardizeCurrencyCode(quoteCurrency));
const CurrencyCodeSet& excludedCurrencies = exchangeInfo.excludedCurrenciesAll();
for (const auto& [baseOrSpecial, asksAndBids] : result.items()) {
if (baseOrSpecial != "payment_currency" && baseOrSpecial != "timestamp") {
const json* asksBids[2];
CurrencyCode baseCurrencyCode;
if (singleMarketQuote && baseOrSpecial == "order_currency") {
// single market quote
baseCurrencyCode = base;
asksBids[0] = std::addressof(result["asks"]);
asksBids[1] = std::addressof(result["bids"]);
} else if (!singleMarketQuote) {
// then it's a base currency
baseCurrencyCode = config.standardizeCurrencyCode(baseOrSpecial);
if (excludedCurrencies.contains(baseCurrencyCode)) {
// Forbidden currency, do not consider its market
log::trace("Discard {} excluded by config", baseCurrencyCode);
continue;
}
asksBids[0] = std::addressof(asksAndBids["asks"]);
asksBids[1] = std::addressof(asksAndBids["bids"]);
} else {
continue;
}
asksBids[0] = std::addressof(asksAndBids["asks"]);
asksBids[1] = std::addressof(asksAndBids["bids"]);
} else {
continue;
}

/*
"bids": [{"quantity" : "6.1189306","price" : "504000"},
{"quantity" : "10.35117828","price" : "503000"}],
"asks": [{"quantity" : "2.67575", "price" : "506000"},
{"quantity" : "3.54343","price" : "507000"}]
*/
using OrderBookVec = vector<OrderBookLine>;
OrderBookVec orderBookLines;
orderBookLines.reserve(static_cast<OrderBookVec::size_type>(asksBids[0]->size() + asksBids[1]->size()));
for (const json* asksOrBids : asksBids) {
const bool isAsk = asksOrBids == asksBids[0];
for (const json& priceQuantityPair : *asksOrBids) {
MonetaryAmount amount(priceQuantityPair["quantity"].get<std::string_view>(), baseCurrencyCode);
MonetaryAmount price(priceQuantityPair["price"].get<std::string_view>(), quoteCurrencyCode);
/*
"bids": [{"quantity" : "6.1189306","price" : "504000"},
{"quantity" : "10.35117828","price" : "503000"}],
"asks": [{"quantity" : "2.67575", "price" : "506000"},
{"quantity" : "3.54343","price" : "507000"}]
*/
using OrderBookVec = vector<OrderBookLine>;
OrderBookVec orderBookLines;
orderBookLines.reserve(static_cast<OrderBookVec::size_type>(asksBids[0]->size() + asksBids[1]->size()));
for (const json* asksOrBids : asksBids) {
const bool isAsk = asksOrBids == asksBids[0];
for (const json& priceQuantityPair : *asksOrBids) {
MonetaryAmount amount(priceQuantityPair["quantity"].get<std::string_view>(), baseCurrencyCode);
MonetaryAmount price(priceQuantityPair["price"].get<std::string_view>(), quoteCurrencyCode);

orderBookLines.emplace_back(amount, price, isAsk);
orderBookLines.emplace_back(amount, price, isAsk);
}
}
Market market(baseCurrencyCode, quoteCurrencyCode);
ret.insert_or_assign(market, MarketOrderBook(market, orderBookLines));
if (singleMarketQuote) {
break;
}
}
Market market(baseCurrencyCode, quoteCurrencyCode);
ret.insert_or_assign(market, MarketOrderBook(market, orderBookLines));
if (singleMarketQuote) {
break;
}
}
}
Expand All @@ -314,16 +323,21 @@ MarketOrderBook BithumbPublic::OrderBookFunc::operator()(Market mk, int depth) {

MonetaryAmount BithumbPublic::TradedVolumeFunc::operator()(Market mk) {
TimePoint t1 = Clock::now();
json result = PublicQuery(_curlHandle, "/public/ticker", mk.base(), mk.quote());
std::string_view last24hVol = result["units_traded_24H"].get<std::string_view>();
std::string_view bithumbTimestamp = result["date"].get<std::string_view>();
int64_t bithumbTimeMs = FromString<int64_t>(bithumbTimestamp);
int64_t t1Ms = TimestampToMs(t1);
int64_t t2Ms = TimestampToMs(Clock::now());
if (t1Ms < bithumbTimeMs && bithumbTimeMs < t2Ms) {
log::debug("Bithumb time is synchronized with us");
} else {
log::error("Bithumb time is not synchronized with us (Bithumb: {}, us: [{} - {}])", bithumbTimestamp, t1Ms, t2Ms);
json result = PublicQuery(_curlHandle, "/public/ticker/", mk.base(), mk.quote());
std::string_view last24hVol;
const auto dateIt = result.find("date");
if (dateIt != result.end()) {
std::string_view bithumbTimestamp = dateIt->get<std::string_view>();

last24hVol = result["units_traded_24H"].get<std::string_view>();
int64_t bithumbTimeMs = FromString<int64_t>(bithumbTimestamp);
int64_t t1Ms = TimestampToMs(t1);
int64_t t2Ms = TimestampToMs(Clock::now());
if (t1Ms < bithumbTimeMs && bithumbTimeMs < t2Ms) {
log::debug("Bithumb time is synchronized with us");
} else {
log::error("Bithumb time is not synchronized with us (Bithumb: {}, us: [{} - {}])", bithumbTimestamp, t1Ms, t2Ms);
}
}

return {last24hVol, mk.base()};
Expand All @@ -340,8 +354,9 @@ TimePoint EpochTime(std::string&& dateStr) {
} // namespace

LastTradesVector BithumbPublic::queryLastTrades(Market mk, [[maybe_unused]] int nbTrades) {
json result = PublicQuery(_curlHandle, "/public/transaction_history", mk.base(), mk.quote());
json result = PublicQuery(_curlHandle, "/public/transaction_history/", mk.base(), mk.quote());
LastTradesVector ret;
ret.reserve(result.size());
for (const json& detail : result) {
MonetaryAmount amount(detail["units_traded"].get<std::string_view>(), mk.base());
MonetaryAmount price(detail["price"].get<std::string_view>(), mk.quote());
Expand Down
Loading

0 comments on commit 292c2e9

Please sign in to comment.