Implement Kraken private api request retry and factorize code

src/api/common/include/ssl_sha.hpp

@@ -12,15 +12,16 @@ namespace cct::ssl {
 
 std::string_view GetOpenSSLVersion();
 
-/// @brief Append Sha256 computed from 'data' to 'str'
-void AppendSha256(std::string_view data, string &str);
-
 /// @brief Helper type containing the number of bytes of the SHA
 enum class ShaType : int16_t { kSha256 = 256 / CHAR_BIT, kSha512 = 512 / CHAR_BIT };
 
-using Md = FixedCapacityVector<char, static_cast<int16_t>(ShaType::kSha512)>;
+using Md256 = FixedCapacityVector<char, static_cast<int16_t>(ShaType::kSha256)>;
+using Md512 = FixedCapacityVector<char, static_cast<int16_t>(ShaType::kSha512)>;
+
+/// @brief Compute Sha256 from 'data'
+Md256 Sha256(std::string_view data);
 
-Md ShaBin(ShaType shaType, std::string_view data, std::string_view secret);
+Md512 ShaBin(ShaType shaType, std::string_view data, std::string_view secret);
 
 string ShaHex(ShaType shaType, std::string_view data, std::string_view secret);
 

src/api/common/src/ssl_sha.cpp

@@ -22,21 +22,22 @@ auto ShaDigestLen(ShaType shaType) { return static_cast<unsigned int>(shaType);
 const EVP_MD* GetEVPMD(ShaType shaType) { return shaType == ShaType::kSha256 ? EVP_sha256() : EVP_sha512(); }
 }  // namespace
 
-void AppendSha256(std::string_view data, string& str) {
+Md256 Sha256(std::string_view data) {
   static_assert(SHA256_DIGEST_LENGTH == static_cast<unsigned int>(ShaType::kSha256));
 
-  str.resize(str.size() + static_cast<string::size_type>(SHA256_DIGEST_LENGTH));
+  Md256 ret(static_cast<string::size_type>(SHA256_DIGEST_LENGTH));
 
-  SHA256(
-      reinterpret_cast<const unsigned char*>(data.data()), data.size(),
-      reinterpret_cast<unsigned char*>(str.data() + str.size() - static_cast<string::size_type>(SHA256_DIGEST_LENGTH)));
+  SHA256(reinterpret_cast<const unsigned char*>(data.data()), data.size(),
+         reinterpret_cast<unsigned char*>(ret.data()));
+
+  return ret;
 }
 
 std::string_view GetOpenSSLVersion() { return OPENSSL_VERSION_TEXT; }
 
-Md ShaBin(ShaType shaType, std::string_view data, std::string_view secret) {
+Md512 ShaBin(ShaType shaType, std::string_view data, std::string_view secret) {
   unsigned int len = ShaDigestLen(shaType);
-  Md binData(static_cast<Md::size_type>(len), 0);
+  Md512 binData(static_cast<Md512::size_type>(len));
 
   HMAC(GetEVPMD(shaType), secret.data(), static_cast<int>(secret.size()),
        reinterpret_cast<const unsigned char*>(data.data()), data.size(),

src/api/common/test/ssl_sha_test.cpp

@@ -11,14 +11,13 @@
 namespace cct::ssl {
 TEST(SSLTest, Version) { EXPECT_NE(GetOpenSSLVersion(), ""); }
 
-TEST(SSLTest, AppendSha256) {
-  string str("test");
-  AppendSha256("thisNonce0123456789Data", str);
-
-  static constexpr char kExpectedData[] = {116,  101,  115, 116,  -98,  74,   -90, 56, -41, 61,   -33, 98,
-                                           -108, -110, -41, -82,  -110, -102, -80, 85, 127, -112, -55, -116,
-                                           38,   36,   10,  -104, -37,  93,   105, 14, 73,  99,   98,  95};
-  EXPECT_TRUE(std::equal(str.begin(), str.end(), std::begin(kExpectedData), std::end(kExpectedData)));
+TEST(SSLTest, Sha256) {
+  auto sha256 = Sha256("thisNonce0123456789Data");
+
+  static constexpr char kExpectedData[] = {-98, 74,   -90,  56,  -41, 61,  -33,  98,  -108, -110, -41,
+                                           -82, -110, -102, -80, 85,  127, -112, -55, -116, 38,   36,
+                                           10,  -104, -37,  93,  105, 14,  73,   99,  98,   95};
+  EXPECT_TRUE(std::equal(sha256.begin(), sha256.end(), std::begin(kExpectedData), std::end(kExpectedData)));
 }
 
 TEST(SSLTest, ShaBin256) {

src/api/exchanges/src/binanceprivateapi.cpp

@@ -103,12 +103,7 @@ void SetNonceAndSignature(const APIKey& apiKey, CurlPostData& postData, Duration
 
   static constexpr std::string_view kSignatureKey = "signature";
 
-  /// Erase signature if present
-  if (postData.back().key() == kSignatureKey) {
-    postData.pop_back();
-  }
-
-  postData.emplace_back(kSignatureKey, ssl::ShaHex(ssl::ShaType::kSha256, postData.str(), apiKey.privateKey()));
+  postData.set_back(kSignatureKey, ssl::ShaHex(ssl::ShaType::kSha256, postData.str(), apiKey.privateKey()));
 }
 
 bool CheckErrorDoRetry(int statusCode, const json& ret, QueryDelayDir& queryDelayDir, Duration& sleepingTime,
@@ -175,7 +170,7 @@ template <class CurlPostDataT = CurlPostData>
 json PrivateQuery(CurlHandle& curlHandle, const APIKey& apiKey, HttpRequestType requestType, std::string_view endpoint,
                   Duration& queryDelay, CurlPostDataT&& curlPostData = CurlPostData(), bool throwIfError = true) {
   CurlOptions opts(requestType, std::forward<CurlPostDataT>(curlPostData));
-  opts.appendHttpHeader("X-MBX-APIKEY", apiKey.key());
+  opts.mutableHttpHeaders().emplace_back("X-MBX-APIKEY", apiKey.key());
 
   Duration sleepingTime = curlHandle.minDurationBetweenQueries();
   int statusCode{};

src/api/exchanges/src/bithumbprivateapi.cpp

@@ -99,11 +99,13 @@ auto GetStrData(std::string_view endpoint, std::string_view postDataStr) {
 }
 
 void SetHttpHeaders(CurlOptions& opts, const APIKey& apiKey, std::string_view signature, const Nonce& nonce) {
-  opts.clearHttpHeaders();
-  opts.appendHttpHeader("API-Key", apiKey.key());
-  opts.appendHttpHeader("API-Sign", signature);
-  opts.appendHttpHeader("API-Nonce", nonce);
-  opts.appendHttpHeader("api-client-type", 1);
+  auto& httpHeaders = opts.mutableHttpHeaders();
+
+  httpHeaders.clear();
+  httpHeaders.emplace_back("API-Key", apiKey.key());
+  httpHeaders.emplace_back("API-Sign", signature);
+  httpHeaders.emplace_back("API-Nonce", nonce);
+  httpHeaders.emplace_back("api-client-type", 1);
 }
 
 template <class ValueType>

src/api/exchanges/src/huobiprivateapi.cpp

@@ -96,12 +96,7 @@ void SetNonceAndSignature(CurlHandle& curlHandle, const APIKey& apiKey, HttpRequ
 
   static constexpr std::string_view kSignatureKey = "Signature";
 
-  /// Erase signature if present
-  if (signaturePostData.back().key() == kSignatureKey) {
-    signaturePostData.pop_back();
-  }
-
-  signaturePostData.emplace_back(
+  signaturePostData.set_back(
       kSignatureKey, URLEncode(B64Encode(ssl::ShaBin(ssl::ShaType::kSha256,
                                                      BuildParamStr(requestType, curlHandle.getNextBaseUrl(), endpoint,
                                                                    signaturePostData.str()),

src/api/exchanges/src/krakenprivateapi.cpp

@@ -3,10 +3,8 @@
 #include <algorithm>
 #include <chrono>
 #include <cstdint>
-#include <memory>
 #include <optional>
 #include <string_view>
-#include <thread>
 #include <tuple>
 #include <utility>
 
@@ -31,7 +29,6 @@
 #include "currencyexchangeflatset.hpp"
 #include "deposit.hpp"
 #include "depositsconstraints.hpp"
-#include "durationstring.hpp"
 #include "exchangeconfig.hpp"
 #include "exchangename.hpp"
 #include "exchangeprivateapi.hpp"
@@ -45,6 +42,7 @@
 #include "orderid.hpp"
 #include "ordersconstraints.hpp"
 #include "permanentcurloptions.hpp"
+#include "request-retry.hpp"
 #include "ssl_sha.hpp"
 #include "stringhelpers.hpp"
 #include "timedef.hpp"
@@ -62,74 +60,71 @@
 namespace cct::api {
 namespace {
 
-string PrivateSignature(const APIKey& apiKey, string data, const Nonce& nonce, std::string_view postdata) {
-  // concatenate nonce and postdata and compute SHA256
-  string noncePostData(nonce.begin(), nonce.end());
-  noncePostData.append(postdata);
-
-  // concatenate path and nonce_postdata (path + ComputeSha256(nonce + postdata))
-  ssl::AppendSha256(noncePostData, data);
-
-  // and compute HMAC
-  return B64Encode(ssl::ShaBin(ssl::ShaType::kSha512, data, B64Decode(apiKey.privateKey())));
-}
-
 enum class KrakenErrorEnum : int8_t { kExpiredOrder, kUnknownWithdrawKey, kUnknownError, kNoError };
 
 template <class CurlPostDataT = CurlPostData>
 std::pair<json, KrakenErrorEnum> PrivateQuery(CurlHandle& curlHandle, const APIKey& apiKey, std::string_view method,
                                               CurlPostDataT&& curlPostData = CurlPostData()) {
-  string path(KrakenPublic::kVersion);
-  path.append(method);
-
   CurlOptions opts(HttpRequestType::kPost, std::forward<CurlPostDataT>(curlPostData));
+  opts.mutableHttpHeaders().emplace_back("API-Key", apiKey.key());
 
-  Nonce nonce = Nonce_TimeSinceEpochInMs();
-  opts.mutablePostData().emplace_back("nonce", nonce);
-  opts.appendHttpHeader("API-Key", apiKey.key());
-  opts.appendHttpHeader("API-Sign", PrivateSignature(apiKey, path, nonce, opts.postData().str()));
-
-  json response = json::parse(curlHandle.query(method, opts));
-  Duration sleepingTime = curlHandle.minDurationBetweenQueries();
+  RequestRetry requestRetry(curlHandle, std::move(opts),
+                            QueryRetryPolicy{.initialRetryDelay = seconds{1}, .nbMaxRetries = 3});
 
   static constexpr std::string_view kErrorKey = "error";
 
-  auto errorIt = response.find(kErrorKey);
-  for (; errorIt != response.end() && !errorIt->empty() &&
-         errorIt->front().get<std::string_view>() == "EAPI:Rate limit exceeded";
-       errorIt = response.find(kErrorKey)) {
-    log::error("Kraken private API rate limit exceeded");
-    sleepingTime *= 2;
-    log::debug("Wait {}", DurationToString(sleepingTime));
-    std::this_thread::sleep_for(sleepingTime);
-
-    // We need to update the nonce
-    nonce = Nonce_TimeSinceEpochInMs();
-    opts.mutablePostData().set("nonce", nonce);
-    opts.setHttpHeader("API-Sign", PrivateSignature(apiKey, path, nonce, opts.postData().str()));
-    response = json::parse(curlHandle.query(method, opts));
-  }
   KrakenErrorEnum err = KrakenErrorEnum::kNoError;
-  if (errorIt != response.end() && !errorIt->empty()) {
-    std::string_view msg = errorIt->front().get<std::string_view>();
-    if (msg.ends_with("Unknown order")) {
-      err = KrakenErrorEnum::kExpiredOrder;
-    } else if (msg.ends_with("Unknown withdraw key")) {
-      err = KrakenErrorEnum::kUnknownWithdrawKey;
-    } else {
-      log::error("Full Kraken json error: '{}'", response.dump());
-      err = KrakenErrorEnum::kUnknownError;
-    }
-  }
-  auto resultIt = response.find("result");
-  const json* pResult;
-  if (resultIt == response.end()) {
-    static const json kEmptyJson{};
-    pResult = std::addressof(kEmptyJson);
-  } else {
-    pResult = std::addressof(*resultIt);
+
+  json ret = requestRetry.queryJson(
+      method,
+      [&err](const json& jsonResponse) {
+        const auto errorIt = jsonResponse.find(kErrorKey);
+        if (errorIt != jsonResponse.end() && !errorIt->empty()) {
+          std::string_view msg = errorIt->front().get<std::string_view>();
+          if (msg == "EAPI:Rate limit exceeded") {
+            log::warn("kraken private API rate limit exceeded");
+            return RequestRetry::Status::kResponseError;
+          }
+          if (msg.ends_with("Unknown order")) {
+            err = KrakenErrorEnum::kExpiredOrder;
+            return RequestRetry::Status::kResponseOK;
+          }
+          if (msg.ends_with("Unknown withdraw key")) {
+            err = KrakenErrorEnum::kUnknownWithdrawKey;
+            return RequestRetry::Status::kResponseOK;
+          }
+          log::error("kraken unknown error {}", msg);
+          return RequestRetry::Status::kResponseError;
+        }
+        return RequestRetry::Status::kResponseOK;
+      },
+      [&apiKey, method](CurlOptions& opts) {
+        Nonce noncePostData = Nonce_TimeSinceEpochInMs();
+        opts.mutablePostData().set("nonce", noncePostData);
+
+        // concatenate nonce and postdata and compute SHA256
+        noncePostData.append(opts.postData().str());
+
+        // concatenate path and nonce_postdata (path + ComputeSha256(nonce + postdata))
+        auto sha256 = ssl::Sha256(noncePostData);
+
+        string path;
+        path.reserve(KrakenPublic::kVersion.size() + method.size() + sha256.size());
+        path.append(KrakenPublic::kVersion).append(method).append(sha256.data(), sha256.data() + sha256.size());
+
+        static constexpr std::string_view kSignatureKey = "API-Sign";
+
+        // and compute HMAC
+        opts.mutableHttpHeaders().set_back(
+            kSignatureKey, B64Encode(ssl::ShaBin(ssl::ShaType::kSha512, path, B64Decode(apiKey.privateKey()))));
+      });
+
+  auto resultIt = ret.find("result");
+  std::pair<json, KrakenErrorEnum> retPair(json::object_t{}, err);
+  if (resultIt != ret.end()) {
+    retPair.first = std::move(*resultIt);
   }
-  return std::make_pair(*pResult, err);
+  return retPair;
 }
 }  // namespace
 

src/api/exchanges/src/kucoinprivateapi.cpp

@@ -82,11 +82,13 @@ json PrivateQuery(CurlHandle& curlHandle, const APIKey& apiKey, HttpRequestType
   string passphrase = B64Encode(ssl::ShaBin(ssl::ShaType::kSha256, apiKey.passphrase(), apiKey.privateKey()));
 
   CurlOptions opts(requestType, std::move(postData), postDataFormat);
-  opts.appendHttpHeader("KC-API-KEY", apiKey.key());
-  opts.appendHttpHeader("KC-API-SIGN", signature);
-  opts.appendHttpHeader("KC-API-TIMESTAMP", std::string_view(strToSign.data(), nonceSize));
-  opts.appendHttpHeader("KC-API-PASSPHRASE", passphrase);
-  opts.appendHttpHeader("KC-API-KEY-VERSION", 2);
+
+  auto& httpHeaders = opts.mutableHttpHeaders();
+  httpHeaders.emplace_back("KC-API-KEY", apiKey.key());
+  httpHeaders.emplace_back("KC-API-SIGN", signature);
+  httpHeaders.emplace_back("KC-API-TIMESTAMP", std::string_view(strToSign.data(), nonceSize));
+  httpHeaders.emplace_back("KC-API-PASSPHRASE", passphrase);
+  httpHeaders.emplace_back("KC-API-KEY-VERSION", 2);
 
   json ret = json::parse(curlHandle.query(endpoint, opts));
   auto errCodeIt = ret.find("code");

src/api/exchanges/src/upbitprivateapi.cpp

@@ -88,7 +88,7 @@ json PrivateQuery(CurlHandle& curlHandle, const APIKey& apiKey, HttpRequestType
   string authStr("Bearer ");
   authStr.append(token.begin(), token.end());
 
-  opts.appendHttpHeader("Authorization", authStr);
+  opts.mutableHttpHeaders().emplace_back("Authorization", authStr);
 
   json ret = json::parse(curlHandle.query(endpoint, opts));
   if (ifError == IfError::kThrow) {

src/http-request/include/curloptions.hpp

@@ -31,6 +31,7 @@ class CurlOptions {
     }
   }
 
+  HttpHeaders &mutableHttpHeaders() { return _httpHeaders; }
   const HttpHeaders &httpHeaders() const { return _httpHeaders; }
 
   const char *proxyUrl() const { return _proxyUrl; }
@@ -51,14 +52,6 @@ class CurlOptions {
 
   HttpRequestType requestType() const { return _requestType; }
 
-  void clearHttpHeaders() { _httpHeaders.clear(); }
-
-  void appendHttpHeader(std::string_view key, std::string_view value) { _httpHeaders.emplace_back(key, value); }
-  void appendHttpHeader(std::string_view key, std::integral auto value) { _httpHeaders.emplace_back(key, value); }
-
-  void setHttpHeader(std::string_view key, std::string_view value) { _httpHeaders.set(key, value); }
-  void setHttpHeader(std::string_view key, std::integral auto value) { _httpHeaders.set(key, value); }
-
   using trivially_relocatable =
       std::bool_constant<is_trivially_relocatable_v<HttpHeaders> && is_trivially_relocatable_v<CurlPostData>>::type;
 

src/http-request/test/curlhandle_test.cpp

@@ -33,7 +33,7 @@ TEST_F(ExampleBaseCurlHandle, CurlVersion) { EXPECT_FALSE(GetCurlVersionInfo().e
 
 TEST_F(ExampleBaseCurlHandle, QueryJsonAndMoveConstruct) {
   CurlOptions opts = kVerboseHttpGetOptions;
-  opts.appendHttpHeader("MyHeaderIsVeryLongToAvoidSSO", "Val1");
+  opts.mutableHttpHeaders().emplace_back("MyHeaderIsVeryLongToAvoidSSO", "Val1");
 
   EXPECT_NE(handle.query("/json", opts).find("slideshow"), std::string_view::npos);