Skip to content

Commit

Permalink
Use glaze for secrets and coincenter info
Browse files Browse the repository at this point in the history
  • Loading branch information
@sjanel
sjanel committed Nov 24, 2024
1 parent f1cddb1 commit d1a1d3c
Show file tree
Hide file tree
Showing 19 changed files with 489 additions and 255 deletions.
2 changes: 1 addition & 1 deletion CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,7 @@ if(CCT_ENABLE_PROTO)
else()
# Check here for a new version: https://protobuf.dev/support/version-support/#cpp
if (NOT PROTOBUF_VERSION)
set(PROTOBUF_VERSION v5.28.2)
set(PROTOBUF_VERSION v5.28.3)
endif()

message(STATUS "Configuring protobuf ${PROTOBUF_VERSION} from sources")
Expand Down
88 changes: 38 additions & 50 deletions src/api-objects/src/apikeysprovider.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,14 @@
#include "accountowner.hpp"
#include "apikey.hpp"
#include "cct_exception.hpp"
#include "cct_json-container.hpp"
#include "cct_log.hpp"
#include "cct_string.hpp"
#include "exchangename.hpp"
#include "exchangesecretsinfo.hpp"
#include "file.hpp"
#include "read-json.hpp"
#include "runmodes.hpp"
#include "secret-schema.hpp"

namespace cct::api {
namespace {
Expand Down Expand Up @@ -73,61 +74,48 @@ APIKeysProvider::APIKeysPerExchange APIKeysProvider::ParseAPIKeys(std::string_vi
const ExchangeSecretsInfo& exchangeSecretsInfo,
settings::RunMode runMode) {
APIKeysProvider::APIKeysPerExchange apiKeysPerExchange;

if (exchangeSecretsInfo.allExchangesWithoutSecrets()) {
log::info("Not loading private keys, using only public exchanges");
} else {
std::string_view secretFileName = GetSecretFileName(runMode);
File secretsFile(dataDir, File::Type::kSecret, secretFileName,
settings::AreTestKeysRequested(runMode) ? File::IfError::kThrow : File::IfError::kNoThrow);
json::container jsonData = secretsFile.readAllJson();
bool atLeastOneKeyFound = false;
for (auto& [publicExchangeName, keyObj] : jsonData.items()) {
const auto& exchangesWithoutSecrets = exchangeSecretsInfo.exchangesWithoutSecrets();
if (std::ranges::find(exchangesWithoutSecrets, ExchangeName(publicExchangeName)) !=
exchangesWithoutSecrets.end()) {
log::info("Not loading {} private keys as requested", publicExchangeName);
return apiKeysPerExchange;
}

std::string_view secretFileName = GetSecretFileName(runMode);
const auto throwOrNoThrow = settings::AreTestKeysRequested(runMode) ? File::IfError::kThrow : File::IfError::kNoThrow;
File secretsFile(dataDir, File::Type::kSecret, secretFileName, throwOrNoThrow);

schema::APIKeysPerExchangeMap apiKeysPerExchangeMap;

ReadJsonOrThrow(secretsFile.readAll(), apiKeysPerExchangeMap);

const auto& exchangesWithoutSecrets = exchangeSecretsInfo.exchangesWithoutSecrets();

bool atLeastOneKeyFound = false;
for (auto& [exchangeNameEnum, apiKeys] : apiKeysPerExchangeMap) {
auto publicExchangeName = kSupportedExchanges[static_cast<int>(exchangeNameEnum)];
if (std::ranges::any_of(exchangesWithoutSecrets, [exchangeNameEnum](const auto& exchangeName) {
return exchangeName.exchangeNameEnum() == exchangeNameEnum;
})) {
log::debug("Not loading {} private keys as requested", publicExchangeName);
continue;
}

for (auto& [keyName, apiKey] : apiKeys) {
if (apiKey.key.empty() || apiKey.priv.empty()) {
log::error("Wrong format for secret.json file. It should contain at least fields 'key' and 'private'");
continue;
}
ExchangeNameEnum exchangeNameEnum = static_cast<ExchangeNameEnum>(
std::find(std::begin(kSupportedExchanges), std::end(kSupportedExchanges), publicExchangeName) -
std::begin(kSupportedExchanges));

for (auto& [name, keySecretObj] : keyObj.items()) {
auto keyIt = keySecretObj.find("key");
auto privateIt = keySecretObj.find("private");
if (keyIt == keySecretObj.end() || privateIt == keySecretObj.end()) {
log::error("Wrong format for secret.json file. It should contain at least fields 'key' and 'private'");
continue;
}
string passphrase;
auto passphraseIt = keySecretObj.find("passphrase");
if (passphraseIt != keySecretObj.end()) {
passphrase = std::move(passphraseIt->get_ref<string&>());
}
std::string_view ownerEnName;
std::string_view ownerKoName;
auto accountOwnerPartIt = keySecretObj.find("accountOwner");
if (accountOwnerPartIt != keySecretObj.end()) {
auto ownerEnNameIt = accountOwnerPartIt->find("enName");
if (ownerEnNameIt != accountOwnerPartIt->end()) {
ownerEnName = ownerEnNameIt->get<std::string_view>();
}
auto ownerKoNameIt = accountOwnerPartIt->find("koName");
if (ownerKoNameIt != accountOwnerPartIt->end()) {
ownerKoName = ownerKoNameIt->get<std::string_view>();
}
}

apiKeysPerExchange[static_cast<int>(exchangeNameEnum)].emplace_back(
publicExchangeName, name, std::move(keyIt->get_ref<string&>()), std::move(privateIt->get_ref<string&>()),
std::move(passphrase), AccountOwner(ownerEnName, ownerKoName));
atLeastOneKeyFound = true;
}
}
if (!atLeastOneKeyFound) {
log::warn("No private api keys file '{}' found. Only public exchange queries will be supported", secretFileName);

apiKeysPerExchange[static_cast<int>(exchangeNameEnum)].emplace_back(
publicExchangeName, keyName, std::move(apiKey.key), std::move(apiKey.priv), std::move(apiKey.passphrase),
AccountOwner(apiKey.accountOwner.enName, apiKey.accountOwner.koName));

atLeastOneKeyFound = true;
}
}
if (!atLeastOneKeyFound) {
log::warn("No private api keys file '{}' found. Only public exchange queries will be supported", secretFileName);
}

return apiKeysPerExchange;
}
Expand Down
49 changes: 41 additions & 8 deletions src/api/common/include/fiatconverter.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,13 @@
#include <optional>
#include <string_view>
#include <unordered_map>
#include <utility>

#include "cache-file-updator-interface.hpp"
#include "cct_fixedcapacityvector.hpp"
#include "cct_flatset.hpp"
#include "cct_string.hpp"
#include "cct_vector.hpp"
#include "curlhandle.hpp"
#include "currencycode.hpp"
#include "file.hpp"
Expand Down Expand Up @@ -36,16 +40,16 @@ class CoincenterInfo;
/// Conversion methods are thread safe.
class FiatConverter : public CacheFileUpdatorInterface {
public:
static File GetRatesCacheFile(std::string_view dataDir);

/// Creates a FiatConverter able to perform live queries to free converter api.
/// @param ratesUpdateFrequency the minimum time needed between two currency rates updates
FiatConverter(const CoincenterInfo &coincenterInfo, Duration ratesUpdateFrequency);

/// Creates a FiatConverter able to perform live queries to free converter api.
/// @param ratesUpdateFrequency the minimum time needed between two currency rates updates
/// @param reader the reader from which to load the initial rates conversion cache
FiatConverter(const CoincenterInfo &coincenterInfo, Duration ratesUpdateFrequency, const Reader &reader);
/// @param fiatsRatesCacheReader the reader from which to load the initial rates conversion cache
/// @param thirdPartySecretReader the reader from which to load the third party secret
FiatConverter(const CoincenterInfo &coincenterInfo, Duration ratesUpdateFrequency,
const Reader &fiatsRatesCacheReader, const Reader &thirdPartySecretReader);

std::optional<double> convert(double amount, CurrencyCode from, CurrencyCode to);

Expand All @@ -64,29 +68,58 @@ class FiatConverter : public CacheFileUpdatorInterface {
private:
struct PriceTimedValue {
double rate;
TimePoint lastUpdatedTime;
int64_t timeepoch;

TimePoint lastUpdatedTime() const { return TimePoint(seconds(timeepoch)); }
};

struct ThirdPartySecret {
string freecurrencyconverter;
};

static ThirdPartySecret LoadCurrencyConverterAPIKey(const Reader &thirdPartySecretReader);

std::optional<double> queryCurrencyRate(Market market);

std::optional<double> queryCurrencyRateSource1(Market market);
std::optional<double> queryCurrencyRateSource2(Market market);

std::optional<double> retrieveRateFromCache(Market market) const;
enum class CacheReadMode : int8_t { kOnlyRecentRates, kUseAllRates };

std::optional<double> retrieveRateFromCache(Market market, CacheReadMode cacheReadMode);

void store(Market market, double rate);

void refreshLastUpdatedTime(Market market);

using PricesMap = std::unordered_map<Market, PriceTimedValue>;

// For the algorithm computing rates
struct Node {
// hard limit to avoid unreasonable long paths and memory allocations
static constexpr std::size_t kMaxCurrencyPathSize = 6U;

using CurrencyPath = FixedCapacityVector<CurrencyCode, kMaxCurrencyPathSize>;

using trivially_relocatable = std::true_type;

CurrencyPath currencyPath;
double rate;
TimePoint oldestTs;
};

vector<Node> _nodes;
using VisitedCurrencyCodesSet = FlatSet<CurrencyCode>;

VisitedCurrencyCodesSet _visitedCurrencies;
vector<std::pair<Market, PriceTimedValue>> _tmpPriceRatesVector;

CurlHandle _curlHandle1;
CurlHandle _curlHandle2;
PricesMap _pricesMap;
Duration _ratesUpdateFrequency;
std::mutex _pricesMutex;
string _apiKey;
ThirdPartySecret _thirdPartySecret;
string _dataDir;
CurrencyCode _baseRateSource2;
};
} // namespace cct
64 changes: 37 additions & 27 deletions src/api/common/src/commonapi.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,25 +1,28 @@
#include "commonapi.hpp"

#include <cstdint>
#include <glaze/glaze.hpp> // IWYU pragma: export
#include <mutex>
#include <optional>
#include <string_view>
#include <utility>

#include "cachedresult.hpp"
#include "cct_const.hpp"
#include "cct_json-container.hpp"
#include "cct_log.hpp"
#include "cct_string.hpp"
#include "cct_vector.hpp"
#include "coincenterinfo.hpp"
#include "curloptions.hpp"
#include "currencycode.hpp"
#include "currencycodeset.hpp"
#include "currencycodevector.hpp"
#include "fiats-cache-schema.hpp"
#include "file.hpp"
#include "httprequesttype.hpp"
#include "monetaryamountbycurrencyset.hpp"
#include "permanentcurloptions.hpp"
#include "read-json.hpp"
#include "timedef.hpp"
#include "withdrawalfees-crawler.hpp"

Expand All @@ -46,18 +49,15 @@ CommonAPI::CommonAPI(const CoincenterInfo& coincenterInfo, Duration fiatsUpdateF
coincenterInfo.getRunMode()),
_withdrawalFeesCrawler(coincenterInfo, withdrawalFeesUpdateFrequency, _cachedResultVault) {
if (atInit == AtInit::kLoadFromFileCache) {
json::container data = GetFiatCacheFile(_coincenterInfo.dataDir()).readAllJson();
if (!data.empty()) {
int64_t timeEpoch = data["timeepoch"].get<int64_t>();
auto& fiatsFile = data["fiats"];
CurrencyCodeSet fiats;
fiats.reserve(static_cast<CurrencyCodeSet::size_type>(fiatsFile.size()));
for (json::container& val : fiatsFile) {
log::trace("Reading fiat {} from cache file", val.get<std::string_view>());
fiats.emplace_hint(fiats.end(), std::move(val.get_ref<string&>()));
schema::FiatsCache fiatsCache;
auto dataStr = GetFiatCacheFile(_coincenterInfo.dataDir()).readAll();
if (!dataStr.empty()) {
ReadJsonOrThrow(dataStr, fiatsCache);
if (fiatsCache.timeepoch != 0) {
CurrencyCodeSet fiats(std::move(fiatsCache.fiats));
log::debug("Loaded {} fiats from cache file", fiats.size());
_fiatsCache.set(std::move(fiats), TimePoint(seconds(fiatsCache.timeepoch)));
}
log::debug("Loaded {} fiats from cache file", fiats.size());
_fiatsCache.set(std::move(fiats), TimePoint(seconds(timeEpoch)));
}
}
}
Expand Down Expand Up @@ -103,7 +103,7 @@ std::optional<MonetaryAmount> CommonAPI::tryQueryWithdrawalFee(std::string_view
}

namespace {
constexpr std::string_view kFiatsUrlSource1 = "https://datahub.io/core/currency-codes/r/codes-all.json";
constexpr std::string_view kFiatsUrlSource1 = "https://datahub.io/core/currency-codes/_r/-/data/codes-all.csv";
constexpr std::string_view kFiatsUrlSource2 = "https://www.iban.com/currency-codes";
} // namespace

Expand All @@ -130,6 +130,14 @@ CurrencyCodeSet CommonAPI::FiatsFunc::operator()() {
}
return fiats;
}
struct CurrencyCSV {
vector<string> Entity;
vector<string> Currency;
vector<string> AlphabeticCode;
vector<string> NumericCode;
vector<string> MinorUnit;
vector<string> WithdrawalDate;
};

CurrencyCodeVector CommonAPI::FiatsFunc::retrieveFiatsSource1() {
CurrencyCodeVector fiatsVec;
Expand All @@ -139,25 +147,27 @@ CurrencyCodeVector CommonAPI::FiatsFunc::retrieveFiatsSource1() {
log::warn("Error parsing currency codes, no fiats found from first source");
return fiatsVec;
}
static constexpr bool kAllowExceptions = false;
json::container dataCSV = json::container::parse(data, nullptr, kAllowExceptions);
if (dataCSV.is_discarded()) {
log::warn("Error parsing json data of currency codes from source 1");

// data is UTF-8 encoded - but the relevant data that we will parse is ASCII normally

CurrencyCSV currencies;
auto ec = json::read<json::opts{.format = glz::CSV, .layout = glz::colwise}>(currencies, data);

if (ec || currencies.AlphabeticCode.size() != currencies.WithdrawalDate.size()) {
log::warn("Error parsing json data of currency codes from source 1: {}", glz::format_error(ec, data));
return fiatsVec;
}
for (const json::container& fiatData : dataCSV) {
static constexpr std::string_view kCodeKey = "AlphabeticCode";
static constexpr std::string_view kWithdrawalDateKey = "WithdrawalDate";

auto codeIt = fiatData.find(kCodeKey);
auto withdrawalDateIt = fiatData.find(kWithdrawalDateKey);
if (codeIt != fiatData.end() && !codeIt->is_null() && withdrawalDateIt != fiatData.end() &&
withdrawalDateIt->is_null()) {
fiatsVec.emplace_back(codeIt->get<std::string_view>());
log::debug("Stored {} fiat", codeIt->get<std::string_view>());

auto nbCurrencies = currencies.AlphabeticCode.size();
for (size_t currencyPos = 0; currencyPos < nbCurrencies; ++currencyPos) {
if (currencies.WithdrawalDate[currencyPos].empty()) {
fiatsVec.emplace_back(currencies.AlphabeticCode[currencyPos]);
log::debug("Stored {} fiat", fiatsVec.back());
}
}

log::info("Found {} fiats from first source", fiatsVec.size());

return fiatsVec;
}

Expand Down
Loading

0 comments on commit d1a1d3c

Please sign in to comment.