From f5ef879f5a8a7c07efb47b52a56f5e83f55df59d Mon Sep 17 00:00:00 2001 From: Stephane Janel Date: Mon, 1 Apr 2024 11:13:26 +0200 Subject: [PATCH] Implement Kraken private api request retry and factorize code --- src/api/common/include/ssl_sha.hpp | 11 +- src/api/common/src/ssl_sha.cpp | 15 +- src/api/common/test/ssl_sha_test.cpp | 15 +- src/api/exchanges/src/binanceprivateapi.cpp | 9 +- src/api/exchanges/src/bithumbprivateapi.cpp | 12 +- src/api/exchanges/src/huobiprivateapi.cpp | 7 +- src/api/exchanges/src/krakenprivateapi.cpp | 113 ++++++------ src/api/exchanges/src/kucoinprivateapi.cpp | 12 +- src/api/exchanges/src/upbitprivateapi.cpp | 2 +- src/http-request/include/curloptions.hpp | 9 +- src/http-request/test/curlhandle_test.cpp | 2 +- .../flat-key-value-string-iterator.hpp | 160 +++++++++++++++++ src/tech/include/flatkeyvaluestring.hpp | 168 ++---------------- src/tech/test/flatkeyvaluestring_test.cpp | 14 ++ 14 files changed, 286 insertions(+), 263 deletions(-) create mode 100644 src/tech/include/flat-key-value-string-iterator.hpp diff --git a/src/api/common/include/ssl_sha.hpp b/src/api/common/include/ssl_sha.hpp index 1ed05486..4c920fd7 100644 --- a/src/api/common/include/ssl_sha.hpp +++ b/src/api/common/include/ssl_sha.hpp @@ -12,15 +12,16 @@ namespace cct::ssl { std::string_view GetOpenSSLVersion(); -/// @brief Append Sha256 computed from 'data' to 'str' -void AppendSha256(std::string_view data, string &str); - /// @brief Helper type containing the number of bytes of the SHA enum class ShaType : int16_t { kSha256 = 256 / CHAR_BIT, kSha512 = 512 / CHAR_BIT }; -using Md = FixedCapacityVector(ShaType::kSha512)>; +using Md256 = FixedCapacityVector(ShaType::kSha256)>; +using Md512 = FixedCapacityVector(ShaType::kSha512)>; + +/// @brief Compute Sha256 from 'data' +Md256 Sha256(std::string_view data); -Md ShaBin(ShaType shaType, std::string_view data, std::string_view secret); +Md512 ShaBin(ShaType shaType, std::string_view data, std::string_view secret); string ShaHex(ShaType shaType, std::string_view data, std::string_view secret); diff --git a/src/api/common/src/ssl_sha.cpp b/src/api/common/src/ssl_sha.cpp index aa8cbc75..93c6d2a5 100644 --- a/src/api/common/src/ssl_sha.cpp +++ b/src/api/common/src/ssl_sha.cpp @@ -22,21 +22,22 @@ auto ShaDigestLen(ShaType shaType) { return static_cast(shaType); const EVP_MD* GetEVPMD(ShaType shaType) { return shaType == ShaType::kSha256 ? EVP_sha256() : EVP_sha512(); } } // namespace -void AppendSha256(std::string_view data, string& str) { +Md256 Sha256(std::string_view data) { static_assert(SHA256_DIGEST_LENGTH == static_cast(ShaType::kSha256)); - str.resize(str.size() + static_cast(SHA256_DIGEST_LENGTH)); + Md256 ret(static_cast(SHA256_DIGEST_LENGTH)); - SHA256( - reinterpret_cast(data.data()), data.size(), - reinterpret_cast(str.data() + str.size() - static_cast(SHA256_DIGEST_LENGTH))); + SHA256(reinterpret_cast(data.data()), data.size(), + reinterpret_cast(ret.data())); + + return ret; } std::string_view GetOpenSSLVersion() { return OPENSSL_VERSION_TEXT; } -Md ShaBin(ShaType shaType, std::string_view data, std::string_view secret) { +Md512 ShaBin(ShaType shaType, std::string_view data, std::string_view secret) { unsigned int len = ShaDigestLen(shaType); - Md binData(static_cast(len), 0); + Md512 binData(static_cast(len)); HMAC(GetEVPMD(shaType), secret.data(), static_cast(secret.size()), reinterpret_cast(data.data()), data.size(), diff --git a/src/api/common/test/ssl_sha_test.cpp b/src/api/common/test/ssl_sha_test.cpp index a109c580..d6862939 100644 --- a/src/api/common/test/ssl_sha_test.cpp +++ b/src/api/common/test/ssl_sha_test.cpp @@ -11,14 +11,13 @@ namespace cct::ssl { TEST(SSLTest, Version) { EXPECT_NE(GetOpenSSLVersion(), ""); } -TEST(SSLTest, AppendSha256) { - string str("test"); - AppendSha256("thisNonce0123456789Data", str); - - static constexpr char kExpectedData[] = {116, 101, 115, 116, -98, 74, -90, 56, -41, 61, -33, 98, - -108, -110, -41, -82, -110, -102, -80, 85, 127, -112, -55, -116, - 38, 36, 10, -104, -37, 93, 105, 14, 73, 99, 98, 95}; - EXPECT_TRUE(std::equal(str.begin(), str.end(), std::begin(kExpectedData), std::end(kExpectedData))); +TEST(SSLTest, Sha256) { + auto sha256 = Sha256("thisNonce0123456789Data"); + + static constexpr char kExpectedData[] = {-98, 74, -90, 56, -41, 61, -33, 98, -108, -110, -41, + -82, -110, -102, -80, 85, 127, -112, -55, -116, 38, 36, + 10, -104, -37, 93, 105, 14, 73, 99, 98, 95}; + EXPECT_TRUE(std::equal(sha256.begin(), sha256.end(), std::begin(kExpectedData), std::end(kExpectedData))); } TEST(SSLTest, ShaBin256) { diff --git a/src/api/exchanges/src/binanceprivateapi.cpp b/src/api/exchanges/src/binanceprivateapi.cpp index c738377f..c45fc8cb 100644 --- a/src/api/exchanges/src/binanceprivateapi.cpp +++ b/src/api/exchanges/src/binanceprivateapi.cpp @@ -103,12 +103,7 @@ void SetNonceAndSignature(const APIKey& apiKey, CurlPostData& postData, Duration static constexpr std::string_view kSignatureKey = "signature"; - /// Erase signature if present - if (postData.back().key() == kSignatureKey) { - postData.pop_back(); - } - - postData.emplace_back(kSignatureKey, ssl::ShaHex(ssl::ShaType::kSha256, postData.str(), apiKey.privateKey())); + postData.set_back(kSignatureKey, ssl::ShaHex(ssl::ShaType::kSha256, postData.str(), apiKey.privateKey())); } bool CheckErrorDoRetry(int statusCode, const json& ret, QueryDelayDir& queryDelayDir, Duration& sleepingTime, @@ -175,7 +170,7 @@ template json PrivateQuery(CurlHandle& curlHandle, const APIKey& apiKey, HttpRequestType requestType, std::string_view endpoint, Duration& queryDelay, CurlPostDataT&& curlPostData = CurlPostData(), bool throwIfError = true) { CurlOptions opts(requestType, std::forward(curlPostData)); - opts.appendHttpHeader("X-MBX-APIKEY", apiKey.key()); + opts.mutableHttpHeaders().emplace_back("X-MBX-APIKEY", apiKey.key()); Duration sleepingTime = curlHandle.minDurationBetweenQueries(); int statusCode{}; diff --git a/src/api/exchanges/src/bithumbprivateapi.cpp b/src/api/exchanges/src/bithumbprivateapi.cpp index 71ecf8e0..a6d872ff 100644 --- a/src/api/exchanges/src/bithumbprivateapi.cpp +++ b/src/api/exchanges/src/bithumbprivateapi.cpp @@ -99,11 +99,13 @@ auto GetStrData(std::string_view endpoint, std::string_view postDataStr) { } void SetHttpHeaders(CurlOptions& opts, const APIKey& apiKey, std::string_view signature, const Nonce& nonce) { - opts.clearHttpHeaders(); - opts.appendHttpHeader("API-Key", apiKey.key()); - opts.appendHttpHeader("API-Sign", signature); - opts.appendHttpHeader("API-Nonce", nonce); - opts.appendHttpHeader("api-client-type", 1); + auto& httpHeaders = opts.mutableHttpHeaders(); + + httpHeaders.clear(); + httpHeaders.emplace_back("API-Key", apiKey.key()); + httpHeaders.emplace_back("API-Sign", signature); + httpHeaders.emplace_back("API-Nonce", nonce); + httpHeaders.emplace_back("api-client-type", 1); } template diff --git a/src/api/exchanges/src/huobiprivateapi.cpp b/src/api/exchanges/src/huobiprivateapi.cpp index 95c2a9f4..4c096cb6 100644 --- a/src/api/exchanges/src/huobiprivateapi.cpp +++ b/src/api/exchanges/src/huobiprivateapi.cpp @@ -96,12 +96,7 @@ void SetNonceAndSignature(CurlHandle& curlHandle, const APIKey& apiKey, HttpRequ static constexpr std::string_view kSignatureKey = "Signature"; - /// Erase signature if present - if (signaturePostData.back().key() == kSignatureKey) { - signaturePostData.pop_back(); - } - - signaturePostData.emplace_back( + signaturePostData.set_back( kSignatureKey, URLEncode(B64Encode(ssl::ShaBin(ssl::ShaType::kSha256, BuildParamStr(requestType, curlHandle.getNextBaseUrl(), endpoint, signaturePostData.str()), diff --git a/src/api/exchanges/src/krakenprivateapi.cpp b/src/api/exchanges/src/krakenprivateapi.cpp index 0fd1b981..537af78b 100644 --- a/src/api/exchanges/src/krakenprivateapi.cpp +++ b/src/api/exchanges/src/krakenprivateapi.cpp @@ -3,10 +3,8 @@ #include #include #include -#include #include #include -#include #include #include @@ -31,7 +29,6 @@ #include "currencyexchangeflatset.hpp" #include "deposit.hpp" #include "depositsconstraints.hpp" -#include "durationstring.hpp" #include "exchangeconfig.hpp" #include "exchangename.hpp" #include "exchangeprivateapi.hpp" @@ -45,6 +42,7 @@ #include "orderid.hpp" #include "ordersconstraints.hpp" #include "permanentcurloptions.hpp" +#include "request-retry.hpp" #include "ssl_sha.hpp" #include "stringhelpers.hpp" #include "timedef.hpp" @@ -62,74 +60,71 @@ namespace cct::api { namespace { -string PrivateSignature(const APIKey& apiKey, string data, const Nonce& nonce, std::string_view postdata) { - // concatenate nonce and postdata and compute SHA256 - string noncePostData(nonce.begin(), nonce.end()); - noncePostData.append(postdata); - - // concatenate path and nonce_postdata (path + ComputeSha256(nonce + postdata)) - ssl::AppendSha256(noncePostData, data); - - // and compute HMAC - return B64Encode(ssl::ShaBin(ssl::ShaType::kSha512, data, B64Decode(apiKey.privateKey()))); -} - enum class KrakenErrorEnum : int8_t { kExpiredOrder, kUnknownWithdrawKey, kUnknownError, kNoError }; template std::pair PrivateQuery(CurlHandle& curlHandle, const APIKey& apiKey, std::string_view method, CurlPostDataT&& curlPostData = CurlPostData()) { - string path(KrakenPublic::kVersion); - path.append(method); - CurlOptions opts(HttpRequestType::kPost, std::forward(curlPostData)); + opts.mutableHttpHeaders().emplace_back("API-Key", apiKey.key()); - Nonce nonce = Nonce_TimeSinceEpochInMs(); - opts.mutablePostData().emplace_back("nonce", nonce); - opts.appendHttpHeader("API-Key", apiKey.key()); - opts.appendHttpHeader("API-Sign", PrivateSignature(apiKey, path, nonce, opts.postData().str())); - - json response = json::parse(curlHandle.query(method, opts)); - Duration sleepingTime = curlHandle.minDurationBetweenQueries(); + RequestRetry requestRetry(curlHandle, std::move(opts), + QueryRetryPolicy{.initialRetryDelay = seconds{1}, .nbMaxRetries = 3}); static constexpr std::string_view kErrorKey = "error"; - auto errorIt = response.find(kErrorKey); - for (; errorIt != response.end() && !errorIt->empty() && - errorIt->front().get() == "EAPI:Rate limit exceeded"; - errorIt = response.find(kErrorKey)) { - log::error("Kraken private API rate limit exceeded"); - sleepingTime *= 2; - log::debug("Wait {}", DurationToString(sleepingTime)); - std::this_thread::sleep_for(sleepingTime); - - // We need to update the nonce - nonce = Nonce_TimeSinceEpochInMs(); - opts.mutablePostData().set("nonce", nonce); - opts.setHttpHeader("API-Sign", PrivateSignature(apiKey, path, nonce, opts.postData().str())); - response = json::parse(curlHandle.query(method, opts)); - } KrakenErrorEnum err = KrakenErrorEnum::kNoError; - if (errorIt != response.end() && !errorIt->empty()) { - std::string_view msg = errorIt->front().get(); - if (msg.ends_with("Unknown order")) { - err = KrakenErrorEnum::kExpiredOrder; - } else if (msg.ends_with("Unknown withdraw key")) { - err = KrakenErrorEnum::kUnknownWithdrawKey; - } else { - log::error("Full Kraken json error: '{}'", response.dump()); - err = KrakenErrorEnum::kUnknownError; - } - } - auto resultIt = response.find("result"); - const json* pResult; - if (resultIt == response.end()) { - static const json kEmptyJson{}; - pResult = std::addressof(kEmptyJson); - } else { - pResult = std::addressof(*resultIt); + + json ret = requestRetry.queryJson( + method, + [&err](const json& jsonResponse) { + const auto errorIt = jsonResponse.find(kErrorKey); + if (errorIt != jsonResponse.end() && !errorIt->empty()) { + std::string_view msg = errorIt->front().get(); + if (msg == "EAPI:Rate limit exceeded") { + log::warn("kraken private API rate limit exceeded"); + return RequestRetry::Status::kResponseError; + } + if (msg.ends_with("Unknown order")) { + err = KrakenErrorEnum::kExpiredOrder; + return RequestRetry::Status::kResponseOK; + } + if (msg.ends_with("Unknown withdraw key")) { + err = KrakenErrorEnum::kUnknownWithdrawKey; + return RequestRetry::Status::kResponseOK; + } + log::error("kraken unknown error {}", msg); + return RequestRetry::Status::kResponseError; + } + return RequestRetry::Status::kResponseOK; + }, + [&apiKey, method](CurlOptions& opts) { + Nonce noncePostData = Nonce_TimeSinceEpochInMs(); + opts.mutablePostData().set("nonce", noncePostData); + + // concatenate nonce and postdata and compute SHA256 + noncePostData.append(opts.postData().str()); + + // concatenate path and nonce_postdata (path + ComputeSha256(nonce + postdata)) + auto sha256 = ssl::Sha256(noncePostData); + + string path; + path.reserve(KrakenPublic::kVersion.size() + method.size() + sha256.size()); + path.append(KrakenPublic::kVersion).append(method).append(sha256.data(), sha256.data() + sha256.size()); + + static constexpr std::string_view kSignatureKey = "API-Sign"; + + // and compute HMAC + opts.mutableHttpHeaders().set_back( + kSignatureKey, B64Encode(ssl::ShaBin(ssl::ShaType::kSha512, path, B64Decode(apiKey.privateKey())))); + }); + + auto resultIt = ret.find("result"); + std::pair retPair(json::object_t{}, err); + if (resultIt != ret.end()) { + retPair.first = std::move(*resultIt); } - return std::make_pair(*pResult, err); + return retPair; } } // namespace diff --git a/src/api/exchanges/src/kucoinprivateapi.cpp b/src/api/exchanges/src/kucoinprivateapi.cpp index 87d24930..c6cf16ee 100644 --- a/src/api/exchanges/src/kucoinprivateapi.cpp +++ b/src/api/exchanges/src/kucoinprivateapi.cpp @@ -82,11 +82,13 @@ json PrivateQuery(CurlHandle& curlHandle, const APIKey& apiKey, HttpRequestType string passphrase = B64Encode(ssl::ShaBin(ssl::ShaType::kSha256, apiKey.passphrase(), apiKey.privateKey())); CurlOptions opts(requestType, std::move(postData), postDataFormat); - opts.appendHttpHeader("KC-API-KEY", apiKey.key()); - opts.appendHttpHeader("KC-API-SIGN", signature); - opts.appendHttpHeader("KC-API-TIMESTAMP", std::string_view(strToSign.data(), nonceSize)); - opts.appendHttpHeader("KC-API-PASSPHRASE", passphrase); - opts.appendHttpHeader("KC-API-KEY-VERSION", 2); + + auto& httpHeaders = opts.mutableHttpHeaders(); + httpHeaders.emplace_back("KC-API-KEY", apiKey.key()); + httpHeaders.emplace_back("KC-API-SIGN", signature); + httpHeaders.emplace_back("KC-API-TIMESTAMP", std::string_view(strToSign.data(), nonceSize)); + httpHeaders.emplace_back("KC-API-PASSPHRASE", passphrase); + httpHeaders.emplace_back("KC-API-KEY-VERSION", 2); json ret = json::parse(curlHandle.query(endpoint, opts)); auto errCodeIt = ret.find("code"); diff --git a/src/api/exchanges/src/upbitprivateapi.cpp b/src/api/exchanges/src/upbitprivateapi.cpp index e09801c3..a445314f 100644 --- a/src/api/exchanges/src/upbitprivateapi.cpp +++ b/src/api/exchanges/src/upbitprivateapi.cpp @@ -88,7 +88,7 @@ json PrivateQuery(CurlHandle& curlHandle, const APIKey& apiKey, HttpRequestType string authStr("Bearer "); authStr.append(token.begin(), token.end()); - opts.appendHttpHeader("Authorization", authStr); + opts.mutableHttpHeaders().emplace_back("Authorization", authStr); json ret = json::parse(curlHandle.query(endpoint, opts)); if (ifError == IfError::kThrow) { diff --git a/src/http-request/include/curloptions.hpp b/src/http-request/include/curloptions.hpp index 9223ce24..521cf5ab 100644 --- a/src/http-request/include/curloptions.hpp +++ b/src/http-request/include/curloptions.hpp @@ -31,6 +31,7 @@ class CurlOptions { } } + HttpHeaders &mutableHttpHeaders() { return _httpHeaders; } const HttpHeaders &httpHeaders() const { return _httpHeaders; } const char *proxyUrl() const { return _proxyUrl; } @@ -51,14 +52,6 @@ class CurlOptions { HttpRequestType requestType() const { return _requestType; } - void clearHttpHeaders() { _httpHeaders.clear(); } - - void appendHttpHeader(std::string_view key, std::string_view value) { _httpHeaders.emplace_back(key, value); } - void appendHttpHeader(std::string_view key, std::integral auto value) { _httpHeaders.emplace_back(key, value); } - - void setHttpHeader(std::string_view key, std::string_view value) { _httpHeaders.set(key, value); } - void setHttpHeader(std::string_view key, std::integral auto value) { _httpHeaders.set(key, value); } - using trivially_relocatable = std::bool_constant && is_trivially_relocatable_v>::type; diff --git a/src/http-request/test/curlhandle_test.cpp b/src/http-request/test/curlhandle_test.cpp index e2f04e2b..70719f85 100644 --- a/src/http-request/test/curlhandle_test.cpp +++ b/src/http-request/test/curlhandle_test.cpp @@ -33,7 +33,7 @@ TEST_F(ExampleBaseCurlHandle, CurlVersion) { EXPECT_FALSE(GetCurlVersionInfo().e TEST_F(ExampleBaseCurlHandle, QueryJsonAndMoveConstruct) { CurlOptions opts = kVerboseHttpGetOptions; - opts.appendHttpHeader("MyHeaderIsVeryLongToAvoidSSO", "Val1"); + opts.mutableHttpHeaders().emplace_back("MyHeaderIsVeryLongToAvoidSSO", "Val1"); EXPECT_NE(handle.query("/json", opts).find("slideshow"), std::string_view::npos); diff --git a/src/tech/include/flat-key-value-string-iterator.hpp b/src/tech/include/flat-key-value-string-iterator.hpp new file mode 100644 index 00000000..2353fd5c --- /dev/null +++ b/src/tech/include/flat-key-value-string-iterator.hpp @@ -0,0 +1,160 @@ +#pragma once + +#include +#include +#include + +namespace cct { + +/// Bi-directional iterator on {key,value} pairs of a FlatKeyValueString. +template +class FlatKeyValueStringIterator { + class FlatKeyValueIteratorValue { + public: + /// Get the key len of this iterator value + auto keyLen() const { return _begValue - _begKey - 1U; } + + /// Get the value len of this iterator value + auto valLen() const { return _endValue - _begValue; } + + /// Access to the key of this iterator value + std::string_view key() const { return {_begKey, static_cast(keyLen())}; } + + /// Access to the value of this iterator value + std::string_view val() const { return {_begValue, _endValue}; } + + /// Get the total size of the key value pair of this iterator value + auto size() const { return _endValue - _begKey; } + + /// a synonym of size() + auto length() const { return size(); } + + private: + template + friend class FlatKeyValueStringIterator; + + template + friend class FlatKeyValueString; + + /// begin() + explicit FlatKeyValueIteratorValue(std::string_view data) + : _begKey(data.data()), _begValue(nullptr), _endValue(nullptr) { + std::size_t assignCharPos = data.find(AssignmentChar); + if (assignCharPos != std::string_view::npos) { + _begValue = _begKey + assignCharPos + 1U; + + std::size_t nextKVCharSep = data.find(KeyValuePairSep, assignCharPos + 1); + _endValue = nextKVCharSep == std::string_view::npos ? data.data() + data.size() : data.data() + nextKVCharSep; + } + } + + /// end() + explicit FlatKeyValueIteratorValue(const char *endData) + : _begKey(endData), _begValue(nullptr), _endValue(nullptr) {} + + void incr(const char *endData) { + if (_endValue == endData) { + // reached the end + _begKey = _endValue; + } else { + _begKey = _endValue + 1; + _begValue = _begKey; + do { + ++_begValue; + } while (*_begValue != AssignmentChar); + ++_begValue; + + _endValue = _begValue; + + while (*_endValue != '\0' && *_endValue != KeyValuePairSep) { + ++_endValue; + } + } + } + + void decr(std::string_view data) { + if (_begKey == data.data() + data.size()) { + _endValue = _begKey; + } else { + _endValue = _begKey - 1; + } + _begValue = _endValue; + do { + --_begValue; + } while (*_begValue != AssignmentChar); + + _begKey = _begValue; + ++_begValue; + + do { + --_begKey; + } while (*_begKey != KeyValuePairSep && _begKey != data.data()); + + if (*_begKey == KeyValuePairSep) { + ++_begKey; + } + } + + const char *_begKey; + const char *_begValue; + const char *_endValue; + }; + + public: + // Needed types for iterators. + using iterator_category = std::bidirectional_iterator_tag; + using value_type = FlatKeyValueIteratorValue; + using difference_type = std::ptrdiff_t; + using pointer = const value_type *; + using reference = const value_type &; + + // Prefix increment, should be called on a valid iterator, otherwise undefined behavior + FlatKeyValueStringIterator &operator++() { + _value.incr(_data.data() + _data.size()); + return *this; + } + + // Postfix increment + FlatKeyValueStringIterator operator++(int) { + auto ret = *this; + ++(*this); + return ret; + } + + // Prefix decrement, should be called on a valid iterator (in range (begin(), end()]), otherwise undefined behavior + FlatKeyValueStringIterator &operator--() { + _value.decr(_data); + return *this; + } + + // Postfix decrement + FlatKeyValueStringIterator operator--(int) { + auto ret = *this; + --(*this); + return ret; + } + + reference operator*() const { return _value; } + + pointer operator->() const { return &this->operator*(); } + + bool operator==(const FlatKeyValueStringIterator &rhs) const noexcept { return _value._begKey == rhs._value._begKey; } + bool operator!=(const FlatKeyValueStringIterator &rhs) const noexcept { return !(*this == rhs); } + + private: + template + friend class FlatKeyValueString; + + /// Create a new FlatKeyValueStringIterator representing begin() + explicit FlatKeyValueStringIterator(std::string_view data) : _data(data), _value(data) {} + + /// Create a new FlatKeyValueStringIterator representing end() + /// bool as second parameter is only here to differentiate both constructors + FlatKeyValueStringIterator(std::string_view data, [[maybe_unused]] bool isEndIt) + : _data(data), _value(_data.data() + _data.size()) {} + + std::string_view _data; + FlatKeyValueIteratorValue _value; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/tech/include/flatkeyvaluestring.hpp b/src/tech/include/flatkeyvaluestring.hpp index ab048659..759cef37 100644 --- a/src/tech/include/flatkeyvaluestring.hpp +++ b/src/tech/include/flatkeyvaluestring.hpp @@ -20,162 +20,12 @@ #include "cct_string.hpp" #include "cct_type_traits.hpp" #include "cct_vector.hpp" +#include "flat-key-value-string-iterator.hpp" #include "unreachable.hpp" #include "url-encode.hpp" namespace cct { -/// Bi-directional iterator on {key,value} pairs of a FlatKeyValueString. -template -class FlatKeyValueStringIterator { - class FlatKeyValueIteratorValue { - public: - /// Get the key len of this iterator value - auto keyLen() const { return _begValue - _begKey - 1U; } - - /// Get the value len of this iterator value - auto valLen() const { return _endValue - _begValue; } - - /// Access to the key of this iterator value - std::string_view key() const { return {_begKey, static_cast(keyLen())}; } - - /// Access to the value of this iterator value - std::string_view val() const { return {_begValue, _endValue}; } - - /// Get the total size of the key value pair of this iterator value - auto size() const { return _endValue - _begKey; } - - /// a synonym of size() - auto length() const { return size(); } - - private: - template - friend class FlatKeyValueStringIterator; - - template - friend class FlatKeyValueString; - - /// begin() - explicit FlatKeyValueIteratorValue(std::string_view data) - : _begKey(data.data()), _begValue(nullptr), _endValue(nullptr) { - std::size_t assignCharPos = data.find(AssignmentChar); - if (assignCharPos != std::string_view::npos) { - _begValue = _begKey + assignCharPos + 1U; - - std::size_t nextKVCharSep = data.find(KeyValuePairSep, assignCharPos + 1); - _endValue = nextKVCharSep == std::string_view::npos ? data.data() + data.size() : data.data() + nextKVCharSep; - } - } - - /// end() - explicit FlatKeyValueIteratorValue(const char *endData) - : _begKey(endData), _begValue(nullptr), _endValue(nullptr) {} - - void incr(const char *endData) { - if (_endValue == endData) { - // reached the end - _begKey = _endValue; - } else { - _begKey = _endValue + 1; - _begValue = _begKey; - do { - ++_begValue; - } while (*_begValue != AssignmentChar); - ++_begValue; - - _endValue = _begValue; - - while (*_endValue != '\0' && *_endValue != KeyValuePairSep) { - ++_endValue; - } - } - } - - void decr(std::string_view data) { - if (_begKey == data.data() + data.size()) { - _endValue = _begKey; - } else { - _endValue = _begKey - 1; - } - _begValue = _endValue; - do { - --_begValue; - } while (*_begValue != AssignmentChar); - - _begKey = _begValue; - ++_begValue; - - do { - --_begKey; - } while (*_begKey != KeyValuePairSep && _begKey != data.data()); - - if (*_begKey == KeyValuePairSep) { - ++_begKey; - } - } - - const char *_begKey; - const char *_begValue; - const char *_endValue; - }; - - public: - // Needed types for iterators. - using iterator_category = std::bidirectional_iterator_tag; - using value_type = FlatKeyValueIteratorValue; - using difference_type = std::ptrdiff_t; - using pointer = const value_type *; - using reference = const value_type &; - - // Prefix increment, should be called on a valid iterator, otherwise undefined behavior - FlatKeyValueStringIterator &operator++() { - _value.incr(_data.data() + _data.size()); - return *this; - } - - // Postfix increment - FlatKeyValueStringIterator operator++(int) { - auto ret = *this; - ++(*this); - return ret; - } - - // Prefix decrement, should be called on a valid iterator (in range (begin(), end()]), otherwise undefined behavior - FlatKeyValueStringIterator &operator--() { - _value.decr(_data); - return *this; - } - - // Postfix decrement - FlatKeyValueStringIterator operator--(int) { - auto ret = *this; - --(*this); - return ret; - } - - reference operator*() const { return _value; } - - pointer operator->() const { return &this->operator*(); } - - bool operator==(const FlatKeyValueStringIterator &rhs) const noexcept { return _value._begKey == rhs._value._begKey; } - bool operator!=(const FlatKeyValueStringIterator &rhs) const noexcept { return !(*this == rhs); } - - private: - template - friend class FlatKeyValueString; - - /// Create a new FlatKeyValueStringIterator representing begin() - explicit FlatKeyValueStringIterator(std::string_view data) : _data(data), _value(data) {} - - /// Create a new FlatKeyValueStringIterator representing end() - /// bool as second parameter is only here to differentiate both constructors - FlatKeyValueStringIterator(std::string_view data, [[maybe_unused]] bool isEndIt) - : _data(data), _value(_data.data() + _data.size()) {} - - std::string_view _data; - FlatKeyValueIteratorValue _value; -}; - /// String Key / Value pairs flattened in a single string. /// It can be used to store URL parameters for instance, or as an optimized key for a map / hashmap based on a list of /// key value pairs. @@ -281,6 +131,9 @@ class FlatKeyValueString { set(key, std::string_view(buf, ret.ptr)); } + /// Like emplace_back, but removes last entry if it has same key as given one. + void set_back(std::string_view key, std::string_view value); + /// Erases given key if present. void erase(std::string_view key); @@ -436,6 +289,19 @@ void FlatKeyValueString::set(std::string_view k } } +template +void FlatKeyValueString::set_back(std::string_view key, std::string_view value) { + if (!_data.empty()) { + auto endIt = --end(); + if (endIt->key() == key) { + _data.replace(_data.end() - endIt->valLen(), _data.end(), value); + return; + } + } + + emplace_back(key, value); +} + template void FlatKeyValueString::erase(std::string_view key) { std::size_t first = find(key); diff --git a/src/tech/test/flatkeyvaluestring_test.cpp b/src/tech/test/flatkeyvaluestring_test.cpp index f69734c0..968386ce 100644 --- a/src/tech/test/flatkeyvaluestring_test.cpp +++ b/src/tech/test/flatkeyvaluestring_test.cpp @@ -31,6 +31,12 @@ TEST(FlatKeyValueStringTest, SetEmpty) { EXPECT_EQ(kvPairs.str(), "timestamp=1621785125200"); } +TEST(FlatKeyValueStringTest, SetBackEmpty) { + KvPairs kvPairs; + kvPairs.set_back("timestamp", "1621785125200"); + EXPECT_EQ(kvPairs.str(), "timestamp=1621785125200"); +} + TEST(FlatKeyValueStringTest, SetAndAppend) { KvPairs kvPairs; kvPairs.emplace_back("abc", "666"); @@ -93,6 +99,14 @@ TEST(FlatKeyValueStringTest, Erase) { EXPECT_TRUE(kvPairs.empty()); } +TEST(FlatKeyValueStringTest, SetBack) { + KvPairs kvPairs{{"abc", "354"}, {"tata", "abc"}, {"rm", "xX"}, {"huhu", "haha"}}; + kvPairs.set_back("abc", "678"); + EXPECT_EQ(kvPairs.str(), "abc=354&tata=abc&rm=xX&huhu=haha&abc=678"); + kvPairs.set_back("abc", "9012"); + EXPECT_EQ(kvPairs.str(), "abc=354&tata=abc&rm=xX&huhu=haha&abc=9012"); +} + TEST(FlatKeyValueStringTest, WithNullTerminatingCharAsSeparator) { using namespace std::literals;