commits
tags
#include "torlinkc/util/net.hpp"
#include <algorithm>
#include <cctype>
#include <chrono>
#include <cmath>
#include <cstdlib>
#include <ctime>
#include <thread>
#include <unordered_set>
#include <curl/curl.h>
namespace torlinkc {
namespace {
const std::unordered_set<int> kRetryStatus = {408, 425, 429, 500, 502, 503, 504};
std::string toLower(std::string s) {
std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return std::tolower(c); });
return s;
}
std::string trim(const std::string& s) {
auto first = s.find_first_not_of(" \t\r\n");
if (first == std::string::npos) return "";
auto last = s.find_last_not_of(" \t\r\n");
return s.substr(first, last - first + 1);
}
bool isAllDigits(const std::string& s) {
return !s.empty() && std::all_of(s.begin(), s.end(), [](unsigned char c) { return std::isdigit(c) != 0; });
}
std::size_t writeBody(char* ptr, std::size_t size, std::size_t nmemb, void* userdata) {
auto* out = static_cast<std::string*>(userdata);
out->append(ptr, size * nmemb);
return size * nmemb;
}
std::size_t writeHeader(char* buffer, std::size_t size, std::size_t nitems, void* userdata) {
auto* headers = static_cast<std::map<std::string, std::string>*>(userdata);
std::string line(buffer, size * nitems);
auto colon = line.find(':');
if (colon != std::string::npos) {
std::string name = toLower(trim(line.substr(0, colon)));
std::string value = trim(line.substr(colon + 1));
(*headers)[name] = value;
}
return size * nitems;
}
// Sleeps up to `ms`, but wakes early (in <= kSliceMs increments) if `token`
// is stop-requested, so a cancelled search doesn't sit out a multi-second
// backoff wait it no longer needs.
void sleepMs(std::int64_t ms, const std::stop_token& token) {
constexpr std::int64_t kSliceMs = 50;
const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms);
while (std::chrono::steady_clock::now() < deadline) {
if (token.stop_requested()) return;
std::this_thread::sleep_for(std::chrono::milliseconds(std::min(kSliceMs, ms)));
}
}
// libcurl's xferinfo callback: returning non-zero aborts the transfer
// in-flight (CURLE_ABORTED_BY_CALLBACK), which is the only way to actually
// interrupt a request already in curl_easy_perform() -- checking the token
// only between attempts would leave a slow single request uncancellable.
int xferInfoCallback(void* clientp, curl_off_t, curl_off_t, curl_off_t, curl_off_t) {
const auto* token = static_cast<const std::stop_token*>(clientp);
return token->stop_requested() ? 1 : 0;
}
} // namespace
std::optional<std::int64_t> parseRetryAfter(const std::string& value) {
if (value.empty()) return std::nullopt;
const std::string trimmed = trim(value);
if (isAllDigits(trimmed)) return std::stoll(trimmed) * 1000;
// curl_getdate parses RFC 1123 / 850 / asctime HTTP-date formats.
time_t date = curl_getdate(trimmed.c_str(), nullptr);
if (date == -1) return std::nullopt;
const std::int64_t nowMs =
std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch())
.count();
return std::max<std::int64_t>(0, static_cast<std::int64_t>(date) * 1000 - nowMs);
}
std::int64_t backoffDelay(int attempt, int baseMs, int capMs, std::optional<std::int64_t> retryAfterMs) {
const double exp = std::min<double>(capMs, static_cast<double>(baseMs) * std::pow(2.0, attempt));
const double jittered = std::floor(static_cast<double>(std::rand()) / RAND_MAX * exp);
if (retryAfterMs) return std::max<std::int64_t>(static_cast<std::int64_t>(jittered), *retryAfterMs);
return static_cast<std::int64_t>(jittered);
}
HttpResponse fetchResilient(const std::string& url, const FetchOptions& opts) {
for (int attempt = 0; attempt <= opts.retries; ++attempt) {
if (opts.stopToken.stop_requested()) throw HttpError(0, "aborted");
CURL* curl = curl_easy_init();
if (!curl) throw HttpError(0, "failed to initialize libcurl");
curl_slist* headerList = nullptr;
for (const auto& h : opts.headers) headerList = curl_slist_append(headerList, h.c_str());
HttpResponse res;
char errbuf[CURL_ERROR_SIZE] = {0};
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_USERAGENT, kUserAgent);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 30L);
if (headerList) curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headerList);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeBody);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &res.body);
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, writeHeader);
curl_easy_setopt(curl, CURLOPT_HEADERDATA, &res.headers);
curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errbuf);
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, xferInfoCallback);
curl_easy_setopt(curl, CURLOPT_XFERINFODATA, &opts.stopToken);
CURLcode code = curl_easy_perform(curl);
curl_slist_free_all(headerList);
if (code != CURLE_OK) {
curl_easy_cleanup(curl);
if (code == CURLE_ABORTED_BY_CALLBACK || opts.stopToken.stop_requested()) throw HttpError(0, "aborted");
if (attempt < opts.retries) {
sleepMs(backoffDelay(attempt, opts.baseMs, opts.capMs), opts.stopToken);
continue;
}
throw HttpError(0, std::string("request to ") + url + " failed: " + errbuf);
}
long status = 0;
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status);
res.status = static_cast<int>(status);
curl_easy_cleanup(curl);
if (!kRetryStatus.count(res.status)) return res;
const std::string server = toLower(res.header("server").value_or(""));
if (res.status == 503 && (server.find("ddos-guard") != std::string::npos ||
server.find("cloudflare") != std::string::npos)) {
throw HttpError(res.status, "request to " + url + " blocked by " + server + " (HTTP " +
std::to_string(res.status) + ")");
}
if (attempt >= opts.retries) {
throw HttpError(res.status, "request to " + url + " failed after " + std::to_string(opts.retries) +
" retries (HTTP " + std::to_string(res.status) + ")");
}
auto retryAfterMs = res.header("retry-after") ? parseRetryAfter(*res.header("retry-after")) : std::nullopt;
sleepMs(backoffDelay(attempt, opts.baseMs, opts.capMs, retryAfterMs), opts.stopToken);
}
throw HttpError(0, "fetchResilient exhausted without a response");
}
} // namespace torlinkc