commits
tags
#include "torlinkc/sources/rss.hpp"
#include <regex>
#include <unordered_set>
#include "torlinkc/util/date_parse.hpp"
#include "torlinkc/util/net.hpp"
#include "torlinkc/util/url_encode.hpp"
namespace torlinkc {
namespace {
constexpr int kWpFeedPageSize = 10;
constexpr int kFeedDepth = 3;
constexpr int kDeepPageRetries = 2;
std::string trim(const std::string& s) {
auto first = s.find_first_not_of(" \t\r\n");
if (first == std::string::npos) return "";
auto last = s.find_last_not_of(" \t\r\n");
return s.substr(first, last - first + 1);
}
std::size_t countOccurrences(const std::string& haystack, const std::string& needle) {
std::size_t count = 0, pos = 0;
while ((pos = haystack.find(needle, pos)) != std::string::npos) {
count++;
pos += needle.size();
}
return count;
}
std::optional<std::string> matchFirst(const std::string& s, const std::regex& re) {
std::smatch m;
if (!std::regex_search(s, m, re)) return std::nullopt;
return m[1].str();
}
std::vector<TorrentResult> parseRssItems(const std::string& xml, const std::string& source) {
static const std::regex kMagnetHref(R"re(href="(magnet:\?xt=urn:btih:[^"]+)")re", std::regex::icase);
static const std::regex kInfoHash(R"(urn:btih:([a-zA-Z0-9]+))");
static const std::regex kTitle(R"(<title>(.*?)</title>)");
static const std::regex kPubDate(R"(<pubDate>(.*?)</pubDate>)");
std::vector<TorrentResult> out;
const std::string marker = "<item>";
std::size_t pos = xml.find(marker);
while (pos != std::string::npos) {
std::size_t start = pos + marker.size();
std::size_t next = xml.find(marker, start);
const std::string item = xml.substr(start, next == std::string::npos ? std::string::npos : next - start);
pos = next;
auto magnetRaw = matchFirst(item, kMagnetHref);
if (!magnetRaw) continue;
const std::string magnet = unescapeEntities(*magnetRaw);
auto hashMatch = matchFirst(magnet, kInfoHash);
if (!hashMatch) continue;
std::string infoHash = *hashMatch;
for (auto& c : infoHash) c = static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
if (infoHash.empty()) continue;
const std::string name = unescapeEntities(matchFirst(item, kTitle).value_or("Unknown Title"));
const std::optional<std::int64_t> added = parseDateToUnixSeconds(matchFirst(item, kPubDate).value_or(""));
TorrentResult r;
r.infoHash = infoHash;
r.name = name;
r.source = source;
r.magnet = magnet;
r.added = added;
out.push_back(std::move(r));
}
return out;
}
std::string feedUrl(const std::string& base, const std::string& query, int page) {
const std::string q = trim(query);
std::string url = q.empty() ? base + "/feed/" : base + "/?s=" + encodeURIComponent(q) + "&feed=rss2";
if (page <= 1) return url;
return url + (q.empty() ? "?" : "&") + "paged=" + std::to_string(page);
}
std::string fetchFeedPage(const std::string& url, const std::string& source, const SearchOptions& opts,
std::optional<int> retries) {
FetchOptions fetchOpts;
fetchOpts.stopToken = opts.stopToken;
if (retries) fetchOpts.retries = *retries;
HttpResponse res = fetchResilient(url, fetchOpts);
if (!res.ok()) throw HttpError(res.status, source + " feed returned " + std::to_string(res.status));
return res.body;
}
} // namespace
std::string unescapeEntities(const std::string& s) {
static const std::pair<std::regex, const char*> kReplacements[] = {
{std::regex(R"(�?38;|&)"), "&"}, {std::regex(R"(–|—)"), "-"},
{std::regex(R"(’|�?39;|')"), "'"}, {std::regex(R"(“|”|")"), "\""},
{std::regex(R"(<)"), "<"}, {std::regex(R"(>)"), ">"},
};
std::string out = s;
for (const auto& [re, replacement] : kReplacements) out = std::regex_replace(out, re, replacement);
return out;
}
std::vector<TorrentResult> fetchWordpressRss(const std::string& base, const std::string& sourceId,
const std::string& query, const SearchOptions& opts) {
const std::string first = fetchFeedPage(feedUrl(base, query, 1), sourceId, opts, std::nullopt);
std::vector<TorrentResult> results = parseRssItems(first, sourceId);
if (countOccurrences(first, "<item>") < static_cast<std::size_t>(kWpFeedPageSize)) return results;
std::unordered_set<std::string> seen;
for (const auto& r : results) seen.insert(r.infoHash);
// Fetched sequentially rather than in parallel (the TS original uses
// Promise.all for these): this already runs on its own per-source thread
// in the Phase 3 SearchAggregator, so a little extra latency here doesn't
// block anything else, and it avoids spinning up nested worker threads for
// what's a rarely-exercised deep-pagination path.
for (int page = 2; page < kFeedDepth + 1; ++page) {
try {
const std::string xml = fetchFeedPage(feedUrl(base, query, page), sourceId, opts, kDeepPageRetries);
for (auto& r : parseRssItems(xml, sourceId)) {
if (seen.insert(r.infoHash).second) results.push_back(std::move(r));
}
} catch (const std::exception&) {
// A deeper page failing doesn't invalidate what page 1 already found.
}
}
return results;
}
} // namespace torlinkc