foxygit / Torlinkc Log in
commits tags

/src/engine/queue.cpp · 20.29 KB

raw
#include "torlinkc/engine/queue.hpp"

#include <algorithm>
#include <chrono>
#include <cmath>
#include <cstdlib>
#include <limits>
#include <thread>

#include "torlinkc/engine/bootguard.hpp"
#include "torlinkc/engine/delete_data.hpp"

namespace torlinkc {

namespace {

constexpr int kStrayTicks = 2;                  // consecutive stray polls before flagging missing (~1s @ 500ms tick)
constexpr std::int64_t kSeedGraceMs = 10'000;   // let libtorrent verify on-disk pieces before watching for strays
constexpr int kFetchMetadataTimeoutMs = 20'000;  // a peerless magnet never fires metadata; give up after this long

std::int64_t nowMs() {
  return std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch())
      .count();
}

int readMaxDownloads() {
  if (const char* v = std::getenv("TORLINK_MAX_DOWNLOADS")) {
    try {
      int n = std::stoi(v);
      if (n > 0) return n;
    } catch (...) {
    }
  }
  return 0;
}

}  // namespace

bool strayDownload(std::int64_t total, double progress, int speed) {
  return total > 0 && progress < 1.0 && speed > 0;
}

DownloadQueue::DownloadQueue(std::optional<int> maxDownloads)
    : maxDownloads_(maxDownloads.value_or(readMaxDownloads())) {}

void DownloadQueue::setTrackers(std::vector<std::string> trackers) { trackers_ = std::move(trackers); }

std::vector<QueueItem> DownloadQueue::getItems() const {
  std::vector<QueueItem> out;
  out.reserve(items_.size());
  for (const auto& [id, it] : items_) out.push_back(it);
  std::sort(out.begin(), out.end(), [](const QueueItem& a, const QueueItem& b) { return a.addedAt > b.addedAt; });
  return out;
}

int DownloadQueue::activeCount() const {
  int n = 0;
  for (const auto& [id, it] : items_)
    if (it.status == DownloadStatus::Downloading) n++;
  return n;
}

bool DownloadQueue::has(const std::string& id) const { return items_.count(id) > 0; }

void DownloadQueue::add(const AddInput& input, const std::string& dir) {
  if (seeds_.count(input.id)) {
    engine_.remove(input.id);
    seeds_.erase(input.id);
    strayHits_.erase(input.id);
    seedStartedAt_.erase(input.id);
    persistSeeds();
  }

  auto existingIt = items_.find(input.id);
  if (existingIt != items_.end() && existingIt->second.status != DownloadStatus::Failed) return;

  QueueItem item;
  if (existingIt != items_.end()) {
    item = existingIt->second;
    // A re-add is a fresh request, so it targets the dir asked for now.
    // Partial data doesn't follow to a new folder, so resume progress only
    // survives when the dir is unchanged.
    const bool sameDir = item.dir == dir;
    item.dir = dir;
    item.error.reset();
    item.speed = 0;
    if (!sameDir) {
      item.progress = 0;
      item.downloadedBytes = 0;
      item.eta.reset();
    }
  } else {
    item.id = input.id;
    item.name = input.name;
    item.source = input.source;
    item.magnet = input.magnet;
    item.dir = dir;
    item.progress = 0;
    item.totalBytes = input.sizeBytes.value_or(0);
    item.downloadedBytes = 0;
    item.speed = 0;
    item.peers = 0;
    item.addedAt = nowMs();
  }

  // Respect the concurrent-download cap: start now if a slot is free, else
  // hold the torrent as "queued" until one frees (see promote()).
  const bool start = maxDownloads_ == 0 || activeCount() < maxDownloads_;
  item.status = start ? DownloadStatus::Downloading : DownloadStatus::Queued;
  items_[item.id] = item;
  if (start) startEngine(items_[item.id]);
  changed();
  persistQueue();
}

void DownloadQueue::startEngine(QueueItem& item) {
  engine_.add(item.id, item.magnet, item.dir, engineHandlers(item.id), trackers_);
}

void DownloadQueue::promote() {
  const int cap = maxDownloads_ == 0 ? std::numeric_limits<int>::max() : maxDownloads_;
  bool started = false;
  while (activeCount() < cap) {
    QueueItem* next = nullptr;
    for (auto& [id, it] : items_) {
      if (it.status != DownloadStatus::Queued) continue;
      if (!next || it.addedAt < next->addedAt) next = &it;
    }
    if (!next) break;
    next->status = DownloadStatus::Downloading;
    next->speed = 0;
    startEngine(*next);
    started = true;
  }
  if (started) {
    changed();
    persistQueue();
  }
}

AddHandlers DownloadQueue::engineHandlers(std::string id) {
  AddHandlers h;
  h.onMetadata = [this, id](const TorrentMeta& meta) {
    // Capture the .torrent metadata as soon as it arrives so a later re-seed
    // can verify the on-disk file locally.
    if (meta.torrentFile) saveTorrentMeta(id, *meta.torrentFile);
    auto it = items_.find(id);
    if (it == items_.end()) return;  // the rest only matters while still downloading
    if (!meta.name.empty()) it->second.name = meta.name;
    if (meta.total > 0) it->second.totalBytes = meta.total;
    it->second.files = meta.files;
    changed();
    persistQueue();
  };
  h.onDone = [this, id]() {
    auto it = items_.find(id);
    if (it != items_.end()) {
      QueueItem completed = it->second;
      if (completed.totalBytes) completed.downloadedBytes = completed.totalBytes;
      completeItem(std::move(completed));
      return;
    }
    // A re-seed (restart / manual resume) passed verification: clear
    // stray-detection state and end its grace period.
    if (seeds_.count(id)) {
      strayHits_[id] = 0;
      seedStartedAt_.erase(id);
    }
  };
  h.onError = [this, id](const std::string& msg) {
    auto it = items_.find(id);
    if (it != items_.end()) {
      it->second.status = DownloadStatus::Failed;
      it->second.error = msg;
      it->second.speed = 0;
      it->second.peers = 0;
      changed();
      persistQueue();
      promote();  // a slot just freed
      return;
    }
    auto sit = seeds_.find(id);
    if (sit != seeds_.end()) {
      sit->second.status = SeedStatus::Missing;
      sit->second.uploadSpeed = 0;
      sit->second.peers = 0;
      seedStartedAt_.erase(id);
      changed();
      persistSeeds();
    }
  };
  return h;
}

void DownloadQueue::completeItem(QueueItem it) {
  recordHistory(it);
  items_.erase(it.id);
  // Opt-out seeding: a finished download is already a complete, verified
  // torrent, so keep it alive and seeding instead of tearing it down.
  beginSeed(it);
  if (onCompleted) onCompleted(it.name);
  changed();
  persistQueue();
  promote();  // a slot just freed
}

void DownloadQueue::beginSeed(const QueueItem& it) {
  if (it.magnet.empty()) return;
  SeedItem s;
  s.id = it.id;
  s.name = it.name;
  s.source = it.source;
  s.magnet = it.magnet;
  s.dir = it.dir;
  s.sizeBytes = it.totalBytes;
  s.status = SeedStatus::Seeding;
  seeds_[it.id] = s;
  strayHits_[it.id] = 0;
  seedStartedAt_[it.id] = nowMs();
  persistSeeds();
}

void DownloadQueue::tick() {
  engine_.pollAlerts();

  bool any = false;
  for (auto& [id, it] : items_) {
    if (it.status != DownloadStatus::Downloading) continue;
    auto s = engine_.stats(id);
    if (!s) continue;
    it.progress = std::min(100, static_cast<int>(std::lround(s->progress * 100)));
    it.downloadedBytes = s->downloaded;
    if (s->total) it.totalBytes = s->total;
    it.speed = s->speed;
    it.peers = s->peers;
    if (s->timeRemaining > 0 && std::isfinite(s->timeRemaining)) {
      it.eta = s->timeRemaining / 1000.0;
    } else {
      it.eta.reset();
    }
    if (!s->name.empty()) it.name = s->name;
    any = true;
  }

  const std::int64_t now = nowMs();
  for (auto& [id, sd] : seeds_) {
    if (sd.status != SeedStatus::Seeding) continue;
    auto s = engine_.stats(id);
    if (!s) continue;
    // Safety-net: a seed that's pulling data has lost its files on disk. Give
    // it a couple of ticks (ignore a one-piece repair blip), then stop it and
    // flag missing, never re-download the whole thing. Skip seeds still
    // inside the grace period: libtorrent needs time to hash-verify on-disk
    // pieces, and during that window progress < 1 with download_rate > 0 is
    // perfectly normal.
    auto startedIt = seedStartedAt_.find(id);
    const std::int64_t age = now - (startedIt != seedStartedAt_.end() ? startedIt->second : 0);
    if (age > kSeedGraceMs && strayDownload(s->total, s->progress, s->speed)) {
      const int hits = (strayHits_.count(id) ? strayHits_[id] : 0) + 1;
      strayHits_[id] = hits;
      if (hits >= kStrayTicks) {
        engine_.remove(id);
        strayHits_.erase(id);
        seedStartedAt_.erase(id);
        sd.status = SeedStatus::Missing;
        sd.uploadSpeed = 0;
        sd.peers = 0;
        persistSeeds();
      }
      any = true;
      continue;
    }
    strayHits_[id] = 0;
    sd.uploadSpeed = s->uploadSpeed;
    sd.uploaded = s->uploaded;
    sd.peers = s->peers;
    any = true;
  }

  if (any) changed();
}

void DownloadQueue::pause(const std::string& id) {
  auto it = items_.find(id);
  if (it == items_.end()) return;
  if (it->second.status != DownloadStatus::Downloading && it->second.status != DownloadStatus::Queued) return;
  const bool wasDownloading = it->second.status == DownloadStatus::Downloading;
  it->second.status = DownloadStatus::Paused;
  it->second.speed = 0;
  it->second.peers = 0;
  it->second.eta.reset();
  if (wasDownloading) engine_.remove(id);
  changed();
  persistQueue();
  if (wasDownloading) promote();  // a slot just freed
}

void DownloadQueue::resume(const std::string& id) {
  auto it = items_.find(id);
  if (it == items_.end() || it->second.status != DownloadStatus::Paused) return;
  const bool start = maxDownloads_ == 0 || activeCount() < maxDownloads_;
  it->second.status = start ? DownloadStatus::Downloading : DownloadStatus::Queued;
  if (start) startEngine(it->second);
  changed();
  persistQueue();
}

void DownloadQueue::togglePause(const std::string& id) {
  auto it = items_.find(id);
  if (it == items_.end()) return;
  if (it->second.status == DownloadStatus::Downloading || it->second.status == DownloadStatus::Queued) pause(id);
  else if (it->second.status == DownloadStatus::Paused) resume(id);
}

std::optional<std::string> DownloadQueue::exportTorrentFile(const std::string& id) {
  if (auto it = items_.find(id); it != items_.end()) return exportTorrentMeta(id, it->second.name, it->second.dir);
  if (auto sit = seeds_.find(id); sit != seeds_.end())
    return exportTorrentMeta(id, sit->second.name, sit->second.dir);
  for (const auto& h : history_) {
    if (h.id == id) return exportTorrentMeta(id, h.name, h.dir);
  }
  return std::nullopt;
}

std::optional<std::string> DownloadQueue::fetchAndExportTorrent(const AddInput& input, const std::string& exportDir) {
  // Fast path: cached from a previous download.
  if (torrentMetaExists(input.id)) return exportTorrentMeta(input.id, input.name, exportDir);
  // Already in the engine (downloading / seeding): its metadata will arrive
  // through the normal queue flow; don't double-add it.
  if (items_.count(input.id) || seeds_.count(input.id)) return std::nullopt;

  const std::string tempKey = "__meta__" + input.id;
  std::optional<TorrentMeta> gotMeta;
  bool gotError = false;

  AddHandlers handlers;
  handlers.onMetadata = [&](const TorrentMeta& meta) { gotMeta = meta; };
  handlers.onError = [&](const std::string&) { gotError = true; };
  engine_.add(tempKey, input.magnet, exportDir, handlers, trackers_);

  const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(kFetchMetadataTimeoutMs);
  while (!gotMeta && !gotError && std::chrono::steady_clock::now() < deadline) {
    engine_.pollAlerts();
    std::this_thread::sleep_for(std::chrono::milliseconds(100));
  }
  // Tear down synchronously before any file data can be written.
  engine_.remove(tempKey);
  if (!gotMeta) return std::nullopt;
  if (gotMeta->torrentFile) saveTorrentMeta(input.id, *gotMeta->torrentFile);
  return exportTorrentMeta(input.id, input.name, exportDir);
}

void DownloadQueue::cancel(const std::string& id) {
  if (!items_.count(id)) return;
  engine_.remove(id);
  items_.erase(id);
  deleteTorrentMeta(id);
  changed();
  persistQueue();
  promote();  // a slot may have freed
}

bool DownloadQueue::remove(const std::string& id, bool deleteFiles) {
  auto it = items_.find(id);
  auto sit = seeds_.find(id);
  const HistoryItem* hist = nullptr;
  for (const auto& h : history_) {
    if (h.id == id) {
      hist = &h;
      break;
    }
  }
  if (it == items_.end() && sit == seeds_.end() && !hist) return false;

  const std::string dir = it != items_.end() ? it->second.dir : sit != seeds_.end() ? sit->second.dir : hist->dir;
  const std::string name = it != items_.end() ? it->second.name : sit != seeds_.end() ? sit->second.name : hist->name;
  const bool hadItem = it != items_.end();

  // Tear down any live engine handle + in-memory record.
  if (it != items_.end() || sit != seeds_.end()) engine_.remove(id);
  if (it != items_.end()) items_.erase(it);
  if (sit != seeds_.end()) {
    seeds_.erase(sit);
    strayHits_.erase(id);
    seedStartedAt_.erase(id);
  }
  deleteTorrentMeta(id);
  removeHistory(id);  // persists history

  if (deleteFiles && !dir.empty() && !name.empty()) deleteSeedData(dir, name);

  changed();
  persistQueue();
  persistSeeds();
  if (hadItem) promote();  // a download slot may have freed
  return true;
}

void DownloadQueue::retry(const std::string& id) {
  auto it = items_.find(id);
  if (it == items_.end() || it->second.status != DownloadStatus::Failed) return;
  it->second.error.reset();
  const bool start = maxDownloads_ == 0 || activeCount() < maxDownloads_;
  it->second.status = start ? DownloadStatus::Downloading : DownloadStatus::Queued;
  if (start) startEngine(it->second);
  changed();
  persistQueue();
}

void DownloadQueue::retryFailed() {
  std::vector<std::string> ids;
  for (const auto& [id, it] : items_) {
    if (it.status == DownloadStatus::Failed) ids.push_back(id);
  }
  for (const auto& id : ids) retry(id);
}

std::optional<SeedItem> DownloadQueue::getSeed(const std::string& id) const {
  auto it = seeds_.find(id);
  if (it == seeds_.end()) return std::nullopt;
  return it->second;
}

std::vector<SeedItem> DownloadQueue::getSeeds() const {
  std::vector<SeedItem> out;
  out.reserve(seeds_.size());
  for (const auto& [id, s] : seeds_) out.push_back(s);
  return out;
}

int DownloadQueue::seedingCount() const {
  int n = 0;
  for (const auto& [id, s] : seeds_)
    if (s.status == SeedStatus::Seeding) n++;
  return n;
}

void DownloadQueue::startSeeding(const HistoryItem& h) {
  if (auto it = seeds_.find(h.id); it != seeds_.end() && it->second.status == SeedStatus::Seeding) return;
  if (items_.count(h.id)) return;  // don't seed a file that's downloading

  SeedItem base;
  base.id = h.id;
  base.name = h.name;
  base.source = h.source;
  base.magnet = h.magnet;
  base.dir = h.dir;
  base.sizeBytes = h.sizeBytes;
  base.status = SeedStatus::Seeding;

  // Only hard guard we can make synchronously and portably: no magnet, no
  // seed. We do NOT guess the on-disk path; we let libtorrent verify the real
  // files and the tick() safety-net flags a missing one.
  if (h.magnet.empty()) {
    SeedItem missing = base;
    missing.status = SeedStatus::Missing;
    seeds_[h.id] = missing;
    changed();
    persistSeeds();
    return;
  }

  seeds_[h.id] = base;
  strayHits_[h.id] = 0;
  seedStartedAt_[h.id] = nowMs();
  // Seed from the stored .torrent metadata when we have it (verifies the
  // local file immediately, no swarm needed); fall back to the magnet
  // otherwise.
  const std::string source = torrentMetaExists(h.id) ? torrentMetaPath(h.id) : h.magnet;
  engine_.add(h.id, source, h.dir, engineHandlers(h.id), trackers_);
  changed();
  persistSeeds();
}

void DownloadQueue::stopSeeding(const std::string& id) {
  auto it = seeds_.find(id);
  if (it == seeds_.end()) return;
  engine_.remove(id);
  strayHits_.erase(id);
  seedStartedAt_.erase(id);
  if (it->second.status == SeedStatus::Seeding) {
    it->second.status = SeedStatus::Paused;
    it->second.uploadSpeed = 0;
    it->second.peers = 0;
  }
  changed();
  persistSeeds();
}

void DownloadQueue::toggleSeeding(const HistoryItem& h) {
  auto it = seeds_.find(h.id);
  if (it != seeds_.end() && it->second.status == SeedStatus::Seeding) stopSeeding(h.id);
  else startSeeding(h);
}

void DownloadQueue::restoreSeeds(const std::vector<SeedRecord>& records, RestoreOptions opts) {
  for (const auto& r : records) {
    const HistoryItem* h = nullptr;
    for (const auto& x : history_) {
      if (x.id == r.id) {
        h = &x;
        break;
      }
    }
    if (!h) continue;
    if (r.status == SeedStatus::Seeding && !opts.safe) startSeeding(*h);
    else restorePaused(*h);
  }
  if (opts.safe) persistSeeds();
}

void DownloadQueue::restorePaused(const HistoryItem& h) {
  if (seeds_.count(h.id)) return;
  SeedItem s;
  s.id = h.id;
  s.name = h.name;
  s.source = h.source;
  s.magnet = h.magnet;
  s.dir = h.dir;
  s.sizeBytes = h.sizeBytes;
  s.status = SeedStatus::Paused;
  seeds_[h.id] = s;
  changed();
}

std::vector<SeedRecord> DownloadQueue::seedRecords() const {
  std::vector<SeedRecord> out;
  out.reserve(seeds_.size());
  for (const auto& [id, s] : seeds_) {
    out.push_back(SeedRecord{s.id, s.status == SeedStatus::Seeding ? SeedStatus::Seeding : SeedStatus::Paused});
  }
  return out;
}

void DownloadQueue::persistSeeds() const { saveSeeds(seedRecords()); }

void DownloadQueue::persistQueue() const { saveQueue(getItems()); }

void DownloadQueue::restore(std::vector<QueueItem> items, RestoreOptions opts) {
  if (opts.safe) {
    // Engines stay cold: pause everything that would have started and keep
    // the rest as saved, then persist so the paused state is the new truth.
    for (auto& raw : items) {
      if (raw.status == DownloadStatus::Downloading || raw.status == DownloadStatus::Queued)
        raw.status = DownloadStatus::Paused;
      items_[raw.id] = raw;
    }
    changed();
    persistQueue();
    return;
  }

  int active = 0;
  for (auto& raw : items) {
    items_[raw.id] = raw;
    if (raw.status != DownloadStatus::Downloading) continue;
    if (maxDownloads_ == 0 || active < maxDownloads_) {
      startEngine(items_[raw.id]);
      active++;
    } else {
      // Over the cap on boot -> hold as queued (promoted as slots free).
      items_[raw.id].status = DownloadStatus::Queued;
    }
  }
  changed();
  // Fill any remaining slots from persisted "queued" items.
  promote();
}

void DownloadQueue::restoreHistory(std::vector<HistoryItem> items) {
  if (items.size() > static_cast<std::size_t>(kHistoryCap)) items.resize(kHistoryCap);
  history_ = std::move(items);
}

std::vector<HistoryItem> DownloadQueue::getHistory() const { return history_; }

void DownloadQueue::recordHistory(const QueueItem& it) {
  HistoryItem rec;
  rec.id = it.id;
  rec.name = it.name;
  rec.source = it.source;
  rec.sizeBytes = it.totalBytes;
  rec.magnet = it.magnet;
  rec.dir = it.dir;
  rec.completedAt = nowMs();

  history_.erase(std::remove_if(history_.begin(), history_.end(), [&](const HistoryItem& h) { return h.id == it.id; }),
                 history_.end());
  history_.insert(history_.begin(), rec);
  if (history_.size() > static_cast<std::size_t>(kHistoryCap)) history_.resize(kHistoryCap);
  saveHistory(history_);
}

void DownloadQueue::removeHistory(const std::string& id) {
  const std::size_t before = history_.size();
  history_.erase(std::remove_if(history_.begin(), history_.end(), [&](const HistoryItem& h) { return h.id == id; }),
                 history_.end());
  if (history_.size() == before) return;

  if (seeds_.count(id)) {
    engine_.remove(id);
    seeds_.erase(id);
    strayHits_.erase(id);
    seedStartedAt_.erase(id);
    persistSeeds();
  }
  deleteTorrentMeta(id);
  saveHistory(history_);
  changed();
}

void DownloadQueue::clearHistory() {
  if (history_.empty()) return;
  for (const auto& h : history_) deleteTorrentMeta(h.id);
  history_.clear();
  if (!seeds_.empty()) {
    for (const auto& [id, s] : seeds_) engine_.remove(id);
    seeds_.clear();
    strayHits_.clear();
    seedStartedAt_.clear();
    persistSeeds();
  }
  saveHistory(history_);
  changed();
}

void DownloadQueue::changed() {
  if (onUpdate) onUpdate();
}

void DownloadQueue::persistSync() {
  saveQueue(getItems());
  saveHistory(history_);
  saveSeeds(seedRecords());
  // A clean flush doubles as proof this run did not die mid-restore, so the
  // crash-boot breaker stands down.
  disarmBootMarker();
}

void DownloadQueue::suspend() {
  // Keep active downloads as "downloading" so restore() resumes them on the
  // next launch; just zero the live stats.
  for (auto& [id, it] : items_) {
    if (it.status == DownloadStatus::Downloading) {
      it.speed = 0;
      it.peers = 0;
      it.eta.reset();
    }
  }
  persistSync();
  engine_.destroy();
}

}  // namespace torlinkc