foxygit / RPA-Remisser Log in
commits tags

/rpa/scraper.py · 2.05 KB

raw
import hashlib
import re
from pathlib import Path
from typing import Optional

from playwright.sync_api import Page


def _stable_id(*parts: str) -> str:
    raw = "|".join(p.strip() for p in parts if p)
    return hashlib.sha1(raw.encode("utf-8")).hexdigest()[:16]


def list_egenremisser(page: Page) -> list[dict]:
    """Extraherar egenremiss-poster från nuvarande sida.

    Generisk, layout-agnostisk extraktion: letar efter listobjekt/tabellrader
    eller <article>-liknande block i huvudinnehållet. Varje post sparas med
    sin text och ev. länkar (för dokument/detaljer). Justera CSS-selectorn
    "main li, main tr, main article" efter vad --discover visar.
    """
    items = []
    candidates = page.locator("main li, main tr, main article")
    count = candidates.count()
    for i in range(count):
        node = candidates.nth(i)
        text = node.inner_text().strip()
        if not text or len(text) < 3:
            continue

        hrefs = node.locator("a").evaluate_all(
            "els => els.map(e => e.href)"
        )

        item = {
            "id": _stable_id(text),
            "text": text,
            "links": hrefs,
        }
        items.append(item)
    return items


def download_attachments(page: Page, item: dict, download_dir: Path) -> list[str]:
    """Laddar ner ev. dokument/PDF-länkar kopplade till en egenremiss-post."""
    saved_paths = []
    doc_links = [
        href
        for href in item.get("links", [])
        if re.search(r"\.pdf($|\?)", href, re.I)
        or re.search(r"dokument|bilaga|attachment", href, re.I)
    ]

    for href in doc_links:
        try:
            with page.expect_download(timeout=15000) as dl_info:
                page.goto(href)
            download = dl_info.value
            filename = f"{item['id']}_{download.suggested_filename}"
            target = download_dir / filename
            download.save_as(str(target))
            saved_paths.append(str(target))
        except Exception as exc:  # noqa: BLE001
            print(f"  Kunde inte ladda ner {href}: {exc}")

    return saved_paths