#!/usr/bin/env python3
"""Scrape Internet Classics Archive HTML section pages for works whose
.mb.txt is only a stub. Writes one Markdown file per work.

Usage: uv run fetch_mit_html.py <jobs.tsv>
jobs.tsv columns: work<TAB>title<TAB>translator
"""
import csv
import html
import pathlib
import re
import subprocess
import sys

BASE = "https://classics.mit.edu/Aristotle/"
NAV = "[Go to home page]"


def get(url: str) -> str:
    proc = subprocess.run(
        ["curl", "-fsSL", "--max-time", "60", "-A", "Mozilla/5.0 (corpus-fetch)", url],
        capture_output=True,
    )
    if proc.returncode != 0:
        raise RuntimeError(proc.stderr.decode("utf-8", "replace")[-160:])
    return proc.stdout.decode("utf-8", "replace")


def sections(work: str) -> list[str]:
    toc = get(f"{BASE}{work}.html")
    seen, out = set(), []
    for href in re.findall(r'HREF="(' + re.escape(work) + r'\.[^"]+\.html)"', toc):
        if href not in seen:
            seen.add(href)
            out.append(href)
    return out or [f"{work}.html"]


def to_text(page_html: str) -> str:
    proc = subprocess.run(
        ["pandoc", "-f", "html", "-t", "plain", "--wrap=none"],
        input=page_html.encode("utf-8"),
        capture_output=True,
    )
    return proc.stdout.decode("utf-8", "replace")


def trim(text: str) -> str:
    lines = text.split("\n")
    starts = [i for i, ln in enumerate(lines) if ln.strip().startswith("Translated by ")]
    start = starts[-1] + 1 if starts else 0
    if not starts:
        dl = [i for i, ln in enumerate(lines) if "text-only version is available" in ln]
        start = dl[-1] + 1 if dl else 0
    ends = [i for i, ln in enumerate(lines) if NAV in ln and i > start]
    end = ends[0] if ends else len(lines)
    body = "\n".join(lines[start:end])
    body = re.sub(r"©\s*1994-2009.*$", "", body, flags=re.M)
    body = re.sub(r"^\s*[-=]{5,}.*$", "", body, flags=re.M)
    body = re.sub(r"\n{3,}", "\n\n", body)
    return body.strip()


def main(jobs_path: str) -> None:
    jobs = list(csv.reader(pathlib.Path(jobs_path).open(), delimiter="\t"))
    out_dir = pathlib.Path("06-aristotle/mit-classics")
    out_dir.mkdir(parents=True, exist_ok=True)
    for work, title, translator in jobs:
        try:
            secs = sections(work)
            chunks = []
            for href in secs:
                chunks.append(trim(to_text(get(BASE + href))))
            body = "\n\n".join(c for c in chunks if c)
        except Exception as exc:  # noqa: BLE001
            print(f"FAIL {work}: {exc}")
            continue
        header = (
            f"# {title}\n\n"
            f"- Author: Aristotle\n"
            f"- Translator: {translator}\n"
            f"- Source: Internet Classics Archive (classics.mit.edu), "
            f"https://classics.mit.edu/Aristotle/{work}.html\n"
            f"- License: Public domain\n\n---\n\n"
        )
        out = out_dir / f"{work.replace('_', '-')}.md"
        out.write_text(header + body + "\n", encoding="utf-8")
        print(f"OK {work} ({len(secs)} sections, {len(body)} chars) -> {out}")


if __name__ == "__main__":
    main(sys.argv[1])