#!/usr/bin/env python3
"""Fetch the Internet Classics Archive (classics.mit.edu) text-only files
(<work>.mb.txt), strip the provider header/footer, reflow, write Markdown.

Usage: uv run fetch_mit.py <jobs.tsv>
jobs.tsv columns: author_dir<TAB>work<TAB>title<TAB>translator
"""
import csv
import pathlib
import re
import subprocess
import sys

FOOTER = re.compile(r"-{10,}\s*\n.*?(?:copyright|Copyright).*$", re.S)


def fetch(author_dir: str, work: str) -> str:
    url = f"https://classics.mit.edu/{author_dir}/{work}.mb.txt"
    proc = subprocess.run(
        ["curl", "-fsSL", "--max-time", "120", "-A", "Mozilla/5.0 (corpus-fetch)", url],
        capture_output=True,
    )
    if proc.returncode != 0 or not proc.stdout:
        raise RuntimeError(f"{author_dir}/{work}: {proc.stderr.decode('utf-8', 'replace')[-160:]}")
    return proc.stdout.decode("utf-8", "replace")


def clean(text: str) -> str:
    lines = text.split("\n")
    # drop the leading "Provided by ... / See bottom ... / Available online ..." block
    while lines and (
        lines[0].startswith("Provided by")
        or lines[0].startswith("See bottom")
        or lines[0].strip().startswith("http")
        or lines[0].strip() == ""
    ):
        lines.pop(0)
    text = "\n".join(lines)
    text = FOOTER.sub("", text)
    # reflow paragraphs
    blocks, cur = [], []
    for line in text.replace("\r\n", "\n").split("\n"):
        if line.strip() == "":
            if cur:
                blocks.append(" ".join(cur).strip())
                cur = []
        else:
            cur.append(line.strip())
    if cur:
        blocks.append(" ".join(cur).strip())
    return "\n\n".join(b for b in blocks if b)


def main(jobs_path: str) -> None:
    jobs = list(csv.reader(pathlib.Path(jobs_path).open(), delimiter="\t"))
    manifest = []
    for author_dir, work, title, translator in jobs:
        out = pathlib.Path("06-aristotle/mit-classics") / f"{work.replace('_', '-')}.md"
        out.parent.mkdir(parents=True, exist_ok=True)
        try:
            body = clean(fetch(author_dir, work))
        except Exception as exc:  # noqa: BLE001
            print(f"FAIL {work}: {exc}")
            manifest.append((work, str(out), "FAIL", ""))
            continue
        header = (
            f"# {title}\n\n"
            f"- Author: Aristotle\n"
            f"- Translator: {translator}\n"
            f"- Source: Internet Classics Archive (classics.mit.edu), "
            f"https://classics.mit.edu/{author_dir}/{work}.html\n"
            f"- License: Public domain\n\n---\n\n"
        )
        out.write_text(header + body + "\n", encoding="utf-8")
        manifest.append((work, str(out), "OK", str(len(body))))
        print(f"OK {work} -> {out} ({len(body)} chars)")
    with pathlib.Path("_manifest_mit.tsv").open("w") as fh:
        for row in manifest:
            fh.write("\t".join(row) + "\n")


if __name__ == "__main__":
    main(sys.argv[1])