#!/usr/bin/env python3
"""Fetch Project Gutenberg plain-text works, strip PG boilerplate, reflow
paragraphs, and write Markdown with a provenance header.

Usage: uv run fetch_gutenberg.py <jobs.tsv>
jobs.tsv columns: pg_id<TAB>out_path<TAB>title<TAB>author
"""
import csv
import pathlib
import re
import subprocess
import sys

START = re.compile(r"\*\*\*\s*START OF (?:THE|THIS) PROJECT GUTENBERG.*?\*\*\*", re.I)
END = re.compile(r"\*\*\*\s*END OF (?:THE|THIS) PROJECT GUTENBERG.*?\*\*\*", re.I)


def fetch(pg_id: int) -> str:
    last = None
    for url in (
        f"https://www.gutenberg.org/cache/epub/{pg_id}/pg{pg_id}.txt",
        f"https://www.gutenberg.org/files/{pg_id}/{pg_id}-0.txt",
        f"https://www.gutenberg.org/files/{pg_id}/{pg_id}.txt",
    ):
        proc = subprocess.run(
            ["curl", "-fsSL", "--max-time", "120", "-A", "Mozilla/5.0 (corpus-fetch)", url],
            capture_output=True,
        )
        if proc.returncode == 0 and proc.stdout:
            return proc.stdout.decode("utf-8", "replace")
        last = proc.stderr.decode("utf-8", "replace")[-200:]
    raise RuntimeError(f"pg{pg_id}: {last}")


def strip_boilerplate(text: str) -> str:
    m = START.search(text)
    if m:
        text = text[m.end():]
    m = END.search(text)
    if m:
        text = text[: m.start()]
    return text.strip()


def reflow(text: str) -> str:
    """Join hard-wrapped lines into paragraphs separated by blank lines."""
    text = text.replace("\r\n", "\n").replace("\r", "\n")
    blocks, cur = [], []
    for line in text.split("\n"):
        if line.strip() == "":
            if cur:
                blocks.append(" ".join(cur).strip())
                cur = []
        else:
            cur.append(line.strip())
    if cur:
        blocks.append(" ".join(cur).strip())
    return "\n\n".join(b for b in blocks if b)


def main(jobs_path: str) -> None:
    jobs = list(csv.reader(pathlib.Path(jobs_path).open(), delimiter="\t"))
    manifest = []
    for pg_id, out_path, title, author in jobs:
        pg_id = int(pg_id)
        out = pathlib.Path(out_path)
        out.parent.mkdir(parents=True, exist_ok=True)
        try:
            body = reflow(strip_boilerplate(fetch(pg_id)))
        except Exception as exc:  # noqa: BLE001
            print(f"FAIL pg{pg_id} {title}: {exc}")
            manifest.append((pg_id, out_path, title, author, "FAIL", ""))
            continue
        header = (
            f"# {title}\n\n"
            f"- Author: {author}\n"
            f"- Source: Project Gutenberg ebook {pg_id} "
            f"(https://www.gutenberg.org/ebooks/{pg_id})\n"
            f"- License: Public domain in the United States\n\n---\n\n"
        )
        out.write_text(header + body + "\n", encoding="utf-8")
        manifest.append((pg_id, out_path, title, author, "OK", str(len(body))))
        print(f"OK pg{pg_id} -> {out} ({len(body)} chars)")
    with pathlib.Path("_manifest_gutenberg.tsv").open("w") as fh:
        for row in manifest:
            fh.write("\t".join(str(x) for x in row) + "\n")


if __name__ == "__main__":
    main(sys.argv[1])