#!/usr/bin/env python3
"""Download archive.org OCR plain text (_djvu.txt) for a list of items, do
light OCR cleanup, reflow into paragraphs, and write Markdown.

Usage: uv run fetch_archive.py <jobs.tsv>
jobs.tsv columns: identifier<TAB>out_path<TAB>title<TAB>author
"""
import csv
import pathlib
import re
import subprocess
import sys

IA_HEADER = re.compile(
    r"^\s*(This is a digital copy of a book|Digitized by the Internet Archive|"
    r"https?://archive\.org/details/|Public Domain|http://www\.archive\.org/details/)",
    re.I,
)


def resolve_item(identifier: str) -> tuple[str, str, str]:
    """Return (djvu_name, server, dir) for an archive.org item."""
    proc = subprocess.run(
        ["curl", "-fsSL", "--max-time", "90", "-A", "Mozilla/5.0 (corpus-fetch)",
         f"https://archive.org/metadata/{identifier}"],
        capture_output=True,
    )
    if proc.returncode == 0 and proc.stdout:
        import json

        meta = json.loads(proc.stdout)
        name = next(
            (f["name"] for f in meta.get("files", []) if f["name"].endswith("_djvu.txt")),
            f"{identifier}_djvu.txt",
        )
        return name, meta.get("server") or "", meta.get("dir") or ""
    return f"{identifier}_djvu.txt", "", ""


def fetch_djvu(identifier: str) -> str:
    name, server, item_dir = resolve_item(identifier)
    urls = [f"https://archive.org/download/{identifier}/{name}"]
    if server and item_dir:
        urls.append(f"https://{server}{item_dir}/{name}")
    last = ""
    for url in urls:
        for _ in range(3):
            proc = subprocess.run(
                ["curl", "-fsSL", "--max-time", "300", "-A", "Mozilla/5.0 (corpus-fetch)", url],
                capture_output=True,
            )
            if proc.returncode == 0 and proc.stdout:
                return proc.stdout.decode("utf-8", "replace")
            last = proc.stderr.decode("utf-8", "replace")[-200:]
    raise RuntimeError(f"{identifier}: {last}")


def dehyphenate(text: str) -> str:
    # join words split across a line break: "exam-\nple" -> "example"
    return re.sub(r"(\w)-\n(\w)", r"\1\2", text)


def reflow(text: str) -> str:
    text = text.replace("\r\n", "\n").replace("\r", "\n")
    lines = [ln for ln in text.split("\n") if not IA_HEADER.match(ln)]
    blocks, cur = [], []
    for line in lines:
        if line.strip() == "":
            if cur:
                blocks.append(" ".join(cur).strip())
                cur = []
        else:
            cur.append(line.strip())
    if cur:
        blocks.append(" ".join(cur).strip())
    return "\n\n".join(b for b in blocks if b)


def main(jobs_path: str) -> None:
    jobs = list(csv.reader(pathlib.Path(jobs_path).open(), delimiter="\t"))
    manifest = []
    for identifier, out_path, title, author in jobs:
        out = pathlib.Path(out_path)
        out.parent.mkdir(parents=True, exist_ok=True)
        try:
            body = reflow(dehyphenate(fetch_djvu(identifier)))
        except Exception as exc:  # noqa: BLE001
            print(f"FAIL {identifier} {title}: {exc}")
            manifest.append((identifier, out_path, title, author, "FAIL", ""))
            continue
        header = (
            f"# {title}\n\n"
            f"- Author: {author}\n"
            f"- Source: Internet Archive item `{identifier}` "
            f"(https://archive.org/details/{identifier})\n"
            f"- Note: OCR plain text from a scanned public-domain edition; "
            f"minor OCR artifacts may remain.\n\n---\n\n"
        )
        out.write_text(header + body + "\n", encoding="utf-8")
        manifest.append((identifier, out_path, title, author, "OK", str(len(body))))
        print(f"OK {identifier} -> {out} ({len(body)} chars)")
    with pathlib.Path("_manifest_archive.tsv").open("w") as fh:
        for row in manifest:
            fh.write("\t".join(str(x) for x in row) + "\n")


if __name__ == "__main__":
    main(sys.argv[1])