#!/usr/bin/env python3
"""Scrape the Summa Theologiae Supplement (QQ 1-99) from New Advent and write
one Markdown file. The New Advent text is the English Dominican translation,
matching the Project Gutenberg parts already fetched.
"""
import html
import pathlib
import re
import subprocess

BASE = "https://www.newadvent.org/summa/"
ELEMENT = re.compile(r"<(h1|h2|p)\b[^>]*>(.*?)</\1>", re.S | re.I)
STOP = re.compile(r"(The Summa Theologi|CONTACT US|ADVERTISE WITH)")


def get(url: str) -> str:
    proc = subprocess.run(
        ["curl", "-fsSL", "--max-time", "60", "-A", "Mozilla/5.0 (corpus-fetch)", url],
        capture_output=True,
    )
    if proc.returncode != 0:
        raise RuntimeError(proc.stderr.decode("utf-8", "replace")[-160:])
    return proc.stdout.decode("utf-8", "replace")


def text_of(raw: str) -> str:
    txt = re.sub(r"<[^>]+>", "", raw)
    return html.unescape(re.sub(r"\s+", " ", txt)).strip()


def question_md(q: int) -> str:
    page = get(f"{BASE}5{q:03d}.htm")
    out = []
    for tag, raw in ELEMENT.findall(page):
        t = text_of(raw)
        if not t or STOP.search(t):
            continue
        tag = tag.lower()
        if tag == "h1":
            out.append(f"\n## {t}\n")
        elif tag == "h2":
            out.append(f"\n### {t}\n")
        else:
            out.append(t + "\n")
    return "\n".join(out).strip()


def main() -> None:
    parts = [
        "# Summa Theologica — Supplement (Supplementum)",
        "",
        "- Author: Thomas Aquinas",
        "- Translation: Fathers of the English Dominican Province (2nd and revised edition, 1920)",
        "- Source: New Advent, https://www.newadvent.org/summa/5.htm (Questions 1-99)",
        "- License: Public domain",
        "- Note: The Supplement is a posthumous compilation drawn from Aquinas's "
        "commentary on the Sentences; the Summa proper ends unfinished in the Tertia Pars.",
        "",
        "---",
        "",
    ]
    for q in range(1, 100):
        try:
            body = question_md(q)
        except Exception as exc:  # noqa: BLE001
            print(f"FAIL Q{q}: {exc}")
            continue
        parts.append(body)
        print(f"OK Q{q} ({len(body)} chars)")
    out = pathlib.Path("03-aquinas-summa/summa-supplement.md")
    out.write_text("\n\n".join(parts) + "\n", encoding="utf-8")
    print("wrote", out, out.stat().st_size, "bytes")


if __name__ == "__main__":
    main()