#!/usr/bin/env python3
"""Top up the DOJ press-release corpus from the public justice.gov API (sorted by created date, newest first).

Fetches pages of 50 until every release on a page was created before STOP (a margin before the
2026-09-15 sweep), writes each raw page to <out>/api/page_NNNNN.json and a slim JSONL
(url, title, date, created, component, number, body_text) to <out>/doj_topup_slim.jsonl.gz.
Usage: topup_api.py <out_dir>
"""
import datetime, gzip, html, json, os, re, subprocess, sys, time

API = "https://www.justice.gov/api/v1/press_releases.json?pagesize=50&sort=created&direction=DESC&page={}"
UA = "PandemicDarlings-research/1.0 (pandemicdarlings@gmail.com)"
STOP = int(datetime.datetime(2026, 9, 13, tzinfo=datetime.timezone.utc).timestamp())


def epoch(v):
    m = re.search(r">(\d+)<", v or "")
    return int(m.group(1)) if m else (int(v) if str(v).isdigit() else 0)


def text(h):
    h = re.sub(r"(?is)<(script|style).*?</\1>", " ", h or "")
    return re.sub(r"\s+", " ", html.unescape(re.sub(r"<[^>]+>", " ", h))).strip()


def main(out):
    os.makedirs(os.path.join(out, "api"), exist_ok=True)
    recs, page = [], 0
    while page < 60:
        p = os.path.join(out, "api", "page_%05d.json" % page)
        if not (os.path.exists(p) and os.path.getsize(p) > 1000):
            r = subprocess.run(["curl", "-s", "-m", "40", "-A", UA, API.format(page)], capture_output=True, text=True)
            open(p, "w", encoding="utf-8").write(r.stdout)
            time.sleep(1)
        d = json.load(open(p, encoding="utf-8"))
        rs = d["results"]
        for x in rs:
            recs.append({"url": x.get("url"), "title": (x.get("title") or "").strip(), "date": int(x.get("date") or 0),
                         "created": epoch(x.get("created")),
                         "component": ", ".join(c.get("name", "") for c in (x.get("component") or []) if isinstance(c, dict)),
                         "number": x.get("number") or "", "body_text": text(x.get("body")), "uuid": x.get("uuid")})
        if not rs or max(epoch(x.get("created")) for x in rs) < STOP:
            break
        page += 1
    with gzip.open(os.path.join(out, "doj_topup_slim.jsonl.gz"), "wt", encoding="utf-8") as fh:
        for r in recs:
            fh.write(json.dumps(r, ensure_ascii=False) + "\n")
    f = lambda t: datetime.datetime.utcfromtimestamp(t).strftime("%Y-%m-%dT%H:%MZ")
    print(f"pages={page + 1} records={len(recs)} created {f(min(r['created'] for r in recs))} -> {f(max(r['created'] for r in recs))}")


if __name__ == "__main__":
    main(sys.argv[1])