#!/usr/bin/env python3
"""Load the DOJ press-release corpus: main harvest + 2026-09-15 sweep + API top-up, deduplicated by URL (last copy wins)."""
import gzip, json, os, datetime


def load(raw_dir, topup):
    files = [os.path.join(raw_dir, "doj_press_releases_2020-2026_slim.jsonl.gz"),
             os.path.join(raw_dir, "doj_press_releases_2026-07-04_2026-09-15_slim.jsonl.gz"), topup]
    seen = {}
    for f in files:
        with gzip.open(f, "rt", encoding="utf-8") as fh:
            for line in fh:
                r = json.loads(line)
                u = (r.get("url") or "").strip().rstrip("/").lower()
                if u:
                    seen[u] = r
    out = []
    for u, r in seen.items():
        d = r.get("date")
        try:
            dt = datetime.datetime.utcfromtimestamp(int(d)).strftime("%Y-%m-%d")
        except Exception:
            dt = ""
        out.append({"url": r.get("url"), "title": (r.get("title") or "").strip(), "date": dt,
                    "component": r.get("component") or "", "body": r.get("body_text") or ""})
    return out