#!/usr/bin/env python3 """Rules for F2 (DOJ pandemic-relief enforcement releases). Stdlib only; imported by reproduce_enforcement.py. scope(): does a release concern a pandemic relief program? (program named in title or body, after the COVID-19 Fraud Enforcement Task Force and fraud-hotline boilerplate is removed) stage(): the release's primary stage from its headline, falling back to the first 400 characters of the body: SENTENCED > CONVICTED (trial) > PLEA > CHARGED; CIVIL_RESOLUTION (settlement, consent judgment, agreement to pay) and CIVIL_ACTION (civil complaint or suit filed); FORFEITURE; OTHER. defendants(): distinct people named in the DOJ style "Name, 45, of Place". settlement_usd(): the amount a civil resolution says will be paid. """ import re BOILER = [ re.compile(r"On May 17, 2021, the Attorney General established the COVID-19 Fraud Enforcement Task Force.*?(?:coronavirus\.|$)", re.S | re.I), re.compile(r"Anyone with information about allegations of (?:attempted )?fraud involving COVID-19.*?(?:\.gov/ncdf|disaster@leo\.gov|\.gov\.|$)", re.S | re.I), re.compile(r"For more information on the (?:Department|department)['’]s response to the pandemic.*?coronavirus\.?", re.S), ] PROG = [ ("PPP", re.compile(r"(?i:paycheck protection)|\bPPP\b")), ("EIDL", re.compile(r"(?i)economic injury disaster|\bEIDL\b")), ("UI", re.compile(r"(?i)pandemic unemployment|federal pandemic unemployment compensation|\bFPUC\b|lost wages assistance|\bPUA\b")), ("ERC", re.compile(r"(?i)employee retention (?:tax )?credit|\bERTC\b")), ("RRF", re.compile(r"(?i)restaurant revitalization")), ("SVOG", re.compile(r"(?i)shuttered venue")), ("PRF", re.compile(r"(?i)provider relief fund|covid-19 uninsured program|hrsa (?:covid-19 )?uninsured")), ("OTHER_RELIEF", re.compile(r"(?i)coronavirus relief fund|emergency rental assistance|economic impact payments?|CARES Act (?:funds?|funding|loans?|benefits|relief|money|grants?|dollars|programs?)|Coronavirus Aid, Relief,? and Economic Security|sick and family leave (?:tax )?credits?|qualified (?:sick|family) leave wages|\bFFCRA\b|Families First Coronavirus Response Act|American Rescue Plan|\bESSER\b|Elementary and Secondary School Emergency Relief|(?:State and Local )?Fiscal Recovery Funds?|Payroll Support Program|Coronavirus Food Assistance Program|\bCFAP\b|Feeding Our Future|COVID-19 relief (?:funds|loans?|benefits|programs?)|pandemic relief (?:funds|loans?|benefits|programs?)|COVID[- ]era (?:loan|relief|benefit)|(?:covid(?:-19)?|pandemic|coronavirus)[- ](?:relief|era)\s+(?:fraud|loans?|funds?|benefits|money|programs?|assistance|aid|grants?)|(?:covid(?:-19)?|pandemic|coronavirus)[- ]relief\b")), ] RENTAL = re.compile(r"(?i)rental assistance|(?:tenants?|rent)\s+(?:relief|assistance)|help tenants") UI_GENERIC = re.compile(r"(?i)unemployment (?:insurance|benefits|compensation|assistance|claims?)") COVID = re.compile(r"(?i)covid|pandemic|coronavirus|CARES Act") CFETF = re.compile(r"(?i)COVID-19 Fraud Enforcement Task Force") T_SENT = re.compile(r"(?i)\bsentenc|\b(?:to|in|behind) (?:federal )?(?:state )?(?:prison|jail)\b|\bgets? (?:\d+ (?:years?|months?)|(?:federal )?prison)|\bimprisonment\b|\bjailed\b|\bprobation\b|\bordered to (?:serve|repay)\b|\bordered to pay (?:more than |over |nearly )?(?:\$[\d.,]+ ?(?:million|m)? ?in )?restitution") T_TRIAL = re.compile(r"(?i)found guilty|\bjury (?:convicts|finds|verdict|returns)|\bconvicted (?:at|after|following|by a (?:federal )?jury)|\bverdict\b|\bconvicted\b|\bconviction\b|\bguilty\b") T_PLEA = re.compile(r"(?i)\bplead(?:s|ed)? guilty|\bpled guilty|\bguilty plea|\bpleads\b|\badmits?\b|\bagrees? to plead") T_CHARGE = re.compile(r"(?i)\bextradit|\bcharged\b|\bcharges?\b|\bindict|\barrest|\bcriminal complaint|\baccused\b|\bsurrender|\bfaces? (?:federal )?(?:charges|counts)|\bcharging\b") T_CIVRES = re.compile(r"(?i)\bsettle|\bagrees? to pay|\bagreed to pay|\bto pay \$|\bpays? \$[\d.,]+ (?:million )?to resolve|\bto resolve (?:civil|false claims|allegations|claims)|\bresolves? (?:civil|false claims|allegations|claims)|\bconsent judge?ment|\bcivil (?:penalt|judge?ment)|\bjudge?ment (?:against|ordered|entered)|\bordered to pay|\bdamages\b|\bfalse claims act (?:settlement|judgment|liability)") T_CIVACT = re.compile(r"(?i)\bcivil (?:complaint|lawsuit|suit|action|fraud (?:lawsuit|suit|complaint))|\bfiles? (?:a )?(?:false claims act|civil)|\bfalse claims act (?:complaint|lawsuit|suit)|\bsues\b|\bintervenes?\b|\blawsuit\b") T_FORF = re.compile(r"(?i)civil forfeiture|forfeit|\bseize|\bseizure|\brecovers? \$|\bmoney (?:is )?returned") STRONG_ROUNDUP = re.compile(r"(?i)calendar year|achievements|highlights key|in civil settlements and judgments|takedown|crack ?down|enforcement action|announces results|\bresults of\b|joins .{0,80}(?:fraud division|surge|takedown|task force)|takes (?:sweeping )?action|this week in fraud|\bfiscal year\b|annual report|milestone|year in review|\bsurpass|settlements and judgments exceed|\bresign|\bdeparture\b|\breadout\b|\bMOU\b|memorandum of understanding|task force (?:secures|announces|provides)|(?:civil and criminal|criminal and civil) (?:actions|collections)") WEAK_ROUNDUP = re.compile(r"(?i)\bcollect(?:s|ions?)\b|\bwarns?\b|\bremind|\bbeware\b|\bguidance\b|\bpartnership\b|\blaunch|\bstrateg|\bawards?\b|\bgrants?\b|\bscams?\b|\bvigilant\b|\balert\b") TRANSL = re.compile(r"(?i)\b(?:años|acusad[oa]s?|condenad[oa]s?|sentenciad[oa]s?|culpable|declara(?:do|da|n)?|fraude|préstamos?|prisión|denúncia|crimes|anos|meses|réu|por|para|con|del|los|las|el|en|em|um|uma|e|y)\b") CRIM = re.compile(r"(?i)\bsentenc|plead|pled guilty|guilty plea|convicted|found guilty|indict|arrest|criminal|\bprison\b|\bjail\b|restitution") def clean(body): b = body or "" for rx in BOILER: b = rx.sub(" ", b) return b def scope(title, body): """Return the list of programs named; empty list = out of scope.""" t = (title or "") + " \n " + clean(body) progs = [name for name, rx in PROG if rx.search(t)] if not progs and UI_GENERIC.search(t) and (COVID.search(t) or CFETF.search(body or "")): progs.append("UI") if "OTHER_RELIEF" not in progs and RENTAL.search(t) and COVID.search(t): progs.append("OTHER_RELIEF") return progs def _stage_of(s): if not s: return None civres = bool(T_CIVRES.search(s)); crim = bool(CRIM.search(s)) if civres and not crim: return "CIVIL_RESOLUTION" if T_SENT.search(s): return "SENTENCED" if T_PLEA.search(s): return "PLEA" if T_TRIAL.search(s): return "CONVICTED" if T_CHARGE.search(s): if T_CIVACT.search(s) and not crim: return "CIVIL_ACTION" return "CHARGED" if civres: return "CIVIL_RESOLUTION" if re.search(r"(?i)civil forfeiture", s): return "FORFEITURE" if T_CIVACT.search(s): return "CIVIL_ACTION" if T_FORF.search(s): return "FORFEITURE" return None def lede(body): b = re.sub(r"\s+", " ", body or "").strip() return b[:400] def is_translation(title): return len(TRANSL.findall(title or "")) >= 3 def stage(title, body): """(stage, basis): basis 'title', 'lede', 'roundup' or 'none'. Round-ups, warnings and office news are OTHER.""" t = title or "" if STRONG_ROUNDUP.search(t): return "OTHER", "roundup" s = _stage_of(t) if s: return s, "title" if WEAK_ROUNDUP.search(t): return "OTHER", "roundup" s = _stage_of(lede(body)) if s: return s, "lede" return "OTHER", "none" NAME = r"[A-Z][A-Za-z'’\-]+(?:\s+(?:[A-Z][A-Za-z'’\-]+|[A-Z]\.|de|del|de la|la|van|von|da|dos|di|le|St\.|Jr\.|Sr\.|II|III|IV)){1,5}" DEF1 = re.compile(r"(" + NAME + r"),\s+(?:age\s+)?(\d{2}),\s+(?:of|from|a resident of|formerly of|a former|a citizen of|who|and\b|both\b|all\b|was\b|were\b|is\b|has\b|an?\b)") DEF2 = re.compile(r"(" + NAME + r")\s+\((?:age\s+)?(\d{2})[,)]") DEF3 = re.compile(r"(" + NAME + r")\s+\([“\"][^”\"]{1,40}[”\"]\)\s*,\s+(?:age\s+)?(\d{2})") STOP = {"The", "A", "An", "U.S.", "United", "Attorney", "According", "In", "On", "Assistant", "Special", "Chief", "Judge"} def defendants(body): names = set() for rx in (DEF1, DEF2, DEF3): for m in rx.finditer(body or ""): nm = m.group(1).strip() toks = nm.split() while toks and toks[0] in STOP: toks = toks[1:] if len(toks) >= 2: names.add(" ".join(toks).lower().strip(" ,")) return sorted(names) MULT = {"thousand": 1e3, "million": 1e6, "billion": 1e9, "k": 1e3, "m": 1e6, "b": 1e9} AMT = r"\$\s?([\d]{1,3}(?:,\d{3})+(?:\.\d+)?|\d+(?:\.\d+)?)\s*(thousand|million|billion|[MmBbKk](?![A-Za-z]))?" PAY = re.compile(r"(?i)(?:agree(?:s|d)? to pay|will pay|to pay|paid|pays|paying|settle(?:s|d)? for|settlement (?:of|for|totaling)|judgment (?:of|for|totaling))\s+(?:the United States\s+|the government\s+|a total of\s+|more than\s+|over\s+|approximately\s+|nearly\s+|up to\s+|about\s+)*" + AMT) ANYAMT = re.compile(AMT, re.I) def _usd(num, mult): v = float(num.replace(",", "")) return v * MULT.get((mult or "").lower(), 1) def settlement_usd(title, body): """(amount, basis). 1) a pay-phrase in the headline; 2) the first $ figure in the headline, replaced by the body's pay-phrase figure when that is within 15% of it (more precise); 3) the body's pay-phrase figure.""" m = PAY.search(title or "") if m: return _usd(m.group(1), m.group(2)), "title_pay" t = ANYAMT.search(title or "") b = PAY.search(clean(body)) tv = _usd(t.group(1), t.group(2)) if t else None bv = _usd(b.group(1), b.group(2)) if b else None if tv is not None and bv is not None: return (bv, "body_pay") if abs(bv - tv) <= 0.15 * tv else (tv, "title_amount") if tv is not None: return tv, "title_amount" if bv is not None: return bv, "body_pay" return None, "none" NUMW = {"two": 2, "three": 3, "four": 4, "five": 5, "six": 6, "seven": 7, "eight": 8, "nine": 9, "ten": 10, "eleven": 11, "twelve": 12, "thirteen": 13, "fourteen": 14, "fifteen": 15, "sixteen": 16, "seventeen": 17, "eighteen": 18, "nineteen": 19, "twenty": 20, "thirty": 30, "forty": 40, "fifty": 50, "sixty": 60, "seventy": 70, "eighty": 80, "ninety": 90} HEAD_N = re.compile(r"(?i)\b(\d{1,3}|(?:twenty|thirty|forty|fifty|sixty|seventy|eighty|ninety)(?:[- ](?:one|two|three|four|five|six|seven|eight|nine))?|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen|eighteen|nineteen)(?!\s*(?:th|st|nd|rd)\b)\s+(?:(?:[A-Za-z-]+\s+){0,3}?(?:individuals|defendants|people|persons|men|women|residents|members|employees|co-?defendants|others|suspects|family members|nationals|citizens)\b|(?:charged|indicted|arrested)\b)") HEAD_2 = re.compile(r"(?i)\b(?:couple|husband and wife|wife and husband|father and son|mother and (?:son|daughter)|father and daughter|brothers|sisters|siblings|duo|pair|twin brothers)\b") def _numword(w): w = w.lower().replace("-", " ") if w.isdigit(): return int(w) parts = w.split() ones = {"one": 1, "two": 2, "three": 3, "four": 4, "five": 5, "six": 6, "seven": 7, "eight": 8, "nine": 9} return NUMW.get(parts[0], 0) + (ones.get(parts[1], 0) if len(parts) > 1 else 0) def headline_count(title): """Number of people a headline says were charged, when it says; else None.""" m = HEAD_N.search(title or "") if m: n = _numword(m.group(1)) if 2 <= n <= 500: return n if HEAD_2.search(title or ""): return 2 return None def lede_count(body): """People counted in the release's opening sentence ('Five residents have been indicted'), when stated.""" b = re.sub(r"\s+", " ", body or "")[:300] m = HEAD_N.search(b) if m: n = _numword(m.group(1)) if 2 <= n <= 500: return n return None def defendants_estimate(title, body): """Lower-bound count of people a charging release announces: max(headline count, opening-sentence count, distinct names given with ages, 1).""" return max(headline_count(title) or 0, lede_count(body) or 0, len(defendants(body)), 1)