"""連邦官報（Federal Register）の全文検索で、署名後の1週間に「super intelligence」を含む文書を探す。

使い方: python3 fr_search.py   → data/federal_register.json
APIは公式（federalregister.gov/api/v1）。検索語はAPI側で語幹処理されるため、ヒットした文書は
本文を取り直し、長形（artificial intelligence / super intelligence）を自分で数え直す。
"""
import json
import re
import urllib.parse
import urllib.request
from pathlib import Path

UA = {"User-Agent": "Mozilla/5.0"}
FROM, TO = "2026-09-30", "2026-10-06"


def get(url):
    return urllib.request.urlopen(urllib.request.Request(url, headers=UA), timeout=60).read().decode("utf8", "replace")


def search(term):
    q = urllib.parse.urlencode({
        "conditions[term]": term,
        "conditions[publication_date][gte]": FROM,
        "conditions[publication_date][lte]": TO,
        "per_page": 100,
        "order": "oldest",
    })
    return json.loads(get("https://www.federalregister.gov/api/v1/documents.json?" + q))


def main():
    found = {}
    for term in ['"artificial intelligence"', '"super intelligence"', "AI", "SI"]:
        for x in search(term)["results"]:
            found[x["document_number"]] = x
    rows = []
    for n in sorted(found):
        d = json.loads(get(f"https://www.federalregister.gov/api/v1/documents/{n}.json"))
        text = re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", get(d["raw_text_url"])))
        rows.append({
            "document_number": n,
            "publication_date": d["publication_date"],
            "type": d["type"],
            "agency": d["agencies"][0]["name"] if d["agencies"] else "",
            "title": d["title"],
            "artificial_intelligence_count": len(re.findall(r"artificial intelligence", text, re.I)),
            "super_intelligence_count": len(re.findall(r"super\s*intelligen", text, re.I)),
        })
        print(rows[-1])
    Path(__file__).parent.joinpath("data", "federal_register.json").write_text(
        json.dumps({"from": FROM, "to": TO, "documents": rows}, ensure_ascii=False, indent=2) + "\n"
    )


if __name__ == "__main__":
    main()