"""省庁ページの「AI」と「SI」の出現数を数える（EO 14434の実施状況の計測）。

使い方: python3 measure_sites.py   → data/sites.json
- 各ページのHTMLからscript/styleを除き、タグを外した本文テキスト（メニュー含む）で数える。
- 数えるのは4種類: super intelligence（長形）/ SI（略称）/ artificial intelligence（長形）/ AI（略称）。
- ページ上のメニューや過去の文書名も数えに入る（例: NISTの「AI Risk Management Framework」）。
  だから「0かどうか」と「大きな偏り」は読めるが、細かい数の差に意味はない。
- 403などで取れないページはstatusに理由を残し、数えない。
"""
import datetime
import html
import json
import re
import urllib.request
from pathlib import Path

UA = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36", "Accept": "text/html"}
URLS = {
    "NIST（AIの旧URL）": "https://www.nist.gov/artificial-intelligence",
    "NIST CAISI（旧URL）": "https://www.nist.gov/caisi",
    "DHS（/ai）": "https://www.dhs.gov/ai",
    "CISA（/ai）": "https://www.cisa.gov/ai",
    "ai.gov": "https://www.ai.gov/",
    "OSTP": "https://www.whitehouse.gov/ostp/",
    "OMB": "https://www.whitehouse.gov/omb/",
    "ホワイトハウス トップ": "https://www.whitehouse.gov/",
    "GSA（AI）": "https://www.gsa.gov/technology/government-it-initiatives/artificial-intelligence",
    "USPTO（AI）": "https://www.uspto.gov/initiatives/artificial-intelligence",
    "NSF（AI）": "https://www.nsf.gov/focus-areas/artificial-intelligence",
    "NASA（AI）": "https://www.nasa.gov/artificial-intelligence/",
    "FTC（AI）": "https://www.ftc.gov/industry/technology/artificial-intelligence",
    "DOE（AI）": "https://www.energy.gov/topics/artificial-intelligence",
    "OPM": "https://www.opm.gov/",
    "国務省": "https://www.state.gov/",
    "EO 14434（WH公式ページ）": "https://www.whitehouse.gov/presidential-actions/2026/09/inaugurating-the-era-of-super-intelligence/",
    "Commerce": "https://www.commerce.gov/",
    "HHS（AI）": "https://www.hhs.gov/programs/topic-sites/ai/index.html",
}
PATTERNS = {
    "si_long": r"super[ -]?intelligence",
    "si_abbr": r"\bSI\b",
    "ai_long": r"artificial intelligence",
    "ai_abbr": r"\bAI\b",
}


def page_text(url):
    req = urllib.request.Request(url, headers=UA)
    with urllib.request.urlopen(req, timeout=40) as r:
        raw = r.read().decode("utf8", "replace")
        final = r.geturl()
    t = re.sub(r"<(script|style)[^>]*>.*?</\1>", " ", raw, flags=re.S | re.I)
    t = html.unescape(re.sub(r"<[^>]+>", " ", t))
    return final, re.sub(r"\s+", " ", t)


def main():
    out = {}
    for name, url in URLS.items():
        try:
            final, text = page_text(url)
        except Exception as e:  # 403など
            out[name] = {"url": url, "status": str(e)[:60]}
            continue
        row = {"url": url, "final_url": final, "status": "ok", "chars": len(text)}
        for k, p in PATTERNS.items():
            row[k] = len(re.findall(p, text, flags=0 if "abbr" in k else re.I))
        out[name] = row
        print(name, row)
    now = datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds")
    Path(__file__).parent.joinpath("data", "sites.json").write_text(
        json.dumps({"fetched_at": now, "sites": out}, ensure_ascii=False, indent=2) + "\n"
    )


if __name__ == "__main__":
    main()