#!/usr/bin/env python3
"""AEO 30分チェックの自動化版。標準ライブラリだけで動く。

使い方:  python3 aeo_check.py https://example.com [https://example.org ...] > result.json

1サイトあたり8〜9回のGETしかしない（robots.txt、llms.txt、トップ、ボットUAで5回）。
"""
import json
import re
import sys
import urllib.error
import urllib.request
from urllib.robotparser import RobotFileParser

BROWSER = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
# 学習用 / 検索用(引用の入口) / ユーザー操作用 のうち、robots.txtで見るトークン
ROBOTS_TOKENS = [
    "GPTBot", "OAI-SearchBot", "ChatGPT-User",
    "ClaudeBot", "Claude-SearchBot", "Claude-User",
    "PerplexityBot", "Google-Extended", "Googlebot", "bingbot",
]
# UAだけを名乗って取りに行く（IPは本物ではないので、UA単位の遮断だけが分かる）
UA_PROBES = {
    "FooBot(対照)": "FooBot/1.0",
    "OAI-SearchBot": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.4; +https://openai.com/searchbot",
    "GPTBot": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.4; +https://openai.com/gptbot",
    "ClaudeBot": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)",
    "PerplexityBot": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)",
}


def get(url, ua=BROWSER, limit=400_000):
    req = urllib.request.Request(url, headers={"User-Agent": ua, "Accept": "text/html,*/*"})
    try:
        with urllib.request.urlopen(req, timeout=20) as r:
            return r.status, dict(r.headers), r.read(limit).decode("utf-8", "ignore"), r.geturl()
    except urllib.error.HTTPError as e:
        return e.code, dict(e.headers or {}), "", url
    except Exception as e:  # 接続失敗など
        return 0, {}, str(e)[:120], url


def check(site):
    site = site.rstrip("/")
    out = {"site": site}

    # 1. robots.txt
    st, _, body, _ = get(site + "/robots.txt")
    out["robots_status"] = st
    if st == 200 and not body.lstrip().lower().startswith(("<!doctype", "<html")):
        rp = RobotFileParser()
        rp.parse(body.splitlines())
        out["robots"] = {t: rp.can_fetch(t, site + "/") for t in ROBOTS_TOKENS}
    else:
        out["robots"] = None  # 無い(404等)=誰も拒否されていない

    # 2. llms.txt（200でもHTMLが返るだけの偽陽性を除く）
    st, hd, body, _ = get(site + "/llms.txt")
    # llms.txtはMarkdown。先頭が「#」で始まらない200は、HTMLやエラー文が返っているだけの偽陽性
    looks_real = body.lstrip().startswith("#") and not body.lstrip().lower().startswith("<")
    out["llms_txt"] = "あり" if st == 200 and looks_real else ("偽陽性(200だが中身がllms.txtではない)" if st == 200 else "なし")

    # 3. トップページ: JSなしの初期HTMLに本文があるか、noindex/nosnippet
    st, hd, html, final = get(site + "/")
    out["home_status"] = st
    out["home_final_url"] = final
    text = re.sub(r"<(script|style)[^>]*>.*?</\1>", " ", html, flags=re.S | re.I)
    text = re.sub(r"<[^>]+>", " ", text)
    text = re.sub(r"\s+", " ", text)
    out["home_text_chars_without_js"] = len(text.strip())
    metas = re.findall(r'<meta[^>]+name=["\']robots["\'][^>]*>', html, flags=re.I)
    out["meta_robots"] = [re.sub(r"\s+", " ", m) for m in metas]
    out["x_robots_tag"] = hd.get("X-Robots-Tag") or hd.get("x-robots-tag")
    out["json_ld_blocks"] = len(re.findall(r'application/ld\+json', html, flags=re.I))

    # 4. UA別の応答: ふつうのブラウザUAと、ボットUAで返事が違うか
    probes = {"browser": out["home_status"]}
    for name, ua in UA_PROBES.items():
        probes[name] = get(site + "/", ua)[0]
    out["status_by_ua"] = probes
    return out


if __name__ == "__main__":
    print(json.dumps([check(s) for s in sys.argv[1:]], ensure_ascii=False, indent=2))