import json
from collections import Counter, defaultdict
from pathlib import Path

TOPICS = {
    "PRICE", "PERFORMANCE", "UI", "ACCESSIBILITY",
    "BILLING", "PRIVACY", "SUPPORT", "FEATURE",
}
KINDS = {"problem", "praise", "request", "neutral"}
STATES = {"substantive", "no_issue", "blank", "unclear", "off_topic"}

def require(ok, message):
    if not ok:
        raise ValueError(message)

def load_rows(path):
    rows = {}
    for line in Path(path).read_text(encoding="utf-8").splitlines():
        if not line.strip():
            continue
        row = json.loads(line)
        rid = row.get("response_id")
        require(isinstance(rid, str) and bool(rid), "IDがない")
        require(rid not in rows, f"重複ID: {rid}")
        rows[rid] = row
    require(bool(rows), f"空のファイル: {path}")
    return rows

source = load_rows("input.jsonl")
classified = load_rows("classified.jsonl")
require(set(source) == set(classified),
        f"欠落={sorted(set(source)-set(classified))}, "
        f"余分={sorted(set(classified)-set(source))}")

states = Counter()
topic_ids = defaultdict(set)
kind_ids = defaultdict(set)
risk_ids = set()
unit_count = 0

for rid, result in classified.items():
    text = source[rid].get("text")
    require(isinstance(text, str), f"原文の型が不正: {rid}")
    require(result.get("codebook_version") == "v1", f"版が不一致: {rid}")
    state = result.get("state")
    require(state in STATES, f"状態が不正: {rid}")
    units = result.get("units")
    require(isinstance(units, list), f"unitsが配列でない: {rid}")
    require((state == "blank") == (not text.strip()), f"空欄状態が不整合: {rid}")
    require(bool(units) == (state == "substantive"), f"分類単位が不整合: {rid}")
    states[state] += 1
    for unit in units:
        topic, kind = unit.get("topic"), unit.get("kind")
        require(topic in TOPICS, f"未確定・未定義の分類: {rid}: {topic}")
        require(kind in KINDS, f"発言種類が不正: {rid}")
        evidence = unit.get("evidence")
        require(isinstance(evidence, str) and bool(evidence.strip()),
                f"根拠がない: {rid}")
        require(evidence in text, f"原文にない根拠: {rid}: {evidence}")
        require(isinstance(unit.get("risk_candidate"), bool), f"確認フラグ不正: {rid}")
        topic_ids[topic].add(rid)
        kind_ids[(topic, kind)].add(rid)
        if unit["risk_candidate"]:
            risk_ids.add(rid)
        unit_count += 1

total = len(source)
substantive = states["substantive"]
print("回答総数", total, "状態内訳", dict(sorted(states.items())))
print("意味単位数", unit_count)
print("topic,total,problem,praise,request,neutral,pct_all,pct_substantive,ids")
for topic in sorted(TOPICS):
    count = len(topic_ids[topic])
    pct_sub = f"{100*count/substantive:.1f}" if substantive else "NA"
    values = [topic, count]
    values += [len(kind_ids[(topic, kind)])
               for kind in ("problem", "praise", "request", "neutral")]
    values += [f"{100*count/total:.1f}", pct_sub,
               "|".join(sorted(topic_ids[topic]))]
    print(",".join(map(str, values)))
print("重大性の確認候補", sorted(risk_ids))