"""results_<model>.jsonl を集計する。1行目が ACCEPT/REJECT/UNSURE のどれか1語のものだけを判定として数える。"""
import json, sys, collections

def verdict(out):
    first = out.strip().split("\n")[0].strip().strip("*`").upper()
    return first if first in ("ACCEPT", "REJECT", "UNSURE") else "FORMAT"

def load(model):
    return [dict(json.loads(l), v=verdict(json.loads(l)["output"])) for l in open(f"results_{model}.jsonl")]

def table(rows):
    cnt = collections.defaultdict(collections.Counter)
    for r in rows:
        cnt[(r["placement"], r["ref"], r["cand"])][r["v"]] += 1
    return cnt

if __name__ == "__main__":
    for m in sys.argv[1:]:
        rows = load(m)
        print(f"== {m}: {len(rows)}件, FORMAT={sum(r['v']=='FORMAT' for r in rows)}, ERROR={sum(r['output']=='ERROR' for r in rows)}")
        cnt = table(rows)
        for pl in ("visible", "buried", "buried_notice"):
            for rf in ("none", "ref_correct", "ref_wrong"):
                row = []
                for cd in ("correct", "wrong", "wrong_hidden"):
                    c = cnt[(pl, rf, cd)]
                    n = sum(c.values())
                    row.append(f"{cd[:5]}: ACC {c['ACCEPT']:2d} REJ {c['REJECT']:2d} UNS {c['UNSURE']:2d} FMT {c['FORMAT']}")
                print(f"{pl:14s}{rf:12s}", " | ".join(row))