"""Score frozen labels against rankings from the actual TypeScript runtime."""
import json
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]


def evaluate(fixture, rankings):
    by_id = {row["id"]: row for row in rankings["rows"]}
    if set(by_id) != {case["id"] for case in fixture["cases"]}:
        raise ValueError("Rankings and fixture do not match")
    rows = []
    for case in fixture["cases"]:
        row = dict(case)
        for method in ("keyword", "semantic"):
            results = by_id[case["id"]][method]
            ids = [result["card"]["id"] for result in results]
            relevant = set(case["relevantIds"])
            first = next((index + 1 for index, value in enumerate(ids) if value in relevant), None)
            row[method] = {"top3": ids[:3], "hitAt3": bool(relevant.intersection(ids[:3])),
                           "top5": ids[:5], "hitAt5": bool(relevant.intersection(ids[:5])),
                           "reciprocalRankAt5": 1 / first if first and first <= 5 else 0,
                           "reciprocalRank": 1 / first if first else 0, "abstained": not ids}
        rows.append(row)
    summary = {}
    for method in ("keyword", "semantic"):
        answerable = [row[method] for row in rows if row["kind"] == "answerable"]
        unsupported = [row[method] for row in rows if row["kind"] == "unsupported"]
        challenge = [row[method] for row in rows if row["kind"] == "challenge"]
        summary[method] = {"answerableCount": len(answerable), "hitAt3Count": sum(r["hitAt3"] for r in answerable),
                           "hitAt5Count": sum(r["hitAt5"] for r in answerable),
                           "mrrAt5": round(sum(r["reciprocalRankAt5"] for r in answerable) / len(answerable), 4),
                           "mrr": round(sum(r["reciprocalRank"] for r in answerable) / len(answerable), 4),
                           "unsupportedCount": len(unsupported), "unsupportedAbstentions": sum(r["abstained"] for r in unsupported),
                           "challengeCount": len(challenge), "challengeHitAt3": sum(r["hitAt3"] for r in challenge)}
    return {"version": 1, "scope": "34 synthetic English queries; independent authored relevance labels, frozen before first run. Not user research or a production benchmark.",
            "protocol": "Hit@3/5 is at least one expected card among the first three/five. MRR@5 reflects the five results shown in the UI; full MRR includes invisible ranks and is diagnostic only. Unsupported abstention requires an empty result. Challenges are reported separately.",
            "provenance": {key: value for key, value in rankings.items() if key != "rows"},
            "summary": summary, "cases": rows}


if __name__ == "__main__":
    fixture = json.loads((ROOT / "evaluation/prompt-cases.json").read_text(encoding="utf-8-sig"))
    rankings = json.loads((ROOT / "evaluation/prompt-rankings.json").read_text(encoding="utf-8"))
    report = evaluate(fixture, rankings)
    (ROOT / "evaluation/prompt-evaluation.json").write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
    print(json.dumps(report["summary"], indent=2))