#!/usr/bin/env python3
"""Synthetic identity-join fixture. Python 3.10+, standard library only.
No model calls, network access, or file writes. Not a production evaluator.
"""
import itertools
import json
import sys
import unittest

CASES = [
    {"case_id": "allow-own", "expected": "ALLOW"},
    {"case_id": "deny-other", "expected": "DENY"},
]
# Both answers are wrong for their actual case. Delivery order is reversed.
RESULTS = [
    {"run_id": "run-1", "case_id": "deny-other", "answer": "ALLOW"},
    {"run_id": "run-1", "case_id": "allow-own", "answer": "DENY"},
]


def positional_score(cases, results):
    pairs = list(zip(cases, results, strict=True))
    return sum(c["expected"] == r["answer"] for c, r in pairs) / len(pairs)


def checked_index(rows, fields):
    index = {}
    for row in rows:
        if type(row) is not dict or set(row) != set(fields):
            raise ValueError("invalid row fields")
        if any(type(row[k]) is not str or not row[k].strip() for k in fields):
            raise ValueError("fields must be nonempty strings")
        key = row["case_id"]
        if key in index:
            raise ValueError("duplicate case_id: " + key)
        index[key] = row
    return index


def report(cases, results, run_id):
    if type(run_id) is not str or not run_id.strip():
        raise ValueError("invalid run_id")
    expected = checked_index(cases, ("case_id", "expected"))
    observed = checked_index(results, ("run_id", "case_id", "answer"))
    if not expected:
        raise ValueError("empty manifest")
    if any(row["run_id"] != run_id for row in observed.values()):
        raise ValueError("wrong run_id")
    missing = sorted(set(expected) - set(observed))
    unexpected = sorted(set(observed) - set(expected))
    if missing or unexpected:
        raise ValueError("case mismatch: " + json.dumps({
            "missing": missing, "unexpected": unexpected}))
    # Validate the entire batch before computing or emitting any score.
    decisions = {key: observed[key]["answer"] == expected[key]["expected"]
                 for key in sorted(expected)}
    passed = sum(decisions.values())
    return {"run_id": run_id, "assessed": len(expected), "passed": passed,
            "accuracy": passed / len(expected), "cases": decisions}


class IdentityTests(unittest.TestCase):
    def test_positional_false_pass(self):
        self.assertEqual(positional_score(CASES, RESULTS), 1.0)
        self.assertEqual(report(CASES, RESULTS, "run-1")["accuracy"], 0.0)

    def test_strict_zip_rejects_short_batch(self):
        with self.assertRaises(ValueError):
            positional_score(CASES, RESULTS[:1])

    def test_permutation_invariance(self):
        for cases in itertools.permutations(CASES):
            for results in itertools.permutations(RESULTS):
                self.assertEqual(report(cases, results, "run-1"),
                                 report(CASES, RESULTS, "run-1"))

    def test_correct_answers(self):
        rows = [{"run_id": "run-1", "case_id": c["case_id"],
                 "answer": c["expected"]} for c in CASES]
        self.assertEqual(report(CASES, rows, "run-1")["accuracy"], 1.0)

    def test_missing_case(self):
        with self.assertRaisesRegex(ValueError, "missing"):
            report(CASES, RESULTS[:1], "run-1")

    def test_duplicate_result(self):
        with self.assertRaisesRegex(ValueError, "duplicate"):
            report(CASES, [RESULTS[0], RESULTS[0]], "run-1")

    def test_duplicate_manifest(self):
        with self.assertRaisesRegex(ValueError, "duplicate"):
            report([CASES[0], CASES[0]], RESULTS, "run-1")

    def test_wrong_case_same_count(self):
        rows = [dict(RESULTS[0], case_id="unknown"), RESULTS[1]]
        with self.assertRaisesRegex(ValueError, "unexpected"):
            report(CASES, rows, "run-1")

    def test_wrong_run(self):
        rows = [dict(r, run_id="old-run") for r in RESULTS]
        with self.assertRaisesRegex(ValueError, "wrong run_id"):
            report(CASES, rows, "run-1")

    def test_empty_manifest(self):
        with self.assertRaisesRegex(ValueError, "empty"):
            report([], [], "run-1")

    def test_malformed_fields(self):
        for bad in (None, {}, dict(RESULTS[0], extra="x"),
                    dict(RESULTS[0], case_id=True),
                    dict(RESULTS[0], answer="")):
            with self.subTest(bad=bad), self.assertRaises(ValueError):
                report(CASES, [bad, RESULTS[1]], "run-1")

    def test_invalid_run_argument(self):
        for run in (None, True, "", " "):
            with self.subTest(run=run), self.assertRaises(ValueError):
                report(CASES, RESULTS, run)


if __name__ == "__main__":
    if sys.argv[1:] == ["--test"]:
        unittest.main(argv=[sys.argv[0]], verbosity=2)
    elif sys.argv[1:]:
        raise SystemExit("usage: agent-eval-result-identity.py [--test]")
    else:
        print("positional_accuracy=" + str(positional_score(CASES, RESULTS)))
        print(json.dumps(report(CASES, RESULTS, "run-1"), sort_keys=True))