"""Synthetic paired binary evaluation; Python 3.9+, standard library only.
Run normally for two examples, or with --test. No model calls or release gate.
Exact two-sided McNemar p-value, conditional on discordant pairs, p=0.5.
Teaching implementation capped at 10,000 pairs, not a statistics package.
"""
import json
import math
import sys
import unittest
from fractions import Fraction


def compare(expected_ids, rows):
    """rows: iterable of (case_id, baseline_bool, candidate_bool)."""
    expected = list(expected_ids)
    if not expected or len(expected) > 10000:
        raise ValueError("require 1..10000 expected cases")
    if any(type(x) is not str or not x for x in expected):
        raise ValueError("expected IDs must be nonempty strings")
    if len(set(expected)) != len(expected):
        raise ValueError("duplicate expected ID")
    expected = set(expected)
    seen = set()
    counts = {"both_pass": 0, "regressions": 0, "improvements": 0, "both_fail": 0}
    for case_id, baseline, candidate in rows:
        if case_id not in expected or case_id in seen:
            raise ValueError("unexpected or duplicate observed ID")
        if type(baseline) is not bool or type(candidate) is not bool:
            raise ValueError("both verdicts must be Boolean; unknowns cannot be dropped")
        seen.add(case_id)
        key = ("both_pass" if candidate else "regressions") if baseline else (
            "improvements" if candidate else "both_fail")
        counts[key] += 1
    if seen != expected:
        raise ValueError("missing paired observation")
    n = len(expected)
    b, c = counts["regressions"], counts["improvements"]
    discordant = b + c
    if discordant:
        tail = sum(math.comb(discordant, k) for k in range(min(b, c) + 1))
        p = min(Fraction(1), Fraction(2 * tail, 2 ** discordant))
    else:
        p = Fraction(1)
    return dict(n=n, **counts, baseline_passes=counts["both_pass"] + b,
                candidate_passes=counts["both_pass"] + c,
                delta_percentage_points=100 * (c - b) / n,
                discordant=discordant, p_two_sided=float(p))


def fixture(both_pass, regressions, improvements, both_fail):
    """Construct explicitly invented outcomes, never measured agent results."""
    rows = []
    for count, baseline, candidate in [(both_pass, True, True),
                                       (regressions, True, False),
                                       (improvements, False, True),
                                       (both_fail, False, False)]:
        for _ in range(count):
            rows.append((f"case-{len(rows):03d}", baseline, candidate))
    return [row[0] for row in rows], rows


class Tests(unittest.TestCase):
    def test_example_a(self):
        r = compare(*fixture(80, 0, 10, 10))
        self.assertEqual((r["baseline_passes"], r["candidate_passes"]), (80, 90))
        self.assertEqual(r["p_two_sided"], 0.001953125)

    def test_example_b(self):
        r = compare(*fixture(70, 10, 20, 0))
        self.assertEqual((r["baseline_passes"], r["candidate_passes"]), (80, 90))
        self.assertAlmostEqual(r["p_two_sided"], 0.09873714670538902)

    def test_equal_discordance(self):
        self.assertEqual(compare(*fixture(1, 4, 4, 1))["p_two_sided"], 1)

    def test_no_discordance(self):
        self.assertEqual(compare(*fixture(5, 0, 0, 5))["p_two_sided"], 1)

    def test_direction_swap(self):
        a, b = compare(*fixture(2, 1, 8, 1)), compare(*fixture(2, 8, 1, 1))
        self.assertEqual(a["p_two_sided"], b["p_two_sided"])
        self.assertEqual(a["delta_percentage_points"], -b["delta_percentage_points"])

    def test_row_order(self):
        ids, rows = fixture(3, 2, 4, 1)
        self.assertEqual(compare(ids, rows), compare(ids, reversed(rows)))

    def test_missing(self):
        with self.assertRaises(ValueError):
            compare(["a", "b"], [("a", True, True)])

    def test_duplicate_observed(self):
        with self.assertRaises(ValueError):
            compare(["a"], [("a", True, True), ("a", True, True)])

    def test_unexpected(self):
        with self.assertRaises(ValueError):
            compare(["a"], [("z", True, True)])

    def test_invalid_verdicts(self):
        for value in [None, 1, 0, "pass"]:
            for row in [("a", value, True), ("a", True, value)]:
                with self.subTest(row=row), self.assertRaises(ValueError):
                    compare(["a"], [row])

    def test_invalid_inventory(self):
        for ids in [[], ["a", "a"], [""], [1], [str(i) for i in range(10001)]]:
            with self.subTest(size=len(ids)), self.assertRaises(ValueError):
                compare(ids, [])

    def test_single_discordance(self):
        r = compare(*fixture(0, 0, 1, 0))
        self.assertEqual(r["p_two_sided"], 1)
        self.assertEqual(r["delta_percentage_points"], 100)


if __name__ == "__main__":
    if sys.argv[1:] == ["--test"]:
        unittest.main(argv=[sys.argv[0]], verbosity=2)
    elif not sys.argv[1:]:
        for name, cells in [("A", (80, 0, 10, 10)), ("B", (70, 10, 20, 0))]:
            print(json.dumps({"synthetic_example": name, **compare(*fixture(*cells))}, sort_keys=True))
    else:
        raise SystemExit("usage: python3 paired-agent-eval.py [--test]")