#!/usr/bin/env python3
"""Standalone verifier for the VAGI Behavior Suite distribution-shift probe.

Imports no CAIN code. Recomputes the per-domain counts from rows_shift.jsonl and
compares them with results_shift.json. It does not prove the variants are a good
benchmark; it proves the published gap map matches the published rows.

    python3 vagi_suite/verify_shift.py artifacts/vagi_suite
"""
from __future__ import annotations

import json
import sys
from pathlib import Path
from typing import Any, Dict, List

DOMAINS = [
    "software_repair", "security_triage", "data_analysis", "business_decision",
    "science_calculation", "planning", "memory_retrieval", "tool_orchestration",
    "recovery", "governance",
]


def main(argv: List[str]) -> int:
    root = Path(argv[0] if argv else "artifacts/vagi_suite")
    try:
        res = json.loads((root / "results_shift.json").read_text(encoding="utf-8"))
        rows = [json.loads(x) for x in (root / "rows_shift.jsonl").read_text(encoding="utf-8").splitlines() if x.strip()]
        tasks = json.loads((root / "tasks_shift_public.json").read_text(encoding="utf-8"))
    except Exception as e:
        print(f"FAIL: cannot read bundle: {e}")
        return 1

    problems: List[str] = []
    checks = []
    ok = len(rows) == 100 and len(tasks) == 100
    checks.append(("row_and_task_count", ok, f"{len(rows)} rows, {len(tasks)} tasks"))
    if not ok:
        problems.append("expected 100 rows and 100 public tasks")

    for d in DOMAINS:
        sub = [r for r in rows if r.get("domain") == d]
        accepted = sum(1 for r in sub if r.get("accepted"))
        stated = (res.get("per_domain") or {}).get(d) or {}
        ok = len(sub) == 10 and stated.get("n") == 10 and stated.get("accepted") == accepted
        checks.append((f"{d}", ok, f"{accepted}/10 vs stated {stated.get('accepted')}/{stated.get('n')}"))
        if not ok:
            problems.append(f"{d}: rows say {accepted}/10, summary says {stated.get('accepted')}/{stated.get('n')}")

    total = sum(1 for r in rows if r.get("accepted"))
    ok = res.get("accepted") == total and res.get("n") == len(rows)
    checks.append(("total", ok, f"{total}/{len(rows)} vs stated {res.get('accepted')}/{res.get('n')}"))
    if not ok:
        problems.append("total does not match rows")

    for name, ok, detail in checks:
        print(f"{'PASS' if ok else 'FAIL':4s} {name:22s} {detail}")
    print()
    if problems:
        print(f"VERDICT: FAIL ({len(problems)} problem(s))")
        for p in problems:
            print(f"  - {p}")
        return 1
    print(f"VERDICT: PASS — {len(checks)} checks; shift {total}/{len(rows)} ({total/len(rows)*100:.1f}%)")
    print("Scope: internal consistency only. A low shift score is the expected gap map, not a CAIN failure to hide.")
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))
