"""Deposit-flight playbook, step 10: the cutoff-ladder backtest under the protocol.

The backtest replays the test accounts at a ladder of cutoffs and asks how
early each arm would have alerted, at the protocol's alert budget: the
with-token model counts every alert (observation-time regime), the tuned
internal quality safeguard only alerts that its event prefixes support.
"""

import json
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))  # playbooks/playbook_api.py
from playbook_api import Api, State

api, state = Api(), State("deposit-flight")

model_id = state["serving_model"]
created = api.post(
    "/backtests",
    {"protocol_id": state["protocol_id"], "model_id": model_id, "seed": 0},
    key="deposit-flight-backtest",
)
detail = api.wait(
    f"/backtests/{created['backtest_id']}",
    lambda item: (item.get("backtest") or {}).get("status") in ("succeeded", "failed"),
    what="the backtest",
)
print(f"backtest: {detail['backtest']['status']}")
if detail["backtest"]["status"] != "succeeded":
    sys.exit(json.dumps(detail)[:3000])
report = detail["report"]
print(f"cutoffs: {len(report['cutoffs'])}, alert budget: {report['alert_budget']}")
block = report["per_label"][state["label_id"]]
if block.get("status") == "completed":
    # The model and your own comparators; the internal reference arms are never printed.
    for arm, row in block["arms"].items():
        if arm == "model" or arm.startswith("cmp_"):
            print(f"  {arm}: regime {row.get('regime')}, alerted {row.get('n_alerted')}")
print(f"lead-time verdict: {(report.get('verdict') or {}).get('overall')}")
state.update(backtest_id=created["backtest_id"])