"""PostHog-loop playbook, step 6: train, then judge the model against the protocol.

The run uses the quickstart's documented CPU CI profile. The evaluation judges
the model against the same protocol and the same bar as step 5; a `fail` or
`incomplete` verdict is the result, and it is published as it is.
"""

import json
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))  # playbooks/playbook_api.py
from playbook_api import Api, State

api, state = Api(), State("posthog-loop")

profile_path = Path(__file__).resolve().parents[2] / "quickstart" / "ci-train-profile.json"
profile = json.loads(profile_path.read_text())
profile.pop("_comment", None)
profile["dataset_id"] = state["dataset_id"]
run = api.post("/runs", profile, key="posthog-loop-train")
run = api.wait(
    f"/runs/{run['run_id']}",
    lambda item: item.get("status") in ("succeeded", "failed", "cancelled"),
    what="the training run",
)
print(f"training: {run['status']}")
if run["status"] != "succeeded":
    sys.exit(json.dumps(run)[:3000])
model = api.wait(
    f"/models/{run['model_id']}",
    lambda item: item.get("status") in ("evaluated", "promoted", "archived", "failed"),
    what="the model's evaluation",
)
print(f"model: {model['status']}")
evaluation = api.post(
    "/evaluations",
    {"model_id": run["model_id"], "protocol_id": state["protocol_id"], "seed": 0},
    key="posthog-loop-evaluation",
)
detail = api.wait(
    f"/evaluations/{evaluation['report_id']}",
    lambda item: (item.get("evaluation") or {}).get("status") in ("succeeded", "failed"),
    what="the protocol evaluation",
)
print(f"evaluation: {detail['evaluation']['status']}")
verdict = (detail.get("report") or {}).get("verdict") or {}
print(f"verdict provenance: {verdict.get('provenance')}")
state.update(model_id=run["model_id"], evaluation_id=evaluation["report_id"])