"""PostHog-loop playbook, step 5: baseline first.

The dataset is built after the protocol, from each user's last 8 events
(the quickstart's CPU-sized window). Before any model exists, the baseline
evaluation fits the protocol's internal quality safeguards on it and reports
what they reach: the bar the model will have to clear, written down first.
"""

import json
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))  # playbooks/playbook_api.py
from playbook_api import Api, State

api, state = Api(), State("posthog-loop")

build = api.post(
    "/datasets",
    {
        "entity_type": "user",
        "labels": [state["label_id"]],
        "window": {
            "max_events_per_entity": 8,
            "time_range": {"start": state["start"], "end": state["end"]},
        },
        "split": {"method": "entity_hash", "train": 0.7, "val": 0.1, "test": 0.2, "seed": 7},
    },
    key="posthog-loop-dataset",
)
dataset = api.wait(
    f"/datasets/{build['dataset_id']}",
    lambda ds: ds.get("status") in ("succeeded", "failed", "aborted_leakage"),
    what="the dataset build",
)
print(f"dataset: {dataset['status']}")
if dataset["status"] != "succeeded":
    sys.exit(json.dumps(dataset)[:3000])
baseline = api.post(
    "/evaluations:baseline",
    {"dataset_id": dataset["dataset_id"], "protocol_id": state["protocol_id"], "seed": 0},
    key="posthog-loop-baseline",
)
detail = api.wait(
    f"/evaluations/{baseline['report_id']}",
    lambda item: (item.get("evaluation") or {}).get("status") in ("succeeded", "failed"),
    what="the baseline evaluation",
)
print(f"baseline evaluation: {detail['evaluation']['status']}")
verdict = (detail.get("report") or {}).get("verdict") or {}
print(f"baseline verdict: {verdict.get('overall')}")
# Reference levels of your own comparators are printed; the internal reference levels are only
# counted (the platform's internal models are never named).
internal = 0
for criterion in verdict.get("criteria") or []:
    if str(criterion.get("arm", "")).startswith("cmp_"):
        print(f"  bar {criterion['metric']} of {criterion['arm']}: {criterion.get('arm_value')}")
    else:
        internal += 1
print(f"internal reference levels recorded: {internal}")
state.update(dataset_id=dataset["dataset_id"], baseline_id=baseline["report_id"])