"""Standalone synthetic teaching example. Run locally with Python, not in the HTML."""
import json
import numpy as np
import pandas as pd
from scipy.special import expit

def clean(value):
    if isinstance(value, dict): return {str(k):clean(v) for k,v in value.items()}
    if isinstance(value,(list,tuple,np.ndarray)): return [clean(v) for v in value]
    if isinstance(value,(np.integer,)): return int(value)
    if isinstance(value,(float,np.floating)): return float(value) if np.isfinite(value) else None
    if isinstance(value,(np.bool_,)): return bool(value)
    return value

def print_result(result):
    print(json.dumps(clean(result),ensure_ascii=False,allow_nan=False))


def make_panel(seed=42, T=180, N=24, k=5, mechanism="linear"):
    """Illustrative weekdays, 6 AR factors, future open-to-open k-day label."""
    if mechanism not in {"null","linear","interaction","regime"}:
        raise ValueError("Unknown mechanism")
    rng=np.random.default_rng(seed)
    dates=pd.bdate_range("2020-01-02",periods=T)
    factors=np.empty((T,N,6)); factors[0]=rng.normal(size=(N,6))
    for t in range(1,T):
        factors[t]=.6*factors[t-1]+.8*rng.normal(size=(N,6))
    factors[:,:,3]=.9*factors[:,:,0]+.15*rng.normal(size=(T,N))
    prices=np.full((T,N),100.)
    for t in range(T-1):
        x=factors[max(0,t-1)]
        signal=np.zeros(N)
        if mechanism!="null": signal=.0015*x[:,0]
        if mechanism in {"interaction","regime"}: signal += .0015*x[:,1]*x[:,2]
        if mechanism=="regime" and t>=int(.65*T): signal -= .002*x[:,0]
        noise=.006*rng.standard_t(5,N)/np.sqrt(5/3)+rng.normal(0,.002)
        prices[t+1]=prices[t]*np.exp(signal+noise)
    panel=pd.DataFrame(factors.reshape(T*N,6),columns=[f"x{i}" for i in range(6)])
    panel["date"]=np.repeat(dates,N); panel["stock"]=np.tile([f"S{i:03}" for i in range(N)],T)
    panel["tpos"]=np.repeat(np.arange(T),N); panel["open"]=prices.ravel()
    panel["signal_time"]=panel.date+pd.Timedelta(hours=15)
    panel["feature_available_time"]=panel.signal_time
    labels=panel.groupby("stock").open
    panel["y"]=labels.shift(-(k+1))/labels.shift(-1)-1
    exit_dates=pd.Series(dates).shift(-(k+1)).to_numpy()[panel.tpos]
    panel["label_available_time"]=exit_dates+pd.Timedelta(hours=9,minutes=30)
    return panel

def validate_panel(frame):
    if frame.duplicated(["date","stock"]).any():
        raise ValueError("Duplicate (date,stock) keys")
    if frame[["signal_time","feature_available_time"]].isna().any().any():
        raise ValueError("Missing availability timestamps")
    if (frame.feature_available_time>frame.signal_time).any():
        raise ValueError("Feature was unavailable at decision time")
    if (frame.y.notna() & frame.label_available_time.isna()).any():
        raise ValueError("Known label has unknown availability time")

def time_fold(frame, start, end, train_start=0):
    """Train at prior signal day's 15:00; dates [start,end) are held out."""
    validate_panel(frame)
    dates=frame.date.drop_duplicates().sort_values().reset_index(drop=True)
    if not 1<=start<end<=len(dates) or not 0<=train_start<start:
        raise ValueError("Invalid date fold")
    cutoff=dates.iloc[start-1]+pd.Timedelta(hours=15)
    train=frame[(frame.date>=dates.iloc[train_start]) & (frame.date<dates.iloc[start])
                & frame.y.notna() & (frame.label_available_time<=cutoff)].copy()
    valid=frame[frame.date.isin(dates.iloc[start:end]) & frame.y.notna()].copy()
    if train.empty or valid.empty: raise ValueError("Empty fold")
    return train,valid,cutoff

def keyed_metrics(frame, predictions):
    """predictions is a (date,stock,pred) table; exact keys required, order free."""
    if frame.duplicated(["date","stock"]).any() or predictions.duplicated(["date","stock"]).any():
        raise ValueError("Duplicate prediction/evaluation keys")
    aligned=frame[["date","stock","y"]].merge(predictions[["date","stock","pred"]],
        on=["date","stock"],how="outer",validate="1:1",indicator=True)
    if not aligned._merge.eq("both").all(): raise ValueError("Prediction keys do not match")
    daily=[]
    for date,group in aligned.groupby("date"):
        finite=np.isfinite(group.y)&np.isfinite(group.pred)
        g=group.loc[finite]; ic=rank_ic=spread=np.nan
        if len(g)>=3 and g.y.nunique()>1 and g.pred.nunique()>1:
            ic=g.y.corr(g.pred); rank_ic=g.y.rank().corr(g.pred.rank())
            ordered=g.sort_values(["pred","stock"]); size=max(1,len(g)//5)
            spread=ordered.tail(size).y.mean()-ordered.head(size).y.mean()
        daily.append({"date":date,"n":len(g),"ic":ic,"rank_ic":rank_ic,"spread":spread})
    table=pd.DataFrame(daily)
    valid=np.isfinite(aligned.y)&np.isfinite(aligned.pred)
    mse=float(np.mean((aligned.loc[valid,"y"]-aligned.loc[valid,"pred"])**2)) if valid.any() else np.nan
    return {"mse":mse,"ic":float(table.ic.mean()),"rank_ic":float(table.rank_ic.mean()),
            "coverage":float(table.ic.notna().mean()),"rows":int(valid.sum()),"dates":len(table)},table

if __name__ == "__main__":
    from sklearn.pipeline import make_pipeline
    from sklearn.preprocessing import StandardScaler
    from sklearn.linear_model import Ridge
    panel=make_panel(T=180,N=24,mechanism="interaction")
    features=[f"x{i}" for i in range(6)];oof=[]
    for start,end in [(80,105),(105,130)]:
        train,valid,_=time_fold(panel,start,end)
        saved=valid[["date","stock","y","label_available_time"]].copy()
        for j,lam in enumerate([.01,1.]):
            model=make_pipeline(StandardScaler(),Ridge(alpha=len(train)*lam)).fit(train[features],train.y)
            saved[f"p{j}"]=model.predict(valid[features])
        oof.append(saved)
    meta_data=pd.concat(oof)
    train,test,cutoff=time_fold(panel,130,174)
    meta_data=meta_data[meta_data.label_available_time<=cutoff]
    meta=Ridge(alpha=.01).fit(meta_data[["p0","p1"]],meta_data.y)
    future=test[["date","stock"]].copy()
    for j,lam in enumerate([.01,1.]):
        model=make_pipeline(StandardScaler(),Ridge(alpha=len(train)*lam)).fit(train[features],train.y)
        future[f"p{j}"]=model.predict(test[features])
    future["pred"]=meta.predict(future[["p0","p1"]])
    metrics,_=keyed_metrics(test,future)
    margin=.7-.2
    pair_loss=np.logaddexp(0,-margin)
    pair_grad=np.array([-expit(-margin),expit(-margin)])
    print_result({"meta_train_rows":len(meta_data),"latest_meta_label":str(meta_data.label_available_time.max()),
        "cutoff":str(cutoff),"stacking_metrics":metrics,"pairwise_loss":pair_loss,"pairwise_gradient":pair_grad})