"""Standalone synthetic teaching example. Run locally with Python, not in the HTML."""
import json
import numpy as np
import pandas as pd
from scipy.special import expit

def clean(value):
    if isinstance(value, dict): return {str(k):clean(v) for k,v in value.items()}
    if isinstance(value,(list,tuple,np.ndarray)): return [clean(v) for v in value]
    if isinstance(value,(np.integer,)): return int(value)
    if isinstance(value,(float,np.floating)): return float(value) if np.isfinite(value) else None
    if isinstance(value,(np.bool_,)): return bool(value)
    return value

def print_result(result):
    print(json.dumps(clean(result),ensure_ascii=False,allow_nan=False))


def cross_section_neutralize(frame, factor="factor", exposures=("size",)):
    """Same-date observed exposures only; residual factor, not portfolio constraints."""
    if frame.duplicated(["date","stock"]).any(): raise ValueError("Duplicate keys")
    result=pd.Series(index=frame.index,dtype=float)
    for _,g in frame.groupby("date"):
        X=g[list(exposures)].to_numpy(float); y=g[factor].to_numpy(float)
        valid=np.isfinite(X).all(axis=1)&np.isfinite(y)
        design=np.column_stack([np.ones(valid.sum()),X[valid]])
        if valid.sum()<=design.shape[1]: continue
        coef=np.linalg.lstsq(design,y[valid],rcond=None)[0]
        result.loc[g.index[valid]]=y[valid]-design@coef
    return result

def keyed_metrics(frame, predictions):
    """predictions is a (date,stock,pred) table; exact keys required, order free."""
    if frame.duplicated(["date","stock"]).any() or predictions.duplicated(["date","stock"]).any():
        raise ValueError("Duplicate prediction/evaluation keys")
    aligned=frame[["date","stock","y"]].merge(predictions[["date","stock","pred"]],
        on=["date","stock"],how="outer",validate="1:1",indicator=True)
    if not aligned._merge.eq("both").all(): raise ValueError("Prediction keys do not match")
    daily=[]
    for date,group in aligned.groupby("date"):
        finite=np.isfinite(group.y)&np.isfinite(group.pred)
        g=group.loc[finite]; ic=rank_ic=spread=np.nan
        if len(g)>=3 and g.y.nunique()>1 and g.pred.nunique()>1:
            ic=g.y.corr(g.pred); rank_ic=g.y.rank().corr(g.pred.rank())
            ordered=g.sort_values(["pred","stock"]); size=max(1,len(g)//5)
            spread=ordered.tail(size).y.mean()-ordered.head(size).y.mean()
        daily.append({"date":date,"n":len(g),"ic":ic,"rank_ic":rank_ic,"spread":spread})
    table=pd.DataFrame(daily)
    valid=np.isfinite(aligned.y)&np.isfinite(aligned.pred)
    mse=float(np.mean((aligned.loc[valid,"y"]-aligned.loc[valid,"pred"])**2)) if valid.any() else np.nan
    return {"mse":mse,"ic":float(table.ic.mean()),"rank_ic":float(table.rank_ic.mean()),
            "coverage":float(table.ic.notna().mean()),"rows":int(valid.sum()),"dates":len(table)},table

if __name__ == "__main__":
    rng=np.random.default_rng(42)
    frame=pd.DataFrame({"date":np.repeat(pd.date_range("2020-01-02",periods=3),12),
        "stock":np.tile([f"S{i}" for i in range(12)],3),"size":rng.normal(size=36)})
    frame["factor"]=2+3*frame["size"]+rng.normal(size=36)
    neutral=cross_section_neutralize(frame)
    orthogonality=[]
    for _,g in frame.groupby("date"):
        design=np.column_stack([np.ones(len(g)),g["size"]])
        orthogonality.append(abs(design.T@neutral.loc[g.index]).max())
    assert max(orthogonality)<1e-10
    panel=frame.rename(columns={"factor":"y"})
    pred=panel[["date","stock"]].assign(pred=panel["size"])
    original,_=keyed_metrics(panel,pred)
    transformed,_=keyed_metrics(panel,pred.assign(pred=7*pred.pred+3))
    np.testing.assert_allclose(original["ic"],transformed["ic"],atol=1e-12)
    print_result({"neutralization_max_inner_product":max(orthogonality),
        "original_ic":original["ic"],"transformed_ic":transformed["ic"],
        "original_mse":original["mse"],"transformed_mse":transformed["mse"]})