from pathlib import Path
import pandas as pd
import numpy as np
from _shared import load
ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT / "outputs"
OUT.mkdir(exist_ok=True)
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
raw=load("panel").copy()
audit=raw.loc[raw.duplicated(["symbol","date"],keep=False)].copy()
audit.to_csv(OUT/"project_panel_duplicates.csv",index=False)
d=raw.drop_duplicates(["symbol","date"],keep="last").copy()
d["date"]=pd.to_datetime(d["date"],errors="raise")
d=d.sort_values(["symbol","date"]).reset_index(drop=True)
assert not d.duplicated(["symbol","date"]).any()
assert d.loc[d["close"].notna(),"close"].gt(0).all()
g=d.groupby("symbol",sort=False)
d["previous_close"]=g["close"].shift(1)
d["gap_days"]=g["date"].diff().dt.days
d["observed_return"]=d["close"]/d["previous_close"]-1
d["mean_3"]=g["close"].transform(lambda s:s.rolling(3,min_periods=3).mean())
d["signal_before_observation"]=d.groupby("symbol")["mean_3"].shift(1)
d["missing_close"]=d["close"].isna()
assert g.head(1)["previous_close"].isna().all()
assert d.groupby("symbol").head(1)["signal_before_observation"].isna().all()
coverage=d.groupby("symbol").agg(rows=("date","size"),missing_close=("missing_close","sum"),first_date=("date","min"),last_date=("date","max"))
industry=d.groupby(["date","industry"])["observed_return"].agg(mean_return="mean",valid_returns="count").reset_index()
d.to_csv(OUT/"project_panel_features.csv",index=False)
coverage.to_csv(OUT/"project_panel_coverage.csv")
industry.to_csv(OUT/"project_panel_industry.csv",index=False)
fig,axes=plt.subplots(2,1,figsize=(8,6),sharex=True)
for symbol,part in d.groupby("symbol"):
    axes[0].plot(part["date"],part["close"],marker="o",label=symbol)
    axes[1].plot(part["date"],part["signal_before_observation"],label=symbol)
axes[0].set_ylabel("Observed close")
axes[1].set_ylabel("Lagged 3-observation mean")
axes[1].set_xlabel("Date")
axes[0].legend();axes[1].legend()
fig.autofmt_xdate();fig.tight_layout()
fig.savefig(OUT/"project_panel.png",dpi=160)
plt.close(fig)
print("原始行 / 清洗行 / 重复审计行:",len(raw),len(d),len(audit))
print(coverage.to_string())
print(d[["symbol","date","gap_days","observed_return","signal_before_observation"]].head(10).to_string(index=False))