"""
Enhanced Strategy: Walk-Forward Factor Selection with Meta-Selection (v2)

Same four-stage design and the same temporal-integrity rules as the baseline
(04-StrategyTemplate.py). What changed:

  1. Ridge mean-variance over ALL factors (EXTENSION POINTS 3-4). Fifteen
     configs are added to the baseline's 36: window {10y, 20y, expanding} x
     shrinkage {0.10, 0.25, 0.50, 0.75, 0.90}, with
     w ~ inv((1-a) S + a tr(S)/n I) mu over every valid factor. Factor
     weights are long-short, normalised to sum |w| = 1 (in the spirit of
     Kozak, Nagel & Santosh 2020). The meta layer chooses among all 51.
  2. Monthly rebalancing of factor selection and meta-selection (was 6).
  3. Exact month-end arithmetic: the decision at month-end rp uses LS rows
     up to the PREVIOUS month-end (the baseline's DateOffset skipped the
     31st after 30-day months), and windows are exactly 120/240 rows.

Tested on 1960-1989 and rejected (the code is kept but off): signed |Sharpe|
selection, n_factors = 20, and volatility targeting.

Every design choice was made on 1960-1989 data only (the period JKP does not
score); see 06-DevelopPre1990.py and results/dev_*.log.

RUN IT
    python 05-StrategyEnhanced.py --data <folder>
"""

import argparse
import time

import numpy as np
import pandas as pd
from scipy.cluster.hierarchy import linkage, fcluster
from scipy.spatial.distance import squareform

# ---------------------------------------------------------------------------
# Hyperparameters, fixed on 1960-1989 development data only.
# ---------------------------------------------------------------------------
REBAL_MONTHS = 1      # rebalance / meta-selection frequency (months)
META_TOP_K = 3        # how many configs the meta layer blends
META_MIN_OBS = 120    # months of config history before meta-selection starts
MIN_OBS = 60          # months of data required for a factor to be usable
MIN_STOCKS = 10       # min stocks per side of a factor portfolio
MIN_HISTORY = 24      # months of history before the first rebalance
SHRINKAGE = 0.3       # covariance shrinkage in MeanVar weighting
N_CLUSTERS_MULT = 3   # clusters = n_factors * this, in ClusterSharpe
RIDGE_SHRINK = (0.1, 0.25, 0.5, 0.75, 0.9)  # ridge shrinkage menu (meta picks)
VOL_TARGET = False    # scale weights by 1/trailing strategy vol
VOL_LOOKBACK = 36     # months of realised strategy returns for vol targeting
VOL_TARGET_ANN = 0.05 # target annualised volatility when VOL_TARGET is on
VOL_CAP = (0.25, 4.0) # bounds on the vol-target scale relative to its median

# Menu dimensions (a full symmetric cross, plus the Ridge configs)
MENU_WINDOWS = (10, 20, None)
MENU_N = (5, 10)
MENU_SELECTION = ("Sharpe", "ClusterSharpe")   # Abs* rules: tested, rejected
MENU_WEIGHTING = ("EW", "RankSharpe", "MeanVar")
MENU_RIDGE = True

# The previous month-end. (The baseline's `rp - DateOffset(months=1)` maps
# 30 June to 30 May and so skipped the 31 May row: harmless but one month
# staler than allowed. With monthly rebalancing that would hit every 30-day
# month, so month-end arithmetic is used instead.)
MONTH_END = pd.offsets.MonthEnd(1)


# ===========================================================================
# Stage 1: factor portfolio returns  (unchanged from the baseline)
# ===========================================================================
def compute_factor_ls_returns(chars, feature_names, min_stocks, min_obs):
    """
    Linear-rank long-short return for every feature, every month.
    The LS value stored at month t uses ret_exc_lead1m, i.e. it is the return
    earned over month t+1 and is KNOWN only at the end of month t+1.
    """
    print("  Stage 1: factor LS returns...")
    t0 = time.time()

    valid_features = [f for f in feature_names if f in chars.columns]
    df = chars[["eom", "ret_exc_lead1m"] + valid_features]
    df = df[df["ret_exc_lead1m"].notna()]

    months = sorted(df["eom"].unique())
    ls_matrix = np.full((len(months), len(valid_features)), np.nan)

    for m_idx, eom in enumerate(months):
        month_data = df[df["eom"] == eom]
        if len(month_data) < min_stocks * 2:
            continue

        ret = month_data["ret_exc_lead1m"].values
        ranks = month_data[valid_features].rank(pct=True).values
        w = np.nan_to_num(ranks, nan=0.5) - 0.5
        not_nan = ~np.isnan(month_data[valid_features].values)
        w = w * not_nan

        num = (ret[:, None] * w).sum(axis=0)
        den = np.abs(w).sum(axis=0)
        ok = (not_nan.sum(axis=0) >= min_stocks * 2) & (den > 1e-10)
        row = np.full(len(valid_features), np.nan)
        row[ok] = num[ok] / den[ok]
        ls_matrix[m_idx] = row

    ls_df = pd.DataFrame(ls_matrix, index=pd.to_datetime(months),
                         columns=valid_features).sort_index()
    ls_df = ls_df[ls_df.columns[ls_df.notna().sum() >= min_obs]]
    print(f"    {ls_df.shape[0]} months x {ls_df.shape[1]} factors "
          f"({time.time()-t0:.0f}s)")
    return ls_df


# ===========================================================================
# Selection rules: history -> ordered list of factor names
# ===========================================================================
def _sharpe_series(tc):
    stds = tc.std().replace(0, np.nan)
    return ((tc.mean() / stds) * np.sqrt(12)).fillna(0)


_LINKAGE_CACHE = {}


def _cluster_labels(tcv, n_factors, cache_key):
    """Average-linkage clusters on 1-|corr|; the linkage is cached per
    (rebalance, window) because it does not depend on n_factors."""
    if cache_key not in _LINKAGE_CACHE:
        corr = tcv.corr().fillna(0).clip(-1, 1)
        dist = (1.0 - corr.abs()).values.copy()
        np.fill_diagonal(dist, 0)
        dist = np.clip((dist + dist.T) / 2, 0, None)
        condensed = np.nan_to_num(squareform(dist, checks=False),
                                  nan=1.0, posinf=1.0, neginf=0.0)
        _LINKAGE_CACHE[cache_key] = linkage(condensed, method="average")
    Z = _LINKAGE_CACHE[cache_key]
    n_clusters = min(max(n_factors * N_CLUSTERS_MULT, 20), tcv.shape[1])
    return fcluster(Z, t=n_clusters, criterion="maxclust")


def _top_by_score(scores, n_factors):
    return list(scores.nlargest(min(n_factors, len(scores))).index)


def _cluster_pick(tc, valid, n_factors, scores, cache_key):
    tcv = tc[valid]
    if len(tcv) < 12 or len(valid) < n_factors:
        return _top_by_score(scores, n_factors)
    try:
        labels = _cluster_labels(tcv, n_factors, cache_key)
    except Exception:
        return _top_by_score(scores, n_factors)
    cdf = pd.DataFrame({"factor": valid, "cluster": labels,
                        "score": scores.reindex(valid).fillna(-999).values})
    best = cdf.loc[cdf.groupby("cluster")["score"].idxmax()]
    best = best[best["score"] > -998].sort_values("score", ascending=False)
    return best.head(n_factors)["factor"].tolist()


def select_sharpe(tc, valid, n_factors, cache_key=None):
    """Top-N factors by annualized Sharpe over the lookback slice."""
    return _top_by_score(_sharpe_series(tc[valid]), n_factors)


def select_cluster_sharpe(tc, valid, n_factors, cache_key=None):
    """Best-Sharpe factor from each of the best N correlation clusters."""
    return _cluster_pick(tc, valid, n_factors,
                         _sharpe_series(tc[valid]), cache_key)


def select_abs_sharpe(tc, valid, n_factors, cache_key=None):
    """Top-N by |Sharpe|; negative-Sharpe factors are later held short."""
    return _top_by_score(_sharpe_series(tc[valid]).abs(), n_factors)


def select_abs_cluster_sharpe(tc, valid, n_factors, cache_key=None):
    """ClusterSharpe on |Sharpe| (clusters use |corr|, so they are
    sign-invariant)."""
    return _cluster_pick(tc, valid, n_factors,
                         _sharpe_series(tc[valid]).abs(), cache_key)


def select_all(tc, valid, n_factors, cache_key=None):
    """Every valid factor (for the all-factor Ridge MVE)."""
    return list(valid)


SELECT_FN = {
    "Sharpe": select_sharpe,
    "ClusterSharpe": select_cluster_sharpe,
    "AbsSharpe": select_abs_sharpe,
    "AbsClusterSharpe": select_abs_cluster_sharpe,
    "All": select_all,
}
SIGNED = {"AbsSharpe", "AbsClusterSharpe"}


# ===========================================================================
# Weighting rules: history + chosen factors -> weight vector
# ===========================================================================
def weight_ew(tc, factors):
    return np.full(len(factors), 1.0 / len(factors))


def weight_rank_sharpe(tc, factors):
    """Best factor by Sharpe gets the largest weight (rank weights)."""
    order = _sharpe_series(tc[factors]).rank(method="first")
    w = order.values.astype(float)
    return w / w.sum()


def weight_mean_var(tc, factors):
    """w ~ Sigma^{-1} mu with diagonal shrinkage; long-only, sums to 1."""
    tcf = tc[factors].fillna(0)
    n = len(factors)
    if len(tcf) < max(12, n + 2):
        return np.full(n, 1.0 / n)
    mu, cov = tcf.mean().values, tcf.cov().values
    cov = (1 - SHRINKAGE) * cov + SHRINKAGE * (np.trace(cov) / n) * np.eye(n)
    try:
        w = np.linalg.inv(cov) @ mu
    except np.linalg.LinAlgError:
        return np.full(n, 1.0 / n)
    w = np.maximum(w, 0)
    return w / w.sum() if w.sum() > 1e-10 else np.full(n, 1.0 / n)


def _make_ridge(alpha):
    def weight_ridge(tc, factors):
        """Long-short tangency weights over all factors:
        w ~ inv((1-a) S + a tr(S)/n I) mu, normalised to sum |w| = 1."""
        tcf = tc[factors].fillna(0)
        n = len(factors)
        mu, cov = tcf.mean().values, tcf.cov().values
        cov = (1 - alpha) * cov + alpha * (np.trace(cov) / n) * np.eye(n)
        try:
            w = np.linalg.solve(cov, mu)
        except np.linalg.LinAlgError:
            return np.full(n, 1.0 / n)
        s = np.abs(w).sum()
        return w / s if s > 1e-12 else np.full(n, 1.0 / n)
    return weight_ridge


WEIGHT_FN = {
    "EW": weight_ew,
    "RankSharpe": weight_rank_sharpe,
    "MeanVar": weight_mean_var,
}


def get_weight_fn(name):
    """'RidgeXX' -> all-factor ridge MVE with shrinkage XX/100."""
    if name.startswith("Ridge"):
        return _make_ridge(int(name[5:]) / 100)
    return WEIGHT_FN[name]


# ===========================================================================
# The candidate menu: a symmetric full cross, plus the Ridge configs
# ===========================================================================
def build_menu():
    grid = [{"window": w, "n_factors": n, "selection": s, "weighting": g}
            for w in MENU_WINDOWS for n in MENU_N
            for s in MENU_SELECTION for g in MENU_WEIGHTING]
    if MENU_RIDGE:
        grid += [{"window": w, "n_factors": 0, "selection": "All",
                  "weighting": f"Ridge{int(a * 100)}"}
                 for w in MENU_WINDOWS for a in RIDGE_SHRINK]
    return grid


def config_label(cfg):
    w = "all" if cfg["window"] is None else f"{cfg['window']}y"
    n = "all" if cfg["selection"] == "All" else cfg["n_factors"]
    return f"{w}/{n}/{cfg['selection']}/{cfg['weighting']}"


# ===========================================================================
# Stage 2: walk-forward simulation of every config
# ===========================================================================
def simulate_configs(ls_returns, configs, rebal_months, min_obs, min_history):
    """
    At each rebalance rp, every config selects factors/weights from LS rows
    <= rp - 1 month, then earns the realized LS returns until the next
    rebalance. Factor weights may be negative (signed / Ridge configs).
    """
    print("  Stage 2: walk-forward config simulation...")
    t0 = time.time()
    _LINKAGE_CACHE.clear()
    all_months = ls_returns.index.sort_values()
    rebal_points = list(all_months[::rebal_months])

    oos = pd.DataFrame(np.nan, index=all_months, columns=range(len(configs)))
    schedule = {}
    sel_cache = {}

    for i, rp in enumerate(rebal_points):
        sel_end = rp - MONTH_END                 # last KNOWN LS row
        ls_avail = ls_returns.loc[:sel_end]
        if len(ls_avail.dropna(how="all")) < min_history:
            continue

        next_rp = rebal_points[i + 1] if i + 1 < len(rebal_points) else None
        block = (all_months[(all_months >= rp) & (all_months < next_rp)]
                 if next_rp is not None else all_months[all_months >= rp])

        threshold = min(min_obs, max(6, len(ls_avail) // 2))
        valid = list(ls_avail.columns[ls_avail.notna().sum() >= threshold])
        if not valid:
            continue

        block_ret = ls_returns.loc[block].fillna(0)
        per_cfg = {}
        tc_cache = {}
        for ci, cfg in enumerate(configs):
            if cfg["window"] not in tc_cache:
                if cfg["window"] is None:
                    tc = ls_avail
                else:   # exactly the last 12*window known monthly rows
                    tc = ls_avail.tail(12 * cfg["window"])
                tc_cache[cfg["window"]] = tc.fillna(0)
            tc = tc_cache[cfg["window"]]
            if len(tc) < 12:
                continue

            key = (rp, cfg["window"], cfg["selection"], cfg["n_factors"])
            if key not in sel_cache:
                sel_cache[key] = SELECT_FN[cfg["selection"]](
                    tc, valid, cfg["n_factors"],
                    cache_key=(rp, cfg["window"]))
            factors = sel_cache[key]
            if not factors:
                continue

            if cfg["selection"] in SIGNED:
                signs = np.sign(_sharpe_series(tc[factors]).values)
                signs[signs == 0] = 1.0
                tcs = tc[factors] * signs
                weights = get_weight_fn(cfg["weighting"])(tcs, factors) * signs
            else:
                weights = get_weight_fn(cfg["weighting"])(tc, factors)
            per_cfg[ci] = (factors, weights)
            oos.loc[block, ci] = block_ret[factors].values @ weights

        if per_cfg:
            schedule[rp] = per_cfg

    print(f"    {len(configs)} configs x {len(schedule)} rebalances "
          f"({time.time()-t0:.0f}s)")
    return rebal_points, schedule, oos


# ===========================================================================
# Stage 3: meta-selection  (unchanged from the baseline)
# ===========================================================================
def meta_select(rebal_points, schedule, oos, configs, top_k, meta_min_obs):
    """
    At each rebalance, blend the top_k configs by the annualized Sharpe of
    their OWN out-of-sample returns up to t-1. Until any config has
    meta_min_obs months of history, hold an equal blend of all configs.
    """
    print("  Stage 3: meta-selection...")
    chosen, log = {}, []
    for rp in rebal_points:
        if rp not in schedule:
            continue
        active = list(schedule[rp].keys())
        hist = oos.loc[:rp - MONTH_END]
        counts = hist.notna().sum()

        eligible = [ci for ci in active if counts[ci] >= meta_min_obs]
        if eligible:
            score = {}
            for ci in eligible:
                s = hist[ci].dropna()
                sd = s.std()
                score[ci] = (s.mean() / sd) * np.sqrt(12) if sd > 0 else -np.inf
            top = sorted(score, key=score.get, reverse=True)[:top_k]
            mode = "meta"
        else:
            top, mode = active, "ew_prior"

        chosen[rp] = top
        log.append({"rebal": rp, "mode": mode,
                    "chosen": ";".join(config_label(configs[ci])
                                       for ci in top)})
    n_meta = sum(1 for r in log if r["mode"] == "meta")
    print(f"    {len(chosen)} rebalances ({n_meta} meta-selected)")
    return chosen, log


# ===========================================================================
# Blended factor-level returns and the volatility-target scale
# ===========================================================================
def blended_factor_returns(ls_returns, schedule, chosen):
    """Return of the meta-chosen blend at each LS row t (realised at t+1),
    using the same 'latest rebalance <= t' rule as Stage 4."""
    rebal = sorted(chosen)
    filled = ls_returns.fillna(0)
    out = pd.Series(np.nan, index=ls_returns.index)
    for i, rp in enumerate(rebal):
        nxt = rebal[i + 1] if i + 1 < len(rebal) else None
        mask = (filled.index >= rp) & ((filled.index < nxt)
                                       if nxt is not None else True)
        block = filled.loc[mask]
        vals = [block[schedule[rp][ci][0]].values @ schedule[rp][ci][1]
                for ci in chosen[rp]]
        out.loc[mask] = np.mean(vals, axis=0)
    return out


def vol_target_scale(blend_ret, months):
    """
    Scale for the weights at month t: target / trailing volatility of the
    strategy's blended returns over LS rows <= t - 1 month (row t-1 is
    realised at the end of month t). Bounded relative to its own expanding
    median so one quiet spell cannot lever the book without limit.
    """
    r = blend_ret.dropna()
    scale = {}
    hist_scales = []
    for t in months:
        h = r.loc[:t - MONTH_END].tail(VOL_LOOKBACK)
        if len(h) < 12 or h.std() <= 0:
            s = 1.0
        else:
            s = (VOL_TARGET_ANN / np.sqrt(12)) / h.std()
            if hist_scales:
                med = float(np.median(hist_scales))
                s = float(np.clip(s, VOL_CAP[0] * med, VOL_CAP[1] * med))
            hist_scales.append(s)
        scale[t] = s
    return pd.Series(scale)


# ===========================================================================
# Stage 4: stock-level weights
# ===========================================================================
def compute_stock_weights(chars, schedule, chosen, scale=None):
    """
    Blend the chosen configs equally and map factor weights to stocks:
      w_i = s_t * (1/K) * sum_cfg sum_k fw_k * (rank_i - 0.5) / sum_j|rank_j - 0.5|
    s_t is the vol-target scale (1 when vol targeting is off).
    """
    print("  Stage 4: stock-level weights...")
    t0 = time.time()
    rebal_dates = sorted(chosen.keys())
    if not rebal_dates:
        return pd.DataFrame(columns=["id", "eom", "w"])

    all_factors = sorted({f for rp in rebal_dates for ci in chosen[rp]
                          for f in schedule[rp][ci][0]})
    available = [f for f in all_factors if f in chars.columns]
    df_work = chars[["id", "eom"] + available]
    months = sorted(df_work["eom"].unique())

    results = []
    for eom in months:
        applicable = [d for d in rebal_dates if d <= eom]
        if not applicable:
            continue
        rp = applicable[-1]
        cfg_ids = chosen[rp]

        month_data = df_work[df_work["eom"] == eom]
        if len(month_data) < 50:
            continue

        needed = sorted({f for ci in cfg_ids for f in schedule[rp][ci][0]
                         if f in month_data.columns})
        if not needed:
            continue
        ranks_all = month_data[needed].rank(pct=True)
        col_pos = {f: j for j, f in enumerate(needed)}
        rv = ranks_all.values
        nan_mask = np.isnan(rv)
        rw = np.where(nan_mask, 0.0, rv - 0.5)
        denom = np.abs(rw).sum(axis=0)
        usable = (denom >= 1e-10) & ((~nan_mask).sum(axis=0) >= MIN_STOCKS * 2)

        # net factor weight per needed column, blended across configs
        fw_net = np.zeros(len(needed))
        blend = 1.0 / len(cfg_ids)
        for ci in cfg_ids:
            factors, weights = schedule[rp][ci]
            for factor, fw in zip(factors, weights):
                j = col_pos.get(factor)
                if j is not None:
                    fw_net[j] += blend * fw
        coef = np.where(usable, fw_net / np.where(usable, denom, 1.0), 0.0)
        stock_w = rw @ coef
        if scale is not None:
            stock_w = stock_w * float(scale.get(eom, 1.0))

        nz = np.abs(stock_w) > 1e-15
        if nz.any():
            results.append(pd.DataFrame({"id": month_data["id"].values[nz],
                                         "eom": eom, "w": stock_w[nz]}))

    output = pd.concat(results, ignore_index=True)
    print(f"    {len(output):,} rows, {output['eom'].nunique()} months "
          f"({time.time()-t0:.0f}s)")
    return output


# ===========================================================================
# Entry point (JKP CTF interface): main(chars, features, daily_ret)
# ===========================================================================
def thresholds(n_months):
    if n_months < 200:   # small validation datasets: loosen thresholds
        return dict(min_obs=max(6, n_months // 5), min_stocks=5,
                    min_history=12, meta_min_obs=max(24, n_months // 3))
    return dict(min_obs=MIN_OBS, min_stocks=MIN_STOCKS,
                min_history=MIN_HISTORY, meta_min_obs=META_MIN_OBS)


def main(chars, features, daily_ret):
    feature_names = features["features"].tolist()
    chars = chars.copy()
    chars["eom"] = pd.to_datetime(chars["eom"])
    n_months = chars["eom"].nunique()
    print(f"Data: {len(chars):,} stock-months, {len(feature_names)} "
          f"features, {n_months} months")
    th = thresholds(n_months)
    configs = build_menu()

    ls = compute_factor_ls_returns(chars, feature_names, th["min_stocks"],
                                   th["min_obs"])
    rebal_points, schedule, oos = simulate_configs(
        ls, configs, REBAL_MONTHS, th["min_obs"], th["min_history"])
    chosen, meta_log = meta_select(
        rebal_points, schedule, oos, configs, META_TOP_K, th["meta_min_obs"])

    scale = None
    if VOL_TARGET:
        blend = blended_factor_returns(ls, schedule, chosen)
        scale = vol_target_scale(blend, sorted(chars["eom"].unique()))

    main.meta_log = meta_log
    main.config_oos = oos
    main.config_labels = [config_label(c) for c in configs]

    output = compute_stock_weights(chars, schedule, chosen, scale)
    assert list(output.columns) == ["id", "eom", "w"]
    assert output["w"].notna().all() and len(output) > 0
    return output


# ===========================================================================
# Stand-alone runner with evaluation
# ===========================================================================
if __name__ == "__main__":
    ap = argparse.ArgumentParser()
    ap.add_argument("--data", required=True,
                    help="directory containing ctff_*.parquet")
    ap.add_argument("--out", default=None,
                    help="path for the weights CSV "
                         "(default: <data>/weights.csv)")
    args = ap.parse_args()

    chars = pd.read_parquet(f"{args.data}/ctff_chars.parquet")
    features = pd.read_parquet(f"{args.data}/ctff_features.parquet")
    weights = main(chars, features,
                   pd.DataFrame(columns=["id", "date", "ret_exc"]))

    # Gross portfolio returns: weights at eom t earn ret_exc_lead1m of t
    chars["eom"] = pd.to_datetime(chars["eom"])
    weights["eom"] = pd.to_datetime(weights["eom"])
    merged = weights.merge(
        chars[["id", "eom", "ret_exc_lead1m"]].dropna(),
        on=["id", "eom"], how="inner")
    port = (merged["w"] * merged["ret_exc_lead1m"]).groupby(
        merged["eom"]).sum().sort_index()

    def sharpe(r):
        return r.mean() / r.std() * np.sqrt(12) if len(r) >= 12 else np.nan

    print("\n  SHARPE RATIOS BY PERIOD (gross)")
    for name, (a, b) in {
        "1960-1989": ("1960", "1989"), "1990-2003": ("1990", "2003"),
        "2004-2013": ("2004", "2013"), "2014-2023": ("2014", "2023"),
        "1990-2023": ("1990", "2023"),
    }.items():
        sub = port.loc[a:b]
        print(f"  {name}: {sharpe(sub):7.3f}  ({len(sub)} months)")

    # One-way turnover, as a fraction of the gross book
    piv = weights.pivot_table(index="eom", columns="id", values="w",
                              fill_value=0.0)
    gross = piv.abs().sum(axis=1)
    to_abs = piv.diff().abs().sum(axis=1) / 2
    to_rel = to_abs / gross.shift(1)
    print(f"\n  Avg monthly one-way turnover: {to_abs.iloc[1:].mean():.3f} "
          f"(same measure as baseline); {to_rel.iloc[1:].mean():.3f} "
          f"as a fraction of the gross book")
    print(f"  Avg gross leverage sum|w|: {gross.mean():.3f}")

    out_path = args.out or f"{args.data}/weights.csv"
    submission = weights[["id", "eom", "w"]].copy()
    submission["eom"] = submission["eom"].dt.strftime("%Y-%m-%d")
    submission.to_csv(out_path, index=False, float_format="%.10g")
    print(f"  Wrote {len(submission):,} weight rows -> {out_path}")
