"""Two Signals, One Risk Model: a blend of size-specialised adaptive ridges and shrunk mean-variance
characteristic portfolios, turned into weights with a statistical risk model.

Authors: Riccardo Molino, Alex Dahan (Yale SOM).

Everything is strictly walk-forward: the portfolio formed at month-end t only uses data known at t.

A) Signal 1 - stock-level ridge ensemble (`predict`)
   * Every characteristic is mapped to its cross-sectional rank in (-0.5, 0.5); missing values -> 0.
   * Nine engineered characteristics are added, all computed from daily returns up to month-end:
     three "trend states" (is the stock in or out of a simple rule: enter when the 5-day return
     exceeds +x, exit when it falls below -x or the price drops more than x from its post-entry
     peak; x = 5%, 10%, 20%), and six industry signals (stock return minus its 2-digit-SIC industry
     return, and the industry return, over 5, 21 and 126 trading days).
   * 20 ridge specifications are updated monthly on past data only: 10 pooled (5 penalties x 2 targets,
     the demeaned return or its rank) and 10 size-specialised (each stock scored by the model of its
     own size group: bottom 50%, next 30%, top 20% by market-cap rank). All past months count equally.
     Each month the 4 specifications with the best out-of-sample Sharpe ratio over their whole
     history are averaged.

B) Signal 2 - shrunk mean-variance over characteristic portfolios (`kns_score`), in the spirit of
   Kozak, Nagel and Santosh (2020)
   * 1,206 signals: 402 ranks, 402 industry-average ranks and 402 centred squared ranks. Each defines
     a long-short portfolio. Mean-variance weights b = (S + g I)^-1 mu are estimated from the
     full, equally weighted history of the portfolios' realised returns.
     The stock rankings are averaged over a grid of shrinkage levels g instead of tuning g.

C) Portfolio (`build_portfolio`)
   * The two signals are z-scored each month and averaged with equal weights.
   * From the trailing two years of daily returns: each stock's variance and K = 100 statistical
     factors (principal components). Covariance: Sigma = B B' + D.
   * Expected return = blended signal x stock volatility. Weights w = Sigma^-1 mu (Woodbury identity),
     rescaled every month to the same ex-ante volatility.

All design choices were made on the 1970-1989 validation sample, before the test period.
Only numpy and pandas are required.
"""
import numpy as np
import pandas as pd

TARGET = "ret_exc_lead1m"

# Stage A
ALPHAS = (1e-3, 1e-2, 1e-1, 1.0, 10.0)   # ridge penalties, relative to the average diagonal of X'X
TARGETS = ("raw", "rank")
HL_POOLED = (0,)                         # memory of the pooled ridges; 0 = no decay, all past months count equally
HL_GROUP = (0,)                          # memory of the size-specialised ridges; 0 = no decay
SIZE_COL = "market_equity"               # size groups come from the monthly market-cap rank
SIZE_CUTS = (0.0, 0.3)                   # rank cut-offs: bottom 50% small, next 30% large, top 20% mega
MIN_HISTORY = 24                         # months of out-of-sample history required before selecting
SEL_WINDOW = 10 ** 6                     # selection uses the whole out-of-sample history
TOP_N = 4                                # specifications averaged each month (of 20)
TREND_THRESHOLDS = (0.05, 0.10, 0.20)    # entry / exit thresholds of the trend-state rule
TREND_WINDOW = 365                       # calendar days of daily returns the rule is run on
TREND_MIN_OBS = 120                      # daily observations required, otherwise the state is missing
IND_HORIZONS = (5, 21, 126)              # trading-day horizons of the industry signals
IND_MIN_STOCKS = 5                       # stocks required to compute an industry average
N_EXTRA = len(TREND_THRESHOLDS) + 2 * len(IND_HORIZONS)

# Signal 2
KNS_HALF_LIFE = 0                        # months; 0 = no decay, all past months count equally
KNS_SHRINK_GRID = np.logspace(-3, 2, 11) # shrinkage levels, relative to the average eigenvalue
KNS_MIN_MONTHS = 24                      # months of history required

# Stage B
WINDOW_DAYS = 730                        # calendar days of daily returns used by the risk model
N_FACTORS = 100                          # principal components kept
MIN_OBS = 60                             # daily observations required to estimate a stock's variance
RESID_FLOOR = 0.25                       # idiosyncratic variance is at least 25% of total variance
TARGET_DAILY_VOL = 0.10 / np.sqrt(252)   # constant ex-ante risk (10% a year)


# ------------------------------------------------------------------------------------------ helpers
def to_bool(col: pd.Series) -> np.ndarray:
    """`ctff_test` may arrive as boolean, number or text."""
    if col.dtype == bool:
        return col.to_numpy()
    if pd.api.types.is_numeric_dtype(col):
        return col.fillna(0).to_numpy() != 0
    txt = col.astype(str).str.strip().str.lower()
    return txt.isin(["true", "t", "1", "1.0", "yes", "y"]).to_numpy()


def to_day(col) -> np.ndarray:
    """Dates -> integer number of days since 1970-01-01."""
    return np.asarray(pd.to_datetime(col)).astype("datetime64[D]").astype(np.int64).astype(np.int32)


def rank_transform(x: pd.DataFrame) -> np.ndarray:
    """Cross-sectional rank in (-0.5, 0.5); missing values -> 0."""
    n = x.notna().sum(axis=0).to_numpy()
    r = x.rank(axis=0, method="average").to_numpy(dtype=np.float64)
    with np.errstate(invalid="ignore", divide="ignore"):
        r = (r - 0.5) / n - 0.5
    return np.nan_to_num(r, nan=0.0, posinf=0.0, neginf=0.0)


def rank_target(y: np.ndarray) -> np.ndarray:
    order = np.argsort(y, kind="stable")
    r = np.empty(len(y))
    r[order] = np.arange(1, len(y) + 1)
    return (r - 0.5) / len(y) - 0.5


def zscore(v: np.ndarray) -> np.ndarray:
    sd = v.std()
    return (v - v.mean()) / sd if sd > 1e-12 else np.zeros_like(v)


def prop_weights(mu: np.ndarray) -> np.ndarray:
    """Dollar-neutral weights proportional to the signal (gross leverage 2); 1/N if there is no signal."""
    s = mu - mu.mean()
    gross = np.abs(s).sum()
    if not np.isfinite(gross) or gross < 1e-12:
        return np.full(len(mu), 1.0 / len(mu))
    return 2.0 * s / gross


# ------------------------------------------------------------------------------------------ engineered features
def window_matrix(ids_m, d_id, d_day, d_ret, m, window):
    """Days x stocks matrix of daily returns over (m - window, m] for the stocks `ids_m` (NaN = missing)."""
    N = len(ids_m)
    lo, hi = np.searchsorted(d_day, m - window, "right"), np.searchsorted(d_day, m, "right")
    if hi <= lo:
        return None
    u_order = np.argsort(ids_m, kind="stable")
    u_sorted = ids_m[u_order]
    pos = np.minimum(np.searchsorted(u_sorted, d_id[lo:hi]), N - 1)
    valid = u_sorted[pos] == d_id[lo:hi]
    if not valid.any():
        return None
    _, dinv = np.unique(d_day[lo:hi][valid], return_inverse=True)
    R = np.full((int(dinv.max()) + 1, N), np.nan, dtype=np.float32)
    R[dinv, u_order[pos[valid]]] = d_ret[lo:hi][valid]
    return R


def trend_states(R: np.ndarray) -> np.ndarray:
    """One column per threshold: 1 = in, 0 = out, NaN = not enough data."""
    D, N = R.shape
    logp = np.cumsum(np.log1p(np.clip(np.nan_to_num(R, nan=0.0).astype(np.float64), -0.95, None)), axis=0)
    out = np.full((N, len(TREND_THRESHOLDS)), np.nan)
    for j, thr in enumerate(TREND_THRESHOLDS):
        inside, peak = np.zeros(N, bool), np.zeros(N)
        for d in range(5, D):
            r5 = np.expm1(logp[d] - logp[d - 5])
            peak = np.where(inside, np.maximum(peak, logp[d]), peak)
            leave = inside & ((r5 <= -thr) | (np.expm1(logp[d] - peak) <= -thr))
            enter = ~inside & (r5 >= thr)
            inside = (inside & ~leave) | enter
            peak = np.where(enter, logp[d], peak)
        out[:, j] = inside
    out[np.isfinite(R).sum(axis=0) < TREND_MIN_OBS] = np.nan
    return out


def sic2(col) -> np.ndarray:
    """Industry code = first two digits of the SIC code (-1 if missing)."""
    return (pd.to_numeric(col, errors="coerce") // 100).fillna(-1).astype(int).to_numpy()


def industry_signals(R: np.ndarray, ind: np.ndarray) -> np.ndarray:
    """Per horizon: stock return minus industry return, then industry return (NaN = not available)."""
    D, N = R.shape
    logr = np.log1p(np.clip(R.astype(np.float64), -0.95, None))
    out = np.full((N, 2 * len(IND_HORIZONS)), np.nan)
    groups = [np.flatnonzero(ind == g) for g in np.unique(ind[ind >= 0])]
    for j, h in enumerate(IND_HORIZONS):
        if D < h:
            continue
        seg = logr[-h:]
        r = np.where(np.isfinite(seg).sum(axis=0) >= 0.7 * h, np.nansum(seg, axis=0), np.nan)
        ind_r = np.full(N, np.nan)
        for rows in groups:
            v = r[rows]
            v = v[np.isfinite(v)]
            if len(v) >= IND_MIN_STOCKS:
                ind_r[rows] = v.mean()
        out[:, j] = r - ind_r
        out[:, len(IND_HORIZONS) + j] = ind_r
    return out


def engineered_features(ids, eom_day, ind, d_id, d_day, d_ret) -> np.ndarray:
    """Trend states and industry signals for every stock-month, from daily returns up to that month-end only."""
    feats = np.full((len(ids), N_EXTRA), np.nan, dtype=np.float32)
    if len(d_day) == 0:
        return feats
    n_tr = len(TREND_THRESHOLDS)
    order_rows = np.argsort(eom_day, kind="stable")
    sorted_days = eom_day[order_rows]
    for m in np.unique(eom_day):
        idx = np.sort(order_rows[np.searchsorted(sorted_days, m, "left"):np.searchsorted(sorted_days, m, "right")])
        R = window_matrix(ids[idx], d_id, d_day, d_ret, m, TREND_WINDOW)
        if R is not None and R.shape[0] > 5:
            feats[idx, :n_tr] = trend_states(R)
            feats[idx, n_tr:] = industry_signals(R, ind[idx])
    return feats


# ------------------------------------------------------------------------------------------ stage A
class RidgeBank:
    """A family of ridge regressions (penalties x memory lengths x targets) updated month by month."""

    def __init__(self, p, half_lives):
        self.p = p
        self.decay = [0.5 ** (1.0 / h) if h > 0 else 1.0 for h in half_lives]
        self.G = [np.zeros((p, p)) for _ in half_lives]            # decayed X'X, one per memory length
        self.c = [np.zeros((p, len(TARGETS))) for _ in half_lives]  # decayed X'y, one column per target
        self.pending, self.n = None, 0
        self.n_cfg = len(half_lives) * len(ALPHAS) * len(TARGETS)

    def absorb(self):
        """Add last month's observations, whose returns are now realised."""
        if self.pending is not None:
            for i, d in enumerate(self.decay):
                self.G[i] = d * self.G[i] + self.pending[0]
                self.c[i] = d * self.c[i] + self.pending[1]
            self.n += 1
        self.pending = None

    def store(self, X, y):
        """Keep this month's observations aside; they are used from next month on."""
        if len(y) > 1:
            self.pending = (X.T @ X, X.T @ np.column_stack([y - y.mean(), rank_target(y)]))

    def betas(self):
        if self.n == 0 or np.trace(self.G[0]) <= 0:
            return None
        out, j, eye = np.empty((self.p, self.n_cfg)), 0, np.eye(self.p)
        for i in range(len(self.decay)):
            scale = np.trace(self.G[i]) / self.p
            for a in ALPHAS:
                out[:, j:j + len(TARGETS)] = np.linalg.solve(self.G[i] + a * scale * eye, self.c[i])
                j += len(TARGETS)
        return out


def predict(chars: pd.DataFrame, features: pd.DataFrame, extra: np.ndarray) -> np.ndarray:
    """Signal for every row of `chars` (0 until a model can be estimated)."""
    feats = [f for f in features["features"].astype(str).tolist() if f in chars.columns]
    p = len(feats) + extra.shape[1]
    j_size = feats.index(SIZE_COL) if SIZE_COL in feats else None
    dates = pd.to_datetime(chars["eom"])
    groups = dates.groupby(dates, sort=True).indices
    months = sorted(groups.keys())
    y_all = chars[TARGET].to_numpy(dtype=np.float64)

    pooled = RidgeBank(p, HL_POOLED)
    special = [RidgeBank(p, HL_GROUP) for _ in range(len(SIZE_CUTS) + 1)]
    n_p, n_s = pooled.n_cfg, special[0].n_cfg
    oos = np.full((len(months), n_p + n_s), np.nan)          # out-of-sample return of each specification
    known, pending_ret = [], None                            # formation months whose return is realised
    pred_all = np.zeros(len(chars))

    for k, m in enumerate(months):
        idx = np.sort(groups[m])
        sub = chars.take(idx)[feats].astype(np.float32)          # single precision: same ranks on any machine
        for j in range(extra.shape[1]):
            sub[f"engineered_{j}"] = extra[idx, j]
        X = rank_transform(sub)
        y = y_all[idx]
        grp = np.digitize(X[:, j_size], SIZE_CUTS) if j_size is not None else np.zeros(len(idx), dtype=int)

        pooled.absorb()
        for bank in special:
            bank.absorb()
        if pending_ret is not None:
            oos[pending_ret[0]] = pending_ret[1]
            known.append(pending_ret[0])

        Z = None
        Bp = pooled.betas()
        if Bp is not None:
            P = X @ Bp
            Z = np.zeros((len(idx), n_p + n_s))
            for j in range(n_p):
                Z[:, j] = zscore(P[:, j])
            for g, bank in enumerate(special):               # each stock is scored by its size group's model
                rows = np.flatnonzero(grp == g)
                Bg = bank.betas()
                if Bg is None or len(rows) < 2:
                    continue
                Pg = X[rows] @ Bg
                for j in range(n_s):
                    Z[rows, n_p + j] = zscore(Pg[:, j])
            chosen = np.arange(n_p)
            if len(known) >= MIN_HISTORY:
                hist = oos[known[-SEL_WINDOW:]]
                sr = hist.mean(axis=0) / (hist.std(axis=0) + 1e-12)
                chosen = np.argsort(-sr, kind="stable")[:TOP_N]
            pred_all[idx] = Z[:, chosen].mean(axis=1)

        ok = np.isfinite(y)
        pending_ret = None
        if ok.sum() > 1:
            pooled.store(X[ok], y[ok])
            for g, bank in enumerate(special):
                sel = ok & (grp == g)
                bank.store(X[sel], y[sel])
            if Z is not None:
                pending_ret = (k, np.array([float(prop_weights(Z[:, j])[ok] @ y[ok]) for j in range(Z.shape[1])]))

        if k % 120 == 0:
            print(f"signal 1: month {k + 1}/{len(months)} ({pd.Timestamp(m).date()})", flush=True)
    return np.nan_to_num(pred_all)


# ------------------------------------------------------------------------------------------ signal 2
def rank_block(x: pd.DataFrame) -> np.ndarray:
    """Cross-sectional rank of each column into [-0.5, 0.5]; missing -> 0."""
    r, n = x.rank(method="average"), x.count()
    return ((r - 1) / (n - 1).where(n > 1) - 0.5).fillna(0.0).to_numpy(np.float64)


def kns_signals(Xr: np.ndarray, ind: np.ndarray) -> np.ndarray:
    """Stock ranks, industry-average ranks and centred squared ranks."""
    XI = pd.DataFrame(Xr).groupby(ind).transform("mean").rank(pct=True).sub(0.5).to_numpy()
    Q = Xr ** 2
    Q -= Q.mean(axis=0)
    return np.hstack([Xr, XI, Q])


def centered_rank(P: np.ndarray) -> np.ndarray:
    return P.argsort(0, kind="stable").argsort(0, kind="stable") / max(P.shape[0] - 1, 1) - 0.5


def kns_score(chars, features, rows) -> np.ndarray:
    """Partner's stock score for every row selected by `rows` (0 elsewhere), using past data only."""
    feats = [f for f in features["features"].astype(str).tolist() if f in chars.columns]
    K = 3 * len(feats)
    dates = pd.to_datetime(chars["eom"])
    groups = dates.groupby(dates, sort=True).indices
    months = sorted(groups.keys())
    ids = chars["id"].to_numpy(dtype=np.float64)
    y_all = chars[TARGET].to_numpy(dtype=np.float64)
    ind_all = sic2(chars["sic"]) if "sic" in chars.columns else np.full(len(chars), -1)
    need = rows

    delta = 0.5 ** (1.0 / KNS_HALF_LIFE) if KNS_HALF_LIFE > 0 else 1.0
    W, s1, S2, prev = 0.0, np.zeros(K), np.zeros((K, K)), None     # decayed sums of portfolio returns
    score = np.zeros(len(chars))
    for k, m in enumerate(months):
        idx = np.sort(groups[m])
        idx = idx[np.argsort(ids[idx], kind="stable")]
        Z = kns_signals(rank_block(chars.take(idx)[feats].astype(np.float32)), ind_all[idx])
        if prev is not None:                                        # last month's portfolio returns are now known
            W, s1, S2 = delta * W + 1.0, delta * s1 + prev, delta * S2 + np.outer(prev, prev)
        if k >= KNS_MIN_MONTHS and W > 0 and need[idx].any():
            mu = s1 / W
            lam, V = np.linalg.eigh(S2 / W - np.outer(mu, mu))
            lam = np.maximum(lam, 0.0)
            ZV, vm, tr = Z @ V, V.T @ mu, lam.sum() / K
            P = np.column_stack([ZV @ (vm / (lam + g * tr)) for g in KNS_SHRINK_GRID])
            score[idx] = centered_rank(P).mean(axis=1)
        prev = Z.T @ np.nan_to_num(y_all[idx]) / len(idx)
        if k % 120 == 0:
            print(f"signal 2: month {k + 1}/{len(months)} ({pd.Timestamp(m).date()})", flush=True)
    return score


def blend_signals(eom_day: np.ndarray, a: np.ndarray, b: np.ndarray) -> np.ndarray:
    """Equal-weight average of the two signals after z-scoring each within the month."""
    out = np.zeros(len(a))
    order = np.argsort(eom_day, kind="stable")
    sorted_days = eom_day[order]
    for m in np.unique(eom_day):
        idx = order[np.searchsorted(sorted_days, m, "left"):np.searchsorted(sorted_days, m, "right")]
        out[idx] = zscore(a[idx]) + zscore(b[idx])
    return out


# ------------------------------------------------------------------------------------------ stage B
def risk_model(R: np.ndarray):
    """Days x stocks matrix of daily returns (NaN = missing) -> total variance and factor loadings."""
    D, N = R.shape
    finite = np.isfinite(R)
    cnt = finite.sum(axis=0)
    good = cnt >= MIN_OBS
    mean = np.where(finite, R, 0.0).astype(np.float64).sum(axis=0) / np.maximum(cnt, 1)
    Rc = np.where(finite, R - mean, 0.0)
    var = (Rc ** 2).sum(axis=0) / np.maximum(cnt, 1)
    if good.sum() < 20:
        return np.ones(N), None
    lo, med, hi = np.quantile(var[good], [0.05, 0.5, 0.95])
    var = np.where(good, np.clip(var, lo, hi), med)          # winsorise; thin histories get the median
    Rc[:, ~good] = 0.0
    _, evecs = np.linalg.eigh(Rc @ Rc.T / D)                 # PCA through the small days x days matrix
    k = min(N_FACTORS, D - 1)
    return var, Rc.T @ evecs[:, ::-1][:, :k] / np.sqrt(D)


def mvo_weights(mu: np.ndarray, var: np.ndarray, B) -> np.ndarray:
    """w = Sigma^-1 mu with Sigma = B B' + D, scaled to constant ex-ante volatility."""
    if B is not None and B.shape[1] > 0:
        resid = np.maximum(var - (B ** 2).sum(axis=1), RESID_FLOOR * var)
        Bd = B / resid[:, None]
        x = mu / resid - Bd @ np.linalg.solve(np.eye(B.shape[1]) + B.T @ Bd, Bd.T @ mu)
    else:
        x = mu / var
    exante_var = float(x @ mu)                               # equals x' Sigma x
    if not np.isfinite(exante_var) or exante_var <= 1e-18:
        return prop_weights(mu)
    return x * TARGET_DAILY_VOL / np.sqrt(exante_var)


def build_portfolio(ids, eom_day, pred, d_id, d_day, d_ret, rows) -> np.ndarray:
    """Weights for every stock-month in `rows`; other rows get 0."""
    w_all = np.zeros(len(ids))
    if len(d_day) > 1 and not np.all(d_day[1:] >= d_day[:-1]):   # daily data must be sorted by date
        order = np.argsort(d_day, kind="stable")
        d_id, d_day, d_ret = d_id[order], d_day[order], d_ret[order]

    months = np.unique(eom_day[rows])
    order_rows = np.argsort(eom_day, kind="stable")
    sorted_days = eom_day[order_rows]
    for k, m in enumerate(months):
        idx = np.sort(order_rows[np.searchsorted(sorted_days, m, "left"):np.searchsorted(sorted_days, m, "right")])
        N = len(idx)
        mu = pred[idx] - pred[idx].mean()
        if np.abs(mu).sum() < 1e-12:
            w_all[idx] = prop_weights(mu)
            continue

        var, B = np.ones(N), None
        lo = np.searchsorted(d_day, m - WINDOW_DAYS, "right")    # daily returns up to month-end m only
        hi = np.searchsorted(d_day, m, "right")
        if hi > lo:
            u_order = np.argsort(ids[idx], kind="stable")
            u_sorted = ids[idx][u_order]
            s_id = d_id[lo:hi]
            pos = np.minimum(np.searchsorted(u_sorted, s_id), N - 1)
            valid = u_sorted[pos] == s_id
            if valid.any():
                _, dinv = np.unique(d_day[lo:hi][valid], return_inverse=True)
                R = np.full((int(dinv.max()) + 1, N), np.nan, dtype=np.float32)
                R[dinv, u_order[pos[valid]]] = d_ret[lo:hi][valid]
                var, B = risk_model(R)

        w_all[idx] = np.nan_to_num(mvo_weights(mu * np.sqrt(var), var, B))
        if k % 60 == 0:
            print(f"portfolio: month {k + 1}/{len(months)} ({np.datetime64(int(m), 'D')})", flush=True)
    return w_all


# ------------------------------------------------------------------------------------------ entry point
def main(chars: pd.DataFrame, features: pd.DataFrame, daily_ret: pd.DataFrame) -> pd.DataFrame:
    np.random.seed(42)                                       # no randomness is used; set for safety
    is_test = to_bool(chars["ctff_test"])
    ids, eom_day = chars["id"].to_numpy(dtype=np.float64), to_day(chars["eom"])

    if daily_ret is None or len(daily_ret) == 0:
        d_id, d_day, d_ret = np.empty(0, np.float64), np.empty(0, np.int32), np.empty(0, np.float32)
    else:
        d_id = daily_ret["id"].to_numpy(dtype=np.float64)
        d_day = to_day(daily_ret["date"])
        d_ret = daily_ret["ret_exc"].to_numpy(dtype=np.float32)
        if not np.all(d_day[1:] >= d_day[:-1]):              # daily data must be sorted by date
            order = np.argsort(d_day, kind="stable")
            d_id, d_day, d_ret = d_id[order], d_day[order], d_ret[order]

    ind = sic2(chars["sic"]) if "sic" in chars.columns else np.full(len(chars), -1)
    print("engineered features...", flush=True)
    extra = engineered_features(ids, eom_day, ind, d_id, d_day, d_ret)
    signal_1 = predict(chars, features, extra)
    signal_2 = kns_score(chars, features, rows=is_test)
    pred = blend_signals(eom_day, signal_1, signal_2)
    w = build_portfolio(ids, eom_day, pred, d_id, d_day, d_ret, rows=is_test)
    id_out = chars["id"].to_numpy()
    if np.issubdtype(id_out.dtype, np.floating) and np.all(id_out == np.round(id_out)):
        id_out = id_out.astype(np.int64)                     # same values, integer type as required by Rule 12
    out = pd.DataFrame({"id": id_out, "eom": chars["eom"].to_numpy(), "w": w})
    return out[is_test].reset_index(drop=True)
