"""
CTF submission: stock picking resized by market regime risk.

At every month-end t, using only data available at t:
  Part 1  what to hold: a market-neutral long-short book from a theme factor
          model of the characteristics;
  Part 2  which regime: P(turbulent) from a two-state hidden Markov model of
          weekly market returns and volatility;
  Part 3  how much to hold: the book scaled by its past volatility over all
          months divided by its past volatility in the current regime.
Every design choice (number of themes, HMM specification, sizing rule) is made
inside main() on the months before the test period. All random draws are
seeded. Dependencies: numpy, pandas, scipy.

CTF Admin Modifications (2026-10-18):
--------------------------------------
1. Left the "all signals" candidate (None) out of theme_grid when the
   CTF_EXECUTION_MODE environment variable is "validation", and added
   "import os" to read it.
   Reason: that candidate regresses returns on all 402 characteristics and
   needs more than 403 stocks in a month. The validation dataset has about
   150 stocks a month, so every month was skipped and the validation run
   stopped with "ValueError: high <= 0". Full runs still try every candidate,
   so the full-run weights are unchanged.
"""

import os

import numpy as np
import pandas as pd
from scipy.cluster.hierarchy import fcluster, linkage
from scipy.spatial.distance import squareform
from scipy.stats import rankdata


# =============================================================================
# Part 1: stock picking, survival-game theme model
# =============================================================================
# Each December, with the returns realized by then (main() uses
# run(..., themes="cofail", game=True, scorer="hero")):
#   signals    one rank-weighted long-short portfolio per characteristic,
#              signed by its past average return
#   themes     Ward clustering of the signals on co-failure similarity (how often
#              two signals rank in the bottom third in the same month); members
#              weighted by a "hero" survival score; exposure = average signed rank
#   factors    monthly cross-sectional regressions on the exposures give theme
#              returns f; mu_f = empirical-Bayes shrunk means (intercept 0),
#              Sigma_f = Ledoit-Wolf covariance, D = 36-month residual variances
#   portfolio  zero-investment tangency Sigma^-1 mu with Sigma = B Sigma_f B' + D,
#              scaled to unit gross exposure
#   n themes   one-standard-error rule on a checkerboard split of pre-test months
# The other options of run() (correlation themes, theme selection, minimum
# variance) are not used by main().


# Constants -------------------------------------------------------------------
# CTF-FIX: validation data has ~150 stocks a month, too few for the all-signals regression
VALIDATION_RUN = os.environ.get("CTF_EXECUTION_MODE", "full").strip().lower() == "validation"
theme_grid = [5, 8, 10, 13, 16, 20, 30, 50, 100] + ([] if VALIDATION_RUN else [None])   # None: no clustering (all signals)
fallback_n = 13               # Used if there is too little pre-test history to choose
min_select_months = 120       # Minimum pre-test months for the checkerboard rule
idio_window = 36              # Months of residuals for idiosyncratic variance
idio_min = 12                 # Minimum residual months, else cross-sectional median
idio_shrink = 0.5             # Weight on the stock's own variance vs the median
roll_months = None            # None: expanding window; else rolling window length


# Helpers ---------------------------------------------------------------------
def month_bounds(eom):
    months, start = np.unique(eom, return_index=True)
    return months, list(zip(start, np.append(start[1:], len(eom))))


def sharpe(r):
    return np.mean(r) / np.std(r, ddof=1) * np.sqrt(12)


def ledoit_wolf(R):
    """Ledoit-Wolf (2004) shrinkage of the covariance of R (T x K) toward a scaled identity."""
    T, K = R.shape
    Xc = R - R.mean(axis=0)
    S = Xc.T @ Xc / T
    nu = np.trace(S) / K
    d2 = np.sum((S - nu * np.eye(K)) ** 2)
    b2 = np.sum((Xc ** 2).T @ (Xc ** 2)) / T ** 2 - np.sum(S ** 2) / T   # sum_t ||x x' - S||^2 / T^2
    delta = min(max(b2 / d2, 0.0), 1.0) if d2 > 0 else 1.0
    return (1 - delta) * S + delta * nu * np.eye(K)


def eb_mean(R, rel=None):
    """Empirical-Bayes shrinkage of the average returns of R (T x K) toward zero.

    Prior: true premia ~ N(0, tau^2). With sample means m_k and standard
    errors se_k, tau^2 = mean(m^2) - mean(se^2) (method of moments, floored at
    0) and each premium is multiplied by kappa_k = tau^2 / (tau^2 + se_k^2),
    as in Jensen, Kelly and Pedersen (2023). With rel (one number per column,
    average 1), the prior variance of premium k is tau^2 * rel_k: premia with
    a larger rel are trusted more (used by the survival game)."""
    m = R.mean(axis=0)
    se2 = R.var(axis=0, ddof=1) / R.shape[0]
    tau2 = max(np.mean(m ** 2) - np.mean(se2), 0.0)
    t2 = tau2 if rel is None else tau2 * rel
    return m * t2 / (t2 + se2)


def checkerboard(r_month, use):
    """Two halves of the months in use, by the month the return is realized:
    (1) odd months in even years and even months in odd years; (2) the rest."""
    yr = np.array([d.astype(object).year for d in r_month])
    mo = np.array([d.astype(object).month for d in r_month])
    h1 = ((mo % 2 == 1) & (yr % 2 == 0)) | ((mo % 2 == 0) & (yr % 2 == 1))
    return use & h1, use & ~h1


# Step 1: signal returns ------------------------------------------------------
def signal_returns(X, ret, bounds, fit=None):
    """Rank-weighted long-short return of every characteristic, by formation month,
    and the equal-weighted market return, using the rows in fit (default: all)."""
    K = X.shape[1]
    F = np.full((len(bounds), K), np.nan)
    mkt = np.full(len(bounds), np.nan)
    for t, (a, b) in enumerate(bounds):
        y = ret[a:b]
        ok = ~np.isnan(y) if fit is None else ~np.isnan(y) & fit[a:b]
        if ok.sum() == 0:
            continue
        x = np.asarray(X[a:b][ok], dtype=np.float64)
        g = np.abs(x).sum(axis=0)
        F[t] = x.T @ y[ok] / np.where(g > 0, g, 1)
        mkt[t] = y[ok].mean()
    return F, mkt


# Step 2: themes --------------------------------------------------------------
cofail_draws = 10000          # months drawn (with replacement) for co-failure themes
cofail_cut = 1 / 3            # a signal "fails" in a month if it ranks in the bottom third


def cofailure_corr(F_hist, seed=0, draws=cofail_draws, cut=cofail_cut):
    """Co-failure similarity from the survival game: draw real months with
    replacement; in each, a signal fails if its relevance (signed,
    volatility-scaled return) ranks in the bottom `cut`. Returns the phi
    coefficient of joint failure for every pair of signals."""
    M = relevance(F_hist)
    rng = np.random.default_rng(seed)
    t = rng.integers(0, len(M), size=draws)
    Mt = M[t]
    B = (Mt <= np.quantile(Mt, cut, axis=1, keepdims=True)).astype(np.float64)
    p = B.mean(axis=0)
    P = B.T @ B / draws
    sd = np.sqrt(p * (1 - p))
    return (P - np.outer(p, p)) / np.where(np.outer(sd, sd) > 0, np.outer(sd, sd), 1)


def cohero_corr(F_hist, seed=0, trials=5000):
    """Co-hero similarity from the hero test: in each trial (three 12-month picks,
    one per third of the history), record which signals are heroes (never below
    -2 of their own standard deviations). Returns the correlation of these hero
    indicators across trials for every pair of signals."""
    M = relevance(F_hist)
    rng = np.random.default_rng(seed)
    T = len(M)
    edges = np.linspace(0, T, 4).astype(int)
    H = np.ones((trials, M.shape[1]), bool)
    for a, b in zip(edges[:-1], edges[1:]):
        H &= (M[rng.integers(a, b, size=(trials, hero_pick))] > hero_threshold).all(axis=1)
    C = np.corrcoef(H.astype(np.float64), rowvar=False)
    return np.nan_to_num(C, nan=0.0)


def form_themes(F_hist, mkt_hist, n, method="corr", seed=0):
    """Sign the signals by their past mean and cluster them into n themes
    (n = None: no clustering). Similarity: correlation of market-adjusted
    signal returns (method="corr", Jensen, Kelly and Pedersen 2023) or
    co-failure in the survival game (method="cofail"). Returns the signs, the
    theme label of every signal, and the loading matrix L (K x n) that
    averages signed members."""
    K = F_hist.shape[1]
    sign = np.sign(np.nanmean(F_hist, axis=0))
    sign[sign == 0] = 1
    if n is None or n >= K:
        return sign, np.arange(1, K + 1), np.diag(sign)
    if method == "cofail":
        C = cofailure_corr(F_hist, seed=seed)
    elif method == "cohero":
        C = cohero_corr(F_hist, seed=seed)
    else:
        Fs = F_hist * sign
        m = mkt_hist - mkt_hist.mean()
        beta = (m @ (Fs - Fs.mean(axis=0))) / (m @ m)
        resid = Fs - np.outer(mkt_hist, beta)
        C = np.nan_to_num(np.corrcoef(resid, rowvar=False), nan=0.0)
    np.fill_diagonal(C, 1.0)
    D = np.sqrt(np.clip(2 * (1 - C), 0, None))     # Euclidean form of 1 - corr, for Ward
    np.fill_diagonal(D, 0.0)
    labels = fcluster(linkage(squareform(D, checks=False), method="ward"), n, criterion="maxclust")
    L = np.zeros((K, labels.max()))
    for k in range(labels.max()):
        idx = labels == k + 1
        L[idx, k] = sign[idx] / idx.sum()
    return sign, labels, L


def exposures(x, L):
    """Theme exposures of one month's stocks: z-scored average signed ranks, plus intercept."""
    S = np.asarray(x, dtype=np.float64) @ L
    sd = S.std(axis=0)
    S = (S - S.mean(axis=0)) / np.where(sd > 0, sd, 1)
    return np.column_stack([np.ones(len(S)), S])


# Step 3: factor model --------------------------------------------------------
def theme_regressions(X, ret, bounds, L, fit=None):
    """Monthly cross-sectional regressions of returns on theme exposures, using
    the rows in fit (default: all). Returns the factor returns f
    (months x (1 + n)) and the residuals of every stock."""
    f = np.full((len(bounds), L.shape[1] + 1), np.nan)
    res = np.full(len(ret), np.nan)
    for t, (a, b) in enumerate(bounds):
        y = ret[a:b]
        ok = ~np.isnan(y)
        use = ok if fit is None else ok & fit[a:b]
        if use.sum() <= L.shape[1] + 1:
            continue
        B = exposures(X[a:b], L)
        coef = np.linalg.lstsq(B[use], y[use], rcond=None)[0]
        f[t] = coef
        res[a:b][ok] = y[ok] - B[ok] @ coef
    return f, res


def factor_tangency(f_hist):
    """Factor-level tangency weights on the theme returns (intercept excluded)."""
    R = f_hist[:, 1:]
    return np.linalg.solve(ledoit_wolf(R), eb_mean(R))


# Optional: the survival game -------------------------------------------------------
game_n = 1000                 # games per tournament
game_stop = 0.10              # the game stops when 10% of the players are left
game_gap = 240                # confirmation month at least 20 years away (past or future)
game_cut = 1 / 3              # a winner loses if it ranks in the bottom third that month


def relevance(R):
    """Relevance of each player in each month: its return, signed by its average
    over the months given and divided by its volatility (months x players)."""
    sgn = np.sign(R.mean(axis=0))
    sgn[sgn == 0] = 1
    sd = R.std(axis=0)
    return R * sgn / np.where(sd > 0, sd, 1)


def game_scores(M, cal, seed=0, n_games=game_n, stop=game_stop, gap=game_gap, cut=game_cut):
    """The survival tournament on the months in M (months x players).

    Each game: every round draws one real month (with replacement) and
    eliminates the player still in play with the lowest relevance that month,
    until stop of the field is left. The winners then face one real month at
    least gap calendar months away (past or future) from the month that ended
    the game: a winner ranking in the bottom cut of all players that month
    loses its win. Score per game: confirmed winner 1, winner that failed 0.5,
    eliminated player half its share of the game survived. Returns the average
    score over n_games games. cal: calendar month number of each row of M."""
    T, K = M.shape
    if K == 1:
        return np.ones(1)
    rng = np.random.default_rng(seed)
    n_stop = max(int(np.ceil(stop * K)), 1)
    rounds = K - n_stop
    total = np.zeros(K)
    for _g in range(n_games):
        alive = np.ones(K, bool)
        lasted = np.zeros(K)
        draws = rng.integers(0, T, size=rounds)
        for r, t in enumerate(draws):
            out = int(np.argmin(np.where(alive, M[t], np.inf)))
            alive[out] = False
            lasted[out] = r / rounds
        sc = lasted / 2
        sc[alive] = 0.5
        far = np.where(np.abs(cal - cal[draws[-1]]) >= gap)[0]
        if len(far):
            check = M[rng.choice(far)]
            sc[alive & (check > np.quantile(check, cut))] = 1.0
        else:
            sc[alive] = 1.0
        total += sc
    return total / n_games


hero_trials = 1000            # trials of the hero test
hero_pick = 12                # months per mini-pick
hero_threshold = -2.0         # shot if the month's return is below -2 of its own standard deviations


def hero_scores(M, seed=0, trials=hero_trials, pick=hero_pick, threshold=hero_threshold):
    """The hero test. Each trial makes three mini-picks of `pick` real months
    (drawn with replacement), one from each third of the history in M (months x
    players, oldest first). A player is shot in a month if its relevance (return
    over its own volatility) is below `threshold`; it is a hero in the trial only
    if it is never shot in any of the three picks. Returns each player's share of
    trials as a hero."""
    rng = np.random.default_rng(seed)
    T = len(M)
    edges = np.linspace(0, T, 4).astype(int)
    hero = np.ones((trials, M.shape[1]), bool)
    for a, b in zip(edges[:-1], edges[1:]):
        draws = rng.integers(a, b, size=(trials, pick))
        hero &= (M[draws] > threshold).all(axis=1)
    return hero.mean(axis=0)


def game_themes(F_hist, cal, labels, sign, seed=0, scorer="survival"):
    """Level 1: one tournament per theme among its member signals. Returns the
    loading matrix L with members weighted by their game score (survival rate,
    or the hero test with scorer="hero")."""
    M = relevance(F_hist)
    n = labels.max()
    L = np.zeros((F_hist.shape[1], n))
    for k in range(n):
        idx = np.where(labels == k + 1)[0]
        sc = (hero_scores(M[:, idx], seed=seed + k) if scorer == "hero"
              else game_scores(M[:, idx], cal, seed=seed + k))
        w = sc if sc.sum() > 0 else np.ones(len(idx))
        L[idx, k] = sign[idx] * w / w.sum()
    return L


# Optional: random-block elimination ----------------------------------------------
def tangency_sr2(R):
    """In-sample tangency squared Sharpe ratio (monthly), with the model's estimators."""
    mu = eb_mean(R)
    return mu @ np.linalg.solve(ledoit_wolf(R), mu)


def eliminate(R, years, n_draws=200, n_top=10, seed=0):
    """Random-block backward elimination of themes (columns of R).

    In each draw, take a random half of the calendar years; starting from all
    themes, repeatedly drop the theme whose removal lowers the tangency squared
    Sharpe ratio the least. Returns each theme's survival: the share of draws in
    which it is among the last n_top themes standing."""
    rng = np.random.default_rng(seed)
    uy = np.unique(years)
    n = R.shape[1]
    top = np.zeros(n)
    for _d in range(n_draws):
        Rd = R[np.isin(years, rng.choice(uy, size=len(uy) // 2, replace=False))]
        alive = list(range(n))
        while len(alive) > n_top:
            base = tangency_sr2(Rd[:, alive])
            cost = [base - tangency_sr2(Rd[:, [a for a in alive if a != k]]) for k in alive]
            alive.remove(alive[int(np.argmin(cost))])
        top[alive] += 1
    return top / n_draws


def select_themes(f_hist, r_month_hist, n_draws=200, n_top=10, keep=0.5, seed=0):
    """Indices of the themes that survive in more than `keep` of the draws."""
    years = np.array([d.astype(object).year for d in r_month_hist])
    surv = eliminate(f_hist[:, 1:], years, n_draws, n_top, seed)
    idx = np.where(surv > keep)[0]
    return idx if len(idx) else np.argsort(-surv)[:n_top], surv


# Step 5: number of themes ----------------------------------------------------
def choose_n_themes(X, ret, bounds, F, mkt, r_month, use, grid=theme_grid, verbose=True,
                    method="corr", game=False, scorer="survival"):
    """One-standard-error rule on a checkerboard split of the months in use.

    For each candidate n and each direction: sign and cluster the signals on one
    half, compute the theme returns, estimate the factor tangency on that half
    and apply it to the other half. The combined out-of-sample series is scored
    by its Sharpe ratio. The smallest n within one standard error of the best
    is chosen (standard error of a Sharpe ratio: sqrt((1 + SR^2/2) / T) per
    month, Lo 2002). method and game select the same theme construction and
    survival game as the deployed model, so the rule scores the model it chooses."""
    h1, h2 = checkerboard(r_month, use)
    cal = np.array([d.astype("datetime64[M]").astype(int) for d in r_month]) - 1
    raw_f = None
    sr = {}
    for n in grid:
        oos = []
        for est, app in ((h1, h2), (h2, h1)):
            sign, labels, L = form_themes(F[est], mkt[est], n, method=method, seed=1)
            if game and n is not None:
                L = game_themes(F[est], cal[est], labels, sign, seed=1, scorer=scorer)
            if n is None:
                # No clustering: regressions do not depend on the signs, so run them once
                if raw_f is None:
                    raw_f, _r = theme_regressions(X, ret, bounds, np.eye(X.shape[1]))
                f = raw_f * np.append(1.0, sign)
            else:
                f, _r = theme_regressions(X, ret, bounds, L)
            # (months with fewer stocks than regressors have no factor returns)
            fe, fa = f[est], f[app]
            oke = ~np.isnan(fe).any(axis=1)
            fe, fa = fe[oke], fa[~np.isnan(fa).any(axis=1)]
            rel = None
            if game:
                sc = game_scores(relevance(fe[:, 1:]), cal[est][oke], seed=2)
                rel = sc / sc.mean() if sc.mean() > 0 else None
            oos.append(fa[:, 1:] @ np.linalg.solve(ledoit_wolf(fe[:, 1:]), eb_mean(fe[:, 1:], rel)))
        comb = np.concatenate(oos)
        sr[n] = (sharpe(comb), len(comb))
        if verbose:
            print(f"  {n if n else 'all'} themes: checkerboard OOS Sharpe {sr[n][0]:.2f}", flush=True)
    best = max(grid, key=lambda n: sr[n][0])
    sr_m = sr[best][0] / np.sqrt(12)
    se = np.sqrt((1 + sr_m ** 2 / 2) / sr[best][1]) * np.sqrt(12)
    size = {n: (n if n else X.shape[1]) for n in grid}
    chosen = min((n for n in grid if sr[n][0] >= sr[best][0] - se), key=lambda n: size[n])
    if verbose:
        print(f"best {best if best else 'all'} ({sr[best][0]:.2f}), one SE = {se:.2f} "
              f"-> {chosen if chosen else 'all'} themes", flush=True)
    return chosen, sr


# Steps 1-4 walked forward ----------------------------------------------------
def run(eom, eom_ret, ids, ret, X, test_start, n="auto", roll=roll_months, start=None,
        end=None, verbose=True, fit=None, apply=None, freeze=False, idio_w=idio_shrink, select=False,
        game=False, themes="corr", portfolio="tangency", scorer="survival"):
    """Walk forward month by month from start (default test_start), re-forming
    the themes each December. Returns (ids, eom, w) and a log per refit.

    Options for the robustness tests: estimate everything on the rows in fit
    and form portfolios only from the rows in apply (default: all rows);
    freeze=True keeps the themes and factor moments estimated at start for
    the whole period; idio_w is the weight on a stock's own residual variance;
    select=True keeps, each December, only the themes that survive the
    random-block elimination (select_themes) on the data available then;
    game=True weights the members of each theme and the trust in each theme's
    premium by the survival game, replayed each December on past months;
    portfolio="gmv" returns the global minimum-variance portfolio
    Sigma^-1 1 / 1'Sigma^-1 1 (fully invested) instead of the tangency."""
    months, bounds = month_bounds(eom)
    m_ret = np.array([eom_ret[a] for a, _ in bounds])           # month each return is realized
    F, mkt = signal_returns(X, ret, bounds, fit)
    uid, id_idx = np.unique(ids, return_inverse=True)
    ok_m = ~np.isnan(F).any(axis=1) & ~np.isnan(mkt)

    if n == "auto":
        pre = ok_m & (m_ret <= test_start)
        if pre.sum() >= min_select_months:
            n, _sr = choose_n_themes(X, ret if fit is None else np.where(fit, ret, np.nan),
                                     bounds, F, mkt, m_ret, pre, verbose=verbose, method=themes, game=game,
                                     scorer=scorer)
        else:
            n = fallback_n

    start = test_start if start is None else start
    run_months = [m for m in months if m >= start and (end is None or m <= end)]
    refits = {m for m in run_months if m.astype(object).month == 12} | {run_months[0]}
    if freeze:
        refits = {run_months[0]}
    raw_f, raw_res = (theme_regressions(X, ret, bounds, np.eye(X.shape[1]), fit) if n is None else (None, None))

    cal_all = months.astype("datetime64[M]").astype(int)
    rel_f = None
    out_id, out_eom, out_w, log = [], [], [], []
    for m in run_months:
        t_now = int(np.searchsorted(months, m))
        avail = ok_m & (m_ret <= (run_months[0] if freeze else m))   # returns realized by m
        if roll is not None:
            avail &= months > (m.astype("datetime64[M]") - roll).astype("datetime64[D]")

        if m in refits:
            sign, labels, L = form_themes(F[avail], mkt[avail], n, method=themes,
                                          seed=int(m.astype(object).year))
            if game and n is not None:
                L = game_themes(F[avail], cal_all[avail], labels, sign, seed=int(m.astype(object).year),
                                scorer=scorer)
            if n is None:
                f_all, res_all = raw_f * np.append(1.0, sign), raw_res
            else:
                f_all, res_all = theme_regressions(X, ret, bounds, L, fit)
            kept = None
            if select:
                fh = f_all[avail]
                okh = ~np.isnan(fh).any(axis=1)
                kept, _surv = select_themes(fh[okh], m_ret[avail][okh], seed=int(m.astype(object).year))
                L = L[:, kept]
                f_all, res_all = theme_regressions(X, ret, bounds, L, fit)
            if game:
                fh = f_all[avail]
                okh = ~np.isnan(fh).any(axis=1)
                sc = game_scores(relevance(fh[okh][:, 1:]), cal_all[avail][okh], seed=int(m.astype(object).year) + 1000)
                rel_f = sc / sc.mean() if sc.mean() > 0 else None
            log.append({"refit": m, "n": L.shape[1], "labels": labels.copy(), "sign": sign.copy(), "kept": kept,
                        "theme_score": None if rel_f is None else rel_f.copy()})
            if verbose:
                print(f"refit {m}: {L.shape[1]} themes", flush=True)

        # Factor moments from the returns realized by m
        f_hist = f_all[avail]
        f_hist = f_hist[~np.isnan(f_hist).any(axis=1)]
        mu_f = np.append(0.0, eb_mean(f_hist[:, 1:], rel_f))     # no bet on the market
        Sf = ledoit_wolf(f_hist)

        # Idiosyncratic variance from each stock's last idio_window months of residuals
        a, b = bounds[t_now]
        anchor = run_months[0] if freeze else m
        win = avail & (months > (anchor.astype("datetime64[M]") - idio_window).astype("datetime64[D]"))
        rows = np.concatenate([np.arange(*bounds[t]) for t in np.where(win)[0]])
        r2 = res_all[rows]
        ok = ~np.isnan(r2)
        ss = np.bincount(id_idx[rows][ok], weights=r2[ok] ** 2, minlength=len(uid))
        cnt = np.bincount(id_idx[rows][ok], minlength=len(uid))
        own = np.where(cnt >= idio_min, ss / np.maximum(cnt, 1), np.nan)[id_idx[a:b]]
        med = np.nanmedian(own) if np.isfinite(own).any() else np.mean(r2[ok] ** 2)
        Dv = np.where(np.isnan(own), med, idio_w * own + (1 - idio_w) * med)

        # Tangency with the zero-investment constraint, via Woodbury:
        # Sigma^-1 v = D^-1 v - D^-1 B (Sf^-1 + B'D^-1 B)^-1 B'D^-1 v
        sel = np.ones(b - a, bool) if apply is None else apply[a:b]
        B = exposures(X[a:b], L)[sel]
        Di = 1 / Dv[sel]
        M = np.linalg.inv(Sf) + B.T @ (B * Di[:, None])

        def sinv(v):
            return Di * v - Di * (B @ np.linalg.solve(M, B.T @ (Di * v)))

        s_mu, s_1 = sinv(B @ mu_f), sinv(np.ones(sel.sum()))
        if portfolio == "gmv":
            out_id.append(ids[a:b][sel])
            out_eom.append(np.full(sel.sum(), m))
            out_w.append(s_1 / s_1.sum())                         # weights sum to 1
            continue
        w = s_mu - (s_mu.sum() / s_1.sum()) * s_1
        out_id.append(ids[a:b][sel])
        out_eom.append(np.full(sel.sum(), m))
        out_w.append(w / np.abs(w).sum())

    return (np.concatenate(out_id), np.concatenate(out_eom), np.concatenate(out_w)), log


def rank_standardize(X, eom):
    """Within each month, map every characteristic to its rank in [-0.5, 0.5]
    (average ranks for ties); missing values get 0, the cross-sectional median.
    Uses only same-month information."""
    _m, bounds = month_bounds(eom)
    for a, b in bounds:
        x = X[a:b]
        r = rankdata(x, axis=0, nan_policy="omit")
        n = np.sum(~np.isnan(x), axis=0)
        z = (r - 1) / np.maximum(n - 1, 1) - 0.5
        z[:, n <= 1] = 0
        X[a:b] = np.nan_to_num(z, nan=0.0)
    return X


# =============================================================================
# Part 2: market regime, two-state HMM with Gaussian-mixture emissions
# =============================================================================
SEED = 42
HMM_CANDIDATES = [        # (Gaussians per state K, rolling window in years W)
    (1, 2), (1, 10),
    (2, 2), (2, 10), (2, 15),
    (3, 20),
]
HMM_DEFAULT = (2, 10)     # used when there is too little pre-test history to choose
HMM_MIN_VAL_YEARS = 5     # validation years needed to choose among the candidates
HMM_REFIT_EVERY = 1       # re-estimate every month-end (12: every December)
THRESHOLD = 0.5           # turbulent when P(turbulent) >= THRESHOLD
BLOCK_DAYS = 5            # trading days per weekly block
MIN_BLOCK_DAYS = 3        # shorter leftovers are merged into the next block
MIN_FIT_BLOCKS = 80       # weekly blocks needed to estimate the HMM
N_INIT = 8                # EM starting points per estimation
MAX_ITER = 300            # EM iterations
TOL = 1e-6                # EM convergence tolerance (log-likelihood per obs)
REG_COVAR = 1e-3          # variance floor on standardized data
NW_LAGS = 12              # Newey-West lags for the selection standard error
MONTH_OFFSET = 5000       # keeps month indices positive inside integer keys


def _to_int(s: pd.Series) -> np.ndarray:
    return s.astype("int64").to_numpy()


def _to_float(s: pd.Series) -> np.ndarray:
    """float64 array with NaN for missing values, for numpy, object (decimal) and
    pyarrow-backed columns alike."""
    try:
        return s.to_numpy(dtype=np.float64, na_value=np.nan)
    except (TypeError, ValueError):
        return pd.to_numeric(s.astype(object), errors="coerce").to_numpy(dtype=np.float64, na_value=np.nan)


def _month_index(s) -> np.ndarray:
    """Months since 1970-01 (Jan 1970 = 0)."""
    d = pd.to_datetime(pd.Series(s)).to_numpy().astype("datetime64[M]")
    return d.astype(np.int64)


def _day_index(s) -> np.ndarray:
    d = pd.to_datetime(pd.Series(s)).to_numpy().astype("datetime64[D]")
    return d.astype(np.int64)


def _dec_month(year: int) -> int:
    """Month index of December of `year`."""
    return (year - 1970) * 12 + 11


def _model_year(month: np.ndarray) -> np.ndarray:
    """HMM estimated in December Y is used for decisions Dec Y ... Nov Y+1."""
    year = month // 12 + 1970
    return np.where(month % 12 == 11, year, year - 1)


def _logsumexp(a: np.ndarray, axis: int) -> np.ndarray:
    m = np.max(a, axis=axis, keepdims=True)
    return np.squeeze(m, axis=axis) + np.log(np.sum(np.exp(a - m), axis=axis))


def newey_west_se(x: np.ndarray, lags: int) -> float:
    """Newey-West standard error of the mean of x."""
    x = np.asarray(x, float) - np.mean(x)
    n = len(x)
    s = np.dot(x, x) / n
    for l in range(1, min(lags, n - 1) + 1):
        s += 2 * (1 - l / (lags + 1)) * np.dot(x[l:], x[:-l]) / n
    return float(np.sqrt(max(s, 0.0) / n))


def build_daily_market(chars: pd.DataFrame, daily_ret: pd.DataFrame) -> pd.DataFrame:
    """Value-weighted daily excess return of the stocks in `chars`.

    Weights on day d of month m are market equity at the end of month m-1,
    drifted with each stock's own returns earlier in month m.
    """
    me = _to_float(chars["market_equity"])
    ok = np.isfinite(me) & (me > 0)
    key_c = _to_int(chars["id"])[ok] * 100000 + _month_index(chars["eom"])[ok] + MONTH_OFFSET
    me = me[ok]
    key_c, first = np.unique(key_c, return_index=True)
    me = me[first]
    index = pd.Index(key_c)

    day = _day_index(daily_ret["date"])
    month = day.astype("datetime64[D]").astype("datetime64[M]").astype(np.int64)
    r = _to_float(daily_ret["ret_exc"])
    key_d = _to_int(daily_ret["id"]) * 100000 + (month - 1) + MONTH_OFFSET
    pos = index.get_indexer(key_d)
    keep = (pos >= 0) & np.isfinite(r)
    day, r, key_d, pos = day[keep], r[keep], key_d[keep], pos[keep]

    # contiguous (stock, month) groups in date order
    sorted_ = np.all((np.diff(key_d) > 0) | ((np.diff(key_d) == 0) & (np.diff(day) > 0)))
    if not sorted_:
        order = np.lexsort((day, key_d))
        day, r, key_d, pos = day[order], r[order], key_d[order], pos[order]

    lr = np.log1p(np.maximum(r, -0.9999))
    cs = np.cumsum(lr)
    start = np.r_[True, key_d[1:] != key_d[:-1]]
    grp = np.cumsum(start) - 1
    base = (cs - lr)[start]
    drift = np.exp(cs - lr - base[grp])           # gross return before day d
    w = me[pos] * drift

    d0 = day.min()
    num = np.bincount(day - d0, weights=w * r)
    den = np.bincount(day - d0, weights=w)
    has = den > 0
    days = np.flatnonzero(has) + d0
    out = pd.DataFrame({
        "date": days.astype("datetime64[D]"),
        "ret": num[has] / den[has],
    })
    out["month"] = out["date"].to_numpy().astype("datetime64[M]").astype(np.int64)
    return out


def build_blocks(mkt: pd.DataFrame) -> pd.DataFrame:
    """Weekly blocks anchored at month-ends; returns (ret, logvol) per block."""
    mkt = mkt.sort_values("date").reset_index(drop=True)
    month = mkt["month"].to_numpy()
    n = mkt.groupby("month")["ret"].transform("size").to_numpy()
    k = n - 1 - mkt.groupby("month").cumcount().to_numpy()   # 0 = last day
    full = n // BLOCK_DAYS
    left = n % BLOCK_DAYS
    b = k // BLOCK_DAYS
    merge = (full >= 1) & (left < MIN_BLOCK_DAYS) & (b == full)
    b = np.where(merge, full - 1, b)

    r = mkt["ret"].to_numpy()
    df = pd.DataFrame({
        "month": month, "b": -b, "date": mkt["date"],
        "lr": np.log1p(np.maximum(r, -0.9999)), "r2": r ** 2,
    })
    g = df.groupby(["month", "b"], sort=True)
    blk = g.agg(end=("date", "max"), n=("lr", "size"), lr=("lr", "sum"), r2=("r2", "mean"))
    blk = blk.reset_index().drop(columns="b")
    blk["ret"] = blk["lr"] * np.sqrt(BLOCK_DAYS / blk["n"])
    blk["logvol"] = 0.5 * np.log(np.maximum(blk["r2"], 1e-12))
    return blk[["month", "end", "n", "ret", "logvol"]]


def _log_gauss(Z: np.ndarray, mu: np.ndarray, cov: np.ndarray) -> np.ndarray:
    L = np.linalg.cholesky(cov)
    sol = np.linalg.solve(L, (Z - mu).T)
    return -0.5 * (Z.shape[1] * np.log(2 * np.pi) + np.sum(sol ** 2, axis=0)) \
        - np.sum(np.log(np.diag(L)))


def _emission_logs(Z, c, mu, cov):
    """log b[s,k,t] = log c_sk + log N(z_t; mu_sk, cov_sk); returns (log b, log B)."""
    S, K = c.shape
    lb = np.empty((S, K, len(Z)))
    for s in range(S):
        for k in range(K):
            lb[s, k] = np.log(max(c[s, k], 1e-300)) + _log_gauss(Z, mu[s, k], cov[s, k])
    return lb, _logsumexp(lb, axis=1)


def _forward(logB: np.ndarray, A: np.ndarray, pi: np.ndarray):
    """Scaled forward pass for 2 states. Returns filtered probs, log c_t."""
    m = logB.max(axis=0)
    B = np.exp(logB - m)
    b0, b1 = B[0].tolist(), B[1].tolist()
    a00, a01, a10, a11 = A[0, 0], A[0, 1], A[1, 0], A[1, 1]
    T = len(b0)
    f = np.empty((T, 2))
    logc = np.empty(T)
    g0, g1 = pi[0] * b0[0], pi[1] * b1[0]
    s = g0 + g1
    f0, f1 = g0 / s, g1 / s
    f[0] = f0, f1
    logc[0] = np.log(s)
    for t in range(1, T):
        g0 = (f0 * a00 + f1 * a10) * b0[t]
        g1 = (f0 * a01 + f1 * a11) * b1[t]
        s = g0 + g1
        f0, f1 = g0 / s, g1 / s
        f[t, 0], f[t, 1] = f0, f1
        logc[t] = np.log(s)
    return f, logc + m, B, np.exp(logc)


def _backward(B: np.ndarray, A: np.ndarray, c: np.ndarray) -> np.ndarray:
    b0, b1 = B[0].tolist(), B[1].tolist()
    a00, a01, a10, a11 = A[0, 0], A[0, 1], A[1, 0], A[1, 1]
    cl = c.tolist()
    T = len(b0)
    beta = np.empty((T, 2))
    e0 = e1 = 1.0
    beta[T - 1] = e0, e1
    for t in range(T - 2, -1, -1):
        u0, u1 = b0[t + 1] * e0, b1[t + 1] * e1
        e0 = (a00 * u0 + a01 * u1) / cl[t + 1]
        e1 = (a10 * u0 + a11 * u1) / cl[t + 1]
        beta[t, 0], beta[t, 1] = e0, e1
    return beta


def _m_step(Z, resp_sk):
    """Mixture weights, means and covariances from responsibilities (S,K,T)."""
    S, K, _ = resp_sk.shape
    D = Z.shape[1]
    nk = resp_sk.sum(axis=2) + 1e-10
    c = nk / nk.sum(axis=1, keepdims=True)
    mu = np.einsum("skt,td->skd", resp_sk, Z) / nk[..., None]
    cov = np.empty((S, K, D, D))
    for s in range(S):
        for k in range(K):
            dz = Z - mu[s, k]
            cov[s, k] = (resp_sk[s, k, :, None] * dz).T @ dz / nk[s, k] + REG_COVAR * np.eye(D)
    return c, mu, cov


def _em(Z: np.ndarray, K: int, resp_s: np.ndarray, rng: np.random.Generator):
    """EM from an initial (hard) state assignment resp_s (2,T)."""
    T = len(Z)
    resp_sk = np.zeros((2, K, T))
    comp = rng.integers(0, K, size=T)
    for k in range(K):
        resp_sk[:, k, :] = resp_s * (comp == k)
    c, mu, cov = _m_step(Z, resp_sk)
    A = np.array([[0.95, 0.05], [0.10, 0.90]])
    pi = np.array([0.5, 0.5])
    ll_old = -np.inf
    for _ in range(MAX_ITER):
        lb, logB = _emission_logs(Z, c, mu, cov)
        f, logc, B, cs = _forward(logB, A, pi)
        ll = logc.sum()
        beta = _backward(B, A, cs)
        gamma = f * beta
        gamma /= gamma.sum(axis=1, keepdims=True)
        X = B[:, 1:].T * beta[1:] / cs[1:, None]
        xi = A * (f[:-1].T @ X)
        A = xi / xi.sum(axis=1, keepdims=True)
        pi = np.clip(gamma[0], 1e-6, None)
        pi /= pi.sum()
        resp_sk = gamma.T[:, None, :] * np.exp(lb - logB[:, None, :])
        c, mu, cov = _m_step(Z, resp_sk)
        if ll - ll_old < TOL * T:
            break
        ll_old = ll
    lb, logB = _emission_logs(Z, c, mu, cov)
    ll = _forward(logB, A, pi)[1].sum()
    return dict(A=A, pi=pi, c=c, mu=mu, cov=cov, ll=ll)


def fit_hmm(X: np.ndarray, K: int, seed: int = SEED) -> dict:
    """Fit the 2-state GMM-HMM on raw observations X (T,2) = (ret, logvol)."""
    mean, std = X.mean(axis=0), X.std(axis=0)
    Z = (X - mean) / std
    rng = np.random.default_rng(seed)
    best = None
    for i in range(N_INIT):
        q = 0.75 if i == 0 else rng.uniform(0.55, 0.90)
        hi = Z[:, 1] > np.quantile(Z[:, 1], q)       # high volatility weeks
        resp_s = np.vstack([~hi, hi]).astype(float)
        m = _em(Z, K, resp_s, rng)
        if best is None or m["ll"] > best["ll"] + 1e-9:
            best = m
    # label: state 1 = turbulent = higher mean log volatility
    vol_mean = (best["c"] * best["mu"][:, :, 1]).sum(axis=1)
    if vol_mean[0] > vol_mean[1]:
        p = [1, 0]
        best["A"] = best["A"][np.ix_(p, p)]
        for key in ("pi", "c", "mu", "cov"):
            best[key] = best[key][p]
    best.update(mean=mean, std=std, K=K, n=len(X))
    return best


def filter_hmm(model: dict, X: np.ndarray):
    """Filtered P(turbulent) and one-step-ahead predictive log density per obs."""
    Z = (X - model["mean"]) / model["std"]
    _, logB = _emission_logs(Z, model["c"], model["mu"], model["cov"])
    f, logc, _, _ = _forward(logB, model["A"], model["pi"])
    return f[:, 1], logc - np.sum(np.log(model["std"]))


def walk_forward(blocks: pd.DataFrame, K: int, W: int, first_month: int, last_month: int,
                 every: int = HMM_REFIT_EVERY):
    """P(turbulent) at the month-ends first_month..last_month (month indices).

    The HMM is re-estimated on the blocks of the last W years up to a month-end
    and used until the next re-estimation: every month-end (every=1) or every
    December (every=12). Estimations with fewer than MIN_FIT_BLOCKS blocks are
    skipped (no regime for those months). Also returns the one-step-ahead
    predictive score of every block, from the model in force before its month."""
    month = blocks["month"].to_numpy()
    X = blocks[["ret", "logvol"]].to_numpy()
    end = blocks["end"].to_numpy()
    last_block = np.r_[month[1:] != month[:-1], True]
    if every == 1:
        refits = list(range(first_month, last_month + 1))
    else:
        refits = list(range(_dec_month(int(_model_year(np.array([first_month]))[0])), last_month + 1, 12))
    probs, scores = [], []
    for i, r in enumerate(refits):
        nxt = refits[i + 1] if i + 1 < len(refits) else r + every
        lo = r - 12 * W + 1
        train = (month >= lo) & (month <= r)
        if train.sum() < MIN_FIT_BLOCKS:
            continue
        model = fit_hmm(X[train], K)
        seq = (month >= lo) & (month <= nxt)
        p, sc = filter_hmm(model, X[seq])
        m_seq = month[seq]
        sel = last_block[seq] & (m_seq >= r) & (m_seq < nxt)
        probs.append(pd.DataFrame({"month": m_seq[sel], "p_turb": p[sel]}))
        out = m_seq > r
        scores.append(pd.DataFrame({"end": end[seq][out], "score": sc[out]}))
    if not probs:
        return pd.DataFrame(columns=["month", "p_turb"]), pd.DataFrame(columns=["end", "score"])
    return pd.concat(probs, ignore_index=True), pd.concat(scores, ignore_index=True)


def select_candidate(val_scores: dict) -> tuple:
    """Best predictive log-likelihood; fewest Gaussians within one NW standard error.
    Candidates are always visited in HMM_CANDIDATES order (deterministic ties)."""
    cands = [c for c in HMM_CANDIDATES if c in val_scores]
    common = np.array(sorted(val_scores[cands[0]]["end"].unique()))
    for c in cands[1:]:
        common = np.intersect1d(common, val_scores[c]["end"].to_numpy())
    al = {}
    for c in cands:
        s = val_scores[c]
        al[c] = s[s["end"].isin(common)].sort_values("end", kind="stable")["score"].to_numpy()
    best = cands[int(np.argmax([al[c].mean() for c in cands]))]
    rows = []
    for c in cands:
        v = al[c]
        diff = al[best] - v
        se = newey_west_se(diff, NW_LAGS) if c != best else 0.0
        rows.append(dict(K=c[0], W=c[1], score=v.mean(), ok=diff.mean() <= se))
    rows = sorted([r for r in rows if r["ok"]], key=lambda r: (r["K"], -r["score"]))
    return rows[0]["K"], rows[0]["W"]


def regime_probabilities(chars: pd.DataFrame, daily_ret: pd.DataFrame, first_decision_month: int,
                         first_test_month: int, last_month: int, verbose: bool = True) -> pd.Series:
    """P(turbulent) at every month-end from first_decision_month to last_month."""
    blocks = build_blocks(build_daily_market(chars, daily_ret))
    first_month = int(blocks["month"].min())
    first_year = first_month // 12 + 1970 + (1 if first_month % 12 != 0 else 0)
    val_first = _dec_month(first_year + max(w for _, w in HMM_CANDIDATES) - 1)
    if first_test_month - val_first >= 12 * HMM_MIN_VAL_YEARS:
        val_scores = {c: walk_forward(blocks, c[0], c[1], val_first, first_test_month - 1)[1]
                      for c in HMM_CANDIDATES}
        K, W = select_candidate(val_scores)
    else:
        K, W = HMM_DEFAULT
    if verbose:
        print(f"HMM: {K} Gaussians per state, {W}-year rolling window, re-estimated every "
              f"{'month' if HMM_REFIT_EVERY == 1 else 'December'}", flush=True)
    probs, _ = walk_forward(blocks, K, W, first_decision_month, last_month)
    return probs.set_index("month")["p_turb"]


# =============================================================================
# Part 3: resize the stock-picking book by regime risk
# =============================================================================
SP_HISTORY_MONTHS = 240   # the book is walked forward from 20 years after the first month (at most from the test start)
SP_MIN_MONTHS = 36        # and never before 36 months of data exist
MIN_REGIME = 24           # past months of a regime needed for its volatility
SIZING_RULES = [          # candidate sizing rules, chosen on pre-test months only
    "book alone",         # size 1 every month
    "cash when turbulent",  # size 1 when calm, 0 when turbulent
    "resize 1/sigma",     # size sigma_all / sigma_regime
    "resize 1/sigma^2",   # size (sigma_all / sigma_regime)^2
]
SIZING_DEFAULT = "resize 1/sigma"   # used when there are too few pre-test months to choose
SIZING_MIN_VAL_MONTHS = 60          # pre-test months of book returns needed to choose


def regime_scale(r: np.ndarray, regime: np.ndarray) -> np.ndarray:
    """Exposure at t = sigma_all / sigma_regime(t) from the book's returns realized
    before t (formation months < t). regime: 1 calm, 0 turbulent, -1 unknown
    (exposure 1). Too little history of the regime: exposure 1."""
    out = np.ones(len(r))
    for t in range(len(r)):
        if regime[t] < 0:
            continue
        past = ~np.isnan(r[:t])
        same = past & (regime[:t] == regime[t])
        if same.sum() >= MIN_REGIME and past.sum() >= MIN_REGIME:
            out[t] = r[:t][past].std(ddof=1) / r[:t][same].std(ddof=1)
    return out


def sizing(rule: str, r: np.ndarray, regime: np.ndarray) -> np.ndarray:
    """Size of the book at every month under one of SIZING_RULES."""
    if rule == "book alone":
        return np.ones(len(r))
    if rule == "cash when turbulent":
        return (regime != 0).astype(float)          # unknown regime: invested
    power = {"resize 1/sigma": 1.0, "resize 1/sigma^2": 2.0}[rule]
    return regime_scale(r, regime) ** power


def choose_sizing(r: np.ndarray, regime: np.ndarray, n_val: int, verbose: bool = True) -> str:
    """The sizing rule with the highest Sharpe ratio of the sized book over the
    first n_val months (the months before the test period; their returns are
    realized before the first test decision). Ties: earlier in SIZING_RULES."""
    if n_val < SIZING_MIN_VAL_MONTHS:
        return SIZING_DEFAULT
    best, best_sr = None, -np.inf
    for rule in SIZING_RULES:
        x = (r * sizing(rule, r, regime))[:n_val]
        sr = x.mean() / x.std(ddof=1) if x.std(ddof=1) > 0 else -np.inf
        if verbose:
            print(f"  sizing rule {rule}: pre-test Sharpe {sr * np.sqrt(12):.3f}", flush=True)
        if sr > best_sr + 1e-12:
            best, best_sr = rule, sr
    return best


def panel_arrays(chars: pd.DataFrame, feats: list):
    """The panel sorted by month and stock, as in theme_model.load_panel: eom,
    eom_ret, id, next-month excess return, test flag and the feature matrix
    (float32)."""
    eom_all = pd.to_datetime(chars["eom"]).to_numpy().astype("datetime64[D]")
    ids_all = _to_int(chars["id"])
    order = np.lexsort((ids_all, eom_all))
    eom = eom_all[order]
    ids = ids_all[order]
    if "eom_ret" in chars.columns:
        eom_ret = pd.to_datetime(chars["eom_ret"]).to_numpy().astype("datetime64[D]")[order]
    else:                                   # month-end one month later
        eom_ret = (eom.astype("datetime64[M]") + 2).astype("datetime64[D]") - 1
    ret = _to_float(chars["ret_exc_lead1m"])[order]
    if "ctff_test" in chars.columns:
        test = chars["ctff_test"].astype(str).str.strip().str.lower().isin(["1", "1.0", "true", "t"]).to_numpy()[order]
    else:
        test = np.zeros(len(ids), bool)
    X = np.empty((len(ids), len(feats)), dtype=np.float32)
    for j, c in enumerate(feats):
        X[:, j] = _to_float(chars[c])[order]
    return eom, eom_ret, ids, ret, test, X


# Competition entry point -----------------------------------------------------
def main(chars: pd.DataFrame, features: pd.DataFrame, daily_ret: pd.DataFrame) -> pd.DataFrame:
    """CTF entry point: stock-picking weights for every test stock-month, scaled
    by the regime-risk factor of that month (every stock-month if none is
    flagged ctff_test, so the output is never empty)."""
    np.random.seed(SEED)
    feats = [str(f) for f in features["features"].tolist()]
    eom, eom_ret, ids, ret, test, X = panel_arrays(chars, feats)
    X = rank_standardize(X, eom)
    months, bounds = month_bounds(eom)
    test_start = eom[test].min() if test.any() else eom.min()

    # Part 1: stock picking, walked forward from sp_start (history for Part 3)
    i_start = min(int(np.searchsorted(months, test_start)), SP_HISTORY_MONTHS)
    i_start = max(i_start, min(SP_MIN_MONTHS, len(months) - 1))
    sp_start = months[i_start]
    (out_id, out_eom, out_w), _log = run(eom, eom_ret, ids, ret, X, test_start, n="auto", themes="cofail",
                                         game=True, scorer="hero", start=sp_start, verbose=True)
    a0 = bounds[i_start][0]
    assert np.array_equal(out_id, ids[a0:]) and np.array_equal(out_eom, eom[a0:])
    m_idx = np.searchsorted(months, out_eom) - i_start               # 0 = sp_start
    r_book = np.bincount(m_idx, weights=out_w * np.nan_to_num(ret[a0:]), minlength=len(months) - i_start)

    # Part 2: regime at every month-end from sp_start
    mi = months.astype("datetime64[M]").astype(np.int64)
    p = regime_probabilities(chars, daily_ret, int(mi[i_start]), int(mi[np.searchsorted(months, test_start)]),
                             int(mi[-1]))
    p = p.reindex(mi[i_start:]).to_numpy()
    regime = np.where(np.isnan(p), -1, (p < THRESHOLD).astype(int))

    # Part 3: size the book; the rule is chosen on the months before the test period
    n_val = int(np.searchsorted(months, test_start)) - i_start      # book months realized before the test
    rule = choose_sizing(r_book, regime, n_val)
    print(f"sizing rule: {rule}", flush=True)
    scale = sizing(rule, r_book, regime)
    w = out_w * scale[m_idx]

    # Output: every stock-month flagged ctff_test (weight 0 before the book starts)
    out = pd.DataFrame({"id": ids, "eom": eom.astype("datetime64[ns]"), "w": 0.0})
    out.loc[a0:, "w"] = w
    return (out[test] if test.any() else out).reset_index(drop=True)
