# Author: Paul Geertsema, 1 Nov 2026
# See "Return Differences Across Stocks Are Not Compensation for Common Risk" on SSRN
# This generates T_1, the main portfolio used for the empirical bound

from __future__ import annotations

import math
import time
from collections.abc import Iterator

import lightgbm as lgb
import numpy as np
import pandas as pd
from scipy import linalg

SEED = 42
REFIT_EVERY = 12
MIN_TRAIN_MONTHS = 36
LGBM_PARAMS = dict(
    n_estimators=1000,
    learning_rate=0.01,
    num_leaves=255,
    min_child_samples=2000,
    random_state=SEED,
    n_jobs=32,
    deterministic=True,                  # with force_row_wise: results independent of the thread count
    force_row_wise=True,
    verbose=-1,
)
SAMPLE_WINDOW_DAYS = 756
MIN_DAILY_OBS = 250
DAYS_PER_MONTH = 21
MIN_NAMES = 50
KAPPA = 1.0                              # Tikhonov ridge as a multiple of the mean eigenvalue
TARGET_COL = 'ret_exc_lead1m'
FEATURE_LIST_COL = 'features'


# Panel ------------------------------------------------------------------------------------------------------

def month_slices(eom: np.ndarray) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
    """eom (n_rows,) sorted ascending -> months (n_months,) datetime64[D], starts and ends (n_months,), and the
    month index of every row (n_rows,)."""
    months, month_ix = np.unique(eom.astype('datetime64[D]'), return_inverse=True)
    starts = np.searchsorted(month_ix, np.arange(len(months)), side='left')
    ends = np.searchsorted(month_ix, np.arange(len(months)), side='right')
    assert np.all(np.diff(month_ix) >= 0), 'panel rows must be sorted by month'
    return months, starts, ends, month_ix


def target(ret: np.ndarray, month_ix: np.ndarray) -> np.ndarray:
    """arctan of the within-month z-score of ret (n_rows,) -> (n_rows,) float32."""
    assert ret.shape == month_ix.shape and np.isfinite(ret).all(), 'ret_exc_lead1m is never missing'
    g = pd.DataFrame({'m': month_ix, 'r': ret}).groupby('m')['r']
    z = ((ret - g.transform('mean')) / g.transform('std')).to_numpy(dtype=np.float32)
    y = np.arctan(z)
    assert np.isfinite(y).all()
    return y


def build_daily_matrix(daily_ret: pd.DataFrame, panel_ids: np.ndarray) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
    """Long daily table -> dates (n_dates,) datetime64[D], ids (n_ids,) ascending and M (n_dates, n_ids) float32
    with NaN where the stock did not trade, restricted to the panel's stocks."""
    rid = daily_ret['id'].to_numpy()
    keep = np.isin(rid, panel_ids)
    rid = rid[keep]
    rdate = daily_ret['date'].to_numpy()[keep].astype('datetime64[D]')
    rret = daily_ret['ret_exc'].to_numpy()[keep].astype(np.float32)
    dates, date_ix = np.unique(rdate, return_inverse=True)
    ids, id_ix = np.unique(rid, return_inverse=True)
    M = np.full((len(dates), len(ids)), np.nan, dtype=np.float32)
    M[date_ix, id_ix] = rret
    return dates, ids, M


def universe(ids: np.ndarray, month: np.datetime64, dates: np.ndarray, daily_ids: np.ndarray,
             M: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
    """Stocks of one formation month that can be held, and their trailing daily window.

    ids (n,) panel stocks of the month; dates (n_dates,), daily_ids (n_ids,), M (n_dates, n_ids).
    Returns usable (n,) bool and X (T, n_usable) float32, column-demeaned, in monthly variance units.
    """
    cols_all = np.clip(np.searchsorted(daily_ids, ids), 0, len(daily_ids) - 1)
    has_daily = daily_ids[cols_all] == ids
    end = int(np.searchsorted(dates, month, side='right'))        # dates[end - 1] <= month
    assert end >= MIN_DAILY_OBS, (month, end)
    X = M[max(0, end - SAMPLE_WINDOW_DAYS):end, cols_all[has_daily]]
    obs = np.isfinite(X)
    enough = obs.sum(axis=0) >= MIN_DAILY_OBS
    X = np.where(obs, X, np.float32(0.0))
    X -= X.mean(axis=0, keepdims=True)
    usable = np.zeros(len(ids), dtype=bool)
    usable[np.flatnonzero(has_daily)[enough]] = True
    assert usable.sum() >= MIN_NAMES, (month, int(usable.sum()))
    return usable, X[:, enough] * np.float32(np.sqrt(DAYS_PER_MONTH))


# Covariance and portfolio -----------------------------------------------------------------------------------

def shrunk_covariance(X: np.ndarray) -> tuple[np.ndarray, float]:
    """Ledoit--Wolf (2004) Sigma = (1 - delta) S + delta (tr(S)/n) I with S = X'X/T, X (T, n) column-demeaned.
    Returns Sigma (n, n) float32 and the intensity delta."""
    T, n = X.shape
    assert T >= 2 and n >= 2 and np.isfinite(X).all(), X.shape
    S = (X.T @ X) / T                                              # (n, n) float32
    tr_S = float(np.trace(S, dtype=np.float64))
    mu_lw = tr_S / n
    norm_S2 = float(np.square(S).sum(dtype=np.float64))
    d2 = norm_S2 - 2.0 * mu_lw * tr_S + n * mu_lw ** 2               # ||S - mu I||_F^2
    row_sq = np.square(X).sum(axis=1, dtype=np.float64)            # (T,)
    b2 = float(np.square(row_sq).sum()) / T ** 2 - norm_S2 / T
    delta = min(max(b2, 0.0), d2) / d2 if d2 > 1e-300 else 1.0
    S *= np.float32(1.0 - delta)
    S[np.diag_indices(n)] += np.float32(delta * mu_lw)
    return S, delta


def tikhonov(Sigma: np.ndarray, kappa: float = KAPPA) -> np.ndarray:
    """Sigma (n, n) float32 plus kappa times its mean eigenvalue on the diagonal, float32."""
    n = Sigma.shape[0]
    assert Sigma.shape == (n, n)
    out = np.array(Sigma, dtype=np.float32, copy=True)
    out[np.diag_indices(n)] += np.float32(kappa * float(np.trace(Sigma, dtype=np.float64)) / n)
    return out


def factorize(Sigma: np.ndarray) -> tuple[np.ndarray, tuple[np.ndarray, bool]]:
    """Sigma (n, n) -> symmetrized float64 copy and its lower Cholesky factor (scipy cho_factor form)."""
    S = np.array(Sigma, dtype=np.float64, order='C', copy=True)
    S = 0.5 * (S + S.T)
    assert np.isfinite(S).all() and np.all(np.diag(S) > 0.0)
    return S, linalg.cho_factor(S, lower=True, overwrite_a=False, check_finite=True)


def solve(S: np.ndarray, factor: tuple[np.ndarray, bool], rhs: np.ndarray) -> np.ndarray:
    """Solve S x = rhs, rhs (n,) or (n, k), with one step of iterative refinement; float64."""
    b = np.array(rhs, dtype=np.float64, copy=True)
    x = linalg.cho_solve(factor, b)
    x = x + linalg.cho_solve(factor, b - S @ x)
    assert np.isfinite(x).all()
    return x


def unit_gross(a: np.ndarray) -> np.ndarray:
    """a (n,) scaled to one dollar of gross exposure, float64 (compensated sum)."""
    assert a.ndim == 1
    s = math.fsum(map(float, np.abs(a)))
    assert s > 0.0
    return np.asarray(a, dtype=np.float64) / s


def lwt_t1(mu: np.ndarray, X: np.ndarray) -> np.ndarray:
    """The main portfolio for one month: mu (n,) and the daily window X (T, n) -> T_1 (n,)."""
    S, factor = factorize(tikhonov(shrunk_covariance(X)[0]))
    return unit_gross(solve(S, factor, mu))


# Forecast ---------------------------------------------------------------------------------------------------

def forecasts(X: np.ndarray, y: np.ndarray, starts: np.ndarray, ends: np.ndarray,
              predict_at: list[int]) -> Iterator[tuple[int, np.ndarray]]:
    """Expanding-window LightGBM forecasts. X (n_rows, 402) float32 and y (n_rows,) float32, rows sorted by month;
    yields (t, mu_t) with mu_t (ends[t] - starts[t],) float32 for every month index t in predict_at."""
    assert X.shape[0] == y.shape[0] and predict_at and predict_at[0] >= MIN_TRAIN_MONTHS
    model = None
    for k, t in enumerate(predict_at):
        if k == 0 or (t - MIN_TRAIN_MONTHS) % REFIT_EVERY == 0:
            t0 = time.perf_counter()
            model = lgb.LGBMRegressor(**LGBM_PARAMS)
            model.fit(X[:starts[t]], y[:starts[t]])
            print(f'refit at month {t}: {int(starts[t]):,} rows in {time.perf_counter() - t0:.0f}s', flush=True)
        mu = model.booster_.predict(X[starts[t]:ends[t]]).astype(np.float32)
        assert np.isfinite(mu).all()
        yield t, mu


# CTF entry point --------------------------------------------------------------------------------------------

def build_weights(chars: pd.DataFrame, features: pd.DataFrame, daily_ret: pd.DataFrame) -> pd.DataFrame:
    """T_1 for every month with at least MIN_TRAIN_MONTHS earlier months. Columns id, eom, w."""
    np.random.seed(SEED)
    feats = [str(v) for v in features[FEATURE_LIST_COL].tolist()]
    assert set(feats) <= set(chars.columns), 'feature list names a column chars lacks'
    panel = chars[['id', 'eom', TARGET_COL] + feats].sort_values(['eom', 'id'], kind='stable').reset_index(drop=True)
    ids = panel['id'].to_numpy()
    eom = panel['eom'].to_numpy()
    months, starts, ends, month_ix = month_slices(eom)
    X = panel[feats].to_numpy(dtype=np.float32)
    y = target(panel[TARGET_COL].to_numpy(), month_ix)
    dates, daily_ids, M = build_daily_matrix(daily_ret, np.unique(ids))
    predict_at = list(range(MIN_TRAIN_MONTHS, len(months)))
    assert predict_at, f'{len(months)} months is fewer than MIN_TRAIN_MONTHS = {MIN_TRAIN_MONTHS}'
    frames = []
    for t, mu in forecasts(X, y, starts, ends, predict_at):
        lo, hi = int(starts[t]), int(ends[t])
        usable, Xd = universe(ids[lo:hi], months[t], dates, daily_ids, M)
        w = np.zeros(hi - lo, dtype=np.float64)
        w[usable] = lwt_t1(mu[usable].astype(np.float64), Xd)
        frames.append(pd.DataFrame({'id': ids[lo:hi], 'eom': eom[lo:hi], 'w': w}))
        print(f'{months[t]} held {int(usable.sum())} of {hi - lo}', flush=True)
    out = pd.concat(frames, ignore_index=True)
    out['id'] = out['id'].astype(np.int64)
    assert list(out.columns) == ['id', 'eom', 'w'] and out.notna().all().all()
    return out


def is_test_row(flag: pd.Series) -> pd.Series:
    """Rows the harness scores; boolean, 0/1 and '0'/'1' flags all occur."""
    if pd.api.types.is_bool_dtype(flag):
        return flag.fillna(False)
    if pd.api.types.is_numeric_dtype(flag):
        return flag.fillna(0).ne(0)
    return flag.astype(str).str.strip().str.lower().isin(['1', '1.0', 'true', 't'])


def main(chars: pd.DataFrame, features: pd.DataFrame, daily_ret: pd.DataFrame) -> pd.DataFrame:
    """Monthly portfolio weights for the test rows (all rows if the panel has no test flag set)."""
    weights = build_weights(chars, features, daily_ret)
    if 'ctff_test' not in chars.columns or not is_test_row(chars['ctff_test']).any():
        return weights
    keys = chars.loc[is_test_row(chars['ctff_test']), ['id', 'eom']].drop_duplicates()
    keys['id'] = keys['id'].astype(np.int64)
    out = weights.merge(keys, on=['id', 'eom'], how='inner')
    assert len(out), 'ctff_test is set but no weights match those rows'
    return out
