"""JKP Common Task Framework submission.

Author: Abhinav Keshri

Interface: main(chars, features, daily_ret) -> pd.DataFrame with columns id, eom, w.

METHOD

    Target     Next month excess return scaled by the stock's PCA risk (756 trading days, eigenvalues
               above the Marchenko-Pastur edge, Ledoit-Wolf shrunk), projected off the PCA factors,
               1/sigma and 1/sigma times SIC2 industry dummies, standardised and clipped at 4.
    Models     XGBoost on the 402 raw characteristics (missing values kept), trained on the target
               above. A feedforward network (tanh) on within month ranks of the same characteristics
               (missing set to 0), trained on the within month z score of the next month return.
               Both refit annually on an expanding window; the tree count and the number of epochs
               are chosen on the last 24 months, then each model is refit on all available data.
    Tuning     Every 36 months an Optuna multivariate TPE search per model scores each trial by mean
               rank IC over four consecutive 24 month validation folds before the tune month. The
               three best trials are refit and their standardised forecasts averaged. The selected
               settings are stored in XGB_PARAMS and NET_PARAMS; TUNE = True reruns the search.
    Blend      The tree forecast is mapped to return units by sigma*sqrt(psi). Both forecasts are
               standardised within month and combined with weight 0.75 on the tree and 0.25 on the
               network.
    Books      Ledoit-Wolf daily covariance with Tikhonov ridge, tangency minus GMV (50%), and a 13
               theme fundamental factor MVO demeaned within SIC2 industry (50%). The combined book is
               made beta neutral.
    Overlay    Gross exposure is scaled each month by the expanding mean of the book's estimated
               volatility over the current estimate, bounded to [0.75, 1.5]. The estimate is an
               exponentially weighted (63 day half life) volatility of the book's daily returns.

CONTACT

    Comments and discussion are welcome:
    LinkedIn  https://www.linkedin.com/in/abhinav-keshri-7507b1150/
    Email     abhinav.keshri@yale.edu, abhinavkeshri1999@gmail.com

"""
from __future__ import annotations

import os
import random
import warnings

import numpy as np
import pandas as pd

warnings.filterwarnings("ignore", message="X does not have valid feature names")
warnings.filterwarnings("ignore", message="Stochastic Optimizer: Maximum iterations")

SEED = 42
ID_COL, DATE_COL, RET_COL, TEST_COL = "id", "eom", "ret_exc_lead1m", "ctff_test"
N_CPU = len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else (os.cpu_count() or 1)

MIN_TRAIN_MONTHS, VAL_MONTHS = 120, 24
SMALL_MIN_TRAIN, SMALL_VAL_MONTHS = 36, 12
RETRAIN_EVERY, TUNE_EVERY = 12, 36
OPTUNA_TRIALS, OPTUNA_STARTUP, OPTUNA_PATIENCE, OPTUNA_MIN_DELTA = 30, 8, 10, 1e-4
TUNE_SUBSAMPLE, CV_FOLDS, TOP_K = 200_000, 4, 3
TUNE = False

XGB_PARAMS = [
    [
        {"learning_rate": 0.005361492042203761, "max_leaves": 488, "max_depth": 10, "min_child_weight": 937, "subsample": 0.7307458068321546, "colsample_bytree": 0.9183326586433281, "reg_lambda": 0.282743289050211},
        {"learning_rate": 0.005620384401502775, "max_leaves": 370, "max_depth": 6, "min_child_weight": 769, "subsample": 0.7246844304357644, "colsample_bytree": 0.8080272084711243, "reg_lambda": 1.5375920235481764},
        {"learning_rate": 0.008805410998786916, "max_leaves": 496, "max_depth": 8, "min_child_weight": 3018, "subsample": 0.803307639753717, "colsample_bytree": 0.6922198821445248, "reg_lambda": 0.45138774845489615},
    ],
    [
        {"learning_rate": 0.005317027151339846, "max_leaves": 244, "max_depth": 8, "min_child_weight": 487, "subsample": 0.7002391268159499, "colsample_bytree": 0.9488999669281977, "reg_lambda": 1.5125148467427891},
        {"learning_rate": 0.005259345758321432, "max_leaves": 289, "max_depth": 9, "min_child_weight": 1711, "subsample": 0.7174824105147295, "colsample_bytree": 0.8610731428199101, "reg_lambda": 8.642292546018083},
        {"learning_rate": 0.006140729982020983, "max_leaves": 130, "max_depth": 11, "min_child_weight": 1075, "subsample": 0.684643763476521, "colsample_bytree": 0.8995676980774353, "reg_lambda": 38.68930559030283},
    ],
    [
        {"learning_rate": 0.005419134889649681, "max_leaves": 234, "max_depth": 11, "min_child_weight": 1101, "subsample": 0.7319233184668592, "colsample_bytree": 0.7837691263473003, "reg_lambda": 0.1808922569204006},
        {"learning_rate": 0.005857261948813926, "max_leaves": 217, "max_depth": 11, "min_child_weight": 1934, "subsample": 0.6456954213490045, "colsample_bytree": 0.7694225282851941, "reg_lambda": 0.2833995329926804},
        {"learning_rate": 0.005041775470646066, "max_leaves": 55, "max_depth": 12, "min_child_weight": 1108, "subsample": 0.7413758837364715, "colsample_bytree": 0.8492591095432286, "reg_lambda": 4.41179577761914},
    ],
    [
        {"learning_rate": 0.005422750435861351, "max_leaves": 39, "max_depth": 12, "min_child_weight": 1758, "subsample": 0.7449004760128024, "colsample_bytree": 0.9527327192043101, "reg_lambda": 0.032608129445503564},
        {"learning_rate": 0.0064149826081652565, "max_leaves": 77, "max_depth": 13, "min_child_weight": 1563, "subsample": 0.6872696644220656, "colsample_bytree": 0.9411081561820739, "reg_lambda": 0.08663424777906456},
        {"learning_rate": 0.007147693686737708, "max_leaves": 144, "max_depth": 12, "min_child_weight": 1668, "subsample": 0.6498600920365465, "colsample_bytree": 0.9624136186108495, "reg_lambda": 0.0398361103800765},
    ],
    [
        {"learning_rate": 0.005775494210671737, "max_leaves": 33, "max_depth": 10, "min_child_weight": 2350, "subsample": 0.8660697986015171, "colsample_bytree": 0.7788327002876111, "reg_lambda": 0.23746810967090787},
        {"learning_rate": 0.008390751319234991, "max_leaves": 60, "max_depth": 14, "min_child_weight": 1377, "subsample": 0.8825146353052767, "colsample_bytree": 0.9286565749613799, "reg_lambda": 36.31249645010902},
        {"learning_rate": 0.0053022161188924, "max_leaves": 85, "max_depth": 13, "min_child_weight": 2615, "subsample": 0.8537537712867919, "colsample_bytree": 0.807916264742613, "reg_lambda": 76.26479825380356},
    ],
    [
        {"learning_rate": 0.006893356021514564, "max_leaves": 101, "max_depth": 10, "min_child_weight": 805, "subsample": 0.7415074756162541, "colsample_bytree": 0.9034435461293118, "reg_lambda": 11.540941047473977},
        {"learning_rate": 0.010240137883474672, "max_leaves": 167, "max_depth": 10, "min_child_weight": 2743, "subsample": 0.7338955534361835, "colsample_bytree": 0.8573068553299898, "reg_lambda": 11.281674534881619},
        {"learning_rate": 0.021377545790244392, "max_leaves": 101, "max_depth": 11, "min_child_weight": 1248, "subsample": 0.6985693697922939, "colsample_bytree": 0.8446340348672071, "reg_lambda": 0.10899436692204616},
    ],
    [
        {"learning_rate": 0.012377889380935168, "max_leaves": 266, "max_depth": 9, "min_child_weight": 694, "subsample": 0.7358886404703455, "colsample_bytree": 0.996921531092323, "reg_lambda": 0.03297767504951523},
        {"learning_rate": 0.005432821471843893, "max_leaves": 484, "max_depth": 12, "min_child_weight": 380, "subsample": 0.6535044781731013, "colsample_bytree": 0.8587139378887917, "reg_lambda": 0.03037965670285859},
        {"learning_rate": 0.006448181558601135, "max_leaves": 430, "max_depth": 8, "min_child_weight": 371, "subsample": 0.720259994598206, "colsample_bytree": 0.9539920814492269, "reg_lambda": 0.027391078175524722},
    ],
    [
        {"learning_rate": 0.008325761823196248, "max_leaves": 332, "max_depth": 12, "min_child_weight": 485, "subsample": 0.9038418561992835, "colsample_bytree": 0.6097646129328174, "reg_lambda": 14.996585839506865},
        {"learning_rate": 0.005691878531494571, "max_leaves": 295, "max_depth": 11, "min_child_weight": 119, "subsample": 0.9145703399116396, "colsample_bytree": 0.6373606980351392, "reg_lambda": 4.161715691350991},
        {"learning_rate": 0.010228018972984752, "max_leaves": 225, "max_depth": 13, "min_child_weight": 1068, "subsample": 0.7995121399633063, "colsample_bytree": 0.7768851131576777, "reg_lambda": 16.207726937190998},
    ],
    [
        {"learning_rate": 0.009217163968410614, "max_leaves": 393, "max_depth": 14, "min_child_weight": 1024, "subsample": 0.8930750435626067, "colsample_bytree": 0.7846990705057728, "reg_lambda": 53.127217473227695},
        {"learning_rate": 0.01971508117200349, "max_leaves": 319, "max_depth": 12, "min_child_weight": 1035, "subsample": 0.9166081994463415, "colsample_bytree": 0.9637701279265789, "reg_lambda": 70.30447764384303},
        {"learning_rate": 0.007548389527708531, "max_leaves": 86, "max_depth": 11, "min_child_weight": 3170, "subsample": 0.8830568508467378, "colsample_bytree": 0.7853895569161397, "reg_lambda": 41.89576294708976},
    ],
    [
        {"learning_rate": 0.006780416131523044, "max_leaves": 122, "max_depth": 14, "min_child_weight": 1293, "subsample": 0.9266556546900605, "colsample_bytree": 0.6869694383685961, "reg_lambda": 0.051000272313623744},
        {"learning_rate": 0.009470933227244538, "max_leaves": 331, "max_depth": 10, "min_child_weight": 1292, "subsample": 0.7327145064761375, "colsample_bytree": 0.8181241185918059, "reg_lambda": 0.014303843830800776},
        {"learning_rate": 0.014152185341532272, "max_leaves": 49, "max_depth": 12, "min_child_weight": 1141, "subsample": 0.7896525809599106, "colsample_bytree": 0.7134526542381222, "reg_lambda": 0.02748231865523355},
    ],
    [
        {"learning_rate": 0.006906038032585417, "max_leaves": 46, "max_depth": 7, "min_child_weight": 1197, "subsample": 0.8631635470346503, "colsample_bytree": 0.6805267325371187, "reg_lambda": 17.13766515014378},
        {"learning_rate": 0.005682538542067017, "max_leaves": 51, "max_depth": 6, "min_child_weight": 541, "subsample": 0.9332757873558751, "colsample_bytree": 0.6477423197944281, "reg_lambda": 18.88152602038497},
        {"learning_rate": 0.009645485086253958, "max_leaves": 45, "max_depth": 6, "min_child_weight": 1104, "subsample": 0.9361221267785198, "colsample_bytree": 0.7283057759818916, "reg_lambda": 23.45551193866391},
    ],
    [
        {"learning_rate": 0.01769669558340315, "max_leaves": 105, "max_depth": 13, "min_child_weight": 49, "subsample": 0.7628825679319307, "colsample_bytree": 0.8609413355115003, "reg_lambda": 1.8663638788270875},
        {"learning_rate": 0.02038809486048989, "max_leaves": 46, "max_depth": 3, "min_child_weight": 60, "subsample": 0.6523494904023821, "colsample_bytree": 0.6667743467710161, "reg_lambda": 0.3075401472925981},
        {"learning_rate": 0.012295067258733257, "max_leaves": 80, "max_depth": 7, "min_child_weight": 41, "subsample": 0.7039909401372236, "colsample_bytree": 0.6190538704677143, "reg_lambda": 0.3777228161190725},
    ],
]

NET_WEIGHT, NET_INITS, NET_MAX_EPOCHS, NET_PATIENCE, NET_MIN_MONTHS = 0.25, 3, 40, 4, 150
NET_DEFAULT = {"hidden": (32, 16), "alpha": 1e-4, "lr": 1e-3, "batch": 8192}

NET_PARAMS = [
    [
        {"hidden": (32, 16, 8), "alpha": 1.0209809498681943e-05, "lr": 0.0029903732741235188, "batch": 32768},
        {"hidden": (32, 16, 8), "alpha": 1.758187147611519e-05, "lr": 0.002312462681359172, "batch": 2048},
        {"hidden": (32, 16, 8), "alpha": 1.7555619942637927e-06, "lr": 0.002929475100916877, "batch": 2048},
    ],
    [
        {"hidden": (128, 64, 32), "alpha": 4.3432503033896554e-05, "lr": 0.002834705780269128, "batch": 2048},
        {"hidden": (128, 64, 32), "alpha": 0.0003808772562212233, "lr": 0.002831493322316422, "batch": 2048},
        {"hidden": (128, 64, 32), "alpha": 0.008528196769274661, "lr": 0.0016608122498321717, "batch": 2048},
    ],
    [
        {"hidden": (32,), "alpha": 0.00675909073123137, "lr": 0.002474248979223204, "batch": 2048},
        {"hidden": (32,), "alpha": 0.0004094125487638287, "lr": 0.0028053455060277747, "batch": 32768},
        {"hidden": (32, 16), "alpha": 0.0017722059664191973, "lr": 0.002769982961321743, "batch": 32768},
    ],
    [
        {"hidden": (32, 16, 8), "alpha": 0.003538449081316857, "lr": 0.0012464096081960217, "batch": 8192},
        {"hidden": (32, 16, 8), "alpha": 9.494266221492696e-05, "lr": 0.0012005068105903362, "batch": 8192},
        {"hidden": (32, 16, 8), "alpha": 0.00030423550959164533, "lr": 0.0018121138959995744, "batch": 8192},
    ],
    [
        {"hidden": (32, 16, 8), "alpha": 0.0020524810011745254, "lr": 0.0005118926728411105, "batch": 2048},
        {"hidden": (32, 16, 8), "alpha": 0.0007475888938627101, "lr": 0.0005432465806516233, "batch": 2048},
        {"hidden": (32, 16, 8), "alpha": 0.0004875939543338893, "lr": 0.0006316082109183872, "batch": 2048},
    ],
    [
        {"hidden": (32, 16, 8), "alpha": 0.0035175991237421434, "lr": 0.00019775428495259504, "batch": 2048},
        {"hidden": (32, 16, 8), "alpha": 0.0013646857070107755, "lr": 0.00023601508152057968, "batch": 2048},
        {"hidden": (32, 16, 8), "alpha": 0.0008245650187008053, "lr": 0.00015813673150430208, "batch": 2048},
    ],
    [
        {"hidden": (32,), "alpha": 6.36543793898443e-05, "lr": 0.0028901660302255777, "batch": 8192},
        {"hidden": (32,), "alpha": 0.0003754422805300362, "lr": 0.0028237463187943184, "batch": 8192},
        {"hidden": (32,), "alpha": 7.841210964034594e-05, "lr": 0.002627301862662698, "batch": 8192},
    ],
    [
        {"hidden": (128, 64, 32), "alpha": 0.005750136354842808, "lr": 0.001831668417986763, "batch": 2048},
        {"hidden": (32,), "alpha": 0.009424685754636083, "lr": 0.0023764245101492577, "batch": 2048},
        {"hidden": (128, 64, 32), "alpha": 0.004795993876083667, "lr": 0.0013534839174981232, "batch": 2048},
    ],
    [
        {"hidden": (64, 32, 16), "alpha": 0.0040160048684699055, "lr": 0.0028107202715478056, "batch": 32768},
        {"hidden": (32,), "alpha": 0.0020991498800268897, "lr": 0.0028236671836115207, "batch": 32768},
        {"hidden": (64, 32, 16), "alpha": 0.0032417852324277867, "lr": 0.0017747478014604901, "batch": 32768},
    ],
    [
        {"hidden": (32,), "alpha": 0.00025811537863915877, "lr": 0.001577146260985407, "batch": 2048},
        {"hidden": (32,), "alpha": 0.0003602105715023333, "lr": 0.0014948752229772128, "batch": 2048},
        {"hidden": (32,), "alpha": 0.0003200933792068207, "lr": 0.001164252004245836, "batch": 2048},
    ],
    [
        {"hidden": (32, 16, 8), "alpha": 2.2427903644801276e-06, "lr": 0.0012673926017000752, "batch": 2048},
        {"hidden": (32, 16, 8), "alpha": 0.00034506419943439577, "lr": 0.001458876033002798, "batch": 2048},
        {"hidden": (32, 16, 8), "alpha": 7.207686342844388e-06, "lr": 0.0016044380327588966, "batch": 2048},
    ],
    [
        {"hidden": (32,), "alpha": 1.0268861585395941e-06, "lr": 0.0013410025315529817, "batch": 32768},
        {"hidden": (32, 16, 8), "alpha": 3.485562798408402e-05, "lr": 0.0007271022828728514, "batch": 2048},
        {"hidden": (32, 16, 8), "alpha": 2.1600889602591333e-06, "lr": 0.0007986727021153458, "batch": 2048},
    ],
]

RISK_DAYS, RISK_MIN_OBS, PSI_FLOOR, MIN_INDUSTRY = 756, 250, 0.05, 10

WINDOW_LONG = 756
MIN_DAILY_OBS, DAYS_PER_MONTH, MIN_NAMES, KAPPA = 250, 21, 50, 1.0
MVO_WINDOW, D_FLOOR_Q = 60, 0.10
VOL_HALFLIFE, VOL_MIN_DAYS, VOL_CLIP = 63, 21, (0.75, 1.5)

THEMES = ["Accruals", "Debt Issuance", "Investment", "Low Leverage", "Low Risk",
          "Momentum", "Profit Growth", "Profitability", "Quality", "Seasonality",
          "Short-Term Reversal", "Size", "Value"]

THEME_MAP = [
    ('age', 'Low Leverage', -1.0),
    ('aliq_at', 'Investment', -1.0),
    ('aliq_mat', 'Low Leverage', -1.0),
    ('ami_126d', 'Size', +1.0),
    ('at_be', 'Low Leverage', -1.0),
    ('at_gr1', 'Investment', -1.0),
    ('at_me', 'Value', +1.0),
    ('at_turnover', 'Quality', +1.0),
    ('be_gr1a', 'Investment', -1.0),
    ('be_me', 'Value', +1.0),
    ('beta_60m', 'Low Risk', -1.0),
    ('beta_dimson_21d', 'Low Risk', -1.0),
    ('betabab_1260d', 'Low Risk', -1.0),
    ('betadown_252d', 'Low Risk', -1.0),
    ('bev_mev', 'Value', +1.0),
    ('bidaskhl_21d', 'Low Leverage', +1.0),
    ('capex_abn', 'Debt Issuance', -1.0),
    ('capx_gr1', 'Investment', -1.0),
    ('capx_gr2', 'Investment', -1.0),
    ('capx_gr3', 'Investment', -1.0),
    ('cash_at', 'Low Leverage', +1.0),
    ('chcsho_12m', 'Value', -1.0),
    ('coa_gr1a', 'Investment', -1.0),
    ('col_gr1a', 'Investment', -1.0),
    ('cop_at', 'Quality', +1.0),
    ('cop_atl1', 'Quality', +1.0),
    ('corr_1260d', 'Seasonality', -1.0),
    ('coskew_21d', 'Seasonality', -1.0),
    ('cowc_gr1a', 'Accruals', -1.0),
    ('dbnetis_at', 'Seasonality', -1.0),
    ('debt_gr3', 'Debt Issuance', -1.0),
    ('debt_me', 'Value', +1.0),
    ('dgp_dsale', 'Quality', +1.0),
    ('div12m_me', 'Value', +1.0),
    ('dolvol_126d', 'Size', -1.0),
    ('dolvol_var_126d', 'Profitability', -1.0),
    ('dsale_dinv', 'Profit Growth', +1.0),
    ('dsale_drec', 'Profit Growth', -1.0),
    ('dsale_dsga', 'Profit Growth', +1.0),
    ('earnings_variability', 'Low Risk', -1.0),
    ('ebit_bev', 'Profitability', +1.0),
    ('ebit_sale', 'Profitability', +1.0),
    ('ebitda_mev', 'Value', +1.0),
    ('emp_gr1', 'Investment', -1.0),
    ('eq_dur', 'Value', -1.0),
    ('eqnetis_at', 'Value', -1.0),
    ('eqnpo_12m', 'Value', +1.0),
    ('eqnpo_me', 'Value', +1.0),
    ('eqpo_me', 'Value', +1.0),
    ('f_score', 'Profitability', +1.0),
    ('fcf_me', 'Value', +1.0),
    ('fnl_gr1a', 'Debt Issuance', -1.0),
    ('gp_at', 'Quality', +1.0),
    ('gp_atl1', 'Quality', +1.0),
    ('inv_gr1', 'Investment', -1.0),
    ('inv_gr1a', 'Investment', -1.0),
    ('iskew_capm_21d', 'Short-Term Reversal', -1.0),
    ('iskew_ff3_21d', 'Short-Term Reversal', -1.0),
    ('iskew_hxz4_21d', 'Short-Term Reversal', -1.0),
    ('ival_me', 'Value', +1.0),
    ('ivol_capm_21d', 'Low Risk', -1.0),
    ('ivol_capm_252d', 'Low Risk', -1.0),
    ('ivol_ff3_21d', 'Low Risk', -1.0),
    ('ivol_hxz4_21d', 'Low Risk', -1.0),
    ('kz_index', 'Seasonality', +1.0),
    ('lnoa_gr1a', 'Investment', -1.0),
    ('lti_gr1a', 'Seasonality', -1.0),
    ('market_equity', 'Size', -1.0),
    ('mispricing_mgmt', 'Investment', +1.0),
    ('mispricing_perf', 'Quality', +1.0),
    ('ncoa_gr1a', 'Investment', -1.0),
    ('ncol_gr1a', 'Debt Issuance', -1.0),
    ('netdebt_me', 'Low Leverage', -1.0),
    ('netis_at', 'Value', -1.0),
    ('nfna_gr1a', 'Debt Issuance', +1.0),
    ('ni_ar1', 'Debt Issuance', +1.0),
    ('ni_be', 'Profitability', +1.0),
    ('ni_inc8q', 'Quality', +1.0),
    ('ni_ivol', 'Low Leverage', +1.0),
    ('ni_me', 'Value', +1.0),
    ('niq_at', 'Quality', +1.0),
    ('niq_at_chg1', 'Profit Growth', +1.0),
    ('niq_be', 'Profitability', +1.0),
    ('niq_be_chg1', 'Profit Growth', +1.0),
    ('niq_su', 'Profit Growth', +1.0),
    ('nncoa_gr1a', 'Investment', -1.0),
    ('noa_at', 'Debt Issuance', -1.0),
    ('noa_gr1a', 'Investment', -1.0),
    ('o_score', 'Profitability', -1.0),
    ('oaccruals_at', 'Accruals', -1.0),
    ('oaccruals_ni', 'Accruals', -1.0),
    ('ocf_at', 'Profitability', +1.0),
    ('ocf_at_chg1', 'Profit Growth', +1.0),
    ('ocf_me', 'Value', +1.0),
    ('ocfq_saleq_std', 'Low Risk', -1.0),
    ('op_at', 'Quality', +1.0),
    ('op_atl1', 'Quality', +1.0),
    ('ope_be', 'Profitability', +1.0),
    ('ope_bel1', 'Profitability', +1.0),
    ('opex_at', 'Quality', +1.0),
    ('pi_nix', 'Seasonality', +1.0),
    ('ppeinv_gr1a', 'Investment', -1.0),
    ('prc', 'Size', -1.0),
    ('prc_highprc_252d', 'Momentum', +1.0),
    ('qmj', 'Quality', +1.0),
    ('qmj_growth', 'Quality', +1.0),
    ('qmj_prof', 'Quality', +1.0),
    ('qmj_safety', 'Quality', +1.0),
    ('rd_me', 'Size', +1.0),
    ('rd_sale', 'Low Leverage', +1.0),
    ('rd5_at', 'Low Leverage', +1.0),
    ('resff3_12_1', 'Momentum', +1.0),
    ('resff3_6_1', 'Momentum', +1.0),
    ('ret_1_0', 'Short-Term Reversal', -1.0),
    ('ret_12_1', 'Momentum', +1.0),
    ('ret_12_7', 'Profit Growth', +1.0),
    ('ret_3_1', 'Momentum', +1.0),
    ('ret_6_1', 'Momentum', +1.0),
    ('ret_60_12', 'Investment', -1.0),
    ('ret_9_1', 'Momentum', +1.0),
    ('rmax1_21d', 'Low Risk', -1.0),
    ('rmax5_21d', 'Low Risk', -1.0),
    ('rmax5_rvol_21d', 'Short-Term Reversal', -1.0),
    ('rskew_21d', 'Short-Term Reversal', -1.0),
    ('rvol_21d', 'Low Risk', -1.0),
    ('sale_bev', 'Quality', +1.0),
    ('sale_emp_gr1', 'Profit Growth', +1.0),
    ('sale_gr1', 'Investment', -1.0),
    ('sale_gr3', 'Investment', -1.0),
    ('sale_me', 'Value', +1.0),
    ('saleq_gr1', 'Investment', -1.0),
    ('saleq_su', 'Profit Growth', +1.0),
    ('seas_1_1an', 'Profit Growth', +1.0),
    ('seas_1_1na', 'Momentum', +1.0),
    ('seas_11_15an', 'Seasonality', +1.0),
    ('seas_11_15na', 'Seasonality', -1.0),
    ('seas_16_20an', 'Seasonality', +1.0),
    ('seas_16_20na', 'Accruals', -1.0),
    ('seas_2_5an', 'Seasonality', +1.0),
    ('seas_2_5na', 'Investment', -1.0),
    ('seas_6_10an', 'Seasonality', +1.0),
    ('seas_6_10na', 'Low Risk', -1.0),
    ('sti_gr1a', 'Seasonality', +1.0),
    ('taccruals_at', 'Accruals', -1.0),
    ('taccruals_ni', 'Accruals', -1.0),
    ('tangibility', 'Low Leverage', +1.0),
    ('tax_gr1a', 'Profit Growth', +1.0),
    ('turnover_126d', 'Low Risk', -1.0),
    ('turnover_var_126d', 'Profitability', -1.0),
    ('z_score', 'Low Leverage', +1.0),
    ('zero_trades_126d', 'Low Risk', +1.0),
    ('zero_trades_21d', 'Low Risk', +1.0),
    ('zero_trades_252d', 'Low Risk', +1.0),
]


def _industry(chars):
    if "sic" in chars.columns:
        return (pd.to_numeric(chars["sic"], errors="coerce") // 100).fillna(-1).astype("int64")
    return pd.Series(-1, index=chars.index, dtype="int64")


def theme_exposures(chars, eom):
    present = [(c, t, d) for c, t, d in THEME_MAP if c in chars.columns]
    cols = [c for c, _, _ in present]
    ranks = chars[cols].groupby(eom, observed=True).rank(pct=True) - 0.5
    sign = {c: d for c, _, d in present}
    out = {}
    for theme in THEMES:
        members = [c for c, t, _ in present if t == theme]
        if members:
            out[theme] = ranks[members].mul([sign[c] for c in members], axis=1).mean(axis=1)
    return pd.DataFrame(out, index=chars.index).fillna(0.0).astype(np.float32)


def prepare(chars, features):
    feat_list = [f for f in features["features"].astype(str).tolist() if f in chars.columns]
    df = chars.copy()
    df[DATE_COL] = pd.to_datetime(df[DATE_COL])
    df = df.sort_values([DATE_COL, ID_COL], kind="stable").reset_index(drop=True)
    eom = df[DATE_COL].to_numpy()
    months, first = np.unique(eom, return_index=True)
    bounds = np.append(first, len(df))
    test = df[TEST_COL].astype(bool).to_numpy() if TEST_COL in df.columns else np.zeros(len(df), bool)
    X = df[feat_list].to_numpy(np.float32)
    Xr = np.empty(X.shape, np.float32)
    for i in range(0, X.shape[1], 50):
        blk = pd.DataFrame(X[:, i:i + 50].astype(np.float64)).groupby(eom).rank(pct=True) - 0.5
        Xr[:, i:i + 50] = np.nan_to_num(blk.to_numpy(np.float32), nan=0.0)
    r = pd.to_numeric(df[RET_COL], errors="coerce")
    g = r.groupby(df[DATE_COL], observed=True)
    return dict(X=X, Xr=Xr, eom=eom, months=months, test=test,
                z=((r - g.transform("mean")) / g.transform("std")).to_numpy(np.float32),
                slc={m: (int(bounds[i]), int(bounds[i + 1])) for i, m in enumerate(months)},
                ids=df[ID_COL].to_numpy(), ret=pd.to_numeric(df[RET_COL], errors="coerce").to_numpy(np.float64),
                themes=theme_exposures(df, df[DATE_COL]).to_numpy(np.float64),
                industry=_industry(df).to_numpy())


def build_daily_matrix(daily_ret, panel_ids):
    dr = daily_ret.rename(columns={c: c.lower() for c in daily_ret.columns})
    rid = dr["id"].to_numpy()
    keep = np.isin(rid, panel_ids)
    rid = rid[keep]
    rdate = pd.to_datetime(dr["date"]).to_numpy()[keep].astype("datetime64[D]")
    rret = dr["ret_exc"].to_numpy()[keep].astype(np.float32)
    dates, di = np.unique(rdate, return_inverse=True)
    ids, ii = np.unique(rid, return_inverse=True)
    M = np.full((len(dates), len(ids)), np.nan, dtype=np.float32)
    M[di, ii] = rret
    return dates, ids, M


def _daily_cols(data, daily_ids):
    ids = data["ids"].astype(np.float64)
    col = np.clip(np.searchsorted(daily_ids, ids), 0, len(daily_ids) - 1)
    return col, daily_ids[col] == ids


def _month_end(dates, t):
    return int(np.searchsorted(dates, np.datetime64(pd.Timestamp(t), "D"), "right"))


def _pca_risk(R):
    T, n = R.shape
    X = R - R.mean(0, keepdims=True)
    ss = (X * X).sum(0)
    sigma = np.sqrt(ss / (T - 1) * DAYS_PER_MONTH)
    Xn = X / np.sqrt(np.maximum(ss, 1e-12))
    G = Xn @ Xn.T
    ev, V = np.linalg.eigh(G)
    B = Xn.T @ V[:, ev > (1.0 + np.sqrt(n / T)) ** 2]
    S2 = float((G * G).sum())
    d2 = S2 - n
    b2 = float(((Xn * Xn).sum(1) ** 2).sum()) - S2 / T
    delta = min(max(b2, 0.0), d2) / d2 if d2 > 0 else 1.0
    B = B * np.sqrt(1.0 - delta)
    return B, np.maximum(1.0 - (B * B).sum(1), PSI_FLOOR), sigma


def _industry_dummies(ind):
    s = pd.Series(np.asarray(ind))
    s = s.where(s.map(s.value_counts()) >= MIN_INDUSTRY, -999)
    return pd.get_dummies(s).to_numpy(np.float64)


def residual_target(data, dates, daily_ids, M):
    n = len(data["ids"])
    y = np.full(n, np.nan, np.float32)
    sig = np.full(n, np.nan, np.float32)
    psi = np.full(n, np.nan, np.float32)
    col, has = _daily_cols(data, daily_ids)
    for t in data["months"]:
        lo, hi = data["slc"][t]
        end = _month_end(dates, t)
        if end < 60:
            continue
        a = max(0, end - RISK_DAYS)
        R = M[a:end][:, np.where(has[lo:hi], col[lo:hi], 0)].astype(np.float64)
        fin = np.isfinite(R)
        ok = has[lo:hi] & (fin.sum(0) >= min(RISK_MIN_OBS, max(20, (end - a) // 2)))
        if ok.sum() < 50:
            continue
        B, p, s = _pca_risk(np.where(fin, R, 0.0)[:, ok])
        rows = np.arange(lo, hi)[ok]
        sig[rows], psi[rows] = s, p
        r = data["ret"][rows]
        f = np.isfinite(r)
        if f.sum() < 50:
            continue
        inv_s = 1.0 / np.maximum(s, 1e-8)
        G = np.column_stack([B, inv_s, _industry_dummies(data["industry"][rows]) * inv_s[:, None]])[f]
        x = r[f] / (s[f] * np.sqrt(p[f]))
        U, sv, _ = np.linalg.svd(G / np.sqrt(p[f])[:, None], full_matrices=False)
        U = U[:, sv > 1e-10 * sv.max()]
        z = x - U @ (U.T @ x)
        sd = z.std()
        if np.isfinite(sd) and sd > 0:
            y[rows[f]] = np.clip(z / sd, -4.0, 4.0)
    return y, sig, psi


def _suggest(trial):
    return {
        "n_estimators": 2000,
        "learning_rate": trial.suggest_float("learning_rate", 5e-3, 1.5e-1, log=True),
        "max_leaves": trial.suggest_int("max_leaves", 15, 511, log=True),
        "max_depth": trial.suggest_int("max_depth", 3, 14),
        "min_child_weight": trial.suggest_int("min_child_weight", 20, 5000, log=True),
        "subsample": trial.suggest_float("subsample", 0.6, 1.0),
        "colsample_bytree": trial.suggest_float("colsample_bytree", 0.6, 1.0),
        "reg_lambda": trial.suggest_float("reg_lambda", 1e-2, 1e2, log=True),
    }


def _default_params():
    return {"n_estimators": 400, "learning_rate": 0.03, "max_leaves": 63, "max_depth": 8,
            "min_child_weight": 200, "subsample": 0.9, "colsample_bytree": 0.8, "reg_lambda": 1.0}


def _fit(Xtr, ytr, Xva, yva, params, threads):
    import xgboost as xgb

    p = dict(params)
    n_est = p.pop("n_estimators", 2000)
    es = {"early_stopping_rounds": 50} if len(Xva) >= 50 else {}
    model = xgb.XGBRegressor(objective="reg:squarederror", tree_method="hist", grow_policy="lossguide",
                             n_jobs=threads, random_state=SEED, verbosity=0, n_estimators=n_est, **es, **p,
                             feature_weights=np.ones(Xtr.shape[1]))
    if es:
        model.fit(Xtr, ytr, eval_set=[(Xva, yva)], verbose=False)
    else:
        model.fit(Xtr, ytr, verbose=False)
    return model


def _fit_refit_full(Xtr, ytr, Xva, yva, params, threads):
    model = _fit(Xtr, ytr, Xva, yva, params, threads)
    n_best = getattr(model, "best_iteration", None)
    if not n_best or len(Xva) < 50:
        return model
    return _fit(np.vstack([Xtr, Xva]), np.concatenate([ytr, yva]), Xva[:0], yva[:0],
                {**params, "n_estimators": int(n_best) + 1}, threads)


def _rank_ic(pred, y, months):
    d = pd.DataFrame({"m": months, "p": pred, "y": y}).dropna()
    return float(d.groupby("m").apply(lambda g: g["p"].corr(g["y"], method="spearman"),
                                      include_groups=False).mean())


def _rows(slc, months):
    return slc[months[0]][0], slc[months[-1]][1]


def _tune_round(X, y, eom, slc, avail, rnd, threads):
    import optuna

    optuna.logging.set_verbosity(optuna.logging.WARNING)
    warnings.filterwarnings("ignore", category=optuna.exceptions.ExperimentalWarning)
    folds = []
    for k in range(1, CV_FOLDS + 1):
        vm = avail[len(avail) - VAL_MONTHS * k: len(avail) - VAL_MONTHS * (k - 1)]
        t_hi = slc[avail[len(avail) - VAL_MONTHS * k - 1]][1]
        folds.append((max(0, t_hi - TUNE_SUBSAMPLE), t_hi) + _rows(slc, vm))
    y64 = y.astype(np.float64)

    def objective(trial):
        params = _suggest(trial)
        ics = []
        for t_lo, t_hi, v_lo, v_hi in folds:
            yt, yv = y64[t_lo:t_hi], y64[v_lo:v_hi]
            mt, mv = np.isfinite(yt), np.isfinite(yv)
            Xv = np.asarray(X[v_lo:v_hi])
            m = _fit(np.asarray(X[t_lo:t_hi])[mt], yt[mt], Xv[mv], yv[mv], params, threads)
            ics.append(_rank_ic(m.predict(Xv), yv, eom[v_lo:v_hi]))
        trial.set_user_attr("params", params)
        return float(np.mean(ics))

    sampler = optuna.samplers.TPESampler(seed=SEED + rnd, n_startup_trials=OPTUNA_STARTUP, multivariate=True)
    study = optuna.create_study(direction="maximize", sampler=sampler)
    d = _default_params()
    study.enqueue_trial({k: v for k, v in d.items() if k != "n_estimators"})
    state = {"best": -np.inf, "since": 0}

    def patience(st, tr):
        if tr.value is None:
            return
        if tr.number < OPTUNA_STARTUP:
            state["best"] = max(state["best"], tr.value)
            return
        if tr.value > state["best"] + OPTUNA_MIN_DELTA:
            state["best"], state["since"] = tr.value, 0
        else:
            state["since"] += 1
            if state["since"] >= OPTUNA_PATIENCE:
                st.stop()

    study.optimize(objective, n_trials=OPTUNA_TRIALS, callbacks=[patience], show_progress_bar=False)
    done = [t for t in study.trials if t.state == optuna.trial.TrialState.COMPLETE]
    done.sort(key=lambda t: (-t.value, t.number))
    print(f"  tune round {rnd}: {len(done)} trials, best IC {done[0].value:.4f}", flush=True)
    return [t.user_attrs["params"] for t in done[:TOP_K]]


def _fit_block(X, y, slc, core, vals, params, pred_rows, threads):
    y64 = y.astype(np.float64)
    c_lo, c_hi = _rows(slc, core)
    v_lo, v_hi = _rows(slc, vals)
    yc, yv = y64[c_lo:c_hi], y64[v_lo:v_hi]
    mc, mv = np.isfinite(yc), np.isfinite(yv)
    model = _fit_refit_full(np.asarray(X[c_lo:c_hi])[mc], yc[mc], np.asarray(X[v_lo:v_hi])[mv], yv[mv],
                            params, threads)
    return model.predict(np.asarray(X[pred_rows]))


def standardize(mu, eom):
    s = pd.Series(mu, index=eom)
    g = s.groupby(level=0)
    return ((s - g.transform("mean")) / g.transform("std")).to_numpy()


def _schedule(data, min_train):
    months, slc = data["months"], data["slc"]
    pred = ([m for m in months if data["test"][slc[m][0]:slc[m][1]].any()] if data["test"].any()
            else list(months[min_train:]))
    pred = [m for m in pred if int(np.searchsorted(months, m)) >= min_train]
    refits = pred[::RETRAIN_EVERY]
    avails = [months[:int(np.searchsorted(months, t))] for t in refits]
    rows = []
    for i, t in enumerate(refits):
        nxt = refits[i + 1] if i + 1 < len(refits) else None
        rows.append(np.concatenate([np.arange(*slc[m]) for m in pred if m >= t and (nxt is None or m < nxt)]))
    return refits, avails, rows


def walk_forward(data, y):
    from joblib import Parallel, delayed

    X, eom, months, slc = data["X"], data["eom"], data["months"], data["slc"]
    small = len(months) < 150
    min_train, val_n = (SMALL_MIN_TRAIN, SMALL_VAL_MONTHS) if small else (MIN_TRAIN_MONTHS, VAL_MONTHS)
    refits, avails, block_rows = _schedule(data, min_train)
    per = TUNE_EVERY // RETRAIN_EVERY

    if small:
        cands = [[_default_params()] for _ in refits]
    elif not TUNE:
        cands = [[{"n_estimators": 2000, **p} for p in XGB_PARAMS[min(i // per, len(XGB_PARAMS) - 1)]]
                 for i in range(len(refits))]
    else:
        tune_at = list(range(0, len(refits), per))
        n_jobs = min(len(tune_at), N_CPU)
        tuned = Parallel(n_jobs=n_jobs)(delayed(_tune_round)(X, y, eom, slc, avails[i], i // per,
                                                              max(1, N_CPU // n_jobs)) for i in tune_at)
        cands = [tuned[i // per] for i in range(len(refits))]

    tasks = []
    for i in range(len(refits)):
        for p in cands[i]:
            tasks.append((i, block_rows[i], avails[i][:-val_n], avails[i][-val_n:], p))
    n_jobs = max(1, N_CPU // 8)
    out = Parallel(n_jobs=n_jobs)(delayed(_fit_block)(X, y, slc, c, v, p, r, max(1, N_CPU // n_jobs))
                                  for _, r, c, v, p in tasks)
    acc, cnt = np.zeros(len(X)), np.zeros(len(X))
    for (i, rows, _, _, _), val in zip(tasks, out):
        mu = np.full(len(X), np.nan)
        mu[rows] = val
        z = standardize(mu, eom) if len(cands[i]) > 1 else mu
        ok = np.isfinite(z)
        acc[ok] += z[ok]
        cnt[ok] += 1
    return np.where(cnt > 0, acc / np.maximum(cnt, 1), np.nan)


def _net_make(params, seed):
    from sklearn.neural_network import MLPRegressor

    return MLPRegressor(hidden_layer_sizes=params["hidden"], activation="tanh", solver="adam",
                        alpha=params["alpha"], learning_rate_init=params["lr"], batch_size=params["batch"],
                        max_iter=1, warm_start=True, shuffle=True, early_stopping=False, random_state=seed)


def _net_epochs(Xt, yt, Xv, yv, params, Xp=None):
    model = _net_make(params, SEED)
    best, epochs, since, pred = np.inf, 1, 0, None
    for ep in range(1, NET_MAX_EPOCHS + 1):
        model.fit(Xt, yt)
        mse = float(np.mean((model.predict(Xv) - yv) ** 2))
        if mse < best - 1e-6:
            best, epochs, since = mse, ep, 0
            if Xp is not None:
                pred = model.predict(Xp)
        else:
            since += 1
            if since >= NET_PATIENCE:
                break
    return epochs, pred


def _net_suggest(trial):
    arch = trial.suggest_categorical("arch", ["32", "32-16", "32-16-8", "64-32-16", "128-64-32"])
    return {"hidden": tuple(int(h) for h in arch.split("-")),
            "alpha": trial.suggest_float("alpha", 1e-6, 1e-2, log=True),
            "lr": trial.suggest_float("lr", 1e-4, 3e-3, log=True),
            "batch": trial.suggest_categorical("batch", [2048, 8192, 32768])}


def _net_tune_round(Xr, z, eom, slc, avail, rnd):
    import optuna

    warnings.filterwarnings("ignore", message="Stochastic Optimizer: Maximum iterations")
    optuna.logging.set_verbosity(optuna.logging.WARNING)
    warnings.filterwarnings("ignore", category=optuna.exceptions.ExperimentalWarning)
    z64 = z.astype(np.float64)
    folds = []
    for k in range(1, CV_FOLDS + 1):
        vm = avail[len(avail) - VAL_MONTHS * k: len(avail) - VAL_MONTHS * (k - 1)]
        t_hi = slc[avail[len(avail) - VAL_MONTHS * k - 1]][1]
        folds.append((max(0, t_hi - TUNE_SUBSAMPLE), t_hi) + _rows(slc, vm))

    def objective(trial):
        params = _net_suggest(trial)
        ics = []
        for t_lo, t_hi, v_lo, v_hi in folds:
            yt, yv = z64[t_lo:t_hi], z64[v_lo:v_hi]
            mt, mv = np.isfinite(yt), np.isfinite(yv)
            Xv = np.asarray(Xr[v_lo:v_hi])
            _, pred = _net_epochs(np.asarray(Xr[t_lo:t_hi])[mt], yt[mt], Xv[mv], yv[mv], params, Xv)
            ics.append(_rank_ic(pred, yv, eom[v_lo:v_hi]))
        trial.set_user_attr("params", params)
        return float(np.mean(ics))

    sampler = optuna.samplers.TPESampler(seed=SEED + rnd, n_startup_trials=OPTUNA_STARTUP, multivariate=True)
    study = optuna.create_study(direction="maximize", sampler=sampler)
    study.enqueue_trial({"arch": "32-16", "alpha": NET_DEFAULT["alpha"], "lr": NET_DEFAULT["lr"],
                         "batch": NET_DEFAULT["batch"]})
    state = {"best": -np.inf, "since": 0}

    def patience(st, tr):
        if tr.value is None:
            return
        if tr.number < OPTUNA_STARTUP:
            state["best"] = max(state["best"], tr.value)
            return
        if tr.value > state["best"] + OPTUNA_MIN_DELTA:
            state["best"], state["since"] = tr.value, 0
        else:
            state["since"] += 1
            if state["since"] >= OPTUNA_PATIENCE:
                st.stop()

    study.optimize(objective, n_trials=OPTUNA_TRIALS, callbacks=[patience], show_progress_bar=False)
    done = [t for t in study.trials if t.state == optuna.trial.TrialState.COMPLETE]
    done.sort(key=lambda t: (-t.value, t.number))
    print(f"  network tune round {rnd}: {len(done)} trials, best IC {done[0].value:.4f}", flush=True)
    return [t.user_attrs["params"] for t in done[:TOP_K]]


def _net_fit(Xr, z, slc, core, vals, params, pred_rows):
    warnings.filterwarnings("ignore", message="Stochastic Optimizer: Maximum iterations")
    z64 = z.astype(np.float64)
    c_lo, c_hi = _rows(slc, core)
    v_lo, v_hi = _rows(slc, vals)
    yc, yv = z64[c_lo:c_hi], z64[v_lo:v_hi]
    mc, mv = np.isfinite(yc), np.isfinite(yv)
    Xc, Xv, yc, yv = np.asarray(Xr[c_lo:c_hi])[mc], np.asarray(Xr[v_lo:v_hi])[mv], yc[mc], yv[mv]
    epochs, _ = _net_epochs(Xc, yc, Xv, yv, params)
    Xa, ya = np.vstack([Xc, Xv]), np.concatenate([yc, yv])
    Xp = np.asarray(Xr[pred_rows])
    preds = []
    for s in range(NET_INITS):
        m = _net_make(params, SEED + s)
        for _ in range(epochs):
            m.fit(Xa, ya)
        preds.append(m.predict(Xp))
    return np.mean(preds, axis=0)


def walk_forward_net(data):
    from joblib import Parallel, delayed

    Xr, eom, months, slc = data["Xr"], data["eom"], data["months"], data["slc"]
    if len(months) < NET_MIN_MONTHS:
        return np.full(len(Xr), np.nan)
    refits, avails, block_rows = _schedule(data, MIN_TRAIN_MONTHS)
    per = TUNE_EVERY // RETRAIN_EVERY
    if TUNE:
        tune_at = list(range(0, len(refits), per))
        tuned = Parallel(n_jobs=min(len(tune_at), N_CPU))(
            delayed(_net_tune_round)(Xr, data["z"], eom, slc, avails[i], i // per) for i in tune_at)
        cands = [tuned[i // per] for i in range(len(refits))]
    else:
        cands = [NET_PARAMS[min(i // per, len(NET_PARAMS) - 1)] for i in range(len(refits))]
    tasks = [(block_rows[i], avails[i][:-VAL_MONTHS], avails[i][-VAL_MONTHS:], p)
             for i in range(len(refits)) for p in cands[i]]
    n_jobs = max(1, N_CPU // 4)
    out = Parallel(n_jobs=n_jobs)(delayed(_net_fit)(Xr, data["z"], slc, c, v, p, r) for r, c, v, p in tasks)
    acc, cnt = np.zeros(len(Xr)), np.zeros(len(Xr))
    for (rows, _, _, _), val in zip(tasks, out):
        mu = np.full(len(Xr), np.nan)
        mu[rows] = val
        z = standardize(mu, eom)
        ok = np.isfinite(z)
        acc[ok] += z[ok]
        cnt[ok] += 1
    return np.where(cnt > 0, acc / np.maximum(cnt, 1), np.nan)


def trailing_window(M, end, cols, window):
    lo = max(0, end - window)
    X = M[lo:end, cols]
    obs = np.isfinite(X)
    enough = obs.sum(axis=0) >= min(MIN_DAILY_OBS, max(20, (end - lo) // 3))
    X = np.where(obs, X, np.float32(0.0))
    X -= X.mean(axis=0, keepdims=True)
    return X, enough


def shrunk_covariance(X):
    T, n = X.shape
    S = (X.T @ X) / T
    tr_S = float(np.trace(S, dtype=np.float64))
    mu_lw = tr_S / n
    norm_S2 = float(np.square(S).sum(dtype=np.float64))
    d2 = norm_S2 - 2.0 * mu_lw * tr_S + n * mu_lw ** 2
    row_sq = np.square(X).sum(axis=1, dtype=np.float64)
    b2 = float(np.square(row_sq).sum()) / T ** 2 - norm_S2 / T
    delta = min(max(b2, 0.0), d2) / d2 if d2 > 1e-300 else 1.0
    S *= np.float32(1.0 - delta)
    S[np.diag_indices(n)] += np.float32(delta * mu_lw)
    return S


def tan_minus_gmv(mu, Sigma, kappa=KAPPA):
    n = mu.shape[0]
    lam = np.float32(kappa * float(np.trace(Sigma, dtype=np.float64)) / n)
    A = Sigma + lam * np.eye(n, dtype=np.float32)
    one = np.ones(n, dtype=np.float32)
    sol = np.linalg.solve(A, np.column_stack([mu, one]))
    a, b = sol[:, 0], sol[:, 1]
    w = a - np.float32(float(one @ a) / float(one @ b)) * b
    den = np.abs(w).sum()
    return w / den if den > 0 else w


def book_daily_cov(data, dates, daily_ids, M, mu, window):
    w_full = np.zeros(len(mu), np.float32)
    col, has = _daily_cols(data, daily_ids)
    for t in data["months"]:
        lo, hi = data["slc"][t]
        if not np.isfinite(mu[lo:hi]).any():
            continue
        cols = col[lo:hi][has[lo:hi]]
        end = _month_end(dates, t)
        if end <= 20 or len(cols) < 2:
            continue
        n_rows = hi - lo
        Xd, enough = trailing_window(M, end, cols, window)
        usable = np.zeros(n_rows, bool)
        usable[np.flatnonzero(has[lo:hi])[enough]] = True
        if usable.sum() < min(MIN_NAMES, max(5, n_rows // 4)):
            continue
        Sig = shrunk_covariance(Xd[:, enough] * np.float32(np.sqrt(DAYS_PER_MONTH)))
        w_full[lo:hi][usable] = tan_minus_gmv(np.nan_to_num(mu[lo:hi], nan=0.0)[usable], Sig)
    return w_full


def _woodbury_sigma_inv_mu(B, Omega, d, mu):
    dinv = 1.0 / d
    BtDinv = B.T * dinv
    Mx = np.linalg.inv(Omega + 1e-8 * np.eye(B.shape[1])) + BtDinv @ B
    return dinv * mu - dinv * (B @ np.linalg.solve(Mx, BtDinv @ mu))


def book_factor_mvo(data, mu):
    months, slc = data["months"], data["slc"]
    Th, ret, ids = data["themes"], data["ret"], data["ids"]
    K = Th.shape[1]
    reg = 1e-4 * np.eye(K)
    F, resid = {}, np.full(len(ret), np.nan)
    for t in months:
        lo, hi = slc[t]
        Xm, rm = Th[lo:hi], ret[lo:hi]
        ok = np.isfinite(rm) & np.isfinite(Xm).all(axis=1)
        if ok.sum() <= K:
            continue
        Xo, ro = Xm[ok], rm[ok]
        f = np.linalg.solve(Xo.T @ Xo + reg, Xo.T @ ro)
        F[t] = f
        resid[np.arange(lo, hi)[ok]] = ro - Xo @ f
    fmonths = np.array(sorted(F), dtype=months.dtype)
    Fv = np.array([F[m] for m in fmonths]) if len(fmonths) else np.zeros((0, K))
    w_full = np.zeros(len(mu), np.float32)
    for t in months:
        lo, hi = slc[t]
        if not np.isfinite(mu[lo:hi]).any():
            continue
        i = int(np.searchsorted(fmonths, t))
        win = fmonths[max(0, i - MVO_WINDOW):i]
        if len(win) < max(12, K):
            continue
        Omega = np.cov(Fv[max(0, i - MVO_WINDOW):i], rowvar=False)
        rows = np.concatenate([np.arange(*slc[m]) for m in win])
        rv = pd.Series(resid[rows], index=ids[rows]).groupby(level=0).var()
        d = pd.Series(ids[lo:hi]).map(rv).to_numpy(np.float64)
        fin = np.isfinite(d)
        if fin.sum() < 5:
            continue
        floor = max(float(np.nanquantile(d[fin], D_FLOOR_Q)), 1e-8)
        d = np.where(fin, np.maximum(d, floor), floor)
        w = _woodbury_sigma_inv_mu(Th[lo:hi].astype(np.float64), Omega, d, np.nan_to_num(mu[lo:hi], nan=0.0))
        w -= w.mean()
        den = np.abs(w).sum()
        if den > 0:
            w_full[lo:hi] = w / den
    return w_full


def industry_neutral(data, w):
    ind = data["industry"]
    out = np.zeros(len(w), np.float32)
    for t in data["months"]:
        lo, hi = data["slc"][t]
        x = w[lo:hi].astype(np.float64)
        if not np.abs(x).sum():
            continue
        x = x - pd.Series(x).groupby(ind[lo:hi]).transform("mean").to_numpy()
        den = np.abs(x).sum()
        if den > 0:
            out[lo:hi] = x / den
    return out


def market_beta(data, dates, daily_ids, M, window=252):
    beta = np.zeros(len(data["ids"]))
    col, has = _daily_cols(data, daily_ids)
    for t in data["months"]:
        lo, hi = data["slc"][t]
        cols = col[lo:hi][has[lo:hi]]
        end = _month_end(dates, t)
        if end < 60 or len(cols) < 5:
            continue
        X, _ = trailing_window(M, end, cols, window)
        mkt = X.mean(axis=1)
        var = float(mkt @ mkt)
        if var > 0:
            beta[np.flatnonzero(has[lo:hi]) + lo] = (X.T @ mkt) / var
    return beta


def beta_neutral(data, w, beta, held):
    out = np.zeros(len(w))
    for t in data["months"]:
        lo, hi = data["slc"][t]
        h = held[lo:hi]
        x = w[lo:hi].astype(np.float64)
        if not np.abs(x).sum() or h.sum() < 3:
            continue
        Bx = np.column_stack([beta[lo:hi][h], np.ones(int(h.sum()))])
        coef, *_ = np.linalg.lstsq(Bx, x[h], rcond=None)
        y = np.zeros(hi - lo)
        y[h] = x[h] - Bx @ coef
        den = np.abs(y).sum()
        if den > 0:
            out[lo:hi] = y / den
    return out


def book_fallback(data, mu):
    s = pd.Series(mu, index=data["eom"])
    w = np.nan_to_num((s.groupby(level=0).rank(pct=True) - 0.5).to_numpy(), nan=0.0)
    gross = pd.Series(np.abs(w), index=data["eom"]).groupby(level=0).transform("sum").to_numpy()
    return np.divide(w, gross, out=np.zeros_like(w), where=gross > 0)


def vol_lever(data, dates, daily_ids, M, w):
    col, has = _daily_cols(data, daily_ids)
    months = [m for m in data["months"] if np.abs(w[slice(*data["slc"][m])]).sum() > 0]
    rd = np.zeros(len(dates))
    for i, t in enumerate(months):
        lo, hi = data["slc"][t]
        d0 = _month_end(dates, t)
        d1 = _month_end(dates, months[i + 1]) if i + 1 < len(months) else len(dates)
        sel = has[lo:hi] & (w[lo:hi] != 0)
        rd[d0:d1] = np.nan_to_num(M[d0:d1][:, col[lo:hi][sel]].astype(np.float64)) @ w[lo:hi][sel]
    if not months:
        return np.ones(len(w))
    start = _month_end(dates, months[0])
    s = pd.Series(rd[start:], index=pd.DatetimeIndex(dates[start:]))
    est = np.sqrt((s ** 2).ewm(halflife=VOL_HALFLIFE, min_periods=VOL_MIN_DAYS).mean())
    est = est.groupby(est.index.to_period("M")).last()
    est.index = est.index.to_timestamp("M")
    est = est.reindex(pd.DatetimeIndex(months))
    lever = (est.expanding().mean() / est).clip(*VOL_CLIP).fillna(1.0)
    return lever.reindex(pd.DatetimeIndex(data["eom"])).fillna(1.0).to_numpy()


def main(chars, features, daily_ret):
    random.seed(SEED)
    np.random.seed(SEED)
    data = prepare(chars, features)
    eom = data["eom"]
    print(f"panel {data['X'].shape}, {len(data['months'])} months", flush=True)
    dates, daily_ids, M = build_daily_matrix(daily_ret, np.unique(data["ids"]))
    y, sig, psi = residual_target(data, dates, daily_ids, M)
    print(f"residual target on {int(np.isfinite(y).sum()):,} rows", flush=True)

    mu_res = walk_forward(data, y)
    held = np.isfinite(mu_res)
    mu = standardize(mu_res * sig.astype(np.float64) * np.sqrt(psi.astype(np.float64)), eom)
    mu = np.where(held, np.nan_to_num(mu), np.nan)
    mu_net = walk_forward_net(data)
    if np.isfinite(mu_net[held]).any():
        b = (1 - NET_WEIGHT) * np.nan_to_num(mu) + NET_WEIGHT * np.nan_to_num(standardize(mu_net, eom))
        mu = np.where(held, np.nan_to_num(standardize(np.where(held, b, np.nan), eom)), np.nan)
        print(f"network blended at weight {NET_WEIGHT}", flush=True)
    print(f"forecasts for {int(held.sum()):,} rows", flush=True)

    w = 0.5 * book_daily_cov(data, dates, daily_ids, M, mu, WINDOW_LONG).astype(np.float64) \
        + 0.5 * industry_neutral(data, book_factor_mvo(data, mu)).astype(np.float64)
    w[~held] = 0.0
    gross = pd.Series(np.abs(w), index=eom).groupby(level=0).transform("sum").to_numpy()
    w = np.divide(w, gross, out=np.zeros_like(w), where=gross > 0)
    empty = pd.Series(gross <= 0, index=eom).groupby(level=0).transform("all").to_numpy()
    if empty.any():
        w = np.where(empty & held, book_fallback(data, np.where(held, mu, 0.0)), w)
    w = beta_neutral(data, w, market_beta(data, dates, daily_ids, M), held)

    w = w * vol_lever(data, dates, daily_ids, M, w)

    out = pd.DataFrame({ID_COL: data["ids"], DATE_COL: pd.to_datetime(eom), "w": w})
    if data["test"].any():
        out = out[data["test"]]
    out = out.dropna(subset=[ID_COL, DATE_COL])
    out["w"] = out["w"].fillna(0.0).replace([np.inf, -np.inf], 0.0)
    print(f"returning {len(out):,} rows", flush=True)
    return out.reset_index(drop=True)
