"""Fixed residual-books + peer ensemble. Self-contained CTF candidate.

All learned states are computed from the input DataFrames. No embedded weights,
local paths, external data, or retrospective volatility normalization.

Source (note added by the CTF administrators):
   This model is derived from the CTF entries "Residual Books (LightGBM)"
   and "Residual Books (LightGBM, industry-neutral)" by Felix Gehling (2026),
   published at jkpfactors.com/ctf/models/residual-books-lightgbm-q54psgdyxg
   and jkpfactors.com/ctf/models/residual-books-lightgbm-industry-neutral-myacq6ha0p.
   Its peer-relative characteristics follow "Peer-Relative Clustering
   LightGBM+Barra" by Andrew Qi et al. (2026), published at
   jkpfactors.com/ctf/models/peer-relative-clustering-lightgbmbarra-1xjz6z5k9d.

CTF Admin Modifications (2026-10-23):
--------------------------------------
1. Added the source note above.
   Reason: the residual-book code builds on the two Residual Books entries
   and the peer-relative characteristics follow the Peer-Relative Clustering
   entry. Entries derived from other entries name their sources in the script.
"""
from __future__ import annotations
import numpy as np
import pandas as pd
import lightgbm as lgb
import torch
from torch import nn
from scipy.linalg import eigh
from scipy.cluster.hierarchy import linkage, fcluster
from threadpoolctl import threadpool_limits
from joblib import Parallel, delayed

SEED=42
TARGET="ret_exc_lead1m"
N_THREADS=4
N_JOBS=3
MIN_INDUSTRY=10
BOOK_NAMES=("open_dollars","neutral","affine_cae")
AFFINE_ANCHOR=int(np.datetime64("1983-12-31").astype("datetime64[D]").astype(np.int64))

def to_days(values) -> np.ndarray:
    return pd.to_datetime(values).to_numpy().astype("datetime64[D]").astype(np.int64)



def mp_edge(p: int, n: int) -> float:
    """Upper edge of the Marchenko-Pastur law for a p-variable correlation matrix from n observations."""
    return (1.0 + np.sqrt(p / n)) ** 2



def daily_matrix(daily_ret: pd.DataFrame, ids: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
    """Dense (days x ids) daily excess returns, NaN where missing; ids must be sorted and unique."""
    did = daily_ret["id"].to_numpy(np.int64)
    col = np.minimum(np.searchsorted(ids, did), len(ids) - 1)
    keep = ids[col] == did
    day = to_days(daily_ret["date"].to_numpy()[keep])
    days, row = np.unique(day, return_inverse=True)
    M = np.full((len(days), len(ids)), np.nan, dtype=np.float32)
    M[row, col[keep]] = daily_ret["ret_exc"].to_numpy(np.float32)[keep]
    return days, M



def residualize(a: np.ndarray, G: np.ndarray, psi: np.ndarray) -> np.ndarray:
    """Q a: the GLS residual of a on G with weights 1/psi."""
    Gw = G / psi[:, None]
    try:
        return a - G @ np.linalg.solve(G.T @ Gw, Gw.T @ a)
    except np.linalg.LinAlgError:                       # rank-deficient G: same projection via least squares
        s = 1.0 / np.sqrt(psi)
        coef = np.linalg.lstsq(G * s[:, None], (a.T * s).T, rcond=None)[0]
        return a - G @ coef



def industry_dummies(codes: np.ndarray) -> np.ndarray:
    """Indicators of two-digit SIC industries; small industries and missing codes form one group."""
    uniq, inv, cnt = np.unique(codes, return_inverse=True, return_counts=True)
    keep = (cnt >= MIN_INDUSTRY) & (uniq >= 0)
    group = np.where(keep[inv], inv, -1)
    labels = np.unique(group)
    return (group[:, None] == labels[None, :]).astype(np.float64)



def book_sharpe_metric(month_codes: np.ndarray, z: np.ndarray, n_months: int):
    """Validation metric: Sharpe ratio of the monthly returns s'z / ||s||."""
    def feval(preds, _data):
        num = np.bincount(month_codes, weights=preds * z, minlength=n_months)
        den = np.sqrt(np.bincount(month_codes, weights=preds * preds, minlength=n_months))
        R = np.where(den > 0, num / np.where(den > 0, den, 1.0), 0.0)
        sd = R.std(ddof=1)
        return "book_sharpe", (R.mean() / sd if sd > 0 else 0.0), True
    return feval



def risk_fast(R):
    """Same PCA subspace as original SVD, using the smaller 252x252 Gram matrix."""
    obs = ~np.isnan(R)
    n_obs = obs.sum(axis=0)
    X = np.where(obs, R, 0.0).astype(np.float64)
    mean = X.sum(axis=0) / np.maximum(n_obs, 1)
    X = np.where(obs, X - mean, 0.0)
    ss = np.square(X).sum(axis=0)
    ok = (n_obs >= 200) & (ss > 0)
    X, ss, n_obs = X[:, ok], ss[ok], n_obs[ok]
    sigma = np.sqrt(ss / (n_obs - 1) * 21.0)
    X /= np.sqrt(ss)
    T, n = X.shape
    eig, U = np.linalg.eigh(X @ X.T)
    take = eig > (1.0 + np.sqrt(n / T)) ** 2
    B = X.T @ U[:, take]
    psi = np.maximum(1.0 - np.square(B).sum(axis=1), 0.05)
    return ok, B, psi, sigma



def char_fast(Xm):
    Z = pd.DataFrame(Xm).rank(pct=True).to_numpy() - 0.5
    Z = np.nan_to_num(Z, nan=0.0)
    Z -= Z.mean(axis=0)
    sd = Z.std(axis=0)
    Z = Z[:, sd > 0] / sd[sd > 0]
    n, p = Z.shape
    eig, V = np.linalg.eigh(Z.T @ Z / n)
    return Z @ V[:, eig > (1.0 + np.sqrt(p / n)) ** 2] / np.sqrt(n)



def projector_basis(G, psi):
    """Orthonormal basis in standardized residual-score coordinates, including rank checks."""
    A = G / np.sqrt(psi)[:, None]
    U, s, _ = np.linalg.svd(A, full_matrices=False)
    rank = (s > s[0] * max(A.shape) * np.finfo(float).eps).sum() if len(s) else 0
    return U[:, :rank]



class AffineFactor(nn.Module):
    def __init__(self,p):
        super().__init__()
        # Retain constructor RNG order of the earlier affine control.
        self.beta=nn.Linear(p,5,bias=False)
        self.factors=nn.Linear(p,5,bias=False)
        self.beta=nn.Linear(p,5,bias=True)

    def loss(self,g):
        X,Y,P=g
        return (self.beta(X)@self.factors(P)-Y).square().mean()+.01*0.



def learn_affine(groups,threads=4):
    torch.set_num_threads(threads)
    p=groups[0]['X'].shape[1]
    sample=[tuple(torch.from_numpy(g[k]) for k in ['X','Y','P']) for g in groups]
    cut=np.datetime64('1978-12-31').astype(np.int64)
    train=[i for i,g in enumerate(groups) if g['day']<cut]
    valid=[i for i,g in enumerate(groups) if g['day']>=cut]
    if not train or not valid:
        split=max(1,len(groups)*2//3);train=list(range(split));valid=list(range(split,len(groups)))
    def stage(indices,max_epochs,select):
        torch.manual_seed(42);rng=np.random.default_rng(42)
        model=AffineFactor(p);optimizer=torch.optim.Adam(model.parameters(),lr=.001,weight_decay=.0001)
        best=float('inf');best_epoch=0;wait=0
        for epoch in range(1,max_epochs+1):
            for i in rng.permutation(indices):
                optimizer.zero_grad(set_to_none=True);loss=model.loss(sample[i]);loss.backward();optimizer.step()
            if select:
                with torch.no_grad():value=float(torch.stack([model.loss(sample[i]) for i in valid]).mean())
                if value<best-1e-7:best=value;best_epoch=epoch;wait=0
                else:wait+=1
                if wait>=10:break
        return model,best_epoch
    _,epochs=stage(train,80,True)
    model,_=stage(list(range(len(groups))),epochs,False)
    return model.train(False),epochs


def peer_features(raw):
    """Current-month cross-section only; all names precede risk eligibility."""
    rank=pd.DataFrame(raw).rank(pct=True).to_numpy(np.float64)-.5
    Z=np.nan_to_num(rank,nan=0.);Z-=Z.mean(0)
    sd=Z.std(0);valid=sd>0;Z=Z[:,valid]/sd[valid]
    pdim=Z.shape[1];k=min(20,pdim)
    if len(Z)<2 or k==0:
        centered=rank-np.nan_to_num(pd.DataFrame(rank).mean().to_numpy(),nan=0.)
    else:
        _,V=eigh(Z.T@Z/max(1,len(Z)),subset_by_index=[pdim-k,pdim-1],check_finite=False)
        P=Z@V
        count=min(len(Z),max(10,min(60,int(np.rint(len(Z)/100)))))
        group=fcluster(linkage(P,method='ward'),count,criterion='maxclust')-1
        sums=np.zeros((group.max()+1,raw.shape[1]))
        np.add.at(sums,group,np.nan_to_num(rank,nan=0.))
        counts=np.zeros_like(sums);np.add.at(counts,group,np.isfinite(rank))
        means=np.divide(sums,counts,out=np.zeros_like(sums),where=counts>0)
        centered=rank-means[group]
    return np.concatenate([raw,centered.astype(np.float32)],axis=1)


def prepare_inputs(chars,features,daily_ret):
    feats=[str(f) for f in features['features'].tolist()]
    if len(feats)!=len(set(feats)) or not feats:
        raise ValueError('Features must be nonempty and unique')
    panel=chars[['id','eom',TARGET,'sic','ctff_test']+feats].copy()
    panel['id']=panel['id'].astype(np.int64);panel['_day']=to_days(panel.eom)
    panel=panel.sort_values(['_day','id'],kind='stable').reset_index(drop=True)
    if panel.duplicated(['id','_day']).any():raise ValueError('Duplicate stock/month')
    months,month_ix=np.unique(panel['_day'].to_numpy(),return_inverse=True)
    starts=np.searchsorted(month_ix,np.arange(len(months)+1))
    ids=panel.id.to_numpy(np.int64)
    X=panel[feats].to_numpy(np.float32);ret=panel[TARGET].to_numpy(np.float64)
    sic=(pd.to_numeric(panel.sic,errors='coerce')//100).fillna(-1).to_numpy(np.int64)
    test=panel.ctff_test.astype(str).str.strip().str.lower().isin(['1','1.0','true']).to_numpy()
    # Small validation inputs may carry no scored rows. Process their final
    # available year with precisely the same historical fitting logic.
    output=test.copy()
    if not output.any():output=month_ix>=max(0,len(months)-12)
    del panel
    uid=np.unique(ids);days,daily=daily_matrix(daily_ret,uid)
    columns=np.searchsorted(uid,ids)
    risk=[None]*len(months)
    XP=np.empty((len(X),2*X.shape[1]),dtype=np.float32)
    with threadpool_limits(limits=1):
        for t,day in enumerate(months):
            lo,hi=starts[t:t+2]
            XP[lo:hi]=peer_features(X[lo:hi])
            end=int(np.searchsorted(days,day,side='right'))
            if end<200:continue
            ok,B,psi,sigma=risk_fast(daily[max(0,end-252):end,columns[lo:hi]])
            if ok.sum()<100:continue
            rows=lo+np.flatnonzero(ok)
            base=np.column_stack([B,1/sigma])
            G=[np.column_stack([base,industry_dummies(sic[rows])/sigma[:,None]]),
               np.column_stack([base,char_fast(X[rows])])]
            U=[projector_basis(g,psi) for g in G]
            risk[t]=dict(rows=rows,psi=psi,sigma=sigma,B=B,G=G,U=U)
            if t%100==0:print(f'Prepared month {t+1}/{len(months)}',flush=True)
    del daily
    groups=[]
    # A representation is enabled only after its fixed historical sample is
    # available, identically with full inputs and future-truncated inputs.
    if len(months) and months[-1]>=AFFINE_ANCHOR:
        cutoff=int(np.datetime64('1983-11-30').astype('datetime64[D]').astype(np.int64))
        with threadpool_limits(limits=1):
            for t,r in enumerate(risk):
                if months[t]>cutoff:break
                if r is None:continue
                rows=r['rows']
                Z=pd.DataFrame(X[rows]).rank(pct=True).fillna(.5).to_numpy(np.float32)-.5
                Y=(ret[rows]/r['sigma']).astype(np.float32)
                P=np.linalg.solve(Z.astype(float).T@Z+.001*len(Z)*np.eye(Z.shape[1]),
                                  Z.T.astype(float)@Y).astype(np.float32)
                groups.append(dict(day=int(months[t]),X=Z,Y=Y,P=P))
    affine=None
    if len(groups)>=384:
        affine,epochs=learn_affine(groups,N_THREADS)
        print(f'Affine representation: {len(groups)} historical months, {epochs} epochs',flush=True)
    del groups
    z=np.full((len(X),3),np.nan);label=np.full_like(z,np.nan)
    with threadpool_limits(limits=1),torch.no_grad():
        for t,r in enumerate(risk):
            if r is None:continue
            rows=r['rows'];sigma=r['sigma'];psi=r['psi']
            if affine is not None:
                Z=pd.DataFrame(X[rows]).rank(pct=True).fillna(.5).to_numpy(np.float32)-.5
                beta=affine.beta(torch.from_numpy(Z)).numpy().astype(float)
                g=np.column_stack([r['B'],1/sigma,beta])
                r['G'].append(g);r['U'].append(projector_basis(g,psi))
            # These are training labels only. fit_model enforces formation <
            # the December refit date, so each label is known by that date.
            x=ret[rows]/sigma/np.sqrt(psi)
            for j,U in enumerate(r['U']):
                value=(residualize(ret[rows]/sigma,r['G'][j],psi)/np.sqrt(psi)
                       if j==1 else x-U@(U.T@x))
                z[rows,j]=value;label[rows,j]=np.clip(value/value.std(),-4,4)
    return dict(X=X,peer=XP,ret=ret,ids=ids,months=months,month_ix=month_ix,
                starts=starts,risk=risk,z=z,label=label,output=output,affine=affine)


def fit_window(data,refit_day,book):
    month_ix=data['month_ix'];months=data['months']
    rows=np.flatnonzero((months[month_ix]<refit_day)&np.isfinite(data['label'][:,book]))
    history=np.unique(month_ix[rows])
    if len(history)<36:return None
    nv=min(60,len(history)//2);cut=int(history[-nv])
    tr=rows[month_ix[rows]<cut];va=rows[month_ix[rows]>=cut]
    return dict(rows=rows,tr=tr,va=va,history=history,nv=nv,cut=cut)


def learner_params(min_leaf,threads):
    return dict(objective='regression',learning_rate=.05,num_leaves=31,
                min_data_in_leaf=int(min_leaf),seed=42,num_threads=threads,
                deterministic=True,force_row_wise=True,verbose=-1,metric='None')


def validation_metric(data,window,book):
    va=window['va'];codes=data['month_ix'][va]-window['cut']
    z=data['z'][va,book]
    if book==1:return book_sharpe_metric(codes,z,window['nv'])
    groups=[]
    for t in np.unique(data['month_ix'][va]):
        idx=np.flatnonzero(data['month_ix'][va]==t)
        r=data['risk'][t]
        np.testing.assert_array_equal(va[idx],r['rows'])
        groups.append((idx,r['U'][book]))
    def metric(pred,_):
        rs=[]
        for idx,U in groups:
            score=pred[idx];p=score-U@(U.T@score);norm=np.linalg.norm(p)
            rs.append(float(score@z[idx]/norm) if norm>1e-14 else 0.)
        values=np.asarray(rs);sd=values.std(ddof=1)
        return 'book_sharpe',float(values.mean()/sd) if sd>0 else 0.,True
    return metric


def fit_model(data,window,book,kind):
    X=data['X'] if kind=='raw' else data['peer']
    tr=window['tr'];va=window['va'];rows=window['rows']
    mi=data['month_ix'];hist=window['history']
    leaf=int(np.median(np.bincount(mi[tr])[hist[hist<window['cut']]]))
    with threadpool_limits(limits=N_THREADS):
        train=lgb.Dataset(X[tr],label=data['label'][tr,book])
        valid=lgb.Dataset(X[va],label=data['label'][va,book],reference=train)
        selection=lgb.train(learner_params(leaf,N_THREADS),train,num_boost_round=2000,
            valid_sets=[valid],feval=validation_metric(data,window,book),
            callbacks=[lgb.early_stopping(200,first_metric_only=True,verbose=False)])
        rounds=max(int(selection.best_iteration),1)
        leaf=int(np.median(np.bincount(mi[rows])[hist]))
        final=lgb.train(learner_params(leaf,N_THREADS),
            lgb.Dataset(X[rows],label=data['label'][rows,book]),num_boost_round=rounds)
    print(f'Fitted {kind}/{BOOK_NAMES[book]}: {len(rows)} rows, {rounds} rounds',flush=True)
    return final


def normalize(w,psi,sigma):
    norm=np.linalg.norm(w*sigma*np.sqrt(psi))
    if not np.isfinite(norm) or norm<=1e-14:raise ValueError('No residual signal')
    return w/norm


def compact_fit_inputs(data,window,book,kind):
    """Share large arrays via joblib's permitted temporary memmaps, not copies.

    Only the validation month's projection bases belong in each worker payload;
    neither all risk matrices nor the affine network is needed for tree fitting.
    """
    risk=[None]*len(data['risk'])
    for t in np.unique(data['month_ix'][window['va']]):
        r=data['risk'][t];U=[None]*3;U[book]=r['U'][book]
        risk[t]=dict(rows=r['rows'],U=U)
    X=data['X'] if kind=='raw' else data['peer']
    return dict(X=X,peer=X,label=data['label'],z=data['z'],
                month_ix=data['month_ix'],risk=risk)


def main(chars:pd.DataFrame,features:pd.DataFrame,daily_ret:pd.DataFrame)->pd.DataFrame:
    np.random.seed(SEED);torch.manual_seed(SEED)
    data=prepare_inputs(chars,features,daily_ret)
    months=data['months'];mi=data['month_ix'];requested=np.unique(mi[data['output']])
    years=pd.to_datetime(months[requested],unit='D').year.to_numpy()
    years=years+(pd.to_datetime(months[requested],unit='D').month.to_numpy()==12)
    models={};frames=[]
    for year in np.unique(years):
        refit=int(np.datetime64(f'{int(year)-1}-12-31').astype('datetime64[D]').astype(np.int64))
        active=[0,1]+([2] if data['affine'] is not None and refit>=AFFINE_ANCHOR else [])
        windows={j:fit_window(data,refit,j) for j in active}
        if any(v is None for v in windows.values()):continue
        print(f'Annual refit for return year {year}',flush=True)
        tasks=[(kind,j) for kind in ('raw','peer') for j in active]
        fitted=Parallel(n_jobs=N_JOBS,backend='loky',max_nbytes='1M')(
            delayed(fit_model)(compact_fit_inputs(data,windows[j],j,kind),windows[j],j,kind)
            for kind,j in tasks)
        models=dict(zip(tasks,fitted))
        for t in requested[years==year]:
            r=data['risk'][t]
            if r is None:continue
            rows=r['rows'];psi=r['psi'];sigma=r['sigma'];W=[]
            for kind in ('raw','peer'):
                X=data['X'] if kind=='raw' else data['peer']
                books=[]
                for j in active:
                    s=models[kind,j].predict(X[rows],num_threads=N_THREADS)
                    U=r['U'][j];p=s-U@(U.T@s)
                    books.append(p/np.linalg.norm(p)/np.sqrt(psi)/sigma)
                W.append(normalize(np.mean(books,axis=0),psi,sigma))
            w=normalize(.5*W[0]+.5*W[1],psi,sigma)*(.1/np.sqrt(12))
            selected=data['output'][rows]
            frames.append(pd.DataFrame(dict(id=data['ids'][rows[selected]],
                eom=str(np.datetime64(int(months[t]),'D')),w=w[selected])))
    if not frames:raise ValueError('No trainable output month with sufficient history')
    result=pd.concat(frames,ignore_index=True)[['id','eom','w']]
    result.id=result.id.astype(np.int64)
    if result.empty or not np.isfinite(result.w).all():raise ValueError('Invalid portfolio')
    # Missing eligible trading histories are represented explicitly by zeros.
    keys=pd.DataFrame(dict(id=data['ids'][data['output']],
        eom=pd.to_datetime(months[mi[data['output']]],unit='D').strftime('%Y-%m-%d')))
    result=keys.merge(result,on=['id','eom'],how='left',validate='one_to_one')
    result['w']=result.w.fillna(0.)
    return result.sort_values(['id','eom'],kind='stable').reset_index(drop=True)


