"""V4 train-only CV, true optimizer-state continuation and controlled batch study."""
from __future__ import annotations

import argparse
from collections import Counter
import copy
import json
import math
from pathlib import Path
import random
import time

import numpy as np
import torch
from torch import nn

from .pose_benchmark import EndToEnd, seed_all
from .pose_benchmark_models import Backbone, PoseHead, PRETRAINED
from .pose_benchmark_v4_data import ROOT, PROJECT, CONFIG, config, load
from .pose_dataset import read_rows
from .cluster_targets import digest
from .pose_model import group_folds, angular_error_degrees
from .pose_refinement import atomic_json, loss_value, decoded, now


def per_error(pred,target):
    position=np.linalg.norm(pred[:,:2]-target[:,:2],axis=1)
    angles=angular_error_degrees(np.degrees(np.arctan2(pred[:,3],pred[:,2])),
                                 np.degrees(np.arctan2(target[:,3],target[:,2])))
    return position,angles


def metrics(pred,target,rows,all_rows):
    position,angles=per_error(pred,target)
    bykey={r['target_key']:float(p/.05+a/5) for r,p,a in zip(rows,position,angles)}
    groups=sorted({r['group_id'] for r in all_rows})
    score=np.mean([np.mean([bykey.get(r['target_key'],config()['preprocessing_failure_score'])
                            for r in all_rows if r['group_id']==group]) for group in groups])
    return dict(samples=len(rows),total=len(all_rows),groups=len(groups),failed_preprocessing=len(all_rows)-len(rows),
                macro_group_selection_score=float(score),position_width_fraction=float(position.mean()),
                angle_mae_degrees=float(angles.mean()))


def trial_root(family,branch,seed=42,arm='base'):
    return ROOT/'trials'/arm/family/branch/str(seed)


def freeze_protocol():
    cfg=config();manifest=json.loads((ROOT/'dataset/manifest.json').read_text())
    assets=PROJECT/cfg['official_assets']
    upstream=json.loads((assets/'upstream_manifest.json').read_text())
    import subprocess
    for row in upstream:
        if subprocess.check_output(['git','rev-parse','HEAD'],cwd=PROJECT/row['path'],text=True).strip()!=row['commit']:
            raise ValueError('Official commit changed')
        if subprocess.check_output(['git','diff','--name-only'],cwd=PROJECT/row['path'],text=True).strip():
            raise ValueError('Official implementation changed')
    weights=json.loads((assets/'weights_manifest.json').read_text())
    for row in weights:
        if digest(assets/'weights'/row['name'])!=row['sha256']:raise ValueError('Official weights changed')
    result=dict(created_at=now(),config=cfg,config_sha256=digest(CONFIG),
                dataset_manifest_sha256=digest(ROOT/'dataset/manifest.json'),
                labels_sha256=manifest['labels_sha256'],upstream=upstream,weights=weights,branches={})
    for branch in ('metric','native_unknown'):
        rows,arrays,all_rows=load(branch)
        groups={r['group_id'] for r in rows};folds=min(cfg['folds'],len(groups))
        if folds<2:raise ValueError('Insufficient independent groups: '+branch)
        details=[]
        for ti,vi in group_folds(rows,folds,cfg['fold_seed']):
            vg={rows[i]['group_id'] for i in vi}
            details.append(dict(fit_keys=[rows[i]['target_key'] for i in ti],validation_keys=[rows[i]['target_key'] for i in vi],
                                validation_groups=sorted(vg),fit_groups=sorted({rows[i]['group_id'] for i in ti})))
        result['branches'][branch]=dict(train_samples=len(rows),full_train_samples=len(all_rows),train_groups=len(groups),
            folds=details,unit='m' if branch=='metric' else 'unverified_native',
            warning='two_group_leave_one_out_exploratory_only' if folds<3 else None)
    path=ROOT/'protocol.json'
    if path.exists():
        previous=json.loads(path.read_text());result['created_at']=previous['created_at']
        if previous!=result:raise ValueError('V4 protocol is already frozen')
    atomic_json(path,result)
    return result


def embeddings(family,branch,split,rows,arrays):
    path=ROOT/'embeddings'/family/(branch+'_'+split+'.npz')
    if path.exists():
        meta=json.loads(path.with_suffix('.json').read_text())
        if digest(path)!=meta['sha256']:raise ValueError('Embedding hash changed')
        with np.load(path,allow_pickle=False) as z:
            if z['keys'].tolist()!=[r['target_key'] for r in rows]:raise ValueError('Embedding cohort changed')
            return z['x'].copy()
    seed_all(42);model=Backbone(family).cuda().eval();cfg=config();parts=[];start=time.monotonic()
    with torch.no_grad():
        for offset in range(0,len(rows),cfg['embedding_batch']):
            stop=offset+cfg['embedding_batch'];views=[]
            for variant in range(3):
                f=torch.from_numpy(arrays['f'][offset:stop,variant]).cuda()
                value=model(f).cpu().numpy()
                views.append(np.concatenate((value,arrays['g'][offset:stop,variant]),-1))
            parts.append(np.stack(views,1))
            if offset%128==0:print(json.dumps(dict(stage='embedding',family=family,branch=branch,split=split,offset=offset,total=len(rows))),flush=True)
    x=np.concatenate(parts)
    if not np.isfinite(x).all():raise ValueError('Nonfinite official encoder output')
    path.parent.mkdir(parents=True,exist_ok=True);np.savez_compressed(path,x=x,keys=[r['target_key'] for r in rows])
    atomic_json(path.with_suffix('.json'),dict(sha256=digest(path),seconds=time.monotonic()-start,
                frozen_official_encoder=True,labels_not_used=True))
    del model;torch.cuda.empty_cache()
    return x


def make_model(family,arrays,ti,x=None):
    if x is not None:return PoseHead(x.shape[-1],x[ti,0].mean(0),x[ti,0].std(0)).cuda()
    return EndToEnd(family,arrays['g'][ti,0],tune=False).cuda()


def optimizer_for(model,frozen):
    cfg=config()
    params=model.parameters() if frozen else [
        dict(params=model.backbone.parameters(),lr=cfg['backbone_learning_rate']),
        dict(params=model.head.parameters(),lr=cfg['head_learning_rate'])]
    return torch.optim.AdamW(params,lr=cfg['head_learning_rate'],weight_decay=cfg['weight_decay'])


def cpu_model(model):return {k:v.detach().cpu().clone() for k,v in model.state_dict().items()}


def checkpoint(path,value):
    path.parent.mkdir(parents=True,exist_ok=True);temporary=path.with_suffix('.tmp')
    torch.save(value,temporary);temporary.replace(path)


def predict(model,arrays,indices,x=None,batch=None):
    batch=batch or config()['evaluation_batch'];model.eval();output=[]
    with torch.no_grad():
        for offset in range(0,len(indices),batch):
            ix=indices[offset:offset+batch]
            if x is not None:p=model(torch.from_numpy(x[ix,0]).cuda())
            else:p=model(torch.from_numpy(arrays['f'][ix,0]).cuda(),torch.from_numpy(arrays['g'][ix,0]).cuda())
            output.append(p.cpu().numpy())
    result=np.concatenate(output)
    if not np.isfinite(result).all():raise ValueError('Nonfinite prediction')
    return result


def train(family,branch,seed,arrays,rows,all_rows,ti,vi,path,*,arm='base',x=None,
          resume=None,checkpoint_epochs=()):
    cfg=config();path.mkdir(parents=True,exist_ok=True);ti=np.asarray(ti,dtype=int);vi=np.asarray(vi,dtype=int)
    if any(rows[i]['split']!='train' for i in np.r_[ti,vi]):raise ValueError('Holdout used for fit/selection')
    tg={rows[i]['group_id'] for i in ti};vg={rows[i]['group_id'] for i in vi}
    if tg&vg:raise ValueError('CV group leakage')
    recipe=next((r for r in cfg['continuation'] if r['name']==arm),None)
    batch=recipe['batch'] if recipe else cfg['base_batch']
    epochs=recipe['total_epochs'] if recipe else cfg['base_epochs']
    spec=dict(family=family,branch=branch,seed=seed,arm=arm,batch=batch,total_epochs=epochs,
        fit_keys=[rows[i]['target_key'] for i in ti],validation_keys=[rows[i]['target_key'] for i in vi],
        protocol_sha256=digest(ROOT/'protocol.json'),frozen_encoder=x is not None,
        resume_parent=str(resume) if resume else None,
        resume_parent_sha256=digest(resume/'last.pt') if resume else None,
        gpu=str(torch.cuda.get_device_properties(0)),checkpoint_epochs=list(checkpoint_epochs))
    if (path/'specification.json').exists():
        previous=json.loads((path/'specification.json').read_text())
        for k in spec:
            if k!='gpu' and spec[k]!=previous[k]:raise ValueError('Trial specification changed: '+k)
    atomic_json(path/'specification.json',spec)
    if (path/'completed.json').exists():return json.loads((path/'completed.json').read_text())
    seed_all(seed);rng=np.random.default_rng(seed)
    model=make_model(family,arrays,ti,x);optimizer=optimizer_for(model,x is not None)
    start_epoch=0;updates=0;history=[];best=float('inf');best_epoch=0;best_state=None
    resume_path=path if (path/'last.pt').exists() else resume
    if resume_path:
        last=torch.load(resume_path/'last.pt',map_location='cuda',weights_only=False)
        parent_spec=last['specification']
        for field in ('fit_keys','validation_keys','family','branch','seed','frozen_encoder','protocol_sha256'):
            if spec[field]!=parent_spec[field]:raise ValueError('Resume cohort/architecture mismatch: '+field)
        if resume_path==resume and last['epoch']!=cfg['base_epochs']:raise ValueError('Continuation must start at epoch80 last, not best')
        model.load_state_dict(last['model'],strict=True);optimizer.load_state_dict(last['optimizer'])
        rng.bit_generator.state=last['numpy_rng'];torch.set_rng_state(last['torch_rng'].cpu())
        torch.cuda.set_rng_state(last['cuda_rng'].cpu())
        start_epoch=last['epoch'];updates=last['optimizer_steps'];history=last['history']
        selected=torch.load(resume_path/'selected.pt',map_location='cpu',weights_only=True)
        best_state=selected['model'];best_epoch=selected['selected_epoch'];best=selected['selection_score']
        if resume_path==resume:
            atomic_json(path/'continuation_binding.json',dict(parent_last_sha256=digest(resume/'last.pt'),
                parent_selected_sha256=digest(resume/'selected.pt'),start_epoch=start_epoch,start_optimizer_steps=updates,
                optimizer_and_rng_restored=True,parent_batch=parent_spec['batch'],new_batch=batch))
    # Keep the small training tensors on GPU; no disk decoding in minibatch loop.
    data_gpu={k:torch.from_numpy(arrays[k]).cuda() for k in ('f','g','y')} if x is None else {
        'x':torch.from_numpy(x).cuda(),'y':torch.from_numpy(arrays['y']).cuda()}
    start=time.monotonic();validation_all=[r for r in all_rows if r['group_id'] in vg]
    export_epochs=set(checkpoint_epochs)
    if resume and not len(vi):
        for epoch in sorted(e for e in export_epochs if e<=start_epoch):
            destination=path/'epochs'/f'{epoch}.pt'
            if epoch>cfg['base_epochs']:
                if not destination.exists():raise ValueError('Own resumed refit epoch missing: '+str(epoch))
                continue
            source=resume/'epochs'/f'{epoch}.pt'
            if not source.exists():raise ValueError('Required refit epoch missing: '+str(epoch))
            destination.parent.mkdir(exist_ok=True)
            destination.write_bytes(source.read_bytes())
    for epoch in range(start_epoch,epochs):
        model.train();order=rng.permutation(ti);losses=[]
        for ix in np.array_split(order,max(1,math.ceil(len(order)/batch))):
            variants=rng.integers(0,3,len(ix))
            if x is not None:prediction=model(data_gpu['x'][ix,variants])
            else:
                f=data_gpu['f'][ix,variants].clone();f[...,:3]+=torch.randn_like(f[...,:3])*.003
                prediction=model(f,data_gpu['g'][ix,variants])
            loss=loss_value(prediction,data_gpu['y'][ix])
            if not torch.isfinite(loss):raise ValueError('Nonfinite training loss')
            optimizer.zero_grad(set_to_none=True);loss.backward();nn.utils.clip_grad_norm_(model.parameters(),5);optimizer.step()
            updates+=1;losses.append(float(loss.detach()))
        item=dict(epoch=epoch+1,train_loss=float(np.mean(losses)),batch=batch,optimizer_steps=updates)
        if len(vi):
            output=predict(model,arrays,vi,x)
            item['validation']=metrics(output,arrays['y'][vi],[rows[i] for i in vi],validation_all)
            score=item['validation']['macro_group_selection_score']
            if score<best:
                best,best_epoch=score,epoch+1;best_state=cpu_model(model)
        else:
            best_epoch=epoch+1;best_state=cpu_model(model);best=0.
        history.append(item)
        selected=dict(model=best_state,family=family,frozen_encoder=x is not None,tune=False,
                      selected_epoch=best_epoch,selection_score=best)
        if epoch+1 in export_epochs:
            checkpoint(path/'epochs'/f'{epoch+1}.pt',dict(model=cpu_model(model),family=family,
                frozen_encoder=x is not None,tune=False,selected_epoch=epoch+1,selection_score=None))
        if (epoch+1)%10==0 or epoch+1==epochs:
            checkpoint(path/'selected.pt',selected)
            checkpoint(path/'last.pt',dict(model=cpu_model(model),optimizer=optimizer.state_dict(),epoch=epoch+1,
                optimizer_steps=updates,history=history,numpy_rng=rng.bit_generator.state,
                torch_rng=torch.get_rng_state(),cuda_rng=torch.cuda.get_rng_state(),specification=spec))
            atomic_json(path/'history.json',history)
            atomic_json(path/'progress.json',dict(epoch=epoch+1,total_epochs=epochs,seconds_this_run=time.monotonic()-start,
                                                best_epoch=best_epoch,optimizer_steps=updates,latest=item))
        if epoch==start_epoch or (epoch+1)%40==0:
            print(json.dumps(dict(stage=str(path),**item)),flush=True)
    if best_state is None:raise ValueError('No completed optimization epoch')
    model.load_state_dict(best_state,strict=True)
    prediction=predict(model,arrays,vi if len(vi) else ti,x)
    np.save(path/'predictions.npy',prediction)
    completed=dict(selected_epoch=best_epoch,last_epoch=epochs,optimizer_steps=updates,
        selected_sha256=digest(path/'selected.pt'),last_sha256=digest(path/'last.pt'),
        seconds_this_run=time.monotonic()-start,parameters=sum(p.numel() for p in model.parameters()),
        trainable_parameters=sum(p.numel() for p in model.parameters() if p.requires_grad))
    atomic_json(path/'completed.json',completed)
    del optimizer,model,data_gpu;torch.cuda.empty_cache()
    return completed


def trial(family,branch,seed=42,arm='base'):
    path=trial_root(family,branch,seed,arm)
    if (path/'completed.json').exists():return json.loads((path/'completed.json').read_text())
    protocol=json.loads((ROOT/'protocol.json').read_text());rows,arrays,all_rows=load(branch)
    x=embeddings(family,branch,'train',rows,arrays) if family in PRETRAINED else None
    prediction=np.zeros_like(arrays['y']);details=[];bykey={r['target_key']:i for i,r in enumerate(rows)}
    for fold,spec in enumerate(protocol['branches'][branch]['folds']):
        ti=[bykey[k] for k in spec['fit_keys']];vi=[bykey[k] for k in spec['validation_keys']]
        parent=trial_root(family,branch,seed)/f'fold_{fold}' if arm!='base' else None
        detail=train(family,branch,seed,arrays,rows,all_rows,ti,vi,path/f'fold_{fold}',arm=arm,x=x,resume=parent)
        prediction[vi]=np.load(path/f'fold_{fold}/predictions.npy');details.append(detail)
    path.mkdir(parents=True,exist_ok=True)
    np.savez_compressed(path/'out_of_fold.npz',predictions=prediction,targets=arrays['y'],keys=[r['target_key'] for r in rows])
    record=dict(family=family,branch=branch,seed=seed,arm=arm,metrics=metrics(prediction,arrays['y'],rows,all_rows),folds=details)
    atomic_json(path/'completed.json',record);print(json.dumps(record),flush=True)
    return record


def shortlist():
    branch=config()['primary_selection_branch'];ranking=[]
    for family in config()['families']:
        value=json.loads((trial_root(family,branch)/'completed.json').read_text());ranking.append(value)
    ranking.sort(key=lambda r:r['metrics']['macro_group_selection_score'])
    result=dict(created_at=now(),branch=branch,ranking=ranking,top_two=[r['family'] for r in ranking[:2]],
                holdout_used=False,unknown_branch_exploratory_not_used_for_selection=True)
    path=ROOT/'shortlist.json'
    if path.exists():
        previous=json.loads(path.read_text());result['created_at']=previous['created_at']
        if result!=previous:raise ValueError('Shortlist changed')
    atomic_json(path,result);return result


def interval(predictions,targets,rows,other_predictions=None):
    baseline=np.zeros_like(targets);baseline[:,2]=1
    def score(p):
        pos,angle=per_error(p,targets);return pos/.05+angle/5
    delta=np.mean([score(p) for p in predictions],0)-(
        np.mean([score(p) for p in other_predictions],0) if other_predictions is not None else score(baseline))
    values=np.array([delta[[r['group_id']==g for r in rows]].mean() for g in sorted({r['group_id'] for r in rows})])
    rng=np.random.default_rng(4242);boot=values[rng.integers(0,len(values),(10000,len(values)))].mean(1)
    return dict(mean=float(values.mean()),lower=float(np.quantile(boot,.025)),upper=float(np.quantile(boot,.975)),
                groups=len(values),descriptive_selected_group_bootstrap_not_independent_significance=True)


def freeze_selection():
    cfg=config();top=json.loads((ROOT/'shortlist.json').read_text())['top_two'];result={}
    for branch in json.loads((ROOT/'protocol.json').read_text())['branches']:
        rows,arrays,all_rows=load(branch);entries=[]
        for family in cfg['families']:
            arms=['base']+([r['name'] for r in cfg['continuation']] if family in top else [])
            for arm in arms:
                seeds=cfg['seeds'] if family in top else [42];records=[];predictions=[];base_predictions=[]
                for seed in seeds:
                    path=trial_root(family,branch,seed,arm);records.append(json.loads((path/'completed.json').read_text()))
                    with np.load(path/'out_of_fold.npz') as z:
                        if z['keys'].tolist()!=[r['target_key'] for r in rows]:raise ValueError('OOF keys changed')
                        predictions.append(z['predictions'].copy())
                    with np.load(trial_root(family,branch,seed)/'out_of_fold.npz') as z:base_predictions.append(z['predictions'].copy())
                scores=[r['metrics']['macro_group_selection_score'] for r in records]
                entry=dict(family=family,arm=arm,seeds=seeds,mean_score=float(np.mean(scores)),std_score=float(np.std(scores)),
                    seed_scores=scores,position_mean_width_fraction=float(np.mean([r['metrics']['position_width_fraction'] for r in records])),
                    angle_mean=float(np.mean([r['metrics']['angle_mae_degrees'] for r in records])),
                    selected_epochs=[f['selected_epoch'] for r in records for f in r['folds']],
                    refit_epoch=max(1,int(np.median([f['selected_epoch'] for r in records for f in r['folds']]))),
                    delta_vs_ROI_geometry=interval(predictions,arrays['y'],rows),
                    delta_vs_same_seed_base=interval(predictions,arrays['y'],rows,base_predictions))
                entries.append(entry)
        # Only three-seed candidates can win final selection; other baseline
        # families remain documented screening/refit results, not mixed seeds.
        stable=[e for e in entries if e['family'] in top]
        best=min(stable,key=lambda r:r['mean_score'])
        result[branch]=dict(best_learning=best,entries=entries,three_seed_selection=True,
            recommendation='learned_candidate_needs_independent_validation' if branch=='metric' and best['delta_vs_ROI_geometry']['upper']<0 else 'retain_geometry',
            limitation='two_independent_groups_only_not_deployment_evidence' if branch=='native_unknown' else None)
    document=dict(created_at=now(),protocol_sha256=digest(ROOT/'protocol.json'),top_two=top,profiles=result,
                  val_test_used=False,previous_test_seen_not_blind=True)
    path=ROOT/'selection_frozen.json'
    if path.exists():
        previous=json.loads(path.read_text());document['created_at']=previous['created_at']
        if previous!=document:raise ValueError('Frozen selection changed')
    atomic_json(path,document);return document


if __name__=='__main__':
    parser=argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--stage',choices=['freeze','trial','shortlist','select'],required=True)
    parser.add_argument('--family');parser.add_argument('--branch',choices=['metric','native_unknown'],default='metric')
    parser.add_argument('--seed',type=int,default=42);parser.add_argument('--arm',default='base')
    args=parser.parse_args()
    if args.stage=='freeze':freeze_protocol()
    elif args.stage=='shortlist':shortlist()
    elif args.stage=='select':freeze_selection()
    else:trial(args.family,args.branch,args.seed,args.arm)
