"""Train-only group CV and frozen final partial-pose regression models.

The two-frame original validation set is diagnostic only. No original test
targets or unassigned/quarantined samples are read for fitting/selection.
"""
from __future__ import annotations

import argparse
from collections import Counter
import fcntl
import io
import json
import os
from pathlib import Path
import random
import time

import numpy as np
import torch

from .cluster_targets import digest
from .devices import pinned_cuda_metadata
from .pose_dataset import PROJECT, protected_write, read_rows, safe_path, write_json
from .pose_model import (TASKS, PointNetResidual, RidgeResidual, base_prediction,
    box_array, group_folds, grouped_metrics, partial_loss, roi_features, target_array)


def atomic_json(path,value):
    path=Path(path);path.parent.mkdir(parents=True,exist_ok=True)
    temporary=path.with_name(path.name+'.tmp')
    temporary.write_text(json.dumps(value,ensure_ascii=False,indent=2,allow_nan=False)+'\n')
    temporary.replace(path)


def atomic_torch(path,value):
    path=Path(path);path.parent.mkdir(parents=True,exist_ok=True)
    temporary=path.with_name(path.name+'.tmp')
    torch.save(value,temporary);temporary.replace(path)


def load_index(root,task,split):
    path=root/f'indices/overlap/{task}/{split}.jsonl'
    rows=read_rows(path)
    for r in rows:
        if r['split']!=split or r['subset']!='overlap' or r['pose']['kind']!=task:
            raise ValueError('Ineligible or wrong-split pose index record')
        if r['formal_instances']!=1 or r['pose']['issues']:
            raise ValueError('Ambiguous or conflicted pose supervision')
        target_array(r,task)
    if len({r['sample_id'] for r in rows})!=len(rows):raise ValueError('Duplicate pose identity')
    return rows


def prepare_samples(root,rows,config,cache,files,*,augment):
    items=[]
    for r in rows:
        path=safe_path(root,r['points'])
        if digest(path)!=files[r['points']]:raise ValueError('Frozen pose tensor changed')
        cache_path=cache/(r['sample_id']+'.npz')
        target=target_array(r,r['pose']['kind'])
        if cache_path.exists():
            with np.load(cache_path,allow_pickle=False) as z:
                fs,gs,ys=z['features'],z['globals'],z['target']
                if not np.array_equal(ys,target):raise ValueError('Cache target differs')
            metadata=json.loads(cache_path.with_suffix('.json').read_text())
        else:
            with np.load(path,allow_pickle=False) as z:
                points,mapping=z['points_xyzi'],z['pixel_to_source_index']
            width,height=r['native_size'];box=box_array(r['box_xywh'])
            fs,gs,metadata=[],[],[]
            variants=5 if augment else 1
            rng=np.random.default_rng(int(r['sample_id'][:8],16))
            for i in range(variants):
                chosen=box.copy()
                if i:
                    jitter=config['crop_jitter']
                    chosen[:2]+=rng.uniform(-jitter,jitter,2)*box[2:]
                    chosen[2:]*=rng.uniform(1-jitter,1+jitter,2)
                try:
                    f,g,meta=roi_features(points,mapping,chosen,width,height,config,
                        scale_to_m=r['pointcloud_scale_to_m'],seed=42+i)
                except ValueError:
                    if not i:raise
                    f,g,meta=fs[0],gs[0],dict(metadata[0],jitter_fallback_to_original=True)
                fs.append(f);gs.append(g);metadata.append(meta)
            fs,gs=np.stack(fs),np.stack(gs)
            buffer=io.BytesIO();np.savez_compressed(buffer,features=fs,globals=gs,target=target)
            protected_write(cache_path,buffer.getvalue());write_json(cache_path.with_suffix('.json'),metadata)
        items.append({'row':r,'features':fs,'globals':gs,'target':target,'metadata':metadata})
    return items


def predict_net(model,items,device,batch=16):
    if not items:return np.empty((0,4 if model.task==TASKS[0] else 6))
    model.eval();values=[]
    with torch.inference_mode():
        for start in range(0,len(items),batch):
            selected=items[start:start+batch]
            f=torch.from_numpy(np.stack([r['features'][0] for r in selected])).to(device)
            g=torch.from_numpy(np.stack([r['globals'][0] for r in selected])).to(device)
            values.append(model(f,g).cpu().numpy())
    return np.concatenate(values)


def load_net(path,device='cpu'):
    state=torch.load(path,map_location='cpu',weights_only=False)
    model=PointNetResidual(state['task'])
    model.load_state_dict(state['model']);model.to(device).eval()
    return model,state


def train_net(train_items,val_items,task,config,output,device,seed,epochs):
    output.mkdir(parents=True,exist_ok=True)
    torch.manual_seed(seed);np.random.seed(seed);random.seed(seed)
    if str(device).startswith('cuda'):torch.cuda.manual_seed_all(seed)
    rng=np.random.default_rng(seed)
    gtrain=np.stack([r['globals'][0] for r in train_items])
    model=PointNetResidual(task,gtrain.mean(axis=0),gtrain.std(axis=0)).to(device)
    optimizer=torch.optim.AdamW(model.parameters(),lr=config['lr'],weight_decay=.001)
    train_ids=[r['row']['sample_id'] for r in train_items]
    val_ids=[r['row']['sample_id'] for r in val_items]
    if set(train_ids)&set(val_ids):raise ValueError('Pose fit/selection IDs overlap')
    if {r['row']['group_id'] for r in train_items}&{r['row']['group_id'] for r in val_items}:
        raise ValueError('Pose fit/selection groups overlap')
    specification={'task':task,'train_ids':train_ids,'validation_ids':val_ids,
        'epochs':epochs,'seed':seed,'config':config,
        'source_hashes':{'training':digest(__file__),'model':digest(Path(__file__).with_name('pose_model.py'))}}
    write_json(output/'specification.json',specification)
    start,best_epoch,best_score,history=0,0,None,[]
    last=output/'last.pt';best_path=output/'best.pt'
    complete=output/'completed.json'
    if complete.exists():
        done=json.loads(complete.read_text());path=Path(done['checkpoint'])
        if digest(path)!=done['checkpoint_sha256']:raise ValueError('Completed pose checkpoint changed')
        model,_=load_net(path,device);return model,done
    if last.exists():
        state=torch.load(last,map_location='cpu',weights_only=False)
        if state['specification']!=specification:raise ValueError('Resume specification mismatch')
        model.load_state_dict(state['model']);optimizer.load_state_dict(state['optimizer'])
        for value in optimizer.state.values():
            for k,v in value.items():
                if isinstance(v,torch.Tensor):value[k]=v.to(device)
        start=state['epoch'];best_epoch=state['best_epoch'];best_score=state['best_score'];history=state['history']
        torch.set_rng_state(state['torch_rng']);rng.bit_generator.state=state['numpy_rng']
        if state['cuda_rng'] is not None:torch.cuda.set_rng_state_all(state['cuda_rng'])
    yval=np.stack([r['target'] for r in val_items]) if val_items else None
    started=time.monotonic()
    for epoch in range(start,epochs):
        began=time.monotonic();model.train();loss_sum=0.;seen=0
        lr=config['lr']*(.01+.99*(1+np.cos(np.pi*epoch/max(epochs-1,1)))/2)
        for group in optimizer.param_groups:group['lr']=lr
        order=rng.permutation(len(train_items))
        for offset in range(0,len(order),config['batch']):
            samples=[train_items[i] for i in order[offset:offset+config['batch']]]
            variants=[int(rng.integers(len(r['features']))) for r in samples]
            f=torch.from_numpy(np.stack([r['features'][j] for r,j in zip(samples,variants)])).to(device)
            g=torch.from_numpy(np.stack([r['globals'][j] for r,j in zip(samples,variants)])).to(device)
            y=torch.from_numpy(np.stack([r['target'] for r in samples])).to(device)
            # Sensor-noise augmentation only; no spatial rotations with stale GT.
            f=f.clone();f[:,:,:3]+=torch.randn_like(f[:,:,:3])*.005
            pred=model(f,g);loss=partial_loss(pred,y,task)
            if not torch.isfinite(loss):raise ValueError('Nonfinite partial-pose loss')
            optimizer.zero_grad(set_to_none=True);loss.backward()
            torch.nn.utils.clip_grad_norm_(model.parameters(),5.);optimizer.step()
            loss_sum+=float(loss.detach())*len(samples);seen+=len(samples)
        row={'epoch':epoch+1,'train_loss':loss_sum/seen,'lr':lr,'seconds':time.monotonic()-began}
        improved=False
        if val_items and ((epoch+1)%5==0 or epoch==0):
            pred=predict_net(model,val_items,device,config['batch'])
            metrics=grouped_metrics(pred,yval,[r['row'] for r in val_items],task)
            score=metrics['overall']['macro_group_selection_score']
            row['cv_validation']=metrics['overall']
            if best_score is None or score<best_score:
                best_score,best_epoch,improved=score,epoch+1,True
        history.append(row)
        state={'model':model.state_dict(),'optimizer':optimizer.state_dict(),'task':task,'epoch':epoch+1,
            'best_epoch':best_epoch,'best_score':best_score,'specification':specification,'history':history,
            'torch_rng':torch.get_rng_state(),'numpy_rng':rng.bit_generator.state,
            'cuda_rng':torch.cuda.get_rng_state_all() if str(device).startswith('cuda') else None}
        atomic_torch(last,state)
        if improved:atomic_torch(best_path,state)
        atomic_json(output/'history.json',history)
        if (epoch+1)%25==0 or epoch==0:
            print(json.dumps({'stage':str(output.name),'task':task,**row},ensure_ascii=False),flush=True)
    chosen=best_path if val_items else last
    final=output/'final.pt'
    protected_write(final,chosen.read_bytes())
    model,state=load_net(final,device)
    result={'state':'complete','task':task,'checkpoint':str(final),'checkpoint_sha256':digest(final),
        'training_samples':len(train_items),'selection_samples':len(val_items),
        'epochs_completed':epochs,'selected_epoch':state['epoch'],
        'selection':'original_train_group_cv' if val_items else 'fixed_epoch_refit_no_heldout_selection',
        'seconds_this_invocation':time.monotonic()-started,'train_ids':train_ids,'validation_ids':val_ids}
    write_json(complete,result)
    return model,result


def arrays(items):
    return np.stack([r['globals'][0] for r in items]),np.stack([r['target'] for r in items])


def run(config_path):
    config_path=Path(config_path).resolve();config=json.loads(config_path.read_text())
    cfg=config['pose'];out=PROJECT/config['output'];root=PROJECT/config['pose_dataset']
    output=out/'pose';output.mkdir(parents=True,exist_ok=True)
    if cfg['unassigned_and_quarantine_allowed'] or cfg['rigid_transforms_allowed']:
        raise ValueError('Unsafe pose training policy')
    if os.environ.get('CUDA_VISIBLE_DEVICES','').lower()!=cfg['gpu_uuid'].lower():
        raise ValueError('Pin CUDA_VISIBLE_DEVICES to the configured pose GPU UUID')
    metadata=pinned_cuda_metadata(cfg['gpu_uuid']);device=torch.device(metadata['device'])
    torch.set_num_threads(2);torch.backends.cudnn.benchmark=False
    torch.use_deterministic_algorithms(True)
    manifest=json.loads((root/'manifest.json').read_text())
    binding={'config_sha256':digest(config_path),'pose_manifest_sha256':digest(root/'manifest.json'),
        'source_hashes':{name:digest(Path(__file__).with_name(name)) for name in ('pose_model.py','pose_training.py')},
        'device':metadata}
    write_json(output/'run_manifest.json',{'binding':binding,'config':config,
        'test_used_for_selection':False,'archive_only_used':False,'quarantine_used':False,
        'global_robot_coordinate_frame_verified':False,'full_six_dof':False})
    with (output/'training.lock').open('a') as lock:
        try:fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
        except BlockingIOError:raise ValueError('Pose training already active')
        if (output/'completed.json').exists():
            completed=json.loads((output/'completed.json').read_text());print(json.dumps(completed));return
        plane_rows=load_index(root,TASKS[0],'train');normal_rows=load_index(root,TASKS[1],'train')
        plane=prepare_samples(root,plane_rows,cfg,output/'cache/plane_train',manifest['files'],augment=True)
        normal=prepare_samples(root,normal_rows,cfg,output/'cache/normal_train',manifest['files'],augment=True)
        folds=group_folds(plane_rows,cfg['folds'],config['seed'])
        g,y=arrays(plane);cv_predictions={k:np.full_like(y,np.nan) for k in ('geometry','ridge','pointnet','train_mean')}
        best_epochs=[];fold_records=[]
        for i,(ti,vi) in enumerate(folds):
            train_items=[plane[j] for j in ti];validation=[plane[j] for j in vi]
            ridge=RidgeResidual(TASKS[0],cfg['ridge_alpha']).fit(g[ti],y[ti])
            net,trained=train_net(train_items,validation,TASKS[0],cfg,output/f'cv_{i}',
                device,config['seed'],cfg['max_epochs'])
            cv_predictions['geometry'][vi]=base_prediction(g[vi],TASKS[0])
            cv_predictions['ridge'][vi]=ridge.predict(g[vi])
            cv_predictions['pointnet'][vi]=predict_net(net,validation,device,cfg['batch'])
            mean=y[ti].mean(axis=0);mean[2:]/=max(np.linalg.norm(mean[2:]),1e-8)
            cv_predictions['train_mean'][vi]=np.tile(mean,(len(vi),1))
            best_epochs.append(trained['selected_epoch'])
            fold_records.append({'fold':i,'fit_ids':trained['train_ids'],'selection_ids':trained['validation_ids'],
                'fit_groups':sorted({plane_rows[j]['group_id'] for j in ti}),
                'selection_groups':sorted({plane_rows[j]['group_id'] for j in vi}),
                'pointnet_selected_epoch':trained['selected_epoch']})
        if any(not np.isfinite(p).all() for p in cv_predictions.values()):raise ValueError('Incomplete OOF predictions')
        cv_metrics={k:grouped_metrics(p,y,plane_rows,TASKS[0]) for k,p in cv_predictions.items()}
        chosen=min(('geometry','ridge','pointnet'),key=lambda k:cv_metrics[k]['overall']['macro_group_selection_score'])
        epochs=max(1,min(cfg['max_epochs'],int(np.median(best_epochs))))
        write_json(output/'cv_results.json',{'folds':fold_records,'metrics':cv_metrics,
            'selected_family':chosen,'final_neural_epoch_budget':epochs,
            'selection_uses_original_val':False,'selection_uses_original_test':False})
        buffer=io.BytesIO();np.savez_compressed(buffer,targets=y,**cv_predictions)
        protected_write(output/'oof_predictions.npz',buffer.getvalue())
        ridge=RidgeResidual(TASKS[0],cfg['ridge_alpha']).fit(g,y)
        write_json(output/'ridge.json',ridge.document())
        net,plane_fit=train_net(plane,[],TASKS[0],cfg,output/'planar_final',device,config['seed'],epochs)
        normal_net,normal_fit=train_net(normal,[],TASKS[1],cfg,output/'normal_final',device,config['seed'],cfg['normal_epochs'])
        ng,ny=arrays(normal)
        normal_pred=predict_net(normal_net,normal,device,cfg['batch'])
        normal_metrics=grouped_metrics(normal_pred,ny,normal_rows,TASKS[1])
        write_json(output/'normal_training_fit_only.json',{'metrics':normal_metrics,
            'independent_generalization_evaluated':False,'resubstitution_only':True,
            'groups':dict(Counter(r['group_id'] for r in normal_rows)),
            'reason':'all_38_original_training_samples_belong_to_one_group_no_val_or_test'})
        weights={'ridge':{'path':str(output/'ridge.json'),'sha256':digest(output/'ridge.json')},
            'pointnet':{'path':plane_fit['checkpoint'],'sha256':plane_fit['checkpoint_sha256']},
            'normal_pointnet':{'path':normal_fit['checkpoint'],'sha256':normal_fit['checkpoint_sha256']}}
        frozen={'state':'frozen_before_test','selected_planar_family':chosen,'weights':weights,
            'cv_results_sha256':digest(output/'cv_results.json'),'config_sha256':digest(config_path),
            'pose_manifest_sha256':digest(root/'manifest.json'),'source_hashes':binding['source_hashes'],
            'detection_confidence':config['detector']['conf'],'match_iou':config['detector']['iou'],
            'success_position_m':.05,'success_angle_degrees':5.,
            'test_selection_or_tuning_allowed':False,'normal_heldout_available':False,
            'full_six_dof_available':False,'robot_deployment_certified':False}
        write_json(output/'frozen_evaluation_plan.json',frozen)
        # Two samples are a diagnostic, never a checkpoint/model selection source.
        val_rows=load_index(root,TASKS[0],'val')
        val=prepare_samples(root,val_rows,cfg,output/'cache/original_val',manifest['files'],augment=False)
        vg,vy=arrays(val)
        pred={'geometry':base_prediction(vg,TASKS[0]),'ridge':ridge.predict(vg),
            'pointnet':predict_net(net,val,device,cfg['batch'])}
        write_json(output/'original_val_diagnostic.json',{'selection_used':False,
            'warning':'two_samples_only_both_pointcloud_units_unverified',
            'metrics':{k:grouped_metrics(p,vy,val_rows,TASKS[0]) for k,p in pred.items()}})
        result={'state':'complete','planar_training_samples':len(plane),'normal_training_samples':len(normal),
            'planar_cv_groups':len({r['group_id'] for r in plane_rows}),
            'selected_planar_family':chosen,'neural_refit_epochs':epochs,'normal_epochs':cfg['normal_epochs'],
            'weights':weights,'device':metadata,'test_used_for_selection':False,
            'archive_only_used':False,'quarantine_used':False,
            'normal_generalization_claimed':False,'frozen_plan_sha256':digest(output/'frozen_evaluation_plan.json'),
            'training_unit_flags':dict(Counter(str(r['pointcloud_scale_to_m']) for r in plane_rows+normal_rows))}
        write_json(output/'completed.json',result);print(json.dumps(result,ensure_ascii=False),flush=True)


def main():
    parser=argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--config',type=Path,default=PROJECT/'configs/pallet_pose_training_v1.json')
    args=parser.parse_args();run(args.config)


if __name__=='__main__':main()
