"""Snapshot-authorized human ruler research labels -> isolated normalized pose models.

Original boxes and automatic geometry alone are inference inputs. Final human
polygon, selected face, ruler, group identity and pose are never network inputs.
Metric and unknown-native branches stay separate. No live annotation DB writes.
"""
from __future__ import annotations

import argparse
from collections import Counter
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
from datetime import datetime, timezone
import csv
import fcntl
import hashlib
import io
import json
import math
import multiprocessing
import os
from pathlib import Path
import random
import sqlite3
import time
from zipfile import ZIP_DEFLATED, ZipFile

import numpy as np
import torch
from torch import nn
from torch.nn import functional as F

from .cluster_targets import digest, load_dataset, protected_snapshot, target_id, target_key
from .devices import pinned_cuda_metadata
from .pose_annotations import bounded_map, independent_output
from .pose_dataset import PROJECT, protected_write, write_json, write_rows, read_rows
from .pose_model import angular_error_degrees, group_folds, normalize_prediction
from .pose_review_geometry import GeometryEngine, canonical, polygon_value, pose_value

SCHEMA = 'human_fork_center_normalized_residual_v2'
REFERENCE = 'fork_face_width_center'
BRANCHES = ('metric', 'native_unknown')
POINT_DIM, GLOBAL_DIM = 7, 16


def now():
    return datetime.now(timezone.utc).isoformat(timespec='seconds')


def atomic_json(path, value):
    path=Path(path);path.parent.mkdir(parents=True,exist_ok=True)
    temporary=path.with_suffix(path.suffix+'.tmp');temporary.write_bytes(canonical(value));temporary.replace(path)


def live_rows(directory):
    path=(Path(directory)/'annotations.sqlite3').resolve()
    with sqlite3.connect(path.as_uri()+'?mode=ro',uri=True) as db:
        return {k:json.loads(v) for k,v in db.execute('SELECT target_key,payload FROM annotations')}


def original_targets(annotations, sources):
    result={}
    for a in annotations:
        source=sources[a['frame_id']]
        for i,box in enumerate(a['boxes']):
            key=target_key(a['frame_id'],i)
            result[key]={'target_key':key,'target_id':target_id(a['frame_id'],i),
                'frame_id':a['frame_id'],'box_index':i,'bbox':box,'split':a['split'],
                'group_id':a['group_id'],'domain':source['domain'],'size':source['size'],
                'source_pcd_sha256':source['source_anchor_hash'],'image':source['image']}
    return result


def validate_review(row, target, review_directory):
    """User authorization changes eligibility in this snapshot only, not UI truth flags."""
    source=row['source'];sensor=row['sensor']
    if any(source.get(k)!=target[k] for k in ('target_key','frame_id','split','group_id','source_pcd_sha256','bbox')):
        raise ValueError('review_source_identity_mismatch')
    if row.get('state')=='deferred':raise ValueError('human_deferred')
    if (sensor['forward_axis'],sensor['up_axis'],sensor['forward_sign'],sensor['up_sign'])!=(0,2,1,1):
        raise ValueError('axes_outside_user_authorized_policy')
    if not row.get('ruler') or not row.get('geometry_token') or not row.get('pose'):
        raise ValueError('missing_ruler_geometry_or_pose')
    recomputed=pose_value(row['ruler'],sensor)
    if recomputed['reference_point']!=REFERENCE:raise ValueError('reference_mismatch')
    for k in ('x','y','theta_degrees'):
        if not isinstance(row['pose'].get(k),(float,int)) or not math.isfinite(row['pose'][k]):
            raise ValueError('nonfinite_pose')
        if not math.isclose(row['pose'][k],recomputed[k],rel_tol=1e-6,abs_tol=1e-7):
            raise ValueError('stored_pose_differs_from_ruler')
    w,h=target['size'];polygon_value(row['polygon'],w,h)
    token=row['geometry_token'];root=Path(review_directory)/'geometry'
    if len(token)!=64 or any(c not in '0123456789abcdef' for c in token):raise ValueError('unsafe_geometry_token')
    meta=json.loads((root/(token+'.json')).read_text())
    if meta['target_key']!=row['target_key'] or meta['polygon']!=row['polygon'] or meta['sensor']!=sensor:
        raise ValueError('edited_region_or_sensor_not_recomputed')
    # Read-only engine selection; no engine constructor or cache mutation here.
    with np.load(root/(token+'.npz'),allow_pickle=False) as z:
        ids=z['source_indices'];xyz=z['xyz'];faces=z['face_ids']
    from .pose_review_geometry import retained
    selected=row['selected_faces']
    available={c['id'] for c in meta['candidates']}|{0}
    if not selected or any(i not in available for i in selected):raise ValueError('invalid_selected_faces')
    mask=(np.ones(len(ids),bool) if 0 in selected else np.isin(faces,selected))&retained(xyz,row.get('exclusions',[]))
    if mask.sum()<16 or int(mask.sum())!=row['selected_measured_points']:
        raise ValueError('too_few_or_inconsistent_measured_face_points')
    return {'pose':recomputed,'selected_source_indices':ids[mask].astype(np.int32),
        'geometry_metadata_sha256':digest(root/(token+'.json')),
        'geometry_points_sha256':digest(root/(token+'.npz'))}


def snapshot(config, out, formal, targets):
    snapshot_root=out/'snapshot';snapshot_root.mkdir(parents=True,exist_ok=True)
    path=snapshot_root/'reviews.sqlite3';review_dir=PROJECT/config['review_directory']
    if not path.exists():
        temporary=path.with_suffix('.tmp')
        source=(review_dir/'annotations.sqlite3').resolve()
        with sqlite3.connect(source.as_uri()+'?mode=ro',uri=True) as reader,sqlite3.connect(temporary) as writer:
            reader.backup(writer)
        os.chmod(temporary,0o600);temporary.replace(path)
    if (snapshot_root/'manifest.json').exists():
        manifest=json.loads((snapshot_root/'manifest.json').read_text())
        if digest(path)!=manifest['database_sha256']:raise ValueError('Frozen snapshot changed')
        return read_rows(snapshot_root/'eligible.jsonl'),manifest
    with sqlite3.connect(path.as_uri()+'?mode=ro',uri=True) as db:
        rows=[json.loads(v) for v, in db.execute('SELECT payload FROM annotations ORDER BY target_key')]
        events=[{'target_key':k,'revision':r,'payload_sha256':hashlib.sha256(v.encode()).hexdigest()}
            for k,r,v in db.execute('SELECT target_key,revision,payload FROM events ORDER BY id')]
    eligible=[];rejected=[]
    for row in rows:
        try:
            key=row['target_key'];t=targets[key];validated=validate_review(row,t,review_dir)
            # Preserve ruler support evidence, but never provide it to model inputs.
            evidence_path=snapshot_root/'support'/(key+'.npz');buffer=io.BytesIO()
            np.savez_compressed(buffer,selected_source_indices=validated['selected_source_indices'])
            protected_write(evidence_path,buffer.getvalue())
            record={**t,'sample_id':key,'review_revision':row['revision'],
                'source_review_sha256':hashlib.sha256(canonical(row)).hexdigest(),
                'source_UI_human_pose_confirmed':row['human_pose_confirmed'],
                'source_UI_axes_confirmed':row['sensor']['axes_confirmed'],
                'label_origin':'user_authorized_saved_ruler_research_label_not_UI_confirmed_GT',
                'sensor':row['sensor'],'pose':validated['pose'],
                'branch':'metric' if row['sensor']['scale_to_m'] is not None else 'native_unknown',
                'selected_measured_points':row['selected_measured_points'],
                'support_sha256':digest(evidence_path),
                'review_geometry_metadata_sha256':validated['geometry_metadata_sha256'],
                'review_geometry_points_sha256':validated['geometry_points_sha256']}
            eligible.append(record)
        except (ValueError,KeyError,OSError) as error:
            rejected.append({'target_key':row.get('target_key'),'reason':str(error),'revision':row.get('revision')})
    write_rows(snapshot_root/'eligible.jsonl',eligible);write_rows(snapshot_root/'rejected.jsonl',rejected)
    write_rows(snapshot_root/'source_reviews.jsonl',rows);write_rows(snapshot_root/'event_fingerprints.jsonl',events)
    protection=protected_snapshot(formal);write_json(snapshot_root/'protected_formal.json',protection)
    for field in ('group_id','source_pcd_sha256'):
        seen={}
        for r in eligible:
            old=seen.setdefault(r[field],r['split'])
            if old!=r['split']:raise ValueError('Original split leakage: '+field)
    manifest={'created_at':now(),'database_sha256':digest(path),'authorization':config['authorization'],
        'saved_targets':len(rows),'eligible':len(eligible),'rejected':rejected,
        'eligible_split_branch_counts':dict(Counter(r['split']+'/'+r['branch'] for r in eligible)),
        'groups':len({r['group_id'] for r in eligible}),'events':len(events),
        'all_saved_keys':[r['target_key'] for r in rows],
        'eligible_sha256':digest(snapshot_root/'eligible.jsonl'),
        'formal_manifest_sha256':digest(formal/'manifest.json'),
        'protected_formal_files':len(protection),'live_database_modified':False,
        'source_UI_flags_changed':False,'old_incompatible_pose_GT_used':False}
    write_json(snapshot_root/'manifest.json',manifest)
    print(json.dumps({'stage':'snapshot',**{k:v for k,v in manifest.items() if k not in ('all_saved_keys','authorization')}},ensure_ascii=False),flush=True)
    return eligible,manifest


def feature_values(points, uv, size, box, candidate, seed, count):
    """All input geometry is automatic and expressed relative to automatic face width."""
    xyz=points[:,:3].astype(float);base=np.asarray(candidate['center'],float)
    normal=np.asarray(candidate['basis'],float)[:,1]
    width=max(candidate['half_width']*2,1e-10)
    low,high=np.quantile(xyz,[.01,.99],axis=0)
    good=((xyz>=low)&(xyz<=high)).all(axis=1)
    if good.sum()>=16:xyz,uv,points=xyz[good],uv[good],points[good]
    intensity=points[:,3];valid=np.isfinite(intensity);encoded=np.zeros(len(points))
    if valid.any():
        lo,hi=np.quantile(intensity[valid],[.05,.95]);encoded[valid]=np.clip((intensity[valid]-lo)/max(hi-lo,1e-8),0,1)
    b=np.array([box[k] for k in ('x','y','width','height')]);w,h=size
    relative_uv=(uv-b[:2])/b[2:]-.5
    local=np.clip((xyz-base)/width,-8,8)
    features=np.column_stack((local,encoded,valid.astype(float),relative_uv)).astype(np.float32)
    selected=np.random.default_rng(seed).choice(len(features),count,replace=len(features)<count)
    spread=(np.quantile(xyz,.75,axis=0)-np.quantile(xyz,.25,axis=0))/width
    g=np.r_[base/width,spread,normal,[b[0]/w,b[1]/h,b[2]/w,b[3]/h],
        np.log1p(len(xyz))/10.,np.std((xyz-base)@normal)/width,0.].astype(np.float32)
    if features.shape[1]!=POINT_DIM or g.shape!=(GLOBAL_DIM,) or not np.isfinite(g).all():raise ValueError('Invalid normalized features')
    return features[selected],g,base,width,normal[:2]/max(np.linalg.norm(normal[:2]),1e-8)


def extract_job(job):
    formal,cache,source,target,count,seed=job
    feature_path=Path(cache)/'features'/(target['target_key']+'.npz')
    if feature_path.exists():
        return json.loads(feature_path.with_suffix('.json').read_text())
    engine=GeometryEngine(formal,Path(cache)/'automatic_geometry')
    b=target['bbox'];x,y,w,h=(b[k] for k in ('x','y','width','height'))
    polygon=[[x,y],[x+w,y],[x+w,y+h],[x,y+h]]
    # Source policy only, never user-edited sensor or final manual face/ruler.
    try:
        meta=engine.compute(target['target_key'],source,polygon,None)
        candidate=meta['candidates'][0] if meta['candidates'] else None
        if candidate is None:raise ValueError('no_automatic_plane_candidate')
        if abs(candidate['normal'][2])>.85:raise ValueError('automatic_candidate_is_horizontal_not_fork_face')
        arrays=engine.selected(meta,[candidate['id']],[])
        points,mapping=engine.registered(canonical(source).decode())
        xyzi=points[arrays['source_indices']].astype(float)
        scale=meta['sensor']['scale_to_m']
        if scale is not None:xyzi[:,:3]*=scale
        fs,gs=[],[]
        for variant in range(3):
            f,g,base,width,direction=feature_values(xyzi,arrays['uv'],target['size'],b,candidate,seed+variant,count)
            fs.append(f);gs.append(g)
        buffer=io.BytesIO();np.savez_compressed(buffer,features=np.stack(fs),globals=np.stack(gs),
            base=base,width=width,direction=direction)
        protected_write(feature_path,buffer.getvalue())
        record={'target_key':target['target_key'],'state':'complete','geometry_token':meta['token'],
            'source_pcd_sha256':source['source_anchor_hash'],'source_image_sha256':source['files'][source['image']],
            'features_sha256':digest(feature_path),'roi_points':meta['roi_points'],
            'automatic_face_points':candidate['points'],'geometry_units':'m' if scale is not None else 'native_unknown',
            'scale_to_m':scale,'source_forward_axis':source['sensor_policy'].get('depth_axis'),
            'input_uses_original_box_only':True,'human_polygon_face_ruler_not_inputs':True}
    except ValueError as error:
        record={'target_key':target['target_key'],'state':'unavailable','reason':str(error),
            'source_pcd_sha256':source['source_anchor_hash']}
    write_json(feature_path.with_suffix('.json'),record)
    return record


def extract_targets(formal,cache,rows,sources,config):
    jobs=((str(formal),str(cache),sources[r['frame_id']],r,config['points'],config['seed']) for r in rows)
    results=[];started=time.monotonic()
    with ProcessPoolExecutor(max_workers=config['workers'],mp_context=multiprocessing.get_context('spawn')) as pool:
        for i,value in enumerate(bounded_map(pool,extract_job,jobs,config['workers']*2),1):
            results.append(value)
            if i==1 or i%500==0 or i==len(rows):
                print(json.dumps({'stage':'automatic_features','completed':i,'total':len(rows),'seconds':round(time.monotonic()-started,1)}),flush=True)
    return results


def residual_target(row,base,width,direction):
    pose=row['pose'];phi=np.deg2rad(pose['theta_degrees'])
    target_direction=np.array([np.cos(phi),np.sin(phi)])
    cosine=float(np.dot(target_direction,direction));sine=float(direction[0]*target_direction[1]-direction[1]*target_direction[0])
    return np.r_[(np.array([pose['x'],pose['y']])-base[:2])/width,cosine,sine].astype(np.float32)


def decoded(residual,base,width,direction):
    p=np.asarray(residual,dtype=float)
    cos,sin=p[...,2],p[...,3];norm=np.maximum(np.hypot(cos,sin),1e-8);cos,sin=cos/norm,sin/norm
    n=np.stack((direction[...,0]*cos-direction[...,1]*sin,direction[...,1]*cos+direction[...,0]*sin),axis=-1)
    return np.concatenate((base[...,:2]+width[...,None]*p[...,:2],n),axis=-1)


class NormalizedPointNet(nn.Module):
    def __init__(self,mean=None,std=None):
        super().__init__()
        self.register_buffer('mean',torch.zeros(GLOBAL_DIM) if mean is None else torch.as_tensor(mean,dtype=torch.float32))
        self.register_buffer('std',torch.ones(GLOBAL_DIM) if std is None else torch.as_tensor(std,dtype=torch.float32).clamp_min(.001))
        self.points=nn.Sequential(nn.Conv1d(POINT_DIM,32,1),nn.GroupNorm(8,32),nn.SiLU(),
            nn.Conv1d(32,64,1),nn.GroupNorm(8,64),nn.SiLU(),nn.Conv1d(64,96,1),nn.GroupNorm(8,96),nn.SiLU())
        self.head=nn.Sequential(nn.Linear(192+GLOBAL_DIM,96),nn.SiLU(),nn.Dropout(.15),nn.Linear(96,4))
        nn.init.zeros_(self.head[-1].weight);nn.init.zeros_(self.head[-1].bias)

    def forward(self,features,globals_):
        h=self.points(features.transpose(1,2));g=((globals_-self.mean)/self.std).clamp(-10,10)
        value=self.head(torch.cat((h.amax(-1),h.mean(-1),g),dim=-1))
        direction=value[:,2:]+value.new_tensor([1.,0.])
        return torch.cat((value[:,:2]*.1,F.normalize(direction,dim=-1)),dim=-1)


def loss_value(pred,target):
    location=F.smooth_l1_loss(pred[:,:2]/.05,target[:,:2]/.05,beta=1.)
    angle=(1-(pred[:,2:]*target[:,2:]).sum(-1).clamp(-1,1)).mean()
    return location+10*angle


def score_values(pred,target,rows):
    position=np.linalg.norm(np.asarray(pred)[:,:2]-np.asarray(target)[:,:2],axis=1)
    pa=np.degrees(np.arctan2(np.asarray(pred)[:,3],np.asarray(pred)[:,2]))
    ta=np.degrees(np.arctan2(np.asarray(target)[:,3],np.asarray(target)[:,2]));angles=angular_error_degrees(pa,ta)
    per=position/.05+angles/5.
    groups=sorted({r['group_id'] for r in rows})
    macro=np.mean([per[[r['group_id']==g for r in rows]].mean() for g in groups])
    return {'samples':len(rows),'groups':len(groups),'normalized_position_mean_width_fraction':float(position.mean()),
        'angle_mae_degrees':float(angles.mean()),'macro_group_selection_score':float(macro)}


def ridge_fit(g,y,alpha):
    mean=g.mean(0);std=np.maximum(g.std(0),.001);x=np.c_[((g-mean)/std).clip(-10,10),np.ones(len(g))]
    base=np.array([0,0,1,0]);penalty=np.eye(x.shape[1])*alpha;penalty[-1,-1]=1e-8
    coef=np.linalg.solve(x.T@x+penalty,x.T@(y-base))
    return {'mean':mean.tolist(),'std':std.tolist(),'coef':coef.tolist()}


def ridge_predict(model,g):
    x=np.c_[np.clip((g-np.array(model['mean']))/np.array(model['std']),-10,10),np.ones(len(g))]
    return normalize_prediction(x@np.array(model['coef'])+np.array([0,0,1,0]),'planar_xy_theta')


def net_predict(model,features,globals_,device,batch):
    output=[];model.eval()
    with torch.inference_mode():
        for offset in range(0,len(features),batch):
            f=torch.from_numpy(features[offset:offset+batch,0]).to(device)
            g=torch.from_numpy(globals_[offset:offset+batch,0]).to(device)
            output.append(model(f,g).cpu().numpy())
    return np.concatenate(output)


def fit_net(f,g,y,train_ids,validation_ids,rows,config,path,device,epochs):
    path.mkdir(parents=True,exist_ok=True)
    seed=config['seed'];random.seed(seed);np.random.seed(seed);torch.manual_seed(seed)
    rng=np.random.default_rng(seed);ti=np.asarray(train_ids);vi=np.asarray(validation_ids,dtype=int)
    model=NormalizedPointNet(g[ti,0].mean(0),g[ti,0].std(0)).to(device)
    optimizer=torch.optim.AdamW(model.parameters(),lr=config['learning_rate'],weight_decay=.005)
    best=None;best_epoch=epochs;history=[];best_state=None
    specification={'schema':SCHEMA,'fit_keys':[rows[i]['target_key'] for i in ti],
        'validation_keys':[rows[i]['target_key'] for i in vi],'epochs':epochs,'seed':seed}
    if {rows[i]['group_id'] for i in ti}&{rows[i]['group_id'] for i in vi}:raise ValueError('CV group leakage')
    write_json(path/'specification.json',specification)
    for epoch in range(epochs):
        model.train();order=rng.permutation(ti);total=0.;seen=0
        for offset in range(0,len(order),config['batch']):
            ix=order[offset:offset+config['batch']];variants=rng.integers(0,f.shape[1],len(ix))
            fs=torch.from_numpy(f[ix,variants]).to(device);gs=torch.from_numpy(g[ix,variants]).to(device)
            target=torch.from_numpy(y[ix]).to(device)
            fs=fs.clone();fs[:,:,:3]+=torch.randn_like(fs[:,:,:3])*.003
            pred=model(fs,gs);loss=loss_value(pred,target)
            if not torch.isfinite(loss):raise ValueError('Nonfinite normalized loss')
            optimizer.zero_grad(set_to_none=True);loss.backward();nn.utils.clip_grad_norm_(model.parameters(),5);optimizer.step()
            total+=float(loss.detach())*len(ix);seen+=len(ix)
        item={'epoch':epoch+1,'train_loss':total/seen}
        if len(vi):
            prediction=net_predict(model,f[vi],g[vi],device,config['batch'])
            metric=score_values(prediction,y[vi],[rows[i] for i in vi]);item['validation']=metric
            if best is None or metric['macro_group_selection_score']<best:
                best=metric['macro_group_selection_score'];best_epoch=epoch+1
                best_state={k:v.detach().cpu().clone() for k,v in model.state_dict().items()}
        history.append(item)
        if epoch==0 or (epoch+1)%20==0:
            print(json.dumps({'stage':str(path.name),**item}),flush=True)
    last={'schema':SCHEMA,'model':{k:v.detach().cpu() for k,v in model.state_dict().items()},
        'optimizer':optimizer.state_dict(),'specification':specification,'history':history,
        'rng':{'torch':torch.get_rng_state(),'numpy':rng.bit_generator.state},'epoch':epochs}
    torch.save(last,path/'last.pt')
    if best_state is not None:model.load_state_dict(best_state)
    state={'schema':SCHEMA,'model':{k:v.detach().cpu() for k,v in model.state_dict().items()},'selected_epoch':best_epoch}
    torch.save(state,path/'selected.pt');write_json(path/'history.json',history)
    return model,best_epoch


def load_features(cache,rows,records):
    fs=[];gs=[];ys=[];bases=[];widths=[];directions=[];accepted=[];rejected=[]
    bykey={r['target_key']:r for r in records}
    for row in rows:
        key=row['target_key'];record=bykey[key]
        if record['state']!='complete':rejected.append({'target_key':key,'reason':record.get('reason')});continue
        path=cache/'features'/(key+'.npz')
        if digest(path)!=record['features_sha256']:raise ValueError('Feature cache identity mismatch')
        with np.load(path,allow_pickle=False) as z:
            f,g,base,width,direction=z['features'],z['globals'],z['base'],float(z['width']),z['direction']
        # Stored human values are in the source review scale. Do not mix differing scales.
        if row['sensor']['scale_to_m']!=record['scale_to_m']:
            rejected.append({'target_key':key,'reason':'human_source_unit_policy_differs'});continue
        y=residual_target(row,base,width,direction)
        if not np.isfinite(y).all() or np.linalg.norm(y[:2])>5:
            rejected.append({'target_key':key,'reason':'human_reference_far_outside_original_box'});continue
        accepted.append(row);fs.append(f);gs.append(g);ys.append(y);bases.append(base);widths.append(width);directions.append(direction)
    return accepted,dict(features=np.stack(fs),globals=np.stack(gs),targets=np.stack(ys),
        bases=np.stack(bases),widths=np.array(widths),directions=np.stack(directions)),rejected


def train_branch(branch,rows,arrays,config,out,device):
    root=out/'training'/branch;root.mkdir(parents=True,exist_ok=True)
    indices=[i for i,r in enumerate(rows) if r['branch']==branch and r['split']=='train']
    if len(indices)<8:raise ValueError('Insufficient branch training samples')
    r=[rows[i] for i in indices];f=arrays['features'][indices];g=arrays['globals'][indices];y=arrays['targets'][indices]
    folds=group_folds(r,config['folds'],config['seed'])
    predictions={name:np.zeros_like(y) for name in ('geometry','ridge','pointnet')}
    predictions['geometry'][:,2]=1;epochs=[];details=[]
    for number,(ti,vi) in enumerate(folds):
        net,epoch=fit_net(f,g,y,ti,vi,r,config,root/f'fold_{number}',device,config['epochs'])
        predictions['pointnet'][vi]=net_predict(net,f[vi],g[vi],device,config['batch'])
        ridge=ridge_fit(g[ti,0],y[ti],config['ridge_alpha']);predictions['ridge'][vi]=ridge_predict(ridge,g[vi,0])
        epochs.append(epoch);details.append({'fold':number,'fit_groups':sorted({r[i]['group_id'] for i in ti}),
            'validation_groups':sorted({r[i]['group_id'] for i in vi}),'selected_epoch':epoch,
            'fit_keys':[r[i]['target_key'] for i in ti],'validation_keys':[r[i]['target_key'] for i in vi]})
    metrics={name:score_values(p,y,r) for name,p in predictions.items()}
    selected=min(metrics,key=lambda name:metrics[name]['macro_group_selection_score'])
    epoch=max(1,int(np.median(epochs)))
    final,_=fit_net(f,g,y,list(range(len(r))),[],r,config,root/'refit',device,epoch)
    ridge=ridge_fit(g[:,0],y,config['ridge_alpha'])
    write_json(root/'ridge.json',ridge)
    write_json(root/'cross_validation.json',{'selection':'original_train_only_group_CV','folds':details,
        'metrics':metrics,'selected_family':selected,'refit_epochs':epoch,
        'original_val_used_for_selection':False,'original_test_used_for_selection':False})
    with (root/'out_of_fold.npz').open('wb') as stream:
        np.savez_compressed(stream,targets=y,**predictions)
    profile={'branch':branch,'selected_family':selected,'training_samples':len(r),'training_groups':len({v['group_id'] for v in r}),
        'training_keys':[v['target_key'] for v in r],'domains':sorted({v['domain'] for v in r}),
        'global_mean':g[:,0].mean(0).tolist(),'global_std':np.maximum(g[:,0].std(0),.001).tolist(),
        'network':str((root/'refit/selected.pt').relative_to(out)),'network_sha256':digest(root/'refit/selected.pt'),
        'ridge':str((root/'ridge.json').relative_to(out)),'ridge_sha256':digest(root/'ridge.json'),
        'cv_sha256':digest(root/'cross_validation.json'),'refit_epochs':epoch,
        'cv_metrics':metrics,'position_reference':REFERENCE,'unit_policy':branch}
    write_json(root/'profile.json',profile)
    return profile


def load_predictors(out,profiles,device='cpu'):
    result={}
    for profile in profiles:
        path=Path(out)/profile['network']
        if digest(path)!=profile['network_sha256']:raise ValueError('Model hash differs')
        state=torch.load(path,map_location='cpu',weights_only=True)
        if state['schema']!=SCHEMA:raise ValueError('Model schema differs')
        net=NormalizedPointNet();net.load_state_dict(state['model']);net.to(device).eval()
        ridge_path=Path(out)/profile['ridge']
        if digest(ridge_path)!=profile['ridge_sha256']:raise ValueError('Ridge hash differs')
        result[profile['branch']]={'profile':profile,'net':net,'ridge':json.loads(ridge_path.read_text())}
    return result


def predict_residual(model,f,g,device='cpu',batch=64):
    family=model['profile']['selected_family']
    if family=='geometry':
        value=np.zeros((len(f),4));value[:,2]=1;return value
    if family=='ridge':return ridge_predict(model['ridge'],g[:,0])
    return net_predict(model['net'],f,g,device,batch)


def evaluate(rows,arrays,profiles,out,device):
    report={'reference':REFERENCE,'labels':'user_authorized_research_rulers_not_independently_blind_GT',
        'old_incompatible_GT_used':False,'selection_after_test_allowed':False,'splits':{}}
    predictions=[];models=load_predictors(out,profiles,device)
    for split in ('val','test'):
        for branch in BRANCHES:
            ix=[i for i,r in enumerate(rows) if r['split']==split and r['branch']==branch]
            if not ix:continue
            f,g=arrays['features'][ix],arrays['globals'][ix]
            residual=predict_residual(models[branch],f,g,device)
            pred=decoded(residual,arrays['bases'][ix],arrays['widths'][ix],arrays['directions'][ix])
            truth=decoded(arrays['targets'][ix],arrays['bases'][ix],arrays['widths'][ix],arrays['directions'][ix])
            distances=np.linalg.norm(pred[:,:2]-truth[:,:2],axis=1)
            angle=angular_error_degrees(np.degrees(np.arctan2(pred[:,3],pred[:,2])),np.degrees(np.arctan2(truth[:,3],truth[:,2])))
            metric=score_values(residual,arrays['targets'][ix],[rows[i] for i in ix])
            metric.update(position_unit='mm' if branch=='metric' else 'native_unknown',
                position_mean=float(distances.mean()*(1000 if branch=='metric' else 1)),
                position_p90=float(np.quantile(distances,.9)*(1000 if branch=='metric' else 1)),
                success_50mm_5deg=float(((distances<=.05)&(angle<=5)).mean()) if branch=='metric' else None)
            report['splits'][split+'/'+branch]=metric
            for j,i in enumerate(ix):
                predictions.append({'target_key':rows[i]['target_key'],'split':split,'branch':branch,
                    'prediction':pred[j].tolist(),'target':truth[j].tolist(),'source_review_revision':rows[i]['review_revision']})
    write_json(out/'evaluation/results.json',report);write_rows(out/'evaluation/predictions.jsonl',predictions)
    return report


def annotate(formal,targets,sources,config,out,manifest,profiles):
    root=out/'annotations';root.mkdir(parents=True,exist_ok=True)
    all_saved=set(manifest['all_saved_keys']);live=live_rows(PROJECT/config['review_directory'])
    candidates=[v for k,v in targets.items() if k not in all_saved and k not in live]
    feature_records=extract_targets(formal,out/'cache',candidates,sources,config)
    models=load_predictors(out,profiles,'cpu');result=[]
    torch.set_num_threads(1)
    for target,record in zip(candidates,feature_records):
        key=target['target_key'];source=sources[target['frame_id']]
        branch='metric' if source['sensor_policy'].get('scale_to_mm') is not None else 'native_unknown'
        model=models[branch];profile=model['profile'];flags=[];pose=None;ruler=None
        if target['domain'] not in profile['domains']:flags.append('sensor_domain_not_in_this_branch_training')
        if target['group_id'] not in {targets[k]['group_id'] for k in profile['training_keys']}:flags.append('capture_group_not_in_branch_training')
        if branch=='native_unknown':flags.append('unit_unknown_native_numeric_estimate_not_metric')
        if source['sensor_policy'].get('depth_axis')!=0:flags.append('forward_X_height_Z_research_assumption_not_sensor_calibration')
        if source['instances']>1:flags.append('multi_target_scene_requires_review')
        if target['split']!='train':flags.append('original_holdout_not_training_GT')
        reason=record.get('reason');zscore=None
        if record['state']=='complete':
            with np.load(out/'cache/features'/(key+'.npz'),allow_pickle=False) as z:
                f,g,base,width,direction=z['features'][None],z['globals'][None],z['base'],float(z['width']),z['direction']
            zscore=float(np.max(np.abs((g[0,0]-profile['global_mean'])/np.array(profile['global_std']))))
            if zscore>config['annotation_feature_zscore_limit']:flags.append('automatic_geometry_outside_training_envelope')
            if 'sensor_domain_not_in_this_branch_training' not in flags:
                residual=predict_residual(model,f,g)[0]
                pred=decoded(residual[None],base[None],np.array([width]),direction[None])[0]
                if np.isfinite(pred).all() and np.linalg.norm(residual[:2])<=2:
                    theta=float(np.degrees(np.arctan2(pred[3],pred[2])))
                    pose={'x':float(pred[0]),'y':float(pred[1]),'theta_degrees':theta,'z':None,
                        'normal':None,'roll':None,'pitch':None,'quaternion':None,'position_unit':'m' if branch=='metric' else 'native_unknown',
                        'reference_point':REFERENCE,'source_scale_to_m':record['scale_to_m'],
                        'coordinate_frame':'source_axes_scaled' if branch=='metric' else 'source_native',
                        'full_six_dof':False,'robot_command_eligible':False}
                    meta=json.loads((out/'cache/automatic_geometry'/(record['geometry_token']+'.json')).read_text());c=meta['candidates'][0]
                    b=np.array(c['basis']);o=np.array(c['center']);delta=np.array([pred[0],pred[1],o[2]])-o
                    local=delta@b;angle=float(np.degrees(np.arctan2(np.array([pred[2],pred[3],0])@b[:,0],np.array([pred[2],pred[3],0])@b[:,1])))
                    ruler={'origin':o.tolist(),'basis':b.tolist(),'w':float(local[0]),'d':float(local[1]),
                        'angle':-angle,'half_width':c['half_width'],'direction_sign':1}
                else:reason='prediction_outside_frozen_residual_limits'
            else:reason='unsupported_sensor_domain_no_new_pose_generated'
        result.append({**target,'branch':branch,'pose_estimate':pose,
            'metric_pose_estimate':pose if branch=='metric' else None,'ruler_suggestion':ruler,
            'geometry_source':'automatic_original_box_only','input_quality':record,'max_training_feature_zscore':zscore,
            'pose_label_origin':'machine_prediction_from_user_authorized_ruler_refinement',
            'human_pose_confirmed':False,'review_required':True,'review_flags':flags,
            'unavailable_reason':reason if pose is None else None,'model_family':profile['selected_family'],
            'model_branch':branch,'pose_reference':REFERENCE,'robot_command_eligible':False,
            'automatic_training_eligible':False,'evaluation_GT_eligible':False,
            'human_review_replaced':False,'source_model_profile_sha256':digest(out/'training'/branch/'profile.json')})
    # Recheck the live DB after long extraction; never publish machine replacements of new user work.
    latest=live_rows(PROJECT/config['review_directory']);newly_saved=set(latest)-set(live)
    dropped=[r['target_key'] for r in result if r['target_key'] in latest]
    result=[r for r in result if r['target_key'] not in latest]
    write_rows(root/'targets.jsonl',result)
    columns=['target_key','frame_id','split','branch','x','y','theta_degrees','position_unit','model_family','review_flags','unavailable_reason']
    stream=io.StringIO(newline='');writer=csv.DictWriter(stream,fieldnames=columns);writer.writeheader()
    for r in result:
        values={k:r.get(k) for k in columns};values.update({k:(r['pose_estimate'] or {}).get(k) for k in ('x','y','theta_degrees','position_unit')})
        values['review_flags']=';'.join(r['review_flags']);writer.writerow(values)
    protected_write(root/'targets.csv',b'\xef\xbb\xbf'+stream.getvalue().encode())
    for split in ('train','val','test'):write_rows(root/f'indices/{split}.jsonl',[r for r in result if r['split']==split])
    write_rows(root/'indices/automatic_training_eligible.jsonl',[])
    # Effective full target index keeps all existing human revisions as references, never copies machine over them.
    bykey={r['target_key']:r for r in result};effective=[]
    for key,target in targets.items():
        effective.append({**target,'label_source':'live_human_review' if key in latest else 'refinement_machine' if key in bykey else 'not_updated',
            'human_revision':latest[key]['revision'] if key in latest else None,
            'machine_pose':bykey[key]['pose_estimate'] if key in bykey else None,'automatic_training_eligible':False})
    write_rows(root/'effective_target_index.jsonl',effective)
    summary={'all_original_targets':len(targets),'frozen_saved_targets_preserved':len(all_saved),
        'live_saved_targets_preserved':len(latest),'machine_target_records':len(result),
        'new_pose_estimates':sum(r['pose_estimate'] is not None for r in result),
        'unavailable_new_pose':sum(r['pose_estimate'] is None for r in result),
        'branch_pose_counts':dict(Counter(r['branch'] for r in result if r['pose_estimate'] is not None)),
        'failure_reasons':dict(Counter(r['unavailable_reason'] for r in result if r['pose_estimate'] is None)),
        'new_human_work_during_annotation_dropped_from_machine':dropped,
        'original_split_preserved':True,'detector_rerun':False,'old_GT_replaced':False,
        'production_review_database_writes':0,'machine_predictions_human_confirmed':False}
    write_json(root/'summary.json',summary)
    return summary


def run(config_path,stage):
    config_path=Path(config_path).resolve();config=json.loads(config_path.read_text())
    if (config['version']!=2 or config['detector_retraining'] or config['old_incompatible_pose_GT_used'] or config['live_review_database_writes']):
        raise ValueError('Unsafe refinement configuration')
    formal=Path(json.loads((PROJECT/config['formal_config']).read_text())['dataset']);out=PROJECT/config['output']
    independent_output(out,[formal,PROJECT/config['review_directory'],PROJECT/'outputs/pallet_pose_training_v1',PROJECT/'outputs/pallet_pose_dataset_v2',PROJECT/'outputs/pallet_structure_human_review_v1'])
    out.mkdir(exist_ok=True,parents=True)
    with (out/'run.lock').open('a') as lock:
        fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
        protected_write(out/'config.json',config_path.read_bytes())
        protected_write(out/'source/pose_refinement.py',Path(__file__).read_bytes())
        annotations,sources=load_dataset(formal);targets=original_targets(annotations,sources)
        rows,manifest=snapshot(config,out,formal,targets)
        if stage=='snapshot':return
        if not (out/'training/completed.json').exists():
            records=extract_targets(formal,out/'cache',rows,sources,config)
            rows,arrays,rejected=load_features(out/'cache',rows,records)
            write_rows(out/'training/eligible_after_features.jsonl',rows);write_rows(out/'training/feature_rejected.jsonl',rejected)
            if os.environ.get('CUDA_VISIBLE_DEVICES')!=config['gpu_uuid']:raise ValueError('Pin configured GPU UUID')
            metadata=pinned_cuda_metadata(config['gpu_uuid']);device=torch.device(metadata['device']);torch.set_num_threads(2)
            torch.backends.cudnn.benchmark=False;torch.use_deterministic_algorithms(True)
            profiles=[train_branch(branch,rows,arrays,config,out,device) for branch in BRANCHES]
            frozen={'state':'frozen_before_original_test','schema':SCHEMA,'created_at':now(),'profiles':profiles,
                'config_sha256':digest(config_path),'snapshot_manifest_sha256':digest(out/'snapshot/manifest.json'),
                'source_sha256':digest(__file__),'annotation_feature_zscore_limit':config['annotation_feature_zscore_limit'],
                'training_split':'train_only','selection':'group_CV_train_only','val_test_used_for_selection':False,
                'unknown_units_never_assumed_metric':True,'reference':REFERENCE,'device':metadata}
            write_json(out/'training/frozen_models.json',frozen)
            report=evaluate(rows,arrays,profiles,out,device)
            write_json(out/'training/completed.json',{'state':'complete','profiles':profiles,
                'eligible_after_features':len(rows),'feature_rejected':rejected,'evaluation':report})
        done=json.loads((out/'training/completed.json').read_text());profiles=done['profiles']
        if stage=='train':return
        summary=annotate(formal,targets,sources,config,out,manifest,profiles)
        if protected_snapshot(formal)!=json.loads((out/'snapshot/protected_formal.json').read_text()):raise ValueError('Protected formal data changed')
        print(json.dumps({'stage':'complete','summary':summary},ensure_ascii=False),flush=True)


def main():
    parser=argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--config',type=Path,default=PROJECT/'configs/pallet_pose_refinement_v2.json')
    parser.add_argument('--stage',choices=('snapshot','train','all'),default='all')
    args=parser.parse_args();run(args.config,args.stage)


if __name__=='__main__':main()
