"""Latest authorized pose-label snapshot and original-box-only v4 input dataset."""
from __future__ import annotations

from collections import Counter
from concurrent.futures import ProcessPoolExecutor
from contextlib import closing
import copy
import io
import json
import multiprocessing
import os
from pathlib import Path
import sqlite3

import numpy as np

from .cluster_targets import digest, load_dataset, protected_snapshot
from .data.orientation import orient_grid
from .data.pcd import read_pcd
from .pose_benchmark_inference import automatic_input
from .pose_dataset import protected_write, read_rows, write_rows
from .pose_refinement import (atomic_json, now, original_targets, residual_target,
                              validate_review, feature_values)
from .pose_review_geometry import canonical, raster_mask, basis_for
from .fork_geometry import decompose_roi, DEFAULTS, valid_xyz

PROJECT = Path(__file__).resolve().parents[1]
CONFIG = PROJECT / 'configs/pallet_pose_benchmark_v4.json'
ROOT = PROJECT / 'outputs/pallet_pose_benchmark_v4'


def config():
    return json.loads(CONFIG.read_text())


def effective_label(record, confirmed):
    """Unit interpretation in the new research version only; never mutate UI data."""
    value = copy.deepcopy(record)
    sensor = value['sensor']
    value['source_sensor'] = copy.deepcopy(sensor)
    value['source_branch'] = value['branch']
    if value['target_key'] in confirmed and sensor['scale_to_m'] is None:
        sensor.update(scale_to_m=1.0, unit_origin='user_confirmed_metre_research_v4')
        value['pose'].update(position_unit='m', source_scale_to_m=1.0,
                             coordinate_frame='source_axes_user_confirmed_metres')
        value['branch'] = 'metric'
        value['unit_evidence'] = 'explicit_user_confirmation_of_previously_audited_823_targets'
    else:
        value['unit_evidence'] = 'source_registered_or_saved_human_policy' if sensor['scale_to_m'] is not None else 'unverified_native_units'
    value['robot_command_eligible'] = False
    return value


def snapshot():
    cfg = config();root = ROOT / 'snapshot';root.mkdir(parents=True, exist_ok=True)
    if (root / 'manifest.json').exists():
        manifest = json.loads((root / 'manifest.json').read_text())
        if manifest['config_sha256'] != digest(CONFIG):raise ValueError('Frozen v4 configuration changed')
        if manifest['database_sha256'] != digest(root / 'reviews.sqlite3'):raise ValueError('Snapshot database changed')
        if manifest['eligible_sha256'] != digest(root / 'eligible.jsonl'):raise ValueError('Frozen labels changed')
        return manifest
    formal = Path(json.loads((PROJECT / cfg['formal_config']).read_text())['dataset'])
    annotations, sources = load_dataset(formal);targets = original_targets(annotations, sources)
    database = root / 'reviews.sqlite3'
    if not database.exists():
        live = (PROJECT / cfg['review_directory'] / 'annotations.sqlite3').resolve()
        with closing(sqlite3.connect(live.as_uri() + '?mode=ro', uri=True)) as src, closing(sqlite3.connect(database)) as dest:
            src.backup(dest)
        os.chmod(database, 0o600)
    with closing(sqlite3.connect(database.resolve().as_uri() + '?mode=ro', uri=True)) as db:
        assert db.execute('PRAGMA quick_check').fetchone()[0] == 'ok'
        reviews = [json.loads(v) for v, in db.execute('SELECT payload FROM annotations ORDER BY target_key')]
        events = [dict(target_key=k,revision=r,payload_sha256=__import__('hashlib').sha256(v.encode()).hexdigest())
                  for k,r,v in db.execute('SELECT target_key,revision,payload FROM events ORDER BY id')]
    old = read_rows(PROJECT / cfg['previous_labels'])
    confirmed = {r['target_key'] for r in old if r['branch'] == 'native_unknown'}
    if len(confirmed) != 823:raise ValueError('Confirmed unit scope changed')
    eligible,rejected = [],[]
    for review in reviews:
        try:
            key=review['target_key'];target=targets[key]
            checked=validate_review(review,target,PROJECT / cfg['review_directory'])
            support=root/'support'/(key+'.npz');stream=io.BytesIO()
            np.savez_compressed(stream,selected_source_indices=checked['selected_source_indices'])
            protected_write(support,stream.getvalue())
            source=sources[target['frame_id']]
            if digest(source['source_paths']['pcd'])!=target['source_pcd_sha256']:raise ValueError('Source PCD identity mismatch')
            sensor=review['sensor']
            row=dict(**target,review_revision=review['revision'],sensor=copy.deepcopy(sensor),pose=checked['pose'],
                branch='metric' if sensor['scale_to_m'] is not None else 'native_unknown',
                source_review_sha256=__import__('hashlib').sha256(canonical(review)).hexdigest(),
                source_UI_human_pose_confirmed=review['human_pose_confirmed'],
                source_UI_axes_confirmed=sensor['axes_confirmed'],
                label_origin='user_authorized_latest_saved_research_draft_not_blind_GT',
                selected_measured_points=review['selected_measured_points'],support_sha256=digest(support),
                **{k:checked[k] for k in ('geometry_metadata_sha256','geometry_points_sha256')})
            eligible.append(effective_label(row,confirmed))
        except (ValueError,KeyError,OSError) as exc:
            rejected.append(dict(target_key=review.get('target_key'),revision=review.get('revision'),reason=str(exc)))
    for field in ('group_id','source_pcd_sha256'):
        seen={}
        for row in eligible:
            if seen.setdefault(row[field],row['split'])!=row['split']:raise ValueError('Original split leakage')
    write_rows(root/'eligible.jsonl',eligible);write_rows(root/'rejected.jsonl',rejected)
    write_rows(root/'source_reviews.jsonl',reviews);write_rows(root/'event_fingerprints.jsonl',events)
    protected=protected_snapshot(formal);atomic_json(root/'protected_formal.json',protected)
    immutable=['configs/active_formal_dataset.json','configs/active_pose_annotations.json','configs/active_pose_refinement.json',
        'outputs/pallet_pose_refinement_v2/pallet_pose_annotations_v2.zip',
        'outputs/pallet_pose_refinement_v2/pallet_pose_refined_models_v2.zip',
        'outputs/pallet_pose_benchmark_v3/pallet_pose_best_learning_models_v3.zip']
    manifest=dict(created_at=now(),config_sha256=digest(CONFIG),database_sha256=digest(database),
        eligible_sha256=digest(root/'eligible.jsonl'),saved_targets=len(reviews),events=len(events),eligible=len(eligible),
        rejected=len(rejected),counts=dict(Counter(r['split']+'/'+r['branch'] for r in eligible)),
        converted_user_confirmed_target_keys=sorted(r['target_key'] for r in eligible if r['source_branch']!=r['branch']),
        formal_manifest_sha256=digest(formal/'manifest.json'),protected_files=len(protected),
        immutable_files={p:digest(PROJECT/p) for p in immutable},authorization=cfg['authorization'],
        unit_policy=cfg['unit_policy'],live_database_writes=0,source_UI_confirmation_fields_modified=False)
    atomic_json(root/'manifest.json',manifest)
    print(json.dumps({k:v for k,v in manifest.items() if k not in ('converted_user_confirmed_target_keys','immutable_files')},ensure_ascii=False),flush=True)
    return manifest


def feature_job(job):
    source,row=job;key=row['target_key'];path=ROOT/'dataset/features'/(key+'.npz')
    if path.exists():
        meta=json.loads(path.with_suffix('.json').read_text())
        if digest(path)!=meta['sha256']:raise ValueError('Feature hash differs')
        return meta
    try:
        old_root=PROJECT/'outputs/pallet_pose_benchmark_v3/roi_features'
        old=old_root/(key+'.npz');old_meta=old.with_suffix('.json')
        # Only numerical feature inputs are reused; labels are always latest snapshot.
        reuse=False
        if old.exists() and old_meta.exists():
            meta=json.loads(old_meta.read_text())
            scale=row['sensor']['scale_to_m'];original=row['source_sensor']['scale_to_m']
            # Unit reclassification changes automatic plane thresholds even
            # when the numeric scale is 1: recompute those inputs under metres.
            reuse=meta.get('source_pcd_sha256')==row['source_pcd_sha256'] and scale==original
            frozen=PROJECT/'outputs/pallet_pose_refinement_v2/cache/features'/(key+'.json')
            if frozen.exists():
                previous=json.loads(frozen.read_text())
                reuse=reuse and previous.get('scale_to_m')==original
            else:reuse=False
            if reuse and digest(old)!=meta['sha256']:raise ValueError('Previous ROI feature hash mismatch')
        if reuse:
            with np.load(old,allow_pickle=False) as z:
                f=z['features'][0].copy();g=z['globals'][0].copy()
                base=z['base'].copy();width=float(z['width']);direction=z['direction'].copy()
            anchor=meta['anchor'];roi_points=meta['roi_points']
        else:
            if digest(source['source_paths']['pcd'])!=row['source_pcd_sha256']:raise ValueError('Source PCD hash mismatch')
            _,points=read_pcd(source['source_paths']['pcd']);w,h=source['size']
            if len(points)!=w*h:raise ValueError('Source grid mismatch')
            mapping=orient_grid(np.arange(len(points),dtype=np.int32).reshape(h,w),source['sensor_policy'])
            box=[row['bbox'][k] for k in ('x','y','width','height')]
            values,anchor,roi_points=automatic_values(points,mapping,box,row['sensor']['scale_to_m'])
            f,g,base,width,direction=values
        # The same three permutation views of the deterministic 512-point sample
        # for new and reused inputs; training additionally adds coordinate jitter.
        fs=[f]
        for variant in (1,2):fs.append(f[np.random.default_rng(42+variant).permutation(len(f))])
        stream=io.BytesIO();np.savez_compressed(stream,features=np.stack(fs),globals=np.stack([g]*3),
            base=base,width=width,direction=direction);raw=stream.getvalue()
        protected_write(path,raw)
        with np.load(path,allow_pickle=False) as z:
            if not np.isfinite(z['features']).all() or not np.isfinite(z['globals']).all():raise ValueError('Nonfinite input features')
            target=residual_target(row,z['base'],float(z['width']),z['direction'])
            if not np.isfinite(target).all():raise ValueError('Nonfinite residual target')
        meta=dict(target_key=key,state='complete',sha256=digest(path),split=row['split'],branch=row['branch'],
            source_pcd_sha256=row['source_pcd_sha256'],bbox=row['bbox'],effective_scale_to_m=row['sensor']['scale_to_m'],
            anchor=anchor,roi_points=roi_points,reused_v3_numerical_inputs=reuse,
            human_polygon_face_ruler_are_inputs=False)
    except (ValueError,OSError) as exc:
        meta=dict(target_key=key,state='failed',reason=str(exc),split=row['split'],branch=row['branch'])
    atomic_json(path.with_suffix('.json'),meta)
    return meta


def automatic_values(points,mapping,box,scale):
    """Same deterministic inference input as v3, plus explicit fallback metadata."""
    height,width=mapping.shape;x,y,w,h=box
    b=dict(zip(('x','y','width','height'),box))
    mask=raster_mask([[x,y],[x+w,y],[x+w,y+h],[x,y+h]],width,height)
    vv,uu=np.nonzero(mask)
    if not len(vv):raise ValueError('Box contains no pixels')
    x0,x1,y0,y1=uu.min(),uu.max()+1,vv.min(),vv.max()+1
    grid=points[mapping[y0:y1,x0:x1],:3].astype(float);grid[~mask[y0:y1,x0:x1]]=np.nan
    result=decompose_roi(grid,dict(DEFAULTS,ransac_max_iterations=180,maximum_planes_per_region=6),
                         scale_to_mm=scale*1000 if scale is not None else None)
    ranked=sorted(result['faces'],key=lambda f:-f['points']*max(.01,abs(f['plane'][0]))**2*
                  (.05 if f['status']!='selected_candidate' or abs(f['plane'][2])>.75 else 1.))
    factor=scale if scale is not None else 1.
    ids=mapping[vv,uu];good=valid_xyz(points[ids,:3]);ids=ids[good]
    full_uv=np.column_stack((uu,vv))[good]
    full=points[ids].astype(float);full[:,:3]*=factor
    if len(full)<16:raise ValueError('Fewer than 16 valid original-box points')
    if not ranked or abs(ranked[0]['plane'][2])>.85:
        center=np.median(full[:,:3],axis=0)
        span=max(float(np.quantile(full[:,1],.99)-np.quantile(full[:,1],.01)),1e-8)
        candidate=dict(center=center,half_width=span/2,basis=np.column_stack(([0,1,0],[1,0,0],[0,0,1])))
        return feature_values(full,full_uv,[width,height],b,candidate,42,512),'roi_median_source_forward_fallback',len(full)
    face=ranked[0];uv=result['uv']+np.array([x0,y0]);ids=mapping[uv[:,1],uv[:,0]]
    xyzi=points[ids].astype(float);xyzi[:,:3]*=factor
    basis=basis_for(face['plane'][:3],dict(up_axis=2,up_sign=1,forward_axis=0,forward_sign=1))
    local=xyzi[face['indices'],:3]@basis;low,high=np.quantile(local,[.01,.99],axis=0)
    center=basis@np.array([(low[0]+high[0])/2,np.median(local[:,1]),np.median(local[:,2])])
    candidate=dict(center=center,basis=basis,half_width=max(float((high[0]-low[0])/2),1e-8))
    _,g,base,span,direction=feature_values(xyzi,uv,[width,height],b,candidate,42,512)
    f,_,_,_,_=feature_values(full,full_uv,[width,height],b,candidate,42,512)
    return (f,g,base,span,direction),'automatic_face_optional',len(full)


def prepare():
    manifest=snapshot();cfg=config()
    dataset_manifest=ROOT/'dataset/manifest.json'
    if dataset_manifest.exists():
        existing=json.loads(dataset_manifest.read_text())
        if existing['snapshot_sha256']!=digest(ROOT/'snapshot/manifest.json'):
            raise ValueError('Dataset snapshot binding changed')
        if existing['labels_sha256']!=digest(ROOT/'dataset/labels.jsonl') or existing['feature_index_sha256']!=digest(ROOT/'dataset/features.jsonl'):
            raise ValueError('Frozen dataset index changed')
        return existing
    formal=Path(json.loads((PROJECT/cfg['formal_config']).read_text())['dataset'])
    _,sources=load_dataset(formal);labels=read_rows(ROOT/'snapshot/eligible.jsonl')
    jobs=[(sources[r['frame_id']],r) for r in labels]
    results=[]
    with ProcessPoolExecutor(max_workers=cfg['workers'],mp_context=multiprocessing.get_context('spawn')) as pool:
        for i,result in enumerate(pool.map(feature_job,jobs),1):
            results.append(result)
            if i==1 or i%200==0:print(json.dumps(dict(stage='features_v4',completed=i,total=len(jobs))),flush=True)
    write_rows(ROOT/'dataset/features.jsonl',results)
    good={r['target_key'] for r in results if r['state']=='complete'}
    write_rows(ROOT/'dataset/labels.jsonl',[r for r in labels if r['target_key'] in good])
    info=dict(schema='latest_saved_pose_research_dataset_v4',created_at=now(),snapshot_sha256=digest(ROOT/'snapshot/manifest.json'),
              eligible_labels=len(labels),usable_features=len(good),failures=[r for r in results if r['state']!='complete'],
              counts=dict(Counter(r['split']+'/'+r['branch'] for r in labels if r['target_key'] in good)),
              labels_sha256=digest(ROOT/'dataset/labels.jsonl'),feature_index_sha256=digest(ROOT/'dataset/features.jsonl'),
              labels_are_authorized_research_drafts=True,robot_command_eligible=False)
    atomic_json(ROOT/'dataset/manifest.json',info)
    print(json.dumps(info,ensure_ascii=False),flush=True)
    return info


def load(branch,split='train'):
    all_rows=[r for r in read_rows(ROOT/'snapshot/eligible.jsonl') if r['branch']==branch and r['split']==split]
    records={r['target_key']:r for r in read_rows(ROOT/'dataset/features.jsonl')}
    accepted=[];parts={k:[] for k in ('f','g','y','base','width','direction')}
    for row in all_rows:
        record=records[row['target_key']]
        if record['state']!='complete':continue
        path=ROOT/'dataset/features'/(row['target_key']+'.npz')
        if digest(path)!=record['sha256']:raise ValueError('Feature identity changed')
        with np.load(path,allow_pickle=False) as z:
            for k,name in [('f','features'),('g','globals'),('base','base'),('width','width'),('direction','direction')]:parts[k].append(z[name].copy())
            parts['y'].append(residual_target(row,z['base'],float(z['width']),z['direction']))
        accepted.append(row)
    if not accepted:raise ValueError(f'No usable {branch}/{split} inputs')
    return accepted,{k:np.stack(v) for k,v in parts.items()},all_rows


if __name__=='__main__':
    import argparse
    parser=argparse.ArgumentParser();parser.add_argument('--stage',choices=['snapshot','prepare'],required=True)
    if parser.parse_args().stage=='snapshot':snapshot()
    else:prepare()
