"""Independent v5 snapshot and original-box XYZ inputs; never writes online data."""
from __future__ import annotations

from collections import Counter
from concurrent.futures import ProcessPoolExecutor
from contextlib import closing
import hashlib
import io
import json
import multiprocessing
import os
from pathlib import Path
import sqlite3

import numpy as np

from .cluster_targets import digest, load_dataset, protected_snapshot
from .data.orientation import orient_grid
from .data.pcd import read_pcd
from .fork_geometry import valid_xyz
from .pose_dataset import protected_write, read_rows, write_rows, write_json
from .pose_refinement import now, original_targets, residual_target, validate_review
from .pose_review_geometry import canonical, raster_mask
from .pose_review_units import MetrePolicy
from .pose_benchmark_v4_data import automatic_values

PROJECT = Path(__file__).resolve().parents[1]
CONFIG = PROJECT / 'configs/pallet_pose_paper_benchmark_v5.json'
ROOT = PROJECT / 'outputs/pallet_pose_paper_benchmark_v5'


def config():
    return json.loads(CONFIG.read_text())


def formal_data():
    formal = Path(json.loads((PROJECT / config()['formal_config']).read_text())['dataset'])
    return formal, *load_dataset(formal)


def snapshot():
    folder = ROOT / 'snapshot'
    folder.mkdir(parents=True, exist_ok=True)
    if (folder / 'manifest.json').exists():
        value = json.loads((folder / 'manifest.json').read_text())
        for name, expected in value['frozen_files'].items():
            if digest(PROJECT / name) != expected:
                raise ValueError('Frozen input changed: ' + name)
        return value
    cfg = config()
    formal, annotations, sources = formal_data()
    policy = MetrePolicy.load(PROJECT / cfg['unit_policy_file'], formal)
    if policy is None:
        raise ValueError('Unit policy does not bind the active formal dataset')
    targets = original_targets(annotations, sources)
    database = folder / 'reviews.sqlite3'
    if not database.exists():
        live = (PROJECT / cfg['review_directory'] / 'annotations.sqlite3').resolve()
        with closing(sqlite3.connect(live.as_uri() + '?mode=ro', uri=True)) as src:
            with closing(sqlite3.connect(database)) as dest:
                src.backup(dest)
        os.chmod(database, 0o600)
    with closing(sqlite3.connect(database.resolve().as_uri() + '?mode=ro', uri=True)) as db:
        if db.execute('PRAGMA quick_check').fetchone()[0] != 'ok':
            raise ValueError('Invalid snapshot database')
        reviews = [json.loads(v) for v, in db.execute('SELECT payload FROM annotations ORDER BY target_key')]
        events = [dict(target_key=k, revision=r, payload_sha256=hashlib.sha256(v.encode()).hexdigest())
                  for k, r, v in db.execute('SELECT target_key,revision,payload FROM events ORDER BY id')]
    accepted, rejected = [], []
    for review in reviews:
        try:
            if review['state'] in ('deleted', 'deferred'):
                raise ValueError('excluded_state_' + review['state'])
            target = targets[review['target_key']]
            source = sources[target['frame_id']]
            checked = validate_review(review, target, PROJECT / cfg['review_directory'])
            if review['sensor']['scale_to_m'] != policy.native_scale(source):
                raise ValueError('saved_scale_differs_from_authorized_policy_needs_explicit_conversion')
            geometry = PROJECT / cfg['review_directory'] / 'geometry'
            meta = json.loads((geometry / (review['geometry_token'] + '.json')).read_text())
            if checked['geometry_points_sha256'] != meta['full_resolution_artifact_sha256']:
                raise ValueError('geometry_evidence_hash_mismatch')
            if digest(source['source_paths']['pcd']) != target['source_pcd_sha256']:
                raise ValueError('source_pcd_changed')
            stream = io.BytesIO()
            np.savez_compressed(stream, selected_source_indices=checked['selected_source_indices'])
            support = folder / 'support' / (review['target_key'] + '.npz')
            protected_write(support, stream.getvalue())
            accepted.append(dict(**target, sensor=review['sensor'], pose=checked['pose'], branch='metric',
                source_sensor=review['sensor'], review_revision=review['revision'],
                source_review_sha256=hashlib.sha256(canonical(review)).hexdigest(),
                source_UI_human_pose_confirmed=review['human_pose_confirmed'],
                source_UI_axes_confirmed=review['sensor']['axes_confirmed'],
                label_origin='user_authorized_saved_research_draft_not_blind_physical_GT',
                support_sha256=digest(support), robot_command_eligible=False,
                **{k: checked[k] for k in ('geometry_metadata_sha256', 'geometry_points_sha256')}))
        except (KeyError, ValueError, OSError) as exc:
            rejected.append(dict(target_key=review.get('target_key'), revision=review.get('revision'), reason=str(exc)))
    for field in ('group_id', 'source_pcd_sha256'):
        seen = {}
        for row in accepted:
            if seen.setdefault(row[field], row['split']) != row['split']:
                raise ValueError('Original split leakage: ' + field)
    write_rows(folder / 'eligible.jsonl', accepted)
    write_rows(folder / 'rejected.jsonl', rejected)
    write_rows(folder / 'source_reviews.jsonl', reviews)
    write_rows(folder / 'event_fingerprints.jsonl', events)
    write_json(folder / 'protected_formal.json', protected_snapshot(formal))
    immutable = [CONFIG, PROJECT / cfg['unit_policy_file'], database,
                 folder / 'eligible.jsonl', folder / 'source_reviews.jsonl', folder / 'event_fingerprints.jsonl']
    # Freeze previous experiments, delivery archives and active pointers by hash.
    immutable += list((PROJECT / 'configs').glob('active*.json'))
    for version in ('pallet_pose_benchmark_v4', 'pallet_pose_benchmark_v3', 'pallet_pose_refinement_v2'):
        previous = PROJECT / 'outputs' / version
        immutable += list(previous.glob('*.zip')) + list(previous.glob('*manifest*.json'))
    value = dict(created_at=now(), saved_targets=len(reviews), events=len(events), eligible=len(accepted),
        rejected=len(rejected), rejection_reasons=dict(Counter(r['reason'] for r in rejected)),
        counts=dict(Counter(r['split'] for r in accepted)),
        groups={s: len({r['group_id'] for r in accepted if r['split'] == s}) for s in ('train','val','test')},
        unit_policy=policy.info(), frozen_files={str(p.relative_to(PROJECT)): digest(p) for p in immutable},
        live_database_writes=0, UI_confirmation_flags_changed=False, robot_command_eligible=False)
    write_json(folder / 'manifest.json', value)
    print(json.dumps(dict(stage='snapshot', **{k:v for k,v in value.items() if k != 'frozen_files'}), ensure_ascii=False), flush=True)
    return value


def feature_job(job):
    source, row = job
    key = row['target_key']
    path = ROOT / 'dataset/features' / (key + '.npz')
    meta_path = path.with_suffix('.json')
    if meta_path.exists():
        meta = json.loads(meta_path.read_text())
        if meta['state'] == 'complete' and digest(path) != meta['sha256']:
            raise ValueError('Feature changed')
        return meta
    try:
        if digest(source['source_paths']['pcd']) != row['source_pcd_sha256']:
            raise ValueError('PCD identity mismatch')
        _, points = read_pcd(source['source_paths']['pcd'])
        w, h = source['size']
        if len(points) != w*h:
            raise ValueError('Unorganized input')
        mapping = orient_grid(np.arange(len(points), dtype=np.int32).reshape(h,w), source['sensor_policy'])
        box = [row['bbox'][k] for k in ('x','y','width','height')]
        x,y,bw,bh = box
        mask = raster_mask([[x,y],[x+bw,y],[x+bw,y+bh],[x,y+bh]],w,h)
        vv, uu = np.nonzero(mask)
        ids = mapping[vv,uu]
        good = valid_xyz(points[ids,:3])
        ids, uu, vv = ids[good], uu[good], vv[good]
        scale = row['sensor']['scale_to_m']
        raw = points[ids].astype(np.float64)
        raw[:,:3] *= scale
        if len(raw) < 16:
            raise ValueError('Fewer than 16 valid ROI points')
        # Only raw ROI and automatic geometry enter these arrays. No ruler or mask.
        values, anchor, count = automatic_values(points, mapping, box, scale)
        f,g,base,width,direction = values
        fs = [f] + [f[np.random.default_rng(42+i).permutation(len(f))] for i in (1,2)]
        stream = io.BytesIO()
        np.savez_compressed(stream, features=np.stack(fs), globals=np.stack([g]*3), base=base,
            width=width, direction=direction)
        protected_write(path, stream.getvalue())
        stream = io.BytesIO()
        np.savez_compressed(stream, xyz=raw[:,:3].astype(np.float32), uv=np.column_stack((uu,vv)),
            source_indices=ids, size=[w,h])
        roi = ROOT / 'dataset/roi' / (key + '.npz')
        protected_write(roi, stream.getvalue())
        meta = dict(target_key=key, state='complete', sha256=digest(path), roi_sha256=digest(roi),
            split=row['split'], branch='metric', source_pcd_sha256=row['source_pcd_sha256'],
            bbox=row['bbox'], effective_scale_to_m=scale, anchor=anchor, roi_points=count,
            human_polygon_face_ruler_are_inputs=False)
    except (ValueError,OSError) as exc:
        meta = dict(target_key=key, state='failed', split=row['split'], branch='metric', reason=str(exc))
    write_json(meta_path, meta)
    return meta


def prepare():
    snapshot()
    path = ROOT / 'dataset/manifest.json'
    if path.exists():
        return json.loads(path.read_text())
    _,_,sources = formal_data()
    labels = read_rows(ROOT / 'snapshot/eligible.jsonl')
    jobs = [(sources[r['frame_id']],r) for r in labels]
    values = []
    with ProcessPoolExecutor(max_workers=config()['workers'], mp_context=multiprocessing.get_context('spawn')) as pool:
        for i,value in enumerate(pool.map(feature_job,jobs),1):
            values.append(value)
            if i == 1 or i % 100 == 0:
                print(json.dumps(dict(stage='prepare_v5', completed=i,total=len(jobs))),flush=True)
    write_rows(ROOT / 'dataset/features.jsonl',values)
    good = {v['target_key'] for v in values if v['state']=='complete'}
    write_rows(ROOT / 'dataset/labels.jsonl',[r for r in labels if r['target_key'] in good])
    result = dict(created_at=now(), snapshot_sha256=digest(ROOT/'snapshot/manifest.json'),
        labels_sha256=digest(ROOT/'dataset/labels.jsonl'),feature_index_sha256=digest(ROOT/'dataset/features.jsonl'),
        total=len(labels),usable_features=len(good),failures=[v for v in values if v['state']!='complete'])
    write_json(path,result)
    print(json.dumps(dict(stage='dataset_complete',**result)),flush=True)
    return result


def load(branch='metric',split='train'):
    if branch != 'metric':
        raise ValueError('Only policy-confirmed metric samples in v5')
    all_rows = [r for r in read_rows(ROOT/'snapshot/eligible.jsonl') if r['split']==split]
    records = {r['target_key']:r for r in read_rows(ROOT/'dataset/features.jsonl')}
    rows, parts = [], {k:[] for k in ('f','g','y','base','width','direction')}
    for row in all_rows:
        record = records[row['target_key']]
        if record['state']!='complete':continue
        path = ROOT/'dataset/features'/(row['target_key']+'.npz')
        if digest(path)!=record['sha256']:raise ValueError('Feature hash mismatch')
        with np.load(path,allow_pickle=False) as z:
            for k,name in [('f','features'),('g','globals'),('base','base'),('width','width'),('direction','direction')]:
                parts[k].append(z[name].copy())
            parts['y'].append(residual_target(row,z['base'],float(z['width']),z['direction']))
        rows.append(row)
    return rows,{k:np.stack(v) for k,v in parts.items()},all_rows


if __name__ == '__main__':
    import argparse
    parser=argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--stage',choices=['snapshot','prepare'],required=True)
    args=parser.parse_args()
    snapshot() if args.stage=='snapshot' else prepare()
