"""Independently check every generated target, original box, split and archive.

Does not promote predictions to GT, modify original data or access the live
review database. Optional real-inference checks use original TRAIN frames only.
"""
from __future__ import annotations

import argparse
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
import hashlib
import json
from pathlib import Path
import sys
from zipfile import ZipFile

import numpy as np
import torch

PROJECT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(PROJECT))
from pallet.cluster_targets import digest, load_dataset, protected_snapshot, target_id, target_key
from pallet.data.orientation import orient_grid
from pallet.data.pcd import read_pcd
from pallet.pose_dataset import read_rows, safe_path, write_json
from pallet.pose_inference import PalletPosePredictor


def verify(config_path, spot_checks=20):
    config_path = Path(config_path).resolve()
    config = json.loads(config_path.read_text())
    out = PROJECT/config['output']
    formal = Path(json.loads((PROJECT/config['formal_config']).read_text())['dataset'])
    active_pose = json.loads((PROJECT/config['pose_dataset_config']).read_text())
    pose_root = PROJECT/active_pose['dataset']
    run = json.loads((out/'run_manifest.json').read_text())
    manifest = json.loads((out/'annotation_manifest.json').read_text())
    delivery = json.loads((out/'delivery_manifest.json').read_text())
    summary = json.loads((out/'summary.json').read_text())
    assert digest(config_path) == run['binding']['config_sha256']
    assert digest(formal/'manifest.json') == run['binding']['formal_manifest_sha256']
    assert digest(pose_root/'manifest.json') == run['binding']['source_pose_manifest_sha256']
    assert digest(out/'run_manifest.json') == manifest['run_manifest_sha256']
    assert summary == manifest['summary'] == delivery['summary']
    assert digest(delivery['archive']) == delivery['archive_sha256']
    for name, expected in run['binding']['code'].items():
        assert digest(PROJECT/'pallet'/name) == expected
        assert digest(out/'reproduction/pallet'/name) == expected
    for name, expected in manifest['files'].items():
        assert digest(safe_path(out, name)) == expected
    with ZipFile(delivery['archive']) as z:
        assert len(z.namelist()) == len(set(z.namelist())) == delivery['archive_members_verified']
        assert set(z.namelist()) == set(manifest['files']) | {'annotation_manifest.json', 'checksums.sha256'}
        for name in z.namelist():
            assert hashlib.sha256(z.read(name)).hexdigest() == digest(safe_path(out, name))
    annotations, sources = load_dataset(formal)
    original_by_id = {a['frame_id']: a for a in annotations}
    targets = read_rows(out/'annotations/targets.jsonl')
    frames = read_rows(out/'annotations/frames.jsonl')
    original_keys = {target_key(a['frame_id'], i) for a in annotations for i in range(len(a['boxes']))}
    assert len(targets) == len(original_keys) and {r['target_key'] for r in targets} == original_keys
    assert len(frames) == len(sources) == len({f['frame_id'] for f in frames})
    assert {r['frame_id'] for r in frames} == set(sources)
    by_frame = {fid: [] for fid in sources}
    known = {r['detection_target_key']: r for r in read_rows(pose_root/'annotations.jsonl') if r['subset'] == 'overlap'}
    for r in targets:
        source = sources[r['frame_id']]
        a = original_by_id[r['frame_id']]
        index = r['box_index']
        assert r['target_id'] == target_id(source['frame_id'], index)
        assert r['target_key'] == target_key(source['frame_id'], index)
        assert r['bbox'] == a['boxes'][index]
        assert r['split'] == r['original_split'] == source['split']
        assert r['group_id'] == source['group_id'] and r['domain'] == source['domain']
        assert r['source_pcd_sha256'] == source['source_anchor_hash']
        assert r['source_pcd'] == source['source_paths']['pcd']
        assert r['image'] == source['image']
        assert r['human_detection_box_confirmed'] and not r['human_pose_confirmed']
        assert not r['automatic_training_eligible'] and not r['evaluation_ground_truth_eligible']
        assert not r['full_six_dof'] and not r['robot_command_eligible']
        assert not r['existing_pose_gt_replaced'] and not r['detector_rerun']
        assert r['review_required'] and not any(r['pose_gt_supervision_mask'].values())
        assert r['model_pose_confidence'] is None
        assert r['point_units_verified'] == (source['sensor_policy'].get('scale_to_mm') is not None)
        gt = known.get(r['target_key'])
        if gt:
            assert r['existing_pose_gt_reference']['sample_id'] == gt['sample_id']
            assert r['existing_pose_gt_reference']['task'] == gt['pose']['kind']
        else:
            assert r['existing_pose_gt_reference'] is None
        pose = r['pose_estimate']
        if pose is None:
            assert not any(r['pose_estimate_mask'].values())
            assert r['metric_pose_estimate'] is None
            assert r['review_state'] == 'unavailable_pending_review'
        else:
            assert r['input_quality']['plane_success']
            assert r['review_state'] == 'machine_pose_pending_review'
            assert all(pose[k] is None for k in ('z', 'normal', 'roll', 'pitch', 'yaw', 'quaternion'))
            assert np.isfinite([pose[k] for k in ('x', 'y', 'theta_degrees')]).all()
            assert -180 <= pose['theta_degrees'] <= 180
            assert not pose['full_six_dof'] and not pose['robot_command_eligible']
            if r['point_units_verified']:
                assert r['metric_pose_estimate'] == pose and pose['position_unit'] == 'm'
            else:
                assert r['metric_pose_estimate'] is None
                assert pose['position_unit'] == 'research_assumed_m_not_calibrated'
                assert 'point_units_unverified_research_assumption' in r['review_flags']
        if source['split'] != 'train':
            assert 'original_holdout_annotation_not_a_training_label' in r['review_flags']
        if source['instances'] > 1:
            assert 'multi_target_scene_not_in_pose_training' in r['review_flags']
        by_frame[r['frame_id']].append(r)
    original_shape_count = 0
    for frame in frames:
        source = sources[frame['frame_id']]
        original = json.loads(safe_path(formal, source['annotation']).read_text())
        sidecar = json.loads(safe_path(out, frame['labelme']).read_text())
        assert frame['target_keys'] == [r['target_key'] for r in by_frame[frame['frame_id']]]
        assert frame['original_status'] == source['status']
        assert frame['original_instances'] == source['instances']
        assert sidecar['pallet_pose_annotation']['split'] == source['split']
        assert not sidecar['pallet_pose_annotation']['automatic_training_eligible']
        assert (safe_path(out, frame['labelme']).parent/sidecar['imagePath']).resolve() == safe_path(formal, source['image']).resolve()
        assert sidecar['imageWidth'] == original['imageWidth'] and sidecar['imageHeight'] == original['imageHeight']
        assert len(sidecar['shapes']) == len(original['shapes'])
        for new, old in zip(sidecar['shapes'], original['shapes']):
            assert all(new[k] == old[k] for k in ('label', 'shape_type', 'points', 'group_id', 'text'))
            for k, value in old['flags'].items():
                assert new['flags'][k] == value
            if 'pose_machine_annotation' in new:
                assert new['pose_machine_annotation'] == by_frame[frame['frame_id']][new['pose_machine_annotation']['box_index']]
                assert not new['flags']['human_pose_confirmed']
                original_shape_count += 1
        if source['status'] == 'negative':
            assert not sidecar['shapes'] and not frame['target_keys']
    assert original_shape_count == len(targets)
    for split in ('train', 'val', 'test'):
        assert read_rows(out/f'indices/{split}.jsonl') == [r for r in targets if r['split'] == split]
    assert (out/'indices/automatic_training_eligible.jsonl').read_bytes() == b''
    computed = {'frames': len(frames), 'targets': len(targets),
        'negative_frames_preserved': sum(not f['target_keys'] for f in frames),
        'pose_estimates': sum(r['pose_estimate'] is not None for r in targets),
        'pose_unavailable': sum(r['pose_estimate'] is None for r in targets),
        'metric_unit_pose_estimates_pending_review': sum(r['metric_pose_estimate'] is not None for r in targets),
        'research_unit_pose_estimates_pending_review': sum(r['pose_estimate'] is not None and not r['point_units_verified'] for r in targets),
        'frame_split_counts': dict(Counter(r['split'] for r in frames)),
        'target_split_counts': dict(Counter(r['split'] for r in targets)),
        'review_flag_counts': dict(Counter(f for r in targets for f in r['review_flags']))}
    for key, value in computed.items():
        assert summary[key] == value, key
    # Third independent read of PCD fingerprints. No changes are written to raw files.
    def source_unchanged(source):
        assert digest(source['source_paths']['pcd']) == source['source_anchor_hash']
    with ThreadPoolExecutor(max_workers=8) as pool:
        list(pool.map(source_unchanged, sources.values()))
    assert protected_snapshot(formal) == run['protected_formal_files']
    # Match current output to the published predictor on stratified original TRAIN
    # targets only. This is implementation equivalence, not model accuracy testing.
    selected = {}
    train = [r for r in targets if r['split'] == 'train' and r['pose_estimate'] is not None]
    rank = lambda r: hashlib.sha256(r['target_key'].encode()).hexdigest()
    for row in sorted(train, key=rank):
        source = sources[row['frame_id']]
        key = (source['domain'], row['point_units_verified'], source['instances'] > 1,
            source['pixel_rotation_degrees'])
        selected.setdefault(key, row)
    checks = list(selected.values())
    used = {r['target_key'] for r in checks}
    for r in sorted(train, key=rank):
        if len(checks) >= spot_checks:
            break
        if r['target_key'] not in used:
            checks.append(r); used.add(r['target_key'])
    if not spot_checks:
        checks = []
    torch.set_num_threads(1)
    if checks:
        predictor = PalletPosePredictor(PROJECT/config['model_bundle'], 'cpu', allow_unverified_units=True)
        for row in checks:
            source = sources[row['frame_id']]
            _, points = read_pcd(source['source_paths']['pcd'])
            width, height = source['size']
            mapping = orient_grid(np.arange(len(points), dtype=np.int32).reshape(height, width), source['sensor_policy'])
            scale = source['sensor_policy'].get('scale_to_mm')
            b = row['bbox']; box = [b[k] for k in ('x', 'y', 'width', 'height')]
            result = predictor.pose(points, mapping, box, width, height,
                scale_to_m=scale/1000 if scale is not None else None)
            assert np.allclose(result['prediction'], row['raw_model_output_diagnostic_only'], atol=2e-6, rtol=1e-6)
    report = {'state': 'passed', 'frames_verified': len(frames), 'human_boxes_verified': len(targets),
        'labelme_rectangles_and_human_2d_flags_unchanged': original_shape_count,
        'all_negatives_preserved': computed['negative_frames_preserved'],
        'original_PCDs_independently_rehashed_unchanged': len(sources),
        'protected_formal_labels_metadata_unchanged': len(run['protected_formal_files']),
        'all_original_splits_preserved': True, 'existing_pose_GT_references_unchanged': len(known),
        'full_six_dof_fabricated': False, 'automatic_training_eligible': 0,
        'archive_sha256': delivery['archive_sha256'], 'archive_members_verified': delivery['archive_members_verified'],
        'train_only_prediction_equivalence_samples': len(checks),
        'train_only_prediction_equivalence_target_keys': [r['target_key'] for r in checks],
        'pose_accuracy_or_generalization_evaluation_performed': False,
        'production_review_database_accessed': False, 'training_started': False,
        'verifier_sha256': digest(__file__)}
    write_json(out/'validation/independent_verification.json', report)
    write_json(out/'validation/model_and_source_provenance.json', run['binding'])
    print(json.dumps(report, ensure_ascii=False), flush=True)
    return report


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--config', type=Path, default=PROJECT/'configs/pallet_pose_annotations_v1.json')
    parser.add_argument('--spot-checks', type=int, default=20)
    args = parser.parse_args()
    verify(args.config, args.spot_checks)
