#!/usr/bin/env python3
"""Portable handoff verifier; no source PCD, project import or CUDA required."""
from __future__ import annotations

import argparse
from collections import Counter, defaultdict
import hashlib
import json
import math
from pathlib import Path, PurePosixPath
import shutil
import stat
import zipfile


CHUNK = 4 * 1024 * 1024
EXPERIMENTS = [
    ('yolov8s', 'png_intensity_rgb'), ('yolo26s', 'png_intensity_rgb'),
    ('rfdetr-small', 'png_intensity_rgb'), ('ecdet-s', 'png_intensity_rgb'),
    ('yolo26s', 'float_intensity_mask'), ('yolo26s', 'float_depth_intensity_masks'),
    ('rfdetr-small', 'float_intensity_mask'), ('rfdetr-small', 'float_depth_intensity_masks'),
]


def digest(path):
    value = hashlib.sha256()
    with Path(path).open('rb') as stream:
        for block in iter(lambda: stream.read(CHUNK), b''):
            value.update(block)
    return value.hexdigest()


def read(path):
    return json.loads(Path(path).read_text())


def safe_path(root, name):
    part = PurePosixPath(name)
    if (not name or not part.parts or '\\' in name or part.is_absolute() or '..' in part.parts
            or ':' in part.parts[0] or str(part) != name):
        raise ValueError(f'Unsafe relative path: {name!r}')
    root = Path(root).resolve()
    target = root / name
    if not target.resolve().is_relative_to(root) or target.is_symlink():
        raise ValueError(f'Path escapes dataset or is a symlink: {name}')
    return target


def validate_config(config):
    if config.get('kind') != 'pallet_baseline_training_handoff':
        raise ValueError('Not a baseline handoff configuration')
    if config.get('implementation_status') != 'pending' or config.get('execution_status') != 'not_started':
        raise ValueError('Handoff must not claim implemented or completed training')
    experiments = config['experiments']
    if ([(x['model'], x['input']) for x in experiments] != EXPERIMENTS
            or len({x['id'] for x in experiments}) != 8
            or any(x.get('implementation_status') != 'pending'
                   or x.get('execution_status') != 'not_started' for x in experiments)):
        raise ValueError('Eight frozen pending experiments do not match the approved task')
    train = config['training']
    if (train['seed'], train['size'], train['effective_batch'], train['epochs'], train['patience']) != (42, 640, 16, 300, 50):
        raise ValueError('Approved training protocol changed')
    if any(train[key] for key in ('mosaic', 'mixup', 'multi_scale', 'p2', 'gated_fusion', 'depth_edge')):
        raise ValueError('Unapproved model or augmentation change')
    if config['dataset']['independent_blind_reference'] is not False:
        raise ValueError('Human-assisted labels cannot be claimed to be blind references')
    for name in config['paths'].values():
        safe_path(Path('.'), name)
    return experiments


def verify_frozen_files(root, manifest_name='manifest.json'):
    root = Path(root)
    manifest = read(root / manifest_name)
    expected = manifest['files']
    entries = (root / 'checksums.sha256').read_text().splitlines()
    parsed = [line.split('  ', 1) for line in entries]
    if any(len(row) != 2 for row in parsed):
        raise ValueError('Malformed checksum table')
    table = {row[1]: row[0] for row in parsed}
    if len(table) != len(parsed) or table != {**expected, manifest_name: digest(root / manifest_name)}:
        raise ValueError('Checksum table does not match frozen manifest')
    for index, (name, wanted) in enumerate(sorted(expected.items()), 1):
        path = safe_path(root, name)
        if not path.is_file() or digest(path) != wanted:
            raise ValueError(f'Missing or changed frozen file: {name}')
        if len(expected) > 10000 and index % 10000 == 0:
            print(json.dumps({'checking_frozen_files': index, 'total': len(expected)}), flush=True)
    return {'state': 'passed', 'files': len(expected) + 1}


def verify_handoff(root):
    root = Path(root)
    manifest = read(root / 'manifest.json')
    if manifest.get('kind') != 'pallet_baseline_training_handoff' or manifest.get('state') != 'frozen':
        raise ValueError('Not a frozen handoff package')
    audit = verify_frozen_files(root)
    config = read(root / 'handoff_config.json')
    validate_config(config)
    archive = safe_path(root, config['paths']['dataset_archive'])
    if manifest['files'][config['paths']['dataset_archive']] != config['dataset']['archive_sha256']:
        raise ValueError('Embedded dataset differs from approved archive hash')
    with zipfile.ZipFile(archive) as zipped:
        dataset_manifest = json.loads(zipped.read('manifest.json'))
        if hashlib.sha256(zipped.read('manifest.json')).hexdigest() != config['dataset']['manifest_sha256']:
            raise ValueError('Embedded dataset manifest hash differs')
        summary = dataset_manifest['summary']
        for key in ('frames', 'boxes', 'groups'):
            if summary[key] != config['dataset'][key]:
                raise ValueError(f'Dataset count changed: {key}')
        if summary['split_counts'] != config['dataset']['splits']:
            raise ValueError('Dataset split counts changed')
    return {**audit, 'experiments': 8, 'dataset': summary['frames'], 'training_started': False}


def dataset_zip_entries(zipped):
    infos = zipped.infolist()
    names = [info.filename for info in infos]
    if len(names) != len(set(names)):
        raise ValueError('Duplicate ZIP entries')
    for info in infos:
        safe_path(Path('.'), info.filename.rstrip('/'))
        if stat.S_ISLNK(info.external_attr >> 16):
            raise ValueError('ZIP symlinks are not supported')
    manifest_raw = zipped.read('manifest.json')
    manifest = json.loads(manifest_raw)
    expected = {**manifest['files'], 'manifest.json': hashlib.sha256(manifest_raw).hexdigest(),
                'checksums.sha256': hashlib.sha256(zipped.read('checksums.sha256')).hexdigest()}
    if set(names) != set(expected):
        raise ValueError('Dataset ZIP membership differs from its manifest')
    return infos, expected


def unpack_dataset(root, *, minimum_free_gib=50):
    root = Path(root).resolve()
    config = read(root / 'handoff_config.json')
    archive = safe_path(root, config['paths']['dataset_archive'])
    if digest(archive) != config['dataset']['archive_sha256']:
        raise ValueError('Dataset archive SHA-256 differs; do not extract')
    destination = safe_path(root, config['paths']['dataset_root'])
    if destination.exists():
        if digest(destination / 'manifest.json') != config['dataset']['manifest_sha256']:
            raise ValueError('Existing dataset has a different manifest; refusing overwrite')
        return {**verify_frozen_files(destination), 'extraction': 'existing_verified'}
    staging = destination.with_name(destination.name + '.unpacking')
    binding = root / '.dataset_unpack_state.json'
    wanted_binding = {'dataset_archive_sha256': config['dataset']['archive_sha256'],
                      'destination': config['paths']['dataset_root']}
    if staging.exists():
        if not binding.is_file() or read(binding) != wanted_binding:
            raise ValueError('Existing extraction staging is not bound to this archive')
    else:
        staging.mkdir(parents=True)
        binding.write_text(json.dumps(wanted_binding, indent=2) + '\n')
    with zipfile.ZipFile(archive) as zipped:
        infos, expected = dataset_zip_entries(zipped)
        required = sum(i.file_size for i in infos if not (staging / i.filename).exists())
        if shutil.disk_usage(root).free < required + minimum_free_gib * 2**30:
            raise RuntimeError('Extraction would leave less than the required disk reserve')
        for index, info in enumerate(infos, 1):
            target = safe_path(staging, info.filename)
            if target.exists():
                if not target.is_file() or digest(target) != expected[info.filename]:
                    raise ValueError(f'Existing extraction payload modified: {info.filename}')
                continue
            target.parent.mkdir(parents=True, exist_ok=True)
            temporary = target.with_name(target.name + '.unpack-part')
            value = hashlib.sha256()
            with zipped.open(info) as source, temporary.open('wb') as output:
                for block in iter(lambda: source.read(CHUNK), b''):
                    value.update(block)
                    output.write(block)
            if value.hexdigest() != expected[info.filename]:
                raise ValueError(f'Extraction payload checksum differs: {info.filename}')
            temporary.replace(target)
            if index % 5000 == 0:
                print(json.dumps({'unpacking_and_hashing': index, 'total': len(infos)}), flush=True)
        actual = {str(p.relative_to(staging)) for p in staging.rglob('*') if p.is_file()}
        if actual != set(expected):
            raise ValueError('Unexpected extraction staging files')
    staging.replace(destination)
    binding.unlink()
    return {'state': 'passed', 'extraction': 'completed', 'files': len(infos),
            'all_member_sha256': 'passed', 'zip_member_crc': 'passed'}


def _same_boxes(actual, expected, *, tolerance=1e-4):
    if len(actual) != len(expected):
        raise ValueError('Box count differs between annotation formats')
    if any(len(a) != 4 or any(not math.isfinite(v) for v in a)
           or any(abs(x-y) > tolerance for x, y in zip(a, e)) for a, e in zip(actual, expected)):
        raise ValueError('Box coordinates differ between annotation formats')


def verify_portable_dataset(root, config, *, hash_payloads=True):
    import numpy as np
    from PIL import Image

    root = Path(root).resolve()
    if digest(root / 'manifest.json') != config['dataset']['manifest_sha256']:
        raise ValueError('Dataset manifest identity differs')
    if digest(root / 'annotations/frames.jsonl') != config['dataset']['annotations_sha256']:
        raise ValueError('Authoritative human labels differ')
    hash_audit = verify_frozen_files(root) if hash_payloads else {'state': 'already_verified_during_extraction'}
    rows = [json.loads(line) for line in (root/'frames.jsonl').read_text().splitlines() if line]
    annotations = [json.loads(line) for line in (root/'annotations/frames.jsonl').read_text().splitlines() if line]
    by_id = {x['frame_id']: x for x in annotations}
    if len(by_id) != len(annotations) or len(rows) != len(by_id) or len({r['frame_id'] for r in rows}) != len(rows):
        raise ValueError('Missing or duplicate frame identity')
    splits = read(root/'splits.json')
    tensor_index = read(root/'tensor_index.json')
    groups, anchors = defaultdict(set), defaultdict(set)
    counts, statuses = Counter(), Counter()
    coco_images, coco_boxes = {}, {}
    for split in ('train', 'val', 'test'):
        coco = read(root/f'annotations/{split}.coco.json')
        if coco['categories'] != [{'id': 1, 'name': 'pallet'}]:
            raise ValueError('COCO single-class mapping differs')
        images = {im['id']: im for im in coco['images']}
        if len(images) != len(coco['images']):
            raise ValueError('Duplicate COCO image IDs')
        for im in images.values():
            if im['frame_id'] in coco_images:
                raise ValueError('COCO image crosses splits')
            coco_images[im['frame_id']] = im
        boxes = defaultdict(list)
        ids = set()
        for box in coco['annotations']:
            if box['id'] in ids or box['category_id'] != 1 or box['iscrowd'] != 0:
                raise ValueError('Duplicate COCO object or changed class/ignore status')
            ids.add(box['id'])
            boxes[images[box['image_id']]['frame_id']].append(box['bbox'])
        coco_boxes.update(boxes)
    samples = {}
    for row in rows:
        fid, split = row['frame_id'], row['split']
        annotation = by_id[fid]
        w, h = annotation['width'], annotation['height']
        if (row['human_reviewed'] is not True or row['proposal_only'] is not False
                or row['independent_blind_reference'] is not False
                or annotation['split'] != split or row['status'] != annotation['status']
                or row['group_id'] != annotation['group_id']
                or splits['assignments'][row['group_id']] != split
                or fid not in splits['frame_ids'][split]):
            raise ValueError(f'Human status/split/group changed: {fid}')
        if not annotation['provenance'] or any(p['source'] != 'human' for p in annotation['provenance']):
            raise ValueError(f'Non-human annotation source: {fid}')
        groups[row['group_id']].add(split)
        anchors[row['source_anchor_hash']].add(split)
        truth = [[b['x'], b['y'], b['width'], b['height']] for b in annotation['boxes']]
        if row['status'] not in {'pos', 'negative'} or bool(truth) != (row['status'] == 'pos'):
            raise ValueError(f'Unresolved or conflicting positive/negative status: {fid}')
        for x, y, bw, bh in truth:
            if not all(math.isfinite(v) for v in (x, y, bw, bh)) or not (0 <= x < w and 0 <= y < h and bw > 0 and bh > 0 and x+bw <= w+1e-4 and y+bh <= h+1e-4):
                raise ValueError(f'Invalid bounding box: {fid}')
        doc = read(safe_path(root, row['annotation']))
        if (doc['imageWidth'], doc['imageHeight']) != (w, h) or doc['imagePath'] != PurePosixPath(row['image']).name:
            raise ValueError(f'AnyLabeling dimensions/path changed: {fid}')
        flags = doc['flags']
        if flags.get('human_reviewed') is not True or any(flags.get(k) for k in ('unreviewed', 'proposal_only', 'ignore', 'label_conflict')) or flags.get('confirmed_negative') != (not truth):
            raise ValueError(f'AnyLabeling human/negative flags changed: {fid}')
        actual = []
        for box in doc['shapes']:
            if box['label'] != 'pallet' or box['shape_type'] != 'rectangle' or len(box['points']) != 2:
                raise ValueError(f'Unknown AnyLabeling shape: {fid}')
            (x1, y1), (x2, y2) = box['points']
            actual.append([min(x1,x2), min(y1,y2), abs(x2-x1), abs(y2-y1)])
        _same_boxes(actual, truth)
        yolo = []
        for line in safe_path(root, row['yolo_label']).read_text().splitlines():
            parts = line.split()
            if len(parts) != 5 or parts[0] != '0':
                raise ValueError(f'Invalid YOLO mapping: {fid}')
            cx, cy, bw, bh = [float(v) for v in parts[1:]]
            yolo.append([(cx-bw/2)*w, (cy-bh/2)*h, bw*w, bh*h])
        _same_boxes(yolo, truth, tolerance=1e-3)
        im = coco_images[fid]
        if (im['file_name'], im['width'], im['height'], im['split']) != (row['image'], w, h, split):
            raise ValueError(f'COCO dimensions/path/split changed: {fid}')
        _same_boxes(coco_boxes.get(fid, []), truth)
        for key in ('image', 'depth', 'depth_validity', 'intensity_validity'):
            with Image.open(safe_path(root, row[key])) as image:
                if image.size != (w, h):
                    raise ValueError(f'PNG native grid changed: {fid}')
        if tensor_index[fid]['path'] != row['sensor'] or not safe_path(root, row['sensor']).is_file():
            raise ValueError(f'NPZ relative index changed: {fid}')
        sample_key = (split, row['domain'], row['depth_available'], row['pixel_rotation_degrees'], row['reconstructed_intensity'])
        samples.setdefault(sample_key, row)
        counts[split] += 1
        statuses[row['status']] += 1
    if any(len(s) != 1 for s in list(groups.values()) + list(anchors.values())):
        raise ValueError('Capture group/source content crosses splits')
    for split in counts:
        if set(splits['frame_ids'][split]) != {r['frame_id'] for r in rows if r['split'] == split}:
            raise ValueError('Frozen split frame coverage differs')
    if set(coco_images) != set(by_id) or set(tensor_index) != set(by_id):
        raise ValueError('COCO or tensor index has omitted/extra frames')
    total_boxes = sum(len(a['boxes']) for a in annotations)
    expected = config['dataset']
    if (len(rows), total_boxes, len(groups), statuses['pos'], statuses['negative']) != (expected['frames'], expected['boxes'], expected['groups'], expected['positives'], expected['negatives']) or dict(counts) != expected['splits']:
        raise ValueError('Portable dataset totals differ from approved task')
    for row in samples.values():
        annotation = by_id[row['frame_id']]
        with np.load(safe_path(root, row['sensor']), allow_pickle=False) as cache:
            data = cache['frame']
            metadata = json.loads(str(cache['metadata']))
            if (data.dtype != np.float32 or data.shape != (annotation['height'], annotation['width'], 4)
                    or not np.isfinite(data).all() or not np.isin(data[..., 2:], (0,1)).all()
                    or np.any(data[..., :2][data[..., 2:] == 0] != 0)
                    or metadata['source_sha256'] != row['source_anchor_hash']
                    or metadata.get('pixel_rotation_degrees',0) != row['pixel_rotation_degrees']
                    or bool(np.any(data[..., 2])) != row['depth_available']):
                raise ValueError(f'Portable NPZ values/identity/masks changed: {row["frame_id"]}')
            for key, channel in [('depth_validity',2), ('intensity_validity',3)]:
                with Image.open(safe_path(root, row[key])) as image:
                    if not np.array_equal(np.asarray(image), data[...,channel]*255):
                        raise ValueError('Portable NPZ mask differs from PNG')
    return {'state': 'passed', 'frames': len(rows), 'boxes': total_boxes, 'groups': len(groups),
            'split_counts': dict(counts), 'three_annotation_formats': 'passed',
            'all_frame_png_dimensions': 'passed', 'npz_stratified_samples': len(samples),
            'npz_sample_splits': sorted({r['split'] for r in samples.values()}),
            'group_and_source_split_overlap': 0, 'raw_source_access_required': False,
            'frozen_payload': hash_audit}


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--root', type=Path, default=Path(__file__).resolve().parents[1])
    parser.add_argument('--unpack-dataset', action='store_true')
    parser.add_argument('--check-data', action='store_true')
    args = parser.parse_args()
    result = {'handoff': verify_handoff(args.root)}
    extraction = None
    if args.unpack_dataset:
        extraction = unpack_dataset(args.root)
        result['extraction'] = extraction
    if args.check_data:
        config = read(args.root/'handoff_config.json')
        result['portable_data'] = verify_portable_dataset(
            safe_path(args.root, config['paths']['dataset_root']), config,
            hash_payloads=not extraction or extraction.get('extraction') != 'completed')
    print(json.dumps(result, ensure_ascii=False, indent=2), flush=True)


if __name__ == '__main__':
    main()
