"""Freeze user-reviewed labels and portable sensor inputs, preserving source splits.

Human acceptance is explicit release evidence; it never fabricates the two blind
machine reviews required by the historical research workflow.
"""
from __future__ import annotations

import argparse
from collections import Counter, defaultdict
from copy import deepcopy
from dataclasses import asdict, replace
from datetime import datetime
import hashlib
import json
from pathlib import Path, PurePosixPath
import shutil
import zipfile
from zoneinfo import ZoneInfo

import numpy as np
from PIL import Image

from pallet.annotation import (FrameAnnotation, Provenance, SceneAttributes,
                               annotation_digest, load_jsonl)
from pallet.clean_intensity_review import reserve, safe_bytes, safe_copy, safe_json
from pallet.data.eligibility import require_eligible
from pallet.data.loader import load_frame
from pallet.data.protocol import file_digest, load_protocol, write_json
from pallet.human_labeling import _load_frame_index, _decode_rectangles
from pallet.manual_update import validate_document
from pallet.review_batches import display


def read(path):
    return json.loads(Path(path).read_text())


def jsonl(path, values):
    safe_bytes(path, ''.join(json.dumps(v, ensure_ascii=False, sort_keys=True) + '\n'
                            for v in values).encode())


def stamp(output, stage, **values):
    data = {'stage': stage, 'updated_at': datetime.now(ZoneInfo('Asia/Shanghai')).isoformat(), **values}
    write_json(Path(output)/'workflow_status.json', data)
    print(json.dumps(data, ensure_ascii=False), flush=True)


def validate_manual_frame(frame, record, split, source):
    frame.validate()
    if (frame.status not in {'pos', 'negative'} or frame.ignore_regions or frame.dimensions_pending
            or not frame.provenance or any(p.source != 'human' for p in frame.provenance)):
        raise ValueError(f'Only resolved human corrections may enter this release: {frame.frame_id}')
    expected = (record.group_id, record.source_sha256, split, record.domain,
                tuple(reversed(record.sensor_config['grid'])))
    actual = (frame.group_id, frame.source_anchor_hash, frame.split, frame.domain,
              (frame.width, frame.height))
    if actual != expected:
        raise ValueError(f'Human label identity/grid mismatch: {frame.frame_id}')
    for key, value in {'group_id': frame.group_id, 'source_anchor_hash': frame.source_anchor_hash,
                       'split': split, 'width': frame.width, 'height': frame.height,
                       'annotation_digest': annotation_digest(frame)}.items():
        if source.get(key) != value:
            raise ValueError(f'Human source receipt mismatch ({key}): {frame.frame_id}')
    if not source.get('archive_sha256'):
        raise ValueError('Missing human source archive receipt')


def import_return(archive, source, by_id, assignments, parent_ids, *, expected_ids):
    """Reject partial, duplicate, changed-identity or out-of-bounds corrections."""
    source = Path(source)
    manifest, index, filenames, manifest_sha, _ = _load_frame_index(source)
    if set(index) != set(expected_ids):
        raise ValueError('Return source is not the exact remaining population')
    incoming, entries, differences = {}, {}, []
    archive_sha = file_digest(archive)
    with zipfile.ZipFile(archive) as z:
        if z.testzip() is not None:
            raise ValueError('Correction ZIP CRC failed')
        seen_names = set()
        for item in z.infolist():
            if item.is_dir():
                continue
            name = PurePosixPath(item.filename)
            if (name.is_absolute() or '..' in name.parts or name.suffix.lower() != '.json'
                    or item.filename in seen_names):
                raise ValueError(f'Unexpected/duplicate/unsafe correction member: {item.filename}')
            seen_names.add(item.filename)
            row = filenames.get(name.name)
            if row is None:
                raise ValueError(f'Unknown returned filename: {item.filename}')
            fid = row['frame_id']
            if fid in parent_ids or fid in incoming or fid not in expected_ids:
                raise ValueError(f'Duplicate or unexpected correction: {fid}')
            record = by_id[fid]
            require_eligible(record)
            document = json.loads(z.read(item))
            baseline = read(source/row['annotation'])
            boxes = validate_document(document, baseline, row, source, record)
            meta = document.get('pallet_prediction') or document.get('pallet_export') or {}
            split = assignments[record.group_id]
            if (meta.get('group_id') != record.group_id or meta.get('original_split') != split
                    or row['split'] != split or meta.get('domain') != record.domain
                    or meta.get('pixel_rotation_degrees', 0) != record.sensor_config.get('pixel_rotation_degrees', 0)):
                raise ValueError(f'Returned split/group/domain/orientation mismatch: {fid}')
            frame = FrameAnnotation(fid, record.group_id, row['image'], *row['size'],
                status='pos' if boxes else 'negative', boxes=boxes, split=split, domain=record.domain,
                scene=SceneAttributes(domain=record.domain), source_anchor_hash=record.source_sha256,
                provenance=[Provenance('human', actor='user_confirmed_archive_import',
                    note=f'Completed remaining mixed corrections returned by user; archive_sha256={archive_sha}')])
            frame.validate()
            unchanged = boxes == _decode_rectangles(baseline, *row['size'])
            incoming[fid] = frame
            entries[fid] = {'frame_id': fid, 'group_id': record.group_id, 'split': split,
                'domain': record.domain, 'width': frame.width, 'height': frame.height,
                'status': frame.status, 'instances': len(boxes), 'source_anchor_hash': record.source_sha256,
                'annotation_digest': annotation_digest(frame), 'archive': str(Path(archive).resolve()),
                'archive_sha256': archive_sha, 'archive_member': item.filename,
                'source_json_sha256': hashlib.sha256(z.read(item)).hexdigest(),
                'baseline_source_json_sha256': file_digest(source/row['annotation']),
                'source_export_manifest_sha256': manifest_sha, 'geometry_unchanged_from_proposal': unchanged}
            differences.append({'frame_id': fid, 'original_split': split,
                'old_boxes_xywh': [asdict(b) for b in _decode_rectangles(baseline, *row['size'])],
                'human_boxes_xywh': [asdict(b) for b in boxes], 'geometry_unchanged_after_review': unchanged,
                'old_flags': baseline.get('flags'), 'returned_flags': document.get('flags'),
                'accepted_status': frame.status})
    if set(incoming) != set(expected_ids):
        raise ValueError('Returned corrections are incomplete')
    return incoming, entries, differences, manifest


def prepare(config_path):
    config_path = Path(config_path).resolve()
    config = read(config_path)
    for key in ('all_returned_frames_reviewed', 'unchanged_boxes_accepted', 'empty_frames_are_confirmed_negative'):
        if config.get(key) is not True:
            raise ValueError(f'Missing established user-review convention: {key}')
    output, base, protocol = map(Path, (config['output'], config['base_output'], config['protocol']))
    source, full = map(Path, (config['correction_source'], config['full_source_export']))
    inputs = [config_path, Path(__file__), Path(config['archive']), protocol/'protocol.json',
        base/'manual_import_manifest.json', base/'manual_partition.json',
        base/'inputs/all_checked_annotations.jsonl', base/'inputs/manual_source_frames.json',
        Path(config['external_source_receipts']), Path(config['cache_index']),
        source/'package_manifest.json', source/'frames.jsonl', full/'package_manifest.json', full/'frames.jsonl',
        Path(config['orientation_manifest']), Path(config['reconstruction_manifest']), Path(config['exclusions'])]
    binding = {str(p.resolve()): file_digest(p) for p in inputs}
    completion = output/'import_manifest.json'
    if completion.exists():
        prior = read(completion)
        if prior['input_binding'] != binding:
            raise ValueError('Release inputs changed; choose a new version')
        for name, digest in prior['artifact_sha256'].items():
            if file_digest(output/name) != digest:
                raise ValueError(f'Preserving changed release snapshot: {name}')
        return config, prior
    records, assignments, proto = load_protocol(protocol)
    by_id = {r.frame_id: r for r in records}
    parent_manifest = read(base/'manual_import_manifest.json')
    # Verify immutable label and raw-return snapshots; the root mixed.zip was
    # intentionally replaced by this new return, so its historical copy is used.
    for name, digest in parent_manifest['artifact_sha256'].items():
        if file_digest(base/name) != digest:
            raise ValueError(f'Parent human snapshot changed: {name}')
    checked = load_jsonl(base/'inputs/all_checked_annotations.jsonl').frames
    entries = read(base/'inputs/manual_source_frames.json')
    external = read(config['external_source_receipts'])
    for fid in set(checked)-set(entries):
        entries[fid] = external[fid]
    expected = set(by_id)-set(checked)
    incoming, new_entries, differences, source_manifest = import_return(
        config['archive'], source, by_id, assignments, set(checked), expected_ids=expected)
    if source_manifest['prediction_binding']['protocol_sha256'] != file_digest(protocol/'protocol.json'):
        raise ValueError('Returned source references a different sensor protocol')
    checked = {**checked, **incoming}
    entries = {**entries, **new_entries}
    if set(checked) != set(by_id) or set(entries) != set(by_id):
        raise ValueError('Human-only release does not cover every active frame')
    _, full_index, _, _, _ = _load_frame_index(full)
    _, correction_index, _, _, _ = _load_frame_index(source)
    if set(full_index) != set(by_id) or set(read(config['cache_index'])) != set(by_id):
        raise ValueError('Native image/tensor sources do not cover every active frame')
    for fid, frame in checked.items():
        require_eligible(by_id[fid])
        validate_manual_frame(frame, by_id[fid], assignments[frame.group_id], entries[fid])
        if fid in incoming and file_digest(full/full_index[fid]['image']) != file_digest(source/correction_index[fid]['image']):
            raise ValueError('Correction pixels differ from the canonical native image')
        checked[fid] = replace(frame, image=full_index[fid]['image'])
    recovered = read(config['reconstruction_manifest'])
    recovered_ids = {row['frame_id'] for row in recovered}
    priority_sha = config['reconstructed_corrections_sha256']
    if any(entries[fid]['archive_sha256'] != priority_sha for fid in recovered_ids):
        raise ValueError('Checked reconstructed-intensity archive lost its priority')
    orientation = read(config['orientation_manifest'])
    if any(by_id[row['frame_id']].sensor_config.get('pixel_rotation_degrees') != 180 for row in orientation):
        raise ValueError('Frozen orientation policy changed')
    excluded_ids = {row['frame_id'] for row in read(config['exclusions'])['frames']}
    if excluded_ids & set(checked):
        raise ValueError('Retired black frame re-entered the release')
    output.mkdir(parents=True, exist_ok=True)
    stamp(output, 'importing', new_frames=len(incoming), total_frames=len(checked))
    bundle = output/'dataset'
    jsonl(bundle/'annotations/frames.jsonl', (asdict(checked[fid]) for fid in sorted(checked)))
    safe_json(bundle/'provenance/human_source_frames.json', entries)
    safe_json(bundle/'provenance/latest_correction_differences.json', differences)
    safe_copy(base/'inputs/all_checked_annotations.jsonl', bundle/'provenance/parent_human_annotations.jsonl')
    safe_copy(base/'manual_import_manifest.json', bundle/'provenance/parent_import_manifest.json')
    safe_copy(config_path, bundle/'provenance/release_config.json')
    safe_json(bundle/'provenance/user_review_convention.json', {
        'literal_current_statement': config['user_statement'], 'reviewer_count_claimed': 1,
        'interpretation': 'User returned the completed remaining package; established session conventions accept unchanged boxes and empty frames as negatives.',
        'gui_flags_are_historical': True, 'independent_blind_reference': False,
        'no_second_reviewer_fabricated': True})
    archive_records = []
    sources = parent_manifest['sources'] + [{'archive': config['archive'], 'archive_sha256': file_digest(config['archive'])}]
    for ordinal, row in enumerate(sources, 1):
        candidate = Path(row['archive'])
        if not candidate.exists() or file_digest(candidate) != row['archive_sha256']:
            candidate = base/'inputs/source_archives'/Path(row['archive']).name
        if not candidate.exists() or file_digest(candidate) != row['archive_sha256']:
            raise ValueError(f'Missing frozen human archive: {row["archive_sha256"]}')
        target = bundle/'provenance/source_archives'/f'{ordinal:02d}_{candidate.stem}_{row["archive_sha256"][:12]}.zip'
        safe_copy(candidate, target)
        archive_records.append({**row, 'frozen_copy': str(target.relative_to(bundle))})
    safe_json(bundle/'provenance/archive_sources.json', archive_records)
    for role, path in [('orientation_manifest', config['orientation_manifest']),
                       ('reconstruction_manifest', config['reconstruction_manifest']), ('exclusions', config['exclusions'])]:
        safe_copy(path, bundle/'provenance'/f'{role}.json')
    for name in proto['files']:
        safe_copy(protocol/name, bundle/'protocol'/name)
    safe_copy(protocol/'protocol.json', bundle/'provenance/parent_sensor_protocol.json')
    if (protocol/'quarantine.json').exists():
        safe_copy(protocol/'quarantine.json', bundle/'provenance/quarantine.json')
    summary = {'frames': len(checked), 'new_frames': len(incoming),
        'status_counts': dict(Counter(f.status for f in checked.values())),
        'boxes': sum(len(f.boxes) for f in checked.values()), 'groups': len(assignments),
        'split_counts': {s: sum(f.split == s for f in checked.values()) for s in ('train', 'val', 'test')},
        'new_positive': sum(f.status == 'pos' for f in incoming.values()),
        'new_negative': sum(f.status == 'negative' for f in incoming.values()),
        'new_boxes': sum(len(f.boxes) for f in incoming.values()),
        'new_geometry_unchanged': sum(d['geometry_unchanged_after_review'] for d in differences),
        'reconstructed_intensity_frames': len(recovered_ids), 'orientation_corrected_frames': len(orientation),
        'irrecoverable_black_excluded': len(excluded_ids), 'other_protocol_exclusions': proto['excluded']-len(excluded_ids),
        'remaining_unreviewed': 0, 'reviewer_count_claimed': 1, 'independent_blind_reference': False}
    prior = {'state': 'imported', 'created_at': datetime.now(ZoneInfo('Asia/Shanghai')).isoformat(),
        'input_binding': binding, 'summary': summary,
        'artifact_sha256': {str(p.relative_to(output)): file_digest(p) for p in bundle.rglob('*') if p.is_file()}}
    safe_json(completion, prior)
    return config, prior


def coco(frames):
    images, boxes = [], []
    for image_id, f in enumerate(frames, 1):
        images.append({'id': image_id, 'file_name': f.image, 'width': f.width, 'height': f.height,
            'frame_id': f.frame_id, 'group_id': f.group_id, 'domain': f.domain, 'split': f.split,
            'status': f.status, 'human_reviewed': True, 'independent_blind_reference': False,
            'annotation_digest': annotation_digest(f)})
        for b in f.boxes:
            boxes.append({'id': len(boxes)+1, 'image_id': image_id, 'category_id': 1,
                          'bbox': [b.x, b.y, b.width, b.height], 'area': b.width*b.height, 'iscrowd': 0})
    return {'images': images, 'annotations': boxes, 'categories': [{'id': 1, 'name': 'pallet'}],
            'info': {'label_origin': 'user_reviewed_corrections', 'reviewer_count_claimed': 1,
                     'independent_blind_reference': False}}


def yolo(frame):
    return ''.join(f'0 {(b.x+b.width/2)/frame.width:.10f} {(b.y+b.height/2)/frame.height:.10f} '
                   f'{b.width/frame.width:.10f} {b.height/frame.height:.10f}\n' for b in frame.boxes)


def document(frame, row):
    return {'version': '0.4.43', 'flags': {'human_reviewed': True, 'confirmed_negative': frame.status == 'negative',
        'unreviewed': False, 'proposal_only': False, 'ignore': False, 'label_conflict': False, 'machine_resolved': False},
        'imagePath': Path(frame.image).name, 'imageData': None, 'imageWidth': frame.width, 'imageHeight': frame.height,
        'shapes': [{'label': 'pallet', 'shape_type': 'rectangle',
            'points': [[b.x, b.y], [b.x+b.width, b.y+b.height]], 'group_id': None, 'text': '',
            'flags': {'human_reviewed': True, 'proposal_only': False}} for b in frame.boxes],
        'pallet_export': {k: row[k] for k in ('frame_id', 'group_id', 'split', 'domain', 'source_anchor_hash',
            'status', 'annotation_digest', 'pixel_rotation_degrees', 'reconstructed_intensity')},
        'pallet_dataset': {'version': 'formal_v1', 'human_reviewed': True, 'reviewer_count_claimed': 1,
            'independent_blind_reference': False, 'label_origin': 'human_corrected_model_assisted'}}


def export(config, imported):
    output = Path(config['output']); bundle = output/'dataset'
    records, assignments, parent = load_protocol(config['protocol'])
    labels = load_jsonl(bundle/'annotations/frames.jsonl').frames
    sources = read(bundle/'provenance/human_source_frames.json')
    _, native, _, _, _ = _load_frame_index(Path(config['full_source_export']))
    caches = read(config['cache_index'])
    reconstructed = {r['frame_id'] for r in read(config['reconstruction_manifest'])}
    orientation = {r['frame_id']: r for r in read(config['orientation_manifest'])}
    frame_rows, tensor_index = [], {}
    stats = {s: {'frames': 0, 'positive': 0, 'negative': 0, 'boxes': 0, 'groups': set()} for s in ('train', 'val', 'test')}
    domains = defaultdict(lambda: {s: {'frames': 0, 'positive': 0, 'negative': 0, 'boxes': 0, 'groups': set()} for s in stats})
    for ordinal, record in enumerate(records, 1):
        reserve()
        fid = record.frame_id; f = labels[fid]; src = native[fid]; cache = caches[fid]
        validate_manual_frame(f, record, assignments[record.group_id], sources[fid])
        data, metadata = load_frame(replace(record, cache_path=cache['path'], cache_sha256=cache['sha256']), return_metadata=True)
        if (not np.isin(data[..., 2:], (0, 1)).all() or np.any(data[..., :2][data[..., 2:] == 0] != 0)):
            raise ValueError(f'Invalid/missing sensor mask semantics: {fid}')
        key = Path(f.image).stem
        sensor = f'sensors/{f.split}/{key}.npz'
        if fid in orientation and cache['sha256'] != orientation[fid]['corrected_tensor_sha256']:
            raise ValueError(f'Corrected tensor lineage changed: {fid}')
        source_image = Path(config['full_source_export'])/src['image']
        image_sha = file_digest(source_image)
        if src.get('view_sha256', {}).get('image') and image_sha != src['view_sha256']['image']:
            raise ValueError(f'Canonical intensity PNG changed: {fid}')
        with Image.open(source_image) as im:
            pixels = np.asarray(im)
            if im.size != (f.width, f.height) or not pixels.any():
                raise ValueError(f'Invalid/black native intensity preview: {fid}')
            if not np.array_equal(pixels, display(data[..., 1], data[..., 3], log=True)):
                raise ValueError(f'Native intensity PNG does not match its floating tensor: {fid}')
        safe_copy(source_image, bundle/f.image)
        safe_copy(cache['path'], bundle/sensor)
        aux = f'auxiliary/{f.split}/{key}'
        depth = f'{aux}.depth.png'; valid_d = f'{aux}.depth_valid.png'; valid_i = f'{aux}.intensity_valid.png'
        # Review previews may expose unverified raw depth; only the NPZ validD
        # mask and frozen sensor policy authorize metric geometry for training.
        safe_copy(Path(config['full_source_export'])/src['depth'], bundle/depth)
        for name, channel in [(valid_d, 2), (valid_i, 3)]:
            from io import BytesIO
            payload = BytesIO(); Image.fromarray((data[..., channel]*255).astype(np.uint8)).save(payload, format='PNG')
            safe_bytes(bundle/name, payload.getvalue())
        row = {'frame_id': fid, 'group_id': f.group_id, 'split': f.split, 'original_split': f.split,
            'domain': f.domain, 'source_anchor_hash': f.source_anchor_hash, 'size': [f.width, f.height],
            'status': f.status, 'instances': len(f.boxes), 'image': f.image, 'annotation': str(Path(f.image).with_suffix('.json')),
            'yolo_label': f'labels/{f.split}/{key}.txt', 'sensor': sensor, 'depth': depth,
            'depth_validity': valid_d, 'intensity_validity': valid_i, 'annotation_digest': annotation_digest(f),
            'pixel_rotation_degrees': record.sensor_config.get('pixel_rotation_degrees', 0),
            'reconstructed_intensity': fid in reconstructed, 'depth_available': bool(data[..., 2].any()),
            'intensity_available': bool(data[..., 3].any()), 'sensor_policy': record.sensor_config,
            'source_paths': {k: str(v) for k, v in record.paths.items()}, 'human_source': sources[fid],
            'human_reviewed': True, 'proposal_only': False, 'independent_blind_reference': False,
            'depth_preview_is_metric_input': False, 'sensor_metadata': metadata}
        safe_json(bundle/row['annotation'], document(f, row))
        safe_bytes(bundle/row['yolo_label'], yolo(f).encode())
        row['files'] = {row[role]: file_digest(bundle/row[role]) for role in
                        ('image', 'annotation', 'yolo_label', 'sensor', 'depth', 'depth_validity', 'intensity_validity')}
        tensor_index[fid] = {'path': sensor, 'sha256': cache['sha256'], 'size': cache['size']}
        frame_rows.append(row)
        for counts in (stats[f.split], domains[f.domain][f.split]):
            counts['frames'] += 1; counts['positive' if f.status == 'pos' else 'negative'] += 1
            counts['boxes'] += len(f.boxes); counts['groups'].add(f.group_id)
        if ordinal % 128 == 0 or ordinal == len(records):
            stamp(output, 'exporting', frames_done=ordinal, frames_total=len(records))
    for s in stats:
        safe_json(bundle/f'annotations/{s}.coco.json', coco([f for f in labels.values() if f.split == s]))
    safe_json(bundle/'splits.json', {'split_seed': parent['split_seed'], 'assignments': assignments,
        'frame_ids': {s: sorted(f.frame_id for f in labels.values() if f.split == s) for s in stats},
        'original_splits_preserved': True, 'original_groups_preserved': True})
    jsonl(bundle/'frames.jsonl', frame_rows)
    safe_json(bundle/'tensor_index.json', tensor_index)
    for counts in list(stats.values())+[counts for domain in domains.values() for counts in domain.values()]:
        counts['groups'] = len(counts['groups'])
    safe_json(bundle/'statistics.json', {'summary': imported['summary'], 'splits': stats, 'domains': dict(domains),
        'depth_verified_frames': sum(r['depth_available'] for r in frame_rows),
        'unknown_scene_attributes_retained': True, 'normalization_fitted': False})
    # Without a path override, Ultralytics resolves relative image directories
    # against yaml_file.parent; "path: ." incorrectly uses the caller's cwd.
    safe_bytes(bundle/'dataset.yaml', b'train: images/train\nval: images/val\ntest: images/test\nnames:\n  0: pallet\n')
    protocol = deepcopy(parent)
    protocol.update(version=8, label_snapshot='../annotations/frames.jsonl',
        label_snapshot_sha256=file_digest(bundle/'annotations/frames.jsonl'),
        label_origin='user_reviewed_corrections', independent_blind_reference=False,
        parent_protocol_sha256=file_digest(Path(config['protocol'])/'protocol.json'))
    protocol['files']['../annotations/frames.jsonl'] = protocol['label_snapshot_sha256']
    safe_json(bundle/'protocol/protocol.json', protocol)
    safe_bytes(bundle/'README_zh.md', (
        '# 托盘检测正式人工数据集 v1\n\n'
        f'冻结时间：{imported["created_at"]}。全量{len(records)}帧，仅采用用户已检查的标签。\n\n'
        '原train/val/test与采集组不变。`images/<split>`为原生强度PNG及AnyLabeling JSON，'
        '`labels/<split>`为YOLO标签，`annotations/<split>.coco.json`为COCO；空TXT是已确认负例。'
        '所有框覆盖可见托盘本体；未经人工确认的场景属性仍为unknown。\n\n'
        '`sensors/<split>/*.npz`中frame为float32 H×W×4，通道顺序：深度(mm)、原始强度、深度有效掩码、强度有效掩码。'
        '保留未归一化数值；未知深度及无效像素为0，掩码显式表示缺测。训练只在train拟合归一化，禁止全库拟合。'
        '主PNG是显示归一化的强度图，不能替代四通道传感器输入；dataset.yaml适用于PNG单模态对照。\n\n'
        'auxiliary含同坐标深度预览和两个有效掩码。深度预览仅供辨认，部分原始深度的单位/配准仍未验证，'
        '不得代替NPZ的深度通道。118帧重建强度采用专题人工修正；300帧240×96已校正180°，不要再次旋转。'
        '148帧不可恢复黑图永久移除，其他不可用源帧见provenance/exclusions.json及protocol/excluded.json。\n\n'
        'labels是人工确认的模型辅助标注，声称1名用户审阅，不冒充双审或独立盲审。历史辅助模型训练/选轮接触过原val/test，'
        '其成绩不能作为独立正式测试；后续研究需从未接触目标划分的初始化训练，仅train拟合、val选轮、test在方案冻结后评分。'
        '本次构建未训练模型、未计算测试指标。\n\n'
        'frames.jsonl保留源身份、组、域、方向、标签/图像/传感器哈希；provenance保存全部人工归档、优先覆盖记录及父标签。'
        'protocol内原始PCD/YAML路径指向外部只读原始库，便携训练使用随包NPZ；无需复制84GiB原始库。\n\n'
        '完整性检查：`sha256sum -c checksums.sha256`。项目入口：'
        '`.venv/bin/python -m pallet.formal_dataset verify --dataset /path/to/dataset`。'
        '更改人工标签应创建新版本，冻结版本不回写历史模型或研究参考。\n'
    ).encode())
    return bundle


def verify_dataset(bundle, *, check_manifest=True):
    bundle = Path(bundle)
    rows = [json.loads(line) for line in (bundle/'frames.jsonl').read_text().splitlines() if line]
    labels = load_jsonl(bundle/'annotations/frames.jsonl').frames
    splits = read(bundle/'splits.json')
    sources = read(bundle/'provenance/human_source_frames.json')
    if len(rows) != len(labels) or {r['frame_id'] for r in rows} != set(labels):
        raise ValueError('Duplicate/missing release frames')
    group_splits, hashes_splits = defaultdict(set), defaultdict(set)
    for row in rows:
        f = labels[row['frame_id']]
        group_splits[f.group_id].add(f.split); hashes_splits[f.source_anchor_hash].add(f.split)
        if (row['annotation_digest'] != annotation_digest(f) or row['status'] != f.status
                or row['group_id'] != f.group_id or row['split'] != f.split
                or splits['assignments'][f.group_id] != f.split or row['human_source'] != sources[f.frame_id]):
            raise ValueError('Frozen frame/split/source identity changed')
        for name, expected in row['files'].items():
            path = bundle/name
            if not path.resolve().is_relative_to(bundle.resolve()) or file_digest(path) != expected:
                raise ValueError(f'Payload modified: {name}')
        doc = read(bundle/row['annotation'])
        if (doc != document(f, row) or (bundle/row['yolo_label']).read_text() != yolo(f)
                or _decode_rectangles(doc, f.width, f.height) != f.boxes):
            raise ValueError('AnyLabeling/YOLO label differs from frozen human boxes')
        for role in ('image', 'depth', 'depth_validity', 'intensity_validity'):
            with Image.open(bundle/row[role]) as image:
                image.load()
                if image.size != (f.width, f.height):
                    raise ValueError('Native PNG grid mismatch')
        with np.load(bundle/row['sensor'], allow_pickle=False) as cache:
            data = cache['frame']; meta = json.loads(str(cache['metadata']))
            if (data.dtype != np.float32 or data.shape != (f.height, f.width, 4)
                    or not np.isfinite(data).all() or not np.isin(data[..., 2:], (0, 1)).all()
                    or np.any(data[..., :2][data[..., 2:] == 0] != 0)
                    or meta['source_sha256'] != f.source_anchor_hash
                    or meta.get('pixel_rotation_degrees', 0) != row['pixel_rotation_degrees']):
                raise ValueError('Floating sensor payload/masks/identity invalid')
            for role, channel in [('depth_validity', 2), ('intensity_validity', 3)]:
                with Image.open(bundle/row[role]) as mask:
                    if not np.array_equal(np.asarray(mask), data[..., channel]*255):
                        raise ValueError('PNG mask differs from floating sensor support')
    if any(len(v) != 1 for v in list(group_splits.values())+list(hashes_splits.values())):
        raise ValueError('Capture group or source hash crosses splits')
    for s in ('train', 'val', 'test'):
        frames = [f for f in labels.values() if f.split == s]
        if set(splits['frame_ids'][s]) != {f.frame_id for f in frames} or read(bundle/f'annotations/{s}.coco.json') != coco(frames):
            raise ValueError('COCO/split membership differs from human snapshot')
    if check_manifest:
        manifest = read(bundle/'manifest.json')
        files = {str(p.relative_to(bundle)) for p in bundle.rglob('*') if p.is_file()}-{'manifest.json', 'checksums.sha256'}
        if set(manifest['files']) != files:
            raise ValueError('Unexpected/missing release payload file')
        for name, digest in manifest['files'].items():
            if file_digest(bundle/name) != digest:
                raise ValueError(f'Release payload changed: {name}')
    return {'state': 'passed', 'frames': len(rows), 'boxes': sum(len(f.boxes) for f in labels.values()),
            'groups': len(group_splits), 'split_counts': dict(Counter(f.split for f in labels.values())),
            'human_only': True, 'native_grid_and_masks': 'passed', 'group_and_source_split_overlap': 0,
            'all_formats_same_boxes': True, 'unreviewed_frames': 0}


def verify_archive(archive, bundle):
    files = {str(p.relative_to(bundle)): p for p in Path(bundle).rglob('*') if p.is_file()}
    with zipfile.ZipFile(archive) as z:
        if (z.testzip() is not None or len(z.infolist()) != len(files)
                or set(z.namelist()) != set(files)):
            raise ValueError('Release ZIP CRC/membership failed')
        for name, path in files.items():
            digest = hashlib.sha256()
            with z.open(name) as stream:
                for chunk in iter(lambda: stream.read(4*1024*1024), b''):
                    digest.update(chunk)
            if digest.hexdigest() != file_digest(path):
                raise ValueError(f'Release ZIP payload mismatch: {name}')
    return {'crc': 'passed', 'all_member_sha256': 'passed', 'members': len(files),
            'sha256': file_digest(archive), 'bytes': Path(archive).stat().st_size}


def verify_frozen_payload(bundle):
    """Check immutable bytes after semantic audit without decoding NPZ again."""
    bundle = Path(bundle)
    manifest = read(bundle/'manifest.json')
    files = {str(p.relative_to(bundle)) for p in bundle.rglob('*') if p.is_file()}-{'manifest.json', 'checksums.sha256'}
    if set(manifest['files']) != files:
        raise ValueError('Unexpected/missing frozen payload file')
    for name, digest in manifest['files'].items():
        if file_digest(bundle/name) != digest:
            raise ValueError(f'Frozen payload changed: {name}')
    entries = (bundle/'checksums.sha256').read_text().splitlines()
    checksums = dict(line.split('  ', 1)[::-1] for line in entries)
    expected = {**manifest['files'], 'manifest.json': file_digest(bundle/'manifest.json')}
    if len(entries) != len(checksums) or checksums != expected:
        raise ValueError('Checksum table differs from frozen payload manifest')
    return {'state': 'passed', 'files': len(expected), 'immutable_payload_unchanged': True}


def build(config_path):
    config, imported = prepare(config_path)
    output = Path(config['output'])
    completed = output/'delivery_manifest.json'
    if completed.exists():
        data = read(completed)
        verify_dataset(data['dataset']); verify_archive(data['archive'], data['dataset'])
        return data
    bundle = export(config, imported)
    stamp(output, 'auditing_dataset')
    audit = verify_dataset(bundle, check_manifest=False)
    safe_json(bundle/'audit.json', audit)
    manifest = {'kind': 'formal_user_reviewed_pallet_dataset', 'version': 1, 'state': 'frozen',
        'created_at': imported['created_at'], 'summary': imported['summary'],
        'input_binding': imported['input_binding'], 'label_origin': 'human_corrected_model_assisted',
        'reviewer_count_claimed': 1, 'independent_blind_reference': False,
        'original_splits_preserved': True, 'original_groups_preserved': True,
        'normalization_fitted': False, 'historical_assistant_final_test_eligible': False,
        'files': {str(p.relative_to(bundle)): file_digest(p) for p in sorted(bundle.rglob('*'))
                  if p.is_file() and p.name not in {'manifest.json', 'checksums.sha256'}}}
    safe_json(bundle/'manifest.json', manifest)
    safe_bytes(bundle/'checksums.sha256', ''.join(f'{file_digest(p)}  {p.relative_to(bundle)}\n'
        for p in sorted(bundle.rglob('*')) if p.is_file() and p.name != 'checksums.sha256').encode())
    archive = output/'pallet_dataset_formal_v1.zip'
    stamp(output, 'packaging', dataset_audit=audit)
    if not archive.exists():
        temporary = archive.with_suffix('.zip.partial')
        with zipfile.ZipFile(temporary, 'w', compression=zipfile.ZIP_DEFLATED, compresslevel=1, allowZip64=True) as z:
            for ordinal, p in enumerate(sorted(bundle.rglob('*')), 1):
                if not p.is_file():
                    continue
                reserve(); z.write(p, str(p.relative_to(bundle)))
                if ordinal % 1024 == 0:
                    stamp(output, 'packaging', paths_visited=ordinal, zip_bytes=temporary.stat().st_size)
        temporary.replace(archive)
    stamp(output, 'auditing_archive')
    archive_audit = verify_archive(archive, bundle)
    # The semantic audit above covered every frame. Byte identity proves the
    # packaging step preserved it; repeating all image/NPZ decoding adds no test.
    verify_frozen_payload(bundle)
    delivery = {'state': 'complete', 'completed_at': datetime.now(ZoneInfo('Asia/Shanghai')).isoformat(),
        'dataset': str(bundle.resolve()), 'archive': str(archive.resolve()), 'summary': imported['summary'],
        'dataset_audit': audit, 'archive_audit': archive_audit,
        'dataset_manifest_sha256': file_digest(bundle/'manifest.json'),
        'annotations_sha256': file_digest(bundle/'annotations/frames.jsonl'),
        'normalization_fitted': False, 'models_trained_this_task': 0, 'final_test_scored': False}
    safe_json(completed, delivery)
    stamp(output, 'complete', **delivery)
    return delivery


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    sub = parser.add_subparsers(dest='command', required=True)
    for command in ('prepare', 'build'):
        p = sub.add_parser(command); p.add_argument('--config', required=True)
    p = sub.add_parser('verify'); p.add_argument('--dataset', required=True); p.add_argument('--archive')
    args = parser.parse_args()
    if args.command == 'prepare':
        result = prepare(args.config)[1]['summary']
    elif args.command == 'build':
        result = build(args.config)
    else:
        result = verify_dataset(args.dataset)
        if args.archive:
            result['zip'] = verify_archive(args.archive, args.dataset)
    print(json.dumps(result, ensure_ascii=False, indent=2))


if __name__ == '__main__':
    main()
