"""Versioned saved-mask refresh; no production writers and no implicit training.

Raw manual coverage, isolated nonconforming review and qualified research
supervision are distinct. Existing internal held-out groups never enter train.
"""
from collections import Counter, defaultdict
from contextlib import closing
from datetime import datetime, timezone
import hashlib
import json
from pathlib import Path
import shutil

import cv2
import numpy as np

from .cluster_targets import digest, protected_snapshot
from .mask_model_improvement_data import preservation, source_binding
from .mask_model_training_data import (PROJECT, FALSE_FLAGS, read, jsonl, write, sha, require,
    global_audit, eligibility, letterbox, build_visual_sheets, verify_dataset)
from .mask_nonconforming_review import FALSE_FLAGS as EXCEPTION_FALSE_FLAGS, validate_target
from .mask_quality_recovery import POSE, GALLERY, fingerprint
from .mask_quota_saved_extension import load_formal_evidence, file_inventory
from .mask_template_refined import snapshot_database, readonly
from .pose_review_geometry import polygon_value, raster_mask

OUTPUT = PROJECT/'outputs/pallet_mask_dataset_update_v1'
SOURCE = PROJECT/'outputs/pallet_mask_continuation_sources_v14'
PREVIOUS = PROJECT/'outputs/pallet_mask_model_improvement_v5'
EXCEPTIONS = PROJECT/'outputs/pallet_mask_nonconforming_requests_v1'
EXCEPTION_STORE = PROJECT/'outputs/pallet_mask_nonconforming_reviews_v1/mask_reviews.sqlite3'
REGULAR_BATCH = PROJECT/'outputs/pallet_mask_under50_requests_v6'


def raw_sha(raw):
    return hashlib.sha256(raw.encode('utf-8')).hexdigest()


def previous_artifacts():
    # All model/data versions and original failed/partial receipts stay immutable.
    paths = [PROJECT/'outputs/pallet_mask_model_training_v1'] + [
        PROJECT/f'outputs/pallet_mask_model_improvement_v{version}' for version in range(1, 6)]
    return {str(path.relative_to(PROJECT)): file_inventory(path) for path in paths}


def exception_snapshot(path, rows, formal):
    """Resolve this isolated schema directly; never route it as gallery pub15."""
    manifest = read(EXCEPTIONS/'manifest.json'); proof = read(EXCEPTIONS/'verification.json')
    require(proof['state'] == 'passed_export_and_isolation_not_quality'
        and proof['manifest_sha256'] == digest(EXCEPTIONS/'manifest.json')
        and proof['proposals_sha256'] == digest(EXCEPTIONS/'proposals.jsonl'), 'Isolated source seal differs')
    proposals = {p['target_key']: p for p in jsonl(EXCEPTIONS/'proposals.jsonl')}
    require(len(proposals) == 7 and all(validate_target(p) is None for p in proposals.values()), 'Seven explicit nonconforming targets required')
    source_identity = {}
    for key, proposal in proposals.items():
        row = rows[key]
        require(proposal['source'] == row and digest(formal/row['image']) == proposal['image_sha256'], 'Isolated target/formal image differs')
        identity = {field: row[field] for field in ('target_key', 'frame_id', 'split', 'box_index', 'bbox', 'size', 'source_pcd_sha256')}
        identity.update(formal_manifest_sha256=manifest['formal_manifest_sha256'], image_sha256=proposal['image_sha256'])
        source_identity[key] = identity
    with closing(readonly(path, True)) as db:
        current = {key: (revision, raw) for key, revision, raw in db.execute('SELECT target_key,revision,payload FROM reviews')}
        events = list(db.execute('SELECT id,target_key,revision,payload FROM events ORDER BY id'))
    require(set(current) == set(proposals), 'All seven reports require real saved current records')
    histories, event_binding = defaultdict(list), []
    expected_binding = sha(dict(formal=manifest['formal_manifest_sha256'], worklist=digest(EXCEPTIONS/'manifest.json'), namespace='nonconforming-v1'))
    for event_id, key, revision, raw in events:
        require(key in proposals, 'Foreign event in isolated live store')
        value = json.loads(raw); proposal = proposals[key]
        require(value['target_key'] == key and type(value['revision']) is int and value['revision'] == revision
            and value['source'] == source_identity[key] and value['source_binding'] == sha(source_identity[key])
            and value['binding'] == expected_binding and value['base_proposal_id'] == sha(proposal)
            and value['proposal_publication'] == 'nonconforming-v1'
            and value['worklist_manifest_sha256'] == digest(EXCEPTIONS/'manifest.json')
            and value['proposal_polygon'] == [] and value['original_vertex_count'] == 0
            and value['event_kind'] == 'human_nonconforming_mask_review' and value['human_edited'] is True
            and value['source_quality'] == 'nonconforming' and value['status'] == 'nonconforming_human_review_not_counted'
            and value['original_rejection_reasons'] == proposal['original_rejection_reasons']
            and all(value.get(flag) is False for flag in EXCEPTION_FALSE_FLAGS), 'Invalid nonconforming event binding or unauthorized promotion')
        if value['quality_decision'] == 'untraceable':
            require(value['polygon'] == [] and value['vertex_count'] == 0, 'Untraceable event cannot count as a contour')
        else:
            require(value['quality_decision'] == 'redrawn_pending_quality_check', 'Unknown isolated review decision')
            polygon_value(value['polygon'], *rows[key]['size'])
            require(len(value['polygon']) == value['vertex_count'], 'Isolated vertex count differs')
        histories[key].append((event_id, revision, raw))
        event_binding.append(dict(event_id=event_id, target_key=key, revision=revision, payload_raw_sha256=raw_sha(raw),
            worklist_manifest_sha256=value['worklist_manifest_sha256'], base_proposal_sha256=value['base_proposal_id'],
            source_namespace='nonconforming-v1', exact_binding_verified=True))
    records, review_only = [], []
    for key, (revision, raw) in current.items():
        require(histories[key][-1][1:] == (revision, raw)
            and [v[1] for v in histories[key]] == list(range(1, revision+1)), 'Isolated current is not recoverable contiguous raw latest history')
        value = json.loads(raw); row = rows[key]
        entry = dict(target_key=key, row=row, polygon=value['polygon'], revision=revision,
            payload_sha256=sha(value), event_raw_sha256=raw_sha(raw), source_event_id=histories[key][-1][0],
            origin='saved_nonconforming_isolated', batch='nonconforming-v1', source_quality='nonconforming',
            qualification='pending_quality_clearance', original_rejection_reasons=value['original_rejection_reasons'],
            review_decision=value['quality_decision'], counts_toward_under50=False, **FALSE_FLAGS)
        (records if value['polygon'] else review_only).append(entry)
    return records, review_only, event_binding


def coverage(catalog, exceptions, formal):
    """Image-global dedup, separate numeric completion from quality clearance."""
    counts = Counter(); isolated = Counter(); keys, frames, image_hashes = set(), set(), set()
    for record in catalog + exceptions:
        key, row = record['target_key'], record['row']; image_sha = digest(formal/row['image'])
        require(key not in keys and row['frame_id'] not in frames and image_sha not in image_hashes, 'Duplicate image cannot increase coverage')
        polygon_value(record['polygon'], *row['size'])
        keys.add(key); frames.add(row['frame_id']); image_hashes.add(image_sha)
        if record.get('source_quality') == 'nonconforming':
            isolated[row['final_class_id']] += 1
        else:
            counts[row['final_class_id']] += 1
    return dict(state='raw_manual_contour_count_not_quality_or_GT_confirmation', total_distinct_images=len(keys),
        classes={c: dict(standard_saved=counts[c], isolated_nonconforming_redrawn=isolated[c],
            all_manual_contours=counts[c]+isolated[c], reaches50_raw_count=counts[c]+isolated[c] >= 50,
            missing_to50_excluding_nonconforming=max(0, 50-counts[c])) for c in range(1, 23)},
        all22_reach50_including_nonconforming=all(counts[c]+isolated[c] >= 50 for c in range(1, 23)),
        all22_reach50_excluding_nonconforming=all(counts[c] >= 50 for c in range(1, 23)),
        source_flags_or_qualification_changed=False, independent_contour_quality_certified=False)


def assign_split(component, old_assignments):
    # New components default to train, never reassign already seen dev/cal to train.
    return old_assignments.get(component, 'train')


def gate(reference, row, neighbors, component, blocked, *, newly_reviewable=False):
    if row['split'] != 'train':
        return ['formal_val_test_label_forbidden'], None, []
    if reference.get('source_quality') == 'nonconforming':
        return ['nonconforming_source_pending_quality_clearance'], None, []
    reasons, roi = eligibility(reference, row, neighbors, component, blocked)
    soft = {'unknown_or_narrow_observation_surface', 'unknown_or_edge_only_structure'} if newly_reviewable else set()
    hard = [r for r in reasons if r not in soft]
    return hard, roi, [r for r in reasons if r in soft]


def prepare(output=OUTPUT):
    output = Path(output).resolve()
    require(output.parent == PROJECT/'outputs' and not output.exists(), 'Fresh dataset version required')
    require(shutil.disk_usage(output.parent).free >= 11*2**30, 'Keep10GiB reserve; no old files may be deleted')
    cv2.setNumThreads(1)
    old_manifest, old_records = verify_dataset(PREVIOUS)
    require(len(old_records) == 263 and digest(PREVIOUS/'dataset_manifest.json') == '8f97f35a42b70d9e967b420d4861890bb9e9cd04c6c44198cbf5316cda51d4db', 'Previous263 dataset pin differs')
    formal, rows, evidence = load_formal_evidence(read(PROJECT/'outputs/pallet_mask_remaining_round5_v1/manifest.json'))
    before = preservation(formal); prior_files = previous_artifacts()
    output.mkdir(mode=0o700); (output/'private').mkdir(mode=0o700)
    write(output/'private/preservation_before.json', before)
    write(output/'private/previous_versions_inventory.json', prior_files)
    snapshot_database(EXCEPTION_STORE, output/'private/nonconforming_snapshot.sqlite3')
    exception_records, review_only, event_binding = exception_snapshot(output/'private/nonconforming_snapshot.sqlite3', rows, formal)
    write(output/'private/nonconforming_source_catalog.jsonl', exception_records, lines=True)
    write(output/'private/nonconforming_review_only.jsonl', review_only, lines=True)
    write(output/'private/nonconforming_event_binding.jsonl', event_binding, lines=True)
    references, binding = source_binding(SOURCE, rows)
    binding.update(nonconforming_snapshot_sha256=digest(output/'private/nonconforming_snapshot.sqlite3'),
        nonconforming_fingerprint=fingerprint(output/'private/nonconforming_snapshot.sqlite3'),
        nonconforming_manifest_sha256=digest(EXCEPTIONS/'manifest.json'),
        nonconforming_contours=len(exception_records), nonconforming_review_only=len(review_only),
        nonconforming_events=len(event_binding))
    write(output/'private/source_binding.json', binding)
    manual_coverage = coverage(jsonl(SOURCE/'private/source_catalog.jsonl'), exception_records, formal)
    write(output/'annotation_coverage.json', manual_coverage)
    labels, blocked, links, hashes = global_audit(rows, evidence)
    old_components = read(PREVIOUS/'private/global_components.json')
    require(labels == old_components['by_target'] and blocked == set(old_components['blocked_by_formal_val_test']),
        'Global components changed: cannot silently carry old split assignments')
    write(output/'private/global_components.json', dict(by_target=labels, blocked_by_formal_val_test=sorted(blocked), links=links,
        all_targets=len(rows), old_component_assignments_preserved=True, all_classes_policies_audited=True))
    write(output/'private/global_native_hashes.jsonl', hashes, lines=True)
    by_frame = defaultdict(list)
    for row in rows.values():
        by_frame[row['frame_id']].append(row)
    current = {r['target_key']: r for r in references}
    old = {r['target_key']: r for r in old_records}; selected_regular = {p['target_key'] for p in jsonl(REGULAR_BATCH/'proposals.jsonl')}
    old_assignments = {}
    for r in old_records:
        require(old_assignments.setdefault(r['component'], r['development_split']) == r['development_split'], 'Old component splits conflict')
    retained, pending, audit = [], [], []
    for key in sorted(set(old) | selected_regular):
        row = rows[key]
        if row['split'] != 'train':
            audit.append(dict(target_key=key, formal_split=row['split'], disposition='excluded', reasons=['formal_val_test_label_forbidden']))
            continue
        require(key in current, 'Selected real manual mask is missing from latest snapshot')
        reference = current[key]; is_new = key not in old
        # Old compatible narrow masks already have an explicit approval; new or
        # changed ones still require the separate visual-decision manifest.
        hard, roi, semantic_notes = gate(reference, row, by_frame[row['frame_id']], labels[key], blocked, newly_reviewable=True)
        require(not (key in old and hard), 'Previously trained source now fails hard gate; preserve failure instead of silently deleting it')
        audit.append(dict(target_key=key, formal_split='train', disposition='excluded' if hard else 'pending_visual' if is_new or reference['polygon'] != old[key]['polygon'] else 'retained',
            reasons=hard, semantic_notes=semantic_notes, source_event_raw_sha256=reference['event_raw_sha256']))
        if hard:
            continue
        record = dict(target_key=key, class_id=row['final_class_id'], component=labels[key], frame_id=row['frame_id'],
            group_id=row['group_id'], native_image=row['image'], native_size=row['size'], original_bbox=row['bbox'], roi=roi,
            polygon=reference['polygon'], source_event_raw_sha256=reference['event_raw_sha256'], source_payload_sha256=reference['payload_sha256'],
            source_event_id=reference['source_event_id'], source_revision=reference['revision'], source_origin=reference['origin'],
            source_batch=reference['batch'], policy_id=row['policy_id'], surface=row['viewpoint_candidate'], structure=row['structure_candidate'],
            formal_split='train', development_split=assign_split(labels[key], old_assignments),
            is_new_since_previous=key not in old, semantic_metadata_notes=semantic_notes,
            occlusion='unknown', reflection='unknown', **FALSE_FLAGS)
        if key in old and reference['polygon'] == old[key]['polygon']:
            record['semantic_review'] = 'prior263_unchanged_polygon_revalidated_latest_event'
            retained.append(record)
        else:
            record['semantic_review'] = 'pending_explicit_visual_review'
            pending.append(record)
    for record in exception_records:
        row = record['row']
        hard, _, _ = gate(record, row, by_frame[row['frame_id']], labels[record['target_key']], blocked)
        audit.append(dict(target_key=record['target_key'], formal_split=row['split'], disposition='quarantined_nonconforming',
            reasons=hard, polygon_present=True, source_event_raw_sha256=record['event_raw_sha256']))
    write(output/'private/eligibility_audit.jsonl', audit, lines=True)
    write(output/'private/retained.jsonl', retained, lines=True)
    write(output/'private/eligible_before_visual_review.jsonl', pending, lines=True)
    write(output/'private/old_split_assignments.json', old_assignments)
    write(output/'research_permit.json', dict(user_authorization='已经完成7张的标注检查是否每类均满足50张的条件，并更新训练数据集。',
        scope='new immutable research-mask dataset only; no training execution or GT promotion',
        inputs='native image pixels in original bbox context only; no human polygon ROI or pose/IDs/class/group features',
        official_val_test_not_model_supervision=True, previous_dev_calibration_never_moved_to_train=True,
        nonconforming_labels_quarantined_until_explicit_quality_clearance=True,
        source_labels_keep_false_flags=True, retraining_started=False, production_writes=0, **FALSE_FLAGS))
    build_visual_sheets(output, pending, formal)
    summary = dict(state='prepared_awaiting_explicit_visual_approval', previous_records=len(old_records), retained=len(retained),
        visual_candidates=len(pending), isolated_drawn_contours=len(exception_records), isolated_review_only=len(review_only),
        isolated_raw_events=len(event_binding), current_manual_total=manual_coverage['total_distinct_images'],
        prior_dataset_sha256=digest(PREVIOUS/'dataset_manifest.json'), candidate_sha256=digest(output/'private/eligible_before_visual_review.jsonl'),
        new_candidate_splits=dict(Counter(r['development_split'] for r in pending)),
        exclusions=dict(Counter(reason for item in audit for reason in item['reasons'])),
        global_components=len(set(labels.values())), source_quota_flags_unchanged=True)
    write(output/'preparation_summary.json', summary)
    print(json.dumps(summary), flush=True)
    return summary


def freeze(decisions_path, output=OUTPUT):
    output = Path(output).resolve(); require(not (output/'dataset_manifest.json').exists(), 'Do not replace a frozen dataset')
    decisions = read(decisions_path); candidates = jsonl(output/'private/eligible_before_visual_review.jsonl')
    require(decisions['candidate_sha256'] == digest(output/'private/eligible_before_visual_review.jsonl')
        and set(decisions['decisions']) == {r['target_key'] for r in candidates}, 'Every candidate requires an explicit bound visual decision')
    records = jsonl(output/'private/retained.jsonl')
    for candidate in candidates:
        decision = decisions['decisions'][candidate['target_key']]
        require(type(decision.get('approved_for_research_mask')) is bool and decision.get('native_image_and_polygon_inspected') is True and decision.get('note'),
            'Visual review must be explicit, never inferred from saved status')
        if decision['approved_for_research_mask']:
            candidate['semantic_review'] = 'visible_single_ring_research_mask_approved_not_surface_or_corner_GT'
            records.append(candidate)
    old = jsonl(PREVIOUS/'private/training_records.jsonl'); old_by_key = {r['target_key']: r for r in old}
    selected = {r['target_key']: r for r in records}
    require(len(records) == len(selected) and set(old_by_key) <= set(selected), 'Old263 keys must remain recoverable in the new version')
    # Retain previous ordering/image IDs; only new selected sources are appended.
    records = [selected[r['target_key']] for r in old]+[selected[k] for k in sorted(set(selected)-set(old_by_key))]
    for record in records:
        if record['target_key'] in old_by_key:
            require(record['development_split'] == old_by_key[record['target_key']]['development_split'], 'Never move an already seen holdout target into train')
    assignments = read(output/'private/old_split_assignments.json')
    for r in records:
        require(r['development_split'] == assign_split(r['component'], assignments) and r['formal_split'] == 'train', 'Formal/internal group leakage')
    formal, _, _ = load_formal_evidence(read(PROJECT/'outputs/pallet_mask_remaining_round5_v1/manifest.json'))
    from pycocotools import mask as mu
    for name in ('dataset', 'dataset_rfdetr', 'private/native_masks'):
        (output/name).mkdir(mode=0o700)
    coco, square, transforms = {}, {}, {}
    for split in ('train', 'valid', 'calibration'):
        for directory in ('dataset', 'dataset_rfdetr'):
            (output/directory/split).mkdir(mode=0o700)
        coco[split] = dict(info=dict(description='Versioned formal-train-only research masks; IDs are audit sidecars only'),
            licenses=[], categories=[dict(id=1, name='fork_face')], images=[], annotations=[])
        square[split] = {**coco[split], 'images': [], 'annotations': []}
    for index, record in enumerate(records):
        image = cv2.imread(str(formal/record['native_image']), cv2.IMREAD_GRAYSCALE)
        require(image is not None and list(image.shape[::-1]) == record['native_size'], 'Wrong native training image')
        x1, y1, x2, y2 = record['roi']; grey = image[y1:y2, x1:x2]
        full = raster_mask(record['polygon'], *record['native_size']).astype(np.uint8); mask = full[y1:y2, x1:x2]
        require(int(mask.sum()) == int(full.sum()) and mask.any(), 'No clipping or empty mask supervision allowed')
        split = 'valid' if record['development_split'] == 'dev' else record['development_split']; name = f'{index:05d}.png'
        cv2.imwrite(str(output/f'dataset/{split}/{name}'), np.repeat(grey[:, :, None], 3, axis=2))
        cv2.imwrite(str(output/f'private/native_masks/{name}'), mask*255)
        def annotation(label):
            rle = mu.encode(np.asfortranarray(label)); require(np.array_equal(mu.decode(rle), label), 'RLE roundtrip differs')
            return dict(id=index+1, image_id=index+1, category_id=1, iscrowd=0,
                segmentation=dict(size=rle['size'], counts=rle['counts'].decode()), area=int(mu.area(rle)), bbox=mu.toBbox(rle).tolist())
        h, w = grey.shape; im = dict(id=index+1, file_name=name, width=w, height=h)
        coco[split]['images'].append(im); coco[split]['annotations'].append(annotation(mask))
        canvas, target, _, transform = letterbox(grey, mask, (576, 576))
        cv2.imwrite(str(output/f'dataset_rfdetr/{split}/{name}'), np.repeat(canvas[:, :, None], 3, axis=2))
        square[split]['images'].append(dict(im, width=576, height=576)); square[split]['annotations'].append(annotation(target))
        transforms[str(index+1)] = transform
        record.update(image=f'dataset/{split}/{name}', native_mask=f'private/native_masks/{name}', roi_size=[w, h], image_id=index+1,
            group_fold={'dev': 0, 'calibration': 1, 'train': 2}[record['development_split']])
    for split in coco:
        write(output/f'dataset/{split}/_annotations.coco.json', coco[split])
        write(output/f'dataset_rfdetr/{split}/_annotations.coco.json', square[split])
    write(output/'private/training_records.jsonl', records, lines=True)
    write(output/'private/rfdetr_transforms.json', transforms)
    write(output/'private/visual_semantic_approval.json', decisions)
    files = {directory+'/'+name: value for directory in ('dataset', 'dataset_rfdetr', 'private/native_masks') for name, value in file_inventory(output/directory).items()}
    for name in ('private/training_records.jsonl', 'private/rfdetr_transforms.json', 'private/visual_semantic_approval.json',
        'private/source_binding.json', 'private/global_components.json', 'private/eligibility_audit.jsonl',
        'private/nonconforming_source_catalog.jsonl', 'annotation_coverage.json', 'research_permit.json'):
        files[name] = digest(output/name)
    components = {s: len({r['component'] for r in records if r['development_split'] == s}) for s in ('train', 'dev', 'calibration')}
    manifest = dict(state='immutable_research_training_dataset', previous_directory=str(PREVIOUS.relative_to(PROJECT)),
        previous_manifest_sha256=digest(PREVIOUS/'dataset_manifest.json'), files=files, record_count=len(records),
        splits=dict(Counter(r['development_split'] for r in records)), independent_components=components,
        by_class={c: dict(Counter(r['development_split'] for r in records if r['class_id'] == c)) for c in range(1, 23)},
        split_rule='Preserve every old component assignment; new components train only. Never formal val/test; no model-metric-based reassignment.',
        group_folds={r['component']: r['group_fold'] for r in records}, formal_val_test_included=False, model_input_metadata=False,
        blind=False, status='updated_research_dataset_not_automatic_training_or_release', selection_split='dev only, calibration not selection',
        nonconforming_labels_in_training=0, old_frozen_datasets_overwritten=False, retraining_started=False,
        release_minimum_groups=10, insufficient_group_support=any(components[s] < 10 for s in ('dev', 'calibration')),
        code_sha256=digest(__file__), **FALSE_FLAGS)
    verify_outputs(output, manifest, records)
    write(output/'dataset_manifest.json', manifest)
    require(preservation(formal) == read(output/'private/preservation_before.json'), 'Old protected data/models/configs changed')
    require(previous_artifacts() == read(output/'private/previous_versions_inventory.json'), 'Prior completed/failed model artifacts changed')
    write(output/'verification.json', dict(state='passed_dataset_exports_split_isolation_and_preservation_not_quality',
        dataset_manifest_sha256=digest(output/'dataset_manifest.json'), records=len(records),
        new_selected=len(records)-len(old), heldout_groups_not_moved_to_train=True, native_masks_and_RLE_replayed=True,
        model_inputs_do_not_include_manual_metadata=True, nonconforming_labels_in_training=0,
        previous_versions_preserved=True, no_training_started=True))
    print(json.dumps(dict(state='updated_new_immutable_dataset_not_retrained', records=len(records), new_selected=len(records)-len(old),
        splits=manifest['splits'], components=components, dataset_manifest_sha256=digest(output/'dataset_manifest.json'))), flush=True)
    return manifest


def verify_outputs(output, manifest, records):
    """Reload every model-facing pixel/RLE, all three group partitions, old IDs."""
    from pycocotools import mask as mu
    output = Path(output)
    require(len(records) == len({r['target_key'] for r in records}) == manifest['record_count'], 'Duplicate frozen record')
    components = defaultdict(set)
    for r in records:
        components[r['component']].add(r['development_split'])
        require(r['formal_split'] == 'train' and r.get('source_quality') != 'nonconforming', 'Held-out/nonconforming label leaked into training')
    require(all(len(v) == 1 for v in components.values()), 'A component spans internal partitions')
    formal, _, _ = load_formal_evidence(read(PROJECT/'outputs/pallet_mask_remaining_round5_v1/manifest.json'))
    maps = {r['image_id']: r for r in records}
    for split in ('train', 'valid', 'calibration'):
        native = read(output/f'dataset/{split}/_annotations.coco.json'); square = read(output/f'dataset_rfdetr/{split}/_annotations.coco.json')
        require(len(native['images']) == len(native['annotations']) == len(square['images']) == len(square['annotations']), 'COCO pairs differ')
        for image, annotation, square_annotation in zip(native['images'], native['annotations'], square['annotations']):
            record = maps[image['id']]
            require(('valid' if record['development_split'] == 'dev' else record['development_split']) == split, 'COCO partition differs')
            full = cv2.imread(str(formal/record['native_image']), cv2.IMREAD_GRAYSCALE)
            x1, y1, x2, y2 = record['roi']; grey = full[y1:y2, x1:x2]
            pixels = cv2.imread(str(output/record['image']))
            require(np.array_equal(pixels, np.repeat(grey[:, :, None], 3, axis=2)), 'Model image pixels differ from original-box ROI')
            target = raster_mask(record['polygon'], *record['native_size']).astype(np.uint8)[y1:y2, x1:x2]
            mask = cv2.imread(str(output/record['native_mask']), cv2.IMREAD_GRAYSCALE) > 0
            def decode(ann):
                rle = ann['segmentation']; return mu.decode(dict(size=rle['size'], counts=rle['counts'].encode()))
            require(np.array_equal(target, mask) and np.array_equal(target, decode(annotation)), 'Native mask/RLE differs from saved polygon')
            canvas, label, _, _ = letterbox(grey, target, (576, 576))
            require(np.array_equal(label, decode(square_annotation)) and np.array_equal(cv2.imread(str(output/f'dataset_rfdetr/{split}/{image["file_name"]}')), np.repeat(canvas[:, :, None], 3, axis=2)), 'RF-DETR aspect-preserving export differs')
    for name, expected in manifest['files'].items():
        require(digest(output/name) == expected, 'New immutable dataset file mismatch')
