"""Private, research-only mask training data with global pre-split isolation.

No production writers, application contexts, pose payload exports or model inputs
other than original-box-localized native grey pixels live in this module.
"""
from __future__ import annotations

from collections import Counter, defaultdict
from datetime import datetime, timezone
import hashlib
import json
import math
import os
from pathlib import Path
import shutil
import subprocess

import cv2
import numpy as np

from .cluster_targets import digest, protected_snapshot
from .mask_quality_recovery import fingerprint, POSE, GALLERY
from .mask_quota_saved_extension import file_inventory, load_formal_evidence
from .pose_review_geometry import canonical, polygon_value, raster_mask

PROJECT = Path(__file__).resolve().parents[1]
SOURCE = PROJECT / 'outputs/pallet_mask_quality_recovery_v2'
OUTPUT = PROJECT / 'outputs/pallet_mask_model_training_v1'
SEAL = '61b73d8e0dbc7b99db01b75e912c797beacc34c8c0b3d31434058eac3d376428'
GPU_UUIDS = {
    'm1': 'GPU-3315c096-fef4-7350-483a-7c5f5ca2b619',
    'm2': 'GPU-09ed84c8-4dee-efc0-dc1d-deda5177e4e8',
}
FALSE_FLAGS = dict(ground_truth_eligible=False, automatic_training_eligible=False,
                  human_mask_confirmed=False, human_pose_confirmed=False,
                  robot_command_eligible=False)


def require(value, message):
    if not value:
        raise ValueError(message)


def read(path):
    return json.loads(Path(path).read_text(encoding='utf-8'))


def jsonl(path):
    with Path(path).open(encoding='utf-8') as stream:
        return [json.loads(line) for line in stream if line.strip()]


def sha(value):
    return hashlib.sha256(canonical(value)).hexdigest()


def write(path, value, *, lines=False):
    """Exclusive immutable audit records; model best/last use a separate API."""
    path = Path(path)
    with path.open('xb') as stream:
        stream.write(b''.join(canonical(v) + b'\n' for v in value)
                     if lines else canonical(value) + b'\n')
    path.chmod(0o600)


def atomic_json(path, value):
    path = Path(path)
    temporary = path.with_name(path.name + '.atomic.tmp')
    with temporary.open('xb') as stream:
        stream.write(canonical(value) + b'\n')
    temporary.chmod(0o600)
    os.replace(temporary, path)


def disk_budget(output):
    # Includes generic downloads, two Adam optimizers, EMA, best/last, one
    # temporary checkpoint, native ROI/RLE images and bounded private reports.
    allocated = dict(generic_weights_gib=1, m1_best_last_optimizer_gib=1,
                     m2_best_last_optimizer_ema_temporary_gib=8,
                     data_logs_reports_gib=2)
    required = sum(allocated.values()) * 2**30
    reserve = 10 * 2**30
    free = shutil.disk_usage(output).free
    temporary_free = shutil.disk_usage('/tmp/opencode').free
    require(free >= required + reserve, 'Insufficient project disk budget/reserve')
    require(temporary_free >= reserve, 'Insufficient approved temporary reserve')
    return dict(project_free_bytes=free, temporary_free_bytes=temporary_free,
                allocation=allocated, maximum_experiment_bytes=required,
                minimum_reserve_bytes=reserve, no_existing_files_deleted=True)


def gpu_receipt(route):
    """Requery physical UUID immediately before each worker uses CUDA."""
    uuid = GPU_UUIDS[route]
    require(os.environ.get('CUDA_VISIBLE_DEVICES') == uuid,
            'Worker must bind its one authorized physical GPU UUID')
    result = subprocess.check_output([
        'nvidia-smi', '--query-gpu=uuid,name,memory.total,memory.used,utilization.gpu',
        '--format=csv,noheader,nounits'], text=True)
    matches = [line.strip() for line in result.splitlines() if line.split(',')[0].strip() == uuid]
    require(len(matches) == 1 and '3090' not in matches[0], 'Authorized non3090 GPU unavailable')
    values = [v.strip() for v in matches[0].split(',')]
    require(int(values[2]) >= (15000 if route == 'm1' else 23000), 'Unexpected physical GPU memory')
    require(int(values[3]) < 2000, 'Authorized GPU is occupied; do not interfere')
    return dict(uuid=uuid, physical_query=matches[0], cuda_visible_devices=uuid,
                logical_device='cuda:0', single_process=True,
                queried_at=datetime.now(timezone.utc).isoformat(), pid=os.getpid())


def verify_sources(*, allow_authorized_supplement_appends=False, trained_keys=()):
    """Compare live logical histories AND frozen snapshots before label use."""
    require(digest(SOURCE / 'independent_verification.json') == SEAL, 'Independent seal changed')
    seal, result = read(SOURCE / 'independent_verification.json'), read(SOURCE / 'result.json')
    require(seal['state'] == 'passed' and digest(SOURCE / 'result.json') == seal['result_sha256'],
            'Unsealed quality recovery result')
    for name, expected in result['output_artifacts'].items():
        require(digest(SOURCE / name) == expected, 'Sealed source artifact changed: ' + name)
    require(digest(SOURCE / 'private/source_catalog.jsonl') == seal['source_catalog_sha256'],
            'Source catalog changed')
    source_fingerprints = {name: fingerprint(path) for name, path in (('pose', POSE), ('gallery', GALLERY))}
    append_audit = None
    for name in ('pose', 'gallery'):
        require(fingerprint(SOURCE / f'private/{name}_snapshot.sqlite3') == seal['source_fingerprints'][name],
                'Frozen raw history fingerprint mismatch: ' + name)
    if source_fingerprints != seal['source_fingerprints']:
        require(allow_authorized_supplement_appends, 'Live stores changed since sealed snapshot')
        require(source_fingerprints['pose'] == seal['source_fingerprints']['pose'],
                'Pose raw history changed; supplement only permits new gallery keys')
        from .mask_model_training_integrity import verify_gallery_extension
        append_audit = verify_gallery_extension(SOURCE/'private/gallery_snapshot.sqlite3', GALLERY, trained_keys)
        # Bind the same transaction that proved raw-prefix preservation, even
        # if another authorized save arrived after the initial quick hash read.
        source_fingerprints['gallery'] = append_audit['observed_logical_fingerprint']
    require(file_inventory(PROJECT / 'configs') == seal['configuration_sha256'], 'Sealed configurations changed')
    for name, expected in seal['frozen_code_sha256'].items():
        require(digest(PROJECT / name) == expected, 'Frozen algorithm changed: ' + name)
    parent = read(PROJECT / 'outputs/pallet_mask_remaining_round5_v1/manifest.json')
    formal, rows, evidence = load_formal_evidence(parent)
    protected = protected_snapshot(formal)
    require(len(rows) == 12344 and len(protected) == 20768, 'Formal denominator changed')
    require(protected == read(SOURCE / 'private/protected_formal_before.json'), 'Protected formal files changed')
    return formal, rows, evidence, dict(
        checked_at=datetime.now(timezone.utc).isoformat(), independent_seal_sha256=SEAL,
        result_sha256=digest(SOURCE / 'result.json'),
        source_catalog_sha256=seal['source_catalog_sha256'], source_fingerprints=source_fingerprints,
        live_fingerprints_identical_to_initial=source_fingerprints == seal['source_fingerprints'],
        authorized_external_append_audit=append_audit,
        configuration_sha256=seal['configuration_sha256'], frozen_code_sha256=seal['frozen_code_sha256'],
        protected_formal_sha256=sha(protected), protected_formal_files=len(protected),
        protected_inventory=protected, source_artifacts_sha256=result['output_artifacts'],
        fixed_formal_targets=len(rows), no_raw_pose_or_access_codes_exported=True)


class Components:
    def __init__(self, keys):
        self.parent = {key: key for key in keys}
        self.links = []

    def find(self, key):
        while self.parent[key] != key:
            self.parent[key] = self.parent[self.parent[key]]
            key = self.parent[key]
        return key

    def union(self, a, b, reason, **details):
        x, y = self.find(a), self.find(b)
        if x != y:
            self.parent[max(x, y)] = min(x, y)
            self.links.append(dict(left=a, right=b, reason=reason, **details))

    def labels(self):
        groups = defaultdict(list)
        for key in self.parent:
            groups[self.find(key)].append(key)
        return {key: sha(sorted(group)) for group in groups.values() for key in group}


def context_roi(box, size):
    """Fixed deployment-available original bbox ONLY, never a polygon bbox."""
    x, y, w, h = (float(box[k]) for k in ('x', 'y', 'width', 'height'))
    require(w > 0 and h > 0 and np.isfinite([x, y, w, h]).all(), 'Invalid original bbox')
    width, height = size
    return [max(0, math.floor(x - .06*w)), max(0, math.floor(y - .22*h)),
            min(width, math.ceil(x + 1.06*w)), min(height, math.ceil(y + 1.22*h))]


def intersects(roi, box):
    x, y, w, h = (box[k] for k in ('x', 'y', 'width', 'height'))
    return min(roi[2], x+w) > max(roi[0], x) and min(roi[3], y+h) > max(roi[1], y)


def signature(image):
    small = cv2.resize(image, (17, 8), interpolation=cv2.INTER_AREA)
    dhash = int.from_bytes(np.packbits(small[:, 1:] > small[:, :-1]).tobytes(), 'big')
    value = cv2.resize(image, (64, 32), interpolation=cv2.INTER_AREA).astype(np.float32).ravel()
    value -= value.mean()
    norm = float(np.linalg.norm(value))
    return dhash, value / max(norm, 1e-12), norm / math.sqrt(len(value))


def near_duplicate_links(items, graph, reason, rows):
    """All-class/all-policy 128-bit Hamming<=2 candidate search (no LSH misses).

    Three disjoint bit bands ensure every pair at distance <=2 shares a band.
    Photometric NCC is then checked; flat patches never establish near links.
    """
    bands = defaultdict(list)
    for index, (key, hash_value, normalized, contrast) in enumerate(items):
        tokens = [(0, hash_value & ((1 << 43)-1)),
                  (1, (hash_value >> 43) & ((1 << 43)-1)), (2, hash_value >> 86)]
        candidates = set(j for token in tokens for j in bands[token])
        if contrast >= 3:
            for j in sorted(candidates):
                other, other_hash, other_normalized, other_contrast = items[j]
                if other_contrast < 3 or (hash_value ^ other_hash).bit_count() > 2:
                    continue
                if graph.find(key) == graph.find(other):
                    continue
                correlation = float(np.dot(normalized, other_normalized))
                if correlation >= .995:
                    graph.union(key, other, reason, ncc=correlation,
                                hamming=(hash_value ^ other_hash).bit_count(),
                                cross_class=rows[key]['final_class_id'] != rows[other]['final_class_id'],
                                cross_policy=rows[key]['policy_id'] != rows[other]['policy_id'])
        for token in tokens:
            bands[token].append(index)


def global_audit(rows, evidence):
    """No polygons: all 12,344 targets, including *unlabelled* held-out targets."""
    graph = Components(rows)
    seen, image_cache, hashes, crops, frames = {}, {}, [], [], []
    for key, row in sorted(rows.items()):
        image = evidence.image(row['frame_id'])
        if row['frame_id'] not in image_cache:
            image_sha = hashlib.sha256(image.tobytes()).hexdigest()
            image_cache[row['frame_id']] = (image_sha, signature(image))
            frames.append((key, *image_cache[row['frame_id']][1]))
        image_sha = image_cache[row['frame_id']][0]
        for kind, value in (('group', row['group_id']), ('frame', row['frame_id']),
                            ('pcd', row['source_pcd_sha256']), ('image_pixels', (image.shape, image_sha))):
            token = (kind, str(value))
            if token in seen:
                graph.union(key, seen[token], kind)
            else:
                seen[token] = key
        x1, y1, x2, y2 = context_roi(row['bbox'], row['size'])
        patch = image[y1:y2, x1:x2]
        require(patch.size > 0, 'Empty original-box ROI')
        sign = signature(patch)
        crops.append((key, *sign))
        hashes.append(dict(target_key=key, formal_split=row['split'],
                           native_image_pixels_sha256=image_sha,
                           native_image_file_sha256=evidence.sources[row['frame_id']]['files'][row['image']],
                           source_pcd_sha256=row['source_pcd_sha256'],
                           crop_pixels_sha256=hashlib.sha256(patch.tobytes()).hexdigest(),
                           crop_dhash128=f'{sign[0]:032x}', roi=[x1, y1, x2, y2]))
    near_duplicate_links(frames, graph, 'global_full_image_near_duplicate', rows)
    near_duplicate_links(crops, graph, 'global_original_bbox_crop_near_duplicate', rows)
    labels = graph.labels()
    splits = defaultdict(set)
    for key, component in labels.items():
        splits[component].add(rows[key]['split'])
    blocked = {component for component, values in splits.items() if values & {'val', 'test'}}
    return labels, blocked, graph.links, hashes


def eligibility(record, row, neighbors, component, blocked):
    """Explicit conservative semantic/quarantine gate, not saved => GT.

    Candidate metadata can only reject/audit; it is NEVER a feature. These are
    weak saved research references, not independently confirmed face semantics.
    A separate visual-approval manifest is required before training starts.
    """
    reasons = []
    if row['split'] != 'train':
        reasons.append('formal_val_test_label_forbidden')
        return reasons, None
    if component in blocked:
        reasons.append('global_component_linked_to_formal_val_test')
    if row['final_class_id'] == 14:
        reasons.append('class14_long_short_surface_ambiguous')
    if row.get('viewpoint_candidate') not in ('近正视候选', '斜视候选'):
        reasons.append('unknown_or_narrow_observation_surface')
    if row.get('structure_candidate') in (None, 'unknown', '低矮窄条边沿面候选'):
        reasons.append('unknown_or_edge_only_structure')
    if any('ambiguous' in str(value) for value in row.get('review_reasons', [])):
        reasons.append('occlusion_depth_or_view_ambiguity')
    if row.get('processing_error') or not row.get('alignment_verified') or row.get('crop_touches_image_edge'):
        reasons.append('invalid_alignment_or_clipped_native_context')
    roi = context_roi(row['bbox'], row['size'])
    for other in neighbors:
        if other['target_key'] != row['target_key'] and intersects(roi, other['bbox']):
            reasons.append('known_neighbor_bbox_intersects_context_roi')
            break
    try:
        p = np.asarray(polygon_value(record['polygon'], *row['size']), float)
        if not ((p[:, 0] >= roi[0]).all() and (p[:, 0] <= roi[2]).all()
                and (p[:, 1] >= roi[1]).all() and (p[:, 1] <= roi[3]).all()):
            reasons.append('supervision_outside_fixed_bbox_context')
        mask = raster_mask(p, *row['size']).astype(np.uint8)
        if cv2.connectedComponents(mask)[0] != 2 or mask.sum() < 16:
            reasons.append('empty_or_disconnected_single_ring_annotation')
    except (ValueError, TypeError):
        reasons.append('invalid_saved_polygon')
    return reasons, roi


def letterbox(image, mask=None, size=(512, 160)):
    """Equal aspect scaling, nearest labels, and zero-weight padding."""
    width, height = size
    h, w = image.shape[:2]
    scale = min(width/w, height/h)
    nw, nh = max(1, round(w*scale)), max(1, round(h*scale))
    left, top = (width-nw)//2, (height-nh)//2
    out = np.zeros((height, width), np.uint8)
    out[top:top+nh, left:left+nw] = cv2.resize(image, (nw, nh), interpolation=cv2.INTER_LINEAR)
    valid = np.zeros_like(out)
    valid[top:top+nh, left:left+nw] = 1
    target = None
    if mask is not None:
        target = np.zeros_like(out)
        target[top:top+nh, left:left+nw] = cv2.resize(mask.astype(np.uint8), (nw, nh), interpolation=cv2.INTER_NEAREST)
    return out, target, valid, dict(native_size=[w, h], scaled_size=[nw, nh],
                                   padding_left_top=[left, top], output_size=[width, height])


def unletterbox(probability, transform):
    left, top = transform['padding_left_top']
    width, height = transform['scaled_size']
    return cv2.resize(probability[top:top+height, left:left+width],
                      tuple(transform['native_size']), interpolation=cv2.INTER_LINEAR)


def prepare(output=OUTPUT):
    output = Path(output).resolve()
    require(output.parent == PROJECT/'outputs' and not output.exists(), 'Fresh private output directory required')
    cv2.setNumThreads(1)
    budget = disk_budget(output.parent)
    formal, rows, evidence, protection = verify_sources()
    output.mkdir(mode=0o700)
    (output/'private').mkdir(mode=0o700)
    write(output/'private/input_protection_before.json', protection)
    write(output/'storage_budget.json', budget)
    write(output/'research_permit.json', dict(
        authorization='Current user explicitly authorizes M1/M2 offline research-mask training only',
        scope='new experiment only; existing sources retain all false eligibility flags',
        automatic_GT_pose_robot_or_autotrain_authorized=False, **FALSE_FLAGS,
        polygon_use='formal-train supervised loss/evaluation only, never model input or ROI/topology oracle',
        metadata_use='IDs/groups/class/sensor/surface only rejection, isolation and stratified audit',
        old_finetuned_weights_banned=True, generic_pretraining_required=True,
        formal_val_test_predictions_or_label_exports=False,
        frozen_OOF_reuse='development; not a fresh blind evaluation',
        model_inputs='native grey replicated to three channels, original-bbox fixed context ROI',
        unknown_unannotated_objects='Known bbox intersections rejected; unknown unboxed faces may remain. '
                                    'Visual single-face approval reduces but does not eliminate this risk.',
        surface_semantics='Candidate metadata is not human confirmation; visual review required before training',
        closed_holes='Single-ring sources supervise only explicitly annotated foreground; no invented holes',
        max_epochs=80, early_stopping_patience=15, seed=42,
        official_val_test_evaluation='Not in this experiment; requires frozen model, thresholds, protocol first'))
    labels, blocked, links, hashes = global_audit(rows, evidence)
    write(output/'private/global_components.json', dict(by_target=labels, links=links,
        blocked_by_formal_val_test=sorted(blocked), all_targets=12344,
        class_or_sensor_restriction=False, near_duplicate=dict(dhash_bits=128, hamming_max=2,
            photometric_ncc_min=.995, minimum_patch_std=3, full_images_and_bbox_context=True),
        limitation='Conservative photometric audit, not exhaustive physical-scene identity proof'))
    write(output/'private/global_native_hashes.jsonl', hashes, lines=True)
    catalog = jsonl(SOURCE/'private/source_catalog.jsonl')
    require(len(catalog) == 1484 and Counter(r['row']['split'] for r in catalog)
            == {'train': 658, 'val': 470, 'test': 356}, 'Sealed source denominator changed')
    by_frame = defaultdict(list)
    for row in rows.values():
        by_frame[row['frame_id']].append(row)
    audit, eligible = [], []
    # Do not extract, rasterize or export ANY official val/test source polygon.
    for record in catalog:
        key = record['target_key']
        row = rows[key]
        reasons, roi = eligibility(record, row, by_frame[row['frame_id']], labels[key], blocked)
        entry = dict(target_key=key, class_id=row['final_class_id'], formal_split=row['split'],
                     component=labels[key], reasons=reasons, roi=roi,
                     source_event_raw_sha256=record['event_raw_sha256'],
                     source_payload_sha256=record['payload_sha256'],
                     research_eligible_after_visual_review=not reasons, **FALSE_FLAGS)
        audit.append(entry)
        if not reasons:
            # A private sidecar, not the model-facing dataset. IDs never become tensors.
            eligible.append(dict(target_key=key, component=labels[key], class_id=row['final_class_id'],
                frame_id=row['frame_id'], group_id=row['group_id'], native_image=row['image'],
                native_size=row['size'], original_bbox=row['bbox'], roi=roi, polygon=record['polygon'],
                source_event_raw_sha256=record['event_raw_sha256'],
                source_payload_sha256=record['payload_sha256'],
                policy_id=row['policy_id'], surface=row['viewpoint_candidate'],
                structure=row['structure_candidate'], formal_split='train', **FALSE_FLAGS))
    write(output/'private/eligibility_audit.jsonl', audit, lines=True)
    write(output/'private/eligible_before_visual_review.jsonl', eligible, lines=True)
    source_summary = dict(total_saved=1484, formal_train=658, formal_val=470, formal_test=356,
        candidate_single_face=len(eligible), candidate_components=len({r['component'] for r in eligible}),
        rejections=dict(Counter(reason for r in audit for reason in r['reasons'])),
        global_components=len(set(labels.values())),
        global_cross_class_near_duplicate_links=sum(bool(r.get('cross_class')) for r in links),
        no_official_val_test_polygons_exported=True)
    write(output/'preparation_summary.json', source_summary)
    # Recheck all protected logical histories/configurations/files after audit.
    _, _, _, after = verify_sources()
    require(protection == {**after, 'checked_at': protection['checked_at']}, 'Inputs changed during preparation')
    write(output/'private/input_protection_after_preparation.json', after)
    build_visual_sheets(output, eligible, formal)
    write(output/'preparation_complete.json', dict(state='awaiting_explicit_visual_single_face_approval',
        summary=source_summary, artifacts=file_inventory(output/'private'),
        code_sha256=digest(__file__), source_inputs_read_only=True))
    print(json.dumps(source_summary), flush=True)
    return source_summary


def build_visual_sheets(output, records, formal):
    from PIL import Image, ImageDraw
    directory = output/'private/semantic_review'
    directory.mkdir(mode=0o700)
    for page in range((len(records)+35)//36):
        sheet = Image.new('RGB', (1440, 960), '#303030')
        draw = ImageDraw.Draw(sheet)
        for local, record in enumerate(records[page*36:(page+1)*36]):
            number = page*36+local
            x, y = (local % 4)*360, (local//4)*106
            image = cv2.imread(str(formal/record['native_image']), cv2.IMREAD_GRAYSCALE)
            x1, y1, x2, y2 = record['roi']
            crop = Image.fromarray(image[y1:y2, x1:x2]).convert('RGB')
            points = [(p[0]-x1, p[1]-y1) for p in record['polygon']]
            overlay = crop.copy()
            ImageDraw.Draw(overlay).line(points+[points[0]], fill='#00ff77', width=1)
            crop.thumbnail((174, 78)); overlay.thumbnail((174, 78))
            sheet.paste(crop, (x+3, y+23)); sheet.paste(overlay, (x+182, y+23))
            draw.text((x+3, y+3), f'{number:03d} C{record["class_id"]} {record["target_key"][:10]}', fill='white')
        sheet.save(directory/f'sheet_{page:02d}.png')


def finalize_dataset(output, approved_indices, reviewer_note):
    """Explicit visual eligibility is separate from actual historical human saves."""
    output = Path(output).resolve()
    require(not (output/'dataset_manifest.json').exists(), 'Dataset is immutable once finalized')
    formal, _, _, _ = verify_sources()
    records = jsonl(output/'private/eligible_before_visual_review.jsonl')
    indices = sorted(set(approved_indices))
    require(indices and all(type(i) is int and 0 <= i < len(records) for i in indices), 'Invalid visual approval indices')
    selected = [dict(records[i]) for i in indices]
    # The split is fixed by seeded component rank after GLOBAL union, never by
    # polygon/score. A hash-modulus trial gave only two dev components after
    # visual quarantine, so use a pretraining count-only 20% group allocation.
    components = sorted({r['component'] for r in selected},
                        key=lambda component: sha(dict(seed=42, component=component)))
    dev_components = set(components[:max(3, math.ceil(.2*len(components)))])
    for record in selected:
        record['development_split'] = 'dev' if record['component'] in dev_components else 'train'
    require({r['development_split'] for r in selected} == {'train', 'dev'}, 'Empty internal train or dev')
    require(len({r['component'] for r in selected if r['development_split'] == 'dev'}) >= 3,
            'Insufficient group-held-out development components')
    write(output/'private/visual_semantic_approval.json', dict(
        approved_indices=indices, rejected_indices=sorted(set(range(len(records)))-set(indices)),
        reviewer='assistant_visual_research_filter_not_human_GT_confirmation', note=reviewer_note,
        reviewed_at=datetime.now(timezone.utc).isoformat(),
        candidate_manifest_sha256=digest(output/'private/eligible_before_visual_review.jsonl'),
        rejected_scope='suspected adjacent/rear/overlapping face, ambiguous material/occlusion, or uncertain single visible face',
        research_only=True, **FALSE_FLAGS))
    dataset = output/'dataset'
    dataset.mkdir(mode=0o700)
    for split in ('train', 'valid'):
        (dataset/split).mkdir(mode=0o700)
    (output/'private/native_masks').mkdir(mode=0o700)
    from pycocotools import mask as mask_utils
    coco = {s: dict(info={'description': 'Internal formal-train-only research ROI split'},
                   licenses=[], images=[], annotations=[], categories=[dict(id=1, name='fork_face')])
            for s in ('train', 'valid')}
    model_records = []
    for i, record in enumerate(selected):
        image = cv2.imread(str(formal/record['native_image']), cv2.IMREAD_GRAYSCALE)
        require(image is not None and list(image.shape[::-1]) == record['native_size'], 'Native image changed')
        x1, y1, x2, y2 = record['roi']
        grey = image[y1:y2, x1:x2]
        full_mask = raster_mask(record['polygon'], *record['native_size']).astype(np.uint8)
        native_mask = full_mask[y1:y2, x1:x2]
        require(native_mask.sum() == full_mask.sum(), 'Fixed crop truncates annotation')
        split = 'valid' if record['development_split'] == 'dev' else 'train'
        name = f'{i:05d}.png'  # no class/group/pose metadata in model-facing names
        cv2.imwrite(str(dataset/split/name), np.repeat(grey[:, :, None], 3, axis=2))
        cv2.imwrite(str(output/'private/native_masks'/name), native_mask*255)
        rle = mask_utils.encode(np.asfortranarray(native_mask))
        require(np.array_equal(mask_utils.decode(rle), native_mask), 'Native COCO RLE is not exact')
        segmentation = dict(size=rle['size'], counts=rle['counts'].decode('ascii'))
        h, w = grey.shape
        coco[split]['images'].append(dict(id=i+1, file_name=name, width=w, height=h))
        coco[split]['annotations'].append(dict(id=i+1, image_id=i+1, category_id=1, iscrowd=0,
            segmentation=segmentation, area=int(mask_utils.area(rle)), bbox=mask_utils.toBbox(rle).tolist()))
        record.update(image=f'dataset/{split}/{name}', native_mask=f'private/native_masks/{name}',
                      roi_size=[w, h])
        model_records.append(record)
    for split in ('train', 'valid'):
        write(dataset/split/'_annotations.coco.json', coco[split])
    # RF-DETR's default square resize distorts long thin faces. Export a separate
    # aspect-preserving 576-square input, with nearest labels and pixel-validity
    # rectangles consumed by our explicit padding-aware loss/matcher adapter.
    m2 = output/'dataset_rfdetr'
    m2.mkdir(mode=0o700)
    transforms = {}
    for split in ('train', 'valid'):
        (m2/split).mkdir(mode=0o700)
        square_coco = dict(info=coco[split]['info'], licenses=[], categories=coco[split]['categories'],
                           images=[], annotations=[])
        for image_record, annotation in zip(coco[split]['images'], coco[split]['annotations']):
            name = image_record['file_name']
            grey = cv2.imread(str(dataset/split/name), cv2.IMREAD_GRAYSCALE)
            native_mask = cv2.imread(str(output/'private/native_masks'/name), cv2.IMREAD_GRAYSCALE) > 0
            square, label, _, transform = letterbox(grey, native_mask, size=(576, 576))
            cv2.imwrite(str(m2/split/name), np.repeat(square[:, :, None], 3, axis=2))
            rle = mask_utils.encode(np.asfortranarray(label))
            require(np.array_equal(mask_utils.decode(rle), label), 'RF-DETR transformed RLE is not exact')
            transforms[str(image_record['id'])] = transform
            square_coco['images'].append(dict(image_record, width=576, height=576))
            square_coco['annotations'].append(dict(annotation,
                segmentation=dict(size=rle['size'], counts=rle['counts'].decode('ascii')),
                bbox=mask_utils.toBbox(rle).tolist(), area=int(mask_utils.area(rle))))
        write(m2/split/'_annotations.coco.json', square_coco)
    write(output/'private/rfdetr_transforms.json', transforms)
    # No official test labels, no aliases copying val as test, and run_test=False.
    write(output/'private/training_records.jsonl', model_records, lines=True)
    files = {**{'dataset/'+k: v for k, v in file_inventory(dataset).items()},
             **{'dataset_rfdetr/'+k: v for k, v in file_inventory(m2).items()},
             **{'private/native_masks/'+k: v for k, v in file_inventory(output/'private/native_masks').items()},
             'private/training_records.jsonl': digest(output/'private/training_records.jsonl'),
             'private/visual_semantic_approval.json': digest(output/'private/visual_semantic_approval.json'),
             'private/rfdetr_transforms.json': digest(output/'private/rfdetr_transforms.json')}
    manifest = dict(state='immutable_research_training_dataset', files=files,
        splits=dict(Counter(r['development_split'] for r in model_records)),
        independent_components={s: len({r['component'] for r in model_records if r['development_split'] == s})
                                for s in ('train', 'dev')},
        by_class={str(c): dict(Counter(r['development_split'] for r in model_records if r['class_id'] == c))
                  for c in range(1, 23)}, formal_val_test_included=False, model_input_metadata=False,
        split_rule='sort sha256({seed:42,component:global_component_hash}); first max(3,ceil(.2*Ncomponents)) => dev',
        split_amendment_before_training='Hash-mod5 left only two dev components after semantic quarantine; '
                                        'seeded rank ensures predeclared independent group support. No predictions/metrics existed.',
        native_rle_exact=True, supervision='saved single-ring research masks, never GT qualification',
        source_catalog_sha256=digest(SOURCE/'private/source_catalog.jsonl'),
        preparation_complete_sha256=digest(output/'preparation_complete.json'), **FALSE_FLAGS)
    write(output/'dataset_manifest.json', manifest)
    print(json.dumps(dict(splits=manifest['splits'], components=manifest['independent_components'])), flush=True)
    return manifest


def verify_dataset(output):
    output = Path(output).resolve()
    manifest = read(output/'dataset_manifest.json')
    require(manifest['state'] == 'immutable_research_training_dataset' and not manifest['formal_val_test_included'],
            'Invalid research dataset')
    for name, expected in manifest['files'].items():
        path = output/name
        require(path.resolve().is_relative_to(output) and not path.is_symlink() and digest(path) == expected,
                'Immutable dataset artifact changed: ' + name)
    records = jsonl(output/'private/training_records.jsonl')
    a = {r['component'] for r in records if r['development_split'] == 'train'}
    b = {r['component'] for r in records if r['development_split'] == 'dev'}
    require(a and b and not a & b and all(r['formal_split'] == 'train' for r in records),
            'Development split leakage')
    return manifest, records
