"""Versioned native-intensity recovery, permanent retirement and GUI review delivery."""
from __future__ import annotations
import argparse
from collections import Counter
from copy import deepcopy
from dataclasses import replace
from datetime import datetime, timezone
import hashlib
import json
from pathlib import Path
import shutil
import zipfile
import cv2
import numpy as np
from PIL import Image
import torch
from torch.utils.data import DataLoader
from pallet.annotation import load_jsonl, save_jsonl
from pallet.data.loader import decoder_digest, load_frame, NormalizationStats
from pallet.data.pcd import read_pcd
from pallet.data.protocol import file_digest, load_protocol, write_json
from pallet.devices import device_metadata, resolve_device
from pallet.export import load_detector_checkpoint
from pallet.experiments.evaluate import predict_batch_boxes
from pallet.human_labeling import _InferenceDataset, _collate_inference, _validated_manual_partition
from pallet.review_batches import display

ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT/'outputs/pallet_clean_intensity_v1'
PROTOCOL = ROOT/'outputs/protocol_v6_clean'
OLD_PROTOCOL = ROOT/'outputs/protocol_v5'
OLD = ROOT/'outputs/human_correction_manual_1716_v2'
SOURCE = ROOT/'outputs/anylabeling/pallet_v5_20260930'
RECOVERY = ROOT/'outputs/black_intensity_recovery_check_20261008_v1'
AUDIT = ROOT/'outputs/black_image_audit_20261008_v1'
UUID = 'GPU-3ce8e1e7-efb2-80a6-de52-75da2f9fa259'


def read(path):
    return json.loads(Path(path).read_text())


def rows(path):
    return [json.loads(line) for line in Path(path).read_text().splitlines() if line]


def jsonl(path, values):
    path.parent.mkdir(parents=True, exist_ok=True)
    payload = ''.join(json.dumps(row, ensure_ascii=False, sort_keys=True)+'\n' for row in values)
    safe_bytes(path, payload.encode())


def reserve():
    if shutil.disk_usage(ROOT).free < 50*2**30:
        raise RuntimeError('Disk below 50 GiB reserve')


def safe_bytes(path, payload):
    path = Path(path); path.parent.mkdir(parents=True, exist_ok=True)
    if path.exists():
        if path.read_bytes() != payload:
            raise ValueError(f'Preserving modified output; select a new version: {path}')
    else:
        reserve(); path.write_bytes(payload)


def safe_json(path, value):
    safe_bytes(path, json.dumps(value, ensure_ascii=False, indent=2).encode())


def safe_copy(source, target):
    target = Path(target); target.parent.mkdir(parents=True, exist_ok=True)
    if target.exists():
        if file_digest(source) != file_digest(target):
            raise ValueError(f'Preserving modified output: {target}')
    else:
        reserve(); shutil.copy2(source, target)


def stamp(stage, **kw):
    write_json(OUT/'workflow_status.json', {'stage': stage, 'updated_at_utc': datetime.now(timezone.utc).isoformat(), **kw})
    print(stage, kw, flush=True)


def prepare():
    OUT.mkdir(parents=True, exist_ok=True)
    binding = {str(p.relative_to(ROOT)): file_digest(p) for p in [
        OLD_PROTOCOL/'protocol.json', OLD/'training/best.pt', OLD/'confidence_threshold.json',
        OLD/'training/completed.json', OLD/'inputs/annotations.jsonl', OLD/'inputs/selection.json',
        OLD/'anylabeling/frames.jsonl', SOURCE/'frames.jsonl',
        RECOVERY/'recoverable_118_raw_intensity_yaml_frames.json', AUDIT/'black_frame_source_trace.json']}
    safe_json(OUT/'input_binding.json', binding)
    original_rows = rows(OLD_PROTOCOL/'manifest.jsonl')
    original = {r['frame_id']: r for r in original_rows}
    bad = read(AUDIT/'black_frame_source_trace.json')
    recovered = read(RECOVERY/'recoverable_118_raw_intensity_yaml_frames.json')
    assert len(bad) == 148 and len(recovered) == 118
    ids = {r['frame_id'] for r in bad}
    assert not ids & {r['frame_id'] for r in recovered}
    candidates = read(RECOVERY/'timestamp_candidates.json')
    retirements = []
    for row in bad:
        fid = row['frame_id']; old = original[fid]
        matched = [c for c in candidates if c['target_frame_id'] == fid and c['kind'] == 'file']
        retirements.append({'frame_id': fid, 'source_sha256': old['source_sha256'],
            'alias_frame_ids': sorted(set(old.get('aliases', [])) | {c['manifest']['frame_id'] for c in matched if c.get('manifest', {}).get('frame_id')}),
            'source_paths': sorted({str(Path(old['paths']['pcd']).resolve())} | {str(Path('/root/private/data/pallet', c['path']).resolve()) for c in matched if c['path'].endswith('.pcd')}),
            'reason': row['cause'], 'manual_partition': row['manual_partition'],
            'excluded_from_future_training_and_annotation': True})
    safe_json(OUT/'exclusions.json', {'authorized_by_user': True, 'frames': retirements,
        'scope': 'All future training, inference annotation and GUI exports; historical artifacts retained',
        'evidence': str(RECOVERY/'summary.json')})
    source_index = {r['frame_id']: r for r in rows(SOURCE/'frames.jsonl')}
    entries = []
    for n, row in enumerate(recovered, 1):
        fid = row['frame_id']; key = Path(source_index[fid]['image']).stem
        if file_digest(row['usable_pcd']) != row['pcd_sha256']:
            raise ValueError(f'PCD hash changed: {fid}')
        header, cloud = read_pcd(row['usable_pcd']); h,w = row['native_grid']
        assert len(cloud) == h*w and int(header['WIDTH'][0]) == w and int(header['HEIGHT'][0]) == h
        intensity = cloud[:,3].reshape(h,w); valid = np.isfinite(intensity) & (intensity >= 0)
        assert valid.any() and np.unique(intensity).size > 1
        view = display(intensity, valid, log=True)
        assert np.array_equal(view, np.asarray(Image.open(SOURCE/source_index[fid]['image'])))
        directory = OUT/'reconstructed'; directory.mkdir(exist_ok=True)
        yaml = directory/f'{key}.intensity.yaml'
        if not yaml.exists():
            fs = cv2.FileStorage(str(yaml), cv2.FILE_STORAGE_WRITE); fs.write('intensity', intensity); fs.release()
        fs = cv2.FileStorage(str(yaml), cv2.FILE_STORAGE_READ); loaded = fs.getFirstTopLevelNode().mat(); fs.release()
        np.testing.assert_array_equal(loaded, intensity)
        npz = directory/f'{key}.npz'
        if not npz.exists(): np.savez_compressed(npz, intensity=intensity, validity=valid)
        with np.load(npz) as data:
            np.testing.assert_array_equal(data['intensity'], intensity); np.testing.assert_array_equal(data['validity'], valid)
        png = directory/f'{key}.png'
        if not png.exists(): Image.fromarray(view).save(png)
        np.testing.assert_array_equal(np.asarray(Image.open(png)), view)
        mask = directory/f'{key}.intensity_validity.png'
        if not mask.exists(): Image.fromarray(valid.astype(np.uint8)*255).save(mask)
        entries.append({**row, 'intensity_yaml': str(yaml), 'intensity_npz': str(npz),
            'intensity_png': str(png), 'intensity_validity_png': str(mask),
            'derived_sha256': {p.name: file_digest(p) for p in [yaml,npz,png,mask]}})
        if n % 30 == 0: print('reconstruct', n, flush=True)
    safe_json(OUT/'reconstruction_manifest.json', entries)
    PROTOCOL.mkdir(exist_ok=True)
    replacement = {r['frame_id']: r for r in entries}
    active_rows = []
    for row in original_rows:
        if row['frame_id'] in ids: continue
        row = deepcopy(row)
        if row['frame_id'] in replacement:
            patch = replacement[row['frame_id']]
            row['original_zero_intensity_path'] = row['paths']['intensity']
            row['paths']['intensity'] = patch['intensity_yaml']
            path = Path(patch['intensity_yaml']); st = path.stat()
            row['source_stat'][str(path)] = {'size': st.st_size, 'mtime_ns': st.st_mtime_ns}
            row['intensity_reconstruction'] = patch
        active_rows.append(row)
    assert len(active_rows) == 10374
    jsonl(PROTOCOL/'manifest.jsonl', active_rows)
    old_protocol = read(OLD_PROTOCOL/'protocol.json')
    groups = {r['group_id'] for r in active_rows}
    for name in old_protocol['files']:
        if name == 'manifest.jsonl': continue
        if name == 'splits.json':
            value = read(OLD_PROTOCOL/name); value['assignments'] = {g:s for g,s in value['assignments'].items() if g in groups}
            safe_json(PROTOCOL/name, value)
        elif name == 'pilot.json':
            safe_json(PROTOCOL/name, {s:[fid for fid in frames if fid not in ids] for s,frames in read(OLD_PROTOCOL/name).items()})
        elif name == 'excluded.json':
            safe_json(PROTOCOL/name, read(OLD_PROTOCOL/name)+retirements)
        else: safe_copy(OLD_PROTOCOL/name, PROTOCOL/name)
    protocol = {**old_protocol, 'version': 6, 'parent_protocol_sha256': file_digest(OLD_PROTOCOL/'protocol.json'),
        'files': {name:file_digest(PROTOCOL/name) for name in old_protocol['files']},
        'frames': len(active_rows), 'groups': len(groups), 'excluded': old_protocol['excluded']+148,
        'split_counts': dict(Counter(r['split'] for r in active_rows)),
        'domains': dict(Counter(r['sensor_config']['domain_id'] for r in active_rows)),
        'depth_verified_frames': sum(r['sensor_config'].get('depth_available',False) for r in active_rows),
        'pilot_counts': {s:len(v) for s,v in read(PROTOCOL/'pilot.json').items()},
        'reconstructed_intensity_frames':118, 'retirement_policy_sha256':file_digest(OUT/'exclusions.json')}
    safe_json(PROTOCOL/'protocol.json', protocol)
    load_protocol(PROTOCOL)
    # Prove the only decoder change is the retirement check, before resealing unchanged cached values.
    previous_seal = read(ROOT/'outputs/cache_v5/seal.json')
    old_loader = (ROOT/'pallet/data/loader.py').read_bytes().replace(
        b'    from .eligibility import require_eligible\n    require_eligible(record)\n', b'')
    old_decoder = hashlib.sha256(old_loader+(ROOT/'pallet/data/pcd.py').read_bytes()).hexdigest()
    if old_decoder != previous_seal['decoder_sha256']:
        raise ValueError('Decoder has additional numerical changes; rebuild the sensor cache')
    index_path = ROOT/'outputs/cache_v5/sealed_index.json'
    if file_digest(index_path) != previous_seal['index_sha256']: raise ValueError('Original cache index changed')
    index = {fid:value for fid,value in read(index_path).items() if fid not in ids}
    assert len(index) == 10374
    cache = ROOT/'outputs/cache_v6_clean'; cache.mkdir(exist_ok=True)
    safe_json(cache/'sealed_index.json', index)
    safe_json(cache/'seal.json', {'protocol_sha256':file_digest(PROTOCOL/'protocol.json'), 'frames':10374,
        'index_sha256':file_digest(cache/'sealed_index.json'), 'decoder_sha256':decoder_digest(),
        'parent_seal_sha256':file_digest(ROOT/'outputs/cache_v5/seal.json'),
        'decoder_note':'Byte-exact proof: removing only require_eligible import/call reproduces the prior sealed decoder digest; sensor decoding unchanged.'})
    filter_manual(ids)
    stamp('prepared', reconstructed=118, retired=148, active_frames=10374)
    return entries, source_index, ids


def filter_manual(ids):
    target = OUT/'human_training'; inputs = target/'inputs'; inputs.mkdir(parents=True, exist_ok=True)
    annotations = load_jsonl(OLD/'inputs/annotations.jsonl')
    removed = sorted(set(annotations.frames) & ids)
    assert len(removed) == 35 and all(annotations.frames[fid].status == 'negative' for fid in removed)
    annotations.frames = {fid:frame for fid,frame in annotations.frames.items() if fid not in ids}
    labels = inputs/'annotations.jsonl'
    if labels.exists():
        assert set(load_jsonl(labels).frames) == set(annotations.frames)
    else: save_jsonl(annotations, labels)
    selection = {s:[fid for fid in frames if fid not in ids] for s,frames in read(OLD/'inputs/selection.json').items()}
    safe_json(inputs/'selection.json', selection)
    for name,role in [('human_training_manifest.json','training'),('human_validation_manifest.json','validation')]:
        manifest = read(OLD/'inputs'/name)
        manifest['protocol_sha256'] = file_digest(PROTOCOL/'protocol.json')
        manifest['annotation_snapshot_sha256'] = file_digest(labels)
        manifest['source_frames'] = {fid:row for fid,row in manifest['source_frames'].items() if fid not in ids}
        manifest[f'{role}_frame_ids'] = sorted(manifest['source_frames'])
        manifest[f'{role}_frames'] = len(manifest['source_frames'])
        manifest['retirement_policy_sha256'] = file_digest(OUT/'exclusions.json')
        safe_json(inputs/name,manifest)
    partition = read(OLD/'manual_partition.json')
    partition.update(protocol_sha256=file_digest(PROTOCOL/'protocol.json'),
        manual_annotations_sha256=file_digest(labels), expected_manual_frames=len(annotations.frames),
        all_manual_frame_ids=sorted(annotations.frames), train_frame_ids=sorted(selection['train']),
        validation_frame_ids=sorted(selection['val']),
        parent_partition_sha256=file_digest(OLD/'manual_partition.json'), removed_manual_negative_frames=removed)
    for role in ['train','val']:
        selected = [annotations.frames[fid] for fid in selection[role]]
        partition['status_counts'][role] = dict(Counter(f.status for f in selected))
        partition['positive_boxes'][role] = sum(len(f.boxes) for f in selected)
    partition['train_groups'] = sorted({annotations.frames[f].group_id for f in selection['train']})
    partition['validation_groups'] = sorted({annotations.frames[f].group_id for f in selection['val']})
    partition['domain_counts'] = {}
    for domain in sorted({frame.domain for frame in annotations.frames.values()}):
        train = [annotations.frames[f] for f in selection['train'] if annotations.frames[f].domain == domain]
        val = [annotations.frames[f] for f in selection['val'] if annotations.frames[f].domain == domain]
        partition['domain_counts'][domain] = {'total_frames':len(train)+len(val),
            'training_frames':len(train),'validation_frames':len(val),
            'training_negatives':sum(f.status=='negative' for f in train),
            'validation_negatives':sum(f.status=='negative' for f in val),
            'note':'Preserved prior group assignments after user-authorized black-frame retirement; no reshuffle.'}
    partition['artifact_sha256'] = {str(p.relative_to(target)):file_digest(p) for p in inputs.iterdir() if p.is_file()}
    safe_json(target/'manual_partition.json',partition)
    _validated_manual_partition(target,PROTOCOL)
    assert len(selection['train']) == 1433 and len(selection['val']) == 248


def infer(entries, batch_size):
    checkpoint = OLD/'training/best.pt'; threshold = read(OLD/'confidence_threshold.json')
    if file_digest(checkpoint) != read(OLD/'training/completed.json')['best_checkpoint_sha256']:
        raise ValueError('Checkpoint completion hash mismatch')
    assert threshold['checkpoint_sha256'] == file_digest(checkpoint)
    records, assignments, _ = load_protocol(PROTOCOL)
    ids = {r['frame_id'] for r in entries}; selected = [r for r in records if r.frame_id in ids]
    binding = {'checkpoint_sha256':file_digest(checkpoint), 'threshold_sha256':file_digest(OLD/'confidence_threshold.json'),
        'protocol_sha256':file_digest(PROTOCOL/'protocol.json'), 'reconstruction_sha256':file_digest(OUT/'reconstruction_manifest.json'),
        'device_uuid':device_metadata(UUID)['uuid'], 'batch_size':batch_size, 'size':640}
    safe_json(OUT/'prediction_binding.json', binding)
    prediction_dir = OUT/'predictions'; prediction_dir.mkdir(exist_ok=True)
    pending=[]; predictions={}
    for record in selected:
        path=prediction_dir/f'{hashlib.sha256(record.frame_id.encode()).hexdigest()[:24]}.json'
        if path.exists():
            pred=read(path)
            if pred['binding']!=binding or pred['source_anchor_hash']!=record.source_sha256: raise ValueError('Prediction resume binding mismatch')
            predictions[record.frame_id]=pred
        else: pending.append(record)
    if pending:
        torch.set_num_threads(4)
        model, config = load_detector_checkpoint(checkpoint)
        model.to(resolve_device(UUID)).eval()
        normalization=NormalizationStats(**torch.load(checkpoint,map_location='cpu',weights_only=False)['normalization'])
        loader=DataLoader(_InferenceDataset(pending,normalization,640),batch_size=batch_size,num_workers=2,
            pin_memory=True,collate_fn=_collate_inference)
        for images,metas in loader:
            results=predict_batch_boxes(model,images.to(resolve_device(UUID),non_blocking=True),metas,
                conf=float(threshold['selected_confidence']),size=640)
            for meta, detections in zip(metas,results):
                h,w=meta.original_size; boxes=[]
                for xyxy,score in detections:
                    coords=np.clip(xyxy,[0,0,0,0],[w,h,w,h]).tolist()
                    if coords[2]>coords[0] and coords[3]>coords[1]: boxes.append({'xyxy':coords,'score':score})
                record=next(r for r in selected if r.frame_id==meta.frame_id)
                pred={'frame_id':record.frame_id,'source_anchor_hash':record.source_sha256,'boxes_xyxy_score':boxes,
                    'width':w,'height':h,'binding':binding,'threshold':threshold['selected_confidence']}
                safe_json(prediction_dir/f'{hashlib.sha256(record.frame_id.encode()).hexdigest()[:24]}.json',pred)
                predictions[record.frame_id]=pred
            stamp('predicting', completed=len(predictions),expected=118)
    assert len(predictions)==118
    safe_json(OUT/'prediction_manifest.json', {'state':'complete','completed_frames':118,'errors':{},'binding':binding,
        'boxes':sum(len(p['boxes_xyxy_score']) for p in predictions.values()),
        'zero_detections':sum(not p['boxes_xyxy_score'] for p in predictions.values())})
    return predictions


def proposal_document(fid, original, prediction, entries, manual_ids):
    w,h=original['size']; boxes=prediction['boxes_xyxy_score']; key=Path(original['image']).name
    return {'version':'0.4.43','flags':{'human_reviewed':False,'unreviewed':True,'proposal_only':True,
        'confirmed_negative':False,'ignore':False,'machine_resolved':False,'label_conflict':False},
        'shapes':[{'label':'pallet','points':[box['xyxy'][:2],box['xyxy'][2:]],'shape_type':'rectangle',
            'group_id':None,'flags':{'proposal_only':True},'text':''} for box in boxes],
        'imagePath':key,'imageData':None,'imageWidth':w,'imageHeight':h,'text':'',
        'pallet_prediction':{'frame_id':fid,'group_id':original['group_id'],'domain':original['domain'],
            'source_anchor_hash':original['source_anchor_hash'],'checkpoint_sha256':prediction['binding']['checkpoint_sha256'],
            'threshold_sha256':prediction['binding']['threshold_sha256'],'threshold':prediction['threshold'],
            'confidence':[b['score'] for b in boxes], 'prediction_only':True,'independent_reference':False,
            'status':'proposal' if boxes else 'zero_detection_unreviewed',
            'reconstructed_intensity':entries[fid], 'previously_manually_checked':fid in manual_ids}}


def audit_workspace(directory, expected_ids, *, proposals):
    index=rows(directory/'frames.jsonl')
    if {r['frame_id'] for r in index}!=expected_ids or len(index)!=len(expected_ids): raise ValueError('Coverage mismatch')
    count=0
    for row in index:
        doc=read(directory/row['annotation']); w,h=row['size']
        if doc['imageWidth']!=w or doc['imageHeight']!=h: raise ValueError('JSON dimensions mismatch')
        for role in ['image','depth','validity']:
            path=directory/row[role]
            if file_digest(path)!=row['view_sha256'][role]: raise ValueError('View hash mismatch')
            with Image.open(path) as image:
                image.load()
                if image.size!=(w,h): raise ValueError('View dimensions mismatch')
                if role=='image' and np.asarray(image).max()==0: raise ValueError('Black primary remains')
        if file_digest(directory/row['annotation'])!=row['baseline_annotation_sha256']: raise ValueError('Label changed after freeze')
        if proposals:
            f=doc['flags']
            if not (f['unreviewed'] and f['proposal_only'] and not f['human_reviewed'] and not f['confirmed_negative'] and not f['ignore']): raise ValueError('Proposal flags conflict')
        for shape in doc['shapes']:
            (x1,y1),(x2,y2)=shape['points']
            if shape['label']!='pallet' or shape['shape_type']!='rectangle' or not (0<=x1<x2<=w and 0<=y1<y2<=h): raise ValueError('Invalid rectangle')
            count+=1
    result={'state':'passed','frames':len(index),'boxes':count,'black_primary_frames':0,'failed':0}
    safe_json(directory/'audit.json',result)
    return result


def publish_subset(source, index, ids, target, archive):
    selected=[r for r in index if r['frame_id'] in ids]
    for row in selected:
        for role in ['image','annotation','depth','validity','intensity_validity']:
            if row.get(role): safe_copy(source/row[role],target/row[role])
    jsonl(target/'frames.jsonl',selected)
    safe_copy(source/'anylabeling_config.yaml',target/'anylabeling_config.yaml')
    safe_copy(source/'labels.txt',target/'labels.txt')
    manifest={'kind':'model_assisted_human_correction_working_labels','frames':len(selected),
        'frames_index_sha256':file_digest(target/'frames.jsonl'), 'split_counts':dict(Counter(r['split'] for r in selected)),
        'prediction_binding':read(OUT/'prediction_binding.json'),'zero_detection_is_negative':False,
        'validation_test_outputs_are_independent_references':False,'excluded_frames':148}
    safe_json(target/'package_manifest.json',manifest)
    safe_bytes(target/'人工复查说明.md', ('打开 images/train、val、test 中的PNG与同名JSON。建议框均待人工复查；空框不代表负例。\n'
        '验证/测试目录只是模型辅助修正工作版本，不能作为独立评估参考。148张不可恢复的黑图已剔除。\n'
        '118帧专题包包含18张此前人工检查过的帧，是再次复查建议；原人工标签保存在独立快照。\n'
        '修正后交回同名JSON，保留pallet_prediction字段；不要更改文件名或图像尺寸。\n').encode())
    audit=audit_workspace(target,ids,proposals=True)
    if not archive.exists():
        reserve(); temp=archive.with_suffix('.zip.tmp')
        with zipfile.ZipFile(temp,'w',compression=zipfile.ZIP_DEFLATED,compresslevel=4) as z:
            for path in sorted(target.rglob('*')):
                if path.is_file(): z.write(path,str(path.relative_to(target)))
        temp.replace(archive)
    with zipfile.ZipFile(archive) as z:
        if z.testzip() is not None: raise ValueError('ZIP CRC failed')
        files={str(p.relative_to(target)):p for p in target.rglob('*') if p.is_file()}
        if set(z.namelist())!=set(files): raise ValueError('ZIP file coverage mismatch')
        for name,path in files.items():
            if hashlib.sha256(z.read(name)).hexdigest()!=file_digest(path): raise ValueError('ZIP payload changed')
    return {**audit,'archive':str(archive),'archive_sha256':file_digest(archive),'archive_bytes':archive.stat().st_size,
        'zip_crc':'passed','zip_payload_hashes':'passed'}


def package(entries, source_index, bad_ids, predictions):
    recovered={e['frame_id']:e for e in entries}
    human=load_jsonl(OUT/'human_training/inputs/annotations.jsonl'); manual_ids=set(human.frames)
    old_remaining={r['frame_id']:r for r in rows(OLD/'anylabeling/frames.jsonl')}
    remaining=set(old_remaining)-bad_ids
    assert len(remaining)==8693 and not remaining & manual_ids
    active=set(source_index)-bad_ids
    directory=OUT/'active_anylabeling'; index=[]
    for n,fid in enumerate(sorted(active),1):
        original=source_index[fid]; row=deepcopy(original)
        if fid in recovered:
            doc=proposal_document(fid,original,predictions[fid],recovered,manual_ids)
        elif fid in remaining:
            doc=read(OLD/'anylabeling'/old_remaining[fid]['annotation'])
        else:
            frame=human.frames[fid]
            doc={'version':'0.4.43','flags':{'human_reviewed':True,'unreviewed':False,'proposal_only':False,
                'confirmed_negative':frame.status=='negative','ignore':False,'machine_resolved':False},
                'shapes':[{'label':'pallet','points':[[b.x,b.y],[b.x+b.width,b.y+b.height]],'shape_type':'rectangle',
                    'group_id':None,'flags':{},'text':''} for b in frame.boxes],
                'imagePath':Path(original['image']).name,'imageData':None,'imageWidth':frame.width,'imageHeight':frame.height,
                'pallet_export':{'frame_id':fid,'source_anchor_hash':frame.source_anchor_hash,'source_status':frame.status,
                    'human_source_snapshot_sha256':file_digest(OUT/'human_training/inputs/annotations.jsonl')}}
        for role in ['image','depth','validity']:
            src=Path(recovered[fid]['intensity_png']) if role=='image' and fid in recovered else SOURCE/original[role]
            safe_copy(src,directory/original[role])
        if fid in recovered:
            mask=Path(original['depth']).with_name(Path(original['image']).stem+'.intensity_validity.png')
            safe_copy(recovered[fid]['intensity_validity_png'],directory/mask); row['intensity_validity']=str(mask)
        safe_json(directory/original['annotation'],doc)
        row.pop('baseline_json_sha256',None); row.pop('files',None)
        row['baseline_annotation_sha256']=file_digest(directory/original['annotation'])
        row['view_sha256']={role:file_digest(directory/original[role]) for role in ['image','depth','validity']}
        row['independent_reference']=False
        row['status']=doc.get('pallet_prediction',{}).get('status', human.frames[fid].status if fid in manual_ids else 'proposal')
        row['proposal_count']=len(doc['shapes']); row['reconstructed_intensity']=fid in recovered
        index.append(row)
        if n%2000==0: stamp('packaging',active_workspace_frames=n,expected=10374)
    jsonl(directory/'frames.jsonl',index)
    safe_copy(SOURCE/'anylabeling_config.yaml',directory/'anylabeling_config.yaml')
    safe_bytes(directory/'labels.txt',b'pallet\n')
    safe_json(directory/'package_manifest.json', {'frames':10374,'kind':'active_clean_native_grid_correction_dataset',
        'frames_index_sha256':file_digest(directory/'frames.jsonl'),
        'prediction_binding':read(OUT/'prediction_binding.json'),'human_snapshot_frames':1681,
        'reconstructed_intensity_frames':118,'retired_frames':148,'independent_reference':False})
    audit_workspace(directory,active,proposals=False)
    stamp('auditing_archives',remaining=8693,recovered=118)
    specific=publish_subset(directory,index,set(recovered),OUT/'reconstructed_118_review',OUT/'reconstructed_intensity_118_review_v1.zip')
    pending=publish_subset(directory,index,remaining,OUT/'remaining_8693_review',OUT/'human_correction_remaining_8693_clean_v1.zip')
    safe_json(OUT/'delivery.json',{'state':'complete','active_frames':10374,'retired':148,'manual_labels':1681,
        'manual_train':1433,'manual_val':248,'remaining':pending,'reconstructed_review':specific,
        'reconstructed_remaining_overlap':100,'reconstructed_previously_manual':18,
        'model_not_retrained':True,'no_test_scoring':True})
    safe_json(ROOT/'configs/active_dataset.json',{'protocol':str(PROTOCOL),
        'source_export':str(directory),'human_training':str(OUT/'human_training'),
        'cache_index':str(ROOT/'outputs/cache_v6_clean/sealed_index.json'),
        'exclusions':str(OUT/'exclusions.json'),'exclusions_sha256':file_digest(OUT/'exclusions.json'),
        'remaining_review_package':str(OUT/'human_correction_remaining_8693_clean_v1.zip'),
        'reconstructed_review_package':str(OUT/'reconstructed_intensity_118_review_v1.zip')})
    stamp('complete',active_frames=10374,retired=148,reconstructed=118,remaining=8693)


def main():
    parser=argparse.ArgumentParser(); parser.add_argument('--batch-size',type=int,default=16)
    args=parser.parse_args()
    entries,index,bad=prepare(); predictions=infer(entries,args.batch_size); package(entries,index,bad,predictions)

if __name__=='__main__': main()
