"""Native mask evaluation, per-face ownership, and hole-aware AnyLabeling export."""
from __future__ import annotations

from collections import Counter, defaultdict
import copy
import gc
import hashlib
import io
import json
import math
from pathlib import Path
import tempfile
import time
import zipfile

import cv2
import numpy as np
from PIL import Image, ImageDraw
from scipy.optimize import linear_sum_assignment

from pallet.fork_face_anylabeling import encoded, mask_to_polygons, protected_write, safe_path
from pallet.fork_face_simplify import check_label_save
from pallet.instance_segmentation_seed import (bbox, digest, event, letterbox, native_image, png,
    read_json, restore_mask, rle, save, source_document, verify_dataset, RESOLUTION)
from pallet.instance_segmentation_training import model_class


def closed_holes(mask):
    background = np.pad(~np.asarray(mask, bool), 1, constant_values=True).astype(np.uint8)
    _, components = cv2.connectedComponents(background, connectivity=4)
    return (components[1:-1,1:-1]!=components[0,0]) & ~mask


def iou(a, b):
    union = (a | b).sum()
    return float((a & b).sum()/union) if union else 0.


def deduplicate(candidates):
    kept = []; areas = []; boxes = []
    for candidate in sorted(candidates, key=lambda c: (-c['score'], c.get('prediction_index', 0))):
        mask = candidate['mask']
        area=int(mask.sum())
        if not area:continue
        box=bbox(mask);x,y,w,h=box;duplicate=False
        for k,other_area,other_box in zip(kept,areas,boxes):
            ox,oy,ow,oh=other_box
            left,top=max(x,ox),max(y,oy);right,bottom=min(x+w,ox+ow),min(y+h,oy+oh)
            if right<=left or bottom<=top:continue
            intersection=int((mask[top:bottom,left:right]&k['mask'][top:bottom,left:right]).sum())
            if intersection/(area+other_area-intersection)>.75 or intersection/min(area,other_area)>.95:
                duplicate=True;break
        if not duplicate:kept.append(candidate);areas.append(area);boxes.append(box)
    return kept


def load_model(ctx, kind):
    import torch
    torch.set_num_threads(4); device=ctx['args'].device
    torch.cuda.set_device(torch.device(device))
    receipt=read_json(ctx['output']/f'training/{kind}/completion.json')
    path=ctx['output']/f'training/{kind}'/receipt['checkpoint']
    if digest(path)!=receipt['checkpoint_sha256']: raise ValueError('Selected model weights changed')
    return model_class(kind)(num_classes=1, resolution=RESOLUTION, device=device,
                             pretrain_weights=str(path), compile=False)


def infer(model, image):
    """Predict on preletterboxed RGB and restore every nonempty mask to native size."""
    import torch
    prepared, transform = letterbox(image)
    with torch.inference_mode():
        detections = model.predict(prepared, threshold=1e-6, include_source_image=False)
    candidates=[]
    if detections.mask is None: return candidates
    for number, (score, label, padded) in enumerate(zip(detections.confidence,detections.class_id,detections.mask)):
        if int(label)!=0: continue
        mask=restore_mask(np.asarray(padded,bool), transform)
        if mask.any(): candidates.append({'score':float(score),'prediction_index':number,'mask':mask,
                                         'padded_mask':np.asarray(padded,bool), 'bbox':bbox(mask)})
    return deduplicate(candidates)


def match_masks(gt, candidates):
    if not gt or not candidates: return []
    matrix=np.array([[iou(g,p['mask']) for p in candidates] for g in gt])
    ii,jj=linear_sum_assignment(1-matrix)
    return [(int(i),int(j),float(matrix[i,j])) for i,j in zip(ii,jj)]


def evaluate_model(ctx, kind):
    from pycocotools.coco import COCO
    from pycocotools.cocoeval import COCOeval
    frames=[f for f in read_json(ctx['output']/'dataset/native_index.json') if f['split']=='valid']
    model=load_model(ctx,kind); predictions=[]; native_results=[]; started=time.monotonic()
    for f in frames:
        candidates=infer(model,Image.open(io.BytesIO(native_image(ctx,f['frame_key']))))
        for p in candidates:
            predictions.append({'image_id':f['image_id'],'category_id':1,'score':p['score'],
                                'segmentation':rle(p['padded_mask'])})
        gt=[np.asarray(Image.open(ctx['output']/i['mask']))>0 for i in f['instances']]
        holes=[np.asarray(Image.open(ctx['output']/i['holes_mask']))>0 for i in f['instances']]
        native_results.append((f,gt,holes,candidates))
    groundtruth=COCO(str(ctx['output']/'dataset/valid/_annotations.coco.json'))
    if predictions:
        predicted=groundtruth.loadRes(predictions)
        evaluator=COCOeval(groundtruth,predicted,'segm');evaluator.params.maxDets=[1,10,100]
        evaluator.evaluate();evaluator.accumulate();evaluator.summarize()
        ap=float(evaluator.stats[0]); ap50=float(evaluator.stats[1])
    else: ap=ap50=0.
    calibration=[]
    for threshold in [.05,.1,.15,.2,.3,.4,.5,.6,.7,.8]:
        tp=fp=fn=0; all_ious=[]; hole_ious=[]; failures=0
        for f,gt,holes,raw in native_results:
            candidates=[p for p in raw if p['score']>=threshold]
            matches=match_masks(gt,candidates);true=sum(value>=.5 for _,_,value in matches)
            tp+=true;fp+=len(candidates)-true;fn+=len(gt)-true
            values={i:(j,value) for i,j,value in matches}
            for i,mask in enumerate(gt):
                value=values.get(i)
                all_ious.append(value[1] if value else 0.)
                failures+=not value or value[1]<.5
                if holes[i].any():
                    candidate=candidates[value[0]]['mask'] if value else np.zeros_like(mask)
                    hole_ious.append(iou(holes[i],closed_holes(candidate)))
        f1=2*tp/max(2*tp+fp+fn,1)
        calibration.append({'threshold':threshold,'true_positives':tp,'false_positives':fp,'false_negatives':fn,
             'f1_at_iou_50':f1,'matched_mask_iou':float(np.mean(all_ious)),
             'hole_background_iou':float(np.mean(hole_ious)) if hole_ious else None,'failed_gt_instances':failures})
    best=max(calibration,key=lambda r:(r['f1_at_iou_50'],r['true_positives'],-r['false_positives'],r['threshold']))
    train=read_json(ctx['output']/f'training/{kind}/completion.json')
    metrics={'model':kind,'mask_ap_50_95':ap,'mask_ap_50':ap50,'calibration':calibration,
             'selected_threshold':best['threshold'], 'matched_mask_iou':best['matched_mask_iou'],
             'hole_background_iou':best['hole_background_iou'],'failed_gt_instances':best['failed_gt_instances'],
             'validation_frames':len(frames),'validation_instances':sum(len(f[1]) for f in native_results),
             'validation_inference_seconds':time.monotonic()-started,
             'training_elapsed_seconds':train['elapsed_seconds'],'peak_cuda_memory_bytes':train['peak_cuda_memory_bytes'],
             'checkpoint_sha256':train['checkpoint_sha256'], 'scope':'internal_group_validation_used_for_selection'}
    save(ctx['output']/f'evaluation/{kind}.json',metrics)
    del model;gc.collect()
    import torch
    torch.cuda.empty_cache()
    return metrics


def select(ctx):
    verify_dataset(ctx)
    comparisons=[]
    for kind in ('small','medium'):
        path=ctx['output']/f'evaluation/{kind}.json'
        metric=read_json(path) if path.exists() else evaluate_model(ctx,kind)
        receipt=read_json(ctx['output']/f'training/{kind}/completion.json')
        if metric['checkpoint_sha256']!=receipt['checkpoint_sha256']:
            raise ValueError('Cached evaluation belongs to another checkpoint')
        comparisons.append(metric)
    winner=max(comparisons,key=lambda m:(m['mask_ap_50_95'],m['hole_background_iou'] or 0.,
                          m['matched_mask_iou'],-m['training_elapsed_seconds']))
    result={'task_hash':ctx['binding']['task_hash'],'model':winner['model'],
            'threshold':winner['selected_threshold'],'checkpoint_sha256':winner['checkpoint_sha256'],
            'comparisons':comparisons,'selection_rule':'mask_AP; then hole_IoU; mask_IoU; training_time'}
    save(ctx['output']/'model_selection.json',result)
    event('select',model=result['model'],threshold=result['threshold'],metrics=comparisons)


def box_pixels(box, size):
    x,y,w,h=box;mask=np.zeros(size,bool)
    x0=max(0,math.floor(x)); y0=max(0,math.floor(y))
    x1=min(size[1],math.ceil(x+w)); y1=min(size[0],math.ceil(y+h))
    mask[y0:y1,x0:x1]=True
    return mask


def ownership(candidates, boxes, selected, threshold, manual=False):
    """Main faces by coverage, side faces by horizontal proximity and vertical overlap."""
    if not candidates: return []
    size=candidates[0]['mask'].shape; box_masks=[box_pixels(b,size) for b in boxes]
    assigned=[];side=[]
    target=boxes[selected]; tx,ty,tw,th=target
    for p in candidates:
        fractions=[float((p['mask']&b).sum()/p['mask'].sum()) for b in box_masks]
        ranked=np.argsort(fractions)[::-1]; best=int(ranked[0]);second=fractions[int(ranked[1])] if len(ranked)>1 else 0.
        score=max(fractions)
        if best==selected and score>=.60 and score-second>=.15:
            assigned.append({**p,'ownership':'main','parent_overlap_fraction':score})
            continue
        if best!=selected and score>=.4: continue
        x,y,w,h=p['bbox'];gap=max(tx-(x+w),x-(tx+tw),0.)
        vertical=max(0,min(y+h,ty+th)-max(y,ty))/max(min(h,th),1)
        if gap<=max(4.,.12*tw) and vertical>=.6 and w<=.6*tw and h<=1.8*th and score<.4:
            side.append({**p,'ownership':'adjacent_side','parent_overlap_fraction':fractions[selected]})
    if manual:
        return assigned+(side if assigned else [])
    regular=[p for p in assigned if p['score']>=threshold]
    if regular: return regular+[p for p in side if p['score']>=threshold]
    if assigned:
        return [{**max(assigned,key=lambda p:p['score']),'low_confidence':True}]
    return []


def parent_boxes(original):
    return [[s['points'][0][0],s['points'][0][1],s['points'][1][0]-s['points'][0][0],
             s['points'][1][1]-s['points'][0][1]] for s in original['shapes'] if s['label']=='pallet' and s['shape_type']=='rectangle']


def crop_retry(model,image,box):
    x,y,w,h=box;padx=max(12,.3*w);pady=max(12,.5*h)
    x0=max(0,math.floor(x-padx)); y0=max(0,math.floor(y-pady))
    x1=min(image.width,math.ceil(x+w+padx));y1=min(image.height,math.ceil(y+h+pady))
    result=infer(model,image.crop((x0,y0,x1,y1)))
    for p in result:
        native=np.zeros((image.height,image.width),bool);native[y0:y1,x0:x1]=p['mask']
        p['mask']=native;p['bbox']=bbox(native);p['inference_mode']='expanded_parent_crop'
    return result


def predict(ctx):
    selected=read_json(ctx['output']/'model_selection.json')
    model=None
    audit={f['frame_key']:f for f in read_json(ctx['output']/'audit/frames.json')}
    records=[]
    for number,row in enumerate(ctx['rows'],1):
        key=Path(row['image']).stem; done=ctx['output']/f'predictions/{key}.json'
        if done.exists():
            r=read_json(done)
            if r['task_hash']!=ctx['binding']['task_hash'] or r['checkpoint_sha256']!=selected['checkpoint_sha256']:
                raise ValueError('Prediction task or model changed')
            for face in r['faces']:
                if digest(ctx['output']/face['mask'])!=face['mask_sha256']:raise ValueError('Cached prediction mask changed')
            records.append(r);continue
        original=source_document(ctx,key);boxes=parent_boxes(original);candidate_manual=[]
        item=audit.get(key)
        if item and item['state']=='manual_accepted':
            for instance in item['instances']:
                candidate_manual.append({**copy.deepcopy(instance), 'mask':np.asarray(Image.open(ctx['output']/instance['mask']))>0,
                                        'score':1.,'bbox':instance['bbox'],'source':'human'})
        faces=ownership(candidate_manual,boxes,row['box_index'],selected['threshold'],True)
        mode='human_preserved'
        if not faces:
            if model is None:model=load_model(ctx,selected['model'])
            image=Image.open(io.BytesIO(native_image(ctx,key))).convert('RGB')
            candidates=infer(model,image)
            for p in candidates:p['source']='model';p['inference_mode']='full_frame'
            faces=ownership(candidates,boxes,row['box_index'],selected['threshold'])
            if not faces:
                candidates=crop_retry(model,image,boxes[row['box_index']])
                for p in candidates:p['source']='model'
                faces=ownership(candidates,boxes,row['box_index'],selected['threshold'])
            mode='model_completion'
        result=[]
        for ordinal,p in enumerate(faces,1):
            relative=f'predictions/masks/{key}__face_{ordinal:03d}.png';data=png(p['mask'])
            protected_write(ctx['output']/relative,data)
            record={k:v for k,v in p.items() if k not in ('mask','padded_mask','holes_mask','mask_sha256','instance_id')}
            record.update({'mask':relative,'mask_sha256':hashlib.sha256(data).hexdigest(),
                           'instance_id':f'{row["target_id"]}::face:{ordinal:03d}', 'face_index':ordinal,
                           'bbox':bbox(p['mask']), 'source':p['source'],
                           'low_confidence':p.get('low_confidence',False)})
            result.append(record)
        r={'task_hash':ctx['binding']['task_hash'],'checkpoint_sha256':selected['checkpoint_sha256'],
           'frame_key':key,'frame_id':row['frame_id'],'target_id':row['target_id'],'appearance_class':row['appearance_class'],
           'box_index':row['box_index'],'original_parent_bbox':row['bbox'],'mode':mode,
           'state':'completed' if result else 'unresolved_no_nonempty_matching_mask', 'faces':result,
           'quarantined_return': bool(item and item['state']=='quarantined')}
        reasons=[]
        if any(f['source']=='model' for f in result):reasons.append('machine_mask_unreviewed')
        if any(f['low_confidence'] for f in result):reasons.append('below_calibrated_confidence_threshold')
        if any(f.get('inference_mode')=='expanded_parent_crop' for f in result):reasons.append('expanded_parent_crop_retry')
        if r['quarantined_return']:reasons.append('returned_human_polygons_quarantined')
        if not result:reasons.append('no_nonempty_spatially_matching_mask_redraw_required')
        r['review_reasons']=reasons
        save(done,r);records.append(r)
        if number%20==0 or number==len(ctx['rows']):event('predict',completed=number,total=len(ctx['rows']),
                         unresolved=sum(not r['faces'] for r in records))
    save(ctx['output']/'prediction_index.json',records)


def polygon_shape(points,label,group,source):
    human=source=='human'
    return {'label':label,'shape_type':'polygon','points':points,'group_id':group,'text':'','description':'',
            'flags':{'human_reviewed':human,'mask_human_reviewed':human,'proposal_only':not human},
            'face_instance_part':{'role':'hole' if label=='__hole__' else 'positive','source':source}}


def machine_shapes(mask,group):
    holes=closed_holes(mask)
    filled=mask|holes
    return ([polygon_shape(points,'pallet',group,'model') for points in mask_to_polygons(filled)]+
            [polygon_shape(points,'__hole__',group,'model') for points in mask_to_polygons(holes)])


def render_shapes(shapes,size,upstream):
    groups=defaultdict(lambda:{'positive':np.zeros(size,bool),'hole':np.zeros(size,bool)})
    for shape in shapes:
        if shape['shape_type']!='polygon' or shape['label'] not in ('pallet','__hole__'):raise ValueError('Non-polygon or unknown label in delivery')
        role='positive' if shape['label']=='pallet' else 'hole'
        groups[shape['group_id']][role] |= upstream(size,shape['points'],'polygon')
    return {g:parts['positive'] & ~parts['hole'] for g,parts in groups.items()}


def export(ctx):
    records=read_json(ctx['output']/'prediction_index.json');delivery=ctx['output']/'delivery'
    audited={f['frame_key']:f for f in read_json(ctx['output']/'audit/frames.json')}
    files={};allfaces=[]; public=[]
    def write(name,data):
        protected_write(delivery/name,data);files[name]=hashlib.sha256(data).hexdigest()
    for r in records:
        key=r['frame_key'];original=source_document(ctx,key);raw=native_image(ctx,key)
        shapes=[];instances=[]; size=(original['imageHeight'],original['imageWidth'])
        for face in r['faces']:
            mask=np.asarray(Image.open(ctx['output']/face['mask']))>0
            group=face['face_index'];human=face['source']=='human'
            if human:
                rings=[('pallet',face['outer_shape'])]+[('__hole__',s) for s in face['hole_shapes']]
                for label,shape in rings:
                    copied=copy.deepcopy(shape);copied['group_id']=group
                    copied.setdefault('text','');copied.setdefault('description','')
                    copied['flags'].update({'human_reviewed':True,'mask_human_reviewed':True,'proposal_only':False})
                    copied['face_instance_part']={'role':'hole' if label=='__hole__' else 'positive','source':'human'}
                    shapes.append(copied)
            else:shapes.extend(machine_shapes(mask,group))
            instance={'instance_id':face['instance_id'],'group_id':group,'face_bbox':bbox(mask),
                'parent_target_id':r['target_id'],'parent_bbox':r['original_parent_bbox'],'source':face['source'],
                'score':face['score'],'low_confidence':face['low_confidence'],'mask_pixels':int(mask.sum()),
                'closed_hole_pixels':int(closed_holes(mask).sum()),'mask_human_reviewed':human,
                'segmentation_training_eligible':human,'mask_rle':rle(mask)}
            instance['ownership']=face['ownership']
            instance['parent_overlap_fraction']=face['parent_overlap_fraction']
            if human:
                instance['source_annotation']={'returned_file':key+'.json',
                    'sha256':audited[key]['source_sha256'],'positive_shape_index':face['source_shape_index'],
                    'hole_shape_indices':face['source_hole_indices']}
            else:
                instance['source_annotation']={'checkpoint_sha256':r['checkpoint_sha256'],
                    'prediction_index':face['prediction_index'],'inference_mode':face['inference_mode']}
            instances.append(instance);allfaces.append({'frame_key':key,**instance})
        rendered=render_shapes(shapes,size,ctx['upstream'])
        for face in r['faces']:
            mask=np.asarray(Image.open(ctx['output']/face['mask']))>0
            if not np.array_equal(rendered[face['face_index']],mask):raise ValueError('Hole polygon export changed native mask')
        allhuman=bool(instances) and all(i['source']=='human' for i in instances)
        doc={'version':'0.4.43','flags':{'human_reviewed':allhuman,'mask_human_reviewed':allhuman,
             'unreviewed':not allhuman,'proposal_only':not allhuman,'confirmed_negative':False,
             'unresolved_positive':not bool(instances),'segmentation_training_eligible':allhuman},
             'imagePath':key+'.png','imageData':None,'imageWidth':size[1],'imageHeight':size[0],
             'text':'','checked':allhuman,'shapes':shapes,
             'pallet_segmentation':{'schema_version':1,'instance_unit':'visible_fork_face','frame_id':r['frame_id'],
                'selected_target_id':r['target_id'],'original_parent_bbox':r['original_parent_bbox'],
                'appearance_class':r['appearance_class'],'original_split':original['pallet_annotation_task']['original_split'],
                'pixel_rotation_degrees':original['pallet_export']['pixel_rotation_degrees'],
                'reconstructed_intensity':original['pallet_export']['reconstructed_intensity'],
                'instances':instances,'hole_semantics':'same_group_positive_union_minus_hole_union',
                'rectangles_as_shapes':False,'original_parent_is_metadata_only':True,
                'checkpoint_sha256':r['checkpoint_sha256'],'task_hash':ctx['binding']['task_hash'],
                'state':r['state'],'quarantined_return':r['quarantined_return']}}
        doc['pallet_segmentation']['review_reasons']=r['review_reasons']
        write(key+'.png',raw);write(key+'.json',encoded(doc))
        public.append({k:v for k,v in r.items() if k!='faces'}|{'face_instances':len(instances),
                      'human_faces':sum(i['source']=='human' for i in instances),
                      'model_faces':sum(i['source']=='model' for i in instances),
                      'low_confidence_faces':sum(i['low_confidence'] for i in instances)})
    summary={'frames':len(records),'selected_targets':len(records),'face_instances':len(allfaces),
             'human_faces':sum(f['source']=='human' for f in allfaces),
             'model_faces':sum(f['source']=='model' for f in allfaces),
             'low_confidence_faces':sum(f['low_confidence'] for f in allfaces),
             'targets_without_mask':sum(not r['faces'] for r in records),
             'frames_with_holes':sum(any(i['closed_hole_pixels']>0 for i in allfaces if i['frame_key']==r['frame_key']) for r in records),
             'rectangles_in_shapes':0,'mask_policy':'holes subtracted; different faces never merged'}
    write('targets.jsonl',b''.join(encoded(r) for r in public))
    write('faces.jsonl',b''.join(encoded(f) for f in allfaces))
    write('review_queue.jsonl',b''.join(encoded(r) for r in public if r['model_faces'] or not r['face_instances']))
    write('labels.txt',b'pallet\n__hole__\n')
    write('summary.json',encoded(summary))
    save(ctx['output']/'export_manifest.json',{'task_hash':ctx['binding']['task_hash'],'files':files,'summary':summary})
    from pallet.instance_segmentation_report import report
    report(ctx,public,summary)
    event('export',**summary)


def verify_sources(ctx):
    for path,expected in ctx['binding']['inputs'].items():
        if digest(path)!=expected:raise ValueError('Frozen input changed')
    formal=ctx['formal'];dataset=Path(formal['binding']['dataset'])
    for relative,expected in formal['protected_files'].items():
        if digest(safe_path(dataset,relative))!=expected:raise ValueError('Formal label/metadata changed')
    for relative,expected in ctx['registry'].items():
        if digest(ctx['source']/'delivery'/relative)!=expected:raise ValueError('Historical rectangle seed artifact changed')
    return len(formal['protected_files'])


def verify(ctx):
    from pallet.instance_segmentation_seed import verify_audit
    verify_dataset(ctx);verify_audit(ctx);delivery=ctx['output']/'delivery'
    records=read_json(ctx['output']/'prediction_index.json')
    if len(records)!=len(ctx['rows']) or len({r['target_id'] for r in records})!=len(records):raise ValueError('Selected target coverage failed')
    instance_ids=[];saved=0;rotated=0;multiple=0;holes=0;human_coordinates=0
    for kind in ('small','medium'):
        receipt=read_json(ctx['output']/f'training/{kind}/completion.json')
        folder=ctx['output']/f'training/{kind}'
        if (receipt['task_hash']!=ctx['binding']['task_hash'] or receipt['state']!='complete'
                or not receipt['weights_reloaded'] or digest(folder/receipt['checkpoint'])!=receipt['checkpoint_sha256']
                or digest(folder/'metrics.csv')!=receipt['metrics_csv_sha256']):
            raise ValueError('Two completed, reloadable model runs required')
    groups=read_json(ctx['output']/'dataset/group_split.json')
    frames=read_json(ctx['output']/'dataset/native_index.json')
    if any(groups[f['group_id']]!=f['split'] for f in frames):raise ValueError('Group split leakage')
    hole_groups={f['group_id'] for f in frames if f['split']=='valid' and any(i['hole_shapes'] for i in f['instances'])}
    if len(hole_groups)<2:raise ValueError('Validation needs two hole acquisition groups')
    from pycocotools.mask import decode
    from rfdetr.datasets.coco import CocoDetection
    from pallet.instance_segmentation_seed import transform_mask
    loader_instances=0
    for split in ('train','valid'):
        coco=read_json(ctx['output']/f'dataset/{split}/_annotations.coco.json')
        lookup={i['instance_id']:(f,i) for f in frames if f['split']==split for i in f['instances']}
        if coco['categories']!=[{'id':1,'name':'pallet'}]:raise ValueError('Holes became a training class')
        for annotation in coco['annotations']:
            f,i=lookup[annotation['native_instance_id']]
            native=np.asarray(Image.open(ctx['output']/i['mask']))>0
            expected=transform_mask(native,f['transform']);actual=decode(annotation['segmentation'])>0
            if not np.array_equal(expected,actual) or bbox(actual)!=annotation['bbox']:
                raise ValueError('COCO RLE/mask/face box mismatch')
        loader=CocoDetection(str(ctx['output']/f'dataset/{split}'),
                str(ctx['output']/f'dataset/{split}/_annotations.coco.json'),
                transforms=None,include_masks=True,remap_category_ids=True)
        for index,image_id in enumerate(loader.ids):
            image,target=loader[index]
            native_annotations=loader.coco.imgToAnns[image_id]
            expected=np.stack([decode(a['segmentation'])>0 for a in native_annotations])
            if image.size!=(RESOLUTION,RESOLUTION) or not np.array_equal(target['masks'].numpy()>0,expected):
                raise ValueError('Official RF-DETR COCO loader changed mask/hole supervision')
            if not (target['labels'].numpy()==0).all():raise ValueError('Training class remapping changed')
            loader_instances+=len(native_annotations)
    with tempfile.TemporaryDirectory(prefix='face-labels-verify-') as directory:
        for r in records:
            path=delivery/(r['frame_key']+'.json');doc=read_json(path)
            size=(doc['imageHeight'],doc['imageWidth']); masks=render_shapes(doc['shapes'],size,ctx['upstream'])
            instances=doc['pallet_segmentation']['instances']
            rotated+=doc['pallet_segmentation']['pixel_rotation_degrees']==180
            multiple+=len(instances)>1
            holes+=any(i['closed_hole_pixels'] for i in instances)
            if set(masks)!={i['group_id'] for i in instances}:raise ValueError('Group identity/instance mismatch')
            for i in instances:
                if not np.array_equal(decode(i['mask_rle'])>0,masks[i['group_id']]):raise ValueError('RLE and AnyLabeling hole masks differ')
                if i['face_bbox']!=bbox(masks[i['group_id']]):raise ValueError('Independent face box changed')
                if i['source']=='model' and i['mask_human_reviewed']:raise ValueError('Machine mask promoted to reviewed')
                instance_ids.append(i['instance_id'])
                if i['source']=='human':
                    source_face=next(f for f in r['faces'] if f['instance_id']==i['instance_id'])
                    original=[source_face['outer_shape']]+source_face['hole_shapes']
                    exported=[s for s in doc['shapes'] if s['group_id']==i['group_id']]
                    if [s['points'] for s in original]!=[s['points'] for s in exported]:
                        raise ValueError('Accepted human polygon coordinates changed')
                    human_coordinates+=1
            if (delivery/doc['imagePath']).read_bytes()!=native_image(ctx,r['frame_key']):raise ValueError('Native PNG changed')
            check_label_save(path,Path(directory)/'saved.json',ctx['label_class']);saved+=1
    if len(instance_ids)!=len(set(instance_ids)):raise ValueError('Duplicate face instance identity')
    protected=verify_sources(ctx)
    assets={str(p.relative_to(delivery)):digest(p) for p in delivery.rglob('*') if p.is_file() and p.name not in ('validation.json','checksums.sha256')}
    save(ctx['output']/'verified_assets.json',{'task_hash':ctx['binding']['task_hash'],'files':assets})
    validation={'state':'passed','task_hash':ctx['binding']['task_hash'],'selected_targets':len(records),
        'face_instances':len(instance_ids),'official_load_save_documents':saved,'hole_masks_pixel_exact':True,
        'independent_face_boxes_verified':True,'rectangles_in_shapes':0,'original_pngs_identical':True,
        'protected_formal_files_unchanged':protected,'historical_rectangle_seed_unchanged':True,
        'two_models_reloaded':True,'groups_do_not_leak':True,'validation_hole_groups':len(hole_groups),
        'human_faces_coordinates_unchanged':human_coordinates,'frames_with_multiple_faces':multiple,
        'frames_with_closed_holes':holes,'frames_using_source_180_degree_correction':rotated,
        'official_rfdetr_rle_loader_instances':loader_instances,
        'gui_application_tested':False,'verified_assets_sha256':digest(ctx['output']/'verified_assets.json')}
    save(ctx['output']/'validation.json',validation);protected_write(delivery/'validation.json',encoded(validation))
    event('verify',**validation)


def package(ctx):
    from datetime import datetime
    from zoneinfo import ZoneInfo
    validation=read_json(ctx['output']/'validation.json');registry=read_json(ctx['output']/'verified_assets.json')
    if validation['verified_assets_sha256']!=digest(ctx['output']/'verified_assets.json'):raise ValueError('Validated asset registry changed')
    delivery=ctx['output']/'delivery';assets=dict(registry['files'])
    for name,expected in assets.items():
        if digest(delivery/name)!=expected:raise ValueError('Human edits or validated assets changed; refusing package')
    verify_sources(ctx)
    receipt_path=ctx['output']/'delivery_manifest.json'
    if receipt_path.exists():
        r=read_json(receipt_path)
        if digest(ctx['output']/r['archive_name'])!=r['archive_sha256']:raise ValueError('Published ZIP changed')
        training_bundle(ctx)
        return r
    assets['validation.json']=digest(delivery/'validation.json')
    protected_write(delivery/'checksums.sha256',''.join(f'{sha}  {name}\n' for name,sha in sorted(assets.items())).encode())
    assets['checksums.sha256']=digest(delivery/'checksums.sha256')
    archive=ctx['output']/'pallet_instance_segmentation_440_anylabeling_v1.zip';partial=archive.with_suffix('.partial')
    with zipfile.ZipFile(partial,'w',compression=zipfile.ZIP_DEFLATED,compresslevel=4) as z:
        for name,expected in sorted(assets.items()):
            data=(delivery/name).read_bytes()
            if hashlib.sha256(data).hexdigest()!=expected:raise ValueError('File changed during ZIP creation')
            info=zipfile.ZipInfo(name,(2026,1,1,0,0,0));info.external_attr=0o644<<16
            z.writestr(info,data,compress_type=zipfile.ZIP_DEFLATED,compresslevel=4)
    with zipfile.ZipFile(partial) as z:
        if z.testzip() is not None or len(z.namelist())!=len(assets):raise ValueError('ZIP CRC or member count failed')
        for name,expected in assets.items():
            if hashlib.sha256(z.read(name)).hexdigest()!=expected:raise ValueError('ZIP member SHA mismatch')
        with tempfile.TemporaryDirectory(prefix='face-labels-portable-') as directory:
            # Sample one frame from every A cluster, using only flat editable pairs.
            seen=set();portable=0
            for r in read_json(ctx['output']/'prediction_index.json'):
                if r['appearance_class'] in seen:continue
                seen.add(r['appearance_class'])
                for suffix in ('png','json'):
                    name=r['frame_key']+'.'+suffix;(Path(directory)/name).write_bytes(z.read(name))
                check_label_save(Path(directory)/(r['frame_key']+'.json'),Path(directory)/'saved.json',ctx['label_class']);portable+=1
    partial.replace(archive)
    receipt={'state':'complete','task_hash':ctx['binding']['task_hash'],'archive_name':archive.name,
        'archive_bytes':archive.stat().st_size,'archive_sha256':digest(archive),'zip_members':len(assets),
        'crc_passed':True,'all_member_sha256_passed':True,'portable_load_save_documents':portable,
        'validation':validation,'summary':read_json(delivery/'summary.json'),
        'finished_at':datetime.now(ZoneInfo('Asia/Shanghai')).isoformat()}
    protected_write(archive.with_suffix('.zip.sha256'),f'{receipt["archive_sha256"]}  {archive.name}\n'.encode())
    save(receipt_path,receipt);event('package',archive=str(archive),bytes=receipt['archive_bytes'])
    training_bundle(ctx)


def training_bundle(ctx):
    """Separate reproducibility ZIP keeps all 440 editable pairs flat and lightweight."""
    import importlib.metadata
    import platform
    import sys
    from pallet.instance_segmentation_seed import verify_audit
    verify_dataset(ctx);verify_audit(ctx)
    output=ctx['output']; receipt_path=output/'training_bundle_manifest.json'
    if receipt_path.exists():
        receipt=read_json(receipt_path)
        if digest(output/receipt['archive_name'])!=receipt['archive_sha256']:
            raise ValueError('Published training bundle changed')
        return receipt
    distributions={}
    for distribution in importlib.metadata.distributions():
        files={}
        for path in distribution.files or []:
            if str(path).endswith(('.dist-info/METADATA','.dist-info/RECORD')):
                file=distribution.locate_file(path)
                if file.is_file():files[str(path)]=digest(file)
        distributions[distribution.metadata['Name']]={'version':distribution.version,'metadata_sha256':files}
    save(output/'environment/distributions.json',{'python':sys.version,'platform':platform.platform(),
                                                 'distributions':distributions})
    protected_write(output/'environment/requirements.lock',
        ''.join(f'{name}=={value["version"]}\n' for name,value in sorted(distributions.items(),key=lambda x:x[0].lower())).encode())
    import rfdetr
    installed=Path(rfdetr.__file__).parent
    save(output/'environment/rfdetr_source_hashes.json',
         {str(p.relative_to(installed)):digest(p) for p in installed.rglob('*.py')})
    save(output/'source_hashes.json',{str(p):digest(p) for p in Path('pallet').rglob('*.py')})
    retrain='''"""Standalone replay from the unpacked training bundle; no original source pack required."""
import argparse, importlib.metadata, os
from pathlib import Path
os.environ.setdefault("CUDA_DEVICE_ORDER", "PCI_BUS_ID")
os.environ.setdefault("OMP_NUM_THREADS", "4")
from rfdetr import RFDETRSegSmall, RFDETRSegMedium
assert importlib.metadata.version("rfdetr") == "1.11.0"
p=argparse.ArgumentParser();p.add_argument("--model",choices=["small","medium"],required=True)
p.add_argument("--device",default="cuda:0");p.add_argument("--output",type=Path,required=True);a=p.parse_args()
assert not a.output.exists(), "Use a new output directory to preserve trained results"
root=Path(__file__).resolve().parent
model={"small":RFDETRSegSmall,"medium":RFDETRSegMedium}[a.model](num_classes=1,resolution=768,device=a.device,gradient_checkpointing=True,compile=False)
model.train(dataset_dir=str(root/"dataset"),output_dir=str(a.output),epochs=150,batch_size=2,grad_accum_steps=4,
 eval_batch_size=1,device="cuda",devices=a.device.split(":")[-1]+",",lr=1e-4,lr_encoder=1e-5,seed=42,num_workers=0,
 multi_scale=False,expanded_scales=False,scale_jitter=False,use_ema=True,early_stopping=True,
 early_stopping_patience=30,early_stopping_use_ema=True,early_stopping_min_delta=.001,run_test=False,
 tensorboard=False,wandb=False,progress_bar=None,checkpoint_interval=50,compute_val_loss=True,
 class_names=["pallet"],square_resize_div_64=True,aug_config=None,eval_masks_head_resolution=False)
'''
    protected_write(output/'retrain.py',retrain.encode())
    files={}
    for directory in ('dataset','audit','inputs','evaluation','environment','logs'):
        for file in (output/directory).rglob('*'):
            if file.is_file():files[str(file.relative_to(output))]=file
    for name in ('run_manifest.json','dataset_manifest.json','model_selection.json','source_hashes.json','retrain.py',
                 'validation.json','delivery_manifest.json'):
        files[name]=output/name
    for input_path in ctx['binding']['inputs']:
        p=Path(input_path)
        if p.name in ('anylabeling_shape.py','anylabeling_label_file.py'):
            files['source/upstream/'+p.name]=p
    files['frozen_sources/clustering_run_manifest.json']=Path('outputs/pallet_target_clustering_v1/run_manifest.json')
    for name in ('run_manifest.json','export_assets.json','delivery_manifest.json'):
        files['frozen_sources/rectangle_seed/'+name]=ctx['source']/name
    files['frozen_sources/selected_targets.jsonl']=ctx['source']/'delivery/targets.jsonl'
    for file in (output/'delivery').rglob('*'):
        if file.is_file():files['results/'+str(file.relative_to(output/'delivery'))]=file
    for kind in ('small','medium'):
        completion=read_json(output/f'training/{kind}/completion.json')
        best=output/f'training/{kind}'/completion['checkpoint']
        if digest(best)!=completion['checkpoint_sha256']:raise ValueError('Trained weights changed')
        for name in (completion['checkpoint'],'completion.json','run_config.json','pretraining.json','training_config.json','metrics.csv'):
            files[f'training/{kind}/{name}']=output/f'training/{kind}/{name}'
        epoch_audit=output/f'training/{kind}/epoch_audit.json'
        if epoch_audit.exists():files[f'training/{kind}/epoch_audit.json']=epoch_audit
    for file in Path('pallet').rglob('*.py'):
        files['source/'+str(file)]=file
    for name in ('test_instance_segmentation_seed.py',):
        files['source/tests/'+name]=Path('tests')/name
    files['source/docs/instance_segmentation_seed.md']=Path('docs/instance_segmentation_seed.md')
    native_frames=[f for f in read_json(output/'audit/frames.json') if f['state']=='manual_accepted']
    native_files={f'native/images/{f["frame_key"]}.png':native_image(ctx,f['frame_key']) for f in native_frames}
    checksums={name:digest(path) for name,path in files.items()}
    checksums.update({name:hashlib.sha256(raw).hexdigest() for name,raw in native_files.items()})
    protected_write(output/'training_bundle_checksums.sha256',
        ''.join(f'{sha}  {name}\n' for name,sha in sorted(checksums.items())).encode())
    files['checksums.sha256']=output/'training_bundle_checksums.sha256'
    checksums['checksums.sha256']=digest(files['checksums.sha256'])
    archive=output/'pallet_instance_segmentation_training_bundle_v1.zip';partial=archive.with_suffix('.partial')
    with zipfile.ZipFile(partial,'w',compression=zipfile.ZIP_DEFLATED,compresslevel=4) as z:
        for name,sha in sorted(checksums.items()):
            raw=native_files[name] if name in native_files else files[name].read_bytes()
            if hashlib.sha256(raw).hexdigest()!=sha:raise ValueError('Training bundle file changed during packaging')
            info=zipfile.ZipInfo(name,(2026,1,1,0,0,0));info.external_attr=0o644<<16
            z.writestr(info,raw,compress_type=zipfile.ZIP_DEFLATED,compresslevel=4)
    with zipfile.ZipFile(partial) as z:
        if z.testzip() is not None or len(z.namelist())!=len(checksums):raise ValueError('Training bundle CRC/coverage failed')
        for name,sha in checksums.items():
            if hashlib.sha256(z.read(name)).hexdigest()!=sha:raise ValueError('Training bundle member SHA mismatch')
    partial.replace(archive)
    receipt={'state':'complete','task_hash':ctx['binding']['task_hash'],'archive_name':archive.name,
             'archive_sha256':digest(archive),'archive_bytes':archive.stat().st_size,
             'zip_members':len(checksums),'crc_passed':True,'all_member_sha256_passed':True,
             'models':['small','medium'],'native_training_frames':len(native_frames)}
    protected_write(archive.with_suffix('.zip.sha256'),f'{receipt["archive_sha256"]}  {archive.name}\n'.encode())
    save(receipt_path,receipt);event('training_bundle',**receipt)
