"""Versioned, resumable extraction of measured pallet fork-face proposals.

This is an independent annotation branch. Frozen detector labels are read only;
no geometric score is interpreted as confirmed forkability or segmentation AP.
"""
from __future__ import annotations

import argparse
from collections import Counter, defaultdict
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed
import hashlib
import json
import multiprocessing
import os
from pathlib import Path
import shutil
import time
import zipfile

import cv2
import numpy as np
from pycocotools import mask as mask_utils

from .cluster_targets import (PROJECT, digest, integer_box, load_dataset,
                              protected_snapshot, read_rows, save_json,
                              target_id, target_key, timestamp, write_rows)
from .data.loader import policy_digest
from .data.orientation import orient_grid
from .data.pcd import read_pcd
from .fork_geometry import DEFAULTS, decompose_roi, resolve_ownership, valid_xyz


VERSION = 1
WORKER = {}


def json_digest(value):
    return hashlib.sha256(json.dumps(value,sort_keys=True,ensure_ascii=False,allow_nan=False).encode()).hexdigest()


def safe_path(root, relative):
    path=Path(relative)
    if path.is_absolute() or '..' in path.parts:
        raise ValueError('Unsafe output relative path')
    result=(Path(root)/path).resolve()
    if not result.is_relative_to(Path(root).resolve()):
        raise ValueError('Output path escapes root')
    return result


def check_space(path, minimum_gib=50.):
    free=shutil.disk_usage(path).free
    if free < minimum_gib*1024**3:
        raise RuntimeError(f'Disk guard: {free/1024**3:.2f} GiB available, {minimum_gib:.2f} required')


def load_registered_cloud(dataset, annotation, source):
    """Hash all used inputs, register the full XYZI and keep source indices."""
    dataset=Path(dataset)
    for key in ('image','sensor','annotation'):
        path=safe_path(dataset,source[key])
        if digest(path)!=source['files'][source[key]]:
            raise ValueError(f'Frozen asset hash mismatch: {source[key]}')
    policy=source['sensor_policy']
    if policy_digest(policy)!=source['sensor_metadata']['policy_sha256']:
        raise ValueError('Sensor policy hash mismatch')
    h,w=annotation['height'],annotation['width']
    if policy.get('grid')!=[h,w] or source['size']!=[w,h]:
        raise ValueError('Registered grid differs from annotation')
    if source['pixel_rotation_degrees']!=policy.get('pixel_rotation_degrees',0):
        raise ValueError('Registered rotation differs from source index')
    pcd=Path(source['source_paths']['pcd'])
    if digest(pcd)!=source['source_anchor_hash']:
        raise ValueError('Raw PCD hash differs from frozen source anchor')
    header,points=read_pcd(pcd)
    if len(points)!=h*w:
        raise ValueError('PCD count differs from registered grid')
    header_grid=(int(header['HEIGHT'][0]),int(header['WIDTH'][0]))
    if header_grid!=(h,w) and header_grid!=(1,h*w):
        raise ValueError('PCD header is incompatible with registered organized grid')
    native=orient_grid(points.reshape(h,w,4),policy)
    source_indices=orient_grid(np.arange(h*w,dtype=np.int32).reshape(h,w),policy)
    if not np.array_equal(native.reshape(-1,4),points[source_indices.ravel()],equal_nan=True):
        raise ValueError('Registered XYZI differs from original source point indices')
    with np.load(safe_path(dataset,source['sensor']),allow_pickle=False) as z:
        sensor=z['frame']
    if sensor.shape!=(h,w,4) or not np.isfinite(sensor).all():
        raise ValueError('Invalid formal float tensor')
    vi=sensor[...,3]>0
    if not vi.any() or not np.allclose(native[...,3][vi],sensor[...,1][vi],rtol=1e-6,atol=1e-6):
        raise ValueError('Raw intensity and formal tensor are not pixel-aligned')
    image=cv2.imread(str(safe_path(dataset,source['image'])),cv2.IMREAD_GRAYSCALE)
    if image is None or image.shape!=(h,w):
        raise ValueError('Invalid formal intensity PNG shape')
    vp=header.get('VIEWPOINT',['0','0','0','1','0','0','0'])
    if len(vp)!=7 or not np.isfinite(np.asarray(vp,dtype=float)).all():
        raise ValueError('Invalid PCD VIEWPOINT')
    origin=np.asarray(vp[:3],dtype=float)
    return image,native,source_indices,origin,header['DATA'][0],int(vi.sum())


def select_pilot(annotations,sources,target_context,seed=42,count=256):
    pool=[a for a in annotations if a['split']=='train' and a['boxes']]
    rank=lambda a: hashlib.sha256(f'{seed}:{a["frame_id"]}'.encode()).hexdigest()
    pool.sort(key=rank)
    selected={}
    strata=defaultdict(list)
    for a in pool:
        s=sources[a['frame_id']]
        strata[('policy',s['sensor_policy']['policy_id'])].append(a)
        strata[('grid',*s['size'])].append(a)
        if s['reconstructed_intensity']:strata[('reconstructed',)].append(a)
        if s['pixel_rotation_degrees']:strata[('rotated',)].append(a)
        if len(a['boxes'])>1:strata[('stack_or_multiple',)].append(a)
        if any(target_context.get(target_id(a['frame_id'],i),{}).get('geometry_status')=='foreground_depth_or_view_ambiguous' for i in range(len(a['boxes']))):
            strata[('previous_background_ambiguity',)].append(a)
    for key in sorted(strata,key=str):
        a=strata[key][0];selected[a['frame_id']]=a
    if len(selected)>count:raise ValueError('Pilot too small for mandatory training strata')
    # Prefer independent groups after covering configuration policies.
    seen={a['group_id'] for a in selected.values()}
    for a in pool:
        if len(selected)>=count:break
        if a['group_id'] not in seen:selected[a['frame_id']]=a;seen.add(a['group_id'])
    for a in pool:
        if len(selected)>=count:break
        selected.setdefault(a['frame_id'],a)
    return sorted(selected,key=lambda fid:rank(selected[fid]))


def source_code_hashes():
    names=('fork_face_masks.py','fork_geometry.py','fork_face_report.py','cluster_targets.py',
           'data/pcd.py','data/orientation.py','data/loader.py')
    return {name:digest(Path(__file__).parent/name) for name in names}


def prepare(args,annotations,sources):
    out=Path(args.output).resolve();dataset=Path(args.dataset).resolve()
    if out==dataset or out.is_relative_to(dataset) or dataset.is_relative_to(out):
        raise ValueError('Derived output must be independent of formal dataset')
    config=dict(DEFAULTS)
    if args.config:
        supplied=json.loads(Path(args.config).read_text())
        unknown=set(supplied)-set(config)
        if unknown:raise ValueError(f'Unknown geometry settings: {sorted(unknown)}')
        config.update(supplied)
    config['seed']=args.seed
    if config['minimum_face_points']<3 or config['minimum_region_points']<3 or config['ransac_max_iterations']<1:
        raise ValueError('Invalid geometric point count or iteration settings')
    units=json.loads(Path(args.unit_registry).read_text())
    context_rows=read_rows(args.target_context)
    context={r['target_id']:r for r in context_rows}
    for a in annotations:
        for i,b in enumerate(a['boxes']):
            old=context[target_id(a['frame_id'],i)]
            if old['bbox']!=b or old['source_pcd_sha256']!=sources[a['frame_id']]['source_anchor_hash']:
                raise ValueError('Existing analysis unit evidence is not bound to current target')
            policy=sources[a['frame_id']]['sensor_policy']
            unit=units[policy['policy_id']]
            if policy.get('scale_to_mm') is not None and unit['scale_to_mm']!=policy['scale_to_mm']:
                raise ValueError('Existing unit evidence conflicts with frozen metric policy')
    snapshot=protected_snapshot(dataset)
    formal=json.loads((dataset/'manifest.json').read_text())
    for relative,actual in snapshot.items():
        if relative in formal['files'] and formal['files'][relative]!=actual:
            raise ValueError(f'Formal metadata differs from frozen manifest: {relative}')
    binding={'version':VERSION,'dataset':str(dataset),'config':config,'source_code':source_code_hashes(),
             'unit_registry_sha256':digest(args.unit_registry),'target_context_sha256':digest(args.target_context),
             'protected_snapshot_sha256':json_digest(snapshot),'frames':len(annotations),
             'targets':sum(len(a['boxes']) for a in annotations)}
    manifest=out/'run_manifest.json'
    if manifest.exists():
        if not args.resume:raise ValueError('Output exists; use --resume for unchanged inputs')
        old=json.loads(manifest.read_text())
        if old['binding']!=binding or old['protected_files']!=snapshot:
            raise ValueError('Resume inputs/configuration/source changed; use a new version')
        stats_path=out/'source_stats.json'
        if stats_path.exists():
            stats=json.loads(stats_path.read_text())
            for fid,s in sources.items():
                p=Path(s['source_paths']['pcd']);stat=p.stat();before=stats[fid]
                if (stat.st_size!=before['size'] or stat.st_mtime_ns!=before['mtime_ns']) and digest(p)!=before['sha256']:
                    raise ValueError('Raw source changed since audit; refusing resume')
    else:
        if out.exists() and any(out.iterdir()):raise ValueError('Output directory is not empty')
        out.mkdir(parents=True,exist_ok=True);check_space(out,args.minimum_free_gib)
        pilot=select_pilot(annotations,sources,context,args.seed,args.pilot_count)
        old={'kind':'measured_fork_face_mask_proposals','created_at':timestamp(),'binding':binding,
             'task_hash':json_digest(binding),'protected_files':snapshot,'pilot_frame_ids':pilot,
             'pilot_selection_sha256':json_digest(pilot),'geometry_semantics':'visible_measured_material_only',
             'proposal_only':True,'mask_human_reviewed':False,'labels_modified':False}
        save_json(manifest,old)
        save_json(out/'geometry_config.json',config)
        save_json(out/'unit_registry.json',units)
        (out/'source').mkdir()
        for name in ('fork_face_masks.py','fork_geometry.py','fork_face_report.py'):
            shutil.copyfile(Path(__file__).parent/name,out/'source'/name)
    for folder in ('extraction','logs','delivery'):(out/folder).mkdir(exist_ok=True)
    args._config=config;args._context=context;args._manifest=old
    return old


def audit(args,annotations,sources):
    out=Path(args.output)
    missing=[];source_stats={};policies={}
    for a in annotations:
        s=sources[a['frame_id']];p=Path(s['source_paths']['pcd'])
        if not p.is_file():missing.append(a['frame_id'])
        else:
            stat=p.stat();source_stats[a['frame_id']]={'size':stat.st_size,'mtime_ns':stat.st_mtime_ns,'sha256':s['source_anchor_hash']}
        policy=s['sensor_policy']
        if policy_digest(policy)!=s['sensor_metadata']['policy_sha256']:raise ValueError('Policy digest mismatch')
        policies[policy['policy_id']]=policy_digest(policy)
    summary={'state':'passed' if not missing else 'failed','finished_at':timestamp(),
             'frames':len(annotations),'targets':sum(len(a['boxes']) for a in annotations),
             'negative_frames':sum(not a['boxes'] for a in annotations),'missing_pcd':missing,
             'split_counts':dict(Counter(a['split'] for a in annotations)),
             'grid_counts':dict(Counter('x'.join(map(str,s['size'])) for s in sources.values())),
             'policies':policies,'protected_files':len(args._manifest['protected_files']),
             'raw_pcd_hash_verification':'every frame during extraction before decoding',
             'task_hash':args._manifest['task_hash']}
    save_json(out/'input_audit.json',summary);save_json(out/'source_stats.json',source_stats)
    if missing:raise ValueError(f'{len(missing)} missing raw point clouds')
    print(json.dumps({k:v for k,v in summary.items() if k not in ('policies','missing_pcd')},ensure_ascii=False),flush=True)
    return summary


def frame_directory(out,annotation,source):
    return Path(out)/'extraction'/annotation['split']/Path(source['image']).stem


def completed_record(out,annotation,source,task_hash,verify=True):
    directory=frame_directory(out,annotation,source);path=directory/'completed.json'
    if not path.exists():return None
    saved=json.loads(path.read_text())
    if saved['task_hash']!=task_hash or saved['frame_id']!=annotation['frame_id']:
        raise ValueError('Frame completion belongs to another task')
    record_path=safe_path(directory,saved['record'])
    if verify:
        for relative,expected in saved['files'].items():
            p=safe_path(directory,relative)
            if not p.is_file() or digest(p)!=expected:
                raise ValueError(f'Completed frame changed (possible human edit): {p}')
    record=json.loads(record_path.read_text())
    if record['mask_human_reviewed']:raise ValueError('Human mask edits are protected')
    return record


def worker_init(dataset,out,config,task_hash,context,minimum_free_gib):
    from threadpoolctl import threadpool_limits
    os.environ['OMP_NUM_THREADS']='1';os.environ['OPENBLAS_NUM_THREADS']='1'
    WORKER.update(dataset=Path(dataset),out=Path(out),config=config,task_hash=task_hash,
                  context=context,minimum_free_gib=minimum_free_gib,limits=threadpool_limits(limits=1))
    cv2.setNumThreads(1)


def _write_image(path,image):
    if not cv2.imwrite(str(path),image):raise OSError(f'Cannot write {path}')


def _mask_iou(a,b):
    union=np.logical_or(a,b).sum()
    return float(np.logical_and(a,b).sum()/union) if union else 1.


def process_frame(annotation,source,pilot=False):
    start=time.perf_counter();w=WORKER;config=w['config']
    directory=frame_directory(w['out'],annotation,source);directory.mkdir(parents=True,exist_ok=True)
    attempt_index=1
    while (directory/f'attempt_{attempt_index:04d}').exists():attempt_index+=1
    attempt=directory/f'attempt_{attempt_index:04d}';attempt.mkdir()
    base={'frame_id':annotation['frame_id'],'frame_key':Path(source['image']).stem,
          'split':annotation['split'],'group_id':annotation['group_id'],'domain':annotation['domain'],
          'width':annotation['width'],'height':annotation['height'],'source_image':source['image'],
          'source_pcd_sha256':source['source_anchor_hash'],'sensor_policy_sha256':source['sensor_metadata']['policy_sha256'],
          'pixel_rotation_degrees':source['pixel_rotation_degrees'],'reconstructed_intensity':source['reconstructed_intensity'],
          'task_hash':w['task_hash'],'proposal_only':True,'mask_human_reviewed':False,
          'independent_blind_reference':False,'targets':[],'faces':[],'pilot':pilot}
    try:
        check_space(w['out'],w['minimum_free_gib'])
        image,native,source_indices,origin,pcd_mode,alignment_count=load_registered_cloud(w['dataset'],annotation,source)
        base.update(alignment_verified=True,intensity_alignment_pixels=alignment_count,pcd_encoding=pcd_mode,
                    camera_origin_raw=origin.tolist())
        h,width=image.shape
        xyz_chunks=[];i_chunks=[];uv_chunks=[];source_chunks=[];parent_chunks=[];owner_chunks=[];residual_chunks=[];plane_chunks=[]
        all_faces=[];cursor=0;next_face=1
        for index,box in enumerate(annotation['boxes']):
            tid=target_id(annotation['frame_id'],index);key=target_key(annotation['frame_id'],index)
            context=w['context'][tid]
            scale=context.get('scale_to_mm');unit_status=context.get('unit_status','unknown')
            x1,y1,x2,y2=integer_box(box,width,h)
            xyz=native[y1:y2,x1:x2,:3]
            seed=int(hashlib.sha256(f'{config["seed"]}:{tid}'.encode()).hexdigest()[:8],16)
            result=decompose_roi(xyz,config,scale,origin,seed)
            local_uv=result['uv'];uv=local_uv+np.array([x1,y1],dtype=np.int32)
            points=native[uv[:,1],uv[:,0],:3];intensity=native[uv[:,1],uv[:,0],3]
            raw_indices=source_indices[uv[:,1],uv[:,0]]
            owners=np.zeros(len(uv),np.int32);plane_labels=np.zeros(len(uv),np.int32)
            normalized_residual=np.full(len(uv),np.inf)
            target={'target_id':tid,'target_key':key,'box_index':index,'bbox':box,'integer_bbox':[x1,y1,x2,y2],
                    'unit_status':unit_status,'scale_to_mm':scale,'unit_reason':context.get('unit_reason'),
                    'valid_roi_points':len(uv),'valid_xyz_zero_intensity_points':int((intensity==0).sum()),
                    'regions':result['regions'],'regions_plane_limit_reached':result['regions_plane_limit_reached'],
                    'normal_valid_fraction':result['normal_valid_fraction'],'faces':[],
                    'review_reasons':['forkability_not_confirmed'],'proposal_only':True,'mask_human_reviewed':False,
                    'previous_geometry_status':context.get('geometry_status')}
            if len(uv)<config['minimum_face_points']:target['review_reasons'].append('insufficient_valid_xyz')
            if unit_status!='verified':target['review_reasons'].append(f'unit_{unit_status}')
            for candidate in result['faces']:
                inds=candidate['indices'];fid=next_face;next_face+=1
                face={k:v for k,v in candidate.items() if k not in ('indices','plane')}
                plane=candidate['plane']
                raw_plane=np.r_[plane[:3],plane[3]/(scale if scale is not None else 1.)]
                face.update(face_id=fid,face_key=f'{key}__face_{candidate["candidate_index"]:03d}',
                            target_id=tid,target_key=key,box_index=index,plane=plane.tolist(),
                            plane_raw=raw_plane.tolist(),geometry_units=result['geometry_units'],
                            unit_status=unit_status,scale_to_mm=scale,proposal_only=True,mask_human_reviewed=False,
                            _indices=inds+cursor)
                plane_labels[inds]=fid
                if face['status']=='selected_candidate':
                    owners[inds]=fid
                    normalized_residual[inds]=np.abs(result['points_work'][inds]@plane[:3]+plane[3])/result['tolerances'][inds]
                target['faces'].append(fid);all_faces.append(face)
            if pilot:
                baseline=owners>0;comparisons=[]
                for multiplier in (.75,1.25):
                    alternate=dict(config);alternate['tolerance_spacing_ratio']*=multiplier
                    changed=decompose_roi(xyz,alternate,scale,origin,seed)
                    chosen=np.zeros(len(uv),bool)
                    for face in changed['faces']:
                        if face['status']=='selected_candidate':chosen[face['indices']]=True
                    comparisons.append({'tolerance_multiplier':multiplier,'mask_iou':_mask_iou(baseline,chosen),
                                        'selected_faces':sum(f['status']=='selected_candidate' for f in changed['faces'])})
                target['parameter_sensitivity']=comparisons
                if min(c['mask_iou'] for c in comparisons)<.85:target['review_reasons'].append('parameter_sensitive_mask')
            base['targets'].append(target)
            xyz_chunks.append(points);i_chunks.append(intensity);uv_chunks.append(uv);source_chunks.append(raw_indices)
            parent_chunks.append(np.full(len(uv),index,np.int16));owner_chunks.append(owners)
            residual_chunks.append(normalized_residual);plane_chunks.append(plane_labels);cursor+=len(uv)
        concatenate=lambda chunks,shape,dtype:np.concatenate(chunks).astype(dtype) if chunks else np.empty(shape,dtype)
        xyz_all=concatenate(xyz_chunks,(0,3),np.float32);uv_all=concatenate(uv_chunks,(0,2),np.int32)
        face_ids=concatenate(owner_chunks,(0,),np.int32);residual=concatenate(residual_chunks,(0,),np.float64)
        assigned,conflicts=resolve_ownership(uv_all,face_ids,residual,config['ownership_residual_margin'])
        # An ownership conflict can leave too few measurements to identify a face.
        for face in all_faces:
            inds=face.pop('_indices');face['diagnostic_points']=len(inds)
            owned=np.flatnonzero(assigned==face['face_id'])
            if face['status']=='selected_candidate' and len(owned)<config['minimum_face_points']:
                assigned[owned]=0;face['status']='diagnostic_only';face['reasons'].append('insufficient_unique_points_after_ownership')
            face['mask_points']=int((assigned==face['face_id']).sum())
        if next_face>65535:raise ValueError('Too many faces for uint16 instance mask')
        instances=np.zeros((h,width),np.uint16)
        selected=assigned>0
        instances[uv_all[selected,1],uv_all[selected,0]]=assigned[selected].astype(np.uint16)
        conflict_mask=np.zeros((h,width),np.uint8)
        if len(conflicts):conflict_mask[conflicts[:,1],conflicts[:,0]]=255
        _write_image(attempt/'instances.png',instances);_write_image(attempt/'conflicts.png',conflict_mask)
        selected_faces=0
        for face in all_faces:
            if face['status']!='selected_candidate':continue
            mask=(instances==face['face_id']).astype(np.uint8)*255
            name=f'{face["face_key"]}.png';_write_image(attempt/name,mask)
            face['mask']=name;selected_faces+=1
        for target in base['targets']:
            ids=[f['face_id'] for f in all_faces if f['target_id']==target['target_id'] and f['status']=='selected_candidate']
            mask=np.isin(instances,ids)&(instances>0)
            name=f'{target["target_key"]}.union.png';_write_image(attempt/name,mask.astype(np.uint8)*255)
            target.update(union_mask=name,selected_face_count=len(ids),mask_points=int(mask.sum()),
                          status='candidate_needs_review' if ids else 'unresolved_no_mask')
            if not ids:target['review_reasons'].append('no_selected_plane')
            if len(conflicts) and ((conflicts[:,0]>=target['integer_bbox'][0])&(conflicts[:,0]<target['integer_bbox'][2])&
                                   (conflicts[:,1]>=target['integer_bbox'][1])&(conflicts[:,1]<target['integer_bbox'][3])).any():
                target['review_reasons'].append('overlapping_box_ownership_conflict')
            target['review_reasons']=sorted(set(target['review_reasons']))
        np.savez_compressed(attempt/'cloud.npz',xyz_raw=xyz_all,
                            intensity=concatenate(i_chunks,(0,),np.float32),uv=uv_all,
                            source_point_index=concatenate(source_chunks,(0,),np.int32),
                            parent_box_index=concatenate(parent_chunks,(0,),np.int16),face_id=assigned,
                            diagnostic_plane_id=concatenate(plane_chunks,(0,),np.int32))
        overlay=cv2.cvtColor(image,cv2.COLOR_GRAY2BGR)
        for face in all_faces:
            if face['status']!='selected_candidate':continue
            mask=instances==face['face_id'];color=np.array(face_color(face['face_id']))
            overlay[mask]=(overlay[mask]*.5+color*.5).astype(np.uint8)
        overlay[conflict_mask>0]=[0,0,255]
        for target in base['targets']:
            x1,y1,x2,y2=target['integer_bbox'];cv2.rectangle(overlay,(x1,y1),(x2-1,y2-1),(0,210,255),1)
        _write_image(attempt/'overlay.jpg',overlay)
        base.update(state='success',faces=all_faces,selected_face_count=selected_faces,conflict_pixels=len(conflicts),
                    roi_points=len(xyz_all),selected_points=int(selected.sum()),
                    frame_status='confirmed_negative' if not annotation['boxes'] else ('candidate_needs_review' if selected_faces else 'unresolved_positive'),
                    elapsed_seconds=time.perf_counter()-start,finished_at=timestamp(),attempt=attempt_index)
        save_json(attempt/'record.json',base)
        files={str(p.relative_to(directory)):digest(p) for p in sorted(attempt.iterdir()) if p.is_file()}
        save_json(directory/'completed.json',{'task_hash':w['task_hash'],'frame_id':annotation['frame_id'],
                                           'record':str((attempt/'record.json').relative_to(directory)),'files':files})
        return {'frame_id':annotation['frame_id'],'state':'success','seconds':base['elapsed_seconds'],
                'faces':selected_faces,'unresolved_targets':sum(t['status']=='unresolved_no_mask' for t in base['targets'])}
    except Exception as exc:
        base.update(state='failed',error=f'{type(exc).__name__}: {exc}',finished_at=timestamp(),attempt=attempt_index)
        save_json(attempt/'failure.json',base)
        save_json(directory/'latest_failure.json',base)
        return {'frame_id':annotation['frame_id'],'state':'failed','error':base['error']}


def face_color(index):
    return [(index*67)%180+60,(index*103)%180+60,(index*139)%180+60]


def extract(args,annotations,sources,pilot=False):
    out=Path(args.output);manifest=args._manifest
    audit_path=out/'input_audit.json'
    if not audit_path.exists() or json.loads(audit_path.read_text())['state']!='passed':
        raise ValueError('Successful input audit is required before extraction')
    if not pilot and not (out/'configuration_freeze.json').exists():
        raise ValueError('Successful 256-frame pilot is required before full extraction')
    if not pilot:
        freeze=json.loads((out/'configuration_freeze.json').read_text())
        if (freeze['task_hash']!=manifest['task_hash'] or freeze['config']!=args._config
                or freeze['pilot_selection_sha256']!=manifest['pilot_selection_sha256']
                or freeze['pilot_frames']!=len(manifest['pilot_frame_ids'])):
            raise ValueError('Pilot configuration freeze differs from task binding')
    selected=[a for a in annotations if (a['frame_id'] in set(manifest['pilot_frame_ids']) if pilot else True)]
    if not pilot:selected=[a for i,a in enumerate(selected) if i%args.shards==args.shard_index]
    if args.limit is not None:
        if not pilot:raise ValueError('--limit is only allowed for a development pilot')
        selected=selected[:args.limit]
    pending=[];previous=0
    for a in selected:
        if completed_record(out,a,sources[a['frame_id']],manifest['task_hash']) is None:pending.append(a)
        else:previous+=1
    start=time.perf_counter();last_report=start;finished=0;failed=[]
    phase='pilot' if pilot else 'extract'
    print(json.dumps({'stage':phase,'selected':len(selected),'already_complete':previous,'pending':len(pending),'workers':args.workers}),flush=True)
    if pending:
        with ProcessPoolExecutor(max_workers=args.workers,mp_context=multiprocessing.get_context('spawn'),
             initializer=worker_init,initargs=(args.dataset,args.output,args._config,manifest['task_hash'],args._context,args.minimum_free_gib)) as pool:
            # Bounded submission avoids holding thousands of frame dictionaries/futures.
            iterator=iter(pending);futures={}
            def submit_next():
                try:a=next(iterator)
                except StopIteration:return False
                futures[pool.submit(process_frame,a,sources[a['frame_id']],pilot)]=a;return True
            for _ in range(args.workers*2):submit_next()
            while futures:
                ready=next(as_completed(futures));a=futures.pop(ready)
                result=ready.result();finished+=1
                if result['state']=='failed':failed.append(result)
                submit_next();now=time.perf_counter()
                if now-last_report>=15 or finished==len(pending):
                    state={'stage':phase,'updated_at':timestamp(),'completed_this_process':finished,
                           'success_this_process':finished-len(failed),'already_complete':previous,
                           'remaining':len(pending)-finished,'failed':len(failed),'elapsed_seconds':now-start,
                           'frames_per_second':finished/max(now-start,1e-6),'shard_index':args.shard_index}
                    save_json(out/'logs'/f'{phase}_{args.shard_index}.json',state)
                    print(json.dumps(state),flush=True);last_report=now
    records=[completed_record(out,a,sources[a['frame_id']],manifest['task_hash']) for a in selected]
    successful=[r for r in records if r is not None]
    summary=summarize_records(successful)
    summary.update(stage=phase,finished_at=timestamp(),selected_frames=len(selected),successful_frames=len(successful),
                   failures=failed,all_selected_read_successfully=len(successful)==len(selected),
                   elapsed_this_process_seconds=time.perf_counter()-start,task_hash=manifest['task_hash'])
    if pilot:
        actual_grids=Counter(f'{r["width"]}x{r["height"]}' for r in successful)
        dataset_grids={f'{s["size"][0]}x{s["size"][1]}' for s in sources.values()}
        train_grids={f'{sources[a["frame_id"]]["size"][0]}x{sources[a["frame_id"]]["size"][1]}' for a in annotations if a['split']=='train' and a['boxes']}
        summary.update(pilot_grid_counts=dict(actual_grids),training_grid_coverage_complete=train_grids<=set(actual_grids),
                       grids_without_positive_training_frames=sorted(dataset_grids-train_grids),
                       supplementary_grid_policy='Frozen-config source alignment checks only; never tune on val/test')
    save_json(out/f'{phase}_summary_{args.shard_index}.json',summary)
    if pilot:
        if args.limit is None and len(successful)==len(manifest['pilot_frame_ids']):
            freeze={'frozen_at':timestamp(),'task_hash':manifest['task_hash'],'config':args._config,
                    'pilot_selection_sha256':manifest['pilot_selection_sha256'],'pilot_summary_sha256':digest(out/f'{phase}_summary_{args.shard_index}.json'),
                    'pilot_frames':len(successful),'all_mappings_verified':all(r['alignment_verified'] for r in successful),
                    'pilot_grid_counts':summary['pilot_grid_counts'],
                    'grids_without_positive_training_frames':summary['grids_without_positive_training_frames'],
                    'semantic_confirmation':'not performed; all generated masks remain proposals'}
            save_json(out/'configuration_freeze.json',freeze)
        from .fork_face_report import write_pilot_report
        write_pilot_report(out,summary,successful)
    if failed:raise RuntimeError(f'{len(failed)} read/extraction failures; rerun unchanged task with --resume')
    return summary


def summarize_records(records):
    targets=[t for r in records for t in r['targets']];faces=[f for r in records for f in r['faces']]
    selected=[f for f in faces if f['status']=='selected_candidate']
    sensitivity=[c['mask_iou'] for t in targets for c in t.get('parameter_sensitivity',[])]
    return {'frames':len(records),'targets':len(targets),'selected_faces':len(selected),
            'diagnostic_faces':sum(f['status']=='diagnostic_only' for f in faces),
            'rejected_background_faces':sum(f['status']=='rejected_background' for f in faces),
            'target_status':dict(Counter(t['status'] for t in targets)),
            'frame_status':dict(Counter(r['frame_status'] for r in records)),
            'unit_status':dict(Counter(t['unit_status'] for t in targets)),
            'split_frames':dict(Counter(r['split'] for r in records)),
            'split_targets':dict(Counter(r['split'] for r in records for _ in r['targets'])),
            'selected_points':sum(f['mask_points'] for f in selected),'roi_points':sum(r['roi_points'] for r in records),
            'conflict_pixels':sum(r['conflict_pixels'] for r in records),
            'pcd_encodings':dict(Counter(r['pcd_encoding'] for r in records)),
            'reconstructed_frames':sum(r['reconstructed_intensity'] for r in records),
            'rotation_corrected_frames':sum(r['pixel_rotation_degrees']==180 for r in records),
            'review_reasons':dict(Counter(reason for t in targets for reason in t['review_reasons'])),
            'pilot_parameter_sensitivity':{'comparisons':len(sensitivity),'mean_mask_iou':float(np.mean(sensitivity)) if sensitivity else None,
                                         'below_0_85':sum(v<.85 for v in sensitivity)},
            'extraction_seconds_sum':sum(r['elapsed_seconds'] for r in records),
            'segmentation_accuracy':None,'forkability_confirmed':False,'proposal_only':True}


def all_records(args,annotations,sources):
    records=[];missing=[]
    for a in annotations:
        r=completed_record(args.output,a,sources[a['frame_id']],args._manifest['task_hash'])
        if r is None:missing.append(a['frame_id'])
        else:records.append(r)
    if missing:raise ValueError(f'{len(missing)} frames lack successful extraction records; no complete export allowed')
    return records


def copy_protected(source,destination,expected=None):
    """Never overwrite an exported asset, including a possible manual edit."""
    source,destination=Path(source),Path(destination)
    expected=digest(source) if expected is None else expected
    if destination.exists():
        if digest(destination)!=expected:raise ValueError(f'Export asset changed; refusing overwrite: {destination}')
        return
    destination.parent.mkdir(parents=True,exist_ok=True)
    temporary=destination.with_name(destination.name+'.tmp')
    shutil.copyfile(source,temporary)
    if digest(temporary)!=expected:raise ValueError('Copied file hash mismatch')
    os.replace(temporary,destination)


def export(args,annotations,sources):
    out=Path(args.output);delivery=out/'delivery';check_space(out,args.minimum_free_gib)
    if (delivery/'export_manifest.json').exists():
        old=json.loads((delivery/'export_manifest.json').read_text())
        if old['task_hash']!=args._manifest['task_hash']:raise ValueError('Export belongs to different task')
        for relative,sha in old['files'].items():
            if digest(safe_path(delivery,relative))!=sha:raise ValueError('Export changed; possible human edit')
        return old
    records=all_records(args,annotations,sources);targets=[];faces=[];frames=[];review=[]
    coco={split:{'info':{'description':'Unreviewed measured fork-face proposals; not independent ground truth'},
                       'images':[],'annotations':[],'categories':[{'id':1,'name':'pallet_fork_face_candidate'}]}
          for split in ('train','val','test')}
    unions={split:{'info':{'description':'Union of selected measured faces per parent pallet'},
                         'images':[],'annotations':[],'categories':[{'id':1,'name':'pallet_fork_surface_candidate'}]}
            for split in coco}
    next_annotation=1
    for image_id,(a,r) in enumerate(zip(annotations,records),1):
        if image_id%100==0:check_space(out,args.minimum_free_gib)
        source=sources[a['frame_id']];directory=frame_directory(out,a,source)/f'attempt_{r["attempt"]:04d}'
        split=r['split'];key=r['frame_key']
        image=source['image'];copy_protected(Path(args.dataset)/image,delivery/image,source['files'][image])
        assets={}
        for name,dest in [('cloud.npz',f'clouds/{split}/{key}.npz'),('instances.png',f'instances/{split}/{key}.png'),
                          ('conflicts.png',f'conflicts/{split}/{key}.png'),('overlay.jpg',f'overlays/{split}/{key}.jpg')]:
            copy_protected(directory/name,delivery/dest);assets[name]=dest
        output_targets=[];output_faces=[]
        image_row={'id':image_id,'file_name':image,'height':r['height'],'width':r['width'],
                   'frame_id':r['frame_id'],'group_id':r['group_id'],'domain':r['domain'],
                   'proposal_only':True,'mask_human_reviewed':False,
                   'segmentation_training_eligible':r['frame_status']=='confirmed_negative',
                   'source_status':r['frame_status']}
        # Positive proposals require human review; missing masks are never negatives.
        coco[split]['images'].append(image_row);unions[split]['images'].append(dict(image_row))
        for f in r['faces']:
            entry=dict(f,frame_id=r['frame_id'],split=split,image=image,cloud=assets['cloud.npz'],
                       group_id=r['group_id'],domain=r['domain'])
            if f['status']=='selected_candidate':
                relative=f'masks/faces/{split}/{f["face_key"]}.png'
                copy_protected(directory/f['mask'],delivery/relative);entry['mask']=relative
                mask=cv2.imread(str(delivery/relative),cv2.IMREAD_GRAYSCALE)>0
                encoded=mask_utils.encode(np.asfortranarray(mask.astype(np.uint8)))
                area=int(mask_utils.area(encoded));bbox=mask_utils.toBbox(encoded).tolist()
                encoded['counts']=encoded['counts'].decode('ascii')
                coco[split]['annotations'].append({'id':next_annotation,'image_id':image_id,'category_id':1,
                   'segmentation':encoded,'area':area,'bbox':bbox,'iscrowd':0,'face_id':f['face_id'],
                   'face_key':f['face_key'],'parent_target_id':f['target_id'],
                   'proposal_only':True,'mask_human_reviewed':False,'segmentation_training_eligible':False})
                next_annotation+=1
            output_faces.append(entry);faces.append(entry)
        for t in r['targets']:
            relative=f'masks/pallets/{split}/{t["target_key"]}.png'
            copy_protected(directory/t['union_mask'],delivery/relative)
            entry=dict(t,union_mask=relative,frame_id=r['frame_id'],split=split,image=image,
                       cloud=assets['cloud.npz'],group_id=r['group_id'],domain=r['domain'])
            targets.append(entry);review.append(entry);output_targets.append(entry)
            if t['mask_points']:
                mask=cv2.imread(str(delivery/relative),cv2.IMREAD_GRAYSCALE)>0
                encoded=mask_utils.encode(np.asfortranarray(mask.astype(np.uint8)));area=int(mask_utils.area(encoded))
                bbox=mask_utils.toBbox(encoded).tolist();encoded['counts']=encoded['counts'].decode('ascii')
                unions[split]['annotations'].append({'id':len(unions[split]['annotations'])+1,'image_id':image_id,'category_id':1,
                    'segmentation':encoded,'area':area,'bbox':bbox,'iscrowd':0,'parent_target_id':t['target_id'],
                    'proposal_only':True,'mask_human_reviewed':False,'segmentation_training_eligible':False})
        frame_entry={k:v for k,v in r.items() if k not in ('targets','faces')}
        frame_entry.update(image=image,cloud=assets['cloud.npz'],instances=assets['instances.png'],
                           conflicts=assets['conflicts.png'],overlay=assets['overlay.jpg'],image_id=image_id)
        frames.append(frame_entry)
        from .fork_face_report import write_cloud_preview
        write_cloud_preview(delivery,frame_entry,output_targets,output_faces)
        if image_id%500==0:print(json.dumps({'stage':'export','frames':image_id,'total':len(records)}),flush=True)
    (delivery/'annotations').mkdir(exist_ok=True)
    for split in coco:
        save_json(delivery/'annotations'/f'faces_{split}.json',coco[split])
        save_json(delivery/'annotations'/f'pallets_{split}.json',unions[split])
    write_rows(delivery/'frames.jsonl',frames);write_rows(delivery/'targets.jsonl',targets)
    write_rows(delivery/'faces.jsonl',faces);write_rows(delivery/'review_queue.jsonl',review)
    summary=summarize_records(records);summary.update(state='exported',task_hash=args._manifest['task_hash'],finished_at=timestamp())
    save_json(delivery/'summary.json',summary)
    for name in ('run_manifest.json','geometry_config.json','unit_registry.json','input_audit.json','configuration_freeze.json','pilot_report_zh.md'):
        copy_protected(out/name,delivery/'provenance'/name)
    for name in ('fork_face_masks.py','fork_geometry.py','fork_face_report.py'):
        copy_protected(out/'source'/name,delivery/'source'/name)
    from .fork_face_report import write_report
    write_report(delivery,summary,frames,targets,faces)
    files={str(p.relative_to(delivery)):digest(p) for p in sorted(delivery.rglob('*')) if p.is_file()}
    manifest={'kind':'measured_fork_face_proposal_delivery','version':VERSION,'created_at':timestamp(),
              'task_hash':args._manifest['task_hash'],'files':files,'summary':summary,'all_paths_relative':True,
              'raw_pcd_copied':False,'proposal_only':True,'mask_human_reviewed':False}
    save_json(delivery/'export_manifest.json',manifest)
    return manifest


def verify(args,annotations,sources):
    out=Path(args.output);delivery=out/'delivery'
    exported=json.loads((delivery/'export_manifest.json').read_text())
    if exported['task_hash']!=args._manifest['task_hash']:raise ValueError('Export task binding differs')
    for relative,expected in exported['files'].items():
        if digest(safe_path(delivery,relative))!=expected:raise ValueError(f'Delivery hash mismatch: {relative}')
    frames=read_rows(delivery/'frames.jsonl');targets=read_rows(delivery/'targets.jsonl');faces=read_rows(delivery/'faces.jsonl')
    if len(frames)!=len(annotations) or {r['frame_id'] for r in frames}!={a['frame_id'] for a in annotations}:
        raise ValueError('Frame coverage differs from dataset')
    expected_targets={target_id(a['frame_id'],i) for a in annotations for i in range(len(a['boxes']))}
    if len(targets)!=len(expected_targets) or {t['target_id'] for t in targets}!=expected_targets:
        raise ValueError('Target coverage differs from dataset')
    by_frame_faces=defaultdict(list);by_frame_targets=defaultdict(list)
    for f in faces:by_frame_faces[f['frame_id']].append(f)
    for t in targets:by_frame_targets[t['frame_id']].append(t)
    pcd_formats=Counter();point_checks=0;mask_checks=0;conflicts=0
    for number,frame in enumerate(frames,1):
        fid=frame['frame_id'];s=sources[fid];h,width=frame['height'],frame['width']
        instances=cv2.imread(str(safe_path(delivery,frame['instances'])),cv2.IMREAD_UNCHANGED)
        conflict=cv2.imread(str(safe_path(delivery,frame['conflicts'])),cv2.IMREAD_GRAYSCALE)>0
        image=cv2.imread(str(safe_path(delivery,frame['image'])),cv2.IMREAD_GRAYSCALE)
        if instances.dtype!=np.uint16 or instances.shape!=(h,width) or image.shape!=(h,width):
            raise ValueError('Native image/instance dimensions differ')
        if np.any(instances[conflict]):raise ValueError('Conflicting pixels were assigned a face')
        with np.load(safe_path(delivery,frame['cloud']),allow_pickle=False) as z:
            xyz,uv,src,assigned,parent,diag=[z[k] for k in ('xyz_raw','uv','source_point_index','face_id','parent_box_index','diagnostic_plane_id')]
            intensity=z['intensity']
        if not (len(xyz)==len(uv)==len(src)==len(assigned)==len(parent)==len(intensity)==len(diag)):
            raise ValueError('Point field lengths differ')
        if len(uv):
            if not valid_xyz(xyz).all() or (uv<0).any() or (uv[:,0]>=width).any() or (uv[:,1]>=h).any():
                raise ValueError('Invalid exported point or pixel')
            expected=uv[:,1]*width+uv[:,0]
            if frame['pixel_rotation_degrees']==180:expected=h*width-1-expected
            if not np.array_equal(expected,src):raise ValueError('Source point index does not map back to pixel')
            for t in by_frame_targets[fid]:
                x1,y1,x2,y2=t['integer_bbox'];subset=uv[parent==t['box_index']]
                if ((subset[:,0]<x1)|(subset[:,0]>=x2)|(subset[:,1]<y1)|(subset[:,1]>=y2)).any():
                    raise ValueError('Point outside its parent box')
            selected=assigned>0
            if len(np.unique(uv[selected],axis=0))!=int(selected.sum()):raise ValueError('Selected pixels have multiple owners')
            if not np.array_equal(instances[uv[selected,1],uv[selected,0]],assigned[selected]):
                raise ValueError('Point assignment and instance raster differ')
            if int(selected.sum())!=int((instances>0).sum()):raise ValueError('Mask contains synthetic points')
            if not np.array_equal(conflict[uv[assigned==-1,1],uv[assigned==-1,0]],np.ones((assigned==-1).sum(),bool)):
                raise ValueError('Point conflict is not represented by conflict mask')
        for f in by_frame_faces[fid]:
            if not f['proposal_only'] or f['mask_human_reviewed']:raise ValueError('Incorrect annotation review flag')
            if f['status']!='selected_candidate':continue
            mask=cv2.imread(str(safe_path(delivery,f['mask'])),cv2.IMREAD_GRAYSCALE)>0
            if not np.array_equal(mask,instances==f['face_id']) or int(mask.sum())!=f['mask_points']:
                raise ValueError('Per-face raster differs from points/instance mask')
            if int(mask.sum())<args._config['minimum_face_points']:raise ValueError('Face below minimum support')
            model=np.asarray(f['plane_raw']);selected_points=xyz[assigned==f['face_id']].astype(float)
            residual=np.abs(selected_points@model[:3]+model[3])*(f['scale_to_mm'] if f['scale_to_mm'] is not None else 1.)
            if abs(np.linalg.norm(model[:3])-1.)>1e-6 or residual.max()>f['maximum_point_tolerance']+1e-5:
                raise ValueError('Exported point is inconsistent with fitted plane')
            mask_checks+=1
        for t in by_frame_targets[fid]:
            ids=[f['face_id'] for f in by_frame_faces[fid] if f['target_id']==t['target_id'] and f['status']=='selected_candidate']
            mask=cv2.imread(str(safe_path(delivery,t['union_mask'])),cv2.IMREAD_GRAYSCALE)>0
            if not np.array_equal(mask,np.isin(instances,ids)&(instances>0)):raise ValueError('Parent union differs')
            mask_checks+=1
        point_checks+=len(xyz);conflicts+=int(conflict.sum());pcd_formats[frame['pcd_encoding']]+=1
        if number%1000==0:print(json.dumps({'stage':'verify','frames':number,'total':len(frames)}),flush=True)
    rle_checks=0
    for split in ('train','val','test'):
        for kind in ('faces','pallets'):
            document=json.loads((delivery/'annotations'/f'{kind}_{split}.json').read_text())
            images={im['id']:im for im in document['images']}
            if len(images)!=sum(f['split']==split for f in frames):raise ValueError('COCO image count differs')
            lookup={(f['frame_id'],f['face_id']):f for f in faces} if kind=='faces' else {t['target_id']:t for t in targets}
            for ann in document['annotations']:
                frame_id=images[ann['image_id']]['frame_id']
                entry=lookup[(frame_id,ann['face_id'])] if kind=='faces' else lookup[ann['parent_target_id']]
                original=cv2.imread(str(safe_path(delivery,entry['mask'] if kind=='faces' else entry['union_mask'])),cv2.IMREAD_GRAYSCALE)>0
                decoded=mask_utils.decode(ann['segmentation']).astype(bool)
                if not np.array_equal(original,decoded) or int(decoded.sum())!=ann['area']:
                    raise ValueError('RLE topology/area differs from original measured mask')
                if not ann['proposal_only'] or ann['mask_human_reviewed'] or ann['segmentation_training_eligible']:
                    raise ValueError('Proposal was incorrectly promoted to reviewed training label')
                rle_checks+=1
    snapshot=protected_snapshot(args.dataset)
    if snapshot!=args._manifest['protected_files']:raise ValueError('Original formal labels or metadata changed')
    source_stats=json.loads((out/'source_stats.json').read_text())
    changed_metadata=0
    for fid,s in sources.items():
        p=Path(s['source_paths']['pcd']);stat=p.stat();before=source_stats[fid]
        if stat.st_size!=before['size'] or stat.st_mtime_ns!=before['mtime_ns']:
            changed_metadata+=1
            if digest(p)!=before['sha256']:raise ValueError('Raw source changed during extraction')
    result={'state':'passed','finished_at':timestamp(),'task_hash':args._manifest['task_hash'],
            'frames_checked':len(frames),'targets_checked':len(targets),'point_rows_checked':point_checks,
            'mask_checks':mask_checks,'rle_checks':rle_checks,'conflict_pixels':conflicts,
            'protected_files_checked':len(snapshot),'protected_files_unchanged':True,
            'raw_sources_unchanged':True,'raw_source_metadata_changes_rehashed':changed_metadata,
            'native_source_index_roundtrip':True,'no_synthetic_mask_pixels':True,
            'no_selected_pixel_multiple_owners':True,'pcd_encodings':dict(pcd_formats),
            'semantic_accuracy_measured':False}
    save_json(out/'validation.json',result)
    return result


def package(args):
    out=Path(args.output);delivery=out/'delivery';validation=json.loads((out/'validation.json').read_text())
    if validation['state']!='passed' or validation['task_hash']!=args._manifest['task_hash']:
        raise ValueError('Successful verification required before packaging')
    archive=out/'pallet_fork_face_masks_v1.zip';receipt=out/'delivery_manifest.json'
    if receipt.exists():
        old=json.loads(receipt.read_text())
        if old['archive_sha256']!=digest(archive):raise ValueError('Completed ZIP changed')
        return old
    check_space(out,args.minimum_free_gib)
    copy_protected(out/'validation.json',delivery/'validation.json')
    files=sorted(p for p in delivery.rglob('*') if p.is_file())
    hashes={str(p.relative_to(delivery)):digest(p) for p in files}
    check_file=delivery/'checksums.sha256'
    check_file.write_text(''.join(f'{sha}  {name}\n' for name,sha in hashes.items()))
    files.append(check_file);hashes[check_file.name]=digest(check_file)
    temporary=archive.with_suffix('.zip.tmp')
    with zipfile.ZipFile(temporary,'w',compression=zipfile.ZIP_DEFLATED,compresslevel=1,allowZip64=True) as z:
        for i,p in enumerate(files,1):
            if i%500==0:check_space(out,args.minimum_free_gib)
            z.write(p,str(p.relative_to(delivery)))
    with zipfile.ZipFile(temporary) as z:
        if z.testzip() is not None:raise ValueError('ZIP CRC verification failed')
        if len(z.infolist())!=len(hashes):raise ValueError('ZIP file count differs')
        for member in z.infolist():
            safe_path(delivery,member.filename)
            sha=hashlib.sha256()
            with z.open(member) as stream:
                for block in iter(lambda:stream.read(4*1024*1024),b''):sha.update(block)
            if sha.hexdigest()!=hashes[member.filename]:raise ValueError('ZIP member SHA mismatch')
    # Relocation smoke uses only packaged assets; it never accesses source PCD.
    relocation=out/'relocation_check';relocation.mkdir(exist_ok=True)
    with zipfile.ZipFile(temporary) as z:
        frames=read_rows(delivery/'frames.jsonl');sample=[]
        for split in ('train','val','test'):
            sample.append(next(f for f in frames if f['split']==split))
        for frame in sample:
            for key in ('image','cloud','instances','conflicts'):
                path=frame[key];z.extract(path,relocation)
                if digest(safe_path(relocation,path))!=hashes[path]:raise ValueError('Relocated asset differs')
            with np.load(safe_path(relocation,frame['cloud']),allow_pickle=False) as cloud:
                if cloud['xyz_raw'].shape[1]!=3:raise ValueError('Relocated NPZ unreadable')
            image=cv2.imread(str(safe_path(relocation,frame['image'])),cv2.IMREAD_GRAYSCALE)
            if image.shape!=(frame['height'],frame['width']):raise ValueError('Relocated PNG unreadable')
    shutil.rmtree(relocation)
    os.replace(temporary,archive);sha=digest(archive)
    archive.with_suffix('.zip.sha256').write_text(f'{sha}  {archive.name}\n')
    result={'state':'complete','finished_at':timestamp(),'task_hash':args._manifest['task_hash'],
            'archive':str(archive.resolve()),'archive_bytes':archive.stat().st_size,'archive_sha256':sha,
            'zip_members':len(hashes),'crc_passed':True,'all_member_sha256_passed':True,
            'relocation_three_splits_passed':True,'validation':validation,
            'summary':json.loads((delivery/'summary.json').read_text()),
            'proposal_only':True,'mask_human_reviewed':False,'formal_dataset_modified':False}
    save_json(receipt,result);save_json(out/'workflow_status.json',result)
    return result


def main(argv=None):
    parser=argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--dataset',default=None)
    parser.add_argument('--output',default='outputs/pallet_fork_face_masks_v1')
    parser.add_argument('--config',default=None)
    parser.add_argument('--unit-registry',default='outputs/pallet_target_clustering_v1/analysis/run_006/unit_registry.json')
    parser.add_argument('--target-context',default='outputs/pallet_target_clustering_v1/analysis/run_006/targets.jsonl')
    parser.add_argument('--stage',choices=('audit','pilot','extract','export','verify','package','all'),default='all')
    parser.add_argument('--resume',action='store_true')
    parser.add_argument('--seed',type=int,default=42)
    parser.add_argument('--workers',type=int,default=min(8,os.cpu_count() or 1))
    parser.add_argument('--pilot-count',type=int,default=256)
    parser.add_argument('--shards',type=int,default=1);parser.add_argument('--shard-index',type=int,default=0)
    parser.add_argument('--limit',type=int,default=None)
    parser.add_argument('--minimum-free-gib',type=float,default=50.)
    args=parser.parse_args(argv)
    if not 1<=args.workers<=8 or not 0<=args.shard_index<args.shards:parser.error('Invalid workers or shard')
    if args.dataset is None:args.dataset=json.loads((PROJECT/'configs/active_formal_dataset.json').read_text())['dataset']
    annotations,sources=load_dataset(args.dataset)
    prepare(args,annotations,sources)
    stages=('audit','pilot','extract','export','verify','package') if args.stage=='all' else (args.stage,)
    for stage in stages:
        if stage=='audit':result=audit(args,annotations,sources)
        elif stage=='pilot':result=extract(args,annotations,sources,True)
        elif stage=='extract':result=extract(args,annotations,sources)
        elif stage=='export':result=export(args,annotations,sources)
        elif stage=='verify':result=verify(args,annotations,sources)
        else:result=package(args)
        print(json.dumps({'stage':stage,'state':result.get('state','done'),'output':args.output},ensure_ascii=False),flush=True)


if __name__=='__main__':
    main()
