"""Independent fresh328 source replay, original-image classK export/protection audit."""
from collections import Counter,defaultdict
from contextlib import closing
import argparse
import hashlib
import json
import math
from pathlib import Path
import sys

import cv2
import numpy as np
from PIL import Image,ImageDraw
from pycocotools import mask as mu
from scipy.ndimage import binary_erosion,distance_transform_edt

PROJECT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(PROJECT));sys.path.insert(0,str(PROJECT/'scripts'))
from pallet.cluster_targets import digest,protected_snapshot
from pallet.mask_quota_saved_extension import load_formal_evidence,file_inventory
from pallet.mask_quality_recovery import fingerprint,POSE,GALLERY
from pallet.mask_remaining_classes import confirmed,original_rectangle
from pallet.mask_template_refined import QUARANTINE,readonly
from pallet.pose_review_geometry import canonical,polygon_value,raster_mask
from scripts.verify_mask_model_requests import roi
from scripts.verify_mask_under50_requests import native_quality,PageInventory

ROOT=PROJECT/'outputs/pallet_mask_annotation_update_requests_v2';MODEL=PROJECT/'outputs/pallet_mask_annotation_update_v2'
FLAGS=('human_edited','human_mask_confirmed','human_pose_confirmed','ground_truth_eligible','automatic_training_eligible',
    'robot_command_eligible','production_saved','counts_as_completed_annotation','accepted_candidate','propagation_seed_eligible')


def read(p):return json.loads(Path(p).read_text())
def lines(p):return [json.loads(s) for s in Path(p).read_text().splitlines() if s.strip()]
def sha(v):return hashlib.sha256(canonical(v)).hexdigest()
def raw_sha(v):return hashlib.sha256(v.encode()).hexdigest() if v else None
def require(v,message):
    if not v:raise ValueError(message)
def write(p,v):
    with Path(p).open('xb') as stream:stream.write(canonical(v)+b'\n')
def decode(rle):return mu.decode(dict(size=rle['size'],counts=rle['counts'].encode())).astype(bool)


def source_state():
    inputs=read(ROOT/'input_manifest.json');training=read(MODEL/'TRAINING_HANDOFF.json');records=lines(MODEL/'private/training_records.jsonl')
    require(training['state']=='verified_both_completed145_warm_updates_all2041_not_quality_release' and not training['errors']
        and training['dataset_manifest_sha256']==inputs['training_dataset_manifest_sha256']==digest(MODEL/'dataset_manifest.json'),'Real2041 training not complete or version differs')
    counts=Counter(r['class_id'] for r in records);require(len(records)==2041 and inputs['real_saved_before']=={str(c):n for c,n in counts.items()},'Actual human counts differ')
    for route,run in training['runs'].items():require(digest(MODEL/route/'final_ema.pt')==run['checkpoint_sha256'],'Updated final EMA changed')
    for path,h in read(MODEL/'private/old_versions_inventory.json').items():require(file_inventory(PROJECT/path)==h,'Old model/data/worklist/failed evidence changed: '+path)
    require(file_inventory(PROJECT/'configs')==inputs['configuration_sha256'] and fingerprint(POSE)==inputs['pose_fingerprint']
        and fingerprint(GALLERY)==inputs['gallery_fingerprint'],'Global configs/pose/gallery changed')
    for path,h in inputs['existing_review_store_fingerprints'].items():require(fingerprint(PROJECT/path)==h,'Real review store changed: '+path)
    formal,rows,evidence=load_formal_evidence(read(PROJECT/'outputs/pallet_mask_remaining_round5_v1/manifest.json'))
    require(protected_snapshot(formal)==inputs['formal_protected'],'Formal labels/PCD/images/GT/metadata changed')
    protected=set(QUARANTINE)|{r['target_key'] for r in records}
    protected.update(r['target_key'] for r in lines(PROJECT/'outputs/pallet_mask_remaining_round5_v1/proposals.jsonl'))
    with closing(readonly(POSE)) as db:
        current=dict(db.execute('SELECT target_key,payload FROM annotations'));history=defaultdict(list)
        for key,raw in db.execute('SELECT target_key,payload FROM events ORDER BY id'):history[key].append(json.loads(raw))
    for key in set(current)|set(history):
        values=history[key]+([json.loads(current[key])] if key in current else [])
        if key not in current or any(v.get('state') in ('deleted','deferred','confirmed') or confirmed(v) or not original_rectangle(v.get('polygon'),rows[key]) for v in values):protected.add(key)
    inventories={}
    for pattern in ('requests.jsonl','shortlist.jsonl','inspection_targets.jsonl'):
        for path in sorted((PROJECT/'outputs').rglob(pattern)):
            if path.is_relative_to(ROOT):continue
            inventories[str(path.relative_to(PROJECT))]=digest(path);protected.update(r['target_key'] for r in lines(path) if r.get('target_key') in rows)
    require(inventories==inputs['prior_worklist_inventories_sha256'],'Prior request/shortlist/inspection files changed')
    for path in inputs['existing_review_store_fingerprints']:
        with closing(readonly(PROJECT/path)) as db:
            for table in ('reviews','events'):protected.update(k for k, in db.execute(f'SELECT DISTINCT target_key FROM {table}') if k in rows)
    require(len(protected)==inputs['previously_protected_targets'],'Protected target inventory differs')
    frames={rows[k]['frame_id'] for k in protected};hashes={digest(formal/name) for name in {rows[k]['image'] for k in protected}}
    return inputs,formal,rows,evidence,current,protected,frames,hashes,counts


def replay_model():
    import torch
    from rfdetr import RFDETRSegSmall
    from pallet.mask_model_training_data import letterbox,gpu_receipt
    from pallet.mask_model_improvement_adapter import highest_class_query,native_logits,mask_logits
    path=ROOT/'private/independent_model_replay.json';require(not path.exists(),'Preserve completed independent GPU replay')
    h=read(MODEL/'TRAINING_HANDOFF.json')['runs']['m2']['checkpoint_sha256']
    require(digest(MODEL/'m2/final_ema.pt')==h and h!='3eb7d659daccd424c2b47b0b70b098f8041273ba8b0f407bc10a57caaa29c524','Must replay newly trained2041 M2, not old predictions')
    gpu=gpu_receipt('m2');torch.set_num_threads(4);cv2.setNumThreads(1);weights=torch.load(MODEL/'m2/final_ema.pt',map_location='cpu',weights_only=True)
    wrapper=RFDETRSegSmall(num_classes=1,resolution=576,device='cuda:0',pretrain_weights=str(PROJECT/'outputs/pallet_mask_model_training_v1/generic_pretraining/rf-detr-seg-small.pt'),gradient_checkpointing=True,compile=False)
    model=wrapper.model.model;model.load_state_dict(weights['model'],strict=True);model.cuda().eval();del weights
    formal=Path(read(PROJECT/'configs/active_formal_dataset.json')['dataset']);staged=lines(ROOT/'private/shortlist.jsonl');predictions={p['target_key']:p for p in lines(ROOT/'private/predictions.jsonl')};matched=[]
    with torch.inference_mode():
        for number,r in enumerate(staged,1):
            row=r['source'];bounds=roi(row);x1,y1,x2,y2=bounds;grey=cv2.imread(str(formal/row['image']),0)[y1:y2,x1:x2]
            canvas,_,_,transform=letterbox(grey,None,(576,576));rgb=np.repeat(canvas[None].astype(np.float32)/255,3,axis=0)
            value=torch.from_numpy((rgb-np.array([.485,.456,.406],np.float32)[:,None,None])/np.array([.229,.224,.225],np.float32)[:,None,None])
            output=model(value[None].cuda());query,score=highest_class_query(output);mask=native_logits(mask_logits(output)[0,query],transform).sigmoid().cpu().numpy()>=.5;p=predictions[r['target_key']]
            require(bounds==p['roi'] and query==p['selected_query'] and np.array_equal(mask,decode(p['raw_native_prediction_rle']))
                and abs(score-p['class_confidence_not_quality_probability'])<1e-7,'Exact newly updated native replay differs: '+r['target_key'])
            matched.append(r['target_key'])
            if number%100==0:print('Independent newly trained2041 M2 replay',number,flush=True)
    write(path,dict(state='all_fresh_native_masks_exactly_replayed_from_new2041_M2',targets=len(matched),checkpoint_sha256=h,
        predictions_sha256=digest(ROOT/'private/predictions.jsonl'),shortlist_sha256=digest(ROOT/'private/shortlist.jsonl'),target_keys_sha256=sha(matched),
        gpu=gpu,human_masks_features_or_query_selection_used=False,quality_certified=False,code_sha256=digest(__file__)))
    print('Independent new2041 model replay complete',len(matched),flush=True)


def verify():
    require(not (ROOT/'verification.json').exists(),'Preserve completed structural seal')
    inputs,formal,rows,evidence,current,protected,frames,hashes,counts=source_state();manifest=read(ROOT/'manifest.json');standard=read(ROOT/'class_keypoint_standard.json')
    staged=lines(ROOT/'private/shortlist.jsonl');predictions=lines(ROOT/'private/predictions.jsonl')
    require(len(staged)==len(predictions)==328 and len({r['target_key'] for r in staged})==328,'Exact328 distinct staged new targets required')
    require(digest(ROOT/'private/shortlist.jsonl')==inputs['shortlist_sha256'] and digest(ROOT/'input_manifest.json')==manifest['input_manifest_sha256']
        and digest(ROOT/'private/predictions.jsonl')==manifest['predictions_sha256'] and manifest['checkpoint_sha256']==digest(MODEL/'m2/final_ema.pt'),'Actual inference/input/checkpoint bytes differ')
    require(digest(ROOT/'class_keypoint_standard.json')==manifest['class_standard_sha256']==inputs['class_standard_sha256']==digest(MODEL/'class_keypoint_standard.json'),'Class standards automatically recomputed')
    decisions=read(ROOT/'private/visual_decisions.json');contact=read(ROOT/'private/contact_sheets/inventory.json');refs={}
    require(decisions['predictions_sha256']==manifest['predictions_sha256']==contact['predictions_sha256']
        and decisions['contact_sheet_inventory_sha256']==digest(ROOT/'private/contact_sheets/inventory.json'),'Visual evidence binding differs')
    for page in contact['pages']:
        require(digest(PROJECT/page['path'])==page['sha256'],'Actually viewed contact sheet changed')
        for key,i in zip(page['target_keys'],page['candidate_indices']):refs[key]=(page['path'],page['sha256'],i)
    require(decisions['actual_sheets_viewed']==len(contact['pages']) and set(decisions['decisions'])=={r['target_key'] for r in staged},'Missing visual scope/decision')
    seenframes=set();seenhashes=set();eligible=[];issued=Counter();by_key={};fidelity=[]
    for order,(r,p) in enumerate(zip(staged,predictions),1):
        key=r['target_key'];row=rows[key];cls=row['final_class_id'];decision=decisions['decisions'][key];h=digest(formal/row['image'])
        require(key==p['target_key'] and r['source']==row and r['class_id']==p['class_id']==cls and r['request_order']==order,'Target/order/class pairing differs')
        require(counts[cls]<100 and key not in protected and row['frame_id'] not in frames and h not in hashes,'Protected or complete-class source generated')
        require(row['frame_id'] not in seenframes and h not in seenhashes and h==r['image_sha256'],'Duplicate frame/image source');seenframes.add(row['frame_id']);seenhashes.add(h)
        require(native_quality(row,evidence)==r['native_quality'] and raw_sha(current.get(key))==r['pose_payload_raw_sha256'] and r['gallery_payload_raw_sha256'] is None,'Native gate/source raw binding differs')
        require(p['roi']==roi(row) and p['threshold']==.5 and p['checkpoint_sha256']==manifest['checkpoint_sha256'],'Fixed inference context or threshold differs')
        mask=decode(p['raw_native_prediction_rle']);x1,y1,x2,y2=p['roi'];require(mask.shape==(y2-y1,x2-x1) and int(mask.sum())==p['raw_mask_area'],'Raw model RLE shape/area differs')
        require(type(decision['publish']) is bool and decision['human_quality_approved'] is False and decision['model_geometry_modified'] is False,'False human QC/geometry claim')
        if not p['polygon']:
            require(not decision['publish'] and p['conversion']['reason'],'Unsupported conversion published');continue
        polygon_value(p['polygon'],*row['size']);q=np.array(p['polygon']);k=standard['classes'][str(cls)]['standard_count']
        require(len(q)==p['standard_count']==k,'Initial machine K differs')
        before=q-np.roll(q,1,axis=0);after=np.roll(q,-1,axis=0)-q;chord=np.roll(q,-1,axis=0)-np.roll(q,1,axis=0)
        turns=np.abs(np.degrees(np.arctan2(before[:,0]*after[:,1]-before[:,1]*after[:,0],(before*after).sum(axis=1))))
        deviations=np.abs(chord[:,0]*before[:,1]-chord[:,1]*before[:,0])/np.maximum(np.linalg.norm(chord,axis=1),1e-12)
        require(np.linalg.norm(after,axis=1).min()>=1.5 and turns.min()>=18 and deviations.min()>=.55 and cv2.contourArea(q.astype(np.float32),oriented=True)>0,'Redundant/collinear/fake K corners')
        _,hierarchy=cv2.findContours(mask.astype(np.uint8),cv2.RETR_CCOMP,cv2.CHAIN_APPROX_SIMPLE)
        require(cv2.connectedComponents(mask.astype(np.uint8),8)[0]==2 and hierarchy is not None and not (hierarchy[0,:,3]>=0).any(),'Components/holes silently discarded')
        rendered=raster_mask(p['polygon'],*row['size'])[y1:y2,x1:x2];iou=float((mask&rendered).sum()/(mask|rendered).sum());a=mask^binary_erosion(mask);b=rendered^binary_erosion(rendered)
        hd=float(np.quantile(np.r_[distance_transform_edt(~a)[b],distance_transform_edt(~b)[a]],.95));recorded=p['conversion']['model_mask_fidelity_not_human_accuracy']
        require(abs(iou-recorded['iou'])<1e-12 and abs(hd-recorded['hd95'])<1e-12 and iou>=.90 and hd<=max(2.,.1*row['bbox']['height'])
            and p['conversion']['inserted_or_collinear_padding_points']==0,'Serialized original-coordinate mask fidelity differs')
        require(key in refs and decision['AI_coarse_crop_inspection'] and refs[key]==(decision['contact_sheet'],decision['contact_sheet_sha256'],decision['candidate_index']),'Actual crop evidence differs')
        if decision['publish'] and issued[cls]<inputs['remaining_to100'][str(cls)]:eligible.append(key);issued[cls]+=1;fidelity.append(iou)
        by_key[key]=(r,p)
    proposals=lines(ROOT/'proposals.jsonl');requests=lines(ROOT/'requests.jsonl')
    require([p['target_key'] for p in proposals]==[r['target_key'] for r in requests]==eligible and len(proposals)==manifest['request_count'],'Published scope differs')
    require(digest(ROOT/'proposals.jsonl')==manifest['proposals_sha256'] and digest(ROOT/'requests.jsonl')==manifest['requests_sha256'],'Task bytes differ')
    for p in proposals:
        key=p['target_key'];r,pred=by_key[key];row=r['source'];polygon=p['polygon'];k=standard['classes'][str(p['class_id'])]['standard_count'];proof=p['model_provenance'];constraint=p['class_keypoint_constraint']
        require(p['schema']=='updated2041_M2_class_standard_keypoint_preannotation_v2' and p['origin']=='updated_M2_prediction'
            and polygon==pred['polygon'] and p['source']==row and p['vertex_count']==k and all(p.get(f) is False for f in FLAGS),'False flags/source/geometry differs')
        require(constraint['class_id']==p['class_id'] and constraint['standard_count']==k and constraint['standard_sha256']==manifest['class_standard_sha256'],'Initial count standard binding differs')
        require(proof['checkpoint_sha256']==manifest['checkpoint_sha256'] and proof['predictions_sha256']==manifest['predictions_sha256'] and proof['native_mask_rle_sha256']==sha(pred['raw_native_prediction_rle'])
            and proof['corner_conversion']==pred['conversion'] and proof['visual_inspection']==decisions['decisions'][key]
            and not proof['human_saved'] and not proof['assistant_redrawn'] and not proof['quality_gate_passed'],'New model/source/inspection proof differs')
        require(p['model_source_image_sha256']==digest(ROOT/'images'/f'{key}.png')==r['image_sha256'] and p['pose_payload_raw_sha256']==r['pose_payload_raw_sha256'],'Original raw bytes differ')
        label=read(ROOT/'labelme'/f'{key}.json');points=[s for s in label['shapes'] if s['shape_type']=='point']
        require(label['num_keypoints']==len(points)==k and label['keypoints_xy']==label['shapes'][0]['points']==polygon and [s['points'][0] for s in points]==polygon
            and [s['label'] for s in points]==[f'kp_{i:02d}' for i in range(1,k+1)] and label['imageWidth']==row['size'][0] and label['imageHeight']==row['size'][1]
            and not label['flags']['human_confirmed'] and not label['flags']['ground_truth_eligible'],'Explicit LabelMe slots differ')
        require(np.array_equal(np.array(Image.open(ROOT/'masks'/f'{key}.png'))>0,raster_mask(polygon,*row['size'])),'PNG export differs')
        image=Image.open(formal/row['image']).convert('RGB');draw=ImageDraw.Draw(image);vertices=[tuple(v) for v in polygon];draw.line(vertices+[vertices[0]],fill='#55d5ff',width=1);box=row['bbox']
        l=max(0,math.floor(box['x']-12));t=max(0,math.floor(box['y']-16));rr=min(row['size'][0],math.ceil(box['x']+box['width']+12));bt=min(row['size'][1],math.ceil(box['y']+box['height']+16))
        expected=image.crop((l,t,rr,bt)).resize(((rr-l)*3,(bt-t)*3),Image.Resampling.NEAREST);draw=ImageDraw.Draw(expected)
        for i,(x,y) in enumerate(polygon,1):
            xx=(x-l)*3;yy=(y-t)*3;draw.ellipse((xx-2,yy-2,xx+2,yy+2),fill='#ffbe43');draw.text((xx+3,yy-9),str(i),fill='white',stroke_width=1,stroke_fill='black')
        require(np.array_equal(np.asarray(expected),np.asarray(Image.open(ROOT/'cases'/f'{key}.png'))),'Exact unedited numbered overlay differs')
    require(manifest['class_counts']=={str(c):n for c,n in issued.items()} and all(issued[c]<=100-counts[c] for c in counts),'Exceeds remaining quota')
    require(manifest['conditional_gap_after_all_drafts_saved']=={str(c):max(0,100-counts[c]-issued[c]) for c in range(1,23)},'Conditional remaining is not actual human completion')
    allowed={'report_zh.html','report-font.otf','FONT_LICENSE.txt','model-editor.js'}
    allowed.update(f'{folder}/{p["target_key"]}.{ext}' for p in proposals for folder,ext in [('images','png'),('cases','png'),('masks','png'),('labelme','json')])
    require(set(manifest['public_assets'])==allowed,'Unexpected public resource')
    for name,h in manifest['public_assets'].items():
        path=ROOT/name;require(path.is_file() and not path.is_symlink() and path.resolve().is_relative_to(ROOT) and digest(path)==h,'Public path/hash differs')
    page=(ROOT/'report_zh.html').read_text();inventory=PageInventory(page)
    require(inventory.articles==[(f'target-{p["target_key"]}',str(p['class_id'])) for p in proposals] and inventory.options==['']+[str(c) for c in range(1,23)]
        and all(n in allowed for n in inventory.assets),'Page inventory differs')
    replay=read(ROOT/'private/independent_model_replay.json')
    require(replay['state']=='all_fresh_native_masks_exactly_replayed_from_new2041_M2' and replay['targets']==328
        and replay['predictions_sha256']==manifest['predictions_sha256'] and replay['shortlist_sha256']==inputs['shortlist_sha256']
        and replay['checkpoint_sha256']==manifest['checkpoint_sha256'],'Actual independently replayed new model required')
    source_state();receipt=dict(state='passed_updated_M2_sources_new_images_classK_exports_not_accuracy',manifest_sha256=digest(ROOT/'manifest.json'),
        public_assets=manifest['public_assets'],real_manual_saved_before=2041,request_count=len(proposals),class_counts=dict(issued),native_model_masks_independently_replayed=328,
        keypoint_conversion_candidates=len(refs),human_events_created=0,old_data_models_worklists_and_human_history_unchanged=True,new_model_retrained=True,
        independent_accuracy_certified=False,class_standard_unchanged=True,production_model_activated=False,conversion_iou_not_human_accuracy_min=min(fidelity),code_sha256=digest(__file__))
    for name in ('verification.json','independent_verification.json'):write(ROOT/name,receipt)
    print(json.dumps({k:v for k,v in receipt.items() if k!='public_assets'}),flush=True)


if __name__=='__main__':
    p=argparse.ArgumentParser(description=__doc__);p.add_argument('--replay-model',action='store_true');a=p.parse_args()
    replay_model() if a.replay_model else verify()
