"""Independent remaining-source protection, updated-checkpoint replay and exports."""
from collections import Counter,defaultdict
from contextlib import closing
import argparse
import hashlib
import json
import math
from pathlib import Path
import sys

import cv2
import numpy as np
from PIL import Image,ImageDraw
from pycocotools import mask as mu
from scipy.ndimage import binary_erosion,distance_transform_edt

PROJECT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(PROJECT));sys.path.insert(0,str(PROJECT/'scripts'))
from pallet.cluster_targets import digest,protected_snapshot
from pallet.mask_quota_saved_extension import load_formal_evidence,file_inventory
from pallet.mask_quality_recovery import fingerprint,POSE,GALLERY
from pallet.mask_remaining_classes import confirmed,original_rectangle
from pallet.mask_template_refined import QUARANTINE,readonly
from pallet.pose_review_geometry import canonical,polygon_value,raster_mask
from scripts.verify_mask_model_requests import roi
from scripts.verify_mask_under50_requests import native_quality,PageInventory

ROOT=PROJECT/'outputs/pallet_mask_annotation_update_requests_v1';MODEL=PROJECT/'outputs/pallet_mask_annotation_update_v1'
FLAGS=('human_edited','human_mask_confirmed','human_pose_confirmed','ground_truth_eligible','automatic_training_eligible',
    'robot_command_eligible','production_saved','counts_as_completed_annotation','accepted_candidate','propagation_seed_eligible')


def read(p):return json.loads(Path(p).read_text())
def lines(p):return [json.loads(s) for s in Path(p).read_text().splitlines() if s.strip()]
def sha(v):return hashlib.sha256(canonical(v)).hexdigest()
def raw_sha(v):return hashlib.sha256(v.encode()).hexdigest() if v else None
def require(v,message):
    if not v:raise ValueError(message)
def write(p,v):
    with Path(p).open('xb') as stream:stream.write(canonical(v)+b'\n')
def decode(rle):return mu.decode(dict(size=rle['size'],counts=rle['counts'].encode())).astype(bool)


def source_state():
    inputs=read(ROOT/'input_manifest.json');training=read(MODEL/'TRAINING_HANDOFF.json')
    require(training['state']=='verified_both_completed337_warm_updates_all1896_not_quality_release' and not training['errors']
        and training['dataset_manifest_sha256']==inputs['training_dataset_manifest_sha256']==digest(MODEL/'dataset_manifest.json'),'Updated training not truly complete')
    records=lines(MODEL/'private/training_records.jsonl');counts=Counter(r['class_id'] for r in records)
    require(len(records)==1896 and inputs['real_saved_before']=={str(c):n for c,n in counts.items()},'Manual saved counts differ')
    for route,run in training['runs'].items():require(digest(MODEL/route/'final_ema.pt')==run['checkpoint_sha256'],'Updated final EMA changed')
    for path,expected in read(MODEL/'private/old_versions_inventory.json').items():require(file_inventory(PROJECT/path)==expected,'Prior model/data/static/failed evidence changed: '+path)
    require(file_inventory(PROJECT/'configs')==inputs['configuration_sha256'],'Global/old configs changed')
    require(fingerprint(POSE)==inputs['pose_fingerprint'] and fingerprint(GALLERY)==inputs['gallery_fingerprint'],'Pose/gallery current or history changed')
    for path,expected in inputs['existing_review_store_fingerprints'].items():require(fingerprint(PROJECT/path)==expected,'Historical real review store changed: '+path)
    formal,rows,evidence=load_formal_evidence(read(PROJECT/'outputs/pallet_mask_remaining_round5_v1/manifest.json'))
    require(protected_snapshot(formal)==inputs['formal_protected'],'Formal labels/images/PCD/GT changed')
    protected=set(QUARANTINE)|{r['target_key'] for r in records}
    protected.update(r['target_key'] for r in lines(PROJECT/'outputs/pallet_mask_remaining_round5_v1/proposals.jsonl'))
    with closing(readonly(POSE)) as db:
        current=dict(db.execute('SELECT target_key,payload FROM annotations'));hist=defaultdict(list)
        for key,raw in db.execute('SELECT target_key,payload FROM events ORDER BY id'):hist[key].append(json.loads(raw))
    for key in set(current)|set(hist):
        values=hist[key]+([json.loads(current[key])] if key in current else [])
        if key not in current or any(v.get('state') in ('deleted','deferred','confirmed') or confirmed(v) or not original_rectangle(v.get('polygon'),rows[key]) for v in values):protected.add(key)
    inventories={}
    for pattern in ('requests.jsonl','shortlist.jsonl','inspection_targets.jsonl'):
        for path in sorted((PROJECT/'outputs').rglob(pattern)):
            if path.is_relative_to(ROOT):continue
            inventories[str(path.relative_to(PROJECT))]=digest(path)
            protected.update(r['target_key'] for r in lines(path) if r.get('target_key') in rows)
    require(inventories==inputs['prior_worklist_inventories_sha256'],'Prior worklist/shortlist/inspection evidence changed')
    for path in inputs['existing_review_store_fingerprints']:
        with closing(readonly(PROJECT/path)) as db:
            for table in ('reviews','events'):protected.update(k for k, in db.execute(f'SELECT DISTINCT target_key FROM {table}') if k in rows)
    require(len(protected)==inputs['previously_protected_targets'],'Protected targets changed')
    frames={rows[k]['frame_id'] for k in protected};hashes={digest(formal/name) for name in {rows[k]['image'] for k in protected}}
    return inputs,formal,rows,evidence,current,protected,frames,hashes,counts


def replay_model():
    import torch
    from rfdetr import RFDETRSegSmall
    from pallet.mask_model_training_data import letterbox,gpu_receipt
    from pallet.mask_model_improvement_adapter import highest_class_query,native_logits,mask_logits
    path=ROOT/'private/independent_model_replay.json';require(not path.exists(),'Preserve completed replay')
    h=read(MODEL/'TRAINING_HANDOFF.json')['runs']['m2']['checkpoint_sha256']
    require(digest(MODEL/'m2/final_ema.pt')==h and h!='92ce35476d4b0887a0e791dc20972c9640aab558ec876df84296b713ebe8ed4c','Must replay the genuinely updated M2, not old predictions')
    gpu=gpu_receipt('m2');torch.set_num_threads(4);cv2.setNumThreads(1)
    weights=torch.load(MODEL/'m2/final_ema.pt',map_location='cpu',weights_only=True)
    wrapper=RFDETRSegSmall(num_classes=1,resolution=576,device='cuda:0',pretrain_weights=str(PROJECT/'outputs/pallet_mask_model_training_v1/generic_pretraining/rf-detr-seg-small.pt'),gradient_checkpointing=True,compile=False)
    model=wrapper.model.model;model.load_state_dict(weights['model'],strict=True);model.cuda().eval();del weights
    formal=Path(read(PROJECT/'configs/active_formal_dataset.json')['dataset']);staged=lines(ROOT/'private/shortlist.jsonl')
    predictions={p['target_key']:p for p in lines(ROOT/'private/predictions.jsonl')};matched=[]
    with torch.inference_mode():
        for number,r in enumerate(staged,1):
            row=r['source'];bounds=roi(row);x1,y1,x2,y2=bounds
            grey=cv2.imread(str(formal/row['image']),0)[y1:y2,x1:x2];canvas,_,_,transform=letterbox(grey,None,(576,576))
            rgb=np.repeat(canvas[None].astype(np.float32)/255,3,axis=0)
            value=torch.from_numpy((rgb-np.array([.485,.456,.406],np.float32)[:,None,None])/np.array([.229,.224,.225],np.float32)[:,None,None])
            output=model(value[None].cuda());query,score=highest_class_query(output)
            mask=native_logits(mask_logits(output)[0,query],transform).sigmoid().cpu().numpy()>=.5;p=predictions[r['target_key']]
            require(bounds==p['roi'] and query==p['selected_query'] and np.array_equal(mask,decode(p['raw_native_prediction_rle']))
                and abs(score-p['class_confidence_not_quality_probability'])<1e-7,'Exact native model replay differs: '+r['target_key'])
            matched.append(r['target_key'])
            if number%200==0:print('Independent updated M2 replay',number,flush=True)
    write(path,dict(state='all_fresh_native_masks_exactly_replayed_from_updated_M2',targets=len(matched),checkpoint_sha256=h,
        predictions_sha256=digest(ROOT/'private/predictions.jsonl'),shortlist_sha256=digest(ROOT/'private/shortlist.jsonl'),
        target_keys_sha256=sha(matched),gpu=gpu,human_masks_features_or_query_selection_used=False,quality_certified=False,code_sha256=digest(__file__)))
    print('Independent exact updated model replay complete',len(matched),flush=True)


def verify():
    require(not (ROOT/'verification.json').exists(),'Preserve structural seal')
    inputs,formal,rows,evidence,current,protected,frames,hashes,counts=source_state()
    manifest=read(ROOT/'manifest.json');standard=read(ROOT/'class_keypoint_standard.json')
    staged=lines(ROOT/'private/shortlist.jsonl');predictions=lines(ROOT/'private/predictions.jsonl')
    require(len(staged)==len(predictions)==921 and len({r['target_key'] for r in staged})==921,'Exact921 distinct fresh targets required')
    require(digest(ROOT/'private/shortlist.jsonl')==inputs['shortlist_sha256'] and digest(ROOT/'input_manifest.json')==manifest['input_manifest_sha256'],'Input binding differs')
    require(digest(ROOT/'private/predictions.jsonl')==manifest['predictions_sha256'] and manifest['checkpoint_sha256']==digest(MODEL/'m2/final_ema.pt'),'Actual updated model provenance differs')
    require(digest(ROOT/'class_keypoint_standard.json')==manifest['class_standard_sha256']==inputs['class_standard_sha256']
        ==digest(MODEL/'class_keypoint_standard.json'),'Frozen class standards changed')
    decisions=read(ROOT/'private/visual_decisions.json');contact=read(ROOT/'private/contact_sheets/inventory.json');refs={}
    require(decisions['predictions_sha256']==manifest['predictions_sha256']==contact['predictions_sha256']
        and decisions['contact_sheet_inventory_sha256']==digest(ROOT/'private/contact_sheets/inventory.json'),'Visual evidence input differs')
    for page in contact['pages']:
        require(digest(PROJECT/page['path'])==page['sha256'],'Paired visual sheet changed')
        for key,i in zip(page['target_keys'],page['candidate_indices']):refs[key]=(page['path'],page['sha256'],i)
    require(decisions['actual_sheets_viewed']==len(contact['pages']) and set(decisions['decisions'])=={r['target_key'] for r in staged},'Missing actual visual evidence scope')
    issued_counts=Counter();eligible=[];fidelity=[];seenframes=set();seenhash=set();by_key={}
    for order,(r,p) in enumerate(zip(staged,predictions),1):
        key=r['target_key'];row=rows[key];cls=row['final_class_id'];decision=decisions['decisions'][key];h=digest(formal/row['image'])
        require(key==p['target_key'] and r['source']==row and r['class_id']==p['class_id']==cls and r['request_order']==order,'Source/order/class pairing differs')
        require(counts[cls]<100 and key not in protected and row['frame_id'] not in frames and h not in hashes,'Protected or complete-class image generated')
        require(row['frame_id'] not in seenframes and h not in seenhash and h==r['image_sha256'],'Duplicate frame or native bytes');seenframes.add(row['frame_id']);seenhash.add(h)
        require(native_quality(row,evidence)==r['native_quality'] and raw_sha(current.get(key))==r['pose_payload_raw_sha256']
            and r['gallery_payload_raw_sha256'] is None,'Native gate/current-source binding differs')
        require(p['roi']==roi(row) and p['threshold']==.5 and p['checkpoint_sha256']==manifest['checkpoint_sha256'],'ROI/threshold/model changed')
        mask=decode(p['raw_native_prediction_rle']);x1,y1,x2,y2=p['roi']
        require(mask.shape==(y2-y1,x2-x1) and int(mask.sum())==p['raw_mask_area'],'Raw RLE shape/area differs')
        require(type(decision['publish']) is bool and decision['human_quality_approved'] is False and decision['model_geometry_modified'] is False,'False human QC or altered geometry claim')
        if not p['polygon']:
            require(not decision['publish'] and p['conversion']['reason'],'Unsupported conversion published');continue
        polygon_value(p['polygon'],*row['size']);k=standard['classes'][str(cls)]['standard_count'];require(len(p['polygon'])==p['standard_count']==k,'Machine initial K differs')
        q=np.array(p['polygon']);before=q-np.roll(q,1,axis=0);after=np.roll(q,-1,axis=0)-q;chord=np.roll(q,-1,axis=0)-np.roll(q,1,axis=0)
        turns=np.abs(np.degrees(np.arctan2(before[:,0]*after[:,1]-before[:,1]*after[:,0],(before*after).sum(axis=1))))
        deviations=np.abs(chord[:,0]*(q-np.roll(q,1,axis=0))[:,1]-chord[:,1]*(q-np.roll(q,1,axis=0))[:,0])/np.maximum(np.linalg.norm(chord,axis=1),1e-12)
        require(np.linalg.norm(after,axis=1).min()>=1.5 and turns.min()>=18 and deviations.min()>=.55
            and cv2.contourArea(q.astype(np.float32),oriented=True)>0,'Padded/redundant/wrong-order initial corners')
        contours,hierarchy=cv2.findContours(mask.astype(np.uint8),cv2.RETR_CCOMP,cv2.CHAIN_APPROX_SIMPLE)
        require(cv2.connectedComponents(mask.astype(np.uint8),8)[0]==2 and hierarchy is not None and not (hierarchy[0,:,3]>=0).any(),'Disconnected component/closed hole lost')
        rendered=raster_mask(p['polygon'],*row['size'])[y1:y2,x1:x2];iou=float((mask&rendered).sum()/(mask|rendered).sum())
        a=mask^binary_erosion(mask);b=rendered^binary_erosion(rendered);hd=float(np.quantile(np.r_[distance_transform_edt(~a)[b],distance_transform_edt(~b)[a]],.95))
        recorded=p['conversion']['model_mask_fidelity_not_human_accuracy']
        require(abs(iou-recorded['iou'])<1e-12 and abs(hd-recorded['hd95'])<1e-12 and iou>=.90
            and hd<=max(2.,.1*row['bbox']['height']) and p['conversion']['inserted_or_collinear_padding_points']==0,'Native conversion fidelity differs')
        require(key in refs and decision['AI_coarse_crop_inspection'] and refs[key]==(decision['contact_sheet'],decision['contact_sheet_sha256'],decision['candidate_index']),'Coarse inspection evidence differs')
        if decision['publish'] and issued_counts[cls]<inputs['remaining_to100'][str(cls)]:eligible.append(key);issued_counts[cls]+=1;fidelity.append(iou)
        by_key[key]=(r,p)
    proposals=lines(ROOT/'proposals.jsonl');requests=lines(ROOT/'requests.jsonl')
    require([p['target_key'] for p in proposals]==[p['target_key'] for p in requests]==eligible and len(proposals)==manifest['request_count'],'Published scope differs')
    require(digest(ROOT/'proposals.jsonl')==manifest['proposals_sha256'] and digest(ROOT/'requests.jsonl')==manifest['requests_sha256'],'Task bytes changed')
    for p in proposals:
        key=p['target_key'];r,pred=by_key[key];row=r['source'];k=standard['classes'][str(p['class_id'])]['standard_count'];proof=p['model_provenance'];constraint=p['class_keypoint_constraint']
        require(p['schema']=='updated1896_M2_class_standard_keypoint_preannotation_v1' and p['origin']=='updated_M2_prediction'
            and p['polygon']==pred['polygon'] and p['source']==row and p['vertex_count']==k and all(p.get(f) is False for f in FLAGS),'Model source/false flags/geometry differs')
        require(constraint['class_id']==p['class_id'] and constraint['standard_count']==k and constraint['standard_sha256']==manifest['class_standard_sha256'],'Per-target standard differs')
        require(proof['checkpoint_sha256']==manifest['checkpoint_sha256'] and proof['predictions_sha256']==manifest['predictions_sha256']
            and proof['native_mask_rle_sha256']==sha(pred['raw_native_prediction_rle']) and proof['corner_conversion']==pred['conversion']
            and proof['visual_inspection']==decisions['decisions'][key] and not proof['human_saved'] and not proof['assistant_redrawn'] and not proof['quality_gate_passed'],'Model/inspection provenance differs')
        require(p['model_source_image_sha256']==digest(ROOT/'images'/f'{key}.png')==r['image_sha256'] and p['pose_payload_raw_sha256']==r['pose_payload_raw_sha256'],'Raw source bytes differ')
        label=read(ROOT/'labelme'/f'{key}.json');points=[s for s in label['shapes'] if s['shape_type']=='point']
        require(label['num_keypoints']==len(points)==k and label['keypoints_xy']==label['shapes'][0]['points']==p['polygon']
            and [s['points'][0] for s in points]==p['polygon'] and [s['label'] for s in points]==[f'kp_{i:02d}' for i in range(1,k+1)]
            and label['imageWidth']==row['size'][0] and label['imageHeight']==row['size'][1]
            and not label['flags']['human_confirmed'] and not label['flags']['ground_truth_eligible'],'Explicit LabelMe points differ')
        require(np.array_equal(np.array(Image.open(ROOT/'masks'/f'{key}.png'))>0,raster_mask(p['polygon'],*row['size'])),'Mask export differs')
        image=Image.open(formal/row['image']).convert('RGB');draw=ImageDraw.Draw(image);vertices=[tuple(v) for v in p['polygon']]
        draw.line(vertices+[vertices[0]],fill='#55d5ff',width=1);box=row['bbox']
        left=max(0,math.floor(box['x']-12));top=max(0,math.floor(box['y']-16))
        right=min(row['size'][0],math.ceil(box['x']+box['width']+12));bottom=min(row['size'][1],math.ceil(box['y']+box['height']+16))
        expected=image.crop((left,top,right,bottom)).resize(((right-left)*3,(bottom-top)*3),Image.Resampling.NEAREST);draw=ImageDraw.Draw(expected)
        for i,(x,y) in enumerate(p['polygon'],1):
            xx=(x-left)*3;yy=(y-top)*3;draw.ellipse((xx-2,yy-2,xx+2,yy+2),fill='#ffbe43');draw.text((xx+3,yy-9),str(i),fill='white',stroke_width=1,stroke_fill='black')
        require(np.array_equal(np.asarray(expected),np.asarray(Image.open(ROOT/'cases'/f'{key}.png'))),'Numbered model overlay differs')
    require(manifest['class_counts']=={str(c):n for c,n in issued_counts.items()} and all(issued_counts[c]<=100-counts[c] for c in counts),'Quota exceeds gaps')
    require(manifest['conditional_gap_after_all_drafts_saved']=={str(c):max(0,100-counts[c]-issued_counts[c]) for c in range(1,23)},'Conditional gap is not actual completion')
    allowed={'report_zh.html','report-font.otf','FONT_LICENSE.txt','model-editor.js'}
    allowed.update(f'{folder}/{p["target_key"]}.{ext}' for p in proposals for folder,ext in [('images','png'),('cases','png'),('masks','png'),('labelme','json')])
    require(set(manifest['public_assets'])==allowed,'Unexpected public resource')
    for name,expected in manifest['public_assets'].items():
        path=ROOT/name;require(path.is_file() and not path.is_symlink() and path.resolve().is_relative_to(ROOT) and digest(path)==expected,'Public bytes/path differ')
    page=(ROOT/'report_zh.html').read_text();inventory=PageInventory(page)
    require(inventory.articles==[(f'target-{p["target_key"]}',str(p['class_id'])) for p in proposals]
        and inventory.options==['']+[str(c) for c in range(1,23)] and all(n in allowed for n in inventory.assets),'Page scope differs')
    replay=read(ROOT/'private/independent_model_replay.json')
    require(replay['state']=='all_fresh_native_masks_exactly_replayed_from_updated_M2' and replay['targets']==921
        and replay['predictions_sha256']==manifest['predictions_sha256'] and replay['checkpoint_sha256']==manifest['checkpoint_sha256'],'Independent updated model replay missing')
    source_state()
    receipt=dict(state='passed_updated_M2_sources_new_images_classK_exports_not_accuracy',manifest_sha256=digest(ROOT/'manifest.json'),
        public_assets=manifest['public_assets'],real_manual_saved_before=1896,request_count=len(proposals),class_counts=dict(issued_counts),
        native_model_masks_independently_replayed=921,keypoint_conversion_candidates=len(refs),human_events_created=0,
        old_data_models_worklists_and_human_history_unchanged=True,new_model_retrained=True,independent_accuracy_certified=False,
        class_standard_unchanged=True,production_model_activated=False,conversion_iou_not_human_accuracy_min=min(fidelity),code_sha256=digest(__file__))
    for name in ('verification.json','independent_verification.json'):write(ROOT/name,receipt)
    print(json.dumps({k:v for k,v in receipt.items() if k!='public_assets'}),flush=True)


if __name__=='__main__':
    p=argparse.ArgumentParser(description=__doc__);p.add_argument('--replay-model',action='store_true');a=p.parse_args()
    replay_model() if a.replay_model else verify()
