"""Independent source/export/protection and real checkpoint replay; no producer import."""
from collections import Counter,defaultdict
from contextlib import closing
import argparse
import hashlib
import json
import math
from pathlib import Path
import re
import sqlite3
import sys

import cv2
import numpy as np
from PIL import Image,ImageDraw
from pycocotools import mask as mu

PROJECT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(PROJECT))
from pallet.cluster_targets import digest,protected_snapshot
from pallet.pose_review_geometry import canonical,polygon_value,raster_mask
from pallet.mask_quality_recovery import fingerprint,POSE,GALLERY
from pallet.mask_quota_saved_extension import load_formal_evidence,file_inventory
from pallet.mask_remaining_classes import confirmed,original_rectangle
from pallet.mask_template_refined import QUARANTINE,readonly
from scripts.verify_mask_under50_requests import native_quality,PageInventory

ROOT=PROJECT/'outputs/pallet_mask_model_requests_v1';MODEL=PROJECT/'outputs/pallet_mask_annotation_full_v1'
SOURCE=PROJECT/'outputs/pallet_mask_continuation_sources_v16'
CHECKPOINT='92ce35476d4b0887a0e791dc20972c9640aab558ec876df84296b713ebe8ed4c'
FALSE=('human_edited','human_mask_confirmed','human_pose_confirmed','ground_truth_eligible','automatic_training_eligible',
    'robot_command_eligible','production_saved','counts_as_completed_annotation','accepted_candidate','propagation_seed_eligible')


def read(p):return json.loads(Path(p).read_text())
def lines(p):return [json.loads(s) for s in Path(p).read_text().splitlines() if s.strip()]
def sha(v):return hashlib.sha256(canonical(v)).hexdigest()
def raw_sha(v):return hashlib.sha256(v.encode('utf-8')).hexdigest() if v is not None else None
def require(v,message):
    if not v:raise ValueError(message)
def write(p,v):
    with Path(p).open('xb') as stream:stream.write(canonical(v)+b'\n')
def index(rs):
    result={r['target_key']:r for r in rs};require(len(result)==len(rs),'Duplicate target key');return result
def decode(rle):return mu.decode(dict(size=rle['size'],counts=rle['counts'].encode('ascii'))).astype(bool)


def roi(row):
    b=row['bbox'];w,h=row['size'];dx=max(48,.2*b['width']);dy=max(16,.6*b['height'])
    return [max(0,math.floor(b['x']-dx)),max(0,math.floor(b['y']-dy)),min(w,math.ceil(b['x']+b['width']+dx)),min(h,math.ceil(b['y']+b['height']+dy))]


def database(path,table):
    with closing(readonly(path)) as db:
        current=dict(db.execute(f'SELECT target_key,payload FROM {table}'));history=defaultdict(list)
        for event_id,key,revision,raw in db.execute('SELECT id,target_key,revision,payload FROM events ORDER BY id'):
            history[key].append((event_id,revision,raw))
    for key,raw in current.items():
        require(key in history and history[key][-1][2]==raw,'Current must equal the last raw event')
        require([e[1] for e in history[key]]==list(range(1,len(history[key])+1)),'History revisions are not contiguous')
    return current,history


def source_state():
    inputs=read(ROOT/'input_manifest.json');audit=read(SOURCE/'audit.json')
    require(digest(SOURCE/'audit.json')==inputs['source_audit_sha256'],'Source audit changed')
    require(digest(SOURCE/'private/source_catalog.jsonl')==audit['source_catalog_sha256'],'Source catalog changed')
    isolated=PROJECT/'outputs/pallet_mask_nonconforming_reviews_v1/mask_reviews.sqlite3'
    fps=dict(pose=fingerprint(POSE),gallery=fingerprint(GALLERY),isolated=fingerprint(isolated))
    require(fps==inputs['source_fingerprints'],'Read-only source rows/schema/history changed since staging')
    require(file_inventory(PROJECT/'configs')==inputs['configuration_sha256'],'Old online/global configuration changed')
    formal,rows,evidence=load_formal_evidence(read(PROJECT/'outputs/pallet_mask_remaining_round5_v1/manifest.json'))
    require(protected_snapshot(formal)==inputs['formal_protected'],'Formal frames/labels/GT/metadata changed')
    current,hist=database(POSE,'annotations');gallery,ghist=database(GALLERY,'reviews');nc,nhist=database(isolated,'reviews')
    catalog=lines(SOURCE/'private/source_catalog.jsonl');sources=index(catalog);training=lines(MODEL/'private/training_records.jsonl')
    pose_refs=index(lines(PROJECT/'outputs/pallet_mask_remaining_round5_v1/all_manual_sources.jsonl'))
    require(len(sources)==1552 and len(nc)==7 and not sources.keys()&nc.keys(),'1552+7 distinct human source keys required')
    for key,r in sources.items():
        raw=(current if r['origin']=='saved_pose' else gallery)[key];value=json.loads(raw)
        require(r['polygon']==value['polygon'] and r['revision']==value['revision'] and r['payload_sha256']==sha(value),
            'Source not the actual saved manual revision')
        history=(hist if r['origin']=='saved_pose' else ghist)[key]
        if r['origin']=='saved_pose':
            # The actual manual polygon event can precede a unit/axis-only
            # revision. Its raw hash is intentionally NOT the latest-event hash.
            ref=pose_refs[key];events=[e for e in history if e[0]==ref['human_event_id']]
            require(len(events)==1 and events[0][2]==ref['human_event_payload_raw']
                and json.loads(events[0][2])['polygon']==r['polygon']
                and json.loads(events[0][2]).get('event_kind') not in ('unit_normalization','axes_normalization','mask_template_proposal'),
                'Actual pose manual polygon event differs')
            require(r['event_raw_sha256']==raw_sha(events[0][2]),'Pose manual event hash not raw UTF-8')
        else:require(r['event_raw_sha256']==raw_sha(history[-1][2]),'Gallery latest event hash not raw UTF-8')
    require(set(nc)=={r['target_key'] for r in training if r['source_origin']=='saved_nonconforming_isolated'},'Seven authorized redraws differ')
    for raw in nc.values():
        value=json.loads(raw);require(value['quality_decision']=='redrawn_pending_quality_check' and value['polygon'],
            'Do not rewrite the historical isolated source flags')
    counts=Counter(rows[k]['final_class_id'] for k in set(sources)|set(nc))
    require(sum(counts.values())==1559 and dict((str(c),n) for c,n in counts.items())==inputs['actual_class_counts'],'Actual human counts differ')
    protected=set(QUARANTINE)|set(sources)|set(nc)|{r['target_key'] for r in training}
    protected.update(r['target_key'] for r in lines(PROJECT/'outputs/pallet_mask_remaining_round5_v1/proposals.jsonl'))
    for k in set(current)|set(hist):
        values=[json.loads(e[2]) for e in hist[k]]+([json.loads(current[k])] if k in current else [])
        if k not in current or any(v.get('state') in ('deleted','deferred','confirmed') or confirmed(v) or not original_rectangle(v.get('polygon'),rows[k]) for v in values):protected.add(k)
    inventories={}
    for pattern in ('requests.jsonl','shortlist.jsonl','inspection_targets.jsonl'):
        for path in sorted((PROJECT/'outputs').rglob(pattern)):
            if path.is_relative_to(ROOT):continue
            inventories[str(path.relative_to(PROJECT))]=digest(path)
            protected.update(r['target_key'] for r in lines(path) if r.get('target_key') in rows)
    require(inventories==inputs['prior_worklist_inventories_sha256'],'Any prior request/shortlist/inspection file changed')
    frames={rows[k]['frame_id'] for k in protected};hashes={digest(formal/name) for name in {rows[k]['image'] for k in protected}}
    return inputs,formal,rows,evidence,current,protected,frames,hashes,counts


def replay_model():
    import torch
    from rfdetr import RFDETRSegSmall
    from pallet.mask_model_training_data import letterbox,gpu_receipt
    from pallet.mask_model_improvement_adapter import highest_class_query,native_logits,mask_logits
    path=ROOT/'private/independent_model_replay.json';require(not path.exists(),'Preserve completed independent GPU replay')
    require(digest(MODEL/'m2/final_ema.pt')==CHECKPOINT,'Checkpoint pin differs')
    receipt=gpu_receipt('m2');torch.set_num_threads(4);cv2.setNumThreads(1)
    weights=torch.load(MODEL/'m2/final_ema.pt',map_location='cpu',weights_only=True)
    wrapper=RFDETRSegSmall(num_classes=1,resolution=576,device='cuda:0',pretrain_weights=str(PROJECT/'outputs/pallet_mask_model_training_v1/generic_pretraining/rf-detr-seg-small.pt'),gradient_checkpointing=True,compile=False)
    network=wrapper.model.model;network.load_state_dict(weights['model'],strict=True);network.cuda().eval();del weights
    formal=Path(read(PROJECT/'configs/active_formal_dataset.json')['dataset']);records=lines(ROOT/'private/shortlist.jsonl')
    predictions=index(lines(ROOT/'private/predictions.jsonl'));matched=[]
    with torch.inference_mode():
        for number,r in enumerate(records,1):
            row=r['source'];bounds=roi(row);x1,y1,x2,y2=bounds
            pixels=cv2.imread(str(formal/row['image']),cv2.IMREAD_GRAYSCALE)[y1:y2,x1:x2]
            canvas,_,_,transform=letterbox(pixels,None,(576,576))
            rgb=np.repeat(canvas[None].astype(np.float32)/255,3,axis=0)
            value=torch.from_numpy((rgb-np.array([.485,.456,.406],np.float32)[:,None,None])/np.array([.229,.224,.225],np.float32)[:,None,None])
            output=network(value[None].cuda());query,score=highest_class_query(output)
            mask=native_logits(mask_logits(output)[0,query],transform).sigmoid().cpu().numpy()>=.5
            p=predictions[r['target_key']]
            require(bounds==p['roi'] and query==p['selected_query'] and np.array_equal(mask,decode(p['raw_native_prediction_rle'])),
                'Actual independent GPU image-only forward differs: '+r['target_key'])
            require(abs(score-p['class_confidence_not_quality_probability'])<1e-7,'Classification score differs')
            matched.append(r['target_key'])
            if number%200==0:print('Independent exact native RLE replay',number,flush=True)
    write(path,dict(state='all908_exact_native_masks_reproduced_from_pinned_M2_checkpoint',targets=len(matched),
        predictions_sha256=digest(ROOT/'private/predictions.jsonl'),shortlist_sha256=digest(ROOT/'private/shortlist.jsonl'),
        checkpoint_sha256=CHECKPOINT,gpu=receipt,target_keys_sha256=sha(matched),
        human_masks_features_or_query_selection_used=False,training_run_started=False,quality_certified=False,code_sha256=digest(__file__)))
    print('Independent checkpoint replay complete:',len(matched),flush=True)


def verify():
    require(not (ROOT/'verification.json').exists() and not (ROOT/'independent_verification.json').exists(),'Never overwrite a sealed package')
    inputs,formal,rows,evidence,current,protected,frames,hashes,counts=source_state()
    manifest=read(ROOT/'manifest.json');staged=lines(ROOT/'private/shortlist.jsonl');predictions=lines(ROOT/'private/predictions.jsonl')
    require(digest(ROOT/'private/shortlist.jsonl')==inputs['shortlist_sha256'],'Stage bytes differ')
    require(len(staged)==len(predictions)==908 and index(staged).keys()==index(predictions).keys(),'Exact908 source/prediction pairing required')
    require(manifest['input_manifest_sha256']==digest(ROOT/'input_manifest.json') and manifest['predictions_sha256']==digest(ROOT/'private/predictions.jsonl'), 'Input/output hash differs')
    require(manifest['checkpoint_sha256']==inputs['checkpoint_sha256']==digest(MODEL/'m2/final_ema.pt')==CHECKPOINT,'Checkpoint pin differs')
    final=read(MODEL/'FINAL_HANDOFF.json')
    require(final['state']=='verified_both_fullpool_training_completed_no_quality_release' and final['errors']==[]
        and final['production_activated'] is False,'Old model training release/activation state differs')
    for name,expected in final['runs'].items():require(digest(MODEL/name/'final_ema.pt')==expected['checkpoint_sha256'],'Old completed model changed')
    decisions=read(ROOT/'private/visual_decisions.json');contact=read(ROOT/'private/coarse_contact_sheets/inventory.json')
    require(decisions['predictions_sha256']==manifest['predictions_sha256'] and decisions['contact_sheet_inventory_sha256']==digest(ROOT/'private/coarse_contact_sheets/inventory.json'), 'Visual input binding differs')
    require(set(decisions['decisions'])==index(staged).keys() and contact['predictions_sha256']==manifest['predictions_sha256'],'Foreign/missing visual decision')
    sheetrefs={}
    for sheet in contact['pages']:
        require(digest(PROJECT/sheet['path'])==sheet['sha256'],'Contact sheet changed')
        for k,i in zip(sheet['target_keys'],sheet['indices']):sheetrefs[k]=(sheet['path'],sheet['sha256'],i)
    require(len(sheetrefs)==803 and len(contact['pages'])==42,'All803 coarse-screened masks /42 actual sheets required')
    prediction_map=index(predictions);staged_map=index(staged);seenframes=set();seenhash=set();derived=[];nonempty=0
    for order,r in enumerate(staged,1):
        key=r['target_key'];row=rows[key];p=prediction_map[key];decision=decisions['decisions'][key]
        require(re.fullmatch('[0-9a-f]{24}',key) and r['source']==row and r['class_id']==row['final_class_id'] and r['request_order']==order,'Source identity/order differs')
        require(counts[r['class_id']]<100 and key not in protected and row['frame_id'] not in frames,'Protected or quota-complete source staged')
        h=digest(formal/row['image']);require(h==r['image_sha256'] and h not in hashes and row['frame_id'] not in seenframes and h not in seenhash,'Repeated source frame/pixels')
        seenframes.add(row['frame_id']);seenhash.add(h)
        require(native_quality(row,evidence)==r['native_quality'],'Unchanged native source gate differs')
        require(r['pose_payload_raw_sha256']==raw_sha(current.get(key)) and r['gallery_payload_raw_sha256'] is None,'Current raw target binding differs')
        require(p['roi']==roi(row) and p['class_id']==row['final_class_id'] and p['checkpoint_sha256']==CHECKPOINT and p['model_threshold']==.5,'Inference context/features/checkpoint differs')
        raw=decode(p['raw_native_prediction_rle']);x1,y1,x2,y2=p['roi']
        require(raw.shape==(y2-y1,x2-x1) and int(raw.sum())==p['raw_mask_area'],'Raw native RLE shape/area differs')
        require(type(decision['publish_as_machine_draft']) is bool and decision['human_QC_pass'] is False and decision['quality_certified'] is False and decision['model_geometry_modified'] is False,'False human/quality or geometry claim')
        if p['polygon']:
            polygon_value(p['polygon'],*row['size']);nonempty+=1
            require(key in sheetrefs and decision['coarse_crop_inspected'] is True and decision['entire_native_image_individually_inspected'] is False,'Coarse inspection scope overstated')
            require(sheetrefs[key]==(decision['contact_sheet'],decision['contact_sheet_sha256'],decision['candidate_index']),'Coarse contact image binding differs')
            contours,hierarchy=cv2.findContours(raw.astype(np.uint8),cv2.RETR_CCOMP,cv2.CHAIN_APPROX_SIMPLE)
            require(cv2.connectedComponents(raw.astype(np.uint8),8)[0]==2 and hierarchy is not None and not (hierarchy[0,:,3]>=0).any(), 'Disconnected/closed-hole raw topology was silently changed')
            local=[[x-x1,y-y1] for x,y in p['polygon']];rendered=raster_mask(local,raw.shape[1],raw.shape[0])
            iou=float((raw&rendered).sum()/(raw|rendered).sum())
            require(iou>=.99 and abs(iou-p['polygonization']['polygon_vs_model_mask_iou'])<1e-12,'Polygon no longer faithfully represents the native model mask')
        else:require(not decision['publish_as_machine_draft'] and p['polygonization']['reason'],'Invalid model geometry issued')
        if decision['publish_as_machine_draft']:derived.append(key)
    proposals=lines(ROOT/'proposals.jsonl');requests=lines(ROOT/'requests.jsonl');issued=index(proposals)
    require([r['target_key'] for r in proposals]==[r['target_key'] for r in requests]==derived and len(proposals)==manifest['request_count']==588,'Exact decision/request/proposal pairing differs')
    require(digest(ROOT/'proposals.jsonl')==manifest['proposals_sha256'] and digest(ROOT/'requests.jsonl')==manifest['requests_sha256'],'Public source hash differs')
    issued_counts=Counter(p['class_id'] for p in proposals)
    require({str(c):n for c,n in issued_counts.items()}==manifest['class_counts'] and all(n<=100-counts[c] for c,n in issued_counts.items()),'Machine work exceeds remaining gap')
    require(sum(counts.values())==1559 and len(issued_counts)==16 and 19 not in issued_counts,'Manual count/safe Class19 capacity misstated')
    require(manifest['conditional_gap_after_all_drafts_saved']=={str(c):max(0,100-counts[c]-issued_counts[c]) for c in range(1,23)},'Conditional remaining is not real completion')
    allowed={'report_zh.html','report-font.otf','FONT_LICENSE.txt','model-editor.js'}
    allowed.update(f'{d}/{k}.{e}' for k in issued for d,e in [('images','png'),('cases','png'),('masks','png'),('labelme','json')])
    require(set(manifest['public_assets'])==allowed,'Private/unexpected public asset')
    for name,expected in manifest['public_assets'].items():
        path=ROOT/name;require(path.is_file() and not path.is_symlink() and path.resolve().is_relative_to(ROOT) and digest(path)==expected,'Asset hash/path differs')
    for key,p in issued.items():
        row=rows[key];prediction=prediction_map[key];r=staged_map[key]
        require(p['schema']=='model_native_mask_preannotation_v1' and p['origin']=='model_prediction' and p['status']=='model_preannotated_pending_human_review'
            and all(p.get(f) is False for f in FALSE) and p['source']==row and p['polygon']==prediction['polygon'] and p['vertex_count']==len(p['polygon']), 'False manual/GT/template source or altered model geometry')
        proof=p['model_provenance'];require(proof['checkpoint_sha256']==CHECKPOINT and proof['predictions_sha256']==manifest['predictions_sha256']
            and proof['native_mask_rle_sha256']==sha(prediction['raw_native_prediction_rle']) and proof['human_saved'] is False and proof['assistant_redrawn'] is False
            and proof['threshold']==.5 and proof['visual_inspection']==decisions['decisions'][key] and proof['quality_gate_passed'] is False,'Model origin or quality attribution differs')
        require(p['formal_manifest_sha256']==inputs['formal_manifest_sha256'] and p['pose_payload_raw_sha256']==r['pose_payload_raw_sha256']
            and p['model_source_image_sha256']==digest(ROOT/'images'/f'{key}.png')==r['image_sha256'],'Original formal/image/target binding differs')
        native=np.asarray(Image.open(formal/row['image']).convert('RGB'));require(np.array_equal(native,np.asarray(Image.open(ROOT/'images'/f'{key}.png').convert('RGB'))),'Original pixels changed')
        mask=raster_mask(p['polygon'],*row['size']);require(np.array_equal(mask,np.asarray(Image.open(ROOT/'masks'/f'{key}.png'))>0),'PNG export differs')
        label=read(ROOT/'labelme'/f'{key}.json');require(label['shapes'][0]['points']==p['polygon'] and label['imageWidth']==row['size'][0] and label['imageHeight']==row['size'][1]
            and label['model_checkpoint_sha256']==CHECKPOINT and label['flags']['human_confirmed'] is False and label['flags']['ground_truth_eligible'] is False,'LabelMe promoted or geometry changed')
        image=Image.fromarray(native);points=[tuple(v) for v in p['polygon']];ImageDraw.Draw(image).line(points+[points[0]],fill='#55d5ff',width=1)
        b=row['bbox'];l=max(0,math.floor(b['x']-12));t=max(0,math.floor(b['y']-16));rr=min(row['size'][0],math.ceil(b['x']+b['width']+12));bt=min(row['size'][1],math.ceil(b['y']+b['height']+16))
        expected=image.crop((l,t,rr,bt)).resize(((rr-l)*3,(bt-t)*3),Image.Resampling.NEAREST)
        require(np.array_equal(np.asarray(expected),np.asarray(Image.open(ROOT/'cases'/f'{key}.png'))),'Preview is not the exact unedited model overlay')
    page=(ROOT/'report_zh.html').read_text();inventory=PageInventory(page)
    require(inventory.articles==[(f'target-{p["target_key"]}',str(p['class_id'])) for p in proposals] and inventory.options==['']+[str(c) for c in range(1,23)],'Page target/class filters differ')
    require(all(n in allowed for n in inventory.assets) and all(t not in page for t in ('access_key','payload_raw','selected_faces','ruler')),'Private raw fields or assets exposed')
    replay=read(ROOT/'private/independent_model_replay.json')
    require(replay['state']=='all908_exact_native_masks_reproduced_from_pinned_M2_checkpoint' and replay['targets']==908 and replay['predictions_sha256']==manifest['predictions_sha256']
        and replay['shortlist_sha256']==inputs['shortlist_sha256'] and replay['checkpoint_sha256']==CHECKPOINT,'Actual independent checkpoint replay missing')
    require(source_state()[0]==inputs,'Source protections changed during verification')
    receipt=dict(state='passed_model_sources_exports_protection_not_accuracy',manifest_sha256=digest(ROOT/'manifest.json'),public_assets=manifest['public_assets'],
        real_human_contours_before=1559,request_count=588,machine_native_masks_replayed=908,polygon_representations_checked=803,
        class_counts=dict(issued_counts),conditional_remaining_after_all_human_saves=manifest['conditional_gap_after_all_drafts_saved'],
        independent_model_replay_sha256=digest(ROOT/'private/independent_model_replay.json'),visual_decisions_sha256=digest(ROOT/'private/visual_decisions.json'),
        source_fingerprints=inputs['source_fingerprints'],formal_and_old_configs_unchanged=True,prior_worklists_unchanged=True,
        automatic_GT_or_training_or_robot_eligibility=False,new_human_events_written=0,independent_accuracy_certified=False,
        production_model_activated=False,new_model_trained=False,code_sha256=digest(__file__))
    for name in ('verification.json','independent_verification.json'):write(ROOT/name,receipt)
    print(json.dumps({k:v for k,v in receipt.items() if k!='public_assets'}),flush=True)


if __name__=='__main__':
    parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('--replay-model',action='store_true');args=parser.parse_args()
    if args.replay_model:replay_model()
    else:verify()
