"""Independent completed337/source/native exports/coverage and final worker audit."""
from collections import Counter,defaultdict
from contextlib import closing
import argparse
import hashlib
import json
import math
from pathlib import Path
import sqlite3
import sys

import cv2
import numpy as np

PROJECT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(PROJECT))
from pallet.cluster_targets import digest
from pallet.mask_quota_saved_extension import load_formal_evidence,file_inventory
from pallet.mask_model_improvement_data import preservation
from pallet.pose_review_geometry import canonical,polygon_value,raster_mask
from pallet.mask_model_training_data import letterbox

ROOT=PROJECT/'outputs/pallet_mask_annotation_update_v1';OLD=PROJECT/'outputs/pallet_mask_annotation_full_v1'


def read(p):return json.loads(Path(p).read_text())
def lines(p):return [json.loads(s) for s in Path(p).read_text().splitlines() if s.strip()]
def sha(v):return hashlib.sha256(canonical(v)).hexdigest()
def raw_sha(raw):return hashlib.sha256(raw.encode('utf-8')).hexdigest()
def require(v,message):
    if not v:raise ValueError(message)
def write(p,v):
    with Path(p).open('xb') as f:f.write(canonical(v)+b'\n')


def preserved(formal):
    for name,expected in read(ROOT/'private/old_versions_inventory.json').items():require(file_inventory(PROJECT/name)==expected,'Old model/data/request/failed evidence changed: '+name)
    require(preservation(formal)==read(ROOT/'private/preservation_before.json'),'Formal/config/old static files changed')


def verify_dataset():
    from pycocotools import mask as mu
    m=read(ROOT/'dataset_manifest.json');records=lines(ROOT/'private/training_records.jsonl')
    require(m['records']==len(records)==1896 and m['components']==240 and m['all_labelled_components_train'] and not m['independent_validation'],'1896/all240 train contract differs')
    for name,expected in m['files'].items():require(digest(ROOT/name)==expected,'Immutable training artifact changed: '+name)
    formal,rows,_=load_formal_evidence(read(PROJECT/'outputs/pallet_mask_remaining_round5_v1/manifest.json'))
    binding=read(ROOT/'private/source_binding.json');source=PROJECT/binding['standard_snapshot']
    require(digest(source/'audit.json')==binding['standard_audit_sha256'],'Standard source snapshot changed')
    catalog={r['target_key']:r for r in lines(source/'private/source_catalog.jsonl')}
    old_records={r['target_key']:r for r in lines(OLD/'private/training_records.jsonl')}
    taskroot=PROJECT/'outputs/pallet_mask_model_requests_v3';proposals={p['target_key']:p for p in lines(taskroot/'proposals.jsonl')}
    require(digest(taskroot/'manifest.json')==binding['completed_task_manifest_sha256'],'Completed task binding differs')
    def snapshot(name,table):
        p=(ROOT/'private'/name).resolve()
        with closing(sqlite3.connect(p.as_uri()+'?mode=ro&immutable=1',uri=True)) as db:
            db.execute('PRAGMA query_only=ON')
            current={k:(r,raw) for k,r,raw in db.execute(f'SELECT target_key,revision,payload FROM {table}')}
            events=list(db.execute('SELECT id,target_key,revision,payload FROM events ORDER BY id'))
        return current,events
    current,events=snapshot('completed337_snapshot.sqlite3','reviews');isolated,isolated_events=snapshot('isolated_snapshot.sqlite3','reviews')
    require(len(current)==337 and set(current)==set(proposals) and len(events)==binding['cutoff_real_review_events']==381,'Exact337 saved/381 event cutoff required')
    require(len(catalog)==1552 and len(isolated)==7 and not set(catalog)&set(isolated) and not (set(catalog)|set(isolated))&set(current),'Human origin partitions overlap')
    histories=defaultdict(list);event_bindings=[]
    for event_id,key,revision,raw in events:
        v=json.loads(raw);p=proposals[key];row=rows[key];history=histories[key]
        identity={n:row[n] for n in ('target_key','frame_id','split','box_index','bbox','size','source_pcd_sha256')}
        identity.update(formal_manifest_sha256=p['formal_manifest_sha256'],image_sha256=digest(formal/row['image']))
        require(revision==len(history)+1==v['revision'] and v['target_key']==key and v['source']==identity and v['source_binding']==sha(identity),'Event relational/native identity differs')
        require(v['proposal_publication']=='model-class-keypoints-v3' and v['base_proposal_id']==sha(p)
            and v['worklist_manifest_sha256']==binding['completed_task_manifest_sha256'] and v['proposal_polygon']==p['polygon']
            and v['model_provenance']==p['model_provenance'] and v['event_kind']=='human_model_mask_review' and v['human_review_save_action'] is True,'Model337 exact namespace/proposal/target provenance differs')
        require(all(v[f] is False for f in ('ground_truth_eligible','automatic_training_eligible','robot_command_eligible','propagation_seed_eligible','human_mask_confirmed','human_pose_confirmed')),'Source flags or GT eligibility rewritten')
        polygon_value(v['polygon'],*row['size']);constraint=p['class_keypoint_constraint'];require(v['class_keypoint_constraint']==constraint,'Initial class standard was overwritten')
        edit=v.get('human_keypoint_editing');count=len(v['polygon']);previous=len(history[-1]['polygon']) if history else len(p['polygon'])
        if edit:
            expected=dict(policy='human_keypoint_fine_adjustment_add_remove_v1',standard_applies_to='initial_machine_proposal_only',
                initial_standard_count=constraint['standard_count'],actual_saved_count=count,delta_from_standard=count-constraint['standard_count'],
                standard_count_matches=count==constraint['standard_count'],previous_saved_or_initial_count=previous,point_count_changed_from_previous=count!=previous)
            require(edit==expected,'Human changed-count provenance differs')
        else:require(count==constraint['standard_count'],'Legacy save cannot have an unrecorded count exception')
        history.append(v);event_bindings.append(dict(event_id=event_id,target_key=key,revision=revision,payload_raw_utf8_sha256=raw_sha(raw)))
    require(event_bindings==lines(ROOT/'private/completed337_event_binding.jsonl'),'Raw UTF-8 event hash audit differs')
    references=dict(catalog)
    for key,(revision,raw) in isolated.items():
        v=json.loads(raw);require(v['polygon'] and key in old_records and old_records[key]['source_origin']=='saved_nonconforming_isolated','Seven prior approved isolated references missing')
        references[key]=dict(polygon=v['polygon'],revision=revision,event_raw_sha256=raw_sha(raw),payload_sha256=sha(v))
    for key,(revision,raw) in current.items():
        v=json.loads(raw);last=next(e for e in reversed(events) if e[1]==key)
        require(last[2:]==(revision,raw),'Current is not the exact final raw event')
        references[key]=dict(polygon=v['polygon'],revision=revision,event_raw_sha256=raw_sha(raw),payload_sha256=sha(v))
    coco=read(ROOT/'dataset/train/_annotations.coco.json');square=read(ROOT/'dataset_rfdetr/train/_annotations.coco.json')
    require(len(coco['images'])==len(coco['annotations'])==len(square['images'])==len(square['annotations'])==1896,'COCO/native image pairing differs')
    components=read(OLD/'private/global_components.json')['by_target'];frames=set();pixels=set();counts=Counter();kchanges=0
    for r,a,b,image in zip(records,coco['annotations'],square['annotations'],coco['images']):
        key=r['target_key'];row=rows[key];ref=references[key];width,height=row['size'];box=row['bbox']
        require(r['polygon']==ref['polygon'] and r['source_revision']==ref['revision'] and r['source_payload_sha256']==ref['payload_sha256']
            and r['source_event_raw_sha256']==ref['event_raw_sha256'],'Actual human supervision/payload/event revision differs')
        dx=max(48,.2*box['width']);dy=max(16,.6*box['height'])
        roi=[max(0,math.floor(box['x']-dx)),max(0,math.floor(box['y']-dy)),min(width,math.ceil(box['x']+box['width']+dx)),min(height,math.ceil(box['y']+box['height']+dy))]
        require(r['roi']==roi and r['development_split']=='train' and r['component']==components[key],'Label oracle crop/split or component differs')
        x1,y1,x2,y2=roi;grey=cv2.imread(str(formal/row['image']),0)[y1:y2,x1:x2]
        full=raster_mask(ref['polygon'],width,height);mask=full[y1:y2,x1:x2]
        require(full.sum()==mask.sum() and mask.any(),'Human foreground clipped')
        def decode(ann):
            v=ann['segmentation'];return mu.decode(dict(size=v['size'],counts=v['counts'].encode())).astype(bool)
        require(np.array_equal(grey,cv2.imread(str(ROOT/r['image']),0)) and np.array_equal(mask,decode(a))
            and np.array_equal(mask,cv2.imread(str(ROOT/r['native_mask']),0)>0),'Native pixels/human masks/RLE differ')
        valid=cv2.imread(str(ROOT/r['valid_mask']),0)>0;require(valid[mask].all(),'Human positives erased by neighbor ignore')
        canvas,label,_,_=letterbox(grey,mask.astype(np.uint8),(576,576))
        require(np.array_equal(label>0,decode(b)) and np.array_equal(canvas,cv2.imread(str(ROOT/'dataset_rfdetr/train'/image['file_name']),0)),'Letterbox supervision mapping differs')
        h=digest(formal/row['image']);require(row['frame_id'] not in frames and h not in pixels,'Repeated distinct human count image')
        frames.add(row['frame_id']);pixels.add(h);counts[row['final_class_id']]+=1
        if key in current and len(ref['polygon'])!=proposals[key]['class_keypoint_constraint']['standard_count']:kchanges+=1
    require(kchanges==10 and len(counts)==22 and len({r['component'] for r in records})==240,'Actual edited counts/all22/all240 lost')
    require(digest(ROOT/'class_keypoint_standard.json')==digest(taskroot/'class_keypoint_standard.json'),'Class standard was automatically recomputed')
    preserved(formal)
    return dict(state='passed1896_exact_human_sources_all240components_exports_not_independent_accuracy',records=1896,reviewed337_events=381,
        classes=dict(counts),components=240,human_changed_count_masks_preserved=10,all_train=True,new_holdout_evaluation=False,
        old_artifacts_unchanged=True,dataset_manifest_sha256=digest(ROOT/'dataset_manifest.json'))


def finish():
    import torch
    from pycocotools import mask as mu
    from pallet.mask_model_training import native_metrics,summarize
    data=verify_dataset();plan=read(ROOT/'execution_plan.json');records={r['target_key']:r for r in lines(ROOT/'private/training_records.jsonl')};runs={}
    for route in ('m1','m2'):
        out=ROOT/route;require(not (out/'failure.json').exists() and (out/'completion.json').is_file(),'Worker incomplete or failed: '+route)
        completion=read(out/'completion.json');require(completion['checkpoint_sha256']==digest(out/'final_ema.pt'),'Worker checkpoint SHA differs')
        checkpoint=torch.load(out/'final_ema.pt',map_location='cpu',weights_only=True)
        require(checkpoint['step']==completion['actual_optimizer_updates']==plan['optimizer_updates'][route]
            and checkpoint['parent_EMA_sha256']==plan['starting_EMA_sha256'][route],'Warm start/budget checkpoint receipt differs');del checkpoint
        exposure=read(out/'exposure_audit.json');require(set(exposure['by_target'])==set(records) and min(exposure['by_target'].values())>=10
            and len(exposure['by_component'])==240,'Any human image/component not trained')
        for epoch in range(1,11):
            e=read(out/'epochs'/f'{epoch:03d}.json');require(e['unique_images_exposed']==1896 and e['unique_components_exposed']==240,'Per-epoch exposure missing')
        predictions=lines(out/'fit_predictions.jsonl');require(len(predictions)==1896 and {p['target_key'] for p in predictions}==set(records),'Fit diagnostic denominator differs')
        for p in predictions:
            rle=p['native_prediction_rle'];prediction=mu.decode(dict(size=rle['size'],counts=rle['counts'].encode())).astype(bool)
            target=cv2.imread(str(ROOT/records[p['target_key']]['native_mask']),0)>0;metrics=native_metrics(prediction,target)
            require(all(v==p[k] if v is None or isinstance(v,bool) else abs(v-p[k])<1e-10 for k,v in metrics.items()),'Native training-fit RLE metric replay differs')
            require(p['trained_on_this_image'] and not p['blind'],'Fit diagnostics falsely marked independent')
        runs[route]=dict(checkpoint_sha256=completion['checkpoint_sha256'],steps=completion['actual_optimizer_updates'],
            training_fit_only=summarize(predictions),all1896_images_and240components_exposed=True,gpu=completion['gpu'])
    result=dict(state='verified_both_completed337_warm_updates_all1896_not_quality_release',dataset=data,runs=runs,
        dataset_manifest_sha256=digest(ROOT/'dataset_manifest.json'),next_annotation_primary_route='m2',
        independent_accuracy=None,production_model_activated=False,all_new_predictions_require_human_review=True,errors=[],code_sha256=digest(__file__))
    write(ROOT/'TRAINING_HANDOFF.json',result);print(json.dumps(result),flush=True)


if __name__=='__main__':
    parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('--finish',action='store_true');args=parser.parse_args()
    if args.finish:finish()
    else:
        result=verify_dataset();write(ROOT/'independent_dataset_verification.json',result);print(json.dumps(result),flush=True)
