"""Independent real-source, full native supervision and actual completed145 training."""
from collections import Counter,defaultdict
from contextlib import closing
import argparse
import hashlib
import json
import math
from pathlib import Path
import sqlite3
import sys

import cv2
import numpy as np

PROJECT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(PROJECT))
from pallet.cluster_targets import digest
from pallet.mask_quota_saved_extension import load_formal_evidence,file_inventory
from pallet.mask_model_improvement_data import preservation
from pallet.mask_quality_recovery import fingerprint
from pallet.pose_review_geometry import canonical,polygon_value,raster_mask
from pallet.mask_model_training_data import letterbox

ROOT=PROJECT/'outputs/pallet_mask_annotation_update_v2';OLD=PROJECT/'outputs/pallet_mask_annotation_update_v1'


def read(p):return json.loads(Path(p).read_text())
def lines(p):return [json.loads(s) for s in Path(p).read_text().splitlines() if s.strip()]
def sha(v):return hashlib.sha256(canonical(v)).hexdigest()
def raw_sha(raw):return hashlib.sha256(raw.encode('utf-8')).hexdigest()
def require(v,message):
    if not v:raise ValueError(message)
def write(p,v):
    with Path(p).open('xb') as stream:stream.write(canonical(v)+b'\n')


def database(name,table):
    path=(ROOT/'private'/f'{name}_snapshot.sqlite3').resolve()
    with closing(sqlite3.connect(path.as_uri()+'?mode=ro&immutable=1',uri=True)) as db:
        db.execute('PRAGMA query_only=ON');current={k:(r,raw) for k,r,raw in db.execute(f'SELECT target_key,revision,payload FROM {table}')}
        events=list(db.execute('SELECT id,target_key,revision,payload FROM events ORDER BY id'))
    history=defaultdict(list)
    for i,k,r,raw in events:history[k].append((i,r,raw))
    for key,(revision,raw) in current.items():require(history[key][-1][1:]==(revision,raw),'Snapshot current is not exact final raw event')
    return current,events,history


def preserved(formal):
    for name,h in read(ROOT/'private/old_versions_inventory.json').items():require(file_inventory(PROJECT/name)==h,'Old static/model/data/failed evidence changed: '+name)
    require(preservation(formal)==read(ROOT/'private/preservation_before.json'),'Formal/config/GT/static source changed')


def verify_dataset():
    from pycocotools import mask as mu
    manifest=read(ROOT/'dataset_manifest.json');records=lines(ROOT/'private/training_records.jsonl');binding=read(ROOT/'private/source_binding.json')
    require(manifest['records']==len(records)==2041 and manifest['components']==246 and not manifest['independent_validation']
        and manifest['splits']=={'train':2041},'All2041 human images /246components train required')
    for name,h in manifest['files'].items():require(digest(ROOT/name)==h,'Frozen training file changed: '+name)
    formal,rows,_=load_formal_evidence(read(PROJECT/'outputs/pallet_mask_remaining_round5_v1/manifest.json'))
    source=PROJECT/binding['standard_snapshot'];source_audit=read(source/'audit.json');standard=lines(source/'private/source_catalog.jsonl')
    require(len(standard)==1552 and digest(source/'audit.json')==binding['standard_audit_sha256']
        and digest(source/'private/source_catalog.jsonl')==binding['standard_catalog_sha256']==source_audit['source_catalog_sha256'],'Fresh standard source/audit binding differs')
    pose,_,pose_events=database('pose','annotations');gallery,_,gallery_events=database('gallery','reviews');isolated,_,_=database('isolated','reviews')
    require(fingerprint(ROOT/'private/pose_snapshot.sqlite3')==binding['pose_fingerprint']==source_audit['pose_fingerprint']
        and fingerprint(ROOT/'private/gallery_snapshot.sqlite3')==binding['gallery_fingerprint']==source_audit['gallery_fingerprint']
        and fingerprint(ROOT/'private/isolated_snapshot.sqlite3')==binding['isolated_fingerprint'],'Snapshot fingerprints differ')
    refs={}
    for r in standard:
        key=r['target_key'];current,history=(pose,pose_events) if r['origin']=='saved_pose' else (gallery,gallery_events)
        revision,raw=current[key];v=json.loads(raw)
        require(r['polygon']==v['polygon'] and r['revision']==revision and r['payload_sha256']==sha(v),'Actual latest standard human polygon differs')
        matches=[e for e in history[key] if raw_sha(e[2])==r['event_raw_sha256']]
        require(len(matches)==1 and json.loads(matches[0][2])['polygon']==r['polygon'],'Manual source raw UTF-8 event lost')
        if r['origin']!='saved_pose':require(matches[0]==history[key][-1],'Gallery must use latest human event')
        refs[key]=r
    previous={r['target_key']:r for r in lines(OLD/'private/training_records.jsonl')}
    require(len(isolated)==7,'Seven previously approved isolated sources required')
    for key,(revision,raw) in isolated.items():
        v=json.loads(raw);require(key in previous and previous[key]['source_origin']=='saved_nonconforming_isolated'
            and v['polygon'] and v['quality_decision']=='redrawn_pending_quality_check','Do not rewrite inherited isolated QC/source flags')
        refs[key]=dict(polygon=v['polygon'],revision=revision,payload_sha256=sha(v),event_raw_sha256=raw_sha(raw))
    batches=[];changed_counts={}
    for b in binding['batches']:
        name=b['name'];tasks=PROJECT/b['tasks'];proposals={p['target_key']:p for p in lines(tasks/'proposals.jsonl')}
        current,events,hist=database(name,'reviews');require(len(current)==b['distinct'] and set(current)==set(proposals),'Incomplete real saved task batch')
        require(digest(tasks/'manifest.json')==b['task_manifest_sha256'] and fingerprint(ROOT/'private'/f'{name}_snapshot.sqlite3')==b['snapshot_fingerprint'],'Model task or raw snapshot changed')
        namespace=read(tasks/'manifest.json')['source_namespace'];seen=Counter();raw_bindings=[];kchanged=0
        for event_id,key,revision,raw in events:
            v=json.loads(raw);p=proposals[key];row=rows[key];seen[key]+=1
            identity={n:row[n] for n in ('target_key','frame_id','split','box_index','bbox','size','source_pcd_sha256')}
            identity.update(formal_manifest_sha256=p['formal_manifest_sha256'],image_sha256=digest(formal/row['image']))
            require(v['target_key']==key and revision==seen[key]==v['revision'] and v['source']==identity and v['source_binding']==sha(identity)
                and v['base_proposal_id']==sha(p) and v['proposal_publication']==namespace and v['worklist_manifest_sha256']==b['task_manifest_sha256']
                and v['model_provenance']==p['model_provenance'] and v['proposal_polygon']==p['polygon']
                and v['event_kind']=='human_model_mask_review' and v['human_review_save_action'] is True,'Model real-event identity/proposal/provenance differs')
            require(all(v[f] is False for f in ('ground_truth_eligible','automatic_training_eligible','robot_command_eligible','propagation_seed_eligible','human_mask_confirmed','human_pose_confirmed')),'Source flags automatically promoted')
            polygon_value(v['polygon'],*row['size']);constraint=p['class_keypoint_constraint'];require(v['class_keypoint_constraint']==constraint,'Class standard rewritten')
            count=len(v['polygon']);edit=v.get('human_keypoint_editing');previous_count=len(json.loads(hist[key][revision-2][2])['polygon']) if revision>1 else len(p['polygon'])
            if edit:
                require(edit==dict(policy='human_keypoint_fine_adjustment_add_remove_v1',standard_applies_to='initial_machine_proposal_only',
                    initial_standard_count=constraint['standard_count'],actual_saved_count=count,delta_from_standard=count-constraint['standard_count'],
                    standard_count_matches=count==constraint['standard_count'],previous_saved_or_initial_count=previous_count,
                    point_count_changed_from_previous=count!=previous_count),'Human added/removed-count history differs')
            else:require(count==constraint['standard_count'],'Legacy untraced count change')
            raw_bindings.append(dict(event_id=event_id,target_key=key,revision=revision,payload_raw_utf8_sha256=raw_sha(raw)))
        require(raw_bindings==lines(ROOT/'private'/f'{name}_event_binding.jsonl') and len(events)==b['events'],'Raw UTF-8 history audit differs')
        for key,(revision,raw) in current.items():
            require(key not in refs,'Duplicate supervision across human stores');v=json.loads(raw)
            refs[key]=dict(polygon=v['polygon'],revision=revision,payload_sha256=sha(v),event_raw_sha256=raw_sha(raw))
            kchanged+=len(v['polygon'])!=proposals[key]['class_keypoint_constraint']['standard_count']
        changed_counts[name]=kchanged;batches.append(dict(name=name,distinct=len(current),events=len(events)))
    require(len(refs)==2041 and [b['distinct'] for b in batches]==[337,145] and changed_counts['completed145']==1,'All sources / actual145 point exception differ')
    coco=read(ROOT/'dataset/train/_annotations.coco.json');square=read(ROOT/'dataset_rfdetr/train/_annotations.coco.json')
    require(len(coco['images'])==len(coco['annotations'])==len(square['images'])==len(square['annotations'])==2041,'Native/square COCO pairing differs')
    components=read(OLD/'private/global_components.json')['by_target'];frames=set();pixels=set();counts=Counter()
    for r,a,b,im in zip(records,coco['annotations'],square['annotations'],coco['images']):
        key=r['target_key'];row=rows[key];ref=refs[key];w,h=row['size'];box=row['bbox']
        require(r['polygon']==ref['polygon'] and r['source_revision']==ref['revision'] and r['source_payload_sha256']==ref['payload_sha256']
            and r['source_event_raw_sha256']==ref['event_raw_sha256'],'Latest actual human supervision differs')
        dx=max(48,.2*box['width']);dy=max(16,.6*box['height'])
        roi=[max(0,math.floor(box['x']-dx)),max(0,math.floor(box['y']-dy)),min(w,math.ceil(box['x']+box['width']+dx)),min(h,math.ceil(box['y']+box['height']+dy))]
        require(r['roi']==roi and r['development_split']=='train' and r['component']==components[key],'Oracle crop or split/component differs')
        x1,y1,x2,y2=roi;grey=cv2.imread(str(formal/row['image']),0)[y1:y2,x1:x2];full=raster_mask(ref['polygon'],w,h);mask=full[y1:y2,x1:x2]
        def decode(ann):
            v=ann['segmentation'];return mu.decode(dict(size=v['size'],counts=v['counts'].encode())).astype(bool)
        require(mask.any() and full.sum()==mask.sum() and np.array_equal(grey,cv2.imread(str(ROOT/r['image']),0))
            and np.array_equal(mask,decode(a)) and np.array_equal(mask,cv2.imread(str(ROOT/r['native_mask']),0)>0),'Native image/mask/RLE/foreground crop differs')
        valid=cv2.imread(str(ROOT/r['valid_mask']),0)>0;require(valid[mask].all(),'Human foreground erased by ignore')
        canvas,label,_,_=letterbox(grey,mask.astype(np.uint8),(576,576))
        require(np.array_equal(label>0,decode(b)) and np.array_equal(canvas,cv2.imread(str(ROOT/'dataset_rfdetr/train'/im['file_name']),0)),'Square native export differs')
        image_sha=digest(formal/row['image']);require(row['frame_id'] not in frames and image_sha not in pixels,'Duplicate frame/pixels inflate counts')
        frames.add(row['frame_id']);pixels.add(image_sha);counts[row['final_class_id']]+=1
    require(len({r['component'] for r in records})==246 and len(counts)==22,'2041/246/22 train coverage differs')
    require(digest(ROOT/'class_keypoint_standard.json')==digest(OLD/'class_keypoint_standard.json'),'Class standard recomputed')
    preserved(formal)
    return dict(state='passed2041_latest_real_human_sources_all246_exports_not_accuracy',records=2041,components=246,classes=dict(counts),batches=batches,
        changed_human_point_counts=changed_counts,all_train=True,independent_validation=False,old_artifacts_unchanged=True,dataset_manifest_sha256=digest(ROOT/'dataset_manifest.json'))


def finish():
    import torch
    from pycocotools import mask as mu
    from pallet.mask_model_training import native_metrics,summarize
    data=verify_dataset();plan=read(ROOT/'execution_plan.json');records={r['target_key']:r for r in lines(ROOT/'private/training_records.jsonl')};runs={}
    for route in ('m1','m2'):
        out=ROOT/route;require(not (out/'failure.json').exists(),'Failed worker evidence present')
        complete=read(out/'completion.json');h=digest(out/'final_ema.pt');checkpoint=torch.load(out/'final_ema.pt',map_location='cpu',weights_only=True)
        require(h==complete['checkpoint_sha256'] and checkpoint['step']==complete['actual_optimizer_updates']==plan['optimizer_updates'][route]==5110
            and checkpoint['parent_EMA_sha256']==plan['starting_EMA_sha256'][route],'Actual warm checkpoint/budget differs');del checkpoint
        exposure=read(out/'exposure_audit.json');require(set(exposure['by_target'])==set(records) and min(exposure['by_target'].values())>=10 and len(exposure['by_component'])==246,'Any image/component omitted')
        for epoch in range(1,11):
            e=read(out/'epochs'/f'{epoch:03d}.json');require(e['unique_images_exposed']==2041 and e['unique_components_exposed']==246,'Per-epoch all-source exposure differs')
        predictions=lines(out/'fit_predictions.jsonl');require(len(predictions)==2041 and {p['target_key'] for p in predictions}==set(records),'Diagnostic denominator differs')
        for p in predictions:
            rle=p['native_prediction_rle'];prediction=mu.decode(dict(size=rle['size'],counts=rle['counts'].encode())).astype(bool)
            mask=cv2.imread(str(ROOT/records[p['target_key']]['native_mask']),0)>0;metrics=native_metrics(prediction,mask)
            require(all(v==p[k] if v is None or isinstance(v,bool) else abs(v-p[k])<1e-10 for k,v in metrics.items())
                and p['trained_on_this_image'] and not p['blind'],'Native training fit replay differs or falsely independent')
        runs[route]=dict(checkpoint_sha256=h,steps=5110,training_fit_only=summarize(predictions),all2041_images_and246components_exposed=True,gpu=complete['gpu'])
    result=dict(state='verified_both_completed145_warm_updates_all2041_not_quality_release',dataset=data,runs=runs,
        dataset_manifest_sha256=digest(ROOT/'dataset_manifest.json'),next_annotation_primary_route='m2',independent_accuracy=None,
        production_model_activated=False,all_new_predictions_require_human_review=True,errors=[],code_sha256=digest(__file__))
    write(ROOT/'TRAINING_HANDOFF.json',result);print(json.dumps(result),flush=True)


if __name__=='__main__':
    p=argparse.ArgumentParser(description=__doc__);p.add_argument('--finish',action='store_true');a=p.parse_args()
    if a.finish:finish()
    else:
        result=verify_dataset();write(ROOT/'independent_dataset_verification.json',result);print(json.dumps(result),flush=True)
