"""New immutable supervision from completed337 reviews; old data/flags stay intact."""
from collections import Counter,defaultdict
from contextlib import closing
import hashlib
import json
from pathlib import Path
import shutil
import sqlite3

import cv2
import numpy as np

from .cluster_targets import digest
from .mask_annotation_full_data import context,supervision_valid
from .mask_model_training_data import PROJECT,FALSE_FLAGS,read,jsonl,write,sha,require,letterbox
from .mask_quota_saved_extension import file_inventory,load_formal_evidence
from .mask_template_refined import snapshot_database,readonly
from .mask_dataset_update import exception_snapshot
from .mask_model_improvement_data import preservation
from .mask_quality_recovery import fingerprint,POSE,GALLERY
from .pose_review_geometry import polygon_value,raster_mask

ROOT=PROJECT/'outputs/pallet_mask_annotation_update_v1'
PREVIOUS=PROJECT/'outputs/pallet_mask_annotation_full_v1'
SOURCE=PROJECT/'outputs/pallet_mask_continuation_sources_v17'
TASKS=PROJECT/'outputs/pallet_mask_model_requests_v3'
REVIEWS=PROJECT/'outputs/pallet_mask_model_reviews_v3/mask_reviews.sqlite3'
ISOLATED=PROJECT/'outputs/pallet_mask_nonconforming_reviews_v1/mask_reviews.sqlite3'
START_SHA={'m1':'4a73f72143c23fdbef375d3412d3cffdf5b2005b5f8ed2d08ef0ce41e37e9800',
    'm2':'92ce35476d4b0887a0e791dc20972c9640aab558ec876df84296b713ebe8ed4c'}


def raw_sha(raw):return hashlib.sha256(raw.encode('utf-8')).hexdigest()


def protected_versions():
    names=['pallet_mask_annotation_full_v1','pallet_mask_dataset_update_v1','pallet_mask_model_requests_v1',
        'pallet_mask_model_requests_v2','pallet_mask_model_requests_v3','pallet_mask_class_keypoint_checks_v1',
        'pallet_mask_class_keypoint_edit_extension_v1']
    return {f'outputs/{n}':file_inventory(PROJECT/'outputs'/n) for n in names}


def completed_review_sources(database,rows,formal):
    # Reuse only the read-only, event-specific consumer, never an editor writer.
    from scripts.audit_mask_model_reviews import audit
    audit_result=audit(TASKS,database)
    require(audit_result['new_distinct_saved_images']==337 and audit_result['events']>=337,'Every337 task needs a genuine saved review')
    proposals={p['target_key']:p for p in jsonl(TASKS/'proposals.jsonl')}
    with closing(readonly(database,True)) as db:
        current={k:(rev,raw) for k,rev,raw in db.execute('SELECT target_key,revision,payload FROM reviews')}
        events=list(db.execute('SELECT id,target_key,revision,payload FROM events ORDER BY id'))
    require(set(current)==set(proposals),'Do not count repeated revisions as more than337 images')
    latest={key:(event_id,rev,raw) for event_id,key,rev,raw in events};result=[]
    for key,(revision,raw) in sorted(current.items()):
        p=proposals[key];row=rows[key];value=json.loads(raw)
        require(row==p['source'] and digest(formal/row['image'])==p['model_source_image_sha256'],'Completed human original-image binding differs')
        require(latest[key][1:]==(revision,raw) and value['polygon'],'Current raw payload is not the exact last genuine event')
        result.append(dict(target_key=key,row=row,polygon=value['polygon'],revision=revision,payload_sha256=sha(value),
            event_raw_sha256=raw_sha(raw),source_event_id=latest[key][0],origin='saved_model_class_keypoint_human_review',
            batch='model-class-keypoints-v3',human_count_editing=value.get('human_keypoint_editing'),
            current_supervision_authorization='user_completed337_and_explicit_model_update_request_not_GT'))
    return result,audit_result,events


def prepare():
    require(not ROOT.exists(),'New immutable annotation update directory required')
    require(shutil.disk_usage(ROOT.parent).free>20*2**30,'Keep >=10GiB reserve, never delete old work')
    cv2.setNumThreads(1)
    formal,rows,evidence=load_formal_evidence(read(PROJECT/'outputs/pallet_mask_remaining_round5_v1/manifest.json'))
    old=protected_versions();before=preservation(formal)
    ROOT.mkdir(mode=0o700);(ROOT/'private').mkdir(mode=0o700)
    write(ROOT/'private/old_versions_inventory.json',old);write(ROOT/'private/preservation_before.json',before)
    snapshot_database(REVIEWS,ROOT/'private/completed337_snapshot.sqlite3')
    snapshot_database(ISOLATED,ROOT/'private/isolated_snapshot.sqlite3')
    snapshot_database(POSE,ROOT/'private/pose_snapshot.sqlite3');snapshot_database(GALLERY,ROOT/'private/gallery_snapshot.sqlite3')
    standard=jsonl(SOURCE/'private/source_catalog.jsonl');source_audit=read(SOURCE/'audit.json')
    require(len(standard)==1552 and source_audit['source_catalog_sha256']==digest(SOURCE/'private/source_catalog.jsonl'),'Fresh standard human source snapshot differs')
    require(fingerprint(ROOT/'private/pose_snapshot.sqlite3')==source_audit['pose_fingerprint']
        and fingerprint(ROOT/'private/gallery_snapshot.sqlite3')==source_audit['gallery_fingerprint'],'Source changed during freeze; create a new version')
    isolated,only,isolated_events=exception_snapshot(ROOT/'private/isolated_snapshot.sqlite3',rows,formal)
    require(len(isolated)==7 and not only,'Seven formerly quality-approved manual redraws must remain')
    completed,audit,events=completed_review_sources(ROOT/'private/completed337_snapshot.sqlite3',rows,formal)
    write(ROOT/'private/completed337_audit.json',audit)
    write(ROOT/'private/completed337_event_binding.jsonl',[
        dict(event_id=i,target_key=k,revision=r,payload_raw_utf8_sha256=raw_sha(raw)) for i,k,r,raw in events],lines=True)
    write(ROOT/'private/isolated_event_binding.jsonl',isolated_events,lines=True)
    source_catalog=standard+isolated+completed;require(len(source_catalog)==1896,'Expected1559+337 distinct manually saved sources')
    write(ROOT/'private/source_catalog.jsonl',source_catalog,lines=True)
    components=read(PREVIOUS/'private/global_components.json');write(ROOT/'private/global_components.json',components)
    write(ROOT/'research_permit.json',dict(schema='completed337_user_authorized_annotation_model_update_v1',
        user_authorization='现已完成337张复核，训练更新模型后继续剩余部分生成',
        old_all_labelled_component_and_former_val_test_training_authorization_continues=True,
        completed337_supervision_authorized_in_this_version_only=True,preserve_actual_human_point_counts_and_order=True,
        former_isolated7_approval_inherited=True,no_live_or_future_saves_dynamically_appended=True,
        purpose='annotation_assistant_training_not_independent_benchmark',independent_validation=False,
        class_standard_preserved_not_recomputed=True,new_online_model_autoactivation=False,**FALSE_FLAGS))
    shutil.copyfile(TASKS/'class_keypoint_standard.json',ROOT/'class_keypoint_standard.json')
    neighbors=defaultdict(list)
    for row in rows.values():neighbors[row['frame_id']].append(row)
    for name in ('dataset/train','dataset_rfdetr/train','private/native_masks','private/valid_masks'):(ROOT/name).mkdir(parents=True,mode=0o700)
    from pycocotools import mask as mu
    coco=dict(info=dict(description='1896 real saved contours, all train, no independent validation'),licenses=[],
        categories=[dict(id=1,name='fork_face')],images=[],annotations=[])
    square={**coco,'images':[],'annotations':[]};records=[];transforms={};keys=set();frames=set();hashes=set();context_notes=[]
    for index,ref in enumerate(sorted(source_catalog,key=lambda r:r['target_key'])):
        key=ref['target_key'];row=rows[key];image_sha=digest(formal/row['image'])
        require(key not in keys and row['frame_id'] not in frames and image_sha not in hashes,'Duplicate target/frame/image cannot add supervision')
        keys.add(key);frames.add(row['frame_id']);hashes.add(image_sha)
        polygon=polygon_value(ref['polygon'],*row['size']);roi=context(row['bbox'],row['size']);x1,y1,x2,y2=roi
        full=raster_mask(polygon,*row['size']).astype(np.uint8);mask=full[y1:y2,x1:x2]
        require(mask.any() and int(full.sum())==int(mask.sum()),'Fixed inference-available context clips real human mask; do not alter labels')
        grey=evidence.image(row['frame_id'])[y1:y2,x1:x2];valid=supervision_valid(roi,neighbors[row['frame_id']],key,mask)
        name=f'{index:05d}.png';height,width=mask.shape
        cv2.imwrite(str(ROOT/'dataset/train'/name),np.repeat(grey[:,:,None],3,axis=2));cv2.imwrite(str(ROOT/'private/native_masks'/name),mask*255)
        cv2.imwrite(str(ROOT/'private/valid_masks'/name),valid*255)
        def ann(label):
            rle=mu.encode(np.asfortranarray(label));require(np.array_equal(mu.decode(rle),label),'Exact manual RLE required')
            return dict(id=index+1,image_id=index+1,category_id=1,iscrowd=0,segmentation=dict(size=rle['size'],counts=rle['counts'].decode()),
                area=int(mu.area(rle)),bbox=mu.toBbox(rle).tolist())
        coco['images'].append(dict(id=index+1,file_name=name,width=width,height=height));coco['annotations'].append(ann(mask))
        image,label,_,transform=letterbox(grey,mask,(576,576));cv2.imwrite(str(ROOT/'dataset_rfdetr/train'/name),np.repeat(image[:,:,None],3,axis=2))
        square['images'].append(dict(id=index+1,file_name=name,width=576,height=576));square['annotations'].append(ann(label));transforms[str(index+1)]=transform
        records.append(dict(target_key=key,component=components['by_target'][key],class_id=row['final_class_id'],frame_id=row['frame_id'],
            group_id=row['group_id'],policy_id=row['policy_id'],native_image=row['image'],native_size=row['size'],original_bbox=row['bbox'],
            roi=roi,roi_size=[width,height],image=f'dataset/train/{name}',native_mask=f'private/native_masks/{name}',valid_mask=f'private/valid_masks/{name}',
            image_id=index+1,development_split='train',former_formal_split=row['split'],polygon=polygon,
            source_event_raw_sha256=ref['event_raw_sha256'],source_payload_sha256=ref['payload_sha256'],source_revision=ref['revision'],
            source_origin=ref['origin'],source_batch=ref['batch'],human_actual_vertex_count=len(polygon),
            current_supervision_authorization='explicit_user_model_update_after337completed_not_GT',
            surface=row.get('viewpoint_candidate','unknown'),structure=row.get('structure_candidate','unknown'),**FALSE_FLAGS))
        if key in {p['target_key'] for p in completed} and len(polygon)!=read(ROOT/'class_keypoint_standard.json')['classes'][str(row['final_class_id'])]['standard_count']:
            context_notes.append(dict(target_key=key,actual_human_points=len(polygon),note='User-added/removed points retained as real mask supervision; class standard not rewritten'))
    write(ROOT/'dataset/train/_annotations.coco.json',coco);write(ROOT/'dataset_rfdetr/train/_annotations.coco.json',square)
    write(ROOT/'private/training_records.jsonl',records,lines=True);write(ROOT/'private/rfdetr_transforms.json',transforms)
    write(ROOT/'private/human_count_variations.json',context_notes)
    write(ROOT/'private/source_binding.json',dict(standard_snapshot=str(SOURCE.relative_to(PROJECT)),standard_audit_sha256=digest(SOURCE/'audit.json'),
        snapshot_fingerprints={n:fingerprint(ROOT/'private'/n) for n in ('pose_snapshot.sqlite3','gallery_snapshot.sqlite3','isolated_snapshot.sqlite3','completed337_snapshot.sqlite3')},
        completed_task_manifest_sha256=digest(TASKS/'manifest.json'),completed_namespace='model-class-keypoints-v3',
        cutoff_real_review_events=len(events),future_revisions_require_new_data_version=True))
    files={d+'/'+n:h for d in ('dataset','dataset_rfdetr','private/native_masks','private/valid_masks') for n,h in file_inventory(ROOT/d).items()}
    for n in ('private/training_records.jsonl','private/rfdetr_transforms.json','private/source_catalog.jsonl','private/global_components.json',
        'private/source_binding.json','research_permit.json','class_keypoint_standard.json'):files[n]=digest(ROOT/n)
    counts=Counter(r['class_id'] for r in records);ncomp=len({r['component'] for r in records})
    require(len(records)==1896 and len(counts)==22 and ncomp==240,'Expected all1896 /22classes /240 labelled components')
    write(ROOT/'dataset_manifest.json',dict(state='immutable_completed337_plus1559_annotation_update_dataset',records=1896,
        classes=dict(counts),components=ncomp,new_reviewed_images=337,new_review_events=len(events),splits={'train':1896},
        former_splits=dict(Counter(r['former_formal_split'] for r in records)),all_labelled_components_train=True,independent_validation=False,
        source_flags_not_rewritten=True,files=files,code_sha256=digest(__file__),**FALSE_FLAGS))
    require(protected_versions()==old and preservation(formal)==before,'Old files/configs/GT changed')
    verify();print(json.dumps(dict(state='frozen1896_actual_human_sources_all240components_train',class_counts=dict(counts),
        remaining_to100={c:max(0,100-counts[c]) for c in range(1,23)},manifest_sha256=digest(ROOT/'dataset_manifest.json'))),flush=True)


def verify():
    from pycocotools import mask as mu
    manifest=read(ROOT/'dataset_manifest.json');records=jsonl(ROOT/'private/training_records.jsonl')
    require(len(records)==manifest['records']==1896 and len({r['component'] for r in records})==manifest['components']==240,'Immutable source coverage differs')
    for name,expected in manifest['files'].items():require(digest(ROOT/name)==expected,'Frozen training input changed: '+name)
    require(all(r['development_split']=='train' for r in records),'No artificial old split exclusions')
    coco=read(ROOT/'dataset/train/_annotations.coco.json');square=read(ROOT/'dataset_rfdetr/train/_annotations.coco.json')
    for r,a,b in zip(records,coco['annotations'],square['annotations']):
        def decode(ann):return mu.decode(dict(size=ann['segmentation']['size'],counts=ann['segmentation']['counts'].encode()))
        mask=cv2.imread(str(ROOT/r['native_mask']),0)>0;valid=cv2.imread(str(ROOT/r['valid_mask']),0)>0;grey=cv2.imread(str(ROOT/r['image']),0)
        image,label,_,_=letterbox(grey,mask.astype(np.uint8),(576,576))
        require(np.array_equal(mask,decode(a)) and valid[mask].all() and np.array_equal(label,decode(b)),'RLE/ignore/square mapping differs')
        require(np.array_equal(image,cv2.imread(str(ROOT/'dataset_rfdetr/train'/Path(r['image']).name),0)),'Square native pixel export differs')
    return manifest,records
