"""Explicit completed145 supervision snapshot; old frozen versions stay immutable."""
from collections import Counter,defaultdict
from contextlib import closing
import hashlib
import json
from pathlib import Path
import shutil

import cv2
import numpy as np

from .cluster_targets import digest
from .mask_annotation_full_data import context,supervision_valid
from .mask_model_training_data import PROJECT,FALSE_FLAGS,read,jsonl,write,sha,require,letterbox
from .mask_quota_saved_extension import file_inventory,load_formal_evidence
from .mask_template_refined import snapshot_database,readonly
from .mask_dataset_update import exception_snapshot
from .mask_model_improvement_data import preservation
from .mask_quality_recovery import fingerprint,POSE,GALLERY
from .pose_review_geometry import polygon_value,raster_mask
from .mask_annotation_update_data import protected_versions as original_versions

ROOT=PROJECT/'outputs/pallet_mask_annotation_update_v2'
PREVIOUS=PROJECT/'outputs/pallet_mask_annotation_update_v1'
SOURCE=PROJECT/'outputs/pallet_mask_continuation_sources_v18'
STANDARD=PROJECT/'outputs/pallet_mask_model_requests_v3/class_keypoint_standard.json'
ISOLATED=PROJECT/'outputs/pallet_mask_nonconforming_reviews_v1/mask_reviews.sqlite3'
TOTAL=2041;COMPONENTS=246
START_SHA={'m1':'153e572357a886783b8a4d9449ada1290dc4b74efd63f3d27a943ac05115db03',
    'm2':'3eb7d659daccd424c2b47b0b70b098f8041273ba8b0f407bc10a57caaa29c524'}
REVIEW_BATCHES=(
    ('completed337','outputs/pallet_mask_model_requests_v3','outputs/pallet_mask_model_reviews_v3/mask_reviews.sqlite3',337),
    ('completed145','outputs/pallet_mask_annotation_update_requests_v1','outputs/pallet_mask_model_reviews_v4/mask_reviews.sqlite3',145),
)


def raw_sha(raw):return hashlib.sha256(raw.encode('utf-8')).hexdigest()


def protected_versions():
    result=original_versions()
    for name in ('pallet_mask_annotation_update_v1','pallet_mask_annotation_update_requests_v1','pallet_mask_annotation_update_checks_v1'):
        result['outputs/'+name]=file_inventory(PROJECT/'outputs'/name)
    return result


def model_sources(name,tasks,database,expected,rows,formal):
    from scripts.audit_mask_model_reviews import audit
    audit_result=audit(tasks,database);proposals={p['target_key']:p for p in jsonl(tasks/'proposals.jsonl')}
    require(audit_result['new_distinct_saved_images']==expected==len(proposals),'Every distinct issued task needs an actual human save')
    with closing(readonly(database,True)) as db:
        current={k:(r,raw) for k,r,raw in db.execute('SELECT target_key,revision,payload FROM reviews')}
        events=list(db.execute('SELECT id,target_key,revision,payload FROM events ORDER BY id'))
    require(set(current)==set(proposals),'Repeated events cannot add new distinct training images')
    latest={k:(i,r,raw) for i,k,r,raw in events};result=[]
    for key,(revision,raw) in sorted(current.items()):
        p=proposals[key];row=rows[key];value=json.loads(raw)
        require(row==p['source'] and digest(formal/row['image'])==p['model_source_image_sha256']
            and latest[key][1:]==(revision,raw),'Real saved target/image/final raw event differs')
        polygon_value(value['polygon'],*row['size'])
        result.append(dict(target_key=key,row=row,polygon=value['polygon'],revision=revision,payload_sha256=sha(value),
            event_raw_sha256=raw_sha(raw),source_event_id=latest[key][0],origin='saved_model_class_keypoint_human_review',
            batch=read(tasks/'manifest.json')['source_namespace'],human_count_editing=value.get('human_keypoint_editing'),
            current_supervision_authorization='explicit_user_completed145_training_request_new_version_only_not_GT'))
    return result,audit_result,events


def prepare():
    require(not ROOT.exists(),'Fresh immutable training version required')
    require(shutil.disk_usage(ROOT.parent).free>20*2**30,'Preserve disk reserve; deletion not authorized')
    cv2.setNumThreads(1);formal,rows,evidence=load_formal_evidence(read(PROJECT/'outputs/pallet_mask_remaining_round5_v1/manifest.json'))
    old=protected_versions();before=preservation(formal);ROOT.mkdir(mode=0o700);(ROOT/'private').mkdir(mode=0o700)
    write(ROOT/'private/old_versions_inventory.json',old);write(ROOT/'private/preservation_before.json',before)
    for name,path in [('pose',POSE),('gallery',GALLERY),('isolated',ISOLATED)]:snapshot_database(path,ROOT/'private'/f'{name}_snapshot.sqlite3')
    standard=jsonl(SOURCE/'private/source_catalog.jsonl');audit=read(SOURCE/'audit.json')
    require(len(standard)==1552 and audit['source_catalog_sha256']==digest(SOURCE/'private/source_catalog.jsonl'),'Fresh1552 human source snapshot differs')
    require(fingerprint(ROOT/'private/pose_snapshot.sqlite3')==audit['pose_fingerprint']
        and fingerprint(ROOT/'private/gallery_snapshot.sqlite3')==audit['gallery_fingerprint'],'Concurrent standard source revision; freeze a new version')
    isolated,only,isolated_events=exception_snapshot(ROOT/'private/isolated_snapshot.sqlite3',rows,formal)
    require(len(isolated)==7 and not only,'Seven previously user-QC-approved redraws required')
    sources=standard+isolated;batch_bindings=[]
    for name,taskpath,dbpath,expected in REVIEW_BATCHES:
        snapshot=ROOT/'private'/f'{name}_snapshot.sqlite3';snapshot_database(PROJECT/dbpath,snapshot)
        new,result,events=model_sources(name,PROJECT/taskpath,snapshot,expected,rows,formal);sources+=new
        write(ROOT/'private'/f'{name}_audit.json',result)
        write(ROOT/'private'/f'{name}_event_binding.jsonl',[
            dict(event_id=i,target_key=k,revision=r,payload_raw_utf8_sha256=raw_sha(raw)) for i,k,r,raw in events],lines=True)
        batch_bindings.append(dict(name=name,tasks=taskpath,live_database=dbpath,distinct=expected,events=len(events),
            task_manifest_sha256=digest(PROJECT/taskpath/'manifest.json'),snapshot_fingerprint=fingerprint(snapshot)))
    require(len(sources)==TOTAL and len({r['target_key'] for r in sources})==TOTAL,'1559+337+145 distinct human sources required')
    write(ROOT/'private/source_catalog.jsonl',sources,lines=True);write(ROOT/'private/isolated_event_binding.jsonl',isolated_events,lines=True)
    components=read(PREVIOUS/'private/global_components.json');write(ROOT/'private/global_components.json',components)
    shutil.copyfile(STANDARD,ROOT/'class_keypoint_standard.json');standard_counts=read(STANDARD)['classes']
    write(ROOT/'research_permit.json',dict(schema='completed145_explicit_annotation_training_update_v2',
        user_authorization='已经完成145审核，训练后继续提供新标注。',purpose='annotation_assistant_only_not_independent_benchmark',
        all_previous_user_authorized_labels_plus_latest145=True,all_previous_labelled_components_train=True,
        preserve_actual_human_polygon_and_vertex_count=True,class_standard_unchanged=True,
        former_isolated7_approval_inherited=True,no_future_live_saves_appended=True,
        old_flags_GT_poses_sources_datasets_and_models_unchanged=True,independent_validation=False,**FALSE_FLAGS))
    neighbors=defaultdict(list)
    for row in rows.values():neighbors[row['frame_id']].append(row)
    for folder in ('dataset/train','dataset_rfdetr/train','private/native_masks','private/valid_masks'):(ROOT/folder).mkdir(parents=True,mode=0o700)
    from pycocotools import mask as mu
    coco=dict(info=dict(description='2041 actual human contours, all train, no independent validation'),licenses=[],
        categories=[dict(id=1,name='fork_face')],images=[],annotations=[])
    square={**coco,'images':[],'annotations':[]};records=[];transforms={};frames=set();hashes=set();variations=[]
    for index,ref in enumerate(sorted(sources,key=lambda r:r['target_key'])):
        key=ref['target_key'];row=rows[key];h=digest(formal/row['image'])
        require(row['frame_id'] not in frames and h not in hashes,'No duplicate frame/pixels can add human completion')
        frames.add(row['frame_id']);hashes.add(h);polygon=polygon_value(ref['polygon'],*row['size']);roi=context(row['bbox'],row['size']);x1,y1,x2,y2=roi
        full=raster_mask(polygon,*row['size']).astype(np.uint8);mask=full[y1:y2,x1:x2]
        require(mask.any() and full.sum()==mask.sum(),'Fixed image-available context clips human mask; do not silently change labels')
        grey=evidence.image(row['frame_id'])[y1:y2,x1:x2];valid=supervision_valid(roi,neighbors[row['frame_id']],key,mask)
        name=f'{index:05d}.png';height,width=mask.shape
        for folder,image in [('dataset/train',np.repeat(grey[:,:,None],3,axis=2)),('private/native_masks',mask*255),('private/valid_masks',valid*255)]:
            require(cv2.imwrite(str(ROOT/folder/name),image),'Image export failed')
        def annotation(label):
            rle=mu.encode(np.asfortranarray(label));require(np.array_equal(mu.decode(rle),label),'Exact RLE supervision required')
            return dict(id=index+1,image_id=index+1,category_id=1,iscrowd=0,segmentation=dict(size=rle['size'],counts=rle['counts'].decode()),area=int(mu.area(rle)),bbox=mu.toBbox(rle).tolist())
        coco['images'].append(dict(id=index+1,file_name=name,width=width,height=height));coco['annotations'].append(annotation(mask))
        image,label,_,transform=letterbox(grey,mask,(576,576));require(cv2.imwrite(str(ROOT/'dataset_rfdetr/train'/name),np.repeat(image[:,:,None],3,axis=2)),'Square image export failed')
        square['images'].append(dict(id=index+1,file_name=name,width=576,height=576));square['annotations'].append(annotation(label));transforms[str(index+1)]=transform
        records.append(dict(target_key=key,component=components['by_target'][key],class_id=row['final_class_id'],frame_id=row['frame_id'],
            group_id=row['group_id'],policy_id=row['policy_id'],native_image=row['image'],native_size=row['size'],original_bbox=row['bbox'],roi=roi,roi_size=[width,height],
            image=f'dataset/train/{name}',native_mask=f'private/native_masks/{name}',valid_mask=f'private/valid_masks/{name}',image_id=index+1,
            development_split='train',former_formal_split=row['split'],polygon=polygon,source_event_raw_sha256=ref['event_raw_sha256'],
            source_payload_sha256=ref['payload_sha256'],source_revision=ref['revision'],source_origin=ref['origin'],source_batch=ref['batch'],human_actual_vertex_count=len(polygon),
            current_supervision_authorization='explicit_completed145_training_request_new_version_only_not_GT',
            surface=row.get('viewpoint_candidate','unknown'),structure=row.get('structure_candidate','unknown'),**FALSE_FLAGS))
        if len(polygon)!=standard_counts[str(row['final_class_id'])]['standard_count']:
            variations.append(dict(target_key=key,source_batch=ref['batch'],actual_human_points=len(polygon),initial_standard_count=standard_counts[str(row['final_class_id'])]['standard_count']))
    write(ROOT/'dataset/train/_annotations.coco.json',coco);write(ROOT/'dataset_rfdetr/train/_annotations.coco.json',square)
    write(ROOT/'private/training_records.jsonl',records,lines=True);write(ROOT/'private/rfdetr_transforms.json',transforms);write(ROOT/'private/human_count_variations.json',variations)
    write(ROOT/'private/source_binding.json',dict(standard_snapshot=str(SOURCE.relative_to(PROJECT)),standard_audit_sha256=digest(SOURCE/'audit.json'),
        standard_catalog_sha256=digest(SOURCE/'private/source_catalog.jsonl'),batches=batch_bindings,
        pose_fingerprint=fingerprint(POSE),gallery_fingerprint=fingerprint(GALLERY),isolated_fingerprint=fingerprint(ISOLATED),
        future_saves_or_revisions_require_a_new_version=True))
    files={d+'/'+n:h for d in ('dataset','dataset_rfdetr','private/native_masks','private/valid_masks') for n,h in file_inventory(ROOT/d).items()}
    for name in ('private/training_records.jsonl','private/rfdetr_transforms.json','private/source_catalog.jsonl','private/global_components.json','private/source_binding.json','class_keypoint_standard.json','research_permit.json'):
        files[name]=digest(ROOT/name)
    counts=Counter(r['class_id'] for r in records);require(len(records)==TOTAL and len(counts)==22 and len({r['component'] for r in records})==COMPONENTS,'All2041 /22classes /246components required')
    write(ROOT/'dataset_manifest.json',dict(state='immutable_completed145_plus1896_human_annotation_update_v2',records=TOTAL,components=COMPONENTS,
        classes=dict(counts),new_reviewed_images=145,new_review_events=batch_bindings[-1]['events'],splits={'train':TOTAL},
        all_labelled_components_train=True,independent_validation=False,source_flags_not_rewritten=True,files=files,code_sha256=digest(__file__),**FALSE_FLAGS))
    require(protected_versions()==old and preservation(formal)==before,'Old data/model/config/GT changed')
    verify();print(dict(state='frozen2041_real_human_images_all246components_train',classes=dict(counts),manifest_sha256=digest(ROOT/'dataset_manifest.json')),flush=True)


def verify():
    manifest=read(ROOT/'dataset_manifest.json');records=jsonl(ROOT/'private/training_records.jsonl')
    require(len(records)==manifest['records']==TOTAL and len({r['component'] for r in records})==manifest['components']==COMPONENTS,'Immutable coverage differs')
    for name,h in manifest['files'].items():require(digest(ROOT/name)==h,'Frozen training bytes changed: '+name)
    require(all(r['development_split']=='train' for r in records),'All labelled images train, no old split exclusion')
    return manifest,records
