"""Latest round4 correct reviews overlay the frozen pool; no source writers."""
from collections import Counter,defaultdict
from contextlib import closing
from copy import deepcopy
from datetime import datetime,timezone
import json
import shutil

import cv2
import numpy as np

from .cluster_targets import digest
from .mask_model_training_data import PROJECT,FALSE_FLAGS,read,write,jsonl,sha,require
from .mask_class_note_integration import snapshot_db
from .mask_template_refined import readonly
from .mask_quality_recovery import fingerprint,POSE
from .mask_annotation_full_data import context,supervision_valid
from .mask_quota_saved_extension import load_formal_evidence
from .mask_taxonomy_data import split_components
from .pallet_description_parallel_guard import ParallelContentSourceGuard
from . import pallet_description_round4_data as previous
from .pallet_description_round4_release import REVIEW_SETS as SOURCE_SETS
from .pallet_description_review import validate_description
from .pose_review_geometry import polygon_value,raster_mask

ROOT=PROJECT/'outputs/pallet_description_round5_training_v1'
PREVIOUS=previous.ROOT
FORMAL_SOURCE=previous.FORMAL_SOURCE
AUTHORIZATION='已经完成复核，继续；执行'
PROMPT='根据图中指定托盘，只输出一个JSON对象，字段依次为"支腿"、"横梁"、"材质"、"结构"、"观察"。支腿为2/3/4或null，横梁为1/2或null，材质为塑料/木质/金属/无法判断。结构和观察必须是带双引号的中文字符串。观察只描述可见遮挡、出视野、视角、图像质量；缺少可靠观察信息时输出空字符串，空字符串不表示无异常。不要猜测不可见结构，不把遮挡或出视野当损坏，不输出Markdown或额外解释。'


def database_inventory():
    return previous.database_inventory()|{str((s['reviews']/'mask_reviews.sqlite3').relative_to(PROJECT)) for s in SOURCE_SETS.values()}


def worklist_inventory():
    return previous.worklist_inventory()|{str((s['root']/'proposals.jsonl').relative_to(PROJECT)) for s in SOURCE_SETS.values()}


def binding_guard():
    b=read(ROOT/'source_binding.json');p=read(ROOT/'private/protection.json')
    return ParallelContentSourceGuard(project=PROJECT,databases=b['database_fingerprints'],files=b['file_hashes'],inventories=p['inventories'],
        discover_databases=database_inventory,discover_inventories=worklist_inventory,extra_databases=[str(POSE.relative_to(PROJECT))],
        pinned_files={str((ROOT/n).relative_to(PROJECT)):digest(ROOT/n) for n in ('dataset_manifest.json','source_binding.json','private/protection.json')})


def sources_current():
    g=binding_guard()
    try:return g.current()
    finally:g.close()


def integrate(base,props,reviews,mode,classes,binding):
    """A newer saved record replaces one existing target; it never adds a copy."""
    result=deepcopy(base);changes=Counter()
    require(set(reviews)==set(props),'Every issued target must have a saved record')
    for key,p in props.items():
        v=reviews[key];require(v['action']=='correct' and v['binding']==binding and v['source_binding']==sha(p) and v['source']==p['source'],'Latest completed/source binding mismatch')
        cls=148 if v['class_id']==149 else v['class_id'];info=classes[str(cls)];poly=deepcopy(v['polygon'])
        polygon_value(poly,*p['source']['size']);require(info['train_enabled'] and len(poly)==info['K'] and v['class_K_matches'],'Invalid completed classK')
        d=deepcopy(v['description']);validate_description(d)
        if mode=='existing':require(key in base,'Existing queue target missing from prior training pool')
        else:require(key not in base,'New target already in prior training pool')
        changes['class_changed']+=cls!=p['class_id'];changes['polygon_changed']+=poly!=p['polygon'];changes['text_changed']+=d['text']!=p['description']['text']
        old=result.get(key);chain=deepcopy(old.get('provenance_chain',[]) if old else [])
        if old:chain.append(dict(epoch='round4_training',revision=old['source_revision'],action=old['original_review_action'],source_payload_sha256=old['source_payload_sha256']))
        result[key]=dict(target_key=key,class_id=cls,K=info['K'],polygon=poly,source=p['source'],image_sha256=p['image_sha256'],mode=mode,
            original_review_action=v['action'],source_revision=v['revision'],source_payload_sha256=sha(v),prior_proposal_sha256=sha(p),
            geometry_source='latest_saved_human_correct_round4',completion_authorization=AUTHORIZATION,description=d,note=v.get('note',''),
            original_human_note=p.get('human_note',''),pose_payload_raw_sha256=p.get('pose_payload_raw_sha256'),provenance_chain=chain,**FALSE_FLAGS)
    return result,dict(changes)


def prepare():
    require(not ROOT.exists(),'Preserve prior round5 attempt');require(shutil.disk_usage(PROJECT/'outputs').free>40*2**30,'Disk reserve required')
    require(previous.sources_current(),'Prior frozen sources changed; reconcile before freeze');previous.verify()
    ROOT.mkdir(mode=0o700);private=ROOT/'private';private.mkdir(mode=0o700)
    for name in ('images','masks','valid','snapshots'):(private/name).mkdir(mode=0o700)
    formal,rows,evidence=load_formal_evidence(read(FORMAL_SOURCE));classes=read(PREVIOUS/'classes.json');parents=read(PREVIOUS/'parents.json')
    base={r['target_key']:r for r in jsonl(PREVIOUS/'private/training.jsonl')};oldsem={r['target_key']:r for r in jsonl(PREVIOUS/'description_data/records.jsonl')}
    records=deepcopy(base);snapshots={};changed={};pins={};updated=set()
    for mode,s in SOURCE_SETS.items():
        path=private/'snapshots'/f'{mode}.sqlite3';snapshot_db(s['reviews']/'mask_reviews.sqlite3',path)
        with closing(readonly(path)) as db:
            meta=dict(db.execute('SELECT key,value FROM metadata'));reviews={k:json.loads(raw) for k,raw in db.execute('SELECT target_key,payload FROM reviews')}
            events=list(db.execute('SELECT target_key,revision,payload FROM events ORDER BY id'))
        latest={}
        for key,rev,raw in events:
            value=json.loads(raw);require(value['revision']==rev and rev==latest.get(key,{}).get('revision',0)+1,'Noncontiguous append history');latest[key]=value
        binding=digest(s['root']/'manifest.json');require(latest==reviews and meta['namespace']==s['namespace'] and meta['formal_manifest_sha256']==binding,'History/namespace mismatch')
        props={p['target_key']:p for p in jsonl(s['root']/'proposals.jsonl')}
        records,changed[mode]=integrate(records,props,reviews,mode,classes,binding);updated.update(props)
        snapshots[mode]=dict(events=len(events),saved=len(reviews),actions=dict(Counter(v['action'] for v in reviews.values())),snapshot_sha256=digest(path))
        for n in ('manifest.json','classes.json','proposals.jsonl'):pins[str((s['root']/n).relative_to(PROJECT))]=digest(s['root']/n)
    records=list(records.values());classes=previous.canonical_classes(classes,records);write(ROOT/'classes.json',classes);write(ROOT/'parents.json',parents)
    components=read(PREVIOUS/'private/components.json');hashes=read(PREVIOUS/'private/image_hashes.json');neighbors=defaultdict(list)
    for row in rows.values():neighbors[row['frame_id']].append(row)
    cv2.setNumThreads(1)
    for r in records:
        key=r['target_key'];row=r['source'];info=classes[str(r['class_id'])]
        require(rows[key]==row and hashes[row['image']]==r['image_sha256'] and digest(formal/row['image'])==r['image_sha256'],'Original source image changed')
        roi=context(row['bbox'],row['size']);x1,y1,x2,y2=roi;full=raster_mask(r['polygon'],*row['size']).astype(np.uint8);mask=full[y1:y2,x1:x2]
        require(mask.any() and full.sum()==mask.sum(),'Fixed bbox ROI clips current human contour: '+key)
        valid=supervision_valid(roi,neighbors[row['frame_id']],key,mask);image=evidence.image(row['frame_id'])[y1:y2,x1:x2]
        for folder,array in (('images',image),('masks',mask*255),('valid',valid*255)):require(cv2.imwrite(str(private/folder/f'{key}.png'),array),'Export failed')
        r.update(fine_index=info['fine_index'],parent=info['parent'],parent_index=info['parent_index'],small_sample=info['small_sample'],component=components['by_target'][key],
            roi=roi,image=f'private/images/{key}.png',mask=f'private/masks/{key}.png',valid=f'private/valid/{key}.png',training_origin='round5_latest_completed_overlay')
    split=split_components(records)
    for r in records:r['evaluation_branch_split']=split[r['target_key']]
    require(all(r['evaluation_branch_split']==base[r['target_key']]['evaluation_branch_split'] for r in records if r['target_key'] in base),'Old component split changed')
    write(private/'training.jsonl',records,lines=True);write(private/'components.json',components);write(private/'image_hashes.json',hashes)
    semantics=[]
    for r in records:
        scope={'text','legs','beams','material','observation'} if r['target_key'] in updated else set(oldsem[r['target_key']]['human_review_scope'])
        value=previous.semantic_record(r,classes[str(r['class_id'])],scope);value['prompt']=PROMPT;semantics.append(value)
    (ROOT/'description_data').mkdir(mode=0o700);write(ROOT/'description_data/records.jsonl',semantics,lines=True)
    write(ROOT/'description_data/conflicts.jsonl',[r for r in semantics if r['semantic_conflicts']],lines=True)
    inventories={n:digest(PROJECT/n) for n in worklist_inventory()};protected=set(read(PREVIOUS/'private/protection.json')['protected_image_hashes'])|{r['image_sha256'] for r in records}
    for n in inventories:
        for p in jsonl(PROJECT/n):
            if p.get('target_key') in rows:protected.add(hashes[rows[p['target_key']]['image']])
    candidates=[r for r in jsonl(PREVIOUS/'private/candidates.jsonl') if r['image_sha256'] not in protected and r['target_key'] not in split]
    write(private/'candidates.jsonl',candidates,lines=True);write(private/'protection.json',dict(inventories=inventories,protected_image_hashes=sorted(protected),no_previous_issued_image_reuse=True))
    b=read(PREVIOUS/'source_binding.json');files=dict(b['file_hashes']);files.update(pins);stores=dict(b['database_fingerprints'])
    for n in ('dataset_manifest.json','classes.json','parents.json','private/components.json','private/candidates.jsonl','private/training.jsonl'):files[str((PREVIOUS/n).relative_to(PROJECT))]=digest(PREVIOUS/n)
    for mode,s in SOURCE_SETS.items():
        path=s['reviews']/'mask_reviews.sqlite3';require(fingerprint(path)==fingerprint(private/'snapshots'/f'{mode}.sqlite3'),'Concurrent review changed during freeze')
        stores[str(path.relative_to(PROJECT))]=fingerprint(path)
    write(ROOT/'source_binding.json',dict(database_fingerprints=stores,file_hashes=files,snapshots=snapshots,created_at=datetime.now(timezone.utc).isoformat(),
        canonical_class_aliases={'149':148},previous_training_manifest_sha256=digest(PREVIOUS/'dataset_manifest.json')))
    write(ROOT/'research_permit.json',dict(user_authorization=AUTHORIZATION,actual_round4_actions=snapshots,source_events_not_rewritten=True,
        no_semantic_normal_negatives_inferred=True,no_automatic_GT_or_robot_promotion=True,**FALSE_FLAGS))
    report=dict(state='round5_latest_completed_reviews_frozen',targets=len(records),distinct_images=len({r['image_sha256'] for r in records}),
        newly_added_targets=len(records)-len(base),updated_existing_targets=len(updated&set(base)),fine_classes=sum(c['train_enabled'] for c in classes.values()),parents=len(parents),
        splits=dict(Counter(split.values())),sources=snapshots,changes_from_round4_task_initialization=changed,
        semantic_supervised_fields={k:sum(r['supervised_fields'][k] for r in semantics) for k in ('支腿','横梁','材质','结构','观察')},
        semantic_conflict_targets=sum(bool(r['semantic_conflicts']) for r in semantics),candidates=len(candidates),candidate_distinct_images=len({r['image_sha256'] for r in candidates}),
        small_classes=[c['class_id'] for c in classes.values() if c['train_enabled'] and c['small_sample']],training_class_counts=dict(Counter(r['class_id'] for r in records)),
        unchanged_component_hash_split=True,not_new_blind_test=True,**FALSE_FLAGS)
    write(ROOT/'preparation_report.json',report)
    write(ROOT/'dataset_manifest.json',dict(schema='round5-latest40fine9parent-v1',records=len(records),classes=40,parents=9,
        files={str(p.relative_to(ROOT)):digest(p) for p in ROOT.rglob('*') if p.is_file()},code_sha256=digest(__file__)))
    verify();require(sources_current(),'Source changed during freeze');print(json.dumps(report,ensure_ascii=False),flush=True)


def verify():
    m=read(ROOT/'dataset_manifest.json')
    for n,h in m['files'].items():require(digest(ROOT/n)==h,'Frozen round5 file changed: '+n)
    records=jsonl(ROOT/'private/training.jsonl');classes=read(ROOT/'classes.json');groups=defaultdict(set);images=defaultdict(set)
    require(len(records)==len({r['target_key'] for r in records})==m['records'],'Duplicate target')
    for r in records:
        c=classes[str(r['class_id'])];require(c['train_enabled'] and len(r['polygon'])==r['K']==c['K'] and r['fine_index']==c['fine_index'] and r['parent']==c['parent'],'Canonical class mismatch')
        groups[r['evaluation_branch_split']].add(r['component']);images[r['evaluation_branch_split']].add(r['image_sha256'])
    require(not groups['train']&groups['holdout'] and not images['train']&images['holdout'],'Image/component leakage')
    return m,records


if __name__=='__main__':prepare()
