"""Freeze completed round7 reviews as a new, append-only research dataset."""
from collections import Counter,defaultdict
from contextlib import closing
from copy import deepcopy
from datetime import datetime,timezone
import json
import shutil

import cv2
import numpy as np

from .cluster_targets import digest
from .mask_model_training_data import PROJECT,FALSE_FLAGS,read,write,jsonl,sha,require
from .mask_class_note_integration import snapshot_db
from .mask_template_refined import readonly
from .mask_quality_recovery import fingerprint,POSE
from .mask_annotation_full_data import context,supervision_valid
from .mask_quota_saved_extension import load_formal_evidence
from .mask_taxonomy_data import split_components
from .pallet_description_parallel_guard import ParallelContentSourceGuard
from . import pallet_description_round7_data as previous
from . import pallet_description_round4_data as policy
from .pallet_description_round7_release import REVIEW_SETS as SOURCE_SETS
from .pallet_description_review import validate_description
from .pose_review_geometry import polygon_value,raster_mask

ROOT=PROJECT/'outputs/pallet_description_round8_training_v1'
PREVIOUS=previous.ROOT
FORMAL_SOURCE=previous.FORMAL_SOURCE
AUTHORIZATION='已经完成复核，继续'
PROMPT=previous.PROMPT


def database_inventory():
    return previous.database_inventory()|{str((s['reviews']/'mask_reviews.sqlite3').relative_to(PROJECT)) for s in SOURCE_SETS.values()}


def worklist_inventory():
    return previous.worklist_inventory()|{str((s['root']/'proposals.jsonl').relative_to(PROJECT)) for s in SOURCE_SETS.values()}


def binding_guard():
    b=read(ROOT/'source_binding.json');p=read(ROOT/'private/protection.json')
    return ParallelContentSourceGuard(project=PROJECT,databases=b['database_fingerprints'],files=b['file_hashes'],inventories=p['inventories'],
        discover_databases=database_inventory,discover_inventories=worklist_inventory,extra_databases=[str(POSE.relative_to(PROJECT))],
        pinned_files={str((ROOT/n).relative_to(PROJECT)):digest(ROOT/n) for n in ('dataset_manifest.json','source_binding.json','private/protection.json')})


def sources_current():
    guard=binding_guard()
    try:return guard.current()
    finally:guard.close()


def integrate(base,props,reviews,mode,classes,binding):
    require(mode in ('new','existing'),'Unknown completed-review mode')
    require(set(reviews)==set(props),'Every issued target must have a saved record')
    result=deepcopy(base);changes=Counter()
    for key,p in props.items():
        v=reviews[key]
        require(v['action']=='correct' and v['binding']==binding and v['source_binding']==sha(p) and v['source']==p['source'],'Latest completed/source binding mismatch')
        cls=148 if v['class_id']==149 else v['class_id'];info=classes[str(cls)];poly=deepcopy(v['polygon'])
        polygon_value(poly,*p['source']['size']);require(info['train_enabled'] and len(poly)==info['K'] and v['class_K_matches'],'Invalid completed classK')
        d=deepcopy(v['description']);validate_description(d)
        require((key in base)==(mode=='existing'),'Completed target new/existing identity mismatch')
        changes['class_changed']+=cls!=p['class_id'];changes['polygon_changed']+=poly!=p['polygon'];changes['text_changed']+=d['text']!=p['description']['text']
        changes['observation_changed']+=d['observation']!=p['description']['observation']
        old=result.get(key);chain=deepcopy(old.get('provenance_chain',[]) if old else [])
        if old:chain.append(dict(epoch='round7_training',revision=old['source_revision'],action=old['original_review_action'],source_payload_sha256=old['source_payload_sha256']))
        result[key]=dict(target_key=key,class_id=cls,K=info['K'],polygon=poly,source=p['source'],image_sha256=p['image_sha256'],mode=mode,
            original_review_action=v['action'],source_revision=v['revision'],source_payload_sha256=sha(v),prior_proposal_sha256=sha(p),
            geometry_source='latest_saved_human_correct_round7',completion_authorization=AUTHORIZATION,description=d,note=v.get('note',''),
            original_human_note=p.get('human_note',''),pose_payload_raw_sha256=p.get('pose_payload_raw_sha256'),provenance_chain=chain,**FALSE_FLAGS)
    return result,dict(changes)


def prepare():
    require(not ROOT.exists(),'Preserve previous round8 attempt');require(shutil.disk_usage(PROJECT/'outputs').free>40*2**30,'Disk reserve required')
    require(previous.sources_current(),'Prior frozen sources changed; reconcile before freeze');previous.verify()
    ROOT.mkdir(mode=0o700);private=ROOT/'private';private.mkdir(mode=0o700)
    for name in ('images','masks','valid','snapshots'):(private/name).mkdir(mode=0o700)
    formal,rows,evidence=load_formal_evidence(read(FORMAL_SOURCE));classes=read(PREVIOUS/'classes.json');parents=read(PREVIOUS/'parents.json')
    base={r['target_key']:r for r in jsonl(PREVIOUS/'private/training.jsonl')};oldsem={r['target_key']:r for r in jsonl(PREVIOUS/'description_data/records.jsonl')}
    records=deepcopy(base);snapshots={};changes={};pins={};updated=set()
    for mode,s in SOURCE_SETS.items():
        path=private/'snapshots'/f'{mode}.sqlite3';snapshot_db(s['reviews']/'mask_reviews.sqlite3',path)
        with closing(readonly(path)) as db:
            require(db.execute('PRAGMA quick_check').fetchall()==[('ok',)],'Damaged source snapshot')
            meta=dict(db.execute('SELECT key,value FROM metadata'));reviews={k:json.loads(raw) for k,raw in db.execute('SELECT target_key,payload FROM reviews')}
            events=list(db.execute('SELECT target_key,revision,payload FROM events ORDER BY id'))
        latest={}
        for key,rev,raw in events:
            value=json.loads(raw);require(value['revision']==rev and rev==latest.get(key,{}).get('revision',0)+1,'Noncontiguous append history');latest[key]=value
        binding=digest(s['root']/'manifest.json');require(latest==reviews and meta['namespace']==s['namespace'] and meta['formal_manifest_sha256']==binding,'History/namespace mismatch')
        props={p['target_key']:p for p in jsonl(s['root']/'proposals.jsonl')};require(not updated&set(props),'Duplicate target across review modes')
        if mode=='existing':
            validate_required(props,reviews,events,read(POLICY))
            pins[str(POLICY.relative_to(PROJECT))]=digest(POLICY)
        records,changes[mode]=integrate(records,props,reviews,mode,classes,binding)
        updated.update(k for k,v in reviews.items() if v['action']=='correct')
        snapshots[mode]=dict(events=len(events),saved=len(reviews),actions=dict(Counter(v['action'] for v in reviews.values())),snapshot_sha256=digest(path))
        for n in ('manifest.json','classes.json','proposals.jsonl'):pins[str((s['root']/n).relative_to(PROJECT))]=digest(s['root']/n)
    records=list(records.values());classes=policy.canonical_classes(classes,records);write(ROOT/'classes.json',classes);write(ROOT/'parents.json',parents)
    components=read(PREVIOUS/'private/components.json');hashes=read(PREVIOUS/'private/image_hashes.json');neighbors=defaultdict(list)
    for row in rows.values():neighbors[row['frame_id']].append(row)
    cv2.setNumThreads(1)
    for r in records:
        key=r['target_key'];row=r['source'];info=classes[str(r['class_id'])]
        require(rows[key]==row and hashes[row['image']]==r['image_sha256'] and digest(formal/row['image'])==r['image_sha256'],'Original source image changed')
        roi=context(row['bbox'],row['size']);x1,y1,x2,y2=roi;full=raster_mask(r['polygon'],*row['size']).astype(np.uint8);mask=full[y1:y2,x1:x2]
        require(mask.any() and full.sum()==mask.sum(),'Fixed bbox ROI clips current human contour: '+key)
        valid=supervision_valid(roi,neighbors[row['frame_id']],key,mask);image=evidence.image(row['frame_id'])[y1:y2,x1:x2]
        for folder,array in (('images',image),('masks',mask*255),('valid',valid*255)):require(cv2.imwrite(str(private/folder/f'{key}.png'),array),'Export failed')
        r.update(fine_index=info['fine_index'],parent=info['parent'],parent_index=info['parent_index'],small_sample=info['small_sample'],component=components['by_target'][key],
            roi=roi,image=f'private/images/{key}.png',mask=f'private/masks/{key}.png',valid=f'private/valid/{key}.png',training_origin='round8_latest_completed_overlay')
    split=split_components(records)
    for r in records:r['evaluation_branch_split']=split[r['target_key']]
    require(all(r['evaluation_branch_split']==base[r['target_key']]['evaluation_branch_split'] for r in records if r['target_key'] in base),'Old component split changed')
    write(private/'training.jsonl',records,lines=True);write(private/'components.json',components);write(private/'image_hashes.json',hashes)
    semantics=[]
    for r in records:
        scope={'text','legs','beams','material','observation'} if r['target_key'] in updated else set(oldsem[r['target_key']]['human_review_scope'])
        value=policy.semantic_record(r,classes[str(r['class_id'])],scope);value['prompt']=PROMPT;semantics.append(value)
    (ROOT/'description_data').mkdir(mode=0o700);write(ROOT/'description_data/records.jsonl',semantics,lines=True)
    write(ROOT/'description_data/conflicts.jsonl',[r for r in semantics if r['semantic_conflicts']],lines=True)
    inventories={n:digest(PROJECT/n) for n in worklist_inventory()};protected=set(read(PREVIOUS/'private/protection.json')['protected_image_hashes'])|{r['image_sha256'] for r in records}
    for n in inventories:
        for p in jsonl(PROJECT/n):
            if p.get('target_key') in rows:protected.add(hashes[rows[p['target_key']]['image']])
    candidates=[r for r in jsonl(PREVIOUS/'private/candidates.jsonl') if r['image_sha256'] not in protected and r['target_key'] not in split]
    write(private/'candidates.jsonl',candidates,lines=True);write(private/'protection.json',dict(inventories=inventories,protected_image_hashes=sorted(protected),no_previous_issued_image_reuse=True))
    b=read(PREVIOUS/'source_binding.json');files=dict(b['file_hashes']);files.update(pins);stores=dict(b['database_fingerprints'])
    for n in ('dataset_manifest.json','classes.json','parents.json','private/components.json','private/candidates.jsonl','private/training.jsonl'):files[str((PREVIOUS/n).relative_to(PROJECT))]=digest(PREVIOUS/n)
    for mode,s in SOURCE_SETS.items():
        path=s['reviews']/'mask_reviews.sqlite3';require(fingerprint(path)==fingerprint(private/'snapshots'/f'{mode}.sqlite3'),'Concurrent review changed during freeze')
        stores[str(path.relative_to(PROJECT))]=fingerprint(path)
    write(ROOT/'source_binding.json',dict(database_fingerprints=stores,file_hashes=files,snapshots=snapshots,created_at=datetime.now(timezone.utc).isoformat(),
        canonical_class_aliases={'149':148},previous_training_manifest_sha256=digest(PREVIOUS/'dataset_manifest.json')))
    write(ROOT/'research_permit.json',dict(user_authorization=AUTHORIZATION,actual_round7_actions=snapshots,source_events_not_rewritten=True,
        no_semantic_normal_negatives_inferred=True,no_automatic_GT_or_robot_promotion=True,**FALSE_FLAGS))
    report=dict(state='round8_latest_completed_reviews_frozen',targets=len(records),distinct_images=len({r['image_sha256'] for r in records}),
        newly_added_targets=len(records)-len(base),updated_existing_targets=len(updated&set(base)),fine_classes=sum(c['train_enabled'] for c in classes.values()),parents=len(parents),
        splits=dict(Counter(split.values())),sources=snapshots,changes_from_round7_task_initialization=changes,
        semantic_supervised_fields={k:sum(r['supervised_fields'][k] for r in semantics) for k in ('支腿','横梁','材质','结构','观察')},
        semantic_conflict_targets=sum(bool(r['semantic_conflicts']) for r in semantics),candidates=len(candidates),candidate_distinct_images=len({r['image_sha256'] for r in candidates}),
        small_classes=[c['class_id'] for c in classes.values() if c['train_enabled'] and c['small_sample']],training_class_counts=dict(Counter(r['class_id'] for r in records)),
        unchanged_component_hash_split=True,not_new_blind_test=True,**FALSE_FLAGS)
    write(ROOT/'preparation_report.json',report)
    write(ROOT/'dataset_manifest.json',dict(schema='round8-latest40fine9parent-v1',records=len(records),classes=40,parents=9,
        files={str(p.relative_to(ROOT)):digest(p) for p in ROOT.rglob('*') if p.is_file()},code_sha256=digest(__file__)))
    verify();require(sources_current(),'Source changed during freeze');print(json.dumps(report,ensure_ascii=False),flush=True)


def verify():
    m=read(ROOT/'dataset_manifest.json')
    for n,h in m['files'].items():require(digest(ROOT/n)==h,'Frozen round8 file changed: '+n)
    records=jsonl(ROOT/'private/training.jsonl');classes=read(ROOT/'classes.json');groups=defaultdict(set);images=defaultdict(set)
    require(len(records)==len({r['target_key'] for r in records})==m['records'],'Duplicate target')
    for r in records:
        c=classes[str(r['class_id'])];require(c['train_enabled'] and len(r['polygon'])==r['K']==c['K'] and r['fine_index']==c['fine_index'] and r['parent']==c['parent'],'Canonical class mismatch')
        groups[r['evaluation_branch_split']].add(r['component']);images[r['evaluation_branch_split']].add(r['image_sha256'])
    require(not groups['train']&groups['holdout'] and not images['train']&images['holdout'],'Image/component leakage')
    return m,records


if __name__=='__main__':prepare()
