"""Versioned latest-review freeze for round4; source stores are always read-only."""
from collections import Counter,defaultdict
from contextlib import closing
from copy import deepcopy
from datetime import datetime,timezone
import json
import re
import shutil

import cv2
import numpy as np

from .cluster_targets import digest
from .mask_model_training_data import PROJECT,FALSE_FLAGS,read,write,jsonl,sha,require
from .mask_class_note_integration import snapshot_db
from .mask_template_refined import readonly
from .mask_quality_recovery import fingerprint,POSE
from .mask_annotation_full_data import context,supervision_valid
from .mask_quota_saved_extension import load_formal_evidence
from .mask_taxonomy_data import split_components
from .mask_taxonomy_source_guard import SourceGuard
from .pallet_description_data import ROOT as PREVIOUS,FORMAL_SOURCE
from .pallet_description_review import SETS,validate_description
from .pallet_description_simple import initial_description
from . import mask_taxonomy_cycle2_data as prior
from .pose_review_geometry import polygon_value,raster_mask
from scripts.summarize_pallet_dataset_stage_20261107 import text_structure,attributes_for,ANOMALY_WORDS

ROOT=PROJECT/'outputs/pallet_description_round4_training_v2'
AUTHORIZATION='现已完成新数据标注和老数据语义标注和部分标签更新；继续VLM模型和标注模型的训练，并在训练后提供下一轮标注与语义的人工复核；执行'
PROMPT='请根据图中指定托盘输出且只输出一个JSON对象，字段依次为"支腿"、"横梁"、"材质"、"结构"、"观察"。支腿为2/3/4或null，横梁为1/2或null，材质为塑料/木质/金属/无法判断，结构为简短中文字符串，观察为可见遮挡、出视野、视角或图像质量的中文字符串，无法确定时为null。所有字符串使用双引号，不输出解释或Markdown。完整结构不等于可见部件，不将遮挡或出视野说成损坏，不推测不可见细节。'


def database_inventory():
    paths=set((PROJECT/'outputs').glob('pallet_mask*_reviews_v*/mask_reviews.sqlite3'))
    paths.update((PROJECT/'outputs').glob('pallet_taxonomy_cycle*_reviews_v*/mask_reviews.sqlite3'))
    return {str(p.relative_to(PROJECT)) for p in paths}|{str((s['reviews']/'mask_reviews.sqlite3').relative_to(PROJECT)) for s in SETS.values()}


def worklist_inventory():
    return prior.worklist_inventory()|{str((s['root']/'proposals.jsonl').relative_to(PROJECT)) for s in SETS.values()}


def binding_guard():
    b=read(ROOT/'source_binding.json');p=read(ROOT/'private/protection.json')
    return SourceGuard(project=PROJECT,databases=b['database_fingerprints'],files=b['file_hashes'],inventories=p['inventories'],
        discover_databases=database_inventory,discover_inventories=worklist_inventory,extra_databases=[str(POSE.relative_to(PROJECT))],
        pinned_files={str((ROOT/n).relative_to(PROJECT)):digest(ROOT/n) for n in ('dataset_manifest.json','source_binding.json','private/protection.json')})


def sources_current():
    guard=binding_guard()
    try:return guard.current()
    finally:guard.close()


def canonical_classes(source,records):
    classes=deepcopy(source);classes['149'].update(train_enabled=False,exclusion_reason='user_authorized_alias_into148',merged_into=148)
    enabled=sorted([c for c in classes.values() if c['train_enabled']],key=lambda c:c['class_id'])
    for index,c in enumerate(enabled):
        rs=[r for r in records if r['class_id']==c['class_id']]
        c.update(fine_index=index,previous_targets=c['targets'],targets=len(rs),distinct_images=len({r['image_sha256'] for r in rs}))
        c['small_sample']=c['distinct_images']<20;c['training_group']=c['parent'] if c['small_sample'] else 'fine-'+str(c['class_id'])
        require(rs,'Empty canonical training class')
    return classes


def observation_text(description,text_confirmed):
    if description['observation'].strip():return description['observation'].strip(),'exact_observation_field'
    if text_confirmed:
        for match in re.finditer(r'[^，,；;。]+',description['text']):
            if any(t in match.group() for t in ANOMALY_WORDS):return description['text'][match.start():].strip(),'exact_contiguous_human_structure_text_suffix'
    return None,'unlabelled_not_normal_negative'


def semantic_record(record,info,scope):
    d=record['description'];selected={'支腿':d['legs'],'横梁':d['beams'],'材质':d['material'],'结构':d['text'],'观察':None}
    expected=attributes_for(info);explicit=text_structure(d['text']);mapping={'legs':'支腿','beams':'横梁','material':'材质'}
    supervised={key:False for key in selected};conflicts={}
    for key,label in mapping.items():
        value=d[key];conflict=(value not in (None,'无法判断') and (value!=expected[key] or key in explicit and value!=explicit[key])) or (key in explicit and explicit[key]!=expected[key])
        if conflict:conflicts[key]=dict(selected=value,class_definition=expected[key],text_explicit=explicit.get(key))
        supervised[label]=key in scope and not conflict and value not in (None,'无法判断')
    supervised['结构']='text' in scope
    note,source=observation_text(d,supervised['结构'])
    supervised['观察']=bool(note and ('observation' in scope or source=='exact_contiguous_human_structure_text_suffix' or record['original_human_note']==note))
    selected['观察']=note if supervised['观察'] else None
    return dict(target_key=record['target_key'],class_id=record['class_id'],image=record['image'],source=record['source'],image_sha256=record['image_sha256'],
        component=record['component'],split=record['evaluation_branch_split'],target=selected,supervised_fields=supervised,observation_supervised=supervised['观察'],
        observation_source=source,original_description=d,original_action=record['original_review_action'],semantic_conflicts=conflicts,
        human_review_scope=sorted(scope),missing_observation_is_normal=False,prompt=PROMPT)


def prepare():
    require(not ROOT.exists(),'Preserve previous round4 attempt; fresh version required')
    require(shutil.disk_usage(PROJECT/'outputs').free>40*2**30,'Disk reserve required')
    from .pallet_description_data import sources_current as previous_current
    require(previous_current(),'Historical sources changed; investigate before freezing')
    ROOT.mkdir(mode=0o700);private=ROOT/'private';private.mkdir(mode=0o700)
    for n in ('images','masks','valid','snapshots'):(private/n).mkdir(mode=0o700)
    formal,rows,evidence=load_formal_evidence(read(FORMAL_SOURCE));components=read(PREVIOUS/'private/components.json');hashes=read(PREVIOUS/'private/image_hashes.json')
    records=[];scopes={};snapshots={};pins={};tax=read(PREVIOUS/'classes.json')
    for mode,s in SETS.items():
        path=private/'snapshots'/f'{mode}.sqlite3';snapshot_db(s['reviews']/'mask_reviews.sqlite3',path)
        with closing(readonly(path)) as db:
            meta=dict(db.execute('SELECT key,value FROM metadata'));saved={k:json.loads(raw) for k,raw in db.execute('SELECT target_key,payload FROM reviews')}
            events=[(k,rev,json.loads(raw)) for k,rev,raw in db.execute('SELECT target_key,revision,payload FROM events ORDER BY id')]
        props=jsonl(s['root']/'proposals.jsonl');binding=digest(s['root']/'manifest.json')
        require(meta['namespace']==s['namespace'] and meta['formal_manifest_sha256']==binding,'Wrong source epoch')
        require(read(s['root']/'classes.json')==tax,'Source taxonomies differ')
        latest={};scope=defaultdict(set)
        for key,rev,value in events:
            require(value['revision']==rev and rev==latest.get(key,{}).get('revision',0)+1,'Invalid revision history');latest[key]=value
            kind=value['event_kind']
            if kind=='human_joint_class_keypoints_description_review':scope[key].update(('text','legs','beams','material','observation'))
            elif kind=='human_batch_structure_text_confirmation':scope[key].add('text')
            elif kind=='human_batch_structure_attributes_confirmation':scope[key].update(('legs','beams','material'))
        require(latest==saved and set(saved)<={p['target_key'] for p in props},'History/current mismatch')
        for p in props:
            key=p['target_key'];v=saved.get(key);effective=v or p
            if v:require(v['binding']==binding and v['source_binding']==sha(p) and v['source']==p['source'] and v['action'] in ('correct','draft'),'Review/source binding or action mismatch')
            cls=148 if effective['class_id']==149 else effective['class_id'];poly=deepcopy(effective['polygon']);info=tax[str(cls)]
            polygon_value(poly,*p['source']['size']);require(info['train_enabled'] and len(poly)==info['K'],'Invalid accepted geometry')
            d=deepcopy(v['description'] if v else initial_description(p));validate_description(d)
            require(rows[key]==p['source'] and digest(formal/p['source']['image'])==p['image_sha256'],'Source image changed')
            records.append(dict(target_key=key,class_id=cls,K=info['K'],polygon=poly,source=p['source'],image_sha256=p['image_sha256'],mode=mode,
                original_review_action=v['action'] if v else 'never_saved',source_revision=v['revision'] if v else None,source_payload_sha256=sha(v or p),
                prior_proposal_sha256=sha(p),geometry_source='latest_saved_human_correct' if v and v['action']=='correct' else 'explicit_user_completed_batch_latest_draft_or_unchanged',
                completion_authorization=AUTHORIZATION,description=d,note=v.get('note','') if v else '',original_human_note=p.get('human_note',''),
                pose_payload_raw_sha256=p.get('pose_payload_raw_sha256'),**FALSE_FLAGS));scopes[key]=scope[key]
        snapshots[mode]=dict(events=len(events),reviews=len(saved),actions=dict(Counter(v['action'] for v in saved.values())),unsaved=len(props)-len(saved),snapshot_sha256=digest(path))
        for n in ('manifest.json','proposals.jsonl','classes.json'):pins[str((s['root']/n).relative_to(PROJECT))]=digest(s['root']/n)
    require(len(records)==len({r['target_key'] for r in records}),'Repeated target across epochs')
    classes=canonical_classes(tax,records);parents=read(PREVIOUS/'parents.json');write(ROOT/'classes.json',classes);write(ROOT/'parents.json',parents)
    neighbors=defaultdict(list)
    for row in rows.values():neighbors[row['frame_id']].append(row)
    cv2.setNumThreads(1)
    for r in records:
        key=r['target_key'];row=r['source'];info=classes[str(r['class_id'])];roi=context(row['bbox'],row['size']);x1,y1,x2,y2=roi
        full=raster_mask(r['polygon'],*row['size']).astype(np.uint8);mask=full[y1:y2,x1:x2]
        require(mask.any() and full.sum()==mask.sum(),'Fixed original bbox ROI clips current human polygon: '+key)
        valid=supervision_valid(roi,neighbors[row['frame_id']],key,mask);image=evidence.image(row['frame_id'])[y1:y2,x1:x2]
        for n,array in (('images',image),('masks',mask*255),('valid',valid*255)):require(cv2.imwrite(str(private/n/f'{key}.png'),array),'Supervision export failed')
        r.update(fine_index=info['fine_index'],parent=info['parent'],parent_index=info['parent_index'],small_sample=info['small_sample'],component=components['by_target'][key],
            roi=roi,image=f'private/images/{key}.png',mask=f'private/masks/{key}.png',valid=f'private/valid/{key}.png',training_origin='round4_latest_joint_review')
    split=split_components(records)
    for r in records:r['evaluation_branch_split']=split[r['target_key']]
    write(private/'training.jsonl',records,lines=True);write(private/'components.json',components);write(private/'image_hashes.json',hashes)
    semantics=[semantic_record(r,classes[str(r['class_id'])],scopes[r['target_key']]) for r in records]
    (ROOT/'description_data').mkdir(mode=0o700);write(ROOT/'description_data/records.jsonl',semantics,lines=True)
    write(ROOT/'description_data/conflicts.jsonl',[r for r in semantics if r['semantic_conflicts']],lines=True)
    inventories={n:digest(PROJECT/n) for n in worklist_inventory()};protected=set(read(PREVIOUS/'private/protection.json')['protected_image_hashes'])|{r['image_sha256'] for r in records}
    for n in inventories:
        for p in jsonl(PROJECT/n):
            if p.get('target_key') in rows:protected.add(hashes[rows[p['target_key']]['image']])
    candidates=[r for r in jsonl(PREVIOUS/'private/candidates.jsonl') if r['image_sha256'] not in protected and r['target_key'] not in split]
    write(private/'candidates.jsonl',candidates,lines=True);write(private/'protection.json',dict(inventories=inventories,protected_image_hashes=sorted(protected),no_previous_issued_image_reuse=True))
    files=dict(read(PREVIOUS/'source_binding.json')['file_hashes']);files.update(pins)
    for n in ('dataset_manifest.json','classes.json','parents.json','private/components.json','private/candidates.jsonl'):files[str((PREVIOUS/n).relative_to(PROJECT))]=digest(PREVIOUS/n)
    stores=dict(read(PREVIOUS/'source_binding.json')['database_fingerprints'])
    for mode,s in SETS.items():
        path=s['reviews']/'mask_reviews.sqlite3';require(fingerprint(path)==fingerprint(private/'snapshots'/f'{mode}.sqlite3'),'Concurrent review change during freeze')
        stores[str(path.relative_to(PROJECT))]=fingerprint(path)
    write(ROOT/'source_binding.json',dict(database_fingerprints=stores,file_hashes=files,snapshots=snapshots,created_at=datetime.now(timezone.utc).isoformat(),
        previous_training_manifest_sha256=digest(PREVIOUS/'dataset_manifest.json'),canonical_class_aliases={'149':148}))
    write(ROOT/'research_permit.json',dict(user_authorization=AUTHORIZATION,source_actions_and_history_preserved=True,unsaved_and_drafts_batch_accepted_not_fake_correct_events=True,
        semantic_loss_only_confirmed_nonconflicting_fields=True,no_automatic_GT_or_robot_promotion=True,**FALSE_FLAGS))
    report=dict(state='round4_latest_annotations_frozen',targets=len(records),distinct_images=len({r['image_sha256'] for r in records}),fine_classes=sum(c['train_enabled'] for c in classes.values()),
        parents=len(parents),splits=dict(Counter(split.values())),sources=snapshots,small_classes=[c['class_id'] for c in classes.values() if c['train_enabled'] and c['small_sample']],
        semantic_supervised_fields={k:sum(r['supervised_fields'][k] for r in semantics) for k in ('支腿','横梁','材质','结构','观察')},
        semantic_conflict_targets=sum(bool(r['semantic_conflicts']) for r in semantics),candidates=len(candidates),candidate_distinct_images=len({r['image_sha256'] for r in candidates}),
        inherited_component_hash_split=True,prior_model_selection_exposure_disclosed=True,not_a_new_prospective_blind_test=True,**FALSE_FLAGS)
    write(ROOT/'preparation_report.json',report)
    write(ROOT/'dataset_manifest.json',dict(schema='round4-latest40fine9parent-v1',records=len(records),classes=40,parents=9,
        files={str(p.relative_to(ROOT)):digest(p) for p in ROOT.rglob('*') if p.is_file()},code_sha256=digest(__file__)))
    verify();require(sources_current(),'Source changed during freeze');print(json.dumps(report,ensure_ascii=False),flush=True)


def verify():
    manifest=read(ROOT/'dataset_manifest.json')
    for n,h in manifest['files'].items():require(digest(ROOT/n)==h,'Frozen round4 file changed: '+n)
    records=jsonl(ROOT/'private/training.jsonl');classes=read(ROOT/'classes.json');groups=defaultdict(set);images=defaultdict(set)
    require(len(records)==len({r['target_key'] for r in records})==manifest['records'],'Unique target count mismatch')
    for r in records:
        c=classes[str(r['class_id'])];require(c['train_enabled'] and r['K']==len(r['polygon'])==c['K'] and r['fine_index']==c['fine_index'] and r['parent']==c['parent'],'Canonical label mismatch')
        groups[r['evaluation_branch_split']].add(r['component']);images[r['evaluation_branch_split']].add(r['image_sha256'])
    require(not groups['train']&groups['holdout'] and not images['train']&images['holdout'],'Component or image leakage')
    return manifest,records


if __name__=='__main__':prepare()
