"""Second taxonomy epoch: explicit batch acceptance, immutable provenance and new images."""
from collections import Counter,defaultdict
from contextlib import closing
from copy import deepcopy
import hashlib
import importlib.util
import json
from pathlib import Path
import shutil

import cv2
import numpy as np

from .cluster_targets import digest
from .mask_model_training_data import PROJECT,FALSE_FLAGS,read,write,jsonl,sha,require
from . import mask_taxonomy_data as old
from .mask_class_note_integration import snapshot_db
from .mask_quality_override import human_protection
from .mask_quality_recovery import POSE,fingerprint
from .mask_template_refined import readonly
from .mask_annotation_full_data import context,supervision_valid
from .mask_quota_saved_extension import load_formal_evidence
from .pose_review_geometry import polygon_value,raster_mask
from .mask_taxonomy_source_guard import SourceGuard

# Separate prefix deliberately avoids changing the legacy service's source inventory.
ROOT=PROJECT/'outputs/pallet_taxonomy_cycle2_training_v1'
REQUESTS=PROJECT/'outputs/pallet_taxonomy_cycle2_requests_v1'
REVIEWS=PROJECT/'outputs/pallet_taxonomy_cycle2_reviews_v1'
MANAGED=PROJECT/'outputs/pallet_taxonomy_cycle2_managed_v1'
NAMESPACE='current-fine41-parent9-cycle2-new-images-v1'
FORMAL_SOURCE=old.FORMAL_SOURCE
AUTHORIZATION='424张未保存的也已经完成了复核，继续'


def engine(path,tag):
    spec=importlib.util.spec_from_file_location('pallet._taxonomy_cycle2_'+tag,PROJECT/path)
    module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module);return module


def database_inventory():
    paths=set((PROJECT/'outputs').glob('pallet_mask*_reviews_v*/mask_reviews.sqlite3'))
    paths.update((PROJECT/'outputs').glob('pallet_taxonomy_cycle*_reviews_v*/mask_reviews.sqlite3'))
    return {str(p.relative_to(PROJECT)) for p in paths if p.parent!=REVIEWS}


def worklist_inventory():
    return {str(p.relative_to(PROJECT)) for prefix in ('pallet_mask*','pallet_taxonomy_cycle*')
        for name in ('requests.jsonl','proposals.jsonl','tasks.jsonl','inspection_targets.jsonl')
        for p in (PROJECT/'outputs').glob(prefix+'/'+name)
        if p.parent not in (ROOT,REQUESTS) and (p.parent/'manifest.json').exists()}


def source_guard(manifest_sha256=None):
    binding=read(ROOT/'source_binding.json');dataset=read(ROOT/'dataset_manifest.json')
    require(dataset['files']['source_binding.json']==digest(ROOT/'source_binding.json'),'Dataset/source binding changed')
    pinned={str((ROOT/name).relative_to(PROJECT)):digest(ROOT/name) for name in ('dataset_manifest.json','source_binding.json','private/protection.json')}
    if manifest_sha256:
        m=read(REQUESTS/'manifest.json')
        require(digest(REQUESTS/'manifest.json')==manifest_sha256 and m['training_dataset_manifest_sha256']==digest(ROOT/'dataset_manifest.json')
            and m['source_binding_sha256']==digest(ROOT/'source_binding.json'),'Task/model data binding changed')
        pinned[str((REQUESTS/'manifest.json').relative_to(PROJECT))]=manifest_sha256
    return SourceGuard(project=PROJECT,databases=binding['database_fingerprints'],files=binding['file_hashes'],
        inventories=read(ROOT/'private/protection.json')['inventories'],discover_databases=database_inventory,
        discover_inventories=worklist_inventory,extra_databases=[str(POSE.relative_to(PROJECT))],pinned_files=pinned)


def sources_current():
    guard=source_guard()
    try:return guard.current()
    finally:guard.close()


def integrate(proposals,reviews,classes,binding,*,accept_unsaved=False):
    """A batch confirmation is separate provenance, never a fabricated saved event."""
    require(set(reviews)<=set(proposals),'Review target outside issued batch')
    result=[];excluded=[]
    for key,p in proposals.items():
        review=reviews.get(key)
        if review:
            require(review['target_key']==key and review['binding']==binding and review['source_binding']==sha(p)
                and review['base_proposal_id']==sha(p) and review['source']==p['source']
                and review['checkpoint_sha256']==p['checkpoint_sha256'],'Latest review/source binding mismatch')
            if review['action']!='correct':
                excluded.append(dict(target_key=key,reason='latest_action_not_completed',action=review['action'],revision=review['revision']));continue
            cls=review['class_id'];poly=review['polygon'];kind='latest_saved_human_correct';action=review['action'];revision=review['revision'];payload_sha=sha(review)
        else:
            if not accept_unsaved:
                excluded.append(dict(target_key=key,reason='no_explicit_batch_confirmation'));continue
            cls=p['class_id'];poly=p['polygon'];kind='explicit_user_batch_acceptance_of_unchanged_proposal';action='never_saved';revision=None;payload_sha=sha(p)
        info=classes.get(str(cls));require(info and info['train_enabled'],'Unresolved fine class cannot enter training')
        polygon_value(poly,*p['source']['size']);require(len(poly)==info['K'],'Accepted human fine-class K differs')
        if review:require(review['vertex_count']==len(poly) and review['expected_K']==info['K'] and review['class_K_matches'] is True,'Saved human K metadata inconsistent')
        result.append(dict(target_key=key,class_id=cls,K=info['K'],polygon=deepcopy(poly),source=p['source'],image_sha256=p['image_sha256'],
            source_revision=revision,source_payload_sha256=payload_sha,geometry_source=kind,original_review_action=action,
            prior_proposal_sha256=sha(p),prior_checkpoint_sha256=p['checkpoint_sha256'],
            batch_confirmation_authorization=AUTHORIZATION if review is None else None,**FALSE_FLAGS))
    return result,excluded


def update_classes(classes,records):
    result=deepcopy(classes)
    for info in result.values():
        members=[r for r in records if r['class_id']==info['class_id']]
        if not info['train_enabled']:continue
        info['previous_targets']=info['targets'];info['previous_distinct_images']=info['distinct_images']
        info['targets']=len(members);info['distinct_images']=len({r['image_sha256'] for r in members});info['small_sample']=info['distinct_images']<20
        info['training_group']=info['parent'] if info['small_sample'] else 'fine-'+str(info['class_id'])
    return result


def prepare():
    require(not ROOT.exists(),'Use a fresh immutable second-cycle dataset')
    require(shutil.disk_usage(PROJECT/'outputs').free>30*2**30,'Preserve disk reserve; no deletions authorized')
    old_manifest,base=old.verify();require(old.sources_current(),'Original taxonomy graph/protection changed')
    _,_,revision,_,_,_,classes,parents=old.source_state();require(revision==5,'Changed taxonomy needs a separate integration audit')
    props={p['target_key']:p for p in jsonl(old.REQUESTS/'proposals.jsonl')};binding=digest(old.REQUESTS/'manifest.json')
    ROOT.mkdir(mode=0o700);(ROOT/'private').mkdir(mode=0o700)
    snapshot_db(old.REVIEWS/'mask_reviews.sqlite3',ROOT/'private/cycle1_reviews.sqlite3')
    with closing(readonly(ROOT/'private/cycle1_reviews.sqlite3')) as db:
        db.execute('BEGIN');meta=dict(db.execute('SELECT key,value FROM metadata'))
        saved={k:json.loads(v) for k,v in db.execute('SELECT target_key,payload FROM reviews')}
        events=list(db.execute('SELECT id,target_key,revision,payload FROM events ORDER BY id'))
    require(meta['formal_manifest_sha256']==binding and meta['namespace']==old.NAMESPACE,'Snapshot belongs to another batch')
    last={key:(rev,raw) for _,key,rev,raw in events}
    require(set(last)==set(saved) and all(saved[k]==json.loads(raw) and saved[k]['revision']==rev for k,(rev,raw) in last.items()),'Latest rows differ from append-only event history')
    require(Counter(v['action'] for v in saved.values())==Counter(correct=1405,draft=2) and len(set(props)-set(saved))==424,'Source count changed since explicit424 confirmation; re-audit before use')
    accepted,excluded=integrate(props,saved,classes,binding,accept_unsaved=True)
    require(len(accepted)==1829 and len(excluded)==2 and not {r['target_key'] for r in base}&{r['target_key'] for r in accepted},'Expected1829 new accepted targets, no repeated training target')
    classes=update_classes(classes,base+accepted)
    write(ROOT/'classes.json',classes);write(ROOT/'parents.json',parents)
    write(ROOT/'private/accepted_cycle1.jsonl',accepted,lines=True);write(ROOT/'private/excluded_cycle1.jsonl',excluded,lines=True)
    write(ROOT/'research_permit.json',dict(user_request='已经完成数据集复核。更新数据集再次训练、增强分类、再提供每类100张严格K新图',
        clarification=AUTHORIZATION,unsaved_424_explicitly_batch_accepted=True,two_latest_drafts_excluded=True,
        original_review_actions_and_events_not_rewritten=True,scope='versioned_research_annotation_assistant_supervision_only',
        formal_labels_GT_robot_eligibility_not_promoted=True,**FALSE_FLAGS))
    formal,rows,evidence=load_formal_evidence(read(FORMAL_SOURCE));labels=read(old.ROOT/'private/components.json')['by_target'];hashes=read(old.ROOT/'private/image_hashes.json')
    require(set(labels)==set(rows),'Inherited global component inventory differs')
    for image,h in hashes.items():require(digest(formal/image)==h,'Original image changed since prior taxonomy')
    write(ROOT/'private/components.json',read(old.ROOT/'private/components.json'));write(ROOT/'private/image_hashes.json',hashes)
    neighbors=defaultdict(list)
    for row in rows.values():neighbors[row['frame_id']].append(row)
    for folder in ('images','masks','valid'):(ROOT/'private'/folder).mkdir(mode=0o700)
    training=[]
    for r in base:
        value=deepcopy(r);value['small_sample']=classes[str(r['class_id'])]['small_sample'];value['training_origin']='prior_2198_human_training_record'
        for field in ('image','mask','valid'):shutil.copyfile(old.ROOT/r[field],ROOT/r[field])
        training.append(value)
    cv2.setNumThreads(1)
    for r in accepted:
        key=r['target_key'];row=rows[key];info=classes[str(r['class_id'])]
        require(row==r['source'] and hashes[row['image']]==r['image_sha256'],'Accepted target original source changed')
        roi=context(row['bbox'],row['size']);x1,y1,x2,y2=roi;full=raster_mask(r['polygon'],*row['size']).astype(np.uint8);mask=full[y1:y2,x1:x2]
        require(mask.any() and full.sum()==mask.sum(),'Fixed original-bbox ROI clips accepted contour; never crop labels silently')
        image=evidence.image(row['frame_id'])[y1:y2,x1:x2];valid=supervision_valid(roi,neighbors[row['frame_id']],key,mask)
        for folder,array in (('images',image),('masks',mask*255),('valid',valid*255)):require(cv2.imwrite(str(ROOT/'private'/folder/f'{key}.png'),array),'Supervision image export failed')
        training.append(dict(**r,fine_index=info['fine_index'],parent=info['parent'],parent_index=info['parent_index'],small_sample=info['small_sample'],
            component=labels[key],roi=roi,image=f'private/images/{key}.png',mask=f'private/masks/{key}.png',valid=f'private/valid/{key}.png',training_origin='cycle1_explicitly_confirmed'))
    split=old.split_components(training)
    for r in training:r['evaluation_branch_split']=split[r['target_key']]
    require(len(training)==4027 and len({r['target_key'] for r in training})==4027,'Keep all4027 target-level supervision records')
    write(ROOT/'private/training.jsonl',training,lines=True)
    protected,pose_current,stores,reasons=human_protection(rows,training)
    inventories={name:digest(PROJECT/name) for name in worklist_inventory()};issued=defaultdict(list)
    for name in inventories:
        for r in jsonl(PROJECT/name):
            if r.get('target_key') in rows:issued[r['target_key']].append(name);protected.add(r['target_key'])
    # Exclude all prior issued images as well as any human/held/deleted history image.
    protected_hashes={hashes[rows[k]['image']] for k in protected if k in rows}
    protected_hashes.update(p['image_sha256'] for p in props.values())
    candidates=[];exclusions=[]
    for key,row in sorted(rows.items()):
        why=[];b=row['bbox'];w,h=row['size']
        if key in protected:why.append('human_history_or_prior_issued_target')
        if hashes[row['image']] in protected_hashes:why.append('image_already_human_or_issued_protected')
        if not np.isfinite([b[n] for n in ('x','y','width','height')]).all() or min(b['width'],b['height'])<=0 or min(w,b['x']+b['width'])<=max(0,b['x']) or min(h,b['y']+b['height'])<=max(0,b['y']):why.append('invalid_original_bbox')
        if why:exclusions.append(dict(target_key=key,reasons=why));continue
        candidates.append(dict(target_key=key,source=row,image_sha256=hashes[row['image']],component=labels[key],pose_payload_raw_sha256=hashlib.sha256(pose_current[key].encode()).hexdigest() if key in pose_current else None))
    write(ROOT/'private/candidates.jsonl',candidates,lines=True);write(ROOT/'private/exclusions.jsonl',exclusions,lines=True)
    write(ROOT/'private/protection.json',dict(human_reasons=reasons,issued_task_membership=dict(issued),inventories=inventories,protected_image_hashes=sorted(protected_hashes)))
    # Bind every raw source DB including the now-completed first taxonomy batch.
    stores[str(POSE.relative_to(PROJECT))]=fingerprint(POSE)
    require(set(stores)-{str(POSE.relative_to(PROJECT))}==database_inventory(),'Source DB inventory changed during freeze')
    require(fingerprint(old.REVIEWS/'mask_reviews.sqlite3')==fingerprint(ROOT/'private/cycle1_reviews.sqlite3'),'Concurrent first-cycle review edit during freeze')
    files=dict(read(old.ROOT/'source_binding.json')['file_hashes'])
    for p in (old.ROOT/'dataset_manifest.json',old.ROOT/'private/training.jsonl',old.ROOT/'classes.json',old.ROOT/'parents.json',old.ROOT/'private/components.json',old.REQUESTS/'manifest.json',old.REQUESTS/'proposals.jsonl',old.REQUESTS/'classes.json'):
        files[str(p.relative_to(PROJECT))]=digest(p)
    files.update(inventories)
    write(ROOT/'source_binding.json',dict(class_graph_revision=revision,database_fingerprints=stores,file_hashes=files,
        prior_training_manifest_sha256=digest(old.ROOT/'dataset_manifest.json'),prior_request_manifest_sha256=binding,
        cycle1_raw_event_count=len(events),cycle1_snapshot_sha256=digest(ROOT/'private/cycle1_reviews.sqlite3'),authorization_sha256=digest(ROOT/'research_permit.json'),
        future_live_source_changes_require_new_version=True))
    report=dict(state='frozen_cycle2_human_confirmed_4027_targets',training_targets=len(training),previous_training_targets=len(base),
        new_saved_correct_targets=1405,new_explicit_batch_confirmed_unsaved_targets=424,excluded_latest_drafts=excluded,
        fine_classes=41,parent_classes=len(parents),excluded_unresolved_classes=old.EXCLUDED,
        distinct_training_images=len({r['image_sha256'] for r in training}),training_class_counts=dict(Counter(r['class_id'] for r in training)),
        small_classes=[c['class_id'] for c in classes.values() if c['train_enabled'] and c['small_sample']],
        evaluation_splits=dict(Counter(r['evaluation_branch_split'] for r in training)),candidate_targets=len(candidates),candidate_distinct_images=len({r['image_sha256'] for r in candidates}),
        older_models_selected_part_of_new_review_data=True,evaluation_is_not_a_new_blind_prospective_test=True,
        source_labels_unchanged=True,source_events_unchanged=True,**FALSE_FLAGS)
    write(ROOT/'preparation_report.json',report)
    files={str(p.relative_to(ROOT)):digest(p) for p in ROOT.rglob('*') if p.is_file()}
    write(ROOT/'dataset_manifest.json',dict(schema=NAMESPACE,files=files,records=len(training),classes=41,parents=9,code_sha256=digest(__file__),**FALSE_FLAGS))
    verify();require(sources_current(),'Concurrent source change during preparation');print(json.dumps(report,ensure_ascii=False),flush=True)


def verify():
    m=read(ROOT/'dataset_manifest.json')
    for name,h in m['files'].items():require(digest(ROOT/name)==h,'Frozen second-cycle data changed: '+name)
    records=jsonl(ROOT/'private/training.jsonl');classes=read(ROOT/'classes.json');sets=defaultdict(set);images=defaultdict(set)
    require(len(records)==len({r['target_key'] for r in records})==m['records']==4027,'Distinct4027 training targets required')
    for r in records:
        info=classes[str(r['class_id'])];require(info['train_enabled'] and len(r['polygon'])==r['K']==info['K'],'Original fine class/K not preserved')
        require(r['parent']==info['parent'] and r['parent_index']==info['parent_index'],'Training parent differs from immutable fine class')
        sets[r['evaluation_branch_split']].add(r['component']);images[r['evaluation_branch_split']].add(r['image_sha256'])
    require(not sets['train']&sets['holdout'] and not images['train']&images['holdout'],'Component/image leakage across generic-start evaluation')
    return m,records
