"""New taxonomy epoch: immutable fine labels, component isolation and protected pool.

Only new files are written. Original class IDs in formal rows remain provenance,
never training labels. Multiple targets in an image retain target-local masks.
"""
from collections import Counter, defaultdict
from contextlib import closing
import hashlib
import json
import re
import shutil

import cv2
import numpy as np

from .cluster_targets import digest
from .mask_annotation_full_data import context, supervision_valid
from .mask_class_correction_review import compute, review_signatures
from .mask_class_note_integration import ACTIVE, snapshot_db
from .mask_model_training_data import (PROJECT, FALSE_FLAGS, Components, jsonl,
    read, require, sha, write)
from .mask_quality_override import human_protection
from .mask_quality_recovery import POSE, fingerprint
from .mask_quota_saved_extension import load_formal_evidence
from .mask_template_refined import readonly
from .pose_review_geometry import polygon_value, raster_mask

ROOT = PROJECT/'outputs/pallet_mask_taxonomy_training_v1'
REQUESTS = PROJECT/'outputs/pallet_mask_taxonomy_requests_v1'
REVIEWS = PROJECT/'outputs/pallet_mask_taxonomy_reviews_v1'
MANAGED = PROJECT/'outputs/pallet_mask_taxonomy_managed_v1'
NAMESPACE = 'current-fine41-parent9-new-images-v1'
EXCLUDED = {37: 'unresolved_anomalous_human_contour', 47: 'unresolved_occlusion_K14'}
COMPONENT_SOURCE = PROJECT/'outputs/pallet_mask_annotation_update_v2/private/global_components.json'
FORMAL_SOURCE = PROJECT/'outputs/pallet_mask_remaining_round5_v1/manifest.json'


def parent_attributes(text):
    """Main crossbars only: reinforcing ribs do not change the beam count."""
    legs = re.findall(r'([234])支腿', text)
    material = [m for m in ('塑料', '木质', '金属') if m in text]
    require(len(legs) == len(material) == 1, 'Ambiguous/missing human parent attributes')
    beams = 2 if '上下横梁' in text else 1 if '上横梁' in text else None
    require(beams is not None, 'Unrecognized human main-crossbar count')
    return dict(legs=int(legs[0]), beams=beams, material=material[0])


def source_state():
    active = read(ACTIVE); source = PROJECT/active['root']; database = PROJECT/active['reviews']/'mask_reviews.sqlite3'
    require(digest(source/'manifest.json') == active['manifest_sha256'], 'Active source package changed')
    with closing(readonly(database)) as db:
        db.execute('BEGIN')
        revision, raw = db.execute("SELECT revision,payload FROM reviews WHERE target_key='class_graph'").fetchone()
        events = [json.loads(v) for v, in db.execute("SELECT payload FROM events WHERE target_key='class_graph' ORDER BY revision")]
    records = jsonl(source/'private/records.jsonl'); state = json.loads(raw)['state']; groups, _ = compute(records, state)
    require(all(state['group_status'].get(f) == 'confirm' for f in groups), 'All current fine classes must be confirmed')
    require(len(records) == 2200 and len(groups) == 43, 'Latest taxonomy changed; re-audit scope before training')
    signatures = review_signatures(state, groups); attributes = read(source/'class_attributes.json')
    # Latest matching human confirmation overrides an older display attribute,
    # but both original texts and the complete immutable graph remain available.
    latest = {}
    for event in events:
        face = event.get('face_id')
        if event['action'] != 'confirm' or face not in groups: continue
        previous, _ = compute(records, event['state'])
        if review_signatures(event['state'], previous).get(face) == signatures[face]: latest[face] = event
    classes = {}; parents = {}
    for face, group in sorted(groups.items(), key=lambda item: item[1]['proposed_class_id']):
        cls = group['proposed_class_id']; original = attributes[face]
        text = latest[face]['note'] if face in latest else original['attributes_text']
        parsed = parent_attributes(re.split(r'注[：:]', text)[0])
        token = f"{parsed['legs']}legs-{parsed['beams']}beams-{parsed['material']}"
        if token not in parents: parents[token] = dict(index=len(parents), **parsed)
        members = [r for r in records if state['assignments'][r['target_key']] == face]
        image_count = len({r['image_sha256'] for r in members})
        classes[str(cls)] = dict(class_id=cls, face_id=face, K=group['K'], targets=len(members),
            distinct_images=image_count, small_sample=image_count < 20, parent=token,
            parent_index=parents[token]['index'], attributes_text=text, original_attributes=original,
            current_confirmation_revision=latest[face]['revision'] if face in latest else None,
            train_enabled=cls not in EXCLUDED, exclusion_reason=EXCLUDED.get(cls))
    enabled = [c for c in classes.values() if c['train_enabled']]
    for index, c in enumerate(enabled): c['fine_index'] = index
    require(len(enabled) == 41 and len(parents) == 9, 'Expected current41 fine classes/9 parents')
    return source, database, revision, raw, records, state, classes, parents


def issued_inventory(rows):
    """Conservative protection of *issued* task inventories, not model shortlists."""
    reasons = defaultdict(set); inventories = {}
    outputs = PROJECT/'outputs'
    for pattern in ('requests.jsonl', 'proposals.jsonl', 'tasks.jsonl', 'inspection_targets.jsonl'):
        for path in sorted(outputs.glob('pallet_mask*/'+pattern)):
            if path.parent in (ROOT, REQUESTS) or not (path.parent/'manifest.json').exists(): continue
            relative = str(path.relative_to(PROJECT)); inventories[relative] = digest(path)
            for item in jsonl(path):
                key = item.get('target_key')
                if key in rows: reasons[key].add(relative)
    return {key: sorted(value) for key, value in reasons.items()}, inventories


def component_labels(rows, image_hashes):
    old = read(COMPONENT_SOURCE); require(set(old['by_target']) == set(rows), 'Association inventory incomplete')
    graph = Components(rows); seen = {}
    for key, row in sorted(rows.items()):
        for kind, value in (('prior_global_association', old['by_target'][key]), ('frame', row['frame_id']),
                            ('group', row['group_id']), ('pcd', row['source_pcd_sha256']),
                            ('original_image_bytes', image_hashes[row['image']])):
            token = kind, str(value)
            if token in seen: graph.union(key, seen[token], kind)
            else: seen[token] = key
    return graph.labels(), graph.links


def split_components(records):
    """Label-independent fixed component hash split; rare missing classes reported."""
    by_component = {r['component']: ('holdout' if int(sha(['taxonomy-v1', r['component']])[:8], 16) % 5 == 0 else 'train') for r in records}
    require(set(by_component.values()) == {'train', 'holdout'}, 'Need separated components')
    return {r['target_key']: by_component[r['component']] for r in records}


def sources_current():
    binding = read(ROOT/'source_binding.json')
    discovered = {str(p.relative_to(PROJECT)) for p in (PROJECT/'outputs').glob('pallet_mask*_reviews_v*/mask_reviews.sqlite3') if p.parent != REVIEWS}
    expected = set(binding['database_fingerprints']) - {str(POSE.relative_to(PROJECT))}
    if discovered != expected: return False
    _, inventories = issued_inventory({})
    frozen = read(ROOT/'private/protection.json')['inventories']
    if inventories != frozen: return False
    return all(fingerprint(PROJECT/p) == h for p, h in binding['database_fingerprints'].items()) and all(
        digest(PROJECT/p) == h for p, h in binding['file_hashes'].items())


def prepare():
    require(not ROOT.exists(), 'Fresh taxonomy training version required')
    require(shutil.disk_usage(PROJECT/'outputs').free > 20*2**30, 'Preserve20GiB disk reserve; no deletion authorized')
    cv2.setNumThreads(1)
    source, database, revision, raw, human, state, classes, parents = source_state()
    formal, rows, evidence = load_formal_evidence(read(FORMAL_SOURCE))
    protected, pose_current, stores, reasons = human_protection(rows, human)
    issued, inventory_hashes = issued_inventory(rows)
    for key in issued:
        protected.add(key); reasons.setdefault(key, []).append('issued_task_inventory_conservative_hold')
    ROOT.mkdir(mode=0o700); (ROOT/'private').mkdir(mode=0o700)
    snapshot_db(database, ROOT/'private/class_reviews.sqlite3')
    require(fingerprint(database) == fingerprint(ROOT/'private/class_reviews.sqlite3'), 'Concurrent class edit during freeze')
    write(ROOT/'private/source_records.jsonl', human, lines=True); write(ROOT/'private/class_state.json', state)
    write(ROOT/'classes.json', classes); write(ROOT/'parents.json', parents)
    hashes = {}; image_sizes = {}
    for row in rows.values():
        if row['image'] not in hashes:
            hashes[row['image']] = digest(formal/row['image']); image_sizes[row['image']] = row['size']
    labels, links = component_labels(rows, hashes)
    write(ROOT/'private/components.json', dict(by_target=labels, additional_links=links,
        inherited_global_association_sha256=digest(COMPONENT_SOURCE), scope='all12344_all_class_group_pcd_bytes_and_prior_near_duplicate_links'))
    write(ROOT/'private/image_hashes.json', hashes)
    manual_hashes = {r['image_sha256'] for r in human}
    require(all(hashes[r['row']['image']] == r['image_sha256'] for r in human), 'Current human/original image bytes differ')
    neighbors = defaultdict(list)
    for row in rows.values(): neighbors[row['frame_id']].append(row)
    for folder in ('images', 'masks', 'valid'): (ROOT/'private'/folder).mkdir(mode=0o700)
    training = []; clipping = []
    for item in human:
        key = item['target_key']; face = state['assignments'][key]
        info = next(c for c in classes.values() if c['face_id'] == face)
        if not info['train_enabled']: continue
        row = rows[key]; require(row == item['row'], 'Frozen formal source row changed')
        polygon = polygon_value(item['polygon'], *row['size']); require(len(polygon) == info['K'], 'Actual fine-class K mismatch')
        roi = context(row['bbox'], row['size']); x1, y1, x2, y2 = roi
        full = raster_mask(polygon, *row['size']).astype(np.uint8); mask = full[y1:y2, x1:x2]
        if full.sum() != mask.sum(): clipping.append(key)
        require(mask.any() and full.sum() == mask.sum(), 'Fixed label-independent ROI clips human contour')
        image = evidence.image(row['frame_id'])[y1:y2, x1:x2]
        valid = supervision_valid(roi, neighbors[row['frame_id']], key, mask)
        for folder, array in (('images', image), ('masks', mask*255), ('valid', valid*255)):
            require(cv2.imwrite(str(ROOT/'private'/folder/f'{key}.png'), array), 'Failed target-local supervision export')
        training.append(dict(target_key=key, class_id=info['class_id'], fine_index=info['fine_index'], parent=info['parent'],
            parent_index=info['parent_index'], K=info['K'], small_sample=info['small_sample'], component=labels[key],
            image_sha256=item['image_sha256'], roi=roi, source=row, polygon=polygon,
            image=f'private/images/{key}.png', mask=f'private/masks/{key}.png', valid=f'private/valid/{key}.png',
            geometry_source=item.get('geometry_source_kind', 'frozen_human_contour'),
            original_note_save_action=item.get('note_integration', {}).get('note_original_action'),
            source_payload_sha256=item['source_payload_sha256'], source_revision=item['source_revision'], **FALSE_FLAGS))
    split = split_components(training)
    for r in training: r['evaluation_branch_split'] = split[r['target_key']]
    require(len(training) == 2198 and len({r['class_id'] for r in training}) == 41, 'Retain all2198 target-level supervision records')
    candidates = []; excluded = Counter(); exclusion_rows = []
    for key, row in sorted(rows.items()):
        why = []
        if key in protected: why.append('human_deleted_quarantine_or_issued_task')
        if hashes[row['image']] in manual_hashes: why.append('image_bytes_already_human_annotated')
        box = row['bbox']; width, height = row['size']
        if not np.isfinite([box[n] for n in ('x','y','width','height')]).all() or box['width'] <= 0 or box['height'] <= 0 or min(width, box['x']+box['width']) <= max(0,box['x']) or min(height,box['y']+box['height']) <= max(0,box['y']):
            why.append('invalid_original_bbox')
        if why:
            excluded.update(why); exclusion_rows.append(dict(target_key=key, reasons=why)); continue
        candidates.append(dict(target_key=key, source=row, image_sha256=hashes[row['image']], component=labels[key],
            pose_payload_raw_sha256=hashlib.sha256(pose_current[key].encode()).hexdigest() if key in pose_current else None))
    write(ROOT/'private/training.jsonl', training, lines=True); write(ROOT/'private/candidates.jsonl', candidates, lines=True)
    write(ROOT/'private/exclusions.jsonl', exclusion_rows, lines=True)
    write(ROOT/'private/protection.json', dict(human_reasons=reasons, issued_task_membership=issued, inventories=inventory_hashes))
    files = {str(p.relative_to(PROJECT)): digest(p) for p in (ACTIVE, source/'manifest.json', source/'private/records.jsonl', source/'class_attributes.json', COMPONENT_SOURCE, FORMAL_SOURCE)}
    files.update(inventory_hashes)
    stores[str(database.relative_to(PROJECT))] = fingerprint(database); stores[str(POSE.relative_to(PROJECT))] = fingerprint(POSE)
    # Integration retains88 authorized human edits and their84 original drafts.
    write(ROOT/'source_binding.json', dict(class_graph_revision=revision, class_graph_raw_sha256=hashlib.sha256(raw.encode()).hexdigest(),
        class_package=str(source.relative_to(PROJECT)), database_fingerprints=stores, file_hashes=files,
        original_draft_actions_not_rewritten=True, source_integration_report=read(source/'integration_report.json'),
        user_authorization='执行计划', future_live_edits_require_new_version=True))
    counts = Counter(r['evaluation_branch_split'] for r in training)
    report = dict(state='frozen_current43_classes_41_train_quota_only', source_graph_revision=revision, human_targets=2200,
        training_targets=2198, fine_classes=41, parent_classes=9, excluded_classes=EXCLUDED,
        unique_human_images=len(manual_hashes), training_unique_images=len({r['image_sha256'] for r in training}),
        multiple_targets_same_image_retained=len(training)-len({r['image_sha256'] for r in training}),
        parent_counts=dict(Counter(r['parent'] for r in training)), small_classes=[c['class_id'] for c in classes.values() if c['train_enabled'] and c['small_sample']],
        evaluation_splits=dict(counts), holdout_scope='generic_pretraining_branch_only_before_any_full_pool_training',
        holdout_class_counts=dict(Counter(r['class_id'] for r in training if r['evaluation_branch_split']=='holdout')),
        candidate_targets=len(candidates), candidate_distinct_image_upper_bound=len({c['image_sha256'] for c in candidates}),
        candidate_class_capacities_unknown_until_inference=True, exclusion_counts=dict(excluded),
        issued_inventory_hold='all published task inventories conservatively protected, not machine-only shortlists',
        original_metadata_is_not_model_input=True, no_existing_labels_models_or_services_modified=True, **FALSE_FLAGS)
    write(ROOT/'preparation_report.json', report)
    assets = {str(p.relative_to(ROOT)): digest(p) for p in ROOT.rglob('*') if p.is_file()}
    write(ROOT/'dataset_manifest.json', dict(schema=NAMESPACE, files=assets, records=2198,
        classes=41, parents=9, code_sha256=digest(__file__), **FALSE_FLAGS))
    require(sources_current(), 'Live source changed during preparation; freeze a new version')
    verify(); print(json.dumps(report, ensure_ascii=False), flush=True)


def verify():
    m = read(ROOT/'dataset_manifest.json')
    for name, expected in m['files'].items(): require(digest(ROOT/name) == expected, 'Frozen taxonomy data changed: '+name)
    records = jsonl(ROOT/'private/training.jsonl'); classes = read(ROOT/'classes.json')
    require(len(records) == len({r['target_key'] for r in records}) == 2198, 'Distinct target-level supervision required')
    sets = defaultdict(set)
    for r in records:
        require(r['class_id'] not in EXCLUDED and len(r['polygon']) == classes[str(r['class_id'])]['K'] == r['K'], 'Fine/K isolation failed')
        sets[r['evaluation_branch_split']].add(r['component'])
    require(not sets['train'] & sets['holdout'], 'Association leakage across new evaluation branch')
    train_images = {r['image_sha256'] for r in records if r['evaluation_branch_split']=='train'}
    require(not train_images & {r['image_sha256'] for r in records if r['evaluation_branch_split']=='holdout'}, 'Image-byte leakage')
    return m, records
