"""All human contours for an annotation assistant, not a held-out benchmark.

The new user permit retires former split boundaries only for this new version.
Historical labels, datasets, source flags and evaluations are never rewritten.
"""
from collections import Counter, defaultdict
from contextlib import closing
import hashlib
import json
import math
from pathlib import Path
import shutil

import cv2
import numpy as np

from .mask_model_training_data import PROJECT, FALSE_FLAGS, read, jsonl, write, sha, require, letterbox, global_audit
from .mask_model_improvement_data import preservation, source_binding
from .mask_dataset_update import previous_artifacts, exception_snapshot
from .mask_quota_saved_extension import load_formal_evidence, file_inventory
from .mask_template_refined import snapshot_database
from .cluster_targets import digest
from .pose_review_geometry import polygon_value, raster_mask

ROOT = PROJECT/'outputs/pallet_mask_annotation_full_v1'
SOURCE = PROJECT/'outputs/pallet_mask_continuation_sources_v15'
ISOLATED = PROJECT/'outputs/pallet_mask_nonconforming_reviews_v1/mask_reviews.sqlite3'


def context(box, size):
    """One label-independent rule, including all borders; no polygon crop oracle."""
    x, y, w, h = [box[k] for k in ('x', 'y', 'width', 'height')]
    dx, dy = max(48, .2*w), max(16, .6*h)
    return [max(0, math.floor(x-dx)), max(0, math.floor(y-dy)), min(size[0], math.ceil(x+w+dx)), min(size[1], math.ceil(y+h+dy))]


def supervision_valid(roi, neighbors, target_key, target_mask):
    """Other known objects are ignore, not false negatives. Human positives stay."""
    x1, y1, x2, y2 = roi; valid = np.ones((y2-y1, x2-x1), np.uint8)
    for row in neighbors:
        if row['target_key'] == target_key:
            continue
        b = row['bbox']; l, t = max(x1, math.floor(b['x'])), max(y1, math.floor(b['y']))
        r, btm = min(x2, math.ceil(b['x']+b['width'])), min(y2, math.ceil(b['y']+b['height']))
        if r > l and btm > t:
            valid[t-y1:btm-y1, l-x1:r-x1] = 0
    valid[target_mask > 0] = 1
    return valid


def prepare():
    require(not ROOT.exists(), 'Fresh immutable full-data version required')
    require(shutil.disk_usage(ROOT.parent).free > 20*2**30, '10GiB run budget plus10GiB reserve required; no deletion')
    cv2.setNumThreads(1)
    formal, rows, evidence = load_formal_evidence(read(PROJECT/'outputs/pallet_mask_remaining_round5_v1/manifest.json'))
    before = preservation(formal); old_runs = previous_artifacts()
    old_update = file_inventory(PROJECT/'outputs/pallet_mask_dataset_update_v1')
    ROOT.mkdir(mode=0o700); (ROOT/'private').mkdir(mode=0o700)
    write(ROOT/'private/preservation_before.json', before)
    write(ROOT/'private/previous_versions_inventory.json', old_runs)
    write(ROOT/'private/previous_update_inventory.json', old_update)
    # Validate the standard source protocol even though former heldout labels are
    # now authorized. Its train-only return value is deliberately NOT the pool.
    _, source_audit = source_binding(SOURCE, rows)
    snapshot_database(ISOLATED, ROOT/'private/isolated_snapshot.sqlite3')
    isolated, review_only, isolated_events = exception_snapshot(ROOT/'private/isolated_snapshot.sqlite3', rows, formal)
    require(len(isolated) == 7 and not review_only, 'All seven quality-approved redraws need actual nonempty events')
    write(ROOT/'private/isolated_event_binding.jsonl', isolated_events, lines=True)
    catalog = jsonl(SOURCE/'private/source_catalog.jsonl') + isolated
    require(len(catalog) == 1559, 'Latest all-human pool changed; do not assume stale counts')
    labels, _, links, hashes = global_audit(rows, evidence)
    write(ROOT/'private/global_components.json', dict(by_target=labels, links=links, all_targets=12344,
        old_formal_split_blocking_retired_by_user=True, scope='component identities used for sampling/audit only; no held-out components'))
    write(ROOT/'private/native_image_hashes.jsonl', hashes, lines=True)
    write(ROOT/'research_permit.json', dict(user_authorization=['7张已经质检通过', '取消原val/test限制', '所有分量均参与训练', '执行'],
        purpose='annotation_assistant_all_manual_fit_not_benchmark', old_formal_split_restriction_cancelled=True,
        all_labelled_components_in_training=True, isolated7_user_quality_approved=True, old_source_flags_unchanged=True,
        independent_test_set=False, validation_or_calibration_set=False, manual_fields_are_supervision_not_features=True,
        future_saves_require_new_version=True, predictions_are_editable_drafts_only=True, **FALSE_FLAGS))
    neighbors = defaultdict(list)
    for row in rows.values(): neighbors[row['frame_id']].append(row)
    for directory in ('dataset/train', 'dataset_rfdetr/train', 'private/native_masks', 'private/valid_masks'):
        (ROOT/directory).mkdir(parents=True, mode=0o700)
    from pycocotools import mask as mu
    coco = dict(info=dict(description='All human contours for annotation assistant; former splits retired, no blind evaluation'),
        licenses=[], categories=[dict(id=1, name='fork_face')], images=[], annotations=[])
    square = {**coco, 'images': [], 'annotations': []}
    transforms = {}; records = []; identities = set(); frames = set(); pixels = set(); boundary_flags = []
    for index, ref in enumerate(sorted(catalog, key=lambda v:v['target_key'])):
        row = rows[ref['target_key']]; key = ref['target_key']; image_sha = digest(formal/row['image'])
        require(key not in identities and row['frame_id'] not in frames and image_sha not in pixels, 'Repeated human target/frame/image')
        identities.add(key); frames.add(row['frame_id']); pixels.add(image_sha)
        poly = polygon_value(ref['polygon'], *row['size']); roi = context(row['bbox'], row['size'])
        x1, y1, x2, y2 = roi; full = raster_mask(poly, *row['size']).astype(np.uint8); mask = full[y1:y2, x1:x2]
        require(mask.any() and int(mask.sum()) == int(full.sum()), 'Fixed context clips actual human supervision')
        if any(not (row['bbox']['x'] <= p[0] <= row['bbox']['x']+row['bbox']['width'] and row['bbox']['y'] <= p[1] <= row['bbox']['y']+row['bbox']['height']) for p in poly):
            boundary_flags.append(dict(target_key=key, note='Manual polygon extends beyond original box; fixed context retained, original box/label unchanged'))
        image = evidence.image(row['frame_id'])[y1:y2, x1:x2]
        valid = supervision_valid(roi, neighbors[row['frame_id']], key, mask)
        name = f'{index:05d}.png'; h, w = image.shape
        cv2.imwrite(str(ROOT/f'dataset/train/{name}'), np.repeat(image[:, :, None], 3, axis=2))
        cv2.imwrite(str(ROOT/f'private/native_masks/{name}'), mask*255)
        cv2.imwrite(str(ROOT/f'private/valid_masks/{name}'), valid*255)
        def annotation(label):
            rle = mu.encode(np.asfortranarray(label)); require(np.array_equal(mu.decode(rle),label), 'COCO label RLE must be exact')
            return dict(id=index+1,image_id=index+1,category_id=1,iscrowd=0,segmentation=dict(size=rle['size'],counts=rle['counts'].decode()),
                bbox=mu.toBbox(rle).tolist(),area=int(mu.area(rle)))
        coco['images'].append(dict(id=index+1,file_name=name,width=w,height=h));coco['annotations'].append(annotation(mask))
        canvas, label, _, transform = letterbox(image,mask,(576,576))
        cv2.imwrite(str(ROOT/f'dataset_rfdetr/train/{name}'),np.repeat(canvas[:,:,None],3,axis=2))
        square['images'].append(dict(id=index+1,file_name=name,width=576,height=576));square['annotations'].append(annotation(label))
        transforms[str(index+1)] = transform
        record = dict(target_key=key, component=labels[key], class_id=row['final_class_id'],frame_id=row['frame_id'],
            group_id=row['group_id'],policy_id=row['policy_id'],native_image=row['image'],native_size=row['size'],original_bbox=row['bbox'],roi=roi,roi_size=[w,h],
            image=f'dataset/train/{name}',native_mask=f'private/native_masks/{name}',valid_mask=f'private/valid_masks/{name}',image_id=index+1,
            development_split='train',former_formal_split=row['split'],former_source_origin=ref['origin'],polygon=poly,
            source_event_raw_sha256=ref['event_raw_sha256'],source_payload_sha256=ref['payload_sha256'],source_revision=ref['revision'],
            source_batch=ref['batch'],source_origin=ref['origin'],surface=row.get('viewpoint_candidate','unknown'),structure=row.get('structure_candidate','unknown'),
            current_supervision_authorization='user_quality_approved_all_human_annotation_training_v1',
            source_quality_history='nonconforming_then_user_quality_approved' if ref in isolated else 'saved_manual_research_contour', **FALSE_FLAGS)
        records.append(record)
    write(ROOT/'dataset/train/_annotations.coco.json',coco);write(ROOT/'dataset_rfdetr/train/_annotations.coco.json',square)
    write(ROOT/'private/training_records.jsonl',records,lines=True);write(ROOT/'private/rfdetr_transforms.json',transforms)
    write(ROOT/'private/source_binding.json',dict(standard=source_audit,isolated_snapshot_sha256=digest(ROOT/'private/isolated_snapshot.sqlite3'),
        quality_approval='User explicitly passed all7; this is a NEW authorization view, original13events keep original false flags'))
    write(ROOT/'private/context_audit.json',dict(fixed_rule='original bbox; dx=max(48px,.2w),dy=max(16px,.6h)',
        labels_outside_original_bbox=boundary_flags,neighbors_ignore_except_manual_positives=True,
        unknown_surface_metadata_never_used_to_drop_a_component=True))
    files = {d+'/'+n:v for d in ('dataset','dataset_rfdetr','private/native_masks','private/valid_masks') for n,v in file_inventory(ROOT/d).items()}
    for name in ('private/training_records.jsonl','private/rfdetr_transforms.json','private/source_binding.json','private/global_components.json','research_permit.json'):
        files[name]=digest(ROOT/name)
    manifest=dict(state='immutable_all_manual_annotation_training_dataset',records=len(records),classes=dict(Counter(r['class_id'] for r in records)),
        components=len({r['component'] for r in records}),splits={'train':len(records)},former_splits=dict(Counter(r['former_formal_split'] for r in records)),
        files=files,all_components_in_train=True,no_permanent_holdouts=True,blind=False,old_val_test_training_authorized=True,
        original_datasets_overwritten=False,model_input_metadata=False,source7_quality_approved=True,code_sha256=digest(__file__), **FALSE_FLAGS)
    require(len(manifest['classes'])==22 and min(manifest['classes'].values())>=50 and manifest['components']==194,'All22classes/all194components must be retained')
    write(ROOT/'dataset_manifest.json',manifest)
    require(preservation(formal)==before and previous_artifacts()==old_runs
        and file_inventory(PROJECT/'outputs/pallet_mask_dataset_update_v1')==old_update,'Old artifacts changed')
    verify()
    write(ROOT/'dataset_verification.json',dict(state='passed_all1559_sources_exports_coverage_and_old_preservation_not_independent_accuracy',
        manifest_sha256=digest(ROOT/'dataset_manifest.json'),records=1559,components=194,classes=22,user_quality_approved7=7,
        all_training=True,new_holdouts=0,original_files_preserved=True,code_sha256=digest(__file__)))
    print(json.dumps(dict(state='frozen_all1559_all194components_train',manifest_sha256=digest(ROOT/'dataset_manifest.json'),classes=manifest['classes'],former_splits=manifest['former_splits'])),flush=True)


def verify():
    from pycocotools import mask as mu
    m=read(ROOT/'dataset_manifest.json');rs=jsonl(ROOT/'private/training_records.jsonl')
    require(len(rs)==m['records']==1559 and all(r['development_split']=='train' for r in rs),'Whole pool must train')
    require(len({r['component'] for r in rs})==194 and len({r['target_key'] for r in rs})==1559,'Component/identity loss')
    for n,h in m['files'].items():require(digest(ROOT/n)==h,'Immutable full-data file changed: '+n)
    coco=read(ROOT/'dataset/train/_annotations.coco.json');sq=read(ROOT/'dataset_rfdetr/train/_annotations.coco.json')
    for r,ann,sann in zip(rs,coco['annotations'],sq['annotations']):
        mask=cv2.imread(str(ROOT/r['native_mask']),0)>0;valid=cv2.imread(str(ROOT/r['valid_mask']),0)>0
        def decode(a):
            v=a['segmentation'];return mu.decode(dict(size=v['size'],counts=v['counts'].encode()))
        require(np.array_equal(mask,decode(ann)) and (valid[mask]).all(),'Native exact supervision or positive validity mismatch')
        im=cv2.imread(str(ROOT/r['image']),0);canvas,label,_,_=letterbox(im,mask,(576,576))
        require(np.array_equal(label,decode(sann)) and np.array_equal(canvas,cv2.imread(str(ROOT/'dataset_rfdetr/train'/Path(r['image']).name),0)),'Square mapping differs')
    return m,rs
