"""Human fork-face masks, RF-DETR-Seg comparison, and native AnyLabeling completion."""
from __future__ import annotations

import argparse
from collections import Counter, defaultdict
import copy
import hashlib
import io
import json
import math
import os
from pathlib import Path
import zipfile

import cv2
import numpy as np
from PIL import Image
from pycocotools import mask as coco_mask
from shapely.geometry import Polygon

from pallet.fork_face_anylabeling import digest, encoded, load_upstream, protected_write, read_checked, safe_path
from pallet.fork_face_simplify import load_label_file


DEFAULT_INPUT = Path('pallet_cluster_annotation_seed_rectangles_v1_partial.zip')
DEFAULT_SOURCE = Path('outputs/pallet_cluster_annotation_seed_rectangles_v1')
DEFAULT_OUTPUT = Path('outputs/pallet_instance_segmentation_seed_v1')
SHAPE_SHA = '77a574e131d4e6bbbd4291859317a4bc59bacb6e64111cdcd0880065472015bc'
LABEL_SHA = '6ffd52de45420952576cf0cf30718ab637e0ad8375c241f1800ee7ccb2f59e92'
RESOLUTION = 768


def read_json(path):
    return json.loads(Path(path).read_text())


def read_jsonl(path):
    return [json.loads(line) for line in Path(path).read_text().splitlines() if line.strip()]


def save(path, value):
    protected_write(path, encoded(value))


def event(stage, **values):
    print(json.dumps({'stage': stage, **values}, ensure_ascii=False), flush=True)


def png(mask):
    buffer = io.BytesIO(); Image.fromarray(np.asarray(mask, np.uint8)*255).save(buffer, format='PNG')
    return buffer.getvalue()


def rle(mask):
    data = coco_mask.encode(np.asfortranarray(mask, dtype=np.uint8))
    data['counts'] = data['counts'].decode('ascii')
    return data


def bbox(mask):
    ys, xs = np.nonzero(mask)
    if not len(xs): raise ValueError('Empty instance has no independent face box')
    return [int(xs.min()), int(ys.min()), int(xs.max()-xs.min()+1), int(ys.max()-ys.min()+1)]


def letterbox(image, size=RESOLUTION):
    """One aspect-preserving resize with recorded integer scales and padding."""
    image = image.convert('RGB'); w, h = image.size
    scale = min(size/w, size/h)
    nw, nh = max(1, round(w*scale)), max(1, round(h*scale))
    left, top = (size-nw)//2, (size-nh)//2
    output = Image.new('RGB', (size, size), (0, 0, 0))
    output.paste(image.resize((nw, nh), Image.Resampling.BILINEAR), (left, top))
    return output, {'width': w, 'height': h, 'resized_width': nw, 'resized_height': nh,
                    'left': left, 'top': top, 'resolution': size}


def transform_mask(mask, transform):
    t = transform
    resized = np.asarray(Image.fromarray(mask.astype(np.uint8)).resize(
        (t['resized_width'], t['resized_height']), Image.Resampling.NEAREST), bool)
    result = np.zeros((t['resolution'], t['resolution']), bool)
    result[t['top']:t['top']+t['resized_height'], t['left']:t['left']+t['resized_width']] = resized
    return result


def restore_mask(mask, transform):
    t = transform
    cropped = mask[t['top']:t['top']+t['resized_height'], t['left']:t['left']+t['resized_width']]
    return np.asarray(Image.fromarray(cropped.astype(np.uint8)).resize(
        (t['width'], t['height']), Image.Resampling.NEAREST), bool)


def init(args):
    source, archive, output = args.source.resolve(), args.input.resolve(), args.output.resolve()
    if output.is_relative_to(source) or source.is_relative_to(output):
        raise ValueError('Output must be separate from frozen source workspace')
    if archive.is_relative_to(output): raise ValueError('Input ZIP must remain outside output workspace')
    registry_path = source/'export_assets.json'
    registry = read_json(registry_path)['files']
    receipt = read_json(source/'delivery_manifest.json')
    if digest(source/receipt['archive_name']) != receipt['archive_sha256']:
        raise ValueError('Frozen rectangle ZIP changed')
    for name in ('targets.jsonl', 'manifest.json'):
        read_checked(source/'delivery', name, registry)
    selected = read_jsonl(source/'delivery/targets.jsonl')
    if len({r['frame_id'] for r in selected}) != len(selected): raise ValueError('Duplicate selected native frame')
    frames = {Path(r['image']).stem: r for r in selected}
    binding_source = read_json(source/'run_manifest.json')
    old_seed = Path(binding_source['seed_source'])
    shape_source = old_seed/'delivery/source/anylabeling_shape.py'
    label_source = old_seed/'delivery/source/anylabeling_label_file.py'
    upstream = load_upstream(shape_source, SHAPE_SHA)
    labels = load_label_file(label_source, LABEL_SHA)
    formal_path = Path('outputs/pallet_target_clustering_v1/run_manifest.json').resolve()
    formal = read_json(formal_path)
    inputs = {str(p): digest(p) for p in (archive, registry_path, source/'delivery_manifest.json',
              source/'run_manifest.json', formal_path, shape_source, label_source)}
    binding = {'kind': 'human_face_instance_segmentation_seed', 'version': 1, 'input': str(archive),
               'source': str(source), 'inputs': inputs, 'seed': args.seed, 'resolution': RESOLUTION,
               'epochs': args.epochs, 'patience': 30, 'lr': 1e-4, 'lr_encoder': 1e-5,
               'effective_batch': 8, 'models': ['small', 'medium'], 'instance_unit': 'visible_fork_face',
               'hole_label': '__hole__', 'classes': ['pallet'], 'selected_targets': len(selected),
               'rectangles_in_shapes': False, 'face_merging': False}
    binding['task_hash'] = hashlib.sha256(encoded(binding)).hexdigest()
    output.mkdir(parents=True, exist_ok=True); save(output/'run_manifest.json', binding)
    return dict(args=args, source=source, archive=archive, output=output, frames=frames, rows=selected,
                registry=registry, binding=binding, upstream=upstream, label_class=labels, formal=formal)


def native_image(ctx, key):
    row = ctx['frames'][key]
    return read_checked(ctx['source']/'delivery', row['image'], ctx['registry'])


def source_document(ctx, key):
    row = ctx['frames'][key]
    return json.loads(read_checked(ctx['source']/'delivery', row['annotation'], ctx['registry']))


def raster_shape(shape, size, upstream):
    points = np.asarray(shape['points'], dtype=float)
    if (shape.get('shape_type') != 'polygon' or points.ndim != 2 or points.shape[1] != 2 or len(points) < 3
            or not np.isfinite(points).all() or points.min() < 0 or points[:, 0].max() >= size[1]
            or points[:, 1].max() >= size[0] or not Polygon(points).is_valid or Polygon(points).area <= 0):
        raise ValueError('Invalid polygon geometry or native coordinates')
    return upstream(size, points.tolist(), 'polygon')


def audit_document(document, size, upstream):
    """Each positive ring is one face; holes belong to one unique positive ring."""
    positives, holes, reasons = [], [], []
    rectangles = 0
    for index, shape in enumerate(document['shapes']):
        if shape.get('shape_type') == 'rectangle': rectangles += 1; continue
        if shape.get('label') not in ('pallet', '__hole__'):
            reasons.append(f'unsupported_label:{index}'); continue
        try: pixels = raster_shape(shape, size, upstream)
        except ValueError as error: reasons.append(f'invalid_shape:{index}:{error}'); continue
        if not pixels.any(): reasons.append(f'empty_shape:{index}'); continue
        item = {'shape_index': index, 'shape': copy.deepcopy(shape), 'pixels': pixels}
        (positives if shape['label'] == 'pallet' else holes).append(item)
    for i, a in enumerate(positives):
        for b in positives[i+1:]:
            fraction = (a['pixels'] & b['pixels']).sum()/min(a['pixels'].sum(), b['pixels'].sum())
            if fraction > .10:
                reasons.append(f'positive_overlap_or_nested:{a["shape_index"]}:{b["shape_index"]}:{fraction:.4f}')
    mapping = defaultdict(list)
    for hole in holes:
        scores = sorted([(float((hole['pixels'] & outer['pixels']).sum()/hole['pixels'].sum()),
                          outer['shape_index']) for outer in positives], reverse=True)
        if not scores or scores[0][0] < .95 or (len(scores) > 1 and scores[1][0] > .1):
            reasons.append(f'ambiguous_hole:{hole["shape_index"]}')
        else:
            mapping[scores[0][1]].append(hole)
    instances = []
    for number, outer in enumerate(positives, 1):
        own_holes = mapping[outer['shape_index']]
        hole_mask = np.zeros(size, bool)
        for hole in own_holes: hole_mask |= hole['pixels']
        pixels = outer['pixels'] & ~hole_mask
        if not pixels.any(): reasons.append(f'empty_after_holes:{outer["shape_index"]}'); continue
        instances.append({'face_index': number, 'source_shape_index': outer['shape_index'],
             'outer_shape': outer['shape'], 'hole_shapes': [h['shape'] for h in own_holes],
             'source_hole_indices': [h['shape_index'] for h in own_holes],
             'mask': pixels, 'hole_mask': hole_mask & outer['pixels'], 'bbox': bbox(pixels)})
    return instances, sorted(set(reasons)), rectangles


def audit(ctx):
    if (ctx['output']/'audit/frames.json').exists():
        verify_audit(ctx)
        return read_json(ctx['output']/'audit/frames.json')
    frames, totals = [], Counter()
    protected_write(ctx['output']/'inputs/returned_partial.zip', ctx['archive'].read_bytes())
    with zipfile.ZipFile(ctx['archive']) as archive:
        names = archive.namelist()
        if len(names) != len(set(names)) or archive.testzip() is not None: raise ValueError('Input ZIP duplicate names or CRC error')
        for name in sorted(names):
            path = Path(name)
            if path.suffix != '.json' or path.name != name: raise ValueError('Unexpected ZIP member path/type')
            key = path.stem
            if key not in ctx['frames']: raise ValueError('Returned frame not in selected source')
            row = ctx['frames'][key]; original = source_document(ctx, key)
            data = archive.read(name); document = json.loads(data)
            if document['imagePath'] != row['image'] or (document['imageWidth'], document['imageHeight']) != (original['imageWidth'], original['imageHeight']):
                raise ValueError('Returned image identity or native grid changed')
            if document['pallet_annotation_task']['selected_target_id'] != row['target_id']:
                raise ValueError('Returned selected parent identity changed')
            size = (document['imageHeight'], document['imageWidth'])
            instances, reasons, rectangles = audit_document(document, size, ctx['upstream'])
            protected_write(ctx['output']/'inputs/returned_json'/name, data)
            record = {'frame_key': key, 'frame_id': row['frame_id'], 'group_id': row['group_id'],
                      'appearance_class': row['appearance_class'], 'domain': original['pallet_export']['domain'],
                      'width': size[1], 'height': size[0], 'source_sha256': hashlib.sha256(data).hexdigest(),
                      'rectangles_removed': rectangles, 'reasons': reasons,
                      'state': 'quarantined' if reasons else ('manual_accepted' if instances else 'no_mask_not_negative'),
                      'instances': []}
            for instance in instances:
                number = instance['face_index']
                relative = f'audit/masks/{key}__face_{number:03d}.png'
                holes_relative = f'audit/holes/{key}__face_{number:03d}.png'
                protected_write(ctx['output']/relative, png(instance.pop('mask')))
                protected_write(ctx['output']/holes_relative, png(instance.pop('hole_mask')))
                record['instances'].append({**instance, 'mask': relative, 'holes_mask': holes_relative,
                                            'instance_id': f'{key}::face:{number:03d}'})
            frames.append(record); totals[record['state']] += 1; totals['rectangles_removed'] += rectangles
            if record['state'] == 'manual_accepted':
                totals['accepted_instances'] += len(instances)
                totals['accepted_holes'] += sum(len(i['hole_shapes']) for i in instances)
    save(ctx['output']/'audit/frames.json', frames)
    summary = {**dict(totals), 'returned_frames': len(frames),
               'accepted_groups': len({f['group_id'] for f in frames if f['state']=='manual_accepted'}),
               'audit_source_sha256': digest(__file__)}
    save(ctx['output']/'audit/summary.json', summary)
    protected_write(ctx['output']/'audit/quarantine.jsonl', b''.join(encoded(f) for f in frames if f['state']!='manual_accepted'))
    event('audit', **summary)
    return frames


def verify_audit(ctx):
    """Recheck cached native masks against the unmodified returned polygons."""
    frames = read_json(ctx['output']/'audit/frames.json')
    if digest(ctx['output']/'inputs/returned_partial.zip') != digest(ctx['archive']):
        raise ValueError('Cached returned archive changed')
    with zipfile.ZipFile(ctx['archive']) as archive:
        if set(archive.namelist()) != {f['frame_key']+'.json' for f in frames}:
            raise ValueError('Audit frame coverage changed')
        for f in frames:
            raw = archive.read(f['frame_key']+'.json')
            if hashlib.sha256(raw).hexdigest() != f['source_sha256']:
                raise ValueError('Audit source identity changed')
            if (ctx['output']/'inputs/returned_json'/(f['frame_key']+'.json')).read_bytes() != raw:
                raise ValueError('Preserved human original changed')
            instances, reasons, rectangles = audit_document(json.loads(raw),(f['height'],f['width']),ctx['upstream'])
            if reasons != f['reasons'] or rectangles != f['rectangles_removed'] or len(instances)!=len(f['instances']):
                raise ValueError('Cached audit geometry changed')
            for instance, cached in zip(instances,f['instances']):
                for kind in ('mask','hole_mask'):
                    path = cached['mask'] if kind=='mask' else cached['holes_mask']
                    if not np.array_equal(np.asarray(Image.open(ctx['output']/path))>0,instance[kind]):
                        raise ValueError('Cached native audit mask changed')
                for kind in ('outer_shape','hole_shapes','source_shape_index','bbox'):
                    if cached[kind]!=instance[kind]:raise ValueError('Cached human contour changed')
    return frames


def split_groups(frames, seed=42):
    groups = defaultdict(list)
    for frame in frames: groups[frame['group_id']].append(frame)
    domain_groups = defaultdict(set)
    for group, values in groups.items():
        for f in values: domain_groups[f['domain']].add(group)
    hole_groups = {g for g, values in groups.items() if any(i['hole_shapes'] for f in values for i in f['instances'])}
    eligible = [g for g in groups if all(len(domain_groups[f['domain']]) >= 2 for f in groups[g])]
    eligible.sort(key=lambda g: hashlib.sha256(f'{seed}\0{g}'.encode()).hexdigest())
    number = max(2, round(len(groups)*.2)); validation = set()
    def allowed(g):
        return all(domain_groups[f['domain']] - validation - {g} for f in groups[g])
    for group in eligible:
        if group in hole_groups and allowed(group): validation.add(group)
        if len(validation)>=2: break
    if len(validation)<2: raise ValueError('Need two independent hole groups for validation')
    target_frames = round(len(frames)*.2)
    while len(validation)<number:
        candidates = [g for g in eligible if g not in validation and allowed(g)]
        if not candidates: break
        candidates.sort(key=lambda g: (abs(sum(len(groups[v]) for v in validation)+len(groups[g])-target_frames),
                                       hashlib.sha256(f'{seed}\0{g}'.encode()).hexdigest()))
        validation.add(candidates[0])
    return {g: 'valid' if g in validation else 'train' for g in sorted(groups)}


def build(ctx):
    verify_audit(ctx)
    if (ctx['output']/'dataset_manifest.json').exists():
        return verify_dataset(ctx)
    frames = [f for f in read_json(ctx['output']/'audit/frames.json') if f['state']=='manual_accepted']
    splits = split_groups(frames, ctx['args'].seed)
    annotations = {s: {'info': {'description': 'human fork-face segmentation; holes subtracted'},
                      'images': [], 'annotations': [], 'categories': [{'id': 1, 'name': 'pallet'}]} for s in ('train', 'valid')}
    index, instance_counter = [], 0
    for image_id, frame in enumerate(frames, 1):
        key, split = frame['frame_key'], splits[frame['group_id']]
        raw = native_image(ctx, key); image, transform = letterbox(Image.open(io.BytesIO(raw)))
        buffer = io.BytesIO(); image.save(buffer, format='PNG')
        protected_write(ctx['output']/f'dataset/{split}/{key}.png', buffer.getvalue())
        annotations[split]['images'].append({'id': image_id, 'file_name': key+'.png',
            'width': RESOLUTION, 'height': RESOLUTION, 'group_id': frame['group_id'], 'native_frame_key': key})
        for instance in frame['instances']:
            native = np.asarray(Image.open(ctx['output']/instance['mask']))>0
            mask = transform_mask(native, transform)
            instance_counter += 1
            ann = {'id': instance_counter, 'image_id': image_id, 'category_id': 1, 'iscrowd': 0,
                   'bbox': bbox(mask), 'area': int(mask.sum()), 'segmentation': rle(mask),
                   'native_instance_id': instance['instance_id']}
            if not np.array_equal(coco_mask.decode(ann['segmentation'])>0, mask): raise ValueError('RLE roundtrip lost holes')
            annotations[split]['annotations'].append(ann)
        index.append({**frame, 'split': split, 'transform': transform, 'image_id': image_id,
                      'native_image_sha256': hashlib.sha256(raw).hexdigest()})
    for split, value in annotations.items(): save(ctx['output']/f'dataset/{split}/_annotations.coco.json', value)
    save(ctx['output']/'dataset/native_index.json', index)
    save(ctx['output']/'dataset/group_split.json', splits)
    counts = {s: {'frames': len(v['images']), 'instances': len(v['annotations']),
                  'groups': sum(role==s for role in splits.values())} for s, v in annotations.items()}
    dataset_files = {str(p.relative_to(ctx['output'])): digest(p) for p in (ctx['output']/'dataset').rglob('*') if p.is_file()}
    save(ctx['output']/'dataset_manifest.json', {'task_hash': ctx['binding']['task_hash'], 'files': dataset_files,
          'split_counts': counts, 'resolution': RESOLUTION, 'rectangles_in_supervision': 0,
          'holes_supervised_as_background': True, 'builder_sha256': digest(__file__)})
    event('build', split_counts=counts)


def verify_dataset(ctx):
    manifest = read_json(ctx['output']/'dataset_manifest.json')
    if manifest['task_hash']!=ctx['binding']['task_hash']: raise ValueError('Dataset task changed')
    for p, expected in manifest['files'].items():
        if digest(ctx['output']/p)!=expected: raise ValueError('Frozen training dataset changed')
    return manifest


def main(argv=None):
    # This host otherwise enumerates its 24 GiB card before the 16 GiB card.
    # Set before importing torch so CUDA indices follow nvidia-smi PCI order.
    os.environ.setdefault('CUDA_DEVICE_ORDER', 'PCI_BUS_ID')
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--input', type=Path, default=DEFAULT_INPUT)
    parser.add_argument('--source', type=Path, default=DEFAULT_SOURCE)
    parser.add_argument('--output', type=Path, default=DEFAULT_OUTPUT)
    parser.add_argument('--seed', type=int, default=42)
    parser.add_argument('--epochs', type=int, default=150)
    parser.add_argument('--device-small', default='cuda:0')
    parser.add_argument('--device-medium', default='cuda:1')
    parser.add_argument('--device', default='cuda:1', help='Inference device')
    parser.add_argument('--model', choices=['small','medium','both'], default='both')
    parser.add_argument('--stage', choices=['audit','build','train','select','predict','export','verify','package','all'], default='all')
    args = parser.parse_args(argv); ctx = init(args)
    stages = ['audit','build','train','select','predict','export','verify','package'] if args.stage=='all' else [args.stage]
    for stage in stages:
        if stage=='audit': audit(ctx)
        elif stage=='build': build(ctx)
        elif stage=='train':
            from pallet.instance_segmentation_training import train
            train(ctx)
        else:
            from pallet.instance_segmentation_completion import select, predict, export, verify, package
            {'select':select,'predict':predict,'export':export,'verify':verify,'package':package}[stage](ctx)


if __name__=='__main__': main()
