"""Traceable structure labels from threshold segmentation and XY projections.

Read-only formal data -> image-processing proposals -> necessary rechecks ->
separate tables / histogram gallery. No feature propagation or model training.
"""
from __future__ import annotations

import argparse
from collections import Counter, defaultdict
from concurrent.futures import ThreadPoolExecutor
import hashlib
import json
from pathlib import Path

import cv2
import numpy as np

from pallet.fork_face_anylabeling import check_space, digest, encoded, protected_write, safe_path
from pallet.merge_appearance_classes import read_json, read_jsonl, write_csv, write_workbook
from pallet.structure_projection import analyze_target
from pallet.structure_mask_fallback import MeasuredFrontMasks, combine_evidence


DEFAULT_SOURCE = Path('outputs/pallet_appearance_center_reclassification_v1')
DEFAULT_FORMAL = Path('configs/active_formal_dataset.json')
DEFAULT_SEEDS = Path('outputs/pallet_cluster_annotation_seed_rectangles_v1/delivery/targets.jsonl')
DEFAULT_POLICY = Path('configs/pallet_structure_attributes_v1.json')
DEFAULT_OUTPUT = Path('outputs/pallet_structure_attributes_v1')
DEFAULT_MASKS = Path('outputs/pallet_fork_face_masks_v1/delivery')
CLOSURES = {'chuan': '川字', 'tian': '田字'}
STRUCTURES = {(n, c): f'{n}腿{CLOSURES[c]}' for n in (2, 3, 4) for c in CLOSURES}


def validate_policy(policy):
    if (policy.get('version') != 1 or policy.get('view_scope') != 'current_primary_fork_face'
            or policy.get('method') != 'intensity_threshold_and_xy_projections'
            or policy.get('support_leg_counts') != [2, 3, 4]
            or policy.get('pointcloud_fallback') is not True
            or set(policy.get('closure_definitions', {})) != set(CLOSURES)
            or policy.get('model_training') is not False):
        raise ValueError('Unsupported structure policy or label definitions')
    offsets = policy.get('threshold_offsets', [])
    if (len(offsets) < 3 or len(set(offsets)) != len(offsets) or 0 not in offsets
            or any(type(v) is not int or abs(v) > 64 for v in offsets)
            or not 3 <= policy['minimum_thresholds'] <= len(offsets)
            or not .5 < policy['minimum_threshold_agreement'] <= 1
            or not 0 < policy['maximum_deskew_degrees'] <= 30
            or not 0 < policy['minimum_bbox_height_width_ratio'] < 1
            or min(policy['minimum_bbox_width'], policy['minimum_bbox_height']) <= 0):
        raise ValueError('Invalid projection / threshold policy')
    for key in ('minimum_top_beam_coverage', 'leg_projection_threshold', 'minimum_leg_width_ratio',
                'beam_gap_coverage', 'open_gap_coverage', 'minimum_beam_thickness_ratio'):
        if not 0 < policy[key] < 1:
            raise ValueError(f'Invalid projection ratio: {key}')


def load_context(args):
    source = args.source.resolve()
    rows = read_jsonl(source/'targets.jsonl')
    by_id = {r['target_id']: r for r in rows}
    keys = {r['target_key']: r for r in rows}
    if not rows or len(by_id) != len(rows) or len(keys) != len(rows):
        raise ValueError('Empty or duplicate target identities')
    previous = read_json(source/'output_manifest.json')
    for name, expected in previous['files'].items():
        if digest(safe_path(source, name)) != expected:
            raise ValueError(f'Frozen 22-class result changed: {name}')
    active = read_json(args.formal)
    dataset = Path(active['dataset']).resolve()
    formal_manifest = read_json(active['delivery'])
    formal_path = Path('outputs/pallet_target_clustering_v1/run_manifest.json').resolve()
    formal = read_json(formal_path)
    if Path(formal['binding']['dataset']).resolve() != dataset:
        raise ValueError('Active formal dataset and frozen clustering disagree')
    frames = {r['frame_id']: r for r in read_jsonl(dataset/'frames.jsonl')}
    annotations = {r['frame_id']: r for r in read_jsonl(dataset/'annotations/frames.jsonl')}
    expected_targets = set()
    for annotation in annotations.values():
        for index, box in enumerate(annotation['boxes']):
            target_id = f'{annotation["frame_id"]}::box:{index:04d}'
            expected_targets.add(target_id)
            row = by_id.get(target_id)
            if (row is None or row['bbox'] != box or row['split'] != annotation['split']
                    or row['group_id'] != annotation['group_id']):
                raise ValueError('Formal native box / target identity disagrees with source')
    if expected_targets != set(by_id) or set(r['final_class_id'] for r in rows) != set(range(1, 23)):
        raise ValueError('All formal targets and all existing 22 appearance classes are required')
    seeds = read_jsonl(args.seeds)
    if len({s['target_id'] for s in seeds}) != len(seeds):
        raise ValueError('Duplicate seed target')
    for seed in seeds:
        row = by_id[seed['target_id']]
        if seed['frame_id'] != row['frame_id'] or seed['bbox'] != row['bbox']:
            raise ValueError('Seed identity / native box disagreement')
    policy = read_json(args.policy)
    validate_policy(policy)
    paths = [source/'targets.jsonl', source/'output_manifest.json', source/'validation.json',
             args.formal.resolve(), Path(active['delivery']), formal_path, args.seeds.resolve(), args.policy.resolve()]
    inputs = {str(p): digest(p) for p in paths}
    measured_masks = MeasuredFrontMasks(args.masks, rows, inputs)
    return {'rows': rows, 'by_id': by_id, 'by_key': keys, 'source': source, 'dataset': dataset,
            'formal': formal, 'formal_delivery': formal_manifest, 'frames': frames,
            'seeds': seeds, 'policy': policy, 'inputs': inputs, 'measured_masks': measured_masks}


def verify_protected(ctx, images=False):
    pairs = [(safe_path(ctx['dataset'], name), sha) for name, sha in ctx['formal']['protected_files'].items()]
    if images:
        pairs += [(safe_path(ctx['dataset'], f['image']), f['files'][f['image']]) for f in ctx['frames'].values()]
    def verify(pair):
        if digest(pair[0]) != pair[1]:
            raise ValueError(f'Protected formal asset changed: {pair[0]}')
    with ThreadPoolExecutor(max_workers=8) as pool:
        list(pool.map(verify, pairs))
    for path, sha in ctx['inputs'].items():
        if digest(path) != sha:
            raise ValueError(f'Frozen structure input changed: {path}')
    for name, sha in ctx['measured_masks'].used_assets.items():
        if digest(safe_path(ctx['measured_masks'].root, name)) != sha:
            raise ValueError(f'Measured fallback source changed: {name}')
    return len(ctx['formal']['protected_files'])


def process_frames(ctx, workers, output):
    by_frame = defaultdict(list)
    for row in ctx['rows']:
        by_frame[row['frame_id']].append(row)
    def process(item):
        frame_id, rows = item
        frame = ctx['frames'][frame_id]
        path = safe_path(ctx['dataset'], frame['image'])
        if digest(path) != frame['files'][frame['image']]:
            raise ValueError(f'Native image hash mismatch: {frame_id}')
        image = cv2.imread(str(path), cv2.IMREAD_GRAYSCALE)
        if image is None:
            raise ValueError(f'Unreadable native intensity image: {frame_id}')
        records = []
        measured_arrays = None
        for row in rows:
            result = analyze_target(image, row, ctx['policy'], include_mask=True)
            raw = result.pop('_raw_threshold_mask')
            foreground = result.pop('_binary_mask')
            relative = f'threshold_masks/{row["target_key"]}.png'
            ok, buffer = cv2.imencode('.png', raw)
            if not ok:
                raise ValueError('Cannot encode threshold mask')
            protected_write(output/relative, buffer.tobytes())
            result['threshold_mask'] = relative
            result['foreground_mask'] = None
            result['foreground_coordinate_scope'] = 'deskewed_native_roi_not_a_segmentation_ground_truth'
            if foreground is not None:
                relative = f'foreground_masks/{row["target_key"]}.png'
                ok, buffer = cv2.imencode('.png', foreground)
                if not ok:
                    raise ValueError('Cannot encode projection foreground')
                protected_write(output/relative, buffer.tobytes())
                result['foreground_mask'] = relative
            result['pointcloud_fallback_attempted'] = False
            result['projection_plot_source'] = 'intensity'
            result['attribute_sources'] = {k: 'intensity_projections' if result[k] is not None else 'unknown'
                                           for k in ('support_leg_count', 'closure_type')}
            if result['review_required']:
                if measured_arrays is None:
                    measured_arrays = ctx['measured_masks'].frame_arrays(frame_id)
                    if measured_arrays[0].shape != image.shape:
                        raise ValueError('Measured mask grid differs from native intensity grid')
                measured, native_mask = ctx['measured_masks'].analyze(row, measured_arrays, ctx['policy'])
                measured_foreground = measured.pop('_binary_mask')
                x1, y1, x2, y2 = result['native_integer_bbox']
                name = f'pointcloud_masks/{row["target_key"]}.png'
                ok, buffer = cv2.imencode('.png', native_mask[y1:y2, x1:x2].astype(np.uint8)*255)
                if not ok: raise ValueError('Cannot encode native measured mask')
                protected_write(output/name, buffer.tobytes())
                measured['native_mask'] = name
                measured['foreground_mask'] = None
                if measured_foreground is not None:
                    name = f'pointcloud_projection_masks/{row["target_key"]}.png'
                    ok, buffer = cv2.imencode('.png', measured_foreground)
                    if not ok: raise ValueError('Cannot encode measured projection mask')
                    protected_write(output/name, buffer.tobytes())
                    measured['foreground_mask'] = name
                result = combine_evidence(result, measured)
                if result['projection_plot_source'] == 'pointcloud_front_mask':
                    result['foreground_mask'] = measured['foreground_mask']
            records.append((row['target_id'], result))
        return records
    with ThreadPoolExecutor(max_workers=workers) as pool:
        return {target: result for batch in pool.map(process, sorted(by_frame.items())) for target, result in batch}


def validate_rechecks(document, rows, results, source_sha256):
    if (document.get('kind') != 'assistant_necessary_projection_rechecks'
            or document.get('human_reviewed') is not False
            or document.get('source_targets_sha256') != source_sha256):
        raise ValueError('Necessary visual rechecks must be bound to the exact frozen source')
    by_key = {r['target_key']: r for r in rows}
    observations = {}
    for item in document['observations']:
        key = item['target_key']
        if key not in by_key or key in observations:
            raise ValueError('Unknown or duplicate recheck target')
        target_id = by_key[key]['target_id']
        if not results[target_id]['review_required']:
            raise ValueError('Visual rechecks are limited to image-processing exceptions')
        leg, closure = item['support_leg_count'], item['closure_type']
        if leg is not None and (type(leg) is not int or leg not in (2, 3, 4)):
            raise ValueError('Invalid rechecked leg count')
        if closure not in (None, *CLOSURES) or len(item.get('evidence', '').strip()) < 10:
            raise ValueError('Explicit recheck evidence / valid closure is required')
        if (leg is None or closure is None) and not item.get('unknown_reason'):
            raise ValueError('Partial rechecks require an unknown reason')
        observations[key] = item
    return observations


def enrich_targets(rows, results, seed_ids, rechecks):
    output = []
    for row in rows:
        result = results[row['target_id']]
        observation = rechecks.get(row['target_key'])
        leg = observation['support_leg_count'] if observation else result['support_leg_count']
        closure = observation['closure_type'] if observation else result['closure_type']
        known = int(leg is not None)+int(closure is not None)
        label = STRUCTURES.get((leg, closure))
        new = dict(row)
        new.update(support_leg_count=leg, closure_type=closure, closure_label=CLOSURES.get(closure, '未知'),
            lower_beam_present=None if closure is None else closure == 'tian',
            structure_label=label, structure_code=f'L{leg}_{closure.upper()}' if label else None,
            structure_attribute_state={0: 'unknown', 1: 'partial', 2: 'complete_proposal'}[known],
            structure_view_scope='current_primary_fork_face',
            structure_label_source='assistant_necessary_visual_recheck' if observation else (
                'pointcloud_mask_projection_fallback' if 'pointcloud_front_mask_projections' in result['attribute_sources'].values()
                else 'threshold_xy_projections'),
            structure_is_seed=row['target_id'] in seed_ids,
            structure_visual_rechecked=observation is not None,
            structure_human_reviewed=False, structure_training_eligible=False,
            structure_review_required=True,
            structure_priority_review=known != 2 or (result['review_required'] and not observation),
            structure_review_flags=result['flags'], structure_algorithm_exception=result['review_required'],
            structure_projection_evidence=result,
            structure_visual_recheck=observation)
        output.append(new)
    return output


def aggregate(rows):
    summary = {'total_targets': len(rows), 'appearance_classes': len({r['final_class_id'] for r in rows}),
        'method': 'threshold_xy_projections', 'states': dict(Counter(r['structure_attribute_state'] for r in rows)),
        'leg_counts': dict(Counter(str(r['support_leg_count']) if r['support_leg_count'] else 'unknown' for r in rows)),
        'closures': dict(Counter(r['closure_type'] or 'unknown' for r in rows)),
        'structures': {label: sum(r['structure_label'] == label for r in rows) for label in STRUCTURES.values()},
        'seed_targets': sum(r['structure_is_seed'] for r in rows),
        'algorithm_exceptions': sum(r['structure_algorithm_exception'] for r in rows),
        'visual_rechecked': sum(r['structure_visual_rechecked'] for r in rows),
        'complete_algorithm_proposals': sum(bool(r['structure_label']) and not r['structure_algorithm_exception'] for r in rows),
        'priority_review_targets': sum(r['structure_priority_review'] for r in rows),
        'human_confirmed': 0, 'training_eligible': 0, 'model_training_performed': False,
        'feature_neighbor_propagation': False, 'independent_classification_accuracy': None}
    summary['pointcloud_fallback_attempted'] = sum(r['structure_projection_evidence']['pointcloud_fallback_attempted'] for r in rows)
    summary['targets_using_pointcloud_attributes'] = sum('pointcloud_front_mask_projections' in
        r['structure_projection_evidence']['attribute_sources'].values() for r in rows)
    groups = []
    for class_id in sorted({r['final_class_id'] for r in rows}):
        own = [r for r in rows if r['final_class_id'] == class_id]
        complete = [r for r in own if r['structure_label']]
        counts = Counter(r['structure_label'] for r in complete)
        top = min(counts, key=lambda label: (-counts[label], label)) if counts else None
        groups.append({'final_class_id': class_id, 'final_class': f'Class {class_id}', 'targets': len(own),
            'complete_proposals': len(complete), 'partial': sum(r['structure_attribute_state'] == 'partial' for r in own),
            'unknown': sum(r['structure_attribute_state'] == 'unknown' for r in own),
            'structure_counts': {label: counts[label] for label in STRUCTURES.values()},
            'main_structure_proposal': top, 'main_count': counts[top] if top else 0,
            'main_fraction_of_known': counts[top]/len(complete) if complete else None,
            'main_fraction_of_all': counts[top]/len(own) if top else None,
            'mixed_known_structures': len(counts) > 1,
            'status': 'candidate_distribution_not_a_group_wide_label'})
    return summary, groups


def validate_results(original, output, rechecks):
    if len(output) != len(original) or len({r['target_id'] for r in output}) != len(original):
        raise ValueError('Missing or duplicate structure target')
    for old, row in zip(original, output):
        if any(row.get(k) != v for k, v in old.items()):
            raise ValueError('Original target metadata changed')
        if row['support_leg_count'] not in (None, 2, 3, 4) or row['closure_type'] not in (None, *CLOSURES):
            raise ValueError('Invalid structure attribute')
        if row['structure_label'] != STRUCTURES.get((row['support_leg_count'], row['closure_type'])):
            raise ValueError('Combined label disagrees with independent attributes')
        if row['lower_beam_present'] != (None if row['closure_type'] is None else row['closure_type'] == 'tian'):
            raise ValueError('Lower beam and closure disagree')
        if row['structure_human_reviewed'] or row['structure_training_eligible'] or not row['structure_review_required']:
            raise ValueError('Machine proposals cannot become human-confirmed training labels')
        if row['structure_visual_rechecked']:
            if row['target_key'] not in rechecks or not row['structure_algorithm_exception']:
                raise ValueError('Only necessary exceptions may have visual rechecks')
        else:
            result = row['structure_projection_evidence']
            if any(row[k] != result[k] for k in ('support_leg_count', 'closure_type')):
                raise ValueError('Non-reviewed label differs from measured projection decision')
    json.dumps(output, allow_nan=False)


def table_exports(output, rows, groups, summary):
    fields = ['target_id', 'target_key', 'final_class_id', 'final_class', 'support_leg_count', 'closure_type',
        'closure_label', 'lower_beam_present', 'structure_label', 'structure_code', 'structure_attribute_state',
        'structure_label_source', 'structure_is_seed', 'structure_visual_rechecked', 'structure_algorithm_exception',
        'structure_priority_review', 'structure_review_required', 'structure_review_flags',
        'structure_human_reviewed', 'structure_training_eligible', 'split', 'domain', 'group_id', 'frame_id', 'image', 'bbox']
    subsets = {'targets': rows, 'seed_targets': [r for r in rows if r['structure_is_seed']],
               'priority_review': [r for r in rows if r['structure_priority_review']],
               'visual_rechecks': [r for r in rows if r['structure_visual_rechecked']]}
    for name, subset in subsets.items():
        protected_write(output/(name+'.jsonl'), b''.join(encoded(r) for r in subset))
        write_csv(output/(name+'.csv'), [{k: r[k] for k in fields} for r in subset])
    protected_write(output/'summary.json', encoded(summary))
    protected_write(output/'groups.json', encoded(groups))
    write_csv(output/'groups.csv', groups)
    header = ['目标ID', '原大组', '当前面腿数', '闭合方式', '结构组合', '属性状态', '判读来源', '原440种子',
              '算法异常', '优先复核', '人工确认', '原划分', '配置域', '异常原因']
    records = [[r['target_id'], r['final_class'], r['support_leg_count'] or '未知', r['closure_label'],
                r['structure_label'] or '未知/部分已知', r['structure_attribute_state'], r['structure_label_source'],
                '是' if r['structure_is_seed'] else '否', '是' if r['structure_algorithm_exception'] else '否',
                '是' if r['structure_priority_review'] else '否', '否', r['split'], r['domain'],
                ','.join(r['structure_review_flags'])] for r in rows]
    group_sheet = [['原大组', '全部目标', '完整组合建议', '部分已知', '全部未知', '主要结构候选', '主要结构/已知',
                    '主要结构/全组', '混合结构', *STRUCTURES.values()]]
    group_sheet += [[g['final_class'], g['targets'], g['complete_proposals'], g['partial'], g['unknown'],
                     g['main_structure_proposal'] or '未知', g['main_fraction_of_known'], g['main_fraction_of_all'],
                     '是' if g['mixed_known_structures'] else '否', *g['structure_counts'].values()] for g in groups]
    definitions = [['属性', '取值', '定义'], ['腿数', '2/3/4', '当前主要叉取面沿宽度方向的支撑腿数量'],
        ['川字', 'chuan', '下部无横梁且下部可见'], ['田字', 'tian', '下部有属于该面的横梁'],
        ['未知', '', '投影不稳定、遮挡、视角、下边界不明确；两个属性独立保留'],
        ['状态', 'proposal', '图像处理/助手复查建议，未人工确认，不自动用于训练']]
    protected_write(output/'托盘结构分类结果.xlsx', write_workbook([
        ('大组结构分布', group_sheet), ('全量结构标签', [header]+records),
        ('440种子结构标签', [header]+[line for line, r in zip(records, rows) if r['structure_is_seed']]),
        ('类别定义', definitions)]))


def run(args):
    from pallet.structure_attribute_report import render_reports
    cv2.setNumThreads(1)
    ctx = load_context(args)
    output = args.output.resolve()
    for source in (ctx['source'], ctx['dataset'], ctx['measured_masks'].root,
                   Path('outputs/pallet_target_clustering_v1').resolve()):
        if output.is_relative_to(source) or source.is_relative_to(output):
            raise ValueError('Output must be separate from frozen inputs')
    code_paths = [Path(__file__).resolve(), Path(__file__).with_name('structure_projection.py').resolve(),
                  Path(__file__).with_name('structure_attribute_report.py').resolve(),
                  Path(__file__).with_name('structure_mask_fallback.py').resolve()]
    from importlib.metadata import version
    binding = {'kind': 'pallet_threshold_projection_structure_attributes', 'version': 1,
        'inputs': ctx['inputs'], 'policy': ctx['policy'], 'code': {str(p): digest(p) for p in code_paths},
        'font_source_sha256': digest(args.font),
        'dependencies': {name: version(name) for name in ('numpy', 'scipy', 'pillow', 'fonttools', 'markdown-it-py', 'opencv-python-headless')},
        'workers': args.workers, 'model_training': False, 'feature_neighbor_propagation': False,
        'all_machine_results_unreviewed': True, 'labels_modified': False,
        'rechecks_sha256': digest(args.rechecks) if args.rechecks else None}
    protected_write(output/'run_manifest.json', encoded(binding))
    check_space(output)
    verify_protected(ctx)
    results = process_frames(ctx, args.workers, output)
    rechecks = validate_rechecks(read_json(args.rechecks), ctx['rows'], results,
        digest(ctx['source']/'targets.jsonl')) if args.rechecks else {}
    rows = enrich_targets(ctx['rows'], results, {s['target_id'] for s in ctx['seeds']}, rechecks)
    validate_results(ctx['rows'], rows, rechecks)
    summary, groups = aggregate(rows)
    table_exports(output, rows, groups, summary)
    protected_write(output/'inputs/used_measured_mask_assets.json', encoded(ctx['measured_masks'].used_assets))
    protected_write(output/'inputs/policy.json', encoded(ctx['policy']))
    if args.rechecks:
        protected_write(output/'inputs/necessary_visual_rechecks.json', args.rechecks.read_bytes())
    ctx['recheck_path'] = str(args.rechecks) if args.rechecks else None
    render_reports(ctx, output, rows, summary, groups, args.font)
    protected = verify_protected(ctx, images=True)
    for path, sha in binding['code'].items():
        if digest(path) != sha:
            raise ValueError('Implementation changed during extraction')
    if args.rechecks and digest(args.rechecks) != binding['rechecks_sha256']:
        raise ValueError('Visual rechecks changed during extraction')
    validation = {'state': 'passed', 'targets': len(rows), 'appearance_classes': len(groups),
        'seed_targets': summary['seed_targets'], 'all_original_fields_unchanged': True,
        'independent_attributes_and_combination_consistent': True, 'all_machine_labels_unreviewed': True,
        'image_processing_only_except_necessary_rechecks': True, 'feature_neighbor_propagation': False,
        'native_images_hash_verified': len(ctx['frames']), 'protected_formal_files_unchanged': protected,
        'visual_rechecks': len(rechecks), 'independent_accuracy_measured': False}
    validation['measured_mask_source_assets_verified'] = len(ctx['measured_masks'].used_assets)
    protected_write(output/'validation.json', encoded(validation))
    files = {str(p.relative_to(output)): digest(p) for p in sorted(output.rglob('*')) if p.is_file()
             and p.name not in ('output_manifest.json', 'checksums.sha256')}
    protected_write(output/'output_manifest.json', encoded({'state': 'complete', 'files': files,
        'summary': summary, 'validation': validation}))
    protected_write(output/'checksums.sha256', ''.join(f'{sha}  {name}\n' for name, sha in files.items()).encode())
    print(json.dumps({'output': str(output), **summary, 'validation': validation}, ensure_ascii=False))


def main(argv=None):
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--source', type=Path, default=DEFAULT_SOURCE)
    parser.add_argument('--formal', type=Path, default=DEFAULT_FORMAL)
    parser.add_argument('--seeds', type=Path, default=DEFAULT_SEEDS)
    parser.add_argument('--policy', type=Path, default=DEFAULT_POLICY)
    parser.add_argument('--masks', type=Path, default=DEFAULT_MASKS)
    parser.add_argument('--output', type=Path, default=DEFAULT_OUTPUT)
    parser.add_argument('--workers', type=int, choices=range(1, 9), default=8)
    parser.add_argument('--rechecks', type=Path)
    parser.add_argument('--font', type=Path, default=Path('/tmp/NotoSansCJKsc-Regular.otf'))
    run(parser.parse_args(argv))


if __name__ == '__main__':
    main()
