"""Read-only class/topology/appearance audit of294 model masks and1559 saves."""
from collections import Counter, defaultdict
from contextlib import closing
import csv
import hashlib
import html
import json
import math
from pathlib import Path
import sqlite3
import sys

import cv2
import numpy as np
from PIL import Image, ImageDraw

PROJECT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(PROJECT))
from pallet.cluster_targets import digest, protected_snapshot
from pallet.mask_annotation_anomalies import (modes, normalize, polygon_features, best_index_alignment,
    feature_diagnostics, medoid_index, robust_upper)
from pallet.mask_model_training_data import read, jsonl, write, verify_dataset
from pallet.mask_quota_saved_extension import load_formal_evidence, file_inventory
from pallet.pose_review_geometry import canonical, polygon_value, raster_mask
from scripts.build_mask_remaining_direct_requests import build_font

DATA = PROJECT/'outputs/pallet_mask_dataset_update_v1'
SOURCE = PROJECT/'outputs/pallet_mask_continuation_sources_v14'
CLUSTER = PROJECT/'outputs/pallet_target_clustering_v1/analysis/run_006'
CENTERS = PROJECT/'outputs/pallet_appearance_center_reclassification_v1/centers.json'
OUT = PROJECT/'outputs/pallet_mask_annotation_anomalies_v2'


def require(value, message):
    if not value:
        raise ValueError(message)


def store_events(path):
    with closing(sqlite3.connect(Path(path).resolve().as_uri()+'?mode=ro', uri=True)) as db:
        db.execute('BEGIN')
        return list(db.execute('SELECT id,target_key,revision,payload FROM events ORDER BY id'))


def feature_index(rows):
    raw = jsonl(CLUSTER/'targets.jsonl'); by_id = {r['target_id']: r for r in raw}
    require(len(by_id) == len(raw) == 12344, 'Unique cached target IDs required')
    with np.load(CLUSTER/'features.npz') as data:
        indices, features = data['appearance_indices'], data['appearance_features']
    require(indices.ndim == 1 and len(set(indices.tolist())) == len(indices) == len(features)
        and features.shape == (12344, 34) and np.isfinite(features).all(), 'Cached image feature shape/identity invalid')
    positions = {raw[int(source_i)]['target_id']: j for j, source_i in enumerate(indices)}
    aligned = {}
    for key, row in rows.items():
        original = by_id[row['target_id']]
        require(original['target_key'] == key and original['frame_id'] == row['frame_id']
            and original['bbox'] == row['bbox'] and original['split'] == row['split'], 'Cached feature/formal source identity differs')
        aligned[key] = np.asarray(features[positions[row['target_id']]], float)
    return aligned


def class_summary(records):
    result = {}; groups = defaultdict(list)
    for r in records:
        groups[r['class_id']].append(r)
    for cls, values in sorted(groups.items()):
        modal, counts = modes([r['vertices'] for r in values]); major_n = max(counts.values())
        strata = defaultdict(list)
        for r in values:
            strata[(r['formal_split'], r['policy_id'], r['surface'], r['structure'])].append(r)
        mixed_strata = [dict(formal_split=k[0], policy=k[1], surface=k[2], structure=k[3],
            n=len(v), counts=dict(Counter(r['vertices'] for r in v))) for k, v in sorted(strata.items()) if len({r['vertices'] for r in v}) > 1]
        result[cls] = dict(n=len(values), modal_vertex_counts=modal, vertex_count_distribution=dict(sorted(counts.items())),
            mixed_vertex_counts=len(counts) > 1, modal_fraction=major_n/len(values),
            multiple_substantial_count_modes=sum(n/len(values) >= .2 for n in counts.values()) >= 2,
            winding_counts=dict(Counter(r['polygon_geometry']['image_coordinate_winding'] for r in values)),
            structures=dict(Counter(r['structure'] for r in values)), surfaces=dict(Counter(r['surface'] for r in values)),
            acquisition_groups=len({r['group_id'] for r in values}), same_policy_surface_structure_mixed=mixed_strata)
    return result


def shape_diagnostics(values):
    # Normalize only for comparison. Saved coordinates/order are never changed.
    masks = []
    for r in values:
        p = normalize(r['polygon'], r['bbox'])*[64, 32]
        masks.append(raster_mask(p, 64, 32).reshape(-1).astype(np.float32))
    matrix = np.stack(masks); intersection = matrix@matrix.T
    areas = matrix.sum(axis=1); union = areas[:, None]+areas[None, :]-intersection
    distance = 1-np.divide(intersection, union, out=np.ones_like(intersection), where=union > 0)
    reference = medoid_index(distance); scores = distance[:, reference]; threshold = robust_upper(scores, .15)
    return reference, scores, threshold


def scoped(records, features):
    records = [dict(r) for r in records]
    x = np.stack([features[r['target_key']] for r in records])
    appearances, reps = feature_diagnostics(x, [r['class_id'] for r in records], [r['group_id'] for r in records],
        [(r['policy_id'], r['surface'], tuple(r['size'])) for r in records])
    stats = class_summary(records); by_class = defaultdict(list)
    for i, r in enumerate(records):
        r['appearance'] = appearances[i]; r['flags'] = []
        if r['vertices'] not in stats[r['class_id']]['modal_vertex_counts']:
            r['flags'].append('nonmodal_vertex_count_not_automatically_wrong')
        if appearances[i]['within_class_outlier']:
            r['flags'].append('cached_image_descriptor_within_class_outlier')
        if appearances[i]['alternative_medoid_candidate']:
            r['flags'].append('closer_to_other_class_image_medoid_candidate')
        if r['classification_priority_review']:
            r['flags'].append('existing_classification_priority_review_not_new_proof')
        by_class[r['class_id']].append(i)
    for cls, indices in by_class.items():
        values = [records[i] for i in indices]; reference_local, scores, threshold = shape_diagnostics(values)
        shape_key = values[reference_local]['target_key']
        winding_mode, _ = modes([r['polygon_geometry']['image_coordinate_winding'] for r in values])
        same_count = defaultdict(list)
        for i in indices:
            same_count[records[i]['vertices']].append(i)
        templates = {}
        for count, count_indices in same_count.items():
            flattened = np.stack([normalize(records[i]['polygon'], records[i]['bbox']).reshape(-1) for i in count_indices])
            d = np.linalg.norm(flattened[:, None, :]-flattened[None, :, :], axis=2)
            templates[count] = count_indices[medoid_index(d)]
        for local, i in enumerate(indices):
            r = records[i]; template = records[templates[r['vertices']]]
            alignment = best_index_alignment(normalize(r['polygon'], r['bbox']), normalize(template['polygon'], template['bbox']))
            r['vertex_index_comparison'] = dict(reference_target_key=template['target_key'], **alignment)
            r['shape_comparison'] = dict(reference_target_key=shape_key, normalized_mask_distance=float(scores[local]),
                diagnostic_threshold=threshold, shape_outlier=bool(scores[local] > threshold))
            r['class_references'] = dict(main_target_key=records[reps[cls]['image_weighted_medoid']]['target_key'],
                group_balanced_target_key=records[reps[cls]['group_balanced_medoid']]['target_key'],
                representative_keys=[records[j]['target_key'] for j in reps[cls]['representative_indices']],
                dominant_group_fraction=reps[cls]['dominant_group_fraction'])
            alt = r['appearance'].get('alternative_medoid_index')
            r['appearance']['alternative_medoid_target_key'] = records[alt]['target_key'] if alt is not None else None
            cross = r['appearance'].get('nearest_same_class_different_group')
            r['appearance']['nearest_same_class_different_group_key'] = records[cross]['target_key'] if cross is not None else None
            if alignment['index_order_candidate']:
                r['flags'].append('cyclic_start_or_winding_convention_candidate')
            if r['polygon_geometry']['image_coordinate_winding'] not in winding_mode:
                r['flags'].append('minority_polygon_winding_not_mask_error')
            if r['shape_comparison']['shape_outlier']:
                r['flags'].append('normalized_shape_differs_from_class_medoid')
            # Count/ordering are not an error score. Appearance+shape concurrence
            # gets higher visual-review priority; none is a relabelling command.
            r['review_priority'] = 4*int(r['appearance']['within_class_outlier'] and r['shape_comparison']['shape_outlier'])
            r['review_priority'] += 2*int(r['appearance']['alternative_medoid_candidate'])
            r['review_priority'] += int('nonmodal_vertex_count_not_automatically_wrong' in r['flags'])
            r['review_priority'] += int(alignment['index_order_candidate'])
    return records, stats


def comparison_image(root, record, reference, rows, evidence, name):
    canvas = Image.new('RGB', (1000, 650), '#14212d'); draw = ImageDraw.Draw(canvas)
    for column, (value, label) in enumerate(((record, 'SAMPLE'), (reference, 'SAME CLASS REFERENCE'))):
        row = rows[value['target_key']]; image = Image.fromarray(evidence.image(row['frame_id'])).convert('RGB')
        top = image.copy(); box = row['bbox']; d = ImageDraw.Draw(top)
        d.rectangle((box['x'], box['y'], box['x']+box['width'], box['y']+box['height']), outline='#ffc247', width=2)
        top.thumbnail((485, 300)); x = column*500+5
        draw.text((x, 4), f'{label}: C{value["class_id"]} / {value["vertices"]} vertices / {row["split"]}', fill='white')
        draw.text((x, 20), value['target_key'], fill='white'); canvas.paste(top, (x, 42))
        bounds = (max(0, math.floor(box['x']-.10*box['width'])), max(0, math.floor(box['y']-.30*box['height'])),
            min(row['size'][0], math.ceil(box['x']+1.10*box['width'])), min(row['size'][1], math.ceil(box['y']+1.30*box['height'])))
        marked = image.copy(); d = ImageDraw.Draw(marked); points = value['polygon']
        d.line([tuple(p) for p in points]+[tuple(points[0])], fill='#55d5ff', width=1)
        crop = marked.crop(bounds); scale = min(6, 485/crop.width, 250/crop.height)
        crop = crop.resize((max(1, round(crop.width*scale)), max(1, round(crop.height*scale))), Image.Resampling.NEAREST)
        d = ImageDraw.Draw(crop)
        for j, p in enumerate(points, 1):
            d.text(((p[0]-bounds[0])*scale, (p[1]-bounds[1])*scale), str(j), fill='#ffdd55', stroke_width=1, stroke_fill='black')
        canvas.paste(crop, (x, 370))
    canvas.save(root/name)


def report(root, full, training, full_stats, training_stats, rows, evidence):
    all_map = {r['target_key']: r for r in full}; train_map = {r['target_key']: r for r in training}
    selected = set()
    # Always show BOTH current-model point-count deviations, and high-ranked
    # image/shape differences; the rest stay in complete CSV/JSONL inventories.
    for values in (training, full):
        by_class = defaultdict(list)
        for r in values:
            by_class[r['class_id']].append(r)
        for cls, items in by_class.items():
            ranked = sorted([r for r in items if r['flags']], key=lambda r: (-r['review_priority'], -r['appearance']['within_class_distance_to_medoid'], r['target_key']))
            selected.update(r['target_key'] for r in ranked[:3])
            nonmodal = [r for r in items if 'nonmodal_vertex_count_not_automatically_wrong' in r['flags']]
            if nonmodal:
                selected.add(min(nonmodal, key=lambda r: (r['vertices'], r['target_key']))['target_key'])
    selected.update(r['target_key'] for r in training if 'nonmodal_vertex_count_not_automatically_wrong' in r['flags'])
    (root/'cases').mkdir(); cards = []; case_index = []
    for key in sorted(selected, key=lambda k: (k not in train_map, all_map[k]['class_id'], -all_map[k]['review_priority'], k)):
        training_scope = key in train_map and bool(train_map[key]['flags'])
        r = train_map[key] if training_scope else all_map[key]
        refs = train_map if training_scope else all_map
        scope = 'current294' if training_scope else 'all1559'
        main_key = r['class_references']['main_target_key']; reference = refs[main_key]
        if key == main_key:
            alternatives = [k for k in r['class_references']['representative_keys'] if k != key]
            if not alternatives:
                alternatives = [other['target_key'] for other in sorted(refs.values(), key=lambda other: other['appearance']['within_class_distance_to_medoid'])
                    if other['class_id'] == r['class_id'] and other['target_key'] != key]
            if alternatives:
                reference = refs[alternatives[0]]
        path = f'cases/{key}.png'; comparison_image(root, r, reference, rows, evidence, path)
        case_index.append(dict(target_key=key, class_id=r['class_id'], in_model_dataset=key in train_map,
            reference_target_key=reference['target_key'], path=path, flags=r['flags'], priority=r['review_priority'], diagnostic_scope=scope))
        notes = '；'.join(r['flags'])
        cards.append(f'<article id="target-{key}" data-class="{r["class_id"]}" data-training="{int(key in train_map)}"><h2>C{r["class_id"]} · {r["vertices"]}点 · {"当前294内" if key in train_map else "全量审计，未纳入当前模型"}</h2><p><code>{key}</code> · 比较范围：{scope}</p><p>{html.escape(notes)}</p><a href="{path}"><img loading="lazy" src="{path}" alt="原图定位和原顺序顶点对照，左样本右同类主要参考"></a><p>左样本 / 右同类主要参考；黄色仅原框，蓝线为真实人工轮廓，数字保持原始顺序。候选差异不等于错标；全量差异不自动等于294内差异。</p></article>')
    table = []
    for cls, s in full_stats.items():
        t = training_stats.get(cls)
        dist = ' / '.join(f'{k}点×{n}' for k, n in s['vertex_count_distribution'].items())
        tdist = '未纳入' if t is None else ' / '.join(f'{k}点×{n}' for k, n in t['vertex_count_distribution'].items())
        table.append(f'<tr><td>{cls}</td><td>{s["n"]}</td><td>{dist}</td><td>{tdist}</td><td>{len(s["same_policy_surface_structure_mixed"])}</td></tr>')
    page = '<!doctype html><html lang="zh-CN"><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>托盘标注异常只读审计</title><style>@font-face{font-family:Audit;src:url("report-font.otf")}*{box-sizing:border-box}body{font:15px/1.65 Audit,sans-serif;background:#10202c;color:#e4edf3;max-width:1220px;margin:auto;padding:14px}a{color:#8dd8ff}table{border-collapse:collapse;min-width:750px}th,td{padding:8px;border:1px solid #597184;text-align:left}.table{overflow:auto}article{border:1px solid #597184;padding:14px;margin:20px 0;overflow-wrap:anywhere}img{width:100%;height:auto;display:block}select,label{font:inherit;padding:6px}code{overflow-wrap:anywhere}.warning{padding:15px;border:2px solid #e4b461}[hidden]{display:none!important}</style>'
    page += '<h1>294份训练研究数据 / 1,559份人工轮廓 · 异常审计</h1><p class="warning">这里的点是多边形顶点，不是已定义稳定语义的关键点。有效四点、不同行为的开口/脚、不同观察面可能合法不同；本报告不自动增删点、改类别、改训练集或重训。所有外观距离仅为只读诊断，不是分类概率或独立测试分数。</p><p>全量范围含正式val/test，仅用于人工标注审计；模型范围另算，不借全量参考进行训练/选模。Class19含7份隔离待质检轮廓，它们仍不进入294份模型数据。</p>'
    page += '<p><a href="summary.json">汇总JSON</a> · <a href="anomalies.csv">完整异常CSV</a> · <a href="all_sources.jsonl">全量逐样本审计</a> · <a href="training_sources.jsonl">294份逐样本审计</a></p><div class="table"><table><tr><th>Class</th><th>全量N</th><th>全量点数分布</th><th>294内点数分布</th><th>同划分/策略/面/结构仍混点数的组数</th></tr>'+''.join(table)+'</table></div>'
    page += '<p><label>类别 <select id="class"><option value="">全部</option>'+''.join(f'<option value="{c}">{c}</option>' for c in full_stats)+'</select></label> <label><input id="training" type="checkbox">只看当前294内</label></p>'+''.join(cards)
    page += '<script>const c=document.getElementById("class"),t=document.getElementById("training");function filter(){for(const a of document.querySelectorAll("article"))a.hidden=!!c.value&&a.dataset.class!==c.value||t.checked&&a.dataset.training!=="1";}c.onchange=t.onchange=filter;</script></html>'
    (root/'report_zh.html').write_text(page); build_font(root, page)
    write(root/'case_index.json', case_index)
    return case_index


def main():
    require(not OUT.exists(), 'Use a fresh immutable diagnostic directory')
    cv2.setNumThreads(1)
    manifest, model = verify_dataset(DATA)
    require(len(model) == 294 and digest(DATA/'dataset_manifest.json') == 'd266590b36bab45c4caf32cb6bcdff233e726a14ebfaf5985ca333ac6eff4425', 'Current dataset version changed')
    formal, rows, evidence = load_formal_evidence(read(PROJECT/'outputs/pallet_mask_remaining_round5_v1/manifest.json'))
    formal_before = protected_snapshot(formal); config_before = file_inventory(PROJECT/'configs')
    stores = [PROJECT/'outputs/pallet_pose_human_review_v1/annotations.sqlite3', PROJECT/'outputs/pallet_mask_gallery_reviews_v1/mask_reviews.sqlite3', PROJECT/'outputs/pallet_mask_nonconforming_reviews_v1/mask_reviews.sqlite3']
    before_events = {str(p.relative_to(PROJECT)): store_events(p) for p in stores}
    inputs = {str(p.relative_to(PROJECT)): digest(p) for p in (DATA/'dataset_manifest.json', DATA/'private/training_records.jsonl',
        DATA/'private/nonconforming_source_catalog.jsonl', SOURCE/'private/source_catalog.jsonl', SOURCE/'audit.json',
        CLUSTER/'features.npz', CLUSTER/'targets.jsonl', CENTERS, Path(__file__).resolve(), PROJECT/'pallet/mask_annotation_anomalies.py')}
    sources = jsonl(SOURCE/'private/source_catalog.jsonl')+jsonl(DATA/'private/nonconforming_source_catalog.jsonl')
    require(len(sources) == len({r['target_key'] for r in sources}) == 1559, 'Frozen manual source scope changed')
    model_map = {r['target_key']: r for r in model}; records = []
    for source in sources:
        row, points = source['row'], source['polygon']; polygon_value(points, *row['size'])
        records.append(dict(target_key=source['target_key'], class_id=row['final_class_id'], formal_split=row['split'],
            in_model_dataset=source['target_key'] in model_map,
            development_split=model_map.get(source['target_key'], {}).get('development_split'),
            polygon=points, vertices=len(points), bbox=row['bbox'], size=row['size'], native_image=row['image'],
            group_id=row['group_id'], policy_id=row['policy_id'], surface=row.get('viewpoint_candidate') or 'unknown',
            structure=row.get('structure_candidate') or 'unknown', source_quality=source.get('source_quality', 'saved_research_reference_not_GT'),
            source_revision=source['revision'], source_payload_sha256=source['payload_sha256'],
            classification_priority_review=bool(row.get('classification_priority_review')),
            classification_review_required=bool(row.get('classification_review_required')),
            frozen_center_distance_ratio=row.get('distance_to_reference_p95_ratio'),
            frozen_nearest_center_class_id=row.get('nearest_center_class_id'),
            polygon_geometry=polygon_features(points, row['bbox'])))
    records.sort(key=lambda r: (r['class_id'], r['target_key']))
    features = feature_index(rows)
    full, full_stats = scoped(records, features)
    training, training_stats = scoped([r for r in records if r['in_model_dataset']], features)
    OUT.mkdir(mode=0o700)
    write(OUT/'all_sources.jsonl', full, lines=True); write(OUT/'training_sources.jsonl', training, lines=True)
    write(OUT/'class_statistics.json', dict(all_saved=full_stats, current_model=training_stats))
    all_flags = Counter(flag for r in full for flag in r['flags']); training_flags = Counter(flag for r in training for flag in r['flags'])
    summary = dict(state='read_only_diagnostic_candidates_not_label_error_ground_truth', manual_sources=1559, model_sources=294,
        model_classes=13, model_mixed_vertex_classes=[c for c, v in training_stats.items() if v['mixed_vertex_counts']],
        all_saved_mixed_vertex_classes=[c for c, v in full_stats.items() if v['mixed_vertex_counts']],
        all_saved_flags=dict(all_flags), model_flags=dict(training_flags),
        no_semantic_keypoint_mapping=True, polygon_vertex_counts_are_not_keypoint_labels=True,
        labels_modified=False, class_assignments_modified=False, dataset_modified=False, training_or_model_inference_started=False,
        appearance_source='frozen run006 34D cached image/PCA+shape features, no new model or PCA fit',
        thresholds=dict(robust_outlier='median + max(floor,3.5*1.4826*MAD), floor=.20 feature/.15 shape distance',
            alternative_medoid_relative_advantage=.15, minimum_own_medoid_feature_distance=.30,
            index_alignment_raw_rms_min=.15, index_alignment_gain_min=.10, best_aligned_rms_max=.12),
        caveats=['Majority is not truth; multiple real views/topologies/sensor domains exist.',
            'Within-policy/surface/size diagnostics need>=8 images; missing support staysunknown.',
            'Raw point order is not stable physical corner semantics; cyclic reverse equivalents have identical masks.',
            'All-saved medoids include formalval/test for annotation audit only; current294 diagnostics computed separately.',
            'Repeated groups can dominate image-majority; group-balanced medoid is also reported.'],
        inputs_sha256=inputs)
    write(OUT/'summary.json', summary)
    with (OUT/'anomalies.csv').open('x', encoding='utf-8-sig', newline='') as stream:
        writer = csv.writer(stream); writer.writerow(['scope','target_key','class_id','formal_split','development_split','vertices','flags','review_priority','same_class_reference','alternative_medoid_class','alternative_reference'])
        for scope, values in (('current294', training), ('all1559', full)):
            for r in values:
                if r['flags']:
                    writer.writerow([scope,r['target_key'],r['class_id'],r['formal_split'],r['development_split'],r['vertices'], '|'.join(r['flags']),r['review_priority'],r['class_references']['main_target_key'],r['appearance']['alternative_medoid_class'],r['appearance']['alternative_medoid_target_key']])
    cases = report(OUT, full, training, full_stats, training_stats, rows, evidence)
    observed_appends = {}
    for path in stores:
        name = str(path.relative_to(PROJECT)); after = store_events(path); before = before_events[name]
        require(after[:len(before)] == before, 'Original raw human event prefix changed')
        observed_appends[name] = len(after)-len(before)
    require(protected_snapshot(formal) == formal_before and file_inventory(PROJECT/'configs') == config_before, 'Protected formal/config changed')
    verify_dataset(DATA)
    for name, expected in inputs.items():
        require(digest(PROJECT/name) == expected, 'Frozen audit input changed: '+name)
    write(OUT/'verification.json', dict(state='passed_recomputable_diagnostics_and_preservation_not_semantic_accuracy',
        sources=1559, model_sources=294, source_point_counts_preserved=True, dataset_files_verified=len(manifest['files']),
        input_sha256=inputs, observed_legal_human_appends_not_incorporated=observed_appends,
        formal_protected_files=len(formal_before), old_labels_and_classes_unchanged=True, GPU_used=False,
        report_cases=len(cases), diagnostic_artifacts_sha256=file_inventory(OUT)))
    print(json.dumps(dict(state=summary['state'], model_mixed_vertex_classes=summary['model_mixed_vertex_classes'],
        all_saved_mixed_vertex_classes=summary['all_saved_mixed_vertex_classes'], model_flags=training_flags,
        all_saved_flags=all_flags, report_cases=len(cases))), flush=True)


if __name__ == '__main__':
    main()
