"""Clustering, group resampling and Chinese report for target attribute discovery."""
from __future__ import annotations

from collections import Counter, defaultdict
import csv
import html
import json
import math
from pathlib import Path
import time

import cv2
import numpy as np

from .cluster_targets import (digest, measure_points, protected_snapshot, read_rows,
                             save_json, target_id, timestamp, write_rows)


GEOMETRY_FIELDS = ['visible_extent_major_mm', 'visible_extent_middle_mm', 'visible_extent_minor_mm',
                   'visible_horizontal_span_mm', 'visible_vertical_span_mm', 'linearity', 'planarity',
                   'scattering', 'principal_extent_ratio', 'range_median_mm']

GEOMETRY_QUALITY_POLICY = {'version': 2, 'minimum_image_aspect': 4.,
                         'maximum_principal_extent_ratio': 3., 'minimum_range_excursion_ratio': .35,
                         'interpretation': 'candidate mask may include aperture background or a strongly oblique surface; exclude from specification clustering'}


def foreground_geometry_quality(points, scale_to_mm, bbox, measurements):
    ranges = np.linalg.norm(np.asarray(points,dtype=float)*scale_to_mm,axis=1)
    excursion = float(np.ptp(np.percentile(ranges,[2,98])))
    ratio = excursion/max(measurements['visible_extent_major_mm'],1.)
    ambiguous = (bbox['width']/bbox['height']>=GEOMETRY_QUALITY_POLICY['minimum_image_aspect']
                 and measurements['principal_extent_ratio']<GEOMETRY_QUALITY_POLICY['maximum_principal_extent_ratio']
                 and ratio>=GEOMETRY_QUALITY_POLICY['minimum_range_excursion_ratio'])
    return {'range_excursion_mm':excursion,'range_excursion_to_major_ratio':ratio,
            'geometry_review_flags':['foreground_depth_or_view_ambiguous'] if ambiguous else [],
            'geometry_measurement_confidence':'unknown' if ambiguous else 'visible_candidate'}


def infer_batch_units(rows):
    groups = defaultdict(list)
    for row in rows:
        groups[row['policy_id']].append(row)
    registry = {}
    for policy, members in sorted(groups.items()):
        known = {r['verified_scale_to_mm'] for r in members if r['verified_scale_to_mm'] is not None}
        if len(known) > 1:
            raise ValueError('Conflicting frozen unit evidence within a policy')
        by_frame = defaultdict(Counter)
        for row in members:
            hint = row.get('unit_observation', {}).get('hint', 'unknown')
            if hint != 'unknown':
                by_frame[row['frame_id']][hint] += 1
        votes = Counter()
        for observations in by_frame.values():
            ranked = observations.most_common()
            if len(ranked) == 1 or ranked[0][1] > ranked[1][1]:
                votes[ranked[0][0]] += 1
        total = sum(votes.values())
        winning, number = votes.most_common(1)[0] if total else ('unknown', 0)
        fraction = number/max(total, 1)
        scale, status, reason = None, 'unknown', 'insufficient_or_mixed_distribution'
        if known:
            scale, status, reason = float(next(iter(known))), 'verified', 'frozen_metric_unit_evidence'
        elif total >= min(3, len({r['frame_id'] for r in members})) and fraction >= .9:
            scale, status, reason = (1000. if winning == 'm' else 1.), 'inferred', 'user_distribution_heuristic_batch_consensus'
        registry[policy] = {'policy_id': policy, 'scale_to_mm': scale, 'unit_status': status,
                            'reason': reason, 'frame_votes': dict(votes), 'vote_fraction': fraction,
                            'voting_frames': total, 'frames': len({r['frame_id'] for r in members}),
                            'targets': len(members), 'scope': 'analysis_only_no_sensor_policy_rewrite'}
    return registry


def resolve_target_unit(row, registry):
    batch = registry[row['policy_id']]
    scale, status = batch['scale_to_mm'], batch['unit_status']
    hint = row.get('unit_observation', {}).get('hint', 'unknown')
    expected_hint = 'm' if scale == 1000 else 'mm' if scale == 1 else 'unknown'
    if status == 'inferred' and hint not in ('unknown', expected_hint):
        return None, 'conflict', 'individual_distribution_conflicts_with_batch'
    return scale, status, batch['reason']


def collect_targets(output, annotations, sources):
    rows, embeddings = [], []
    pending = []
    for annotation in annotations:
        if not annotation['boxes']:
            continue
        path = output/'frames'/(Path(annotation['image']).stem+'.npz')
        if not path.exists():
            pending.append(annotation['frame_id'])
            continue
        with np.load(path, allow_pickle=False) as archive:
            saved = json.loads(str(archive['records_json']))
            appearance = archive['appearance']
            expected = [target_id(annotation['frame_id'], i) for i in range(len(annotation['boxes']))]
            if [r['target_id'] for r in saved] != expected or len(appearance) != len(saved):
                raise ValueError('Extraction cache target identities differ from formal annotations')
            for i, row in enumerate(saved):
                if row['bbox'] != annotation['boxes'][i] or row['source_pcd_sha256'] != sources[row['frame_id']]['source_anchor_hash']:
                    raise ValueError('Cached box/source binding differs from formal annotations')
                row['_cache_path'] = str(path)
                rows.append(row)
                embeddings.append(appearance[i].copy())
    if pending:
        raise ValueError(f'Analysis requires complete extraction: {len(pending)} positive frames missing; first={pending[0]}')
    expected_total = sum(len(r['boxes']) for r in annotations)
    if len(rows) != expected_total or len({r['target_id'] for r in rows}) != expected_total:
        raise ValueError('Incomplete or duplicate target coverage')
    return rows, np.asarray(embeddings, dtype=np.float32)


def add_geometry(rows, registry):
    by_cache = defaultdict(list)
    for row in rows:
        by_cache[row['_cache_path']].append(row)
    for path, members in by_cache.items():
        with np.load(path, allow_pickle=False) as archive:
            for row in members:
                scale, status, reason = resolve_target_unit(row, registry)
                row.update(scale_to_mm=scale, unit_status=status, unit_reason=reason)
                row['geometry_measurement_confidence']='unknown'
                if scale is None:
                    if row['geometry_status'] in ('eligible', 'partial_visible_surface'):
                        row['geometry_status'] = 'unit_unknown'
                    continue
                if row['geometry_status'] not in ('eligible', 'partial_visible_surface'):
                    continue
                try:
                    i = row['box_index']
                    measurements = measure_points(archive[f'points_{i}'], archive[f'uv_{i}'], scale)
                    row.update(measurements)
                    row.update(foreground_geometry_quality(archive[f'points_{i}'], scale, row['bbox'], measurements))
                    if row['geometry_status'] == 'eligible':
                        row['geometry_status'] = ('foreground_depth_or_view_ambiguous' if row['geometry_review_flags'] else 'ok')
                except Exception as exc:
                    row['geometry_status'] = 'measurement_failed'
                    row['geometry_error'] = f'{type(exc).__name__}: {exc}'


def appearance_matrix(rows, embeddings, seed):
    from sklearn.decomposition import PCA
    from sklearn.preprocessing import RobustScaler
    valid = np.isfinite(embeddings).all(axis=1)
    indices = np.flatnonzero(valid)
    if len(indices) < 2:
        raise ValueError('Insufficient valid appearance features')
    components = min(32, len(indices)-1, embeddings.shape[1])
    pca = PCA(n_components=components, random_state=seed, svd_solver='randomized')
    reduced = pca.fit_transform(embeddings[valid]).astype(np.float64)
    reduced /= np.maximum(np.linalg.norm(reduced, axis=1, keepdims=True), 1e-9)
    shape = np.array([[math.log(r['bbox']['width']/r['bbox']['height']), r.get('mask_fraction', .5)] for r in rows])[valid]
    scaler = RobustScaler().fit(shape)
    shape = np.clip(scaler.transform(shape), -3, 3)*.15
    matrix = np.column_stack([reduced, shape])
    return indices, matrix, reduced, {'pca_components': components,
        'pca_explained_variance': pca.explained_variance_ratio_.tolist(),
        'shape_features': ['log_bbox_aspect_ratio', 'foreground_mask_fraction'], 'shape_weight': .15,
        'pca_components_array': pca.components_, 'pca_mean_array': pca.mean_,
        'shape_center_array': scaler.center_, 'shape_scale_array': scaler.scale_}


def geometry_matrix(rows):
    from sklearn.preprocessing import RobustScaler
    indices = np.array([i for i, r in enumerate(rows) if r['geometry_status'] == 'ok'], dtype=int)
    if not len(indices):
        return indices, np.empty((0, 7)), {}
    numeric = np.array([[math.log(max(rows[i][name], 2.)) for name in GEOMETRY_FIELDS[:5]]
                        + [rows[i]['planarity'], rows[i]['scattering']] for i in indices])
    scaler = RobustScaler().fit(numeric)
    return indices, np.clip(scaler.transform(numeric), -5, 5), {
        'features': ['log_'+k for k in GEOMETRY_FIELDS[:5]]+['planarity', 'scattering'],
        'center': scaler.center_.tolist(), 'scale': scaler.scale_.tolist()}


def stable_labels(labels):
    counts = Counter(int(v) for v in labels if v >= 0)
    remap = {old: new for new, old in enumerate(sorted(counts, key=lambda v: (-counts[v], v)))}
    return np.array([remap.get(int(v), int(v)) for v in labels], dtype=int)


def fit_clusters(matrix, min_cluster_size=50):
    from sklearn.cluster import HDBSCAN
    if len(matrix) < min_cluster_size:
        return np.full(len(matrix), -1, dtype=int), np.zeros(len(matrix)), None
    model = HDBSCAN(min_cluster_size=min_cluster_size, min_samples=10, n_jobs=4,
                    cluster_selection_method='eom', copy=True).fit(matrix)
    return stable_labels(model.labels_), model.probabilities_, model


def cluster_statistics(matrix, labels, probabilities, sample_rows, seed):
    from sklearn.metrics import silhouette_score, normalized_mutual_info_score
    assigned = labels >= 0
    counts = Counter(int(v) for v in labels if v >= 0)
    result = {'targets': len(labels), 'clusters': len(counts), 'noise': int((labels == -1).sum()),
              'assigned_fraction': float(assigned.mean()) if len(labels) else 0.,
              'cluster_sizes': dict(sorted(counts.items())), 'silhouette_assigned': None,
              'domain_nmi_assigned': None, 'group_nmi_assigned': None,
              'mean_membership_assigned': float(np.mean(probabilities[assigned])) if assigned.any() else None}
    if len(counts) >= 2 and assigned.sum() > len(counts):
        selected = np.flatnonzero(assigned)
        if len(selected) > 2500:
            selected = np.random.default_rng(seed).choice(selected, 2500, replace=False)
        if len(set(labels[selected])) >= 2:
            result['silhouette_assigned'] = float(silhouette_score(matrix[selected], labels[selected]))
    if assigned.any():
        domains = [sample_rows[i]['domain'] for i in np.flatnonzero(assigned)]
        groups = [sample_rows[i]['group_id'] for i in np.flatnonzero(assigned)]
        result['domain_nmi_assigned'] = float(normalized_mutual_info_score(domains, labels[assigned]))
        result['group_nmi_assigned'] = float(normalized_mutual_info_score(groups, labels[assigned]))
    return result


def bootstrap_clusters(matrix, labels, rows, rounds, seed, log):
    from sklearn.metrics import adjusted_rand_score
    groups = defaultdict(list)
    for i, row in enumerate(rows):
        groups[row['group_id']].append(i)
    names = sorted(groups)
    rng = np.random.default_rng(seed)
    results = []
    if not names:
        return {'rounds': [], 'mean_ari': None, 'mean_ari_assigned': None}
    for number in range(rounds):
        chosen = rng.choice(names, len(names), replace=True)
        sampled = np.concatenate([groups[name] for name in chosen])
        new_labels, _, _ = fit_clusters(matrix[sampled])
        unique, positions = np.unique(sampled, return_index=True)
        original, new = labels[unique], new_labels[positions]
        both = (original >= 0) & (new >= 0)
        result = {'round': number, 'sampled_groups': len(chosen), 'unique_groups': len(set(chosen)),
                  'unique_targets': len(unique), 'fit_targets_including_repeated_groups': len(sampled),
                  'ari': float(adjusted_rand_score(original, new)),
                  'ari_assigned': float(adjusted_rand_score(original[both], new[both])) if both.sum() >= 2 else None,
                  'shared_assigned_fraction': float(both.mean()), 'clusters': len(set(new_labels[new_labels>=0])),
                  'noise_fraction': float((new_labels<0).mean())}
        results.append(result)
        log({'stage': 'bootstrap', **result})
    nonempty = [r['ari_assigned'] for r in results if r['ari_assigned'] is not None]
    return {'rounds': results, 'mean_ari': float(np.mean([r['ari'] for r in results])) if results else None,
            'mean_ari_assigned': float(np.mean(nonempty)) if nonempty else None,
            'ari_p05_p95': np.percentile([r['ari'] for r in results], [5, 95]).tolist() if results else None,
            'interpretation': 'group bootstrap with replacement; noise-inclusive ARI and jointly assigned ARI both reported'}


def describe_clusters(rows, indices, matrix, labels, probabilities, kind):
    descriptions = []
    for label in sorted(set(labels[labels >= 0])):
        local = np.flatnonzero(labels == label)
        center = np.median(matrix[local], axis=0)
        distances = np.linalg.norm(matrix[local]-center, axis=1)
        ranked = local[np.argsort(distances, kind='stable')]
        representatives, group_counts = [], Counter()
        for position in ranked:
            global_index = int(indices[position])
            group = rows[global_index]['group_id']
            if group_counts[group] < 2:
                representatives.append(global_index)
                group_counts[group] += 1
            if len(representatives) >= 16:
                break
        boundary = local[np.argsort(probabilities[local], kind='stable')[:8]]
        members = [rows[int(indices[i])] for i in local]
        aspect = float(np.median([r['bbox']['width']/r['bbox']['height'] for r in members]))
        # Descriptive phenotype names avoid treating guessed materials as truth.
        morphology = ('宽扁可见面' if aspect >= 6 else '中等长宽比可见面' if aspect >= 3 else '窄面或斜视候选')
        name = f'{"A" if kind=="appearance" else "G"}{int(label):02d} · {morphology}'
        dimensions = {}
        for field in GEOMETRY_FIELDS[:5]:
            values = [r[field] for r in members if field in r]
            dimensions[field] = {'median': float(np.median(values)), 'p10': float(np.percentile(values, 10)),
                                 'p90': float(np.percentile(values, 90)), 'targets': len(values)} if values else None
        if kind == 'geometry' and dimensions['visible_extent_major_mm']:
            a, b = dimensions['visible_extent_major_mm']['median'], dimensions['visible_extent_middle_mm']['median']
            name = f'G{int(label):02d} · 可见主轴约 {a:.0f}×{b:.0f} mm'
        groups = Counter(r['group_id'] for r in members)
        domains = Counter(r['domain'] for r in members)
        description = {'cluster_id': int(label), 'kind': kind, 'name': name, 'targets': len(members),
                       'groups': len(groups), 'domains': dict(domains), 'splits': dict(Counter(r['split'] for r in members)),
                       'unit_status_counts': dict(Counter(r['unit_status'] for r in members)),
                       'dominant_group_fraction': max(groups.values())/len(members),
                       'dominant_domain_fraction': max(domains.values())/len(members),
                       'bbox_aspect_median': aspect, 'dimensions': dimensions,
                       'mean_membership': float(np.mean(probabilities[local])),
                       'representative_target_ids': [rows[i]['target_id'] for i in representatives],
                       'boundary_target_ids': [rows[int(indices[i])]['target_id'] for i in boundary],
                       'structure_candidate': morphology if kind=='appearance' else 'unknown',
                       'material_candidate': 'unknown', 'viewpoint_candidate': 'unknown',
                       'evidence_level': 'unsupervised_candidate_requires_review'}
        descriptions.append(description)
    return descriptions


def assign_candidates(rows, descriptions, reviewed=None):
    reviewed = reviewed or {}
    cross = Counter((r['appearance_cluster'], r['geometry_cluster']) for r in rows)
    extremes = {field: np.percentile([r[field] for r in rows if r['geometry_status']=='ok'], [.5, 99.5])
                for field in GEOMETRY_FIELDS[:2] if any(r['geometry_status']=='ok' for r in rows)}
    for cluster in descriptions['appearance']:
        entry=reviewed.get(f"A{cluster['cluster_id']:02d}")
        if entry:
            for field in ('structure_candidate','material_candidate','viewpoint_candidate'):
                if entry.get(field):cluster[field]=entry[field]
            if entry.get('name'):cluster['name']=f"A{cluster['cluster_id']:02d} · {entry['name']}"
            cluster['evidence_level'] = 'assistant_sample_visual_candidate'
            cluster['attribute_review_basis'] = entry.get('evidence')
            cluster['visually_reviewed_target_ids'] = entry.get('sampled_target_ids', [])
            cluster['structure_definition'] = entry.get('definition', '当前可见观察面的形态候选')
    appearance = {r['cluster_id']: r for r in descriptions['appearance']}
    geometry = {r['cluster_id']: r for r in descriptions['geometry']}
    for row in rows:
        a, g = appearance.get(row['appearance_cluster']), geometry.get(row['geometry_cluster'])
        reasons = []
        if a:
            row['structure_candidate'] = f"A{a['cluster_id']:02d}:{a['structure_candidate']}"
            row['viewpoint_candidate'] = '宽面候选' if row['bbox']['width']/row['bbox']['height'] >= 6 else '窄面或斜面候选'
            if a['groups'] == 1:
                reasons.append('appearance_cluster_single_acquisition_group')
            if a['dominant_domain_fraction'] >= .9:
                reasons.append('appearance_cluster_domain_dependent')
            entry = reviewed.get(f"A{a['cluster_id']:02d}")
            if entry:
                for field in ('structure_candidate', 'material_candidate', 'viewpoint_candidate'):
                    if entry.get(field):
                        row[field] = entry[field]
                row['attribute_review_basis'] = entry.get('evidence', 'assistant_sample_visual_review')
        elif row['appearance_cluster'] == -1:
            reasons.append('appearance_noise')
        else:
            reasons.append('appearance_extraction_failed')
        row['geometry_candidate'] = f"G{g['cluster_id']:02d}" if g else 'unknown'
        row['physical_specification_candidate'] = 'unknown'
        row['attribute_status'] = 'candidate_not_confirmed'
        row['attribute_evidence_level'] = ('assistant_sample_visual_candidate' if row.get('attribute_review_basis')
                                           else 'unsupervised_phenotype_candidate')
        if row['geometry_status'] != 'ok':
            reasons.append('geometry_'+row['geometry_status'])
        elif row['geometry_cluster'] == -1:
            reasons.append('geometry_noise')
        if row['unit_status'] in ('unknown', 'conflict'):
            reasons.append('unit_'+row['unit_status'])
        if row['appearance_membership'] is not None and 0 < row['appearance_membership'] < .5:
            reasons.append('low_appearance_membership')
        if row['geometry_membership'] is not None and 0 < row['geometry_membership'] < .5:
            reasons.append('low_geometry_membership')
        if row['processing_error']:
            reasons.append('processing_error')
        if a and g and cross[(row['appearance_cluster'], row['geometry_cluster'])] < 5:
            reasons.append('rare_attribute_cross_cell')
        if row['geometry_status']=='ok' and any(row[f] < low or row[f] > high for f,(low,high) in extremes.items()):
            reasons.append('rare_visible_extent_candidate')
        row['review_reasons'] = sorted(set(reasons))
    hints = Counter()
    for row in rows:
        path = row['group_id'].lower().split('/')
        for term in ('wood', 'plastic', '3leg', '3legs', '4leg', '4legs'):
            if any(p == term or p.startswith(term+'_') for p in path):
                hints[term] += 1
    families=defaultdict(list)
    for row in rows:
        families[row['structure_candidate']].append(row)
    family_summary=[]
    for name,members in sorted(families.items(),key=lambda pair:-len(pair[1])):
        ac=sorted({r['appearance_cluster'] for r in members if r['appearance_cluster']>=0})
        gc=Counter(r['geometry_cluster'] for r in members if r['geometry_cluster']>=0)
        family_summary.append({'structure_candidate':name,'targets':len(members),
            'appearance_clusters':ac,'geometry_clusters':dict(gc),
            'groups':len({r['group_id'] for r in members}),
            'splits':dict(Counter(r['split'] for r in members)),
            'merge_policy':'candidate_semantic_family_keep_original_clusters_for_geometry_review'})
    return {'status': 'candidate_taxonomy_not_formal_labels',
            'structure_families':family_summary,
            'facets': {
                'structure': {'values': sorted(set(r['structure_candidate'] for r in rows)),
                              'definition': '可见结构候选；外观簇不保证对应唯一物理型号'},
                'material': {'values': sorted(set(r['material_candidate'] for r in rows)),
                             'definition': '仅在代表样本支持时提出候选；单通道强度不足时为unknown'},
                'geometry': {'values': [f"G{r['cluster_id']:02d}" for r in descriptions['geometry']]+['unknown'],
                             'definition': '可见点云空间范围，不能直接替代完整托盘规格'},
                'viewpoint': {'values': sorted(set(r['viewpoint_candidate'] for r in rows)),
                              'definition': '当前可见观察面候选，未恢复统一相机姿态'}},
            'cross_counts': [{'appearance_cluster': a, 'geometry_cluster': g, 'targets': n}
                             for (a,g),n in sorted(cross.items())],
            'source_directory_semantic_hints': dict(hints),
            'structure_cluster_merge_candidates': [
                {'structure_candidate':name,'appearance_clusters':[f"A{c['cluster_id']:02d}" for c in descriptions['appearance'] if c['structure_candidate']==name],
                 'targets':sum(r['structure_candidate']==name for r in rows),
                 'definition':next((c.get('structure_definition','可见形态候选') for c in descriptions['appearance'] if c['structure_candidate']==name),'可见形态候选'),
                 'interpretation':'共享可见结构候选，需结合几何、观察面与边界样本决定属性合并'}
                for name in sorted(set(r['structure_candidate'] for r in rows)) if name!='unknown'],
            'clusters_without_training_targets': [f"{'A' if kind=='appearance' else 'G'}{c['cluster_id']:02d}"
                 for kind,clusters in descriptions.items() for c in clusters if c['splits'].get('train',0)==0],
            'source_hint_policy': '解释辅助线索；未用于聚类特征，也未直接当作材质或结构真值',
            'membership_policy': '聚类隶属分数不是细分类正确概率',
            'confirmed_subclass_labels': 0}


def comparison_groups(rows, minimum_cell_size=10):
    """Show cross-view examples without claiming complete dimensions or identity."""
    cells = defaultdict(list)
    for row in rows:
        if row['appearance_cluster'] >= 0 and row['geometry_cluster'] >= 0:
            cells[(row['appearance_cluster'], row['geometry_cluster'])].append(row)

    def sample(members):
        values = np.array([[math.log(max(r[f], 2.)) for f in GEOMETRY_FIELDS[:3]] for r in members])
        distances = np.linalg.norm(values-np.median(values, axis=0), axis=1)
        chosen = min(range(len(members)), key=lambda i:(distances[i], members[i]['target_id']))
        row = members[chosen]
        return {k: row[k] for k in ('target_id', 'target_key', 'appearance_cluster', 'geometry_cluster',
                                   'structure_candidate', 'unit_status', *GEOMETRY_FIELDS[:3])}

    appearance, geometry = defaultdict(list), defaultdict(list)
    for (a, g), members in sorted(cells.items()):
        if len(members) >= minimum_cell_size:
            appearance[a].append((g, members))
        for row in members:
            if row['structure_candidate'] != 'unknown':
                geometry[(g, row['structure_candidate'])].append(row)
    different_geometry = []
    for a, groups in sorted(appearance.items()):
        if len(groups) < 2:
            continue
        ordered = sorted(groups, key=lambda p:np.median([r['visible_extent_major_mm'] for r in p[1]]))
        left, right = ordered[0], ordered[-1]
        ratio = np.median([r['visible_extent_major_mm'] for r in right[1]])/max(np.median([r['visible_extent_major_mm'] for r in left[1]]), 1.)
        if ratio >= 1.15:
            different_geometry.append({'kind': 'same_appearance_different_visible_geometry', 'appearance_cluster': a,
                'cell_targets': [len(left[1]), len(right[1])], 'median_visible_major_ratio': float(ratio),
                'samples': [sample(left[1]), sample(right[1])],
                'interpretation': '同一外观簇的不同可见几何分组；差异也可能来自单位推断、遮挡或观察面'})
    different_geometry.sort(key=lambda p:(-p['median_visible_major_ratio'], p['appearance_cluster']))
    by_geometry = defaultdict(list)
    for (g, structure), members in sorted(geometry.items()):
        if len(members) >= minimum_cell_size:
            by_geometry[g].append((structure, members))
    different_structure = []
    for g, groups in sorted(by_geometry.items()):
        groups.sort(key=lambda p:(-len(p[1]), p[0]))
        if len(groups) >= 2:
            left, right = groups[:2]
            different_structure.append({'kind': 'same_visible_geometry_different_structure_candidate',
                'geometry_cluster': g, 'cell_targets': [len(left[1]), len(right[1])],
                'samples': [sample(left[1]), sample(right[1])],
                'interpretation': '同一可见几何簇中的不同结构候选；样本尚未逐目标确认'})
    different_structure.sort(key=lambda p:(-min(p['cell_targets']), p['geometry_cluster']))
    return different_geometry[:6]+different_structure[:6]


def write_csv(path, rows):
    excluded = {'_cache_path', 'source_pcd_sha256'}
    fields = sorted(set().union(*(r.keys() for r in rows))-excluded)
    with Path(path).open('w', encoding='utf-8-sig', newline='') as stream:
        writer = csv.DictWriter(stream, fieldnames=fields)
        writer.writeheader()
        for row in rows:
            writer.writerow({k:json.dumps(v, ensure_ascii=False) if isinstance(v, (dict, list)) else v
                             for k,v in row.items() if k in fields})


def plots(result, rows, descriptions):
    import matplotlib
    matplotlib.use('Agg')
    import matplotlib.pyplot as plt
    figures = result/'figures'
    figures.mkdir(exist_ok=True)
    with np.load(result/'features.npz') as features:
        for kind in ('appearance', 'geometry'):
            indices, matrix = features[kind+'_indices'], features[kind+'_features']
            if len(matrix) < 2:
                continue
            from sklearn.decomposition import PCA
            xy = (PCA(n_components=2, random_state=42).fit_transform(matrix)
                  if np.max(np.ptp(matrix,axis=0)) > 1e-12 else np.zeros((len(matrix),2)))
            labels = np.array([rows[int(i)][kind+'_cluster'] for i in indices])
            fig, axis = plt.subplots(figsize=(10, 7))
            axis.scatter(xy[labels<0, 0], xy[labels<0, 1], s=4, c='#b7bcc4', alpha=.35, label='Noise')
            for label in sorted(set(labels[labels>=0])):
                sel = labels==label
                axis.scatter(xy[sel, 0], xy[sel, 1], s=5, alpha=.6, label=f'{"A" if kind=="appearance" else "G"}{label:02d}')
            if len(set(labels)) <= 30:
                axis.legend(markerscale=3, fontsize=7, ncol=2, loc='best')
            axis.set(xlabel='PC 1 (display only)', ylabel='PC 2 (display only)', title=f'{kind.title()} clusters / PCA display')
            fig.tight_layout(); fig.savefig(figures/f'{kind}_pca.png', dpi=150); plt.close(fig)
    a_ids = [d['cluster_id'] for d in descriptions['appearance']]
    g_ids = [d['cluster_id'] for d in descriptions['geometry']]
    if a_ids and g_ids:
        counts = Counter((r['appearance_cluster'],r['geometry_cluster']) for r in rows)
        cross = np.array([[counts[(a,g)] for g in g_ids] for a in a_ids])
        fig, axis = plt.subplots(figsize=(max(8,min(16,len(g_ids)*.45)), max(5,min(18,len(a_ids)*.3))))
        im = axis.imshow(np.log1p(cross), aspect='auto', cmap='Blues')
        axis.set_xticks(range(len(g_ids)), [f'G{g:02d}' for g in g_ids], rotation=90)
        axis.set_yticks(range(len(a_ids)), [f'A{a:02d}' for a in a_ids])
        axis.set(xlabel='Visible geometry cluster', ylabel='Appearance cluster', title='Cross-distribution / log(1 + targets)')
        fig.colorbar(im, ax=axis); fig.tight_layout(); fig.savefig(figures/'cross_distribution.png', dpi=150); plt.close(fig)
    if g_ids:
        fig, axes = plt.subplots(1, 2, figsize=(13, 5))
        for axis, field, title in zip(axes, GEOMETRY_FIELDS[:2], ['Visible principal extent (mm)', 'Visible second extent (mm)']):
            values = [[r[field] for r in rows if r['geometry_cluster']==g and field in r] for g in g_ids]
            axis.boxplot(values, showfliers=False, tick_labels=[f'G{g:02d}' for g in g_ids])
            axis.set_title(title); axis.tick_params(axis='x', rotation=90)
        fig.tight_layout(); fig.savefig(figures/'visible_dimensions.png', dpi=150); plt.close(fig)
    for name in ('appearance_pca', 'geometry_pca', 'cross_distribution', 'visible_dimensions'):
        path = figures/f'{name}.png'
        if not path.exists():
            fig, axis = plt.subplots(figsize=(8,4))
            axis.text(.5,.5,'Insufficient valid targets / clusters for this plot',ha='center',va='center')
            axis.set_axis_off(); fig.tight_layout(); fig.savefig(path,dpi=120); plt.close(fig)


def representative_panels(result, output, rows, descriptions, comparisons=()):
    """Numbered panels are suitable for a subsequent, explicitly candidate-only review."""
    by_id = {r['target_id']:r for r in rows}
    panels = result/'panels'
    clouds = result/'clouds'
    panels.mkdir(exist_ok=True); clouds.mkdir(exist_ok=True)
    cloud_ids = {sample['target_id'] for pair in comparisons for sample in pair['samples']}
    for kind, clusters in descriptions.items():
        for cluster in clusters:
            prefix = f'{"A" if kind=="appearance" else "G"}{cluster["cluster_id"]:02d}'
            representative_ids = cluster['representative_target_ids']
            ids = representative_ids+cluster['boundary_target_ids']
            canvas = np.full((math.ceil(len(ids)/3)*160, 1080, 3), 245, np.uint8)
            for index, identity in enumerate(ids):
                row = by_id[identity]; key = row['target_key']
                crop = cv2.imread(str(output/'media'/f'{key}.jpg'))
                if crop is None:
                    continue
                h,w = crop.shape[:2]; scale = min(345/w, 105/h)
                resized = cv2.resize(crop, (max(1,int(w*scale)), max(1,int(h*scale))))
                oy,ox = (index//3)*160+30, (index%3)*360+7
                canvas[oy:oy+resized.shape[0], ox:ox+resized.shape[1]] = resized
                label = f'{prefix} #{index:02d} {"REP" if index<len(representative_ids) else "BOUNDARY"} {row["target_key"][:8]}'
                cv2.putText(canvas,label,(ox,oy-10),cv2.FONT_HERSHEY_SIMPLEX,.42,(25,25,25),1,cv2.LINE_AA)
                geometry = row.get('visible_extent_major_mm')
                caption = f'{row["domain"]} | {row["split"]}'
                cv2.putText(canvas,caption,(ox,(index//3)*160+145),cv2.FONT_HERSHEY_SIMPLEX,.34,(50,50,50),1,cv2.LINE_AA)
                cloud_ids.add(identity)
            cv2.imwrite(str(panels/f'{prefix}.jpg'), canvas, [cv2.IMWRITE_JPEG_QUALITY, 92])
            cluster['panel'] = f'panels/{prefix}.jpg'
    by_cache = defaultdict(list)
    for identity in cloud_ids:
        row = by_id[identity]
        by_cache[row['_cache_path']].append(row)
    for path, targets in by_cache.items():
        with np.load(path, allow_pickle=False) as archive:
            for row in targets:
                points = archive[f'points_{row["box_index"]}'].copy()
                if len(points) > 1000:
                    points = points[np.linspace(0,len(points)-1,1000,dtype=int)]
                scale = row['scale_to_mm']
                if scale is not None:
                    points *= scale
                save_json(clouds/f'{row["target_key"]}.json', {'target_id':row['target_id'],
                    'units':'mm' if scale is not None else 'raw_unknown', 'unit_status':row['unit_status'],
                    'points':np.round(points,3).tolist(), 'measurement_scope':'visible_foreground_only'})


def generate_reports(result, output, dataset, rows, descriptions, summary, sensitivity, proposals):
    plots(result, rows, descriptions)
    representative_panels(result, output, rows, descriptions, proposals['comparison_examples'])
    save_json(result/'clusters.json', descriptions)
    gallery_html(result, output, dataset, rows, descriptions, summary, proposals['comparison_examples'])
    report_markdown(result, rows, descriptions, summary, sensitivity, proposals)
    from .cluster_report_html import build_html_report
    build_html_report(result/'report_zh.md')


def gallery_html(result, output, dataset, rows, descriptions, summary, comparisons=()):
    import os
    import shutil
    font_cache=Path.home()/'.cache/pallet_analysis'
    font_style=''
    if (font_cache/'PalletReportSans.woff').exists() and (font_cache/'FONT_LICENSE.txt').exists():
        for name in ('PalletReportSans.woff','FONT_LICENSE.txt'):
            shutil.copyfile(font_cache/name,result/name)
        font_style="@font-face{font-family:PalletReportSans;src:url('PalletReportSans.woff')}body{font-family:PalletReportSans,system-ui,sans-serif}"
    dataset_relative = os.path.relpath(dataset, result)
    compact = []
    for row in rows:
        compact.append({k:row.get(k) for k in ['target_id', 'target_key', 'frame_id', 'box_index', 'group_id',
            'domain', 'split', 'image', 'bbox', 'appearance_cluster', 'geometry_cluster', 'appearance_membership',
            'geometry_membership', 'structure_candidate', 'material_candidate', 'viewpoint_candidate', 'unit_status',
            'geometry_status', 'review_reasons', 'mask_score', 'visible_extent_major_mm', 'visible_extent_middle_mm',
            'visible_extent_minor_mm', 'visible_horizontal_span_mm', 'visible_vertical_span_mm', 'scale_to_mm']})
        compact[-1]['cloud'] = (result/'clouds'/f'{row["target_key"]}.json').exists()
    cards = []
    for kind in ('appearance', 'geometry'):
        for c in descriptions[kind]:
            prefix = f'{"A" if kind=="appearance" else "G"}{c["cluster_id"]:02d}'
            representative = next((r for r in rows if r['target_id']==c['representative_target_ids'][0]), None)
            photo = f'../../media/{representative["target_key"]}.jpg' if representative else ''
            cards.append(f'<button class="cluster" onclick="choose(\'{kind}\',{c["cluster_id"]})">'
                         f'<img loading="lazy" src="{photo}"><b>{html.escape(c["name"])}</b>'
                         f'<span>{c["targets"]:,} 个目标 · {c["groups"]} 个采集组</span>'
                         f'<span>最大配置域占比 {c["dominant_domain_fraction"]:.0%}</span></button>')
    comparison_cards = []
    for pair in comparisons:
        same_appearance = pair['kind']=='same_appearance_different_visible_geometry'
        title = (f"A{pair['appearance_cluster']:02d}：外观相近，可见几何不同" if same_appearance
                 else f"G{pair['geometry_cluster']:02d}：可见几何相近，结构候选不同")
        samples = []
        for sample in pair['samples']:
            key = sample['target_key']
            samples.append(f'<button class="cluster" onclick="openTarget(rows.find(r=>r.target_key===\'{key}\'))">'
                f'<img loading="lazy" src="../../media/{key}.jpg">'
                f'<b>A{sample["appearance_cluster"]:02d} / G{sample["geometry_cluster"]:02d} · {html.escape(sample["structure_candidate"])}</b>'
                f'<span>可见主轴 {sample["visible_extent_major_mm"]:.0f} × {sample["visible_extent_middle_mm"]:.0f} mm · {sample["unit_status"]}</span></button>')
        comparison_cards.append(f'<article><h3>{title}</h3><div class="clusters">'+''.join(samples)+
            f'</div><p class="hint">{html.escape(pair["interpretation"])}；两侧交叉单元分别含 {pair["cell_targets"][0]} / {pair["cell_targets"][1]} 个目标。</p></article>')
    data = json.dumps(compact, ensure_ascii=False, allow_nan=False).replace('</', '<\\/')
    overview = json.dumps(summary['validation'], ensure_ascii=False).replace('</', '<\\/')
    cloud_data = {p.stem: json.loads(p.read_text()) for p in sorted((result/'clouds').glob('*.json'))}
    (result/'cloud_data.js').write_text('window.PALLET_CLOUDS = '+json.dumps(cloud_data, ensure_ascii=False)+';\n')
    template = '''<!doctype html><html lang="zh-CN"><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<title>托盘目标多属性聚类图册</title><style>
*{box-sizing:border-box}body{margin:0;background:#f3f6fa;color:#172333;font:15px/1.65 system-ui,sans-serif}header{padding:30px max(24px,5vw);background:#17344b;color:white}h1{margin:0;font-size:28px}header p{max-width:1100px;color:#d9e5ef}.wrap{margin:auto;max-width:1450px;padding:24px}.kpis{display:flex;flex-wrap:wrap;gap:14px}.kpi{background:white;border-radius:10px;padding:16px;flex:1;min-width:160px}.kpi b{display:block;font-size:27px}h2{font-size:21px;margin:28px 0 12px}.clusters{display:grid;grid-template-columns:repeat(auto-fill,minmax(260px,1fr));gap:12px}.cluster{background:white;border:1px solid #dae2e9;border-radius:9px;padding:12px;text-align:left;cursor:pointer}.cluster img{width:100%;height:72px;object-fit:contain;background:#172333}.cluster b,.cluster span{display:block}.cluster:hover,.target:hover{border-color:#3786af}.filters{display:flex;flex-wrap:wrap;gap:9px;background:white;padding:16px;border-radius:9px;position:sticky;top:0;z-index:2}select,input,button{font:inherit}select,input{padding:7px;border:1px solid #c9d4df;border-radius:5px;max-width:240px}button{cursor:pointer}.targets{display:grid;grid-template-columns:repeat(auto-fill,minmax(300px,1fr));gap:12px;margin-top:14px}.target{padding:12px;background:white;border:1px solid #d8e2eb;border-radius:9px;cursor:pointer}.target img{width:100%;height:90px;object-fit:contain;background:#172333}.target small{display:block;color:#556777;word-break:break-all}.tag{display:inline-block;background:#e4eff7;padding:2px 7px;border-radius:4px;margin:2px;font-size:12px}.warn{background:#fff0d4}.plots{display:grid;grid-template-columns:repeat(auto-fit,minmax(420px,1fr));gap:12px}.plots img{width:100%;background:white;border-radius:8px}.pager{display:flex;align-items:center;gap:14px;margin:16px 0}dialog{width:min(1150px,96vw);max-height:94vh;border:0;border-radius:12px;padding:22px;box-shadow:0 10px 100px #0005}dialog::backdrop{background:#14273788}dialog .close{float:right}#original{max-width:100%;max-height:400px;object-fit:contain;background:#172333}#cloud{background:#132838;width:100%;height:330px;border-radius:8px;touch-action:none}#metadata{white-space:pre-wrap;word-break:break-all;font-size:13px;font-family:inherit;background:#f3f6fa;padding:12px}.modal-grid{display:grid;grid-template-columns:1fr 1fr;gap:18px}@media(max-width:750px){.modal-grid{grid-template-columns:1fr}.plots{grid-template-columns:1fr}}.hint{color:#627384;font-size:13px}a{color:#246d99}.search{flex:1;min-width:170px}
</style><header><h1>托盘目标多属性聚类图册</h1><p>外观结构与可见几何共同支持细分类候选。A 表示外观表型簇，G 表示可见几何簇；所有属性均待确认。点击目标可定位原标注框，代表与边界样本提供可旋转点云。单位推断、缺测和离群目标均保留。</p></header><main class="wrap">
<div class="kpis" id="kpis"></div><p><a href="report_zh.html">中文报告</a> · <a href="targets.csv">逐目标 CSV</a> · <a href="attribute_taxonomy.json">候选属性体系</a> · <a href="review_queue.jsonl">复核清单</a> · <a href="validation.json">完整性核验</a></p>
<h2>外观与几何交叉样本</h2><p class="hint">点击对比目标查看原图、分割及点云。样本反映可见部分，仍需检查单位推断和观察面。</p>__COMPARISONS__
<h2>簇概览</h2><div class="clusters">__CARDS__</div><h2>分布与可见尺寸</h2><p class="hint">二维图仅用于展示；聚类在原特征空间完成。几何量为可见部分，第三轴范围不是托盘厚度。</p><div class="plots"><img src="figures/appearance_pca.png"><img src="figures/geometry_pca.png"><img src="figures/cross_distribution.png"><img src="figures/visible_dimensions.png"></div>
<h2>逐目标浏览</h2><div class="filters"><select id="appearance"><option value="">全部外观簇</option></select><select id="geometry"><option value="">全部几何簇</option></select><select id="domain"><option value="">全部配置域</option></select><select id="split"><option value="">全部划分</option></select><select id="unit"><option value="">全部单位状态</option></select><select id="material"><option value="">全部材质候选</option></select><select id="structure"><option value="">全部结构候选</option></select><select id="viewpoint"><option value="">全部观察面候选</option></select><label><input type="checkbox" id="review">优先复核</label><input class="search" id="search" placeholder="帧、采集组或目标ID"></div>
<div class="pager"><button id="prev">上一页</button><span id="count"></span><button id="next">下一页</button></div><div class="targets" id="targets"></div></main>
<dialog id="detail"><button class="close" onclick="document.getElementById('detail').close()">关闭</button><h2 id="detail-title"></h2><div class="modal-grid"><div><canvas id="original"></canvas><p><a id="source" target="_blank">查看原始强度图</a></p><img id="overlay" style="width:100%;max-height:130px;object-fit:contain;background:#172333"></div><div><canvas id="cloud" width="650" height="330"></canvas><p class="hint" id="cloud-note">拖动旋转点云</p><pre id="metadata"></pre></div></div></dialog>
<script src="cloud_data.js"></script><script>
const rows=__DATA__, validation=__VALIDATION__, datasetRoot=__DATASET_ROOT__;
const $=id=>document.getElementById(id);const esc=v=>String(v??'').replace(/[&<>"']/g,c=>({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[c]));
let page=0, filtered=rows;const pageSize=48;
const attributes={appearance:'appearance_cluster',geometry:'geometry_cluster',domain:'domain',split:'split',unit:'unit_status',material:'material_candidate',structure:'structure_candidate',viewpoint:'viewpoint_candidate'};
for(const [id,key] of Object.entries(attributes)){const values=[...new Set(rows.map(r=>r[key]))].sort((a,b)=>typeof a==='number'?a-b:String(a).localeCompare(String(b)));for(const value of values){const op=document.createElement('option');op.value=String(value);op.textContent=(id==='appearance'?'A':id==='geometry'?'G':'')+(value===-1?'离群':value===-2?'缺测':value);$(id).appendChild(op)}$(id).addEventListener('change',()=>{page=0;render()})}
$('review').addEventListener('change',()=>{page=0;render()});$('search').addEventListener('input',()=>{page=0;render()});$('prev').onclick=()=>{page=Math.max(0,page-1);render()};$('next').onclick=()=>{page=Math.min(Math.ceil(filtered.length/pageSize)-1,page+1);render()};
function choose(kind,id){$(kind).value=String(id);page=0;render();$('targets').scrollIntoView({behavior:'smooth',block:'start'})}
function render(){const term=$('search').value.toLowerCase();filtered=rows.filter(r=>Object.entries(attributes).every(([id,key])=>$(id).value===''||String(r[key])===$(id).value)&&(!$('review').checked||r.review_reasons.length)&&(!term||(r.target_id+' '+r.group_id).toLowerCase().includes(term)));const pages=Math.max(1,Math.ceil(filtered.length/pageSize));page=Math.max(0,Math.min(page,pages-1));$('count').textContent=`${filtered.length.toLocaleString()} 个目标 · 第 ${page+1}/${pages} 页`;$('prev').disabled=page===0;$('next').disabled=page===pages-1;$('targets').innerHTML=filtered.slice(page*pageSize,(page+1)*pageSize).map(r=>`<div class="target" data-key="${r.target_key}"><img loading="lazy" src="../../media/${r.target_key}.jpg"><b>${esc(r.structure_candidate)}</b><div><span class="tag">A${r.appearance_cluster}</span><span class="tag">G${r.geometry_cluster}</span><span class="tag ${r.unit_status==='inferred'?'warn':''}">${esc(r.unit_status)}</span><span class="tag">${esc(r.material_candidate)}</span></div><small>${esc(r.frame_id)} · 框 ${r.box_index}</small><small>${esc(r.domain)} · ${esc(r.geometry_status)}</small></div>`).join('');for(const card of $('targets').children)card.onclick=()=>openTarget(rows.find(r=>r.target_key===card.dataset.key))}
const kpis=[['目标总数',validation.actual_targets],['外观有效',validation.appearance_status.ok||0],['几何可聚类',validation.geometry_status.ok||0],['单位推断',validation.unit_status.inferred||0],['已核验单位',validation.unit_status.verified||0]];$('kpis').innerHTML=kpis.map(([title,value])=>`<div class="kpi"><b>${value.toLocaleString()}</b>${title}</div>`).join('');
let points=[],yaw=.35,pitch=.35,currentKey='';const cloudCanvas=$('cloud'),ctx=cloudCanvas.getContext('2d');
function drawCloud(){ctx.fillStyle='#132838';ctx.fillRect(0,0,650,330);if(!points.length){ctx.fillStyle='#c6d4df';ctx.font='15px sans-serif';ctx.fillText('代表与边界样本提供点云视图',155,165);return}const cy=Math.cos(yaw),sy=Math.sin(yaw),cp=Math.cos(pitch),sp=Math.sin(pitch);const projected=points.map(p=>{const x=p[0]*cy+p[2]*sy,z=-p[0]*sy+p[2]*cy,y=p[1]*cp-z*sp,depth=p[1]*sp+z*cp;return[x,y,depth]}).sort((a,b)=>a[2]-b[2]);for(const p of projected){ctx.fillStyle=`hsl(${180+p[2]*55},75%,65%)`;ctx.fillRect(325+p[0]*240,165-p[1]*240,2.5,2.5)}}
let dragging=false,previous;cloudCanvas.onpointerdown=e=>{dragging=true;previous=[e.clientX,e.clientY];cloudCanvas.setPointerCapture(e.pointerId)};cloudCanvas.onpointerup=()=>dragging=false;cloudCanvas.onpointermove=e=>{if(dragging){yaw+=(e.clientX-previous[0])*.01;pitch+=(e.clientY-previous[1])*.01;previous=[e.clientX,e.clientY];drawCloud()}};
async function openTarget(r){currentKey=r.target_key;$('detail-title').textContent=`目标 ${r.target_key} · A${r.appearance_cluster} / G${r.geometry_cluster}`;$('detail').showModal();const url=datasetRoot+'/'+r.image;$('source').href=url;$('overlay').src='../../media/'+r.target_key+'.overlay.jpg';const im=new Image();im.onload=()=>{if(currentKey!==r.target_key)return;const c=$('original');c.width=im.width;c.height=im.height;const context=c.getContext('2d');context.drawImage(im,0,0);context.strokeStyle='#fb5545';context.lineWidth=Math.max(2,im.width/250);const b=r.bbox;context.strokeRect(b.x,b.y,b.width,b.height)};im.src=url;$('metadata').textContent=JSON.stringify({target_id:r.target_id,结构候选:r.structure_candidate,材质候选:r.material_candidate,观察面候选:r.viewpoint_candidate,单位状态:r.unit_status,几何状态:r.geometry_status,可见主轴_mm:r.visible_extent_major_mm,可见第二轴_mm:r.visible_extent_middle_mm,第三轴范围_mm:r.visible_extent_minor_mm,外观隶属分数:r.appearance_membership,复核原因:r.review_reasons},null,2);points=[];drawCloud();$('cloud-note').textContent=r.cloud?'加载点云…':'该目标的原始点保存在帧缓存；本图册为代表与边界样本提供点云视图。';if(r.cloud){try{const data=window.PALLET_CLOUDS[r.target_key];if(!data)throw new Error('Point cloud data missing');if(currentKey!==r.target_key)return;if(data.points.length){const center=[0,1,2].map(k=>data.points.reduce((s,p)=>s+p[k],0)/data.points.length);const centered=data.points.map(p=>p.map((v,k)=>v-center[k]));const size=Math.max(...centered.flat().map(Math.abs),1e-8);points=centered.map(p=>p.map(v=>v/size));drawCloud()}$('cloud-note').textContent=`拖动旋转 · ${data.points.length} 个可见前景点 · ${data.units} / ${data.unit_status}`}catch(e){$('cloud-note').textContent='点云视图无法加载：'+e.message}}}
render();drawCloud();
const initialTarget=new URLSearchParams(location.search).get('target');if(initialTarget){const target=rows.find(r=>r.target_key===initialTarget);if(target)openTarget(target)}
</script></html>'''
    template = template.replace('__CARDS__',''.join(cards)).replace('__DATA__',data).replace('__VALIDATION__',overview)
    template = template.replace('__COMPARISONS__',''.join(comparison_cards) or '<p>当前没有足够交叉样本。</p>')
    template = template.replace('__DATASET_ROOT__',json.dumps(dataset_relative))
    template = template.replace('</style>',font_style+'</style>')
    (result/'gallery.html').write_text(template)


def report_markdown(result, rows, descriptions, summary, sensitivity, proposals):
    validation, clustering = summary['validation'], summary['clustering']
    a, g = clustering['appearance'], clustering['geometry']
    lines = ['# 托盘目标多属性聚类与细分类候选报告', '',
        f"完成时间：{summary['finished_at']}。输入为正式人工数据集，分析全部 **{len(rows):,} 个托盘框**；"
        f"{validation['negative_frames_excluded']} 张负例未参与目标聚类。", '',
        f"默认设置得到 **{a['clusters']} 个外观结构簇、{g['clusters']} 个可见几何簇**。"
        f"外观离群 {a['noise']:,} 个；几何可聚类 {g['targets']:,} 个，其中离群 {g['noise']:,} 个。"
        '簇是候选分组，不能直接当作已确认托盘型号。', '',
        '[打开可筛选图册](gallery.html) · [逐目标CSV](targets.csv) · [逐目标JSONL](targets.jsonl) · '
        '[候选属性体系](attribute_taxonomy.json) · [优先复核清单](review_queue.jsonl)', '',
        '## 数据覆盖与几何证据', '',
        '| 指标 | 数量 |', '|---|---:|',
        f"| 覆盖正例图像 | {validation['actual_positive_frames']:,} |",
        f"| 唯一目标 | {validation['unique_targets']:,} |",
        f"| 点云—正式强度逐帧对应通过的目标 | {validation['alignment_verified_targets']:,} |",
        f"| 提取处理错误 | {validation['processing_errors']:,} |",
        f"| 几何可聚类 | {validation['geometry_status'].get('ok',0):,} |",
        f"| 仅能测量部分可见面 | {validation['geometry_status'].get('partial_visible_surface',0):,} |",
        f"| 背景回波或斜视使几何证据存疑 | {validation['geometry_status'].get('foreground_depth_or_view_ambiguous',0):,} |",
        f"| 正式标签及元数据核验文件 | {validation['protected_files_checked']:,} |", '',
        '单位采用冻结证据优先，其余按用户指定分布规则逐批推断：整数比例至少95%且有效ROI量级不小于10时提示mm；'
        '具有小数且ROI距离90分位不超过10时提示m。至少3帧或批次全部帧参与投票，批次一致率至少90%；'
        '单目标与推断批次矛盾时为未知。统计只使用有效前景点，原传感器政策未改写。', '',
        '| 单位状态 | 目标数 | 含义 |', '|---|---:|---|']
    unit_text = {'verified':'已有冻结单位证据', 'inferred':'按数值分布推断，属于候选测量', 'unknown':'无足够一致证据', 'conflict':'与批次推断矛盾'}
    for status,n in sorted(validation['unit_status'].items()):
        lines.append(f'| {status} | {n:,} | {unit_text.get(status,status)} |')
    lines += ['', '[逐批单位证据](unit_registry.json) · [全部质量状态](validation.json)', '',
        '三维数据经框提示分割、有限值/零点过滤和离群过滤后测量。报告给出可见点云主轴范围、可见水平/垂直跨度和形状描述；'
        '**第三轴范围不等于托盘厚度，单视角可见范围不等于完整长宽高。** 遮挡、裁切和有效点不足的样本保留缺测状态。', '',
        '原图复核发现开口内背景可能混入候选掩码。为保守处理，二维框长宽比至少4、三维主轴比例小于3，'
        '且距离2—98分位范围超过可见第一轴35%的目标标为`foreground_depth_or_view_ambiguous`。'
        '这也可能由明显斜视造成，因此仅标为证据存疑：保留诊断测量和点云，将几何候选设为unknown并排除规格聚类。'
        '该筛查规则不是已验证的背景分割准确率。[规则记录](geometry_quality_policy.json)', '',
        '## 外观结构与几何分组', '',
        '外观采用本地DINOv3-L/16的全局与目标区域池化特征，保持裁剪长宽比，PCA最多32维；'
        '辅以低权重的框长宽比和前景占比。几何采用对数可见尺寸及平面度/散乱度，稳健标准化。'
        '采集批次、目录语义、配置域及原划分均未作为聚类特征。', '',
        f"两路均采用HDBSCAN，最小簇大小50、min_samples=10、EOM选择，随机种子{summary['seed']}。"
        '2D图仅用于展示；聚类隶属分数不是分类正确率。', '',
        '![外观分布](figures/appearance_pca.png)', '', '![几何分布](figures/geometry_pca.png)', '',
        '| 外观簇 | 目标 | train/val/test | 采集组 | 长宽比中位数 | 最大配置域占比 | 属性证据 | 样本 |',
        '|---|---:|---|---:|---:|---:|---|---|']
    for c in descriptions['appearance']:
        counts='/'.join(str(c['splits'].get(s,0)) for s in ('train','val','test'))
        evidence = f"助手抽样{len(c.get('visually_reviewed_target_ids',[]))}个，待确认" if c.get('visually_reviewed_target_ids') else '无监督表型，待确认'
        lines.append(f"| {c['name']} | {c['targets']:,} | {counts} | {c['groups']} | {c['bbox_aspect_median']:.2f} | "
                     f"{c['dominant_domain_fraction']:.1%} | {evidence} | [图板]({c['panel']}) |")
    lines += ['', '| 几何簇 | 目标 | train/val/test | 采集组 | 可见第一/第二轴中位数(mm) | 单位推断占比 | 单位证据：已核验/推断 | 样本 |',
              '|---|---:|---|---:|---:|---:|---|---|']
    for c in descriptions['geometry']:
        dims = c['dimensions']; members = [r for r in rows if r['geometry_cluster']==c['cluster_id']]
        fraction = sum(r['unit_status']=='inferred' for r in members)/max(len(members),1)
        counts='/'.join(str(c['splits'].get(s,0)) for s in ('train','val','test'))
        lines.append(f"| G{c['cluster_id']:02d} | {c['targets']:,} | {counts} | {c['groups']} | "
                     f"{dims['visible_extent_major_mm']['median']:.0f} / {dims['visible_extent_middle_mm']['median']:.0f} | "
                     f"{fraction:.1%} | {c['unit_status_counts'].get('verified',0)} / {c['unit_status_counts'].get('inferred',0)} | [图板]({c['panel']}) |")
    lines += ['', '![可见尺寸分布](figures/visible_dimensions.png)', '', '## 多属性候选与细分类建议', '',
        '| 属性 | 本轮提供 | 后续确认要点 |', '|---|---|---|',
        '| 结构 | 外观表型簇及代表图；抽样观察支持时提出结构候选 | 支撑结构、插孔、面板形式，同型不同观察面之间的联系 |',
        '| 材质 | 无法可靠辨认时保留unknown；目录wood/plastic只作为辅助线索 | 强度纹理可能随传感器、距离、曝光改变，需明确样本证据 |',
        '| 几何规格 | 可见几何簇G及测量区间 | 单位推断、遮挡和观察面；完整规格目前均为unknown |',
        '| 观察面 | 宽面或窄面/斜面候选；不宣称已恢复统一姿态 | 同一物理托盘的多观察面及负载/堆叠上下文 |', '',
        '建议保留独立属性，先确认外观结构和可见几何的交叉单元，再决定需要合并哪些观察面。'
        '仅由一个采集组支撑、或90%以上来自一个配置域的簇应优先复核，避免把采集条件当成托盘类型。', '',
        '![外观—几何交叉分布](figures/cross_distribution.png)', '',
        '| 外观簇 | 几何簇 | 目标数 | 采集组数 |', '|---|---|---:|---:|']
    cross = defaultdict(list)
    for r in rows:
        if r['appearance_cluster'] >= 0 and r['geometry_cluster'] >= 0:
            cross[(r['appearance_cluster'],r['geometry_cluster'])].append(r)
    for (aa,gg), members in sorted(cross.items(), key=lambda pair:-len(pair[1]))[:40]:
        lines.append(f'| A{aa:02d} | G{gg:02d} | {len(members):,} | {len({r["group_id"] for r in members})} |')
    if summary.get('assistant_visually_reviewed_targets'):
        lines += ['', f"助手共查看{summary['assistant_visually_reviewed_targets']}个不同代表/边界目标（每个外观簇最多取3个代表及1个边界样本，重叠目标去重）。"
                  '据此提出以下可见结构家族候选，并保留局部可见或暗部信息不足目标为unknown。'
                  '抽样观察不代表逐目标确认或整簇纯度，全部材质仍为unknown。[观察依据](candidate_attribute_review.json)', '',
                  '| 可见结构家族候选 | 定义 | 目标数 | train/val/test | 对应外观簇 |', '|---|---|---:|---|---|']
        for family in proposals['structure_cluster_merge_candidates']:
            splits=next(f['splits'] for f in proposals['structure_families'] if f['structure_candidate']==family['structure_candidate'])
            counts='/'.join(str(splits.get(s,0)) for s in ('train','val','test'))
            lines.append(f"| {family['structure_candidate']} | {family['definition']} | {family['targets']:,} | {counts} | {', '.join(family['appearance_clusters'])} |")
        lines += ['', '双开口、中支撑等描述对应当前可见观察面，不能据此断言本体腿数。'
                  '窄条边沿是可见形态；层状横梁候选需检查原图，区分本体纹理与层间边界。']
    lines += ['', '完整交叉计数见[属性体系JSON](attribute_taxonomy.json)。同一外观簇跨多个几何簇可以支持规格细分候选；'
              '同一几何簇跨多个外观簇提示不同结构、观察面或成像差异，不能仅因尺寸相近而合并。', '',
              '| 交叉比较 | 左侧样本（可见第一/第二轴mm） | 右侧样本（可见第一/第二轴mm） |', '|---|---|---|']
    for pair in proposals['comparison_examples']:
        title = (f"A{pair['appearance_cluster']:02d}：外观相近，几何不同" if pair['kind']=='same_appearance_different_visible_geometry'
                 else f"G{pair['geometry_cluster']:02d}：几何相近，结构候选不同")
        cells = [f"[A{s['appearance_cluster']:02d}/G{s['geometry_cluster']:02d}](gallery.html?target={s['target_key']}) "
                 f"{s['visible_extent_major_mm']:.0f}/{s['visible_extent_middle_mm']:.0f}" for s in pair['samples']]
        lines.append(f"| {title} | {' | '.join(cells)} |")
    missing_train = ', '.join(proposals['clusters_without_training_targets']) or '无'
    lines += ['', f'原train划分中未覆盖的簇：**{missing_train}**。逐簇train/val/test计数见上表。'
              '在冻结划分之外确认属性并建立独立版本前，不宜直接训练细分类器或调整现有划分。', '',
              '## 稳定性、复核与使用边界', '',
              '| 特征路 | 最小簇25：簇/离群 | 最小簇50：簇/离群 | 最小簇100：簇/离群 | 组重采样ARI均值 |',
              '|---|---|---|---|---:|']
    for kind in ('appearance','geometry'):
        stability = summary['group_stability'][kind]['mean_ari']
        sizes = [f"{sensitivity[kind][str(size)]['clusters']} / {sensitivity[kind][str(size)]['noise']:,}" for size in (25,50,100)]
        lines.append(f"| {'外观' if kind=='appearance' else '几何'} | {' | '.join(sizes)} | {stability:.3f} |" if stability is not None
                     else f"| {kind} | {' | '.join(sizes)} | 未运行 |")
    lines += ['', f"每路进行{summary['bootstrap_rounds_per_feature_view']}次采集组有放回重采样。ARI包含离群样本；"
        '另报告两次均有归属目标的ARI及覆盖率。重采样改变密度，稳定性仅表示候选分组对采集组组成的敏感程度，'
        '不能解释为分类精度。[重采样明细](group_bootstrap.json) · [粒度敏感性](sensitivity.json)', '',
        f"优先复核清单包含 **{summary['review_targets']:,} 个目标**。全部候选属性尚未正式确认；复核清单突出离群、缺测、低隶属分数和配置域依赖。"
        '另提供[罕见候选清单](rare_candidates.jsonl)：交叉单元少于5个目标，或可见前两轴落在有效目标的0.5%/99.5%分位之外。'
        '罕见性可能来自观察面、遮挡或误分割，需要原图复核。', '',
        '| 复核原因 | 目标数（可重叠） |', '|---|---:|']
    for reason,n in sorted(summary['review_reasons'].items(), key=lambda p:-p[1]):
        lines.append(f'| {reason} | {n:,} |')
    lines += ['', f"正式标签、划分与冻结元数据的{validation['protected_files_checked']:,}个文件前后哈希"
              f"{'全部一致' if validation['protected_files_unchanged'] else '存在变化，须调查'}；目标覆盖无遗漏或重复。"
              '本轮不训练细分类模型，不提供细分类测试精度，不自动写回正式标签。', '',
              '## 复现与浏览', '',
              '从项目根目录运行，首次使用新的输出目录；恢复同一输入时追加`--resume`。', '',
              '```bash', '.venv/bin/python -m pallet.cluster_targets --device cuda:1',
              '.venv/bin/python -m pallet.cluster_targets --stage analyze --resume \\\n  --attribute-review outputs/pallet_target_clustering_v1/candidate_attribute_review.json', '```', '',
              '每次分析写入新的`analysis/run_NNN`，保留历史结果。使用`--shards 2 --shard-index 0|1`可在两张GPU分别提取，'
              '全部帧缓存齐备后再执行分析。图册与代表点云可离线浏览；保持图册、相对路径的media和正式数据集目录位置。'
              '复用已有样本的候选观察记录时使用`--attribute-review`参数；该参数只影响候选属性解释。', '',
              '[运行汇总](summary.json) · [核验结果](validation.json) · [分析校验和](checksums.sha256) · '
              '[特征变换](feature_transforms.json) · [处理失败](failures.jsonl)', '']
    (result/'report_zh.md').write_text('\n'.join(lines))


def run_analysis(args, annotations, sources):
    output, dataset = Path(args.output).resolve(), Path(args.dataset).resolve()
    manifest = json.loads((output/'run_manifest.json').read_text())
    if manifest['protected_files'] != protected_snapshot(dataset):
        raise ValueError('Formal labels changed before analysis')
    existing = sorted((output/'analysis').glob('run_*'))
    result = output/'analysis'/f'run_{len(existing)+1:03d}'
    result.mkdir()
    started = time.perf_counter()
    def log(message):
        message = {'timestamp': timestamp(), **message}
        with (result/'analysis.log').open('a') as stream:
            stream.write(json.dumps(message)+'\n')
        print(json.dumps(message), flush=True)
    rows, embeddings = collect_targets(output, annotations, sources)
    registry = infer_batch_units(rows)
    save_json(result/'unit_registry.json', registry)
    save_json(result/'geometry_quality_policy.json', GEOMETRY_QUALITY_POLICY)
    add_geometry(rows, registry)
    log({'stage': 'geometry_complete', 'targets': len(rows), 'geometry_states': dict(Counter(r['geometry_status'] for r in rows))})
    appearance_i, appearance_x, reduced, transform = appearance_matrix(rows, embeddings, args.seed)
    geometry_i, geometry_x, geometry_transform = geometry_matrix(rows)
    numeric_transform = {k: v for k, v in transform.items() if k.endswith('_array')}
    with (result/'features.npz').open('wb') as stream:
        np.savez_compressed(stream, appearance_indices=appearance_i, appearance_features=appearance_x,
                            appearance_pca=reduced, geometry_indices=geometry_i, geometry_features=geometry_x,
                            **numeric_transform)
    transform = {k:v for k,v in transform.items() if not k.endswith('_array')}
    save_json(result/'feature_transforms.json', {'appearance': transform, 'geometry': geometry_transform})
    for row in rows:
        row.update(appearance_cluster=-2, geometry_cluster=-2, appearance_membership=None, geometry_membership=None)
    summaries, descriptions, sensitivity, bootstraps = {}, {}, {}, {}
    for kind, indices, matrix in [('appearance', appearance_i, appearance_x), ('geometry', geometry_i, geometry_x)]:
        subset = [rows[int(i)] for i in indices]
        labels, probabilities, _ = fit_clusters(matrix)
        for position, index in enumerate(indices):
            rows[int(index)][kind+'_cluster'] = int(labels[position])
            rows[int(index)][kind+'_membership'] = float(probabilities[position])
        summaries[kind] = cluster_statistics(matrix, labels, probabilities, subset, args.seed)
        descriptions[kind] = describe_clusters(rows, indices, matrix, labels, probabilities, kind)
        log({'stage': 'clusters', 'kind': kind, **summaries[kind]})
        sensitivity[kind] = {}
        for size in (25, 50, 100):
            alt_labels, alt_probability, _ = (labels, probabilities, None) if size == 50 else fit_clusters(matrix, size)
            sensitivity[kind][str(size)] = cluster_statistics(matrix, alt_labels, alt_probability, subset, args.seed)
        bootstraps[kind] = bootstrap_clusters(matrix, labels, subset, args.bootstrap_rounds, args.seed, log)
    review_document = json.loads(Path(args.attribute_review).read_text()) if getattr(args, 'attribute_review', None) else {}
    if review_document:
        reviewed_ids = set(review_document.get('reviewed_target_ids', []))
        if not reviewed_ids or not reviewed_ids.issubset({r['target_id'] for r in rows}):
            raise ValueError('Attribute review must identify actual reviewed targets')
    proposals = assign_candidates(rows, descriptions, review_document.get('clusters', {}))
    proposals['comparison_examples'] = comparison_groups(rows)
    save_json(result/'attribute_taxonomy.json', proposals)
    save_json(result/'clusters.json', descriptions)
    save_json(result/'sensitivity.json', sensitivity)
    save_json(result/'group_bootstrap.json', bootstraps)
    validation = {'expected_targets': sum(len(r['boxes']) for r in annotations), 'actual_targets': len(rows),
                  'unique_targets': len({r['target_id'] for r in rows}),
                  'expected_positive_frames': sum(bool(r['boxes']) for r in annotations),
                  'actual_positive_frames': len({r['frame_id'] for r in rows}),
                  'negative_frames_excluded': sum(r['status']=='negative' for r in annotations),
                  'protected_files_checked': len(manifest['protected_files']),
                  'protected_files_unchanged': manifest['protected_files'] == protected_snapshot(dataset),
                  'alignment_verified_targets': sum(r['alignment_verified'] for r in rows),
                  'processing_errors': sum(bool(r['processing_error']) for r in rows),
                  'unit_status': dict(Counter(r['unit_status'] for r in rows)),
                  'geometry_status': dict(Counter(r['geometry_status'] for r in rows)),
                  'appearance_status': dict(Counter(r['appearance_status'] for r in rows)),
                  'all_targets_accounted_for': len(rows)==sum(len(r['boxes']) for r in annotations)}
    save_json(result/'validation.json', validation)
    write_rows(result/'targets.jsonl', [{k:v for k,v in row.items() if not k.startswith('_')} for row in rows])
    write_csv(result/'targets.csv', rows)
    review = [r for r in rows if r['review_reasons']]
    write_rows(result/'review_queue.jsonl', [{k:v for k,v in row.items() if not k.startswith('_')} for row in review])
    write_rows(result/'rare_candidates.jsonl', [{k:v for k,v in row.items() if not k.startswith('_')}
               for row in review if any(reason.startswith('rare_') for reason in row['review_reasons'])])
    write_rows(result/'failures.jsonl', [r for r in rows if r['processing_error']])
    summary = {'created_at': timestamp(), 'finished_at': timestamp(), 'seed': args.seed,
               'analysis_source_sha256': digest(__file__), 'input_manifest_sha256': digest(output/'run_manifest.json'),
               'validation': validation, 'clustering': summaries,
               'bootstrap_rounds_per_feature_view': args.bootstrap_rounds,
               'group_stability': {k: {f:v for f,v in obj.items() if f!='rounds'} for k,obj in bootstraps.items()},
               'review_targets': len(review), 'review_reasons': dict(Counter(reason for r in rows for reason in r['review_reasons'])),
               'elapsed_seconds': time.perf_counter()-started, 'cluster_selection_method': 'eom',
               'min_cluster_size': 50, 'min_samples': 10, 'labels_modified': False,
               'geometry_quality_policy': GEOMETRY_QUALITY_POLICY,
               'attribute_review_sha256': digest(args.attribute_review) if review_document else None,
               'assistant_visually_reviewed_targets': len(review_document.get('reviewed_target_ids',[]))}
    if review_document:
        save_json(result/'candidate_attribute_review.json', review_document)
    save_json(result/'summary.json', summary)
    generate_reports(result, output, dataset, rows, descriptions, summary, sensitivity, proposals)
    log({'stage': 'analysis_complete', 'output': str(result), 'targets': len(rows)})
    files = sorted(p for p in result.rglob('*') if p.is_file() and p.name!='checksums.sha256')
    (result/'checksums.sha256').write_text(''.join(f'{digest(p)}  {p.relative_to(result)}\n' for p in files))
    save_json(output/'latest.json', {'analysis': str(result), 'report': str(result/'report_zh.md'),
                                   'report_html': str(result/'report_zh.html'),
                                   'gallery': str(result/'gallery.html'), 'finished_at': timestamp()})
    return summary
