"""Read-only, resumable discovery of attributes of human-labelled pallet targets.

Only this analysis' output directory is written. Cluster IDs and proposed
attributes are exploratory; they are never imported into the formal labels.
"""
from __future__ import annotations

import argparse
from collections import Counter, defaultdict
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
import hashlib
import json
import math
import os
from pathlib import Path
import sys
import time
from zoneinfo import ZoneInfo

import cv2
import numpy as np

from .data.orientation import orient_grid
from .data.pcd import read_pcd


PROJECT = Path(__file__).resolve().parents[1]
DEFAULT_MODEL_REPO = Path('/root/private/project/zero_shot_segmentation/third_party/dinov3')
DEFAULT_WEIGHTS = Path('/root/.cache/torch/hub/checkpoints/dinov3_vitl16_pretrain_lvd1689m-8aa4cbdd.pth')
EXTRACT_VERSION = 1
APPEARANCE_DIM = 2048


def timestamp():
    return datetime.now(ZoneInfo('Asia/Shanghai')).isoformat()


def digest(path):
    value = hashlib.sha256()
    with Path(path).open('rb') as stream:
        for block in iter(lambda: stream.read(4 * 1024 * 1024), b''):
            value.update(block)
    return value.hexdigest()


def save_json(path, value):
    path = Path(path)
    temporary = path.with_suffix(path.suffix + '.tmp')
    temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False) + '\n')
    os.replace(temporary, path)


def read_rows(path):
    with Path(path).open() as stream:
        return [json.loads(line) for line in stream if line.strip()]


def write_rows(path, rows):
    path = Path(path)
    temporary = path.with_suffix(path.suffix + '.tmp')
    with temporary.open('w') as stream:
        for row in rows:
            stream.write(json.dumps(row, ensure_ascii=False, allow_nan=False) + '\n')
    os.replace(temporary, path)


def integer_box(box, width, height):
    values = np.array([box[k] for k in ('x', 'y', 'width', 'height')], dtype=float)
    if not np.isfinite(values).all() or min(values[2:]) <= 0:
        raise ValueError('nonfinite or nonpositive annotation box')
    x, y, w, h = values
    if x < 0 or y < 0 or x+w > width+1e-5 or y+h > height+1e-5:
        raise ValueError('annotation outside native grid')
    return (max(0, math.floor(x)), max(0, math.floor(y)),
            min(width, math.ceil(x+w)), min(height, math.ceil(y+h)))


def target_id(frame_id, box_index):
    return f'{frame_id}::box:{box_index:04d}'


def target_key(frame_id, box_index):
    return hashlib.sha256(target_id(frame_id, box_index).encode()).hexdigest()[:24]


def load_dataset(dataset):
    dataset = Path(dataset).resolve()
    annotations = read_rows(dataset / 'annotations/frames.jsonl')
    sources = {r['frame_id']: r for r in read_rows(dataset / 'frames.jsonl')}
    if len(sources) != len(annotations) or len({r['frame_id'] for r in annotations}) != len(annotations):
        raise ValueError('duplicate or missing formal frame identities')
    for annotation in annotations:
        source = sources[annotation['frame_id']]
        if (annotation['split'] != source['split'] or annotation['group_id'] != source['group_id']
                or len(annotation['boxes']) != source['instances']
                or annotation['status'] not in ('pos', 'negative')
                or bool(annotation['boxes']) != (annotation['status'] == 'pos')):
            raise ValueError('formal annotation/source disagreement')
        for box in annotation['boxes']:
            integer_box(box, annotation['width'], annotation['height'])
    return annotations, sources


def protected_snapshot(dataset):
    """Hash actual labels and metadata, rather than only trusting the manifest."""
    dataset = Path(dataset)
    files = set(dataset.glob('*.json')) | set(dataset.glob('*.jsonl'))
    files |= set((dataset / 'annotations').rglob('*'))
    files |= set((dataset / 'labels').rglob('*'))
    files |= set((dataset / 'protocol').rglob('*'))
    files |= set((dataset / 'images').rglob('*.json'))
    files |= {dataset / 'checksums.sha256'}
    paths = sorted(p for p in files if p.is_file())
    with ThreadPoolExecutor(max_workers=8) as pool:
        hashes = list(pool.map(digest, paths))
    return {str(p.relative_to(dataset)): h for p, h in zip(paths, hashes)}


def prepare_run(args, annotations):
    from .proposals import SAM3_WEIGHTS
    out = Path(args.output).resolve()
    binding = {'dataset': str(Path(args.dataset).resolve()), 'seed': args.seed,
               'extract_version': EXTRACT_VERSION, 'dinov3_weights': str(Path(args.weights).resolve()),
               'dinov3_sha256': digest(args.weights), 'sam3_weights': str(SAM3_WEIGHTS),
               'sam3_sha256': digest(SAM3_WEIGHTS), 'appearance_long_side': 512,
               'points_per_target': 4096, 'mask_min_score': 0.5,
               'targets': sum(len(r['boxes']) for r in annotations), 'frames': len(annotations)}
    manifest_path = out / 'run_manifest.json'
    if manifest_path.exists():
        if not args.resume:
            raise ValueError('Output already exists; use --resume for the same inputs or choose a new output.')
        old = json.loads(manifest_path.read_text())
        if old['binding'] != binding:
            raise ValueError('Resume binding differs from the original extraction.')
        if old['dinov3_implementation_sha256'] != digest(Path(args.model_repo)/'dinov3/models/vision_transformer.py'):
            raise ValueError('DINOv3 implementation changed since the extraction started.')
        snapshot = protected_snapshot(args.dataset)
        if old['protected_files'] != snapshot:
            raise ValueError('Formal labels or frozen metadata changed since the run started.')
    else:
        if out.exists() and any(out.iterdir()):
            raise ValueError('Choose an empty output directory.')
        out.mkdir(parents=True, exist_ok=True)
        snapshot = protected_snapshot(args.dataset)
        from importlib.metadata import version
        old = {'kind': 'exploratory_pallet_multiattribute_clustering', 'created_at': timestamp(),
               'binding': binding, 'protected_files': snapshot,
               'source_code_sha256': digest(__file__),
               'dinov3_implementation_sha256': digest(Path(args.model_repo) / 'dinov3/models/vision_transformer.py'),
               'packages': {k: version(k) for k in ('numpy', 'scipy', 'scikit-learn', 'torch', 'torchvision')},
               'labels_modified': False, 'independent_classification_accuracy': None}
        save_json(manifest_path, old)
    for name in ('frames', 'media', 'logs', 'analysis'):
        (out / name).mkdir(exist_ok=True)
    return old


def load_native(dataset, annotation, source):
    image_path, sensor_path = dataset / source['image'], dataset / source['sensor']
    for path in (image_path, sensor_path):
        expected = source['files'][str(path.relative_to(dataset))]
        if digest(path) != expected:
            raise ValueError(f'Frozen asset hash mismatch: {path}')
    with np.load(sensor_path, allow_pickle=False) as archive:
        sensor = archive['frame']
    if sensor.shape != (annotation['height'], annotation['width'], 4) or not np.isfinite(sensor).all():
        raise ValueError('Invalid formal sensor shape or values')
    path = Path(source['source_paths']['pcd'])
    if digest(path) != source['source_anchor_hash']:
        raise ValueError('Raw PCD differs from the frozen source anchor')
    _, points = read_pcd(path)
    if len(points) != annotation['width'] * annotation['height']:
        raise ValueError('PCD point count differs from registered grid')
    # The explicit registry restores flattened grids; never guess a grid from point count here.
    native = orient_grid(points.reshape(annotation['height'], annotation['width'], 4), source['sensor_policy'])
    valid_i = sensor[..., 3] > 0
    if not valid_i.any() or not np.allclose(native[..., 3][valid_i], sensor[..., 1][valid_i], rtol=1e-6, atol=1e-6):
        raise ValueError('Raw PCD intensity and formal tensor are not pixel-aligned')
    image = cv2.imread(str(image_path), cv2.IMREAD_GRAYSCALE)
    if image is None or image.shape != sensor.shape[:2]:
        raise ValueError('Invalid formal intensity PNG')
    return image, native[..., :3], valid_i


def unit_hint(points):
    points = np.asarray(points, dtype=float)
    good = np.isfinite(points).all(axis=1) & (np.linalg.norm(np.nan_to_num(points), axis=1) > 1e-8)
    points = points[good]
    if len(points) < 32:
        return {'hint': 'unknown', 'reason': 'too_few_points', 'points': len(points)}
    fractional = float((np.abs(points-np.rint(points)) > 1e-4).mean())
    ranges = np.linalg.norm(points, axis=1)
    q50, q90 = np.percentile(ranges, [50, 90])
    hint = 'unknown'
    if fractional <= 0.05 and q50 >= 10:
        hint = 'mm'
    elif fractional >= 0.05 and q90 <= 10:
        hint = 'm'
    return {'hint': hint, 'noninteger_fraction': fractional, 'range_q50_raw': float(q50),
            'range_q90_raw': float(q90), 'points': len(points), 'reason': 'user_distribution_heuristic'}


def clean_points(points, uv):
    """Remove invalid returns and distant contamination, retaining visible surfaces."""
    from scipy.spatial import cKDTree
    points, uv = np.asarray(points), np.asarray(uv)
    good = np.isfinite(points).all(axis=1) & (np.linalg.norm(np.nan_to_num(points), axis=1) > 1e-8)
    points, uv = points[good], uv[good]
    before = len(points)
    if before < 64:
        return points, uv, {'points_before': before, 'points_after': before, 'retained_fraction': 1.0}
    ranges = np.linalg.norm(points, axis=1)
    median = np.median(ranges)
    mad = np.median(np.abs(ranges-median))
    keep = np.abs(ranges-median) <= max(4.5*1.4826*mad, abs(median)*0.025, 1e-6)
    if keep.sum() >= 64:
        points, uv = points[keep], uv[keep]
    distances, _ = cKDTree(points).query(points, k=min(8, len(points)), workers=1)
    local = distances[:, -1]
    threshold = max(float(np.percentile(local, 97.5)), 1e-10)
    keep = local <= threshold
    points, uv = points[keep], uv[keep]
    return points, uv, {'points_before': before, 'points_after': len(points),
                        'retained_fraction': float(len(points)/before)}


def measure_points(points, uv, scale_to_mm):
    """Visible surface estimates; the third extent is never called pallet thickness."""
    points = np.asarray(points, dtype=np.float64) * scale_to_mm
    if len(points) < 64:
        raise ValueError('Too few valid foreground points for geometry')
    centered = points - np.median(points, axis=0)
    values, vectors = np.linalg.eigh(np.cov(centered, rowvar=False))
    order = np.argsort(values)[::-1]
    values, vectors = np.maximum(values[order], 0), vectors[:, order]
    projections = centered @ vectors
    extents = np.diff(np.percentile(projections, [2, 98], axis=0), axis=0)[0]
    denominator = max(values[0], 1e-12)
    uv = np.asarray(uv, dtype=float)
    design = np.column_stack([uv-uv.mean(axis=0), np.ones(len(uv))])
    coefficients = np.linalg.lstsq(design, centered, rcond=None)[0]
    visible_uv = np.diff(np.percentile(uv, [2, 98], axis=0), axis=0)[0]
    face = np.linalg.norm(coefficients[:2], axis=1) * visible_uv
    return {'visible_extent_major_mm': float(extents[0]), 'visible_extent_middle_mm': float(extents[1]),
            'visible_extent_minor_mm': float(extents[2]), 'visible_horizontal_span_mm': float(face[0]),
            'visible_vertical_span_mm': float(face[1]),
            'linearity': float((values[0]-values[1])/denominator),
            'planarity': float((values[1]-values[2])/denominator),
            'scattering': float(values[2]/denominator),
            'principal_extent_ratio': float(extents[0]/max(extents[1], 1e-6)),
            'range_median_mm': float(np.median(np.linalg.norm(points, axis=1))),
            'measurement_scope': 'visible_foreground_only', 'physical_full_dimensions': None}


class FeatureModels:
    def __init__(self, args):
        import torch
        from .proposals import _sam3_service
        self.torch, self.device = torch, args.device
        torch.set_num_threads(4)
        torch.manual_seed(args.seed)
        np.random.seed(args.seed)
        if args.device.startswith('cuda'):
            torch.cuda.set_device(torch.device(args.device))
        self.sam = _sam3_service(args.device)
        self.sam._lazy_init()
        sys.path.insert(0, str(Path(args.model_repo)))
        from dinov3.hub.backbones import dinov3_vitl16
        self.dino = dinov3_vitl16(pretrained=False)
        self.dino.load_state_dict(torch.load(args.weights, map_location='cpu', weights_only=True), strict=True)
        self.dino.eval().to(args.device)
        self.mean = torch.tensor([.485, .456, .406], device=args.device)[:, None, None]
        self.std = torch.tensor([.229, .224, .225], device=args.device)[:, None, None]

    def mask_frame(self, image, boxes):
        """Reuse the image encoder across boxes without accumulating prompts."""
        from PIL import Image
        from zss_demo.mask_utils import choose_best_candidate, cleanup_mask, xyxy_to_cxcywh_norm
        sam = self.sam
        results = []
        with sam._inference_context():
            original = sam._processor.set_image(Image.fromarray(image).convert('RGB'))
            for box in boxes:
                state = dict(original)
                state['backbone_out'] = dict(original['backbone_out'])
                state.pop('geometric_prompt', None)
                state = sam._processor.add_geometric_prompt(
                    box=xyxy_to_cxcywh_norm(box, image.shape[1], image.shape[0]), label=True, state=state)
                best = choose_best_candidate(sam._state_to_candidates(state), box_xyxy=box, points=[])
                if best is None:
                    results.append((np.zeros(image.shape, bool), 0.0))
                else:
                    mask = cleanup_mask(best.mask, box_xyxy=box, points=[])
                    bounded = np.zeros(image.shape, bool)
                    x1, y1, x2, y2 = box
                    bounded[y1:y2, x1:x2] = mask[y1:y2, x1:x2]
                    results.append((bounded, float(best.score)))
        return results

    def appearance(self, crop, mask):
        torch = self.torch
        height, width = crop.shape
        scale = 512 / max(height, width)
        h, w = max(1, round(height*scale)), max(1, round(width*scale))
        resized = cv2.resize(crop, (w, h), interpolation=cv2.INTER_CUBIC)
        valid = cv2.resize(mask.astype(np.float32), (w, h), interpolation=cv2.INTER_NEAREST)
        # Gray padding is excluded from the patch pool; preserve the thin native target's aspect ratio.
        ph, pw = max(64, math.ceil(h/16)*16), max(64, math.ceil(w/16)*16)
        canvas = np.full((ph, pw), 127, np.uint8)
        region = np.zeros((ph, pw), np.float32)
        canvas[:h, :w], region[:h, :w] = resized, valid
        if not region.any():
            region[:h, :w] = 1  # Appearance remains available when segmentation fails.
        tensor = torch.as_tensor(canvas.copy(), device=self.device, dtype=torch.float32)
        tensor = tensor[None].expand(3, -1, -1) / 255
        tensor = ((tensor-self.mean)/self.std)[None]
        with torch.inference_mode(), torch.autocast(device_type=self.device.split(':')[0],
                                                    dtype=torch.float16, enabled=self.device.startswith('cuda')):
            output = self.dino.forward_features(tensor)
        patch = output['x_norm_patchtokens'][0].float()
        weights = torch.as_tensor(cv2.resize(region, (pw//16, ph//16), interpolation=cv2.INTER_AREA).ravel(),
                                  device=self.device)
        pooled = (patch*weights[:, None]).sum(0)/weights.sum().clamp_min(1e-6)
        cls = output['x_norm_clstoken'][0].float()
        return torch.cat([cls/cls.norm().clamp_min(1e-6), pooled/pooled.norm().clamp_min(1e-6)]).cpu().numpy()


def media_assets(output, key, crop, mask):
    target = output / 'media'
    if not cv2.imwrite(str(target / f'{key}.jpg'), crop, [cv2.IMWRITE_JPEG_QUALITY, 88]):
        raise OSError('Could not write target crop')
    overlay = cv2.cvtColor(crop, cv2.COLOR_GRAY2BGR)
    if mask.any():
        overlay[mask] = (overlay[mask].astype(float)*.55 + np.array([70, 200, 60])*.45).astype(np.uint8)
    if not cv2.imwrite(str(target / f'{key}.overlay.jpg'), overlay, [cv2.IMWRITE_JPEG_QUALITY, 88]):
        raise OSError('Could not write target overlay')


def extract_frame(dataset, output, annotation, source, models):
    features, rows, arrays = [], [], {}
    frame_error = None
    try:
        image, xyz, valid_i = load_native(dataset, annotation, source)
        boxes = [integer_box(b, annotation['width'], annotation['height']) for b in annotation['boxes']]
        masks = models.mask_frame(image, boxes)
    except Exception as exc:
        frame_error = f'{type(exc).__name__}: {exc}'
    for index, box in enumerate(annotation['boxes']):
        key = target_key(annotation['frame_id'], index)
        row = {'target_id': target_id(annotation['frame_id'], index), 'target_key': key, 'box_index': index,
               'frame_id': annotation['frame_id'], 'group_id': annotation['group_id'],
               'domain': annotation['domain'], 'split': annotation['split'], 'image': source['image'],
               'bbox': box, 'policy_id': source['sensor_policy']['policy_id'],
               'verified_scale_to_mm': source['sensor_policy'].get('scale_to_mm'),
               'pixel_rotation_degrees': source['pixel_rotation_degrees'],
               'source_pcd_sha256': source['source_anchor_hash'], 'alignment_verified': frame_error is None,
               'appearance_status': 'failed', 'geometry_status': 'unknown', 'processing_error': frame_error,
               'structure_candidate': 'unknown', 'material_candidate': 'unknown', 'viewpoint_candidate': 'unknown'}
        feature = np.full(APPEARANCE_DIM, np.nan, np.float32)
        arrays[f'points_{index}'] = np.empty((0, 3), np.float32)
        arrays[f'uv_{index}'] = np.empty((0, 2), np.float32)
        if frame_error is None:
            try:
                x1, y1, x2, y2 = boxes[index]
                mask, score = masks[index]
                crop, local_mask = image[y1:y2, x1:x2], mask[y1:y2, x1:x2]
                feature = models.appearance(crop, local_mask)
                if feature.shape != (APPEARANCE_DIM,) or not np.isfinite(feature).all():
                    raise ValueError('Invalid appearance embedding')
                media_assets(output, key, crop, local_mask)
                row.update(appearance_status='ok', integer_bbox=list(boxes[index]), mask_score=score,
                           mask_pixels=int(mask.sum()), mask_fraction=float(local_mask.mean()),
                           crop_touches_image_edge=bool(x1==0 or y1==0 or x2==image.shape[1] or y2==image.shape[0]))
                good = np.isfinite(xyz).all(axis=2) & (np.linalg.norm(np.nan_to_num(xyz), axis=2)>1e-8) & valid_i
                selected = mask & good
                vv, uu = np.nonzero(selected)
                p = xyz[selected]
                uv = np.column_stack([uu, vv]).astype(np.float32)
                row['foreground_valid_fraction'] = float(len(p)/max(mask.sum(), 1))
                if len(p) > 4096:
                    sample = np.linspace(0, len(p)-1, 4096, dtype=int)
                    p, uv = p[sample], uv[sample]
                row['unit_observation'] = unit_hint(p)
                p, uv, cleaning = clean_points(p, uv)
                arrays[f'points_{index}'], arrays[f'uv_{index}'] = p.astype(np.float32), uv
                row.update(cleaning)
                row['geometry_status'] = ('eligible' if score >= .5 and len(p) >= 64
                                          and row['foreground_valid_fraction'] >= .25
                                          and row['mask_fraction'] >= .03 and cleaning['retained_fraction'] >= .6
                                          else 'insufficient_foreground')
                if row['crop_touches_image_edge'] and row['geometry_status'] == 'eligible':
                    row['geometry_status'] = 'partial_visible_surface'
            except Exception as exc:
                row['processing_error'] = f'{type(exc).__name__}: {exc}'
        rows.append(row)
        features.append(feature)
    arrays.update(appearance=np.asarray(features, dtype=np.float32),
                  records_json=np.array(json.dumps(rows, ensure_ascii=False, allow_nan=False)))
    destination = output / 'frames' / (Path(source['image']).stem+'.npz')
    temporary = destination.with_suffix('.npz.tmp')
    with temporary.open('wb') as stream:
        np.savez_compressed(stream, **arrays)
    os.replace(temporary, destination)
    return rows


def extract(args, annotations, sources):
    dataset, output = Path(args.dataset).resolve(), Path(args.output).resolve()
    selected = [r for i, r in enumerate(annotations) if r['boxes'] and i % args.shards == args.shard_index]
    if args.limit is not None:
        selected = selected[:args.limit]
    pending = [r for r in selected if not (output/'frames'/(Path(r['image']).stem+'.npz')).exists()]
    progress = output/'logs'/f'extract_{args.shard_index}.json'
    start, completed, errors = time.perf_counter(), 0, 0
    print(json.dumps({'stage': 'extract_start', 'shard': args.shard_index, 'pending_frames': len(pending)}), flush=True)
    models = FeatureModels(args) if pending else None
    for annotation in pending:
        rows = extract_frame(dataset, output, annotation, sources[annotation['frame_id']], models)
        errors += sum(bool(r['processing_error']) for r in rows)
        completed += 1
        if completed % 20 == 0 or completed == len(pending):
            seconds = time.perf_counter()-start
            state = {'stage': 'extract', 'updated_at': timestamp(), 'shard': args.shard_index,
                     'completed_this_process': completed, 'selected_frames': len(selected),
                     'pending_at_start': len(pending), 'remaining_this_process': len(pending)-completed,
                     'target_errors_this_process': errors, 'elapsed_seconds': seconds,
                     'frames_per_second': completed/max(seconds, 1e-6)}
            save_json(progress, state)
            print(json.dumps(state), flush=True)
    save_json(output/'logs'/f'extract_{args.shard_index}_complete.json',
              {'finished_at': timestamp(), 'frames': len(selected), 'errors_this_process': errors,
               'limit': args.limit, 'shards': args.shards})


def analysis(args, annotations, sources):
    from .cluster_report import run_analysis
    return run_analysis(args, annotations, sources)


def main(argv=None):
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--dataset', default=None)
    parser.add_argument('--output', default='outputs/pallet_target_clustering_v1')
    parser.add_argument('--device', default='cuda:1')
    parser.add_argument('--seed', type=int, default=42)
    parser.add_argument('--model-repo', default=str(DEFAULT_MODEL_REPO))
    parser.add_argument('--weights', default=str(DEFAULT_WEIGHTS))
    parser.add_argument('--stage', choices=('prepare', 'extract', 'analyze', 'all'), default='all')
    parser.add_argument('--resume', action='store_true')
    parser.add_argument('--shards', type=int, default=1)
    parser.add_argument('--shard-index', type=int, default=0)
    parser.add_argument('--limit', type=int, default=None, help='Extraction smoke limit; analysis still requires all targets.')
    parser.add_argument('--bootstrap-rounds', type=int, default=20)
    parser.add_argument('--attribute-review', default=None, help='Optional candidate-only visual review JSON; never rewrites labels.')
    args = parser.parse_args(argv)
    if not (args.shards >= 1 and 0 <= args.shard_index < args.shards):
        parser.error('Invalid shard index/count')
    if args.dataset is None:
        args.dataset = json.loads((PROJECT/'configs/active_formal_dataset.json').read_text())['dataset']
    annotations, sources = load_dataset(args.dataset)
    prepare_run(args, annotations)
    if args.stage in ('extract', 'all'):
        extract(args, annotations, sources)
    if args.stage in ('analyze', 'all'):
        analysis(args, annotations, sources)


if __name__ == '__main__':
    main()
