#!/usr/bin/env python3
"""Trace a completed raster scan to canonical PCD, sealed tensors and packages.

Produces audit artifacts only; does not alter images, labels, masks or protocols.
Run with PYTHONPATH=. .venv/bin/python scripts/trace_black_image_sources.py.
"""
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
import hashlib
import json
from pathlib import Path
from zoneinfo import ZoneInfo

import numpy as np
from PIL import Image

from pallet.data.pcd import read_pcd
from pallet.review_batches import display


def sha(path):
    h = hashlib.sha256()
    with Path(path).open('rb') as stream:
        for chunk in iter(lambda: stream.read(4 * 1024**2), b''):
            h.update(chunk)
    return h.hexdigest()


def read(path):
    return json.loads(Path(path).read_text())


def write(path, value):
    Path(path).write_text(json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False) + '\n')


def main():
    project = Path(__file__).resolve().parent.parent
    output = project / 'outputs/black_image_audit_20261008_v1'
    source = project / 'outputs/anylabeling/pallet_v5_20260930'
    latest = project / 'outputs/human_correction_manual_1716_v2'
    records = {r['frame_id']: r for r in map(json.loads,
               (project / 'outputs/protocol_v5/manifest.jsonl').read_text().splitlines())}
    index = {r['frame_id']: r for r in map(json.loads, (source / 'frames.jsonl').read_text().splitlines())}
    cached = read(project / 'outputs/cache_v5/sealed_index.json')
    scan = read(output / 'raster_scan.json')
    manual = read(latest / 'manual_partition.json')
    train_ids, val_ids = set(manual['train_frame_ids']), set(manual['validation_frame_ids'])
    annotations = {r['frame_id']: r for r in map(json.loads,
                   (latest / 'inputs/annotations.jsonl').read_text().splitlines())}
    assert len(records) == len(index) == scan['summary']['unique_frames']

    def trace(item):
        fid = item['frame_id']
        record, entry, views = records[fid], cached[fid], index[fid]
        pcd_path = Path(record['paths']['pcd'])
        assert sha(pcd_path) == record['source_sha256'] == views['source_anchor_hash']
        assert sha(entry['path']) == entry['sha256']
        with np.load(entry['path'], allow_pickle=False) as saved:
            array, metadata = saved['frame'], json.loads(str(saved['metadata']))
        assert metadata['source_sha256'] == record['source_sha256']
        _, points = read_pcd(pcd_path)
        intensity = points[:, 3]
        finite = intensity[np.isfinite(intensity)]
        valid_i = array[..., 3] > 0
        values = array[..., 1][valid_i]
        rendered = (display(array[..., 1], array[..., 3], log=True) if valid_i.any()
                    else display(array[..., 0], array[..., 2], invert=True))
        assert not np.any(rendered), 'Scanned black PNG no longer agrees with sealed tensor'
        with Image.open(source / views['image']) as im:
            assert np.array_equal(np.asarray(im), rendered)
        image_sha = sha(source / views['image'])
        assert image_sha == views['files'][views['image']]
        has_signal = bool(np.any(array[..., :2]))
        if valid_i.any() and len(np.unique(values)) == 1:
            cause = 'constant_intensity_selected_as_primary'
        elif not has_signal and not np.any(array[..., 2:]):
            cause = 'all_model_channels_and_masks_zero'
        else:
            cause = 'other_render_degeneracy'
        result = {'frame_id': fid, 'group_id': record['group_id'], 'domain': views['domain'],
                  'original_split': views['split'], 'cause': cause,
                  'manual_partition': 'train' if fid in train_ids else 'validation' if fid in val_ids else None,
                  'manual_status': annotations[fid]['status'] if fid in annotations else None,
                  'manual_boxes': len(annotations[fid]['boxes']) if fid in annotations else None,
                  'in_remaining_package': item['in_remaining_package'],
                  'image': views['image'], 'depth_view': views['depth'], 'validity_view': views['validity'],
                  'source_paths': record['paths'], 'source_pcd_sha256': record['source_sha256'],
                  'source_primary_png_sha256': image_sha, 'cache_sha256': entry['sha256'],
                  'cache_metadata': metadata, 'policy': record['sensor_config'],
                  'raw_intensity_min': float(finite.min()) if len(finite) else None,
                  'raw_intensity_max': float(finite.max()) if len(finite) else None,
                  'raw_intensity_unique_values': int(len(np.unique(finite))),
                  'raw_xyz_nonzero_pixels': int(np.count_nonzero(np.any(np.nan_to_num(points[:, :3]) != 0, axis=1))),
                  'raw_xyz_finite_pixels': int(np.isfinite(points[:, :3]).all(axis=1).sum()),
                  'raw_points': int(len(points)), 'depth_valid_pixels': int(np.count_nonzero(array[..., 2])),
                  'intensity_valid_pixels': int(np.count_nonzero(array[..., 3])),
                  'display_percentiles_log_intensity': np.percentile(np.log1p(values), [1, 99]).tolist() if len(values) else None}
        if 'depth' in record['paths']:
            with Image.open(record['paths']['depth']) as im:
                raw_depth = np.asarray(im)
            result['raw_depth'] = {'dtype': str(raw_depth.dtype), 'shape': list(raw_depth.shape),
                                   'min': int(raw_depth.min()), 'max': int(raw_depth.max()),
                                   'positive_pixels': int(np.count_nonzero(raw_depth > 0))}
        if item['in_remaining_package']:
            assert sha(latest / 'anylabeling' / views['image']) == image_sha
            prediction = read(latest / 'predictions' / (hashlib.sha256(fid.encode()).hexdigest()[:24] + '.json'))
            result['current_model_proposal_count'] = len(prediction['boxes_xyxy_score'])
            result['current_model_status'] = prediction['status']
        return result

    with ThreadPoolExecutor(max_workers=2) as pool:
        traced = list(pool.map(trace, scan['black_primary_frames']))
    write(output / 'black_frame_source_trace.json', traced)
    summary = {'checked_at': datetime.now(ZoneInfo('Asia/Shanghai')).isoformat(),
               'black_primary_frames': len(traced),
               'counts_by_cause': dict(Counter(r['cause'] for r in traced)),
               'remaining_black_frames': sum(r['in_remaining_package'] for r in traced),
               'manual_train_black_frames': sum(r['manual_partition'] == 'train' for r in traced),
               'manual_validation_black_frames': sum(r['manual_partition'] == 'validation' for r in traced),
               'remaining_black_with_proposals': sum(r.get('current_model_proposal_count', 0) > 0 for r in traced),
               'remaining_black_proposal_boxes': sum(r.get('current_model_proposal_count', 0) for r in traced),
               'all_pcd_cache_and_image_hashes_verified': True,
               'all_primary_pngs_reproduced_from_sealed_tensors': True,
               'source_images_labels_masks_and_snapshots_changed': False}
    write(output / 'source_trace_summary.json', summary)
    binding = {'source_frame_index_sha256': sha(source / 'frames.jsonl'),
               'protocol_manifest_sha256': sha(project / 'outputs/protocol_v5/manifest.jsonl'),
               'sealed_index_sha256': sha(project / 'outputs/cache_v5/sealed_index.json'),
               'manual_partition_sha256': sha(latest / 'manual_partition.json'),
               'raster_scan_sha256': sha(output / 'raster_scan.json'),
               'trace_script_sha256': sha(Path(__file__))}
    write(output / 'input_binding.json', binding)
    print(json.dumps(summary, ensure_ascii=False, indent=2), flush=True)


if __name__ == '__main__':
    main()
