#!/usr/bin/env python3
"""Search same-capture disk/archive modalities for recoverable black-view intensity.

Read-only on the dataset. Requires the previous complete black-image PCD audit.
Archive payloads are hashed as streams, without extracting large source copies.
"""
from collections import Counter, defaultdict
from datetime import datetime
import hashlib
import json
from pathlib import Path
import re
import subprocess
import tarfile
import zipfile
from zoneinfo import ZoneInfo


def read(path):
    return json.loads(Path(path).read_text())


def digest_stream(stream):
    h = hashlib.sha256()
    for block in iter(lambda: stream.read(4 * 1024**2), b''):
        h.update(block)
    return h.hexdigest()


def digest(path):
    with Path(path).open('rb') as stream:
        return digest_stream(stream)


def write(path, value):
    Path(path).write_text(json.dumps(value, ensure_ascii=False, indent=2) + '\n')


def header(path):
    values = {}
    with Path(path).open('rb') as stream:
        while True:
            raw = stream.readline()
            if not raw:
                raise ValueError('PCD DATA header missing')
            parts = raw.decode('ascii').strip().split()
            if not parts or parts[0].startswith('#'):
                continue
            values[parts[0].upper()] = parts[1:]
            if parts[0].upper() == 'DATA':
                return values


def main():
    project = Path(__file__).resolve().parent.parent
    data = Path('/root/private/data/pallet')
    prior = project / 'outputs/black_image_audit_20261008_v1'
    output = project / 'outputs/black_intensity_recovery_check_20261008_v1'
    output.mkdir(parents=True, exist_ok=True)
    trace = read(prior / 'black_frame_source_trace.json')
    by_id = {r['frame_id']: r for r in trace}
    by_stamp = {r['frame_id'].split('#')[0].rsplit('/', 1)[-1]: r['frame_id'] for r in trace}
    assert len(by_id) == len(by_stamp) == 148
    pattern = re.compile('|'.join(re.escape(t) for t in by_stamp))
    inventory_path = project / 'outputs/inventory_v3/items.jsonl'
    inventory = [json.loads(line) for line in inventory_path.read_text().splitlines()]
    disk_index = {r['path']: r for r in inventory if r['kind'] == 'file'}
    current_files = {str(Path(p).relative_to(data)) for p in subprocess.check_output(
        ['rg', '--files', '--hidden', '--no-ignore', str(data)], text=True).splitlines()}
    assert current_files == set(disk_index), 'Dataset file catalog changed; rebuild inventory first'
    assert all((data / name).stat().st_size == row['size'] for name, row in disk_index.items()), 'Dataset sizes changed'
    selected = []
    for item in inventory:
        name = item.get('path') or item.get('member') or ''
        match = pattern.search(name)
        if match:
            selected.append(dict(item, target_frame_id=by_stamp[match.group()]))
    write(output / 'timestamp_candidates.json', selected)
    disks, archives = [], []
    for row in selected:
        if row['kind'] != 'file':
            continue
        path = data / row['path']
        actual = digest(path)
        fid = row['target_frame_id']
        entry = {'frame_id': fid, 'path': row['path'], 'size': path.stat().st_size,
                 'sha256': actual, 'kind': 'file', 'extension': path.suffix.lower()}
        if path.suffix.lower() == '.pcd':
            pcd_header = header(path)
            entry['pcd_header'] = pcd_header
            # SHA equality binds to the prior FULL payload intensity/XYZ reading,
            # not a header sample. No alternate intensity fields may go unnoticed.
            assert actual == by_id[fid]['source_pcd_sha256'], 'Different same-frame PCD needs a new full payload audit'
            assert set(pcd_header['FIELDS']) == {'x', 'y', 'z', 'intensity'}, 'Additional PCD fields need inspection'
            entry.update(canonical_content_identical=True,
                         intensity_min=by_id[fid]['raw_intensity_min'],
                         intensity_max=by_id[fid]['raw_intensity_max'],
                         intensity_unique_values=by_id[fid]['raw_intensity_unique_values'])
        disks.append(entry)
    by_disk_name = {r['path']: r for r in disks}
    archived = defaultdict(dict)
    for row in selected:
        if row['kind'] == 'archive_member':
            archived[row['archive']][row['member']] = row
    archive_hashes = {}
    for archive_name, wanted in sorted(archived.items()):
        archive_path = data / archive_name
        archive_hashes[archive_name] = digest(archive_path)

        def inspect_member(name, stream, size):
            row = wanted[name]
            fid = row['target_frame_id']
            actual = digest_stream(stream)
            matches = [relative for relative in row.get('matches', [])
                       if relative in by_disk_name and by_disk_name[relative]['sha256'] == actual]
            assert matches, f'Archive member differs from all verified disk candidates: {archive_name}/{name}'
            reference = by_disk_name[matches[0]]
            entry = {'frame_id': fid, 'kind': 'archive_member', 'archive': archive_name,
                     'member': name, 'size': size, 'sha256': actual, 'verified_disk_matches': matches,
                     'extension': Path(name).suffix.lower(), 'content_identical': True}
            if entry['extension'] == '.pcd':
                entry.update(intensity_min=reference['intensity_min'], intensity_max=reference['intensity_max'],
                             intensity_unique_values=reference['intensity_unique_values'],
                             fields=reference['pcd_header']['FIELDS'])
            archives.append(entry)
        seen = set()
        if archive_name.endswith('.tar.gz'):
            with tarfile.open(archive_path, 'r|gz') as source:
                for member in source:
                    if member.name in wanted and member.isfile():
                        with source.extractfile(member) as stream:
                            inspect_member(member.name, stream, member.size)
                        seen.add(member.name)
        elif archive_name.endswith('.zip'):
            with zipfile.ZipFile(archive_path) as source:
                for name in wanted:
                    with source.open(name) as stream:
                        inspect_member(name, stream, source.getinfo(name).file_size)
                    seen.add(name)
        else:
            raise ValueError('Additional archive decoder required: ' + archive_name)
        assert seen == set(wanted), 'Expected candidate archive members missing'
    write(output / 'disk_candidate_checks.json', disks)
    write(output / 'archive_candidate_checks.json', archives)
    per_frame = []
    for fid, black in by_id.items():
        disk_rows = [r for r in disks if r['frame_id'] == fid]
        archive_rows = [r for r in archives if r['frame_id'] == fid]
        intensity_rasters = [r for r in disk_rows if any(token in Path(r['path']).name.lower()
                             for token in ('amplitude', 'intensity', 'reflectivity')) and r['extension'] != '.pcd']
        assert not intensity_rasters, 'Found same-frame intensity raster; inspect before declaring missing'
        per_frame.append({'frame_id': fid, 'in_remaining_package': black['in_remaining_package'],
                          'disk_pc_candidates': sum(r['extension'] == '.pcd' for r in disk_rows),
                          'archive_pc_candidates': sum(r['extension'] == '.pcd' for r in archive_rows),
                          'same_frame_intensity_rasters': len(intensity_rasters),
                          'raw_intensity_min': black['raw_intensity_min'],
                          'raw_intensity_max': black['raw_intensity_max'],
                          'raw_intensity_unique_values': black['raw_intensity_unique_values'],
                          'real_intensity_recoverable': False,
                          'depth_view_recoverable': black['depth_valid_pixels'] > 0,
                          'rgb_available': any(r['extension'] == '.jpg' for r in disk_rows),
                          'depth_view': black['depth_view'],
                          'reason': 'only_constant_one_intensity_copies' if black['raw_intensity_min'] == 1
                                    else 'all_zero_original_and_archive_duplicate',
                          'source_paths': black['source_paths']})
    write(output / 'frame_recovery_results.json', per_frame)
    summary = {'checked_at': datetime.now(ZoneInfo('Asia/Shanghai')).isoformat(),
               'scope': '148 black primary frames in canonical protocol, current complete disk inventory and timestamp-matched archive entries',
               'disk_files_searched': len(current_files),
               'archive_catalog_rows_searched': sum(r['kind'] == 'archive_member' for r in inventory),
               'same_timestamp_disk_candidates': len(disks),
               'same_timestamp_archive_candidates': len(archives),
               'disk_pc_candidates': sum(r['extension'] == '.pcd' for r in disks),
               'archive_pc_candidates': sum(r['extension'] == '.pcd' for r in archives),
               'archive_container_sha256': archive_hashes,
               'pc_field_layouts': dict(Counter(' '.join(r['pcd_header']['FIELDS']) for r in disks if r['extension'] == '.pcd')),
               'real_intensity_recoverable_frames': 0,
               'depth_view_recoverable_frames': sum(r['depth_view_recoverable'] for r in per_frame),
               'rgb_available_frames': sum(r['rgb_available'] for r in per_frame),
               'same_frame_intensity_raster_files': 0,
               'all_disk_pcs_identical_to_full_audited_canonical_pc': True,
               'all_checked_archive_candidates_match_disk_content': True,
               'dataset_originals_labels_snapshots_zip_modified': False}
    write(output / 'summary.json', summary)
    write(output / 'input_binding.json', {
        'inventory_sha256': digest(inventory_path),
        'prior_full_pc_trace_sha256': digest(prior / 'black_frame_source_trace.json'),
        'source_script_sha256': digest(Path(__file__)), 'archive_sha256': archive_hashes})
    print(json.dumps(summary, ensure_ascii=False, indent=2), flush=True)


if __name__ == '__main__':
    main()
