#!/usr/bin/env python3
"""Read-only audit of source intensity rasters against the frozen formal dataset."""
from __future__ import annotations

import argparse
from collections import Counter, defaultdict
from concurrent.futures import ThreadPoolExecutor, as_completed
import csv
from datetime import datetime
import hashlib
import json
from pathlib import Path
import tarfile
import zipfile
from zoneinfo import ZoneInfo

import cv2
import numpy as np

from pallet.data.inventory import _candidates, _type
from pallet.data.manifest import scan_dataset


def digest(path):
    h = hashlib.sha256()
    with Path(path).open('rb') as source:
        for block in iter(lambda: source.read(4 * 1024 * 1024), b''):
            h.update(block)
    return h.hexdigest()


def intensity_name(name):
    path = Path(name.replace('\\', '/'))
    return (path.suffix.lower() in {'.png', '.yaml', '.yml', '.tif', '.tiff', '.bmp', '.npy', '.npz'}
            and any(word in path.name.lower() for word in ('amplitude', 'intensity', 'reflectivity', 'infrared')))


def matrix(path):
    if path.suffix.lower() in {'.yaml', '.yml'}:
        storage = cv2.FileStorage(str(path), cv2.FILE_STORAGE_READ)
        try:
            array = storage.getFirstTopLevelNode().mat() if storage.isOpened() else None
        finally:
            storage.release()
    else:
        array = cv2.imread(str(path), cv2.IMREAD_UNCHANGED)
    if array is None or array.ndim != 2 or not array.size:
        raise ValueError(f'Not a readable single-channel intensity raster: {path}')
    return array


def save_json(path, value):
    Path(path).write_text(json.dumps(value, ensure_ascii=False, indent=2) + '\n')


def save_rows(path, rows):
    with Path(path).open('w') as target:
        for row in rows:
            target.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + '\n')


def audit(root, dataset, output, workers):
    root, dataset, output = map(lambda p: Path(p).resolve(), (root, dataset, output))
    if output.exists():
        raise ValueError('Choose a new audit output version; existing records are preserved.')
    output.mkdir(parents=True)
    frozen_manifest = json.loads((dataset / 'manifest.json').read_text())
    for name in ('frames.jsonl', 'protocol/manifest.jsonl', 'protocol/excluded.json'):
        if digest(dataset / name) != frozen_manifest['files'][name]:
            raise ValueError(f'Frozen dataset metadata changed: {name}')
    files = {p: p.stat() for p in root.rglob('*') if p.is_file()}
    rows = [json.loads(line) for line in (dataset / 'frames.jsonl').open()]
    formal = {row['frame_id']: row for row in rows}
    protocol = [json.loads(line) for line in (dataset / 'protocol/manifest.jsonl').open()]
    aliases, direct = {}, {}
    for row in protocol:
        for fid in row.get('aliases', []) + [row['frame_id']]:
            if fid in aliases and aliases[fid]['frame_id'] != row['frame_id']:
                raise ValueError(f'Ambiguous alias: {fid}')
            aliases[fid] = row
        for p in row['paths'].values():
            direct[p] = row
        if row.get('original_zero_intensity_path'):
            direct[row['original_zero_intensity_path']] = row
    raw_records = scan_dataset(root)
    raw_paths = {str(p): row for row in raw_records for p in row.paths.values()}
    excluded = json.loads((dataset / 'protocol/excluded.json').read_text())
    excluded_paths = {p: row for row in excluded for p in row.get('record', {}).get('paths', {}).values()}
    sources = sorted(p for p in files if intensity_name(p.name))
    print(json.dumps({'stage': 'disk_rasters', 'files': len(files), 'intensity_sources': len(sources)}), flush=True)
    disk, raster_hashes = [], {}
    for i, path in enumerate(sources, 1):
        raw = raw_paths.get(str(path))
        target = direct.get(str(path)) or (aliases.get(raw.frame_id) if raw else None)
        entry = {'path': str(path.relative_to(root)), 'suffix': path.suffix.lower(),
                 'size': files[path].st_size, 'mtime_ns': files[path].st_mtime_ns,
                 'raw_frame_id': raw.frame_id if raw else None}
        try:
            sha = digest(path)
            raster_hashes[path] = sha
            arr = matrix(path)
            valid = np.isfinite(arr)
            entry.update(sha256=sha, dimensions=[int(arr.shape[1]), int(arr.shape[0])],
                         dtype=str(arr.dtype), finite_fraction=float(valid.mean()),
                         all_zero=bool(np.all(arr == 0)),
                         minimum=float(arr[valid].min()) if valid.any() else None,
                         maximum=float(arr[valid].max()) if valid.any() else None)
            if target:
                fid = target['frame_id']
                released = formal[fid]
                reconstructed = str(path) == target.get('original_zero_intensity_path')
                status = 'included_reconstructed' if reconstructed else (
                    'included' if str(path) == target['paths'].get('intensity') else 'duplicate_alias')
                entry.update(status=status, frame_id=fid, image=released['image'],
                             split=released['split'], domain=released['domain'],
                             intensity_source=released['sensor_metadata']['intensity_source'],
                             pixel_rotation_degrees=released['pixel_rotation_degrees'])
                if entry['dimensions'] != released['size']:
                    raise ValueError('Source raster dimensions differ from the released frame')
                frozen_stat = target['source_stat'].get(str(path))
                if frozen_stat and frozen_stat != {'size': files[path].st_size, 'mtime_ns': files[path].st_mtime_ns}:
                    raise ValueError('Source stat differs from the frozen protocol')
                if status == 'duplicate_alias':
                    canonical_name = target['paths'].get('intensity')
                    entry['canonical_intensity_path'] = canonical_name
                    entry['byte_identical_to_canonical_intensity'] = (
                        sha == digest(canonical_name) if canonical_name else None)
                    if raw is None or raw.pcd is None or not target['paths'].get('pcd'):
                        raise ValueError('Alias lacks primary-source evidence')
                    entry['alias_primary_source_sha256'] = digest(raw.pcd)
                    if (entry['alias_primary_source_sha256'] != target['source_sha256']
                            or digest(target['paths']['pcd']) != target['source_sha256']):
                        raise ValueError('Alias point cloud differs from the frozen primary source')
                if reconstructed:
                    if not entry['all_zero'] or not released['reconstructed_intensity']:
                        raise ValueError('Reconstruction lineage conflicts with the original zero raster')
                    if digest(target['paths']['pcd']) != target['source_sha256']:
                        raise ValueError('Reconstruction source point cloud changed')
                entry['coverage_verified'] = True
            elif str(path) in excluded_paths:
                entry.update(status='excluded', coverage_verified=False,
                             exclusion_reason=excluded_paths[str(path)]['reason'])
            else:
                entry.update(status='unmatched', coverage_verified=False)
            after = path.stat()
            if (after.st_size, after.st_mtime_ns) != (files[path].st_size, files[path].st_mtime_ns):
                raise ValueError('Source changed while auditing')
        except Exception as exc:
            entry.update(status='error', coverage_verified=False, error=str(exc))
        disk.append(entry)
        if i % 500 == 0:
            print(json.dumps({'disk_rasters_checked': i, 'total': len(sources)}), flush=True)
    save_rows(output / 'disk_intensity_files.jsonl', disk)
    disk_by_path = {row['path']: row for row in disk}
    filename_hash_paths = defaultdict(list)
    for path, sha in raster_hashes.items():
        filename_hash_paths[(path.name, sha)].append(path)
    selected = {row['frame_id'] for row in disk if row.get('coverage_verified')}
    artifact_checks = []
    for fid in sorted(selected):
        row = formal[fid]
        path = dataset / row['image']
        check = {'frame_id': fid, 'image': row['image']}
        try:
            if digest(path) != row['files'][row['image']]:
                raise ValueError('Released PNG hash differs from the frozen file index')
            array = matrix(path)
            if [array.shape[1], array.shape[0]] != row['size']:
                raise ValueError('Released PNG dimensions differ')
            if not (dataset / row['sensor']).is_file() or not (dataset / row['annotation']).is_file():
                raise ValueError('Released sensor/annotation pair missing')
            if row['reconstructed_intensity'] and not np.any(array > 0):
                raise ValueError('Reconstructed main image has no intensity texture')
            check['state'] = 'passed'
        except Exception as exc:
            check.update(state='failed', error=str(exc))
        artifact_checks.append(check)
    save_rows(output / 'released_frame_checks.jsonl', artifact_checks)
    print(json.dumps({'stage': 'archives', 'released_frames_checked': len(artifact_checks)}), flush=True)

    archives = sorted(p for p in files if _type(p))

    def archive_check(path):
        rel = str(path.relative_to(root))
        meta = {'archive': rel, 'type': _type(path), 'size': files[path].st_size,
                'mtime_ns': files[path].st_mtime_ns, 'member_count': 0, 'intensity_members': 0}
        records = []

        def member(name, size, stream):
            entry = {'archive': rel, 'member': name, 'size': size}
            h = hashlib.sha256()
            with stream:
                for chunk in iter(lambda: stream.read(1024 * 1024), b''):
                    h.update(chunk)
            entry['sha256'] = h.hexdigest()
            candidates, issue = _candidates(root, path, name, size, files)
            matches = [p for p in candidates if raster_hashes.get(p) == entry['sha256']]
            method = 'relative_path_and_current_sha256'
            if not matches and not issue:
                # Renamed scene directories cannot be resolved by path alone.
                # Exact name + bytes proves image-content coverage; it does not
                # establish a new independent capture from the raster alone.
                matches = filename_hash_paths.get((Path(name).name, entry['sha256']), [])
                method = 'same_filename_and_current_sha256'
            entry['matches'] = [str(p.relative_to(root)) for p in matches]
            entry['match_method'] = method
            covered = [disk_by_path[n] for n in entry['matches'] if disk_by_path[n].get('coverage_verified')]
            entry.update(status='covered_disk_duplicate' if covered else 'unresolved',
                         frame_ids=sorted({row['frame_id'] for row in covered}),
                         reason=issue or ('current_member_sha256_matches_covered_disk_raster' if covered
                                          else 'no_identity_confirmed_covered_disk_match'))
            records.append(entry)
            meta['intensity_members'] += 1

        try:
            meta['sha256'] = digest(path)
            if meta['type'] == 'tar.gz':
                with tarfile.open(path, 'r|gz') as archive:
                    for info in archive:
                        if info.isfile():
                            meta['member_count'] += 1
                            if intensity_name(info.name):
                                member(info.name, info.size, archive.extractfile(info))
                        elif intensity_name(info.name):
                            records.append({'archive': rel, 'member': info.name,
                                            'status': 'unresolved_nonregular_intensity_member',
                                            'reason': 'Intensity symlink/hardlink needs explicit lineage'})
                            meta['intensity_members'] += 1
            elif meta['type'] == 'zip':
                with zipfile.ZipFile(path) as archive:
                    for info in archive.infolist():
                        if not info.is_dir():
                            meta['member_count'] += 1
                            if intensity_name(info.filename):
                                member(info.filename, info.file_size, archive.open(info))
            elif meta['type'] == 'rar':
                import rarfile
                rarfile.UNRAR_TOOL = str(Path('outputs/inventory_v3/tooling/usr/bin/unrar-nonfree').resolve())
                with rarfile.RarFile(path) as archive:
                    for info in archive.infolist():
                        if not info.isdir():
                            meta['member_count'] += 1
                            if intensity_name(info.filename):
                                member(info.filename, info.file_size, archive.open(info))
            else:
                raise ValueError('Unsupported archive; cannot claim complete member inspection')
            after = path.stat()
            if (after.st_size, after.st_mtime_ns) != (files[path].st_size, files[path].st_mtime_ns):
                raise ValueError('Archive changed while auditing')
            meta['state'] = 'passed'
        except Exception as exc:
            meta.update(state='failed', error=str(exc))
        return meta, records

    archive_meta, archive_members = [], []
    with ThreadPoolExecutor(max_workers=workers) as executor:
        futures = [executor.submit(archive_check, p) for p in archives]
        for i, future in enumerate(as_completed(futures), 1):
            meta, records = future.result()
            archive_meta.append(meta)
            archive_members.extend(records)
            print(json.dumps({'archives_checked': i, 'total': len(archives),
                              'archive': meta['archive'], 'state': meta['state'],
                              'intensity_members': meta['intensity_members']}), flush=True)
    archive_meta.sort(key=lambda x: x['archive'])
    archive_members.sort(key=lambda x: (x['archive'], x['member']))
    save_json(output / 'archive_checks.json', archive_meta)
    save_rows(output / 'archive_intensity_members.jsonl', archive_members)
    with (output / 'source_to_dataset.csv').open('w') as target:
        keys = ['path', 'suffix', 'status', 'frame_id', 'image', 'split', 'domain', 'sha256',
                'dimensions', 'all_zero', 'byte_identical_to_canonical_intensity', 'error']
        writer = csv.DictWriter(target, fieldnames=keys, extrasaction='ignore')
        writer.writeheader()
        writer.writerows(disk)
    problems = [row for row in disk if not row.get('coverage_verified')]
    problems += [row for row in archive_members if row['status'] != 'covered_disk_duplicate']
    problems += [row for row in archive_meta if row['state'] != 'passed']
    problems += [row for row in artifact_checks if row['state'] != 'passed']
    changed_inventory = {p: (s.st_size, s.st_mtime_ns) for p, s in files.items()} != {
        p: (s.st_size, s.st_mtime_ns) for p in root.rglob('*') if p.is_file() for s in [p.stat()]}
    counts = defaultdict(Counter)
    for row in disk:
        counts[row['suffix']][row['status']] += 1
    summary = {
        'state': 'passed' if not problems and not changed_inventory else 'requires_attention',
        'checked_at': datetime.now(ZoneInfo('Asia/Shanghai')).isoformat(),
        'source_root': str(root), 'formal_dataset': str(dataset),
        'source_files': len(files), 'directory_changed_during_audit': changed_inventory,
        'disk_intensity_rasters': len(disk), 'disk_by_suffix_and_status': dict(counts),
        'disk_status_counts': dict(Counter(row['status'] for row in disk)),
        'distinct_represented_formal_frames': len(selected), 'formal_total_frames': len(formal),
        'other_formal_frames_without_dedicated_source_intensity_raster': len(formal) - len(selected),
        'duplicate_alias_byte_equal': sum(row.get('byte_identical_to_canonical_intensity') is True for row in disk),
        'duplicate_alias_without_canonical_raster': sum(
            row['status'] == 'duplicate_alias' and row.get('canonical_intensity_path') is None for row in disk),
        'archives_checked': len(archive_meta), 'archive_types': dict(Counter(row['type'] for row in archive_meta)),
        'archive_member_count': sum(row['member_count'] for row in archive_meta),
        'archived_intensity_members': len(archive_members),
        'archived_intensity_status_counts': dict(Counter(row['status'] for row in archive_members)),
        'archived_intensity_match_methods': dict(Counter(row.get('match_method') for row in archive_members)),
        'missing_or_failed_items': len(problems), 'problems': problems,
        'released_png_hash_dimensions_and_pair_checks': dict(Counter(row['state'] for row in artifact_checks)),
        'input_binding': {str(dataset / n): digest(dataset / n) for n in
                          ['manifest.json', 'frames.jsonl', 'protocol/manifest.jsonl', 'protocol/excluded.json']},
        'audit_script_sha256': digest(__file__),
        'coverage_definition': 'All source intensity raster files are associated with released capture identities; '
                               'display PNGs are regenerated from preferred native intensity, not byte copies of raw rasters.',
        'archive_check_scope': 'Current archive directories enumerated; every intensity member content hashed; '
                               'non-intensity archive payloads are outside this audit.',
    }
    save_json(output / 'summary.json', summary)
    print(json.dumps({k: v for k, v in summary.items() if k not in ('problems', 'input_binding')},
                     ensure_ascii=False), flush=True)
    return summary


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--root', default='/root/private/data/pallet')
    parser.add_argument('--dataset', default='outputs/pallet_dataset_formal_v1/dataset')
    parser.add_argument('--output', default='outputs/intensity_coverage_audit_20261012_v1')
    parser.add_argument('--workers', type=int, default=4)
    args = parser.parse_args()
    result = audit(args.root, args.dataset, args.output, args.workers)
    raise SystemExit(0 if result['state'] == 'passed' else 1)
