"""Audited removal of redundant legacy review caches and reproducible previews.

Only old review cache/*.npz and views/<frame hash>/{depth,intensity,validity}.png
are eligible. Human records, task bindings, live stores and delivery artifacts
are never removed. Every deleted byte has an exact-copy or verified regeneration
recipe; restore never overwrites an existing path.
"""
from __future__ import annotations

import argparse
from collections import Counter
from contextlib import closing
from datetime import datetime, timezone
import hashlib
import io
import json
import os
from pathlib import Path
import re
import shutil
import sqlite3
import subprocess
import sys

import numpy as np
import PIL
from PIL import Image

PROJECT = Path(__file__).resolve().parents[1]
OUTPUT = PROJECT/'outputs/pallet_review_intermediate_cleanup_v1'
SPECIAL_ROOTS = {'review_v3', 'review_paired_v5', 'review_paired_missing_v5', 'conflict_review_train_v5'}
VIEW_NAMES = {'intensity.png', 'depth.png', 'validity.png'}
STORES = (
    'outputs/pallet_pose_human_review_v1/annotations.sqlite3',
    'outputs/pallet_mask_gallery_reviews_v1/mask_reviews.sqlite3',
    'outputs/pallet_structure_human_review_v1/reviews.sqlite3',
)


def require(value, message):
    if not value:
        raise ValueError(message)


def canonical(value):
    return json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False).encode()


def sha(value):
    return hashlib.sha256(canonical(value)).hexdigest()


def digest(path):
    with Path(path).open('rb') as stream:
        return hashlib.file_digest(stream, 'sha256').hexdigest()


def write(path, value):
    with Path(path).open('xb') as stream:
        stream.write(canonical(value)+b'\n')


def read(path):
    return json.loads(Path(path).read_text())


def relative(path):
    return str(Path(path).relative_to(PROJECT))


def legacy_name(name):
    return (name in SPECIAL_ROOTS or bool(re.fullmatch(r'(?:audit_)?review_round_\d{3}(?:_small|_reblind)?', name))
        or bool(re.fullmatch(r'full_review_(?:train|val|test)_\d{3}_v5', name)))


def safe_path(value):
    value = Path(value)
    require(not value.is_absolute() and '..' not in value.parts, 'Unsafe relative path')
    path = PROJECT/value
    for parent in (path, *path.parents):
        if parent == PROJECT:
            break
        require(not parent.is_symlink(), 'Symlinks are never cleanup targets or sources')
    return path


def eligible_target(value):
    parts = Path(value).parts
    if len(parts) < 4 or parts[0] != 'outputs' or not legacy_name(parts[1]):
        return False
    return ((len(parts) == 4 and parts[2] == 'cache' and parts[3].endswith('.npz'))
        or (len(parts) == 5 and parts[2] == 'views' and re.fullmatch(r'[a-f0-9]{20}', parts[3])
            and parts[4] in VIEW_NAMES))


def stat_value(path):
    st = path.stat()
    return dict(bytes=st.st_size, allocated_bytes=st.st_blocks*512, device=st.st_dev,
        inode=st.st_ino, links=st.st_nlink, mtime_ns=st.st_mtime_ns, mode=st.st_mode & 0o777)


def database_fingerprint(path):
    with closing(sqlite3.connect(path.resolve().as_uri()+'?mode=ro', uri=True)) as db:
        db.execute('PRAGMA query_only=ON'); db.execute('BEGIN')
        schema = list(db.execute('SELECT type,name,tbl_name,sql FROM sqlite_master ORDER BY type,name'))
        tables = {}
        for kind, name, *_ in schema:
            if kind == 'table':
                require(name.replace('_', '').isalnum(), 'Unexpected SQL table name')
                tables[name] = sha(sorted(canonical(list(row)).decode() for row in db.execute('SELECT * FROM "'+name+'"')))
        return dict(schema_sha256=sha(schema), tables=tables)


def protection(roots, excluded):
    """Content hashes for every retained legacy file, configs and live human rows.

Formal protection checks namespace/stat identity, not a fresh full-byte rehash.
No mutating operation in this script can target the formal namespace.
"""
    kept = {}
    for root in roots:
        for path in root.rglob('*'):
            if path.is_file() and not path.is_symlink() and relative(path) not in excluded:
                kept[relative(path)] = digest(path)
    configs = {relative(p): digest(p) for p in (PROJECT/'configs').rglob('*') if p.is_file()}
    formal = {relative(p): stat_value(p) for p in (PROJECT/'outputs/pallet_dataset_formal_v1/dataset').rglob('*') if p.is_file()}
    stores = {name: database_fingerprint(PROJECT/name) for name in STORES}
    return dict(retained_legacy_files_sha256=kept, configurations_sha256=configs,
        formal_file_stat_identity=formal, human_database_logical_fingerprints=stores,
        formal_protection_scope='all-file namespace/stat identity; not a new full-byte audit')


def external_view_references():
    result = set()
    expression = re.compile(r'outputs/([^\s"\'<>]+/views/[a-f0-9]{20}/(?:intensity|depth|validity)\.png)')
    for name in ('configs', 'docs', 'pallet', 'scripts', 'tests'):
        for path in (PROJECT/name).rglob('*'):
            if path.is_file() and path.suffix in ('.py', '.json', '.jsonl', '.md', '.html', '.js', '.yaml', '.yml'):
                for match in expression.finditer(path.read_text(errors='replace')):
                    result.add('outputs/'+match.group(1))
    return result


def render_preview(frame, name):
    def display(values, valid, log=False, invert=False):
        valid = (valid > 0) & np.isfinite(values)
        if not valid.any():
            return np.zeros(values.shape, np.uint8)
        data = np.log1p(np.maximum(values, 0)) if log else values.copy()
        low, high = np.percentile(data[valid], [1, 99])
        scaled = np.clip((data-low)/max(high-low, 1e-8), 0, 1)
        if invert:
            scaled = 1-scaled
        return np.where(valid, scaled*255, 0).astype(np.uint8)
    if name == 'intensity.png':
        pixels = display(frame[..., 1], frame[..., 3], log=True)
    elif name == 'depth.png':
        pixels = display(frame[..., 0], frame[..., 2], invert=True)
    elif name == 'validity.png':
        pixels = (np.stack((frame[..., 2], frame[..., 3], np.zeros(frame.shape[:2])), axis=-1)*255).astype(np.uint8)
    else:
        raise ValueError('Not a derived review preview')
    stream = io.BytesIO(); Image.fromarray(pixels).save(stream, format='PNG')
    return stream.getvalue()


def open_paths():
    paths = set()
    for process in Path('/proc').glob('[0-9]*'):
        try:
            for fd in (process/'fd').iterdir():
                try:
                    target = fd.readlink()
                    if target.is_absolute():
                        paths.add(str(target))
                except (FileNotFoundError, PermissionError, OSError):
                    pass
        except (FileNotFoundError, PermissionError, ProcessLookupError):
            pass
    return paths


def usage():
    completed = subprocess.run(['du', '-x', '-B1', '--max-depth=1', str(PROJECT/'outputs')],
        check=True, capture_output=True, text=True)
    by_directory = {}
    total = 0
    for line in completed.stdout.splitlines():
        size, path = line.split('\t', 1); size = int(size)
        if Path(path) == PROJECT/'outputs':
            total = size
        else:
            by_directory[relative(Path(path))] = size
    groups = Counter()
    for path, size in by_directory.items():
        name = Path(path).name
        if legacy_name(name):
            groups['legacy_full_review' if name.startswith('full_review_') else 'legacy_round_and_paired_review'] += size
        elif 'validation' in name:
            groups['validation_evidence_retained'] += size
        elif name.startswith('recipe_') and 'smoke' in name:
            groups['smoke_experiments_retained'] += size
        elif name in ('pallet_pose_human_review_v1', 'pallet_mask_gallery_reviews_v1', 'pallet_structure_human_review_v1'):
            groups['live_human_stores_retained'] += size
    return dict(outputs_allocated_bytes=total, free_bytes=shutil.disk_usage(PROJECT/'outputs').free,
        groups={name: dict(allocated_bytes=size, outputs_fraction=size/total) for name, size in groups.items()},
        legacy_directories={name: size for name, size in by_directory.items() if legacy_name(Path(name).name)})


def plan(out=OUTPUT):
    out = Path(out).absolute()
    require(out.parent == PROJECT/'outputs' and not out.exists(), 'Fresh outputs audit directory required')
    roots = sorted(p for p in (PROJECT/'outputs').iterdir() if p.is_dir() and not p.is_symlink() and legacy_name(p.name))
    before_usage = usage(); referenced = external_view_references(); opened = open_paths()
    candidates, skipped = [], Counter(); hash_cache = {}
    def hashed(path):
        key = relative(path)
        if key not in hash_cache:
            hash_cache[key] = digest(path)
        return hash_cache[key]
    def add(path, method, source, **details):
        key = relative(path)
        require(eligible_target(key), 'Not a whitelisted legacy derived target')
        if str(path) in opened or key in referenced or path.stat().st_nlink != 1:
            skipped['open_externally_referenced_or_hardlinked'] += 1
            return
        candidates.append(dict(path=key, sha256=hashed(path), stat=stat_value(path), method=method,
            source=relative(source), source_sha256=hashed(source), **details))
    for root in roots:
        for path in sorted((root/'cache').glob('*.npz')):
            safe_path(relative(path))
            source = PROJECT/'outputs/cache_v5/frames'/path.name
            if not source.is_file() or source.is_symlink() or source.stat().st_size != path.stat().st_size:
                skipped['cache_without_same_size_master'] += 1
                continue
            if hashed(source) != hashed(path):
                skipped['cache_not_byte_identical_to_master'] += 1
                continue
            add(path, 'exact_master_cache_copy', source)
        print(json.dumps(dict(stage='cache_plan', directory=root.name, candidates=len(candidates))), flush=True)
    cache_replacements = {c['path']: c['source'] for c in candidates}
    for root in roots:
        index_path = root/'cache_index.json'
        if not index_path.exists():
            continue
        for frame_id, entry in read(index_path).items():
            path = Path(entry['path'] if isinstance(entry, dict) else entry)
            if not path.is_absolute():
                path = PROJECT/path
            try:
                key = relative(path)
                source = safe_path(cache_replacements.get(key, key))
            except ValueError:
                skipped['preview_source_outside_project'] += 1
                continue
            if not source.is_file():
                skipped['preview_source_missing'] += 1
                continue
            if not (source.parent == PROJECT/'outputs/cache_v5/frames' or source.parent == root/'cache'
                    or source.is_relative_to(PROJECT/'outputs/cache_v6_clean') or source.is_relative_to(PROJECT/'outputs/cache_v7_oriented')):
                skipped['preview_source_not_allowed_cache'] += 1
                continue
            view_root = root/'views'/hashlib.sha256(frame_id.encode()).hexdigest()[:20]
            views = [view_root/name for name in sorted(VIEW_NAMES) if (view_root/name).is_file()]
            if not views:
                continue
            with np.load(source, allow_pickle=False) as value:
                frame = value['frame']
            for view in views:
                safe_path(relative(view))
                expected = hashlib.sha256(render_preview(frame, view.name)).hexdigest()
                if hashed(view) != expected:
                    skipped['preview_not_exactly_regenerable'] += 1
                    continue
                add(view, 'verified_native_preview_regeneration', source, preview_name=view.name,
                    frame_id_sha256=hashlib.sha256(frame_id.encode()).hexdigest())
        print(json.dumps(dict(stage='preview_plan', directory=root.name, candidates=len(candidates))), flush=True)
    deleted_paths = {c['path'] for c in candidates}
    require(all(c['source'] not in deleted_paths for c in candidates), 'Restoration source is a deletion target')
    protected = protection(roots, deleted_paths)
    result = dict(schema='legacy_review_intermediate_cleanup_v1', state='planned_not_deleted',
        created_at=datetime.now(timezone.utc).isoformat(), script_sha256=digest(__file__),
        pillow_version=PIL.__version__, numpy_version=np.__version__, usage_before=before_usage,
        legacy_roots=[relative(p) for p in roots], candidates=candidates, skipped=dict(skipped),
        deletion_counts=dict(Counter(c['method'] for c in candidates)),
        deletion_allocated_bytes=sum(c['stat']['allocated_bytes'] for c in candidates),
        no_human_annotation_or_task_deletion=True, no_active_configuration_changes=True,
        no_checkpoint_or_delivery_archive_deletion=True,
        legacy_UI_requires_restore_for_removed_views_or_local_caches=True)
    out.mkdir(mode=0o700)
    write(out/'protection_before.json', protected); result['protection_before_sha256'] = digest(out/'protection_before.json')
    write(out/'plan.json', result)
    print(json.dumps(dict(state=result['state'], plan_sha256=digest(out/'plan.json'),
        counts=result['deletion_counts'], reclaim_gib=result['deletion_allocated_bytes']/2**30,
        skipped=result['skipped'])), flush=True)
    return result


def load_plan(out, expected_sha256=None):
    out = Path(out).absolute()
    require(out.parent == PROJECT/'outputs' and not out.is_symlink(), 'Use outputs audit directory')
    if expected_sha256 is not None:
        require(digest(out/'plan.json') == expected_sha256, 'Plan hash mismatch')
    value = read(out/'plan.json')
    require(value['schema'] == 'legacy_review_intermediate_cleanup_v1', 'Unknown cleanup schema')
    require(value['script_sha256'] == digest(__file__), 'Cleanup/restore code changed')
    require(value['pillow_version'] == PIL.__version__ and value['numpy_version'] == np.__version__, 'Renderer environment changed')
    require(digest(out/'protection_before.json') == value['protection_before_sha256'], 'Protected evidence changed')
    targets = {c['path'] for c in value['candidates']}
    require(len(targets) == len(value['candidates']), 'Duplicate cleanup target')
    for candidate in value['candidates']:
        require(eligible_target(candidate['path']), 'Deletion outside allowlist')
        safe_path(candidate['path']); safe_path(candidate['source'])
        require(candidate['source'] not in targets, 'Restoration source may not be deleted')
    return out, value


def apply(out, expected_sha256):
    out, value = load_plan(out, expected_sha256)
    require(not (out/'deletion_journal.jsonl').exists() and not (out/'result.json').exists(), 'Cleanup already started; inspect journal or restore')
    roots = [safe_path(p) for p in value['legacy_roots']]
    targets = {c['path'] for c in value['candidates']}
    require(protection(roots, targets) == read(out/'protection_before.json'), 'Retained sources changed since planning')
    opened = open_paths(); sources = {}
    for candidate in value['candidates']:
        path = safe_path(candidate['path']); source = safe_path(candidate['source'])
        require(str(path) not in opened, 'Target is open in another process')
        require(stat_value(path) == candidate['stat'] and digest(path) == candidate['sha256'], 'Target changed since planning')
        if candidate['source'] not in sources:
            sources[candidate['source']] = digest(source)
        require(sources[candidate['source']] == candidate['source_sha256'], 'Restoration source changed')
    write(out/'preflight_passed.json', dict(state='passed', plan_sha256=expected_sha256,
        targets_verified=len(targets), distinct_restoration_sources_verified=len(sources)))
    removed = []; free_before = shutil.disk_usage(PROJECT/'outputs').free
    try:
        with (out/'deletion_journal.jsonl').open('xb') as journal:
            for candidate in value['candidates']:
                path = safe_path(candidate['path'])
                require(stat_value(path) == candidate['stat'], 'Target changed immediately before deletion')
                journal.write(canonical(dict(state='intent', path=candidate['path'], sha256=candidate['sha256']))+b'\n')
                journal.flush()
                path.unlink()
                removed.append(candidate['path'])
                journal.write(canonical(dict(state='deleted', path=candidate['path']))+b'\n')
                if len(removed) % 1000 == 0:
                    journal.flush(); os.fsync(journal.fileno())
                    print(json.dumps(dict(stage='delete', completed=len(removed), total=len(targets))), flush=True)
            journal.flush(); os.fsync(journal.fileno())
        require(protection(roots, targets) == read(out/'protection_before.json'), 'Retained files or human stores changed during cleanup')
        require(all(not safe_path(path).exists() for path in removed), 'Deletion incomplete')
        for source, source_sha in sources.items():
            require(digest(safe_path(source)) == source_sha, 'Retained restoration source changed during cleanup')
        after = usage()
        result = dict(state='passed', plan_sha256=expected_sha256, deleted_files=len(removed),
            deletion_counts=value['deletion_counts'], removed_allocated_bytes=value['deletion_allocated_bytes'],
            measured_filesystem_free_increase_bytes=shutil.disk_usage(PROJECT/'outputs').free-free_before,
            usage_after=after, retained_legacy_files_content_verified=len(read(out/'protection_before.json')['retained_legacy_files_sha256']),
            formal_files_stat_identity_verified=len(read(out/'protection_before.json')['formal_file_stat_identity']),
            all_human_stores_logically_unchanged=True, all_configurations_unchanged=True,
            restoration_sources_byte_verified=len(sources), production_annotation_writes=0,
            checkpoint_and_delivery_deletions=0, source_image_and_PCD_deletions=0,
            legacy_UI_requires_restore=True, completed_at=datetime.now(timezone.utc).isoformat())
        write(out/'result.json', result)
        print(json.dumps({k: result[k] for k in ('state', 'deleted_files', 'deletion_counts', 'removed_allocated_bytes', 'measured_filesystem_free_increase_bytes')}), flush=True)
        return result
    except BaseException as exc:
        write(out/'failure.json', dict(state='interrupted_or_failed', deleted_files=len(removed), error=str(exc),
            restore_available_from_original_plan=True))
        raise


def restore(out):
    out, value = load_plan(out)
    require(not (out/'restoration.json').exists(), 'Restoration already recorded')
    restored = 0
    for candidate in value['candidates']:
        path = safe_path(candidate['path']); source = safe_path(candidate['source'])
        if path.exists():
            require(digest(path) == candidate['sha256'], 'Refuse to overwrite an existing different file')
            continue
        require(digest(source) == candidate['source_sha256'], 'Restoration source changed')
        if candidate['method'] == 'exact_master_cache_copy':
            require(candidate['source_sha256'] == candidate['sha256'], 'Not an exact-copy recovery recipe')
            with source.open('rb') as src, path.open('xb') as dst:
                shutil.copyfileobj(src, dst)
        elif candidate['method'] == 'verified_native_preview_regeneration':
            with np.load(source, allow_pickle=False) as arrays:
                payload = render_preview(arrays['frame'], candidate['preview_name'])
            require(hashlib.sha256(payload).hexdigest() == candidate['sha256'], 'Regeneration did not reproduce original bytes')
            with path.open('xb') as stream:
                stream.write(payload)
        else:
            raise ValueError('Unknown recovery recipe')
        require(digest(path) == candidate['sha256'], 'Restored bytes differ')
        path.chmod(candidate['stat']['mode'])
        os.utime(path, ns=(candidate['stat']['mtime_ns'], candidate['stat']['mtime_ns']))
        restored += 1
    write(out/'restoration.json', dict(state='passed', restored_files=restored, all_original_hashes_verified=True))
    print(json.dumps(dict(state='passed', restored_files=restored)), flush=True)


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('action', choices=('plan', 'apply', 'restore'))
    parser.add_argument('--directory', type=Path, default=OUTPUT)
    parser.add_argument('--plan-sha256')
    args = parser.parse_args()
    if args.action == 'plan':
        plan(args.directory)
    elif args.action == 'apply':
        require(args.plan_sha256 is not None, 'Apply requires the exact reviewed plan hash')
        apply(args.directory, args.plan_sha256)
    else:
        restore(args.directory)
