"""Keep frozen train-CV best refits; prune v4 training checkpoints only.

Never changes the deployed bundle, delivery ZIPs, selection, histories or labels.
Removed optimizer/RNG checkpoints are NOT recoverable from the model-only ZIP.
"""
from __future__ import annotations

import argparse
from collections import Counter
from contextlib import contextmanager
from datetime import datetime, timezone
import fcntl
import hashlib
import importlib.util
import json
import os
from pathlib import Path
import re
import shutil
import sys

PROJECT = Path(__file__).resolve().parents[1]
ROOT_NAME = 'outputs/pallet_pose_benchmark_v4'
OUTPUT_NAME = 'outputs/pallet_pose_v4_checkpoint_cleanup_v1'
CONFIG_NAME = 'configs/pallet_pose_benchmark_v4.json'
BRANCHES = ('metric', 'native_unknown')
FROZEN_ENCODERS = {'utonia', 'sonata', 'concerto', 'litept', 'pointmae'}


def require(value, message):
    if not value:
        raise ValueError(message)


def canonical(value):
    return json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False).encode()


def digest(path):
    with Path(path).open('rb') as stream:
        return hashlib.file_digest(stream, 'sha256').hexdigest()


def read(path):
    return json.loads(Path(path).read_text())


def write(path, value):
    with Path(path).open('xb') as stream:
        stream.write(canonical(value)+b'\n')


def relative(path):
    return str(Path(path).relative_to(PROJECT))


def safe_path(value):
    value = Path(value)
    require(not value.is_absolute() and '..' not in value.parts, 'Unsafe path')
    path = PROJECT/value
    for parent in (path, *path.parents):
        if parent == PROJECT:
            break
        require(not parent.is_symlink(), 'Symlink paths are not permitted')
    return path


def identity(path):
    st = path.stat()
    return dict(bytes=st.st_size, allocated_bytes=st.st_blocks*512, device=st.st_dev,
        inode=st.st_ino, links=st.st_nlink, mtime_ns=st.st_mtime_ns)


def helpers():
    path = Path(__file__).resolve().with_name('cleanup_legacy_review_intermediates.py')
    spec = importlib.util.spec_from_file_location('v4_cleanup_guards', path)
    module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module)
    module.PROJECT = PROJECT
    return module


def eligible(path, families, arms):
    """Only explicit training .pt layouts, never deployment/ZIP/data/metadata."""
    try:
        parts = Path(path).relative_to(ROOT_NAME).parts
    except ValueError:
        return False
    if not parts:
        return False
    if parts[0] == 'trials' and len(parts) == 7:
        _, arm, family, branch, seed, fold, name = parts
        return (arm in arms and family in families and branch in BRANCHES and seed.isdigit()
            and bool(re.fullmatch(r'fold_\d+', fold)) and name in ('last.pt', 'selected.pt'))
    if parts[0] == 'refit' and len(parts) in (5, 6):
        _, family, branch, arm, *tail = parts
        if family not in families or branch not in BRANCHES or arm not in arms:
            return False
        return ((len(tail) == 1 and tail[0] in ('last.pt', 'selected.pt'))
            or (len(tail) == 2 and tail[0] == 'epochs' and bool(re.fullmatch(r'[1-9]\d*\.pt', tail[1]))))
    return False


def best_entries(selection, families):
    require(selection['val_test_used'] is False, 'Cannot select from val/test')
    require(set(selection['profiles']) == set(BRANCHES), 'Keep distinct metric and native-unknown models')
    result = []
    for branch in BRANCHES:
        entries = selection['profiles'][branch]['entries']
        require(set(e['family'] for e in entries) == set(families), 'Incomplete frozen family catalog')
        for family in families:
            group = [e for e in entries if e['family'] == family]
            require(len({e['arm'] for e in group}) == len(group), 'Duplicate recipe')
            # Stable frozen entry order breaks exact CV-score ties; no holdout,
            # no seed/fold cherry-picking, and no new numerical evaluation.
            chosen = min(group, key=lambda e: e['mean_score'])
            result.append(dict(family=family, branch=branch, arm=chosen['arm'],
                frozen_train_cv_mean_score=chosen['mean_score'], refit_epoch=chosen['refit_epoch']))
    return result


def validate_weight(path, family, epoch):
    import torch
    state = torch.load(path, map_location='cpu', weights_only=True)
    require(state['family'] == family and state['selected_epoch'] == epoch, 'Best checkpoint identity mismatch')
    require(state['frozen_encoder'] == (family in FROZEN_ENCODERS), 'Backbone/head semantics mismatch')
    require(state.get('tune') is False and not {'optimizer', 'numpy_rng', 'cuda_rng'} & set(state), 'Expected model-only weight')
    model = state['model']
    require(isinstance(model, dict) and model and all(isinstance(v, torch.Tensor) for v in model.values()), 'Invalid state_dict')
    require(all(torch.isfinite(t).all().item() for t in model.values()), 'Nonfinite best model weight')
    return dict(cpu_weights_only_load_verified=True, tensor_count=len(model),
        tensor_elements=sum(t.numel() for t in model.values()), frozen_encoder=state['frozen_encoder'],
        contains_optimizer_state=False, numerical_inference_or_training_performed=False)


def retained_models(root, config, selection, bundle):
    result = []
    for best in best_entries(selection, config['families']):
        family, branch, arm = (best[k] for k in ('family', 'branch', 'arm'))
        records = read(root/'refit'/family/branch/'profiles.json')
        actual = [r for r in records if r['arm'] == arm]
        require(len(actual) == 1, 'Missing exact frozen refit profile')
        actual = actual[0]
        require(actual['refit_epoch'] == best['refit_epoch'] and actual['mean_score'] == best['frozen_train_cv_mean_score'], 'Frozen refit choice changed')
        path = safe_path(ROOT_NAME+'/'+actual['checkpoint'])
        require(eligible(relative(path), config['families'], {'base', *[r['name'] for r in config['continuation']]}), 'Unexpected best-weight path')
        expected = actual['checkpoint_sha256']
        require(digest(path) == expected, 'Best refit bytes changed')
        deployed = bundle['model_catalog'][family][branch][arm]
        deployed_path = root/'deployment'/deployed['checkpoint']
        require(bundle['files'][deployed['checkpoint']] == expected and digest(deployed_path) == expected, 'Best refit and delivered model disagree')
        require(deployed['refit_epoch'] == best['refit_epoch'], 'Deployed refit epoch mismatch')
        spec_path = path.parent.parent/'specification.json'
        specification = read(spec_path)
        require(not specification['validation_keys'] and specification['family'] == family
            and specification['branch'] == branch and specification['arm'] == arm, 'Not an all-train refit')
        require(specification['protocol_sha256'] == digest(root/'protocol.json'), 'Foreign training protocol')
        loaded = validate_weight(path, family, best['refit_epoch'])
        result.append(dict(**best, checkpoint=relative(path), checkpoint_sha256=expected,
            delivered_checkpoint=relative(deployed_path), train_samples=actual['train_samples'],
            train_groups=actual['train_groups'], training_domains=actual['training_domains'],
            specification=relative(spec_path), specification_sha256=digest(spec_path), **loaded))
    return result


def protections(root, targets):
    guard = helpers()
    return dict(v4_retained_files_sha256={relative(p): digest(p) for p in root.rglob('*')
            if p.is_file() and not p.is_symlink() and relative(p) not in targets},
        unrelated=guard.protection([], set()))


@contextmanager
def pipeline_lock(root):
    lock = root/'pipeline.lock'
    require(lock.is_file() and not lock.is_symlink(), 'Expected pre-existing v4 pipeline lock')
    with lock.open('rb') as stream:
        try:
            fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
        except BlockingIOError as exc:
            raise ValueError('V4 pipeline is active; do not prune training checkpoints') from exc
        yield


def confirm_out(out):
    out = Path(out).absolute()
    require(out.parent == PROJECT/'outputs' and not out.is_symlink(), 'Fresh top-level outputs audit required')
    return out


def plan(out):
    out = confirm_out(out); require(not out.exists(), 'Do not overwrite existing cleanup evidence')
    root = safe_path(ROOT_NAME); config = read(PROJECT/CONFIG_NAME)
    selection = read(root/'selection_frozen.json'); bundle = read(root/'deployment/bundle_config.json')
    require(bundle['schema'] == 'pose_benchmark_v4', 'Foreign deployment bundle')
    arms = {'base', *[r['name'] for r in config['continuation']]}
    with pipeline_lock(root):
        models = retained_models(root, config, selection, bundle)
        keep = {m['checkpoint'] for m in models}
        require(len(keep) == len(config['families'])*len(BRANCHES), 'Keep one best refit per model and unit branch')
        opened = helpers().open_paths(); candidates = []
        for tree in ('trials', 'refit'):
            for path in sorted((root/tree).rglob('*.pt')):
                key = relative(path); safe_path(key)
                require(eligible(key, config['families'], arms), 'Unexpected training weight; manual review required: '+key)
                if key in keep:
                    continue
                require(str(path) not in opened and path.stat().st_nlink == 1, 'Open or hardlinked checkpoint: '+key)
                folder = path.parent.parent if path.parent.name == 'epochs' else path.parent
                completed_path = folder/'completed.json'; completed = read(completed_path)
                specification = read(folder/'specification.json')
                require(completed['last_epoch'] == specification['total_epochs'], 'Incomplete training run')
                if path.name in ('last.pt', 'selected.pt'):
                    expected = completed['last_sha256' if path.name == 'last.pt' else 'selected_sha256']
                else:
                    match = [r for r in read(folder.parent/'profiles.json') if r['checkpoint'] == str(path.relative_to(root))]
                    require(len(match) == 1, 'Unbound epoch weight')
                    expected = match[0]['checkpoint_sha256']
                require(digest(path) == expected, 'Historical checkpoint hash mismatch: '+key)
                candidates.append(dict(path=key, sha256=expected, stat=identity(path),
                    category=path.name if path.name in ('last.pt', 'selected.pt') else 'nonwinning_refit_epoch.pt',
                    completed_record=relative(completed_path), completed_record_sha256=digest(completed_path),
                    recovery='not_retained' if path.name in ('last.pt', 'selected.pt') else 'original_model_delivery_zip'))
                if len(candidates) % 50 == 0:
                    print(json.dumps(dict(stage='hash_training_checkpoints', verified=len(candidates))), flush=True)
        targets = {c['path'] for c in candidates}
        require(not targets & keep, 'Selected best weight cannot be a deletion target')
        guard = protections(root, targets)
        for name, expected in bundle['files'].items():
            require(guard['v4_retained_files_sha256'].get(ROOT_NAME+'/deployment/'+name) == expected, 'Published bundle is incomplete or changed')
        delivery = read(root/'delivery_manifest.json')
        for entry in (delivery['models'], delivery['annotations']):
            require(guard['v4_retained_files_sha256'].get(entry['archive']) == entry['sha256'], 'Delivery ZIP changed')
        result = dict(schema='pose_v4_best_refit_checkpoint_compaction_v1', state='planned_not_deleted',
            user_authorization='v4的last.pt仅保存每个模型的最佳权重用于后训练，其余权重释放',
            created_at=datetime.now(timezone.utc).isoformat(), script_sha256=digest(__file__),
            protection_helper_sha256=digest(Path(__file__).resolve().with_name('cleanup_legacy_review_intermediates.py')),
            config_sha256=digest(PROJECT/CONFIG_NAME), selection_sha256=digest(root/'selection_frozen.json'),
            families=config['families'], arms=sorted(arms), retained_best_models=models,
            published_deployment_kept_intact=True, all_delivery_archives_kept_intact=True,
            candidates=candidates, counts=dict(Counter(c['category'] for c in candidates)),
            allocated_bytes_to_remove=sum(c['stat']['allocated_bytes'] for c in candidates),
            free_bytes_before=shutil.disk_usage(PROJECT/'outputs').free,
            selection_scope='frozen original_train_CV; stable frozen-order tie break; no val/test reselection',
            exact_optimizer_rng_resume_will_be_lost=True,
            old_full_trial_checkpoint_audit_will_no_longer_pass=True,
            new_post_training_run_requires_fresh_output_and_new_optimizer=True,
            training_or_inference_launched=False)
        out.mkdir(mode=0o700); write(out/'protection_before.json', guard)
        result['protection_before_sha256'] = digest(out/'protection_before.json')
        write(out/'plan.json', result)
        write(out/'best_weights_index.json', dict(schema='v4_best_weights_for_future_finetuning_v1',
            models=models, source_selection_sha256=result['selection_sha256'],
            backbone_weights_directory=ROOT_NAME+'/deployment/weights',
            upstream_source_directory=ROOT_NAME+'/deployment/upstream',
            architecture_source=ROOT_NAME+'/deployment/pallet/pose_benchmark_models.py',
            full_deployment_config=ROOT_NAME+'/deployment/bundle_config.json',
            preserve_unit_branch_separation=True, fresh_optimizer_required=True,
            training_started=False, automatic_training_authorization=False))
        print(json.dumps(dict(state=result['state'], plan_sha256=digest(out/'plan.json'),
            keep_best_models=len(models), deletions=result['counts'], reclaim_gib=result['allocated_bytes_to_remove']/2**30)), flush=True)
        return result


def apply(out, expected_sha256):
    out = confirm_out(out)
    require(digest(out/'plan.json') == expected_sha256, 'Frozen plan hash mismatch')
    value = read(out/'plan.json')
    require(value['schema'] == 'pose_v4_best_refit_checkpoint_compaction_v1', 'Unknown cleanup plan')
    require(value['script_sha256'] == digest(__file__), 'Pruner changed since planning')
    require(value['protection_helper_sha256'] == digest(Path(__file__).resolve().with_name('cleanup_legacy_review_intermediates.py')), 'Protection helper changed')
    require(digest(out/'protection_before.json') == value['protection_before_sha256'], 'Protected evidence changed')
    require(not (out/'deletion_journal.jsonl').exists() and not (out/'result.json').exists(), 'Cleanup already started')
    root = safe_path(ROOT_NAME); targets = {c['path'] for c in value['candidates']}
    keep = {m['checkpoint'] for m in value['retained_best_models']}
    require(len(targets) == len(value['candidates']) and not targets & keep, 'Unsafe or repeated target')
    with pipeline_lock(root):
        before = read(out/'protection_before.json')
        require(protections(root, targets) == before, 'Protected sources changed since planning')
        require(digest(PROJECT/CONFIG_NAME) == value['config_sha256'] and digest(root/'selection_frozen.json') == value['selection_sha256'], 'Frozen training choice changed')
        config = read(PROJECT/CONFIG_NAME)
        expected_models = retained_models(root, config, read(root/'selection_frozen.json'), read(root/'deployment/bundle_config.json'))
        require(expected_models == value['retained_best_models'], 'Best weight catalog changed')
        opened = helpers().open_paths()
        for candidate in value['candidates']:
            require(eligible(candidate['path'], value['families'], value['arms']), 'Deletion outside training scope')
            path = safe_path(candidate['path'])
            require(str(path) not in opened and identity(path) == candidate['stat'], 'Training checkpoint opened or changed')
            require(digest(path) == candidate['sha256'], 'Checkpoint bytes changed since planning')
        write(out/'preflight_passed.json', dict(state='passed', plan_sha256=expected_sha256,
            best_weights_verified=len(keep), delete_targets_verified=len(targets)))
        removed = 0; free_before = shutil.disk_usage(PROJECT/'outputs').free
        try:
            with (out/'deletion_journal.jsonl').open('xb') as journal:
                for candidate in value['candidates']:
                    path = safe_path(candidate['path'])
                    require(identity(path) == candidate['stat'], 'Checkpoint changed immediately before unlink')
                    journal.write(canonical(dict(state='intent', path=candidate['path'], sha256=candidate['sha256']))+b'\n'); journal.flush()
                    path.unlink(); removed += 1
                    journal.write(canonical(dict(state='deleted', path=candidate['path']))+b'\n')
                    if removed % 25 == 0:
                        journal.flush(); os.fsync(journal.fileno())
                        print(json.dumps(dict(stage='delete_training_checkpoints', completed=removed, total=len(targets))), flush=True)
                journal.flush(); os.fsync(journal.fileno())
            require(protections(root, targets) == before, 'Retained artifacts changed during cleanup')
            require(all(not safe_path(c['path']).exists() for c in value['candidates']), 'Deletion incomplete')
            require({relative(p) for tree in ('trials', 'refit') for p in (root/tree).rglob('*.pt')} == keep, 'Unexpected remaining training weight')
            for model in value['retained_best_models']:
                path = safe_path(model['checkpoint'])
                require(digest(path) == model['checkpoint_sha256'], 'Best weight missing or changed')
                validate_weight(path, model['family'], model['refit_epoch'])
            result = dict(state='passed', plan_sha256=expected_sha256, deleted_files=removed,
                deletion_counts=value['counts'], retained_best_training_weights=len(keep),
                removed_allocated_bytes=value['allocated_bytes_to_remove'],
                measured_filesystem_free_increase_bytes=shutil.disk_usage(PROJECT/'outputs').free-free_before,
                free_bytes_after=shutil.disk_usage(PROJECT/'outputs').free,
                all_retained_v4_artifacts_content_unchanged=True, deployment_and_archives_byte_unchanged=True,
                configurations_and_human_stores_unchanged=True,
                formal_stat_files_verified=len(before['unrelated']['formal_file_stat_identity']),
                formal_integrity_scope='all-file namespace/stat identity; not a new full-byte audit',
                exact_optimizer_rng_resume_available=False, old_full_checkpoint_audit_rerunnable=False,
                training_started=False, inference_rerun=False, production_annotation_writes=0,
                completed_at=datetime.now(timezone.utc).isoformat())
            write(out/'result.json', result)
            print(json.dumps(result), flush=True)
            return result
        except BaseException as exc:
            write(out/'failure.json', dict(state='failed_or_interrupted', deleted_files=removed, error=str(exc),
                exact_optimizer_rng_restore_unavailable=True))
            raise


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('action', choices=('plan', 'apply'))
    parser.add_argument('--directory', type=Path, default=PROJECT/OUTPUT_NAME)
    parser.add_argument('--plan-sha256')
    args = parser.parse_args()
    if args.action == 'plan':
        plan(args.directory)
    else:
        require(args.plan_sha256 is not None, 'Apply requires exact reviewed plan hash')
        apply(args.directory, args.plan_sha256)
