#!/usr/bin/env python3
"""Snapshot current source and frozen portable data for eight independent runs."""
from __future__ import annotations

import argparse
from datetime import datetime
import importlib.metadata
import json
from pathlib import Path
import platform
import shutil
import subprocess
import sys
import tempfile
import urllib.request
import zipfile
from zoneinfo import ZoneInfo

try:
    from .verify_8x4090_handoff import digest, read, safe_path, validate_config, verify_handoff, verify_portable_dataset, unpack_dataset
except ImportError:
    from verify_8x4090_handoff import digest, read, safe_path, validate_config, verify_handoff, verify_portable_dataset, unpack_dataset


NAME = 'pallet_baselines_8x4090_handoff_v1'
SOURCE_DIRS = ('pallet', 'configs', 'docs', 'tests', 'scripts')
SOURCE_ROOT_FILES = ('README.md', 'pyproject.toml', '.gitignore')
SKIP_PARTS = {'.git', '.venv', '__pycache__', '.pytest_cache', '.mypy_cache', '.ruff_cache', 'node_modules'}
SOURCE_SUFFIXES = {'.py', '.json', '.jsonl', '.md', '.txt', '.toml', '.yaml', '.yml', '.sh', '.csv', '.patch'}
PROJECT_REQUIREMENTS = '''# Clean Linux/Python3.11 training environment; install torch/cu124 separately.
# Use PYTHONPATH source loading; do not install the historic headless project metadata.
torch==2.6.0+cu124
torchvision==0.21.0+cu124
numpy==1.26.4
ultralytics==8.4.163
rfdetr==1.11.0
torch-hungarian==0.1.0rc0
ultralytics-thop==2.1.6
opencv-python==4.11.0.86
scipy==1.15.3
pycocotools==2.0.11
pillow==11.0.0
onnx==1.18.0
onnxruntime==1.22.1
pytorch-lightning==2.6.6
torchmetrics[detection]==1.8.2
lightning-utilities==0.15.3
faster-coco-eval==1.8.0
hotcoco==1.0.1
ultrafast-pycocotools==0.1.11
vernier==0.5.4
simplejpeg==1.9.0
transformers==5.17.0
timm==1.0.30
supervision==0.30.5
av==14.2.0
matplotlib==3.10.7
pandas==2.2.3
psutil==7.0.0
PyYAML==6.0.2
packaging==24.2
pytest==9.0.2
'''
ECDET_REQUIREMENTS = '''# Separate ECDet training environment; torch/cu124 installed separately.
# Original pinned upstream requirements retained alongside this training-only list.
torch==2.6.0+cu124
torchvision==0.21.0+cu124
numpy==2.0.1
scipy==1.16.0
pycocotools==2.0.11
PyYAML==6.0.2
tensorboard==2.21.0
calflops==0.3.2
transformers==5.17.0
timm==1.0.30
tabulate==0.10.0
opencv-python==4.11.0.86
pillow==11.0.0
pytest==9.0.2
'''


def now():
    return datetime.now(ZoneInfo('Asia/Shanghai')).isoformat()


def json_file(path, value):
    path = Path(path)
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + '\n')


def stage(output, name, **values):
    value = {'stage': name, 'updated_at': now(), 'training_started': False, **values}
    json_file(Path(output)/'workflow_status.json', value)
    print(json.dumps(value, ensure_ascii=False), flush=True)


def source_files(repository):
    repository = Path(repository)
    paths = [repository/name for name in SOURCE_ROOT_FILES if (repository/name).is_file()]
    for directory in SOURCE_DIRS:
        for path in (repository/directory).rglob('*'):
            relative = path.relative_to(repository)
            if (path.is_file() and not path.is_symlink() and not SKIP_PARTS.intersection(relative.parts)
                    and not any(p.endswith('.egg-info') for p in relative.parts)
                    and path.suffix.lower() in SOURCE_SUFFIXES):
                paths.append(path)
    for path in paths:
        if path.is_symlink() or not path.resolve().is_relative_to(repository.resolve()):
            raise ValueError(f'Source file is not a regular workspace file: {path}')
    return sorted(set(paths))


def git(repository, *arguments):
    return subprocess.check_output(['git', '-C', str(repository), *arguments], text=True)


def fetch_reference(url, target):
    request = urllib.request.Request(url, headers={'User-Agent': 'pallet-handoff/1.0'})
    with urllib.request.urlopen(request, timeout=30) as response:
        raw = response.read()
    target.parent.mkdir(parents=True, exist_ok=True)
    target.write_bytes(raw)
    return {'url': url, 'sha256': digest(target), 'bytes': len(raw)}


def verify_outer_archive(archive, bundle):
    expected = {str(p.relative_to(bundle)): digest(p) for p in bundle.rglob('*') if p.is_file()}
    prefix = bundle.name + '/'
    with zipfile.ZipFile(archive) as zipped:
        infos = zipped.infolist()
        if len(infos) != len(expected) or {i.filename for i in infos} != {prefix+n for n in expected}:
            raise ValueError('Outer ZIP has missing/extra/duplicate members')
        for info in infos:
            import hashlib
            value = hashlib.sha256()
            with zipped.open(info) as stream:
                for block in iter(lambda: stream.read(4*1024*1024), b''):
                    value.update(block)
            if value.hexdigest() != expected[info.filename[len(prefix):]]:
                raise ValueError(f'Outer ZIP member hash mismatch: {info.filename}')
    return {'state': 'passed', 'crc': 'passed', 'all_member_sha256': 'passed',
            'members': len(expected), 'sha256': digest(archive), 'bytes': Path(archive).stat().st_size}


def build(repository, output):
    repository, output = Path(repository).resolve(), Path(output).resolve()
    config_path = repository/'configs/baselines_8x4090_handoff.json'
    config = read(config_path)
    validate_config(config)
    if output.exists():
        delivery = output/'delivery_manifest.json'
        if delivery.exists() and read(delivery).get('state') == 'complete':
            previous = read(delivery)
            if previous['configuration_sha256'] != digest(config_path):
                raise ValueError('Completed handoff configuration changed; create a new version')
            if digest(output/(NAME+'.zip')) != previous['archive_audit']['sha256']:
                raise ValueError('Completed handoff ZIP changed')
            print(json.dumps(previous, ensure_ascii=False, indent=2))
            return previous
        raise ValueError('Output already exists without completed delivery; preserve it and use a new output directory')
    output.mkdir(parents=True)
    if shutil.disk_usage(output).free < 50*2**30:
        raise RuntimeError('Less than50GiB free disk; large output creation paused')
    bundle = output/NAME
    bundle.mkdir()
    stage(output, 'binding_frozen_dataset')
    archive = repository/'outputs/pallet_dataset_formal_v1/pallet_dataset_formal_v1.zip'
    delivery_source = repository/'outputs/pallet_dataset_formal_v1/delivery_manifest.json'
    original_delivery = read(delivery_source)
    if (digest(archive) != config['dataset']['archive_sha256']
            or original_delivery['state'] != 'complete'
            or original_delivery['archive_audit']['sha256'] != config['dataset']['archive_sha256']):
        raise ValueError('Formal dataset archive/delivery identity mismatch')
    dataset_target = safe_path(bundle, config['paths']['dataset_archive'])
    dataset_target.parent.mkdir(parents=True)
    shutil.copy2(archive, dataset_target)
    json_file(bundle/'handoff_config.json', config)
    (bundle/'TASK_zh.md').write_bytes((repository/'docs/baselines_8x4090_task_zh.md').read_bytes())
    (bundle/'ENVIRONMENT_zh.md').write_bytes((repository/'docs/baselines_8x4090_environment_zh.md').read_bytes())
    shutil.copy2(delivery_source, bundle/'formal_dataset_delivery_original.json')

    stage(output, 'snapshot_current_workspace')
    paths = source_files(repository)
    source_hashes = {}
    for path in paths:
        relative = str(path.relative_to(repository))
        target = bundle/'source'/relative
        target.parent.mkdir(parents=True, exist_ok=True)
        before = digest(path)
        shutil.copy2(path, target)
        if digest(target) != before or digest(path) != before:
            raise ValueError(f'Source changed while snapshotting: {relative}')
        source_hashes[relative] = before
    snapshot = {'captured_at': now(), 'base_git_commit': git(repository, 'rev-parse', 'HEAD').strip(),
                'source_files': len(source_hashes), 'files': source_hashes,
                'contains_current_modified_and_untracked_source': True,
                'excluded': ['.git', '.venv', 'raw_sensor_data', 'outputs', 'cache', 'historical_weights', 'root_correction_archives'],
                'source_scope': list(SOURCE_DIRS)+list(SOURCE_ROOT_FILES),
                'history_configs_are_provenance_not_target_runtime': True}
    json_file(bundle/'source_snapshot.json', snapshot)
    (bundle/'workspace_status.txt').write_text(git(repository, 'status', '--short', '--untracked-files=all'))
    (bundle/'workspace_tracked_diff.patch').write_text(git(repository, 'diff', '--binary', 'HEAD', '--', *SOURCE_DIRS, *SOURCE_ROOT_FILES))
    stage(output, 'record_environment_and_model_sources', source_files=len(paths))
    (bundle/'environment').mkdir()
    (bundle/'environment/requirements-project.lock').write_text(PROJECT_REQUIREMENTS)
    (bundle/'environment/requirements-ecdet.lock').write_text(ECDET_REQUIREMENTS)
    distributions = sorted({(d.metadata['Name'], d.version) for d in importlib.metadata.distributions()})
    json_file(bundle/'environment/source_environment_reference.json', {
        'kind': 'reference_only_not_installation_lock', 'python': platform.python_version(),
        'platform': platform.platform(), 'packages': [{'name': name, 'version': version} for name,version in distributions],
        'note': 'Includes inherited source environment packages. Use clean target training lists; retain target full freeze.'})
    check = subprocess.run([sys.executable, '-m', 'pip', 'check'], capture_output=True, text=True)
    (bundle/'environment/source_pip_check.txt').write_text(
        f'Reference source environment only; exit_code={check.returncode}\n'+check.stdout+check.stderr)
    for name in ('regression.json', 'project_dependency_resolution_audit.json', 'ecdet_dependency_resolution_audit.json'):
        evidence = repository/'outputs/handoff_checks_v1'/name
        if evidence.is_file():
            shutil.copy2(evidence, bundle/'environment'/name)
    edge_commit = config['environment']['edgecrafter_commit']
    upstream = {}
    for path in ('requirements.txt', 'LICENSE.md', 'ecdetseg/configs/ecdet/ecdet_s.yml', 'ecdetseg/configs/ecdet/ecdet.yml'):
        url = f'https://raw.githubusercontent.com/Intellindust-AI-Lab/EdgeCrafter/{edge_commit}/{path}'
        upstream[path] = fetch_reference(url, bundle/'environment/upstream_edgecrafter'/path)
    json_file(bundle/'environment/upstream_sources.json', upstream)
    (bundle/'weights').mkdir()
    model_sources = {'download_stage': 'prepare_once_before_parallel_training', 'models': {}}
    for name in ('yolov8s', 'yolo26s'):
        path = repository/(name+'.pt')
        shutil.copy2(path, bundle/'weights'/path.name)
        model_sources['models'][name] = {'status': 'included_existing_official_detection_weight',
            'path': 'weights/'+path.name, 'sha256': digest(path), 'bytes': path.stat().st_size,
            'official_source': 'https://docs.ultralytics.com/models/'+('yolov8' if name == 'yolov8s' else 'yolo26')+'/',
            'no_historical_assistant_weight': True}
    model_sources['models']['rfdetr-small'] = {'status': 'download_required', 'filename': 'rf-detr-small.pth',
        'url': 'https://storage.googleapis.com/rfdetr/small_coco/checkpoint_best_regular.pth',
        'upstream_md5': 'fb37061c1af7bace359c91b723a8d5c1',
        'upstream_md5_source': 'rfdetr==1.11.0/rfdetr/assets/model_weights.py',
        'target_sha256': None, 'require_target_sha256_before_training': True,
        'official_source': 'https://github.com/roboflow/rf-detr'}
    model_sources['models']['ecdet-s'] = {'status': 'download_required', 'filename': 'ecdet_s.pth',
        'url': 'https://github.com/capsule2077/edgecrafter/releases/download/edgecrafterv1/ecdet_s.pth',
        'repository': config['environment']['edgecrafter_repository'], 'commit': edge_commit,
        'config': 'ecdetseg/configs/ecdet/ecdet_s.yml', 'target_sha256': None,
        'require_target_sha256_before_training': True, 'objects365_detection_weight': False,
        'official_source': 'https://github.com/Intellindust-AI-Lab/EdgeCrafter'}
    json_file(bundle/'model_sources.json', model_sources)
    paper = repository/'Logistics Pallets Detection Based on Improved YOLOv8 with Depth Image.docx'
    (bundle/'reference').mkdir()
    shutil.copy2(paper, bundle/'reference'/paper.name)
    (bundle/'tools').mkdir()
    shutil.copy2(repository/'scripts/verify_8x4090_handoff.py', bundle/'tools/verify_handoff.py')
    (bundle/'README_zh.md').write_text(
        '# 托盘检测8卡4090训练交接包 v1\n\n'
        '包含冻结正式数据ZIP、当前源码、两份官方YOLO初始化权重和完整任务/环境说明。'
        '这是数据与实施交接，不是已实现八项训练入口的发行版；训练状态全部not_started。\n\n'
        '先阅读TASK_zh.md与ENVIRONMENT_zh.md，配置见handoff_config.json。'
        '源码快照在source，继续实现复制到work/source，勿覆盖冻结证据。\n\n'
        '标准库校验与解压：`python3.11 tools/verify_handoff.py --root . --unpack-dataset`。'
        '安装NumPy/Pillow后：`python3.11 tools/verify_handoff.py --root . --check-data`。\n\n'
        '数据解压一次供八卡共享；不需要84GiB原始点云库。原10374帧、12344框、428组，'
        'train/val/test=6552/1455/2367，人工确认的模型辅助参考。'
        '历史绝对路径仅作溯源，目标训练以dataset根目录和相对路径NPZ为准。\n\n'
        '外层逐文件摘要见checksums.sha256，整体ZIP摘要由外部.sha256文件提供。'
        '外层清单保持冻结；目标运行产物放runs，目标代码放work，不覆盖原包。\n')

    stage(output, 'freeze_bundle')
    files = {str(p.relative_to(bundle)): digest(p) for p in sorted(bundle.rglob('*')) if p.is_file()}
    manifest = {'schema_version': 1, 'kind': config['kind'], 'state': 'frozen', 'created_at': now(),
                'scope': config['scope'], 'training_started': False, 'baseline_results_available': False,
                'configuration_sha256': digest(config_path), 'dataset': config['dataset'],
                'source_snapshot_sha256': digest(bundle/'source_snapshot.json'), 'files': files}
    json_file(bundle/'manifest.json', manifest)
    (bundle/'checksums.sha256').write_text(''.join(f'{value}  {name}\n' for name,value in sorted(
        {**files, 'manifest.json': digest(bundle/'manifest.json')}.items())))
    bundle_audit = verify_handoff(bundle)
    stage(output, 'package_and_crc')
    archive_target = output/(NAME+'.zip')
    temporary = archive_target.with_suffix('.zip.partial')
    with zipfile.ZipFile(temporary, 'w', allowZip64=True) as zipped:
        for index, path in enumerate(sorted(bundle.rglob('*')), 1):
            if not path.is_file():
                continue
            compression = zipfile.ZIP_STORED if path.suffix in {'.zip', '.pt', '.docx'} else zipfile.ZIP_DEFLATED
            zipped.write(path, str(path.relative_to(bundle.parent)), compress_type=compression, compresslevel=6)
    temporary.replace(archive_target)
    archive_audit = verify_outer_archive(archive_target, bundle)
    (output/(NAME+'.zip.sha256')).write_text(f'{archive_audit["sha256"]}  {archive_target.name}\n')
    stage(output, 'verify_relocation', archive_bytes=archive_audit['bytes'])
    with tempfile.TemporaryDirectory(prefix='pallet-handoff-relocated-', dir=output) as temp:
        relocation = Path(temp)/'arbitrary-target-parent'
        relocation.mkdir()
        with zipfile.ZipFile(archive_target) as zipped:
            for info in zipped.infolist():
                safe_path(relocation, info.filename)
            zipped.extractall(relocation)
        moved = relocation/NAME
        relocated_bundle_audit = verify_handoff(moved)
        extraction = unpack_dataset(moved)
        portable_audit = verify_portable_dataset(moved/'dataset', config, hash_payloads=False)
        # All payloads were stream-hashed during extraction; semantic checks use only relocated files.
        relocation_audit = {'state': 'passed', 'different_parent_path': True,
                            'verifier': relocated_bundle_audit, 'dataset_extraction': extraction,
                            'portable_data': portable_audit, 'temporary_copy_cleaned_after_audit': True}
    for relative, wanted in source_hashes.items():
        if digest(repository/relative) != wanted:
            raise ValueError(f'Live source changed during packaging: {relative}; frozen snapshot preserved')
    delivery = {'state': 'complete', 'completed_at': now(), 'archive': str(archive_target),
                'task': str(bundle/'TASK_zh.md'), 'configuration_sha256': digest(config_path),
                'bundle_manifest_sha256': digest(bundle/'manifest.json'), 'archive_audit': archive_audit,
                'bundle_audit': bundle_audit, 'relocation_audit': relocation_audit,
                'source_files': len(source_hashes), 'base_git_commit': snapshot['base_git_commit'],
                'dataset_summary': config['dataset'], 'baseline_training_started': False,
                'baseline_interface_implementation_status': 'pending'}
    json_file(output/'delivery_manifest.json', delivery)
    stage(output, 'complete', state='complete', archive_sha256=archive_audit['sha256'],
          archive_bytes=archive_audit['bytes'], source_files=len(source_hashes),
          portable_data_frames=portable_audit['frames'])
    return delivery


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--repository', type=Path, default=Path(__file__).resolve().parents[1])
    parser.add_argument('--output', type=Path)
    args = parser.parse_args()
    output = args.output or args.repository/'outputs/baselines_8x4090_handoff_v1'
    try:
        result = build(args.repository, output)
        print(json.dumps(result, ensure_ascii=False, indent=2), flush=True)
    except Exception as error:
        if Path(output).exists() and not (Path(output)/'delivery_manifest.json').exists():
            stage(output, 'failed', state='failed', error=str(error))
        raise


if __name__ == '__main__':
    main()
