"""Archive current research evidence and regenerate the training ledger."""
from __future__ import annotations

import argparse
from collections import Counter
from datetime import datetime, timezone
import hashlib
import json
from pathlib import Path
import shutil
import subprocess
import sys

from pallet.annotation import is_resolved, load_jsonl
from pallet.annotation.workflow import annotation_digest
from pallet.data.protocol import file_digest, load_protocol, write_json


def read_json(path):
    return json.loads(Path(path).read_text())


def run_category(path):
    parts = Path(path).parts
    if 'pilot600_v5' in parts:
        return '600帧先导对照'
    if 'paired_learning_60_v5' in parts:
        return '60帧配对模态检查'
    if 'learning_120_v4' in parts:
        return '120帧早期学习检查'
    if 'rounds_v5' in parts:
        return '标注教师'
    return '配方工程检查'


def collect_run(directory):
    directory = Path(directory)
    run_path = directory / 'run.json'
    run = read_json(run_path)
    result = {'directory': str(directory), 'category': run_category(directory),
              'run_sha256': file_digest(run_path), 'config': run,
              'state': 'no_completion_marker', 'validation': None,
              'test_score_present': (directory / 'test.json').exists()}
    completion_path = directory / 'completed.json'
    if completion_path.exists():
        completion = read_json(completion_path)
        best = directory / 'best.pt'
        checks = {
            'run_hash_matches': completion.get('run_config_sha256') == file_digest(run_path),
            'checkpoint_hash_matches': best.exists() and
                completion.get('best_checkpoint_sha256') == file_digest(best),
        }
        result.update(completion=completion,
                      completion_sha256=file_digest(completion_path), checks=checks,
                      state='complete_verified' if all(checks.values()) else 'completion_mismatch')
    metrics_path = directory / 'metrics.jsonl'
    rows = []
    malformed = 0
    if metrics_path.exists():
        for line in metrics_path.read_text().splitlines():
            if not line.strip():
                continue
            try:
                rows.append(json.loads(line))
            except json.JSONDecodeError:
                malformed += 1
        scores = [r.get('validation', {}).get('AP50_95') for r in rows]
        result.update(metrics_sha256=file_digest(metrics_path), metric_rows=len(rows),
                      malformed_metric_rows=malformed,
                      best_monitor_AP50_95=max((v for v in scores if v is not None), default=None))
    validation_path = directory / 'val.json'
    if validation_path.exists():
        result['validation'] = read_json(validation_path)
        result['validation_sha256'] = file_digest(validation_path)
    recipe_path = directory / 'recipe_report.json'
    if recipe_path.exists():
        result['recipe_report_sha256'] = file_digest(recipe_path)
    return result


def collect_references(root, *, protocol=None):
    root = Path(root)
    paths = list((root / 'outputs/protocol_v5/references').glob('*.manifest.json'))
    paths += list((root / 'outputs').glob('full_review_*_v5/reference*.manifest.json'))
    references = []
    unique = {'val': {}, 'test': {}}
    payloads = {'val': {}, 'test': {}}
    records = assignments = None
    if protocol is not None:
        items, assignments, _ = load_protocol(protocol)
        records = {item.frame_id: item for item in items}
    for path in sorted(paths):
        manifest = read_json(path)
        split = manifest.get('split')
        if split not in unique:
            continue
        labels_path = path.with_name(path.name.replace('.manifest.json', '.jsonl'))
        if not labels_path.exists():
            continue
        labels = load_jsonl(labels_path)
        counts = Counter(f.status for f in labels.frames.values())
        verified = (file_digest(labels_path) == manifest.get('label_sha256')
                    and len(labels.frames) == manifest.get('frames')
                    and all(f.split == split and is_resolved(f, require_blind=True)
                            for f in labels.frames.values())
                    and ('counts' not in manifest or all(
                        counts.get(status, 0) == manifest['counts'].get(status, 0)
                        for status in ('pos', 'negative', 'ignore'))))
        identity_verified = None
        if records is not None:
            identity_verified = (manifest.get('protocol_sha256') == file_digest(Path(protocol) / 'protocol.json')
                                 and all(fid in records and f.group_id == records[fid].group_id
                                         and f.source_anchor_hash == records[fid].source_sha256
                                         and f.split == split == assignments.get(f.group_id)
                                         for fid, f in labels.frames.items()))
            verified = verified and identity_verified
        references.append({'path': str(labels_path.relative_to(root)),
                           'manifest_sha256': file_digest(path),
                           'label_sha256': file_digest(labels_path), 'split': split,
                           'frames': len(labels.frames), 'counts': dict(counts),
                           'groups': len({f.group_id for f in labels.frames.values()}),
                           'verified': verified, 'protocol_identity_verified': identity_verified})
        if verified:
            for fid, frame in labels.frames.items():
                unique[split].setdefault(fid, set()).add(frame.status)
                payloads[split].setdefault(fid, set()).add(annotation_digest(frame))
    # Frame counts are deduplicated; disagreements remain explicit and are not scorable counts.
    coverage = {split: {'unique_frozen_frames': len(items),
                       'status_conflicts': sum(len(values) != 1 for values in items.values()),
                       'payload_conflicts': sum(len(values) != 1 for values in payloads[split].values()),
                       'counts': dict(Counter(next(iter(values)) for values in items.values()
                                              if len(values) == 1))}
                for split, items in unique.items()}
    for split, items in unique.items():
        conflicts = sorted(fid for fid, values in payloads[split].items() if len(values) != 1)
        conflict_ids = set(conflicts)
        consistent = Counter(next(iter(values)) for fid, values in items.items()
                             if len(values) == 1 and fid not in conflict_ids)
        coverage[split].update(
            conflicting_frame_ids=conflicts,
            consistent_frames=sum(consistent.values()),
            consistent_counts=dict(consistent),
            scorable_consistent_frames=consistent['pos'] + consistent['negative'])
    return references, coverage


def command_result(command, cwd):
    try:
        result = subprocess.run(command, cwd=cwd, capture_output=True, text=True, timeout=20)
        return {'command': command, 'exit_code': result.returncode,
                'stdout': result.stdout.strip(), 'stderr': result.stderr.strip()}
    except (OSError, subprocess.TimeoutExpired) as error:
        return {'command': command, 'error': str(error)}


def collect(root):
    root = Path(root).resolve()
    progress_path = root / 'outputs/progress_v5.json'
    progress = read_json(progress_path)
    references, reference_coverage = collect_references(root, protocol=root / 'outputs/protocol_v5')
    runs = [collect_run(p.parent) for p in sorted((root / 'outputs').rglob('run.json'))]
    for run in runs:
        run['directory'] = str(Path(run['directory']).relative_to(root))
    evidence = {'outputs/progress_v5.json': file_digest(progress_path)}
    queue_path = (progress.get('full_review_queue') or {}).get('manifest')
    source_paths = ['outputs/protocol_v5/protocol.json',
                     'configs/experiment_matrix.json', 'docs/paper_analysis.md',
                     'docs/pilot_validation_v5.md', 'docs/paired_modality_pilot_v5.md',
                     'docs/official_recipe_gap.md', 'docs/session_summary.md',
                     'docs/debug_log.md', 'docs/change_log.md', 'docs/experiment_status.md',
                     'README.md', 'pyproject.toml',
                     'pallet/research_log.py', 'tests/test_research_log.py']
    if queue_path:
        source_paths.append(queue_path)
    for relative in source_paths:
        path = root / relative
        if path.exists():
            evidence[relative] = file_digest(path)
    exports = []
    for path in sorted((root / 'outputs/pilot600_v5/exports').glob('*.json')):
        report = read_json(path)
        model_path = Path(report.get('onnx', ''))
        exports.append({'report': str(path.relative_to(root)),
                        'report_sha256': file_digest(path), 'details': report,
                        'onnx_sha256': file_digest(model_path) if model_path.is_file() else None})
    checks = {str(p.relative_to(root)): p.read_text() for p in
              sorted((root / 'outputs/session_logs/checks').glob('*.log'))}
    source_hashes = {str(path.relative_to(root)): file_digest(path)
                     for directory in ('pallet', 'tests', 'configs')
                     for path in sorted((root / directory).rglob('*'))
                     if path.is_file() and path.suffix in ('.py', '.json', '.yaml', '.yml')}
    return {'captured_at_utc': datetime.now(timezone.utc).isoformat(),
            'scope': 'Current artifacts plus session history documented in docs/session_summary.md; '
                     'artifact modification time is not a training start/end time.',
            'evidence_sha256': evidence, 'progress': progress, 'runs': runs,
            'current_source_sha256': source_hashes,
            'current_source_manifest_sha256': hashlib.sha256(
                json.dumps(source_hashes, sort_keys=True, separators=(',', ':')).encode()).hexdigest(),
            'run_counts': dict(Counter(r['state'] for r in runs)),
            'verified_run_categories': dict(Counter(r['category'] for r in runs
                                                    if r['state'] == 'complete_verified')),
            'references': references, 'reference_coverage': reference_coverage,
            'pilot_exports': exports,
            'verification_logs': checks,
            'git_head': command_result(['git', 'rev-parse', 'HEAD'], root),
            'git_worktree': command_result(['git', 'status', '--short'], root),
            'git_history': command_result(['git', 'log', '-8', '--format=%h %aI %s'], root),
            'gpu_observation': command_result(['nvidia-smi', '--query-gpu=index,name,uuid,memory.used,utilization.gpu',
                                               '--format=csv'], root),
            'gpu_processes': command_result(['nvidia-smi', '--query-compute-apps=pid,process_name,gpu_uuid,used_memory',
                                             '--format=csv'], root),
            'disk_free_gib': round(shutil.disk_usage(root).free / 1024**3, 2)}


def render_training_log(data, snapshot_path):
    lines = ['# 训练与验证日志', '', f'本次核验时间：{data["captured_at_utc"]}。', '',
             f'完整配置、SHA256、指标和现场状态：[不可变快照](../{snapshot_path})。', '',
             '本表从 `run.json`、`completed.json`、`metrics.jsonl` 和 `val.json` 重建。'
             '完成条件同时核对运行配置和最佳检查点哈希；仅有 `best.pt` 不视为完成。'
             '原始文件没有提供的起止时间不补写。AP 使用 0–1；教师数值是训练监控，不能作为研究对照。', '',
             f'运行记录总数 {len(data["runs"])}；完成且哈希匹配 '
             f'{sum(r["state"] == "complete_verified" for r in data["runs"])}；'
             f'未通过完成核验 {sum(r["state"] != "complete_verified" for r in data["runs"])}。', '',
             '| 类别 | 运行 | 完成状态 | epoch | 配方 | 验证 AP50:95 / 教师监控最优 |',
             '|---|---|---|---:|---|---:|']
    for run in data['runs']:
        directory = run['directory']
        name = directory.split('/within/within/')[-1] if '/within/within/' in directory else directory.removeprefix('outputs/')
        completion = run.get('completion', {})
        validation = run.get('validation') or {}
        ap = validation.get('AP50_95')
        if ap is None and run['category'] == '标注教师':
            ap = run.get('best_monitor_AP50_95')
        score = f'{ap:.4f}' if ap is not None else '—'
        state = {'complete_verified': '完成且哈希匹配', 'completion_mismatch': '完成清单不匹配',
                 'no_completion_marker': '未完成记录'}.get(run['state'], run['state'])
        recipe = run['config'].get('recipe') or '历史未显式记名（见下文）'
        lines.append(f'| {run["category"]} | [{name}](../{directory}/run.json) | {state} | '
                     f'{completion.get("completed_epochs", "—")} | {recipe} | {score} |')
    lines += ['', '早期学习、600 帧先导、60 帧配对和教师 001–004 的 run.json 未显式写出 recipe。'
              '已有实验文档将其描述为项目 AdamW 先导配方；不能追认成官方配方。'
              '配方检查和教师 005–006 显式记录 `official_research_constrained`。', '',
              '600 帧先导的有效训练帧为 282，验证有效帧为 106，八条对照均为 seed 42、640 输入。'
              '最佳 YOLO26s AP50:95=0.414；完整改造版=0.368，当前没有观察到结构收益。'
              '三个可信配对模态运行使用 42 张有效训练帧与 46 张有效验证帧，域分布失衡。', '',
              '## 标注教师与审计', '',
              '| 轮次 | 候选状态 | 候选已解决 / 计划 | 实质审计变化 / 49 | 待裁决 |',
              '|---|---|---:|---:|---:|']
    progress = data['progress']
    for key, item in progress.get('candidate_rounds', {}).items():
        audit = progress.get('training_audits', {}).get(key, {})
        lines.append(f'| {key} | {item.get("round_state")} | {item.get("resolved")}/{item.get("planned")} | '
                     f'{audit.get("substantive_changed", "—")} | {item.get("pending_adjudication", "—")} |')
    for key, candidate in progress.get('candidate_rounds', {}).items():
        review_progress = candidate.get('review_progress') or {}
        if not review_progress:
            continue
        lines += ['', f'轮次 {key} 已核验提交覆盖；提交数可能领先于 latest.json 导入，不能直接相加到冻结总量。', '',
                  '| 侧 | 已核验任务 | 已核验帧 | 尚未覆盖候选帧 |', '|---|---:|---:|---:|']
        for side, item in review_progress.get('reviewers', {}).items():
            lines.append(f'| {side.upper()} | {item["tasks_submitted"]} | {item["frames_reviewed"]} | {item["pending_frames"]} |')
    lines += ['', 'C 只需裁决 A/B 分歧；尚未覆盖候选帧数不是必须由 C 审阅的剩余量。', '',
              '固定 49 帧的停止要求是连续两轮实质改动均为 0；六轮尚未收敛。'
              '教师 006 共 125 epoch，最佳监控 AP50:95=0.4150，因 patience 停止；'
              '会话记录两次中断并恢复，训练结束权重与完成清单已核验。', '',
              '## 独立参考与测试', '', '| split | 各版冻结参考唯一帧 | 正例 | 负例 | ignore | 状态冲突 | 标签内容冲突 |',
              '|---|---:|---:|---:|---:|---:|---:|']
    for split, item in data['reference_coverage'].items():
        counts = item['counts']
        lines.append(f'| {split} | {item["unique_frozen_frames"]} | {counts.get("pos", 0)} | '
                     f'{counts.get("negative", 0)} | {counts.get("ignore", 0)} | {item["status_conflicts"]} | {item["payload_conflicts"]} |')
    lines += ['', '参考按 frame_id 去重；“已解决”包括 ignore，不能等同于可评分帧数。'
              '有标签内容冲突时，分版参考不能合并成一个评分集；同状态也可能存在框界分歧。'
              '队列中的训练盲审补充、临时导入和冻结参考采用不同口径，详见会话总结。', '',
              '| split | 排除跨版本标签分歧后的唯一帧 | 其中可评分帧 |',
              '|---|---:|---:|']
    for split, item in data['reference_coverage'].items():
        lines.append(f'| {split} | {item["consistent_frames"]} | {item["scorable_consistent_frames"]} |')
    lines += ['', '这里的无分歧覆盖只用于进度统计，不构成新的冻结评分集；分歧 frame_id 保存在快照中。', '',
              f'运行目录中已有 test.json 的数量：{sum(r["test_score_present"] for r in data["runs"])}。'
              '本命令不调用模型、不生成测试预测。正式三种子、全量结构/模态消融及留域实验尚未完成。', '',
              '640 ONNX 已有先导训练权重的 YOLO26、完整改造、RF-DETR 三项导出验证；'
              '最终模型的重新导出和测试仍待正式实验。RF 数值波动与偶发输出对应差异见调试日志。', '',
              '## 现场状态与证据', '',
              f'磁盘可用 {data.get("disk_free_gib", "未知")} GiB；GPU现场查询如下。'
              '历史运行设备以其run.json的UUID为准，当前CUDA序号不能回填历史设备型号。', '',
              '```text', data.get('gpu_observation', {}).get('stdout', '无设备查询记录'), '```', '',
              'GPU计算进程：', '', '```text',
              data.get('gpu_processes', {}).get('stdout') or '查询无输出；失败详情查看证据快照。', '```', '',
              'Git提交、工作区差异、当前源码逐文件哈希、参考分歧frame_id、ONNX清单及原始回归输出均存入快照。'
              '日志中的事实来源为既有产物或会话记录，不按文件mtime重建训练耗时。', '',
              '## 更新方式', '', '```bash',
              'PYTHONPATH=. .venv/bin/python -m pallet.research_log --refresh-progress', '```', '',
              '每次生成新的 UTC 命名快照和 latest 指针，更新本训练日志；调试与修改日志保留历史，按条目追加。', '']
    return '\n'.join(lines)


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--root', default='.')
    parser.add_argument('--refresh-progress', action='store_true')
    args = parser.parse_args()
    root = Path(args.root).resolve()
    if args.refresh_progress:
        subprocess.run([sys.executable, '-m', 'pallet.progress_v5'], cwd=root,
                       check=True, stdout=subprocess.DEVNULL)
    data = collect(root)
    stamp = datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S%fZ')
    relative = f'outputs/session_logs/{stamp}.json'
    path = root / relative
    path.parent.mkdir(parents=True, exist_ok=True)
    with path.open('x') as output:
        json.dump(data, output, ensure_ascii=False, indent=2)
        output.write('\n')
    pointer = {'snapshot': relative, 'snapshot_sha256': file_digest(path),
               'captured_at_utc': data['captured_at_utc']}
    log_path = root / 'docs/training_log.md'
    log_path.parent.mkdir(parents=True, exist_ok=True)
    temp = log_path.with_suffix('.md.tmp')
    temp.write_text(render_training_log(data, relative))
    temp.replace(log_path)
    write_json(path.parent / 'latest.json', pointer)
    print(json.dumps(pointer, ensure_ascii=False))


if __name__ == '__main__':
    main()
