#!/usr/bin/env python3
"""Publish read-only live training evidence without changing a frozen experiment."""
import argparse
from datetime import datetime
import json
import os
from pathlib import Path
import statistics
import subprocess
import time
from zoneinfo import ZoneInfo

ROOT = Path(__file__).resolve().parents[1]
START = '<!-- mixed-assistant-live:start -->'
END = '<!-- mixed-assistant-live:end -->'


def read(path):
    return json.loads(Path(path).read_text())


def stamp():
    return datetime.now(ZoneInfo('Asia/Shanghai')).isoformat()


def write(path, data):
    temporary = path.with_suffix(path.suffix+'.tmp')
    temporary.write_text(json.dumps(data, ensure_ascii=False, indent=2)+'\n')
    temporary.replace(path)


def alive(pid):
    try:
        return Path(f'/proc/{pid}/stat').read_text().split(') ', 1)[1][0] != 'Z'
    except FileNotFoundError:
        return False


def observe(output):
    delivery = output/'delivery_manifest.json'
    if delivery.exists():
        completed = read(delivery)
        return {'state':'complete', 'updated_at':stamp(), 'archive':completed['archive'],
                'delivery':str(delivery), 'new_archive_created':True}
    config = read(output/'inputs/corrections_config.json')
    partition = read(output/'manual_partition.json')
    result = {'state':'running', 'updated_at':stamp(), 'scope':config['scope'],
              'training_frames':len(partition['train_frame_ids']),
              'validation_frames':len(partition['validation_frame_ids']),
              'runtime':config['runtime'], 'device_label':config.get('device_label'),
              'report_path':config.get('report_path', 'docs/human_correction_7086_report.md'),
              'new_archive_created':False,
              'final_research_test_scored':False}
    workflow = read(output/'workflow_process.json')
    result['workflow_alive'] = alive(workflow['pid'])
    status_path = output/'workflow_status.json'
    state = read(status_path) if status_path.exists() else {}
    run_path = output/'training/run.json'
    result['stage'] = state.get('stage', 'preparing')
    result['completed_epochs'] = 0
    if result['stage']=='training' and not run_path.exists():
        result['stage'] = 'source_checks_and_training_normalization'
    if run_path.exists():
        run = read(run_path)
        result['normalization'] = run['normalization']
        result['device_uuid'] = run['device_uuid']
    metrics_path = output/'training/metrics.jsonl'
    metrics = []
    if metrics_path.exists():
        lines = metrics_path.read_text().splitlines()
        for index,line in enumerate(lines):
            try:
                metrics.append(json.loads(line))
            except json.JSONDecodeError:
                if index != len(lines)-1:
                    raise ValueError('Invalid metric JSON before final appending line')
    if metrics:
        best = max(metrics,key=lambda r:r['validation']['AP50_95'])
        last = metrics[-1]
        result.update(completed_epochs=len(metrics),latest_epoch=last['epoch'],
                      best_epoch=best['epoch'],best_internal_validation={
                          k:best['validation'][k] for k in ('images','objects','AP50','AP75','AP50_95',
                          'precision','recall_at_threshold_iou_0.5','false_positives_per_frame','per_domain')
                          if k in best['validation']},
                      latest_epoch_seconds=last['epoch_seconds'],latest_train_loss=last['train_loss'])
        if len(metrics)>=10:
            first = metrics[:10]
            seconds = statistics.median(r['epoch_seconds'] for r in first)
            train_seconds = statistics.median(r['train_seconds'] for r in first)
            val_seconds = statistics.median(r['validation_seconds'] for r in first)
            result['first_ten_real_epochs'] = {
                'median_epoch_seconds':seconds,'median_train_seconds':train_seconds,
                'median_validation_seconds':val_seconds,
                'training_frames_per_second':result['training_frames']/train_seconds,
                'validation_frames_per_second':result['validation_frames']/val_seconds,
                'max_remaining_training_hours':max(0,300-len(metrics))*seconds/3600,
                'earliest_patience_stop_remaining_hours_if_no_improvement':
                    max(0,50-(last['epoch']-best['epoch']))*seconds/3600,
                'estimate_excludes_prediction_packaging_and_future_improvements':True}
    try:
        query = subprocess.run(['nvidia-smi','--id='+config['device'],
            '--query-gpu=uuid,name,memory.used,utilization.gpu,power.draw',
            '--format=csv,noheader,nounits'],capture_output=True,text=True,timeout=10,check=True)
        uuid,name,memory,utilization,power = [s.strip() for s in query.stdout.strip().split(',')]
        result['gpu'] = {'uuid':uuid,'name':name,'memory_used_mib':float(memory),
                         'utilization_percent':float(utilization),'power_watts':float(power)}
    except (subprocess.SubprocessError,ValueError) as error:
        result['gpu_sampling_error'] = str(error)
    if not result['workflow_alive']:
        result['state'] = 'failed_resumable' if not (output/'delivery_manifest.json').exists() else 'complete'
        result['failure'] = state.get('error','Workflow exited before delivery')
    if (output/'disk_reserve_status.json').exists():
        disk = read(output/'disk_reserve_status.json')
        if disk['state']=='paused_for_disk_reserve':
            result['state'] = disk['state']
    return result


def publish(output, current):
    write(output/'live_progress.json',current)
    if current['state']=='complete':
        return
    with (output/'gpu_observations.jsonl').open('a') as stream:
        stream.write(json.dumps({k:current.get(k) for k in ('updated_at','state','stage','completed_epochs','gpu')})+'\n')
    text = (f'{START}\n实时状态（{current["updated_at"]}）：**{current["state"]} / {current["stage"]}**。'
            f'已完成{current["completed_epochs"]}个真实epoch；{current["training_frames"]}训练、'
            f'{current["validation_frames"]}内部人工验证。新ZIP尚未生成。\n\n')
    if current.get('best_internal_validation'):
        v = current['best_internal_validation']
        text += (f'当前最佳第{current["best_epoch"]+1}轮，内部验证AP50/AP75/AP50:95='
                 f'{v["AP50"]:.5f}/{v["AP75"]:.5f}/{v["AP50_95"]:.5f}。这是选轮指标，不是独立test结果。\n\n')
    if current.get('gpu'):
        g = current['gpu']
        runtime = current['runtime']
        label = current.get('device_label') or g['name']
        text += (f'{label}本次采样利用率{g["utilization_percent"]:.0f}%，显存{g["memory_used_mib"]:.0f}MiB；'
                 f'实际batch{runtime["batch_size"]}/effective{runtime["effective_batch"]}、'
                 f'workers{runtime["workers"]}、验证batch{runtime["validation_batch_size"]}。\n\n')
    if current.get('first_ten_real_epochs'):
        timing = current['first_ten_real_epochs']
        text += (f'前10轮实测中位数{timing["median_epoch_seconds"]:.1f}秒/轮，'
                 f'训练吞吐{timing["training_frames_per_second"]:.1f}帧/秒。'
                 f'若跑满300轮，剩余训练约{timing["max_remaining_training_hours"]:.2f}小时；'
                 f'若后续不提升，最早早停约{timing["earliest_patience_stop_remaining_hours_if_no_improvement"]:.2f}小时。'
                 '估计不含后续预测、打包时间，也不能预知指标继续改善。\n\n')
    text += f'可恢复运行与交付由后台任务继续执行；完整实时记录见`{output.relative_to(ROOT)}/live_progress.json`。\n{END}\n'
    report = ROOT/current.get('report_path', 'docs/human_correction_7086_report.md')
    report.parent.mkdir(parents=True, exist_ok=True)
    original = report.read_text() if report.exists() else '# 人工标签训练与混合修正包\n'
    if START in original and END in original:
        first = original.index(START)
        last = original.index(END,first)+len(END)
        updated = original[:first]+text+original[last:].lstrip('\n')
    else:
        title,rest = original.split('\n',1)
        updated = title+'\n\n'+text+'\n'+rest.lstrip('\n')
    # The training workflow owns the final report after delivery publication.
    if not (output/'delivery_manifest.json').exists():
        report.write_text(updated)
    if current.get('first_ten_real_epochs'):
        marker = f'<!-- first-ten-actual:{output.name} -->'
        timing = current['first_ten_real_epochs']
        runtime = current['runtime']
        label = current.get('device_label') or current.get('gpu',{}).get('name','配置的GPU')
        body = (f'\n\n{marker}\n## {current["updated_at"]}：前10轮真实训练吞吐\n\n'
                f'累计人工分支batch{runtime["batch_size"]}/effective{runtime["effective_batch"]}'
                f'/workers{runtime["workers"]}/val{runtime["validation_batch_size"]}，{label} UUID固定。'
                f'前10轮中位数{timing["median_epoch_seconds"]:.2f}s/epoch；'
                f'训练{timing["median_train_seconds"]:.2f}s、验证{timing["median_validation_seconds"]:.2f}s；'
                f'训练吞吐{timing["training_frames_per_second"]:.2f}帧/s。'
                '实际训练仍在进行，早停和完成时间尚未知；不是合成试跑，也未开展独立研究test评分。\n')
        for name in ['training_log.md','debug_log.md','change_log.md','session_summary.md']:
            path = ROOT/'docs'/name
            existing = path.read_text()
            if marker not in existing:
                path.write_text(existing+body)


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--output',default='outputs/human_correction_manual_7086_v1')
    parser.add_argument('--once',action='store_true')
    args = parser.parse_args()
    output = (ROOT/args.output).resolve()
    while True:
        current = observe(output)
        publish(output,current)
        print(json.dumps(current,ensure_ascii=False),flush=True)
        if args.once or current['state'] in {'complete','failed_resumable'}:
            return
        time.sleep(30)


if __name__=='__main__':
    main()
