"""Resumable latest-label benchmark and optimizer-state batch/duration study."""
from __future__ import annotations

import argparse
from concurrent.futures import ThreadPoolExecutor
import fcntl
import json
import os
from pathlib import Path
from queue import Queue, Empty
import subprocess
import sys
import traceback

PROJECT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(PROJECT))
from pallet.pose_benchmark_v4_data import ROOT,config
from pallet.pose_benchmark_v4 import trial_root
from pallet.pose_refinement import atomic_json,now


def state(stage,**details):
    value=dict(stage=stage,updated_at=now(),pid=os.getpid(),**details)
    atomic_json(ROOT/'pipeline_state.json',value);print(json.dumps(value),flush=True)


def environment(gpu=None):
    env=dict(os.environ,OPENBLAS_NUM_THREADS='1',OMP_NUM_THREADS='2',PYTHONDONTWRITEBYTECODE='1',
        PYTHONPATH='/tmp/opencode/pose_benchmark_small_deps:/tmp/opencode/pose_benchmark_deps')
    if gpu:env['CUDA_VISIBLE_DEVICES']=gpu
    return env


def run(label,args,gpu=None,extra_environment=None):
    path=ROOT/'logs'/(label+'.log');path.parent.mkdir(parents=True,exist_ok=True)
    env=environment(gpu)
    if extra_environment:env.update(extra_environment)
    with path.open('a') as log:
        process=subprocess.run([sys.executable,'-u',*args],cwd=PROJECT,env=env,stdout=log,stderr=subprocess.STDOUT)
    if process.returncode:raise RuntimeError(f'{label}: exit {process.returncode}; see {path}')


def trial_worker(gpu,queue):
    while True:
        try:job=queue.get_nowait()
        except Empty:return
        family,branch,seed,arm=job
        if (trial_root(*job[:3],arm)/'completed.json').exists():continue
        label='_'.join(map(str,job));atomic_json(ROOT/'jobs'/(label+'.json'),dict(state='running',gpu=gpu,job=job,started_at=now()))
        print(json.dumps(dict(start=job,gpu=gpu)),flush=True)
        try:
            run(label,['-m','pallet.pose_benchmark_v4','--stage','trial','--family',family,'--branch',branch,'--seed',str(seed),'--arm',arm],gpu)
            atomic_json(ROOT/'jobs'/(label+'.json'),dict(state='complete',gpu=gpu,job=job,finished_at=now()))
        except Exception:
            atomic_json(ROOT/'jobs'/(label+'.json'),dict(state='failed',gpu=gpu,job=job,error=traceback.format_exc()))
            raise


def execute_jobs(jobs):
    queue=Queue()
    for job in jobs:queue.put(job)
    with ThreadPoolExecutor(max_workers=2) as pool:
        futures=[pool.submit(trial_worker,gpu,queue) for gpu in config()['preferred_gpu_uuids']]
        for future in futures:future.result()


def screen():
    # Freeze and extraction are idempotent; snapshot timestamps do not advance on resume.
    run('prepare',['-m','pallet.pose_benchmark_v4_data','--stage','prepare'])
    run('protocol',['-m','pallet.pose_benchmark_v4','--stage','freeze'])
    state('all_eight_models_retraining')
    execute_jobs([(family,branch,42,'base') for branch in ('metric','native_unknown') for family in config()['families']])
    run('shortlist',['-m','pallet.pose_benchmark_v4','--stage','shortlist'])


def rest():
    cfg=config();top=json.loads((ROOT/'shortlist.json').read_text())['top_two']
    state('top_two_three_seed_baselines',top_two=top)
    execute_jobs([(family,branch,seed,'base') for branch in ('metric','native_unknown') for family in top for seed in cfg['seeds']])
    state('controlled_continuation_from_epoch80',top_two=top,arms=cfg['continuation'])
    execute_jobs([(family,branch,seed,recipe['name']) for recipe in cfg['continuation']
                  for branch in ('metric','native_unknown') for family in top for seed in cfg['seeds']])
    state('freeze_selection_train_only')
    run('selection',['-m','pallet.pose_benchmark_v4','--stage','select'])
    state('last_epoch_train_OOF_analysis_without_reselection')
    with ThreadPoolExecutor(max_workers=2) as pool:
        futures=[pool.submit(run,'endpoints_'+branch,['-m','pallet.pose_benchmark_v4_results','--stage','endpoints','--branch',branch],gpu)
                 for branch,gpu in zip(('metric','native_unknown'),cfg['preferred_gpu_uuids'])]
        for future in futures:future.result()
    state('refit_all_eight_and_selected_continuations')
    queue=Queue()
    for branch in ('metric','native_unknown'):
        for family in cfg['families']:queue.put((family,branch))
    def refit_worker(gpu):
        while True:
            try:family,branch=queue.get_nowait()
            except Empty:return
            run('refit_'+family+'_'+branch,['-m','pallet.pose_benchmark_v4_results','--stage','refit',
                                          '--family',family,'--branch',branch],gpu)
    with ThreadPoolExecutor(max_workers=2) as pool:
        futures=[pool.submit(refit_worker,gpu) for gpu in cfg['preferred_gpu_uuids']]
        for future in futures:future.result()
    state('fixed_holdout_evaluation_after_all_choices_frozen')
    with ThreadPoolExecutor(max_workers=2) as pool:
        futures=[pool.submit(run,'evaluate_'+branch,['-m','pallet.pose_benchmark_v4_results','--stage','evaluate','--branch',branch],gpu)
                 for branch,gpu in zip(('metric','native_unknown'),cfg['preferred_gpu_uuids'])]
        for future in futures:future.result()
    state('report_package_integrity_and_inference_checks')
    run('report',['-m','pallet.pose_benchmark_v4_results','--stage','report'])
    run('delivery',['scripts/deliver_pose_benchmark_v4.py'])
    run('verification',['scripts/verify_pose_benchmark_v4.py'],cfg['preferred_gpu_uuids'][0])
    run('regressions',['-m','pytest','tests/test_pose_benchmark_v4.py','tests/test_pose_benchmark.py',
                       'tests/test_pose_refinement.py','tests/test_pose_review.py','tests/test_pose_model.py','-q'],
                       cfg['preferred_gpu_uuids'][0],dict(RUN_V4_GPU_TEST='1'))
    state('complete',report=str(ROOT/'REPORT_zh.md'),source_annotation_writes=0,active_machine_labels_modified=False)


def main():
    parser=argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--stage',choices=['screen','continue','all'],default='all')
    parser.add_argument('--wait-for-pipeline',action='store_true',help='Block on the existing pipeline lock without polling')
    args=parser.parse_args();ROOT.mkdir(parents=True,exist_ok=True)
    with (ROOT/'pipeline.lock').open('a') as lock:
        fcntl.flock(lock,fcntl.LOCK_EX|(0 if args.wait_for_pipeline else fcntl.LOCK_NB))
        try:
            if args.stage in ('screen','all'):screen()
            if args.stage in ('continue','all'):rest()
            if args.stage=='screen':state('screening_complete_ready_for_continuation')
        except Exception:
            state('failed',error=traceback.format_exc());raise


if __name__=='__main__':main()
