#!/usr/bin/env python3
"""Resume-safe v5 CPU/GPU experiment driver; never schedules a 3090."""
from __future__ import annotations

import argparse
from concurrent.futures import ThreadPoolExecutor
import os
from pathlib import Path
import subprocess
import sys

PROJECT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(PROJECT))
from pallet.pose_paper_data import ROOT, config
from pallet.pose_refinement import atomic_json,now


def run(module,stage,*,gpu=None,seed=None,tag=None):
    env=dict(os.environ,PYTHONDONTWRITEBYTECODE='1',OPENBLAS_NUM_THREADS='1',OMP_NUM_THREADS='1')
    deps='/tmp/opencode/pallet_paper_tools:/tmp/opencode/pose_benchmark_small_deps'
    env['PYTHONPATH']=deps+(':'+env['PYTHONPATH'] if env.get('PYTHONPATH') else '')
    if gpu:env['CUDA_VISIBLE_DEVICES']=gpu
    args=[sys.executable,'-u','-m',module,'--stage',stage]
    if seed is not None:args+=['--seed',str(seed)]
    logs=ROOT/'logs';logs.mkdir(parents=True,exist_ok=True)
    path=logs/(tag or module.rsplit('.',1)[-1]+'_'+stage)
    with path.with_suffix('.log').open('a') as out:
        subprocess.run(args,cwd=PROJECT,env=env,stdout=out,stderr=subprocess.STDOUT,check=True)
    print(f'Completed {module}/{stage} seed={seed}',flush=True)


def worker(gpu,seeds):
    for seed in seeds:run('pallet.pose_paper_learning','trial',gpu=gpu,seed=seed,tag=f'pointnext_seed_{seed}')


def main():
    parser=argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--skip-cpu-cv',action='store_true',help='CPU CV already running separately; require its selection file before holdouts')
    args=parser.parse_args()
    ROOT.mkdir(parents=True,exist_ok=True)
    atomic_json(ROOT/'run_status.json',dict(state='running',started_at=now(),robot_command_eligible=False))
    try:
        run('pallet.pose_paper_data','prepare')
        run('pallet.pose_paper_learning','freeze')
        g0,g1=config()['preferred_gpu_uuids']
        with ThreadPoolExecutor(max_workers=3) as pool:
            tasks=[pool.submit(worker,g0,[42,242]),pool.submit(worker,g1,[142])]
            if not args.skip_cpu_cv:tasks.append(pool.submit(run,'pallet.pose_paper_benchmark','cv'))
            for task in tasks:task.result()
        run('pallet.pose_paper_learning','select')
        run('pallet.pose_paper_benchmark','freeze')
        # Holdout choices fixed before refit and before any holdout scores.
        run('pallet.pose_paper_learning','refit',gpu=g1)
        with ThreadPoolExecutor(max_workers=2) as pool:
            tasks=[pool.submit(run,'pallet.pose_paper_benchmark','evaluate'),
                   pool.submit(run,'pallet.pose_paper_learning','evaluate',gpu=g1)]
            for task in tasks:task.result()
        run('pallet.pose_paper_results','report')
        run('pallet.pose_paper_results','audit')
        atomic_json(ROOT/'run_status.json',dict(state='completed',completed_at=now(),robot_command_eligible=False))
    except Exception as exc:
        atomic_json(ROOT/'run_status.json',dict(state='failed',failed_at=now(),error=str(exc),robot_command_eligible=False))
        raise


if __name__=='__main__':main()
