"""Execute independent benchmark jobs on the two preferred stable GPUs."""
import argparse
from concurrent.futures import ThreadPoolExecutor
import json
import os
from queue import Queue, Empty
from pathlib import Path
import subprocess
import sys

PROJECT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(PROJECT))
from pallet.pose_benchmark import ROOT, atomic_json,refinement_recipe
from pallet.pose_benchmark_models import PRETRAINED

GPUS = ['GPU-3315c096-fef4-7350-483a-7c5f5ca2b619',
        'GPU-09ed84c8-4dee-efc0-dc1d-deda5177e4e8']


def jobs_refine():
    shortlist = json.loads((ROOT / 'shortlist.json').read_text())
    result = []
    for branch, info in shortlist.items():
        for family in info['shortlist']:
            variants = [('residual', False), ('roi', family in PRETRAINED)]
            if family in PRETRAINED:
                variants.append(('residual', True))
            for route, tune in variants:
                for seed in (42, 142, 242):
                    result.append(dict(family=family, branch=branch, route=route, tune=tune, seed=seed,recipe=refinement_recipe()))
    return result


def worker(gpu, jobs):
    output = []
    while True:
        try:
            job = jobs.get_nowait()
        except Empty:
            return output
        label = '_'.join(str(job[k]) for k in ('family','branch','route','tune','seed','recipe'))
        from pallet.pose_benchmark_results import result_path
        root=result_path(job['family'],job['branch'],job['seed'],job['route'],job['tune'],job['recipe'])
        if (root/'completed.json').exists():
            output.append(dict(job=job,state='already_complete'))
            continue
        cmd = [sys.executable,'-u','-m','pallet.pose_benchmark','--stage','trial',
               '--family',job['family'],'--branch',job['branch'],'--route',job['route'],'--seed',str(job['seed']),
               '--recipe',job['recipe']]
        if job['tune']:cmd.append('--tune')
        env = dict(os.environ, CUDA_VISIBLE_DEVICES=gpu, OPENBLAS_NUM_THREADS='1', OMP_NUM_THREADS='2',
                   PYTHONPATH='/tmp/opencode/pose_benchmark_small_deps:/tmp/opencode/pose_benchmark_deps')
        log = ROOT/'logs'/(label+'.log');log.parent.mkdir(exist_ok=True)
        print(json.dumps(dict(start=job,gpu=gpu,log=str(log))),flush=True)
        with log.open('w') as stream:
            process = subprocess.run(cmd,cwd=PROJECT,env=env,stdout=stream,stderr=subprocess.STDOUT)
        result = dict(job=job,gpu=gpu,returncode=process.returncode,state='complete' if process.returncode==0 else 'failed',log=str(log))
        atomic_json(ROOT/'jobs'/(label+'.json'),result)
        print(json.dumps(result),flush=True);output.append(result)


def main():
    parser=argparse.ArgumentParser();parser.add_argument('--stage',choices=['refine'],required=True)
    args=parser.parse_args()
    jobs=jobs_refine()
    # Long jobs first; workers take the next available job rather than leaving
    # one GPU with every expensive native-unit fold.
    jobs.sort(key=lambda job: (job['branch']=='native_unknown', job['family']=='ptv3',
                               job['family']=='pointnext'), reverse=True)
    queue=Queue()
    for job in jobs:queue.put(job)
    with ThreadPoolExecutor(max_workers=len(GPUS)) as pool:
        futures=[pool.submit(worker,gpu,queue) for gpu in GPUS]
        results=[r for f in futures for r in f.result()]
    atomic_json(ROOT/'refinement_jobs.json',results)
    if any(r['state']=='failed' for r in results):raise SystemExit(1)


if __name__=='__main__':main()
