"""Execute independent benchmark jobs on the three available GPUs."""
import argparse
from concurrent.futures import ThreadPoolExecutor
import json
import os
from pathlib import Path
import subprocess
import sys

PROJECT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(PROJECT))
from pallet.pose_benchmark import ROOT, atomic_json
from pallet.pose_benchmark_models import PRETRAINED

GPUS = ['GPU-3315c096-fef4-7350-483a-7c5f5ca2b619',
        'GPU-09ed84c8-4dee-efc0-dc1d-deda5177e4e8',
        'GPU-3ce8e1e7-efb2-80a6-de52-75da2f9fa259']


def jobs_refine():
    shortlist = json.loads((ROOT / 'shortlist.json').read_text())
    result = []
    for branch, info in shortlist.items():
        for family in info['shortlist']:
            variants = [('residual', False), ('roi', family in PRETRAINED)]
            if family in PRETRAINED:
                variants.append(('residual', True))
            for route, tune in variants:
                for seed in (42, 142, 242):
                    result.append(dict(family=family, branch=branch, route=route, tune=tune, seed=seed))
    return result


def worker(gpu, jobs):
    output = []
    for job in jobs:
        label = '_'.join(str(job[k]) for k in ('family','branch','route','tune','seed'))
        root = ROOT / 'trials' / job['route'] / ('tune' if job['tune'] else 'screen') / job['family'] / job['branch'] / str(job['seed'])
        if (root/'completed.json').exists():
            output.append(dict(job=job,state='already_complete'))
            continue
        cmd = [sys.executable,'-u','-m','pallet.pose_benchmark','--stage','trial',
               '--family',job['family'],'--branch',job['branch'],'--route',job['route'],'--seed',str(job['seed'])]
        if job['tune']:cmd.append('--tune')
        env = dict(os.environ, CUDA_VISIBLE_DEVICES=gpu, OPENBLAS_NUM_THREADS='1', OMP_NUM_THREADS='2',
                   PYTHONPATH='/tmp/opencode/pose_benchmark_small_deps:/tmp/opencode/pose_benchmark_deps')
        log = ROOT/'logs'/(label+'.log');log.parent.mkdir(exist_ok=True)
        print(json.dumps(dict(start=job,gpu=gpu,log=str(log))),flush=True)
        with log.open('w') as stream:
            process = subprocess.run(cmd,cwd=PROJECT,env=env,stdout=stream,stderr=subprocess.STDOUT)
        result = dict(job=job,gpu=gpu,returncode=process.returncode,state='complete' if process.returncode==0 else 'failed',log=str(log))
        atomic_json(ROOT/'jobs'/(label+'.json'),result)
        print(json.dumps(result),flush=True);output.append(result)
    return output


def main():
    parser=argparse.ArgumentParser();parser.add_argument('--stage',choices=['refine'],required=True)
    args=parser.parse_args()
    jobs=jobs_refine()
    with ThreadPoolExecutor(max_workers=len(GPUS)) as pool:
        futures=[pool.submit(worker,gpu,jobs[i::len(GPUS)]) for i,gpu in enumerate(GPUS)]
        results=[r for f in futures for r in f.result()]
    atomic_json(ROOT/'refinement_jobs.json',results)
    if any(r['state']=='failed' for r in results):raise SystemExit(1)


if __name__=='__main__':main()
