"""Generate and resume the preregistered three-seed experiment matrix."""

from __future__ import annotations

import argparse
from concurrent.futures import ThreadPoolExecutor
import hashlib
import json
from pathlib import Path
import shutil
import subprocess
import sys
import threading

from pallet.annotation import load_jsonl, is_resolved
from pallet.devices import device_metadata
from pallet.status import summarize


def _sha256(path):
    return hashlib.sha256(Path(path).read_bytes()).hexdigest()


def _is_completed(output, target_epochs):
    marker = output / "completed.json"
    checkpoint = output / "best.pt"
    if not marker.exists() or not checkpoint.exists():
        return False
    try:
        status = json.loads(marker.read_text(encoding="utf-8"))
        if status.get("target_epochs") != target_epochs or status.get("completed_epochs", 0) < 1:
            return False
        if status.get("stopped_reason") not in {"epochs", "patience"}:
            return False
        if status["stopped_reason"] == "epochs" and status["completed_epochs"] < target_epochs:
            return False
        if status["best_checkpoint_sha256"] != _sha256(checkpoint):
            return False
        run_file = output / "run.json"
        if run_file.exists() and status.get("run_config_sha256") != _sha256(run_file):
            return False
        return bool(status.get("config_digest"))
    except (OSError, ValueError, KeyError, TypeError):
        return False


def build_matrix(config_path, annotations_path, output_root):
    config = json.loads(Path(config_path).read_text())
    annotations = load_jsonl(annotations_path)
    domains = sorted({frame.domain for frame in annotations.frames.values()
                      if frame.domain != "unknown; FOV uncertain"})
    named = {variant["name"]: variant for variant in config["within_domain"]}
    specs = []
    for variant in config["within_domain"]:
        for seed in config["seeds"]:
            specs.append({"scope": "within", "variant": variant["name"],
                          "family": variant["family"], "flags": variant["flags"],
                          "seed": seed, "holdout_domain": None})
    for domain in domains:
        for name in config["leave_one_configuration"]:
            variant = named[name]
            for seed in config["seeds"]:
                specs.append({"scope": "leave_one_configuration", "variant": name,
                              "family": variant["family"], "flags": variant["flags"],
                              "seed": seed, "holdout_domain": domain})
    for spec in specs:
        domain = spec["holdout_domain"] or "within"
        safe_domain = "".join(char if char.isalnum() else "_" for char in domain)
        spec["output"] = str(Path(output_root) / spec["scope"] / safe_domain /
                             spec["variant"] / f"seed_{spec['seed']}")
    return config, specs


def _run_one(spec, config, annotation_path, root, device, lock):
    with lock:
        if shutil.disk_usage(Path.cwd()).free < 50 * 2**30:
            raise RuntimeError("less than 50 GiB available; experiment queue paused")
        output = Path(spec["output"])
        output.mkdir(parents=True, exist_ok=True)
        commands = []
        checkpoint = output / "best.pt"
        run_path=output/'run.json'
        if run_path.exists():
            prior=json.loads(run_path.read_text())
            expected={'annotation_sha256':_sha256(annotation_path), 'family':spec['family'],
                      'seed':spec['seed'], 'holdout_domain':spec['holdout_domain'],
                      'recipe':config.get('recipe','project_adamw_pilot'),
                      'protocol_sha256':_sha256(Path(config['protocol'])/'protocol.json'),
                      'subset_sha256':_sha256(config['subset']) if config.get('subset') else None}
            if any(prior.get(key,'project_adamw_pilot' if key=='recipe' else None)!=value
                   for key,value in expected.items()):
                raise ValueError(f'completed/resumed run belongs to a different data snapshot: {output}')
        needs_training = not _is_completed(output, config["epochs"])
        if needs_training:
            command = [sys.executable, "-m", "pallet.cli", "train", "--root", root,
                       "--annotations", annotation_path, "--output", str(output),
                       "--family", spec["family"], "--seed", str(spec["seed"]),
                       "--epochs", str(config["epochs"]), "--size", str(config["image_size"]),
                       "--batch-size", str(config.get('physical_batch_by_family',{}).get(spec['family'],16)),
                       "--effective-batch", str(config["effective_batch"]),
                       "--recipe", config.get('recipe','project_adamw_pilot'),
                       "--device", device] + spec["flags"]
            command += ['--protocol',config['protocol']]
            if config.get('subset'):command += ['--subset',config['subset']]
            if config.get('cache_index'):command += ['--cache-index',config['cache_index']]
            if spec["holdout_domain"]:
                command += ["--holdout-domain", spec["holdout_domain"]]
            commands.append(command)
        eval_split='val' if config.get('phase')=='pilot' else 'test'
        if needs_training or not (output / f"{eval_split}.json").exists():
            commands.append([sys.executable, "-m", "pallet.cli", "evaluate", "--root", root,
                             "--annotations", annotation_path, "--checkpoint", str(checkpoint),
                             "--split", eval_split, "--device", device,
                             '--protocol',config['protocol'],
                             '--bootstrap-samples',str(config.get('bootstrap_samples',0)),
                             "--output", str(output / f"{eval_split}.json")]
                            + (['--subset',config['subset']] if config.get('subset') else [])
                            + (['--cache-index',config['cache_index']] if config.get('cache_index') else []))
        for command in commands:
            with (output / "runner.log").open("a") as log:
                log.write("COMMAND " + json.dumps(command) + "\n")
                log.flush()
                result = subprocess.run(command, stdout=log, stderr=subprocess.STDOUT, check=False)
                log.write(f"EXIT {result.returncode}\n")
            if result.returncode:
                raise RuntimeError(f"experiment failed ({result.returncode}): {output}")
            if command[3] == "train" and not _is_completed(output, config["epochs"]):
                raise RuntimeError(f"training did not produce a valid completion marker: {output}")
        return {"output": str(output), "commands_executed": len(commands)}


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--config", default="configs/experiment_matrix.json")
    parser.add_argument("--annotations", default="outputs/annotations_reviewed_v2.jsonl")
    parser.add_argument("--root", default="/root/private/data/pallet")
    parser.add_argument("--output-root", default="outputs/experiments")
    parser.add_argument("--manifest", default="outputs/matrix_plan.json")
    parser.add_argument("--execute", action="store_true")
    parser.add_argument('--protocol',default='outputs/protocol_v5')
    parser.add_argument('--subset')
    parser.add_argument('--phase',choices=('pilot','full'),default='full')
    parser.add_argument('--recipe',choices=('project_adamw_pilot','official_research_constrained','official_default'))
    parser.add_argument('--cache-index')
    args = parser.parse_args()
    config, specs = build_matrix(args.config, args.annotations, args.output_root)
    config.update(protocol=args.protocol,subset=args.subset,phase=args.phase,
                  recipe=args.recipe or config.get('recipe') or
                  ('project_adamw_pilot' if args.phase=='pilot' else 'official_research_constrained'),
                  cache_index=args.cache_index)
    progress = summarize(args.annotations,output=str(Path(args.manifest).with_suffix('.progress.json')),
                         queue_path=str(Path(args.manifest).with_suffix('.queue.jsonl')))
    eligible=load_jsonl(args.annotations).frames
    domain_counts={domain:{'blind_pos':0,'blind_negative':0,'blind_ignore':0}
                   for domain in {frame.domain for frame in eligible.values()}}
    for frame in eligible.values():
        if frame.split=='test' and is_resolved(frame,require_blind=True):
            domain_counts[frame.domain]['blind_'+frame.status]+=1
    scored_domains={domain for domain,counts in domain_counts.items()
                    if counts['blind_pos']>=10 and counts['blind_negative']>=5}
    if args.phase=='full':
        specs=[spec for spec in specs if not spec['holdout_domain'] or spec['holdout_domain'] in scored_domains]
    # Planned configurations remain distinct from completed runs; don't imply
    # a holdout result where the reference domain has no scorable pos/negative.
    required=set(eligible)
    if args.phase=='pilot':
        if not args.subset:raise ValueError('pilot runs require an explicit frozen subset')
        ids=json.loads(Path(args.subset).read_text());required=set(ids['train']) | set(ids['val'])
    ready=all(fid in eligible and is_resolved(eligible[fid],require_blind=eligible[fid].split in {'val','test'})
              for fid in required)
    plan = {"experiments": len(specs), "reference_ready": ready,
            'domain_reference_counts':domain_counts,
            'excluded_unscorable_domains':sorted({f.domain for f in eligible.values()}-scored_domains),
            'phase':args.phase,'protocol_sha256':_sha256(Path(args.protocol)/'protocol.json'),
            'recipe':config['recipe'],
            'cache_index_sha256':_sha256(args.cache_index) if args.cache_index else None,
            'annotations_sha256':_sha256(args.annotations),
            "specs": specs}
    Path(args.manifest).parent.mkdir(parents=True, exist_ok=True)
    Path(args.manifest).write_text(json.dumps(plan, indent=2), encoding="utf-8")
    print(json.dumps({"experiments": len(specs), "reference_ready": plan["reference_ready"],
                      "manifest": args.manifest}))
    if not args.execute:
        return
    if not plan["reference_ready"]:
        raise ValueError("full reviewed train, validation and held-out reference labels are required")
    devices = config["devices"]
    identities=[device_metadata(device).get('uuid') or 'cpu' for device in devices]
    if len(set(identities))!=len(identities):raise ValueError('device aliases map to the same physical GPU')
    locks = {device: threading.Lock() for device in devices}
    with ThreadPoolExecutor(max_workers=len(devices)) as pool:
        futures = [pool.submit(_run_one, spec, config, args.annotations, args.root,
                               devices[idx % len(devices)], locks[devices[idx % len(devices)]])
                   for idx, spec in enumerate(specs)]
        for future in futures:
            print(json.dumps(future.result()))


if __name__ == "__main__":
    main()
