"""Reproducible official-backbone benchmark on the frozen v2 train cohort."""
from __future__ import annotations

import argparse
import copy
import hashlib
import json
import random
import time
import traceback
from pathlib import Path

import numpy as np
import torch
from torch import nn

from .pose_benchmark_models import Backbone, PoseHead, ROOT, FAMILIES, PRETRAINED
from .pose_model import group_folds
from .pose_refinement import residual_target, score_values, loss_value, atomic_json

SOURCE = Path('outputs/pallet_pose_refinement_v2')
CONFIG = Path('configs/pallet_pose_benchmark_v3.json')
ACCELERATION = Path('configs/pallet_pose_benchmark_v3_acceleration.json')


def sha(path):
    return hashlib.sha256(Path(path).read_bytes()).hexdigest()


def seed_all(seed):
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)


def read_rows(path):
    return [json.loads(line) for line in Path(path).read_text().splitlines() if line]


def refinement_recipe():
    return 'accelerated' if (ROOT / 'refinement_protocol.json').exists() else 'screen'


def recipe_config(recipe='screen'):
    config = json.loads((ROOT / 'protocol.json').read_text())['config']
    if recipe == 'accelerated':
        phase = json.loads((ROOT / 'refinement_protocol.json').read_text())
        if phase['parent_protocol_sha256'] != sha(ROOT / 'protocol.json') or phase['config_sha256'] != sha(ACCELERATION):
            raise ValueError('Accelerated protocol binding changed')
        config.update({k: phase['config'][k] for k in
                       ('end_to_end_batch', 'batch', 'evaluation_batch', 'embedding_batch', 'epochs')})
        config['recipe'] = recipe
        config['refinement_protocol_sha256'] = sha(ROOT / 'refinement_protocol.json')
    return config


def freeze_refinement_protocol():
    config = json.loads(ACCELERATION.read_text())
    value = dict(parent_protocol_sha256=sha(ROOT / 'protocol.json'),
                 config_sha256=sha(ACCELERATION), config=config,
                 evidence={p: sha(p) for p in config['memory_and_speed_evidence']})
    path = ROOT / 'refinement_protocol.json'
    if path.exists() and json.loads(path.read_text()) != value:
        raise ValueError('Accelerated refinement protocol is already frozen')
    atomic_json(path, value)
    return value


def load_data(branch, split='train', route='residual'):
    row_path = SOURCE / ('snapshot/eligible.jsonl' if route == 'roi' else 'training/eligible_after_features.jsonl')
    rows = [r for r in read_rows(row_path)
            if r['branch'] == branch and r['split'] == split]
    features, globals_, targets, bases, widths, directions = [], [], [], [], [], []
    accepted = []
    for row in rows:
        key = row['target_key']
        path = (ROOT / 'roi_features' if route == 'roi' else SOURCE / 'cache/features') / (key + '.npz')
        if route == 'roi' and not path.exists():
            manifest = {r['target_key']: r for r in json.loads((ROOT / ('roi_' + split + '.json')).read_text())}
            if manifest[key]['state'] != 'failed':
                raise ValueError('Missing ROI feature without recorded failure: ' + key)
            continue  # Original full denominator is retained in evaluation_stats.
        record = json.loads(path.with_suffix('.json').read_text())
        if sha(path) != record['sha256' if route == 'roi' else 'features_sha256']:
            raise ValueError('v2 feature hash mismatch: ' + key)
        with np.load(path) as data:
            f, g = data['features'].copy(), data['globals'].copy()
            base, width, direction = data['base'].copy(), float(data['width']), data['direction'].copy()
        features.append(f)
        globals_.append(g)
        targets.append(residual_target(row, base, width, direction))
        bases.append(base)
        widths.append(width)
        directions.append(direction)
        accepted.append(row)
    rows = accepted
    if split == 'train':
        expected = json.loads((SOURCE / 'training' / branch / 'profile.json').read_text())['training_keys']
        if [r['target_key'] for r in rows] != expected:
            raise ValueError('Frozen training cohort/order changed')
    return rows, dict(f=np.stack(features), g=np.stack(globals_), y=np.stack(targets),
                      base=np.stack(bases), width=np.array(widths), direction=np.stack(directions))


def freeze_protocol():
    config = json.loads(CONFIG.read_text())
    path = ROOT / 'protocol.json'
    value = dict(config=config, config_sha256=sha(CONFIG),
                 label_sha256=sha(SOURCE / 'snapshot/eligible.jsonl'),
                 eligible_sha256=sha(SOURCE / 'training/eligible_after_features.jsonl'),
                 upstream=json.loads((ROOT / 'upstream_manifest.json').read_text()),
                 weights=json.loads((ROOT / 'weights_manifest.json').read_text()),
                 original_train_counts={}, folds={})
    for branch in config['branches']:
        rows, _ = load_data(branch)
        value['original_train_counts'][branch] = len(rows)
        value['folds'][branch] = []
        for ti, vi in group_folds(rows, 3, 42):
            assert not ({rows[i]['group_id'] for i in ti} & {rows[i]['group_id'] for i in vi})
            value['folds'][branch].append(dict(fit=[rows[i]['target_key'] for i in ti],
                                              validation=[rows[i]['target_key'] for i in vi]))
    if path.exists() and json.loads(path.read_text()) != value:
        raise ValueError('Protocol is already frozen with different contents')
    atomic_json(path, value)
    return config


def encoded_features(family, rows, arrays, route, split):
    path = ROOT / 'embeddings' / route / family / (rows[0]['branch'] + '_' + split + '.npz')
    keys = [r['target_key'] for r in rows]
    if path.exists():
        with np.load(path) as data:
            if data['keys'].tolist() != keys:
                raise ValueError('Embedding key mismatch')
            return data['x'].copy()
    seed_all(42)
    model = Backbone(family).cuda().eval()
    started = time.monotonic()
    batch = recipe_config(refinement_recipe()).get('embedding_batch', 8)
    parts = []
    with torch.inference_mode():
        for offset in range(0, len(rows), batch):
            chunk = []
            for variant in range(arrays['f'].shape[1]):
                f = torch.from_numpy(arrays['f'][offset:offset + batch, variant]).cuda()
                embedding = model(f).cpu().numpy()
                chunk.append(np.concatenate((embedding, arrays['g'][offset:offset + batch, variant]), -1))
            parts.append(np.stack(chunk, 1))
            if offset % 128 == 0:
                print(json.dumps(dict(stage='embedding', family=family, route=route, split=split,
                                      offset=offset, total=len(rows))), flush=True)
    x = np.concatenate(parts)
    if not np.isfinite(x).all():
        raise ValueError('Nonfinite embedding')
    path.parent.mkdir(parents=True, exist_ok=True)
    np.savez_compressed(path, x=x, keys=keys)
    atomic_json(path.with_suffix('.json'), dict(seconds=time.monotonic() - started,
                sha256=sha(path), family=family, frozen=True, no_cohort_fitted_preprocessing=True))
    del model
    torch.cuda.empty_cache()
    return x


class EndToEnd(nn.Module):
    def __init__(self, family, g, tune=False):
        super().__init__()
        self.backbone = Backbone(family)
        self.partial = tune and family in PRETRAINED
        if self.partial:
            self.backbone.requires_grad_(False)
            net = self.backbone.model
            if family == 'pointmae':
                net.blocks.blocks[-1].requires_grad_(True)
                net.norm.requires_grad_(True)
            else:
                list(net.enc.children())[-1].requires_grad_(True)
        self.head = PoseHead(self.backbone.dim + g.shape[-1],
                            np.r_[np.zeros(self.backbone.dim), g.mean(0)],
                            np.r_[np.ones(self.backbone.dim), g.std(0)])

    def train(self, mode=True):
        super().train(mode)
        if self.partial:
            self.backbone.eval()  # freeze BN statistics and upstream dropout
        return self

    def forward(self, f, g):
        return self.head(torch.cat((self.backbone(f), g), -1))


def predict(model, arrays, indices, x=None, batch=16):
    model.eval()
    parts = []
    with torch.no_grad():
        for offset in range(0, len(indices), batch):
            ix = indices[offset:offset + batch]
            if x is not None:
                p = model(torch.from_numpy(x[ix, 0]).cuda())
            else:
                p = model(torch.from_numpy(arrays['f'][ix, 0]).cuda(),
                          torch.from_numpy(arrays['g'][ix, 0]).cuda())
            parts.append(p.cpu().numpy())
    result = np.concatenate(parts)
    if not np.isfinite(result).all():
        raise ValueError('Nonfinite prediction')
    return result


def fit(family, rows, arrays, ti, vi, seed, path, config, x=None, tune=False, epochs=None):
    path.mkdir(parents=True, exist_ok=True)
    ti, vi = np.asarray(ti), np.asarray(vi, dtype=int)
    if any(rows[i]['split'] != 'train' for i in ti) or any(rows[i]['split'] != 'train' for i in vi):
        raise ValueError('Non-train rows passed to model selection')
    if {rows[i]['group_id'] for i in ti} & {rows[i]['group_id'] for i in vi}:
        raise ValueError('Group leakage')
    seed_all(seed)
    rng = np.random.default_rng(seed)
    epochs = epochs or config['epochs']
    if x is not None:
        model = PoseHead(x.shape[-1], x[ti, 0].mean(0), x[ti, 0].std(0)).cuda()
        params = model.parameters()
    else:
        model = EndToEnd(family, arrays['g'][ti, 0], tune=tune).cuda()
        params = [{'params': [p for p in model.backbone.parameters() if p.requires_grad], 'lr': config['backbone_learning_rate']},
                  {'params': model.head.parameters(), 'lr': config['head_learning_rate']}]
    optimizer = torch.optim.AdamW(params, lr=config['head_learning_rate'], weight_decay=config['weight_decay'])
    history, best_state, best, best_epoch = [], None, float('inf'), epochs
    started = time.monotonic()
    batch = config['batch'] if x is not None else config.get('end_to_end_batch', 8)
    evaluation_batch = config.get('evaluation_batch', batch)
    atomic_json(path / 'specification.json', dict(family=family, seed=seed, frozen_encoder=x is not None,
                tune=tune, fit_keys=[rows[i]['target_key'] for i in ti], validation_keys=[rows[i]['target_key'] for i in vi],
                epochs=epochs, batch=batch, evaluation_batch=evaluation_batch,
                recipe=config.get('recipe', 'screen'),
                refinement_protocol_sha256=config.get('refinement_protocol_sha256'),
                gpu=str(torch.cuda.get_device_properties(0)),
                protocol_sha256=sha(ROOT / 'protocol.json')))
    for epoch in range(epochs):
        model.train()
        order = rng.permutation(ti)
        losses = []
        # Avoid singleton BN batches without discarding any example.
        batches = list(np.array_split(order, max(1, int(np.ceil(len(order) / batch)))))
        for ix in batches:
            variant = rng.integers(0, 3, len(ix))
            if x is not None:
                pred = model(torch.from_numpy(x[ix, variant]).cuda())
            else:
                f = torch.from_numpy(arrays['f'][ix, variant]).cuda()
                f[..., :3] += torch.randn_like(f[..., :3]) * .003
                pred = model(f, torch.from_numpy(arrays['g'][ix, variant]).cuda())
            loss = loss_value(pred, torch.from_numpy(arrays['y'][ix]).cuda())
            if not torch.isfinite(loss):
                raise ValueError('Nonfinite training loss')
            optimizer.zero_grad(set_to_none=True)
            loss.backward()
            nn.utils.clip_grad_norm_(model.parameters(), 5)
            optimizer.step()
            losses.append(float(loss.detach()))
        item = dict(epoch=epoch + 1, loss=float(np.mean(losses)))
        if len(vi):
            p = predict(model, arrays, vi, x, evaluation_batch)
            item['validation'] = score_values(p, arrays['y'][vi], [rows[i] for i in vi])
            score = item['validation']['macro_group_selection_score']
            if score < best:
                best, best_epoch = score, epoch + 1
                best_state = {k: v.detach().cpu().clone() for k, v in model.state_dict().items()}
        history.append(item)
        if epoch == 0 or (epoch + 1) % 20 == 0:
            atomic_json(path / 'progress.json', dict(epoch=epoch + 1, epochs=epochs, seconds=time.monotonic() - started, **{'latest': item}))
            print(json.dumps(dict(path=str(path), **item)), flush=True)
    if best_state is not None:
        model.load_state_dict(best_state)
    torch.save(dict(model={k: v.detach().cpu() for k, v in model.state_dict().items()},
                    family=family, selected_epoch=best_epoch, frozen_encoder=x is not None, tune=tune), path / 'selected.pt')
    atomic_json(path / 'history.json', history)
    p = predict(model, arrays, vi if len(vi) else ti, x, evaluation_batch)
    np.save(path / 'predictions.npy', p)
    info = dict(selected_epoch=best_epoch, seconds=time.monotonic() - started,
                parameters=sum(p.numel() for p in model.parameters()),
                trainable_parameters=sum(p.numel() for p in model.parameters() if p.requires_grad),
                checkpoint_sha256=sha(path / 'selected.pt'))
    atomic_json(path / 'completed.json', info)
    del model, optimizer
    torch.cuda.empty_cache()
    return p, info


def trial(family, branch, seed=42, route='residual', tune=False, recipe='screen'):
    config = recipe_config(recipe)
    stage = 'accelerated_tune' if recipe == 'accelerated' and tune else ('accelerated' if recipe == 'accelerated' else ('tune' if tune else 'screen'))
    root = ROOT / 'trials' / route / stage / family / branch / str(seed)
    if (root / 'completed.json').exists():
        return json.loads((root / 'completed.json').read_text())
    rows, arrays = load_data(branch, route=route)
    x = encoded_features(family, rows, arrays, route, 'train') if family in PRETRAINED and not tune else None
    oof = np.zeros_like(arrays['y'])
    infos = []
    for fold, (ti, vi) in enumerate(group_folds(rows, 3, 42)):
        path = root / f'fold_{fold}'
        if (path / 'completed.json').exists():
            p = np.load(path / 'predictions.npy')
            info = json.loads((path / 'completed.json').read_text())
        else:
            p, info = fit(family, rows, arrays, ti, vi, seed, path, config, x=x, tune=tune)
        oof[vi] = p
        infos.append(info)
    root.mkdir(parents=True, exist_ok=True)
    np.savez_compressed(root / 'out_of_fold.npz', predictions=oof, targets=arrays['y'], keys=[r['target_key'] for r in rows])
    result = dict(family=family, branch=branch, seed=seed, route=route, tune=tune, recipe=recipe,
                  metrics=score_values(oof, arrays['y'], rows), folds=infos,
                  seconds=sum(v['seconds'] for v in infos))
    atomic_json(root / 'completed.json', result)
    print(json.dumps(result), flush=True)
    return result


def smoke(family):
    seed_all(42)
    rows, arrays = load_data('metric')
    f = torch.from_numpy(arrays['f'][:2, 0]).cuda()
    torch.cuda.reset_peak_memory_stats()
    started = time.monotonic()
    model = Backbone(family).cuda().eval()
    result = model(f)
    assert result.shape == (2, model.dim) and torch.isfinite(result).all()
    result.square().mean().backward()
    assert any(p.grad is not None and torch.isfinite(p.grad).all() for p in model.parameters())
    with torch.no_grad():
        a, b = model(f), model(f)
        repeat_error = float((a - b).abs().max())
    if repeat_error > 1e-4:
        raise ValueError(('nondeterministic eval', repeat_error))
    torch.cuda.synchronize()
    record = dict(family=family, state='passed', output_dim=model.dim, seconds=time.monotonic() - started,
                  peak_memory=torch.cuda.max_memory_allocated(), repeat_error=repeat_error,
                  parameters=sum(p.numel() for p in model.parameters()), official_weights=model.pretrained)
    atomic_json(ROOT / 'compatibility' / (family + '.json'), record)
    print(json.dumps(record), flush=True)


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--stage', choices=['freeze', 'smoke', 'trial'], required=True)
    parser.add_argument('--family', choices=FAMILIES)
    parser.add_argument('--branch', choices=['metric', 'native_unknown'], default='metric')
    parser.add_argument('--seed', type=int, default=42)
    parser.add_argument('--route', choices=['residual', 'roi'], default='residual')
    parser.add_argument('--tune', action='store_true')
    parser.add_argument('--recipe', choices=['screen', 'accelerated'], default='screen')
    args = parser.parse_args()
    if args.stage == 'freeze':
        freeze_protocol()
    elif args.stage == 'smoke':
        try:
            smoke(args.family)
        except Exception:
            atomic_json(ROOT / 'compatibility' / (args.family + '.json'), dict(family=args.family, state='failed', traceback=traceback.format_exc()))
            raise
    else:
        trial(args.family, args.branch, args.seed, args.route, args.tune, args.recipe)


if __name__ == '__main__':
    main()
