"""Reproducible official-backbone benchmark on the frozen v2 train cohort."""
from __future__ import annotations

import argparse
import copy
import hashlib
import json
import random
import time
import traceback
from pathlib import Path

import numpy as np
import torch
from torch import nn

from .pose_benchmark_models import Backbone, PoseHead, ROOT, FAMILIES, PRETRAINED
from .pose_model import group_folds
from .pose_refinement import residual_target, score_values, loss_value, atomic_json

SOURCE = Path('outputs/pallet_pose_refinement_v2')
CONFIG = Path('configs/pallet_pose_benchmark_v3.json')


def sha(path):
    return hashlib.sha256(Path(path).read_bytes()).hexdigest()


def seed_all(seed):
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)


def read_rows(path):
    return [json.loads(line) for line in Path(path).read_text().splitlines() if line]


def load_data(branch, split='train', route='residual'):
    row_path = SOURCE / ('snapshot/eligible.jsonl' if route == 'roi' else 'training/eligible_after_features.jsonl')
    rows = [r for r in read_rows(row_path)
            if r['branch'] == branch and r['split'] == split]
    features, globals_, targets, bases, widths, directions = [], [], [], [], [], []
    for row in rows:
        key = row['target_key']
        path = (ROOT / 'roi_features' if route == 'roi' else SOURCE / 'cache/features') / (key + '.npz')
        record = json.loads(path.with_suffix('.json').read_text())
        if sha(path) != record['sha256' if route == 'roi' else 'features_sha256']:
            raise ValueError('v2 feature hash mismatch: ' + key)
        with np.load(path) as data:
            f, g = data['features'].copy(), data['globals'].copy()
            base, width, direction = data['base'].copy(), float(data['width']), data['direction'].copy()
        features.append(f)
        globals_.append(g)
        targets.append(residual_target(row, base, width, direction))
        bases.append(base)
        widths.append(width)
        directions.append(direction)
    if split == 'train':
        expected = json.loads((SOURCE / 'training' / branch / 'profile.json').read_text())['training_keys']
        if [r['target_key'] for r in rows] != expected:
            raise ValueError('Frozen training cohort/order changed')
    return rows, dict(f=np.stack(features), g=np.stack(globals_), y=np.stack(targets),
                      base=np.stack(bases), width=np.array(widths), direction=np.stack(directions))


def freeze_protocol():
    config = json.loads(CONFIG.read_text())
    path = ROOT / 'protocol.json'
    value = dict(config=config, config_sha256=sha(CONFIG),
                 label_sha256=sha(SOURCE / 'snapshot/eligible.jsonl'),
                 eligible_sha256=sha(SOURCE / 'training/eligible_after_features.jsonl'),
                 upstream=json.loads((ROOT / 'upstream_manifest.json').read_text()),
                 weights=json.loads((ROOT / 'weights_manifest.json').read_text()),
                 original_train_counts={}, folds={})
    for branch in config['branches']:
        rows, _ = load_data(branch)
        value['original_train_counts'][branch] = len(rows)
        value['folds'][branch] = []
        for ti, vi in group_folds(rows, 3, 42):
            assert not ({rows[i]['group_id'] for i in ti} & {rows[i]['group_id'] for i in vi})
            value['folds'][branch].append(dict(fit=[rows[i]['target_key'] for i in ti],
                                              validation=[rows[i]['target_key'] for i in vi]))
    if path.exists() and json.loads(path.read_text()) != value:
        raise ValueError('Protocol is already frozen with different contents')
    atomic_json(path, value)
    return config


def encoded_features(family, rows, arrays, route, split):
    path = ROOT / 'embeddings' / route / family / (rows[0]['branch'] + '_' + split + '.npz')
    keys = [r['target_key'] for r in rows]
    if path.exists():
        with np.load(path) as data:
            if data['keys'].tolist() != keys:
                raise ValueError('Embedding key mismatch')
            return data['x'].copy()
    seed_all(42)
    model = Backbone(family).cuda().eval()
    started = time.monotonic()
    batch = 8
    parts = []
    with torch.inference_mode():
        for offset in range(0, len(rows), batch):
            chunk = []
            for variant in range(arrays['f'].shape[1]):
                f = torch.from_numpy(arrays['f'][offset:offset + batch, variant]).cuda()
                embedding = model(f).cpu().numpy()
                chunk.append(np.concatenate((embedding, arrays['g'][offset:offset + batch, variant]), -1))
            parts.append(np.stack(chunk, 1))
            if offset % 128 == 0:
                print(json.dumps(dict(stage='embedding', family=family, route=route, split=split,
                                      offset=offset, total=len(rows))), flush=True)
    x = np.concatenate(parts)
    if not np.isfinite(x).all():
        raise ValueError('Nonfinite embedding')
    path.parent.mkdir(parents=True, exist_ok=True)
    np.savez_compressed(path, x=x, keys=keys)
    atomic_json(path.with_suffix('.json'), dict(seconds=time.monotonic() - started,
                sha256=sha(path), family=family, frozen=True, no_cohort_fitted_preprocessing=True))
    del model
    torch.cuda.empty_cache()
    return x


class EndToEnd(nn.Module):
    def __init__(self, family, g, tune=False):
        super().__init__()
        self.backbone = Backbone(family)
        self.partial = tune and family in PRETRAINED
        if self.partial:
            self.backbone.requires_grad_(False)
            net = self.backbone.model
            if family == 'pointmae':
                net.blocks.blocks[-1].requires_grad_(True)
                net.norm.requires_grad_(True)
            else:
                list(net.enc.children())[-1].requires_grad_(True)
        self.head = PoseHead(self.backbone.dim + g.shape[-1],
                            np.r_[np.zeros(self.backbone.dim), g.mean(0)],
                            np.r_[np.ones(self.backbone.dim), g.std(0)])

    def train(self, mode=True):
        super().train(mode)
        if self.partial:
            self.backbone.eval()  # freeze BN statistics and upstream dropout
        return self

    def forward(self, f, g):
        return self.head(torch.cat((self.backbone(f), g), -1))


def predict(model, arrays, indices, x=None, batch=16):
    model.eval()
    parts = []
    with torch.no_grad():
        for offset in range(0, len(indices), batch):
            ix = indices[offset:offset + batch]
            if x is not None:
                p = model(torch.from_numpy(x[ix, 0]).cuda())
            else:
                p = model(torch.from_numpy(arrays['f'][ix, 0]).cuda(),
                          torch.from_numpy(arrays['g'][ix, 0]).cuda())
            parts.append(p.cpu().numpy())
    result = np.concatenate(parts)
    if not np.isfinite(result).all():
        raise ValueError('Nonfinite prediction')
    return result


def fit(family, rows, arrays, ti, vi, seed, path, config, x=None, tune=False, epochs=None):
    path.mkdir(parents=True, exist_ok=True)
    ti, vi = np.asarray(ti), np.asarray(vi, dtype=int)
    if any(rows[i]['split'] != 'train' for i in ti) or any(rows[i]['split'] != 'train' for i in vi):
        raise ValueError('Non-train rows passed to model selection')
    if {rows[i]['group_id'] for i in ti} & {rows[i]['group_id'] for i in vi}:
        raise ValueError('Group leakage')
    seed_all(seed)
    rng = np.random.default_rng(seed)
    epochs = epochs or config['epochs']
    if x is not None:
        model = PoseHead(x.shape[-1], x[ti, 0].mean(0), x[ti, 0].std(0)).cuda()
        params = model.parameters()
    else:
        model = EndToEnd(family, arrays['g'][ti, 0], tune=tune).cuda()
        params = [{'params': [p for p in model.backbone.parameters() if p.requires_grad], 'lr': config['backbone_learning_rate']},
                  {'params': model.head.parameters(), 'lr': config['head_learning_rate']}]
    optimizer = torch.optim.AdamW(params, lr=config['head_learning_rate'], weight_decay=config['weight_decay'])
    history, best_state, best, best_epoch = [], None, float('inf'), epochs
    started = time.monotonic()
    batch = config['batch'] if x is not None else 8
    atomic_json(path / 'specification.json', dict(family=family, seed=seed, frozen_encoder=x is not None,
                tune=tune, fit_keys=[rows[i]['target_key'] for i in ti], validation_keys=[rows[i]['target_key'] for i in vi],
                epochs=epochs, batch=batch, protocol_sha256=sha(ROOT / 'protocol.json')))
    for epoch in range(epochs):
        model.train()
        order = rng.permutation(ti)
        losses = []
        # Avoid singleton BN batches without discarding any example.
        batches = list(np.array_split(order, max(1, int(np.ceil(len(order) / batch)))))
        for ix in batches:
            variant = rng.integers(0, 3, len(ix))
            if x is not None:
                pred = model(torch.from_numpy(x[ix, variant]).cuda())
            else:
                f = torch.from_numpy(arrays['f'][ix, variant]).cuda()
                f[..., :3] += torch.randn_like(f[..., :3]) * .003
                pred = model(f, torch.from_numpy(arrays['g'][ix, variant]).cuda())
            loss = loss_value(pred, torch.from_numpy(arrays['y'][ix]).cuda())
            if not torch.isfinite(loss):
                raise ValueError('Nonfinite training loss')
            optimizer.zero_grad(set_to_none=True)
            loss.backward()
            nn.utils.clip_grad_norm_(model.parameters(), 5)
            optimizer.step()
            losses.append(float(loss.detach()))
        item = dict(epoch=epoch + 1, loss=float(np.mean(losses)))
        if len(vi):
            p = predict(model, arrays, vi, x, batch)
            item['validation'] = score_values(p, arrays['y'][vi], [rows[i] for i in vi])
            score = item['validation']['macro_group_selection_score']
            if score < best:
                best, best_epoch = score, epoch + 1
                best_state = {k: v.detach().cpu().clone() for k, v in model.state_dict().items()}
        history.append(item)
        if epoch == 0 or (epoch + 1) % 20 == 0:
            atomic_json(path / 'progress.json', dict(epoch=epoch + 1, epochs=epochs, seconds=time.monotonic() - started, **{'latest': item}))
            print(json.dumps(dict(path=str(path), **item)), flush=True)
    if best_state is not None:
        model.load_state_dict(best_state)
    torch.save(dict(model={k: v.detach().cpu() for k, v in model.state_dict().items()},
                    family=family, selected_epoch=best_epoch, frozen_encoder=x is not None, tune=tune), path / 'selected.pt')
    atomic_json(path / 'history.json', history)
    p = predict(model, arrays, vi if len(vi) else ti, x, batch)
    np.save(path / 'predictions.npy', p)
    info = dict(selected_epoch=best_epoch, seconds=time.monotonic() - started,
                parameters=sum(p.numel() for p in model.parameters()),
                trainable_parameters=sum(p.numel() for p in model.parameters() if p.requires_grad),
                checkpoint_sha256=sha(path / 'selected.pt'))
    atomic_json(path / 'completed.json', info)
    del model, optimizer
    torch.cuda.empty_cache()
    return p, info


def trial(family, branch, seed=42, route='residual', tune=False):
    config = json.loads((ROOT / 'protocol.json').read_text())['config']
    root = ROOT / 'trials' / route / ('tune' if tune else 'screen') / family / branch / str(seed)
    if (root / 'completed.json').exists():
        return json.loads((root / 'completed.json').read_text())
    rows, arrays = load_data(branch, route=route)
    x = encoded_features(family, rows, arrays, route, 'train') if family in PRETRAINED and not tune else None
    oof = np.zeros_like(arrays['y'])
    infos = []
    for fold, (ti, vi) in enumerate(group_folds(rows, 3, 42)):
        path = root / f'fold_{fold}'
        if (path / 'completed.json').exists():
            p = np.load(path / 'predictions.npy')
            info = json.loads((path / 'completed.json').read_text())
        else:
            p, info = fit(family, rows, arrays, ti, vi, seed, path, config, x=x, tune=tune)
        oof[vi] = p
        infos.append(info)
    root.mkdir(parents=True, exist_ok=True)
    np.savez_compressed(root / 'out_of_fold.npz', predictions=oof, targets=arrays['y'], keys=[r['target_key'] for r in rows])
    result = dict(family=family, branch=branch, seed=seed, route=route, tune=tune,
                  metrics=score_values(oof, arrays['y'], rows), folds=infos,
                  seconds=sum(v['seconds'] for v in infos))
    atomic_json(root / 'completed.json', result)
    print(json.dumps(result), flush=True)
    return result


def smoke(family):
    seed_all(42)
    rows, arrays = load_data('metric')
    f = torch.from_numpy(arrays['f'][:2, 0]).cuda()
    torch.cuda.reset_peak_memory_stats()
    started = time.monotonic()
    model = Backbone(family).cuda().eval()
    result = model(f)
    assert result.shape == (2, model.dim) and torch.isfinite(result).all()
    result.square().mean().backward()
    assert any(p.grad is not None and torch.isfinite(p.grad).all() for p in model.parameters())
    with torch.no_grad():
        a, b = model(f), model(f)
        repeat_error = float((a - b).abs().max())
    if repeat_error > 1e-4:
        raise ValueError(('nondeterministic eval', repeat_error))
    torch.cuda.synchronize()
    record = dict(family=family, state='passed', output_dim=model.dim, seconds=time.monotonic() - started,
                  peak_memory=torch.cuda.max_memory_allocated(), repeat_error=repeat_error,
                  parameters=sum(p.numel() for p in model.parameters()), official_weights=model.pretrained)
    atomic_json(ROOT / 'compatibility' / (family + '.json'), record)
    print(json.dumps(record), flush=True)


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--stage', choices=['freeze', 'smoke', 'trial'], required=True)
    parser.add_argument('--family', choices=FAMILIES)
    parser.add_argument('--branch', choices=['metric', 'native_unknown'], default='metric')
    parser.add_argument('--seed', type=int, default=42)
    parser.add_argument('--route', choices=['residual', 'roi'], default='residual')
    parser.add_argument('--tune', action='store_true')
    args = parser.parse_args()
    if args.stage == 'freeze':
        freeze_protocol()
    elif args.stage == 'smoke':
        try:
            smoke(args.family)
        except Exception:
            atomic_json(ROOT / 'compatibility' / (args.family + '.json'), dict(family=args.family, state='failed', traceback=traceback.format_exc()))
            raise
    else:
        trial(args.family, args.branch, args.seed, args.route, args.tune)


if __name__ == '__main__':
    main()
