"""Generic-start shared fine/parent/attribute and mask-boundary learning.

Independent component holdout is evaluated BEFORE full-pool annotation fitting.
No old pallet checkpoint or old validation prediction is used in either phase.
"""
from collections import Counter, defaultdict
from copy import deepcopy
import math
import os
import shutil
import time
import traceback

import cv2
import numpy as np
import torch
from torch import nn
from torch.nn import functional as F

from .cluster_targets import digest
from .mask_annotation_full_train import normalize, augment, update_ema
from .mask_model_training import ROIMaskBoundaryUNet, seed_all, atomic_torch, native_metrics, summarize
from .mask_model_training_data import FALSE_FLAGS, PROJECT, read, write, jsonl, require, gpu_receipt, letterbox, unletterbox, sha
from .mask_taxonomy_data import ROOT, NAMESPACE, verify, sources_current

CONFIG = dict(seed=42, input_size=[512,160], batch_size=16, evaluation_epochs=24, full_pool_epochs=20,
    lr=4e-4, encoder_lr=8e-5, weight_decay=1e-4, extra_balanced_fraction=.5,
    small_fine_loss_weight=.35, ema_decay=.98, boundary_loss_weight=.2,
    checkpoint_selection='fixed_final_EMA_no_holdout_model_selection',
    fine_probability_threshold=.75, fine_margin_threshold=.25, parent_probability_threshold=.75,
    small_fine_probability_threshold=.85, prototype_cosine_minimum=.65,
    thresholds_are_heuristics_not_calibrated_quality_probabilities=True)
GENERIC = PROJECT/'outputs/pallet_mask_model_training_v1/generic_pretraining/resnet18-f37072fd.pth'
RUN = ROOT/'run_v2_bfloat16'
CONFIG['precision'] = 'bfloat16_no_gradient_scaling'


class TaxonomyMaskNet(ROIMaskBoundaryUNet):
    """Shared pixel-only backbone; heads do not receive class IDs or true K."""
    def __init__(self, fine_count=41, parent_count=9, encoder_weights=None):
        super().__init__(encoder_weights)
        self.class_features = nn.Sequential(nn.Conv2d(512,64,1), nn.GroupNorm(8,64), nn.ReLU(),
            nn.AdaptiveAvgPool2d((2,8)), nn.Flatten(), nn.Linear(1024,256), nn.ReLU(), nn.Dropout(.15))
        self.fine_head = nn.Linear(256, fine_count)
        self.parent_head = nn.Linear(256, parent_count)
        self.legs_head = nn.Linear(256,3)
        self.beams_head = nn.Linear(256,2)
        self.material_head = nn.Linear(256,3)

    def forward(self,x):
        raw = self.raw(x); stem = self.encoder.relu(self.encoder.bn1(self.encoder.conv1(x)))
        a = self.encoder.layer1(self.encoder.maxpool(stem)); b = self.encoder.layer2(a)
        c = self.encoder.layer3(b); d = self.encoder.layer4(c)
        embedding = self.class_features(d)
        y = self.up4(self.join(d,c)); y = self.up3(self.join(y,b))
        y = self.up2(self.join(y,a)); y = self.up1(self.join(y,stem)); y = self.up0(self.join(y,raw))
        return dict(mask=self.mask_head(y)[:,0], boundary=self.boundary_head(y)[:,0],
            fine=self.fine_head(embedding), parent=self.parent_head(embedding),
            legs=self.legs_head(embedding), beams=self.beams_head(embedding), material=self.material_head(embedding),
            embedding=F.normalize(embedding,dim=1))


def class_tables():
    classes = read(ROOT/'classes.json'); parents = read(ROOT/'parents.json')
    fine = sorted([c for c in classes.values() if c['train_enabled']], key=lambda c:c['fine_index'])
    ordered = sorted(parents.values(), key=lambda p:p['index'])
    return fine, ordered


class Inputs:
    def __init__(self,records):
        self.records = records; self.items = []
        classes, parents = class_tables(); materials = ['塑料','木质','金属']
        self.targets = []
        for r in records:
            grey = cv2.imread(str(ROOT/r['image']),0); mask = (cv2.imread(str(ROOT/r['mask']),0)>0).astype(np.uint8)
            valid = (cv2.imread(str(ROOT/r['valid']),0)>0).astype(np.uint8)
            image, target, canvas, transform = letterbox(grey,mask,tuple(CONFIG['input_size']))
            _, weight, _, _ = letterbox(grey,valid,tuple(CONFIG['input_size']))
            edge = target - cv2.erode(target,np.ones((3,3),np.uint8),borderType=cv2.BORDER_CONSTANT,borderValue=0)
            # Keep uint8 in RAM; normalized float input is made only per batch.
            self.items.append((image,target,weight,canvas,edge,transform,mask))
            parent = parents[r['parent_index']]
            self.targets.append([r['fine_index'],r['parent_index'],parent['legs']-2,parent['beams']-1,materials.index(parent['material'])])

    def batch(self,indices):
        tensors = [torch.from_numpy(np.stack([self.items[i][j] for i in indices])).cuda() for j in range(5)]
        grey = tensors[0][:,None].float()/255
        mean = grey.new_tensor([.485,.456,.406])[None,:,None,None]; std = grey.new_tensor([.229,.224,.225])[None,:,None,None]
        image = (grey.expand(-1,3,-1,-1)-mean)/std
        return image, *[t.float() for t in tensors[1:]], torch.tensor([self.targets[i] for i in indices],device='cuda')


def sample_indices(records,epoch):
    rng = np.random.default_rng(42+epoch); groups = defaultdict(list)
    for i,r in enumerate(records): groups[r['parent_index'],r['component']].append(i)
    # Guaranteed base coverage, plus balanced parent/association examples.
    base = list(rng.permutation(len(records))); extra = []
    keys = sorted(groups)
    for _ in range(round(len(records)*CONFIG['extra_balanced_fraction'])):
        parent = int(rng.choice(sorted({k[0] for k in keys})))
        options = [k for k in keys if k[0]==parent]; group = options[int(rng.integers(len(options)))]
        extra.append(int(rng.choice(groups[group])))
    result = base+extra; rng.shuffle(result); return result


def loss(output,target,valid,edge,labels,fine_weights,small_weights):
    with torch.autocast('cuda',enabled=False):
        logits = output['mask'].float(); probability = logits.sigmoid()*valid
        bce = (F.binary_cross_entropy_with_logits(logits,target,reduction='none')*valid).sum()/valid.sum().clamp_min(1)
        dice = 1-(2*(probability*target).sum((1,2))+1)/((probability+target*valid).sum((1,2))+1)
        boundary = (F.binary_cross_entropy_with_logits(output['boundary'].float(),edge,reduction='none',pos_weight=logits.new_tensor(4.))*valid).sum()/valid.sum().clamp_min(1)
        fine = F.cross_entropy(output['fine'].float(),labels[:,0],weight=fine_weights,reduction='none')
        fine = (fine*small_weights[labels[:,0]]).mean()
        parent = F.cross_entropy(output['parent'].float(),labels[:,1])
        attributes = sum(F.cross_entropy(output[name].float(),labels[:,j]) for j,name in ((2,'legs'),(3,'beams'),(4,'material')))/3
        total = bce+dice.mean()+CONFIG['boundary_loss_weight']*boundary+fine+parent+.5*attributes
    return total, dict(mask_bce=float(bce.detach()),dice=float(dice.mean().detach()),boundary=float(boundary.detach()),fine=float(fine.detach()),parent=float(parent.detach()),attributes=float(attributes.detach()))


@torch.inference_mode()
def predictions(model,inputs,indices):
    model.eval(); results=[]
    for offset in range(0,len(indices),CONFIG['batch_size']):
        batch = indices[offset:offset+CONFIG['batch_size']]; x,*_ = inputs.batch(batch)
        with torch.autocast('cuda',dtype=torch.bfloat16): output = model(x)
        for j,i in enumerate(batch):
            r=inputs.records[i]; transform=inputs.items[i][5]; truth=inputs.items[i][6]
            predicted=unletterbox(output['mask'][j].float().sigmoid().cpu().numpy(),transform)>=.5
            fine=output['fine'][j].float().softmax(0).cpu().numpy(); parent=output['parent'][j].float().softmax(0).cpu().numpy()
            results.append(dict(target_key=r['target_key'],class_id=r['class_id'],fine_index=r['fine_index'],parent_index=r['parent_index'],component=r['component'],
                fine_probabilities=fine.tolist(),parent_probabilities=parent.tolist(),embedding=output['embedding'][j].float().cpu().tolist(),
                predicted_fine_index=int(fine.argmax()),predicted_parent_index=int(parent.argmax()),**native_metrics(predicted,truth)))
    return results


def summarize_classification(results):
    return dict(records=len(results), fine_top1=sum(r['predicted_fine_index']==r['fine_index'] for r in results)/max(1,len(results)),
        parent_top1=sum(r['predicted_parent_index']==r['parent_index'] for r in results)/max(1,len(results)),
        segmentation=summarize(results), per_class={str(c):dict(count=len([r for r in results if r['class_id']==c]),
            fine_top1=sum(r['predicted_fine_index']==r['fine_index'] for r in results if r['class_id']==c)/len([r for r in results if r['class_id']==c])) for c in sorted({r['class_id'] for r in results})})


def run_phase(name,model,records,epochs):
    directory=RUN/name; directory.mkdir(mode=0o700); inputs=Inputs(records)
    classes,_=class_tables(); counts=Counter(r['fine_index'] for r in records)
    weights=np.array([min(3.,math.sqrt(len(records)/41/max(1,counts[c['fine_index']]))) for c in classes],np.float32)
    weights/=np.mean([weights[r['fine_index']] for r in records])
    fine_weights=torch.from_numpy(weights).cuda(); small_weights=torch.tensor([CONFIG['small_fine_loss_weight'] if c['small_sample'] else 1. for c in classes],device='cuda')
    groups=[dict(params=model.encoder.parameters(),lr=CONFIG['encoder_lr']),
        dict(params=[p for n,p in model.named_parameters() if not n.startswith('encoder.')],lr=CONFIG['lr'])]
    optimizer=torch.optim.AdamW(groups,weight_decay=CONFIG['weight_decay']); scaler=torch.amp.GradScaler('cuda',enabled=False); ema=deepcopy(model).eval()
    for p in ema.parameters():p.requires_grad_(False)
    started=time.monotonic(); steps=0; history=[]
    for epoch in range(epochs):
        model.train(); indices=sample_indices(records,epoch); sums=Counter(); epoch_steps=0
        fraction=(1+math.cos(math.pi*epoch/epochs))/2
        for group,base in zip(optimizer.param_groups,[CONFIG['encoder_lr'],CONFIG['lr']]):group['lr']=base*(.1+.9*fraction)
        for offset in range(0,len(indices),CONFIG['batch_size']):
            batch=indices[offset:offset+CONFIG['batch_size']]
            x,target,valid,canvas,edge,labels=inputs.batch(batch); x=augment(x,canvas[:,None])
            optimizer.zero_grad(set_to_none=True)
            with torch.autocast('cuda',dtype=torch.bfloat16): output=model(x); total,values=loss(output,target,valid,edge,labels,fine_weights,small_weights)
            require(torch.isfinite(total).item(),'Nonfinite joint training loss')
            scaler.scale(total).backward(); scaler.unscale_(optimizer); norm=torch.nn.utils.clip_grad_norm_(model.parameters(),5.)
            require(torch.isfinite(norm).item(),'Nonfinite training gradient')
            scaler.step(optimizer);scaler.update();update_ema(ema,model,CONFIG['ema_decay'])
            steps+=1;epoch_steps+=1;sums.update(values)
        entry=dict(epoch=epoch+1,optimizer_steps=steps,epoch_updates=epoch_steps,samples=len(indices),
            target_exposure_sha256=sha([records[i]['target_key'] for i in indices]),
            class_exposures=dict(Counter(records[i]['class_id'] for i in indices)),
            component_exposures=dict(Counter(records[i]['component'] for i in indices)),
            mean_losses={k:v/epoch_steps for k,v in sums.items()},elapsed_seconds=time.monotonic()-started)
        write(directory/f'epoch_{epoch+1:03d}.json',entry); history.append(entry)
        if epoch+1 in (1,epochs) or (epoch+1)%4==0:
            print(dict(phase=name,epoch=epoch+1,updates=steps,seconds=round(time.monotonic()-started),losses=entry['mean_losses']),flush=True)
        require(shutil.disk_usage(ROOT).free>10*2**30,'Disk reserve reached; preserve old files')
    atomic_torch(directory/'final_ema.pt',dict(model={k:v.detach().cpu() for k,v in ema.state_dict().items()},
        dataset_manifest_sha256=digest(ROOT/'dataset_manifest.json'),phase=name,config=CONFIG,
        optimization_steps=steps,trained_targets=[r['target_key'] for r in records]))
    write(directory/'training_receipt.json',dict(phase=name,epochs=epochs,optimization_steps=steps,records=len(records),
        components=len({r['component'] for r in records}),checkpoint_sha256=digest(directory/'final_ema.pt'),
        elapsed_seconds=time.monotonic()-started,all_targets_exposed_every_epoch=True,
        original_small_fine_classes_preserved=True,shared_parent_and_mask_learning=True,history_sha256=sha(history)))
    model.load_state_dict(ema.state_dict()); del optimizer,ema,scaler;torch.cuda.empty_cache()
    return model,inputs


def train():
    require(not RUN.exists(),'Pinned run cannot be silently restarted or overwritten')
    _,records=verify();require(sources_current(),'Latest taxonomy source changed')
    require(GENERIC.exists() and digest(GENERIC).startswith('f37072fd'),'Official generic ResNet18 starting weights required')
    gpu=gpu_receipt('m2');torch.set_num_threads(4);cv2.setNumThreads(1);seed_all()
    require(torch.cuda.is_bf16_supported(),'BF16-capable authorized device required')
    RUN.mkdir(mode=0o700); snapshots=RUN/'code_snapshot';snapshots.mkdir(mode=0o700)
    paths=[PROJECT/'pallet'/n for n in ('mask_taxonomy_data.py','mask_taxonomy_train.py','mask_model_training.py','mask_annotation_full_train.py','mask_model_training_data.py')]
    for p in paths:shutil.copyfile(p,snapshots/p.name)
    write(RUN/'execution_plan.json',dict(config=CONFIG,gpu=gpu,generic_weights_sha256=digest(GENERIC),generic_weights=str(GENERIC.relative_to(PROJECT)),
        prior_attempt_failure_preserved=str((ROOT/'training_failure.json').relative_to(PROJECT)),
        old_pallet_checkpoints_used=False,dataset_manifest_sha256=digest(ROOT/'dataset_manifest.json'),
        code_sha256={str(p.relative_to(PROJECT)):digest(p) for p in paths},
        evaluation='fixed_component_holdout_from_generic_only_frozen_before_full_pool',
        production='all2198_explicit_user_authorized_contours_after_independent_evaluation',
        model_inputs='original grayscale pixels in original-bbox fixed context; no IDs/classes/K/pose/group/metadata',**FALSE_FLAGS))
    try:
        model=TaxonomyMaskNet(encoder_weights=GENERIC).cuda()
        train_records=[r for r in records if r['evaluation_branch_split']=='train']
        model,_=run_phase('independent_branch',model,train_records,CONFIG['evaluation_epochs'])
        all_inputs=Inputs(records); holdout_indices=[i for i,r in enumerate(records) if r['evaluation_branch_split']=='holdout']
        held=predictions(model,all_inputs,holdout_indices)
        write(RUN/'independent_branch/holdout_predictions.jsonl',held,lines=True)
        missing=sorted({r['class_id'] for r in records}-{r['class_id'] for r in train_records})
        write(RUN/'independent_branch/holdout_metrics.json',dict(scope='component_holdout_generic_branch_before_full_pool_training',
            checkpoint_sha256=digest(RUN/'independent_branch/final_ema.pt'),old_val_test_not_claimed_independent=True,
            no_old_pallet_checkpoint=True,classes_absent_from_training=missing,
            split_chosen_without_holdout_labels=True,thresholds_not_tuned_on_holdout=True,
            **summarize_classification(held)))
        print(dict(phase='independent_holdout_frozen',**summarize_classification(held)),flush=True)
        del all_inputs
        require(sources_current(),'Sources changed before full-pool phase')
        model,inputs=run_phase('full_pool',model,records,CONFIG['full_pool_epochs'])
        fitted=predictions(model,inputs,list(range(len(records))))
        write(RUN/'full_pool/fit_predictions.jsonl',fitted,lines=True)
        write(RUN/'full_pool/fit_metrics.json',dict(scope='resubstitution_only_NOT_independent_accuracy',**summarize_classification(fitted)))
        prototypes=[]
        for cls in range(41):
            vectors=np.array([r['embedding'] for r in fitted if r['fine_index']==cls],np.float32)
            center=vectors.mean(0);center/=max(np.linalg.norm(center),1e-12);prototypes.append(center.tolist())
        write(RUN/'full_pool/prototypes.json',dict(centers=prototypes,scope='full_pool_training_embeddings_only',
            leave_one_out_or_test_accuracy_not_claimed=True,checkpoint_sha256=digest(RUN/'full_pool/final_ema.pt')))
        require(sources_current(),'Live sources changed during training; do not publish stale tasks')
        write(RUN/'training_complete.json',dict(state='actual_new41fine9parent_joint_mask_training_complete',
            independent_branch=digest(RUN/'independent_branch/final_ema.pt'),full_pool=digest(RUN/'full_pool/final_ema.pt'),
            independent_metrics_sha256=digest(RUN/'independent_branch/holdout_metrics.json'),
            fit_metrics_sha256=digest(RUN/'full_pool/fit_metrics.json'),gpu=gpu,config=CONFIG,
            original_datasets_models_services_unchanged=True,**FALSE_FLAGS))
        print(dict(state='taxonomy_joint_training_complete',checkpoint_sha256=digest(RUN/'full_pool/final_ema.pt')),flush=True)
    except Exception:
        write(RUN/'training_failure.json',dict(traceback=traceback.format_exc(),sources_preserved=True));raise
