"""One-epoch format repair: mask missing observation VALUE, not structural delimiters/EOS."""
import gc
import json
import random
import time

import torch
from peft import PeftModel

from .cluster_targets import digest
from .mask_model_training_data import read,write,jsonl,sha,require,gpu_receipt
from .pallet_description_data import ROOT,sources_current
from .pallet_description_vlm import load,inputs,score,RUN as PREVIOUS

RUN=ROOT/'vlm_run_v2_format_repair'


def label_mask(processor,full,prompt,answer,annotated):
    start=prompt['input_ids'].shape[1];labels=full['input_ids'].clone()
    require(torch.equal(full['input_ids'][:,:start],prompt['input_ids']),'Training/generation prefix mismatch')
    labels[:,:start]=-100
    if not annotated:
        marker='"观察":';pos=answer.index(marker)+len(marker);require(answer[pos:pos+4]=='null','Missing observation must be an unlabelled value')
        # Mask any tokens overlapping the null value, including boundary-merged tokens.
        encoded=processor.tokenizer(answer,add_special_tokens=False,return_offsets_mapping=True)
        for i,(a,b) in enumerate(encoded['offset_mapping']):
            if a<pos+4 and b>pos:labels[:,start+i]=-100
    return labels


def phase(name,records):
    folder=RUN/name;require(not folder.exists(),'Preserve previous repair attempt');folder.mkdir(mode=0o700)
    processor,base=load('qwen');initial=PREVIOUS/name/'adapter';model=PeftModel.from_pretrained(base,initial,is_trainable=True)
    model.gradient_checkpointing_enable();model.enable_input_require_grads();model.config.use_cache=False
    params=[p for p in model.parameters() if p.requires_grad];optimizer=torch.optim.AdamW(params,lr=5e-5,weight_decay=.01)
    write(folder/'execution_plan.json',dict(epochs=1,records=len(records),masked_missing_observations=True,frozen_vision=True,
        initial_adapter_sha256=digest(initial/'adapter_model.safetensors'),same_component_branch_no_full_pool_warm_start=name=='component_branch',
        label_mask='only_tokens_overlapping_missing_null_value_keep_structural_terminators_and_EOS',source_code_sha256=digest(__file__),trainable_parameters=sum(p.numel() for p in params)))
    order=list(records);random.Random(43).shuffle(order);optimizer.zero_grad();steps=0;loss_sum=0.;started=time.monotonic();model.train()
    for i,r in enumerate(order):
        answer=json.dumps(r['target'],ensure_ascii=False,separators=(',',':'));full=inputs(processor,r,answer);prompt=inputs(processor,r)
        labels=label_mask(processor,full,prompt,answer,r['observation_supervised']);loss=model(**full,labels=labels).loss
        require(torch.isfinite(loss).item(),'Nonfinite format-repair loss');loss_sum+=float(loss.detach());(loss/8).backward()
        if (i+1)%8==0 or i+1==len(order):
            norm=torch.nn.utils.clip_grad_norm_(params,1.);require(torch.isfinite(norm).item(),'Nonfinite repair gradient');optimizer.step();optimizer.zero_grad();steps+=1
        if (i+1)%200==0:print(dict(phase=name,format_repair_samples=i+1,total=len(order),steps=steps,seconds=round(time.monotonic()-started)),flush=True)
    model.save_pretrained(folder/'adapter');processor.save_pretrained(folder/'adapter');model.eval();model.config.use_cache=True
    entry=dict(epoch=1,steps=steps,samples=len(order),mean_loss=loss_sum/len(order),exposure_sha256=sha([r['target_key'] for r in order]),seconds=time.monotonic()-started)
    write(folder/'epoch_001.json',entry);write(folder/'training_receipt.json',dict(optimizer_steps=steps,epochs=1,records=len(records),history_sha256=sha([entry]),seconds=time.monotonic()-started,
        adapter_sha256=digest(folder/'adapter/adapter_model.safetensors'),vision_frozen=True,source_code_sha256=digest(__file__)))
    return model,processor


def train():
    require(not RUN.exists(),'Fresh format-repair run required');RUN.mkdir(mode=0o700);gpu=gpu_receipt('m1');torch.set_num_threads(4);torch.manual_seed(43)
    records=jsonl(ROOT/'description_data/pilot.jsonl');subset=[r for r in records if r['split']=='train'];held=[r for r in records if r['split']=='holdout']
    model,processor=phase('component_branch',subset);processor.tokenizer.padding_side='left'
    from scripts.infer_pallet_descriptions import batch_predict
    preds=[]
    for i in range(0,len(held),16):preds.extend(batch_predict(model,processor,held[i:i+16]))
    for p in preds:p['elapsed_ms']=p['batch_ms']/p['batch_size']
    write(RUN/'component_branch/holdout_predictions.jsonl',preds,lines=True)
    metrics=score(preds,held);write(RUN/'component_branch/holdout_metrics.json',dict(scope='same_component_holdout_after_format_repair_NOT_new_blind_test',latency_policy='batch_average_not_single_image_latency',**metrics))
    print(dict(phase='format_repair_holdout',**metrics),flush=True)
    del model,processor;gc.collect();torch.cuda.empty_cache();model,processor=phase('full_pool',records)
    require(sources_current(),'Sources changed during repair')
    write(RUN/'training_complete.json',dict(state='actual_tiny_VLM_format_repair_complete',gpu=gpu,model='Qwen3.5-0.8B',
        adapter_sha256=digest(RUN/'full_pool/adapter/adapter_model.safetensors'),component_metrics_sha256=digest(RUN/'component_branch/holdout_metrics.json'),
        previous_attempt_fully_preserved=True,training_scope='same1211_target_balanced_pilot_one_extra_format_epoch',code_sha256=digest(__file__)))
    print(dict(state='actual_tiny_VLM_format_repair_complete'),flush=True)


if __name__=='__main__':train()
