"""Batched partial-label LoRA with complete JSON delimiter/EOS supervision."""
from collections import Counter
import gc
import json
import random
import shutil
import time
import traceback

import torch
from PIL import Image

from .cluster_targets import digest
from .mask_model_training_data import PROJECT,read,write,jsonl,sha,require,gpu_receipt
from .pallet_description_round4_data import ROOT,PREVIOUS,PROMPT,verify,sources_current
from .pallet_description_vlm import load
from .pallet_description_output import decode_output

RUN=ROOT/'vlm_run_v1'
FIELDS=('支腿','横梁','材质','结构','观察')
CONFIG=dict(epochs=2,batch_size=4,accumulation=2,learning_rate=1e-4,LoRA_rank=8,max_new_tokens=256,seed=44,
    vision_frozen=True,missing_or_conflicting_field_value_loss_masked=True,structural_delimiters_and_EOS_supervised=True)


def answer_spans(target):
    text='{';spans={}
    for i,key in enumerate(FIELDS):
        if i:text+=','
        text+=json.dumps(key,ensure_ascii=False)+':';start=len(text)
        text+=json.dumps(target[key],ensure_ascii=False,separators=(',',':'));spans[key]=(start,len(text))
    return text+'}',spans


def parse(raw):
    try:value=json.loads(raw.strip())
    except (ValueError,TypeError):return dict(valid=False,fields=None,reason='invalid_JSON',raw=raw)
    valid=isinstance(value,dict) and set(value)==set(FIELDS)
    if valid:
        valid=(type(value['支腿']) in (int,type(None)) and value['支腿'] in (None,2,3,4)
            and type(value['横梁']) in (int,type(None)) and value['横梁'] in (None,1,2)
            and value['材质'] in ('塑料','木质','金属','无法判断') and isinstance(value['结构'],str)
            and (value['观察'] is None or isinstance(value['观察'],str)))
    return dict(valid=valid,fields=value if valid else None,reason=None if valid else 'schema_mismatch',raw=raw)


def image_for(r):
    if 'image' in r and (ROOT/r['image']).exists():return Image.open(ROOT/r['image']).convert('RGB')
    from .mask_annotation_full_data import context
    formal=PROJECT/'outputs/pallet_dataset_formal_v1/dataset'
    return Image.open(formal/r['source']['image']).convert('RGB').crop(context(r['source']['bbox'],r['source']['size']))


def messages(image,answer=None,prompt=PROMPT):
    value=[dict(role='user',content=[dict(type='image',image=image),dict(type='text',text=prompt)])]
    if answer is not None:value.append(dict(role='assistant',content=[dict(type='text',text=answer)]))
    return value


def training_batch(processor,records):
    images=[image_for(r) for r in records];answers=[answer_spans(r['target']) for r in records]
    def texts(full):
        return [processor.apply_chat_template(messages(im,a[0] if full else None),tokenize=False,add_generation_prompt=not full,enable_thinking=False) for im,a in zip(images,answers)]
    full=processor(text=texts(True),images=images,return_tensors='pt',padding=True)
    prompt=processor(text=texts(False),images=images,return_tensors='pt',padding=True)
    labels=full['input_ids'].clone();labels[full['attention_mask']==0]=-100
    audits=[]
    for j,(r,(answer,spans)) in enumerate(zip(records,answers)):
        start=int(prompt['attention_mask'][j].sum());require(torch.equal(full['input_ids'][j,:start],prompt['input_ids'][j,:start]),'Prompt/full prefix mismatch')
        labels[j,:start]=-100
        encoded=processor.tokenizer(answer,add_special_tokens=False,return_offsets_mapping=True)
        ids=torch.tensor(encoded['input_ids'],dtype=full['input_ids'].dtype)
        require(torch.equal(full['input_ids'][j,start:start+len(ids)],ids),'Tokenizer answer alignment differs')
        masked=[]
        for key,(left,right) in spans.items():
            if not r['supervised_fields'][key]:
                for i,(a,b) in enumerate(encoded['offset_mapping']):
                    if a<right and b>left:labels[j,start+i]=-100;masked.append(i)
        require((labels[j]!=-100).any(),'All semantic and format labels masked')
        audits.append(dict(target_key=r['target_key'],prompt_tokens=start,answer_tokens=len(ids),masked_value_tokens=sorted(set(masked)),
            eos_supervised=bool((labels[j,start+len(ids):]!=-100).any()),supervised_fields=r['supervised_fields']))
    return full.to('cuda'),labels.to('cuda'),audits


@torch.inference_mode()
def predict_batch(model,processor,records,prompt=PROMPT):
    images=[image_for(r) for r in records]
    texts=[processor.apply_chat_template(messages(im,prompt=prompt),tokenize=False,add_generation_prompt=True,enable_thinking=False) for im in images]
    data=processor(text=texts,images=images,return_tensors='pt',padding=True).to('cuda');started=time.perf_counter()
    ids=model.generate(**data,max_new_tokens=CONFIG['max_new_tokens'],do_sample=False,pad_token_id=processor.tokenizer.pad_token_id)
    torch.cuda.synchronize();ms=1000*(time.perf_counter()-started);tail=ids[:,data['input_ids'].shape[1]:]
    texts=processor.tokenizer.batch_decode(tail,skip_special_tokens=True);result=[]
    for r,raw,tokens in zip(records,texts,tail):
        p=parse(raw);recovered=decode_output(raw)
        result.append(dict(target_key=r['target_key'],image_sha256=r['image_sha256'],**p,
            recovered_fields=recovered.get('recovered_fields') if not p['valid'] else None,
            structure_recovered=bool(not p['valid'] and recovered.get('structure_recovered')),
            observation_withheld=not p['valid'],generated_tokens=int((tokens!=processor.tokenizer.pad_token_id).sum()),
            hit_token_limit=int((tokens!=processor.tokenizer.pad_token_id).sum())>=CONFIG['max_new_tokens'],batch_ms=ms,batch_size=len(records),prompt_sha256=sha(prompt)))
    return result


def score(predictions,records):
    bykey={r['target_key']:r for r in records};defined=Counter();correct=Counter();observation_reference=0
    for p in predictions:
        r=bykey[p['target_key']]
        for key in FIELDS[:3]:
            if r['supervised_fields'][key]:defined[key]+=1;correct[key]+=bool(p['valid'] and p['fields'][key]==r['target'][key])
        observation_reference+=r['supervised_fields']['观察']
    return dict(targets=len(predictions),strict_complete_JSON_valid=sum(p['valid'] for p in predictions),
        strict_complete_JSON_fraction=sum(p['valid'] for p in predictions)/max(1,len(predictions)),
        attribute_exact={k:dict(correct=correct[k],labelled=defined[k],accuracy=correct[k]/max(1,defined[k])) for k in FIELDS[:3]},
        token_limit_hits=sum(p['hit_token_limit'] for p in predictions),human_observation_reference_targets=observation_reference,
        observation_correctness_not_inferred_from_JSON_validity=True,no_normal_negative_labels_available=True,
        semantic_observation_generalization_certified=False)


def evaluate(model,processor,records,folder,prompt=PROMPT):
    processor.tokenizer.padding_side='left';preds=[];started=time.monotonic()
    for i in range(0,len(records),16):
        preds.extend(predict_batch(model,processor,records[i:i+16],prompt))
        if i%256==0:print(dict(evaluation=str(folder),inferred=len(preds),total=len(records),seconds=round(time.monotonic()-started)),flush=True)
    write(folder/'holdout_predictions.jsonl',preds,lines=True);metrics=score(preds,records)
    write(folder/'holdout_metrics.json',dict(scope='same_component_holdout_of_latest_partial_human_semantics_not_new_blind_test',prompt_sha256=sha(prompt),**metrics))
    return metrics


def phase(name,records,initial=None):
    from peft import LoraConfig,get_peft_model,PeftModel
    folder=RUN/name;require(not folder.exists(),'Preserve previous VLM phase');folder.mkdir(mode=0o700)
    processor,base=load('qwen');processor.tokenizer.padding_side='right'
    if initial:model=PeftModel.from_pretrained(base,initial,is_trainable=True)
    else:
        targets=[n for n,m in base.named_modules() if isinstance(m,torch.nn.Linear) and ('language_model' in n or 'text_model' in n) and n.endswith(('q_proj','k_proj','v_proj','o_proj','in_proj_qkv','out_proj','gate_proj','up_proj','down_proj'))]
        model=get_peft_model(base,LoraConfig(r=8,lora_alpha=16,lora_dropout=.05,target_modules=targets,bias='none',task_type='CAUSAL_LM'))
    model.config.use_cache=False;model.gradient_checkpointing_enable();model.enable_input_require_grads()
    params=[p for p in model.parameters() if p.requires_grad];require(params,'No trainable LoRA parameters')
    optimizer=torch.optim.AdamW(params,lr=CONFIG['learning_rate'],weight_decay=.01);steps=0;started=time.monotonic();history=[]
    write(folder/'execution_plan.json',dict(config=CONFIG,records=len(records),trainable_parameters=sum(p.numel() for p in params),
        initial_adapter_sha256=digest(initial/'adapter_model.safetensors') if initial else None,independent_base_start=initial is None,
        prompt_sha256=sha(PROMPT),dataset_manifest_sha256=digest(ROOT/'dataset_manifest.json'),source_code_sha256=digest(__file__)))
    for epoch in range(CONFIG['epochs']):
        order=list(records);random.Random(CONFIG['seed']+epoch).shuffle(order);model.train();optimizer.zero_grad();losses=[]
        batches=[order[i:i+CONFIG['batch_size']] for i in range(0,len(order),CONFIG['batch_size'])]
        for i,group in enumerate(batches):
            full,labels,audits=training_batch(processor,group)
            if epoch==0 and i==0:write(folder/'token_alignment_audit.json',audits)
            loss=model(**full,labels=labels).loss;require(torch.isfinite(loss).item(),'Nonfinite VLM loss');losses.append(float(loss.detach()))
            # Normalize the final partial accumulation group by its actual batch count.
            scale=min(CONFIG['accumulation'],len(batches)-(i//CONFIG['accumulation'])*CONFIG['accumulation'])
            (loss/scale).backward()
            if (i+1)%CONFIG['accumulation']==0 or i+1==len(batches):
                norm=torch.nn.utils.clip_grad_norm_(params,1.);require(torch.isfinite(norm).item(),'Nonfinite VLM gradient');optimizer.step();optimizer.zero_grad();steps+=1
            if (i+1)%50==0:print(dict(phase=name,epoch=epoch+1,samples=min((i+1)*CONFIG['batch_size'],len(order)),total=len(order),steps=steps,seconds=round(time.monotonic()-started),loss=round(sum(losses[-50:])/50,4)),flush=True)
        model.save_pretrained(folder/f'epoch_{epoch+1:03d}_adapter')
        entry=dict(epoch=epoch+1,records=len(order),optimizer_steps=steps,mean_loss=sum(losses)/len(losses),exposure_sha256=sha([r['target_key'] for r in order]),seconds=time.monotonic()-started)
        write(folder/f'epoch_{epoch+1:03d}.json',entry);history.append(entry)
    model.save_pretrained(folder/'adapter');processor.save_pretrained(folder/'adapter');model.eval();model.config.use_cache=True
    write(folder/'training_receipt.json',dict(records=len(records),epochs=CONFIG['epochs'],optimizer_steps=steps,seconds=time.monotonic()-started,
        adapter_sha256=digest(folder/'adapter/adapter_model.safetensors'),history_sha256=sha(history),vision_frozen=True,all_targets_exposed_every_epoch=True))
    del optimizer;torch.cuda.empty_cache();return model,processor


def train():
    from peft import PeftModel
    require(not RUN.exists(),'Preserve prior VLM attempt');verify();require(sources_current(),'Source changed before VLM training')
    RUN.mkdir(mode=0o700);(RUN/'code_snapshot').mkdir(mode=0o700)
    for n in ('pallet_description_round4_data.py','pallet_description_round4_vlm.py'):
        shutil.copyfile(PROJECT/'pallet'/n,RUN/'code_snapshot'/n)
    gpu=gpu_receipt('m1');torch.set_num_threads(4);random.seed(CONFIG['seed']);torch.manual_seed(CONFIG['seed'])
    records=[r for r in jsonl(ROOT/'description_data/records.jsonl') if any(r['supervised_fields'].values())]
    subset=[r for r in records if r['split']=='train'];held=[r for r in records if r['split']=='holdout']
    require(not {r['component'] for r in subset}&{r['component'] for r in held},'VLM component leakage')
    write(RUN/'execution_plan.json',dict(config=CONFIG,gpu=gpu,train_targets=len(subset),holdout_targets=len(held),full_pool_targets=len(records),
        independent_branch_from_pinned_pretrained_base=True,full_pool_continues_previous_full_adapter=True,conflicts_masked_not_template_replaced=True,
        dataset_manifest_sha256=digest(ROOT/'dataset_manifest.json'),prompt_sha256=sha(PROMPT),no_class_K_polygon_or_note_input=True))
    try:
        old=RUN/'old_component_comparison';old.mkdir(mode=0o700);processor,base=load('qwen')
        model=PeftModel.from_pretrained(base,PREVIOUS/'vlm_run_v2_format_repair/component_branch/adapter').merge_and_unload().eval()
        from .pallet_description_supervision import PROMPT as oldprompt
        evaluate(model,processor,held,old,oldprompt);del model,processor,base;gc.collect();torch.cuda.empty_cache()
        model,processor=phase('component_branch',subset);metrics=evaluate(model,processor,held,RUN/'component_branch')
        print(dict(phase='round4_vlm_holdout',**metrics),flush=True)
        del model,processor;gc.collect();torch.cuda.empty_cache();require(sources_current(),'Source changed before full VLM phase')
        model,processor=phase('full_pool',records,PREVIOUS/'vlm_run_v2_format_repair/full_pool/adapter')
        require(sources_current(),'Source changed during VLM training')
        write(RUN/'training_complete.json',dict(state='round4_actual_partial_human_semantic_LoRA_complete',gpu=gpu,model='Qwen3.5-0.8B',
            adapter_sha256=digest(RUN/'full_pool/adapter/adapter_model.safetensors'),component_metrics_sha256=digest(RUN/'component_branch/holdout_metrics.json'),
            config=CONFIG,full_pool_targets=len(records),semantics_not_certified_by_format_alone=True))
    except Exception:
        write(RUN/'failure.json',dict(traceback=traceback.format_exc()));raise


def infer(mode):
    from peft import PeftModel
    require((RUN/'training_complete.json').exists(),'VLM training must finish first');require(sources_current(),'Source changed before inference')
    out=ROOT/f'descriptions_{mode}.jsonl';require(not out.exists(),'Preserve VLM predictions')
    records=jsonl(ROOT/('private/release_selected.jsonl' if mode=='new' else 'private/semantic_review_selected.jsonl'))
    gpu=gpu_receipt('m1');torch.set_num_threads(4);processor,base=load('qwen')
    adapter=RUN/'full_pool/adapter';adapter_sha=digest(adapter/'adapter_model.safetensors')
    require(adapter_sha==read(RUN/'training_complete.json')['adapter_sha256'],'Changed VLM adapter')
    model=PeftModel.from_pretrained(base,adapter).merge_and_unload().eval();processor.tokenizer.padding_side='left';preds=[]
    parts=ROOT/f'description_parts_{mode}';parts.mkdir(mode=0o700,exist_ok=True)
    for offset in range(0,len(records),64):
        path=parts/f'part_{offset:06d}.json';group=records[offset:offset+64]
        if path.exists():
            part=read(path);require(part['targets']==[r['target_key'] for r in group] and part['adapter_sha256']==adapter_sha and part['code_sha256']==digest(__file__),'Shard differs')
        else:
            ps=[]
            for j in range(0,len(group),16):ps.extend(predict_batch(model,processor,group[j:j+16]))
            for p in ps:p.update(adapter_sha256=adapter_sha,status='machine_draft',model='Qwen3.5-0.8B-round4-LoRA')
            part=dict(targets=[r['target_key'] for r in group],predictions=ps,adapter_sha256=adapter_sha,code_sha256=digest(__file__));write(path,part)
        preds.extend(part['predictions'])
    write(out,preds,lines=True);write(ROOT/f'description_inference_{mode}.json',dict(targets=len(preds),schema_valid=sum(p['valid'] for p in preds),
        structure_only_recovered=sum(p['structure_recovered'] for p in preds),observation_withheld=sum(p['observation_withheld'] for p in preds),
        adapter_sha256=adapter_sha,gpu=gpu,no_label_features_as_input=True,actual_model_raw_retained=True))


if __name__=='__main__':
    import argparse
    p=argparse.ArgumentParser();p.add_argument('phase',choices=('train','infer-new','infer-semantic'));a=p.parse_args()
    train() if a.phase=='train' else infer(a.phase.split('-')[1])
