"""Round5 LoRA: strings retain quote supervision even when observation is missing."""
import gc
import json
import random
import shutil
import time
import traceback

import torch

from .mask_model_training_data import PROJECT,read,write,jsonl,sha,require,gpu_receipt
from .mask_taxonomy_cycle2_data import engine
from .cluster_targets import digest
from .pallet_description_round5_data import ROOT,PREVIOUS,PROMPT,verify,sources_current
from .pallet_description_round4_vlm import FIELDS,answer_spans,parse
from .pallet_description_vlm import load

RUN=ROOT/'vlm_run_v1'
CONFIG=dict(epochs=2,batch_size=4,accumulation=2,learning_rate=1e-4,LoRA_rank=8,max_new_tokens=256,seed=45,
    vision_frozen=True,missing_or_conflicting_field_value_loss_masked=True,structural_delimiters_and_EOS_supervised=True,
    observation_always_JSON_string=True,missing_observation_empty_string_is_unlabelled_not_normal=True)
learning=engine('pallet/pallet_description_round4_vlm.py','round5_vlm_learning')
learning.ROOT=ROOT;learning.RUN=RUN;learning.CONFIG=CONFIG;learning.PROMPT=PROMPT;learning.sources_current=sources_current


def format_target(record):
    target=dict(record['target'])
    if not record['supervised_fields']['观察']:target['观察']=''
    else:require(isinstance(target['观察'],str) and bool(target['观察'].strip()),'Supervised observation must be real human text')
    return target


def mask_spans(record,spans):
    result=[]
    for field,(start,end) in spans.items():
        if not record['supervised_fields'][field]:
            # For missing observation the value is an empty string: no semantic
            # characters exist. Supervise both quotes, not a fabricated normal.
            if field=='观察':continue
            result.append((start,end))
    return result


def training_batch(processor,records):
    images=[learning.image_for(r) for r in records];answers=[answer_spans(format_target(r)) for r in records]
    def texts(full):return [processor.apply_chat_template(learning.messages(im,a[0] if full else None,prompt=PROMPT),tokenize=False,add_generation_prompt=not full,enable_thinking=False) for im,a in zip(images,answers)]
    full=processor(text=texts(True),images=images,return_tensors='pt',padding=True);prompt=processor(text=texts(False),images=images,return_tensors='pt',padding=True)
    labels=full['input_ids'].clone();labels[full['attention_mask']==0]=-100;audits=[]
    for j,(r,(answer,spans)) in enumerate(zip(records,answers)):
        start=int(prompt['attention_mask'][j].sum());require(torch.equal(full['input_ids'][j,:start],prompt['input_ids'][j,:start]),'Training/generation prefix differs');labels[j,:start]=-100
        encoded=processor.tokenizer(answer,add_special_tokens=False,return_offsets_mapping=True);ids=torch.tensor(encoded['input_ids'],dtype=full['input_ids'].dtype)
        require(torch.equal(full['input_ids'][j,start:start+len(ids)],ids),'Answer alignment differs');masked=[]
        for left,right in mask_spans(r,spans):
            for i,(a,b) in enumerate(encoded['offset_mapping']):
                if a<right and b>left:labels[j,start+i]=-100;masked.append(i)
        quote_tokens=[]
        if not r['supervised_fields']['观察']:
            left,right=spans['观察'];require(answer[left:right]=='""','Missing observation has invented semantic content')
            for i,(a,b) in enumerate(encoded['offset_mapping']):
                if a<right and b>left:quote_tokens.append(i);require(labels[j,start+i]!=-100,'Missing observation quotes were masked')
        audits.append(dict(target_key=r['target_key'],prompt_tokens=start,answer_tokens=len(ids),masked_value_tokens=sorted(set(masked)),
            eos_supervised=bool((labels[j,start+len(ids):]!=-100).any()),observation_quote_tokens_supervised=quote_tokens,
            missing_observation_has_zero_semantic_characters=not r['supervised_fields']['观察'],supervised_fields=r['supervised_fields']))
    return full.to('cuda'),labels.to('cuda'),audits


learning.training_batch=training_batch


def evaluate(model,processor,records,folder,prompt=PROMPT):
    metrics=learning.evaluate(model,processor,records,folder,prompt)
    preds=jsonl(folder/'holdout_predictions.jsonl');bykey={r['target_key']:r for r in records};stats={k:dict(labelled=0,correct=0,unrecoverable=0) for k in FIELDS[:3]}
    observed=0;missing=0
    for p in preds:
        fields=p['fields'] if p['valid'] else p.get('recovered_fields');ref=bykey[p['target_key']]
        observed+=bool(p['valid'] and p['fields'].get('观察'));missing+=bool(p['valid'] and not p['fields'].get('观察'))
        for key,value in stats.items():
            if ref['supervised_fields'][key]:value['labelled']+=1;value['correct']+=bool(fields and fields[key]==ref['target'][key]);value['unrecoverable']+=not bool(fields)
    for value in stats.values():value['accuracy']=value['correct']/max(1,value['labelled'])
    write(folder/'structure_recovery_audit.json',dict(attribute_exact_including_explicit_structure_recovery=stats,strict_valid_observation_nonempty=observed,
        strict_valid_observation_empty_unknown=missing,no_normal_negative_labels=True,raw_outputs_retained=True,not_observation_accuracy=True))
    return metrics


def train():
    from peft import PeftModel
    require(not RUN.exists(),'Preserve VLM attempt');verify();require(sources_current(),'Source changed before VLM training')
    RUN.mkdir(mode=0o700);snapshot=RUN/'code_snapshot';snapshot.mkdir(mode=0o700)
    for n in ('pallet_description_round5_data.py','pallet_description_round5_vlm.py','pallet_description_round4_vlm.py'):shutil.copyfile(PROJECT/'pallet'/n,snapshot/n)
    gpu=gpu_receipt('m1');torch.set_num_threads(4);random.seed(CONFIG['seed']);torch.manual_seed(CONFIG['seed'])
    records=[r for r in jsonl(ROOT/'description_data/records.jsonl') if any(r['supervised_fields'].values())];subset=[r for r in records if r['split']=='train'];held=[r for r in records if r['split']=='holdout']
    require(not {r['component'] for r in subset}&{r['component'] for r in held},'Semantic component leakage')
    write(RUN/'execution_plan.json',dict(config=CONFIG,gpu=gpu,train_targets=len(subset),holdout_targets=len(held),full_pool_targets=len(records),
        independent_branch_from_pinned_base=True,full_pool_continues_previous_full_adapter=True,prompt_sha256=sha(PROMPT),
        format_fix='supervise_quotes_for_zero_content_unlabelled_observation_not_mask_null_token',missing_observation_not_normal_GT=True,
        dataset_manifest_sha256=digest(ROOT/'dataset_manifest.json'),source_code_sha256=digest(__file__)))
    try:
        old=RUN/'old_component_comparison';old.mkdir(mode=0o700);processor,base=load('qwen')
        model=PeftModel.from_pretrained(base,PREVIOUS/'vlm_run_v1/component_branch/adapter').merge_and_unload().eval()
        from .pallet_description_round4_data import PROMPT as oldprompt
        evaluate(model,processor,held,old,oldprompt);del model,processor,base;gc.collect();torch.cuda.empty_cache()
        model,processor=learning.phase('component_branch',subset);metrics=evaluate(model,processor,held,RUN/'component_branch')
        print(dict(phase='round5_vlm_holdout',**metrics),flush=True);del model,processor;gc.collect();torch.cuda.empty_cache()
        require(sources_current(),'Source changed before full VLM phase')
        model,processor=learning.phase('full_pool',records,PREVIOUS/'vlm_run_v1/full_pool/adapter')
        require(sources_current(),'Source changed during VLM training')
        write(RUN/'training_complete.json',dict(state='round5_actual_human_semantic_format_supervised_LoRA_complete',model='Qwen3.5-0.8B',gpu=gpu,config=CONFIG,
            adapter_sha256=digest(RUN/'full_pool/adapter/adapter_model.safetensors'),component_metrics_sha256=digest(RUN/'component_branch/holdout_metrics.json'),full_pool_targets=len(records),
            schema_validity_not_semantic_accuracy=True,missing_observations_not_normal_labels=True))
    except Exception:write(RUN/'failure.json',dict(traceback=traceback.format_exc()));raise


def infer(mode):
    from peft import PeftModel
    require((RUN/'training_complete.json').exists() and sources_current(),'Actual VLM training/current sources required')
    out=ROOT/f'descriptions_{mode}.jsonl';require(not out.exists(),'Preserve raw VLM inference')
    records=jsonl(ROOT/('private/release_selected.jsonl' if mode=='new' else 'private/semantic_review_selected.jsonl'))
    gpu=gpu_receipt('m1');torch.set_num_threads(4);processor,base=load('qwen');adapter=RUN/'full_pool/adapter';h=digest(adapter/'adapter_model.safetensors')
    require(h==read(RUN/'training_complete.json')['adapter_sha256'],'Adapter changed');model=PeftModel.from_pretrained(base,adapter).merge_and_unload().eval();processor.tokenizer.padding_side='left'
    parts=ROOT/f'description_parts_{mode}';parts.mkdir(mode=0o700,exist_ok=True);preds=[]
    for offset in range(0,len(records),64):
        path=parts/f'part_{offset:06d}.json';group=records[offset:offset+64]
        if path.exists():
            part=read(path);require(part['targets']==[r['target_key'] for r in group] and part['adapter_sha256']==h and part['code_sha256']==digest(__file__),'Inference shard differs')
        else:
            ps=[]
            for j in range(0,len(group),16):ps.extend(learning.predict_batch(model,processor,group[j:j+16],PROMPT))
            for p in ps:p.update(adapter_sha256=h,status='machine_draft',model='Qwen3.5-0.8B-round5-LoRA',empty_observation_means_unknown_not_normal=True)
            part=dict(targets=[r['target_key'] for r in group],predictions=ps,adapter_sha256=h,code_sha256=digest(__file__));write(path,part)
        preds.extend(part['predictions'])
    require(sources_current(),'Sources changed during inference');write(out,preds,lines=True)
    write(ROOT/f'description_inference_{mode}.json',dict(targets=len(preds),schema_valid=sum(p['valid'] for p in preds),structure_only_recovered=sum(p['structure_recovered'] for p in preds),
        observation_withheld=sum(p['observation_withheld'] for p in preds),strict_valid_empty_observation=sum(p['valid'] and not p['fields'].get('观察') for p in preds),
        adapter_sha256=h,gpu=gpu,actual_model_raw_retained=True,no_label_features_as_input=True))


if __name__=='__main__':
    import argparse
    p=argparse.ArgumentParser();p.add_argument('phase',choices=('train','infer-new','infer-semantic'));a=p.parse_args()
    train() if a.phase=='train' else infer(a.phase.split('-')[1])
