"""Length sensitivity only: rerun every capped free response in the frozen audit subset."""
import argparse
from pathlib import Path
import sys
import time

PROJECT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(PROJECT))
from pallet.mask_model_training_data import read,write,jsonl,require,gpu_receipt
from pallet.cluster_targets import digest
from pallet.pallet_vlm_native_intensity import ROOT


def main(name):
    import torch
    from PIL import Image
    from transformers import AutoProcessor,AutoModelForImageTextToText
    torch.set_num_threads(4);torch.manual_seed(20261105);gpu=gpu_receipt('m1' if name=='qwen08b' else 'm2')
    original=ROOT/'runs'/name;complete=read(original/'complete.json');require(complete['model_unchanged'],'Need completed native main run')
    ids=set(read(ROOT/'qualitative_plan.json')['sample_ids']);tasks={t['task_id']:t for t in jsonl(ROOT/'sample/tasks.jsonl')}
    selected=[p for p in jsonl(original/'predictions.jsonl') if p['control'] is None and p['sample_id'] in ids and p['kind'] in ('scene','crop') and p['hit_token_limit']]
    out=ROOT/'length_check'/name;require(not out.exists(),'Preserve previous length check');out.mkdir(parents=True,mode=0o700);(out/'predictions').mkdir(mode=0o700)
    write(out/'plan.json',dict(rule='all capped scene/crop responses in the frozen23 audit subset; no semantic selection',
        tasks=[p['task_id'] for p in selected],max_new_tokens=384,prompt_unchanged=True,main_predictions_sha256=complete['predictions_sha256']))
    folder=PROJECT/'outputs/pallet_description_vlm_models_v1/qwen' if name=='qwen08b' else ROOT/'models/qwen4b'
    receipt=folder.parent/('qwen_download.json' if name=='qwen08b' else 'qwen4b_download.json');pinned=read(receipt)
    require(all(digest(folder/n)==h for n,h in pinned['files'].items()),'Pretrained model changed')
    processor=AutoProcessor.from_pretrained(folder,local_files_only=True);processor.image_processor.min_pixels=128*128;processor.image_processor.max_pixels=1024*1024
    model=AutoModelForImageTextToText.from_pretrained(folder,dtype=torch.bfloat16,attn_implementation='sdpa',local_files_only=True).cuda().eval();model.requires_grad_(False)
    results=[];start=time.monotonic()
    with torch.inference_mode():
        for p in selected:
            t=tasks[p['task_id']];image=Image.open(ROOT/t['image']).convert('RGB')
            messages=[dict(role='user',content=[dict(type='image',image=image),dict(type='text',text=t['prompt'])])]
            text=processor.apply_chat_template(messages,tokenize=False,add_generation_prompt=True,enable_thinking=False)
            data=processor(text=[text],images=[image],return_tensors='pt').to('cuda');ids=model.generate(**data,max_new_tokens=384,do_sample=False,pad_token_id=processor.tokenizer.pad_token_id)
            new=ids[0,data['input_ids'].shape[1]:];raw=processor.tokenizer.decode(new,skip_special_tokens=True)
            result=dict(model=name,task_id=t['task_id'],raw=raw,generated_tokens=int(len(new)),hit_token_limit=len(new)>=384,
                same_prefix=raw.startswith(p['raw']),main_output_sha256=digest(original/'predictions'/(t['task_id']+'.json')),
                prompt=t['prompt'],input_image_sha256=t['image_sha256'])
            write(out/'predictions'/(t['task_id']+'.json'),result);results.append(result)
    write(out/'predictions.jsonl',results,lines=True)
    write(out/'complete.json',dict(state='native_length_sensitivity_complete',model=name,tasks=len(results),gpu=gpu,
        same_prefix_count=sum(p['same_prefix'] for p in results),still_capped=sum(p['hit_token_limit'] for p in results),
        no_prompt_change=True,no_finetuning=True,seconds=time.monotonic()-start,predictions_sha256=digest(out/'predictions.jsonl'),code_sha256=digest(__file__)))
    print(read(out/'complete.json'),flush=True)


if __name__=='__main__':
    p=argparse.ArgumentParser();p.add_argument('model',choices=('qwen08b','qwen4b'));a=p.parse_args();main(a.model)
