"""Summarize native predictions without turning unlabeled observations into negatives."""
from collections import Counter,defaultdict
import csv
import html
import io
import json
from pathlib import Path
import statistics
import sys

PROJECT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(PROJECT))
from pallet.mask_model_training_data import read,write,jsonl,require
from pallet.cluster_targets import digest
from pallet.pallet_vlm_native_intensity import ROOT


def agreement(rows,predictions):
    out={}
    for key in ('支腿','横梁','材质'):
        counts=Counter();references=Counter(r['reference'][key] for r in rows)
        for r in rows:
            p=predictions[r['sample_id']];parsed=p['parsed']
            if not parsed['valid']:counts['invalid']+=1;continue
            value=parsed['fields'][key]
            if value is None:counts['abstain']+=1
            else:
                counts['answered']+=1;counts['correct']+=int(value==r['reference'][key]);counts['incorrect']+=int(value!=r['reference'][key])
        out[key]=dict(total=len(rows),**{k:counts[k] for k in ('answered','correct','incorrect','abstain','invalid')},
            agreement_all=counts['correct']/len(rows),agreement_answered=counts['correct']/counts['answered'] if counts['answered'] else None,
            majority_baseline_count=references.most_common(1)[0][1],majority_baseline_value=references.most_common(1)[0][0])
    return out


def metrics(rows,preds):
    real=[p for p in preds if p['control'] is None];qa={p['sample_id']:p for p in real if p['kind']=='structure'}
    require(set(qa)=={r['sample_id'] for r in rows},'Missing real structured response')
    observations={}
    for label,field,choose in (
        ('occlusion','遮挡',lambda r:'occlusion' in r['observation_tags']),
        ('poor_quality','质量差',lambda r:'low_quality' in r['observation_tags']),
        ('actual_out_of_view','出视野',lambda r:'超出视野' in (r['reference']['观察'] or ''))):
        selected=[r for r in rows if choose(r)];counts=Counter()
        for r in selected:
            parsed=qa[r['sample_id']]['parsed'];value=parsed['fields'][field] if parsed['valid'] else 'invalid'
            counts['positive' if value is True else 'negative' if value is False else 'abstain' if value is None else 'invalid']+=1
        observations[label]=dict(human_positive_targets=len(selected),**{k:counts[k] for k in ('positive','negative','abstain','invalid')},
            positive_detection_fraction=counts['positive']/len(selected) if selected else None,negative_labels_available=False)
    controls={}
    for kind in ('blank','shuffled'):
        ps=[p for p in preds if p['control']==kind and p['kind']=='structure'];valid=[p for p in ps if p['parsed']['valid']]
        controls[kind]=dict(total=len(ps),valid=len(valid),pallet_true=sum(p['parsed']['fields']['托盘'] is True for p in valid),
            pallet_false=sum(p['parsed']['fields']['托盘'] is False for p in valid),pallet_abstain=sum(p['parsed']['fields']['托盘'] is None for p in valid),
            nonnull_structure=sum(any(p['parsed']['fields'][k] is not None for k in ('支腿','横梁','材质')) for p in valid))
    return dict(real_tasks=len(real),structure_responses=len(qa),structure_JSON_valid=sum(p['parsed']['valid'] for p in qa.values()),
        structured_pallet_true=sum(p['parsed']['valid'] and p['parsed']['fields']['托盘'] is True for p in qa.values()),
        weak_class_reference_agreement=agreement(rows,qa),
        all_three_weak_attributes_correct=sum(p['parsed']['valid'] and all(p['parsed']['fields'][k]==next(r for r in rows if r['sample_id']==p['sample_id'])['reference'][k] for k in ('支腿','横梁','材质')) for p in qa.values()),
        fine_class_stratum_agreement=agreement([r for r in rows if r['stratum']=='fine_class_stratum'],qa),
        known_human_positive_observations=observations,controls=controls,
        structured_response_distributions={key:dict(Counter(str(p['parsed']['fields'][key]) for p in qa.values() if p['parsed']['valid'])) for key in ('托盘','支腿','横梁','材质','遮挡','出视野','质量差')},
        per_parent={parent:agreement([r for r in rows if r['parent']==parent],qa) for parent in sorted({r['parent'] for r in rows})},
        per_fine_class={cls:agreement([r for r in rows if r['class_id']==cls],qa) for cls in sorted({r['class_id'] for r in rows})},
        hit_token_limit=dict(Counter(p['kind'] for p in real if p['hit_token_limit'])),
        latency_ms_median=statistics.median(p['elapsed_ms'] for p in real),latency_ms_mean=statistics.mean(p['elapsed_ms'] for p in real))


def report():
    rows=jsonl(ROOT/'sample/references.jsonl');models={};predictions={}
    for name in ('qwen08b','qwen4b'):
        out=ROOT/'runs'/name;complete=read(out/'complete.json');require(complete['model_unchanged'],'Model changed')
        ps=jsonl(out/'predictions.jsonl');require(digest(out/'predictions.jsonl')==complete['predictions_sha256'],'Predictions changed')
        models[name]=dict(execution=read(out/'execution.json'),completion=complete,metrics=metrics(rows,ps));predictions[name]={p['task_id']:p for p in ps}
        models[name]['length_sensitivity']=read(ROOT/'length_check'/name/'complete.json')
    audit=read(ROOT/'qualitative_audit.json');plan=read(ROOT/'qualitative_plan.json')
    require(set(audit['sample_ids'])==set(plan['sample_ids']),'Qualitative audit subset changed after predictions')
    for name in models:
        entries=[v for v in audit['entries'] if v['model']==name]
        require(len(entries)==len(plan['sample_ids']) and {v['sample_id'] for v in entries}==set(plan['sample_ids']),'Missing paired qualitative audits')
        for v in entries:
            require(all(v[k] in (0,1,2) for k in ('scene_score','crop_score')),'Invalid audit rating')
            for kind,h in v['prediction_sha256'].items():require(digest(ROOT/'runs'/name/'predictions'/(v['sample_id']+'_'+kind+'.json'))==h,'Audited output changed')
        models[name]['qualitative_scores']={kind:dict(total=len(entries),grounded_without_major_error=sum(v[kind+'_score']==2 for v in entries),
            partially_grounded_with_major_error=sum(v[kind+'_score']==1 for v in entries),wrong_or_ungrounded=sum(v[kind+'_score']==0 for v in entries)) for kind in ('scene','crop')}
    summary=dict(state='native_intensity_understanding_evaluated',sample=read(ROOT/'sample/manifest.json'),models=models,qualitative_audit=audit,
        length_interpretation=read(ROOT/'length_interpretation.json'),cold_control=read(ROOT/'cold_control/complete.json'),
        actual_generation_calls=648+sum(models[n]['length_sensitivity']['tasks'] for n in models)+5,
        parameter_accounting=read(ROOT/'models/parameter_accounting.json'),
        caveats=['不是新建的现实盲测集；96张原图来自既有数据，属于59个关联分量，不能按96个完全独立场景解释。',
            '未提供或要求模型预测细类ID；覆盖41细类不等于做了41分类准确率测试。',
            '项目托盘目录包含模拟件和料架式载具，项目内语义与通用木托盘概念不完全相同。',
            '支腿／横梁／材质参考来自已确认的类别定义，不是逐图可见结构GT。',
            '没有备注是缺失异常标签，不是正常负例；线上空描述默认正常的应用规则不用于本次评价。',
            '真正出视野仅1条、画质极差仅3条，不能据此认证泛化能力或精确率。',
            '全黑／打乱对照检验提示偏置与图像依赖，不代替真实无托盘场景的特异度评估。',
            '主实验自由描述上限160新token，问答192；截断计数已列出，不把省略的末段当主动弃答。',
            '定性评分者为本助手，不是新增独立人类标注者。'],
        source_no_writes=True,no_finetuning=True,no_private_data_uploaded=True)
    write(ROOT/'evaluation.json',summary)
    lines=['# 强度图场景与托盘：VLM原生推理诊断','',
        '96张不同原图：41细类各2张（82张）＋全部14条已有人工观察备注；覆盖9大类。未经微调、无LoRA、无类别／关键点／人工备注输入。',
        '每模型324次独立对话：整图自由描述96次、目标裁剪自由描述96次、结构问答96次、空白／像素打乱对照36次。整图与裁剪自由描述均不提示“托盘”或“强度图”；结构问答明确告知传感器强度图。','',
        '| 指标 | Qwen3.5-0.8B 原始权重 | Qwen3.5-4B 原始权重 |','|---|---:|---:|']
    def add(label,values):lines.append('| '+label+' | '+' | '.join(values)+' |')
    ms=[models[n]['metrics'] for n in models]
    add('结构问答完整字段有效',[f"{m['structure_JSON_valid']}/96" for m in ms])
    add('问答中判断有托盘（已提示目标问题，不是盲识别率）',[f"{m['structured_pallet_true']}/96" for m in ms])
    for key in ('支腿','横梁','材质'):
        add(key+'与类别定义一致（弱参考；分母含弃答／格式失败）',[f"{m['weak_class_reference_agreement'][key]['correct']}/96" for m in ms])
        add(key+'有回答时一致／回答数',[f"{m['weak_class_reference_agreement'][key]['correct']}/{m['weak_class_reference_agreement'][key]['answered']}" for m in ms])
    add('三项同时与类别定义一致',[f"{m['all_three_weak_attributes_correct']}/96" for m in ms])
    for label,chinese in (('occlusion','已知遮挡阳性检出'),('poor_quality','已知质量差阳性检出'),('actual_out_of_view','已知真正出视野阳性检出')):
        add(chinese,[f"{m['known_human_positive_observations'][label]['positive']}/{m['known_human_positive_observations'][label]['human_positive_targets']}" for m in ms])
    add('空白图仍判断有托盘',[f"{m['controls']['blank']['pallet_true']}/{m['controls']['blank']['total']}" for m in ms])
    add('打乱图仍判断有托盘（仅敏感性对照）',[f"{m['controls']['shuffled']['pallet_true']}/{m['controls']['shuffled']['total']}" for m in ms])
    add('单次生成耗时中位数',[f"{m['latency_ms_median']/1000:.2f}s" for m in ms])
    add('整图／裁剪描述命中160-token上限',[f"{m['hit_token_limit'].get('scene',0)} / {m['hit_token_limit'].get('crop',0)}" for m in ms])
    add('固定审计子集截断续查：384-token重跑数／仍截断数',[f"{models[name]['length_sensitivity']['tasks']} / {models[name]['length_sensitivity']['still_capped']}" for name in models])
    for kind,label in (('scene','23张定性子集：整图核心描述贴图且无主要错误'),('crop','23张定性子集：裁剪核心描述贴图且无主要错误')):
        add(label,[f"{models[name]['qualitative_scores'][kind]['grounded_without_major_error']}/23" for name in models])
    lines+=['','按本样本类别定义，始终猜多数值的支腿／横梁／材质一致数分别为60／56／48（各分母96），仅作弱参考基线；谨慎弃答不能与无依据猜测混为同一种错误。',
        '','## 定性观察与结论','',audit['conclusion_zh'],'','## 长度与冷启动复查','',summary['length_interpretation']['findings_zh'],
        '',f"主实验648次生成，长度复查{sum(models[n]['length_sensitivity']['tasks'] for n in models)}次，冷启动复现5次，共{summary['actual_generation_calls']}次。",
        '', '冷启动新进程先输入全黑图、再换实图、再回到同一全黑图，5次输出均逐字复现主实验；首次全黑输入也虚构4腿／2横梁／金属，排除了前一请求残留这一解释。',
        '','## 限制','']+['- '+v for v in summary['caveats']]
    lines+=['','参数口径：原始权重张量元素数分别为873,438,784／4,659,865,088；Transformers原生推理未实例化MTP分支，运行时参数数以`execution.json`为准。这不是裁剪适配器或任务微调。']
    lines+=['','长度敏感性检查：对固定23张审计子集中所有触及上限的自由描述，不改提示、贪心设置或图像，只把上限增至384新token。保留主实验指标不替换、不择优；补充原文在`length_check/`及HTML各样本中展开查看。']
    lines+=['','全部原图、裁剪、提示、模型原文、控制组、逐类统计与逐例视觉审计均保留在本目录。`report.html`提供并排浏览；评估不修改线上标注或默认正常策略。']
    (ROOT/'REPORT.md').write_text('\n'.join(lines)+'\n')
    esc=lambda v:html.escape(str(v));cards=[]
    for r in rows:
        cells=[]
        for name in models:
            ps=[predictions[name][r['sample_id']+'_'+kind] for kind in ('scene','crop','structure')]
            longer=[]
            for p in ps:
                path=ROOT/'length_check'/name/'predictions'/(p['task_id']+'.json')
                if path.exists():
                    v=read(path);longer.append('<details><summary>'+esc(p['kind'])+' · 384-token长度检查'+(' · 仍截断' if v['hit_token_limit'] else '')+'</summary><pre>'+esc(v['raw'])+'</pre></details>')
            cells.append('<section><h3>'+esc(name)+'</h3>'+''.join('<h4>'+esc(p['kind'])+(' · TOKEN LIMIT' if p['hit_token_limit'] else '')+'</h4><pre>'+esc(p['raw'])+'</pre>' for p in ps)+''.join(longer)+'</section>')
        note=r['reference']['观察'] or '无逐图观察标签（不能当正常负例）'
        cards.append('<article><h2>'+esc(r['sample_id'])+' · Class'+str(r['class_id'])+' · '+esc(r['attributes_text'])+'</h2><p>参考：'+esc(note)+'</p><div class="images"><img src="'+esc(r['paths']['images'])+'"><img src="'+esc(r['paths']['crops'])+'"></div><div class="models">'+''.join(cells)+'</div></article>')
    controls=[]
    for name in models:
        for p in predictions[name].values():
            if p['control'] is not None:controls.append('<details><summary>'+esc(name+' '+p['task_id'])+'</summary><pre>'+esc(p['raw'])+'</pre></details>')
    text='<!doctype html><html lang="zh"><meta charset="utf-8"><title>强度图VLM原生理解诊断</title><style>body{font-family:sans-serif;max-width:1450px;margin:auto;padding:20px;background:#eee;color:#222}article{background:white;margin:20px 0;padding:18px;border-radius:8px}.images,.models{display:flex;gap:16px}.images img{object-fit:contain;max-width:48%;max-height:300px}.models section{width:50%;min-width:0}pre{white-space:pre-wrap;overflow-wrap:anywhere;background:#f6f6f6;padding:10px}h4{margin-bottom:2px}@media(max-width:700px){.models{display:block}.models section{width:auto}.images{display:block}.images img{max-width:100%}}</style><h1>强度图VLM原生理解诊断</h1><p>96张原图／41细类／9大类；无微调、无LoRA、无标签输入。结构问答已提示传感器强度图，另两种自由描述不提示物体名或图像类型。</p><p>材质／结构采用类别定义弱参考；无备注不代表正常。此为诊断实验，不能据此晋升GT或机器人资格。</p><h2>结论</h2><pre>'+esc(audit['conclusion_zh'])+'</pre><h2>长度补查</h2><pre>'+esc(summary['length_interpretation']['findings_zh'])+'</pre>'+''.join(cards)+'<h2>空白／打乱对照原文</h2>'+''.join(controls)+'</html>'
    text=text.replace('<style>',"<style>@font-face{font-family:ReportCJK;src:url('report-font.otf')}body,pre{font-family:ReportCJK,sans-serif!important}")
    from scripts.build_mask_remaining_direct_requests import build_font
    build_font(ROOT,text)
    (ROOT/'report.html').write_text(text)
    csvout=io.StringIO();writer=csv.writer(csvout);writer.writerow(['model','class_id','attribute','n','answered','correct','incorrect','abstain','invalid'])
    for name,m in models.items():
        for cls,attributes in m['metrics']['per_fine_class'].items():
            for attribute,v in attributes.items():writer.writerow([name,cls,attribute,v['total'],v['answered'],v['correct'],v['incorrect'],v['abstain'],v['invalid']])
    (ROOT/'per_class.csv').write_text(csvout.getvalue())
    print(dict(state=summary['state'],models={n:m['metrics']['structure_JSON_valid'] for n,m in models.items()}),flush=True)


if __name__=='__main__':report()
