"""Independent correct-only source replay and real optimizer/paired-model audit."""
from collections import Counter,defaultdict
from contextlib import closing
from datetime import datetime,timezone
import json
from pathlib import Path
import sys
import types

import cv2
import numpy as np

PROJECT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(PROJECT));sys.path.insert(0,str(PROJECT/'scripts'))
from pallet.cluster_targets import digest
from pallet.mask_model_training_data import read,write,jsonl,require
from pallet.mask_template_refined import readonly
from pallet.pose_review_geometry import raster_mask
from pallet.pallet_description_round7_data import ROOT,PREVIOUS,SOURCE_SETS,verify,sources_current,validate_required,POLICY,REINSPECTION
from pallet.pallet_description_round7_vlm import format_target


def source_replay():
    _,records=verify();baseline={r['target_key']:r for r in jsonl(PREVIOUS/'private/training.jsonl')};oldkeys=set(baseline);scopes={};accepted=set();retained=set()
    for mode,s in SOURCE_SETS.items():
        props={p['target_key']:p for p in jsonl(s['root']/'proposals.jsonl')}
        with closing(readonly(ROOT/f'private/snapshots/{mode}.sqlite3')) as db:
            require(db.execute('PRAGMA quick_check').fetchall()==[('ok',)],'Snapshot damaged')
            current={k:json.loads(raw) for k,raw in db.execute('SELECT target_key,payload FROM reviews')};events=list(db.execute('SELECT target_key,revision,payload FROM events ORDER BY id'));last={}
        for k,rev,raw in events:
            v=json.loads(raw);require(rev==v['revision']==last.get(k,{}).get('revision',0)+1,'Noncontiguous source history');last[k]=v
        require(last==current and set(current)<=set(props),'Source history/current mismatch')
        if mode=='new':require(set(current)==set(props) and all(v['action']=='correct' for v in current.values()),'New targets incomplete')
        else:validate_required(props,current,events,read(POLICY),read(REINSPECTION))
        for key,p in props.items():
            v=current.get(key)
            if v and v['action']=='correct':
                baseline[key]=dict(**{f:v[f] for f in ('polygon','class_id','description')},image_sha256=p['image_sha256']);accepted.add(key)
            else:
                require(mode=='existing' and key in baseline,'Cannot accept unsaved new model output');retained.add(key)
        scopes[mode]=dict(issued=len(props),accepted_correct=sum(v['action']=='correct' for v in current.values()),retained_prior_baselines=sum(k not in accepted for k in props),events=len(events))
    require(set(baseline)=={r['target_key'] for r in records},'Merged coverage mismatch')
    for r in records:
        ref=baseline[r['target_key']];require(all(r[f]==ref[f] for f in ('polygon','class_id','description','image_sha256')),'Latest correct or retained baseline changed')
        full=raster_mask(r['polygon'],*r['source']['size']);x1,y1,x2,y2=r['roi'];mask=cv2.imread(str(ROOT/r['mask']),0)>0;valid=cv2.imread(str(ROOT/r['valid']),0)>0
        require(np.array_equal(mask,full[y1:y2,x1:x2]) and mask.sum()==full.sum() and valid[mask].all(),'Mask clipped or target validity wrong')
    require(sources_current(),'Sources changed during replay')
    write(ROOT/'source_replay.json',dict(state='round7_completed_latest_labels_and_native_masks_replayed',targets=len(records),new_targets=len(set(baseline)-oldkeys),sources=scopes,
        retained_optional_prior_baselines=len(retained),required_pending=0,current_equals_history=True,no_draft_or_defer_acceptance=True,source_actions_not_rewritten=True,production_annotation_POSTs=0))
    print(dict(state='round7_source_replay_passed',targets=len(records),retained_optional_prior_baselines=len(retained)))


def training():
    # Exact existing optimizer, token quote/EOS and checkpoint checks, with only
    # round identifiers changed; replay above accounts for partial optional data.
    path=PROJECT/'scripts/finalize_pallet_description_round5.py';source=path.read_text().replace('round5','round7')
    line='from pallet.pallet_description_round7_release import REVIEW_SETS,RELEASE,description_draft\n'
    require(source.count(line)==1,'Training audit import anchor changed');source=source.replace(line,'')
    module=types.ModuleType('_round7_training_audit');module.__file__=__file__;exec(compile(source,str(path),'exec'),module.__dict__)
    module.training()
    (ROOT/'adapted_training_audit.py').write_text(source)


def bundles():
    from pallet.pallet_description_round7_release import REVIEW_SETS,RELEASE,description_draft
    from pallet.pallet_description_round7_vlm import parse
    baseline={r['target_key']:r for r in jsonl(ROOT/'private/training.jsonl')};protected=set(read(ROOT/'private/protection.json')['protected_image_hashes']);results={}
    for mode,s in REVIEW_SETS.items():
        props=jsonl(s['root']/'proposals.jsonl');m=read(s['root']/'manifest.json');preds={p['target_key']:p for p in jsonl(ROOT/f'descriptions_{"new" if mode=="new" else "semantic"}.jsonl')}
        require(len(props)==len({p['target_key'] for p in props})==m['request_count'] and m['proposals_sha256']==digest(s['root']/'proposals.jsonl'),'Wrong task identities')
        for p in props:
            pred=preds[p['target_key']];require(p['description_raw']==pred['raw'] and parse(pred['raw'])['valid']==pred['valid'],'Model raw output does not replay')
            if mode=='new':require(p['target_key'] not in baseline and p['image_sha256'] not in protected and p['description']==description_draft(pred),'Old image or altered machine draft')
            else:require(all(p[f]==baseline[p['target_key']][f] for f in ('polygon','class_id','description')),'Existing accepted human baseline overwritten')
        if mode=='new':require(len(props)==len({p['image_sha256'] for p in props}),'Repeated new original')
        for n,h in m['public_assets'].items():require(digest(s['root']/n)==h,'Task asset changed')
        if mode=='existing':
            policy=read(RELEASE/mode/'policy.json');require(set(policy['entries'])=={p['target_key'] for p in props} and policy['actual_audit_count']==50 and policy['completed_round6_Class13_scope_not_reopened'],'Bounded policy coverage/audit changed')
        results[mode]=dict(targets=len(props),unique_images=len({p['image_sha256'] for p in props}),manifest_sha256=digest(s['root']/'manifest.json'))
    require(sources_current(),'Source changed during bundle replay');write(ROOT/'bundle_source_replay.json',dict(state='round7_next_tasks_latest_human_baseline_and_model_raw_replayed',bundles=results,production_annotation_POSTs=0))
    print(dict(state='round7_bundles_replayed',bundles=results))


def paired_metrics():
    classes=read(ROOT/'classes.json');indices={c['fine_index']:c['class_id'] for c in classes.values() if c['train_enabled']}
    old={r['target_key']:r for r in jsonl(ROOT/'mask_run_v1/old_component_branch_predictions.jsonl')};new={r['target_key']:r for r in jsonl(ROOT/'mask_run_v1/independent_branch/holdout_predictions.jsonl')}
    require(set(old)==set(new),'Unpaired mask comparison');perclass=defaultdict(lambda:dict(n=0,old_correct=0,new_correct=0));changed=[];confusions=Counter()
    for k,p in new.items():
        q=old[k];ref=indices[p['fine_index']];a=indices[q['predicted_fine_index']];b=indices[p['predicted_fine_index']];require(q['fine_index']==p['fine_index'],'Reference mismatch')
        v=perclass[ref];v['n']+=1;v['old_correct']+=a==ref;v['new_correct']+=b==ref
        if b!=ref:confusions[(ref,b)]+=1
        if a!=b:changed.append(dict(target_key=k,reference=ref,old_prediction=a,new_prediction=b,old_correct=a==ref,new_correct=b==ref))
    om=read(ROOT/'mask_run_v1/old_component_branch_comparison.json');nm=read(ROOT/'mask_run_v1/independent_branch/holdout_metrics.json')
    write(ROOT/'mask_regression_audit.json',dict(matched_holdout_targets=len(new),old_fine_top1=om['fine_top1'],new_fine_top1=nm['fine_top1'],fine_delta=nm['fine_top1']-om['fine_top1'],
        old_parent_top1=om['parent_top1'],new_parent_top1=nm['parent_top1'],old_iou_median=om['segmentation']['iou_median'],new_iou_median=nm['segmentation']['iou_median'],
        old_correct_new_wrong=sum(v['old_correct'] and not v['new_correct'] for v in changed),old_wrong_new_correct=sum(not v['old_correct'] and v['new_correct'] for v in changed),per_class=dict(perclass),paired_changed_targets=changed,
        confusions=[dict(reference=a,predicted=b,count=n) for (a,b),n in confusions.most_common()],automatic_formal_model_replacement=False,not_new_blind_test=True))
    sem={r['target_key']:r for r in jsonl(ROOT/'description_data/records.jsonl')};old={r['target_key']:r for r in jsonl(ROOT/'vlm_run_v1/old_component_comparison/holdout_predictions.jsonl')};new={r['target_key']:r for r in jsonl(ROOT/'vlm_run_v1/component_branch/holdout_predictions.jsonl')}
    require(set(old)==set(new),'Unpaired VLM comparison');changes=[]
    for k,p in new.items():
        q=old[k];a=q['fields'] if q['valid'] else q.get('recovered_fields');b=p['fields'] if p['valid'] else p.get('recovered_fields');r=sem[k]
        fields={f:dict(reference=r['target'][f],old=a.get(f) if a else None,new=b.get(f) if b else None) for f in ('支腿','横梁','材质') if r['supervised_fields'][f] and (a.get(f) if a else None)!=(b.get(f) if b else None)}
        if fields or p['valid']!=q['valid']:changes.append(dict(target_key=k,attributes=fields,old_JSON_valid=q['valid'],new_JSON_valid=p['valid']))
    write(ROOT/'vlm_regression_audit.json',dict(matched_holdout_targets=len(new),metrics={label:read(ROOT/f'vlm_run_v1/{phase}/holdout_metrics.json') for label,phase in (('old','old_component_comparison'),('new','component_branch'))},
        attribute_recovery={label:read(ROOT/f'vlm_run_v1/{phase}/structure_recovery_audit.json') for label,phase in (('old','old_component_comparison'),('new','component_branch'))},paired_changed_targets=changes,
        human_observation_references=sum(sem[k]['observation_supervised'] for k in new),nonempty_is_not_semantic_accuracy=True,no_normal_negative_labels_inferred=True,automatic_formal_model_replacement=False))


def report():
    require(read(ROOT/'training_verification.json')['state']=='round7_actual_training_and_latest_source_replay_passed','Actual training audit required')
    paired_metrics();data=read(ROOT/'preparation_report.json');mask=read(ROOT/'mask_regression_audit.json');vlm=read(ROOT/'vlm_regression_audit.json')
    lines=['# 第七轮真实训练与同标签比较','',f"冻结{data['targets']}目标／{data['distinct_images']}原图，40细类／9大类；新增408，更新752已有目标，393项非必做诊断保留上轮人工基线。训练{data['splits']['train']}、组件留出{data['splits']['holdout']}。",'',
        '来源为第六轮真实保存：新图408/408 correct；已有752 correct；优先／抽检和Class13再次逐图复查待办均为零。不把1,145项全部说成完成，不制造correct事件，不接受草稿／暂缓。','',
        '## 同一份最新留出标签','', '| 指标 | 第六轮独立分支 | 第七轮独立分支 |','|---|---:|---:|',
        f"| 细类Top1 | {mask['old_fine_top1']:.2%} | {mask['new_fine_top1']:.2%} |",f"| 大类Top1 | {mask['old_parent_top1']:.2%} | {mask['new_parent_top1']:.2%} |",f"| IoU中位数 | {mask['old_iou_median']:.4f} | {mask['new_iou_median']:.4f} |",'',
        f"VLM完整JSON：{vlm['metrics']['old']['strict_complete_JSON_valid']}/{vlm['metrics']['old']['targets']} → {vlm['metrics']['new']['strict_complete_JSON_valid']}/{vlm['metrics']['new']['targets']}。结构字段一致率包含显式恢复："]
    for f in ('支腿','横梁','材质'):
        a=vlm['attribute_recovery']['old']['attribute_exact_including_explicit_structure_recovery'][f]['accuracy'];b=vlm['attribute_recovery']['new']['attribute_exact_including_explicit_structure_recovery'][f]['accuracy'];lines.append(f'- {f}：{a:.2%} → {b:.2%}。')
    lines+=['','保留全部退步与逐类／逐图证据，不按留出指标挑选训练检查点，不自动替换正式模型。独立分类／轮廓从固定通用ResNet18起训24轮，评估后全量20轮；独立VLM从固定基座起训2轮，全量仅续训第六轮全量适配器2轮。优化器、权重、所有记录暴露及引号／EOS监督已核验。',
        f"结构字段冲突{data['semantic_conflict_targets']}项，仅遮蔽冲突字段监督，保留原标签与文字。缺失观察不制造正常GT，模型非空观察不等于正确。",'',
        '组件／同图划分继承，反复迭代留出并非新采集盲测。旧冻结数据、模型、库、任务、原ptunnel地址和草稿namespace保持；没有自动晋升正式GT／机器人资格，没有生产模拟标注POST或提交／推送。',
        '本报告只证明训练和比较；下一轮候选推理／人工任务若未完成验收，不宣称已发布。']
    require(sources_current(),'Source changed before report');(ROOT/'TRAINING_REPORT.md').write_text('\n'.join(lines)+'\n')
    write(ROOT/'TRAINING_HANDOFF.json',dict(state='round7_actual_training_source_replay_and_matched_comparison_complete',data=data,mask=mask,vlm=vlm,
        source_replay_sha256=digest(ROOT/'source_replay.json'),training_verification_sha256=digest(ROOT/'training_verification.json'),report_sha256=digest(ROOT/'TRAINING_REPORT.md'),
        old_models_data_and_histories_preserved=True,production_annotation_POSTs=0,no_commit_or_push=True))
    print(dict(state='round7_training_audited',old_fine=mask['old_fine_top1'],new_fine=mask['new_fine_top1']))


def deliver():
    from urllib.request import urlopen
    from pallet.pallet_description_round7_release import REVIEW_SETS,RELEASE
    from pallet.pallet_review_ptunnel import profiles
    from serve_structure_review import process_identity
    from serve_pallet_description_round7 import configure
    from pallet.mask_class_note_integration import snapshot_db
    from reload_mask_taxonomy_keyboard_ui import state
    require(read(ROOT/'TRAINING_HANDOFF.json')['state']=='round7_actual_training_source_replay_and_matched_comparison_complete','Training report required')
    prior=read(ROOT/'before_round7_publication.json');routes=profiles();services={};public={}
    require(all(all(routes[k].get(f)==v.get(f) for f in ('id','host','port','target','url','tunnelID')) for k,v in prior['saved_ptunnel_profiles'].items()),'Old ptunnel route changed')
    for mode,v in prior['old_services'].items():
        current=read(PROJECT/f'outputs/pallet_description_round6_{mode}_managed_v1/supervisor.json')
        require(current['pid']==v['pid'] and current['app']['pid']==v['app']['pid'] and current['public_url']==v['public_url'] and current['process_identity']==process_identity(current['pid']),'Round6 app/supervisor/URL changed')
    for mode,s in REVIEW_SETS.items():
        configure(mode);current=read(s['managed']/'supervisor.json');proof=read(s['managed']/'round7_public_verification.json')
        require(current['state']=='connected' and current['process_identity']==process_identity(current['pid']),'New supervisor unhealthy')
        require(proof['state']=='round7_ptunnel_authenticated_public_readonly_passed' and not proof['page_errors'] and not proof['blocked_writes'] and proof['production_annotation_POSTs']==proof['production_decision_POSTs']==0,'Public read-only verification required')
        require(proof['implementation_sha256']==digest(PROJECT/'scripts/verify_pallet_description_round7_public.py'),'Public checker changed')
        snapshot_db(s['reviews']/'mask_reviews.sqlite3',RELEASE/mode/'final.sqlite3');bm,be,_=state(RELEASE/mode/'before_public.sqlite3');am,ae,ar=state(RELEASE/mode/'final.sqlite3')
        require(bm==am and ae[:len(be)]==be and ar=={k:(rev,raw) for _,k,rev,raw in ae},'New current rows/raw history changed')
        require(all('SYNTHETIC' not in raw for _,k,rev,raw in ae),'Synthetic annotation present in production')
        services[mode]=dict(public_url=current['public_url'],targets=read(s['root']/'manifest.json')['request_count'],namespace=s['namespace'],events=len(ae),browser_verification_sha256=digest(RELEASE/mode/'browser_verification.json'));public[mode]=proof
    health={}
    for port in range(3413,3424):
        with urlopen(f'http://127.0.0.1:{port}/healthz',timeout=10) as response:h=json.load(response)
        require(h['status']=='ok','Old/new service unhealthy');health[str(port)]=h['publication_version']
    require(sources_current(),'Frozen sources changed before delivery');selection=read(ROOT/'release_selection.json');priority=read(ROOT/'review_priority_selection.json');data=read(ROOT/'preparation_report.json')
    lines=[(ROOT/'TRAINING_REPORT.md').read_text(),'','## 第七轮人工复核入口已验收','',
        f"- 新图类别／轮廓／语义：{services['new']['targets']}项，{services['new']['public_url']} 。",
        f"- 已有定向复核：默认重点{priority['required']}项（优先{priority['tier_counts'].get('priority',0)}＋固定50抽检），{services['existing']['public_url']} 。其余{priority['tier_counts'].get('diagnostic',0)}项模型诊断非必做；全部{services['existing']['targets']}项及历史可查。",'',
        '沿用原访问码，新域名需重新登录；3422／3423、独立任务／库／草稿namespace和独立ptunnel profiles，第六轮和所有旧网址／进度保持。没有搬迁或删除旧域名草稿。',
        f"每类100新图总目标4,000，严格K且跨轮原图去重实际{services['new']['targets']}，缺口{selection['gap']}；未放宽门槛、复制模板、补点或混入旧图。",'',
        '## 界面与安全验收','',
        '- Ctrl+S／⌘+S完成并停留；D完成后下一条，A不提交；默认复核人1、空异常默认正常、右键平移／左键修点、严格K、409、本机草稿／备份与原子追加历史保留。',
        '- 独立覆盖／结构选项／描述项增加／删除／替换，保留完整中文逗号分项精确匹配、去重、不匹配子串和不覆盖独有项；全部批量范围与D／A序列在打开详情时固定。',
        '- 桌面／390宽实际任务临时库验证快捷键、严格K、草稿／备份、409、存储失败、忙锁、D／A、所有批量预览、替换丢回复幂等及整批409、全队列替换重放；已有入口另验优先／抽检、不补入新抽检、独立疑点结论幂等与409、所有细类状态切换。完成的第六轮Class13临时组不重新开启。',
        '- 公网仅登录、看图、读取与只读替换预览；生产模拟标注和疑点结论POST均为0。未测试真实MacOS／iPhone Safari实机；保存的ptunnel profile不等于永久在线／永久域名，未配置开机自启。',
        '- 398项CPU回归通过；原旧服务和profiles保持，冻结数据／旧模型／正式标签不覆盖，不自动晋升GT或机器人资格，不提交／推送。','',
        '训练和逐项比较证据见本目录TRAINING_HANDOFF.json、mask_regression_audit.json、vlm_regression_audit.json；新任务完整验收位于outputs/pallet_description_simplified_v1/round7_v1/。']
    (ROOT/'REPORT.md').write_text('\n'.join(lines)+'\n')
    write(ROOT/'HANDOFF.json',dict(state='round7_real_training_and_bounded_next_review_delivered_ptunnel',data=data,services=services,priority=priority,quota_gap=selection['gap'],
        training_handoff_sha256=digest(ROOT/'TRAINING_HANDOFF.json'),training_verification_sha256=digest(ROOT/'training_verification.json'),bundle_source_replay_sha256=digest(ROOT/'bundle_source_replay.json'),
        report_sha256=digest(ROOT/'REPORT.md'),public_proofs={m:digest(s['managed']/'round7_public_verification.json') for m,s in REVIEW_SETS.items()},healthy_ports=health,regression_tests=398,
        old_models_data_histories_services_profiles_preserved=True,production_annotation_POSTs=0,production_decision_POSTs=0,no_commit_or_push=True))
    print(dict(state='round7_delivered',services=services,priority=priority['tier_counts'],production_annotation_POSTs=0))


if __name__=='__main__':
    import argparse
    p=argparse.ArgumentParser();p.add_argument('phase',choices=('source','training','bundles','report','deliver'));a=p.parse_args();{'source':source_replay,'training':training,'bundles':bundles,'report':report,'deliver':deliver}[a.phase]()
