"""Model-native editable prelabels toward100/class; never fabricated human saves."""
from collections import Counter, defaultdict
from contextlib import closing
import hashlib
import json
import math
from pathlib import Path
import re
import shutil
import time

import cv2
import numpy as np
from PIL import Image, ImageDraw

from .cluster_targets import digest, protected_snapshot
from .mask_model_training_data import PROJECT, read, jsonl, write, sha, require, letterbox, gpu_receipt
from .mask_model_improvement_adapter import mask_logits, native_logits, highest_class_query
from .mask_annotation_full_data import context
from .mask_annotation_full_train import normalize
from .mask_quality_recovery import POSE, GALLERY, fingerprint
from .mask_quota_saved_extension import file_inventory, load_formal_evidence
from .mask_remaining_classes import confirmed, original_rectangle
from .mask_template_refined import QUARANTINE, readonly
from .pose_review_geometry import polygon_value, raster_mask

ROOT=PROJECT/'outputs/pallet_mask_model_requests_v1'
MODEL=PROJECT/'outputs/pallet_mask_annotation_full_v1'
SOURCE=PROJECT/'outputs/pallet_mask_continuation_sources_v16'
CHECKPOINT_SHA='92ce35476d4b0887a0e791dc20972c9640aab558ec876df84296b713ebe8ed4c'
SCHEMA='model_native_mask_preannotation_v1'
FLAGS=('human_edited','human_mask_confirmed','human_pose_confirmed','ground_truth_eligible','automatic_training_eligible',
       'robot_command_eligible','production_saved','counts_as_completed_annotation','accepted_candidate','propagation_seed_eligible')


def raw_sha(raw):return hashlib.sha256(raw.encode('utf-8')).hexdigest()


def source_context():
    summary=read(SOURCE/'audit.json')
    require(summary['source_catalog_sha256']==digest(SOURCE/'private/source_catalog.jsonl'),'Changed source snapshot')
    require(fingerprint(POSE)==summary['pose_fingerprint'] and fingerprint(GALLERY)==summary['gallery_fingerprint'],'New saves arrived; refresh source version')
    formal,rows,ev=load_formal_evidence(read(PROJECT/'outputs/pallet_mask_remaining_round5_v1/manifest.json'))
    records=jsonl(MODEL/'private/training_records.jsonl');catalog=jsonl(SOURCE/'private/source_catalog.jsonl')
    isolated=PROJECT/'outputs/pallet_mask_nonconforming_reviews_v1/mask_reviews.sqlite3'
    # The isolated source remains explicitly user-quality-approved in MODEL permit.
    with closing(readonly(isolated)) as db:
        nc={k:json.loads(v) for k,v in db.execute('SELECT target_key,payload FROM reviews')}
    require(set(nc)=={r['target_key'] for r in records if r['source_origin']=='saved_nonconforming_isolated'},'Isolated completed source inventory changed')
    counts=Counter(r['row']['final_class_id'] for r in catalog)
    for k,v in nc.items():
        require(v['polygon'] and v['quality_decision']=='redrawn_pending_quality_check','QC-approved isolated contour missing')
        counts[rows[k]['final_class_id']]+=1
    protected=set(QUARANTINE)|{r['target_key'] for r in catalog}|set(nc)|{r['target_key'] for r in records}
    protected.update(p['target_key'] for p in jsonl(PROJECT/'outputs/pallet_mask_remaining_round5_v1/proposals.jsonl'))
    with closing(readonly(POSE)) as db:
        current=dict(db.execute('SELECT target_key,payload FROM annotations'));histories=defaultdict(list)
        for k,v in db.execute('SELECT target_key,payload FROM events ORDER BY id'):histories[k].append(json.loads(v))
    for k in set(current)|set(histories):
        values=histories[k]+([json.loads(current[k])] if k in current else [])
        if k not in current or any(v.get('state') in ('deleted','deferred','confirmed') or confirmed(v) or not original_rectangle(v.get('polygon'),rows[k]) for v in values):protected.add(k)
    inventories={}
    for pattern in ('requests.jsonl','shortlist.jsonl','inspection_targets.jsonl'):
        for path in sorted((PROJECT/'outputs').rglob(pattern)):
            if path.is_relative_to(ROOT):continue
            for r in jsonl(path):
                if r.get('target_key') in rows:protected.add(r['target_key'])
            inventories[str(path.relative_to(PROJECT))]=digest(path)
    frames={rows[k]['frame_id'] for k in protected};hashes={digest(formal/name) for name in {rows[k]['image'] for k in protected}}
    return dict(formal=formal,rows=rows,evidence=ev,counts=counts,gaps={c:max(0,100-counts[c]) for c in range(1,23)},
        protected=protected,frames=frames,image_hashes=hashes,pose_current_raw=current,history=histories,inventories=inventories,
        fingerprints=dict(pose=fingerprint(POSE),gallery=fingerprint(GALLERY),isolated=fingerprint(isolated)))


def stage():
    from scripts.build_mask_template_requests import native_quality
    require(not ROOT.exists(),'New immutable model-native worklist required')
    data=source_context();rows=data['rows'];available={};skips=Counter()
    for k,row in sorted(rows.items()):
        if not data['gaps'][row['final_class_id']]:continue
        if k in data['protected'] or row['frame_id'] in data['frames']:skips['protected_history_or_frame']+=1;continue
        q,reason=native_quality(row,data['evidence'])
        if reason:skips[reason]+=1;continue
        h=digest(data['formal']/row['image'])
        if h in data['image_hashes']:skips['protected_image_bytes']+=1;continue
        available[k]=(q,h)
    selected=[];frames=set(data['frames']);hashes=set(data['image_hashes']);groups=Counter();policies=Counter();capacity=Counter()
    fseen=set(frames);hseen=set(hashes)
    for k,(q,h) in sorted(available.items()):
        row=rows[k]
        if row['frame_id'] not in fseen and h not in hseen:
            capacity[row['final_class_id']]+=1;fseen.add(row['frame_id']);hseen.add(h)
    for cls,gap in data['gaps'].items():
        for _ in range(math.ceil(gap*1.5)+3 if gap else 0):
            options=[k for k,(_,h) in available.items() if rows[k]['final_class_id']==cls and rows[k]['frame_id'] not in frames and h not in hashes]
            if not options:break
            key=min(options,key=lambda k:(groups[cls,rows[k]['group_id']],policies[cls,rows[k]['policy_id']],
                -available[k][0]['bbox_height_native_pixels'],-available[k][0]['clarity_score'],k))
            row=rows[key];selected.append(key);frames.add(row['frame_id']);hashes.add(available[key][1]);groups[cls,row['group_id']]+=1;policies[cls,row['policy_id']]+=1
    ROOT.mkdir(mode=0o700);(ROOT/'private').mkdir(mode=0o700)
    staged=[]
    for order,k in enumerate(selected,1):
        row=rows[k];raw=data['pose_current_raw'].get(k)
        staged.append(dict(target_key=k,class_id=row['final_class_id'],source=row,request_order=order,native_quality=available[k][0],image_sha256=available[k][1],
            pose_payload_raw_sha256=raw_sha(raw) if raw else None,gallery_payload_raw_sha256=None,existing_pose_revision=json.loads(raw)['revision'] if raw else 0))
    write(ROOT/'private/shortlist.jsonl',staged,lines=True)
    require(digest(MODEL/'m2/final_ema.pt')==CHECKPOINT_SHA,'Completed M2 checkpoint pin differs')
    write(ROOT/'input_manifest.json',dict(actual_class_counts=dict(data['counts']),remaining_to100=data['gaps'],current_manual_total=sum(data['counts'].values()),
        prior_worklist_inventories_sha256=data['inventories'],native_capacity=dict(capacity),source_audit_sha256=digest(SOURCE/'audit.json'),
        source_fingerprints=data['fingerprints'],configuration_sha256=file_inventory(PROJECT/'configs'),formal_protected=protected_snapshot(data['formal']),
        formal_manifest_sha256=read(PROJECT/'outputs/pallet_mask_remaining_round5_v1/manifest.json')['formal_manifest_sha256'],
        checkpoint_sha256=CHECKPOINT_SHA,model_selection='M2 final EMA: stronger training-fit P10 and fractionIoU>=.90; NOT blind-best or new-image quality guarantee',
        shortage_policy='No repeats/old rejects; Class19 capacity0 remains shortage; predictions do not count as completed',
        staged_counts=dict(Counter(r['class_id'] for r in staged)),shortlist_sha256=digest(ROOT/'private/shortlist.jsonl'),skips=dict(skips)))
    print(json.dumps(dict(state='staged_unprocessed_for_actual_M2_inference',count=len(staged),by_class=dict(Counter(r['class_id'] for r in staged)),gaps=data['gaps'],capacity=dict(capacity))),flush=True)


def polygonize(mask):
    """Pixel-cell union, no convex hull/filling holes/forced vertex counts."""
    from shapely.geometry import box
    from shapely.ops import unary_union
    if not mask.any():return None,dict(reason='empty_model_mask')
    n,labels,stats,_=cv2.connectedComponentsWithStats(mask.astype(np.uint8),8)
    if n!=2:return None,dict(reason='multiple_disconnected_model_components',components=n-1)
    cells=[]
    for y,row in enumerate(mask):
        padded=np.r_[False,row,False].astype(np.int8);starts=np.where(np.diff(padded)==1)[0];ends=np.where(np.diff(padded)==-1)[0]
        cells.extend(box(int(l),y,int(r),y+1) for l,r in zip(starts,ends))
    shape=unary_union(cells)
    if shape.geom_type!='Polygon' or not shape.is_valid:return None,dict(reason='invalid_or_touching_model_topology')
    if shape.interiors:return None,dict(reason='closed_model_holes_not_single_ring',holes=len(shape.interiors))
    for tolerance in (.2,.4,.65,.9):
        simplified=shape.simplify(tolerance,preserve_topology=True)
        points=[[float(x),float(y)] for x,y in list(simplified.exterior.coords)[:-1]]
        if not 3<=len(points)<=128:continue
        try:polygon_value(points,mask.shape[1],mask.shape[0])
        except ValueError:continue
        rendered=raster_mask(points,mask.shape[1],mask.shape[0]);iou=float((rendered&mask).sum()/(rendered|mask).sum())
        if iou>=.99:return points,dict(reason=None,polygon_vs_model_mask_iou=iou,simplification_tolerance_native_pixels=tolerance,vertices=len(points))
    return None,dict(reason='cannot_represent_model_boundary_without_large_change')


def infer():
    import torch
    from rfdetr import RFDETRSegSmall
    from .mask_model_improvement import predicted_rle
    require(not (ROOT/'private/predictions.jsonl').exists(),'Preserve existing inference/failure evidence')
    gpu=gpu_receipt('m2');torch.set_num_threads(4);cv2.setNumThreads(1)
    checkpoint=MODEL/'m2/final_ema.pt';require(digest(checkpoint)==CHECKPOINT_SHA,'Model checkpoint changed')
    weights=torch.load(checkpoint,map_location='cpu',weights_only=True)
    generic=PROJECT/'outputs/pallet_mask_model_training_v1/generic_pretraining/rf-detr-seg-small.pt'
    wrapper=RFDETRSegSmall(num_classes=1,resolution=576,device='cuda:0',pretrain_weights=str(generic),gradient_checkpointing=True,compile=False)
    model=wrapper.model.model;model.load_state_dict(weights['model'],strict=True);model.cuda().eval()
    del weights
    data=source_context();outcomes=[];started=time.monotonic()
    write(ROOT/'private/inference_start.json',dict(checkpoint_sha256=CHECKPOINT_SHA,gpu=gpu,model_eval=True,no_target_labels_input=True,threshold=.5))
    with torch.inference_mode():
        for index,r in enumerate(jsonl(ROOT/'private/shortlist.jsonl')):
            row=r['source'];roi=context(row['bbox'],row['size']);x1,y1,x2,y2=roi
            grey=data['evidence'].image(row['frame_id'])[y1:y2,x1:x2];canvas,_,_,transform=letterbox(grey,None,(576,576))
            outputs=model(normalize(canvas)[None].cuda());q,score=highest_class_query(outputs)
            probability=native_logits(mask_logits(outputs)[0,q],transform).sigmoid().cpu().numpy();mask=probability>=.5
            reason=None;b=row['bbox'];area=int(mask.sum())
            if mask[0].any() or mask[-1].any() or mask[:,0].any() or mask[:,-1].any():reason='model_mask_touches_context_edge'
            bx0=max(0,math.floor(b['x']-x1));by0=max(0,math.floor(b['y']-y1));bx1=min(mask.shape[1],math.ceil(b['x']+b['width']-x1));by1=min(mask.shape[0],math.ceil(b['y']+b['height']-y1))
            overlap=float(mask[by0:by1,bx0:bx1].sum()/max(1,area));ratio=area/(b['width']*b['height'])
            if not .05<=ratio<=1.25:reason=reason or 'implausible_mask_area_for_target_box'
            if overlap<.75:reason=reason or 'model_mask_not_localized_to_requested_target'
            polygon,metrics=polygonize(mask) if reason is None else (None,dict(reason=reason))
            if polygon:polygon=[[x+x1,y+y1] for x,y in polygon]
            outcome=dict(target_key=r['target_key'],class_id=r['class_id'],roi=roi,raw_native_prediction_rle=predicted_rle(mask),
                selected_query=q,class_confidence_not_quality_probability=score,raw_mask_area=area,raw_mask_bbox_area_ratio=ratio,
                raw_mask_fraction_inside_original_bbox=overlap,polygon=polygon,polygonization=metrics,checkpoint_sha256=CHECKPOINT_SHA,
                model_threshold=.5,inference_input='image_pixels_in_fixed_original_bbox_context_only')
            outcomes.append(outcome)
            if index==0:write(ROOT/'private/first_actual_inference.json',dict(state='actual_M2_checkpoint_forward_succeeded',target_key=r['target_key'],gpu=gpu,checkpoint_sha256=CHECKPOINT_SHA))
            if (index+1)%100==0:print('M2 inferred',index+1,'new targets',flush=True)
    write(ROOT/'private/predictions.jsonl',outcomes,lines=True)
    write(ROOT/'private/inference_complete.json',dict(state='actual_M2_inference_completed_not_quality_pass',targets=len(outcomes),seconds=time.monotonic()-started,
        nonempty_polygons=sum(bool(r['polygon']) for r in outcomes),rejections=dict(Counter(r['polygonization']['reason'] for r in outcomes if not r['polygon'])),gpu=gpu))
    print(read(ROOT/'private/inference_complete.json'),flush=True)


def validate_proposal(p):
    require(p['schema']==SCHEMA and p['origin']=='model_prediction' and p['status']=='model_preannotated_pending_human_review'
        and all(p.get(k) is False for k in FLAGS) and p['vertex_count']==len(p['polygon'])
        and p['model_provenance']['checkpoint_sha256']==CHECKPOINT_SHA and p['model_provenance']['model_family']=='RF-DETR-Seg Small'
        and p['model_provenance']['human_saved'] is False
        and re.fullmatch('[0-9a-f]{24}',p['target_key']) and p['target_key']==p['source']['target_key'],
        'Model-native source must not masquerade as assistant drawing/human save/GT')
    polygon_value(p['polygon'],*p['source']['size'])


def build():
    from scripts.build_mask_remaining_direct_requests import build_font
    from .mask_model_review import editor_js
    require(not (ROOT/'manifest.json').exists(),'Never overwrite a completed publication')
    data=source_context();inputs=read(ROOT/'input_manifest.json');preds={r['target_key']:r for r in jsonl(ROOT/'private/predictions.jsonl')}
    choices=read(ROOT/'private/visual_decisions.json')
    require(choices['predictions_sha256']==digest(ROOT/'private/predictions.jsonl'),'Visual decision inputs differ')
    for d in ('images','cases','masks','labelme'):(ROOT/d).mkdir()
    selected=[];withheld=[];counts=Counter()
    for record in jsonl(ROOT/'private/shortlist.jsonl'):
        key=record['target_key'];prediction=preds[key];cls=record['class_id'];decision=choices['decisions'].get(key,{})
        if not prediction['polygon'] or not decision.get('publish_as_machine_draft') or counts[cls]>=data['gaps'][cls]:
            withheld.append(dict(target_key=key,class_id=cls,reason=prediction['polygonization']['reason'] or decision.get('reason','class_quota_or_visual_withheld')));continue
        row=record['source'];proof=dict(model_family='RF-DETR-Seg Small',training_version=str(MODEL.relative_to(PROJECT)),checkpoint='m2/final_ema.pt',checkpoint_sha256=CHECKPOINT_SHA,
            predictions_sha256=digest(ROOT/'private/predictions.jsonl'),native_mask_rle_sha256=sha(prediction['raw_native_prediction_rle']),
            threshold=.5,query_selected_by='classification_only_no_target_GT',human_saved=False,assistant_redrawn=False,
            polygonization=prediction['polygonization'],visual_inspection=decision,quality_gate_passed=False,
            count_before_real_human_save=False)
        p=dict(schema=SCHEMA,origin='model_prediction',target_key=key,class_id=cls,source=row,polygon=prediction['polygon'],vertex_count=len(prediction['polygon']),
            status='model_preannotated_pending_human_review',formal_manifest_sha256=inputs['formal_manifest_sha256'],model_provenance=proof,
            model_source_image_sha256=record['image_sha256'],pose_payload_raw_sha256=record['pose_payload_raw_sha256'],gallery_payload_raw_sha256=None,
            mask_semantics='model_visible_fork_face_draft_requires_human_review_not_GT',**dict.fromkeys(FLAGS,False))
        validate_proposal(p);selected.append(p);counts[cls]+=1
        shutil.copyfile(data['formal']/row['image'],ROOT/'images'/f'{key}.png')
        native=Image.open(ROOT/'images'/f'{key}.png').convert('RGB');d=ImageDraw.Draw(native);poly=p['polygon'];d.line([tuple(v) for v in poly]+[tuple(poly[0])],fill='#55d5ff',width=1)
        b=row['bbox'];l=max(0,math.floor(b['x']-12));t=max(0,math.floor(b['y']-16));r=min(row['size'][0],math.ceil(b['x']+b['width']+12));bt=min(row['size'][1],math.ceil(b['y']+b['height']+16))
        native.crop((l,t,r,bt)).resize(((r-l)*3,(bt-t)*3),Image.Resampling.NEAREST).save(ROOT/'cases'/f'{key}.png')
        Image.fromarray(raster_mask(poly,*row['size']).astype(np.uint8)*255).save(ROOT/'masks'/f'{key}.png')
        write(ROOT/'labelme'/f'{key}.json',dict(version='5.0.0',imagePath=f'../images/{key}.png',imageData=None,imageWidth=row['size'][0],imageHeight=row['size'][1],
            flags=dict(model_preannotation=True,human_confirmed=False,ground_truth_eligible=False),annotation_scope='single_target_not_full_frame',
            model_checkpoint_sha256=CHECKPOINT_SHA,shapes=[dict(label='pallet',shape_type='polygon',points=poly,flags=dict(human_confirmed=False))]))
    require(selected,'No publishable machine drafts')
    write(ROOT/'proposals.jsonl',selected,lines=True);write(ROOT/'requests.jsonl',[dict(target_key=p['target_key'],class_id=p['class_id'],status='machine_draft_not_completed') for p in selected],lines=True)
    write(ROOT/'private/withheld.json',withheld)
    page='<!doctype html><html lang="zh-CN"><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>22类累计100张 · M2新图预标注复核</title><style>@font-face{font-family:Review;src:url("report-font.otf")}*{box-sizing:border-box}body{font:16px/1.6 Review,sans-serif;background:#101720;color:#e7edf5;max-width:1100px;margin:auto;padding:12px}article{border:1px solid #536479;padding:12px;margin:18px 0;min-width:0}p,h2,td{overflow-wrap:anywhere}img{display:block;max-width:100%;height:auto}a{color:#80ceff}table{width:100%;border-collapse:collapse}td,th{padding:5px;border:1px solid #536479}@media(max-width:480px){h1{font-size:23px}body{padding:8px}}</style>'
    page+=f'<h1>累计每类100张 · 新模型预标注复核</h1><p>本批{len(selected)}张来自M2最终EMA真实预测，不是助手手画、严格传播通过或人工保存。先检查并保存，机器草稿未保存不计完成。有效四点保持四点，不强迫增点；不自动获得GT、训练、传播模板或机器人资格。</p><p>沿用原访问码，新增独立可追溯模型掩膜历史库，全部旧入口/旧人工历史保持。来源旧划分保留审计，当前全量训练授权不改变原文件。Class19没有安全新图，缺口不凑数。</p><p>筛选 <select id="request-filter"><option value="">全部</option>'+''.join(f'<option value="{c}">Class {c}</option>' for c in range(1,23))+'</select></p><table><tr><th>类别</th><th>真实保存</th><th>本批草稿</th><th>若全部保存后仍缺</th></tr>'
    for c in range(1,23):page+=f'<tr><td>Class {c}</td><td>{data["counts"][c]}</td><td>{counts[c]}</td><td>{max(0,100-data["counts"][c]-counts[c])}</td></tr>'
    page+='</table>'
    for p in selected:
        key=p['target_key'];page+=f'<article id="target-{key}" data-class="{p["class_id"]}"><h2>Class {p["class_id"]} · 模型粗预标注，待人工复核</h2><img loading="lazy" src="cases/{key}.png" alt="模型轮廓，不是已确认标签"><p>梁缘、脚底、孔口及错面仍需人工判断；该模型没有独立新图质量证明。</p><a href="images/{key}.png">原图</a> · <a href="masks/{key}.png">机器掩膜</a> · <a href="labelme/{key}.json">LabelMe</a></article>'
    page+='<script>const f=document.getElementById("request-filter");f.onchange=()=>document.querySelectorAll("article").forEach(a=>a.hidden=!!f.value&&a.dataset.class!==f.value);</script></html>'
    (ROOT/'report_zh.html').write_text(page);build_font(ROOT,page)
    (ROOT/'model-editor.js').write_text(editor_js())
    assets={n:digest(ROOT/n) for n in ('report_zh.html','report-font.otf','FONT_LICENSE.txt','model-editor.js')}
    for p in selected:
        for d,e in (('images','png'),('cases','png'),('masks','png'),('labelme','json')):
            n=f'{d}/{p["target_key"]}.{e}';assets[n]=digest(ROOT/n)
    write(ROOT/'manifest.json',dict(schema='model_native_mask_worklist_v1',request_count=len(selected),class_counts=dict(counts),filter_classes=list(range(1,23)),
        actual_human_saved_before=dict(data['counts']),target_per_class=100,conditional_gap_after_all_drafts_saved={c:max(0,100-data['counts'][c]-counts[c]) for c in range(1,23)},
        checkpoint_sha256=CHECKPOINT_SHA,input_manifest_sha256=digest(ROOT/'input_manifest.json'),proposals_sha256=digest(ROOT/'proposals.jsonl'),requests_sha256=digest(ROOT/'requests.jsonl'),
        predictions_sha256=digest(ROOT/'private/predictions.jsonl'),public_assets=assets,source_fingerprints=data['fingerprints'],
        qualified_or_confirmed_labels_created=0,production_human_events_created=0,global_configuration_writes=0,models_retrained=False,
        source_schema=SCHEMA,code_sha256=digest(__file__),quality_pass_not_certified=True))
    print(json.dumps(dict(state='built_model_native_drafts_pending_readonly_verification',count=len(selected),classes=dict(counts),conditional_remaining=sum(max(0,100-data['counts'][c]-counts[c]) for c in range(1,23)))),flush=True)
