"""Package/verify refinement results and publish read-only pending-review sidecars.

Never import machine predictions into the live human SQLite database. Rechecks
current saved targets immediately before publishing and excludes all of them.
"""
from __future__ import annotations

import argparse
from collections import Counter
import csv
import hashlib
import io
import json
import os
from pathlib import Path
import shutil
import sqlite3
import sys
import tempfile
from zipfile import ZipFile,ZIP_DEFLATED

import numpy as np

PROJECT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(PROJECT))
from pallet.cluster_targets import digest,load_dataset,protected_snapshot
from pallet.pose_dataset import protected_write,read_rows,write_json,write_rows
from pallet.pose_refinement import SCHEMA,REFERENCE,live_rows,original_targets
from pallet.pose_review_geometry import canonical


def archive_verified(root,paths,destination):
    temporary=destination.with_suffix('.tmp')
    with ZipFile(temporary,'w',ZIP_DEFLATED,compresslevel=6) as z:
        for path in sorted(paths):z.write(path,str(path.relative_to(root)))
    temporary.replace(destination)
    with ZipFile(destination) as z:
        assert set(z.namelist())=={str(p.relative_to(root)) for p in paths}
        for name in z.namelist():
            assert hashlib.sha256(z.read(name)).hexdigest()==digest(root/name)
    return {'archive':str(destination),'bytes':destination.stat().st_size,'sha256':digest(destination),'members_verified':len(paths)}


def make_bundle(out,config,done):
    root=out/'deployment';root.mkdir(exist_ok=True)
    for profile in done['profiles']:
        for field in ('network','ridge'):
            path=out/profile[field];protected_write(root/profile[field],path.read_bytes())
        protected_write(root/'training'/profile['branch']/'profile.json',
            (out/'training'/profile['branch']/'profile.json').read_bytes())
    detector=PROJECT/'outputs/pallet_pose_training_v1/deployment/detector.pt'
    protected_write(root/'detector.pt',detector.read_bytes())
    # Include only inference dependencies; original data, access keys and DBs are excluded.
    dependencies=['__init__.py','pose_refinement.py','pose_refinement_inference.py','pose_review_geometry.py',
        'fork_geometry.py','cluster_targets.py','pose_annotations.py','pose_dataset.py','pose_model.py',
        'pose_inference.py','devices.py','data/orientation.py','data/pcd.py']
    for name in dependencies:protected_write(root/'pallet'/name,(PROJECT/'pallet'/name).read_bytes())
    protected_write(root/'pallet/data/__init__.py',b'"""Portable registered-cloud I/O namespace."""\n')
    for name in ('results.json','predictions.jsonl'):
        protected_write(root/'reports'/name,(out/'evaluation'/name).read_bytes())
    protected_write(root/'reports/frozen_models.json',(out/'training/frozen_models.json').read_bytes())
    protected_write(root/'requirements.txt',b'numpy==1.26.4\ntorch==2.6.0\nopencv-python-headless==4.11.0.86\nscipy==1.15.3\npillow==11.0.0\nultralytics==8.4.163\n')
    files={str(p.relative_to(root)):digest(p) for p in root.rglob('*') if p.is_file() and p.name not in ('bundle_config.json','README_zh.md')}
    spec={'schema':SCHEMA,'reference':REFERENCE,'profiles':done['profiles'],'files':files,
        'points':config['points'],'seed':config['seed'],'detector_retrained':False,
        'unknown_units_require_explicit_allow_native':True,'robot_command_eligible':False,
        'full_six_dof':False,'original_test_used_for_selection':False}
    write_json(root/'bundle_config.json',spec)
    return root


def verify_provenance(out,config,targets):
    snapshot=json.loads((out/'snapshot/manifest.json').read_text())
    assert digest(out/'snapshot/reviews.sqlite3')==snapshot['database_sha256']
    assert digest(out/'snapshot/eligible.jsonl')==snapshot['eligible_sha256']
    frozen=json.loads((out/'training/frozen_models.json').read_text())
    assert frozen['schema']==SCHEMA and frozen['reference']==REFERENCE
    assert not frozen['val_test_used_for_selection']
    assert digest(PROJECT/'pallet/pose_refinement.py')==frozen['source_sha256']
    eligible={r['target_key']:r for r in read_rows(out/'training/eligible_after_features.jsonl')}
    cv_details=[]
    for profile in frozen['profiles']:
        assert digest(out/profile['network'])==profile['network_sha256']
        assert digest(out/profile['ridge'])==profile['ridge_sha256']
        cv=json.loads((out/'training'/profile['branch']/'cross_validation.json').read_text())
        assert cv['selected_family']==profile['selected_family']
        allkeys=set(profile['training_keys']);val_seen=[]
        assert all(eligible[k]['split']=='train' and eligible[k]['branch']==profile['branch'] for k in allkeys)
        for fold in cv['folds']:
            ti,vi=set(fold['fit_keys']),set(fold['validation_keys'])
            assert not ti&vi and ti|vi==allkeys
            assert not set(fold['fit_groups'])&set(fold['validation_groups'])
            val_seen.extend(vi)
        assert len(val_seen)==len(set(val_seen))==len(allkeys)
        cv_details.append({'branch':profile['branch'],'training_samples':len(allkeys),
            'groups':profile['training_groups'],'selected_family':profile['selected_family']})
    with sqlite3.connect((out/'snapshot/reviews.sqlite3').resolve().as_uri()+'?mode=ro',uri=True) as db:
        rows={k:json.loads(v) for k,v in db.execute('SELECT target_key,payload FROM annotations')}
    for key,value in eligible.items():
        assert hashlib.sha256(canonical(rows[key])).hexdigest()==value['source_review_sha256']
        assert value['review_revision']==rows[key]['revision']
        assert value['source_pcd_sha256']==targets[key]['source_pcd_sha256']
    return snapshot,cv_details


def publish_labels(out,config,targets,snapshot):
    root=out/'published_labels';root.mkdir(exist_ok=True)
    live=live_rows(PROJECT/config['review_directory'])
    rows=read_rows(out/'annotations/targets.jsonl')
    filtered=[r for r in rows if r['target_key'] not in live]
    saved_keys=set(snapshot['all_saved_keys'])
    for row in filtered:
        key=row['target_key'];assert key not in saved_keys
        assert row['bbox']==targets[key]['bbox'] and row['split']==targets[key]['split']
        assert row['source_pcd_sha256']==targets[key]['source_pcd_sha256']
        assert not row['human_pose_confirmed'] and not row['automatic_training_eligible']
        pose=row.get('pose_estimate')
        if pose:
            assert pose['reference_point']==REFERENCE and pose['z'] is None and pose['normal'] is None
            assert pose['quaternion'] is None and not pose['robot_command_eligible']
            if row['branch']=='native_unknown':assert row['metric_pose_estimate'] is None and pose['position_unit']=='native_unknown'
    write_rows(root/'targets.jsonl',filtered)
    write_rows(root/'preserved_human_references.jsonl',[{'target_key':k,'revision':v['revision'],
        'source_UI_state':v['state'],'source_UI_human_pose_confirmed':v['human_pose_confirmed'],
        'pose_source':'live_human_review','replaced':False} for k,v in sorted(live.items())])
    effective=[];updated={r['target_key']:r for r in filtered}
    for k,t in targets.items():
        effective.append({**t,'pose_label_source':'live_human_review' if k in live else 'updated_machine' if k in updated else 'not_updated',
            'human_revision':live[k]['revision'] if k in live else None,
            'machine_pose':updated[k]['pose_estimate'] if k in updated else None})
    write_rows(root/'effective_target_index.jsonl',effective)
    for split in ('train','val','test'):write_rows(root/f'indices/{split}.jsonl',[r for r in filtered if r['split']==split])
    write_rows(root/'indices/automatic_training_eligible.jsonl',[])
    stream=io.StringIO(newline='');cols=['target_key','frame_id','split','branch','x','y','theta_degrees','position_unit','model_family','unavailable_reason']
    writer=csv.DictWriter(stream,fieldnames=cols);writer.writeheader()
    for r in filtered:
        row={k:r.get(k) for k in cols};row.update({k:(r['pose_estimate'] or {}).get(k) for k in ('x','y','theta_degrees','position_unit')});writer.writerow(row)
    protected_write(root/'targets.csv',b'\xef\xbb\xbf'+stream.getvalue().encode())
    stats={'original_targets':len(targets),'snapshot_saved':snapshot['saved_targets'],'live_saved_preserved':len(live),
        'machine_target_records':len(filtered),'new_pose_estimates':sum(r['pose_estimate'] is not None for r in filtered),
        'new_poses_by_branch':dict(Counter(r['branch'] for r in filtered if r['pose_estimate'])),
        'no_pose_reason_counts':dict(Counter(r['unavailable_reason'] for r in filtered if not r['pose_estimate'])),
        'preserved_human_revisions':{k:v['revision'] for k,v in live.items()},
        'additional_saved_targets_since_snapshot':len(set(live)-saved_keys),
        'new_human_work_filtered_at_publication':len(rows)-len(filtered),
        'source_database_writes':0,'old_GT_replaced':False,'original_split_preserved':True}
    write_json(root/'summary.json',stats)
    return root,stats


def main():
    parser=argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--config',type=Path,default=PROJECT/'configs/pallet_pose_refinement_v2.json')
    parser.add_argument('--activate',action='store_true')
    args=parser.parse_args();config=json.loads(args.config.read_text());out=PROJECT/config['output']
    formal=Path(json.loads((PROJECT/config['formal_config']).read_text())['dataset'])
    annotations,sources=load_dataset(formal);targets=original_targets(annotations,sources)
    done=json.loads((out/'training/completed.json').read_text());snapshot,cv=verify_provenance(out,config,targets)
    assert protected_snapshot(formal)==json.loads((out/'snapshot/protected_formal.json').read_text())
    bundle=make_bundle(out,config,done);labels,summary=publish_labels(out,config,targets,snapshot)
    # Both archives are independent, do not contain source data or production DBs/access keys.
    note='''# 人工标尺修正模型与预标注 v2

研究标签经用户本轮明确授权从已保存草稿快照提取；没有批量改写UI确认字段，也不是独立盲标真值。
只用原train训练，原train内采集组CV选模型/轮数，原val/test不选模型。原检测器保持v1不重训。
新参考点统一为叉取面宽度中心；旧154条不同源表口径GT不混入。
metric和native_unknown两路独立训练；未知单位只做无量纲位置残差学习，输出仍为原始单位，不能当作米制真值。
输入仅原人工框和注册点云的自动聚类平面特征，不读取最终人工多边形、选择面或标尺作为模型输入。
所有新预测为待复核机器建议，不自动回流训练、非完整6D、不能直接用于机器人控制。
已有及发布时新增的人工草稿均不覆盖；原val/test不回流训练；原263帧负例保持不变。

模型目录deployment可独立移走运行：
OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 python -m pallet.pose_refinement_inference --bundle . --image 原图.png --cloud 注册点云.npz --domain 传感器域 --scale-to-m 原登记比例 --output 预测.json
NPZ包含points_xyzi与pixel_to_source_index；未知单位去掉scale-to-m并显式加--allow-native。
模型包不包含原图/PCD，不依赖GT或人工标尺输入。标注包是JSONL/CSV轻量侧车，不重复原图。
'''
    protected_write(out/'README_zh.md',note.encode());protected_write(bundle/'README_zh.md',note.encode());protected_write(labels/'README_zh.md',note.encode())
    models_archive=archive_verified(bundle,[p for p in bundle.rglob('*') if p.is_file() and '__pycache__' not in p.parts],out/'pallet_pose_refined_models_v2.zip')
    labels_archive=archive_verified(labels,[p for p in labels.rglob('*') if p.is_file()],out/'pallet_pose_annotations_v2.zip')
    verification={'state':'passed','cv_protocol':cv,'snapshot_labels_verified':snapshot['eligible'],
        'snapshot_database_sha256':snapshot['database_sha256'],'protected_formal_files_unchanged':snapshot['protected_formal_files'],
        'snapshot_human_UI_flags_modified':False,'test_used_for_model_selection':False,
        'all_original_target_identities_preserved':len(targets),'source_review_database_writes':0,
        'machine_targets_verified':summary['machine_target_records'],'labels_archive':labels_archive,
        'models_archive':models_archive,'human_precedence_checked_at_publication':True}
    write_json(out/'validation/provenance_verification.json',verification)
    if args.activate:
        active={'kind':'pending_machine_fork_face_center_pose_update','version':2,'schema':SCHEMA,
            'targets':str((labels/'targets.jsonl').relative_to(PROJECT)),'targets_sha256':digest(labels/'targets.jsonl'),
            'formal_manifest_sha256':digest(formal/'manifest.json'),'snapshot_manifest':str((out/'snapshot/manifest.json').relative_to(PROJECT)),
            'models':str(bundle.relative_to(PROJECT)),'machine_targets':summary['machine_target_records'],
            'human_review_precedence':'always_live_store_before_machine','source_review_database_writes':False,
            'full_six_dof':False,'robot_command_eligible':False}
        config_path=PROJECT/'configs/active_pose_refinement.json'
        if config_path.exists():protected_write(out/'previous_active_pose_refinement.json',config_path.read_bytes())
        temporary=config_path.with_suffix('.tmp');temporary.write_text(json.dumps(active,ensure_ascii=False,indent=2)+'\n');temporary.replace(config_path)
    report={'state':'complete','summary':summary,'models':models_archive,'labels':labels_archive,'activated':args.activate}
    delivery=out/'delivery_manifest.json'
    if delivery.exists() and json.loads(delivery.read_text())!=report:
        protected_write(out/('prior_delivery_'+digest(delivery)[:12]+'.json'),delivery.read_bytes())
        temporary=delivery.with_suffix('.tmp');temporary.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n');temporary.replace(delivery)
    else:write_json(delivery,report)
    print(json.dumps({**report,'summary':{k:v for k,v in summary.items() if k!='preserved_human_revisions'}},ensure_ascii=False),flush=True)


if __name__=='__main__':main()
