"""Read-only, post-hoc paper-vs-v5 analysis. Never retunes frozen experiments."""
from __future__ import annotations

from collections import Counter
from concurrent.futures import ThreadPoolExecutor
import hashlib
import json
from pathlib import Path
import sys
import xml.etree.ElementTree as ET

import numpy as np
from scipy.spatial import cKDTree

PROJECT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(PROJECT))
from pallet.cluster_targets import digest
from pallet.pose_dataset import protected_write, read_rows, write_json
from pallet.pose_paper_data import ROOT
from pallet.pose_paper_methods import plane_fit, plane_regions

OUTPUT = PROJECT / 'outputs/pallet_pose_paper_gap_analysis_v1'
METHODS = ('xiao_plane_template', 'pila_geometry', 'shao_agwf', 'fpfh_ablation')


def stats(values):
    a = np.asarray(values, dtype=float)
    if not len(a):
        return dict(n=0, mean=None, median=None, p90=None, p95=None, min=None, max=None)
    return dict(n=len(a), mean=float(a.mean()), median=float(np.median(a)),
        p90=float(np.quantile(a,.9)), p95=float(np.quantile(a,.95)),
        min=float(a.min()), max=float(a.max()))


def errors(record, label):
    p = np.asarray(record['pose'], dtype=float)
    delta = (p[:2] - [label['pose']['x'],label['pose']['y']])*1000
    position = float(np.linalg.norm(delta))
    angle = float(abs((np.rad2deg(p[2])-label['pose']['theta_degrees']+180)%360-180))
    return dict(position_mm=position, angle_deg=angle, abs_source_X_mm=float(abs(delta[0])),
        abs_source_Y_mm=float(abs(delta[1])), position_pass=position<=50,
        angle_pass=angle<=5, joint_pass=position<=50 and angle<=5)


def summarize(records, labels):
    good = [r for r in records if r['state']=='complete']
    values = [errors(r,labels[r['target_key']]) for r in good]
    failures = Counter(r['reason'] for r in records if r['state']!='complete')
    return dict(total=len(records), outputs=len(good), no_output=len(records)-len(good),
        joint_pass=sum(v['joint_pass'] for v in values),
        output_but_joint_fail=sum(not v['joint_pass'] for v in values),
        position_pass=sum(v['position_pass'] for v in values),angle_pass=sum(v['angle_pass'] for v in values),
        full_denominator_joint_rate=sum(v['joint_pass'] for v in values)/len(records),
        position=stats([v['position_mm'] for v in values]),angle=stats([v['angle_deg'] for v in values]),
        pose_XY_RMSE_mm=float(np.sqrt(np.mean([v['position_mm']**2 for v in values]))),
        MAE_source_X_mm=float(np.mean([v['abs_source_X_mm'] for v in values])),
        MAE_source_Y_mm=float(np.mean([v['abs_source_Y_mm'] for v in values])),
        failures=dict(failures),
        position_pass_subset=stats([v['position_mm'] for v in values if v['position_pass']]),
        position_fail_subset=stats([v['position_mm'] for v in values if not v['position_pass']]))


def paper_tables():
    p=ROOT/'paper_sources/shao_2023_fulltext.xml'
    tree=ET.fromstring(p.read_bytes())
    tables={}
    for table in tree.findall('.//table-wrap'):
        rows=[]
        for row in table.findall('.//tr'):
            rows.append([' '.join(' '.join(c.itertext()).split()) for c in row if c.tag in ('th','td')])
        tables[table.attrib['id']]=dict(label=table.findtext('label'),rows=rows)
    # Transcription from Table 2, not inferred from the article's rounded means.
    cases=[('a',.0102,.0267,.1124,True),('b',.0109,.0340,.1344,True),
           ('c',.0212,.0534,.1900,True),('d',.0536,.0687,6.5300,False),
           ('e',.0035,.0005,.4060,True),('f',.0096,.0012,1.0620,True),
           ('g',.0036,.0023,1.2014,True),('h',.0072,.0421,3.2503,False)]
    derived=[]
    for name,x,y,angle,included in cases:
        pos=float(np.hypot(x,y)*1000)
        derived.append(dict(case=name,absolute_error_x_m=x,absolute_error_y_m=y,
            angle_error_deg=angle,included_in_paper_average=included,
            reconstructed_planar_error_mm=pos,passes_50mm_5deg=pos<=50 and angle<=5))
    selected=[c for c in derived if c['included_in_paper_average']]
    return dict(source=str(p.relative_to(PROJECT)),source_sha256=digest(p),tables=tables,
        ground_scene_reported=dict(mean_abs_x_mm=9.8,mean_abs_y_mm=19.4,mean_angle_deg=.501,accuracy_percent=97.3,
            exclusions=['d','h'],accuracy_is_not_50mm_5deg_joint_rate=True),
        ground_scene_reconstructed=dict(cases=derived,included_n=len(selected),
            included_planar_error_mm=float(np.mean([c['reconstructed_planar_error_mm'] for c in selected])),
            included_angle_error_deg=float(np.mean([c['angle_error_deg'] for c in selected])),
            included_passes_50mm_5deg=sum(c['passes_50mm_5deg'] for c in selected),
            all_n=len(derived),all_passes_50mm_5deg=sum(c['passes_50mm_5deg'] for c in derived),
            threshold_recalculation_only_not_equivalent_reference_point_or_protocol=True),
        table4=dict(AGWF_point_registration_RMSE=.011715,FPFH_point_registration_RMSE=.017559,
            AGWF_feature_seconds=.952,FPFH_feature_seconds=1.430,
            printed_error_reduction_vs_FPFH_percent=36.23,
            recomputed_error_reduction_vs_FPFH_percent=(1-.011715/.017559)*100,
            printed_time_reduction_vs_FPFH_percent=48.15,
            recomputed_time_reduction_vs_FPFH_percent=(1-.952/1.430)*100))


def input_diagnostic(key):
    meta=json.loads((ROOT/'paper_features'/(key+'.json')).read_text())
    with np.load(ROOT/'dataset/roi'/(key+'.npz'),allow_pickle=False) as z:
        raw_count=len(z['xyz'])
    if meta['state']!='complete':
        return dict(target_key=key,raw_ROI_points=raw_count,state='failed',reason=meta['reason'])
    with np.load(ROOT/'paper_features'/(key+'.npz'),allow_pickle=False) as z:
        p=z['points'];r=z['agwf_radii'];features=z['agwf'];keys=z['agwf_keys']
        tree=cKDTree(p)
        nearest=tree.query(p,k=2)[0][:,1]
        neighbors=tree.query_ball_point(p,r,return_length=True)
        empty=np.linalg.norm(features,axis=1)<1e-12
        return dict(target_key=key,state='complete',raw_ROI_points=raw_count,
            registration_points=len(p),ISS_keypoints=len(keys),
            mean_nearest_neighbor_mm=float(nearest[nearest>1e-9].mean()*1000),
            median_adaptive_radius_mm=float(np.median(r)*1000),
            neighbors_below_4_including_self_fraction=float((neighbors<4).mean()),
            zero_feature_fraction=float(empty.mean()),
            zero_ISS_feature_fraction=float(empty[keys].mean()) if len(keys) else None)


def region_diagnostic(job):
    key,dimensions=job
    with np.load(ROOT/'dataset/roi'/(key+'.npz'),allow_pickle=False) as z:
        p=z['xyz'].astype(float);uv=z['uv']
    regions=plane_regions(p,uv,.008)
    sizes=[];compatible=0
    for origin,normal,mask in regions[:6]:
        direction=normal.copy();direction[2]=0;direction/=np.linalg.norm(direction)
        lateral=np.cross(direction,[0.,0.,1.])
        local=(p[mask]-origin)@np.column_stack((lateral,[0.,0.,1.]))
        lo,hi=np.quantile(local,[.01,.99],axis=0);width,height=hi-lo
        sizes.append([float(width),float(height),int(mask.sum())])
        if width<.04 or height<.015:continue
        if any(.5<width/tw<1.8 and .3<height/th<3. for tw,th in dimensions):compatible+=1
    return dict(target_key=key,front_regions=len(regions),tested_top6_sizes=sizes,
        size_compatible_regions=compatible,
        posthoc_exact_frozen_preprocessing_only_not_model_retuning=True)


def generate():
    if OUTPUT.exists() and any(OUTPUT.iterdir()):
        raise ValueError('Choose a new analysis version; do not overwrite evidence')
    OUTPUT.mkdir(parents=True)
    labels_list=read_rows(ROOT/'snapshot/eligible.jsonl');labels={r['target_key']:r for r in labels_list}
    consumed=[ROOT/'final_results.json',ROOT/'snapshot/eligible.jsonl',ROOT/'paper_selection.json',
        ROOT/'selection_frozen.json',ROOT/'protocol.json',ROOT/'paper_protocol.json',
        ROOT/'templates/full_train/manifest.json',ROOT/'pallet_pose_paper_comparison_v5.zip',
        ROOT/'pallet_pose_paper_comparison_v5r2.zip',ROOT/'paper_features/index.jsonl',ROOT/'dataset/features.jsonl',
        ROOT/'paper_sources/shao_2023_fulltext.xml',PROJECT/'pallet/pose_paper_methods.py',
        PROJECT/'pallet/pose_paper_benchmark.py']
    bank=json.loads((ROOT/'templates/full_train/manifest.json').read_text())
    consumed += [ROOT/'templates/full_train'/t['file'] for t in bank['templates']]
    for row in labels_list:
        if row['split']!='test':continue
        key=row['target_key'];consumed.append(ROOT/'dataset/roi'/(key+'.npz'))
        meta_path=ROOT/'paper_features'/(key+'.json');consumed.append(meta_path)
        if json.loads(meta_path.read_text())['state']=='complete':consumed.append(meta_path.with_suffix('.npz'))
    predictions={}
    for split in ('train','val','test'):
        if split=='train':continue
        predictions[split]={}
        for method in METHODS:
            path=ROOT/'predictions'/method/(split+'.json');consumed.append(path)
            predictions[split][method]=json.loads(path.read_text())
    before={str(p.relative_to(PROJECT)):digest(p) for p in consumed}
    current={split:{method:summarize(records,labels) for method,records in pred.items()}
        for split,pred in predictions.items()}
    frozen=json.loads((ROOT/'final_results.json').read_text())
    for split,methods in current.items():
        for method,s in methods.items():
            old=frozen['evaluation'][split][method]
            if not np.isclose(s['position']['mean'],old['position_mean_mm']) or s['joint_pass']!=old['success_count']:
                raise ValueError('Native metrics do not reproduce')
    with ThreadPoolExecutor(max_workers=4) as pool:
        inputs=list(pool.map(input_diagnostic,[r['target_key'] for r in labels_list if r['split']=='test']))
    good_inputs=[r for r in inputs if r['state']=='complete']
    input_stats={name:stats([r[name] for r in good_inputs if r[name] is not None]) for name in (
        'raw_ROI_points','registration_points','ISS_keypoints','mean_nearest_neighbor_mm',
        'median_adaptive_radius_mm','neighbors_below_4_including_self_fraction',
        'zero_feature_fraction','zero_ISS_feature_fraction')}
    input_stats['failed_preprocessing']=len(inputs)-len(good_inputs)
    bank=json.loads((ROOT/'templates/full_train/manifest.json').read_text())
    templates=[];dimensions=[]
    for row in bank['templates']:
        path=ROOT/'templates/full_train'/row['file'];consumed.append(path)
        with np.load(path,allow_pickle=False) as z:
            points=z['points'];dims=z['dimensions'];dimensions.append(dims.tolist())
            origin,normal,mask=plane_fit(points,.015)
            templates.append(dict(index=row['index'],source_train_target=row['source_train_target'],
                source_train_group=row['source_train_group'],domain=labels[row['source_train_target']]['domain'],
                dimensions_m=dims.tolist(),points=len(points),ISS=len(z['agwf_keys']),
                plane_origin_from_assigned_label_origin=origin.tolist(),
                signed_plane_offset_from_assigned_label_origin_m=float(normal@origin),
                canonical_X_quantiles_m=np.quantile(points[:,0],[.01,.5,.99]).tolist(),
                cannot_disambiguate_automatic_wrong_face_from_draft_label_error=True))
    records=predictions['test']['xiao_plane_template']
    # Investigate the dominant failure, without changing thresholds or producing new predictions.
    jobs=[(r['target_key'],dimensions) for r in records if r.get('reason')=='no_compatible_structure_template']
    with ThreadPoolExecutor(max_workers=4) as pool:
        regions=list(pool.map(region_diagnostic,jobs))
    template_outcomes={}
    for method in ('shao_agwf','fpfh_ablation'):
        template_outcomes[method]={}
        for template in templates:
            rows=[r for r in predictions['test'][method] if r['state']=='complete' and r['evidence']['template']==template['index']]
            values=[errors(r,labels[r['target_key']]) for r in rows]
            template_outcomes[method][str(template['index'])]=dict(outputs=len(rows),
                joint_pass=sum(v['joint_pass'] for v in values),position=stats([v['position_mm'] for v in values]),
                angle=stats([v['angle_deg'] for v in values]))
    agwf={r['target_key']:r for r in predictions['test']['shao_agwf'] if r['state']=='complete'}
    fpfh={r['target_key']:r for r in predictions['test']['fpfh_ablation'] if r['state']=='complete'}
    keys=sorted(set(agwf)&set(fpfh))
    a=[errors(agwf[k],labels[k]) for k in keys];f=[errors(fpfh[k],labels[k]) for k in keys]
    common=dict(n=len(keys),AGWF_position_mm=float(np.mean([v['position_mm'] for v in a])),
        FPFH_position_mm=float(np.mean([v['position_mm'] for v in f])),
        AGWF_angle_deg=float(np.mean([v['angle_deg'] for v in a])),
        FPFH_angle_deg=float(np.mean([v['angle_deg'] for v in f])),
        AGWF_joint_pass=sum(v['joint_pass'] for v in a),FPFH_joint_pass=sum(v['joint_pass'] for v in f))
    common['position_mean_relative_reduction_percent']=(1-common['AGWF_position_mm']/common['FPFH_position_mm'])*100
    # sqrt(2*huber) is only a Huber-derived score proxy, NOT the paper's RMSE.
    low_huber=[]
    for key,record in agwf.items():
        v=errors(record,labels[key])
        if record['evidence']['huber']<.0002 and v['position_mm']>50:low_huber.append(key)
    paper=paper_tables()
    report=dict(schema='posthoc_paper_gap_analysis_v1',current_native=current,
        paper_AGWF=paper,template_bank=templates,template_outcomes=template_outcomes,
        test_input_stats=input_stats,test_input_records=inputs,
        xiao_size_failure_diagnostics=regions,AGWF_FPFH_same_output_comparison=common,
        AGWF_low_Huber_but_position_error_above_50mm=dict(count=len(low_huber),
            threshold_huber_m2=.0002,target_keys=low_huber,
            inference='low_alignment_objective_does_not_guarantee_correct_fork_center',
            not_equivalent_to_original_RMSE=True),
        models_or_parameters_changed=False,new_training=False,GT_used_only_for_posthoc_error_analysis=True,
        not_independent_causal_ablation=True,robot_command_eligible=False)
    for p in consumed:
        before.setdefault(str(p.relative_to(PROJECT)),digest(p))
    after={name:digest(PROJECT/name) for name in before}
    if before!=after:raise ValueError('Frozen input changed during analysis')
    write_json(OUTPUT/'analysis.json',report)
    write_json(OUTPUT/'input_manifest.json',dict(inputs=before,all_inputs_unchanged=True,
        script_sha256=digest(Path(__file__)),online_database_opened=False,robot_command_eligible=False))
    print(json.dumps(dict(current_native=current,input_stats=input_stats,templates=templates,
        common=common,xiao_size_failures_rechecked=len(regions),
        AGWF_low_Huber_but_position_error_above_50mm=len(low_huber)),ensure_ascii=False,indent=2),flush=True)


if __name__=='__main__':
    generate()
