"""Common original holdout cohort comparison; no prospective/blind-test claim."""
from collections import Counter
from pathlib import Path
import sys

import numpy as np

PROJECT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(PROJECT))
from pallet.mask_model_training_data import read,write,jsonl,require
from pallet.cluster_targets import digest
from pallet.mask_taxonomy_cycle2_data import ROOT
from pallet.mask_taxonomy_cycle2_train import RUN
from pallet.mask_taxonomy_train import RUN as PRIOR_RUN


def metrics(rows):
    return dict(targets=len(rows),fine_top1=sum(r['fine_index']==r['predicted_fine_index'] for r in rows)/len(rows),
        parent_top1=sum(r['parent_index']==r['predicted_parent_index'] for r in rows)/len(rows),
        mask_iou_median=float(np.median([r['iou'] for r in rows])))


def analyze():
    out=ROOT/'classification_comparison.json';require(not out.exists(),'Preserve prior comparison')
    old={r['target_key']:r for r in jsonl(PRIOR_RUN/'independent_branch/holdout_predictions.jsonl')};new={r['target_key']:r for r in jsonl(RUN/'independent_branch/holdout_predictions.jsonl')}
    require(set(old)<=set(new),'Original fixed holdout cohort moved to training or disappeared')
    for k,r in old.items():require(r['fine_index']==new[k]['fine_index'] and r['parent_index']==new[k]['parent_index'],'Original cohort labels changed')
    keys=sorted(old);before=metrics([old[k] for k in keys]);after=metrics([new[k] for k in keys])
    corrected=sum(old[k]['predicted_fine_index']!=old[k]['fine_index'] and new[k]['predicted_fine_index']==new[k]['fine_index'] for k in keys)
    regressed=sum(old[k]['predicted_fine_index']==old[k]['fine_index'] and new[k]['predicted_fine_index']!=new[k]['fine_index'] for k in keys)
    write(out,dict(scope='same_original_component_holdout_labels_generic_start_models_NOT_new_blind_test',
        old=before,new=after,fine_errors_corrected=corrected,new_fine_errors=regressed,
        new_whole_holdout=read(RUN/'independent_branch/holdout_metrics.json'),
        original_cohort_selected_before_second_training=True,prior_full_pool_model_selected_new_review_data=True,
        no_new_blind_prospective_generalization_or_calibrated_probability_claim=True,
        previous_prediction_sha256=digest(PRIOR_RUN/'independent_branch/holdout_predictions.jsonl'),
        new_prediction_sha256=digest(RUN/'independent_branch/holdout_predictions.jsonl'),code_sha256=digest(__file__)))
    print(dict(common_cohort=len(keys),old_fine_top1=before['fine_top1'],new_fine_top1=after['fine_top1'],corrected=corrected,regressed=regressed),flush=True)


if __name__=='__main__':analyze()
