"""Independently verify CV cohorts, result hashes, ranking and delivery evidence."""
import json
from pathlib import Path
import sys

PROJECT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(PROJECT))
import numpy as np
from pallet.pose_benchmark import ROOT,SOURCE,load_data,sha,atomic_json
from pallet.pose_benchmark_results import result_path,per_sample_score
from pallet.pose_refinement import score_values


def main():
    protocol=json.loads((ROOT/'protocol.json').read_text())
    assert sha(PROJECT/'configs/pallet_pose_benchmark_v3.json')==protocol['config_sha256']
    assert sha(SOURCE/'snapshot/eligible.jsonl')==protocol['label_sha256']
    assert sha(SOURCE/'training/eligible_after_features.jsonl')==protocol['eligible_sha256']
    selection=json.loads((ROOT/'selection_frozen.json').read_text())
    assert selection['test_used'] is False and selection['protocol_sha256']==sha(ROOT/'protocol.json')
    accelerated=selection.get('recipe')=='accelerated'
    if accelerated:
        assert selection['refinement_protocol_sha256']==sha(ROOT/'refinement_protocol.json')
        phase=json.loads((ROOT/'refinement_protocol.json').read_text())
        assert sha(PROJECT/'configs/pallet_pose_benchmark_v3_acceleration.json')==phase['config_sha256']
        assert phase['config']['rtx3090_new_jobs'] is False
    shortlist=json.loads((ROOT/'shortlist.json').read_text())
    checked=[];folds_checked=0
    for branch,info in shortlist.items():
        rows,arrays=load_data(branch)
        lookup={r['target_key']:r for r in rows}
        expected_shortlist=[r['family'] for r in sorted(info['screening'],key=lambda x:x['metrics']['macro_group_selection_score'])[:3]]
        assert info['shortlist']==expected_shortlist
        for root in sorted((ROOT/'trials').glob('*/*/*/'+branch+'/*')):
            if not (root/'completed.json').exists():continue
            record=json.loads((root/'completed.json').read_text())
            oof=np.zeros_like(arrays['y']);seen=[]
            for fold,expected in enumerate(protocol['folds'][branch]):
                path=root/f'fold_{fold}'
                spec=json.loads((path/'specification.json').read_text())
                assert spec['fit_keys']==expected['fit'] and spec['validation_keys']==expected['validation']
                if spec.get('recipe')=='accelerated':
                    assert spec['batch']==32 and spec['evaluation_batch']==64
                    assert spec['refinement_protocol_sha256']==sha(ROOT/'refinement_protocol.json')
                    assert '3ce8e1e7-efb2-80a6-de52-75da2f9fa259' not in spec['gpu']
                assert all(lookup[k]['split']=='train' for k in spec['fit_keys']+spec['validation_keys'])
                assert not ({lookup[k]['group_id'] for k in spec['fit_keys']} &
                            {lookup[k]['group_id'] for k in spec['validation_keys']})
                completed=json.loads((path/'completed.json').read_text())
                assert sha(path/'selected.pt')==completed['checkpoint_sha256']
                history=json.loads((path/'history.json').read_text())
                assert len(history)==80
                epoch=min(history,key=lambda h:h['validation']['macro_group_selection_score'])['epoch']
                assert epoch==completed['selected_epoch']
                indices=[i for i,r in enumerate(rows) if r['target_key'] in set(expected['validation'])]
                p=np.load(path/'predictions.npy');assert np.isfinite(p).all()
                oof[indices]=p;seen+=indices;folds_checked+=1
            assert sorted(seen)==list(range(len(rows)))
            with np.load(root/'out_of_fold.npz') as z:
                np.testing.assert_array_equal(z['keys'],[r['target_key'] for r in rows])
                np.testing.assert_allclose(z['targets'],arrays['y'],atol=1e-7)
                np.testing.assert_allclose(z['predictions'],oof,atol=1e-7)
            actual=score_values(oof,arrays['y'],rows)
            for field in ('macro_group_selection_score','angle_mae_degrees','normalized_position_mean_width_fraction'):
                np.testing.assert_allclose(actual[field],record['metrics'][field],atol=1e-7)
            checked.append(dict(family=record['family'],branch=branch,seed=record['seed'],route=record['route'],passed=True))
        candidates=selection['profiles'][branch]['candidates']
        assert selection['profiles'][branch]['best_learning']==min(candidates,key=lambda c:c['mean_score'])
        for c in candidates:
            assert c.get('recipe','screen')==selection.get('recipe','screen')
            scores=[]
            for seed in c['seeds']:
                record=json.loads((result_path(c['family'],branch,seed,c['route'],c['tune'])/'completed.json').read_text())
                scores.append(record['metrics']['macro_group_selection_score'])
            np.testing.assert_allclose(c['mean_score'],np.mean(scores),atol=1e-9)
        refit=json.loads((ROOT/'refit'/branch/'specification.json').read_text())
        assert refit['validation_keys']==[]
        assert set(refit['fit_keys'])==set(lookup)
        assert refit['epochs']==selection['profiles'][branch]['best_learning']['refit_epochs']
    for file in ('inference_verification.json','portable_inference_verification.json','integrity_audit.json'):
        assert json.loads((ROOT/file).read_text())['state']=='passed'
    with np.load(next((ROOT/'trials').glob('residual/screen/*/metric/42/out_of_fold.npz'))) as z:
        assert np.isfinite(z['predictions']).all()
    delivery=json.loads((ROOT/'delivery_manifest.json').read_text())
    assert sha(delivery['archive'])==delivery['sha256']
    report=dict(state='passed',trial_runs_checked=len(checked),folds_checked=folds_checked,
                trials=checked,selection_train_only=True,all_original_holdout_splits_preserved=True,
                delivery_sha256=delivery['sha256'],source_database_writes=0)
    atomic_json(ROOT/'final_acceptance.json',report)
    print(json.dumps(report),flush=True)


if __name__=='__main__':main()
