"""Merge explicitly returned human corrections with priority and retired-frame receipts."""
from __future__ import annotations
import argparse
from collections import Counter
from dataclasses import replace
from datetime import datetime, timezone
import json
from pathlib import Path, PurePosixPath
import shutil
import zipfile

from PIL import Image
from pallet.annotation import AnnotationSet, FrameAnnotation, Provenance, SceneAttributes, annotation_digest, load_jsonl, save_jsonl
from pallet.data.eligibility import require_eligible
from pallet.data.protocol import file_digest, load_protocol, write_json
from pallet.human_labeling import _load_frame_index, _decode_rectangles, _sha, create_manual_validation_partition, _validated_manual_partition

ROOT = Path(__file__).resolve().parents[1]
OUTPUT = ROOT/'outputs/human_correction_manual_6552_v1'
PROTOCOL = ROOT/'outputs/protocol_v6_clean'
BASE = ROOT/'outputs/pallet_clean_intensity_v1/human_training'
OLD_SOURCE = ROOT/'outputs/human_correction_manual_1716_v2/anylabeling'
RECON_SOURCE = ROOT/'outputs/pallet_clean_intensity_v1/reconstructed_118_review'


def read(path):
    return json.loads(Path(path).read_text())


def validate_document(document, baseline, row, source, record):
    """GUI flags are historical; identity, pixels and coordinates remain binding."""
    metadata=document.get('pallet_prediction') or document.get('pallet_export') or {}
    prior=baseline.get('pallet_prediction') or baseline.get('pallet_export') or {}
    if (metadata.get('frame_id')!=row['frame_id'] or prior.get('frame_id')!=row['frame_id']
        or metadata.get('source_anchor_hash')!=row['source_anchor_hash']
        or record.source_sha256!=row['source_anchor_hash'] or record.group_id!=row['group_id']):
        raise ValueError(f'Correction identity mismatch: {row["frame_id"]}')
    w,h=row['size']
    if (document.get('imageWidth'),document.get('imageHeight'))!=(w,h) or document.get('imagePath')!=baseline.get('imagePath'):
        raise ValueError(f'Correction dimensions/imagePath changed: {row["frame_id"]}')
    expected=row.get('baseline_annotation_sha256') or row.get('baseline_json_sha256')
    if expected and file_digest(source/row['annotation'])!=expected:
        raise ValueError('Source baseline JSON changed')
    expected_image=row.get('view_sha256',{}).get('image') or row.get('files',{}).get(row['image'])
    if expected_image and file_digest(source/row['image'])!=expected_image:
        raise ValueError('Source PNG changed')
    with Image.open(source/row['image']) as image:
        if image.size!=(w,h):raise ValueError('Native image dimensions mismatch')
    return _decode_rectangles(document,w,h)


def merge_reviewed(base, incoming, *, replace_existing):
    """Never silently resolve overlaps: the priority archive must explicitly replace."""
    overlap=set(base)&set(incoming)
    if overlap and not replace_existing:
        raise ValueError('Unexpected duplicate human frame')
    return {**base,**incoming}, sorted(overlap)


def ingest():
    active,assignments,_=load_protocol(PROTOCOL); by_id={r.frame_id:r for r in active}
    legacy,_,_=load_protocol(ROOT/'outputs/protocol_v5'); legacy_by_id={r.frame_id:r for r in legacy}
    excluded=set(legacy_by_id)-set(by_id)
    config={'kind':'user_confirmed_manual_update_v1','user_statement':'completed_remaining_train_and_checked_reconstructed_archive; reconstructed_archive_has_priority',
        'base_annotations':str(BASE/'inputs/annotations.jsonl'), 'sources':[
          {'archive':str(ROOT/'train_4984.zip'),'source_export':str(OLD_SOURCE),'labels':4984,'priority':1},
          {'archive':str(ROOT/'reconstructed_intensity_118_review_v1_checked.zip'),'source_export':str(RECON_SOURCE),'labels':118,'priority':2}],
        'all_frames_reviewed_by_user':True,'unchanged_proposals_accepted_after_manual_review':True,
        'empty_frames_confirmed_negative_by_user':True,'retired_frames_remain_excluded':True,'protocol':str(PROTOCOL)}
    input_files=[PROTOCOL/'protocol.json',BASE/'inputs/annotations.jsonl',BASE/'inputs/human_training_manifest.json',BASE/'inputs/human_validation_manifest.json']
    for source in config['sources']:
        input_files += [Path(source['archive']),Path(source['source_export'])/'frames.jsonl',Path(source['source_export'])/'package_manifest.json']
    binding={str(p):file_digest(p) for p in input_files}
    completion=OUTPUT/'manual_import_manifest.json'
    if completion.exists():
        prior=read(completion)
        if prior['input_binding']!=binding:raise ValueError('Import resume inputs changed')
        for name,digest in prior['artifact_sha256'].items():
            if file_digest(OUTPUT/name)!=digest:raise ValueError('Import snapshot changed')
        return prior
    if OUTPUT.exists() and any(OUTPUT.iterdir()):raise ValueError('Unbound partial output; choose new version')
    labels=load_jsonl(BASE/'inputs/annotations.jsonl').frames.copy()
    source_entries={**read(BASE/'inputs/human_training_manifest.json')['source_frames'],**read(BASE/'inputs/human_validation_manifest.json')['source_frames']}
    summaries=[]; traces=[]; retired=[]; raw_docs=[]
    for source in config['sources']:
        archive=Path(source['archive']); src=Path(source['source_export']); archive_sha=file_digest(archive)
        _,index,by_filename,manifest_sha,index_sha=_load_frame_index(src)
        incoming={}; entries={}; seen=set()
        with zipfile.ZipFile(archive) as z:
            if z.testzip() is not None:raise ValueError('ZIP CRC failed')
            members=[]
            for item in z.infolist():
                path=PurePosixPath(item.filename)
                if path.is_absolute() or '..' in path.parts:raise ValueError('Unsafe archive member path')
                if item.is_dir() or path.suffix.lower()!='.json':continue
                d=json.loads(z.read(item))
                if 'shapes' not in d:continue
                members.append((item,d))
            if len(members)!=source['labels']:raise ValueError('Unexpected correction count')
            for item,doc in members:
                filename=PurePosixPath(item.filename).name
                row=by_filename.get(filename)
                if row is None:raise ValueError(f'Unknown correction filename: {filename}')
                fid=row['frame_id']
                if fid in seen:raise ValueError('Duplicate frame in single archive')
                seen.add(fid); record=legacy_by_id.get(fid)
                if record is None:raise ValueError('Unknown source identity')
                baseline=read(src/row['annotation'])
                boxes=validate_document(doc,baseline,row,src,record)
                # If PNGs were returned alongside checked JSON, verify their exact source pixels.
                paired=str(PurePosixPath(item.filename).with_suffix('.png'))
                if paired in z.namelist() and _sha(z.read(paired))!=file_digest(src/row['image']):
                    raise ValueError('Returned PNG modified or mismatched')
                raw_docs.append((source['priority'],filename,z.read(item)))
                if fid in excluded:
                    retired.append({'frame_id':fid,'archive':str(archive),'returned_annotation_sha256':_sha(z.read(item)),
                        'reason':'user-retired irrecoverable intensity; checked annotation retained as evidence only'})
                    continue
                record=by_id[fid];require_eligible(record)
                split=assignments[record.group_id]
                frame=FrameAnnotation(fid,record.group_id,row['image'],row['size'][0],row['size'][1],
                    status='pos' if boxes else 'negative',boxes=boxes,split=split,domain=record.domain,
                    scene=SceneAttributes(domain=record.domain),source_anchor_hash=record.source_sha256,
                    provenance=[Provenance('human',actor='user_confirmed_archive_import',
                        note=f'user-returned correction; archive={archive.name}; sha256={archive_sha}')])
                frame.validate();incoming[fid]=frame
                entries[fid]={'frame_id':fid,'group_id':record.group_id,'source_anchor_hash':record.source_sha256,
                    'split':split,'width':frame.width,'height':frame.height,'domain':frame.domain,'status':frame.status,
                    'instances':len(boxes),'annotation_digest':annotation_digest(frame),'archive':str(archive),
                    'archive_sha256':archive_sha,'source_json_sha256':_sha(z.read(item)),
                    'baseline_source_json_sha256':file_digest(src/row['annotation']),
                    'source_export_manifest_sha256':manifest_sha,
                    'geometry_unchanged_from_proposal':frame.boxes==_decode_rectangles(baseline,*row['size'])}
            labels,overlap=merge_reviewed(labels,incoming,replace_existing=source['priority']==2)
            for fid in overlap:
                traces.append({'frame_id':fid,'superseded':source_entries[fid], 'accepted':entries[fid],
                    'reason':'user explicitly prioritizes checked reconstructed-intensity archive'})
            source_entries.update(entries)
            summaries.append({'archive':str(archive),'archive_sha256':archive_sha,'frames':len(members),
                'accepted_active_frames':len(incoming),'retired_frames':len(members)-len(incoming),
                'overridden_frames':len(overlap),'status_counts':dict(Counter(f.status for f in incoming.values())),
                'boxes':sum(len(f.boxes) for f in incoming.values())})
    assert len(retired)==113 and len(labels)==6586 and len(traces)==84
    train={fid:f for fid,f in labels.items() if f.split=='train'}
    external={fid:f for fid,f in labels.items() if f.split in {'val','test'}}
    assert len(train)==6552 and len(external)==34
    assert set(train)=={r.frame_id for r in active if assignments[r.group_id]=='train'}
    OUTPUT.mkdir(parents=True); inputs=OUTPUT/'inputs';inputs.mkdir()
    save_jsonl(AnnotationSet(train.values()),inputs/'manual_annotations.jsonl')
    save_jsonl(AnnotationSet(labels.values()),inputs/'all_checked_annotations.jsonl')
    save_jsonl(AnnotationSet(external.values()),inputs/'checked_external_annotations.jsonl')
    write_json(inputs/'manual_source_frames.json',{fid:source_entries[fid] for fid in sorted(train)})
    write_json(inputs/'all_checked_source_frames.json',source_entries)
    write_json(inputs/'priority_overrides.json',traces);write_json(inputs/'returned_retired_frames.json',retired)
    write_json(inputs/'corrections_config.json',config);write_json(inputs/'input_binding.json',binding)
    for priority,filename,payload in raw_docs:
        p=inputs/'raw_returned_json'/str(priority)/filename;p.parent.mkdir(parents=True,exist_ok=True);p.write_bytes(payload)
    receipt={'frame_ids':sorted(external),'protocol_sha256':file_digest(PROTOCOL/'protocol.json'),
        'annotation_snapshot_sha256':file_digest(inputs/'checked_external_annotations.jsonl'),
        'role':'Already human-checked original val/test frames; exclude from prediction; never fit/select/calibrate or score final test',
        'source_archive_sha256':file_digest(ROOT/'reconstructed_intensity_118_review_v1_checked.zip')}
    write_json(inputs/'external_review_receipt.json',receipt)
    write_json(inputs/'reviewed_external_exclusion.json',{'frame_ids':sorted(external),
        'protocol_sha256':file_digest(PROTOCOL/'protocol.json'),'receipt_path':'inputs/external_review_receipt.json',
        'receipt_sha256':file_digest(inputs/'external_review_receipt.json')})
    archive_sources=read(BASE/'manual_partition.json')['source_archives']+summaries
    digest=_sha(json.dumps(binding,sort_keys=True).encode())
    manifest={'kind':'multi_archive_user_confirmed_manual_labels','state':'imported','input_binding':binding,
        'source_archive_set_sha256':digest,'protocol_sha256':file_digest(PROTOCOL/'protocol.json'),
        'frame_count':6552,'all_checked_frames':6586,'status_counts':dict(Counter(f.status for f in train.values())),
        'box_count':sum(len(f.boxes) for f in train.values()),'group_count':len({f.group_id for f in train.values()}),
        'sources':archive_sources,'retired_returned_frames':113,'priority_overrides':84,
        'external_checked_frames':34,'all_frames_reviewed_by_user':True,'reviewer_count_claimed':1,
        'artifact_sha256':{str(p.relative_to(OUTPUT)):file_digest(p) for p in OUTPUT.rglob('*') if p.is_file()}}
    write_json(completion,manifest)
    write_json(ROOT/'configs/human_correction_6552.json',{**config,'input_hashes':binding,'output':str(OUTPUT),
        'runtime':{'batch_size':16,'effective_batch':16,'workers':4,'validation_batch_size':16,'prediction_batch_size':16}})
    return manifest


def main():
    parser=argparse.ArgumentParser();parser.add_argument('--prepare-only',action='store_true');args=parser.parse_args()
    result=ingest()
    partition=create_manual_validation_partition(source_output=OUTPUT,output=OUTPUT,protocol=PROTOCOL,seed=42,validation_fraction=.15)
    _validated_manual_partition(OUTPUT,PROTOCOL)
    write_json(OUTPUT/'merge_summary.json',{'train_eligible_frames':6552,'all_checked_frames':6586,'external_checked':34,
        'retired_returned_frames':113,'priority_overrides':84,'train':len(partition['train_frame_ids']),
        'val':len(partition['validation_frame_ids']),'status_counts':partition['status_counts'],
        'positive_boxes':partition['positive_boxes'],'remaining':{'val':1439,'test':2349,'total':3788}})
    print(json.dumps(read(OUTPUT/'merge_summary.json'),ensure_ascii=False,indent=2))

if __name__=='__main__':main()
