from collections import Counter
from dataclasses import asdict
import json
from pathlib import Path
from types import SimpleNamespace
import zipfile

import pytest
from PIL import Image

from pallet.annotation import (AnnotationSet, Box, FrameAnnotation, Provenance, SceneAttributes,
                               annotation_digest, save_jsonl)
from pallet.data.loader import NormalizationStats
from pallet.data.manifest import FrameRecord
from pallet.data.protocol import file_digest
from pallet.experiments.dataset import PalletDataset
import pallet.human_labeling as human
import pallet.mixed_manual_assistant as mixed


def frame(fid, split, domain='d'):
    return FrameAnnotation(fid, f'g-{fid}', f'{fid}.png', 32, 24, status='pos',
        boxes=[Box(2, 3, 10, 8)], split=split, domain=domain,
        scene=SceneAttributes(domain=domain), source_anchor_hash=f'h-{fid}',
        provenance=[Provenance('human', actor='user_confirmed_archive_import')])


def manifests(tmp_path, monkeypatch):
    protocol = tmp_path/'protocol'
    protocol.mkdir()
    (protocol/'protocol.json').write_text('{}')
    frames = [frame('tr', 'train'), frame('te', 'test')]
    records = [FrameRecord(f.frame_id, f.group_id, f.domain, {}, {'grid': [24, 32]}, f.source_anchor_hash)
               for f in frames]
    monkeypatch.setattr(human, 'load_protocol', lambda _: (records, {'g-tr': 'train', 'g-te': 'test'}, {}))
    labels = AnnotationSet(frames)
    path = tmp_path/'annotations.jsonl'
    save_jsonl(labels, path)
    common = {'protocol_sha256': file_digest(protocol/'protocol.json'),
        'annotation_snapshot_sha256': file_digest(path), 'scope': human.AUXILIARY_SCOPE,
        'approver': 'user_confirmed_in_conversation', 'original_splits_preserved': True,
        'eligible_for_research_evaluation': False}
    def entry(f):
        return {'group_id': f.group_id, 'split': f.split, 'source_anchor_hash': f.source_anchor_hash,
                'annotation_digest': annotation_digest(f)}
    training = {**common, 'kind': human.AUXILIARY_TRAIN_KIND, 'source_frames': {'te': entry(frames[1])},
                'training_frame_ids': ['te'], 'training_frames': 1}
    validation = {**common, 'kind': human.AUXILIARY_VAL_KIND, 'source_frames': {'tr': entry(frames[0])},
                  'validation_frame_ids': ['tr'], 'validation_frames': 1}
    return protocol, labels, records, training, validation


def test_original_test_identity_accepted_only_in_isolated_assistant(tmp_path, monkeypatch):
    protocol, labels, records, train, val = manifests(tmp_path, monkeypatch)
    assert human.validate_human_training_manifest(tmp_path/'human.json', train, labels, protocol) == {'te'}
    assert human.validate_human_validation_manifest(tmp_path/'human.json', val, labels, protocol, {'te'}) == {'tr'}
    assert labels.frames['te'].split == 'test'
    for field in ('scope', 'original_splits_preserved', 'eligible_for_research_evaluation', 'approver'):
        changed = {k: v for k, v in train.items() if k != field}
        with pytest.raises(ValueError, match='unrecognized'):
            human.validate_human_training_manifest(tmp_path/'human.json', changed, labels, protocol)
    changed = {**train, 'kind': 'explicit_user_confirmed_manual_training_labels'}
    with pytest.raises(ValueError, match='identity/label mismatch'):
        human.validate_human_training_manifest(tmp_path/'human.json', changed, labels, protocol)
    train['source_frames']['te']['split'] = 'train'
    with pytest.raises(ValueError, match='identity/label mismatch'):
        human.validate_human_training_manifest(tmp_path/'human.json', train, labels, protocol)


def test_cross_split_dataset_has_explicit_scope_and_disjoint_roles(tmp_path, monkeypatch):
    _, labels, records, _, _ = manifests(tmp_path, monkeypatch)
    record = records[1]
    kwargs = dict(manual_training_ids={'te'}, manual_validation_ids={'tr'}, manual_role='train')
    with pytest.raises(ValueError, match='non-train'):
        PalletDataset([record], labels, NormalizationStats(1, 1), **kwargs)
    data = PalletDataset([record], labels, NormalizationStats(1, 1),
                         manual_annotation_scope=human.AUXILIARY_SCOPE, **kwargs)
    assert data.rows[0][1].split == 'test'
    with pytest.raises(ValueError, match='wrong role'):
        PalletDataset([record], labels, NormalizationStats(1, 1), manual_training_ids={'te'},
                      manual_role='val', manual_annotation_scope=human.AUXILIARY_SCOPE)


def test_single_group_domain_is_never_split_to_create_validation():
    frames = [frame('one', 'test')]
    records = {'one': SimpleNamespace(group_id='g-one', source_sha256='h-one', domain='d')}
    with pytest.raises(ValueError, match='cannot create'):
        human._choose_group_holdout(frames, records, seed=42, fraction=.15)
    ids, groups, summary = human._choose_group_holdout(
        frames, records, seed=42, fraction=.15, allow_train_only_domains=True)
    assert ids == [] and groups == set()
    assert summary['d']['training_frames'] == 1 and summary['d']['validation_frames'] == 0
    assert 'validation_unavailable_reason' in summary['d']


def test_mixed_queue_spreads_source_population_without_changing_groups():
    items = [{'frame_id': f'f{i}', 'group_id': f'g{i % 9}', 'split': 'val' if i < 90 else 'test',
              'domain': 'a' if i % 5 else 'b'} for i in range(200)]
    ordered = mixed.mixed_order(items)
    assert ordered == mixed.mixed_order(items)
    assert sorted(r['frame_id'] for r in ordered) == sorted(r['frame_id'] for r in items)
    assert Counter((r['split'], r['domain']) for r in ordered) == Counter((r['split'], r['domain']) for r in items)
    assert {r['frame_id']: r['group_id'] for r in ordered} == {r['frame_id']: r['group_id'] for r in items}
    for stop in (20, 50, 100, 150):
        assert abs(sum(r['split'] == 'val' for r in ordered[:stop]) - stop*90/200) <= 2
        assert abs(sum(r['domain'] == 'b' for r in ordered[:stop]) - stop*.2) <= 2


def test_mixed_zip_verification_checks_payload_and_duplicate_members(tmp_path):
    workspace = tmp_path/'workspace'
    workspace.mkdir()
    (workspace/'one.json').write_text('checked')
    archive = tmp_path/'bad.zip'
    with zipfile.ZipFile(archive, 'w') as z:
        z.writestr('one.json', 'different')
    with pytest.raises(ValueError, match='payload differs'):
        mixed.validate_zip_payload(archive, workspace)
    with zipfile.ZipFile(archive, 'w') as z:
        z.writestr('one.json', 'checked')
    assert mixed.validate_zip_payload(archive, workspace)['member_payload_sha256'] == 'passed'


def test_import_requires_user_confirmation_before_cross_split_training(tmp_path):
    config = tmp_path/'config.json'
    config.write_text(json.dumps({'scope': human.AUXILIARY_SCOPE, 'all_frames_reviewed_by_user': True}))
    with pytest.raises(ValueError, match='Awaiting explicit user confirmation'):
        mixed.ingest(config)


def test_only_machine_roundoff_at_boundary_is_normalized_and_recorded():
    doc = {'imageWidth': 352, 'imageHeight': 264, 'shapes': [
        {'points': [[43.8, -7.105e-15], [284.1, 57.5]]},
        {'points': [[-0.01, 20], [30, 40]]}]}
    fixed, receipt = mixed.normalize_coordinate_roundoff(doc)
    assert fixed['shapes'][0]['points'][0][1] == 0
    assert len(receipt) == 1 and receipt[0]['original'] == -7.105e-15
    assert fixed['shapes'][1]['points'][0][0] == -.01
    assert doc['shapes'][0]['points'][0][1] == -7.105e-15


def test_mixed_package_keeps_pngs_boxes_and_original_identity(tmp_path, monkeypatch):
    output = tmp_path/'run'
    source = output/'anylabeling'
    source.mkdir(parents=True)
    items = []
    for i, split in enumerate(['test', 'val', 'test', 'val']):
        fid = f'f{i}'
        name = f'{i:024d}'
        row = {'frame_id': fid, 'group_id': f'g{i}', 'split': split, 'domain': f'd{i % 2}',
               'source_anchor_hash': f'h{i}', 'image': f'images/{split}/{name}.png',
               'annotation': f'images/{split}/{name}.json', 'depth': f'auxiliary/{split}/{name}.depth.png',
               'validity': f'auxiliary/{split}/{name}.valid.png'}
        doc = {'imageWidth': 32, 'imageHeight': 24, 'imagePath': name+'.png', 'imageData': None,
               'shapes': [{'label': 'pallet', 'shape_type': 'rectangle', 'points': [[2, 3], [12, 11]]}],
               'flags': {'human_reviewed': False, 'unreviewed': True, 'proposal_only': True},
               'pallet_prediction': {'frame_id': fid, 'group_id': f'g{i}', 'source_anchor_hash': f'h{i}'}}
        for role in ['image', 'depth', 'validity']:
            path = source/row[role]
            path.parent.mkdir(parents=True, exist_ok=True)
            Image.new('L', (32, 24), 50+i).save(path)
        (source/row['annotation']).write_text(json.dumps(doc))
        row['baseline_annotation_sha256'] = file_digest(source/row['annotation'])
        row['view_sha256'] = {role: file_digest(source/row[role]) for role in ['image', 'depth', 'validity']}
        items.append(row)
    (source/'frames.jsonl').write_text(''.join(json.dumps(row)+'\n' for row in items))
    (source/'labels.txt').write_text('pallet\n')
    (source/'anylabeling_config.yaml').write_text('labels: [pallet]\n')
    (source/'package_manifest.json').write_text(json.dumps({'frames': 4, 'frames_index_sha256': file_digest(source/'frames.jsonl')}))
    (output/'inputs').mkdir()
    save_jsonl(AnnotationSet([]), output/'inputs/all_checked_annotations.jsonl')
    result = mixed.make_mixed_package(output, tmp_path/'protocol')
    assert result['frames'] == 4 and result['split_counts'] == {'test': 2, 'val': 2}
    rows = mixed.rows(output/'mixed_anylabeling/frames.jsonl')
    assert all(Path(r['image']).parent == Path('images/mixed') for r in rows)
    assert all(r['original_split'] == r['split'] for r in rows)
    assert mixed.make_mixed_package(output, tmp_path/'protocol') == result
    altered = output/'mixed_anylabeling'/rows[0]['annotation']
    doc = json.loads(altered.read_text())
    doc['shapes'] = []
    altered.write_text(json.dumps(doc))
    with pytest.raises(ValueError, match='Preserving modified'):
        mixed.make_mixed_package(output, tmp_path/'protocol')


@pytest.mark.parametrize('new_splits', [('test', 'test'), ('val', 'test')])
def test_import_partition_resume_preserves_source_splits_and_excludes_external(tmp_path, monkeypatch, new_splits):
    protocol = tmp_path/'protocol'
    protocol.mkdir()
    (protocol/'protocol.json').write_text('{}')
    base, source, output = (tmp_path/name for name in ['base', 'source', 'output'])
    (base/'inputs').mkdir(parents=True)
    source.mkdir()
    originals = [frame(f't{i}', 'train') for i in range(4)]
    external = frame('external', 'val')
    new = [frame('n0', new_splits[0], 'new'), frame('n1', new_splits[1], 'new')]
    all_frames = originals+[external]+new
    records = [FrameRecord(f.frame_id, f.group_id, f.domain, {}, {'grid': [24, 32]}, f.source_anchor_hash)
               for f in all_frames]
    assignments = {f.group_id: f.split for f in all_frames}
    load = lambda _: (records, assignments, {})
    monkeypatch.setattr(human, 'load_protocol', load)
    monkeypatch.setattr(mixed, 'load_protocol', load)
    save_jsonl(AnnotationSet(originals), base/'inputs/annotations.jsonl')
    save_jsonl(AnnotationSet(originals+[external]), base/'inputs/all_checked_annotations.jsonl')
    save_jsonl(AnnotationSet([external]), base/'inputs/checked_external_annotations.jsonl')
    entry = lambda f: human._manual_source_entry(f, next(r for r in records if r.frame_id==f.frame_id))
    (base/'inputs/human_training_manifest.json').write_text(json.dumps({'source_frames':{f.frame_id:entry(f) for f in originals}}))
    (base/'inputs/human_validation_manifest.json').write_text(json.dumps({'source_frames':{}}))
    (base/'manual_partition.json').write_text(json.dumps({'source_archives':[]}))
    receipt = base/'inputs/external_review_receipt.json'
    receipt.write_text(json.dumps({'frame_ids':['external']}))
    (base/'inputs/reviewed_external_exclusion.json').write_text(json.dumps({'frame_ids':['external'],
        'protocol_sha256':file_digest(protocol/'protocol.json'), 'receipt_path':'inputs/external_review_receipt.json',
        'receipt_sha256':file_digest(receipt)}))
    archive = tmp_path/'test.zip'
    index = []
    with zipfile.ZipFile(archive, 'w') as z:
        for f in new:
            row = {'frame_id':f.frame_id,'group_id':f.group_id,'split':f.split,'domain':f.domain,
                   'source_anchor_hash':f.source_anchor_hash,'size':[32,24],
                   'image':f'{f.frame_id}.png','annotation':f'{f.frame_id}.json'}
            Image.new('L',(32,24),100).save(source/row['image'])
            doc = {'imageWidth':32,'imageHeight':24,'imagePath':row['image'],'flags':{'unreviewed':True},
                   'shapes':[{'label':'pallet','shape_type':'rectangle','points':[[2,3],[12,11]]}],
                   'pallet_prediction':{'frame_id':f.frame_id,'source_anchor_hash':f.source_anchor_hash,
                                        'group_id':f.group_id,'original_split':f.split}}
            (source/row['annotation']).write_text(json.dumps(doc))
            row['baseline_annotation_sha256'] = file_digest(source/row['annotation'])
            row['view_sha256'] = {'image':file_digest(source/row['image'])}
            index.append(row)
            # An unchanged box and an empty negative still carry old GUI flags.
            if f.frame_id == 'n1':doc['shapes']=[]
            z.writestr(row['annotation'],json.dumps(doc))
    (source/'frames.jsonl').write_text(''.join(json.dumps(r)+'\n' for r in index))
    (source/'package_manifest.json').write_text(json.dumps({'frames':2,'frames_index_sha256':file_digest(source/'frames.jsonl'),
        'prediction_binding':{'protocol_sha256':file_digest(protocol/'protocol.json')}}))
    config = tmp_path/'config.json'
    confirmation = tmp_path/'confirmation.json'
    confirmation.write_text(json.dumps({'user_statement':'returned corrected subset',
        'convention':'unchanged accepted and empty negative'}))
    config.write_text(json.dumps({'all_frames_reviewed_by_user':True,'unchanged_proposals_accepted_after_manual_review':True,
        'empty_frames_confirmed_negative_by_user':True,'scope':human.AUXILIARY_SCOPE,'use_original_test_for_assistance':True,
        'protocol':str(protocol),'output':str(output),'base_output':str(base),'source_export':str(source),
        'archive':str(archive),'expected_new_frames':2,'allowed_source_splits':['val','test'],
        'confirmation_record':str(confirmation),'confirmation_record_sha256':file_digest(confirmation)}))
    imported = mixed.ingest(config)
    assert imported['frame_count']==6 and imported['all_checked_frames']==7
    assert imported['new_source']['status_counts']=={'pos':1,'negative':1}
    partition = human.create_manual_validation_partition(source_output=output, output=output, protocol=protocol)
    labels, train, val, _, _ = human._validated_manual_partition(output, protocol)
    assert labels.frames['n0'].split==new_splits[0] and labels.frames['n1'].split==new_splits[1]
    assert 'external' not in train|val
    assert set(partition['train_groups']).isdisjoint(partition['validation_groups'])
    assert mixed.ingest(config)==imported
    assert human.create_manual_validation_partition(source_output=output, output=output, protocol=protocol)==partition
    assert str(confirmation.resolve()) in imported['input_binding']
    for field in ['group_id','original_split']:
        bad_archive = tmp_path/f'bad_{field}.zip'
        with zipfile.ZipFile(archive) as original, zipfile.ZipFile(bad_archive,'w') as changed:
            for ordinal, name in enumerate(original.namelist()):
                document = json.loads(original.read(name))
                if ordinal==0:document['pallet_prediction'][field]='changed'
                changed.writestr(name,json.dumps(document))
        bad_config = tmp_path/f'bad_{field}.json'
        bad_config.write_text(json.dumps({**json.loads(config.read_text()),'archive':str(bad_archive),
                                         'output':str(tmp_path/f'bad_output_{field}')}))
        with pytest.raises(ValueError,match=f'{field} identity mismatch'):
            mixed.ingest(bad_config)
    config.write_text(config.read_text()+' ')
    with pytest.raises(ValueError, match='inputs changed'):
        mixed.ingest(config)
    confirmation.write_text('{"convention":"changed"}')
    with pytest.raises(ValueError, match='review convention record changed'):
        mixed.ingest(config)


def test_delivery_resumes_after_manifest_publication_and_keeps_prior_report(tmp_path, monkeypatch):
    monkeypatch.setattr(mixed,'ROOT',tmp_path)
    output=tmp_path/'outputs'/'new_round'
    (output/'training').mkdir(parents=True)
    (tmp_path/'configs').mkdir()
    (tmp_path/'docs').mkdir()
    for name in ['training_log.md','debug_log.md','change_log.md','session_summary.md']:
        (tmp_path/'docs'/name).write_text('# History\n')
    prior_report=tmp_path/'docs'/'human_correction_7086_report.md'
    prior_report.write_text('Prior completed round')
    (tmp_path/'README.md').write_text('# Project\n')
    checkpoint=output/'training/best.pt'
    checkpoint.write_bytes(b'unit-test-checkpoint')
    digest=file_digest(checkpoint)
    def write(name,value):
        path=output/name
        path.write_text(json.dumps(value))
    write('mixed_package_audit.json',{'frames':1,'archive':str(output/'mixed.zip'),'archive_sha256':'archive'})
    write('training/completed.json',{'completed_epochs':1,'best_checkpoint_sha256':digest})
    write('confidence_threshold.json',{'checkpoint_sha256':digest,'selected_confidence':.7,
        'selected_validation_metrics':{'recall':.96}})
    write('manual_partition.json',{'expected_manual_frames':2,'train_frame_ids':['a'],
        'validation_frame_ids':['b'],'train_groups':['ga'],'validation_groups':['gb']})
    write('prediction_manifest.json',{'state':'complete','errors':0,'completed_frames':1,
        'checkpoint_sha256':digest})
    write('merge_summary.json',{'all_checked_frames':2,'new_source':{'source_splits':{'val':1}}})
    (output/'training/metrics.jsonl').write_text(json.dumps({'epoch':0,'validation':{
        'AP50':.99,'AP75':.95,'AP50_95':.90}})+'\n')
    control=tmp_path/'configs/active_dataset.json'
    control.write_text(json.dumps({'protocol':'protocol'}))
    config={'protocol':'protocol','expected_new_frames':1,'report_path':'docs/new_report.md',
        'previous_report':'docs/human_correction_7086_report.md'}
    original_write=mixed.write_json
    def interrupt(path,value):
        if path==control:raise RuntimeError('simulated publication interruption')
        return original_write(path,value)
    monkeypatch.setattr(mixed,'write_json',interrupt)
    monkeypatch.setattr(mixed,'now',lambda:'first-completion-time')
    with pytest.raises(RuntimeError,match='simulated publication interruption'):
        mixed.deliver(output,config)
    initial_digest=file_digest(output/'delivery_manifest.json')
    monkeypatch.setattr(mixed,'write_json',original_write)
    monkeypatch.setattr(mixed,'now',lambda:'later-resume-time')
    result=mixed.deliver(output,config)
    assert result['completed_at']=='first-completion-time'
    assert file_digest(output/'delivery_manifest.json')==initial_digest
    assert prior_report.read_text()=='Prior completed round'
    assert (tmp_path/'docs/new_report.md').exists()
    assert "{'val': 1}" in (tmp_path/'docs/new_report.md').read_text()
    assert json.loads(control.read_text())['human_training']==str(output)
    assert 'human_correction_7086_report.md' in (tmp_path/'docs/human_correction_latest_report.md').read_text()
