"""Read-only diagnostic of direct human-template coverage; never a publication."""
from __future__ import annotations

from collections import Counter, defaultdict
from concurrent.futures import ThreadPoolExecutor
from contextlib import closing
from datetime import datetime, timezone
import json
from pathlib import Path
import sys

import cv2

PROJECT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(PROJECT))
from pallet.cluster_targets import digest, load_dataset, target_key
from pallet.mask_gallery_review import VERSION as EDITOR_VERSION
from pallet.mask_template_continue import attempt_target, sha
from pallet.mask_template_refined import (CLASSES, QUARANTINE, readonly,
    read_jsonl, select_diverse, snapshot_database, write_jsonl)
from pallet.mask_template_transfer import (ImageEvidence, annotation_inventory,
    compatible, write_json)
from pallet.pose_review_geometry import polygon_value

OUT = PROJECT/'outputs/pallet_mask_quota_feasibility_v1'
PARENT = PROJECT/'outputs/pallet_mask_template_refined_v3'
PUBLICATIONS = {3: PROJECT/'outputs/pallet_mask_template_bulk_v5',
                4: PROJECT/'outputs/pallet_mask_template_refined_v2', 5: PARENT}


def main():
    cv2.setNumThreads(1)
    if OUT.exists():
        raise ValueError('Diagnostic requires a new output directory')
    formal = Path(json.loads((PROJECT/'configs/active_formal_dataset.json').read_text())['dataset'])
    classification = PROJECT/'outputs/pallet_appearance_center_reclassification_v1/targets.jsonl'
    formal_sha, classification_sha = digest(formal/'manifest.json'), digest(classification)
    parent_manifest = json.loads((PARENT/'manifest.json').read_text())
    code = {'pallet/mask_template_continue.py': parent_manifest['attempt_code_sha256'],
            'pallet/mask_template_transfer.py': parent_manifest['transfer_code_sha256'],
            'pallet/mask_boundary_review.py': parent_manifest['boundary_code_sha256']}
    assert all(digest(PROJECT/name) == expected for name, expected in code.items())
    assert formal_sha == parent_manifest['formal_manifest_sha256']
    assert classification_sha == parent_manifest['classification_sha256']
    previous = read_jsonl(PARENT/'proposals.jsonl')
    published = {p['target_key']: p for p in previous}
    previous_frames = {p['source']['frame_id'] for p in previous}
    assert len(previous) == len(published) == len(previous_frames) == 140
    annotations, sources = load_dataset(formal)
    rows = {r['target_key']: r for r in read_jsonl(classification)}
    identities = {target_key(a['frame_id'], i): (a, b) for a in annotations for i, b in enumerate(a['boxes'])}
    assert set(identities) == set(rows)
    for key, row in rows.items():
        a, b = identities[key]
        source = sources[row['frame_id']]
        assert row['bbox'] == b and all(row[k] == a[k] for k in ('frame_id', 'split', 'group_id'))
        assert row['policy_id'] == source['sensor_policy']['policy_id']
        assert row['source_pcd_sha256'] == source['source_anchor_hash']
        row['size'] = source['size']
    OUT.mkdir()
    pose_live = PROJECT/'outputs/pallet_pose_human_review_v1/annotations.sqlite3'
    gallery_live = PROJECT/'outputs/pallet_mask_gallery_reviews_v1/mask_reviews.sqlite3'
    pose_snapshot, gallery_snapshot = OUT/'pose_snapshot.sqlite3', OUT/'gallery_snapshot.sqlite3'
    pose_sha = snapshot_database(pose_live, pose_snapshot)
    gallery_sha = snapshot_database(gallery_live, gallery_snapshot)
    with closing(readonly(pose_snapshot, True)) as db:
        current, original_templates, status = annotation_inventory(db, rows)
        pose_events = db.execute('SELECT COUNT(*) FROM events').fetchone()[0]
        assert json.loads(db.execute("SELECT value FROM metadata WHERE key='binding'").fetchone()[0])['formal_manifest_sha256'] == formal_sha
    with closing(readonly(gallery_snapshot, True)) as db:
        reviews = {k: json.loads(v) for k, v in db.execute('SELECT target_key,payload FROM reviews')}
        events = [json.loads(v) for v, in db.execute('SELECT payload FROM events ORDER BY id')]
        assert db.execute("SELECT value FROM metadata WHERE key='formal_manifest_sha256'").fetchone()[0] == formal_sha
    gallery_protected = set(reviews) | {v['target_key'] for v in events}
    histories = defaultdict(list)
    for event in events:
        histories[event['target_key']].append(event)
    publication_records = {}
    for version, path in PUBLICATIONS.items():
        proposal_sha = digest(path/'proposals.jsonl')
        seal = json.loads((path/'verification.json').read_text())
        assert seal['state'] == 'passed' and seal['proposals_sha256'] == proposal_sha
        publication_records[version] = ({p['target_key']: p for p in read_jsonl(path/'proposals.jsonl')},
            sha(dict(formal=formal_sha, proposals=proposal_sha, editor_version=EDITOR_VERSION)))
    templates = {}
    for key, t in original_templates.items():
        if key in QUARANTINE:
            continue
        templates['pose:'+key] = dict(t, origin='original_pose_human_polygon',
            provenance=dict(store='pose', target_key=key, snapshot_sha256=pose_sha,
                current_payload_sha256=t['current_payload_sha256'], human_event_sha256=t['human_event_sha256'],
                current_revision=t['revision'], human_polygon_revision=t['human_polygon_revision'],
                polygon_sha256=sha(t['polygon'])))
    for key, review in reviews.items():
        assert review['event_kind'] == 'human_mask_gallery_revision' and review['human_edited'] is True
        assert max(histories[key], key=lambda v: v['revision']) == review
        if len(review.get('polygon', [])) <= 4 or key in QUARANTINE:
            continue
        row = rows[key]
        polygon_value(review['polygon'], *row['size'])
        source_proposals, binding = publication_records[review['proposal_publication']]
        p = source_proposals[key]
        assert p == published[key] and sha(p) == review['base_proposal_id'] and binding == review['binding']
        assert review['proposal_polygon'] == p['polygon'] and review['template_provenance'] == p['template']
        identity = {k: row[k] for k in ('target_key', 'frame_id', 'split', 'box_index', 'bbox', 'size', 'source_pcd_sha256')}
        identity.update(formal_manifest_sha256=formal_sha, image_sha256=digest(PARENT/'images'/f'{key}.png'))
        assert review['source'] == identity and review['source_binding'] == sha(identity)
        assert len(review['polygon']) == review['vertex_count']
        templates['gallery:'+key] = dict(row=row, polygon=review['polygon'], origin='mask_gallery_human_revision',
            provenance=dict(store='gallery', target_key=key, snapshot_sha256=gallery_sha,
                current_revision=review['revision'], human_polygon_revision=review['revision'],
                payload_sha256=sha(review), human_event_sha256=sha(review),
                source_binding=review['source_binding'], polygon_sha256=sha(review['polygon']),
                proposal_publication=review['proposal_publication']))
    # Namespaces retain both actual latest manual polygons if stores share a key.
    evidence = ImageEvidence(formal, rows, sources)
    class NamespacedEvidence:
        def patch(self, key):
            return evidence.patch(templates[key]['row']['target_key'] if key in templates else key)
        def image(self, frame):
            return evidence.image(frame)
    named_evidence = NamespacedEvidence()
    write_jsonl(OUT/'human_templates.jsonl', [dict(template_id=k, **t) for k, t in sorted(templates.items())])
    write_jsonl(OUT/'gallery_saved_records.jsonl', list(reviews.values()))
    write_json(OUT/'protected_targets.json', dict(already_published=sorted(published),
        gallery_history=sorted(gallery_protected), quarantine=sorted(QUARANTINE), pose_status=status))
    coverage, work, outcomes = [], [], []
    for key, row in sorted(rows.items()):
        reason = ('previously_quarantined' if key in QUARANTINE else 'already_published' if key in published else
                  'gallery_history_protected' if key in gallery_protected else status[key])
        if reason == 'eligible' and row['final_class_id'] not in CLASSES:
            reason = 'class_not_requested'
        if reason != 'eligible':
            outcomes.append(dict(target_key=key, class_id=row['final_class_id'], reason=reason))
            continue
        options = [k for k, t in templates.items() if compatible(t['row'], row)]
        gallery_options = [k for k in options if k.startswith('gallery:')]
        pose_options = [k for k in options if k.startswith('pose:')]
        coverage.append(dict(target_key=key, frame_id=row['frame_id'], class_id=row['final_class_id'],
            group_id=row['group_id'], split=row['split'], policy_id=row['policy_id'],
            viewpoint_candidate=row.get('viewpoint_candidate'), novel_frame=row['frame_id'] not in previous_frames,
            gallery_compatible=bool(gallery_options), pose_compatible=bool(pose_options), union_compatible=bool(options),
            compatible_template_ids=options, coverage_added_by_pose=bool(pose_options) and not gallery_options))
        work.append((key, row))
    write_jsonl(OUT/'target_compatibility.jsonl', coverage)
    print(json.dumps(dict(state='diagnostic_scan', saved_gallery=len(reviews), gallery_events=len(events),
        pose_saved=len(current), pose_events=pose_events, original_human_templates=len(original_templates),
        usable_template_variants=len(templates), eligible_targets=len(work),
        eligible_by_class=dict(Counter(row['final_class_id'] for _, row in work)))), flush=True)
    accepted, attempts = [], []
    def worker(item):
        return item, attempt_target(*item, templates, named_evidence)
    with ThreadPoolExecutor(max_workers=8) as pool:
        for number, ((key, row), (match, trials, outcome)) in enumerate(pool.map(worker, work), 1):
            attempts.extend(trials)
            if match is not None:
                tid = match['template_key']; t = templates[tid]
                old = current.get(key)
                accepted.append(dict(schema='human_union_quota_feasibility_diagnostic_v1', target_key=key,
                    class_id=row['final_class_id'], source=row, polygon=match['polygon'], vertex_count=len(match['polygon']),
                    metrics=match['metrics'], status='diagnostic_only_pending_independent_verification_and_human_review',
                    template=dict(template_id=tid, target_key=t['row']['target_key'], origin=t['origin'],
                        source=t['row'], vertex_count=len(t['polygon']), provenance=t['provenance'],
                        canonical_template_sha256=sha(t)), template_rank=match['template_rank'],
                    target_expected_revision=old['revision'] if old else 0,
                    target_snapshot_payload_sha256=sha(old) if old else None,
                    snapshot_sha256=pose_sha, gallery_snapshot_sha256=gallery_sha,
                    formal_manifest_sha256=formal_sha, class_assignments_sha256=classification_sha,
                    human_mask_confirmed=False, human_pose_confirmed=False, automatic_training_eligible=False,
                    robot_command_eligible=False, ground_truth_eligible=False, production_saved=False))
                outcome['reason'] = 'diagnostic_accepted'
            outcomes.append(outcome)
            if number % 500 == 0:
                print(json.dumps(dict(scanned=number, accepted=len(accepted), attempts=len(attempts))), flush=True)
    selected, dispositions = select_diverse(previous, accepted)
    for outcome in outcomes:
        if outcome['target_key'] in dispositions:
            outcome['selection_diagnostic'] = dispositions[outcome['target_key']]
    write_jsonl(OUT/'accepted_candidates.jsonl', accepted)
    write_jsonl(OUT/'selected_novel_matches.jsonl', selected)
    write_jsonl(OUT/'registration_attempts.jsonl', attempts)
    write_jsonl(OUT/'outcomes.jsonl', outcomes)
    accepted_keys = {p['target_key'] for p in accepted}
    selected_keys = {p['target_key'] for p in selected}
    def counts(items):
        return dict(eligible_targets=len(items), eligible_distinct_frames=len({v['frame_id'] for v in items}),
            eligible_novel_frames=len({v['frame_id'] for v in items if v['novel_frame']}),
            gallery_compatible_targets=sum(v['gallery_compatible'] for v in items),
            pose_compatible_targets=sum(v['pose_compatible'] for v in items),
            union_compatible_targets=sum(v['union_compatible'] for v in items),
            union_compatible_novel_frames=len({v['frame_id'] for v in items if v['union_compatible'] and v['novel_frame']}),
            coverage_added_by_pose_targets=sum(v['coverage_added_by_pose'] for v in items),
            coverage_added_by_pose_novel_frames=len({v['frame_id'] for v in items if v['coverage_added_by_pose'] and v['novel_frame']}),
            accepted_targets=sum(v['target_key'] in accepted_keys for v in items),
            accepted_novel_frames=len({v['frame_id'] for v in items if v['novel_frame'] and v['target_key'] in accepted_keys}),
            selected_novel_frames=sum(v['target_key'] in selected_keys for v in items))
    classes = {}
    for cls in CLASSES:
        items = [v for v in coverage if v['class_id'] == cls]
        before = len({p['source']['frame_id'] for p in previous if p['class_id'] == cls})
        values = counts(items)
        values.update(existing_distinct_frames=before, initial_shortfall=50-before,
            gallery_templates=sum(t['row']['final_class_id'] == cls and k.startswith('gallery:') for k, t in templates.items()),
            pose_templates=sum(t['row']['final_class_id'] == cls and k.startswith('pose:') for k, t in templates.items()),
            template_distinct_target_keys=len({t['row']['target_key'] for t in templates.values() if t['row']['final_class_id'] == cls}),
            attainable_with_direct_frozen_union=before+values['selected_novel_frames'],
            remaining_shortfall=50-before-values['selected_novel_frames'],
            outcomes=dict(Counter(v['reason'] for v in outcomes if v['class_id'] == cls)),
            rejection_attempts=dict(Counter(v['metrics']['rejection'] for v in attempts if v['class_id'] == cls and v['metrics']['rejection'])))
        classes[cls] = values
    grouped = []
    for fields in [('class_id', 'group_id', 'split', 'policy_id', 'viewpoint_candidate'),
                   ('class_id', 'split'), ('class_id', 'policy_id'), ('class_id', 'viewpoint_candidate')]:
        buckets = defaultdict(list)
        for item in coverage:
            buckets[tuple(item[k] for k in fields)].append(item)
        for identity, items in sorted(buckets.items(), key=lambda pair: str(pair[0])):
            grouped.append(dict(dimensions=list(fields), **dict(zip(fields, identity)), **counts(items)))
    write_jsonl(OUT/'coverage_by_group_split_policy_view.jsonl', grouped)
    template_groups = defaultdict(list)
    for tid, t in templates.items():
        r = t['row']
        template_groups[(r['final_class_id'], r['group_id'], r['split'], r['policy_id'], r.get('viewpoint_candidate'))].append(tid)
    write_jsonl(OUT/'template_groups.jsonl', [dict(class_id=k[0], group_id=k[1], split=k[2], policy_id=k[3],
        viewpoint_candidate=k[4], template_ids=v) for k, v in sorted(template_groups.items(), key=lambda pair: str(pair[0]))])
    # Fresh read records whether concurrent saves would require a later rerun.
    with closing(readonly(gallery_live)) as db:
        live_reviews = {k: json.loads(v) for k, v in db.execute('SELECT target_key,payload FROM reviews')}
        live_gallery_events = db.execute('SELECT COUNT(*) FROM events').fetchone()[0]
    with closing(readonly(pose_live)) as db:
        live_current, live_templates, live_status = annotation_inventory(db, rows)
        live_pose_events = db.execute('SELECT COUNT(*) FROM events').fetchone()[0]
    changed_inputs = (live_reviews != reviews or live_templates != original_templates or live_status != status)
    assert all(digest(PROJECT/name) == expected for name, expected in code.items())
    summary = dict(state='diagnostic_complete_not_publication_verification', created_at=datetime.now(timezone.utc).isoformat(),
        classes=classes, saved_gallery_at_snapshot=len(reviews), gallery_events_at_snapshot=len(events),
        saved_gallery_at_finish=len(live_reviews), gallery_events_at_finish=live_gallery_events,
        pose_saved_at_snapshot=len(current), pose_events_at_snapshot=pose_events, pose_events_at_finish=live_pose_events,
        template_or_eligibility_inputs_changed_during_scan=changed_inputs,
        original_pose_human_templates=len(original_templates), gallery_template_variants=sum(k.startswith('gallery:') for k in templates),
        template_variants=len(templates), distinct_template_target_keys=len({t['row']['target_key'] for t in templates.values()}),
        overlapping_pose_gallery_template_keys=sorted(set(original_templates) & set(reviews)),
        eligible_scanned=len(work), registration_attempts=len(attempts), accepted_targets=len(accepted),
        selected_novel_frames=len(selected), existing_frames=len(previous_frames),
        attainable_cumulative_frames=len(previous_frames)+len(selected), remaining_to_250=250-len(previous_frames)-len(selected),
        all_five_classes_reach_50=all(v['remaining_shortfall'] == 0 for v in classes.values()),
        pose_snapshot_sha256=pose_sha, gallery_snapshot_sha256=gallery_sha,
        formal_manifest_sha256=formal_sha, classification_sha256=classification_sha,
        frozen_gate_code_sha256=code, diagnostic_code_sha256=digest(__file__),
        thresholds_relaxed=False, cross_split_transfers=0, production_database_writes=0,
        recursive_candidate_propagation=False, publication_verified=False, ground_truth_eligible=False,
        scan_policy='Unchanged attempt_target: every compatible template >=0.55 initial NCC until first transfer+beam success; all eligible targets scanned. Both stores retained as separate human polygon variants.',
        pose_eligibility_policy='Exact annotation_inventory; non-original mask history/deleted/deferred/confirmed protected; unchanged-rectangle-only pose/ruler saves remain sidecar eligible.')
    write_json(OUT/'summary.json', summary)
    write_json(OUT/'used_image_hashes.json', evidence.used)
    lines = ['# Direct human-template quota feasibility — diagnostic only', '',
        '| Class | Existing | Gallery / pose templates | Compatible targets | Added coverage from pose | Accepted | Selected novel | Attainable | Gap |',
        '|---|---:|---:|---:|---:|---:|---:|---:|---:|']
    for cls, v in classes.items():
        lines.append(f"| {cls} | {v['existing_distinct_frames']} | {v['gallery_templates']} / {v['pose_templates']} | {v['union_compatible_targets']} | {v['coverage_added_by_pose_targets']} | {v['accepted_targets']} | {v['selected_novel_frames']} | {v['attainable_with_direct_frozen_union']} | {v['remaining_shortfall']} |")
    lines += ['', f"Snapshot: {len(reviews)} saved gallery edits; {len(original_templates)} original pose-human templates. Actual gate-passing candidates and complete canonical provenance are in `accepted_candidates.jsonl`; quota-selected novel frames are in `selected_novel_matches.jsonl`.", '',
        'Compatibility coverage is an upper bound, not evidence of passing image/vertex/beam gates. Group/split/policy/view details are in `coverage_by_group_split_policy_view.jsonl` and `template_groups.jsonl`.', '',
        f"All eligible targets were scanned. Concurrent template/eligibility changes: {changed_inputs}. Results are diagnostic drafts; no publication verification, GT, training eligibility, or production writes."]
    (OUT/'README.md').write_text('\n'.join(lines)+'\n')
    print(json.dumps(summary), flush=True)


if __name__ == '__main__':
    main()
