"""Independent D0/D1 freeze. Historical artifacts and live stores are read-only."""
from __future__ import annotations

from collections import Counter, defaultdict
from contextlib import closing
from datetime import datetime, timezone
from pathlib import Path
import hashlib
import json
import shutil

import cv2
import numpy as np

from .cluster_targets import protected_snapshot
from .mask_model_training_data import (
    PROJECT, FALSE_FLAGS, digest, read, jsonl, write, require, sha, context_roi,
    eligibility, global_audit, letterbox, build_visual_sheets, verify_dataset,
)
from .mask_model_training_history_v2 import compare_history
from .mask_model_training_integrity import database_rows
from .mask_quality_recovery import POSE, GALLERY, PUBLICATIONS, fingerprint
from .mask_quota_saved_extension import file_inventory, load_formal_evidence
from .mask_quota_completion import publication_inventory
from .mask_saved_preannotations_v2 import prelabel_registry, combine_registries, resolve_event, validate_gallery_event
from .mask_template_refined import readonly
from .pose_review_geometry import raster_mask

OUTPUT = PROJECT/'outputs/pallet_mask_model_improvement_v3'
OLD = PROJECT/'outputs/pallet_mask_model_training_v1'
BASE = PROJECT/'outputs/pallet_mask_quality_recovery_v2'


def budget(output):
    free = shutil.disk_usage(output).free
    temp = shutil.disk_usage('/tmp/opencode').free
    require(free >= 35 * 2**30 and temp >= 10 * 2**30, '25GiB experiment +10GiB reserve required')
    return dict(project_free_bytes=free, temporary_free_bytes=temp,
                checkpoint_budget_gib=24, minimum_reserve_gib=10,
                existing_files_deleted=False, maximum_optimizer_updates_per_run=1000)


def preservation(formal):
    """Static artifacts only: live current revisions are NOT immutable artifacts."""
    seal = read(BASE/'independent_verification.json')
    require(digest(BASE/'independent_verification.json') ==
            '61b73d8e0dbc7b99db01b75e912c797beacc34c8c0b3d31434058eac3d376428', 'Old seal changed')
    require(all(digest(PROJECT/k) == h for k, h in seal['frozen_code_sha256'].items()), 'Old frozen code changed')
    protected = protected_snapshot(formal)
    require(protected == read(BASE/'private/protected_formal_before.json') and len(protected) == 20768,
            'Protected formal files changed')
    manifest, _ = verify_dataset(OLD)
    require(len(manifest['files']) == 694 and digest(OLD/'dataset_manifest.json') ==
            '563326cddcd97962a8f2cd8ceb5fe760cb76a0c5f0572a6e9535d01b2ccc13db', 'Old 694-file freeze changed')
    handoff = read(PROJECT/'outputs/pallet_mask_model_training_history_v2_m2/TRAINING_FINAL_HANDOFF.json')
    old_models = {}
    for name, value in handoff['models'].items():
        path = OLD/name/value['checkpoint']
        require(digest(path) == value['checkpoint_sha256'], 'Old model changed')
        old_models[str(path.relative_to(PROJECT))] = digest(path)
    # Include old seals, reports and public packages, never copy/delete them.
    public = {}
    for directory in [PROJECT/'outputs'/name for name in PUBLICATIONS.values()]:
        public[str(directory.relative_to(PROJECT))] = file_inventory(directory)
    return dict(configurations=file_inventory(PROJECT/'configs'), frozen_code=seal['frozen_code_sha256'],
                protected_formal_files=20768, protected_formal_inventory_sha256=sha(protected),
                old_dataset_files=manifest['files'], old_models=old_models, old_public_files=public,
                old_experiment_artifacts=file_inventory(OLD),
                old_history_audits={name: file_inventory(PROJECT/'outputs'/name) for name in
                    ('pallet_mask_model_training_history_v2_checked', 'pallet_mask_model_training_history_v2_m2')})


def provenance_registry(rows, pose_snapshot):
    parent = read(PROJECT/'outputs/pallet_mask_remaining_round5_v1/manifest.json')
    formal_sha = parent['formal_manifest_sha256']
    publications = publication_inventory({v: PROJECT/'outputs'/n for v, n in PUBLICATIONS.items()}, formal_sha)
    with closing(readonly(pose_snapshot, True)) as db:
        current = {k: json.loads(raw) for k, raw in db.execute('SELECT target_key,payload FROM annotations')}
        keys = {k for k, in db.execute('SELECT DISTINCT target_key FROM events')}
    paths = [(PROJECT/f'outputs/pallet_mask_preannotated_requests_v{n}', v, f'v{n}')
             for n, v in ((1, 13), (2, 14), (3, 15))]
    for directory in sorted((PROJECT/'outputs').glob('pallet_mask_under50_requests_v*')):
        if all((directory/n).is_file() for n in ('verification.json', 'independent_verification.json')):
            paths.append((directory, 15, directory.name))
    registry = combine_registries(*(prelabel_registry(p, rows, formal_sha, publications[v], current, keys, v, ns)
                                    for p, v, ns in paths))
    cache = {}

    def validate(value, event):
        ref = resolve_event(value, rows, formal_sha, publications, registry, cache)
        validate_gallery_event(value, rows[event['target_key']], ref, event['revision'])
        require(value['proposal_polygon'] == ref['proposal']['polygon'], 'Human proposal binding changed')
        return dict(namespace=ref.get('namespace'), publication=value['proposal_publication'],
                    task_manifest_sha256=ref.get('task_manifest_sha256'),
                    task_verification_sha256=ref.get('task_verification_sha256'),
                    base_proposal_sha256=value['base_proposal_id'], exact_triple_verified=True)
    return validate


def source_binding(source, rows):
    """Revalidate raw-prefix/latest/revision and actual publication-task triple."""
    source = Path(source).resolve()
    audit = read(source/'audit.json')
    require(audit['state'] == 'passed_event_provenance_and_distinct_saved_count_not_contour_accuracy', 'Source audit failed')
    require(digest(source/'private/source_catalog.jsonl') == audit['source_catalog_sha256'], 'Source catalog changed')
    for kind in ('pose', 'gallery'):
        require(fingerprint(source/f'private/{kind}_snapshot.sqlite3') == audit[kind+'_fingerprint'], 'Source snapshot changed')
    require(fingerprint(POSE) == audit['pose_fingerprint'], 'Pose history changed')
    baseline = database_rows(BASE/'private/gallery_snapshot.sqlite3', True)
    observed = database_rows(source/'private/gallery_snapshot.sqlite3', True)
    validator = provenance_registry(rows, source/'private/pose_snapshot.sqlite3')
    history = compare_history(baseline, observed, validator)
    # Revalidate *every* frozen event, not merely the newer suffix.
    ec = observed['columns']['events']
    for raw in observed['tables']['events']:
        event = dict(zip(ec, raw))
        validator(json.loads(event['payload']), event)
    current = {r[0]: r for r in observed['tables']['reviews']}
    selected = []
    stores = {k: database_rows(source/f'private/{k}_snapshot.sqlite3', True) for k in ('pose', 'gallery')}
    for reference in jsonl(source/'private/source_catalog.jsonl'):
        key = reference['target_key']
        # Held-out polygons are never accessed, rasterized, exported or scored.
        if rows[key]['split'] != 'train':
            continue
        kind = 'pose' if reference['origin'] == 'saved_pose' else 'gallery'
        store = stores[kind]
        columns = store['columns']['events']
        hits = [dict(zip(columns, e)) for e in store['tables']['events']
                if e[columns.index('target_key')] == key and
                hashlib.sha256(e[columns.index('payload')].encode()).hexdigest() == reference['event_raw_sha256']]
        require(len(hits) == 1, 'Research reference lacks unique recoverable raw event')
        event = hits[0]
        payload = json.loads(event['payload'])
        table = 'annotations' if kind == 'pose' else 'reviews'
        cc = store['columns'][table]
        current_rows = [dict(zip(cc, r)) for r in store['tables'][table] if r[cc.index('target_key')] == key]
        require(len(current_rows) == 1 and sha(json.loads(current_rows[0]['payload'])) == reference['payload_sha256'],
                'Research reference/current payload differs')
        # Pose stores can have later pose-only revisions. Bind the human polygon
        # event independently from current payload metadata, never export pose.
        require(payload['polygon'] == reference['polygon'] == json.loads(current_rows[0]['payload'])['polygon'],
                'Research mask/event polygon differs')
        reference['source_event_id'] = event['id']
        selected.append(reference)
    return selected, dict(source=str(source.relative_to(PROJECT)), source_audit_sha256=digest(source/'audit.json'),
        source_catalog_sha256=audit['source_catalog_sha256'], summary=audit,
        raw_history=history['summary'], train_references=len(selected), source_files={
            n: digest(source/n) for n in ('audit.json', 'private/source_catalog.jsonl',
                'private/event_binding_audit.jsonl', 'private/pose_snapshot.sqlite3', 'private/gallery_snapshot.sqlite3')})


def prepare(source, output=OUTPUT):
    output = Path(output).resolve()
    require(output.parent == PROJECT/'outputs' and not output.exists(), 'Fresh version required; retain failures')
    cv2.setNumThreads(1)
    formal, rows, evidence = load_formal_evidence(read(PROJECT/'outputs/pallet_mask_remaining_round5_v1/manifest.json'))
    require(len(rows) == 12344, 'Formal denominator changed')
    storage = budget(output.parent)
    before = preservation(formal)
    references, binding = source_binding(source, rows)
    output.mkdir(mode=0o700)
    (output/'private').mkdir(mode=0o700)
    write(output/'storage_budget.json', storage)
    write(output/'private/preservation_before.json', before)
    write(output/'private/source_binding.json', binding)
    write(output/'research_permit.json', dict(
        authorization='User now authorizes D0/D1/D2 and bounded A0/A1/A2/A3-freeze/A3-photometric/B0/B1 actual training',
        model_inputs='image pixels only; original bbox context; no human/pose/class/group/ID input',
        polygons='formal train only, supervised research mask loss/development metrics, not GT promotion',
        future_appends='Never dynamically appended; frozen raw revisions remain recoverable when superseded',
        official_val_test_polygon_exports=0, official_val_test_predictions=0,
        old_task_checkpoints_used=False, production_writes=0, max_updates_per_run=1000,
        release_minimum_groups=10, semantic_unknown_is_unknown=True, closed_holes_invented=False, **FALSE_FLAGS))
    labels, blocked, links, hashes = global_audit(rows, evidence)
    write(output/'private/global_components.json', dict(by_target=labels, links=links,
        blocked_by_formal_val_test=sorted(blocked), all_targets=12344,
        all_classes_and_policies=True, audit='group/frame/PCD/image-pixel + dHash128<=2,NCC>=.995 full/crop',
        limitation='Not exhaustive physical-scene identity proof'))
    write(output/'private/global_native_hashes.jsonl', hashes, lines=True)
    by_frame = defaultdict(list)
    for row in rows.values():
        by_frame[row['frame_id']].append(row)
    old = {r['target_key']: r for r in jsonl(OLD/'private/training_records.jsonl')}
    new_keys = set(binding['summary']['new_distinct_saved_keys'])
    audit, candidates, retained = [], [], []
    for reference in references:
        key, row = reference['target_key'], rows[reference['target_key']]
        reasons, roi = eligibility(reference, row, by_frame[row['frame_id']], labels[key], blocked)
        # Narrow/unknown metadata is not automatically resolved by a human save.
        # It may enter ONLY explicit visual review for stable observed-mask
        # supervision; Class14 ambiguity/neighbor/leakage remain hard exclusions.
        reviewable = {'unknown_or_narrow_observation_surface', 'unknown_or_edge_only_structure'}
        if key in new_keys:
            hard = [r for r in reasons if r not in reviewable]
        else:
            hard = reasons
        entry = dict(target_key=key, class_id=row['final_class_id'], component=labels[key],
            reasons=reasons, hard_exclusions=hard, source_event_raw_sha256=reference['event_raw_sha256'],
            revision=reference['revision'], is_new_distinct=key in new_keys,
            prior_approved=key in old, **FALSE_FLAGS)
        audit.append(entry)
        if hard:
            continue
        record = dict(target_key=key, component=labels[key], class_id=row['final_class_id'],
            frame_id=row['frame_id'], group_id=row['group_id'], native_image=row['image'],
            native_size=row['size'], original_bbox=row['bbox'], roi=roi, polygon=reference['polygon'],
            source_event_raw_sha256=reference['event_raw_sha256'], source_payload_sha256=reference['payload_sha256'],
            source_event_id=reference['source_event_id'], source_revision=reference['revision'],
            source_origin=reference['origin'], source_batch=reference['batch'],
            policy_id=row['policy_id'], surface=row['viewpoint_candidate'], structure=row['structure_candidate'],
            formal_split='train', occlusion='unknown', reflection='unknown', **FALSE_FLAGS)
        if key in old and reference['polygon'] == old[key]['polygon']:
            record['semantic_review'] = 'prior_approved_unchanged_polygon_requalified_global_audit'
            retained.append(record)
        elif key in old or key in new_keys:
            record['semantic_review'] = 'pending_new_or_revised_visual_mask_compatibility_not_surface_GT'
            candidates.append(record)
        # Previously visually rejected samples are never silently readmitted.
    write(output/'private/eligibility_audit.jsonl', audit, lines=True)
    write(output/'private/retained_requalified.jsonl', retained, lines=True)
    write(output/'private/eligible_before_visual_review.jsonl', candidates, lines=True)
    build_visual_sheets(output, candidates, formal)
    summary = dict(train_source_references=len(references), retained=len(retained), visual_candidates=len(candidates),
        new_distinct_source_count=len(new_keys), new_distinct_formal_train=sum(r['is_new_distinct'] for r in audit),
        hard_exclusions=dict(Counter(reason for r in audit for reason in r['hard_exclusions'])),
        global_components=len(set(labels.values())), cross_class_near_links=sum(bool(r.get('cross_class')) for r in links))
    write(output/'preparation_summary.json', summary)
    print(json.dumps(summary), flush=True)
    return summary


def balanced_split(records):
    """Class-coverage-first grouped holdout, no metrics/label-shape use.

    Reserve train support for EVERY selected class, then dev support for each
    multi-component class. Grow dev to 10 groups where at least 2 calibration
    groups remain. The tiny calibration side is honestly insufficient to release.
    Fold0=dev, fold1=calibration, fold2=train; same component never crosses.
    """
    groups = defaultdict(list)
    for r in records:
        groups[r['component']].append(r)
    classes = {g: {r['class_id'] for r in values} for g, values in groups.items()}
    support = Counter(c for values in classes.values() for c in values)
    order = sorted(support, key=lambda c: (support[c], c))
    assignment, covered = {}, set()
    rank = lambda g: sha(dict(seed=42, component=g))
    for c in order:
        if c in covered:
            continue
        candidates = [g for g in groups if g not in assignment and c in classes[g]]
        g = min(candidates, key=lambda g: (-len(classes[g]-covered), rank(g)))
        assignment[g] = 2
        covered.update(classes[g])
    dev_covered = set()
    for c in order:
        if c in dev_covered or support[c] < 2:
            continue
        candidates = [g for g in groups if g not in assignment and c in classes[g]]
        if candidates:
            g = min(candidates, key=lambda g: (-len(classes[g]-dev_covered), rank(g)))
            assignment[g] = 0
            dev_covered.update(classes[g])
    remaining = sorted(set(groups)-set(assignment), key=rank)
    target_dev = min(10, sum(v == 0 for v in assignment.values()) + max(0, len(remaining)-2))
    while remaining and sum(v == 0 for v in assignment.values()) < target_dev:
        g = remaining.pop(0)
        assignment[g] = 0
    for i, g in enumerate(remaining):
        assignment[g] = 1 if i < max(2, len(remaining)//2) else 2
    return assignment


def freeze(approved_indices, output=OUTPUT):
    output = Path(output)
    require(not (output/'dataset_manifest.json').exists(), 'Already frozen')
    formal, rows, _ = load_formal_evidence(read(PROJECT/'outputs/pallet_mask_remaining_round5_v1/manifest.json'))
    candidates = jsonl(output/'private/eligible_before_visual_review.jsonl')
    indices = sorted(set(approved_indices))
    require(all(type(i) is int and 0 <= i < len(candidates) for i in indices), 'Invalid visual approval')
    records = jsonl(output/'private/retained_requalified.jsonl') + [candidates[i] for i in indices]
    require(len(records) == len({r['target_key'] for r in records}), 'Duplicate frozen target')
    for record in records:
        if record['semantic_review'].startswith('pending'):
            record['semantic_review'] = 'assistant_visual_compatible_observed_single_ring_mask_only_not_human_GT'
    folds = balanced_split(records)
    for record in records:
        record['group_fold'] = folds[record['component']]
        record['development_split'] = {0: 'dev', 1: 'calibration', 2: 'train'}[record['group_fold']]
    require({r['development_split'] for r in records} == {'train', 'dev', 'calibration'}, 'Empty split')
    write(output/'private/visual_semantic_approval.json', dict(approved_indices=indices,
        rejected_indices=sorted(set(range(len(candidates))) - set(indices)),
        candidate_sha256=digest(output/'private/eligible_before_visual_review.jsonl'),
        reviewer='assistant_visual_research_compatibility_not_human_GT',
        note='Original bbox context. Only visible stable manual topology; unknown semantic corners/holes remain unknown. '
             'No re-admission of previously visually rejected sources. No predictions existed at split time.', **FALSE_FLAGS))
    from pycocotools import mask as mask_utils
    for name in ('dataset', 'dataset_rfdetr', 'private/native_masks'):
        (output/name).mkdir(mode=0o700)
    transforms = {}
    coco = {}
    for s in ('train', 'valid', 'calibration'):
        for name in ('dataset', 'dataset_rfdetr'):
            (output/name/s).mkdir(mode=0o700)
        coco[s] = dict(info=dict(description='Formal-train-only grouped development research'),
            licenses=[], images=[], annotations=[], categories=[dict(id=1, name='fork_face')])
    square = {s: {**coco[s], 'images': [], 'annotations': []} for s in coco}
    for i, record in enumerate(records):
        image = cv2.imread(str(formal/record['native_image']), cv2.IMREAD_GRAYSCALE)
        require(image is not None and list(image.shape[::-1]) == record['native_size'], 'Native image invalid')
        x1, y1, x2, y2 = record['roi']
        grey = image[y1:y2, x1:x2]
        full = raster_mask(record['polygon'], *record['native_size']).astype(np.uint8)
        mask = full[y1:y2, x1:x2]
        require(mask.sum() == full.sum(), 'Original bbox crop clips mask supervision')
        split = 'valid' if record['development_split'] == 'dev' else record['development_split']
        name = f'{i:05d}.png'
        cv2.imwrite(str(output/f'dataset/{split}/{name}'), np.repeat(grey[:, :, None], 3, axis=2))
        cv2.imwrite(str(output/f'private/native_masks/{name}'), mask * 255)
        h, w = grey.shape
        im = dict(id=i+1, file_name=name, width=w, height=h)
        def annotation(label):
            rle = mask_utils.encode(np.asfortranarray(label))
            require(np.array_equal(mask_utils.decode(rle), label), 'RLE roundtrip differs')
            return dict(id=i+1, image_id=i+1, category_id=1, iscrowd=0,
                segmentation=dict(size=rle['size'], counts=rle['counts'].decode('ascii')),
                area=int(mask_utils.area(rle)), bbox=mask_utils.toBbox(rle).tolist())
        coco[split]['images'].append(im)
        coco[split]['annotations'].append(annotation(mask))
        canvas, label, _, transform = letterbox(grey, mask, (576, 576))
        cv2.imwrite(str(output/f'dataset_rfdetr/{split}/{name}'), np.repeat(canvas[:, :, None], 3, axis=2))
        transforms[str(i+1)] = transform
        square[split]['images'].append(dict(im, width=576, height=576))
        square[split]['annotations'].append(annotation(label))
        record.update(image=f'dataset/{split}/{name}', native_mask=f'private/native_masks/{name}',
                      roi_size=[w, h], image_id=i+1)
    for split in coco:
        write(output/f'dataset/{split}/_annotations.coco.json', coco[split])
        write(output/f'dataset_rfdetr/{split}/_annotations.coco.json', square[split])
    write(output/'private/training_records.jsonl', records, lines=True)
    write(output/'private/rfdetr_transforms.json', transforms)
    files = {str(p.relative_to(output)): digest(p) for directory in
             ('dataset', 'dataset_rfdetr', 'private/native_masks') for p in sorted((output/directory).rglob('*')) if p.is_file()}
    for name in ('private/training_records.jsonl', 'private/rfdetr_transforms.json', 'private/visual_semantic_approval.json',
                 'private/source_binding.json', 'private/global_components.json'):
        files[name] = digest(output/name)
    components = {s: len({r['component'] for r in records if r['development_split'] == s})
                  for s in ('train', 'dev', 'calibration')}
    strata = {field: {str(v): {s: dict(images=sum(r[field] == v and r['development_split'] == s for r in records),
                groups=len({r['component'] for r in records if r[field] == v and r['development_split'] == s}))
                for s in components} for v in sorted({r[field] for r in records}, key=str)}
              for field in ('class_id', 'structure', 'policy_id', 'surface')}
    manifest = dict(state='immutable_research_training_dataset', files=files,
        splits=dict(Counter(r['development_split'] for r in records)), independent_components=components,
        strata=strata, group_folds=folds, split_rule='metadata-only class-coverage-first train reserve, multi-group dev coverage then dev10 if possible, calibration>=2; seed42 hash ties',
        selection_split='dev only; calibration never early-stopping/architecture/threshold selection',
        blind=False, formal_val_test_included=False, model_input_metadata=False,
        release_minimum_groups=10, insufficient_group_support=any(components[s] < 10 for s in ('dev', 'calibration')),
        status='exploratory_group_heldout_not_blind', **FALSE_FLAGS)
    write(output/'dataset_manifest.json', manifest)
    require(preservation(formal) == read(output/'private/preservation_before.json'), 'Old protected inputs changed')
    write(output/'private/preservation_after_freeze.json', read(output/'private/preservation_before.json'))
    print(json.dumps(dict(splits=manifest['splits'], components=components, release_blocked=manifest['insufficient_group_support'])), flush=True)
    return manifest


def audit_after(output=OUTPUT):
    output = Path(output)
    formal, rows, _ = load_formal_evidence(read(PROJECT/'outputs/pallet_mask_remaining_round5_v1/manifest.json'))
    same = preservation(formal) == read(output/'private/preservation_before.json')
    require(same, 'Old static artifacts changed')
    verify_dataset(output)
    binding = read(output/'private/source_binding.json')
    source = PROJECT/binding['source']
    require(all(digest(source/n) == h for n, h in binding['source_files'].items()), 'Frozen source files changed')
    validator = provenance_registry(rows, source/'private/pose_snapshot.sqlite3')
    history = compare_history(database_rows(source/'private/gallery_snapshot.sqlite3', True), database_rows(GALLERY), validator)
    require(fingerprint(POSE) == binding['summary']['pose_fingerprint'], 'Pose changed')
    selected = {r['target_key']: r for r in jsonl(output/'private/training_records.jsonl')}
    superseded = [dict(target_key=k, development_split=selected[k]['development_split'],
                       frozen_revision=selected[k]['source_revision'],
                       observed_revision=history['new_current'][k]['revision'])
                  for k in sorted(history['changed_keys']) if k in selected]
    result = dict(state='passed_static_preservation_and_append_aware_history',
        old_dataset_files_verified=694, old_models_verified=2, all_static_before_after_identical=True,
        frozen_source_revision_is_recoverable=True, live_gallery_history=history['summary'],
        superseded_selected_references=superseded, dynamic_training_append=False,
        observed_at=datetime.now(timezone.utc).isoformat(), formal_val_test_predictions=0, production_writes=0)
    write(output/'preservation_final.json', result)
    write(output/'private/concurrent_human_appends.jsonl', history['delta_events'], lines=True)
    return result
