"""Versioned audit of legitimate concurrent human revisions, not new training.

Unlike v1's strictly unchanged-current policy, this separately authorized v2
proves immutable old event prefixes and fully bound later human revisions. It
does not alter v1 receipts, data, selection, training or any live store.
"""
from __future__ import annotations

from collections import Counter, defaultdict
from datetime import datetime, timezone
import hashlib
import json
from pathlib import Path

import cv2
import numpy as np

from .cluster_targets import protected_snapshot
from .mask_model_training_data import (
    PROJECT, SOURCE, OUTPUT, SEAL, POSE, GALLERY, digest, file_inventory, jsonl,
    read, require, sha, verify_dataset, write,
)
from .mask_model_training_integrity import database_rows
from .mask_quality_metrics import compare_polygons
from .mask_quota_saved_extension import load_formal_evidence
from .mask_quota_completion import publication_inventory
from .mask_saved_preannotations_v2 import (
    prelabel_registry, registry_key, validate_gallery_event,
)
from .pose_review_geometry import raster_mask

TASK_V3 = PROJECT/'outputs/pallet_mask_preannotated_requests_v3'
TASK_V3_PIN = '265974c1fa43a950e813e19987e6d99e5a870321115e4a965807b037e1037fe0'
SUPPLEMENT = PROJECT/'outputs/pallet_mask_under50_requests_v1'
SUPPLEMENT_PIN = '87d868e53d396acf37a161019b71144fbff906c62091e7d4b9d68458f9083e5a'
DEFAULT_AUDIT = PROJECT/'outputs/pallet_mask_model_training_history_v2'


def raw_sha(value):
    return hashlib.sha256(value.encode('utf-8')).hexdigest()


def table_index(store, table, key):
    columns = store['columns'][table]
    index = columns.index(key)
    result = {row[index]: dict(zip(columns, row)) for row in store['tables'][table]}
    require(len(result) == len(store['tables'][table]), 'Duplicate relational key in '+table)
    return result


def compare_history(baseline, current, validate_delta):
    """Raw prefix/schema/metadata proof plus every current==latest event pair.

    validate_delta checks the actual immutable task/publication/proposal triple;
    no whitelist alone suffices to accept a newly appended human revision.
    """
    require(baseline['schema'] == current['schema'] and baseline['columns'] == current['columns'],
            'Gallery schema/columns changed')
    old_tables, new_tables = baseline['tables'], current['tables']
    require(old_tables.keys() == new_tables.keys(), 'Gallery table inventory changed')
    for name in old_tables:
        if name not in ('events', 'reviews', 'sqlite_sequence'):
            require(sorted(old_tables[name]) == sorted(new_tables[name]), 'Gallery metadata changed: '+name)
    event_columns = baseline['columns']['events']
    event_id_index = event_columns.index('id')
    old_raw = sorted(old_tables['events'], key=lambda row: row[event_id_index])
    new_raw = sorted(new_tables['events'], key=lambda row: row[event_id_index])
    require(new_raw[:len(old_raw)] == old_raw, 'Old raw event prefix was rewritten or removed')
    require(len({r[event_id_index] for r in new_raw}) == len(new_raw), 'Duplicate gallery event ID')
    old_current = table_index(baseline, 'reviews', 'target_key')
    new_current = table_index(current, 'reviews', 'target_key')
    histories, latest, validated = defaultdict(list), {}, []
    old_last = max((r[event_id_index] for r in old_raw), default=0)
    for index, raw_event in enumerate(new_raw):
        event = dict(zip(event_columns, raw_event))
        key, revision, payload = event['target_key'], event['revision'], event['payload']
        value = json.loads(payload)
        require(value.get('target_key') == key and type(value.get('revision')) is int
                and type(revision) is int and value['revision'] == revision,
                'Gallery raw payload/relational identity differs')
        require(not histories[key] or revision > histories[key][-1]['revision'],
                'Nonmonotonic/duplicate human event revision')
        if index >= len(old_raw):
            require(event['id'] > old_last, 'New event ID precedes frozen prefix')
            binding = validate_delta(value, event)
            validated.append(dict(event_id=event['id'], target_key=key, revision=revision,
                payload_raw_sha256=raw_sha(payload), payload_canonical_sha256=sha(value),
                polygon_sha256=sha(value['polygon']), updated_at=value.get('updated_at'), **binding))
        histories[key].append(event)
        latest[key] = event
    # All old current rows must still exist *as raw historical payloads*. Their
    # replacement by a later legitimate current revision is explicitly reported.
    for key, row in old_current.items():
        require(key in new_current, 'Old gallery current key was removed')
        require(any(e['revision'] == row['revision'] and e['payload'] == row['payload']
                    for e in histories[key] if e['id'] <= old_last),
                'Frozen baseline current payload is absent from old raw history')
    require(set(new_current) == set(latest), 'Current/latest event key coverage differs')
    for key, row in new_current.items():
        require((row['revision'], row['payload']) == (latest[key]['revision'], latest[key]['payload']),
                'Current is not exact raw latest appended event')
    if 'sqlite_sequence' in old_tables:
        old_seq, new_seq = dict(old_tables['sqlite_sequence']), dict(new_tables['sqlite_sequence'])
        require(old_seq.keys() == new_seq.keys(), 'Sequence inventory changed')
        for name, value in old_seq.items():
            expected = max((r[event_id_index] for r in new_raw), default=0) if name == 'events' else value
            require(new_seq[name] == expected, 'Sequence differs from append-only history')
    changed = {key for key, row in old_current.items() if new_current[key] != row}
    require(changed <= {e['target_key'] for e in validated}, 'Current changed without validated appended event')
    return dict(old_current=old_current, new_current=new_current, histories=histories,
        delta_events=validated, changed_keys=changed,
        summary=dict(baseline_current_records=len(old_current), observed_current_records=len(new_current),
            baseline_events=len(old_raw), observed_events=len(new_raw), appended_events=len(validated),
            appended_distinct_keys=len({r['target_key'] for r in validated}),
            superseded_baseline_current_keys=len(changed),
            baseline_current_raw_payloads_unchanged=not changed,
            baseline_current_raw_payloads_preserved_as_historical_events=True,
            baseline_raw_event_prefix_identical=True, schema_metadata_preserved=True,
            baseline_event_rows_sha256=sha(old_raw), observed_event_rows_sha256=sha(new_raw)))


def frozen_inputs(output):
    """Frozen source proof independent of the intentionally changing live DB."""
    require(digest(SOURCE/'independent_verification.json') == SEAL, 'Frozen independent source seal changed')
    seal, result = read(SOURCE/'independent_verification.json'), read(SOURCE/'result.json')
    require(seal['state'] == 'passed' and digest(SOURCE/'result.json') == seal['result_sha256'],
            'Frozen source result binding changed')
    for name, expected in result['output_artifacts'].items():
        require(digest(SOURCE/name) == expected, 'Frozen source artifact changed: '+name)
    require(file_inventory(PROJECT/'configs') == seal['configuration_sha256'], 'Configuration changed')
    for name, expected in seal['frozen_code_sha256'].items():
        require(digest(PROJECT/name) == expected, 'Old frozen code changed: '+name)
    parent = read(PROJECT/'outputs/pallet_mask_remaining_round5_v1/manifest.json')
    formal, rows, evidence = load_formal_evidence(parent)
    protected = protected_snapshot(formal)
    require(len(protected) == 20768 and protected == read(SOURCE/'private/protected_formal_before.json'),
            'Protected formal inventory changed')
    manifest, records = verify_dataset(output)
    catalog = {r['target_key']: r for r in jsonl(SOURCE/'private/source_catalog.jsonl')}
    snapshots = {kind: database_rows(SOURCE/f'private/{kind}_snapshot.sqlite3', True)
                 for kind in ('pose', 'gallery')}
    for kind, store in snapshots.items():
        require(store['logical_fingerprint'] == seal['source_fingerprints'][kind],
                'Frozen store logical history changed: '+kind)
    source_audit = []
    for record in records:
        key = record['target_key']
        reference = catalog[key]
        require(reference['row']['split'] == record['formal_split'] == 'train'
                and record['polygon'] == reference['polygon']
                and record['original_bbox'] == reference['row']['bbox']
                and record['source_payload_sha256'] == reference['payload_sha256']
                and record['source_event_raw_sha256'] == reference['event_raw_sha256'],
                'Selected training record differs from sealed source catalog: '+key)
        kind = 'pose' if reference['origin'] == 'saved_pose' else 'gallery'
        store, table = snapshots[kind], ('annotations' if kind == 'pose' else 'reviews')
        frozen_current = table_index(store, table, 'target_key')[key]
        require(sha(json.loads(frozen_current['payload'])) == record['source_payload_sha256'],
                'Training source canonical payload differs from frozen current: '+key)
        ec = store['columns']['events']
        matches = [dict(zip(ec, event)) for event in store['tables']['events']
                   if event[ec.index('target_key')] == key
                   and raw_sha(event[ec.index('payload')]) == record['source_event_raw_sha256']]
        require(len(matches) == 1 and json.loads(matches[0]['payload'])['polygon'] == record['polygon'],
                'Frozen training supervision event/polygon binding differs: '+key)
        x1, y1, x2, y2 = record['roi']
        original = raster_mask(record['polygon'], *record['native_size'])
        saved_mask = cv2.imread(str(output/record['native_mask']), cv2.IMREAD_GRAYSCALE) > 0
        require(np.array_equal(original[y1:y2, x1:x2], saved_mask), 'Frozen polygon/native training mask mismatch')
        evidence.image(record['frame_id'])  # source-native file hash, not model inference
        source_audit.append(dict(target_key=key, development_split=record['development_split'],
            frozen_source_revision=frozen_current['revision'],
            frozen_supervision_event_id=matches[0]['id'], source_store=kind,
            frozen_payload_sha256=record['source_payload_sha256'],
            frozen_event_raw_sha256=record['source_event_raw_sha256'], polygon_sha256=sha(record['polygon'])))
    return formal, rows, manifest, records, snapshots, seal, result, source_audit


def event_validator(rows, snapshots, source_result):
    formal_sha = read(PROJECT/'outputs/pallet_mask_remaining_round5_v1/manifest.json')['formal_manifest_sha256']
    publication = publication_inventory({15: PROJECT/'outputs/pallet_mask_remaining_round5_v1'}, formal_sha)[15]
    expected_publication = source_result['source_summary']['publication_inputs']['15']
    require(all(publication[k] == expected_publication[k] for k in
                ('manifest_sha256', 'proposals_sha256', 'verification_sha256')),
            'Publication15 immutable triple changed')
    pose_current = table_index(snapshots['pose'], 'annotations', 'target_key')
    current = {key: json.loads(row['payload']) for key, row in pose_current.items()}
    ec = snapshots['pose']['columns']['events']
    pose_event_keys = {event[ec.index('target_key')] for event in snapshots['pose']['tables']['events']}
    require(digest(TASK_V3/'manifest.json') == TASK_V3_PIN
            and source_result['source_summary']['task_inputs']['outputs/pallet_mask_preannotated_requests_v3']['manifest']
                == TASK_V3_PIN, 'Original v3 task manifest changed')
    old_registry = prelabel_registry(TASK_V3, rows, formal_sha, publication, current, pose_event_keys, 15, 'v3')
    require(len(old_registry) == 26, 'Original v3 has a different task denominator')
    require(digest(SUPPLEMENT/'verification.json') == SUPPLEMENT_PIN, 'New under50 seal changed')
    new_registry = prelabel_registry(SUPPLEMENT, rows, formal_sha, publication, current, pose_event_keys, 15, 'under50_v1')
    require(len(new_registry) == 10, 'New under50 task denominator changed')
    registry = {**old_registry, **new_registry}

    def validate(value, event):
        triple = registry_key(value)
        require(triple in registry, 'New event is not an explicitly sealed old-v3/new-under50 task triple')
        reference = registry[triple]
        require(triple[0] == 15 and sha(reference['proposal']) == triple[1]
                and triple[2] == event['target_key'], 'New event publication/proposal/target triple differs')
        validate_gallery_event(value, rows[event['target_key']], reference, event['revision'])
        return dict(publication=15, base_proposal_sha256=triple[1],
            task_manifest_sha256=reference['task_manifest_sha256'],
            task_verification_sha256=reference['task_verification_sha256'],
            source_binding=value['source_binding'], namespace=reference['namespace'],
            real_human_event_and_false_GT_pose_robot_training_flags_validated=True)

    return validate, old_registry, new_registry


def route_binding(output, route, manifest, records):
    """Only caller-requested completed route; no probing of any other worker."""
    directory = output/route
    completion = read(directory/'completion.json')
    require(completion['state'] == 'completed_internal_research_training_only', 'Completed route required')
    start = read(directory/'worker_start.json')
    require(start['dataset_manifest_sha256'] == digest(output/'dataset_manifest.json'), 'Worker dataset changed')
    for name, expected in start['code_sha256'].items():
        require(digest(directory/'code_snapshot'/Path(name).name) == expected, 'Launch code snapshot changed')
    require(digest(directory/completion['checkpoint']) == completion['checkpoint_sha256'], 'Checkpoint changed')
    selection = read(directory/'frozen_selection.json')
    require(selection['sha256'] == completion['checkpoint_sha256'], 'Frozen selection/checkpoint differs')
    predictions = jsonl(directory/'internal_development_predictions.jsonl')
    dev = {r['target_key'] for r in records if r['development_split'] == 'dev'}
    require({r['target_key'] for r in predictions} == dev and len(predictions) == len(dev)
            and len(dev) == manifest['splits']['dev']
            and all(r['formal_split'] == 'train' and r['development_split'] == 'dev' for r in predictions),
            'Reported dev membership differs from private/training_records.jsonl')
    # Existing M1 CPU verification remains immutable; bind it rather than rewrite.
    original_verification = directory/'verification.json'
    return dict(route=route, completion_sha256=digest(directory/'completion.json'),
        checkpoint_sha256=completion['checkpoint_sha256'], epochs=completion['epochs_trained'],
        frozen_selection_sha256=digest(directory/'frozen_selection.json'),
        internal_metrics_sha256=digest(directory/'internal_development_predictions.jsonl'),
        original_verification_sha256=digest(original_verification) if original_verification.exists() else None,
        internal_quality=completion['internal_quality'], dev_predictions=len(predictions),
        evaluated_reference_version='unchanged frozen source catalog / dataset manifest, NOT latest live revisions',
        new_current_revision_metrics_recomputed=False, automatic_retrain=False)


def audit(output=OUTPUT, audit_output=DEFAULT_AUDIT, route='m1'):
    output, audit_output = Path(output).resolve(), Path(audit_output).resolve()
    require(audit_output.parent == PROJECT/'outputs' and not audit_output.exists(), 'Fresh private audit directory required')
    baseline_artifacts = {str(p.relative_to(output)): digest(p) for p in (
        output/'dataset_manifest.json', output/'private/training_records.jsonl',
        output/'m1/completion.json', output/'m1/verification.json',
        output/'training_handoff.json', output/'training_handoff_v2.json')}
    formal, rows, manifest, records, snapshots, seal, source_result, source_audit = frozen_inputs(output)
    live_pose = database_rows(POSE)
    require(live_pose['logical_fingerprint'] == snapshots['pose']['logical_fingerprint'], 'Pose store changed')
    live_gallery = database_rows(GALLERY)  # one consistent read-only transaction; no status polling
    gallery_observed_at = datetime.now(timezone.utc).isoformat()
    validate, old_registry, new_registry = event_validator(rows, snapshots, source_result)
    history = compare_history(snapshots['gallery'], live_gallery, validate)
    selected = {r['target_key']: r for r in records}  # correct PRIVATE immutable membership
    changed_records = []
    for key in sorted(history['changed_keys']):
        old, latest = history['old_current'][key], history['new_current'][key]
        old_value, latest_value = json.loads(old['payload']), json.loads(latest['payload'])
        row = rows[key]
        membership = selected[key]['development_split'] if key in selected else 'outside_selected_model_dataset'
        metrics = compare_polygons(old_value['polygon'], latest_value['polygon'], row['size'], row['bbox']['height'])
        changed_records.append(dict(target_key=key, class_id=row['final_class_id'], formal_split=row['split'],
            model_dataset_membership=membership, frozen_gallery_revision=old['revision'],
            current_gallery_revision=latest['revision'],
            frozen_payload_raw_sha256=raw_sha(old['payload']), current_payload_raw_sha256=raw_sha(latest['payload']),
            frozen_polygon_sha256=sha(old_value['polygon']), current_polygon_sha256=sha(latest_value['polygon']),
            polygon_changed=old_value['polygon'] != latest_value['polygon'], polygon_delta_metrics=metrics,
            historical_human_revision_difference_not_new_model_prediction=True,
            baseline_payload_still_in_raw_event_prefix=True,
            source_data_were_not_overwritten_by_training=True))
    old_keys = {triple[2] for triple in old_registry}
    new_keys = {triple[2] for triple in new_registry}
    delta = history['delta_events']
    old_v3_delta = [r for r in delta if r['namespace'] == 'v3']
    new_saved = set(history['new_current']) & new_keys
    superseded = [r for r in changed_records if r['model_dataset_membership'] in ('train', 'dev')]
    route_receipt = route_binding(output, route, manifest, records)
    audit_output.mkdir(mode=0o700)
    audit_code = {}
    for path in (Path(__file__), PROJECT/'scripts/verify_mask_model_training_history_v2.py'):
        destination = audit_output/('audit_source_'+path.name)
        with destination.open('xb') as stream:
            stream.write(path.read_bytes())
        destination.chmod(0o600)
        audit_code[str(path.relative_to(PROJECT))] = digest(destination)
    write(audit_output/'frozen_training_source_bindings.jsonl', source_audit, lines=True)
    write(audit_output/'concurrent_human_event_appends.jsonl', delta, lines=True)
    write(audit_output/'superseded_current_references.jsonl', changed_records, lines=True)
    write(audit_output/'trained_or_dev_superseded_references.jsonl', superseded, lines=True)
    require(verify_dataset(output)[0] == manifest, 'Frozen dataset changed during v2 audit')
    require(all(digest(output/name) == expected for name, expected in baseline_artifacts.items()),
            'Original training receipts/dataset/handoffs changed')
    require(file_inventory(PROJECT/'configs') == seal['configuration_sha256']
            and protected_snapshot(formal) == read(SOURCE/'private/protected_formal_before.json'),
            'Configs/formal protection changed during v2 audit')
    require(all(digest(SOURCE/name) == expected for name, expected in source_result['output_artifacts'].items()),
            'Frozen source artifact changed during v2 audit')
    require(all(digest(PROJECT/name) == expected for name, expected in seal['frozen_code_sha256'].items()),
            'Old frozen code changed during v2 audit')
    result = dict(state='passed_frozen_label_version_and_append_history_provenance_only',
        audit_version='concurrent_human_revision_v2_separate_authorization_not_v1_policy_waiver',
        audited_at=datetime.now(timezone.utc).isoformat(), independent_source_seal_sha256=SEAL,
        model_dataset_manifest_sha256=digest(output/'dataset_manifest.json'),
        training_records_path='private/training_records.jsonl',
        training_records_sha256=digest(output/'private/training_records.jsonl'),
        frozen_supervision_records_verified=len(source_audit), train_dev_counts=manifest['splits'],
        frozen_dataset_unchanged=True, frozen_dataset_new_records_added=0,
        frozen_source_catalog_sha256=digest(SOURCE/'private/source_catalog.jsonl'),
        protected_formal_files_verified=20768, configs_and_19_old_frozen_code_unchanged=True,
        source_fingerprints=dict(baseline=seal['source_fingerprints'],
            current=dict(pose=live_pose['logical_fingerprint'], gallery=live_gallery['logical_fingerprint'])),
        pose_store_unchanged=True, gallery_transaction_observed_at=gallery_observed_at,
        current_gallery_entire_fingerprint_unchanged=(
            live_gallery['logical_fingerprint'] == snapshots['gallery']['logical_fingerprint']),
        history=history['summary'],
        old_v3=dict(task_manifest_sha256=TASK_V3_PIN, original_tasks=26,
            appended_events=len(old_v3_delta), appended_distinct_keys=len({r['target_key'] for r in old_v3_delta}),
            latest_saved_original_tasks=len(set(history['new_current']) & old_keys)),
        new_under50=dict(independent_seal_sha256=SUPPLEMENT_PIN, issued=10,
            actually_saved_distinct_targets=len(new_saved), unsaved=10-len(new_saved),
            saved_keys=sorted(new_saved), new_events=sum(r['namespace'] == 'under50_v1' for r in delta),
            old_v3_events_not_counted_as_new_under50_completion=True),
        superseded_model_references=dict(total=len(superseded),
            by_split=dict(Counter(r['model_dataset_membership'] for r in superseded)),
            polygon_changed=sum(r['polygon_changed'] for r in superseded)),
        model_route=route_receipt,
        score_qualification='Scores agree with frozen label version only; superseded references are NOT current truth. '
                            'No dynamic append, numerical training patch, restart, automatic retrain or latest-label rescore.',
        quality_does_not_authorize_GT_production_or_robot=True,
        original_v1_policy_and_receipts_unchanged=True, original_training_artifacts_sha256=baseline_artifacts,
        no_raw_pose_payloads_review_notes_reviewers_or_access_codes_exported=True,
        audit_artifacts_sha256=file_inventory(audit_output), code_sha256=digest(__file__))
    result['audit_code_snapshots_sha256'] = audit_code
    write(audit_output/'verification.json', result)
    print(json.dumps(dict(state=result['state'], history=result['history'],
        superseded_model_references=result['superseded_model_references'],
        new_under50=result['new_under50'], route=route)), flush=True)
    return result
