"""Independent semantic checks against the source archive and frozen detector.

Unlike checksum-only verification this re-reads GT tables, maps every exported
sample to its audit decision and verifies that no unlabelled/conflicted frame
or ambiguous detector target entered the supervised indices.
"""
from __future__ import annotations

import argparse
from collections import Counter, defaultdict
import hashlib
from importlib.metadata import PackageNotFoundError, version
import json
from pathlib import Path
import sys
from zipfile import ZipFile

import numpy as np

PROJECT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(PROJECT))
from pallet.pose_dataset import (compare_clouds, csv_rows, digest, label_from_table,
    parse_table, protected_snapshot, read_rows, relative_name, safe_path, verify,
    write_json)
from pallet.data.orientation import orient_grid
from pallet.data.pcd import read_pcd


def package_version(name):
    try:return version(name)
    except PackageNotFoundError:return 'not_installed_optional'


def check(output,archive):
    output=Path(output).resolve()
    archive=Path(archive).resolve()
    dataset=output/'dataset'
    delivery=json.loads((output/'delivery_manifest.json').read_text())
    run=json.loads((output/'run_manifest.json').read_text())
    if digest(PROJECT/'pallet/pose_dataset.py')!=run['binding']['code_sha256']:
        raise ValueError('Dataset builder source differs from frozen binding')
    if digest(archive)!=run['binding']['archive_sha256']:
        raise ValueError('Source archive differs')
    if digest(delivery['archive'])!=delivery['archive_sha256']:
        raise ValueError('Delivery archive differs')
    formal=Path(run['formal_dataset'])
    formal_sources={r['frame_id']:r for r in read_rows(formal/'frames.jsonl')}
    formal_annotations={r['frame_id']:r for r in read_rows(formal/'annotations/frames.jsonl')}
    result=verify(dataset,delivery['archive'])
    if protected_snapshot(formal)!=run['protected_formal_files']:
        raise ValueError('Formal dataset was modified')
    audits=read_rows(dataset/'audit/samples.jsonl')
    exported=read_rows(dataset/'annotations.jsonl')
    bykey={r['sample_key']:r for r in audits}
    if len(bykey)!=len(audits) or len({r['sample_key'] for r in exported})!=len(exported):
        raise ValueError('Duplicate sample key')
    usable={r['sample_key'] for r in audits if not r['issues']}
    if usable!={r['sample_key'] for r in exported}:
        raise ValueError('Usable samples were lost or quarantine samples were exported')
    lookup={r['member']:r for r in json.loads((dataset/'audit/archive_inventory.json').read_text())}
    table_records={}
    unindexed=[]
    cloud_inventory=read_rows(dataset/'audit/cloud_inventory.jsonl')
    exact_frames={fid for cloud in cloud_inventory for fid in cloud['exact_formal_frame_ids']}
    primary_frames={r['overlap']['frame_id'] for r in audits if r['overlap'].get('frame_id')}
    alias_only=[]
    for audit in audits:
        if audit['overlap'].get('frame_id'):
            continue
        aliases=[cloud for cloud in cloud_inventory if cloud['sample_key']==audit['sample_key']
            and cloud['exact_formal_frame_ids']]
        if aliases:
            alias_only.append({'sample_key':audit['sample_key'],
                'primary_pcd':audit['source_row']['pcd_packed'],
                'primary_comparison':audit['overlap']['kind'],
                'exact_original_pcds':[c['pcd_packed'] for c in aliases],
                'exact_formal_frame_ids':sorted({f for c in aliases for f in c['exact_formal_frame_ids']}),
                'pose_assignment':'not_transferred_from_cropped_or_transformed_cloud'})
    with ZipFile(archive) as z:
        if len(z.namelist())!=len(lookup) or set(z.namelist())!=set(lookup):
            raise ValueError('Archive inventory member mismatch')
        for name,expected in lookup.items():
            raw=z.read(name)
            if len(raw)!=expected['bytes'] or hashlib.md5(raw).hexdigest()!=expected['md5'] or hashlib.sha256(raw).hexdigest()!=expected['sha256']:
                raise ValueError('Archive CRC/hash/size mismatch')
            if '/GT表格/' in name:
                table=name.split('/GT表格/',1)[1]
                table_records[table]=parse_table(raw,table)
                if raw!=safe_path(dataset,'provenance/'+name).read_bytes():
                    raise ValueError('Archived table copy differs')
        base=next(name.rsplit('/',1)[0] for name in lookup if name.endswith('/gt_manifest.csv'))
        original=csv_rows(z.read(base+'/gt_manifest.csv'))
        if {r['sample_key'] for r in original}!=set(bykey):
            raise ValueError('GT manifest coverage differs')
        for row in original:
            record=bykey[row['sample_key']]
            if record['source_row']!=row:
                raise ValueError('Original GT row changed')
            table=relative_name(row['authoritative_table']) if row['authoritative_table'] else ''
            if label_from_table(row,table_records.get(table,[]))!=record['label']:
                raise ValueError('Re-parsed source GT differs')
        for table,ev in table_records.items():
            # Diagnostic only: source tables contain many repeated historical rows.
            for e in ev:
                if e['kind']=='source_reference_point_and_normal' and e['sample_key'] not in bykey:
                    unindexed.append(e)
    pixel_checks=0
    geometry=defaultdict(list)
    from PIL import Image
    for example in exported:
        audit=bykey[example['sample_key']]
        if example['pose']!=audit['label'] or example['pose']['issues']:
            raise ValueError('Supervised label differs from audited source')
        label_path=Path(example['pointcloud']).with_suffix('.json')
        if json.loads((dataset/label_path).read_text())!=example:
            raise ValueError('Per-sample JSON differs from annotation index')
        header,raw=read_pcd(dataset/example['pointcloud'])
        xyz=raw[:,:3].copy()
        xyz[xyz==0]=0  # Canonicalize signed zero and NaN bit representations.
        xyz[np.isnan(xyz)]=np.nan
        origin=np.asarray(header.get('VIEWPOINT',['0','0','0','1','0','0','0']),dtype=np.float64)
        origin[origin==0]=0
        geometry_hash=hashlib.sha256(np.ascontiguousarray(xyz).tobytes()
            +origin.tobytes()).hexdigest()
        geometry[geometry_hash].append({'sample_key':example['sample_key'],
            'subset':example['subset'],'split':example['split']})
        if example['subset']=='archive_only':
            if example['split']!='unassigned' or example['frame_id'] or example['box_xywh']:
                raise ValueError('Archive-only samples acquired invented detection/split labels')
            continue
        f=formal_sources[example['frame_id']]
        ann=formal_annotations[example['frame_id']]
        if len(ann['boxes'])!=1 or example['box_xywh']!=ann['boxes'][0] or example['group_id']!=f['group_id'] or example['split']!=f['split']:
            raise ValueError('Detection identity/box/group/split changed')
        if digest(dataset/example['image'])!=f['files'][f['image']] or digest(dataset/example['sensor'])!=f['files'][f['sensor']]:
            raise ValueError('Formal image/sensor copy differs')
        packed=safe_path(output/'inputs',base+'/'+relative_name(audit['source_row']['pcd_packed']))
        if audit['overlap']['kind']=='sha256_exact':
            if digest(packed)!=example['pointcloud_sha256']:
                raise ValueError('False exact PCD match')
        elif compare_clouds(packed,dataset/example['pointcloud'])!=audit['overlap']['kind']:
            raise ValueError('False geometric match')
        h,w=ann['height'],ann['width']
        registered=orient_grid(raw.reshape(h,w,4),f['sensor_policy'])
        with np.load(dataset/example['points'],allow_pickle=False) as points:
            if not np.array_equal(raw[points['pixel_to_source_index']],registered,equal_nan=True):
                raise ValueError('Pixel/source mapping changed XYZ/I')
        with np.load(dataset/example['sensor'],allow_pickle=False) as tensor:
            native=tensor['frame']
            valid=native[...,3]>0
            if native.shape!=(h,w,4) or not np.isfinite(native).all():
                raise ValueError('Invalid formal sensor tensor')
            if not np.allclose(native[...,1][valid],registered[...,3][valid],rtol=1e-6,atol=1e-6):
                raise ValueError('Native raw intensity is not aligned with formal tensor')
        with Image.open(dataset/example['image']) as image:
            if image.size!=(w,h):raise ValueError('Image dimensions differ')
        pixel_checks+=1
    geometry_duplicates=[samples for samples in geometry.values() if len(samples)>1]
    for samples in geometry_duplicates:
        splits={r['split'] for r in samples if r['subset']=='overlap'}
        if len(splits)>1:
            raise ValueError('Exact XYZ geometry leaks across supervised splits')
    for subset in ('overlap','archive_only'):
        for kind in ('planar_xy_theta','reference_xyz_normal'):
            for split in (('train','val','test') if subset=='overlap' else ('unassigned',)):
                path=dataset/f'indices/{subset}/{kind}/{split}.jsonl'
                expected=[r for r in exported if r['subset']==subset and r['pose']['kind']==kind and r['split']==split]
                if read_rows(path)!=expected:raise ValueError('Training index is incomplete or contaminated')
    if protected_snapshot(formal)!=run['protected_formal_files']:
        raise ValueError('Formal labels changed during verification')
    result.update(source_archive_all_members_verified=len(lookup),
        source_gt_tables_reparsed=len(table_records),original_gt_rows_rechecked=len(original),
        unique_detection_target_and_native_pixels_verified=pixel_checks,
        task_indices_exact_and_quarantine_disjoint=True,formal_detection_labels_unchanged=True,
        unindexed_horizontal_gt_labels=unindexed,
        archive_wide_unique_overlapping_formal_frames=len(exact_frames|primary_frames),
        full_file_exact_unique_frames=len(exact_frames),
        ordered_xyz_extra_unique_frames=len(primary_frames-exact_frames),
        exported_ordered_xyz_duplicate_groups=geometry_duplicates,
        cross_split_ordered_xyz_leakage=0,
        exact_originals_present_but_primary_is_different=alias_only,
        reference_point_definition_not_assumed_center=True,full_6d_pose_available=False,
        subset_counts=dict(Counter(r['subset'] for r in exported)),
        verifier_source_sha256=digest(__file__),
        packages={name:package_version(name) for name in ('numpy','openpyxl','xlrd')},
        human_review_database_accessed=False,training_started=False)
    write_json(output/'independent_verification.json',result)
    return result


def main():
    parser=argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--output',type=Path,default=PROJECT/'outputs/pallet_pose_dataset_v2')
    parser.add_argument('--archive',type=Path,default=PROJECT/'2026_10_17.zip')
    args=parser.parse_args()
    print(json.dumps(check(args.output,args.archive),ensure_ascii=False))


if __name__=='__main__':main()
