"""Build an independent, partially supervised pallet pose dataset.

The supplied GT manifest is an index, not a licence to invent 6-DoF labels.
Source tables, ordered XYZ equivalence, target association and old splits are
audited separately. Neither the formal detector dataset nor the live review DB
is written. The source archive is extracted only inside this run's input area.
"""
from __future__ import annotations

import argparse
from collections import Counter, defaultdict
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
import csv
import hashlib
import io
import json
import math
from pathlib import Path, PurePosixPath
import re
import shutil
import stat
import tempfile
from zipfile import ZIP_DEFLATED, ZipFile

import numpy as np

from .cluster_targets import digest, load_dataset, protected_snapshot, target_key
from .data.orientation import orient_grid
from .data.pcd import read_pcd

PROJECT = Path(__file__).resolve().parents[1]
GT_FIELDS = ('x_gt', 'y_gt', 'z_gt', 'theta_d_gt', 'roll_gt', 'pitch_gt', 'yaw_gt')
REQUIRED_POLICY = {
    'overlap': 'sha256_or_exact_ordered_float32_xyz_and_viewpoint',
    'conflicts': 'quarantine_without_automatic_resolution',
    'target_association': 'exactly_one_existing_human_box_only',
    'split': 'inherit_formal_no_resplit_archive_only_unassigned',
    'coordinates': 'preserve_source_values_no_unverified_transform',
    'missing_fields': 'null_with_explicit_supervision_mask_never_zero_fill',
}


def now():
    return datetime.now(timezone.utc).isoformat(timespec='seconds')


def json_bytes(value):
    return (json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2, allow_nan=False)+'\n').encode()


def protected_write(path, raw):
    path = Path(path)
    if path.exists():
        if path.read_bytes() != raw:
            raise ValueError(f'Refusing to overwrite changed output: {path}')
        return
    path.parent.mkdir(parents=True, exist_ok=True)
    temporary = path.with_name(path.name+'.tmp')
    temporary.write_bytes(raw)
    temporary.replace(path)


def write_json(path, value):
    protected_write(path, json_bytes(value))


def write_rows(path, rows):
    raw = ''.join(json.dumps(r, ensure_ascii=False, sort_keys=True, allow_nan=False)+'\n' for r in rows)
    protected_write(path, raw.encode())


def read_rows(path):
    return [json.loads(s) for s in Path(path).read_text().splitlines() if s.strip()]


def relative_name(name):
    value = str(name).replace('\\', '/')
    path = PurePosixPath(value)
    if (not value or path.is_absolute() or '..' in path.parts or ':' in value
            or value != str(path) or '\x00' in value):
        raise ValueError(f'Unsafe or noncanonical path: {name}')
    return value


def safe_path(root, name):
    root = Path(root).resolve()
    path = root/relative_name(name)
    if not path.resolve().is_relative_to(root):
        raise ValueError('Path escapes output root')
    return path


def csv_rows(raw):
    return list(csv.DictReader(io.StringIO(raw.decode('utf-8-sig'))))


def number(value):
    if value is None or isinstance(value, bool):
        return None
    try:
        result = float(str(value).strip())
        return result if math.isfinite(result) else None
    except (ValueError, TypeError):
        return None


def vector(value):
    if value is None:
        return None
    values = [number(s) for s in str(value).strip(' ()[]').split(';')]
    if len(values) != 3 or any(v is None for v in values):
        return None
    return values


def sample_key(value):
    text = str(value)
    m = re.search(r'(\d{2})-(\d{2})-(\d{4})_(\d{2})-(\d{2})-(\d{2})', text)
    if m:
        d, mo, y, h, mi, s = m.groups()
        return 'T:'+y+mo+d+h+mi+s
    m = re.search(r'(\d{4})-(\d{2})-(\d{2})[ T](\d{2}):(\d{2}):(\d{2})', text)
    if m:
        return 'T:'+''.join(m.groups())
    return None


def normalize_header(value):
    return re.sub(r'[^a-z0-9]', '', str(value or '').lower())


def parse_sheet(rows, table, sheet):
    """Only explicit GT columns or the known horizontal GT layout are labels.

    In particular x/y/theta, trans_*, icp_* and unnamed normals in result tables
    are predictions, not extra dimensions of ground truth.
    """
    evidence = []
    header = None
    for i, row in enumerate(rows):
        names = [normalize_header(v) for v in row]
        if 'filename' in names and 'xgt' in names and 'ygt' in names and 'thetadgt' in names:
            header = {k: names.index(k) for k in names if k}
            continue
        if header:
            get = lambda k: row[header[k]] if k in header and header[k] < len(row) else None
            key = sample_key(get('filename'))
            if key is None:
                continue
            values = {k: number(get(normalize_header(k))) for k in GT_FIELDS}
            if any(values[k] is not None for k in GT_FIELDS):
                evidence.append({'sample_key': key, 'table': table, 'sheet': sheet, 'row': i+1,
                    'filename_cell': str(get('filename')), 'values': values,
                    'normal_gt': vector(get('normalgt')), 'kind': 'explicit_gt_columns'})
    if Path(table).name == 'ground_truth.xlsx' and len(rows) >= 6:
        # Sheet columns are samples: category, filename, X, Y, Z, normal.
        for c, cell in enumerate(rows[1]):
            key = sample_key(cell)
            if key is None:
                continue
            values = {k: None for k in GT_FIELDS}
            for r, field, prefix in ((2, 'x_gt', 'X:'), (3, 'y_gt', 'Y:'), (4, 'z_gt', 'Z:')):
                value = str(rows[r][c] if c < len(rows[r]) else '').strip()
                if value.startswith(prefix):
                    values[field] = number(value[len(prefix):])
            normal = vector(rows[5][c] if c < len(rows[5]) else None)
            evidence.append({'sample_key': key, 'table': table, 'sheet': sheet, 'column': c+1,
                'row': 2, 'filename_cell': str(cell), 'values': values, 'normal_gt': normal,
                'kind': 'source_reference_point_and_normal'})
    return evidence


def parse_table(raw, name):
    if name.endswith('.csv'):
        return parse_sheet(list(csv.reader(io.StringIO(raw.decode('utf-8-sig')))), name, 'CSV')
    if name.endswith('.xls'):
        import xlrd
        book = xlrd.open_workbook(file_contents=raw)
        return [item for s in book.sheets() for item in parse_sheet(
            [s.row_values(i) for i in range(s.nrows)], name, s.name)]
    import openpyxl
    book = openpyxl.load_workbook(io.BytesIO(raw), read_only=True, data_only=True)
    try:
        return [item for s in book for item in parse_sheet(list(s.iter_rows(values_only=True)), name, s.title)]
    finally:
        book.close()


def values_close(left, right, fields):
    return all(left.get(k) is not None and right.get(k) is not None
        and abs(left[k]-right[k]) <= (1e-3 if k.endswith(('d_gt', 'roll_gt', 'pitch_gt', 'yaw_gt')) else 1e-4)
        for k in fields)


def label_from_table(row, evidence):
    manifest_values = {k: number(row.get(k)) for k in GT_FIELDS}
    normal = vector(row.get('normal_gt'))
    planar = all(manifest_values[k] is not None for k in ('x_gt', 'y_gt', 'theta_d_gt'))
    kind = 'planar_xy_theta' if planar else 'reference_xyz_normal' if normal else 'no_usable_gt'
    candidates = [e for e in evidence if e['sample_key'] == row['sample_key']]
    issues = []
    correction = None
    values = dict(manifest_values)
    chosen = None
    required = ('x_gt', 'y_gt', 'theta_d_gt') if planar else ('x_gt', 'y_gt', 'z_gt')
    if kind != 'no_usable_gt':
        for item in candidates:
            if planar and values_close(manifest_values, item['values'], required):
                chosen = item
                break
            if normal and item['normal_gt'] is not None and np.allclose(normal, item['normal_gt'], rtol=0, atol=1e-6):
                chosen = item
                break
        if chosen is None and kind == 'reference_xyz_normal':
            # The supplied index flattened one scientific-notation component.
            # The authoritative horizontal GT table is primary; accept its
            # explicit values only when all matching source columns agree.
            raw_candidates = [e for e in candidates if e['kind']=='source_reference_point_and_normal'
                and e['normal_gt'] is not None and .99 <= np.linalg.norm(e['normal_gt']) <= 1.01
                and all(e['values'][k] is not None for k in required)]
            if raw_candidates:
                first=raw_candidates[0]
                consistent=all(values_close(first['values'],e['values'],required)
                    and np.allclose(first['normal_gt'],e['normal_gt'],rtol=0,atol=1e-6)
                    for e in raw_candidates)
                if consistent:
                    chosen=first
                    correction={'field':'normal_gt','manifest_value':normal,
                        'authoritative_table_value':chosen['normal_gt'],
                        'reason':'index_value_differs_from_explicit_authoritative_table',
                        'source_table_values_used_without_estimating_pose':True}
                    normal=chosen['normal_gt'].copy()
        if chosen is None:
            issues.append('manifest_not_verified_in_authoritative_table')
        else:
            # Horizontal source table contains XYZ which the manifest omitted.
            if not planar:
                values = chosen['values'].copy()
            if any(values[k] is None for k in required):
                issues.append('incomplete_source_label')
            for item in candidates:
                if all(item['values'][k] is not None for k in required):
                    if not values_close(values, item['values'], required):
                        issues.append('conflicting_rows_in_authoritative_table')
                    if normal and item['normal_gt'] is not None and not np.allclose(normal,item['normal_gt'],rtol=0,atol=1e-4):
                        issues.append('conflicting_normals_in_authoritative_table')
        if normal and not .99 <= np.linalg.norm(normal) <= 1.01:
            issues.append('invalid_normal_length')
    else:
        issues.append('no_usable_gt')
    if row.get('has_conflict') == 'Y' or int(row.get('n_value_variants') or 0) > 1:
        issues.append('source_gt_conflict_requires_human_resolution')
    result = {'kind': kind, 'values': values, 'normal_gt': normal,
        'supervision_mask': {**{k: v is not None for k, v in values.items()}, 'normal_gt': normal is not None},
        'evidence': chosen, 'issues': sorted(set(issues)),
        'coordinate_frame': 'source_table_native_untransformed',
        'position_unit': 'm_as_source_package', 'angle_unit': 'degree_as_source_package',
        'reference_point_definition': 'source_table_not_assumed_geometric_center',
        'six_dof_complete': False, 'rigid_transform_available': False}
    if correction:result['index_correction']=correction
    return result


def viewpoint(header):
    result = np.asarray(header.get('VIEWPOINT', ['0','0','0','1','0','0','0']), dtype=float)
    if result.shape != (7,) or not np.isfinite(result).all():
        raise ValueError('Invalid PCD VIEWPOINT')
    return result


def compare_clouds(left, right):
    ha, a = read_pcd(left)
    hb, b = read_pcd(right)
    if a.shape != b.shape or not np.array_equal(viewpoint(ha), viewpoint(hb)):
        return None
    if not np.array_equal(a[:, :3], b[:, :3], equal_nan=True):
        return None
    return 'ordered_xyzi_equal' if np.array_equal(a, b, equal_nan=True) else 'ordered_xyz_equal_intensity_differs'


def find_overlap(path, sha, key, by_hash, by_time, checked_hashes):
    exact = by_hash.get(sha, [])
    if len(exact) > 1:
        return {'kind': 'ambiguous_exact_match', 'frame_id': None, 'candidates': [f['frame_id'] for f in exact]}
    if exact:
        candidates = exact
    else:
        candidates = by_time.get(key, [])
    matches = []
    for f in candidates:
        source_path = Path(f['source_paths']['pcd'])
        if str(source_path) not in checked_hashes:
            actual = digest(source_path)
            if actual != f['source_anchor_hash']:
                raise ValueError(f'Formal raw source changed: {source_path}')
            checked_hashes[str(source_path)] = actual
        kind = 'sha256_exact' if exact else compare_clouds(path, source_path)
        if kind:
            matches.append((f, kind))
    if len(matches) == 1:
        f, kind = matches[0]
        return {'kind': kind, 'frame_id': f['frame_id'], 'split': f['split'],
            'group_id': f['group_id'], 'instances': f['instances'],
            'formal_pcd_sha256': f['source_anchor_hash'], 'candidates': [f['frame_id']]}
    return {'kind': 'ambiguous_geometry_match' if matches else 'same_timestamp_not_equivalent' if candidates else 'not_in_formal_dataset',
        'frame_id': None, 'candidates': [f['frame_id'] for f in candidates]}


def extract_inputs(archive, root):
    inventory = []
    with ZipFile(archive) as z:
        names = [i.filename for i in z.infolist() if not i.is_dir()]
        if len(names) != len(set(names)):
            raise ValueError('Duplicate ZIP paths')
        for info in z.infolist():
            if info.is_dir():
                continue
            if stat.S_ISLNK(info.external_attr >> 16):
                raise ValueError('ZIP symlinks are not accepted')
            path = safe_path(root, info.filename)
            # ZipFile.read consumes the whole member and validates its CRC.
            raw = z.read(info)
            if len(raw) != info.file_size:
                raise ValueError('ZIP member size mismatch')
            protected_write(path, raw)
            inventory.append({'member': info.filename, 'bytes': len(raw),
                'sha256': hashlib.sha256(raw).hexdigest(), 'md5': hashlib.md5(raw).hexdigest()})
    return inventory


def inventory_metadata(root, inventory):
    manifests = [r for r in inventory if Path(r['member']).name == 'gt_manifest.csv']
    if len(manifests) != 1:
        raise ValueError('Expected one GT manifest')
    base = str(PurePosixPath(manifests[0]['member']).parent)
    lookup = {r['member']: r for r in inventory}
    load = lambda name: csv_rows(safe_path(root, base+'/'+name).read_bytes())
    rows, clouds, tables = load('gt_manifest.csv'), load('清单_点云.csv'), load('清单_GT表格.csv')
    if len({r['sample_key'] for r in rows}) != len(rows):
        raise ValueError('GT sample keys are not unique')
    for records, field in ((clouds, 'pcd_packed'), (tables, 'table_packed')):
        members = []
        for row in records:
            name = base+'/'+relative_name(row[field])
            actual = lookup[name]
            if int(row['bytes']) != actual['bytes'] or row['md5'].lower() != actual['md5']:
                raise ValueError(f'Archive inventory disagreement: {name}')
            members.append(name)
        if len(members) != len(set(members)):
            raise ValueError('Duplicate inventory path')
    actual_pcd = {r['member'] for r in inventory if r['member'].endswith('.pcd')}
    if actual_pcd != {base+'/'+relative_name(r['pcd_packed']) for r in clouds}:
        raise ValueError('Cloud inventory does not cover the archive')
    for row in rows:
        if not row['pcd_packed']:
            continue
        actual = lookup[base+'/'+relative_name(row['pcd_packed'])]
        if int(row['pcd_bytes']) != actual['bytes'] or not actual['md5'].startswith(row['pcd_md5']):
            raise ValueError('GT manifest PCD identity disagreement')
        if sample_key(row['pcd_packed']) != row['sample_key']:
            raise ValueError('GT manifest timestamp disagreement')
    return base, rows, clouds, tables, lookup


def cloud_metadata(path):
    header, points = read_pcd(path)
    xyz_valid = np.isfinite(points[:, :3]).all(axis=1) & (np.linalg.norm(np.nan_to_num(points[:, :3]),axis=1) > 1e-8)
    return {'header': header, 'point_count': len(points), 'valid_xyz_count': int(xyz_valid.sum()),
        'intensity_present': 'intensity' in [s.lower() for s in header.get('FIELDS',[])],
        'viewpoint': viewpoint(header).tolist(), 'coordinate_unit': 'source_pcd_native_no_rescaling'}


def copy_bound(source, destination, expected):
    raw = Path(source).read_bytes()
    if hashlib.sha256(raw).hexdigest() != expected:
        raise ValueError(f'Source asset changed: {source}')
    protected_write(destination, raw)


def export_example(dataset, inputs, base, row, label, overlap, formal, source, annotation, category):
    fid = source['frame_id'] if source else None
    key = Path(source['image']).stem if source else hashlib.sha256(row['sample_key'].encode()).hexdigest()[:24]
    split = source['split'] if source else 'unassigned'
    prefix = f'{category}/{split}/{key}'
    raw_path = Path(source['source_paths']['pcd']) if source else safe_path(inputs,base+'/'+relative_name(row['pcd_packed']))
    raw_sha = source['source_anchor_hash'] if source else digest(raw_path)
    copy_bound(raw_path, dataset/(prefix+'.pcd'), raw_sha)
    header, raw = read_pcd(raw_path)
    valid = np.isfinite(raw[:, :3]).all(axis=1) & (np.linalg.norm(np.nan_to_num(raw[:,:3]),axis=1) > 1e-8)
    # Unmodified native coordinates and order. A mask handles invalid XYZ/I.
    output = {'points_xyzi': raw, 'valid_xyz': valid, 'valid_intensity': np.isfinite(raw[:,3]),
        'source_point_index': np.arange(len(raw),dtype=np.int32)}
    record = {'sample_id': key, 'sample_key': row['sample_key'], 'capture_time': row['capture_time'],
        'subset': category, 'split': split, 'frame_id': fid,
        'group_id': source['group_id'] if source else 'archive/'+row['capture_time'][:10],
        'source_archive_pcd': row['pcd_packed'], 'pointcloud': prefix+'.pcd', 'pointcloud_sha256': raw_sha,
        'points': prefix+'.npz', 'pose': label,
        'overlap': overlap, 'detection_target_key': None, 'box_xywh': None,
        'target_association': 'source_sample_only_no_existing_box',
        'pointcloud_coordinate_unit': 'source_pcd_native_no_rescaling',
        'pointcloud_scale_to_m': None,
        'coordinate_transform_to_robot': None, 'six_dof_training_eligible': False,
        'source_task_supervision_eligible': category in ('overlap','archive_only'),
        'negative_example': False, 'source_reference_not_assumed_center': True,
        'input_header': header}
    if source:
        w, h = source['size']
        if len(raw) != w*h:
            raise ValueError('Formal cloud differs from native grid')
        indices = orient_grid(np.arange(len(raw),dtype=np.int32).reshape(h,w),source['sensor_policy'])
        output['pixel_to_source_index'] = indices
        record.update(image=prefix+'.png', sensor=prefix+'.sensor.npz', native_size=[w,h],
            pixel_rotation_degrees=source['pixel_rotation_degrees'], original_split=source['split'],
            pointcloud_scale_to_m=(source['sensor_policy'].get('scale_to_mm')/1000
                if source['sensor_policy'].get('scale_to_mm') is not None else None),
            sensor_policy=source['sensor_policy'],
            formal_instances=source['instances'], formal_boxes=annotation['boxes'])
        for field, suffix in (('image','.png'),('sensor','.sensor.npz')):
            copy_bound(safe_path(formal,source[field]),dataset/(prefix+suffix),source['files'][source[field]])
        if category == 'overlap':
            if len(annotation['boxes']) != 1:
                raise ValueError('A pose target cannot be assigned to ambiguous detector boxes')
            record.update(detection_target_key=target_key(fid,0),box_xywh=annotation['boxes'][0],
                target_association='unique_existing_human_box_frame_level_assumption')
    # ZIP timestamps inside NPZ vary; protect an existing resume file by semantic equality.
    npz_path = dataset/(prefix+'.npz')
    if npz_path.exists():
        with np.load(npz_path,allow_pickle=False) as old:
            if set(old.files) != set(output) or any(not np.array_equal(old[k],v,equal_nan=True) for k,v in output.items()):
                raise ValueError('Existing point tensor differs')
    else:
        # A cancelled build must never leave a final-named, truncated NPZ.
        buffer=io.BytesIO()
        np.savez_compressed(buffer,**output)
        protected_write(npz_path,buffer.getvalue())
    write_json(dataset/(prefix+'.json'),record)
    return record


def write_csv(path, records):
    names = ['sample_key','capture_time','label_kind','overlap_kind','frame_id','split','group_id',
        'instances','subset','issues','x_gt','y_gt','z_gt','theta_d_gt','normal_gt','source_table']
    stream = io.StringIO(newline='')
    writer = csv.DictWriter(stream,fieldnames=names)
    writer.writeheader()
    for r in records:
        label, overlap = r['label'], r['overlap']
        writer.writerow({'sample_key':r['sample_key'],'capture_time':r['capture_time'],
            'label_kind':label['kind'],'overlap_kind':overlap['kind'],'frame_id':overlap.get('frame_id'),
            'split':overlap.get('split'),'group_id':overlap.get('group_id'),'instances':overlap.get('instances'),
            'subset':r['subset'],'issues':';'.join(r['issues']),
            **{k:label['values'][k] for k in ('x_gt','y_gt','z_gt','theta_d_gt')},
            'normal_gt':json.dumps(label['normal_gt']),'source_table':r['source_row']['authoritative_table']})
    protected_write(path,('\ufeff'+stream.getvalue()).encode())


def verify(dataset, archive=None):
    dataset = Path(dataset).resolve()
    manifest = json.loads((dataset/'manifest.json').read_text())
    files = manifest['files']
    for relative, expected in files.items():
        if digest(safe_path(dataset,relative)) != expected:
            raise ValueError(f'Dataset checksum mismatch: {relative}')
    actual = {str(p.relative_to(dataset)) for p in dataset.rglob('*') if p.is_file()}
    if actual != set(files)|{'manifest.json','checksums.sha256'}:
        raise ValueError('Unexpected or missing dataset files')
    expected_sums = ''.join(f'{h}  {name}\n' for name,h in sorted({**files,'manifest.json':digest(dataset/'manifest.json')}.items()))
    if (dataset/'checksums.sha256').read_text() != expected_sums:
        raise ValueError('Checksum list differs from manifest')
    rows = read_rows(dataset/'annotations.jsonl')
    assigned = [r for r in rows if r['subset']=='overlap']
    if len({r['sample_id'] for r in rows}) != len(rows) or len({r['pointcloud_sha256'] for r in rows}) != len(rows):
        raise ValueError('Duplicate sample or pointcloud')
    groups = defaultdict(set)
    source_hashes = defaultdict(set)
    for r in rows:
        label = r['pose']
        if label['issues'] or label['six_dof_complete'] or r['six_dof_training_eligible']:
            raise ValueError('Invalid eligibility')
        if r['subset']=='archive_only' and r['split']!='unassigned':
            raise ValueError('Archive-only samples must not get unverified splits')
        if r['subset']=='overlap':
            if r['original_split']!=r['split'] or r['formal_instances']!=1 or not r['box_xywh']:
                raise ValueError('Formal split/association changed')
            groups[r['group_id']].add(r['split'])
            source_hashes[r['pointcloud_sha256']].add(r['split'])
        header, raw = read_pcd(safe_path(dataset,r['pointcloud']))
        with np.load(safe_path(dataset,r['points']),allow_pickle=False) as tensor:
            if not np.array_equal(tensor['points_xyzi'],raw,equal_nan=True):
                raise ValueError('Point tensor differs from PCD')
            valid=np.isfinite(raw[:,:3]).all(axis=1)&(np.linalg.norm(np.nan_to_num(raw[:,:3]),axis=1)>1e-8)
            if not np.array_equal(valid,tensor['valid_xyz']) or not np.array_equal(tensor['source_point_index'],np.arange(len(raw))):
                raise ValueError('Point validity or order changed')
            if r['subset']=='overlap':
                w,h=r['native_size']
                index=orient_grid(np.arange(len(raw),dtype=np.int32).reshape(h,w),r['sensor_policy'])
                if not np.array_equal(index,tensor['pixel_to_source_index']):
                    raise ValueError('Registered pixels do not map back to native PCD')
        if header != r['input_header']:
            raise ValueError('PCD header changed')
        if r['pose']['kind']=='planar_xy_theta' and not all(r['pose']['supervision_mask'][k] for k in ('x_gt','y_gt','theta_d_gt')):
            raise ValueError('Missing planar label')
    if any(len(v)>1 for v in groups.values()) or any(len(v)>1 for v in source_hashes.values()):
        raise ValueError('Cross-split source/group leakage')
    if archive:
        with ZipFile(archive) as z:
            names={n for n in z.namelist() if not n.endswith('/')}
            if names!=actual or len(names)!=len(z.namelist()):
                raise ValueError('Archive member set differs')
            for name in sorted(names):
                raw=z.read(name)
                expected=digest(dataset/name) if name in ('manifest.json','checksums.sha256') else files[name]
                if hashlib.sha256(raw).hexdigest()!=expected:
                    raise ValueError('Archive member hash differs')
    return {'state':'passed','files_verified':len(actual),'examples_verified':len(rows),
        'overlap_examples':len(assigned),'cross_split_group_leakage':0,'cross_split_source_leakage':0,
        'native_pcd_tensor_and_pixel_mapping_verified':True,'archive_verified':bool(archive)}


def build(config_path, resume=False):
    config_path=Path(config_path).resolve()
    config=json.loads(config_path.read_text())
    if config['version']!=1 or config['policy']!=REQUIRED_POLICY:
        raise ValueError('Unsupported pose dataset policy')
    archive=(PROJECT/config['archive']).resolve()
    formal_config=(PROJECT/config['formal_dataset_config']).resolve()
    formal=Path(json.loads(formal_config.read_text())['dataset']).resolve()
    out=(PROJECT/config['output']).resolve()
    if out==formal or out.is_relative_to(formal) or formal.is_relative_to(out):
        raise ValueError('Pose output must be separate from formal data')
    binding={'config_sha256':digest(config_path),'archive_sha256':digest(archive),
        'formal_config_sha256':digest(formal_config),'formal_manifest_sha256':digest(formal/'manifest.json'),
        'code_sha256':digest(__file__),'pcd_reader_sha256':digest(Path(__file__).parent/'data/pcd.py'),
        'orientation_code_sha256':digest(Path(__file__).parent/'data/orientation.py')}
    run_path=out/'run_manifest.json'
    if run_path.exists():
        run=json.loads(run_path.read_text())
        if not resume or run['binding']!=binding:
            raise ValueError('Existing run: use --resume with identical inputs/code, or a new output version')
    else:
        if out.exists() and any(out.iterdir()):
            raise ValueError('Output is not empty')
        out.mkdir(parents=True,exist_ok=True)
        if shutil.disk_usage(out).free<config['minimum_free_gib']*1024**3:
            raise ValueError('Insufficient free space')
        snapshot=protected_snapshot(formal)
        frozen=json.loads((formal/'manifest.json').read_text())['files']
        if any(frozen.get(p,h)!=h for p,h in snapshot.items() if p!='manifest.json'):
            raise ValueError('Formal labels or metadata differ from frozen manifest')
        run={'kind':'partial_pallet_pose_dataset','created_at':now(),'binding':binding,
            'protected_formal_files':snapshot,'archive':str(archive),'formal_dataset':str(formal)}
        write_json(run_path,run)
    if protected_snapshot(formal)!=run['protected_formal_files']:
        raise ValueError('Protected formal data changed')
    inputs=out/'inputs'
    inventory=extract_inputs(archive,inputs)
    base,rows,clouds,tables,lookup=inventory_metadata(inputs,inventory)
    print(json.dumps({'stage':'archive_verified','files':len(inventory),'gt_rows':len(rows),'clouds':len(clouds),'tables':len(tables)}),flush=True)
    dataset=out/'dataset'
    dataset.mkdir(exist_ok=True)
    evidence_by_table={}
    for table in tables:
        member=base+'/'+relative_name(table['table_packed'])
        raw=safe_path(inputs,member).read_bytes()
        name=relative_name(table['table_packed'])[len('GT表格/'):]
        evidence_by_table[name]=parse_table(raw,name)
        protected_write(dataset/'provenance'/member,raw)
    for name in ('README.md','gt_manifest.csv','清单_点云.csv','清单_GT表格.csv'):
        protected_write(dataset/'provenance'/base/name,safe_path(inputs,base+'/'+name).read_bytes())
    annotations,sources=load_dataset(formal)
    annotations={r['frame_id']:r for r in annotations}
    by_hash,by_time=defaultdict(list),defaultdict(list)
    for f in sources.values():
        by_hash[f['source_anchor_hash']].append(f)
        key=sample_key(Path(f['source_paths']['pcd']).name)
        if key:by_time[key].append(f)
    checked_hashes={}
    cloud_audit=[]
    # Full archive identity audit: same-named cropped/segmented files are not matches.
    for c in clouds:
        member=base+'/'+relative_name(c['pcd_packed'])
        sha=lookup[member]['sha256']
        exact=by_hash.get(sha,[])
        cloud_audit.append({**c,'sha256':sha,'exact_formal_frame_ids':[f['frame_id'] for f in exact],
            'same_timestamp_formal_frame_ids':[f['frame_id'] for f in by_time.get(c['sample_key'],[])],
            'content_match':bool(exact),
            'derived_path_hint':bool(re.search(r'(?:_seg|_normal|_crop|_face|_filtered|_label)(?:/|\.|$)|/(?:seg|normal)/',relative_name(c['pcd_packed']))),
            'declared_is_derived':c['is_derived_output']=='Y'})
    audits,examples=[],[]
    for i,row in enumerate(rows):
        table=relative_name(row['authoritative_table']) if row['authoritative_table'] else ''
        label=label_from_table(row,evidence_by_table.get(table,[]))
        overlap={'kind':'missing_pcd','frame_id':None,'candidates':by_time.get(row['sample_key'],[]) and [f['frame_id'] for f in by_time[row['sample_key']]] or []}
        meta=None
        if row['pcd_packed']:
            member=base+'/'+relative_name(row['pcd_packed'])
            path=safe_path(inputs,member)
            overlap=find_overlap(path,lookup[member]['sha256'],row['sample_key'],by_hash,by_time,checked_hashes)
            meta=cloud_metadata(path)
        f=sources.get(overlap.get('frame_id'))
        issues=list(label['issues'])
        if not row['pcd_packed']:issues.append('missing_pcd')
        if meta and not meta['valid_xyz_count']:issues.append('no_valid_xyz_points')
        if f and f['instances']!=1:issues.append('ambiguous_multiple_boxes' if f['instances']>1 else 'formal_negative_conflicts_with_pose')
        if overlap['kind'] in ('same_timestamp_not_equivalent','ambiguous_exact_match','ambiguous_geometry_match'):
            issues.append('sample_identity_not_proven')
        subset='quarantine' if issues else 'overlap' if f else 'archive_only'
        record={'sample_key':row['sample_key'],'capture_time':row['capture_time'],
            'source_row':row,'label':label,'overlap':overlap,'cloud':meta,'subset':subset,'issues':sorted(set(issues))}
        audits.append(record)
        if not issues:
            example=export_example(dataset,inputs,base,row,label,overlap,formal,f,
                annotations.get(f['frame_id']) if f else None,subset)
            examples.append(example)
        if (i+1)%50==0:print(json.dumps({'stage':'matched','samples':i+1,'examples':len(examples)}),flush=True)
    summary={'archive_members':len(inventory),'pointcloud_files':len(clouds),
        'declared_raw_clouds':sum(c['is_derived_output']=='N' for c in clouds),
        'declared_derived_clouds':sum(c['is_derived_output']=='Y' for c in clouds),
        'source_gt_tables':len(tables),'gt_manifest_samples':len(rows),
        'source_label_kinds':dict(Counter(a['label']['kind'] for a in audits)),
        'all_cloud_exact_overlap_files':sum(a['content_match'] for a in cloud_audit),
        'all_cloud_exact_unique_formal_frames':len({f for a in cloud_audit for f in a['exact_formal_frame_ids']}),
        'manifest_overlap_kinds':dict(Counter(a['overlap']['kind'] for a in audits)),
        'manifest_verified_overlapping_samples':sum(bool(a['overlap'].get('frame_id')) for a in audits),
        'subsets':dict(Counter(a['subset'] for a in audits)),
        'quarantine_reasons':dict(Counter(k for a in audits for k in a['issues'])),
        'exported_examples':len(examples),
        'overlap_split_counts':dict(Counter(r['split'] for r in examples if r['subset']=='overlap')),
        'overlap_task_split_counts':{kind:dict(Counter(r['split'] for r in examples if r['subset']=='overlap' and r['pose']['kind']==kind))
            for kind in ('planar_xy_theta','reference_xyz_normal')},
        'archive_only_task_counts':dict(Counter(r['pose']['kind'] for r in examples if r['subset']=='archive_only')),
        'authoritative_table_index_corrections':sum('index_correction' in a['label'] for a in audits),
        'six_dof_examples':0,'normalization_fitted':False,'training_started':False,
        'original_detection_labels_modified':False,'human_review_database_accessed':False}
    write_rows(dataset/'annotations.jsonl',examples)
    for subset in ('overlap','archive_only'):
        for kind in ('planar_xy_theta','reference_xyz_normal'):
            for split in (('train','val','test') if subset=='overlap' else ('unassigned',)):
                write_rows(dataset/f'indices/{subset}/{kind}/{split}.jsonl',
                    [r for r in examples if r['subset']==subset and r['pose']['kind']==kind and r['split']==split])
    write_rows(dataset/'audit/samples.jsonl',audits)
    write_rows(dataset/'audit/cloud_inventory.jsonl',cloud_audit)
    write_rows(dataset/'audit/quarantine.jsonl',[a for a in audits if a['issues']])
    write_rows(dataset/'audit/table_evidence.jsonl',[e for rows_ in evidence_by_table.values() for e in rows_])
    write_json(dataset/'audit/archive_inventory.json',inventory)
    write_csv(dataset/'overlap_and_quality.csv',audits)
    write_json(dataset/'statistics.json',summary)
    write_json(dataset/'provenance/run_manifest.json',run)
    protected_write(dataset/'README_zh.md',readme(summary,out.name).encode())
    paths=sorted(p for p in dataset.rglob('*') if p.is_file() and p.name not in ('manifest.json','checksums.sha256'))
    with ThreadPoolExecutor(max_workers=8) as pool:hashes=list(pool.map(digest,paths))
    files={str(p.relative_to(dataset)):h for p,h in zip(paths,hashes)}
    write_json(dataset/'manifest.json',{'kind':'partial_pallet_pose_dataset','version':1,'state':'frozen',
        'revision':out.name,'created_at':run['created_at'],'binding':binding,'policy':config['policy'],'summary':summary,'files':files})
    sums={**files,'manifest.json':digest(dataset/'manifest.json')}
    protected_write(dataset/'checksums.sha256',''.join(f'{h}  {name}\n' for name,h in sorted(sums.items())).encode())
    package=out/(out.name+'.zip')
    if not package.exists():
        temporary=package.with_suffix('.zip.tmp')
        with ZipFile(temporary,'w',ZIP_DEFLATED,compresslevel=6) as z:
            for p in sorted(dataset.rglob('*')):
                if p.is_file():z.write(p,str(p.relative_to(dataset)))
        temporary.replace(package)
    result=verify(dataset,package)
    if protected_snapshot(formal)!=run['protected_formal_files']:
        raise ValueError('Protected formal labels changed during build')
    for source_path,expected in checked_hashes.items():
        if digest(source_path)!=expected:raise ValueError('Used raw source changed')
    result.update(protected_formal_files_unchanged=len(run['protected_formal_files']),used_formal_pcds_unchanged=len(checked_hashes))
    write_json(out/'verification.json',result)
    delivery={'state':'complete','created_at':run['created_at'],'dataset':str(dataset),'archive':str(package),
        'archive_sha256':digest(package),'archive_bytes':package.stat().st_size,
        'dataset_manifest_sha256':digest(dataset/'manifest.json'),'summary':summary,'verification':result}
    write_json(out/'delivery_manifest.json',delivery)
    print(json.dumps(delivery,ensure_ascii=False),flush=True)
    return delivery


def readme(summary,revision='pallet_pose_dataset_v2'):
    return f'''# 托盘位姿数据集（{revision}；部分位姿监督）

来源：2026_10_17.zip；原检测集保持不变。本包独立构建，不训练模型、不修改当前活动检测数据入口。

## 数量

```json
{json.dumps(summary,ensure_ascii=False,indent=2)}
```

## 使用与边界

- `indices/overlap/planar_xy_theta/train|val|test.jsonl`：与正式检测集核验重叠、唯一人工框、无冲突的x/y/theta监督。
- `indices/overlap/reference_xyz_normal/`：原表中的参考点XYZ和法向量监督；与平面位姿分开使用。法向量不能确定绕法线的旋转，不能当完整6D姿态。
- `indices/archive_only/`：有明确GT但未证实属于当前正式集的扩展样本。无检测框、split=unassigned；未确认完整目标/采集组划分前，不自动进入训练或测试。
- `audit/quarantine.jsonl`：全部排除原因、冲突原值和来源引用。无GT不等于负例，多框不擅自绑定其中一框，缺失PCD不猜测对应文件。
- 每例有原生PCD、float32 `points_xyzi`（不删点/不改变顺序）、有效掩码和源点索引；重叠例另有原PNG、四通道sensor NPZ和 `pixel_to_source_index`。原PNG可经过180°方向校正；PCD未旋转，映射显式记录，不给GT角度擅加180°。
- 标签见每例JSON及 `annotations.jsonl`，缺失维度为null，`supervision_mask`控制损失。缺少z/roll/pitch时不填0；不导出虚构四元数/刚体变换或YOLO人体关键点标签。
- 权威原表优先于派生CSV索引；索引的法向量科学计数法损坏时，只在原表同一样本明确数值无歧义且各列一致时恢复，并记录index_correction与索引原值。真实GT冲突不自动裁决。
- `x_gt/y_gt/theta_d_gt`仅来自明确GT列，不读取x/y/theta、ICP输出或theta_d_convert_gt作为真值。单位按来源包为m/degree；点云原单位保留，只有原正式政策已有核验时才给scale_to_m。
- 参考系、目标参考点和角度方向保持来源语义，尚无统一传感器→机器人外参，不声称所有表已统一到同一个机器人坐标系，也不默认位置就是几何中心。用于跨域绝对位姿前须补充标定和定义。
- 原划分完整继承，没有为凑比例把val/test迁入train；只有一个框时按帧级唯一目标关联，仍需注意人工GT是否针对该目标的物理参考点。
- archive中的同名点云包含裁剪/分割/强度修改版本；只有SHA-256相同或逐点有序XYZ与VIEWPOINT完全相同才认定重叠。强度不同时保留正式集输入，记录差异，不覆盖输入。

## 核验

`sha256sum -c checksums.sha256`；或 `.venv/bin/python -m pallet.pose_dataset verify --dataset 解压目录`。
完整输入/逐表逐行证据见provenance和audit。CSV可直接在Excel中查看重叠和排除清单。
'''


def main():
    parser=argparse.ArgumentParser(description=__doc__)
    sub=parser.add_subparsers(dest='command',required=True)
    p=sub.add_parser('build');p.add_argument('--config',type=Path,default=PROJECT/'configs/pallet_pose_dataset_v2.json')
    p.add_argument('--resume',action='store_true')
    p=sub.add_parser('verify');p.add_argument('--dataset',type=Path,required=True);p.add_argument('--archive',type=Path)
    args=parser.parse_args()
    if args.command=='build':build(args.config,args.resume)
    else:print(json.dumps(verify(args.dataset,args.archive),ensure_ascii=False))


if __name__=='__main__':main()
