"""Immutable dataset identity, continuous-session splits and pilot sampling."""
from __future__ import annotations

import argparse
from collections import Counter, defaultdict, deque
from dataclasses import replace
import hashlib
import json
from pathlib import Path
import random
import shutil

from .dedup import hash_sensor_files, unify_duplicate_groups
from .manifest import FrameRecord, canonical_records, scan_dataset
from .sensors import attach_registry, build_registry


def file_digest(path):
    digest = hashlib.sha256()
    with Path(path).open('rb') as stream:
        for chunk in iter(lambda: stream.read(4 * 1024 * 1024), b''):
            digest.update(chunk)
    return digest.hexdigest()


def write_json(path, value):
    path = Path(path)
    path.parent.mkdir(parents=True, exist_ok=True)
    temp = path.with_name(path.name + '.tmp')
    temp.write_text(json.dumps(value, indent=2, ensure_ascii=False))
    temp.replace(path)


def grouped_assignments(records, seed=42):
    groups, domain_groups = defaultdict(list), defaultdict(set)
    for row in records:
        groups[row.group_id].append(row)
        domain_groups[row.domain].add(row.group_id)
    target_only = {domain for domain, members in domain_groups.items() if len(members) < 3}
    held_groups = set().union(*(domain_groups[d] for d in target_only)) if target_only else set()
    main = [r for r in records if r.group_id not in held_groups]
    totals = Counter(r.domain for r in main)
    fractions = {'train': .7, 'val': .15, 'test': .15}
    counts = {s: Counter() for s in fractions}
    all_counts = Counter()
    ordered = list(groups)
    random.Random(seed).shuffle(ordered)
    ordered.sort(key=lambda g: -len(groups[g]))
    assignments = {}
    for group in ordered:
        if group in held_groups:
            assignments[group] = 'test'
            continue
        bucket = Counter(r.domain for r in groups[group])
        size = len(groups[group])
        def cost(split):
            target = max(1., len(main) * fractions[split])
            change = ((all_counts[split] + size - target)**2 - (all_counts[split]-target)**2) / target
            for domain, number in bucket.items():
                domain_target = max(1., totals[domain] * fractions[split])
                change += ((counts[split][domain] + number - domain_target)**2
                           - (counts[split][domain]-domain_target)**2) / domain_target
            return change
        split = min(fractions, key=cost)
        assignments[group] = split
        all_counts[split] += size
        counts[split].update(bucket)
    return assignments, sorted(target_only)


def balanced_subset(records, assignments, split, count, seed=42):
    """Round-robin domains and sessions; do not fill the pilot with adjacent frames."""
    rng = random.Random(seed)
    by_domain = defaultdict(lambda: defaultdict(list))
    for row in records:
        if assignments[row.group_id] == split:
            by_domain[row.domain][row.group_id].append(row.frame_id)
    domains = {}
    for domain, groups in sorted(by_domain.items()):
        shuffled = []
        for group, frames in sorted(groups.items()):
            rng.shuffle(frames)
            shuffled.append(deque(frames))
        rng.shuffle(shuffled)
        domains[domain] = deque(shuffled)
    result = []
    while len(result) < count and domains:
        for domain in list(domains):
            groups = domains[domain]
            group = groups.popleft()
            result.append(group.popleft())
            if group:
                groups.append(group)
            if not groups:
                del domains[domain]
            if len(result) == count:
                break
    return result


def validate_splits(records, assignments):
    if set(assignments) != {r.group_id for r in records}:
        raise ValueError('split manifest and source groups differ')
    digests = defaultdict(set)
    for record in records:
        if record.source_sha256:
            digests[record.source_sha256].add(assignments[record.group_id])
    if any(len(splits) > 1 for splits in digests.values()):
        raise ValueError('identical primary source spans train/val/test')


def build_protocol(root, output, *, registry_path=None, old_hash_index='outputs/content_hashes.json'):
    output = Path(output).resolve()
    if (output / 'protocol.json').exists():
        return load_protocol(output)[2]
    output.mkdir(parents=True, exist_ok=True)
    records = canonical_records(scan_dataset(root))
    hashes = output / 'content_hashes.json'
    if not hashes.exists() and Path(old_hash_index).exists():
        shutil.copyfile(old_hash_index, hashes)
    index = hash_sensor_files(records, hashes)
    records = unify_duplicate_groups(records, index)
    unique, aliases = {}, defaultdict(list)
    for row in records:
        source = row.pcd or row.paths.get('intensity') or row.paths.get('depth')
        if source is None:
            continue
        digest = index[str(source)]['sha256']
        aliases[digest].append(row.to_dict())
        unique.setdefault(digest, replace(row, source_sha256=digest))
    records = sorted(unique.values(), key=lambda row: row.frame_id)
    sensors = output / 'sensors.json'
    if registry_path and Path(registry_path).resolve() != sensors.resolve():
        shutil.copyfile(registry_path, sensors)
    registry = json.loads(sensors.read_text()) if sensors.exists() else build_registry(records, sensors)
    records = attach_registry(records, registry)
    usable, excluded = [], []
    for row in records:
        if not row.sensor_config.get('grid'):
            excluded.append({'frame_id': row.frame_id, 'reason': 'No registered sensor grid',
                             'record': row.to_dict()})
        else:
            usable.append(row)
    assignments, target_only = grouped_assignments(usable)
    validate_splits(usable, assignments)
    manifest = output / 'manifest.jsonl'
    with manifest.open('w') as stream:
        for row in usable:
            data = row.to_dict()
            data['split'] = assignments[row.group_id]
            data['source_stat'] = {str(p): {'size': p.stat().st_size, 'mtime_ns': p.stat().st_mtime_ns}
                                   for p in row.paths.values()}
            data['aliases'] = [a['frame_id'] for a in aliases[row.source_sha256]]
            stream.write(json.dumps(data, sort_keys=True) + '\n')
    write_json(output / 'splits.json', {'split_seed': 42, 'assignments': assignments,
                                      'target_only_domains': target_only})
    write_json(output / 'excluded.json', excluded)
    pilot = {split: balanced_subset(usable, assignments, split, count)
             for split, count in [('train', 600), ('val', 150), ('test', 150)]}
    write_json(output / 'pilot.json', pilot)
    protocol = {'version': 3, 'root': str(Path(root).resolve()), 'split_seed': 42,
                'files': {name: file_digest(output / name) for name in
                          ['manifest.jsonl', 'splits.json', 'sensors.json', 'content_hashes.json', 'pilot.json', 'excluded.json']},
                'frames': len(usable), 'excluded': len(excluded),
                'groups': len(assignments), 'split_counts': dict(Counter(assignments[r.group_id] for r in usable)),
                'domains': dict(Counter(r.domain for r in usable)),
                'depth_verified_frames': sum(r.sensor_config['depth_available'] for r in usable),
                'pilot_counts': {s: len(ids) for s, ids in pilot.items()},
                'label_snapshot': None}
    write_json(output / 'protocol.json', protocol)
    return protocol


def load_protocol(directory, *, verify_sources=True):
    directory = Path(directory)
    protocol = json.loads((directory / 'protocol.json').read_text())
    for name, expected in protocol['files'].items():
        if file_digest(directory / name) != expected:
            raise ValueError(f'frozen protocol file changed: {name}')
    records = []
    for line in (directory / 'manifest.jsonl').read_text().splitlines():
        row = json.loads(line)
        if verify_sources:
            for path, expected in row['source_stat'].items():
                stat = Path(path).stat()
                if stat.st_size != expected['size'] or stat.st_mtime_ns != expected['mtime_ns']:
                    raise ValueError(f'frozen raw source changed: {path}')
        records.append(FrameRecord(row['frame_id'], row['group_id'], row['variant'],
                                   {k: Path(v) for k, v in row['paths'].items()}, row['sensor_config'],
                                   row['source_sha256'], row.get('cache_path')))
    assignments = json.loads((directory / 'splits.json').read_text())['assignments']
    validate_splits(records, assignments)
    return records, assignments, protocol


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--root', default='/root/private/data/pallet')
    parser.add_argument('--output', default='outputs/protocol_v3')
    args = parser.parse_args()
    print(json.dumps(build_protocol(args.root, args.output), ensure_ascii=False))
