"""Read-only, bounded inventory of a pallet dataset and its archive members."""

from __future__ import annotations

from collections import Counter, defaultdict
import argparse
import gzip
import hashlib
import json
import os
from pathlib import Path, PurePosixPath
import shutil
import struct
import tarfile
import zipfile
import zlib

from pallet.data.manifest import _TIME, scan_dataset


READ_LIMIT = 64 * 1024 * 1024
HEADER_LIMIT = 16 * 1024
EXTRACT_LIMIT = 20 * 1024 * 1024 * 1024
FREE_FLOOR = 50 * 1024 * 1024 * 1024
SENSOR_SUFFIXES = {".pcd", ".yaml", ".png", ".jpg"}


def _type(path):
    name = path.name.lower()
    if name.endswith(".tar.gz"):
        return "tar.gz"
    if name.endswith(".zip"):
        return "zip"
    if name.endswith(".rar"):
        return "rar"
    if name.endswith(".gz"):
        return "gz"
    return None


def _hash(stream):
    digest = hashlib.sha256()
    while chunk := stream.read(1024 * 1024):
        digest.update(chunk)
    return digest.hexdigest()


def _crc(path):
    crc = 0
    with path.open("rb") as stream:
        while chunk := stream.read(1024 * 1024):
            crc = zlib.crc32(chunk, crc)
    return crc & 0xffffffff


def _pcd_header(path):
    """Validate metadata and minimum binary payload without reading point arrays."""
    header = {}
    with path.open("rb") as stream:
        consumed = 0
        while consumed < HEADER_LIMIT:
            line = stream.readline(min(HEADER_LIMIT - consumed, 4096))
            consumed += len(line)
            if not line or not line.endswith(b"\n"):
                return None, "incomplete_pcd_header"
            try:
                parts = line.decode("ascii").strip().split()
            except UnicodeDecodeError:
                return None, "non_ascii_pcd_header"
            if parts and not parts[0].startswith("#"):
                header[parts[0].upper()] = parts[1:]
            if parts and parts[0].upper() == "DATA":
                break
        else:
            return None, "pcd_header_limit"
        try:
            fields = header.get("FIELDS", header.get("FIELD", []))
            sizes = [int(x) for x in header["SIZE"]]
            types = header["TYPE"]
            counts = [int(x) for x in header.get("COUNT", ["1"] * len(fields))]
            width = int(header["WIDTH"][0])
            height = int(header["HEIGHT"][0])
            points = int(header["POINTS"][0])
            mode = header["DATA"][0].lower()
            if (not {"x", "y", "z"}.issubset(set(fields)) or width < 1 or height < 1
                    or points != width * height or points < 1 or mode not in
                    {"ascii", "binary", "binary_compressed"} or
                    not len(fields) == len(sizes) == len(types) == len(counts) or
                    any(x < 1 for x in sizes + counts)):
                return None, "invalid_pcd_metadata"
            for size, kind in zip(sizes, types):
                if (kind == "F" and size not in (4, 8)) or (kind in ("I", "U") and size not in (1, 2, 4, 8)) or kind not in ("F", "I", "U"):
                    return None, "unsupported_pcd_field"
            remaining = path.stat().st_size - stream.tell()
            if mode == "binary" and remaining < points * sum(a * b for a, b in zip(sizes, counts)):
                return None, "truncated_binary_pcd"
            if mode == "binary_compressed":
                sizes_header = stream.read(8)
                if len(sizes_header) != 8:
                    return None, "truncated_compressed_pcd"
                compressed, raw = struct.unpack("<II", sizes_header)
                if remaining < compressed + 8 or raw != points * sum(a * b for a, b in zip(sizes, counts)):
                    return None, "invalid_compressed_pcd_size"
            if mode == "ascii" and remaining == 0:
                return None, "empty_ascii_pcd_payload"
            return {"width": width, "height": height, "points": points, "data": mode}, None
        except (KeyError, IndexError, ValueError, OverflowError):
            return None, "invalid_pcd_metadata"


def _candidates(root, archive, name, size, files):
    member = PurePosixPath(name.replace("\\", "/"))
    if member.is_absolute() or ".." in member.parts or not member.parts or name.endswith("/"):
        return [], "unsafe_member_path"
    relative = Path(*member.parts)
    stem = archive.name.removesuffix(".tar.gz").removesuffix(".zip").removesuffix(".gz")
    guesses = [archive.parent / relative, root / relative, archive.parent / stem / relative]
    found = []
    for guess in guesses:
        if guess in files and files[guess].st_size == size and guess not in found:
            found.append(guess)
    # Archives sometimes store an enclosing directory that is already present.
    parts = relative.parts
    if len(parts) > 1 and parts[0] == stem:
        guess = archive.parent / Path(*parts[1:])
        if guess in files and files[guess].st_size == size and guess not in found:
            found.append(guess)
    return found, None


def _member(root, archive, name, size, files, expected_crc=None, stream=None):
    matches, issue = _candidates(root, archive, name, size, files)
    row = {"kind": "archive_member", "archive": str(archive.relative_to(root)),
           "member": name, "size": size, "classification": "pending", "reason": issue or "no_extracted_size_match"}
    if issue or not matches:
        return row
    row["candidate_paths"] = [str(p.relative_to(root)) for p in matches]
    if size > READ_LIMIT:
        row["reason"] = "content_check_limit_exceeded"
        return row
    if expected_crc is not None:
        verified = [p for p in matches if _crc(p) == expected_crc]
        method = "size_and_crc32"
    elif stream is not None:
        digest = _hash(stream)
        verified = []
        for path in matches:
            with path.open("rb") as source:
                if _hash(source) == digest:
                    verified.append(path)
        method = "size_and_sha256"
    else:
        verified = []
        method = None
    if verified:
        row.update(classification="duplicate", reason="matches_extracted_content",
                   evidence=method, matches=[str(p.relative_to(root)) for p in verified])
    else:
        row["reason"] = "content_mismatch" if method else "content_unverified"
    return row


def _archive_members(root, archive, files):
    kind = _type(archive)
    try:
        if kind == "zip":
            with zipfile.ZipFile(archive) as source:
                for info in source.infolist():
                    if not info.is_dir():
                        yield _member(root, archive, info.filename, info.file_size, files, expected_crc=info.CRC)
        elif kind == "tar.gz":
            with tarfile.open(archive, "r|gz") as source:
                for info in source:
                    if info.isfile():
                        # tar metadata has no member checksum; hash small matched members.
                        matches, issue = _candidates(root, archive, info.name, info.size, files)
                        stream = source.extractfile(info) if matches and not issue and info.size <= READ_LIMIT else None
                        yield _member(root, archive, info.name, info.size, files, stream=stream)
                    elif not info.isdir():
                        yield {"kind": "archive_member", "archive": str(archive.relative_to(root)),
                               "member": info.name, "size": info.size, "classification": "pending",
                               "reason": "non_regular_tar_member"}
        elif kind == "gz":
            name = archive.name[:-3]
            with gzip.open(archive, "rb") as source:
                # A plain gzip member has no reliable stored path. The output is
                # bounded; larger streams remain pending without decompression.
                with archive.open("rb") as compressed:
                    compressed.seek(-4, os.SEEK_END)
                    raw_size = struct.unpack("<I", compressed.read(4))[0]
                if raw_size <= READ_LIMIT:
                    yield _member(root, archive, name, raw_size, files, stream=source)
                else:
                    yield {"kind": "archive_member", "archive": str(archive.relative_to(root)),
                           "member": name, "size": raw_size, "classification": "pending",
                           "reason": "content_check_limit_exceeded"}
        elif kind == "rar":
            try:
                import rarfile
            except ImportError:
                yield {"kind": "archive_member", "archive": str(archive.relative_to(root)),
                       "member": None, "classification": "pending", "reason": "rar_listing_decoder_unavailable"}
                return
            with rarfile.RarFile(archive) as source:
                for info in source.infolist():
                    if not info.isdir():
                        yield _member(root, archive, info.filename, info.file_size, files,
                                      expected_crc=info.CRC if info.CRC is not None else None)
    except (OSError, EOFError, ValueError, tarfile.TarError, zipfile.BadZipFile) as exc:
        yield {"kind": "archive_member", "archive": str(archive.relative_to(root)),
               "member": None, "classification": "pending", "reason": "archive_read_error",
               "error": str(exc)[:200]}


def inventory(root, output, hash_cache=None):
    root, output = Path(root).resolve(), Path(output)
    if not root.is_dir():
        raise FileNotFoundError(root)
    files = {p: p.stat() for p in root.rglob("*") if p.is_file()}
    records = scan_dataset(root)
    manifest = {}
    for record in records:
        for modality, path in record.paths.items():
            manifest[path] = {"frame_id": record.frame_id, "group_id": record.group_id, "modality": modality}
    hashes = json.loads(Path(hash_cache).read_text()) if hash_cache and Path(hash_cache).exists() else {}
    valid_hashes = {}
    groups = defaultdict(list)
    for name, item in hashes.items():
        path = Path(name)
        stat = files.get(path)
        if stat and stat.st_size == item.get("size") and stat.st_mtime_ns == item.get("mtime_ns"):
            valid_hashes[path] = item["sha256"]
            groups[item["sha256"]].append(path)
    output.mkdir(parents=True, exist_ok=True)
    counts, reasons = Counter(), Counter()
    by_kind = defaultdict(Counter)
    non_timestamp = []
    members = 0
    with (output / "items.jsonl").open("w") as report:
        def emit(row):
            counts[row["classification"]] += 1
            by_kind[row["kind"]][row["classification"]] += 1
            reasons[row["reason"]] += 1
            report.write(json.dumps(row, sort_keys=True, ensure_ascii=False) + "\n")

        for path, stat in sorted(files.items()):
            rel = str(path.relative_to(root))
            kind = _type(path)
            row = {"kind": "file", "path": rel, "size": stat.st_size}
            if stat.st_size == 0:
                row.update(classification="damaged", reason="zero_bytes")
            elif path.suffix.lower() == ".pcd":
                header, error = _pcd_header(path)
                row["pcd_header"] = header
                if error:
                    row.update(classification="damaged", reason=error)
                elif not _TIME.search(path.name):
                    row.update(classification="pending", reason="non_timestamp_pcd_candidate")
                    non_timestamp.append(rel)
                else:
                    row.update(classification="used", reason="valid_timestamped_pcd")
            elif kind:
                row.update(classification="pending", reason="archive_requires_member_review")
            elif rel.startswith("2024_11_27/") and path.name.startswith("xyz_image") and path.suffix.lower() == ".yaml":
                row.update(classification="damaged", reason="reported_truncated_xyz_yaml")
            elif rel.startswith("2023_10_7/") and path.name.startswith("amplitude_image") and path.suffix.lower() == ".yaml":
                row.update(classification="damaged", reason="reported_all_zero_intensity_yaml")
            elif path in manifest:
                row.update(classification="used", reason="manifest_sensor_file")
            else:
                row.update(classification="pending", reason="outside_manifest")
            if path in manifest:
                row["manifest"] = manifest[path]
            if path in valid_hashes:
                digest = valid_hashes[path]
                row["sha256"] = digest
                peers = sorted(str(p.relative_to(root)) for p in groups[digest] if p != path)
                if peers and rel > peers[0] and row["classification"] != "damaged":
                    row.update(classification="duplicate", reason="same_sha256_as_file", matches=[peers[0]])
            emit(row)
            if kind:
                for member in _archive_members(root, path, files):
                    emit(member)
                    members += 1
    summary = {"root": str(root), "files": len(files), "manifest_records": len(records),
               "archive_member_rows": members, "classifications": dict(counts),
               "classifications_by_kind": {kind: dict(values) for kind, values in by_kind.items()},
               "reasons": dict(reasons), "non_timestamp_pcd_candidates": non_timestamp,
               "hash_cache_valid_entries": len(valid_hashes), "content_check_limit_bytes": READ_LIMIT}
    (output / "summary.json").write_text(json.dumps(summary, indent=2, ensure_ascii=False) + "\n")
    return summary


def resolve_pending(root, output, hash_cache=None, rar_decoder=None):
    """Resolve pending archive members by content and cache only unique sensors."""
    root, output = Path(root).resolve(), Path(output).resolve()
    report_path = output / "items.jsonl"
    rows = [json.loads(line) for line in report_path.open()]
    for row in rows:
        if row.get("extracted_path") and "sensor_role" not in row:
            name = row["member"].lower()
            if name.endswith(".pcd"):
                header, error = _pcd_header(Path(row["extracted_path"]))
                row["pcd_header"] = header
                if error:
                    role = "invalid_pcd"
                elif ".section." in name or (header["height"] == 1 and header["width"] not in (307200, 23040)):
                    role = "partial_or_unorganized_cloud"
                else:
                    role = "frame_candidate"
            else:
                role = "frame_candidate"
            row["sensor_role"] = role
            if role != "frame_candidate":
                row.update(classification="pending", reason="unique_archived_sensor_needs_review")
    files_by_size = defaultdict(list)
    for path in root.rglob("*"):
        if path.is_file():
            files_by_size[path.stat().st_size].append(path)
    known = json.loads(Path(hash_cache).read_text()) if hash_cache and Path(hash_cache).exists() else {}
    disk_digests = {}
    for name, item in known.items():
        path = Path(name)
        if path.is_file():
            stat = path.stat()
            if stat.st_size == item.get("size") and stat.st_mtime_ns == item.get("mtime_ns"):
                disk_digests[path] = item["sha256"]
    extract_dir = output / "extracted"
    extract_dir.mkdir(exist_ok=True)
    extracted_bytes = sum(p.stat().st_size for p in extract_dir.iterdir() if p.is_file())

    def candidates(size, digest):
        found = []
        for path in files_by_size[size]:
            if path not in disk_digests:
                with path.open("rb") as source:
                    disk_digests[path] = _hash(source)
            if disk_digests[path] == digest:
                found.append(str(path.relative_to(root)))
        return found

    def resolve(row, stream):
        nonlocal extracted_bytes
        name, size = row["member"], row["size"]
        suffix = PurePosixPath(name).suffix.lower()
        if suffix not in SENSOR_SUFFIXES:
            row.update(classification="pending", reason="ancillary_archive_member")
            return
        if size == 0:
            row.update(classification="damaged", reason="zero_byte_archive_sensor")
            return
        if size > READ_LIMIT:
            row.update(classification="pending", reason="content_check_limit_exceeded")
            return
        if stream is None:
            row.update(classification="pending", reason="archive_decoder_unavailable")
            return
        if extracted_bytes + size > EXTRACT_LIMIT or shutil.disk_usage(output).free - size < FREE_FLOOR:
            row.update(classification="pending", reason="extraction_budget_exceeded")
            return
        identifier = hashlib.sha256((row["archive"] + "\0" + name).encode()).hexdigest()
        temporary = extract_dir / (identifier + ".part")
        digest = hashlib.sha256()
        written = 0
        try:
            with temporary.open("wb") as target:
                while chunk := stream.read(1024 * 1024):
                    written += len(chunk)
                    if written > size:
                        raise ValueError("member_exceeds_declared_size")
                    digest.update(chunk)
                    target.write(chunk)
            if written != size:
                raise ValueError("member_shorter_than_declared_size")
            content_hash = digest.hexdigest()
            row["sha256"] = content_hash
            matches = candidates(size, content_hash)
            if matches:
                row.update(classification="duplicate", reason="same_sha256_as_disk_file",
                           evidence="size_and_sha256", matches=matches)
                temporary.unlink()
                return
            destination = extract_dir / (content_hash + suffix)
            if destination.exists():
                row.update(classification="duplicate", reason="same_sha256_as_extracted_member",
                           evidence="size_and_sha256", matches=[str(destination)])
                temporary.unlink()
                return
            temporary.replace(destination)
            extracted_bytes += size
            role = "frame_candidate"
            if suffix == ".pcd":
                header, error = _pcd_header(destination)
                row["pcd_header"] = header
                if error:
                    role = "invalid_pcd"
                elif ".section." in name.lower() or (header["height"] == 1 and header["width"] not in (307200, 23040)):
                    role = "partial_or_unorganized_cloud"
            row.update(classification="used" if role == "frame_candidate" else "pending",
                       reason="unique_archived_sensor" if role == "frame_candidate" else "unique_archived_sensor_needs_review",
                       sensor_role=role, extracted_path=str(destination), evidence="size_and_sha256")
        except (OSError, ValueError, EOFError) as exc:
            temporary.unlink(missing_ok=True)
            row.update(classification="pending", reason="archive_decode_error", error=str(exc)[:200])

    # Replace the placeholder rows emitted when no RAR header parser was present.
    rar_indexes = [i for i, row in enumerate(rows) if row.get("reason") == "rar_listing_decoder_unavailable"]
    if rar_indexes:
        try:
            import rarfile
        except ImportError:
            rarfile = None
        if rarfile:
            if rar_decoder:
                rarfile.UNRAR_TOOL = str(Path(rar_decoder).resolve())
            replacements = {}
            for i in rar_indexes:
                archive = root / rows[i]["archive"]
                try:
                    with rarfile.RarFile(archive) as source:
                        replacements[i] = [{"kind": "archive_member", "archive": rows[i]["archive"],
                                            "member": info.filename, "size": info.file_size,
                                            "classification": "pending", "reason": "content_unverified"}
                                           for info in source.infolist() if not info.isdir()]
                except Exception as exc:
                    rows[i]["error"] = str(exc)[:200]
            rows = [item for i, row in enumerate(rows) for item in replacements.get(i, [row])]

    pending = defaultdict(list)
    for row in rows:
        if (row["kind"] == "archive_member" and row["classification"] == "pending"
                and row.get("member") and not row.get("extracted_path")):
            pending[row["archive"]].append(row)
    for rel_archive, members in pending.items():
        archive = root / rel_archive
        kind = _type(archive)
        lookup = defaultdict(list)
        for row in members:
            lookup[(row["member"], row["size"])].append(row)
        try:
            if kind == "zip":
                with zipfile.ZipFile(archive) as source:
                    for info in source.infolist():
                        bucket = lookup[(info.filename, info.file_size)]
                        if bucket:
                            row = bucket.pop(0)
                            if row["size"] and PurePosixPath(row["member"]).suffix.lower() in SENSOR_SUFFIXES:
                                with source.open(info) as stream:
                                    resolve(row, stream)
                            else:
                                resolve(row, None)
            elif kind == "tar.gz":
                with tarfile.open(archive, "r|gz") as source:
                    for info in source:
                        bucket = lookup[(info.name, info.size)]
                        if bucket:
                            row = bucket.pop(0)
                            if row["size"] and PurePosixPath(row["member"]).suffix.lower() in SENSOR_SUFFIXES:
                                with source.extractfile(info) as stream:
                                    resolve(row, stream)
                            else:
                                resolve(row, None)
            elif kind == "rar":
                import rarfile
                if rar_decoder:
                    rarfile.UNRAR_TOOL = str(Path(rar_decoder).resolve())
                with rarfile.RarFile(archive) as source:
                    for info in source.infolist():
                        bucket = lookup[(info.filename, info.file_size)]
                        if bucket:
                            row = bucket.pop(0)
                            if row["size"] and PurePosixPath(row["member"]).suffix.lower() in SENSOR_SUFFIXES:
                                with source.open(info) as stream:
                                    resolve(row, stream)
                            else:
                                resolve(row, None)
        except Exception as exc:
            for bucket in lookup.values():
                for row in bucket:
                    row.update(reason="archive_decode_error", error=str(exc)[:200])

    nested_seen = {row["archive"] for row in rows if row["kind"] == "archive_member"}
    nested_errors = []
    for row in list(rows):
        if (row["kind"] != "archive_member" or row.get("member") is None or
                not row["member"].lower().endswith(".tar.gz") or not row["archive"].lower().endswith(".zip")):
            continue
        nested_id = row["archive"] + "::" + row["member"]
        if nested_id in nested_seen:
            continue
        try:
            with zipfile.ZipFile(root / row["archive"]) as outer:
                with outer.open(row["member"]) as compressed:
                    with tarfile.open(fileobj=compressed, mode="r|gz") as inner:
                        for info in inner:
                            if not info.isfile():
                                continue
                            member = {"kind": "archive_member", "archive": nested_id,
                                      "member": info.name, "size": info.size,
                                      "classification": "pending", "reason": "nested_archive_member"}
                            with inner.extractfile(info) as stream:
                                resolve(member, stream)
                            rows.append(member)
        except Exception as exc:
            nested_errors.append((nested_id, str(exc)[:200]))
    for nested_id, error in nested_errors:
        rows.append({"kind": "archive_member", "archive": nested_id, "member": None,
                     "classification": "pending", "reason": "nested_archive_read_error", "error": error})

    counts = Counter(row["classification"] for row in rows)
    by_kind = defaultdict(Counter)
    reasons = Counter()
    unique = []
    for row in rows:
        by_kind[row["kind"]][row["classification"]] += 1
        reasons[row["reason"]] += 1
        if row.get("extracted_path"):
            name = PurePosixPath(row["member"])
            archive = PurePosixPath(row["archive"].split("::", 1)[0])
            collection = name.parts[0] if name.parts and name.parts[0].startswith("20") else archive.parts[0].removesuffix(".zip").removesuffix(".tar.gz")
            unique.append({"extracted_path": row["extracted_path"], "archive": row["archive"],
                           "member": row["member"], "collection": collection,
                           "scene": str(name.parent), "original_name": name.name,
                           "size": row["size"], "sha256": row["sha256"],
                           "sensor_role": row.get("sensor_role", "frame_candidate"),
                           "usable_frame_candidate": row.get("sensor_role", "frame_candidate") == "frame_candidate"})
    temporary = report_path.with_suffix(".jsonl.tmp")
    with temporary.open("w") as stream:
        for row in rows:
            stream.write(json.dumps(row, sort_keys=True, ensure_ascii=False) + "\n")
    temporary.replace(report_path)
    with (output / "unique_sensor_records.jsonl").open("w") as stream:
        for row in unique:
            stream.write(json.dumps(row, sort_keys=True, ensure_ascii=False) + "\n")
    summary = json.loads((output / "summary.json").read_text())
    summary.update(archive_member_rows=sum(row["kind"] == "archive_member" for row in rows),
                   classifications=dict(counts), classifications_by_kind={k: dict(v) for k, v in by_kind.items()},
                   reasons=dict(reasons), unique_archived_sensors=len(unique),
                   usable_new_frame_candidates=sum(row["usable_frame_candidate"] for row in unique),
                   extracted_bytes=extracted_bytes, extraction_limit_bytes=EXTRACT_LIMIT,
                   free_space_floor_bytes=FREE_FLOOR)
    (output / "summary.json").write_text(json.dumps(summary, indent=2, ensure_ascii=False) + "\n")
    return summary


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("root", type=Path)
    parser.add_argument("output", type=Path)
    parser.add_argument("--hash-cache", type=Path)
    parser.add_argument("--resolve-pending", action="store_true")
    parser.add_argument("--rar-decoder", type=Path)
    args = parser.parse_args()
    run = resolve_pending if args.resolve_pending else inventory
    if args.resolve_pending:
        result = run(args.root, args.output, args.hash_cache, args.rar_decoder)
    else:
        result = run(args.root, args.output, args.hash_cache)
    print(json.dumps(result, indent=2))


if __name__ == "__main__":
    main()
