§
    ­¸j =  ã                  ó  — U d Z ddlmZ ddlZddlmZ ddlmZ ddlm	Z	 ddl
mZmZ ddlmZ d	Zd
ZdZded<   ed         Zd,d„Zd-d„Zd.d„Zd/d0d!„Zd.d"„Z ed#¬$¦  «         G d%„ d&¦  «        ¦   «         Z G d'„ d(e¦  «        Zd1d+„ZdS )2a  WebDataset shard-index schema, validation and on-disk access.

Purpose: Define the stable contract shared by packing and streaming WebDataset splits. Scope: shard names, split
validation, index serialization and loading. Usage: import ShardIndex and read_shard_index from this module. Outputs:
parsed or serialized JSON index data and verified shard-relative paths. Failure: rejects malformed names, incompatible
schemas and unavailable split indexes. Used by: webdataset pack, load, RF-DETR dataset construction and the CLI.
é    )ÚannotationsN)ÚIterable)Ú	dataclass)ÚPath)ÚAnyÚLiteral)Úfilter_parent_categoriesé   i  @)ÚjpgÚjpegÚpngÚwebpÚbmpútuple[str, ...]ÚIMAGE_EXTENSIONS©ÚremapÚrawÚ
categoriesúIterable[dict[str, Any]]Ú	cat2labelúdict[int, int] | NoneÚreturnú	list[str]c                óB  — d„ | D ¦   «         }|€;|sg S dgt          |¦  «        dz   z  }|                     ¦   «         D ]
\  }}|||<   Œ|S |sg S dgt          |                     ¦   «         ¦  «        dz   z  }|                     ¦   «         D ]\  }}||v r||         ||<   Œ|S )ub  Return category names indexed by the label the dataset emits for them.

    Every entry sits at its own label index, so ``names[label]`` is always the emitted label's name. Under a
    ``"remap"`` mapping that is the contiguous 0-based index; with no mapping (``"raw"``) it is the source
    ``category_id`` itself â€” raw labels skip whatever gaps the id range has, so the list carries an empty string at
    every skipped index rather than shifting later names down to fill the gap. A mapping whose labels leave a slot
    unnamed gets the same empty-string placeholder.

    Args:
        categories: COCO ``categories`` entries; ``id`` and ``name`` are read from each.
        cat2label: ``category_id`` to label-index mapping, or ``None`` when source ids are used as labels.

    Returns:
        The category names, indexed by label, with an empty string at every label with no category.

    Examples:
        >>> categories = [{"id": 3, "name": "cat"}, {"id": 9, "name": "dog"}]
        >>> category_names_by_label(categories, {3: 0, 9: 1})
        ['cat', 'dog']
        >>> category_names_by_label(categories, None)
        ['', '', '', 'cat', '', '', '', '', '', 'dog']
        >>> category_names_by_label(categories, {3: 0, 9: 2})
        ['cat', '', 'dog']
        >>> category_names_by_label([], None), category_names_by_label(categories, {})
        ([], [])
    c                ó`   — i | ]+}t          |d          ¦  «        t          |d         ¦  «        “Œ,S )ÚidÚname)ÚintÚstr)Ú.0Úcategorys     úp/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/rfdetr/datasets/webdataset/index.pyú
<dictcomp>z+category_names_by_label.<locals>.<dictcomp>?   s3   € ÐYÐYÐYÀ(•3�x ”~Ñ&Ô&­¨H°VÔ,<Ñ(=Ô(=ÐYÐYÐYó    NÚ r
   )ÚmaxÚitemsÚvalues)r   r   Únames_by_idÚnamesÚcategory_idr   Úlabels          r#   Úcategory_names_by_labelr.   $   så   € ð6 ZÐYÈjÐYÑYÔY€KØÐØð 	ØˆIØ��˜KÑ(Ô(¨1Ñ,Ñ-ˆØ!,×!2Ò!2Ñ!4Ô!4ð 	&ð 	&ÑˆK˜Ø!%ˆE�+ÑÐØˆØð Øˆ	ØˆD•C˜	×(Ò(Ñ*Ô*Ñ+Ô+¨aÑ/Ñ0€EØ'ŸošoÑ/Ô/ð 4ð 4Ñˆ�UØ˜+Ð%Ð%Ø& {Ô3ˆE�%‰LøØ€Lr%   Úbaser   r   r    c                ó²   — |                       ¦   «         }| |z                        ¦   «         }|                     |¦  «        st          d|›d| › d�¦  «        ‚|S )a—  Resolve *name* under *base*, rejecting any path that escapes it.

    Shard file names come from ordinary on-disk JSON. Both shard reading and
    stale-shard cleanup trust those entries, so traversal must fail before an
    open or unlink reaches a path outside the shard directory.

    Args:
        base: Directory *name* must resolve inside.
        name: Untrusted path-like string taken from a shard index.

    Returns:
        The resolved absolute path to *name* under *base*.

    Raises:
        ValueError: If *name* is absolute or resolves outside *base*.

    Examples:
        >>> resolve_within(Path("/data/shards"), "train-000000.tar").name
        'train-000000.tar'
        >>> resolve_within(Path("/data/shards"), "../../etc/passwd")  # doctest: +ELLIPSIS
        Traceback (most recent call last):
            ...
        ValueError: shard entry '../../etc/passwd' resolves outside ...shards.
    zshard entry z resolves outside ú.)ÚresolveÚis_relative_toÚ
ValueError)r/   r   Úbase_resolvedÚ	candidates       r#   Úresolve_withinr7   P   se   € ð2 —L’L‘N”N€MØ˜‘×%Ò%Ñ'Ô'€IØ×#Ò# MÑ2Ô2ð KÝÐI¨ÐIÐIÀ$ÐIÐIÐIÑJÔJÐJØÐr%   Úsplitc                óþ   — | rd| v sd| v sd| v s| dv rt          d| ›d�¦  «        ‚t          t          d¦  «        t          | ¦  «        z  ¦  «        }|r)t          d| ›dd	                     |¦  «        › d
�¦  «        ‚| S )uÆ  Reject a split name that could escape the shard directory as a path component.

    *split* becomes part of a shard or index file name with no other sanitisation, so a value containing a path
    separator or a ``..`` segment would let :func:`_shard_name`/:func:`index_name` resolve outside the shard
    directory â€” writing shards outside ``output_dir`` when packing, or reading an index from outside ``shard_dir``
    when loading.

    Args:
        split: Split name supplied by a caller (CLI argument or library call).

    Returns:
        *split*, unchanged, once validated.

    Raises:
        ValueError: If *split* is empty, contains a path separator or a ``.``/``..`` segment, or carries a
            glob metacharacter. The last one matters because the name is also matched against existing shard
            files: a split called ``*`` would match â€” and then delete â€” the shards of every other split.

    Examples:
        >>> _validate_split_name("train")
        'train'
        >>> _validate_split_name("../escape")  # doctest: +ELLIPSIS
        Traceback (most recent call last):
            ...
        ValueError: split '../escape' must not contain a path separator, drive prefix, or '..'.
    ú/ú\ú:)r1   z..zsplit z: must not contain a path separator, drive prefix, or '..'.z*?[]z must not contain r&   zm: the name is matched against existing shard files, so a glob metacharacter would reach other splits' shards.)r4   ÚsortedÚsetÚjoin)r8   Ú	forbiddens     r#   Ú_validate_split_namerA   p   s¼   € ð6 ð g�C˜5�L�L D¨E M M°S¸E°\°\ÀUÈkÐEYÐEYÝÐe %ÐeÐeÐeÑfÔfÐfÝ•s˜6‘{”{¥S¨¡Z¤ZÑ/Ñ0Ô0€IØð 
ÝðO�Uð Oð O°·²¸	Ñ0BÔ0Bð Oð Oð Oñ
ô 
ð 	
ð €Lr%   Úindexr   Ú
generationú
str | Nonec                óN   — t          | ¦  «         |€	| › d|d›d�S | › d|› d|d›d�S )a™  Return the file name of shard *index* of *split*.

    Args:
        split: Split name the shard belongs to.
        index: Zero-based shard number.
        generation: Token identifying one pack run. Included in the name so that re-packing a split writes new
            files instead of overwriting the ones the published index still points at.

    Returns:
        Shard file name.

    Raises:
        ValueError: If *split* would escape the shard directory (see :func:`_validate_split_name`).

    Examples:
        >>> _shard_name("train", 7)
        'train-000007.tar'
        >>> _shard_name("train", 7, "a1b2c3d4")
        'train-a1b2c3d4-000007.tar'
    Nú-Ú06dz.tar©rA   )r8   rB   rC   s      r#   Ú_shard_namerI   –   sT   € õ* ˜ÑÔÐØÐØÐ)Ð)˜%Ð)Ð)Ð)Ð)Ð)ØÐ2Ð2�jÐ2Ð2 5Ð2Ð2Ð2Ð2Ð2r%   c                ó*   — t          | ¦  «         | › d�S )aq  Return the file name of the shard index of *split*.

    Args:
        split: Split name the index describes.

    Returns:
        Index file name, relative to the shard directory.

    Raises:
        ValueError: If *split* would escape the shard directory (see :func:`_validate_split_name`).

    Examples:
        >>> index_name("val")
        'val-index.json'
    z-index.jsonrH   )r8   s    r#   Ú
index_namerK   ±   s!   € õ  ˜ÑÔÐØÐ Ð Ð Ð r%   T)Úfrozenc                  óŽ   — e Zd ZU dZded<   ded<   ded<   ded	<   d
ed<   ded<   dZd
ed<   dd„Zedd„¦   «         Zdd„Z	dd„Z
dS )Ú
ShardIndexaä  Everything a reader needs about one packed split without opening a shard.

    Args:
        split: Split name, matching the shard and index file names.
        shards: Shard file names in packing order, relative to the index's directory.
        num_samples: Total samples across all shards.
        categories: COCO ``categories`` entries copied verbatim from the source annotation file.
        annotated_category_ids: Category ids carrying at least one annotation in this split.
        category_ids: ``"remap"`` when labels are contiguous 0-based indices, ``"raw"`` when they are the source
            ``category_id`` values.
        samples_per_shard: Sample count of each shard, aligned by position with *shards*. Shards are cut by byte
            size, not sample count, so this can vary widely between shards of the same split; an empty tuple means
            the index predates this field (or was built without it), and readers fall back to assuming a uniform
            split.
    r    r8   r   Úshardsr   Únum_samplesztuple[dict[str, Any], ...]r   ztuple[int, ...]Úannotated_category_idsÚCategoryIdPolicyÚcategory_ids© Úsamples_per_shardr   údict[str, Any]c           
     óÐ   — t           | j        t          | j        ¦  «        | j        t          | j        ¦  «        t          | j        ¦  «        | j        t          | j        ¦  «        dœS )aG  Return the JSON-serialisable form written to disk.

        Returns:
            Mapping stamped with :data:`INDEX_VERSION`.

        Examples:
            >>> index = ShardIndex("val", ("val-000000.tar",), 1, ({"id": 1, "name": "a"},), (1,), "remap", (1,))
            >>> index.to_json()["num_samples"]
            1
        )Úversionr8   rO   rP   r   rQ   rS   rU   )	ÚINDEX_VERSIONr8   ÚlistrO   rP   r   rQ   rS   rU   ©Úselfs    r#   Úto_jsonzShardIndex.to_jsonß   s]   € õ %Ø”ZÝ˜4œ;Ñ'Ô'ØÔ+Ý˜tœÑ/Ô/Ý&*¨4Ô+FÑ&GÔ&GØ Ô-Ý!% dÔ&<Ñ!=Ô!=ð	
ð 	
ð 		
r%   Úpayloadc                ó†  — t          |                     dd¦  «        ¦  «        }|t          k    rt          d|› dt          › d�¦  «        ‚|                     dd¦  «        }|dvrt          d	|›d
�¦  «        ‚ | t	          |d         ¦  «        t          d„ |d         D ¦   «         ¦  «        t          |d         ¦  «        t          |                     dd¦  «        ¦  «        t          d„ |                     dd¦  «        D ¦   «         ¦  «        |t          d„ |                     dd¦  «        D ¦   «         ¦  «        ¬¦  «        S )ac  Rebuild an index from its on-disk mapping.

        Args:
            payload: Parsed index JSON.

        Returns:
            The reconstructed index.

        Raises:
            ValueError: If the index was written by an incompatible schema version.

        Examples:
            >>> payload = {
            ...     "version": 1, "split": "val", "shards": ["val-000000.tar"], "num_samples": 1,
            ...     "categories": [{"id": 1, "name": "a"}], "annotated_category_ids": [1], "category_ids": "remap",
            ... }
            >>> ShardIndex.from_json(payload).split
            'val'
        rX   r   zShard index schema version z4 is not supported by this RF-DETR release (expected z/); re-pack the dataset with the current packer.rS   r   r   z1Shard index declares unknown category_ids policy z; expected 'remap' or 'raw'.r8   c              3  ó4   K  — | ]}t          |¦  «        V — Œd S ©N)r    )r!   Úshards     r#   ú	<genexpr>z'ShardIndex.from_json.<locals>.<genexpr>  s(   è è € ÐCÐC¨�˜U™œÐCÐCÐCÐCÐCÐCr%   rO   rP   r   rT   c              3  ó4   K  — | ]}t          |¦  «        V — Œd S ra   ©r   )r!   Úcids     r#   rc   z'ShardIndex.from_json.<locals>.<genexpr>  s(   è è € Ð(gÐ(g°c­¨S©¬Ð(gÐ(gÐ(gÐ(gÐ(gÐ(gr%   rQ   c              3  ó4   K  — | ]}t          |¦  «        V — Œd S ra   re   )r!   Úcounts     r#   rc   z'ShardIndex.from_json.<locals>.<genexpr>  s(   è è € Ð#aÐ#a°5¥C¨¡J¤JÐ#aÐ#aÐ#aÐ#aÐ#aÐ#ar%   rU   )r8   rO   rP   r   rQ   rS   rU   )r   ÚgetrY   r4   r    Útuple)Úclsr^   rX   Úpolicys       r#   Ú	from_jsonzShardIndex.from_jsonõ   si  € õ* �g—k’k )¨QÑ/Ô/Ñ0Ô0ˆØ•mÒ#Ð#Ýð\¨gð \ð \Ý*ð\ð \ð \ñô ð ð —’˜^¨WÑ5Ô5ˆØÐ)Ð)Ð)ÝÐwÐQWÐwÐwÐwÑxÔxÐxØˆsÝ�g˜gÔ&Ñ'Ô'ÝÐCÐC°¸Ô1BÐCÑCÔCÑCÔCÝ˜G MÔ2Ñ3Ô3Ý˜WŸ[š[¨°rÑ:Ô:Ñ;Ô;Ý#(Ð(gÐ(g¸W¿[º[ÐIaÐceÑ=fÔ=fÐ(gÑ(gÔ(gÑ#gÔ#gØÝ#Ð#aÐ#a¸G¿KºKÐH[Ð]_Ñ<`Ô<`Ð#aÑ#aÔ#aÑaÔað
ñ 
ô 
ð 	
r%   r   c                ó¶   — | j         dk    rdS t          t          | j        ¦  «        t	          | j        ¦  «        ¦  «        }d„ t          |¦  «        D ¦   «         S )a-  Return the ``category_id`` to label-index mapping implied by this index.

        Mirrors :class:`~rfdetr.datasets.coco.CocoDetection`'s detection branch: unannotated grouping categories are
        dropped by :func:`~rfdetr.datasets.coco.filter_parent_categories` before indices are assigned, so a synthetic
        Roboflow root category consumes no output slot.

        Returns:
            The mapping, or ``None`` when the index declares the ``"raw"`` policy and source ids are used as labels.

        Examples:
            >>> categories = ({"id": 3, "name": "a"}, {"id": 9, "name": "b"})
            >>> ShardIndex("t", (), 0, categories, (3, 9), "remap").cat2label()
            {3: 0, 9: 1}
            >>> ShardIndex("t", (), 0, categories, (3, 9), "raw").cat2label() is None
            True
        r   Nc                ó@   — i | ]\  }}t          |d          ¦  «        |“ŒS )r   re   )r!   r-   r"   s      r#   r$   z(ShardIndex.cat2label.<locals>.<dictcomp>1  s*   € ÐRÐRÐR©¨u°h•�H˜T”NÑ#Ô# UÐRÐRÐRr%   )rS   r	   rZ   r   r>   rQ   Ú	enumerate)r\   Úkepts     r#   r   zShardIndex.cat2label  sU   € ð" Ô Ò%Ð%Ø�4Ý'­¨T¬_Ñ(=Ô(=½sÀ4ÔC^Ñ?_Ô?_Ñ`Ô`ˆØRÐRÅ)ÈDÁ/Ä/ÐRÑRÔRÐRr%   r   c                óP   — t          | j        |                      ¦   «         ¦  «        S )a°  Return the category names in label order, as a reader of this split would emit them.

        The packed index is the only place a shard stream keeps its category list, so this is what
        :attr:`~rfdetr.datasets.webdataset.load.WebDatasetDetection.class_names` reports for a train split and what
        :meth:`~rfdetr.detr.RFDETR.train` records before the datamodule has built one. See
        :func:`category_names_by_label` for the ``"remap"`` versus ``"raw"`` layout.

        Returns:
            The category names, indexed by label, with an empty string at every label with no category.

        Examples:
            >>> categories = ({"id": 3, "name": "a"}, {"id": 9, "name": "b"})
            >>> ShardIndex("t", (), 0, categories, (3, 9), "remap").class_names()
            ['a', 'b']
            >>> ShardIndex("t", (), 0, categories, (3, 9), "raw").class_names()
            ['', '', '', 'a', '', '', '', '', '', 'b']
        )r.   r   r   r[   s    r#   Úclass_nameszShardIndex.class_names3  s    € õ$ ' t¤¸¿ºÑ8HÔ8HÑIÔIÐIr%   N)r   rV   )r^   rV   r   rN   )r   r   )r   r   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú__annotations__rU   r]   Úclassmethodrm   r   rs   rT   r%   r#   rN   rN   Å   sà   € € € € € € ðð ð  €J€J�JØÐÐÑØÐÐÑØ*Ð*Ð*Ñ*Ø+Ð+Ð+Ñ+Ø"Ð"Ð"Ñ"Ø)+ÐÐ+Ð+Ð+Ñ+ð
ð 
ð 
ð 
ð, ð%
ð %
ð %
ñ „[ð%
ðNSð Sð Sð Sð,Jð Jð Jð Jð Jð Jr%   rN   c                  ó   — e Zd ZdZdS )ÚWebDatasetSplitUnavailableErrora•  Raised when a shard directory carries no index for the requested split.

    A subclass of :class:`FileNotFoundError`, so callers that only care that the split is missing keep working, while
    :meth:`~rfdetr.training.module_data.RFDETRDataModule._build_test_dataset` can tell "this split was never packed"
    apart from any other missing file and fall back to ``val`` rather than aborting the run.
    N)rt   ru   rv   rw   rT   r%   r#   r{   r{   H  s   € € € € € ðð ð ð r%   r{   Ú	shard_dirú
str | Pathc           	     ó¦  — t          | ¦  «        t          |¦  «        z  }|                     ¦   «         st          d|›d|› d|› d�¦  «        ‚|                     d¬¦  «        5 }t          j        |¦  «        }ddd¦  «         n# 1 swxY w Y   t                               |¦  «        }|j	        |k    rt          |› d|›d	|j	        ›d
�¦  «        ‚|S )a8  Load the shard index of *split* from *shard_dir*.

    Args:
        shard_dir: Directory holding the shards and their index files.
        split: Split name to read.

    Returns:
        The parsed index.

    Raises:
        WebDatasetSplitUnavailableError: If the split has no index file in *shard_dir*.
    zNo WebDataset index for split z at z@. Pack the split first: python -m rfdetr.cli.webdataset --split z ...zutf-8)ÚencodingNz is named for split z but its own recorded split is u�    â€” the index file was likely copied or renamed rather than produced by packing this split; repack it instead of renaming an existing index.)r   rK   Úexistsr{   ÚopenÚjsonÚloadrN   rm   r8   r4   )r|   r8   ÚpathÚhandler^   rB   s         r#   Úread_shard_indexr†   Q  sS  € õ �	‰?Œ?�Z¨Ñ.Ô.Ñ.€DØ�;Š;‰=Œ=ð 
Ý-ðY¨Uð Yð Y¸$ð Yð YØMRðYð Yð Yñ
ô 
ð 	
ð 
�Š˜GˆÑ	$Ô	$ð $¨Ý”)˜FÑ#Ô#ˆð$ð $ð $ñ $ô $ð $ð $ð $ð $ð $ð $øøøð $ð $ð $ð $å× Ò  Ñ)Ô)€EØ„{�eÒÐÝØð ?ð ?¨ð ?ð ?ÐQVÔQ\ð ?ð ?ð ?ñ
ô 
ð 	
ð
 €Ls   Á#BÂBÂB)r   r   r   r   r   r   )r/   r   r   r    r   r   )r8   r    r   r    ra   )r8   r    rB   r   rC   rD   r   r    )r|   r}   r8   r    r   rN   )rw   Ú
__future__r   r‚   Úcollections.abcr   Údataclassesr   Úpathlibr   Útypingr   r   Úrfdetr.datasets.cocor	   rY   ÚDEFAULT_MAX_SHARD_BYTESr   rx   rR   r.   r7   rA   rI   rK   rN   ÚFileNotFoundErrorr{   r†   rT   r%   r#   ú<module>r�      sÈ  ððð ð ð #Ð "Ð "Ð "Ð "Ð "à €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð à 9Ð 9Ð 9Ð 9Ð 9Ð 9ð €ð ,Ð ð %JÐ Ð IÐ IÐ IÑ Ià˜>Ô*Ð ð)ð )ð )ð )ðXð ð ð ð@#ð #ð #ð #ðL3ð 3ð 3ð 3ð 3ð6!ð !ð !ð !ð( €�$ÐÑÔðJð Jð Jð Jð Jñ Jô Jñ ÔðJðDð ð ð ð Ð&7ñ ô ð ðð ð ð ð ð r%   