§
    ­¸jóª  ã            	      óä  — d Z ddlmZ ddlZddlZddlmZmZ ddlm	Z	 ddl
mZ ddlmZmZ ddlmZ ddlZddlZddlZdd	lmZ dd
lmZ ddlmZ ddlmZmZmZmZmZ ddl m!Z! ddl"m#Z#m$Z$ ddl%m&Z&m'Z'm(Z(m)Z)m*Z* ddl+m,Z,  e,¦   «         Z-dZ.dZ/dZ0dZ1dEd„Z2dFd„Z3dFd„Z4dGd„Z5dHd"„Z6 G d#„ d$ej7        j8        j9        e:eef                  ¦  «        Z;d%d%d&œdId+„Z<dJd1„Z=d2d2ddd3ddd%d4œdKd@„Z>dLdD„Z?dS )Ma8  Load WebDataset shards into RF-DETR streaming datasets and loaders.

Purpose: Read indexed tar shards with RF-DETR transforms and deterministic distributed worker planning. Scope: the
optional WebDataset dependency, iterable dataset, epoch sizing and DataLoader construction. Usage: build a dataset with
build_webdataset and a loader with build_webdataset_loader. Outputs: RF-DETR-compatible dataset samples and DataLoaders.
Failure: rejects missing indexes, incompatible distribution plans and unsupported keypoints. Used by: datasets, detr and
training module data.
é    )ÚannotationsN)ÚCallableÚIterator)Úpartial)Úgcd)ÚPathÚPurePath)ÚAny)ÚImage)Ú
DataLoader)Ú
MultiScale)ÚConvertCocoÚdraft_size_for_transformsÚmake_coco_transformsÚ"make_coco_transforms_square_div_64Úscale_coco_annotation)Údecode_image_bytes)Úis_gpu_postprocessÚresolve_backend_for_build)ÚIMAGE_EXTENSIONSÚWebDatasetSplitUnavailableErrorÚcategory_names_by_labelÚread_shard_indexÚresolve_within)Ú
get_loggeriè  gš™™™™™©?g333333Ó?éd   Úreturnr
   c                 óV   — 	 ddl } n"# t          $ r}t          d¦  «        |‚d}~ww xY w| S )zLImport and return the WebDataset module, or raise an actionable ImportError.r   NzeStreaming WebDataset shards requires the webdataset package. Install with: pip install 'rfdetr[data]')Ú
webdatasetÚImportError)r   Úexcs     úo/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/rfdetr/datasets/webdataset/load.pyÚ_require_webdatasetr#   A   sU   € ðØÐÐÐÐøÝð ð ð ÝØsñ
ô 
àð	øøøøðøøøð Ðs   ‚ ‡
&‘!¡&Úintc                 óÔ   — t           j                             ¦   «         rIt           j                             ¦   «         r+t	          t           j                             ¦   «         ¦  «        S dS )zQReturn the distributed world size, or ``1`` outside an initialised process group.é   )ÚtorchÚdistributedÚis_availableÚis_initializedr$   Úget_world_size© ó    r"   Ú_distributed_world_sizer.   L   sQ   € åÔ×%Ò%Ñ'Ô'ð 7­EÔ,=×,LÒ,LÑ,NÔ,Nð 7Ý•5Ô$×3Ò3Ñ5Ô5Ñ6Ô6Ð6Øˆ1r-   c                 óÔ   — t           j                             ¦   «         rIt           j                             ¦   «         r+t	          t           j                             ¦   «         ¦  «        S dS )zKReturn the distributed rank, or ``0`` outside an initialised process group.r   )r'   r(   r)   r*   r$   Úget_rankr,   r-   r"   Ú_distributed_rankr1   S   sQ   € åÔ×%Ò%Ñ'Ô'ð 1­EÔ,=×,LÒ,LÑ,NÔ,Nð 1Ý•5Ô$×-Ò-Ñ/Ô/Ñ0Ô0Ð0Øˆ1r-   ÚrankÚ
world_sizeú(Callable[[Iterator[Any]], Iterator[Any]]c                ó   ‡ ‡— dˆ ˆfd„}|S )u~  Return a ``webdataset`` ``nodesplitter`` fixed to *rank* and *world_size* at closure-creation time.

    ``webdataset.split_by_node`` re-resolves rank and world size inside whichever process calls it, preferring
    the ``RANK``/``WORLD_SIZE`` environment variables and falling back to ``torch.distributed`` only when those
    are unset. Under a ``spawn``-started DataLoader worker (a fresh interpreter, distributed not yet initialised
    in the child) combined with a launcher that exports neither variable, that resolution can silently disagree
    with the world size this module's own epoch planning already used in the main process â€” every rank then
    streams the entire split, duplicated, with no error. Capturing the value the loader already resolved and
    applying it with a plain positional stride removes that second, independently-resolved source of truth.

    Args:
        rank: This process's rank, resolved once when the loader is built.
        world_size: Total ranks sharing the split, resolved once when the loader is built.

    Returns:
        A splitter usable as ``webdataset.WebDataset(..., nodesplitter=...)``.
    ÚsourceúIterator[Any]r   c              3  óp   •K  — ‰dk    r$t          | ¦  «        D ]\  }}|‰z  ‰k    r|V — Œd S | E d {V —† d S )Nr&   )Ú	enumerate)r6   ÚindexÚitemr2   r3   s      €€r"   Ú_splitz$_fixed_node_splitter.<locals>._splitm   so   øè è € Ø˜Š>ˆ>Ý(¨Ñ0Ô0ð ð ‘��tØ˜:Ñ%¨Ò-Ð-Ø�J�J�Jøðð ð ÐÐÐÐÐÐÐÐÐr-   )r6   r7   r   r7   r,   )r2   r3   r<   s   `` r"   Ú_fixed_node_splitterr=   Z   s.   øø€ ð&ð ð ð ð ð ð ð €Mr-   Úpathr	   Ústrc                ó0   — d|                       ¦   «         › �S )u®  Return the ``file:`` URL naming *path* to ``webdataset``'s shard opener.

    ``webdataset`` calls ``urlparse`` on every shard string and, for the empty and ``file`` schemes, opens
    ``urlparse(url).path`` verbatim â€” no percent-decoding, and no conversion from URL syntax back to a native path.
    Two spellings that look right therefore fail on Windows: a bare ``str(path)`` parses ``C:\shards\train.tar``
    as scheme ``c``, which has no handler, and :meth:`pathlib.Path.as_uri` yields ``file:///C:/shards/train.tar``,
    whose path keeps a leading slash that ``open`` rejects with ``[Errno 22] Invalid argument``. An authority-less
    ``file:`` URL over the forward-slash form of the path leaves ``C:/shards/train.tar`` on Windows and
    ``/shards/train.tar`` elsewhere, both of which ``open`` takes as-is. Percent-encoding is deliberately omitted
    for the same reason: nothing downstream reverses it, so a directory containing spaces has to pass through
    literally.

    Args:
        path: Path to one shard, absolute or relative to the working directory.

    Returns:
        The URL to hand to ``webdataset``.

    Examples:
        >>> from pathlib import PurePosixPath, PureWindowsPath
        >>> _shard_url(PureWindowsPath(r"C:\data\shards\train-000000.tar"))
        'file:C:/data/shards/train-000000.tar'
        >>> _shard_url(PurePosixPath("/data/shards/train-000000.tar"))
        'file:/data/shards/train-000000.tar'
    zfile:)Úas_posix)r>   s    r"   Ú
_shard_urlrB   x   s   € ð4 %�4—=’=‘?”?Ð$Ð$Ð$r-   c                  ó¤   ‡ — e Zd ZdZdddddddœd/ˆ fd„Zed0d„¦   «         Zed1d„¦   «         Zd2d„Zd3d!„Z	d0d"„Z
d4d$„Zd5d(„Zd1d)„Zd6d,„Zd7d.„Zˆ xZS )8ÚWebDatasetDetectionu¾
  Streaming COCO-style detection dataset reading WebDataset tar shards.

    Yields the same ``(image, target)`` pairs as :class:`~rfdetr.datasets.coco.CocoDetection`: the sidecar JSON goes
    through :class:`~rfdetr.datasets.coco.ConvertCoco` and the result through *transforms*, both inside the DataLoader
    worker that read the shard.

    Sizing follows WebDataset's own convention and has two modes, chosen by :meth:`configure_epoch`:

    - **Unplanned** (the default, and what evaluation uses): every worker drains its own shards exactly once, so the
      split is seen once per epoch with no sample repeated or dropped. The dataset is unsized â€” ``len()`` raises
      :class:`TypeError`, as it does for any un-lengthed iterable â€” because the per-worker batch tails make the batch
      count depend on how shards happen to divide across workers.
    - **Planned** (what training uses): every worker yields exactly ``samples_per_worker`` samples, wrapping around
      its own shards if that subset is shorter. The epoch length is then fixed and ``len()`` is exact, which is what
      ``trainer.estimated_stepping_batches`` â€” and through it the LR schedule â€” needs. The trade-off is that a
      worker holding fewer shards than average repeats some of its samples within the epoch.

    Args:
        shard_dir: Directory holding the shards and the split's index file.
        split: Split name to read.
        transforms: Transform pipeline applied to ``(image, target)`` after annotation conversion, or ``None``.
        include_masks: Decode polygon/RLE segmentation into binary mask tensors.
        cat2label: ``category_id`` to label-index mapping. ``None`` uses the mapping implied by the shard index.
        shuffle_buffer: Reservoir size for within-shard shuffling; ``0`` disables it.
        shard_shuffle: Shards held in the shard-order shuffle buffer; ``0`` visits shards in packing order.
            Together with *shuffle_buffer* this is the streaming counterpart of ``shuffle=True`` on a map-style
            loader â€” a local shuffle, not a global permutation.
        seed: Rank-independent base seed for shard order. The loader uses a dedicated generator so rank-local
            random draws cannot change the pre-split permutation; see :meth:`_epoch_seeds`.
        draft_size: Smallest source extent the transform pipeline can consume without upscaling, applied by
            :func:`~rfdetr.datasets.io_utils.decode_image_bytes` the same way the loose-file loaders apply it, or
            ``None`` to decode at full resolution. See :func:`~rfdetr.datasets.coco.draft_size_for_transforms`
            for when a non-``None`` value is actually correct â€” only the train split, never a mask dataset.
    FNr   )Úinclude_masksÚ	cat2labelÚshuffle_bufferÚshard_shuffleÚseedÚ
draft_sizeÚ	shard_dirú
str | PathÚsplitr?   Ú
transformsú
Any | NonerE   ÚboolrF   údict[int, int] | NonerG   r$   rH   rI   rJ   ú
int | Noner   ÚNonec               óV  •— t          ¦   «                              ¦   «          t          |¦  «        | _        || _        || _        || _        || _        || _        |	| _	        t          | j        |¦  «        | _        | j        j        | _        |€| j                             ¦   «         nt          |¦  «        | _        | j        €d n"d„ | j                             ¦   «         D ¦   «         | _        t%          || j        ¬¦  «        | _        d | _        d| _        d| _        d | _        d | _        d S )Nc                ó   — i | ]\  }}||“Œ	S r,   r,   )Ú.0Úcat_idÚlabels      r"   ú
<dictcomp>z0WebDatasetDetection.__init__.<locals>.<dictcomp>Ñ   s!   € Ð=xÐ=xÐ=xÑP]ÐPVÐX]¸eÀVÐ=xÐ=xÐ=xr-   )rE   rF   r&   éÿÿÿÿ)ÚsuperÚ__init__r   Ú
_shard_dirr<   Ú_transformsÚ_shuffle_bufferÚ_shard_shuffleÚ_seedÚ_draft_sizer   r:   Ú
categoriesÚ_label_categoriesrF   ÚdictÚitemsÚ	label2catr   ÚprepareÚ_samples_per_workerÚ_planned_workersÚ_epoch_counterÚ_rankÚ_world_size)ÚselfrK   rM   rN   rE   rF   rG   rH   rI   rJ   Ú	__class__s             €r"   r\   zWebDatasetDetection.__init__¹   s  ø€ õ 	‰Œ×ÒÑÔÐÝ˜y™/œ/ˆŒØˆŒØ%ˆÔØ-ˆÔØ+ˆÔØˆŒ
Ø%ˆÔÝ% d¤o°uÑ=Ô=ˆŒ
Ø!%¤Ô!6ˆÔØ3<Ð3D˜œ×-Ò-Ñ/Ô/Ð/Í$ÈyÉ/Ì/ˆŒØ!%¤Ð!7˜˜Ð=xÐ=xÐaeÔao×auÒauÑawÔawÐ=xÑ=xÔ=xˆŒÝ"°È$Ì.ÐYÑYÔYˆŒØ/3ˆÔ Ø !ˆÔØ ˆÔØ!%ˆŒ
Ø'+ˆÔÐÐr-   c                ó   — | j         j        S )zNSamples the shard index reports for this split, independent of any epoch plan.)r:   Únum_samples©rn   s    r"   Útotal_samplesz!WebDatasetDetection.total_samplesÙ   s   € ð ŒzÔ%Ð%r-   ú	list[str]c                ó6   — t          | j        | j        ¦  «        S )uù  Category names in label order, using train metadata when built for evaluation.

        The map-style datasets expose the same thing through their ``coco`` object, which a shard stream has no
        equivalent of; the packed index carries the category list instead. The layout â€” one entry per emitted
        label, ``"remap"`` versus ``"raw"`` â€” is :func:`~rfdetr.datasets.webdataset.index.category_names_by_label`,
        applied to whichever category list and mapping this dataset was built with (its own index's, or the train
        split's when built for evaluation), so an explicit ``cat2label`` is honoured here too.

        Returns:
            The category names, indexed by label, with an empty string at every label with no category.
        )r   rd   rF   rr   s    r"   Úclass_nameszWebDatasetDetection.class_namesÞ   s   € õ ' tÔ'=¸t¼~ÑNÔNÐNr-   Úsamples_per_workerÚnum_workersc               ó†   — |dk     rt          d|› d�¦  «        ‚|dk     rt          d|› d�¦  «        ‚|| _        || _        dS )aE  Fix the epoch length so ``len()`` is exact.

        Args:
            samples_per_worker: Samples each DataLoader worker yields per epoch.
            num_workers: Workers the loader will run; ``0`` and ``1`` both mean one iterating process.

        Raises:
            ValueError: If either argument is below one.
        r&   z%samples_per_worker must be >= 1, got ú.znum_workers must be >= 1, got N)Ú
ValueErrorri   rj   )rn   rw   rx   s      r"   Úconfigure_epochz#WebDatasetDetection.configure_epochí   sc   € ð  Ò!Ð!ÝÐZÐEWÐZÐZÐZÑ[Ô[Ð[Ø˜Š?ˆ?ÝÐL¸kÐLÐLÐLÑMÔMÐMØ#5ˆÔ Ø +ˆÔÐÐr-   r2   r3   c               ó"   — || _         || _        dS )a  Fix this dataset's node split to a *rank*/*world_size* resolved once, in the main process.

        Not required for single-process use: :meth:`__iter__` falls back to ``webdataset.split_by_node``'s own
        per-process resolution when this is never called. See :func:`_fixed_node_splitter` for why a distributed
        loader should call this instead of relying on that per-process resolution.

        Args:
            rank: This process's rank.
            world_size: Total ranks sharing the split.
        N)rl   rm   )rn   r2   r3   s      r"   Úconfigure_distributionz*WebDatasetDetection.configure_distributionþ   s   € ð ˆŒ
Ø%ˆÔÐÐr-   c                ó^   — | j         €t          d| j        ›d�¦  «        ‚| j         | j        z  S )z¾Return the planned per-epoch sample count for this rank.

        Raises:
            TypeError: If no epoch was planned, matching how ``len()`` behaves on any un-lengthed iterable.
        NzWebDatasetDetection(u˜   ) has no planned epoch length. Call configure_epoch() â€” build_webdataset_loader() does it for the training loader â€” or treat the dataset as unsized.)ri   Ú	TypeErrorr<   rj   rr   s    r"   Ú__len__zWebDatasetDetection.__len__  sJ   € ð Ô#Ð+Ýð3 t¤{ð 3ð 3ð 3ñô ð ð
 Ô'¨$Ô*?Ñ?Ð?r-   útuple[int, int]c                ó  — | xj         dz  c_         t          j        j                             ¦   «         }|€| j        | j         z   dz  }||fS |j        |j        z
  dz  }|| j         z   dz  }|t          j        ¦   «         | j         z   dz  fS )a´  Return this epoch's ``(shard_order_seed, buffer_seed)``.

        The loader's dedicated generator starts identically on every rank. Restarted workers receive its next
        base seed; persistent workers retain their original WorkerInfo seed and advance a local epoch counter.
        Removing the worker ID from that immutable seed recovers the same pre-split permutation on all ranks.
        The initialization hook changes torch's current seed for rank-specific augmentation and buffer randomness
        without changing WorkerInfo. Main-process iteration instead advances the dataset's own base seed.

        All ranks must iterate the same epochs with the same worker configuration. Direct DataLoader users must
        likewise supply synchronized generators; build_webdataset_loader establishes this contract automatically.

        Returns:
            Seed shared by every worker this epoch, and a seed unique to this worker.
        r&   Nl        )	rk   r'   ÚutilsÚdataÚget_worker_infora   rI   ÚidÚinitial_seed)rn   Úworker_infoÚsharedÚ	base_seeds       r"   Ú_epoch_seedsz WebDatasetDetection._epoch_seeds  s™   € ð 	ÐÔ˜qÑ ÐÔÝ”kÔ&×6Ò6Ñ8Ô8ˆØÐØ”j 4Ô#6Ñ6¸5ÑAˆFØ˜6�>Ð!à Ô%¨¬Ñ6¸5ÑAˆ	Ø˜dÔ1Ñ1°eÑ<ˆØ�Ô*Ñ,Ô,¨tÔ/BÑBÀuÑMÐMÐMr-   Úsampleúdict[str, Any]útuple[Any, dict[str, Any]]c           	     óÌ  ‡‡	‡
— t          ˆfd„t          D ¦   «         d¦  «        }|€^t          d„ ‰D ¦   «         ¦  «        }t          d‰                     dd¦  «        › dd                     t          ¦  «        › d	|› d
�¦  «        ‚d‰vr't          d‰                     dd¦  «        › d�¦  «        ‚t          ‰|         | j        ¦  «        \  }\  Š	Š
t          j	        |¦  «        }t          j        ‰d         ¦  «        }|d         }‰	‰
fdk    rˆ	ˆ
fd„|D ¦   «         }|d         |dœ}|                      ||¦  «        \  }}| j        �|                      ||¦  «        \  }}||fS )u,  Turn one raw WebDataset sample into the ``(image, target)`` pair the transform pipeline expects.

        Decodes through :func:`~rfdetr.datasets.io_utils.decode_image_bytes`, the same entry point the loose-file
        loaders use, so a shard member gets the same ``simplejpeg``-or-Pillow choice and the same power-of-two
        reduced-scale decode when ``self._draft_size`` is set (train split only â€” see
        :func:`draft_size_for_transforms`), plus the same annotation rescale when a draft actually reduces the
        image.

        Args:
            sample: Raw sample dict keyed by member extension.

        Returns:
            Converted and transformed ``(image, target)``.

        Raises:
            KeyError: If the sample carries no recognised image member or no ``json`` sidecar.
        c              3  ó$   •K  — | ]
}|‰v ¯|V — Œd S ©Nr,   )rV   Ú	candidater�   s     €r"   ú	<genexpr>z.WebDatasetDetection._decode.<locals>.<genexpr>E  s.   øè è € Ð]Ð]¨	ÈÐV\ÐI\ÐI\˜)ÐI\ÐI\ÐI\ÐI\Ð]Ð]r-   Nc              3  óD   K  — | ]}|                      d ¦  «        °|V — ŒdS )Ú__N)Ú
startswith)rV   Úkeys     r"   r”   z.WebDatasetDetection._decode.<locals>.<genexpr>G  s3   è è € ÐOÐO S¸#¿.º.ÈÑ:NÔ:NÐO˜SÐOÐOÐOÐOÐOÐOr-   zWebDataset sample Ú__key__Ú?z! has no image member (looked for z, z	); found rz   Újsonz" has no 'json' annotation sidecar.r   )ç      ð?rœ   c                ó2   •— g | ]}t          |‰‰¦  «        ‘ŒS r,   )r   )rV   Ú
annotationÚx_scaleÚy_scales     €€r"   ú
<listcomp>z/WebDatasetDetection._decode.<locals>.<listcomp>T  s'   ø€ ÐmÐmÐmÐS]Õ0°¸WÀgÑNÔNÐmÐmÐmr-   Úimage_id)r¢   r   )Únextr   ÚsortedÚKeyErrorÚgetÚjoinr   rb   r   Ú	fromarrayr›   Úloadsrh   r^   )rn   r�   Ú	extensionÚpresentÚpixelsÚimageÚmetadatar   ÚtargetrŸ   r    s    `       @@r"   Ú_decodezWebDatasetDetection._decode3  s¹  øøø€ õ$ Ð]Ð]Ð]Ð]Õ5EÐ]Ñ]Ô]Ð_cÑdÔdˆ	ØÐÝÐOÐO¨FÐOÑOÔOÑOÔOˆGÝðP V§Z¢Z°	¸3Ñ%?Ô%?ð Pð PØ#ŸyšyÕ)9Ñ:Ô:ðPð PØELðPð Pð Pñô ð ð ˜ÐÐÝÐn°·
²
¸9ÀcÑ0JÔ0JÐnÐnÐnÑoÔoÐoå%7¸¸yÔ8IÈ4ÔK[Ñ%\Ô%\Ñ"ˆÑ"�˜'Ý” Ñ'Ô'ˆÝ”:˜f VœnÑ-Ô-ˆØ˜}Ô-ˆØ�WÐ Ò+Ð+ØmÐmÐmÐmÐmÐalÐmÑmÔmˆKØ.6°zÔ.BÐS^Ð!_Ð!_ˆØŸš U¨FÑ3Ô3‰ˆˆvØÔÐ'Ø ×,Ò,¨U°FÑ;Ô;‰MˆE�6Ø�fˆ}Ðr-   c                ó4   ‡ — ˆ fd„‰ j         j        D ¦   «         S )u
  Return this split's shards as ``file:`` URLs.

        See :func:`_shard_url` for why a shard is named as a ``file:`` URL rather than by its plain path, and
        :func:`resolve_within` for why each entry is validated against ``self._shard_dir`` before being opened â€”
        the index is on-disk JSON, not a trusted value.

        Returns:
            One ``file:`` URL per shard, in index order.

        Raises:
            ValueError: If a shard entry is absolute or resolves outside ``self._shard_dir``.
        c                óT   •— g | ]$}t          t          ‰j        |¦  «        ¦  «        ‘Œ%S r,   )rB   r   r]   )rV   Úshardrn   s     €r"   r¡   z3WebDatasetDetection._shard_urls.<locals>.<listcomp>h  s-   ø€ ÐbÐbÐbÀu•
�>¨$¬/¸5ÑAÔAÑBÔBÐbÐbÐbr-   )r:   Úshardsrr   s   `r"   Ú_shard_urlszWebDatasetDetection._shard_urls[  s'   ø€ ð cÐbÐbÐbÐPTÔPZÔPaÐbÑbÔbÐbr-   Úshard_orderú	list[int]c                óÖ  ‡
‡‡— | j         �| j         nt          ¦   «         }t          |¦  «        }|t          d|¦  «        z  Š| j        j        Št          ‰¦  «        |k    rVˆfd„|D ¦   «         Š
ˆ
ˆfd„t          ‰¦  «        D ¦   «         }t          |¦  «        }| j        ‰z  }d|dk    r||z  ndz
  }d}	nd|‰z  |‰z  z  z
  }d}	|t          k    rJt          d	| j        j        ›d
|› d|› dt          d|¦  «        › d|	rdnd› |dz  d›dt          d›d�¦  «        ‚|t          k    rBt                               d| j        j        ||t          d|¦  «        |	rdnd|dz  ‰¦  «         dS dS )a×  Reject or report imbalance in the shard order workers will actually consume.

        All ranks check every slot before any data is read, so an unsafe shuffled epoch fails consistently.
        Missing per-shard counts retain the existing shard-count estimate.

        Args:
            shard_order: Index positions in the pre-split epoch order, shared by all ranks and workers.
            num_workers: DataLoader workers per rank; zero means the main process.
        Nr&   c                ó    •— g | ]
}‰|         ‘ŒS r,   r,   )rV   ÚpositionÚ	per_shards     €r"   r¡   z9WebDatasetDetection._check_shard_skew.<locals>.<listcomp>}  s   ø€ ÐNÐNÐN°h˜i¨Ô1ÐNÐNÐNr-   c                ó@   •— g | ]}t          ‰|d ‰…         ¦  «        ‘ŒS r’   ©Úsum)rV   rº   Úordered_countsÚslotss     €€r"   r¡   z9WebDatasetDetection._check_shard_skew.<locals>.<listcomp>~  s,   ø€ Ð]Ð]Ð]ÀH�3˜~¨h¨o¸¨oÔ>Ñ?Ô?Ð]Ð]Ð]r-   rœ   r   TFúSplit ú has ú shard(s) for ú rank(s) x z- worker(s), so the worst-served worker holds Ú zan estimated r   z.0fu3   % fewer samples than the epoch asks of it â€” past z.0%zÖ, this is no longer a tuning nuisance worth a log line nobody watches live. Re-pack with a smaller --max-shard-mb (aim for a shard count that divides the rank/worker count, or simply many more shards than workers).am  Split %r has %d shards for %d rank(s) x %d worker(s), so the worst-served worker holds %s%.0f%% fewer samples than the epoch asks of it: it repeats some of its own while better-supplied workers leave some unseen, which measurably costs accuracy. Re-pack with a smaller --max-shard-mb (aim for a shard count that divides %d, or simply many more shards than workers).)rm   r.   ÚlenÚmaxr:   Úsamples_per_shardÚrangeÚminrs   ÚSHARD_SKEW_RAISE_FRACTIONr{   rM   ÚSHARD_SKEW_WARN_FRACTIONÚloggerÚwarning)rn   r¶   rx   Úranksr´   Úslot_totalsÚworstÚaverageÚdeficitÚmeasuredr¿   r»   rÀ   s             @@@r"   Ú_check_shard_skewz%WebDatasetDetection._check_shard_skewj  s  øøø€ ð %)Ô$4Ð$@�Ô Ð ÕF]ÑF_ÔF_ˆÝ�[Ñ!Ô!ˆØ�˜A˜{Ñ+Ô+Ñ+ˆØ”JÔ0ˆ	Ýˆy‰>Œ>˜VÒ#Ð#ð
 OÐNÐNÐNÀ+ÐNÑNÔNˆNØ]Ð]Ð]Ð]Ð]ÕPUÐV[ÑP\ÔP\Ð]Ñ]Ô]ˆKÝ˜Ñ$Ô$ˆEØÔ(¨5Ñ0ˆGØ°¸!²°˜U W™_˜_ÀÑEˆGØˆHˆHð ˜V u™_°¸%±Ñ@Ñ@ˆGØˆHØÕ.Ò.Ð.Ýð[˜œÔ)ð [ð [°&ð [ð [Èð [ð [Ý�q˜+Ñ&Ô&ð[ð [à!Ð6�2�2 ð[à8?À#¹Ðrð[ð [õ  9Ðpð[ð [ð [ñô ð ð Õ-Ò-Ð-Ý�NŠNðð
 ”
Ô ØØÝ�A�{Ñ#Ô#ØÐ3�� OØ˜#‘Øñô ð ð ð ð .Ð-r-   úIterator[tuple[Any, Any]]c                ó  ‡— t          ¦   «         }|                      ¦   «         \  }}|                      ¦   «         Š| j        dk    r€t	          t          t          ‰¦  «        ¦  «        ¦  «        }t          j        |¦  «         	                    |¦  «         ˆfd„|D ¦   «         Š| j
        �|                      || j        ¦  «         | j        �| j        €|j        nt!          | j        | j        ¦  «        }|                     ‰d| j        ||j        |¬¦  «        }| j        dk    r| 	                    | j        |¬¦  «        }|                     | j        ¦  «        }| j
        �|                     | j
        ¦  «        }t/          |¦  «        }|S )u3  Iterate this worker's share of the split.

        ``webdataset``'s own pipeline applies ``nodesplitter`` and ``workersplitter`` *before* its shard-order
        shuffler (``compat.WebDataset.__init__``, checked against the installed ``webdataset==1.0.2`` source): the
        node/worker split is a plain positional stride (``islice(src, rank, None, world_size)``, then the same
        over workers) over whatever order the shard list already has, and only the surviving per-worker subset
        gets shuffled afterwards. Passing ``shardshuffle=`` to :class:`webdataset.WebDataset` therefore never
        changes *which* shards a worker owns â€” it only reorders shards the split already fixed for the entire
        run, every epoch. Shuffling ``urls`` ourselves first, identically across every worker via the shared
        epoch seed, changes the input to the stride split itself, so the shard-to-worker assignment actually
        rotates every epoch instead of being frozen for the run.

        Returns:
            Iterator over ``(image, target)`` pairs.
        r   c                ó    •— g | ]
}‰|         ‘ŒS r,   r,   )rV   rº   Úurlss     €r"   r¡   z0WebDatasetDetection.__iter__.<locals>.<listcomp>¹  s   ø€ Ð?Ð?Ð? x�D˜”NÐ?Ð?Ð?r-   NF)Úempty_checkÚshardshuffleÚnodesplitterÚworkersplitterrI   )rI   )r#   rŒ   rµ   r`   ÚlistrÉ   rÆ   ÚrandomÚRandomÚshuffleri   rÕ   rj   rl   rm   Úsplit_by_noder=   Ú
WebDatasetÚsplit_by_workerr_   Úmapr°   Ú
with_epochÚiter)	rn   ÚwdsÚ
shard_seedÚbuffer_seedr¶   rÜ   ÚpipelineÚiteratorrÙ   s	           @r"   Ú__iter__zWebDatasetDetection.__iter__¡  s�  ø€ õ  "Ñ#Ô#ˆØ"&×"3Ò"3Ñ"5Ô"5Ñˆ
�KØ×ÒÑ!Ô!ˆØÔ Ò"Ð"õ �u¥S¨¡Y¤YÑ/Ô/Ñ0Ô0ˆKÝŒM˜*Ñ%Ô%×-Ò-¨kÑ:Ô:Ð:Ø?Ð?Ð?Ð?°;Ð?Ñ?Ô?ˆDØÔ'Ð3Ø×&Ò& {°DÔ4IÑJÔJÐJð ŒzÐ! TÔ%5Ð%=ð ÔÐå% d¤j°$Ô2BÑCÔCð 	ð
 —>’>Øð
 ØÔ,Ø%ØÔ.Øð "ñ 
ô 
ˆð Ô !Ò#Ð#Ø×'Ò'¨Ô(<À;Ð'ÑOÔOˆHØ—<’< ¤Ñ-Ô-ˆØÔ#Ð/Ø×*Ò*¨4Ô+CÑDÔDˆHÝ.2°8©n¬nˆØˆr-   )rK   rL   rM   r?   rN   rO   rE   rP   rF   rQ   rG   r$   rH   r$   rI   r$   rJ   rR   r   rS   ©r   r$   )r   rt   )rw   r$   rx   r$   r   rS   )r2   r$   r3   r$   r   rS   )r   r‚   )r�   rŽ   r   r�   )r¶   r·   rx   r$   r   rS   )r   rÖ   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r\   Úpropertyrs   rv   r|   r~   r�   rŒ   r°   rµ   rÕ   rí   Ú__classcell__)ro   s   @r"   rD   rD   •   sY  ø€ € € € € ð!ð !ðR $Ø+/ØØØØ!%ð,ð ,ð ,ð ,ð ,ð ,ð ,ð ,ð@ ð&ð &ð &ñ „Xð&ð ðOð Oð Oñ „XðOð,ð ,ð ,ð ,ð"&ð &ð &ð &ð@ð @ð @ð @ðNð Nð Nð Nð2&ð &ð &ð &ðPcð cð cð cð5ð 5ð 5ð 5ðn2ð 2ð 2ð 2ð 2ð 2ð 2ð 2r-   rD   r&   )r3   Úgrad_accum_stepsrs   Ú
batch_sizerx   rõ   c               óþ   — t          d|¦  «        }t          d|¦  «        }||t          ||¦  «        z  z  }| t          d|¦  «        |z  z  }|||z  z  }||k     r"t          d| › d|› d|› d|› d|› d|› d�¦  «        ‚|S )	uÆ  Return the per-worker epoch length that makes every emitted batch full.

    Each worker emits full micro-batches; accumulation windows belong to the rank, not each worker. Flooring to
    ``batch_size * grad_accum_steps / gcd(workers, grad_accum_steps)`` makes the aggregate rank batch count a
    multiple of the accumulation factor, so ``drop_last=True`` never actually drops anything and PTL never
    fires the optimizer on a partial accumulation window
    (https://github.com/Lightning-AI/pytorch-lightning/issues/19987) â€” the streaming counterpart of what
    :class:`~rfdetr.training.module_data.GradAccumAlignedDataset` pads the map-style loader to.

    Args:
        total_samples: Samples in the split, across all ranks.
        batch_size: Per-rank micro-batch size.
        num_workers: DataLoader workers per rank; ``0`` counts as one iterating process.
        world_size: Number of distributed ranks sharing the split.
        grad_accum_steps: Micro-batches accumulated per optimizer step.

    Returns:
        Samples each worker yields per epoch.

    Raises:
        ValueError: If the split cannot fill full worker batches and complete rank accumulation windows.

    Examples:
        >>> plan_samples_per_worker(1000, batch_size=4, num_workers=2)
        500
        >>> plan_samples_per_worker(1000, batch_size=16, num_workers=3)
        320
        >>> plan_samples_per_worker(1000, batch_size=4, num_workers=2, grad_accum_steps=8)
        496
    r&   zA split of z] samples cannot fill one accumulation window across workers: each worker needs a multiple of z samples (batch_size=z, grad_accum_steps=z	) across rÄ   zT worker(s). Lower num_workers, batch_size or grad_accum_steps, or pack more samples.)rÇ   r   r{   )	rs   rö   rx   r3   rõ   ÚworkersÚaccumulationÚwindowÚ
per_workers	            r"   Úplan_samples_per_workerrü   Ö  sã   € õB �!�[Ñ!Ô!€GÝ�qÐ*Ñ+Ô+€LØ˜<­3¨w¸Ñ+EÔ+EÑEÑF€FØ¥3 q¨*Ñ#5Ô#5¸Ñ#?Ñ@€JØ�*˜vÑ%Ñ%€JØ�FÒÐÝð˜-ð ð Ø/5ðð à%ðð à:Jðð àU_ðð ð !ðð ð ñ
ô 
ð 	
ð Ðr-   Ú	worker_idrø   Ú
initializeúCallable[[int], None] | NonerS   c               óð   — t          j        ¦   «         ||z  z   dz  }t          j        |¦  «         t          j                             |dz  ¦  «         t	          j        |¦  «         |� || ¦  «         dS dS )a5  Separate rank-local augmentation RNGs from the loader's shared shard-order seed.

    PyTorch stores its original seed in WorkerInfo before this callback. Preserve that immutable value for shard
    partitioning, then seed torch, NumPy and random for this global worker and invoke the caller's hook last.
    l            l        N)r'   rˆ   Úmanual_seedÚnprß   rI   )rý   r2   rø   rþ   rI   s        r"   Ú_seed_streaming_workerr    s}   € õ ÔÑ Ô  4¨'¡>Ñ1°eÑ<€DÝ	Ô�dÑÔÐÝ„I‡N‚N�4˜5‘>Ñ"Ô"Ð"Ý
„K�ÑÔÐØÐØˆ
�9ÑÔÐÐÐð Ðr-   FT)Ú
pin_memoryÚpersistent_workersÚprefetch_factorÚworker_init_fnÚfixed_epochr3   r2   rõ   ÚdatasetÚ
collate_fnú2Callable[[list[tuple[Any, Any]]], tuple[Any, ...]]r  rP   r  r  rR   r  r  úDataLoader[Any]c               ó   ‡‡— t          ¦   «          |	€t          ¦   «         n|	Š|
€t          ¦   «         n|
}|                      |‰¬¦  «         t	          | j        j        ¦  «        }‰|k    r#t          d| j        j        ›d|› d‰› d�¦  «        ‚|�rZ|}‰t          d|¦  «        z  }||k    r4t          d| j        j        ›d|› d‰› d	t          d|¦  «        › d
�	¦  «        ‚| j
        dk    r0|                      t          t          |¦  «        ¦  «        |¦  «         t          | j        ||‰|¬¦  «        }t                                d| j        j        ||||z  | j        ||¦  «         ||z  }d| j        dk    r
|| j        z  ndz
  }|t$          k    r4t                                d| j        j        | j        ||||dz  ||¦	  «	         |                      |t          d|¦  «        ¬¦  «         nŸ‰dk    r™| j        j        Št	          ‰¦  «        |k    rzˆˆfd„t          ‰¦  «        D ¦   «         }t          |¦  «        t-          |¦  «        }}|dk    r:||z
  |z  t$          k    r)t                                d| j        j        ‰||‰¦  «         |||||t/          t0          |t          d|¦  «        |¬¦  «        t3          j        ¦   «                              | j        ¦  «        dœ}|dk    r||d<   |�||d<   t;          | fi |¤ŽS )uÏ
  Build the loader that streams *dataset*.

    This returns a stock :class:`~torch.utils.data.DataLoader` rather than ``webdataset.WebLoader``. ``WebLoader`` is
    a fluid-interface wrapper that *constructs* the very same ``DataLoader`` and then hides it behind a
    ``DataPipeline``, which drops both ``__len__`` and the ``DataLoader`` type. RF-DETR needs the length:
    ``trainer.estimated_stepping_batches`` derives the LR schedule and the drop schedule from it, and ``WebLoader``
    only offers ``with_length()``, which fakes a length without guaranteeing it. The value ``WebLoader`` adds on top â€”
    post-loader ``unbatched().shuffle().batched()`` rebatching across workers â€” would break exactly that length
    guarantee. Streaming behaviour is identical either way: it comes from the pipeline inside *dataset*, not from the
    loader class.

    ``webdataset`` is imported here even though the loader itself does not use it, so a missing optional extra fails
    in the main process with an actionable message instead of inside a worker on the first batch.

    Args:
        dataset: The streaming dataset to wrap.
        batch_size: Per-rank micro-batch size.
        collate_fn: Batch collation callable, normally the DataModule's block-size-aware one.
        num_workers: DataLoader worker processes.
        pin_memory: Stage batches in pinned host memory before the device copy.
        persistent_workers: Keep workers alive between epochs.
        prefetch_factor: Batches prefetched per worker, or ``None`` for the DataLoader default.
        worker_init_fn: Per-worker initialisation hook. The CPU augmentation stack draws from NumPy and ``random``,
            which PyTorch does not seed per worker, so this needs the same seeding hook the map-style loaders use.
        fixed_epoch: Plan a fixed-length epoch and drop partial batches. ``True`` for training, where a known length
            drives the LR schedule; ``False`` for evaluation, where every sample must be seen exactly once.
        world_size: Distributed ranks sharing the split. ``None`` reads it from the active process group.
        rank: This process's rank. ``None`` reads it from the active process group. See
            :func:`_fixed_node_splitter` for why the loader resolves this once here rather than letting each
            DataLoader worker re-resolve it independently.
        grad_accum_steps: Micro-batches accumulated per optimizer step. Only used when *fixed_epoch* is ``True``;
            see :func:`plan_samples_per_worker`.

    Returns:
        A ``DataLoader`` over *dataset*.

    Raises:
        ValueError: If the split has fewer shards than distributed ranks, or if a planned epoch would leave a
            worker with no shard to read.
    N)r2   r3   rÁ   rÂ   rÃ   zÊ rank(s): a rank with no shard never reaches the step function while the others do, which deadlocks the process group. Re-pack with a smaller --max-shard-mb so the split has at least one shard per rank.r&   z shard(s), which cannot cover rÄ   zµ worker(s): a worker left with no shard would yield nothing and silently shorten the epoch. Lower num_workers, or re-pack with a smaller --max-shard-mb so the split has more shards.r   )rö   rx   r3   rõ   z—Split %r: fixed training epoch plans %d sample(s)/worker x %d slot(s) = %d of %d total sample(s) seen this epoch (batch_size=%d x grad_accum_steps=%d).rœ   a{  Split %r has %d samples, but a fixed epoch of %d worker(s) x %d sample(s) sees only %d (%.0f%% never seen this epoch): flooring to a whole accumulation window (batch_size=%d x grad_accum_steps=%d) discards the remainder every epoch, unlike the map-style loader's padding. Lower num_workers, batch_size or grad_accum_steps, or pack more samples, to shrink the discarded remainder.r   )rw   rx   c                ó@   •— g | ]}t          ‰|d ‰…         ¦  «        ‘ŒS r’   r½   )rV   rº   r»   rÏ   s     €€r"   r¡   z+build_webdataset_loader.<locals>.<listcomp>Ÿ  s,   ø€ ÐXÐXÐX¸x�3˜y¨¨°5¨Ô9Ñ:Ô:ÐXÐXÐXr-   aâ  Split %r has an uneven per-rank sample split for evaluation across %d rank(s): the busiest rank holds %d samples against the quietest rank's %d, so they produce different numbers of evaluation batches. This module does not equalize per-rank evaluation batch counts, so a rank that reaches epoch-end collectives before its busiest sibling can wait indefinitely there. A simple mean-of-per-rank-means reduction over a logged metric (not mAP, which this training loop accumulates through a dedicated, correctly sample-weighted metric object) would also under- or over-weight the quieter rank's batches relative to its actual sample count. Re-pack with a smaller --max-shard-mb (aim for a shard count that divides %d) to reduce the imbalance.)r2   rø   rþ   )rö   Ú	drop_lastr
  rx   r  r  Ú	generatorr  r  )r#   r.   r1   r~   rÆ   r:   r´   r{   rM   rÇ   r`   rÕ   rÞ   rÉ   rü   rs   rÍ   ÚinforÌ   rÎ   r|   rÈ   rÊ   r   r  r'   Ú	Generatorr  ra   r   )r	  rö   r
  rx   r  r  r  r  r  r3   r2   rõ   Úresolved_rankÚshard_countr´   rÀ   rw   ÚseenÚ
floor_lossÚrank_totalsÚbusiestÚquietestÚloader_kwargsr»   rÏ   s                          @@r"   Úbuild_webdataset_loaderr    sä  øø€ õn ÑÔÐØ)3Ð);Õ#Ñ%Ô%Ð%À€EØ+/¨<Õ%Ñ'Ô'Ð'¸T€MØ×"Ò"¨À%Ð"ÑHÔHÐHÝ�g”mÔ*Ñ+Ô+€KØˆ{ÒÐõ
 ðZ�W”]Ô(ð Zð Z°ð Zð ZÈEð Zð Zð Zñ
ô 
ð 	
ð
 ñ TØˆØ�˜A˜{Ñ+Ô+Ñ+ˆØ�6Š>ˆ>Ýð?˜œÔ,ð ?ð ?°Vð ?ð ?Øð?ð ?Ý%(¨¨KÑ%8Ô%8ð?ð ?ð ?ñô ð ð Ô! QÒ&Ð&Ø×%Ò%¥d­5°©=¬=Ñ&9Ô&9¸;ÑGÔGÐGÝ4ØÔ!Ø!Ø#ØØ-ð
ñ 
ô 
Ðõ 	�ŠðOàŒMÔØØØ Ñ&ØÔ!ØØñ
	
ô 
	
ð 
	
ð " EÑ)ˆð ¸GÔ<QÐTUÒ<UÐ<U˜D 7Ô#8Ñ8Ð8Ð[^Ñ_ˆ
ØÕ0Ò0Ð0Ý�NŠNð>ð
 ”Ô#ØÔ%ØØ"ØØ˜SÑ ØØ ñô ð ð 	×ÒÐ3EÕSVÐWXÐZeÑSfÔSfÐÑgÔgÐgÐgØ	�Šˆð ”MÔ3ˆ	Ýˆy‰>Œ>˜[Ò(Ð(ØXÐXÐXÐXÐXÍ5ÐQVÉ<Ì<ÐXÑXÔXˆKÝ # KÑ 0Ô 0µ#°kÑ2BÔ2B�XˆGØ˜Š{ˆ{ ¨(Ñ 2°gÑ=Õ@XÒXÐXÝ—’ð;ð ”MÔ'ØØØØñô ð ð" !Ø Ø Ø"Ø Ý!Ý"¨ÅÀAÀ{Ñ@SÔ@SÐ`nð
ñ 
ô 
õ ”_Ñ&Ô&×2Ò2°7´=ÑAÔAð%ð %€Mð �Q‚€Ø.@ˆÐ*Ñ+ØÐ&Ø/>ˆMÐ+Ñ,Ý�gÐ/Ð/ Ð/Ð/Ð/r-   Ú	image_setÚargsÚ
resolutionc                óô  — t          |j        ¦  «        }|                     ¦   «         st          d|› d�¦  «        ‚t	          |dd¦  «        rt          d¦  «        ‚|                      dd¬¦  «        d	         }|d
k    }t	          |dd¦  «        }t	          |dd¦  «        }t	          |dd¦  «        }t          t          t	          |dd¦  «        ¦  «        ¦  «        }	t	          |dd¦  «        rt          nt          }
t          j        t	          |dd¦  «        ¦  «        } |
| ||t          j        ut	          |dd¦  «        |t          j        ut	          |dd¦  «        t	          |dd¦  «        |||	d¬¦  «        }|rd}n•	 t          |d
¦  «        }n,# t           $ r}t!          d|›d|› d|›d�¦  «        |‚d}~ww xY wt          ||¦  «        }|j        |j        k    r#t%          d|›d|j        ›d |j        ›d!�¦  «        ‚|                     ¦   «         }t)          | |t          j        t	          |dd¦  «        ¦  «        t          j        ut	          |dd¦  «        t	          |dd¦  «        t	          |dd¦  «        ||¬"¦  «        }t*                               d#| ||¦  «         t/          ||||||rt0          nd	|rt2          nd	t5          t	          |d$d	¦  «        pd	¦  «        |¬%¦	  «	        }|s|j        |_        |S )&u  Build the WebDataset-backed dataset for *image_set*.

    Reuses the loose-file transform pipeline unchanged, so a packed split trains through exactly the same CPU
    augmentation stack â€” including the optional Albumentations backend â€” as the directory it was packed from.

    Non-train splits adopt the train split's label mapping, for the same reason
    :func:`~rfdetr.datasets.coco.build_roboflow_from_coco` does: deriving indices per split shifts them whenever a
    split's annotation coverage of a grouping category differs from the train split's.

    Args:
        image_set: Split identifier, optionally suffixed (``"val_speed"`` reads the ``val`` shards).
        args: Merged model/train namespace, as built by :func:`rfdetr._namespace._namespace_from_configs`.
        resolution: Target square resolution in pixels.

    Returns:
        The streaming dataset for that split.

    Raises:
        FileNotFoundError: If ``args.dataset_dir`` does not exist.
        NotImplementedError: If keypoint training is requested.
    zWebDataset shard directory z does not existÚuse_grouppose_keypointsFzídataset_file='webdataset' does not support keypoint training: the keypoint label space is inferred from a whole parsed COCO annotation file, which a shard index does not carry. Use dataset_file='coco', 'roboflow' or 'yolo' for keypoints.Ú_r&   )Úmaxsplitr   ÚtrainÚsegmentation_headÚ
aug_configNÚscale_jitterTÚaugmentation_backendÚcpuÚsquare_resize_div_64Úmulti_scaleÚexpanded_scalesÚ
patch_sizeé   Únum_windowsé   )	r*  r+  Úskip_random_resizer,  r.  r%  r&  Úgpu_postprocessÚkeypoint_flip_pairszEvaluating split z! needs a packed 'train' index in u4    to adopt its label space (category_ids policy) â€” z… being packed is not enough by itself. Pack 'train' too, or pass cat2label explicitly to WebDatasetDetection to bypass this adoption.rÁ   z was packed with category_ids=z% but the train split was packed with z°. The two label spaces do not match, so evaluation would score predictions against different class indices than training used. Re-pack both splits with the same --category-ids.)r*  r+  r,  r.  r&  rE   z7Building WebDataset %s dataset at resolution %d from %srI   )rN   rE   rF   rG   rH   rI   rJ   )r   Údataset_dirÚexistsÚFileNotFoundErrorÚgetattrÚNotImplementedErrorrM   r   r   r   r   r   Ú
from_valueÚOFFÚ
PER_SAMPLEr   r   Úcategory_idsr{   rF   r   rÍ   r  rD   ÚDEFAULT_SHUFFLE_BUFFERÚDEFAULT_SHARD_SHUFFLEr$   rc   rd   )r  r  r  ÚrootrM   Úis_trainrE   r%  r&  r1  Útransform_factoryr*  rN   rF   Útrain_indexr!   Úsplit_indexrJ   r	  s                      r"   Úbuild_webdatasetrC  Å  s«  € õ, �Ô Ñ!Ô!€DØ�;Š;‰=Œ=ð UÝÐ S¸dÐ SÐ SÐ SÑTÔTÐTÝˆtÐ.°Ñ6Ô6ð 
Ý!ð2ñ
ô 
ð 	
ð �OŠO˜C¨!ˆOÑ,Ô,¨QÔ/€EØ˜Ò€HÝ˜DÐ"5°uÑ=Ô=€MÝ˜˜|¨TÑ2Ô2€JÝ˜4 °Ñ6Ô6€LÝ(Õ)BÅ7È4ÐQgÐinÑCoÔCoÑ)pÔ)pÑqÔq€Oå.5°dÐ<RÐTYÑ.ZÔ.ZÐtÕ*Ð*Õ`tð õ Ô'­°°mÀUÑ(KÔ(KÑLÔL€KØ"Ð"ØØØ¥z¤~Ð5Ý Ð&7¸Ñ?Ô?Ø&­jÔ.CÐCÝ˜4 ¨rÑ2Ô2Ý˜D -°Ñ3Ô3ØØ!Ø'Ø ðñ ô €Jð ð ,Øˆ	ˆ	ð	Ý*¨4°Ñ9Ô9ˆKˆKøÝ.ð 	ð 	ð 	Ý1ð] Eð ]ð ]Èdð ]ð ]Ø-2ð]ð ]ð ]ñô ð ð	øøøøð	øøøõ ' t¨UÑ3Ô3ˆØÔ# {Ô'?Ò?Ð?Ýð0˜ð 0ð 0ÀÔ@Xð 0ð 0Ø#.Ô#;ð0ð 0ð 0ñô ð ð  ×)Ò)Ñ+Ô+ˆ	Ý*ØØÝÔ)­'°$¸ÀuÑ*MÔ*MÑNÔNÕV`ÔVdÐdÝ Ð&7¸Ñ?Ô?Ý˜4 ¨rÑ2Ô2Ý˜D -°Ñ3Ô3Ø!Ø#ð	ñ 	ô 	€Jõ ‡K‚KÐIÈ9ÐV`ÐbfÑgÔgÐgÝ!ØØØØ#ØØ19Ð@Õ-Ð-¸qØ/7Ð>Õ+Ð+¸QÝ•˜˜v qÑ)Ô)Ð.¨QÑ/Ô/Øð
ñ 
ô 
€Gð ð ;à$/Ô$:ˆÔ!Ø€Ns   Æ F Æ
F:ÆF5Æ5F:)r   r
   rî   )r2   r$   r3   r$   r   r4   )r>   r	   r   r?   )rs   r$   rö   r$   rx   r$   r3   r$   rõ   r$   r   r$   )
rý   r$   r2   r$   rø   r$   rþ   rÿ   r   rS   )r	  rD   rö   r$   r
  r  rx   r$   r  rP   r  rP   r  rR   r  rÿ   r  rP   r3   rR   r2   rR   rõ   r$   r   r  )r  r?   r  r
   r  r$   r   rD   )@rò   Ú
__future__r   r›   rß   Úcollections.abcr   r   Ú	functoolsr   Úmathr   Úpathlibr   r	   Útypingr
   Únumpyr  r'   Útorch.utils.dataÚPILr   r   Úrfdetr.configr   Úrfdetr.datasets.cocor   r   r   r   r   Úrfdetr.datasets.io_utilsr   Ú!rfdetr.datasets.kornia_transformsr   r   Ú rfdetr.datasets.webdataset.indexr   r   r   r   r   Úrfdetr.utilities.loggerr   rÍ   r<  rÌ   rË   r=  r#   r.   r1   r=   rB   r„   r…   ÚIterableDatasetÚtuplerD   rü   r  r  rC  r,   r-   r"   ú<module>rU     s5  ððð ð #Ð "Ð "Ð "Ð "Ð "à €€€Ø €€€Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø "Ð "Ð "Ð "Ð "Ð "Ð "Ð "Ø Ð Ð Ð Ð Ð à Ð Ð Ð Ø €€€Ø Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø 'Ð 'Ð 'Ð 'Ð 'Ð 'à $Ð $Ð $Ð $Ð $Ð $ðð ð ð ð ð ð ð ð ð ð ð ð ð ð 8Ð 7Ð 7Ð 7Ð 7Ð 7Ø [Ð [Ð [Ð [Ð [Ð [Ð [Ð [ðð ð ð ð ð ð ð ð ð ð ð ð ð ð /Ð .Ð .Ð .Ð .Ð .à	ˆ‰Œ€ð Ð ð  Ð ð !Ð ð Ð ðð ð ð ðð ð ð ðð ð ð ðð ð ð ð<%ð %ð %ð %ð:~ð ~ð ~ð ~ð ~˜%œ+Ô*Ô:¸5ÀÀcÀ¼?ÔKñ ~ô ~ð ~ðD
 QRÐklð.ð .ð .ð .ð .ð .ðbð ð ð ð, Ø$Ø"&Ø37ØØ!ØØðk0ð k0ð k0ð k0ð k0ð k0ð\lð lð lð lð lð lr-   