§
    ·¯¸j(K  ã            ]       ó^  — d Z ddlZddlmZ ddlmZ ddlmZmZm	Z	m
Z
mZmZmZ ddlZddlmZmZ ddlmZmZ dd	lmZmZ dd
lmZ ddlmZ  G d„ d¦  «        Zddddddddddddddddddddddddddeedddddddddddej         ej        d¦  «        dddf,deeeeef         ef                  dee
e                  dee
e                   deed f         d!ed"ed#e!d$ee         d%e!d&e d'e"d(ed)e!d*ee"         d+eee e f                  d,eee e f                  d-e d.e d/e d0ee          d1e d2e d3ee"         d4ed5ed6e"d7ee d f         d8ee d f         d9ee          d:ee"         d;ee         d<ed=e!d>ed?ed@edAedBe!dCe!dDej#        dEee"ej        f         dFe!dGe"dHe!dIeej$        j%        j&        ef         fZdJ„Z'dS )Ka¸  NaFlex data loader for dynamic sequence length training.

This module provides a specialized data loader for Vision Transformer models that supports:
- Dynamic sequence length sampling during training for improved efficiency
- Variable patch size training with probabilistic selection
- Patch-level random erasing augmentation
- Efficient GPU prefetching with normalization

Hacked together by / Copyright 2025, Ross Wightman, Hugging Face
é    N)Úsuppress)Úpartial)ÚCallableÚDictÚIteratorÚListÚOptionalÚTupleÚUnioné   )ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)Ú_worker_initÚadapt_to_chs)ÚNaFlexMapDatasetWrapperÚNaFlexCollator)ÚPatchRandomErasing)Úcreate_transformc                   ót  — e Zd ZdZeed ej        d¦  «        dddddd	f
d
ej        j	        j
        deedf         deedf         dedej        deej                 dedededededdfd„Zdeeeeej        f         ej        f                  fd„Zdefd„Zed„ ¦   «         Zed„ ¦   «         Zed„ ¦   «         ZdS )ÚNaFlexPrefetchLoaderz;Data prefetcher for NaFlex format which normalizes patches.é   ÚcudaNç        Úconstr   r   TÚloaderÚmean.ÚstdÚchannelsÚdeviceÚ	img_dtypeÚre_probÚre_modeÚre_countÚre_num_splitsÚpatchify_channels_lastÚreturnc                 óÈ  — || _         || _        |pt          j        | _        t          ||¦  «        }t          ||¦  «        }|| _        || _        |rdd|fnd|df}t          j        d„ |D ¦   «         || j        ¬¦  «         	                    |¦  «        | _
        t          j        d„ |D ¦   «         || j        ¬¦  «         	                    |¦  «        | _        |dk    rt          |||	|
|¬¦  «        | _        nd| _        |j        dk    ot          j                             ¦   «         | _        |j        d	k    ot          j                             ¦   «         | _        d| _        dS )
a“  Initialize NaFlexPrefetchLoader.

        Args:
            loader: DataLoader to prefetch from.
            mean: Mean values for normalization.
            std: Standard deviation values for normalization.
            channels: Number of image channels.
            device: Device to move tensors to.
            img_dtype: Data type for image tensors.
            re_prob: Random erasing probability.
            re_mode: Random erasing mode.
            re_count: Maximum number of erasing rectangles.
            re_num_splits: Number of augmentation splits.
            patchify_channels_last: Per-patch flat layout produced by the
                upstream Patchify. ``True`` (default): channel index is the
                innermost (P-P-C). ``False``: channel index is the outermost
                (C-P-P). Controls how ``patches`` tensors are viewed for
                normalization and erasing.
        r   c                 ó   — g | ]}|d z  ‘ŒS ©éÿ   © ©Ú.0Úxs     úg/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/data/naflex_loader.pyú
<listcomp>z1NaFlexPrefetchLoader.__init__.<locals>.<listcomp>M   s   € Ð#Ð#Ð#˜ˆQ�‰WÐ#Ð#Ð#ó    )r   Údtypec                 ó   — g | ]}|d z  ‘ŒS r)   r+   r,   s     r/   r0   z1NaFlexPrefetchLoader.__init__.<locals>.<listcomp>O   s   € Ð"Ð"Ð"˜ˆQ�‰WÐ"Ð"Ð"r1   r   )Ú
erase_probÚmodeÚ	max_countÚ
num_splitsr   Nr   Únpu)r   r   ÚtorchÚfloat32r    r   r   r%   ÚtensorÚviewr   r   r   Úrandom_erasingÚtyper   Úis_availableÚis_cudar8   Úis_npuÚstream)Úselfr   r   r   r   r   r    r!   r"   r#   r$   r%   Únormalization_shapes                r/   Ú__init__zNaFlexPrefetchLoader.__init__   sƒ  € ðB ˆŒØˆŒØ"Ð3¥e¤mˆŒõ ˜D (Ñ+Ô+ˆÝ˜3 Ñ)Ô)ˆØ ˆŒØ&<ˆÔ#Ø2HÐ^˜q ! XÐ.Ð.ÈqÐRZÐ\]ÐN^ÐÝ”LØ#Ð#˜dÐ#Ñ#Ô#¨F¸$¼.ðJñ Jô JßJNÊ$ÐObÑJcÔJcð 	Œ	å”<Ø"Ð"˜cÐ"Ñ"Ô"¨6¸¼ðIñ Iô IßIMÊÐNaÑIbÔIbð 	Œð �RŠ<ˆ<Ý"4Ø"ØØ"Ø(Øð#ñ #ô #ˆDÔÐð #'ˆDÔð ”{ fÒ,ÐJµ´×1HÒ1HÑ1JÔ1JˆŒØ”k UÒ*ÐG­u¬y×/EÒ/EÑ/GÔ/GˆŒØˆŒˆˆr1   c           
   #   ó”  K  — d}d}| j         rt          j        }n| j        rt          j        }|�^| j        € |                     | j        ¬¦  «        | _        t          |j        | j        ¬¦  «        }| 	                    | j        ¬¦  «        }n	t          }d}| j        D �]\  }}|�| j                             |¦  «          |¦   «         5  |                     ¦   «         D ]T\  }}t          |t          j        ¦  «        r5|dk    r| j        nd}	||                              | j        d|	¬¦  «        ||<   ŒU|                     | j        d¬¦  «        }|d         }
|
j        }|
j        dk    rI|\  }}}| j        r|
                     ||d	| j        ¦  «        }né|
                     ||| j        d	¦  «        }nË|
j        d
k    r¨|dd…         \  }}| j        rJ|d	         | j        k    sJ d| j        › d|d	         › �¦   «         ‚|
                     ||d	| j        ¦  «        }nb|d         | j        k    sJ d| j        › d|d         › �¦   «         ‚|
                     ||| j        d	¦  «        }nt/          d|
j        › d�¦  «        ‚|                     | j        ¦  «                             | j        ¦  «        }| j        ��| j        s(|                     dd	¦  «                             ¦   «         }|                      ||d         |                     dd¦  «        ¬¦  «        }| j        s(|                     dd	¦  «                             ¦   «         }|                     |¦  «        |d<   ddd¦  «         n# 1 swxY w Y   |s||fV — nd}|�5| 	                    | j        ¬¦  «        }|                     | j        ¦  «         |}|}�Œ |s||fV — dS dS )zœIterate through the loader with prefetching and normalization.

        Yields:
            Tuple of (input_dict, targets) with normalized patches.
        TN)r   )rB   Úpatches)r   Únon_blockingr2   )r   rH   r   éÿÿÿÿé   é   z	Expected z channels, got z&Unexpected patches tensor dimensions: z. Expected 3 or 5.éþÿÿÿÚpatch_coordÚpatch_valid)rM   rN   F) r@   r9   r   rA   r8   rB   ÚStreamr   r   Úcurrent_streamr   r   Úwait_streamÚitemsÚ
isinstanceÚTensorr    ÚtoÚshapeÚndimr%   r<   r   Ú
ValueErrorÚsubr   Údivr   r=   Ú	transposeÚ
contiguousÚget)rC   ÚfirstÚ
device_modÚstream_contextÚconsumer_streamÚnext_input_dictÚnext_targetÚkÚvr2   Úpatches_tensorÚoriginal_shapeÚ
batch_sizeÚnum_patchesÚ_rG   Ú
input_dictÚtargets                     r/   Ú__iter__zNaFlexPrefetchLoader.__iter__a   s¹  è è € ð ˆàˆ
ØŒ<ð 	#ÝœˆJˆJØŒ[ð 	#ÝœˆJàÐ!ØŒ{Ð"ð )×/Ò/°t´{Ð/ÑCÔC�”Ý$ ZÔ%6¸t¼{ÐKÑKÔKˆNØ(×7Ò7¸t¼{Ð7ÑKÔKˆOˆOå%ˆNØ"ˆOà,0¬Kð O	!ñ O	!Ñ(ˆO˜[ØÐ*ð ”×'Ò'¨Ñ8Ô8Ð8à�Ñ!Ô!ð =Jð =Jà+×1Ò1Ñ3Ô3ð ð ‘D�A�qÝ! !¥U¤\Ñ2Ô2ð Ø23°y².°. ¤ Àd˜Ø-<¸QÔ-?×-BÒ-BØ#'¤;Ø)-Ø"'ð .Cñ .ô .˜¨Ñ*øð *Ÿnšn°D´KÈd˜nÑSÔS�ð
 "1°Ô!;�Ø!/Ô!5�à!Ô&¨!Ò+Ð+à1?Ñ.�J ¨QØÔ2ð bà"0×"5Ò"5°jÀ+ÈrÐSWÔS`Ñ"aÔ"a˜˜ð #1×"5Ò"5°jÀ+ÈtÌ}Ð^`Ñ"aÔ"a˜˜Ø#Ô(¨AÒ-Ð-à.<¸R¸a¸RÔ.@Ñ+�J ØÔ2ð 	bà-¨bÔ1°T´]ÒBÐBÐBØZ¨¬ÐZÐZÀnÐUWÔFXÐZÐZñ  CÔBÐBà"0×"5Ò"5°jÀ+ÈrÐSWÔS`Ñ"aÔ"a˜˜ð  .¨aÔ0°D´MÒAÐAÐAØY¨¬ÐYÐYÀnÐUVÔFWÐYÐYñ  BÔAÐAà"0×"5Ò"5°jÀ+ÈtÌ}Ð^`Ñ"aÔ"a˜˜å$Ð%uÈnÔNaÐ%uÐ%uÐ%uÑvÔvÐvð "Ÿ+š+ d¤iÑ0Ô0×4Ò4°T´XÑ>Ô>�àÔ&Ð2ð  Ô6ð IØ")×"3Ò"3°B¸Ñ";Ô";×"FÒ"FÑ"HÔ"H˜Ø"×1Ò1ØØ$3°MÔ$BØ$3×$7Ò$7¸ÀtÑ$LÔ$Lð 2ñ ô �Gð
  Ô6ð IØ")×"3Ò"3°B¸Ñ";Ô";×"FÒ"FÑ"HÔ"H˜ð .5¯\ª\¸.Ñ-IÔ-I� 	Ñ*ð{=Jð =Jð =Jñ =Jô =Jð =Jð =Jð =Jð =Jð =Jð =Jøøøð =Jð =Jð =Jð =Jð~ ð Ø  &Ð(Ð(Ð(Ð(Ð(à�àÐ*Ø",×";Ò";À4Ä;Ð";Ñ"OÔ"O�Ø×+Ò+¨D¬KÑ8Ô8Ð8à(ˆJØ ˆF‰Fàð 	%Ø˜fÐ$Ð$Ð$Ð$Ð$Ð$ð	%ð 	%s   Ã
JM+Í+M/	Í2M/	c                 ó*   — t          | j        ¦  «        S )zhGet length of underlying loader.

        Returns:
            Number of batches in the loader.
        )Úlenr   ©rC   s    r/   Ú__len__zNaFlexPrefetchLoader.__len__Î   s   € õ �4”;ÑÔÐr1   c                 ó   — | j         j        S )zrGet sampler from underlying loader.

        Returns:
            Sampler from the underlying DataLoader.
        )r   Úsamplerrp   s    r/   rs   zNaFlexPrefetchLoader.samplerÖ   ó   € ð Œ{Ô"Ð"r1   c                 ó   — | j         j        S )z)Get batch sampler from underlying loader.)r   Úbatch_samplerrp   s    r/   rv   z"NaFlexPrefetchLoader.batch_samplerß   s   € ð Œ{Ô(Ð(r1   c                 ó   — | j         j        S )zrGet dataset from underlying loader.

        Returns:
            Dataset from the underlying DataLoader.
        )r   Údatasetrp   s    r/   rx   zNaFlexPrefetchLoader.datasetä   rt   r1   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r9   r   ÚutilsÚdataÚ
DataLoaderr
   ÚfloatÚintr	   r2   ÚstrÚboolrE   r   r   rT   rm   rq   Úpropertyrs   rv   rx   r+   r1   r/   r   r      s´  € € € € € ØEÐEð
 '<Ø%9ØØ#/ 5¤<°Ñ#7Ô#7Ø/3ØØ"ØØ!"Ø+/ðAð Aà”KÔ$Ô/ðAð ˜˜s˜
Ô#ðAð �u˜c�zÔ"ð	Að
 ðAð ”LðAð   ¤Ô,ðAð ðAð ðAð ðAð ðAð %)ðAð ðAð Að Að AðFk%˜( 5¨¨c°5´<Ð.?Ô)@À%Ä,Ð)NÔ#OÔPð k%ð k%ð k%ð k%ðZ ˜ð  ð  ð  ð  ð ð#ð #ñ „Xð#ð ð)ð )ñ „Xð)ð ð#ð #ñ „Xð#ð #ð #r1   r   )é€   é   é@  i  i   r‡   é    Fr   r   g      à?gš™™™™™Ù?Úbilinearé   é*   Tr   ÚallÚ
patch_sizeÚpatch_size_choicesÚpatch_size_choice_probsÚtrain_seq_lens.Úmax_seq_lenrh   Úis_trainingÚmixup_fnÚno_augr!   r"   r#   Úre_splitÚtrain_crop_modeÚscaleÚratioÚhflipÚvflipÚcolor_jitterÚcolor_jitter_probÚgrayscale_probÚgaussian_blur_probÚauto_augmentÚnum_aug_repeatsÚnum_aug_splitsÚinterpolationr   r   Úcrop_pctÚ	crop_modeÚcrop_border_pixelsÚnum_workersÚdistributedÚrankÚ
world_sizeÚseedÚepochÚuse_prefetcherÚ
pin_memoryr    r   Úpersistent_workersÚworker_seedingr%   r&   c-                 óÀ  — |�r)|dk    s
J d¦   «         ‚t          t          fi dd“d|	“d|“d|“d|“d	|“d
|“d|“d|“d|“d|“d|“d|“d|“d|“d|“d|“d|“d|
“d|“d|“d|&“dd“d|,“Ž}-t          |¦  «        }.||.z  }/t          | t          j        j        j        ¦  «        r
J d¦   «         ‚t          | |-|||||/||$|!|"|#d|%|,¬¦  «        }0t          j        j         	                    |0dd| d|'t          t          |+¬ ¦  «        |*o| dk    ¬!¦  «        }1|&rt          |1|||(|)|
|||,¬"¦	  «	        }1n±t          d||||&d||d|,¬#¦
  «
        | _        t          |¬$¦  «        }2d}3|!r5t          | t          j        j        j        ¦  «        sdd%lm}4  |4| ¦  «        }3t          j        j         	                    | |d| |3|2|'d|*o| dk    ¬&¦	  «	        }1|&rt          |1|||(|)|,¬'¦  «        }1|1S )(u¢
  Create a data loader with dynamic sequence length sampling for training.

    Args:
        dataset: Dataset to load from.
        patch_size: Single patch size to use.
        patch_size_choices: List of patch sizes for variable patch size training.
        patch_size_choice_probs: Probabilities for each patch size choice.
        train_seq_lens: Training sequence lengths for dynamic batching.
        max_seq_len: Fixed sequence length for validation.
        batch_size: Batch size for validation and max training sequence length.
        is_training: Whether this is for training (enables dynamic batching).
        mixup_fn: Optional mixup function.
        no_aug: Disable augmentation.
        re_prob: Random erasing probability.
        re_mode: Random erasing mode.
        re_count: Maximum number of erasing rectangles.
        re_split: Random erasing split flag.
        train_crop_mode: Training crop mode.
        scale: Scale range for random resize crop.
        ratio: Aspect ratio range for random resize crop.
        hflip: Horizontal flip probability.
        vflip: Vertical flip probability.
        color_jitter: Color jitter factor.
        color_jitter_prob: Color jitter probability.
        grayscale_prob: Grayscale conversion probability.
        gaussian_blur_prob: Gaussian blur probability.
        auto_augment: AutoAugment policy.
        num_aug_repeats: Number of augmentation repeats.
        num_aug_splits: Number of augmentation splits.
        interpolation: Interpolation method.
        mean: Normalization mean values.
        std: Normalization standard deviation values.
        crop_pct: Crop percentage for validation.
        crop_mode: Crop mode.
        crop_border_pixels: Crop border pixels.
        num_workers: Number of data loading workers.
        distributed: Whether using distributed training.
        rank: Process rank for distributed training.
        world_size: Total number of processes.
        seed: Random seed.
        epoch: Starting epoch.
        use_prefetcher: Whether to use prefetching.
        pin_memory: Whether to pin memory.
        img_dtype: Image data type.
        device: Device to move tensors to.
        persistent_workers: Whether to use persistent workers when ``num_workers`` is greater than zero.
        worker_seeding: Worker seeding mode.
        patchify_channels_last: Per-patch flat layout. ``True`` (default):
            channel index varies fastest (NaFlex default). ``False``: C-P-P
            (channels first â€” Gemma4 / HF native layout). Forwarded to the
            upstream ``Patchify`` and to the prefetcher's normalization layout.

    Returns:
        DataLoader or NaFlexPrefetchLoader instance.
    r   z=Augmentation repeats not currently supported in NaFlex loaderr’   Tr”   r–   r—   r˜   r™   rš   r›   rœ   r�   rž   rŸ   r¢   r   r   r£   r¤   r¥   r!   r"   r#   r¬   Únaflexr%   Fz IterableDataset Wrapper is a WIP)Útransform_factoryr�   rŽ   r�   Úseq_lensÚmax_tokens_per_batchr“   rª   r§   r¨   r©   Úshuffler«   r%   N)r¯   )rh   rµ   r¦   rs   r­   Úworker_init_fnr®   )r   r   r    r   r!   r"   r#   r%   )
r’   r¢   r   r   r¬   r±   r�   r‘   Úpatchifyr%   )r‘   )ÚOrderedDistributedSampler)rh   rµ   r¦   rs   Ú
collate_fnr­   Ú	drop_lastr®   )r   r   r    r   r%   )r   r   ÚmaxrS   r9   r}   r~   ÚIterableDatasetr   r   r   r   Ú	transformr   Útimm.data.distributed_samplerr¸   )5rx   r�   rŽ   r�   r�   r‘   rh   r’   r“   r”   r!   r"   r#   r•   r–   r—   r˜   r™   rš   r›   rœ   r�   rž   rŸ   r    r¡   r¢   r   r   r£   r¤   r¥   r¦   r§   r¨   r©   rª   r«   r¬   r­   r    r   r®   r¯   r%   r²   Úmax_train_seq_lenr´   Únaflex_datasetr   r¹   rs   r¸   s5                                                        r/   Úcreate_naflex_loaderrÁ   î   s}  € ðR ñ à !Ò#Ð#Ð#Ð%dÑ#Ô#Ð#å#Ýð
ð 
ð 
à˜ð
ð �6ð
ð ,˜Oð	
ð
 �%ð
ð �%ð
ð �%ð
ð �%ð
ð &˜ð
ð 0Ð/ð
ð *˜>ð
ð  2Ð1ð
ð &˜ð
ð (˜-ð
ð �ð
ð  �ð!
ð" �Xð#
ð$  �ið%
ð&  2Ð1ð'
ð( �Gð)
ð* �Gð+
ð, �Xð-
ð. *˜>ð/
ð0 �4ð1
ð2 $:Ð#9ð3
Ðõ8   Ñ/Ô/ÐØ)Ð,=Ñ=Ðå�g�uœ{Ô/Ô?Ñ@Ô@ð 	=Ø<Ð<Ñ<Ô<�5å0ØØ/Ø!Ø1Ø$;Ø#Ø!5ØØØ#ØØ!ØØØ#9ð
ñ 
ô 
ˆõ& ”Ô!×,Ò,ØØØØ#ØØ!Ý"¥<ÀÐOÑOÔOØ1ÐE°kÀA²oð -ñ 	
ô 	
ˆð ð 	Ý)ØØØØ#ØØØØ!Ø'=ð
ñ 
ô 
ˆFøõ -ØØ'ØØà)ØØ!Ø#ØØ#9ð
ñ 
ô 
ˆÔõ $°Ð<Ñ<Ô<ˆ
ð ˆØð 	9�z¨'µ5´;Ô3CÔ3SÑTÔTð 	9àOÐOÐOÐOÐOÐOØ/Ð/°Ñ8Ô8ˆGå”Ô!×,Ò,ØØ!ØØ#ØØ!Ø!ØØ1ÐE°kÀA²oð -ñ 

ô 

ˆð ð 	Ý)ØØØØ#ØØ'=ðñ ô ˆFð €Mr1   )(r|   ÚmathÚ
contextlibr   Ú	functoolsr   Útypingr   r   r   r   r	   r
   r   r9   Ú	constantsr   r   r   r   r   rÀ   r   r   Únaflex_random_erasingr   Útransforms_factoryr   r   r:   r   r�   r€   rƒ   r‚   r2   r}   r~   r   rÁ   r+   r1   r/   ú<module>rÉ      s†  ðð	ð 	ð €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ Ið €€€à BÐ BÐ BÐ BÐ BÐ BÐ BÐ BØ .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø CÐ CÐ CÐ CÐ CÐ CÐ CÐ CØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0ðP#ð P#ð P#ð P#ð P#ñ P#ô P#ð P#ðj =AØ26Ø9=Ø*DØØØ!Ø'+àØØØØØ)-Ø/3Ø/3ØØØ!Ø-1Ø "Ø$&Ø&*Ø ØØ'Ø"7Ø!5Ø$(Ø#'Ø,0àØ!ØØØØØ#ØØ!&¤Ø+7¨5¬<¸Ñ+?Ô+?Ø#'Ø#Ø'+ð_jð jà˜U 5¨¨c¨¤?°CÐ#7Ô8Ô9ðjð % T¨#¤YÔ/ðjð "*¨$¨u¬+Ô!6ð	jð
 ˜c 3˜hœðjð ðjð ðjð ðjð ˜8Ô$ðjð ðjð ðjð ðjð ðjð ðjð  " #œð!jð" ˜˜e U˜lÔ+Ô,ð#jð$ ˜˜e U˜lÔ+Ô,ð%jð& ð'jð( ð)jð* ð+jð, $ Eœ?ð-jð. ð/jð0 "ð1jð2 ˜s”mð3jð4 ð5jð6 ð7jð8 ð9jð: �E˜3�JÔð;jð< �5˜#�:Ôð=jð> ˜5”/ð?jð@ ˜C”=ðAjðB % SœMðCjðF ðGjðH ðIjðJ ðKjðL ðMjðN ðOjðP ðQjðR ðSjðT ðUjðV ”;ðWjðX �c˜5œ<Ð'Ô(ðYjðZ !ð[jð\ ð]jð^ !%ð_jð` 
ˆuŒ{ÔÔ*Ð,@Ð@Ô	Aðajð jð jð jð jð jr1   