§
    ·¯¸jPB  ã                   óº  — d Z ddlZddlZddlZddlZddlZddlZddlZddlm	Z	 ddl
mZ ddlmZ ddlmZmZmZmZmZmZ ddlZddlmZ ddlZddlmZ ddlmZmZmZ 	 ddl Z!dd	l"m#Z#m$Z$ dd
l%m&Z& ddl'm(Z(m)Z)m*Z*m+Z+ n# e,$ r dZ!dZ&Y nw xY wddl-m.Z. ddl/m0Z0 ddl1m2Z2  ej3        e4¦  «        Z5 e6ej7         8                    dd¦  «        ¦  «        Z9 e6ej7         8                    dd¦  «        ¦  «        Z:d(d„Z;e	 G d„ d¦  «        ¦   «         Z<de=defd„Z>d„ Z?	 	 	 	 d)d „Z@d!„ ZAe!� G d"„ d#e!jB        ¦  «        ZCndZC G d$„ d%e¦  «        ZD G d&„ d'e0¦  «        ZEdS )*zR Dataset reader for webdataset

Hacked together by / Copyright 2022 Ross Wightman
é    N)Ú	dataclass)Úpartial)Úislice)ÚAnyÚCallableÚDictÚListÚOptionalÚTuple)ÚImage)ÚDatasetÚIterableDatasetÚget_worker_info)Ú_shuffleÚgetfirst)Úexpand_urls)Úbase_plus_extÚ
url_openerÚtar_file_expanderÚvalid_sampleé   )Úload_class_map)ÚReader)ÚSharedCountÚWDS_SHUFFLE_SIZEi    ÚWDS_INITIAL_SIZEi   ©z
_info.jsonz	info.jsonc                 ó  — t          |t          ¦  «        r|f}g }d}|D ]Ë}t          j                             | |¦  «        }	 |                     |¦  «         t          j        |¦  «        5 }|                     d¦  «        rt          j
        |¦  «        }nt          j        |¦  «        }d d d ¦  «         n# 1 swxY w Y   |c S # t          $ r}t          |¦  «        }Y d }~ŒÄd }~ww xY wt                               d|› d|› d�¦  «         i S )NÚ z.jsonzDataset info file not found at z	. Error: z.. Falling back to provided split and size arg.)Ú
isinstanceÚstrÚosÚpathÚjoinÚappendÚwdsÚgopenÚendswithÚjsonÚloadÚyamlÚ	safe_loadÚ	ExceptionÚ_loggerÚwarning)	ÚrootÚnamesÚtriedÚerr_strÚnÚ	full_pathÚfÚ	info_dictÚes	            úl/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/data/readers/reader_wds.pyÚ
_load_infor:   *   sw  € Ý�%�ÑÔð Ø�ˆØ€EØ€GØð ð ˆÝ”G—L’L  qÑ)Ô)ˆ	ð		Ø�LŠL˜Ñ#Ô#Ð#Ý”˜9Ñ%Ô%ð 2¨Ø—:’:˜gÑ&Ô&ð 2Ý $¤	¨!¡¤�I�Iå $¤¨qÑ 1Ô 1�Ið	2ð 2ð 2ñ 2ô 2ð 2ð 2ð 2ð 2ð 2ð 2øøøð 2ð 2ð 2ð 2ð
 ÐÐÐøÝð 	ð 	ð 	Ý˜!‘f”fˆGˆGˆGˆGˆGˆGøøøøð	øøøõ ‡O‚Oð	7¨%ð 	7ð 	7¸'ð 	7ð 	7ð 	7ñ8ô 8ð 8ð €Is<   Á)CÁ+?B6Â*CÂ6B:	Â:CÂ=B:	Â>CÃ
C(ÃC#Ã#C(c                   óf   — e Zd ZU eed<   ee         ed<   dZee         ed<   dZeed<   dZ	eed<   dS )	Ú	SplitInfoÚnum_samplesÚ	filenames© Úshard_lengthsr   Ú	alt_labelÚnameN)
Ú__name__Ú
__module__Ú__qualname__ÚintÚ__annotations__r   r!   r@   rA   rB   r?   ó    r9   r<   r<   B   s]   € € € € € € àÐÐÑØ�SŒzÐÐÑØ "€M�5˜”:Ð"Ð"Ñ"Ø€IˆsÐÐÑØ€Dˆ#€N€N�N€N€NrH   r<   ÚsplitÚinfoc           	      ó¬  ‡	— d„ }d| v sd| v �ra|                       d¦  «        } d}d}t          | ¦  «        dk    rt          | d         ¦  «        }| d         } d| vrF|                       d	d
¦  «        }t          |¦  «        dz
  }|rd|v r||         |d         v r||         }t          | ¦  «        }|rª|d         |         }|sŽd„ t	          |d         |d         ¦  «        D ¦   «         Š	t          ˆ	fd„|D ¦   «         ¦  «        }t          ‰	                     ¦   «         ¦  «        |d<   t          ‰	                     ¦   «         ¦  «        |d<   ||d<    ||¦  «        }nxt          |||¬¦  «        }ned|vs
| |d         vr<t          d| › d|                     di ¦  «                             ¦   «         › d�¦  «        ‚| } |d         |          } ||¦  «        }|S )Nc           	      ó¶   — t          | d         t          | d         ¦  «        t          | d         ¦  «        |                      dd¦  «        | d         ¬¦  «        S )Nr=   r>   r@   rA   r   rB   )r=   r>   r@   rA   rB   )r<   ÚtupleÚget)Ú	dict_infos    r9   Ú_info_convertz(_parse_split_info.<locals>._info_convertL   sZ   € ÝØ! -Ô0Ý˜I kÔ2Ñ3Ô3Ý 	¨/Ô :Ñ;Ô;Ø—m’m K°Ñ4Ô4Ø˜6Ô"ð
ñ 
ô 
ð 	
rH   Útarz..ú|r   r   r   z::ú-é   Úsplitsc                 ó   — i | ]\  }}||“Œ	S r?   r?   )Ú.0r6   Úcs      r9   ú
<dictcomp>z%_parse_split_info.<locals>.<dictcomp>h   s   € ÐbÐbÐb¡  1�q˜!ÐbÐbÐbrH   r>   r@   c              3   ó(   •K  — | ]}‰|         V — Œd S ©Nr?   )rW   r6   Ú_fcs     €r9   ú	<genexpr>z$_parse_split_info.<locals>.<genexpr>i   s'   øè è € Ð!BÐ!B¨Q # a¤&Ð!BÐ!BÐ!BÐ!BÐ!BÐ!BrH   r=   )rB   r=   r>   zsplit z not found in info (ú))rI   ÚlenrF   r   ÚzipÚsumrM   ÚkeysÚvaluesr<   ÚRuntimeErrorrN   )
rI   rJ   rP   r=   Ú
split_nameÚsplit_partsÚ	split_idxÚsplit_filenamesÚ
split_infor\   s
            @r9   Ú_parse_split_inforj   K   s,  ø€ ð
ð 
ð 
ð �€~€~˜ ˜™ð —’˜CÑ Ô ˆØˆØˆ
Ýˆu‰:Œ:˜Š>ˆ>Ý˜e Aœh™-œ-ˆKØ�a”ˆØ�uÐÐØŸ+š+ c¨1Ñ-Ô-ˆKÝ˜KÑ(Ô(¨1Ñ,ˆIØð 4˜X¨Ð-Ð-°+¸iÔ2HÈDÐQYÌNÐ2ZÐ2ZØ(¨Ô3�
å% eÑ,Ô,ˆØð 	Ø˜hœ¨
Ô3ˆJØð 8ØbÐb­¨J°{Ô,CÀZÐP_ÔE`Ñ(aÔ(aÐbÑbÔb�Ý!Ð!BÐ!BÐ!BÐ!B°/Ð!BÑ!BÔ!BÑBÔB�Ý*/°·²±
´
Ñ*;Ô*;�
˜;Ñ'Ý.3°C·J²J±L´LÑ.AÔ.A�
˜?Ñ+Ø,7�
˜=Ñ)Ø&˜ zÑ2Ô2ˆJˆJå"ØØ'Ø)ðñ ô ˆJˆJð ˜4ÐÐ 5°°X´Ð#>Ð#>ÝÐc¨ÐcÐcÀ4Ç8Â8ÈHÐVXÑCYÔCY×C^ÒC^ÑC`ÔC`ÐcÐcÐcÑdÔdÐdØˆØ˜(”^ EÔ*ˆ
Ø"�] :Ñ.Ô.ˆ
àÐrH   c                 óŠ   — t                                dt          | ¦  «        › d�¦  «         t          | t          ¦  «        r| ‚dS )zQCall in an exception handler to ignore exceptions, issue a warning, and continue.zHandling webdataset error (z). Ignoring.T)r.   r/   Úreprr    Ú	TypeError)Úexns    r9   Úlog_and_continuero   ~   sB   € å‡O‚OÐIµ$°s±)´)ÐIÐIÐIÑJÔJÐJå�#•yÑ!Ô!ð Øˆ	Øˆ4rH   ÚjpgÚRGBÚclsr   c                 óæ  — |r8t          j        | d         ¦  «        }t          ||         ¦  «        }|dk     rdS nt          | |         ¦  «        }t          | |¦  «        }t	          j        |¦  «        5 }t          j        |¦  «        }|                     ¦   «          ddd¦  «         n# 1 swxY w Y   |r| 	                    |¦  «        }t          |||                      dd¦  «        ¬¦  «        }	|	S )z¡ Custom sample decode
    * decode and convert PIL Image
    * cls byte string label to int
    * pass through JSON byte string (if it exists) without parse
    r)   r   N)rp   rr   r)   )r)   ÚloadsrF   r   ÚioÚBytesIOr   Úopenr*   ÚconvertÚdictrN   )
ÚsampleÚ	image_keyÚ
image_modeÚ
target_keyrA   ÚmetaÚclass_labelÚimgÚbÚdecodeds
             r9   Ú_decoderƒ   ‡   s&  € ð ð .åŒz˜& œ.Ñ)Ô)ˆÝ˜$˜yœ/Ñ*Ô*ˆØ˜Š?ˆ?à�4ð õ ˜& Ô,Ñ-Ô-ˆõ �6˜9Ñ
%Ô
%€CÝ	Œ�C‰Œð ˜AÝŒj˜‰mŒmˆØ�Š‰
Œ
ˆ
ðð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð ð ð &Ø�kŠk˜*Ñ%Ô%ˆõ �s °&·*²*¸VÀTÑ2JÔ2JÐKÑKÔK€GØ€Ns   Á4)B)Â)B-Â0B-c                  ól   — t          ¦   «         } | �| j        S t          j                             ¦   «         S )z'get dataloader worker seed from pytorch)r   Úseedr&   ÚutilsÚpytorch_worker_seed)Úworker_infos    r9   r‡   r‡   «   s0   € å!Ñ#Ô#€KØÐàÔÐåŒ9×(Ò(Ñ*Ô*Ð*rH   c                   ó$   — e Zd Z	 	 	 	 dd„Zd„ ZdS )	Údetshuffle2éè  éd   r   éÿÿÿÿc                 ó>   — || _         || _        || _        || _        d S r[   )ÚbufsizeÚinitialr…   Úepoch)Úselfr�   r�   r…   r‘   s        r9   Ú__init__zdetshuffle2.__init__¹   s$   € ð #ˆDŒLØ"ˆDŒLØˆDŒIØˆDŒJˆJˆJrH   c                 ó,  — t          | j        t          ¦  «        r| j        j        }n| xj        dz  c_        | j        }| j        dk     rt          ¦   «         |z   }n
| j        |z   }t          j        |¦  «        }t          || j	        | j
        |¦  «        S )Nr   r   )r    r‘   r   Úvaluer…   r‡   ÚrandomÚRandomr   r�   r�   )r’   Úsrcr‘   r…   Úrngs        r9   Úrunzdetshuffle2.runÅ   sŠ   € Ý˜$œ*¥kÑ2Ô2ð #Øœ
Ô(��ð �
”
˜a‘�
”
Øœ
�àŒy˜1Š}ˆ}Ý*Ñ,Ô,¨uÑ4��à”y 5Ñ(�å”- Ñ%Ô%ˆCÝ˜C ¤¨t¬|¸SÑAÔAÐArH   N)r‹   rŒ   r   r�   )rC   rD   rE   r“   rš   r?   rH   r9   rŠ   rŠ   ¸   sJ   € € € € € ð ØØØð
	ð 
	ð 
	ð 
	ð	Bð 	Bð 	Bð 	Bð 	BrH   rŠ   c                   ó<   ‡ — e Zd ZdZej        dddfˆ fd„	Zd„ Zˆ xZS )ÚResampledShards2z,An iterable dataset yielding a list of urls.NTr�   c                 óR  •— t          ¦   «                              ¦   «          t          j                             |¦  «        }|| _        t          | j        d         t          ¦  «        sJ ‚|| _        t          j
        ¦   «         | _        |€t          n|| _        || _        || _        dS )z‹Sample shards from the shard list with replacement.

        :param urls: a list of URLs as a Python list or brace notation string
        r   N)Úsuperr“   r&   Ú
shardlistsr   Úurlsr    r!   Únshardsr–   r—   r™   r‡   Úworker_seedÚdeterministicr‘   )r’   r    r¡   r¢   r£   r‘   Ú	__class__s         €r9   r“   zResampledShards2.__init__Ý   s�   ø€ õ 	‰Œ×ÒÑÔÐÝŒ~×)Ò)¨$Ñ/Ô/ˆØˆŒ	Ý˜$œ) Aœ,­Ñ,Ô,Ð,Ð,Ð,ØˆŒÝ”=‘?”?ˆŒØ2=Ð2EÕ.Ð.È;ˆÔØ*ˆÔØˆŒ
ˆ
ˆ
rH   c              #   ó¸  K  — t          | j        t          ¦  «        r| j        j        }n| xj        dz  c_        | j        }| j        r.t          j        |                      ¦   «         |z   ¦  «        | _        t          | j
        ¦  «        D ]O}| j                             dt          | j        ¦  «        dz
  ¦  «        }t          | j        |         ¬¦  «        V — ŒPdS )z#Return an iterator over the shards.r   r   )ÚurlN)r    r‘   r   r•   r£   r–   r—   r¢   r™   Úranger¡   Úrandintr_   r    ry   )r’   r‘   Ú_Úindexs       r9   Ú__iter__zResampledShards2.__iter__ó   sÔ   è è € å�d”j¥+Ñ.Ô.ð 	Ø”JÔ$ˆEˆEð ˆJŒJ˜!‰OˆJŒJØ”JˆEàÔð 	Aå”} T×%5Ò%5Ñ%7Ô%7¸%Ñ%?Ñ@Ô@ˆDŒHå�t”|Ñ$Ô$ð 	-ð 	-ˆAØ”H×$Ò$ Q­¨D¬I©¬¸Ñ(:Ñ;Ô;ˆEÝ˜4œ9 UÔ+Ð,Ñ,Ô,Ð,Ð,Ð,Ð,ð	-ð 	-rH   )	rC   rD   rE   Ú__doc__ÚsysÚmaxsizer“   r«   Ú__classcell__©r¤   s   @r9   rœ   rœ   Ú   sb   ø€ € € € € Ø6Ð6ð
 ”ØØØðð ð ð ð ð ð,-ð -ð -ð -ð -ð -ð -rH   rœ   c            "       ó  ‡ — e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d'dedee         dededee         dedededee         dedededededee         dee         f ˆ fd„Zd„ Z		 d(dee         fd„Z
d „ Zd!„ Zd"„ Zd#„ Zd$„ Zd)d%„Zd)d&„Zˆ xZS )*Ú	ReaderWdsNÚtrainFr   r   é*   újpg;png;webprq   rr   r   Úfilenamer0   rB   rI   Úis_trainingr=   Ú
batch_sizeÚrepeatsr…   Ú	class_mapÚ	input_keyÚinput_img_moder}   Útarget_img_modeÚfilename_keyÚsample_shuffle_sizeÚsample_initial_sizec                 óö  •— t          ¦   «                              ¦   «          t          €t          d¦  «        ‚|| _        || _        || _        || _        || _        d| _	        |pt          | _        |pt          | _        |
| _        || _        || _        || _        d| _        t'          | j        ¦  «        | _        t+          || j        ¦  «        | _        |�|| _        n| j        j        | _        |r| j        st          d¦  «        ‚d| _        |	rt3          |	¦  «        | _        d| _        ni | _        d| _        d| _        t;          j        ¦   «         rZt;          j        ¦   «         rGt;          j         ¦   «         dk    r0t;          j!        ¦   «         | _        t;          j         ¦   «         | _        d | _"        d| _#        || _$        d| _%        d| _&        d| _'        d| _(        tS          ¦   «         | _*        d | _+        d S )	NzcPlease install webdataset 0.2.x package `pip install git+https://github.com/webdataset/webdataset`.iô  z.JPEGzBInvalid split definition, num_samples not specified in train mode.FTr   r   ),rž   r“   r&   rd   r0   r·   r¸   r¹   Úcommon_seedÚshard_shuffle_sizeÚSAMPLE_SHUFFLE_SIZEr¿   ÚSAMPLE_INITIAL_SIZErÀ   r»   r¼   r}   r¾   Úkey_extr:   rJ   rj   ri   r=   Úremap_classr   Úclass_to_idxÚ	dist_rankÚdist_num_replicasÚdistÚis_availableÚis_initializedÚget_world_sizeÚget_rankrˆ   Ú	worker_idr¢   Únum_workersÚglobal_worker_idÚglobal_num_workersÚ
init_countr   Úepoch_countÚds)r’   r0   rB   rI   r·   r=   r¸   r¹   r…   rº   r»   r¼   r}   r½   r¾   r¿   rÀ   r¤   s                    €r9   r“   zReaderWds.__init__  só  ø€ õ& 	‰Œ×ÒÑÔÐÝˆ;ÝØuñwô wð wàˆŒ	Ø&ˆÔØ$ˆŒØˆŒØˆÔØ"%ˆÔØ#6Ð#MÕ:MˆÔ Ø#6Ð#MÕ:MˆÔ à"ˆŒØ,ˆÔØ$ˆŒØ(ˆÔØˆŒå˜tœyÑ)Ô)ˆŒ	Ý+¨E°4´9Ñ=Ô=ˆŒØÐ"Ø*ˆDÔÐà#œÔ:ˆDÔØð 	f˜tÔ/ð 	fÝÐdÑeÔeÐeØ ˆÔØð 	#Ý .¨yÑ 9Ô 9ˆDÔØ#ˆDÔÐà "ˆDÔð ˆŒØ!"ˆÔÝÔÑÔð 	;¥4Ô#6Ñ#8Ô#8ð 	;½TÔ=PÑ=RÔ=RÐUVÒ=VÐ=VÝ!œ]™_œ_ˆDŒNÝ%)Ô%8Ñ%:Ô%:ˆDÔ"ð  ˆÔØˆŒØˆÔØˆÔØ !ˆÔØ"#ˆÔØˆŒÝ&™=œ=ˆÔð ˆŒˆˆrH   c                 ó   — || j         _        d S r[   )rÕ   r•   )r’   Úcounts     r9   Ú	set_epochzReaderWds.set_epochQ  s   € Ø!&ˆÔÔÐÐrH   rÑ   c                 óV   — | j         �d S |�|| _        | j        | j        z  | _        d S d S r[   )rÖ   rÑ   rÊ   rÓ   )r’   rÑ   s     r9   Úset_loader_cfgzReaderWds.set_loader_cfgT  s@   € ð Œ7ÐØˆFØÐ"Ø*ˆDÔØ&*Ô&<¸tÔ?OÑ&OˆDÔ#Ð#Ð#ð #Ð"rH   c                 ó  ‡ — ‰ j         €€t          j        j                             ¦   «         }|�+|‰ _         |j        ‰ _        |j        ‰ _        |j	        ‰ _	        ‰ j
        ‰ j	        z  ‰ _        ‰ j        ‰ j	        z  ‰ j        z   ‰ _        ˆ fd„‰ j        j        D ¦   «         }t!          j        |¦  «        g}‰ j        rŒ|                     t)          ‰ j        ‰ j        ‰ j        ¬¦  «        ‰ j        t!          j        t4          ¬¦  «        t!          j        ‰ j        ‰ j        t=          j        ‰ j        ¦  «        ¬¦  «        g¦  «         n4|                     ‰ j        t!          j        t4          ¬¦  «        g¦  «         |                     t!          j         tC          tD          ‰ j#        ‰ j$        ‰ j        j%        ¬¦  «        t4          ¬¦  «        t!          j&        ‰ j#        ‰ j'        ¬¦  «        g¦  «         t!          j(        |Ž ‰ _)        dS )z8 Lazily initialize worker (in worker processes)
        Nc                 óZ   •— g | ]'}t           j                             ‰j        |¦  «        ‘Œ(S r?   )r"   r#   r$   r0   )rW   r6   r’   s     €r9   ú
<listcomp>z(ReaderWds._lazy_init.<locals>.<listcomp>l  s+   ø€ Ð]Ð]Ð]¸a�rœwŸ|š|¨D¬I°qÑ9Ô9Ð]Ð]Ð]rH   )r…   r‘   )Úhandler)r�   r�   r™   )r{   r|   rA   )ÚimageÚtarget)*rˆ   Útorchr†   Údatar   ÚidrÐ   r…   r¢   rÑ   rÊ   rÓ   rÉ   rÒ   ri   r>   r&   ÚSimpleShardListr·   ÚextendrŠ   rÃ   rÂ   rÕ   Ú_split_by_node_and_workerÚtarfile_to_samplesro   Úshuffler¿   rÀ   r–   r—   Úmapr   rƒ   r»   r¼   rA   Úrenamer}   ÚDataPipelinerÖ   )r’   rˆ   Úabs_shard_filenamesÚpipelines   `   r9   Ú
_lazy_initzReaderWds._lazy_init^  s  ø€ ð ÔÐ#Ýœ+Ô*×:Ò:Ñ<Ô<ˆKØÐ&Ø#.�Ô Ø!,¤�”Ø#.Ô#3�Ô Ø#.Ô#:�Ô Ø&*Ô&<¸tÔ?OÑ&OˆDÔ#Ø$(¤N°TÔ5EÑ$EÈÌÑ$VˆDÔ!ð ^Ð]Ð]Ð]À4Ä?ÔC\Ð]Ñ]Ô]ÐÝÔ'Ð(;Ñ<Ô<Ð=ˆàÔð 	Ø�OŠOÝØÔ+ØÔ)ØÔ*ðñ ô ð
 Ô.åÔ&Õ/?Ð@Ñ@Ô@Ý”Ø Ô4Ø Ô4Ýœ dÔ&6Ñ7Ô7ðñ ô ðñ ô ð ð ð  �OŠOØÔ.åÔ&Õ/?Ð@Ñ@Ô@ðñ ô ð ð
 	�ŠÝŒGÝÝØ"œnØ#Ô2Ø"œoÔ7ð	ñ ô õ )ðñ ô õ ŒJ˜Tœ^°D´OÐDÑDÔDð
ñ 	ô 	ð 	õ Ô" HÐ-ˆŒˆˆrH   c              #   ó|   K  — | j         dk    r%t          || j        d | j         ¦  «        D ]}|V — Œd S |D ]}|V — Œd S ©Nr   )rÓ   r   rÒ   )r’   r˜   Úss      r9   rç   z#ReaderWds._split_by_node_and_worker“  sr   è è € ØÔ" QÒ&Ð&Ý˜C Ô!6¸¸dÔ>UÑVÔVð ð �Ø����ðð ð ð ð �Ø����ðð rH   c                 ó  — | j         t          | j        | j        ¦  «        z  }| j        s| j        dk    rt          j        |¦  «        }| j        r$t          j        || j        z  ¦  «        | j        z  }t          |¦  «        S rñ   )	r=   ÚmaxrÓ   rÊ   r·   ÚmathÚceilr¸   rF   )r’   Únum_worker_sampless     r9   Ú_num_samples_per_workerz!ReaderWds._num_samples_per_worker›  s…   € Ø!Ô-µ°DÔ4KÈTÔMcÑ0dÔ0dÑdÐØÔð 	?˜tÔ5¸Ò9Ð9Ý!%¤Ð+=Ñ!>Ô!>ÐØÔð 	cÝ!%¤Ð+=ÀÄÑ+OÑ!PÔ!PÐSWÔSbÑ!bÐÝÐ%Ñ&Ô&Ð&rH   c              #   ó8  K  — | j         €|                      ¦   «          |                      ¦   «         }| j        s| j        dk    r| j                              |¦  «        }n| j         }d}|D ]/}|d         }| j        r| j        |         }|d         |fV — |dz  }Œ0d S )Nr   r   rá   rà   )rÖ   rï   rø   r·   rÊ   Ú
with_epochrÇ   rÈ   )r’   r÷   rÖ   Úirz   rá   s         r9   r«   zReaderWds.__iter__£  sÈ   è è € ØŒ7ˆ?Ø�OŠOÑÔÐà!×9Ò9Ñ;Ô;ÐØÔð 	˜tÔ5¸Ò9Ð9ð
 ”×#Ò#Ð$6Ñ7Ô7ˆBˆBà”ˆBàˆàð 	ð 	ˆFØ˜HÔ%ˆFØÔð 3ØÔ*¨6Ô2�Ø˜”/ 6Ð)Ð)Ð)Ð)Ø�‰FˆAˆAð	ð 	rH   c                 ó>   — |                       ¦   «         | j        z  }|S r[   )rø   rÑ   )r’   r=   s     r9   Ú__len__zReaderWds.__len__»  s!   € Ø×2Ò2Ñ4Ô4°tÔ7GÑGˆØÐrH   c                 ó   — J d¦   «         ‚)NFzNot supportedr?   )r’   rª   ÚbasenameÚabsolutes       r9   Ú	_filenamezReaderWds._filename¿  s   € Ø%�oÑ%Ô%ˆurH   c                 ó  — | j         €|                      ¦   «          g }| j         D ]g}| j        |v r|| j                 }nd|v r|d         | j        z   }n
J d¦   «         ‚|                     |¦  «         t          |¦  «        | j        k    r nŒh|S )z0 Return all filenames in dataset, overrides baseNÚ__key__FzNo supported name field present)rÖ   rï   r¾   rÆ   r%   r_   r=   )r’   rÿ   r   r1   rz   rB   s         r9   r>   zReaderWds.filenamesÂ  s­   € àŒ7ˆ?Ø�OŠOÑÔÐàˆØ”gð 		ð 		ˆFØÔ  FÐ*Ð*Ø˜dÔ/Ô0��Ø˜fÐ$Ð$Ø˜iÔ(¨4¬<Ñ7��à?Ð?Ñ?Ô?�uØ�LŠL˜ÑÔÐÝ�5‰zŒz˜TÔ-Ò-Ð-Ø�ð .àˆrH   )Nr³   FNr   r   r´   Nrµ   rq   rr   r   r¶   NNr[   )FF)rC   rD   rE   r!   r
   ÚboolrF   ry   r“   rÙ   rÛ   rï   rç   rø   r«   rý   r  r>   r¯   r°   s   @r9   r²   r²     sî  ø€ € € € € ð #'Ø Ø %Ø)-ØØØØ(,Ø+Ø"'Ø#Ø#%Ø *Ø15Ø15ð#Hð HàðHð ˜3”-ðHð ð	Hð
 ðHð " #œðHð ðHð ðHð ðHð   ”~ðHð ðHð  ðHð ðHð !ðHð ðHð  "*¨#¤ð!Hð" "*¨#¤ð#Hð Hð Hð Hð Hð HðT'ð 'ð 'ð
 *.ðPð Pà! #œðPð Pð Pð Pð3.ð 3.ð 3.ðjð ð ð'ð 'ð 'ðð ð ð0ð ð ð&ð &ð &ð &ðð ð ð ð ð ð ð rH   r²   )r   )rp   rq   rr   r   )Fr¬   ru   r)   Úloggingrõ   r"   r–   r­   Údataclassesr   Ú	functoolsr   Ú	itertoolsr   Útypingr   r   r   r	   r
   r   râ   Útorch.distributedÚdistributedrË   r+   ÚPILr   Útorch.utils.datar   r   r   Ú
webdatasetr&   Úwebdataset.filtersr   r   Úwebdataset.shardlistsr   Úwebdataset.tariteratorsr   r   r   r   ÚImportErrorrº   r   Úreaderr   Úshared_countr   Ú	getLoggerrC   r.   rF   ÚenvironrN   rÄ   rÅ   r:   r<   r!   rj   ro   rƒ   r‡   ÚPipelineStagerŠ   rœ   r²   r?   rH   r9   ú<module>r     s  ððð ð 
€	€	€	Ø €€€Ø €€€Ø €€€Ø 	€	€	€	Ø €€€Ø 
€
€
€
Ø !Ð !Ð !Ð !Ð !Ð !Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø =Ð =Ð =Ð =Ð =Ð =Ð =Ð =Ð =Ð =Ð =Ð =Ð =Ð =Ð =Ð =à €€€Ø  Ð  Ð  Ð  Ð  Ð  Ø €€€Ø Ð Ð Ð Ð Ð Ø FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FðØÐÐÐØ5Ð5Ð5Ð5Ð5Ð5Ð5Ð5Ø1Ð1Ð1Ð1Ð1Ð1ØbÐbÐbÐbÐbÐbÐbÐbÐbÐbÐbÐbÐbøØð ð ð Ø
€CØ€K€K€Kðøøøð &Ð %Ð %Ð %Ð %Ð %Ø Ð Ð Ð Ð Ð Ø %Ð %Ð %Ð %Ð %Ð %à
ˆ'Ô
˜HÑ
%Ô
%€à�c˜"œ*Ÿ.š.Ð);¸TÑBÔBÑCÔCÐ Ø�c˜"œ*Ÿ.š.Ð);¸TÑBÔBÑCÔCÐ ðð ð ð ð0 ðð ð ð ð ñ ô ñ „ðð0˜Sð 0¨ð 0ð 0ð 0ð 0ðfð ð ð ØØØð!ð !ð !ð !ðH+ð +ð +ð €?ðBð Bð Bð Bð B�cÔ'ñ Bô Bð Bð Bð> €Kð)-ð )-ð )-ð )-ð )-�ñ )-ô )-ð )-ðXLð Lð Lð Lð L�ñ Lô Lð Lð Lð Ls   Á A? Á?	BÂ
B