§
    ·¯¸jµ'  ã                   ó�  — d Z ddlZddlZddlZddlZddlmZ ddlmZmZm	Z	m
Z
mZmZ ddlZddlmZ ddlmZ ddlmZ dd	lmZ  ej        e¦  «        Zd
Z G d„ dej        ¦  «        Z G d„ d¦  «        Zdej        de	de
e         fd„Z 	 	 	 	 ddee	         dee!         deeeee
f                  de!fd„Z" G d„ de¦  «        Z#dS )aS   A dataset reader that reads tarfile based datasets

This reader can extract image samples from:
* a single tar of image files
* a folder of multiple tarfiles containing imagefiles
* a tar of tars containing image files

Labels are based on the combined folder and/or tar name structure.

Hacked together by / Copyright 2020 Ross Wightman
é    N)Úglob)ÚListÚTupleÚDictÚSetÚOptionalÚUnion)Únatural_keyé   )Úload_class_map)Úget_img_extensions)ÚReaderz_tarinfos.picklec                   ó2   ‡ — e Zd ZdZdhZdedefˆ fd„Zˆ xZS )Ú_TarInfoUnpicklera±  Restricted unpickler for the tarinfo cache file.

    The cache only ever contains plain containers (dict/list/str/int/None) plus
    ``tarfile.TarInfo`` instances. Built-in containers are handled by pickle opcodes
    and never reach ``find_class``, so allowing only ``tarfile.TarInfo`` is sufficient
    to load valid caches while blocking arbitrary code execution from a crafted cache
    file (CWE-502, GHSA-pqww-w348-m542).
    )ÚtarfileÚTarInfoÚmoduleÚnamec                 ó”   •— ||f| j         v r"t          ¦   «                              ||¦  «        S t          j        d|› d|› d�¦  «        ‚)NzGlobal ú.z+ is not permitted in a tar info cache file.)Ú_allowed_globalsÚsuperÚ
find_classÚpickleÚUnpicklingError)Úselfr   r   Ú	__class__s      €úu/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/data/readers/reader_image_in_tar.pyr   z_TarInfoUnpickler.find_class*   sT   ø€ Ø�Dˆ>˜TÔ2Ð2Ð2Ý‘7”7×%Ò% f¨dÑ3Ô3Ð3ÝÔ$Ð%i¨vÐ%iÐ%i¸Ð%iÐ%iÐ%iÑjÔjÐjó    )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Ústrr   Ú__classcell__©r   s   @r   r   r      sl   ø€ € € € € ðð ð /Ð/Ððk ð k¨Cð kð kð kð kð kð kð kð kð kð kr   r   c                   ó:   — e Zd Zddej        dej        fd„Zd„ ZdS )ÚTarStateNÚtfÚtic                 ó0   — || _         || _        i | _        d S ©N)r)   r*   Úchildren)r   r)   r*   s      r   Ú__init__zTarState.__init__2   s   € Ø#%ˆŒØ#%ˆŒØ-/ˆŒˆˆr   c                 ó   — d | _         d S r,   )r)   ©r   s    r   ÚresetzTarState.reset7   s   € ØˆŒˆˆr   )NN)r    r!   r"   r   ÚTarFiler   r.   r1   © r   r   r(   r(   0   sN   € € € € € ð0ð 0˜7œ?ð 0°w´ð 0ð 0ð 0ð 0ð
ð ð ð ð r   r(   r)   Úparent_infoÚ
extensionsc                 óF  — d}t          | ¦  «        D �]�\  }}|                     ¦   «         sŒt          j                             |j        ¦  «        \  }}t          j                             |¦  «        \  }}	|	                     ¦   «         }	|	dk    rët          j        |  	                    |¦  «        d¬¦  «        5 }
t          |j        t          j                             |d         |¦  «        |g g ¬¦  «        }|t          |
||¬¦  «        z  }t                               |› d|j        › d	t!          |d
         ¦  «        › d�¦  «         |d                              |¦  «         d d d ¦  «         n# 1 swxY w Y   �Œi|	|v r |d
                              |¦  «         |dz  }�Œ�|S )Nr   ú.tarúr|)ÚfileobjÚmodeÚpath©r   r;   r*   r-   Úsamples©r5   z"/?. Extracted child tarinfos from ú. r=   z images.r-   r   )Ú	enumerateÚisfileÚosr;   ÚsplitÚsplitextÚlowerr   ÚopenÚextractfileÚdictr   ÚjoinÚ_extract_tarinfoÚ_loggerÚdebugÚlenÚappend)r)   r4   r5   Úsample_countÚir*   ÚdirnameÚbasenamer   ÚextÚctfÚ
child_infos               r   rJ   rJ   ;   sÓ  € Ø€LÝ˜2‘”ð ñ ‰ˆˆ2Ø�yŠy‰{Œ{ð 	ØÝœGŸMšM¨"¬'Ñ2Ô2Ñˆ�Ý”G×$Ò$ XÑ.Ô.‰	ˆˆcØ�iŠi‰kŒkˆØ�&Š=ˆ=Ý” b§n¢n°RÑ&8Ô&8¸tÐDÑDÔDð ;ÈÝ!Øœ¥r¤w§|¢|°KÀÔ4GÈÑ'NÔ'NÐSUÐ`bÐlnðpñ pô p�
àÕ 0°°jÈZÐ XÑ XÔ XÑX�Ý—’ ÐuÐuÀbÄgÐuÐuÕQTÐU_Ð`iÔUjÑQkÔQkÐuÐuÐuÑvÔvÐvØ˜JÔ'×.Ò.¨zÑ:Ô:Ð:ð;ð ;ð ;ñ ;ô ;ð ;ð ;ð ;ð ;ð ;ð ;øøøð ;ð ;ð ;ð ;ùð �JÐÐØ˜	Ô"×)Ò)¨"Ñ-Ô-Ð-Ø˜AÑˆLùØÐs   Â9B)E.Å.E2	Å5E2	TÚclass_name_to_idxÚcache_tarinfoÚsortc                 óŽ  ‡‡‡ ‡!‡"— |st          d¬¦  «        nt          |¦  «        }d}t          j                             | ¦  «        rŠt          j                             | ¦  «        d                              ¦   «         dk    sJ ‚| g}t          j                             | ¦  «        \  } }t          j                             |¦  «        d         }d}n{|                      t          j        j	        ¦  «                             t          j        j	        ¦  «        d         }t          t          j                             | d¦  «        d¬¦  «        }t          |¦  «        }t          d	„ |D ¦   «         ¦  «        }	|sJ d
| › d�¦   «         ‚t                               d|	dz  d›d�¦  «         t!          g ¬¦  «        }
d}|€
|	dk    rdnd}|r-d|z   t"          z   }t          j                             | |¦  «        }t          j                             |¦  «        rÜt                               d|› d�¦  «         t'          |d¦  «        5 }t)          |¦  «                             ¦   «         }
d d d ¦  «         n# 1 swxY w Y   t-          |
t           ¦  «        r(t-          |
                     d¦  «        t0          ¦  «        st3          d|› d�¦  «        ‚t          |
d         ¦  «        |k    s
J d¦   «         ‚�n…t5          |¦  «        D �]\  }}|rdnAt          j                             t          j                             |¦  «        ¦  «        d         }t9          j        |d¬¦  «        5 }t!          t          j                             || ¦  «        |d g g ¬¦  «        }t=          |||¬¦  «        }t          |d         ¦  «        }t                               |› d|› d |› d!|› d"|› d#�
¦  «         d d d ¦  «         n# 1 swxY w Y   |
d                               |¦  «         �Œ|r[t                               d$|› d�¦  «         t'          |d%¦  «        5 }tC          j"        |
|¦  «         d d d ¦  «         n# 1 swxY w Y   g Š"g Š!dŠ ‰€dŠ g }dd&œd'„Šˆˆ ˆˆ!ˆ"fd(„}t                               d)¦  «         |
d         D ]Š}|rd n|d*         }tG          ¦   «         }d}|d         D ]:} |||¬+¦  «        }|r$tG          |d,         ¬-¦  «        |j$        |d*         <   ||z  }Œ;| |||¬+¦  «        z  }|r|                      ||f¦  «         Œ‹~
‰ rIt1          tK          t          ‰!¦  «        tL          ¬.¦  «        ¦  «        }d/„ t5          |¦  «        D ¦   «         Št                               d0¦  «         ˆfd1„tO          ‰"‰!¦  «        D ¦   «         }|rtK          |d2„ ¬.¦  «        }tO          |Ž \  Š"}tQ          j)        ‰"¦  «        Š"tQ          j)        |¦  «        }t                               d3t          ‰"¦  «        › d4t          |¦  «        › d5�¦  «         ‰"|‰|fS )6NT)Úas_setFéÿÿÿÿr7   r   z*.tar)Ú	recursivec                 óL   — g | ]!}t           j                             |¦  «        ‘Œ"S r3   )rB   r;   Úgetsize)Ú.0Úfs     r   ú
<listcomp>z$extract_tarinfos.<locals>.<listcomp>c   s&   € Ð?Ð?Ð?¨A•R”W—_’_ QÑ'Ô'Ð?Ð?Ð?r   z'No .tar files found at specified path (z).z	Scanning i   z.2fzMB of tar files...)ÚtartreesÚ l       
 Ú_z!Reading tar info from cache file r   Úrbrb   z'Invalid or corrupt tar info cache file z3Cached tartree len doesn't match number of tarfilesr8   )r:   r<   r>   r-   ú/z. Extracted tarinfos from r?   z children, z	 samples.zWriting tar info to cache file Úwb)Ú	leaf_onlyc                 ó  — t          j        j        |Ž                      t           j        j        ¦  «        }| r*|                     t           j        j        ¦  «        d         n$|                     t           j        j        d¦  «        S )Nr[   rd   )rB   r;   rI   ÚstripÚseprC   Úreplace)rh   r;   s     r   Ú_label_from_pathsz+extract_tarinfos.<locals>._label_from_paths�   sZ   € ÝŒwŒ|˜TÐ"×(Ò(­¬¬Ñ5Ô5ˆØ.7Ð[ˆt�zŠz�"œ'œ+Ñ&Ô& rÔ*Ð*¸T¿\º\Í"Ì'Ì+ÐWZÑ=[Ô=[Ð[r   c                 ó  •— d}| d         D ]u} ‰| d         t           j                             |j        ¦  «        ¦  «        }‰s|‰vrŒ=‰	                     ||| d         f¦  «         ‰                     |¦  «         |dz  }Œv|S )Nr   r=   r;   r*   r   )rB   r;   rQ   rN   )
ÚinfoÚfnÚaddedÚsÚlabelrm   Úbuild_class_maprV   Úlabelsr=   s
        €€€€€r   Ú_add_samplesz&extract_tarinfos.<locals>._add_samples”   s–   ø€ ØˆØ�i”ð 	ð 	ˆAØ%Ð% d¨6¤lµB´G·O²OÀAÄFÑ4KÔ4KÑLÔLˆEØ"ð  uÐ4EÐ'EÐ'EØØ�NŠN˜A˜r 4¨¤:Ð.Ñ/Ô/Ð/Ø�MŠM˜%Ñ Ô Ð Ø�Q‰JˆEˆEØˆr   z+Collecting samples and building tar states.r   )rp   r*   )r*   )Úkeyc                 ó   — i | ]\  }}||“Œ	S r3   r3   )r_   ÚidxÚcs      r   ú
<dictcomp>z$extract_tarinfos.<locals>.<dictcomp>²   s   € ÐKÐKÐK©¨¨Q˜Q ÐKÐKÐKr   z$Mapping targets and sorting samples.c                 ó2   •— g | ]\  }}|‰v ¯	|‰|         f‘ŒS r3   r3   )r_   rr   ÚlrV   s      €r   ra   z$extract_tarinfos.<locals>.<listcomp>µ   s4   ø€ ÐpÐpÐp¹¸¸AÐYZÐ^oÐYoÐYo˜AÐ0°Ô3Ð4ÐYoÐYoÐYor   c                 óB   — t          | d         d         j        ¦  «        S ©Nr   )r
   r;   )Úks    r   ú<lambda>z"extract_tarinfos.<locals>.<lambda>·   s   € ÍÐTUÐVWÔTXÐYZÔT[ÔT`ÑHaÔHa€ r   zFinished processing z samples across z tar files.)*r   ÚsetrB   r;   rA   rD   rE   rC   rj   rk   r   rI   rM   ÚsumrK   ro   rH   ÚCACHE_FILENAME_SUFFIXÚexistsrF   r   ÚloadÚ
isinstanceÚgetÚlistÚ
ValueErrorr@   rR   r   ÚrelpathrJ   rL   rN   r   Údumpr(   r-   Úsortedr
   ÚzipÚnpÚarray)#ÚrootrV   rW   r5   rX   Úroot_is_tarÚtar_filenamesÚ	root_nameÚnum_tarsÚ	tar_bytesro   Ú
cache_pathÚcache_filenameÚpfrP   rp   r;   r)   r4   Únum_samplesÚnum_childrenÚtarfilesrv   Útar_nameÚ	tar_stateÚparent_addedrU   Úchild_addedÚsorted_labelsÚsamples_and_targetsÚtargetsrm   rt   ru   r=   s#    `                             @@@@r   Úextract_tarinfosr¤   P   sk  øøøøø€ ð 9CÐWÕ#¨4Ð0Ñ0Ô0Ð0ÍÈJÉÌ€JØ€KÝ	„w‡~‚~�dÑÔð JÝŒw×Ò Ñ%Ô% bÔ)×/Ò/Ñ1Ô1°VÒ;Ð;Ð;Ð;Ø˜ˆÝœ'Ÿ-š-¨Ñ-Ô-‰ˆˆiÝ”G×$Ò$ YÑ/Ô/°Ô2ˆ	Øˆˆà—J’J�rœwœ{Ñ+Ô+×1Ò1µ"´'´+Ñ>Ô>¸rÔBˆ	Ý�RœWŸ\š\¨$°Ñ8Ô8ÀDÐIÑIÔIˆÝ�=Ñ!Ô!€HÝÐ?Ð?°Ð?Ñ?Ô?Ñ@Ô@€IØÐGÐGÐG¸tÐGÐGÐGÑGÔGˆ8å‡L‚LÐF˜Y wÑ.ÐFÐFÐFÐFÑGÔGÐGÝ˜ÐÑÔ€DØ€JØÐØ )¨JÒ 6Ð 6˜˜¸EˆØð 8Ø˜y™Õ+@Ñ@ˆÝ”W—\’\ $¨Ñ7Ô7ˆ
Ý	„w‡~‚~�jÑ!Ô!ð &Ý�ŠÐF¸ÐFÐFÐFÑGÔGÐGÝ�*˜dÑ#Ô#ð 	0 rÝ$ RÑ(Ô(×-Ò-Ñ/Ô/ˆDð	0ð 	0ð 	0ñ 	0ô 	0ð 	0ð 	0ð 	0ð 	0ð 	0ð 	0øøøð 	0ð 	0ð 	0ð 	0å˜$¥Ñ%Ô%ð 	V­Z¸¿ºÀÑ8LÔ8LÍdÑ-SÔ-Sð 	VÝÐTÀzÐTÐTÐTÑUÔUÐUÝ�4˜
Ô#Ñ$Ô$¨Ò0Ð0Ð0Ð2gÑ0Ô0Ð0Ñ0å˜}Ñ-Ô-ð 	1ñ 	1‰EˆAˆrØ$ÐS�2�2­"¬'×*:Ò*:½2¼7×;KÒ;KÈBÑ;OÔ;OÑ*PÔ*PÐQRÔ*SˆDÝ”˜b tÐ,Ñ,Ô,ð v°Ý"­¬¯ª¸¸DÑ(AÔ(AÈÐQUÐ`bÐlnÐoÑoÔo�Ý.¨r°;È:ÐVÑVÔV�Ý" ;¨zÔ#:Ñ;Ô;�Ý—’ØÐtÐt˜8ÐtÐt¸rÐtÐtÀ\ÐtÐtÐ^iÐtÐtÐtñvô vð vð	vð vð vñ vô vð vð vð vð vð vð vøøøð vð vð vð vð �Ô×#Ò# KÑ0Ô0Ð0Ñ0Øð 	&Ý�LŠLÐH¸:ÐHÐHÐHÑIÔIÐIÝ�j $Ñ'Ô'ð &¨2Ý”˜D "Ñ%Ô%Ð%ð&ð &ð &ñ &ô &ð &ð &ð &ð &ð &ð &øøøð &ð &ð &ð &ð €GØ€FØ€OØÐ Øˆð €Hà+/ð \ð \ð \ð \ð \ð	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	õ ‡L‚LÐ?Ñ@Ô@Ð@Ø˜JÔ'ð 3ð 3ˆà&Ð?�4�4¨K¸Ô,?ˆÝ‘J”Jˆ	ØˆØ% jÔ1ð 	(ð 	(ˆJØ&˜, z°hÐ?Ñ?Ô?ˆKØð WÝ9AÀZÐPTÔEUÐ9VÑ9VÔ9V�	Ô" :¨fÔ#5Ñ6Ø˜KÑ'ˆLˆLØ˜˜ [°XÐ>Ñ>Ô>Ñ>ˆØð 	3Ø�OŠO˜X yÐ1Ñ2Ô2Ð2øØàð Lå�V¥C¨¡K¤Kµ[ÐAÑAÔAÑBÔBˆØKÐKµ)¸MÑ2JÔ2JÐKÑKÔKÐå‡L‚LÐ8Ñ9Ô9Ð9ØpÐpÐpÐpÅÀWÈfÑAUÔAUÐpÑpÔpÐØð cÝ$Ð%8Ð>aÐ>aÐbÑbÔbÐÝÐ/Ð0Ñ€GˆWÝŒh�wÑÔ€GÝŒh�wÑÔ€GÝ‡L‚LÐ`­¨G©¬Ð`Ð`ÅcÈ(ÁmÄmÐ`Ð`Ð`ÑaÔaÐaØ�GÐ.°Ð8Ð8s7   É"I/É/I3É6I3Í BO/Ï/O3	Ï6O3	ÑQ)Ñ)Q-Ñ0Q-c                   ó8   ‡ — e Zd ZdZd
ˆ fd„	Zd„ Zd„ Zdd	„Zˆ xZS )ÚReaderImageInTarzI Multi-tarfile dataset reader where there is one .tar file per class
    rc   TNc                 óà  •— t          ¦   «                              ¦   «          d }|rt          ||¦  «        }|| _        t	          | j        ||¬¦  «        \  | _        | _        | _        }d„ | j                             ¦   «         D ¦   «         | _	        t          |¦  «        dk    r)|d         d         €d| _        |d         d         | _        nd| _        t          |¦  «        | _        || _        d S )N)rV   rW   c                 ó   — i | ]\  }}||“Œ	S r3   r3   )r_   r€   Úvs      r   r{   z-ReaderImageInTar.__init__.<locals>.<dictcomp>Ï   s   € Ð!RÐ!RÐ!R©4¨1¨a ! QÐ!RÐ!RÐ!Rr   r   r   TF)r   r.   r   r‘   r¤   r=   r£   rV   ÚitemsÚclass_idx_to_namerM   r’   rž   rH   Úcache_tarfiles)r   r‘   Ú	class_mapr¬   rW   rV   rœ   r   s          €r   r.   zReaderImageInTar.__init__Ã   só   ø€ Ý‰Œ×ÒÑÔÐà ÐØð 	@Ý .¨y¸$Ñ ?Ô ?ÐØˆŒ	ÝGWØŒIØ/Ø'ðH
ñ H
ô H
ÑDˆŒ�d”l DÔ$:¸Hð
 "SÐ!R°4Ô3I×3OÒ3OÑ3QÔ3QÐ!RÑ!RÔ!RˆÔÝˆx‰=Œ=˜AÒÐ (¨1¤+¨a¤.Ð"8Ø#ˆDÔØ% aœ[¨œ^ˆDŒNˆNà$ˆDÔÝ! (™^œ^ˆDŒNØ,ˆÔÐÐr   c                 ó*   — t          | j        ¦  «        S r,   )rM   r=   r0   s    r   Ú__len__zReaderImageInTar.__len__Ø   s   € Ý�4”<Ñ Ô Ð r   c                 óF  — | j         |         }| j        |         }|\  }}}|r%t          j                             | j        |¦  «        n| j        }d }d }	| j        r"| j        r| j        n| j        |         }	|	j	        }|€"t          j        |¦  «        }| j        r||	_	        |�j| j        r|	j        |j                 j	        nd }
|
€Ft          j        |                     |¦  «        ¬¦  «        }
| j        r|
|	j        |j                 _	        |
}|                     |¦  «        |fS )N)r9   )r=   r£   rB   r;   rI   r‘   r¬   r’   rž   r)   r   rF   r-   r   rG   )r   ÚindexÚsampleÚtargetÚ	sample_tiÚ	parent_fnÚchild_tiÚ
parent_absr)   Úcache_staterT   s              r   Ú__getitem__zReaderImageInTar.__getitem__Û   s)  € Ø”˜eÔ$ˆØ”˜eÔ$ˆØ)/Ñ&ˆ	�9˜hØ;DÐS•R”W—\’\ $¤)¨YÑ7Ô7Ð7È$Ì)ˆ
àˆØˆØÔð 	 Ø,0Ô,<Ð[˜$œ.˜.À$Ä.ÐQZÔB[ˆKØ”ˆBØˆ:Ý”˜jÑ)Ô)ˆBØÔ"ð $Ø!#�”ØÐØ<@Ô<OÐY�+Ô& x¤}Ô5Ô8Ð8ÐUYˆCØˆ{Ý”l¨2¯>ª>¸(Ñ+CÔ+CÐDÑDÔD�ØÔ&ð AØ=@�KÔ(¨¬Ô7Ô:ØˆBà�~Š~˜iÑ(Ô(¨&Ð0Ð0r   Fc                 óx   — | j         |         d         j        }|rt          j                             |¦  «        }|S r   )r=   r   rB   r;   rR   )r   r±   rR   ÚabsoluteÚfilenames        r   Ú	_filenamezReaderImageInTar._filenameô   s8   € Ø”< Ô& qÔ)Ô.ˆØð 	2Ý”w×'Ò'¨Ñ1Ô1ˆHØˆr   )rc   TN)FF)	r    r!   r"   r#   r.   r¯   r¹   r½   r%   r&   s   @r   r¦   r¦   ¿   sy   ø€ € € € € ðð ð-ð -ð -ð -ð -ð -ð*!ð !ð !ð1ð 1ð 1ð2ð ð ð ð ð ð ð r   r¦   )NNNT)$r#   ÚloggingrB   r   r   r   Útypingr   r   r   r   r   r	   Únumpyr�   Útimm.utils.miscr
   r­   r   Úimg_extensionsr   Úreaderr   Ú	getLoggerr    rK   r„   Ú	Unpicklerr   r(   r2   r$   rJ   Úboolr¤   r¦   r3   r   r   ú<module>rÇ      s  ðð
ð 
ð €€€Ø 	€	€	€	Ø €€€Ø €€€Ø Ð Ð Ð Ð Ð Ø :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :à Ð Ð Ð à 'Ð 'Ð 'Ð 'Ð 'Ð 'à %Ð %Ð %Ð %Ð %Ð %Ø .Ð .Ð .Ð .Ð .Ð .Ø Ð Ð Ð Ð Ð à
ˆ'Ô
˜HÑ
%Ô
%€Ø*Ð ðkð kð kð kð k˜Ô(ñ kô kð kð"ð ð ð ð ñ ô ð ð˜œð °tð ÈÈSÌð ð ð ð ð. -1Ø(,Ø8<Øðl9ð l9à# Dœ>ðl9ð   ”~ðl9ð ˜U 4¨°Ð#3Ô4Ô5ð	l9ð
 ðl9ð l9ð l9ð l9ð^9ð 9ð 9ð 9ð 9�vñ 9ô 9ð 9ð 9ð 9r   