§
    ·¯¸jÂt  ã                   ó<  — d Z dgZddlZddlmZ ddlmZmZmZm	Z	m
Z
mZmZ ddlZddlmZ ddlmc mZ ddlmZmZ ddlmZmZmZmZmZmZmZmZ dd	lm Z  dd
l!m"Z" ddl#m$Z$ ddl%m&Z&m'Z' ddl(m)Z)m*Z*  G d„ dej        j+        ¦  «        Z, G d„ dej-        ¦  «        Z. G d„ dej-        ¦  «        Z/ G d„ dej-        ¦  «        Z0 G d„ dej-        ¦  «        Z1 G d„ dej-        ¦  «        Z2 G d„ dej        j-        ¦  «        Z3 G d„ dej-        ¦  «        Z4 e$e4¦  «          G d„ dej-        ¦  «        Z5 G d „ dej-        ¦  «        Z6d!„ Z7d8d#„Z8 e* e8d$d%¬&¦  «         e8d$¬'¦  «         e8d$¬'¦  «         e8d$d%¬&¦  «         e8d$¬'¦  «         e8d$¬'¦  «         e8d$d%¬&¦  «         e8d$¬'¦  «         e8d$¬'¦  «         e8d$d(d)d*¬+¦  «         e8d$d,d-d*d.¬/¦  «        d0œ¦  «        Z9d9d2„Z:e)d9d3„¦   «         Z;e)d9d4„¦   «         Z<e)d9d5„¦   «         Z=e)d9d6„¦   «         Z>e)d9d7„¦   «         Z?dS ):zÓ TinyViT

Paper: `TinyViT: Fast Pretraining Distillation for Small Vision Transformers`
    - https://arxiv.org/abs/2207.10666

Adapted from official impl at https://github.com/microsoft/Cream/tree/main/TinyViT
ÚTinyVité    N)Úpartial)ÚDictÚListÚOptionalÚTupleÚUnionÚTypeÚAny©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)ÚLayerNorm2dÚNormMlpClassifierHeadÚDropPathÚtrunc_normal_Úresize_rel_pos_bias_table_levitÚuse_fused_attnÚcalculate_drop_path_ratesÚget_device_dtypeé   )Úbuild_model_with_cfg)Úfeature_take_indices)Úregister_notrace_module)Ú
checkpointÚcheckpoint_seq)Úregister_modelÚgenerate_default_cfgsc                   ó‚   ‡ — e Zd Z	 	 	 	 	 	 	 	 ddededededed	ed
edefˆ fd„Z ej        ¦   «         d„ ¦   «         Zˆ xZ	S )ÚConvNormr   r   NÚin_chsÚout_chsÚksÚstrideÚpadÚdilationÚgroupsÚbn_weight_initc           	      óz  •— |	|
dœ}t          ¦   «                              ¦   «          t          j        |||||||fddi|¤Ž| _        t          j        |fi |¤Ž| _        t          j        j         	                    | j        j
        |¦  «         t          j        j         	                    | j        j        d¦  «         d S )N©ÚdeviceÚdtypeÚbiasFr   )ÚsuperÚ__init__ÚnnÚConv2dÚconvÚBatchNorm2dÚbnÚtorchÚinitÚ	constant_Úweightr-   )Úselfr!   r"   r#   r$   r%   r&   r'   r(   r+   r,   ÚddÚ	__class__s               €úd/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/models/tiny_vit.pyr/   zConvNorm.__init__   s¯   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ”I˜f g¨r°6¸3ÀÈ&ÐcÐcÐW\ÐcÐ`bÐcÐcˆŒ	Ý”. Ð/Ð/¨BÐ/Ð/ˆŒÝŒŒ×Ò ¤¤°Ñ?Ô?Ð?ÝŒŒ×Ò ¤¤¨aÑ0Ô0Ð0Ð0Ð0ó    c           	      ón  — | j         | j        }}|j        |j        |j        z   dz  z  }|j        |d d …d d d f         z  }|j        |j        |j        z  |j        |j        z   dz  z  z
  }t          j         	                    | 
                    d¦  «        | j         j        z  | 
                    d¦  «        |j        dd …         | j         j        | j         j        | j         j        | j         j        ¬¦  «        }|j        j                             |¦  «         |j        j                             |¦  «         |S )Ng      à?r   r   é   )r$   Úpaddingr&   r'   )r2   r4   r8   Úrunning_varÚepsr-   Úrunning_meanr5   r0   r1   Úsizer'   Úshaper$   r@   r&   ÚdataÚcopy_)r9   Úcr4   ÚwÚbÚms         r<   ÚfusezConvNorm.fuse2   s  € à”	˜4œ7ˆ2ˆØŒI˜œ¨"¬&Ñ0°SÑ8Ñ8ˆØŒH�q˜˜˜˜D $¨Ð,Ô-Ñ-ˆØŒG�b”o¨¬	Ñ1ØŒ^˜bœfÑ$¨Ñ,ñ-ñ -ˆåŒH�OŠOØ�FŠF�1‰IŒI˜œ	Ô(Ñ(¨!¯&ª&°©)¬)°Q´W¸Q¸R¸R´[Ø”9Ô#¨T¬YÔ->ÈÌÔI[ÐdhÔdmÔdtð ñ vô vˆð 	
ŒŒ×Ò˜AÑÔÐØ	ŒŒ×Ò˜!ÑÔÐØˆr=   )r   r   r   r   r   r   NN)
Ú__name__Ú
__module__Ú__qualname__ÚintÚfloatr/   r5   Úno_gradrL   Ú__classcell__©r;   s   @r<   r    r       sÍ   ø€ € € € € ð
 ØØØØØ$%ØØð1ð 1àð1ð ð1ð ð	1ð
 ð1ð ð1ð ð1ð ð1ð "ð1ð 1ð 1ð 1ð 1ð 1ð( €U„]�_„_ðð ñ „_ðð ð ð ð r=   r    c                   óN   ‡ — e Zd Z	 	 ddededeej                 fˆ fd„Zd„ Zˆ xZ	S )Ú
PatchEmbedNr!   r"   Ú	act_layerc                 óâ   •— ||dœ}t          ¦   «                              ¦   «          d| _        t          ||dz  dddfi |¤Ž| _         |¦   «         | _        t          |dz  |dddfi |¤Ž| _        d S )Nr*   é   r?   é   r   )r.   r/   r$   r    Úconv1ÚactÚconv2)r9   r!   r"   rW   r+   r,   r:   r;   s          €r<   r/   zPatchEmbed.__init__B   s†   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØˆŒÝ˜f g°¡l°A°q¸!ÐBÐB¸rÐBÐBˆŒ
Ø�9‘;”;ˆŒÝ˜g¨™l¨G°Q¸¸1ÐCÐCÀÐCÐCˆŒ
ˆ
ˆ
r=   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S ©N)r[   r\   r]   ©r9   Úxs     r<   ÚforwardzPatchEmbed.forwardQ   s4   € Ø�JŠJ�q‰MŒMˆØ�HŠH�Q‰KŒKˆØ�JŠJ�q‰MŒMˆØˆr=   ©NN©
rM   rN   rO   rP   r
   r0   ÚModuler/   rb   rS   rT   s   @r<   rV   rV   A   s…   ø€ € € € € ð ØðDð DàðDð ðDð ˜BœI”ð	Dð Dð Dð Dð Dð Dðð ð ð ð ð ð r=   rV   c                   óV   ‡ — e Zd Z	 	 d	dedededeej                 def
ˆ fd„Zd„ Z	ˆ xZ
S )
ÚMBConvNr!   r"   Úexpand_ratiorW   Ú	drop_pathc                 ó°  •— ||dœ}t          ¦   «                              ¦   «          t          ||z  ¦  «        }	t          ||	fddi|¤Ž| _         |¦   «         | _        t          |	|	fddd|	dœ|¤Ž| _         |¦   «         | _        t          |	|fdddœ|¤Ž| _         |¦   «         | _	        |dk    rt          |¦  «        nt          j        ¦   «         | _        d S )Nr*   r#   r   rZ   ©r#   r$   r%   r'   ç        )r#   r(   )r.   r/   rP   r    r[   Úact1r]   Úact2Úconv3Úact3r   r0   ÚIdentityri   )r9   r!   r"   rh   rW   ri   r+   r,   r:   Úmid_chsr;   s             €r<   r/   zMBConv.__init__Y   sê   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ�f˜|Ñ+Ñ,Ô,ˆÝ˜f gÐ:Ð:°!Ð:°rÐ:Ð:ˆŒ
Ø�I‘K”KˆŒ	Ý˜g wÐ\°1¸QÀAÈgÐ\Ð\ÐY[Ð\Ð\ˆŒ
Ø�I‘K”KˆŒ	Ý˜g wÐO°1ÀSÐOÐOÈBÐOÐOˆŒ
Ø�I‘K”KˆŒ	Ø09¸B²°� )Ñ,Ô,Ð,ÅBÄKÁMÄMˆŒˆˆr=   c                 ó:  — |}|                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }||z  }|                      |¦  «        }|S r_   )r[   rm   r]   rn   ro   ri   rp   )r9   ra   Úshortcuts      r<   rb   zMBConv.forwardn   s€   € ØˆØ�JŠJ�q‰MŒMˆØ�IŠI�a‰LŒLˆØ�JŠJ�q‰MŒMˆØ�IŠI�a‰LŒLˆØ�JŠJ�q‰MŒMˆØ�NŠN˜1ÑÔˆØ	ˆX‰ˆØ�IŠI�a‰LŒLˆØˆr=   rc   )rM   rN   rO   rP   rQ   r
   r0   re   r/   rb   rS   rT   s   @r<   rg   rg   X   s›   ø€ € € € € ð ØðRð RàðRð ðRð  ð	Rð
 ˜BœI”ðRð ðRð Rð Rð Rð Rð Rð*
ð 
ð 
ð 
ð 
ð 
ð 
r=   rg   c                   óN   ‡ — e Zd Z	 	 ddededeej                 fˆ fd„Zd„ Zˆ xZ	S )ÚPatchMergingNÚdimÚout_dimrW   c                 ó  •— ||dœ}t          ¦   «                              ¦   «          t          ||dddfi |¤Ž| _         |¦   «         | _        t          ||dddfd|i|¤Ž| _         |¦   «         | _        t          ||dddfi |¤Ž| _        d S )Nr*   r   r   rZ   r?   r'   )r.   r/   r    r[   rm   r]   rn   ro   )r9   rw   rx   rW   r+   r,   r:   r;   s          €r<   r/   zPatchMerging.__init__|   s¨   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ˜c 7¨A¨q°!Ð:Ð:°rÐ:Ð:ˆŒ
Ø�I‘K”KˆŒ	Ý˜g w°°1°aÐNÐNÀÐNÈ2ÐNÐNˆŒ
Ø�I‘K”KˆŒ	Ý˜g w°°1°aÐ>Ð>¸2Ð>Ð>ˆŒ
ˆ
ˆ
r=   c                 óØ   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r_   )r[   rm   r]   rn   ro   r`   s     r<   rb   zPatchMerging.forwardŒ   sR   € Ø�JŠJ�q‰MŒMˆØ�IŠI�a‰LŒLˆØ�JŠJ�q‰MŒMˆØ�IŠI�a‰LŒLˆØ�JŠJ�q‰MŒMˆØˆr=   rc   rd   rT   s   @r<   rv   rv   {   s{   ø€ € € € € ð Øð?ð ?àð?ð ð?ð ˜BœI”ð	?ð ?ð ?ð ?ð ?ð ?ð ð ð ð ð ð ð r=   rv   c                   óv   ‡ — e Zd Z	 	 	 	 ddededeej                 deee	e         f         def
ˆ fd	„Z
d
„ Zˆ xZS )Ú	ConvLayerrl   ç      @Nrw   ÚdepthrW   ri   Úconv_expand_ratioc                 óØ   •‡‡‡‡‡— ||dœŠt          ¦   «                              ¦   «          ‰| _        || _        t	          j        ˆˆˆˆˆfd„t          |¦  «        D ¦   «         Ž | _        d S )Nr*   c                 ól   •— g | ]0}t          ‰‰‰‰t          ‰t          ¦  «        r‰|         n‰fi ‰¤Ž‘Œ1S © )rg   Ú
isinstanceÚlist)Ú.0ÚirW   r   r:   rw   ri   s     €€€€€r<   ú
<listcomp>z&ConvLayer.__init__.<locals>.<listcomp>¤   sj   ø€ ð 
&
ð 
&
ð 
&
ð õ ØØØ!ØÝ *¨9µdÑ ;Ô ;ÐJ�	˜!”�Àðð ð ðð ð
&
ð 
&
ð 
&
r=   )r.   r/   rw   r~   r0   Ú
SequentialÚrangeÚblocks)
r9   rw   r~   rW   ri   r   r+   r,   r:   r;   s
    ` ```  @€r<   r/   zConvLayer.__init__–   s�   øøøøøø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØˆŒØˆŒ
Ý”mð 
&
ð 
&
ð 
&
ð 
&
ð 
&
ð 
&
ð 
&
ð 
&
õ ˜5‘\”\ð
&
ñ 
&
ô 
&
ð 
ˆŒˆˆr=   c                 ó0   — |                       |¦  «        }|S r_   )rŠ   r`   s     r<   rb   zConvLayer.forward°   s   € Ø�KŠK˜‰NŒNˆØˆr=   )rl   r}   NN)rM   rN   rO   rP   r
   r0   re   r	   rQ   r   r/   rb   rS   rT   s   @r<   r|   r|   •   s£   ø€ € € € € ð 46Ø')ØØðð àðð ðð ˜BœI”ð	ð
 ˜U D¨¤KÐ/Ô0ðð  %ðð ð ð ð ð ð4ð ð ð ð ð ð r=   r|   c                   ó¦   ‡ — e Zd Zddej        ej        dddfdedee         dee         deej	                 deej	                 de
fˆ fd	„Zd
„ Zˆ xZS )ÚNormMlpNrl   Úin_featuresÚhidden_featuresÚout_featuresÚ
norm_layerrW   Údropc	                 ó`  •— ||dœ}	t          ¦   «                              ¦   «          |p|}|p|} ||fi |	¤Ž| _        t          j        ||fi |	¤Ž| _         |¦   «         | _        t          j        |¦  «        | _        t          j        ||fi |	¤Ž| _	        t          j        |¦  «        | _
        d S )Nr*   )r.   r/   Únormr0   ÚLinearÚfc1r\   ÚDropoutÚdrop1Úfc2Údrop2)r9   rŽ   r�   r�   r‘   rW   r’   r+   r,   r:   r;   s             €r<   r/   zNormMlp.__init__¶   s¾   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ#Ð2 {ˆØ)Ð8¨[ˆØ�J˜{Ð1Ð1¨bÐ1Ð1ˆŒ	Ý”9˜[¨/Ð@Ð@¸RÐ@Ð@ˆŒØ�9‘;”;ˆŒÝ”Z Ñ%Ô%ˆŒ
Ý”9˜_¨lÐAÐA¸bÐAÐAˆŒÝ”Z Ñ%Ô%ˆŒ
ˆ
ˆ
r=   c                 ó  — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r_   )r”   r–   r\   r˜   r™   rš   r`   s     r<   rb   zNormMlp.forwardÌ   sa   € Ø�IŠI�a‰LŒLˆØ�HŠH�Q‰KŒKˆØ�HŠH�Q‰KŒKˆØ�JŠJ�q‰MŒMˆØ�HŠH�Q‰KŒKˆØ�JŠJ�q‰MŒMˆØˆr=   )rM   rN   rO   r0   Ú	LayerNormÚGELUrP   r   r
   re   rQ   r/   rb   rS   rT   s   @r<   r�   r�   µ   sº   ø€ € € € € ð .2Ø*.Ø*,¬,Ø)+¬ØØØð&ð &àð&ð & cœ]ð&ð # 3œ-ð	&ð
 ˜RœYœð&ð ˜BœI”ð&ð ð&ð &ð &ð &ð &ð &ð,ð ð ð ð ð ð r=   r�   c                   ó  ‡ — e Zd ZU ej        j        e         ed<   ee	ej
        f         ed<   	 	 	 	 	 ddeded	ed
edeeef         f
ˆ fd„Z ej        ¦   «         dˆ fd„	¦   «         Zdej        dej
        fd„Zd„ Zdd„Zdd„Zdd„Zˆ xZS )Ú	AttentionÚ
fused_attnÚattention_bias_cacheé   rY   ©é   r¤   Nrw   Úkey_dimÚ	num_headsÚ
attn_ratioÚ
resolutionc                 ón  •— ||dœ}t          ¦   «                              ¦   «          t          |t          ¦  «        rt	          |¦  «        dk    sJ ‚|| _        |dz  | _        || _        t          ||z  ¦  «        | _	        | j	        |z  | _
        || _        || _        t          ¦   «         | _        t          j        |fi |¤Ž| _        t          j        ||| j	        d|z  z   z  fi |¤Ž| _        t          j        | j
        |fi |¤Ž| _        |d         |d         z  }	|d         |d         z  }
t*          j                             t+          j        ||
fi |¤Ž¦  «        | _        |                      dt+          j        |	|	f|t*          j        ¬¦  «        d¬¦  «         i | _        |                      ¦   «          d S )	Nr*   r?   g      à¿r   r   Úattention_bias_idxsF)Ú
persistent)r.   r/   rƒ   ÚtupleÚlenr¦   Úscaler¥   rP   Úval_dimrx   r§   r¨   r   r    r0   rœ   r”   r•   ÚqkvÚprojr5   Ú	ParameterÚemptyÚattention_biasesÚregister_bufferÚlongr¡   Úreset_parameters)r9   rw   r¥   r¦   r§   r¨   r+   r,   r:   ÚNÚnum_offsetsr;   s              €r<   r/   zAttention.__init__Ú   s·  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ˜*¥eÑ,Ô,ÐEµ°Z±´ÀAÒ1EÐ1EÐ1EÐEØ"ˆŒØ ‘_ˆŒ
ØˆŒÝ˜:¨Ñ/Ñ0Ô0ˆŒØ”| iÑ/ˆŒØ$ˆŒØ$ˆŒÝ(Ñ*Ô*ˆŒå”L Ð+Ð+¨Ð+Ð+ˆŒ	Ý”9˜S )¨t¬|¸aÀ'¹kÑ/IÑ"JÐQÐQÈbÐQÐQˆŒÝ”I˜dœl¨CÐ6Ð6°2Ð6Ð6ˆŒ	à�qŒM˜J qœMÑ)ˆØ  ”m j°¤mÑ3ˆÝ %¤× 2Ò 2µ5´;¸yÈ+Ð3\Ð3\ÐY[Ð3\Ð3\Ñ ]Ô ]ˆÔØ×ÒØ!ÝŒK˜˜A˜ vµU´ZÐ@Ñ@Ô@Øð 	ñ 	
ô 	
ð 	
ð
 %'ˆÔ!ð 	×ÒÑÔÐÐÐr=   Tc                 ój   •— t          ¦   «                              |¦  «         | j        r	i | _        d S d S r_   )r.   Útrainr¡   )r9   Úmoder;   s     €r<   r»   zAttention.train  s<   ø€ å‰Œ�Š�dÑÔÐØÔ$ð 	+Ø(*ˆDÔ%Ð%Ð%ð	+ð 	+r=   r+   Úreturnc                 óþ   — t           j                             ¦   «         s| j        r| j        d d …| j        f         S t          |¦  «        }|| j        vr| j        d d …| j        f         | j        |<   | j        |         S r_   )r5   ÚjitÚ
is_tracingÚtrainingr´   rª   Ústrr¡   )r9   r+   Ú
device_keys      r<   Úget_attention_biaseszAttention.get_attention_biases  s„   € ÝŒ9×ÒÑ!Ô!ð 	9 T¤]ð 	9ØÔ(¨¨¨¨DÔ,DÐ)DÔEÐEå˜V™œˆJØ Ô!:Ð:Ð:Ø8<Ô8MÈaÈaÈaÐQUÔQiÐNiÔ8j�Ô)¨*Ñ5ØÔ,¨ZÔ8Ð8r=   c                 ó  — |                       |j        ¦  «        }|j        \  }}}|                      |¦  «        }|                      |¦  «        }|                     ||| j        d¦  «                             | j        | j        | j	        gd¬¦  «        \  }}}	| 
                    dddd¦  «        }| 
                    dddd¦  «        }|	 
                    dddd¦  «        }	| j        rt          j        |||	|¬¦  «        }nC|| j        z  }||                     dd¦  «        z  }
|
|z   }
|
                     d¬¦  «        }
|
|	z  }|                     dd¦  «                             ||| j        ¦  «        }|                      |¦  «        }|S )	NéÿÿÿÿrZ   )rw   r   r?   r   )Ú	attn_maskéþÿÿÿ)rÄ   r+   rE   r”   r°   Úviewr¦   Úsplitr¥   r¯   Úpermuter    ÚFÚscaled_dot_product_attentionr®   Ú	transposeÚsoftmaxÚreshaperx   r±   )r9   ra   Ú	attn_biasÚBr¸   Ú_r°   ÚqÚkÚvÚattns              r<   rb   zAttention.forward  sr  € Ø×-Ò-¨a¬hÑ7Ô7ˆ	Ø”'‰ˆˆ1ˆaà�IŠI�a‰LŒLˆØ�hŠh�q‰kŒkˆà—(’(˜1˜a ¤°Ñ4Ô4×:Ò:¸D¼LÈ$Ì,ÐX\ÔXdÐ;eÐklÐ:ÑmÔm‰ˆˆ1ˆaà�IŠI�a˜˜A˜qÑ!Ô!ˆØ�IŠI�a˜˜A˜qÑ!Ô!ˆØ�IŠI�a˜˜A˜qÑ!Ô!ˆàŒ?ð 	ÝÔ.¨q°!°QÀ)ÐLÑLÔLˆAˆAà�D”J‘ˆAØ�q—{’{ 2 rÑ*Ô*Ñ*ˆDØ˜)Ñ#ˆDØ—<’< B�<Ñ'Ô'ˆDØ�q‘ˆAØ�KŠK˜˜1ÑÔ×%Ò% a¨¨D¬LÑ9Ô9ˆØ�IŠI�a‰LŒLˆØˆr=   c                 óv   — t           j                             | j        ¦  «         |                      ¦   «          dS )z"Initialize parameters and buffers.N)r0   r6   Úzeros_r´   Ú_init_buffers©r9   s    r<   r·   zAttention.reset_parameters)  s1   € å
Œ�Š�tÔ,Ñ-Ô-Ð-Ø×ÒÑÔÐÐÐr=   c           	      ó~  — | j         j        }t          t          j        t          | j        d         ¦  «        t          | j        d         ¦  «        ¦  «        ¦  «        }t          |¦  «        }i }g }|D ]t}|D ]o}t          |d         |d         z
  ¦  «        t          |d         |d         z
  ¦  «        f}||vrt          |¦  «        ||<   | 	                    ||         ¦  «         ŒpŒu| j          
                    t          j        ||t          j        ¬¦  «                             ||¦  «        ¦  «         i | _        dS )z.Compute and fill non-persistent buffer values.r   r   r*   N)rª   r+   r„   Ú	itertoolsÚproductr‰   r¨   r­   ÚabsÚappendrG   r5   Útensorr¶   rÉ   r¡   )	r9   r+   Úpointsr¸   Úattention_offsetsÚidxsÚp1Úp2Úoffsets	            r<   rÚ   zAttention._init_buffers.  s8  € àÔ)Ô0ˆÝ•iÔ'­¨d¬o¸aÔ.@Ñ(AÔ(AÅ5ÈÌÐYZÔI[ÑC\ÔC\Ñ]Ô]Ñ^Ô^ˆÝ�‰KŒKˆØÐØˆØð 	7ð 	7ˆBØð 7ð 7�Ý˜b œe b¨¤e™mÑ,Ô,­c°"°Q´%¸"¸Q¼%±-Ñ.@Ô.@ÐA�ØÐ!2Ð2Ð2Ý03Ð4EÑ0FÔ0FÐ% fÑ-Ø—’Ð-¨fÔ5Ñ6Ô6Ð6Ð6ð	7ð
 	Ô ×&Ò&¥u¤|°DÀÍuÌzÐ'ZÑ'ZÔ'Z×'_Ò'_Ð`aÐcdÑ'eÔ'eÑfÔfÐfØ$&ˆÔ!Ð!Ð!r=   c                 ó.   — |                       ¦   «          dS )z"Initialize non-persistent buffers.N)rÚ   rÛ   s    r<   Úinit_non_persistent_buffersz%Attention.init_non_persistent_buffers>  s   € à×ÒÑÔÐÐÐr=   )r¢   rY   r£   NN©T)r½   N)rM   rN   rO   r5   r¿   ÚFinalÚboolÚ__annotations__r   rÂ   ÚTensorrP   r   r/   rR   r»   r+   rÄ   rb   r·   rÚ   ré   rS   rT   s   @r<   rŸ   rŸ   Ö   s^  ø€ € € € € € Ø”	” Ô%Ð%Ð%Ñ%Ø˜s E¤LÐ0Ô1Ð1Ð1Ñ1ð ØØ*2ØØð% ð % àð% ð ð% ð ð	% ð
 ð% ð ˜c 3˜hœð% ð % ð % ð % ð % ð % ðN €U„]�_„_ð+ð +ð +ð +ð +ñ „_ð+ð
9¨5¬<ð 9¸E¼Lð 9ð 9ð 9ð 9ðð ð ð2ð ð ð ð
'ð 'ð 'ð 'ð ð ð ð ð ð ð ð r=   rŸ   c                   óˆ   ‡ — e Zd ZdZdddddej        ddfdeded	ed
ededededeej	                 fˆ fd„Z
d„ Zdefd„Zˆ xZS )ÚTinyVitBlocka5   TinyViT Block.

    Args:
        dim (int): Number of input channels.
        num_heads (int): Number of attention heads.
        window_size (int): Window size.
        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
        drop (float, optional): Dropout rate. Default: 0.0
        drop_path (float, optional): Stochastic depth rate. Default: 0.0
        local_conv_size (int): the kernel size of the convolution between
                               Attention and MLP. Default: 3
        act_layer: the activation function. Default: nn.GELU
    é   r}   rl   rZ   Nrw   r¦   Úwindow_sizeÚ	mlp_ratior’   ri   Úlocal_conv_sizerW   c                 óJ  •— |	|
dœ}t          ¦   «                              ¦   «          || _        || _        |dk    s
J d¦   «         ‚|| _        || _        ||z  dk    s
J d¦   «         ‚||z  }||f}t          |||fd|dœ|¤Ž| _        |dk    rt          |¦  «        nt          j
        ¦   «         | _        t          d|t          ||z  ¦  «        ||dœ|¤Ž| _        |dk    rt          |¦  «        nt          j
        ¦   «         | _        |d	z  }t!          ||f|d||d
œ|¤Ž| _        d S )Nr*   r   z"window_size must be greater than 0z"dim must be divisible by num_headsr   )r§   r¨   rl   )rŽ   r�   rW   r’   r?   rk   r‚   )r.   r/   rw   r¦   rò   ró   rŸ   r×   r   r0   rq   Ú
drop_path1r�   rP   ÚmlpÚ
drop_path2r    Ú
local_conv)r9   rw   r¦   rò   ró   r’   ri   rô   rW   r+   r,   r:   Úhead_dimÚwindow_resolutionr%   r;   s                  €r<   r/   zTinyVitBlock.__init__R  s{  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØˆŒØ"ˆŒØ˜QŠˆˆÐ D‰ŒˆØ&ˆÔØ"ˆŒà�Y‰ !Ò#Ð#Ð#Ð%IÑ#Ô#Ð#Ø˜)Ñ#ˆà(¨+Ð6ÐÝØØØð
ð Ø(ð
ð 
ð ð
ð 
ˆŒ	ð 2;¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒåð 
ØÝ  i¡Ñ0Ô0ØØð	
ð 
ð
 ð
ð 
ˆŒð 2;¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒà Ñ"ˆÝ" 3¨Ðe°ÈÈsÐ[^ÐeÐeÐbdÐeÐeˆŒˆˆr=   c           	      ó  — |j         \  }}}}||z  }|}|| j        k    rQ|| j        k    rF|                     |||¦  «        }|                      |¦  «        }|                     ||||¦  «        }�nc| j        || j        z  z
  | j        z  }| j        || j        z  z
  | j        z  }	|dk    p|	dk    }
|
rt          j        |ddd|	d|f¦  «        }||z   ||	z   }}|| j        z  }|| j        z  }|                     ||| j        || j        |¦  «                             dd¦  «                             ||z  |z  | j        | j        z  |¦  «        }|                      |¦  «        }|                     |||| j        | j        |¦  «                             dd¦  «                             ||||¦  «        }|
r#|d d …d |…d |…f                              ¦   «         }||  	                    |¦  «        z   }| 
                    dddd¦  «        }|                      |¦  «        }|                     |||¦  «                             dd¦  «        }||                      |                      |¦  «        ¦  «        z   }|                     ||||¦  «        S ©Nr   r?   rZ   r   )rE   rò   rÐ   r×   rÉ   rÌ   r%   rÎ   Ú
contiguousrö   rË   rù   rø   r÷   )r9   ra   rÒ   ÚHÚWÚCÚLrt   Úpad_bÚpad_rr@   ÚpHÚpWÚnHÚnWs                  r<   rb   zTinyVitBlock.forward�  s›  € Ø”W‰
ˆˆ1ˆa�Ø�‰EˆàˆØ�Ô Ò Ð  Q¨$Ô*:Ò%:Ð%:Ø—	’	˜!˜Q Ñ"Ô"ˆAØ—	’	˜!‘”ˆAØ—’�q˜!˜Q Ñ"Ô"ˆA‰AàÔ%¨¨DÔ,<Ñ(<Ñ<ÀÔ@PÑPˆEØÔ%¨¨DÔ,<Ñ(<Ñ<ÀÔ@PÑPˆEØ˜a’iÐ, 5¨1¢9ˆGØð 9Ý”E˜!˜a  A u¨a°Ð7Ñ8Ô8�ð ˜‘Y  E¡	�ˆBØ�tÔ'Ñ'ˆBØ�tÔ'Ñ'ˆBØ—’�q˜"˜dÔ.°°DÔ4DÀaÑHÔH×RÒRÐSTÐVWÑXÔX×`Ò`Ø�B‘˜‘˜TÔ-°Ô0@Ñ@À!ñô ˆAð —	’	˜!‘”ˆAð —’�q˜"˜b $Ô"2°DÔ4DÀaÑHÔH×RÒRÐSTÐVWÑXÔX×`Ò`ÐabÐdfÐhjÐlmÑnÔnˆAàð .Ø�a�a�a˜˜!˜˜R˜a˜R�i”L×+Ò+Ñ-Ô-�Ø�t—’ qÑ)Ô)Ñ)ˆà�IŠI�a˜˜A˜qÑ!Ô!ˆØ�OŠO˜AÑÔˆØ�IŠI�a˜˜AÑÔ×(Ò(¨¨AÑ.Ô.ˆà�—’ §¢¨¡¤Ñ,Ô,Ñ,ˆØ�vŠv�a˜˜A˜qÑ!Ô!Ð!r=   r½   c                 óF   — d| j         › d| j        › d| j        › d| j        › �S )Núdim=z, num_heads=z, window_size=z, mlp_ratio=)rw   r¦   rò   ró   rÛ   s    r<   Ú
extra_reprzTinyVitBlock.extra_repr©  sQ   € ðM�d”hð Mð M¨D¬Nð Mð MØ"Ô.ðMð MØ<@¼NðMð Mð 	Mr=   )rM   rN   rO   Ú__doc__r0   r�   rP   rQ   r
   re   r/   rb   rÂ   r  rS   rT   s   @r<   rð   rð   C  s   ø€ € € € € ðð ð$  !Ø!ØØ!Ø#$Ø)+¬ØØð-fð -fàð-fð ð-fð ð	-fð
 ð-fð ð-fð ð-fð !ð-fð ˜BœI”ð-fð -fð -fð -fð -fð -fð^&"ð &"ð &"ðPM˜Cð Mð Mð Mð Mð Mð Mð Mð Mr=   rð   c                   óÒ   ‡ — e Zd ZdZdddddej        ddfdededed	ed
edededeee	e         f         de
eej                          dedeej                 fˆ fd„Zd„ Zdefd„Zˆ xZS )ÚTinyVitStagea   A basic TinyViT layer for one stage.

    Args:
        dim (int): Number of input channels.
        out_dim: the output dimension of the layer
        depth (int): Number of blocks.
        num_heads (int): Number of attention heads.
        window_size (int): Local window size.
        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
        drop (float, optional): Dropout rate. Default: 0.0
        drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0
        downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None
        local_conv_size: the kernel size of the depthwise convolution between attention and MLP. Default: 3
        act_layer: the activation function. Default: nn.GELU
    r}   rl   NrZ   rw   rx   r~   r¦   rò   ró   r’   ri   Ú
downsamplerô   rW   c                 óP  •‡‡‡‡‡‡‡
‡‡— ||dœŠt          ¦   «                              ¦   «          || _        ‰| _        |	� |	d|‰‰dœ‰¤Ž| _        n t          j        ¦   «         | _        |‰k    sJ ‚t          j        ˆˆˆˆˆ
ˆˆˆˆf	d„t          |¦  «        D ¦   «         Ž | _	        d S )Nr*   )rw   rx   rW   c                 ót   •	— g | ]4}t          d‰	‰‰
‰‰t          ‰t          ¦  «        r‰|         n‰‰‰d œ‰¤Ž‘Œ5S ))rw   r¦   rò   ró   r’   ri   rô   rW   r‚   )rð   rƒ   r„   )r…   r†   rW   r:   r’   ri   rô   ró   r¦   rx   rò   s     €€€€€€€€€r<   r‡   z)TinyVitStage.__init__.<locals>.<listcomp>ä  sx   ø€ ð &#ð &#ð &#ð õ ð 
ØØ#Ø'Ø#ØÝ*4°YÅÑ*EÔ*EÐT˜) Aœ,˜,È9Ø /Ø#ð
ð 
ð ð
ð 
ð&#ð &#ð &#r=   r‚   )
r.   r/   r~   rx   r  r0   rq   rˆ   r‰   rŠ   )r9   rw   rx   r~   r¦   rò   ró   r’   ri   r  rô   rW   r+   r,   r:   r;   s     ` ````` ``  @€r<   r/   zTinyVitStage.__init__Â  sÿ   øøøøøøøøøø€ ð  ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØˆŒ
ØˆŒð Ð!Ø(˜jð ØØØ#ðð ð ð	ð ˆDŒOˆOõ !œk™mœmˆDŒOØ˜'’>�>�>�>õ ”mð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#õ ˜5‘\”\ð&#ñ &#ô &#ð $ˆŒˆˆr=   c                 óº   — |                       |¦  «        }|                     dddd¦  «        }|                      |¦  «        }|                     dddd¦  «        }|S rý   )r  rË   rŠ   r`   s     r<   rb   zTinyVitStage.forwardò  sU   € Ø�OŠO˜AÑÔˆØ�IŠI�a˜˜A˜qÑ!Ô!ˆØ�KŠK˜‰NŒNˆØ�IŠI�a˜˜A˜qÑ!Ô!ˆØˆr=   r½   c                 ó&   — d| j         › d| j        › �S )Nr
  z, depth=)rx   r~   rÛ   s    r<   r  zTinyVitStage.extra_reprù  s   € Ø8�d”lÐ8Ð8¨D¬JÐ8Ð8Ð8r=   )rM   rN   rO   r  r0   r�   rP   rQ   r	   r   r   r
   re   r/   rb   rÂ   r  rS   rT   s   @r<   r  r  ±  s  ø€ € € € € ðð ð.  "ØØ35Ø48Ø#$Ø)+¬ØØð.$ð .$àð.$ð ð.$ð ð	.$ð
 ð.$ð ð.$ð ð.$ð ð.$ð ˜U D¨¤KÐ/Ô0ð.$ð !  b¤i¤Ô1ð.$ð !ð.$ð ˜BœI”ð.$ð .$ð .$ð .$ð .$ð .$ð`ð ð ð9˜Cð 9ð 9ð 9ð 9ð 9ð 9ð 9ð 9r=   r  c                   ó  ‡ — e Zd Zddddddddd	d
dddej        ddfdedededeedf         deedf         deedf         deedf         dededede	dedede
ej                 fˆ fd„Zd9de	fd„Zd9d ej        de	fd!„Zej        j        d"„ ¦   «         Zej        j        d#„ ¦   «         Zej        j        d:d$„¦   «         Zej        j        d9d%„¦   «         Zej        j        d&ej        fd'„¦   «         Zd;dedee         fd(„Z	 	 	 	 	 d<d*ej        d+eeeee         f                  d,e	d-e	d.ed/e	d&eeej                 eej        eej                 f         f         fd0„Z	 	 	 d=d+eeee         f         d2e	d3e	fd4„Zd5„ Zd:d6e	fd7„Zd8„ Z ˆ xZ!S )>r   rZ   éè  Úavg)é`   éÀ   é€  i   ©r?   r?   é   r?   )rZ   r  é   é   ©rñ   rñ   r¤   rñ   r}   rl   çš™™™™™¹?FNÚin_chansÚnum_classesÚglobal_poolÚ
embed_dims.Údepthsr¦   Úwindow_sizesró   Ú	drop_rateÚdrop_path_rateÚuse_checkpointÚmbconv_expand_ratiorô   rW   c                 óV  •— t          ¦   «                              ¦   «          ||dœ}|| _        || _        || _        t          |¦  «        | _        || _        || _        t          d||d         |dœ|¤Ž| _
        t          |
t          |¦  «        ¦  «        }t          j        ¦   «         | _        | j
        j        }|d         }g | _        t%          | j        ¦  «        D ]ð}|dk    r't'          d|||         ||d ||         …         |dœ|¤Ž}nƒ||         }|t          |d |…         ¦  «        t          |d |dz   …         ¦  «        …         }
t)          d||dz
           |||         ||         ||         | j        |	||
t*          |dœ|¤Ž}|}|dz  }| j                             |¦  «         | xj        t/          ||d|› �¬	¦  «        gz  c_        Œñ|d
         x| _        | _        t5          t6          d¬¦  «        }t9          | j        |f||dœ|¤Ž| _        |                      d¬¦  «         d S )Nr*   r   )r!   r"   rW   )rw   r~   rW   ri   r   r   )rw   rx   r~   r¦   rò   ró   r’   rô   ri   r  rW   r?   zstages.)Únum_chsÚ	reductionÚmodulerÆ   gñhãˆµøä>)rB   )Ú	pool_typer‘   F©Úneeds_resetr‚   )r.   r/   r!  r   r$  r­   Ú
num_stagesró   Úgrad_checkpointingrV   Úpatch_embedr   Úsumr0   rˆ   Ústagesr$   Úfeature_infor‰   r|   r  rv   rà   ÚdictÚnum_featuresÚhead_hidden_sizer   r   r   ÚheadÚinit_weights)r9   r   r!  r"  r#  r$  r¦   r%  ró   r&  r'  r(  r)  rô   rW   r+   r,   r:   Údprr$   Úprev_dimÚ	stage_idxÚstagerx   Únorm_layer_cfr;   s                            €r<   r/   zTinyVit.__init__þ  s²  ø€ õ& 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆà&ˆÔØ ˆŒØˆŒÝ˜f™+œ+ˆŒØ"ˆŒØ"0ˆÔå%ð 
ØØ˜q”MØð
ð 
ð ð	
ð 
ˆÔõ (¨½¸F¹¼ÑDÔDˆõ ”m‘o”oˆŒØÔ!Ô(ˆØ˜a”=ˆØˆÔÝ˜tœÑ/Ô/ð 	jð 	jˆIØ˜AŠ~ˆ~Ý!ð Ø Ø  Ô+Ø'Ø!Ð"4 6¨)Ô#4Ð"4Ô5Ø&9ðð ð ðð ��ð % YÔ/�Ø!$¥S¨°
°°
Ô);Ñ%<Ô%<½SÀÈÈÐUVÉÈÔAWÑ=XÔ=XÐ%XÔ!Y�Ý$ð Ø" 9¨q¡=Ô1Ø#Ø  Ô+Ø'¨	Ô2Ø ,¨YÔ 7Ø"œnØ"Ø$3Ø,Ý+Ø'ðð ð ðð �ð #�Ø˜!‘�ØŒK×Ò˜uÑ%Ô%Ð%ØÐÔ¥$¨xÀ6ÐRgÐ\eÐRgÐRgÐ"hÑ"hÔ"hÐ!iÑiÐÔÐð 5?¸r´NÐBˆÔ˜DÔ1å¥°Ð6Ñ6Ô6ˆÝ)ØÔØð
ð "Ø$ð	
ð 
ð
 ð
ð 
ˆŒ	ð 	×Ò eÐÑ,Ô,Ð,Ð,Ð,r=   Tr0  c                 óX   — |                       t          | j        |¬¦  «        ¦  «         d S )Nr/  )Úapplyr   Ú_init_weights)r9   r0  s     r<   r;  zTinyVit.init_weightsY  s*   € Ø�
Š
•7˜4Ô-¸;ÐGÑGÔGÑHÔHÐHÐHÐHr=   rK   c                 ó  — t          |t          j        ¦  «        rFt          |j        d¬¦  «         |j        �'t          j                             |j        d¦  «         d S d S |r&t          |d¦  «        r| 	                    ¦   «          d S d S d S )Ng{®Gáz”?)Ústdr   r·   )
rƒ   r0   r•   r   r8   r-   r6   r7   Úhasattrr·   )r9   rK   r0  s      r<   rC  zTinyVit._init_weights\  s¡   € Ý�a�œÑ#Ô#ð 	!Ý˜!œ(¨Ð,Ñ,Ô,Ð,ØŒvÐ!Ý”×!Ò! !¤&¨!Ñ,Ô,Ð,Ð,Ð,ð "Ð!àð 	!�W QÐ(:Ñ;Ô;ð 	!Ø×ÒÑ Ô Ð Ð Ð ð	!ð 	!ð 	!ð 	!r=   c                 ó   — dhS )Nr´   r‚   rÛ   s    r<   Úno_weight_decay_keywordsz TinyVit.no_weight_decay_keywordsd  s   € à"Ð#Ð#r=   c                 ób   — d„ |                       ¦   «                              ¦   «         D ¦   «         S )Nc                 ó   — h | ]}d |v ¯|’Œ	S )r´   r‚   )r…   ra   s     r<   ú	<setcomp>z*TinyVit.no_weight_decay.<locals>.<setcomp>j  s#   € ÐOÐOÐO�aÐ7IÈQÐ7NÐ7N�Ð7NÐ7NÐ7Nr=   )Ú
state_dictÚkeysrÛ   s    r<   Úno_weight_decayzTinyVit.no_weight_decayh  s,   € àOÐO˜4Ÿ?š?Ñ,Ô,×1Ò1Ñ3Ô3ÐOÑOÔOÐOr=   c                 ó4   — t          d|rdnddg¬¦  «        }|S )Nz^patch_embedz^stages\.(\d+))z^stages\.(\d+).downsample)r   )z^stages\.(\d+)\.\w+\.(\d+)N)ÚstemrŠ   )r7  )r9   ÚcoarseÚmatchers      r<   Úgroup_matcherzTinyVit.group_matcherl  s9   € åØ Ø(.ð Ð$Ð$Ø4Ø5ð5ð
ñ 
ô 
ˆð ˆr=   c                 ó   — || _         d S r_   )r2  )r9   Úenables     r<   Úset_grad_checkpointingzTinyVit.set_grad_checkpointingw  s   € à"(ˆÔÐÐr=   r½   c                 ó   — | j         j        S r_   )r:  ÚfcrÛ   s    r<   Úget_classifierzTinyVit.get_classifier{  s   € àŒyŒ|Ðr=   c                 ó\   — t          | ¦  «        }|| _         | j        j        |fd|i|¤Ž d S )Nr.  )r   r!  r:  Úreset)r9   r!  r"  r:   s       r<   Úreset_classifierzTinyVit.reset_classifier  s>   € Ý˜dÑ#Ô#ˆØ&ˆÔØˆŒ	Œ˜ÐAÐA¨{ÐA¸bÐAÐAÐAÐAÐAr=   ÚNCHWra   Úindicesr”   Ú
stop_earlyÚ
output_fmtÚintermediates_onlyc                 óø  — |dv s
J d¦   «         ‚g }t          t          | j        ¦  «        |¦  «        \  }}	|                      |¦  «        }t          j                             ¦   «         s|s| j        }
n| j        d|	dz   …         }
t          |
¦  «        D ]_\  }}| j        r/t          j                             ¦   «         st          ||¦  «        }n ||¦  «        }||v r| 
                    |¦  «         Œ`|r|S ||fS )aï   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )r]  zOutput shape must be NCHW.Nr   )r   r­   r5  r3  r5   r¿   Úis_scriptingÚ	enumerater2  r   rà   )r9   ra   r^  r”   r_  r`  ra  ÚintermediatesÚtake_indicesÚ	max_indexr5  Úfeat_idxr?  s                r<   Úforward_intermediateszTinyVit.forward_intermediates„  s"  € ð* ˜YÐ&Ð&Ð&Ð(DÑ&Ô&Ð&ØˆÝ"6µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�ið ×Ò˜QÑÔˆÝŒ9×!Ò!Ñ#Ô#ð 	1¨:ð 	1Ø”[ˆFˆFà”[  )¨a¡- Ô0ˆFå(¨Ñ0Ô0ð 	(ð 	(‰OˆH�eØÔ&ð ­u¬y×/EÒ/EÑ/GÔ/Gð Ý˜u aÑ(Ô(��à�E˜!‘H”H�Ø˜<Ð'Ð'Ø×$Ò$ QÑ'Ô'Ð'øàð 	!Ø Ð à�-ÐÐr=   r   Ú
prune_normÚ
prune_headc                 ó®   — t          t          | j        ¦  «        |¦  «        \  }}| j        d|dz   …         | _        |r|                      dd¦  «         |S )z@ Prune layers not required for specified intermediates.
        Nr   r   Ú )r   r­   r5  r\  )r9   r^  rj  rk  rf  rg  s         r<   Úprune_intermediate_layersz!TinyVit.prune_intermediate_layers±  s\   € õ #7µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�iØ”k . 9¨q¡= .Ô1ˆŒØð 	)Ø×!Ò! ! RÑ(Ô(Ð(ØÐr=   c                 óÐ   — |                       |¦  «        }| j        r4t          j                             ¦   «         st          | j        |¦  «        }n|                      |¦  «        }|S r_   )r3  r2  r5   r¿   rc  r   r5  r`   s     r<   Úforward_featureszTinyVit.forward_features¿  s\   € Ø×Ò˜QÑÔˆØÔ"ð 	­5¬9×+AÒ+AÑ+CÔ+Cð 	Ý˜tœ{¨AÑ.Ô.ˆAˆAà—’˜A‘”ˆAØˆr=   Ú
pre_logitsc                 ób   — |r|                       ||¬¦  «        n|                       |¦  «        }|S )N)rq  )r:  )r9   ra   rq  s      r<   Úforward_headzTinyVit.forward_headÇ  s0   € Ø3=ÐOˆD�IŠI�a JˆIÑ/Ô/Ð/À4Ç9Â9ÈQÁ<Ä<ˆØˆr=   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S r_   )rp  rs  r`   s     r<   rb   zTinyVit.forwardË  s-   € Ø×!Ò! !Ñ$Ô$ˆØ×Ò˜aÑ Ô ˆØˆr=   rê   ©Fr_   )NFFr]  F)r   FT)"rM   rN   rO   r0   r�   rP   rÂ   r   rQ   rì   r
   re   r/   r;  rC  r5   r¿   ÚignorerH  rN  rS  rV  rY  r   r\  rî   r	   r   ri  rn  rp  rs  rb   rS   rT   s   @r<   r   r   ý  s¯  ø€ € € € € ð Ø#Ø$Ø*=Ø&2Ø)7Ø,9Ø!Ø!Ø$'Ø#(Ø),Ø#$Ø)+¬ØØð#Y-ð Y-àðY-ð ðY-ð ð	Y-ð
 ˜c 3˜hœðY-ð ˜#˜s˜(”OðY-ð ˜S #˜X”ðY-ð    S œ/ðY-ð ðY-ð ðY-ð "ðY-ð !ðY-ð "'ðY-ð !ðY-ð ˜BœI”ðY-ð Y-ð Y-ð Y-ð Y-ð Y-ðvIð I¨ð Ið Ið Ið Ið!ð !˜rœyð !°tð !ð !ð !ð !ð „YÔð$ð $ñ Ôð$ð „YÔðPð Pñ ÔðPð „YÔðð ð ñ Ôðð „YÔð)ð )ð )ñ Ôð)ð „YÔð ¤	ð ð ð ñ ÔððBð B¨Cð B¸hÀs¼mð Bð Bð Bð Bð 8<ØØ$Ø$Ø',ð+ ð + àŒ|ð+ ð ˜e C¨¨c¬ NÔ3Ô4ð+ ð ð	+ ð
 ð+ ð ð+ ð !%ð+ ð 
ˆt�E”LÔ! 5¨¬°t¸E¼LÔ7IÐ)IÔ#JÐJÔ	Kð+ ð + ð + ð + ð^ ./Ø$Ø#ð	ð à˜3  S¤	˜>Ô*ðð ðð ð	ð ð ð ðð ð ðð ¨$ð ð ð ð ðð ð ð ð ð ð r=   c                 ó<  — d|                       ¦   «         v r| d         } |                     ¦   «         }i }|                      ¦   «         D ]R\  }}|                     d¦  «        rŒd|v r.t	          |j        ||         j        d d d…         ¦  «        j        }|||<   ŒS|S )NÚmodelrª   r´   rÆ   )rM  rL  ÚitemsÚendswithr   ÚTrE   )rL  rx  Ú	target_sdÚout_dictrÕ   rÖ   s         r<   Úcheckpoint_filter_fnr~  Ñ  s¯   € Ø�*—/’/Ñ#Ô#Ð#Ð#Ø Ô(ˆ
Ø× Ò Ñ"Ô"€IØ€HØ× Ò Ñ"Ô"ð ð ‰ˆˆ1Ø�:Š:Ð+Ñ,Ô,ð 	ØØ Ð"Ð"å/°´°Y¸q´\Ô5GÈÈÈ"ÈÔ5MÑNÔNÔPˆAØˆ�‰ˆØ€Or=   rm  c                 ó4   — | dt           t          dddddddœ
|¥S )	Nr  zpatch_embed.conv1.convzhead.fc)rñ   rñ   )rZ   éà   r€  gffffffî?z
apache-2.0)
Úurlr!  ÚmeanrE  Ú
first_convÚ
classifierÚ	pool_sizeÚ
input_sizeÚcrop_pctÚlicenser   )r�  Úkwargss     r<   Ú_cfgrŠ  à  s9   € àØÝ%Ý#Ø.ØØØ#ØØðð ð ðð r=   ztimm/iQU  )Ú	hf_hub_idr!  )r‹  )rZ   r  r  )r  r  g      ð?)r‹  r†  r…  r‡  )rZ   é   rŒ  )é   r�  Úsquash)r‹  r†  r…  r‡  Ú	crop_mode)ztiny_vit_5m_224.dist_in22kz"tiny_vit_5m_224.dist_in22k_ft_in1kztiny_vit_5m_224.in1kztiny_vit_11m_224.dist_in22kz#tiny_vit_11m_224.dist_in22k_ft_in1kztiny_vit_11m_224.in1kztiny_vit_21m_224.dist_in22kz#tiny_vit_21m_224.dist_in22k_ft_in1kztiny_vit_21m_224.in1kz#tiny_vit_21m_384.dist_in22k_ft_in1kz#tiny_vit_21m_512.dist_in22k_ft_in1kFc                 óˆ   — |                      dd¦  «        }t          t          | |ft          d|¬¦  «        t          dœ|¤Ž}|S )NÚout_indices)r   r   r?   rZ   T)Úflatten_sequentialr‘  )Úfeature_cfgÚpretrained_filter_fn)Úpopr   r   r7  r~  )ÚvariantÚ
pretrainedr‰  r‘  rx  s        r<   Ú_create_tiny_vitr˜  %  s]   € Ø—*’*˜]¨LÑ9Ô9€KÝ ÝØØðõ ¨D¸kÐJÑJÔJÝ1ðð ð ðð €Eð €Lr=   c                 ó€   — t          g d¢g d¢g d¢g d¢d¬¦  «        }|                     |¦  «         t          d| fi |¤ŽS )N)é@   é€   é    i@  r  )r?   rY   é   é
   r  rl   ©r#  r$  r¦   r%  r'  Útiny_vit_5m_224©r7  Úupdater˜  ©r—  r‰  Úmodel_kwargss      r<   r   r   2  sf   € åØ&Ð&Ð&Øˆ|ˆ|Ø�-�-Ø"�]�]Øðñ ô €Lð ×Ò˜ÑÔÐÝÐ-¨zÐJÐJ¸\ÐJÐJÐJr=   c                 ó€   — t          g d¢g d¢g d¢g d¢d¬¦  «        }|                     |¦  «         t          d| fi |¤ŽS )N)rš  r›  é   iÀ  r  )r?   rY   r¢   r¤   r  r  rŸ  Útiny_vit_11m_224r¡  r£  s      r<   r§  r§  ?  sf   € åØ&Ð&Ð&Øˆ|ˆ|Ø�-�-Ø"�]�]Øðñ ô €Lð ×Ò˜ÑÔÐÝÐ.°
ÐKÐK¸lÐKÐKÐKr=   c                 ó€   — t          g d¢g d¢g d¢g d¢d¬¦  «        }|                     |¦  «         t          d| fi |¤ŽS )N©r  r  r  i@  r  ©rZ   r  r  é   r  gš™™™™™É?rŸ  Útiny_vit_21m_224r¡  r£  s      r<   r¬  r¬  L  sf   € åØ&Ð&Ð&Øˆ|ˆ|Ø �.�.Ø"�]�]Øðñ ô €Lð ×Ò˜ÑÔÐÝÐ.°
ÐKÐK¸lÐKÐKÐKr=   c                 ó€   — t          g d¢g d¢g d¢g d¢d¬¦  «        }|                     |¦  «         t          d| fi |¤ŽS )Nr©  r  rª  )r  r  r  r  r  rŸ  Útiny_vit_21m_384r¡  r£  s      r<   r®  r®  Y  óh   € åØ&Ð&Ð&Øˆ|ˆ|Ø �.�.Ø%Ð%Ð%Øðñ ô €Lð ×Ò˜ÑÔÐÝÐ.°
ÐKÐK¸lÐKÐKÐKr=   c                 ó€   — t          g d¢g d¢g d¢g d¢d¬¦  «        }|                     |¦  «         t          d| fi |¤ŽS )Nr©  r  rª  )r�  r�  é    r�  r  rŸ  Útiny_vit_21m_512r¡  r£  s      r<   r²  r²  f  r¯  r=   )rm  ru  )@r  Ú__all__rÝ   Ú	functoolsr   Útypingr   r   r   r   r	   r
   r   r5   Útorch.nnr0   Útorch.nn.functionalÚ
functionalrÌ   Ú	timm.datar   r   Útimm.layersr   r   r   r   r   r   r   r   Ú_builderr   Ú	_featuresr   Ú_features_fxr   Ú_manipulater   r   Ú	_registryr   r   rˆ   r    re   rV   rg   rv   r|   r�   rŸ   rð   r  r   r~  rŠ  Údefault_cfgsr˜  r   r§  r¬  r®  r²  r‚   r=   r<   ú<module>rÁ     sÙ  ððð ð ˆ+€à Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @à €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð à AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aðpð pð pð pð pð pð pð pð pð pð pð pð pð pð pð pð pð pð pð pà *Ð *Ð *Ð *Ð *Ð *Ø +Ð +Ð +Ð +Ð +Ð +Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø <Ð <Ð <Ð <Ð <Ð <Ð <Ð <ð!ð !ð !ð !ð !ˆuŒxÔ"ñ !ô !ð !ðHð ð ð ð �”ñ ô ð ð. ð  ð  ð  ð  ˆRŒYñ  ô  ð  ðFð ð ð ð �2”9ñ ô ð ð4ð ð ð ð �”	ñ ô ð ð@ð ð ð ð ˆbŒiñ ô ð ðBjð jð jð jð j�””ñ jô jð jðZhMð hMð hMð hMð hM�2”9ñ hMô hMð hMðV Ð ˜Ñ %Ô %Ð %ðI9ð I9ð I9ð I9ð I9�2”9ñ I9ô I9ð I9ðXQð Qð Qð Qð QˆbŒiñ Qô Qð Qðhð ð ðð ð ð ð  %Ð$Ø"& $Øàð#ñ #ô #ð
 +/¨$Øð+ñ +ô +ð !˜DØðñ ô ð $( 4Øàð$ñ $ô $ð
 ,0¨4Øð,ñ ,ô ,ð "˜TØðñ ô ð $( 4Øàð$ñ $ô $ð
 ,0¨4Øð,ñ ,ô ,ð "˜TØðñ ô ð ,0¨4Øà ¨H¸sð,ñ ,ô ,ð
 ,0¨4Øà ¨H¸sÈhð,ñ ,ô ,ð[2&ð 2&ñ 2ô 2€ðj
ð 
ð 
ð 
ð ð	Kð 	Kð 	Kñ „ð	Kð ð	Lð 	Lð 	Lñ „ð	Lð ð	Lð 	Lð 	Lñ „ð	Lð ð	Lð 	Lð 	Lñ „ð	Lð ð	Lð 	Lð 	Lñ „ð	Lð 	Lð 	Lr=   