§
    ·¯¸j's  ã            
       ó
  — d Z dgZddlZddlmZ ddlmZ ddlmZm	Z	m
Z
mZmZmZ ddlZddlmZ ddlmZmZ ddlmZmZmZmZmZ d	d
lmZ d	dlmZ d	dlmZm Z  d	dl!m"Z"m#Z#  G d„ dej        j$        ¦  «        Z% G d„ dej        j$        ¦  «        Z& G d„ dej        j'        ¦  «        Z( G d„ dej        j'        ¦  «        Z) G d„ dej        j'        ¦  «        Z* G d„ dej        j'        ¦  «        Z+ G d„ dej        j'        ¦  «        Z, G d„ dej        j'        ¦  «        Z- G d„ dej        j'        ¦  «        Z. G d „ d!ej        j$        ¦  «        Z/ G d"„ dej'        ¦  «        Z0d0d$„Z1 e# e1d%¬&¦  «         e1d%¬&¦  «         e1d%¬&¦  «         e1d%¬&¦  «         e1d%¬&¦  «         e1d%¬&¦  «        d'œ¦  «        Z2d1d)„Z3e"d1d*„¦   «         Z4e"d1d+„¦   «         Z5e"d1d,„¦   «         Z6e"d1d-„¦   «         Z7e"d1d.„¦   «         Z8e"d1d/„¦   «         Z9dS )2zò EfficientViT (by MSRA)

Paper: `EfficientViT: Memory Efficient Vision Transformer with Cascaded Group Attention`
    - https://arxiv.org/abs/2305.07027

Adapted from official impl at https://github.com/microsoft/Cream/tree/main/EfficientViT
ÚEfficientVitMsraé    N)ÚOrderedDict)Úpartial)ÚDictÚListÚOptionalÚTupleÚTypeÚUnion©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)ÚSqueezeExciteÚSelectAdaptivePool2dÚtrunc_normal_Ú_assertÚget_device_dtypeé   )Úbuild_model_with_cfg)Úfeature_take_indices)Ú
checkpointÚcheckpoint_seq)Úregister_modelÚgenerate_default_cfgsc                   ó‚   ‡ — e Zd Z	 	 	 	 	 	 	 	 ddededededed	ed
edefˆ fd„Z ej        ¦   «         d„ ¦   «         Zˆ xZ	S )ÚConvNormr   r   NÚin_chsÚout_chsÚksÚstrideÚpadÚdilationÚgroupsÚbn_weight_initc           	      ó  •— |	|
dœ}t          ¦   «                              ¦   «          t          j        |||||||fddi|¤Ž| _        t          j        |fi |¤Ž| _        t          j        j         	                    | j        j
        |¦  «         d S )N©ÚdeviceÚdtypeÚbiasF)ÚsuperÚ__init__ÚnnÚConv2dÚconvÚBatchNorm2dÚbnÚtorchÚinitÚ	constant_Úweight)Úselfr   r   r   r    r!   r"   r#   r$   r'   r(   ÚddÚ	__class__s               €úm/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/models/efficientvit_msra.pyr+   zConvNorm.__init__   s‘   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ”I˜f g¨r°6¸3ÀÈ&ÐcÐcÐW\ÐcÐ`bÐcÐcˆŒ	Ý”. Ð/Ð/¨BÐ/Ð/ˆŒÝŒŒ×Ò ¤¤°Ñ?Ô?Ð?Ð?Ð?ó    c           	      ón  — | j         | j        }}|j        |j        |j        z   dz  z  }|j        |d d …d d d f         z  }|j        |j        |j        z  |j        |j        z   dz  z  z
  }t          j         	                    | 
                    d¦  «        | j         j        z  | 
                    d¦  «        |j        dd …         | j         j        | j         j        | j         j        | j         j        ¬¦  «        }|j        j                             |¦  «         |j        j                             |¦  «         |S )Nç      à?r   r   é   )r    Úpaddingr"   r#   )r.   r0   r4   Úrunning_varÚepsr)   Úrunning_meanr1   r,   r-   Úsizer#   Úshaper    r=   r"   ÚdataÚcopy_)r5   Úcr0   ÚwÚbÚms         r8   ÚfusezConvNorm.fuse.   s  € à”	˜4œ7ˆ2ˆØŒI˜œ¨"¬&Ñ0°3Ñ6Ñ6ˆØŒH�q˜˜˜˜D $¨Ð,Ô-Ñ-ˆØŒG�b”o¨¬	Ñ1ØŒ^˜bœfÑ$ sÑ*ñ+ñ +ˆåŒH�OŠOØ�FŠF�1‰IŒI˜œ	Ô(Ñ(¨!¯&ª&°©)¬)°Q´W¸Q¸R¸R´[Ø”9Ô#¨T¬YÔ->ÈÌÔI[ÐdhÔdmÔdtð ñ vô vˆð 	
ŒŒ×Ò˜AÑÔÐØ	ŒŒ×Ò˜!ÑÔÐØˆr9   )r   r   r   r   r   r   NN)
Ú__name__Ú
__module__Ú__qualname__ÚintÚfloatr+   r1   Úno_gradrI   Ú__classcell__©r7   s   @r8   r   r      sÜ   ø€ € € € € ð
 ØØØØØ$%ØØð@ð @àð@ð ð@ð ð	@ð
 ð@ð ð@ð ð@ð ð@ð "ð@ð @ð @ð @ð @ð @ð& €U„]�_„_ðð ñ „_ðð ð ð ð r9   r   c                   óp   ‡ — e Zd Z	 	 	 	 	 ddedededed	ef
ˆ fd
„Z ej        ¦   «         d„ ¦   «         Z	ˆ xZ
S )Ú
NormLinearTç{®Gáz”?ç        NÚin_featuresÚout_featuresr)   ÚstdÚdropc                 óŒ  •— ||dœ}t          ¦   «                              ¦   «          t          j        |fi |¤Ž| _        t          j        |¦  «        | _        t          j        ||fd|i|¤Ž| _        t          | j        j
        |¬¦  «         | j        j        �,t          j                             | j        j        d¦  «         d S d S )Nr&   r)   )rX   r   )r*   r+   r,   ÚBatchNorm1dr0   ÚDropoutrY   ÚLinearÚlinearr   r4   r)   r2   r3   )
r5   rV   rW   r)   rX   rY   r'   r(   r6   r7   s
            €r8   r+   zNormLinear.__init__>   sÀ   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ”. Ð3Ð3°Ð3Ð3ˆŒÝ”J˜tÑ$Ô$ˆŒ	Ý”i ¨\ÐKÐKÀÐKÈÐKÐKˆŒå�d”kÔ(¨cÐ2Ñ2Ô2Ð2ØŒ;ÔÐ'ÝŒG×Ò˜dœkÔ.°Ñ2Ô2Ð2Ð2Ð2ð (Ð'r9   c                 óŽ  — | j         | j        }}|j        |j        |j        z   dz  z  }|j        | j         j        | j         j        z  |j        |j        z   dz  z  z
  }|j        |d d d …f         z  }|j        €|| j        j        j        z  }n4|j        |d d …d f         z                       d¦  «        | j        j        z   }t          j
                             |                     d¦  «        |                     d¦  «        ¦  «        }|j        j                             |¦  «         |j        j                             |¦  «         |S )Nr;   éÿÿÿÿr   r   )r0   r^   r4   r>   r?   r)   r@   ÚTÚviewr1   r,   r]   rA   rC   rD   )r5   r0   r^   rF   rG   rH   s         r8   rI   zNormLinear.fuseR   s   € à”W˜dœkˆFˆØŒI˜œ¨"¬&Ñ0°3Ñ6Ñ6ˆØŒG�d”gÔ*ØŒGŒNñØ œn¨r¬vÑ5¸Ñ;ñ<ñ <ˆàŒM˜A˜d A A A˜gœJÑ&ˆØŒ;ÐØ�D”KÔ&Ô(Ñ(ˆAˆAà”  1 1 1 d 7¤Ñ+×1Ò1°"Ñ5Ô5¸¼Ô8HÑHˆAÝŒH�OŠO˜AŸFšF 1™IœI q§v¢v¨a¡y¤yÑ1Ô1ˆØ	ŒŒ×Ò˜AÑÔÐØ	ŒŒ×Ò˜!ÑÔÐØˆr9   )TrT   rU   NN)rJ   rK   rL   rM   ÚboolrN   r+   r1   rO   rI   rP   rQ   s   @r8   rS   rS   =   s¦   ø€ € € € € ð
 ØØØØð3ð 3àð3ð ð3ð ð	3ð
 ð3ð ð3ð 3ð 3ð 3ð 3ð 3ð( €U„]�_„_ðð ñ „_ðð ð ð ð r9   rS   c                   ó4   ‡ — e Zd Z	 	 ddedefˆ fd„Zd„ Zˆ xZS )ÚPatchMergingNÚdimÚout_dimc                 ój  •— ||dœ}t          ¦   «                              ¦   «          t          |dz  ¦  «        }t          ||dddfi |¤Ž| _        t
          j                             ¦   «         | _        t          ||dddfd|i|¤Ž| _	        t          |dfi |¤Ž| _        t          ||dddfi |¤Ž| _        d S )	Nr&   é   r   r   é   r<   r#   g      Ð?)r*   r+   rM   r   Úconv1r1   r,   ÚReLUÚactÚconv2r   ÚseÚconv3)r5   rf   rg   r'   r(   r6   Úhid_dimr7   s          €r8   r+   zPatchMerging.__init__d   sÇ   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ�c˜A‘g‘,”,ˆÝ˜c 7¨A¨q°!Ð:Ð:°rÐ:Ð:ˆŒ
Ý”8—=’=‘?”?ˆŒÝ˜g w°°1°aÐNÐNÀÐNÈ2ÐNÐNˆŒ
Ý ¨Ð3Ð3°Ð3Ð3ˆŒÝ˜g w°°1°aÐ>Ð>¸2Ð>Ð>ˆŒ
ˆ
ˆ
r9   c                 óî   — |                       |                      |                      |                      |                      |                      |¦  «        ¦  «        ¦  «        ¦  «        ¦  «        ¦  «        }|S ©N)rp   ro   rm   rn   rk   ©r5   Úxs     r8   ÚforwardzPatchMerging.forwardt   sR   € Ø�JŠJ�t—w’w˜tŸxšx¨¯
ª
°4·8²8¸D¿JºJÀq¹M¼MÑ3JÔ3JÑ(KÔ(KÑLÔLÑMÔMÑNÔNˆØˆr9   ©NN©rJ   rK   rL   rM   r+   rv   rP   rQ   s   @r8   re   re   c   sk   ø€ € € € € ð
 Øð?ð ?àð?ð ð?ð ?ð ?ð ?ð ?ð ?ð ð ð ð ð ð ð r9   re   c                   ó:   ‡ — e Zd Zddej        defˆ fd„Zd„ Zˆ xZS )ÚResidualDroprU   rH   rY   c                 ód   •— t          ¦   «                              ¦   «          || _        || _        d S rs   )r*   r+   rH   rY   )r5   rH   rY   r7   s      €r8   r+   zResidualDrop.__init__z   s+   ø€ Ý‰Œ×ÒÑÔÐØˆŒØˆŒ	ˆ	ˆ	r9   c           	      ót  — | j         rš| j        dk    r�||                      |¦  «        t          j        |                     d¦  «        ddd|j        ¬¦  «                             | j        ¦  «                             d| j        z
  ¦  «         	                    ¦   «         z  z   S ||                      |¦  «        z   S )Nr   r   )r'   )
ÚtrainingrY   rH   r1   ÚrandrA   r'   Úge_ÚdivÚdetachrt   s     r8   rv   zResidualDrop.forward   s¢   € ØŒ=ð 	!˜TœY¨š]˜]Ø�t—v’v˜a‘y”y¥5¤:Ø—’�q‘	”	˜1˜a ¨1¬8ð$5ñ $5ô $5ß58²S¸¼±^´^ÇCÂCÈÈDÌIÉÑDVÔDV×W]ÒW]ÑW_ÔW_ñ`ñ `ð `ð �t—v’v˜a‘y”y‘=Ð r9   )rU   )	rJ   rK   rL   r,   ÚModulerN   r+   rv   rP   rQ   s   @r8   rz   rz   y   sa   ø€ € € € € ðð ˜"œ)ð ¨5ð ð ð ð ð ð ð
!ð !ð !ð !ð !ð !ð !r9   rz   c                   ó4   ‡ — e Zd Z	 	 ddedefˆ fd„Zd„ Zˆ xZS )ÚConvMlpNÚedÚhc                 óè   •— ||dœ}t          ¦   «                              ¦   «          t          ||fi |¤Ž| _        t          j                             ¦   «         | _        t          ||fddi|¤Ž| _        d S )Nr&   r$   r   )	r*   r+   r   Úpw1r1   r,   rl   rm   Úpw2)r5   r…   r†   r'   r(   r6   r7   s         €r8   r+   zConvMlp.__init__ˆ   st   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ˜B Ð(Ð( RÐ(Ð(ˆŒÝ”8—=’=‘?”?ˆŒÝ˜A˜rÐ:Ð:°!Ð:°rÐ:Ð:ˆŒˆˆr9   c                 ó|   — |                       |                      |                      |¦  «        ¦  «        ¦  «        }|S rs   )r‰   rm   rˆ   rt   s     r8   rv   zConvMlp.forward•   s.   € Ø�HŠH�T—X’X˜dŸhšh q™kœkÑ*Ô*Ñ+Ô+ˆØˆr9   rw   rx   rQ   s   @r8   r„   r„   ‡   sk   ø€ € € € € ð
 Øð;ð ;àð;ð ð;ð ;ð ;ð ;ð ;ð ;ðð ð ð ð ð ð r9   r„   c                   óö   ‡ — e Zd ZU eeej        f         ed<   	 	 	 	 	 	 	 ddeded	ed
edede	edf         fˆ fd„Z
dd„Zdd„Zdd„Z ej        ¦   «         dˆ fd„	¦   «         Zdej        dej        fd„Zd„ Zˆ xZS )ÚCascadedGroupAttentionÚattention_bias_cacheé   ri   é   ©é   r‘   r‘   r‘   Nrf   Úkey_dimÚ	num_headsÚ
attn_ratioÚ
resolutionÚkernels.c	                 óL  •— ||dœ}	t          ¦   «                              ¦   «          || _        |dz  | _        || _        t          ||z  ¦  «        | _        || _        g }
g }t          |¦  «        D ]z}|
 	                    t          ||z  | j        dz  | j        z   fi |	¤Ž¦  «         | 	                    t          | j        | j        ||         d||         dz  fd| j        i|	¤Ž¦  «         Œ{t          j                             |
¦  «        | _        t          j                             |¦  «        | _        t          j                             t          j                             ¦   «         t          | j        |z  |fddi|	¤Ž¦  «        | _        || _        ||z  }||z  }t          j                             t          j        ||fi |	¤Ž¦  «        | _        |                      dt          j        ||f|t          j        ¬¦  «        d	¬
¦  «         i | _        |                      ¦   «          d S )Nr&   g      à¿r<   r   r#   r$   r   Úattention_bias_idxsF)Ú
persistent)r*   r+   r“   Úscaler’   rM   Úval_dimr”   ÚrangeÚappendr   r1   r,   Ú
ModuleListÚqkvsÚdwsÚ
Sequentialrl   Úprojr•   Ú	ParameterÚemptyÚattention_biasesÚregister_bufferÚlongr�   Úreset_parameters)r5   rf   r’   r“   r”   r•   r–   r'   r(   r6   rŸ   r    ÚiÚNÚnum_offsetsr7   s                  €r8   r+   zCascadedGroupAttention.__init__§   s   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ"ˆŒØ ‘_ˆŒ
ØˆŒÝ˜:¨Ñ/Ñ0Ô0ˆŒØ$ˆŒàˆØˆÝ�yÑ!Ô!ð 	xð 	xˆAØ�KŠK� ¨	Ñ!1°4´<À!Ñ3CÀdÄlÑ3RÐYÐYÐVXÐYÐYÑZÔZÐZØ�JŠJ•x ¤¨d¬l¸GÀA¼JÈÈ7ÐSTÌ:ÐYZÉ?ÐvÐvÐcgÔcoÐvÐsuÐvÐvÑwÔwÐwÐwÝ”H×'Ò'¨Ñ-Ô-ˆŒ	Ý”8×&Ò& sÑ+Ô+ˆŒÝ”H×'Ò'ÝŒH�MŠM‰OŒOÝ�T”\ IÑ-¨sÐKÐKÀ1ÐKÈÐKÐKñ
ô 
ˆŒ	ð
 %ˆŒØ˜Ñ#ˆà  :Ñ-ˆÝ %¤× 2Ò 2µ5´;¸yÈ+Ð3\Ð3\ÐY[Ð3\Ð3\Ñ ]Ô ]ˆÔØ×ÒØ!ÝŒK˜˜A˜ vµU´ZÐ@Ñ@Ô@Øð 	ñ 	
ô 	
ð 	
ð
 %'ˆÔ!ð 	×ÒÑÔÐÐÐr9   Úreturnc                 ó€   — t           j        j                             | j        ¦  «         |                      ¦   «          dS )z"Initialize parameters and buffers.N)r1   r,   r2   Úzeros_r¥   Ú_init_buffers©r5   s    r8   r¨   z'CascadedGroupAttention.reset_parametersÕ   s5   € åŒŒ×Ò˜TÔ2Ñ3Ô3Ð3Ø×ÒÑÔÐÐÐr9   c                 óT  — t          t          j        t          | j        ¦  «        t          | j        ¦  «        ¦  «        ¦  «        }i }g }|D ]t}|D ]o}t          |d         |d         z
  ¦  «        t          |d         |d         z
  ¦  «        f}||vrt          |¦  «        ||<   |                     ||         ¦  «         ŒpŒu| j         	                    t          j        |t          j        ¬¦  «                             t          |¦  «        t          |¦  «        ¦  «        ¦  «         dS )z.Compute and fill non-persistent buffer values.r   r   )r(   N)ÚlistÚ	itertoolsÚproductrœ   r•   ÚabsÚlenr�   r˜   rD   r1   Útensorr§   rb   )r5   ÚpointsÚattention_offsetsÚidxsÚp1Úp2Úoffsets          r8   r¯   z$CascadedGroupAttention._init_buffersÚ   s  € å•iÔ'­¨d¬oÑ(>Ô(>ÅÀdÄoÑ@VÔ@VÑWÔWÑXÔXˆØÐØˆØð 	7ð 	7ˆBØð 7ð 7�Ý˜b œe b¨¤e™mÑ,Ô,­c°"°Q´%¸"¸Q¼%±-Ñ.@Ô.@ÐA�ØÐ!2Ð2Ð2Ý03Ð4EÑ0FÔ0FÐ% fÑ-Ø—’Ð-¨fÔ5Ñ6Ô6Ð6Ð6ð	7ð
 	Ô ×&Ò&¥u¤|°DÅÄ
Ð'KÑ'KÔ'K×'PÒ'PÕQTÐU[ÑQ\ÔQ\Õ^aÐbhÑ^iÔ^iÑ'jÔ'jÑkÔkÐkÐkÐkr9   c                 ó.   — |                       ¦   «          dS )z"Initialize non-persistent buffers.N)r¯   r°   s    r8   Úinit_non_persistent_buffersz2CascadedGroupAttention.init_non_persistent_buffersç   s   € à×ÒÑÔÐÐÐr9   Tc                 ój   •— t          ¦   «                              |¦  «         | j        r	i | _        d S d S rs   )r*   Útrainr�   )r5   Úmoder7   s     €r8   rÁ   zCascadedGroupAttention.trainë   s<   ø€ å‰Œ�Š�dÑÔÐØÔ$ð 	+Ø(*ˆDÔ%Ð%Ð%ð	+ð 	+r9   r'   c                 óþ   — t           j                             ¦   «         s| j        r| j        d d …| j        f         S t          |¦  «        }|| j        vr| j        d d …| j        f         | j        |<   | j        |         S rs   )r1   ÚjitÚ
is_tracingr}   r¥   r˜   Ústrr�   )r5   r'   Ú
device_keys      r8   Úget_attention_biasesz+CascadedGroupAttention.get_attention_biasesñ   s„   € ÝŒ9×ÒÑ!Ô!ð 	9 T¤]ð 	9ØÔ(¨¨¨¨DÔ,DÐ)DÔEÐEå˜V™œˆJØ Ô!:Ð:Ð:Ø8<Ô8MÈaÈaÈaÐQUÔQiÐNiÔ8j�Ô)¨*Ñ5ØÔ,¨ZÔ8Ð8r9   c                 óÜ  — |j         \  }}}}|                     t          | j        ¦  «        d¬¦  «        }g }|d         }|                      |j        ¦  «        }	t          t          | j        | j        ¦  «        ¦  «        D �]A\  }
\  }}|
dk    r|||
         z   } ||¦  «        }| 	                    |d||¦  «         
                    | j        | j        | j        gd¬¦  «        \  }}} ||¦  «        }|                     d¦  «        |                     d¦  «        |                     d¦  «        }}}|| j        z  }|                     dd¦  «        |z  }||	|
         z   }|                     d¬¦  «        }||                     dd¦  «        z  }| 	                    || j        ||¦  «        }|                     |¦  «         �ŒC|                      t'          j        |d¦  «        ¦  «        }|S )Nr   )rf   r   r`   r<   éþÿÿÿ)rB   Úchunkr¶   rŸ   rÈ   r'   Ú	enumerateÚzipr    rb   Úsplitr’   r›   Úflattenrš   Ú	transposeÚsoftmaxr�   r¢   r1   Úcat)r5   ru   ÚBÚCÚHÚWÚfeats_inÚ	feats_outÚfeatÚ	attn_biasÚhead_idxÚqkvr    ÚqÚkÚvÚattns                    r8   rv   zCascadedGroupAttention.forwardú   sË  € Ø”W‰
ˆˆ1ˆa�Ø—7’7�3˜tœy™>œ>¨q�7Ñ1Ô1ˆØˆ	Ø˜Œ{ˆØ×-Ò-¨a¬hÑ7Ô7ˆ	Ý$-­c°$´)¸T¼XÑ.FÔ.FÑ$GÔ$Gð 	#ñ 	#Ñ ˆH‘j�s˜CØ˜!Š|ˆ|Ø˜h xÔ0Ñ0�Ø�3�t‘9”9ˆDØ—i’i  2 q¨!Ñ,Ô,×2Ò2°D´LÀ$Ä,ÐPTÔP\Ð3]ÐcdÐ2ÑeÔe‰GˆAˆq�!Ø��A‘”ˆAØ—i’i ‘l”l A§I¢I¨a¡L¤L°!·)²)¸A±,´,�!ˆqˆAØ�D”J‘ˆAØ—;’;˜r 2Ñ&Ô&¨Ñ*ˆDØ˜) HÔ-Ñ-ˆDØ—<’< B�<Ñ'Ô'ˆDØ�t—~’~ b¨"Ñ-Ô-Ñ-ˆDØ—9’9˜Q ¤¨a°Ñ3Ô3ˆDØ×Ò˜TÑ"Ô"Ð"Ñ"Ø�IŠI•e”i 	¨1Ñ-Ô-Ñ.Ô.ˆØˆr9   )rŽ   ri   r�   r�   NN)r¬   N©T)rJ   rK   rL   r   rÆ   r1   ÚTensorÚ__annotations__rM   r	   r+   r¨   r¯   r¿   rO   rÁ   r'   rÈ   rv   rP   rQ   s   @r8   rŒ   rŒ   š   s_  ø€ € € € € € Ø˜s E¤LÐ0Ô1Ð1Ð1Ñ1ð	ð ØØ Ø'3ØØð, ð , àð, ð ð, ð ð	, ð
 ð, ð ð, ð ˜3 ˜8”_ð, ð , ð , ð , ð , ð , ð\ð ð ð ð
lð lð lð lðð ð ð ð €U„]�_„_ð+ð +ð +ð +ð +ñ „_ð+ð
9¨5¬<ð 9¸E¼Lð 9ð 9ð 9ð 9ðð ð ð ð ð ð r9   rŒ   c                   óf   ‡ — e Zd ZdZ	 	 	 	 	 	 	 dded	ed
ededededeedf         fˆ fd„Zd„ Zˆ xZS )ÚLocalWindowAttentiona¹   Local Window Attention.

    Args:
        dim (int): Number of input channels.
        key_dim (int): The dimension for query and key.
        num_heads (int): Number of attention heads.
        attn_ratio (int): Multiplier for the query dim for value dimension.
        resolution (int): Input resolution.
        window_resolution (int): Local window resolution.
        kernels (List[int]): The kernel size of the dw conv on query.
    rŽ   ri   r�   é   r�   Nrf   r’   r“   r”   r•   Úwindow_resolutionr–   .c
                 óú   •— ||	dœ}
t          ¦   «                              ¦   «          || _        || _        || _        |dk    s
J d¦   «         ‚|| _        t          ||¦  «        }t          |||f|||dœ|
¤Ž| _        d S )Nr&   r   z"window_size must be greater than 0)r”   r•   r–   )	r*   r+   rf   r“   r•   rç   ÚminrŒ   rà   ©r5   rf   r’   r“   r”   r•   rç   r–   r'   r(   r6   r7   s              €r8   r+   zLocalWindowAttention.__init__  s©   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØˆŒØ"ˆŒØ$ˆŒØ  1Ò$Ð$Ð$Ð&JÑ$Ô$Ð$Ø!2ˆÔÝÐ 1°:Ñ>Ô>ÐÝ*Ø�˜)ð
à!Ø(Øð	
ð 
ð
 ð
ð 
ˆŒ	ˆ	ˆ	r9   c           	      ó„  — | j         x}}|j        \  }}}}t          ||k    d||f› d||f› �¦  «         t          ||k    d||f› d||f› �¦  «         || j        k    r"|| j        k    r|                      |¦  «        }�nÁ|                     dddd¦  «        }| j        || j        z  z
  | j        z  }| j        || j        z  z
  | j        z  }	t          j        j         	                    |ddd|	d|f¦  «        }||z   ||	z   }}
|
| j        z  }|| j        z  }| 
                    ||| j        || j        |¦  «                             dd¦  «        }|                     ||z  |z  | j        | j        |¦  «                             dddd¦  «        }|                      |¦  «        }|                     dddd¦  «         
                    |||| j        | j        |¦  «        }|                     dd¦  «                             ||
||¦  «        }|d d …d |…d |…f                              ¦   «         }|                     dddd¦  «        }|S )Nz%input feature has wrong size, expect z, got r   r<   rj   r   )r•   rB   r   rç   rà   Úpermuter1   r,   Ú
functionalr!   rb   rÐ   ÚreshapeÚ
contiguous)r5   ru   rÕ   rÖ   rÓ   rÔ   ÚH_ÚW_Úpad_bÚpad_rÚpHÚpWÚnHÚnWs                 r8   rv   zLocalWindowAttention.forward:  sk  € Ø”ÐˆˆAØ”w‰ˆˆ1ˆb�"å��R’ÐYÀ!ÀQÀÐYÐYÐPRÐTVÈxÐYÐYÑZÔZÐZÝ��R’ÐYÀ!ÀQÀÐYÐYÐPRÐTVÈxÐYÐYÑZÔZÐZØ�Ô&Ò&Ð&¨1°Ô0FÒ+FÐ+FØ—	’	˜!‘”ˆA‰Aà—	’	˜!˜Q  1Ñ%Ô%ˆAØÔ+¨a°$Ô2HÑ.HÑHÈDÔLbÑbˆEØÔ+¨a°$Ô2HÑ.HÑHÈDÔLbÑbˆEÝ”Ô#×'Ò'¨¨A¨q°!°U¸A¸uÐ+EÑFÔFˆAà˜‘Y  E¡	�ˆBØ�tÔ-Ñ-ˆBØ�tÔ-Ñ-ˆBà—’�q˜"˜dÔ4°b¸$Ô:PÐRSÑTÔT×^Ò^Ð_`ÐbcÑdÔdˆAØ—	’	˜!˜b™& 2™+ tÔ'=¸tÔ?UÐWXÑYÔY×aÒaÐbcÐefÐhiÐklÑmÔmˆAØ—	’	˜!‘”ˆAà—	’	˜!˜Q  1Ñ%Ô%×*Ò*¨1¨b°"°dÔ6LÈdÔNdÐfgÑhÔhˆAØ—’˜A˜qÑ!Ô!×)Ò)¨!¨R°°QÑ7Ô7ˆAØ�!�!�!�R�a�R˜˜!˜�)”×'Ò'Ñ)Ô)ˆAØ—	’	˜!˜Q  1Ñ%Ô%ˆAØˆr9   )rŽ   ri   r�   ræ   r�   NN)	rJ   rK   rL   Ú__doc__rM   r	   r+   rv   rP   rQ   s   @r8   rå   rå     s¾   ø€ € € € € ð
ð 
ð ØØ Ø%&Ø'3ØØð
ð 
àð
ð ð
ð ð	
ð
 ð
ð ð
ð  #ð
ð ˜3 ˜8”_ð
ð 
ð 
ð 
ð 
ð 
ð8ð ð ð ð ð ð r9   rå   c                   óf   ‡ — e Zd ZdZddddg d¢ddfded	ed
ededededee         fˆ fd„Zd„ Zˆ xZS )ÚEfficientVitBlockaÕ   A basic EfficientVit building block.

    Args:
        dim (int): Number of input channels.
        key_dim (int): Dimension for query and key in the token mixer.
        num_heads (int): Number of attention heads.
        attn_ratio (int): Multiplier for the query dim for value dimension.
        resolution (int): Input resolution.
        window_resolution (int): Local window resolution.
        kernels (List[int]): The kernel size of the dw conv on query.
    rŽ   ri   r�   ræ   r�   Nrf   r’   r“   r”   r•   rç   r–   c
           
      óö  •— ||	dœ}
t          ¦   «                              ¦   «          t          t          ||dddf|ddœ|
¤Ž¦  «        | _        t          t          |t          |dz  ¦  «        fi |
¤Ž¦  «        | _        t          t          |||f||||dœ|
¤Ž¦  «        | _	        t          t          ||dddf|ddœ|
¤Ž¦  «        | _
        t          t          |t          |dz  ¦  «        fi |
¤Ž¦  «        | _        d S )Nr&   rj   r   rU   )r#   r$   r<   )r”   r•   rç   r–   )r*   r+   rz   r   Údw0r„   rM   Úffn0rå   ÚmixerÚdw1Úffn1rê   s              €r8   r+   zEfficientVitBlock.__init__c  s)  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐå¥¨¨c°1°a¸Ð `À3ÐWYÐ `Ð `Ð]_Ð `Ð `ÑaÔaˆŒÝ ¥¨­c°#¸±'©l¬lÐ!AÐ!A¸bÐ!AÐ!AÑBÔBˆŒ	å!Ý Ø�W˜iðà%Ø%Ø"3Øðð ð ðð ñ	
ô 	
ˆŒ
õ  ¥¨¨c°1°a¸Ð `À3ÐWYÐ `Ð `Ð]_Ð `Ð `ÑaÔaˆŒÝ ¥¨­c°#¸±'©l¬lÐ!AÐ!A¸bÐ!AÐ!AÑBÔBˆŒ	ˆ	ˆ	r9   c                 óÄ   — |                       |                      |                      |                      |                      |¦  «        ¦  «        ¦  «        ¦  «        ¦  «        S rs   )r   rÿ   rþ   rý   rü   rt   s     r8   rv   zEfficientVitBlock.forwardƒ  sB   € Ø�yŠy˜Ÿš $§*¢*¨T¯YªY°t·x²xÀ±{´{Ñ-CÔ-CÑ"DÔ"DÑEÔEÑFÔFÐFr9   )	rJ   rK   rL   rø   rM   r   r+   rv   rP   rQ   s   @r8   rú   rú   W  sÔ   ø€ € € € € ð
ð 
ð ØØ Ø%&Ø!-  ØØðCð CàðCð ðCð ð	Cð
 ðCð ðCð  #ðCð ˜#”YðCð Cð Cð Cð Cð Cð@Gð Gð Gð Gð Gð Gð Gr9   rú   c                   ó‚   ‡ — e Zd Zdddddg d¢dddf	d	ed
ededeeef         dededededee         defˆ fd„Zd„ Zˆ xZ	S )ÚEfficientVitStage©Ú r   rŽ   ri   r�   ræ   r�   r   NÚin_dimrg   r’   Ú
downsampler“   r”   r•   rç   r–   Údepthc                 ó  •— ||dœ}t          ¦   «                              ¦   «          |d         dk    �rZ|dz
  |d         z  dz   | _        g }|                     dt          j                             t          t          ||dddfd|i|¤Ž¦  «        t          t          |t          |dz  ¦  «        fi |¤Ž¦  «        ¦  «        f¦  «         |                     d	t          ||fi |¤Žf¦  «         |                     d
t          j                             t          t          ||dddfd|i|¤Ž¦  «        t          t          |t          |dz  ¦  «        fi |¤Ž¦  «        ¦  «        f¦  «         t          j        t          |¦  «        ¦  «        | _        n'||k    sJ ‚t          j        ¦   «         | _        || _        g }t          |
¦  «        D ]-}|                     t!          ||||| j        ||	fi |¤Ž¦  «         Œ.t          j        |Ž | _        d S )Nr&   r   Ú	subsampler   Úres1rj   r#   r<   Ú
patchmergeÚres2)r*   r+   r•   r�   r1   r,   r¡   rz   r   r„   rM   re   r   r  ÚIdentityrœ   rú   Úblocks)r5   r  rg   r’   r  r“   r”   r•   rç   r–   r  r'   r(   r6   Údown_blocksr  Údr7   s                    €r8   r+   zEfficientVitStage.__init__ˆ  sL  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ�aŒ=˜KÒ'Ñ'Ø)¨A™~°*¸Q´-Ñ?À!ÑCˆDŒOØˆKØ×ÒØÝ”×#Ò#Ý ¥¨&°&¸!¸QÀÐ!WÐ!WÈ&Ð!WÐTVÐ!WÐ!WÑXÔXÝ ¥¨µ°V¸a±Z±´Ð!GÐ!GÀBÐ!GÐ!GÑHÔHñô ð ñ ô ð ð ×Ò ­l¸6À7Ð.QÐ.QÈbÐ.QÐ.QÐRÑSÔSÐSØ×ÒØÝ”×#Ò#Ý ¥¨'°7¸A¸qÀ!Ð!ZÐ!ZÈGÐ!ZÐWYÐ!ZÐ!ZÑ[Ô[Ý ¥¨µ#°gÀ±kÑ2BÔ2BÐ!IÐ!IÀbÐ!IÐ!IÑJÔJñô ð ñ ô ð õ !œm­K¸Ñ,DÔ,DÑEÔEˆDŒOˆOà˜WÒ$Ð$Ð$Ð$Ý œk™mœmˆDŒOØ(ˆDŒOàˆÝ�u‘”ð 
	ð 
	ˆAØ�MŠMÕ+ØØØØØ”Ø!Øð	ð 	ð ð	ð 	ñ 	ô 	ð 	ð 	õ ”m VÐ,ˆŒˆˆr9   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S rs   )r  r  rt   s     r8   rv   zEfficientVitStage.forward¿  s'   € Ø�OŠO˜AÑÔˆØ�KŠK˜‰NŒNˆØˆr9   )
rJ   rK   rL   rM   r	   rÆ   r   r+   rv   rP   rQ   s   @r8   r  r  ‡  sá   ø€ € € € € ð +2ØØØ Ø%&Ø!-  ØØØð5-ð 5-àð5-ð ð5-ð ð	5-ð
 ˜c 3˜hœð5-ð ð5-ð ð5-ð ð5-ð  #ð5-ð ˜#”Yð5-ð ð5-ð 5-ð 5-ð 5-ð 5-ð 5-ðnð ð ð ð ð ð r9   r  c                   ó.   ‡ — e Zd Z	 	 ddedefˆ fd„Zˆ xZS )ÚPatchEmbeddingNÚin_chansrf   c           
      óØ  •— t          ¦   «                              ¦   «          ||dœ}|                      dt          ||dz  dddfi |¤Ž¦  «         |                      dt          j                             ¦   «         ¦  «         |                      dt          |dz  |d	z  dddfi |¤Ž¦  «         |                      d
t          j                             ¦   «         ¦  «         |                      dt          |d	z  |dz  dddfi |¤Ž¦  «         |                      dt          j                             ¦   «         ¦  «         |                      dt          |dz  |dddfi |¤Ž¦  «         d| _        d S )Nr&   rk   rŽ   rj   r<   r   Úrelu1rn   ri   Úrelu2rp   Úrelu3Úconv4é   )r*   r+   Ú
add_moduler   r1   r,   rl   Ú
patch_size)r5   r  rf   r'   r(   r6   r7   s         €r8   r+   zPatchEmbedding.__init__Æ  sU  ø€ õ 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆØ�Š˜¥¨(°C¸1±H¸aÀÀAÐ!LÐ!LÈÐ!LÐ!LÑMÔMÐMØ�Š˜¥¤§¢¡¤Ñ1Ô1Ð1Ø�Š˜¥¨#°©(°C¸1±H¸aÀÀAÐ!LÐ!LÈÐ!LÐ!LÑMÔMÐMØ�Š˜¥¤§¢¡¤Ñ1Ô1Ð1Ø�Š˜¥¨#°©(°C¸1±H¸aÀÀAÐ!LÐ!LÈÐ!LÐ!LÑMÔMÐMØ�Š˜¥¤§¢¡¤Ñ1Ô1Ð1Ø�Š˜¥¨#°©(°C¸¸A¸qÐ!GÐ!GÀBÐ!GÐ!GÑHÔHÐHØˆŒˆˆr9   rw   )rJ   rK   rL   rM   r+   rP   rQ   s   @r8   r  r  Å  s\   ø€ € € € € ð
 Øðð àðð ðð ð ð ð ð ð ð ð ð r9   r  c                   ó   ‡ — e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 d8dedededeedf         deedf         deedf         deedf         deedf         deedf         deeeef         df         dedefˆ fd„Zd9defd„Z	d9de
j        ded dfd!„Zej        j        d"„ ¦   «         Zej        j        d:d$„¦   «         Zej        j        d9d%„¦   «         Zej        j        d e
j        fd&„¦   «         Zd;dedee         fd'„Z	 	 	 	 	 d<d)ej        d*eeeee         f                  d+ed,ed-ed.ed eeej                 eej        eej                 f         f         fd/„Z	 	 	 d=d*eeee         f         d1ed2efd3„Zd4„ Zd:d5efd6„Zd7„ Zˆ xZS )>r   éà   rj   éè  ©é@   é€   éÀ   ©r  r  r  ©r   r<   rj   ©ri   ri   ri   ©ræ   ræ   ræ   r�   ©r  ©r
  r<   r*  ÚavgrU   NÚimg_sizer  Únum_classesÚ	embed_dim.r’   r  r“   Úwindow_sizer–   Údown_opsÚglobal_poolÚ	drop_ratec                 ó4  •‡‡‡— t          ¦   «                              ¦   «          ||dœ}d| _        || _        || _        || _        t          |‰d         fi |¤Ž| _        | j        j        }|| j        j        z  }ˆˆˆfd„t          t          ‰¦  «        ¦  «        D ¦   «         }g | _        g }‰d         }t          t          ‰‰|‰|||
¦  «        ¦  «        D ]ƒ\  }\  }}}}}}}t          d|||||||||	|dœ
|¤Ž}|}|d         dk    r|dk    r||d         z  }|j        }|                     |¦  «         | xj        t#          ||d|› �¬	¦  «        gz  c_        Œ„t%          j        |Ž | _        |d
k    rt+          |d¬¦  «        | _        n |dk    sJ ‚t%          j        ¦   «         | _        ‰d         x| _        | _        |dk    rt5          | j        |fd| j        i|¤Žnt6          j                             ¦   «         | _        |                      d¬¦  «         d S )Nr&   Fr   c                 óD   •— g | ]}‰|         ‰|         ‰|         z  z  ‘ŒS © r5  )Ú.0r©   r.  r’   r“   s     €€€r8   ú
<listcomp>z-EfficientVitMsra.__init__.<locals>.<listcomp>ö  s/   ø€ Ð`Ð`Ð`ÀQ�i ”l g¨a¤j°9¸Q´<Ñ&?Ñ@Ð`Ð`Ð`r9   )
r  rg   r’   r  r“   r”   r•   rç   r–   r  r
  r   zstages.)Únum_chsÚ	reductionÚmoduler+  T©Ú	pool_typerÏ   r`   rY   ©Úneeds_resetr5  )r*   r+   Úgrad_checkpointingr-  r  r2  r  Úpatch_embedr  rœ   r¶   Úfeature_inforÌ   rÍ   r  r•   r�   Údictr,   r¡   Ústagesr   r1  r  Únum_featuresÚhead_hidden_sizerS   r1   ÚheadÚinit_weights)r5   r,  r  r-  r.  r’   r  r“   r/  r–   r0  r1  r2  r'   r(   r6   r    r•   r”   rC  Úpre_edr©   r…   ÚkdÚdpthÚnhÚarÚwdÚdoÚstager7   s       `` `                      €r8   r+   zEfficientVitMsra.__init__Ú  s¬  øøøø€ õ" 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆØ"'ˆÔØ&ˆÔØ ˆŒØ"ˆŒõ *¨(°I¸a´LÐGÐGÀBÐGÐGˆÔØÔ!Ô,ˆØ Ô!1Ô!<Ñ<ˆ
Ø`Ð`Ð`Ð`Ð`Ð`Í%ÕPSÐT]ÑP^ÔP^ÑJ_ÔJ_Ð`Ñ`Ô`ˆ
ð ˆÔØˆØ˜1”ˆÝ1:Ý�I˜w¨¨y¸*ÀkÐS[Ñ\Ô\ñ2^ô 2^ð 	\ð 	\Ñ-ˆAÑ-��B˜˜b " b¨"å%ð ØØØØØØØ%Ø"$ØØðð ð ðð ˆEð ˆFØ�!Œu˜Ò#Ð#¨¨Qª¨Ø˜"˜Qœ%‘�ØÔ)ˆJØ�MŠM˜%Ñ Ô Ð ØÐÔ¥$¨r¸VÈMÐVWÈMÈMÐ"ZÑ"ZÔ"ZÐ![Ñ[ÐÔÐÝ”m VÐ,ˆŒà˜%ÒÐÝ3¸kÐSWÐXÑXÔXˆDÔÐà !Ò#Ð#Ð#Ð#Ý!œ{™}œ}ˆDÔØ4=¸b´MÐAˆÔ˜DÔ1àJUÐXYÊ/È/õ ØÔ˜{ðGð GØ15´ðGØCEðGð Gð GÝ_dÔ_g×_pÒ_pÑ_rÔ_rð 	Œ	ð 	×Ò eÐÑ,Ô,Ð,Ð,Ð,r9   Tr>  c                 óX   — |                       t          | j        |¬¦  «        ¦  «         d S )Nr=  )Úapplyr   Ú_init_weights)r5   r>  s     r8   rG  zEfficientVitMsra.init_weights  s*   € Ø�
Š
•7˜4Ô-¸;ÐGÑGÔGÑHÔHÐHÐHÐHr9   rH   r¬   c                 óZ   — |r&t          |d¦  «        r|                     ¦   «          d S d S d S )Nr¨   )Úhasattrr¨   )r5   rH   r>  s      r8   rR  zEfficientVitMsra._init_weights"  sH   € Øð 	!�7 1Ð&8Ñ9Ô9ð 	!Ø×ÒÑ Ô Ð Ð Ð ð	!ð 	!ð 	!ð 	!r9   c                 ób   — d„ |                       ¦   «                              ¦   «         D ¦   «         S )Nc                 ó   — h | ]}d |v ¯|’Œ	S )r¥   r5  )r6  ru   s     r8   ú	<setcomp>z3EfficientVitMsra.no_weight_decay.<locals>.<setcomp>(  s#   € ÐOÐOÐO�aÐ7IÈQÐ7NÐ7N�Ð7NÐ7NÐ7Nr9   )Ú
state_dictÚkeysr°   s    r8   Úno_weight_decayz EfficientVitMsra.no_weight_decay&  s,   € àOÐO˜4Ÿ?š?Ñ,Ô,×1Ò1Ñ3Ô3ÐOÑOÔOÐOr9   Fc                 ó4   — t          d|rdnddg¬¦  «        }|S )Nz^patch_embedz^stages\.(\d+))z^stages\.(\d+).downsample)r   )z^stages\.(\d+)\.\w+\.(\d+)N)Ústemr  )rB  )r5   ÚcoarseÚmatchers      r8   Úgroup_matcherzEfficientVitMsra.group_matcher*  s9   € åØ Ø(.ð Ð$Ð$Ø4Ø5ð5ð
ñ 
ô 
ˆð ˆr9   c                 ó   — || _         d S rs   )r?  )r5   Úenables     r8   Úset_grad_checkpointingz'EfficientVitMsra.set_grad_checkpointing5  s   € à"(ˆÔÐÐr9   c                 ó   — | j         j        S rs   )rF  r^   r°   s    r8   Úget_classifierzEfficientVitMsra.get_classifier9  s   € àŒyÔÐr9   c                 óð  — t          | ¦  «        }|| _        |�{|dk    r6t          |d¬¦  «        | _        | j                             | j        ¦  «         n?|dk    sJ ‚t          j        ¦   «         | _        | j                             | j        ¦  «         |dk    rt          | j	        |fd| j
        i|¤Žnt          j                             ¦   «         | _        | j                             | j        ¦  «         d S )Nr+  Tr;  r   rY   )r   r-  r   r1  rÁ   r}   r,   r  rS   rD  r2  r1   rF  )r5   r-  r1  r6   s       r8   Úreset_classifierz!EfficientVitMsra.reset_classifier=  s
  € Ý˜dÑ#Ô#ˆØ&ˆÔØÐ"Ø˜eÒ#Ð#Ý#7À+ÐW[Ð#\Ñ#\Ô#\�Ô ØÔ ×&Ò& t¤}Ñ5Ô5Ð5Ð5à" aÒ'Ð'Ð'Ð'Ý#%¤;¡=¤=�Ô ØÔ ×&Ò& t¤}Ñ5Ô5Ð5àJUÐXYÊ/È/õ ØÔ˜{ðGð GØ15´ðGØCEðGð Gð GÝ_dÔ_g×_pÒ_pÑ_rÔ_rð 	Œ	àŒ	�Š˜œÑ&Ô&Ð&Ð&Ð&r9   ÚNCHWru   ÚindicesÚnormÚ
stop_earlyÚ
output_fmtÚintermediates_onlyc                 óø  — |dv s
J d¦   «         ‚g }t          t          | j        ¦  «        |¦  «        \  }}	|                      |¦  «        }t          j                             ¦   «         s|s| j        }
n| j        d|	dz   …         }
t          |
¦  «        D ]_\  }}| j        r/t          j                             ¦   «         st          ||¦  «        }n ||¦  «        }||v r| 
                    |¦  «         Œ`|r|S ||fS )aï   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )rg  zOutput shape must be NCHW.Nr   )r   r¶   rC  r@  r1   rÄ   Úis_scriptingrÌ   r?  r   r�   )r5   ru   rh  ri  rj  rk  rl  ÚintermediatesÚtake_indicesÚ	max_indexrC  Úfeat_idxrO  s                r8   Úforward_intermediatesz&EfficientVitMsra.forward_intermediatesL  s"  € ð* ˜YÐ&Ð&Ð&Ð(DÑ&Ô&Ð&ØˆÝ"6µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�ið ×Ò˜QÑÔˆåŒ9×!Ò!Ñ#Ô#ð 	1¨:ð 	1Ø”[ˆFˆFà”[  )¨a¡- Ô0ˆFå(¨Ñ0Ô0ð 	(ð 	(‰OˆH�eØÔ&ð ­u¬y×/EÒ/EÑ/GÔ/Gð Ý˜u aÑ(Ô(��à�E˜!‘H”H�Ø˜<Ð'Ð'Ø×$Ò$ QÑ'Ô'Ð'øàð 	!Ø Ð à�-ÐÐr9   r   Ú
prune_normÚ
prune_headc                 ó®   — t          t          | j        ¦  «        |¦  «        \  }}| j        d|dz   …         | _        |r|                      dd¦  «         |S )z@ Prune layers not required for specified intermediates.
        Nr   r   r  )r   r¶   rC  rf  )r5   rh  rt  ru  rp  rq  s         r8   Úprune_intermediate_layersz*EfficientVitMsra.prune_intermediate_layersz  s\   € õ #7µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�iØ”k . 9¨q¡= .Ô1ˆŒØð 	)Ø×!Ò! ! RÑ(Ô(Ð(ØÐr9   c                 óÐ   — |                       |¦  «        }| j        r4t          j                             ¦   «         st          | j        |¦  «        }n|                      |¦  «        }|S rs   )r@  r?  r1   rÄ   rn  r   rC  rt   s     r8   Úforward_featuresz!EfficientVitMsra.forward_featuresˆ  s\   € Ø×Ò˜QÑÔˆØÔ"ð 	­5¬9×+AÒ+AÑ+CÔ+Cð 	Ý˜tœ{¨AÑ.Ô.ˆAˆAà—’˜A‘”ˆAØˆr9   Ú
pre_logitsc                 ó^   — |                       |¦  «        }|r|n|                      |¦  «        S rs   )r1  rF  )r5   ru   rz  s      r8   Úforward_headzEfficientVitMsra.forward_head�  s.   € Ø×Ò˜QÑÔˆØÐ0ˆqˆq D§I¢I¨a¡L¤LÐ0r9   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S rs   )ry  r|  rt   s     r8   rv   zEfficientVitMsra.forward”  s-   € Ø×!Ò! !Ñ$Ô$ˆØ×Ò˜aÑ Ô ˆØˆr9   )r  rj   r   r!  r%  r&  r'  r(  r�   r)  r+  rU   NNrá   ©Frs   )NFFrg  F)r   FT)rJ   rK   rL   rM   r	   rÆ   rN   r+   rc   rG  r,   r‚   rR  r1   rÄ   ÚignorerZ  r_  rb  rd  r   rf  râ   r   r   rs  rw  ry  r|  rv   rP   rQ   s   @r8   r   r   Ù  sŒ  ø€ € € € € ð  ØØ#Ø)7Ø'3Ø%.Ø)2Ø+4Ø'3Ø4aØ$Ø!ØØðC-ð C-àðC-ð ðC-ð ð	C-ð
 ˜S #˜X”ðC-ð ˜3 ˜8”_ðC-ð ˜˜c˜”?ðC-ð ˜S #˜X”ðC-ð ˜s C˜xœðC-ð ˜3 ˜8”_ðC-ð ˜E # s (œO¨SÐ0Ô1ðC-ð ðC-ð ðC-ð C-ð C-ð C-ð C-ð C-ðJIð I¨ð Ið Ið Ið Ið!ð !˜rœyð !°tð !Àtð !ð !ð !ð !ð „YÔðPð Pñ ÔðPð „YÔðð ð ñ Ôðð „YÔð)ð )ð )ñ Ôð)ð „YÔð  ¤	ð  ð  ð  ñ Ôð ð'ð '¨Cð '¸hÀs¼mð 'ð 'ð 'ð 'ð$ 8<ØØ$Ø$Ø',ð, ð , àŒ|ð, ð ˜e C¨¨c¬ NÔ3Ô4ð, ð ð	, ð
 ð, ð ð, ð !%ð, ð 
ˆt�E”LÔ! 5¨¬°t¸E¼LÔ7IÐ)IÔ#JÐJÔ	Kð, ð , ð , ð , ð` ./Ø$Ø#ð	ð à˜3  S¤	˜>Ô*ðð ðð ð	ð ð ð ðð ð ð1ð 1¨$ð 1ð 1ð 1ð 1ðð ð ð ð ð ð r9   r  c           
      ó2   — | dt           t          ddddddœ	|¥S )Nr   zpatch_embed.conv1.convzhead.linearT)ri   ri   Úmit)	Úurlr-  ÚmeanrX   Ú
first_convÚ
classifierÚfixed_input_sizeÚ	pool_sizeÚlicenser   )r‚  Úkwargss     r8   Ú_cfgrŠ  ¼  s6   € àØÝ%Ý#Ø.Ø#Ø ØØðð ð ðð r9   ztimm/)Ú	hf_hub_id)zefficientvit_m0.r224_in1kzefficientvit_m1.r224_in1kzefficientvit_m2.r224_in1kzefficientvit_m3.r224_in1kzefficientvit_m4.r224_in1kzefficientvit_m5.r224_in1kFc                 ó|   — |                      dd¦  «        }t          t          | |fdt          d|¬¦  «        i|¤Ž}|S )NÚout_indices)r   r   r<   Úfeature_cfgT)Úflatten_sequentialr�  )Úpopr   r   rB  )ÚvariantÚ
pretrainedr‰  r�  Úmodels        r8   Ú_create_efficientvit_msrar”  ç  sZ   € Ø—*’*˜]¨IÑ6Ô6€KÝ ÝØØðð õ ¨D¸kÐJÑJÔJð	ð
 ðð €Eð €Lr9   c           	      ór   — t          dg d¢g d¢g d¢g d¢g d¢¬¦  «        }t          d
d	| it          |fi |¤Ž¤ŽS )Nr  r!  r&  r'  r(  r�   ©r,  r.  r  r“   r/  r–   Úefficientvit_m0r’  )r—  ©rB  r”  ©r’  r‰  Ú
model_argss      r8   r—  r—  ó  sj   € åØØ �.�.ØˆiˆiØ�)�)Ø�I�IØ��ðñ ô €Jõ %ÐlÐlÀ:ÐlÕQUÐV`ÐQkÐQkÐdjÐQkÐQkÐlÐlÐlr9   c           	      ór   — t          dg d¢g d¢g d¢g d¢g d¢¬¦  «        }t          d
d	| it          |fi |¤Ž¤ŽS )Nr  )r#  é�   r$  r&  )r<   rj   rj   r(  ©ræ   r‘   rj   rj   r–  Úefficientvit_m1r’  )rž  r˜  r™  s      r8   rž  rž     ój   € åØØ!�/�/ØˆiˆiØ�)�)Ø�I�IØ��ðñ ô €Jõ %ÐlÐlÀ:ÐlÕQUÐV`ÐQkÐQkÐdjÐQkÐQkÐlÐlÐlr9   c           	      ór   — t          dg d¢g d¢g d¢g d¢g d¢¬¦  «        }t          d
d	| it          |fi |¤Ž¤ŽS )Nr  )r#  r$  r  r&  )ri   rj   r<   r(  r�  r–  Úefficientvit_m2r’  )r¡  r˜  r™  s      r8   r¡  r¡    rŸ  r9   c           	      ór   — t          dg d¢g d¢g d¢g d¢g d¢¬¦  «        }t          d
d	| it          |fi |¤Ž¤ŽS )Nr  )r#  éð   i@  r&  )ri   rj   ri   r(  r�   r–  Úefficientvit_m3r’  )r¤  r˜  r™  s      r8   r¤  r¤    rŸ  r9   c           	      ór   — t          dg d¢g d¢g d¢g d¢g d¢¬¦  «        }t          d
d	| it          |fi |¤Ž¤ŽS )Nr  )r#  é   é€  r&  r'  r(  r�  r–  Úefficientvit_m4r’  )r¨  r˜  r™  s      r8   r¨  r¨  '  rŸ  r9   c           	      ór   — t          dg d¢g d¢g d¢g d¢g d¢¬¦  «        }t          d
d	| it          |fi |¤Ž¤ŽS )Nr  )r$  i   r§  )r   rj   ri   )rj   rj   ri   r(  r�  r–  Úefficientvit_m5r’  )rª  r˜  r™  s      r8   rª  rª  4  rŸ  r9   )r  r~  ):rø   Ú__all__r³   Úcollectionsr   Ú	functoolsr   Útypingr   r   r   r	   r
   r   r1   Útorch.nnr,   Ú	timm.datar   r   Útimm.layersr   r   r   r   r   Ú_builderr   Ú	_featuresr   Ú_manipulater   r   Ú	_registryr   r   r¡   r   rS   r‚   re   rz   r„   rŒ   rå   rú   r  r  r   rŠ  Údefault_cfgsr”  r—  rž  r¡  r¤  r¨  rª  r5  r9   r8   ú<module>r·     s
  ððð ð Ð
€Ø Ð Ð Ð Ø #Ð #Ð #Ð #Ð #Ð #Ø Ð Ð Ð Ð Ð Ø ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;à €€€Ø Ð Ð Ð Ð Ð à AÐ AÐ AÐ AÐ AÐ AÐ AÐ AØ eÐ eÐ eÐ eÐ eÐ eÐ eÐ eÐ eÐ eÐ eÐ eÐ eÐ eØ *Ð *Ð *Ð *Ð *Ð *Ø +Ð +Ð +Ð +Ð +Ð +Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø <Ð <Ð <Ð <Ð <Ð <Ð <Ð <ð ð  ð  ð  ð  ˆuŒxÔ"ñ  ô  ð  ðF#ð #ð #ð #ð #�”Ô$ñ #ô #ð #ðLð ð ð ð �5”8”?ñ ô ð ð,!ð !ð !ð !ð !�5”8”?ñ !ô !ð !ðð ð ð ð ˆeŒhŒoñ ô ð ð&uð uð uð uð u˜UœXœ_ñ uô uð uðpBð Bð Bð Bð B˜5œ8œ?ñ Bô Bð BðJ-Gð -Gð -Gð -Gð -G˜œœñ -Gô -Gð -Gð`;ð ;ð ;ð ;ð ;˜œœñ ;ô ;ð ;ð|ð ð ð ð �U”XÔ(ñ ô ð ð(~ð ~ð ~ð ~ð ~�r”yñ ~ô ~ð ~ðFð ð ð ð %Ð$Ø!% Øð"ñ "ô "ð "& Øð"ñ "ô "ð "& Øð"ñ "ô "ð "& Øð"ñ "ô "ð "& Øð"ñ "ô "ð "& Øð"ñ "ô "ð+&ð &ñ ô €ð8	ð 	ð 	ð 	ð ð	mð 	mð 	mñ „ð	mð ð	mð 	mð 	mñ „ð	mð ð	mð 	mð 	mñ „ð	mð ð	mð 	mð 	mñ „ð	mð ð	mð 	mð 	mñ „ð	mð ð	mð 	mð 	mñ „ð	mð 	mð 	mr9   