§
    ·¯¸jZ  ã                   ó”  — d Z ddlZddlmZ ddlmZmZmZmZm	Z	m
Z
 ddlZddlmZ ddlmZmZ ddlmZmZmZmZmZmZmZmZmZmZmZmZmZ ddlm Z  dd	l!m"Z" dd
l#m$Z$ ddl%m&Z&m'Z' dgZ(dddddœZ)dddddœZ*dddddœZ+ G d„ dej,        ¦  «        Z- G d„ dej        j,        ¦  «        Z. G d„ dej        j,        ¦  «        Z/ G d „ d!ej        j,        ¦  «        Z0 G d"„ d#ej,        ¦  «        Z1 G d$„ d%ej,        ¦  «        Z2 G d&„ d'ej,        ¦  «        Z3 G d(„ d)ej4        ¦  «        Z5 G d*„ d+ej,        ¦  «        Z6 G d,„ dej,        ¦  «        Z7d9d.„Z8 e& e8d/¬0¦  «         e8d/¬0¦  «         e8d/¬0¦  «         e8d/¬0¦  «        d1œ¦  «        Z9d:d3„Z:e'd:d4e7fd5„¦   «         Z;e'd:d4e7fd6„¦   «         Z<e'd:d4e7fd7„¦   «         Z=e'd:d4e7fd8„¦   «         Z>dS );aJ   EfficientFormer-V2

@article{
    li2022rethinking,
    title={Rethinking Vision Transformers for MobileNet Size and Speed},
    author={Li, Yanyu and Hu, Ju and Wen, Yang and Evangelidis, Georgios and Salahi, Kamyar and Wang, Yanzhi and Tulyakov, Sergey and Ren, Jian},
    journal={arXiv preprint arXiv:2212.08059},
    year={2022}
}

Significantly refactored and cleaned up for timm from original at: https://github.com/snap-research/EfficientFormer

Original code licensed Apache 2.0, Copyright (c) 2022 Snap Inc.

Modifications and timm support by / Copyright 2023, Ross Wightman
é    N)Úpartial)ÚDictÚListÚOptionalÚTupleÚTypeÚUnion©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)Úget_device_dtypeÚcreate_conv2dÚcreate_norm_layerÚget_act_layerÚget_norm_layerÚConvNormActÚLayerScale2dÚDropPathÚcalculate_drop_path_ratesÚtrunc_normal_Ú	to_2tupleÚ	to_ntupleÚndgridé   )Úbuild_model_with_cfg)Úfeature_take_indices)Úcheckpoint_seq)Úgenerate_default_cfgsÚregister_modelÚEfficientFormerV2)é(   éP   éÀ   é€  )é    é@   é�   i   )r%   é0   éx   éà   )r%   r(   é`   é°   )ÚLÚS2ÚS1ÚS0)é   r1   é   é
   )é   r4   é   é   )é   r7   é	   é   )é   r:   r9   r4   )r4   r4   )r4   r4   r4   r4   r7   r7   r7   r7   r7   r7   r7   r4   r4   r4   r4   )
r4   r4   r4   r7   r7   r7   r7   r4   r4   r4   )r4   r4   )r4   r4   r7   r7   r7   r7   r7   r7   r4   r4   r4   r4   )r4   r4   r7   r7   r7   r7   r4   r4   )r4   r4   )	r4   r4   r7   r7   r7   r7   r4   r4   r4   )r4   r4   r7   r7   r4   r4   )r4   r4   )r4   r7   r7   r7   r4   r4   )r4   r7   r7   r4   c                   ó€   ‡ — e Zd Z	 	 	 	 	 	 	 	 	 	 ddededed	ed
eeef         dededededee         fˆ fd„Z	d„ Z
ˆ xZS )ÚConvNormr   Ú TÚbatchnorm2dNÚin_channelsÚout_channelsÚkernel_sizeÚstrideÚpaddingÚdilationÚgroupsÚbiasÚ
norm_layerÚnorm_kwargsc           	      ó¸   •— ||dœ}|
pi }
t          ¦   «                              ¦   «          t          |||f|||||dœ|¤Ž| _        t	          |	|fi |
¤|¤Ž| _        d S )N©ÚdeviceÚdtype)rB   rC   rD   rE   rF   )ÚsuperÚ__init__r   Úconvr   Úbn)Úselfr?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rK   rL   ÚddÚ	__class__s                 €ún/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/models/efficientformer_v2.pyrN   zConvNorm.__init__G   s•   ø€ ð ¨Ð/Ð/ˆØ!Ð' RˆÝ‰Œ×ÒÑÔÐÝ!ØØØð

ð ØØØØð

ð 

ð ð

ð 

ˆŒ	õ $ J°ÐRÐRÀÐRÈrÐRÐRˆŒˆˆó    c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S ©N)rO   rP   ©rQ   Úxs     rT   ÚforwardzConvNorm.forwardf   s%   € Ø�IŠI�a‰LŒLˆØ�GŠG�A‰JŒJˆØˆrU   )
r   r   r=   r   r   Tr>   NNN)Ú__name__Ú
__module__Ú__qualname__Úintr	   ÚstrÚboolr   r   rN   rZ   Ú__classcell__©rS   s   @rT   r<   r<   F   sð   ø€ € € € € ð
  !ØØ')ØØØØ+Ø*.ØØðSð SàðSð ðSð ð	Sð
 ðSð ˜3 ˜8”_ðSð ðSð ðSð ðSð ðSð " $œðSð Sð Sð Sð Sð Sð>ð ð ð ð ð ð rU   r<   c                   óB  ‡ — e Zd ZU eeej        f         ed<   dddddej	        dddf	de
d	e
d
e
de
dee
ee
e
f         f         deej                 dee
         fˆ fd„Z ej        ¦   «         dˆ fd„	¦   «         Zdd„Zdd„Zdd„Zdd„Zdej        dej        fd„Zd„ Zˆ xZS )ÚAttention2dÚattention_bias_cacher$   r%   r6   r4   é   NÚdimÚkey_dimÚ	num_headsÚ
attn_ratioÚ
resolutionÚ	act_layerrB   c
                 ó  •‡— ||	dœ}
t          ¦   «                              ¦   «          || _        |dz  | _        || _        t          |¦  «        }‰�Nt          ˆfd„|D ¦   «         ¦  «        }t          ||fd‰|dœ|
¤Ž| _        t          j
        ‰d¬¦  «        | _        nd | _        d | _        || _        | j        d         | j        d	         z  | _        t          ||z  ¦  «        | _        t          ||z  ¦  «        |z  | _        || _        | j        | j        z  }t          ||fi |
¤Ž| _        t          ||fi |
¤Ž| _        t          || j        fi |
¤Ž| _        t          | j        | j        fd| j        d
œ|
¤Ž| _        t          j        | j        | j        fdd	i|
¤Ž| _        t          j        | j        | j        fdd	i|
¤Ž| _         |¦   «         | _        t          | j        |d	fi |
¤Ž| _        t6          j	                             t7          j        || j        fi |
¤Ž¦  «        | _        |                      dt7          j        | j        | j        f|t6          j         ¬¦  «        d¬¦  «         i | _!        |  "                    ¦   «          d S )NrJ   ç      à¿c                 ó>   •— g | ]}t          j        |‰z  ¦  «        ‘ŒS © ©ÚmathÚceil)Ú.0ÚrrB   s     €rT   ú
<listcomp>z(Attention2d.__init__.<locals>.<listcomp>ƒ   s'   ø€ ÐJÐJÐJ¸!¥¤	¨!¨f©*Ñ 5Ô 5ÐJÐJÐJrU   r7   ©rA   rB   rE   Úbilinear)Úscale_factorÚmoder   r   )rA   rE   rA   Úattention_bias_idxsF©Ú
persistent)#rM   rN   ri   Úscalerh   r   Útupler<   Ústride_convÚnnÚUpsampleÚupsamplerk   ÚNr^   ÚdÚdhrj   ÚqÚkÚvÚv_localÚConv2dÚtalking_head1Útalking_head2ÚactÚprojÚtorchÚ	ParameterÚemptyÚattention_biasesÚregister_bufferÚlongre   Úreset_parameters)rQ   rg   rh   ri   rj   rk   rl   rB   rK   rL   rR   ÚkhrS   s          `    €rT   rN   zAttention2d.__init__o   s§  øø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ"ˆŒØ ‘_ˆŒ
ØˆŒå˜zÑ*Ô*ˆ
ØÐÝÐJÐJÐJÐJ¸zÐJÑJÔJÑKÔKˆJÝ'¨¨SÐa¸aÈÐWZÐaÐaÐ^`ÐaÐaˆDÔÝœK°VÀ*ÐMÑMÔMˆDŒMˆMà#ˆDÔØ ˆDŒMà$ˆŒØ” Ô# d¤o°aÔ&8Ñ8ˆŒÝ�Z 'Ñ)Ñ*Ô*ˆŒÝ�j 7Ñ*Ñ+Ô+¨iÑ7ˆŒØ$ˆŒØŒ\˜DœNÑ*ˆå˜#˜rÐ(Ð( RÐ(Ð(ˆŒÝ˜#˜rÐ(Ð( RÐ(Ð(ˆŒÝ˜#˜tœwÐ-Ð-¨"Ð-Ð-ˆŒÝ ¤¨¬ÐV¸aÈÌÐVÐVÐSUÐVÐVˆŒÝœY t¤~°t´~Ð[Ð[ÐSTÐ[ÐXZÐ[Ð[ˆÔÝœY t¤~°t´~Ð[Ð[ÐSTÐ[ÐXZÐ[Ð[ˆÔà�9‘;”;ˆŒÝ˜TœW c¨1Ð3Ð3°Ð3Ð3ˆŒ	å %¤× 2Ò 2µ5´;¸yÈ$Ì&Ð3WÐ3WÐTVÐ3WÐ3WÑ XÔ XˆÔØ×ÒØ!ÝŒK˜œ ¤Ð(°½u¼zÐJÑJÔJØð 	ñ 	
ô 	
ð 	
ð
 %'ˆÔ!ð 	×ÒÑÔÐÐÐrU   Tc                 ój   •— t          ¦   «                              |¦  «         | j        r	i | _        d S d S rW   ©rM   Útrainre   ©rQ   rz   rS   s     €rT   rš   zAttention2d.train¦   ó<   ø€ å‰Œ�Š�dÑÔÐØÔ$ð 	+Ø(*ˆDÔ%Ð%Ð%ð	+ð 	+rU   Úreturnc                 óv   — t           j                             | j        ¦  «         |                      ¦   «          dS ©z"Initialize parameters and buffers.N©r�   ÚinitÚzeros_r“   Ú_init_buffers©rQ   s    rT   r–   zAttention2d.reset_parameters¬   ó1   € å
Œ�Š�tÔ,Ñ-Ô-Ð-Ø×ÒÑÔÐÐÐrU   c           
      ó°  — t          j        t          t          j        | j        d         |t           j        ¬¦  «        t          j        | j        d         |t           j        ¬¦  «        ¦  «        ¦  «                             d¦  «        }|ddd…df         |dddd…f         z
                       ¦   «         }|d         | j        d         z  |d         z   }|S )ú5Compute relative position indices for attention bias.r   rJ   r   .N©r�   Ústackr   Úarangerk   r•   ÚflattenÚabs)rQ   rK   ÚposÚrel_poss       rT   Ú_compute_attention_bias_idxsz(Attention2d._compute_attention_bias_idxs±   sÂ   € åŒk�&ÝŒL˜œ¨Ô+°FÅ%Ä*ÐMÑMÔMÝŒL˜œ¨Ô+°FÅ%Ä*ÐMÑMÔMñ
ô 
ñ ô ÷ Š7�1‰:Œ:ð 	ð �s˜A˜A˜A˜t�|Ô$ s¨3°°a°a°a¨<Ô'8Ñ8×=Ò=Ñ?Ô?ˆØ˜1”: ¤°Ô 2Ñ2°g¸a´jÑ@ˆØˆrU   c                 ó„   — | j                              |                      | j         j        ¬¦  «        ¦  «         i | _        dS ©z.Compute and fill non-persistent buffer values.)rK   N©r{   Úcopy_r¯   rK   re   r¤   s    rT   r£   zAttention2d._init_buffers»   óG   € àÔ ×&Ò&Ø×-Ò-°TÔ5MÔ5TÐ-ÑUÔUñ	
ô 	
ð 	
ð %'ˆÔ!Ð!Ð!rU   c                 ó.   — |                       ¦   «          dS ©z"Initialize non-persistent buffers.N©r£   r¤   s    rT   Úinit_non_persistent_buffersz'Attention2d.init_non_persistent_buffersÂ   ó   € à×ÒÑÔÐÐÐrU   rK   c                 óþ   — t           j                             ¦   «         s| j        r| j        d d …| j        f         S t          |¦  «        }|| j        vr| j        d d …| j        f         | j        |<   | j        |         S rW   ©r�   ÚjitÚ
is_tracingÚtrainingr“   r{   r_   re   ©rQ   rK   Ú
device_keys      rT   Úget_attention_biasesz Attention2d.get_attention_biasesÆ   ó„   € ÝŒ9×ÒÑ!Ô!ð 	9 T¤]ð 	9ØÔ(¨¨¨¨DÔ,DÐ)DÔEÐEå˜V™œˆJØ Ô!:Ð:Ð:Ø8<Ô8MÈaÈaÈaÐQUÔQiÐNiÔ8j�Ô)¨*Ñ5ØÔ,¨ZÔ8Ð8rU   c                 óD  — |j         \  }}}}| j        �|                      |¦  «        }|                      |¦  «                             || j        d| j        ¦  «                             dddd¦  «        }|                      |¦  «                             || j        d| j        ¦  «                             dddd¦  «        }|                      |¦  «        }|  	                    |¦  «        }	|                     || j        d| j        ¦  «                             dddd¦  «        }||z  | j
        z  }
|
|                      |j        ¦  «        z   }
|                      |
¦  «        }
|
                     d¬¦  «        }
|                      |
¦  «        }
|
|z                       dd¦  «        }|                     || j        | j        d         | j        d         ¦  «        |	z   }| j        �|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S ©Néÿÿÿÿr   r   r7   r:   ©rg   )Úshaper€   r‡   Úreshaperi   r„   Úpermuterˆ   r‰   rŠ   r~   rÁ   rK   rŒ   Úsoftmaxr�   Ú	transposer†   rk   rƒ   rŽ   r�   ©rQ   rY   ÚBÚCÚHÚWr‡   rˆ   r‰   rŠ   Úattns              rT   rZ   zAttention2d.forwardÏ   sÔ  € Ø”W‰
ˆˆ1ˆa�ØÔÐ'Ø× Ò  Ñ#Ô#ˆAà�FŠF�1‰IŒI×Ò˜a ¤°°T´VÑ<Ô<×DÒDÀQÈÈ1ÈaÑPÔPˆØ�FŠF�1‰IŒI×Ò˜a ¤°°T´VÑ<Ô<×DÒDÀQÈÈ1ÈaÑPÔPˆØ�FŠF�1‰IŒIˆØ—,’,˜q‘/”/ˆØ�IŠI�a˜œ¨¨T¬VÑ4Ô4×<Ò<¸QÀÀ1ÀaÑHÔHˆà�A‘˜œÑ#ˆØ�d×/Ò/°´Ñ9Ô9Ñ9ˆØ×!Ò! $Ñ'Ô'ˆØ�|Š| ˆ|Ñ#Ô#ˆØ×!Ò! $Ñ'Ô'ˆà�A‰X× Ò   AÑ&Ô&ˆØ�IŠI�a˜œ $¤/°!Ô"4°d´oÀaÔ6HÑIÔIÈGÑSˆØŒ=Ð$Ø—’˜aÑ Ô ˆAà�HŠH�Q‰KŒKˆØ�IŠI�a‰LŒLˆØˆrU   ©T©r�   NrW   )r[   r\   r]   r   r_   r�   ÚTensorÚ__annotations__r�   ÚGELUr^   r	   r   r   ÚModuler   rN   Úno_gradrš   r–   r¯   r£   r¸   rK   rÁ   rZ   ra   rb   s   @rT   rd   rd   l   s“  ø€ € € € € € Ø˜s E¤LÐ0Ô1Ð1Ð1Ñ1ð ØØØØ67Ø)+¬Ø$(ØØð5 ð 5 àð5 ð ð5 ð ð	5 ð
 ð5 ð ˜c 5¨¨c¨¤?Ð2Ô3ð5 ð ˜BœI”ð5 ð ˜S”Mð5 ð 5 ð 5 ð 5 ð 5 ð 5 ðn €U„]�_„_ð+ð +ð +ð +ð +ñ „_ð+ð
ð ð ð ð
ð ð ð ð'ð 'ð 'ð 'ðð ð ð ð9¨5¬<ð 9¸E¼Lð 9ð 9ð 9ð 9ðð ð ð ð ð ð rU   rd   c                   ó4   ‡ — e Zd Z	 	 ddedefˆ fd„Zd„ Zˆ xZS )ÚLocalGlobalQueryNÚin_dimÚout_dimc                 óê   •— ||dœ}t          ¦   «                              ¦   «          t          j        ddd¦  «        | _        t          j        ||fddd|dœ|¤Ž| _        t          ||dfi |¤Ž| _        d S )NrJ   r   r:   r   r7   )rA   rB   rC   rE   )	rM   rN   r�   Ú	AvgPool2dÚpoolr‹   Úlocalr<   r�   )rQ   rÛ   rÜ   rK   rL   rR   rS   s         €rT   rN   zLocalGlobalQuery.__init__ë   s…   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ”L  A qÑ)Ô)ˆŒ	Ý”Y˜v vÐg¸1ÀQÐPQÐZ`ÐgÐgÐdfÐgÐgˆŒ
Ý˜V W¨aÐ6Ð6°2Ð6Ð6ˆŒ	ˆ	ˆ	rU   c                 óŽ   — |                       |¦  «        }|                      |¦  «        }||z   }|                      |¦  «        }|S rW   )rà   rß   r�   )rQ   rY   Úlocal_qÚpool_qr‡   s        rT   rZ   zLocalGlobalQuery.forwardø   s>   € Ø—*’*˜Q‘-”-ˆØ—’˜1‘”ˆØ�fÑˆØ�IŠI�a‰LŒLˆØˆrU   )NN)r[   r\   r]   r^   rN   rZ   ra   rb   s   @rT   rÚ   rÚ   ê   sk   ø€ € € € € ð
 Øð7ð 7àð7ð ð7ð 7ð 7ð 7ð 7ð 7ðð ð ð ð ð ð rU   rÚ   c                   óB  ‡ — e Zd ZU eeej        f         ed<   ddddddej	        ddf	de
d	e
d
e
de
dee
ee
e
f         f         dee
         deej                 fˆ fd„Z ej        ¦   «         dˆ fd„	¦   «         Zdd„Zdd„Zdd„Zdd„Zdej        dej        fd„Zd„ Zˆ xZS )ÚAttention2dDownsamplere   r$   é   r6   r4   rf   Nrg   rh   ri   rj   rk   rÜ   rl   c
                 óP  •— ||	dœ}
t          ¦   «                              ¦   «          || _        |dz  | _        || _        t          |¦  «        | _        t          d„ | j        D ¦   «         ¦  «        | _        | j        d         | j        d         z  | _	        | j        d         | j        d         z  | _
        t          ||z  ¦  «        | _        t          ||z  ¦  «        |z  | _        || _        |p|| _        | j        | j        z  }t!          ||fi |
¤Ž| _        t%          ||dfi |
¤Ž| _        t%          || j        dfi |
¤Ž| _        t%          | j        | j        fdd| j        dœ|
¤Ž| _         |¦   «         | _        t%          | j        | j        dfi |
¤Ž| _        t1          j        t5          j        || j	        fi |
¤Ž¦  «        | _        |                      d	t5          j        | j
        | j	        f|t4          j        ¬¦  «        d
¬¦  «         i | _        |                       ¦   «          d S )NrJ   rn   c                 ó<   — g | ]}t          j        |d z  ¦  «        ‘ŒS ©r:   rq   ©rt   ru   s     rT   rv   z2Attention2dDownsample.__init__.<locals>.<listcomp>  s&   € Ð!LÐ!LÐ!L°q¥$¤)¨A°©EÑ"2Ô"2Ð!LÐ!LÐ!LrU   r   r   r7   r:   rw   r{   Fr|   )!rM   rN   ri   r~   rh   r   rk   r   Úresolution2r„   ÚN2r^   r…   r†   rj   rÜ   rÚ   r‡   r<   rˆ   r‰   rŠ   rŽ   r�   r�   r‘   r�   r’   r“   r”   r•   re   r–   )rQ   rg   rh   ri   rj   rk   rÜ   rl   rK   rL   rR   r—   rS   s               €rT   rN   zAttention2dDownsample.__init__  s*  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐà"ˆŒØ ‘_ˆŒ
ØˆŒÝ# JÑ/Ô/ˆŒÝ Ð!LÐ!L¸D¼OÐ!LÑ!LÔ!LÑMÔMˆÔØ” Ô# d¤o°aÔ&8Ñ8ˆŒØÔ" 1Ô%¨Ô(8¸Ô(;Ñ;ˆŒå�Z 'Ñ)Ñ*Ô*ˆŒÝ�j 7Ñ*Ñ+Ô+¨iÑ7ˆŒØ$ˆŒØ�~ #ˆŒØŒ\˜DœNÑ*ˆå! # rÐ0Ð0¨RÐ0Ð0ˆŒÝ˜#˜r 1Ð+Ð+¨Ð+Ð+ˆŒÝ˜#˜tœw¨Ð0Ð0¨RÐ0Ð0ˆŒÝ ¤¨¬Ð`¸aÈÐRVÔRYÐ`Ð`Ð]_Ð`Ð`ˆŒà�9‘;”;ˆŒÝ˜TœW d¤l°AÐ<Ð<¸Ð<Ð<ˆŒ	å "¤­U¬[¸ÀDÄFÐ-QÐ-QÈbÐ-QÐ-QÑ RÔ RˆÔØ×ÒØ!ÝŒK˜œ $¤&Ð)°&ÅÄ
ÐKÑKÔKØð 	ñ 	
ô 	
ð 	
ð
 %'ˆÔ!ð 	×ÒÑÔÐÐÐrU   Tc                 ój   •— t          ¦   «                              |¦  «         | j        r	i | _        d S d S rW   r™   r›   s     €rT   rš   zAttention2dDownsample.train3  rœ   rU   r�   c                 óv   — t           j                             | j        ¦  «         |                      ¦   «          dS rŸ   r    r¤   s    rT   r–   z&Attention2dDownsample.reset_parameters9  r¥   rU   c                 óÊ  — t          j        t          t          j        | j        d         |t           j        ¬¦  «        t          j        | j        d         |t           j        ¬¦  «        ¦  «        ¦  «                             d¦  «        }t          j        t          t          j        d| j        d         d|t           j        ¬¦  «        t          j        d| j        d         d|t           j        ¬¦  «        ¦  «        ¦  «                             d¦  «        }|ddd…df         |dddd…f         z
                       ¦   «         }|d         | j        d         z  |d         z   }|S )r§   r   rJ   r   r:   )ÚsteprK   rL   .Nr¨   )rQ   rK   Úk_posÚq_posr®   s        rT   r¯   z2Attention2dDownsample._compute_attention_bias_idxs>  s3  € å”�FÝŒL˜œ¨Ô+°FÅ%Ä*ÐMÑMÔMÝŒL˜œ¨Ô+°FÅ%Ä*ÐMÑMÔMñ
ô 
ñ ô ÷ Š7�1‰:Œ:ð 	õ ”�FÝŒL˜˜DœO¨AÔ.°Q¸vÍUÌZÐXÑXÔXÝŒL˜˜DœO¨AÔ.°Q¸vÍUÌZÐXÑXÔXñ
ô 
ñ ô ÷ Š7�1‰:Œ:ð 	ð ˜˜a˜a˜a ˜Ô&¨¨s°D¸!¸!¸!¨|Ô)<Ñ<×AÒAÑCÔCˆØ˜1”: ¤°Ô 2Ñ2°g¸a´jÑ@ˆØˆrU   c                 ó„   — | j                              |                      | j         j        ¬¦  «        ¦  «         i | _        dS r±   r²   r¤   s    rT   r£   z#Attention2dDownsample._init_buffersL  r´   rU   c                 ó.   — |                       ¦   «          dS r¶   r·   r¤   s    rT   r¸   z1Attention2dDownsample.init_non_persistent_buffersS  r¹   rU   rK   c                 óþ   — t           j                             ¦   «         s| j        r| j        d d …| j        f         S t          |¦  «        }|| j        vr| j        d d …| j        f         | j        |<   | j        |         S rW   r»   r¿   s      rT   rÁ   z*Attention2dDownsample.get_attention_biasesW  rÂ   rU   c                 ó€  — |j         \  }}}}|                      |¦  «                             || j        d| j        ¦  «                             dddd¦  «        }|                      |¦  «                             || j        d| j        ¦  «                             dddd¦  «        }|                      |¦  «        }|  	                    |¦  «        }	|                     || j        d| j        ¦  «                             dddd¦  «        }||z  | j
        z  }
|
|                      |j        ¦  «        z   }
|
                     d¬¦  «        }
|
|z                       dd¦  «        }|                     || j        | j        d         | j        d         ¦  «        |	z   }|                      |¦  «        }|                      |¦  «        }|S rÄ   )rÇ   r‡   rÈ   ri   rì   rÉ   rˆ   r„   r‰   rŠ   r~   rÁ   rK   rÊ   rË   r†   rë   rŽ   r�   rÌ   s              rT   rZ   zAttention2dDownsample.forward`  s{  € Ø”W‰
ˆˆ1ˆa�à�FŠF�1‰IŒI×Ò˜a ¤°°T´WÑ=Ô=×EÒEÀaÈÈAÈqÑQÔQˆØ�FŠF�1‰IŒI×Ò˜a ¤°°T´VÑ<Ô<×DÒDÀQÈÈ1ÈaÑPÔPˆØ�FŠF�1‰IŒIˆØ—,’,˜q‘/”/ˆØ�IŠI�a˜œ¨¨T¬VÑ4Ô4×<Ò<¸QÀÀ1ÀaÑHÔHˆà�A‘˜œÑ#ˆØ�d×/Ò/°´Ñ9Ô9Ñ9ˆØ�|Š| ˆ|Ñ#Ô#ˆà�A‰X× Ò   AÑ&Ô&ˆØ�IŠI�a˜œ $Ô"2°1Ô"5°tÔ7GÈÔ7JÑKÔKÈgÑUˆØ�HŠH�Q‰KŒKˆØ�IŠI�a‰LŒLˆØˆrU   rÒ   rÓ   rW   )r[   r\   r]   r   r_   r�   rÔ   rÕ   r�   rÖ   r^   r	   r   r   r   r×   rN   rØ   rš   r–   r¯   r£   r¸   rK   rÁ   rZ   ra   rb   s   @rT   rå   rå      s“  ø€ € € € € € Ø˜s E¤LÐ0Ô1Ð1Ð1Ñ1ð ØØØØ67Ø%)Ø)+¬ØØð. ð . àð. ð ð. ð ð	. ð
 ð. ð ˜c 5¨¨c¨¤?Ð2Ô3ð. ð ˜c”]ð. ð ˜BœI”ð. ð . ð . ð . ð . ð . ð` €U„]�_„_ð+ð +ð +ð +ð +ñ „_ð+ð
ð ð ð ð
ð ð ð ð'ð 'ð 'ð 'ðð ð ð ð9¨5¬<ð 9¸E¼Lð 9ð 9ð 9ð 9ðð ð ð ð ð ð rU   rå   c                   ó*  ‡ — e Zd Zdddddej        ej        ddf	deded	eeeeef         f         d
eeeeef         f         deeeeef         f         deeeeef         f         de	de
ej                 dee
ej                          fˆ fd„Zd„ Zˆ xZS )Ú
Downsampler7   r:   r   rf   FNÚin_chsÚout_chsrA   rB   rC   rk   Úuse_attnrl   rG   c                 óH  •— |
|dœ}t          ¦   «                              ¦   «          t          |¦  «        }t          |¦  «        }t          |¦  «        }|	pt          j        ¦   «         }	t          ||f||||	dœ|¤Ž| _        |rt          d||||dœ|¤Ž| _        d S d | _        d S )NrJ   )rA   rB   rC   rG   )rg   rÜ   rk   rl   rp   )	rM   rN   r   r�   ÚIdentityr<   rO   rå   rÑ   )rQ   rù   rú   rA   rB   rC   rk   rû   rl   rG   rK   rL   rR   rS   s                €rT   rN   zDownsample.__init__u  sä   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐå Ñ,Ô,ˆÝ˜6Ñ"Ô"ˆÝ˜GÑ$Ô$ˆØÐ0¥2¤;¡=¤=ˆ
ÝØØð
ð $ØØØ!ð
ð 
ð ð
ð 
ˆŒ	ð ð 		Ý-ð ØØØ%Ø#ð	ð ð
 ðð ˆDŒIˆIˆIð ˆDŒIˆIˆIrU   c                 ón   — |                       |¦  «        }| j        �|                      |¦  «        |z   S |S rW   )rO   rÑ   )rQ   rY   Úouts      rT   rZ   zDownsample.forwardŸ  s3   € Ø�iŠi˜‰lŒlˆØŒ9Ð Ø—9’9˜Q‘<”< #Ñ%Ð%Øˆ
rU   )r[   r\   r]   r�   rÖ   ÚBatchNorm2dr^   r	   r   r`   r   r×   r   rN   rZ   ra   rb   s   @rT   rø   rø   t  s$  ø€ € € € € ð
 89Ø23Ø34Ø67Ø"Ø)+¬Ø46´NØØð(ð (àð(ð ð(ð ˜s E¨#¨s¨(¤OÐ3Ô4ð	(ð
 ˜#˜u S¨# XœÐ.Ô/ð(ð ˜3  c¨3 h¤Ð/Ô0ð(ð ˜c 5¨¨c¨¤?Ð2Ô3ð(ð ð(ð ˜BœI”ð(ð !  b¤i¤Ô1ð(ð (ð (ð (ð (ð (ðTð ð ð ð ð ð rU   rø   c                   ó°   ‡ — e Zd ZdZddej        ej        ddddfdedee         dee         de	ej
                 d	e	ej
                 d
edefˆ fd„Zd„ Zˆ xZS )ÚConvMlpWithNormz`
    Implementation of MLP with 1*1 convolutions.
    Input: tensor with shape [B, C, H, W]
    Nç        FÚin_featuresÚhidden_featuresÚout_featuresrl   rG   ÚdropÚmid_convc
                 óŠ  •— ||	dœ}
t          ¦   «                              ¦   «          |p|}|p|}t          ||dfd||dœ|
¤Ž| _        |rt          ||df|d||dœ|
¤Ž| _        nt          j        ¦   «         | _        t          j        |¦  «        | _        t          ||dfd|i|
¤Ž| _
        t          j        |¦  «        | _        d S )NrJ   r   T)rF   rG   rl   r7   )rE   rF   rG   rl   rG   )rM   rN   r   Úfc1Úmidr�   rý   ÚDropoutÚdrop1r<   Úfc2Údrop2)rQ   r  r  r  rl   rG   r  r  rK   rL   rR   rS   s              €rT   rN   zConvMlpWithNorm.__init__¬  s  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ#Ð2 {ˆØ)Ð8¨[ˆÝØØØð
ð Ø!Øð
ð 
ð ð
ð 
ˆŒð ð 	%Ý"ØØØð	ð 'ØØ%Ø#ð	ð 	ð ð	ð 	ˆDŒHˆHõ ”{‘}”}ˆDŒHÝ”Z Ñ%Ô%ˆŒ
Ý˜O¨\¸1ÐZÐZÈÐZÐWYÐZÐZˆŒÝ”Z Ñ%Ô%ˆŒ
ˆ
ˆ
rU   c                 óØ   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S rW   )r
  r  r  r  r  rX   s     rT   rZ   zConvMlpWithNorm.forwardÖ  sR   € Ø�HŠH�Q‰KŒKˆØ�HŠH�Q‰KŒKˆØ�JŠJ�q‰MŒMˆØ�HŠH�Q‰KŒKˆØ�JŠJ�q‰MŒMˆØˆrU   )r[   r\   r]   Ú__doc__r�   rÖ   r   r^   r   r   r×   Úfloatr`   rN   rZ   ra   rb   s   @rT   r  r  ¦  sÒ   ø€ € € € € ðð ð .2Ø*.Ø)+¬Ø*,¬.ØØ"ØØð(&ð (&àð(&ð & cœ]ð(&ð # 3œ-ð	(&ð
 ˜BœI”ð(&ð ˜RœYœð(&ð ð(&ð ð(&ð (&ð (&ð (&ð (&ð (&ðTð ð ð ð ð ð rU   r  c                   óÞ   ‡ — e Zd Zdej        ej        ddddddddfdeded	eej	                 d
eej	                 dedede
e         deeeeef         f         de
e         defˆ fd„Zd„ Zˆ xZS )ÚEfficientFormerV2Blockç      @r  çñhãˆµøä>rf   NTrg   Ú	mlp_ratiorl   rG   Ú	proj_dropÚ	drop_pathÚlayer_scale_init_valuerk   rB   rû   c           
      óR  •— ||dœ}t          ¦   «                              ¦   «          |
rlt          |f|||	dœ|¤Ž| _        |�t	          ||fi |¤Žnt          j        ¦   «         | _        |dk    rt          |¦  «        nt          j        ¦   «         | _	        nd | _        d | _        d | _	        t          d|t          ||z  ¦  «        |||ddœ|¤Ž| _        |�t	          ||fi |¤Žnt          j        ¦   «         | _        |dk    rt          |¦  «        nt          j        ¦   «         | _        d S )NrJ   )rk   rl   rB   r  T)r  r  rl   rG   r  r  rp   )rM   rN   rd   Útoken_mixerr   r�   rý   Úls1r   Ú
drop_path1r  r^   ÚmlpÚls2Ú
drop_path2)rQ   rg   r  rl   rG   r  r  r  rk   rB   rû   rK   rL   rR   rS   s                 €rT   rN   zEfficientFormerV2Block.__init__à  s‹  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐàð 	#Ý*Øð à%Ø#Øð	 ð  ð
 ð ð  ˆDÔð 7MÐ6Xõ $ØÐ+ð3ð 3Ø/1ð3ð 3ð 3Ý^`Ô^iÑ^kÔ^kð ŒHà5>À²^°^�h yÑ1Ô1Ð1ÍÌÉÌˆDŒOˆOà#ˆDÔØˆDŒHØ"ˆDŒOå"ð 
ØÝ  i¡Ñ0Ô0ØØ!ØØð
ð 
ð ð
ð 
ˆŒð 3IÐ2Tõ  ØÐ'ð/ð /Ø+-ð/ð /ð /ÝZ\ÔZeÑZgÔZgð 	Œà1:¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒˆˆrU   c                 ó  — | j         �>||                      |                      |                       |¦  «        ¦  «        ¦  «        z   }||                      |                      |                      |¦  «        ¦  «        ¦  «        z   }|S rW   )r  r  r  r!  r   r  rX   s     rT   rZ   zEfficientFormerV2Block.forward  sl   € ØÔÐ'Ø�D—O’O D§H¢H¨T×-=Ò-=¸aÑ-@Ô-@Ñ$AÔ$AÑBÔBÑBˆAØ�—’ §¢¨¯ª°!©¬Ñ 5Ô 5Ñ6Ô6Ñ6ˆØˆrU   )r[   r\   r]   r�   rÖ   r   r^   r  r   r×   r   r	   r   r`   rN   rZ   ra   rb   s   @rT   r  r  ß  s  ø€ € € € € ð  "Ø)+¬Ø*,¬.Ø!Ø!Ø6:Ø67Ø$(Ø!ØØð-Sð -Sàð-Sð ð-Sð ˜BœI”ð	-Sð
 ˜RœYœð-Sð ð-Sð ð-Sð %-¨U¤Oð-Sð ˜c 5¨¨c¨¤?Ð2Ô3ð-Sð ˜S”Mð-Sð ð-Sð -Sð -Sð -Sð -Sð -Sð^ð ð ð ð ð ð rU   r  c            
       óz   ‡ — e Zd Zej        ej        ddfdededeej                 deej                 fˆ fd„Z	ˆ xZ
S )ÚStem4Nrù   rú   rl   rG   c           
      óÔ   •— ||dœ}t          ¦   «                              ¦   «          d| _        t          ||dz  fdddd||dœ|¤Ž| _        t          |dz  |fdddd||dœ|¤Ž| _        d S )NrJ   r4   r:   r7   r   T)rA   rB   rC   rF   rG   rl   )rM   rN   rB   r   Úconv1Úconv2)	rQ   rù   rú   rl   rG   rK   rL   rR   rS   s	           €rT   rN   zStem4.__init__  sº   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØˆŒÝ ØØ�q‰Lð	
ð Ø˜aØØ!Øð	
ð 	
ð ð	
ð 	
ˆŒ
õ !Ø�q‰LØð

ð ØØØØ!Øð

ð 

ð ð

ð 

ˆŒ
ˆ
ˆ
rU   )r[   r\   r]   r�   rÖ   r   r^   r   r×   rN   ra   rb   s   @rT   r$  r$    s†   ø€ € € € € ð
 *,¬Ø*,¬.ØØð 
ð  
àð 
ð ð 
ð ˜BœI”ð	 
ð
 ˜RœYœð 
ð  
ð  
ð  
ð  
ð  
ð  
ð  
ð  
ð  
rU   r$  c                    ó4  ‡ — e Zd Zddddddddddej        ej        ddfd	ed
ededeeeeef         f         de	de
e         de	de	dedeeeedf         f         dedeeee         f         de
e         deej                 deej                 fˆ fd„Zd„ Zˆ xZS )ÚEfficientFormerV2Stagerf   TNFr   r  r  r  rg   Údim_outÚdepthrk   Ú
downsampleÚblock_strideÚdownsample_use_attnÚblock_use_attnÚnum_vitr  .r  r  r  rl   rG   c                 ó  •— ||dœ}t          ¦   «                              ¦   «          d| _         t          |¦  «        |
¦  «        }
t	          |¦  «        }|r4t          ||f||||dœ|¤Ž| _        |}t          d„ |D ¦   «         ¦  «        }n ||k    sJ ‚t          j	        ¦   «         | _        g }t          |¦  «        D ]9}||	z
  dz
  }t          |f|||
|         |o||k    |||         |||dœ	|¤Ž}||gz  }Œ:t          j        |Ž | _        d S )NrJ   F)rû   rk   rG   rl   c                 ó<   — g | ]}t          j        |d z  ¦  «        ‘ŒS ré   rq   rê   s     rT   rv   z3EfficientFormerV2Stage.__init__.<locals>.<listcomp>a  s&   € ÐEÐEÐE°Q¥¤	¨!¨a©%Ñ 0Ô 0ÐEÐEÐErU   r   )	rk   rB   r  rû   r  r  r  rl   rG   )rM   rN   Úgrad_checkpointingr   r   rø   r,  r   r�   rý   Úranger  Ú
SequentialÚblocks)rQ   rg   r*  r+  rk   r,  r-  r.  r/  r0  r  r  r  r  rl   rG   rK   rL   rR   r6  Ú	block_idxÚ
remain_idxÚbrS   s                          €rT   rN   zEfficientFormerV2Stage.__init__<  sr  ø€ ð( ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ"'ˆÔØ$•I˜eÑ$Ô$ YÑ/Ô/ˆ	Ý˜zÑ*Ô*ˆ
àð 	,Ý(ØØðð -Ø%Ø%Ø#ðð ð ðð ˆDŒOð ˆCÝÐEÐE¸*ÐEÑEÔEÑFÔFˆJˆJà˜'’>�>�>�>Ý œk™mœmˆDŒOàˆÝ˜u™œð 	ð 	ˆIØ ™¨1Ñ,ˆJÝ&Øðà%Ø#Ø# IÔ.Ø'ÐB¨I¸
Ò,BØ#Ø# IÔ.Ø'=Ø#Ø%ðð ð ðð ˆAð �q�c‰MˆFˆFÝ”m VÐ,ˆŒˆˆrU   c                 óÐ   — |                       |¦  «        }| j        r4t          j                             ¦   «         st          | j        |¦  «        }n|                      |¦  «        }|S rW   )r,  r3  r�   r¼   Úis_scriptingr   r6  rX   s     rT   rZ   zEfficientFormerV2Stage.forwardy  sZ   € Ø�OŠO˜AÑÔˆØÔ"ð 	­5¬9×+AÒ+AÑ+CÔ+Cð 	Ý˜tœ{¨AÑ.Ô.ˆAˆAà—’˜A‘”ˆAØˆrU   )r[   r\   r]   r�   rÖ   r   r^   r	   r   r`   r   r  r   r   r×   rN   rZ   ra   rb   s   @rT   r)  r)  :  s]  ø€ € € € € ð 78Ø#Ø*.Ø(-Ø#(ØØ9;Ø!Ø35Ø6:Ø)+¬Ø*,¬.ØØð%;-ð ;-àð;-ð ð;-ð ð	;-ð
 ˜c 5¨¨c¨¤?Ð2Ô3ð;-ð ð;-ð # 3œ-ð;-ð "&ð;-ð !ð;-ð ð;-ð ˜U E¨%°¨*Ô$5Ð5Ô6ð;-ð ð;-ð ˜U D¨¤KÐ/Ô0ð;-ð %-¨U¤Oð;-ð ˜BœI”ð;-ð  ˜RœYœð!;-ð ;-ð ;-ð ;-ð ;-ð ;-ðzð ð ð ð ð ð rU   r)  c            #       ól  ‡ — e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d;deedf         dedeeeeef         f         dedeeedf                  deeedf                  dee	ee	df         eee	df         df         f         dede	dedede	de	de	dee	         dedef"ˆ fd„Z
d<d efd!„Zd<d efd"„Zej        j        d#„ ¦   «         Zej        j        d=d%„¦   «         Zej        j        d<d&„¦   «         Zej        j        d'ej        fd(„¦   «         Zd>dedee         fd)„Zej        j        d<d*„¦   «         Z	 	 	 	 	 d?d,ej        d-eeeee         f                  d.ed/ed0ed1ed'eeej                 eej        eej                 f         f         fd2„Z	 	 	 d@d-eeee         f         d4ed5efd6„Zd7„ Zd=d8efd9„Zd:„ Zˆ xZS )Ar    r7   r*   ÚavgNr4   r>   r  Úgeluéè  r  r   TÚdepths.Úin_chansÚimg_sizeÚglobal_poolÚ
embed_dimsÚdownsamplesÚ
mlp_ratiosrG   Únorm_epsrl   Únum_classesÚ	drop_rateÚproj_drop_rateÚdrop_path_rater  r0  Údistillationc                 óz  •‡— t          ¦   «                              ¦   «          ||dœ}|dv sJ ‚|| _        || _        || _        g | _        t          |¦  «        }t          t          |¦  «        |	¬¦  «        }t          |
¦  «        }
t          ||d         f|
|dœ|¤Ž| _        |d         }dŠt          |¦  «        }t          ||d¬¦  «        }|pd	d
t          |¦  «        dz
  z  z   } t          |¦  «        |¦  «        }g }t          |¦  «        D ]µ}t!          ˆfd„|D ¦   «         ¦  «        }t#          |||         f||         |||         |dk    rdnd |dk    |dk    |||         |||         ||
|dœ|¤Ž}||         r‰dz  Š||         }| xj        t%          |‰d|› �¬¦  «        gz  c_        |                     |¦  «         Œ¶t)          j        |Ž | _        |d         x| _        | _         ||d         fi |¤Ž| _        t)          j        |¦  «        | _        |dk    rt)          j        |d         |fi |¤Žnt)          j        ¦   «         | _        || _        | j        r8|dk    rt)          j        |d         |fi |¤Žnt)          j        ¦   «         | _         nd | _         |  !                    d¬¦  «         d| _"        d S )NrJ   )r=  r=   )Úepsr   )rl   rG   r4   T)Ú	stagewise©FrÒ   r   c                 ó>   •— g | ]}t          j        |‰z  ¦  «        ‘ŒS rp   rq   )rt   ÚsrB   s     €rT   rv   z.EfficientFormerV2.__init__.<locals>.<listcomp>®  s'   ø€ Ð$MÐ$MÐ$M¸q¥T¤Y¨q°6©zÑ%:Ô%:Ð$MÐ$MÐ$MrU   r:   r7   )r+  rk   r,  r-  r.  r/  r0  r  r  r  r  rl   rG   zstages.)Únum_chsÚ	reductionÚmodulerÅ   F©Úneeds_reset)#rM   rN   rH  rA  rC  Úfeature_infor   r   r   r   r$  ÚstemÚlenr   r   r4  r   r)  ÚdictÚappendr�   r5  ÚstagesÚnum_featuresÚhead_hidden_sizeÚnormr  Ú	head_dropÚLinearrý   ÚheadÚdistÚ	head_distÚinit_weightsÚdistilled_training)rQ   r@  rA  rB  rC  rD  rE  rF  rG   rG  rl   rH  rI  rJ  rK  r  r0  rL  rK   rL   rR   Úprev_dimÚ
num_stagesÚdprr]  ÚiÚcurr_resolutionÚstagerB   rS   s                               @€rT   rN   zEfficientFormerV2.__init__ƒ  s3  øø€ õ, 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆØ˜kÐ)Ð)Ð)Ð)Ø&ˆÔØ ˆŒØ&ˆÔØˆÔÝ˜XÑ&Ô&ˆÝ�^¨JÑ7Ô7¸XÐFÑFÔFˆ
Ý! )Ñ,Ô,ˆ	å˜( J¨q¤MÐd¸YÐS]ÐdÐdÐacÐdÐdˆŒ	Ø˜a”=ˆØˆå˜‘[”[ˆ
Ý'¨¸È$ÐOÑOÔOˆØ!ÐK X°½3¸v¹;¼;È¹?Ñ0KÑ%KˆØ*•Y˜zÑ*Ô*¨:Ñ6Ô6ˆ
ØˆÝ�zÑ"Ô"ð 	!ð 	!ˆAÝ#Ð$MÐ$MÐ$MÐ$MÀHÐ$MÑ$MÔ$MÑNÔNˆOÝ*ØØ˜1”ðð ˜Q”iØ*Ø& qœ>Ø"# q¢& &˜Q˜Q¨dØ$%¨¢FØ  AšvØØ$ Qœ-Ø(Ø˜aœ&Ø'=Ø#Ø%ðð ð  ð!ð ˆEð$ ˜1Œ~ð Ø˜!‘�Ø! !”}ˆHØÐÔ¥$¨xÀ6ÐR_Ð\]ÐR_ÐR_Ð"`Ñ"`Ô"`Ð!aÑaÐÔØ�MŠM˜%Ñ Ô Ð Ð Ý”m VÐ,ˆŒð 5?¸r´NÐBˆÔ˜DÔ1Ø�J˜z¨"œ~Ð4Ð4°Ð4Ð4ˆŒ	Ýœ IÑ.Ô.ˆŒØDOÐRSÂOÀO•B”I˜j¨œn¨kÐ@Ð@¸RÐ@Ð@Ð@ÕY[ÔYdÑYfÔYfˆŒ	Ø ˆŒ	ØŒ9ð 	"ØMXÐ[\Ê_È_�RœY z°"¤~°{ÐIÐIÀbÐIÐIÐIÕbdÔbmÑboÔboˆDŒNˆNà!ˆDŒNð 	×Ò eÐÑ,Ô,Ð,à"'ˆÔÐÐrU   rW  c                 ó  — t          |t          j        ¦  «        rFt          |j        d¬¦  «         |j        �'t          j                             |j        d¦  «         d S d S |r&t          |d¦  «        r| 	                    ¦   «          d S d S d S )Nç{®Gáz”?)Ústdr   r–   )
Ú
isinstancer�   rb  r   ÚweightrF   r¡   Ú	constant_Úhasattrr–   )rQ   ÚmrW  s      rT   Ú_init_weightszEfficientFormerV2._init_weightsØ  s¡   € Ý�a�œÑ#Ô#ð 	!Ý˜!œ(¨Ð,Ñ,Ô,Ð,ØŒvÐ!Ý”×!Ò! !¤&¨!Ñ,Ô,Ð,Ð,Ð,ð "Ð!àð 	!�W QÐ(:Ñ;Ô;ð 	!Ø×ÒÑ Ô Ð Ð Ð ð	!ð 	!ð 	!ð 	!rU   c                 óX   — |                       t          | j        |¬¦  «        ¦  «         d S )NrV  )Úapplyr   rv  )rQ   rW  s     rT   rf  zEfficientFormerV2.init_weightsà  s*   € Ø�
Š
•7˜4Ô-¸;ÐGÑGÔGÑHÔHÐHÐHÐHrU   c                 ó>   — d„ |                       ¦   «         D ¦   «         S )Nc                 ó    — h | ]\  }}d |v ¯	|’ŒS )r“   rp   )rt   rˆ   Ú_s      rT   ú	<setcomp>z4EfficientFormerV2.no_weight_decay.<locals>.<setcomp>å  s'   € ÐQÐQÐQ‘d�a˜Ð9KÈqÐ9PÐ9P�Ð9PÐ9PÐ9PrU   )Únamed_parametersr¤   s    rT   Úno_weight_decayz!EfficientFormerV2.no_weight_decayã  s"   € àQÐQ˜d×3Ò3Ñ5Ô5ÐQÑQÔQÐQrU   Fc                 ó,   — t          dddg¬¦  «        }|S )Nz^stem)z^stages\.(\d+)N)z^norm)iŸ† )rY  r6  )r[  )rQ   ÚcoarseÚmatchers      rT   Úgroup_matcherzEfficientFormerV2.group_matcherç  s)   € åØØ-Ð/CÐDð
ñ 
ô 
ˆð ˆrU   c                 ó(   — | j         D ]	}||_        Œ
d S rW   )r]  r3  )rQ   ÚenablerR  s      rT   Úset_grad_checkpointingz(EfficientFormerV2.set_grad_checkpointingï  s(   € à”ð 	*ð 	*ˆAØ#)ˆAÔ Ð ð	*ð 	*rU   r�   c                 ó   — | j         | j        fS rW   ©rc  re  r¤   s    rT   Úget_classifierz EfficientFormerV2.get_classifierô  s   € àŒy˜$œ.Ð(Ð(rU   c                 ó˜  — t          | ¦  «        }|| _        |�|| _        |dk    rt          j        | j        |fi |¤Žnt          j        ¦   «         | _        | j                             | j	        ¦  «         |dk    rt          j        | j        |fi |¤Žnt          j        ¦   «         | _
        | j
                             | j	        ¦  «         d S )Nr   )r   rH  rC  r�   rb  r^  rý   rc  rš   r¾   re  )rQ   rH  rC  rR   s       rT   Úreset_classifierz"EfficientFormerV2.reset_classifierø  sÅ   € Ý˜dÑ#Ô#ˆØ&ˆÔØÐ"Ø*ˆDÔØGRÐUVÂÀ•B”I˜dÔ/°ÐCÐCÀÐCÐCÐCÕ\^Ô\gÑ\iÔ\iˆŒ	ØŒ	�Š˜œÑ&Ô&Ð&ØLWÐZ[ÊOÈO�œ 4Ô#4°kÐHÐHÀRÐHÐHÐHÕacÔalÑanÔanˆŒØŒ×Ò˜Tœ]Ñ+Ô+Ð+Ð+Ð+rU   c                 ó   — || _         d S rW   )rg  )rQ   r„  s     rT   Úset_distilled_trainingz(EfficientFormerV2.set_distilled_training  s   € à"(ˆÔÐÐrU   ÚNCHWrY   Úindicesr`  Ú
stop_earlyÚ
output_fmtÚintermediates_onlyc                 óZ  — |dv s
J d¦   «         ‚g }t          t          | j        ¦  «        |¦  «        \  }}	|                      |¦  «        }t          | j        ¦  «        dz
  }
t          j                             ¦   «         s|s| j        }n| j        d|	dz   …         }t          |¦  «        D ]^\  }} ||¦  «        }||v rJ||
k    r/|r|                      |¦  «        n|}| 	                    |¦  «         ŒI| 	                    |¦  «         Œ_|r|S ||
k    r|                      |¦  «        }||fS )aï   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )r�  zOutput shape must be NCHW.r   N)
r   rZ  r]  rY  r�   r¼   r;  Ú	enumerater`  r\  )rQ   rY   rŽ  r`  r�  r�  r‘  ÚintermediatesÚtake_indicesÚ	max_indexÚlast_idxr]  Úfeat_idxrm  Úx_inters                  rT   Úforward_intermediatesz'EfficientFormerV2.forward_intermediates  sU  € ð* ˜YÐ&Ð&Ð&Ð(DÑ&Ô&Ð&ØˆÝ"6µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�ið �IŠI�a‰LŒLˆå�t”{Ñ#Ô# aÑ'ˆÝŒ9×!Ò!Ñ#Ô#ð 	1¨:ð 	1Ø”[ˆFˆFà”[  )¨a¡- Ô0ˆFå(¨Ñ0Ô0ð 	,ð 	,‰OˆH�eØ��a‘”ˆAØ˜<Ð'Ð'Ø˜xÒ'Ð'Ø.2Ð9˜dŸiši¨™lœl˜l¸�GØ!×(Ò(¨Ñ1Ô1Ð1Ð1à!×(Ò(¨Ñ+Ô+Ð+øàð 	!Ø Ð à�xÒÐØ—	’	˜!‘”ˆAà�-ÐÐrU   r   Ú
prune_normÚ
prune_headc                 óâ   — t          t          | j        ¦  «        |¦  «        \  }}| j        d|dz   …         | _        |rt          j        ¦   «         | _        |r|                      dd¦  «         |S )z@ Prune layers not required for specified intermediates.
        Nr   r   r=   )r   rZ  r]  r�   rý   r`  rŠ  )rQ   rŽ  r›  rœ  r•  r–  s         rT   Úprune_intermediate_layersz+EfficientFormerV2.prune_intermediate_layers9  sq   € õ #7µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�iØ”k . 9¨q¡= .Ô1ˆŒØð 	&Ýœ™œˆDŒIØð 	)Ø×!Ò! ! RÑ(Ô(Ð(ØÐrU   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S rW   )rY  r]  r`  rX   s     rT   Úforward_featuresz"EfficientFormerV2.forward_featuresI  s4   € Ø�IŠI�a‰LŒLˆØ�KŠK˜‰NŒNˆØ�IŠI�a‰LŒLˆØˆrU   Ú
pre_logitsc                 ó:  — | j         dk    r|                     d¬¦  «        }|                      |¦  «        }|r|S |                      |¦  «        |                      |¦  «        }}| j        r)| j        r"t          j         	                    ¦   «         s||fS ||z   dz  S )Nr=  )r:   r7   rÆ   r:   )
rC  Úmeanra  rc  re  rg  r¾   r�   r¼   r;  )rQ   rY   r¡  Úx_dists       rT   Úforward_headzEfficientFormerV2.forward_headO  s¤   € ØÔ˜uÒ$Ð$Ø—’˜6�Ñ"Ô"ˆAØ�NŠN˜1ÑÔˆØð 	ØˆHØ—I’I˜a‘L”L $§.¢.°Ñ"3Ô"3ˆ6ˆØÔ"ð 	$ t¤}ð 	$½U¼Y×=SÒ=SÑ=UÔ=Uð 	$à�f�9Ðð ˜‘J !Ñ#Ð#rU   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S rW   )r   r¥  rX   s     rT   rZ   zEfficientFormerV2.forward]  s-   € Ø×!Ò! !Ñ$Ô$ˆØ×Ò˜aÑ Ô ˆØˆrU   )r7   r*   r=  NNr4   r>   r  r>  r?  r  r  r  r  r   TNNrÒ   rP  rW   )NFFr�  F)r   FT) r[   r\   r]   r   r^   r	   r_   r   r`   r  rN   rv  rf  r�   r¼   Úignorer~  r‚  r…  r�   r×   rˆ  rŠ  rŒ  rÔ   r   rš  rž  r   r¥  rZ   ra   rb   s   @rT   r    r    ‚  s  ø€ € € € € ð Ø47Ø$Ø48Ø6:ØYZØ+Ø"Ø#Ø#Ø!Ø$&Ø$&Ø6:ØØ!%ØØð)S(ð S(à˜#˜s˜(”OðS(ð ðS(ð ˜C  s¨C x¤Ð0Ô1ð	S(ð
 ðS(ð !  s¨C x¤Ô1ðS(ð " %¨¨c¨	Ô"2Ô3ðS(ð ˜e U¨5°#¨:Ô%6¸¸eÀEÈ3ÀJÔ>OÐQTÐ>TÔ8UÐUÔVðS(ð ðS(ð ðS(ð ðS(ð ðS(ð ðS(ð "ðS(ð "ðS(ð  %-¨U¤Oð!S(ð" ð#S(ð$ ð%S(ð S(ð S(ð S(ð S(ð S(ðj!ð !¨Dð !ð !ð !ð !ðIð I¨ð Ið Ið Ið Ið „YÔðRð Rñ ÔðRð „YÔðð ð ñ Ôðð „YÔð*ð *ð *ñ Ôð*ð „YÔð) ¤	ð )ð )ð )ñ Ôð)ð,ð ,¨Cð ,¸hÀs¼mð ,ð ,ð ,ð ,ð „YÔð)ð )ð )ñ Ôð)ð 8<ØØ$Ø$Ø',ð1 ð 1 àŒ|ð1 ð ˜e C¨¨c¬ NÔ3Ô4ð1 ð ð	1 ð
 ð1 ð ð1 ð !%ð1 ð 
ˆt�E”LÔ! 5¨¬°t¸E¼LÔ7IÐ)IÔ#JÐJÔ	Kð1 ð 1 ð 1 ð 1 ðj ./Ø$Ø#ð	ð à˜3  S¤	˜>Ô*ðð ðð ð	ð ð ð ð ð ð ð$ð $¨$ð $ð $ð $ð $ðð ð ð ð ð ð rU   r=   c                 ó8   — | ddd dddt           t          dddd	œ|¥S )
Nr?  )r7   r*   r*   Tgffffffî?Úbicubicr‡  zstem.conv1.convz
apache-2.0)ÚurlrH  Ú
input_sizeÚ	pool_sizeÚfixed_input_sizeÚcrop_pctÚinterpolationr£  rp  Ú
classifierÚ
first_convÚlicenser
   )rª  Úkwargss     rT   Ú_cfgr´  c  s<   € àØ¨=ÀtÐaeØ¨)Ý%Õ.BØ+Ð;LØðð ð ðð rU   ztimm/)Ú	hf_hub_id)z#efficientformerv2_s0.snap_dist_in1kz#efficientformerv2_s1.snap_dist_in1kz#efficientformerv2_s2.snap_dist_in1kz"efficientformerv2_l.snap_dist_in1kFc                 ó|   — |                      dd¦  «        }t          t          | |fdt          d|¬¦  «        i|¤Ž}|S )NÚout_indices)r   r   r:   r7   Úfeature_cfgT)Úflatten_sequentialr·  )Úpopr   r    r[  )ÚvariantÚ
pretrainedr³  r·  Úmodels        rT   Ú_create_efficientformerv2r¾    sV   € Ø—*’*˜]¨LÑ9Ô9€KÝ Ý˜7 Jðð å¨D¸kÐJÑJÔJðð ðð €Eð €LrU   r�   c           	      óž   — t          t          d         t          d         ddt          d         ¬¦  «        }t	          dd| it          |fi |¤Ž¤ŽS )Nr0   r:   r  ©r@  rD  r0  rK  rF  Úefficientformerv2_s0r¼  )rÁ  ©r[  ÚEfficientFormer_depthÚEfficientFormer_widthÚ EfficientFormer_expansion_ratiosr¾  ©r¼  r³  Ú
model_argss      rT   rÁ  rÁ  ˆ  ób   € åÝ$ TÔ*Ý(¨Ô.ØØÝ3°DÔ9ðñ ô €Jõ %ÐqÐqÈ
ÐqÕVZÐ[eÐVpÐVpÐioÐVpÐVpÐqÐqÐqrU   c           	      óž   — t          t          d         t          d         ddt          d         ¬¦  «        }t	          dd| it          |fi |¤Ž¤ŽS )Nr/   r:   r  rÀ  Úefficientformerv2_s1r¼  )rÊ  rÂ  rÆ  s      rT   rÊ  rÊ  ”  rÈ  rU   c           	      óž   — t          t          d         t          d         ddt          d         ¬¦  «        }t	          dd| it          |fi |¤Ž¤ŽS )Nr.   r4   ro  rÀ  Úefficientformerv2_s2r¼  )rÌ  rÂ  rÆ  s      rT   rÌ  rÌ     sb   € åÝ$ TÔ*Ý(¨Ô.ØØÝ3°DÔ9ðñ ô €Jõ %ÐqÐqÈ
ÐqÕVZÐ[eÐVpÐVpÐioÐVpÐVpÐqÐqÐqrU   c           	      óž   — t          t          d         t          d         ddt          d         ¬¦  «        }t	          dd| it          |fi |¤Ž¤ŽS )Nr-   r9   gš™™™™™¹?rÀ  Úefficientformerv2_lr¼  )rÎ  rÂ  rÆ  s      rT   rÎ  rÎ  ¬  sb   € åÝ$ SÔ)Ý(¨Ô-ØØÝ3°CÔ8ðñ ô €Jõ %ÐpÐpÀzÐpÕUYÐZdÐUoÐUoÐhnÐUoÐUoÐpÐpÐprU   )r=   rP  )?r  rr   Ú	functoolsr   Útypingr   r   r   r   r   r	   r�   Útorch.nnr�   Ú	timm.datar   r   Útimm.layersr   r   r   r   r   r   r   r   r   r   r   r   r   Ú_builderr   Ú	_featuresr   Ú_manipulater   Ú	_registryr   r   Ú__all__rÄ  rÃ  rÅ  r×   r<   rd   rÚ   rå   rø   r  r  r5  r$  r)  r    r´  Údefault_cfgsr¾  rÁ  rÊ  rÌ  rÎ  rp   rU   rT   ú<module>rÚ     s8  ððð ð  €€€Ø Ð Ð Ð Ð Ð Ø ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;à €€€Ø Ð Ð Ð Ð Ð à AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð +Ð *Ð *Ð *Ð *Ð *Ø +Ð +Ð +Ð +Ð +Ð +Ø 'Ð 'Ð 'Ð 'Ð 'Ð 'Ø <Ð <Ð <Ð <Ð <Ð <Ð <Ð <ð Ð
€ð 
Ø
Ø
Ø
ð	ð Ð ð 
Ø
Ø
Ø
ð	ð Ð ð 
_Ø
PØ
AØ
2ð	$ð $Ð  ð#ð #ð #ð #ð #ˆrŒyñ #ô #ð #ðL{ð {ð {ð {ð {�%”(”/ñ {ô {ð {ð|ð ð ð ð �u”x”ñ ô ð ð,qð qð qð qð q˜EœHœOñ qô qð qðh/ð /ð /ð /ð /�”ñ /ô /ð /ðd6ð 6ð 6ð 6ð 6�b”iñ 6ô 6ð 6ðr4ð 4ð 4ð 4ð 4˜RœYñ 4ô 4ð 4ðn!
ð !
ð !
ð !
ð !
ˆBŒMñ !
ô !
ð !
ðHEð Eð Eð Eð E˜RœYñ Eô Eð EðP^ð ^ð ^ð ^ð ^˜œ	ñ ^ô ^ð ^ðB	ð 	ð 	ð 	ð %Ð$Ø+/¨4Øð,ñ ,ô ,ð ,0¨4Øð,ñ ,ô ,ð ,0¨4Øð,ñ ,ô ,ð +/¨$Øð+ñ +ô +ð&ð &ñ ô €ð ð ð ð ð ðrð rÐ8Ið rð rð rñ „ðrð ðrð rÐ8Ið rð rð rñ „ðrð ðrð rÐ8Ið rð rð rñ „ðrð ðqð qÐ7Hð qð qð qñ „ðqð qð qrU   