§
    ·¯¸j.¡  ã                   óü  — d Z ddlZddlmZ ddlmZ ddlmZmZ ddl	m
Z
mZmZmZmZmZ ddlZddlmZ ddlmZmZ dd	lmZmZmZmZmZmZmZ d
dlmZ d
dlm Z  d
dl!m"Z" d
dl#m$Z$ d
dl%m&Z&m'Z' ddgZ(e G d„ d¦  «        ¦   «         Z)d„ Z* G d„ dej+        ¦  «        Z,e"	 dVdee-         de.deej/        eej/                 f         fd„¦   «         Z0e"	 dWde-deej/                 deej/        ee-         f         fd„¦   «         Z1e"dej/        dej/        de.d ee-         d!ee-         d"ej/        d#ej/        fd$„¦   «         Z2 G d%„ d&ej+        ¦  «        Z3 G d'„ d(ej+        ¦  «        Z4 G d)„ d*ej+        ¦  «        Z5 G d+„ d,ej+        ¦  «        Z6 G d-„ dej+        ¦  «        Z7d.„ Z8 e9 e)d/¬0¦  «         e)d1¬0¦  «         e)d2¬0¦  «         e)d3d4d5d6¬7¦  «         e)d1d¬8¦  «         e)d2d¬8¦  «         e)d3d4d5dd¬9¦  «         e)d:d;d<dd¬9¦  «        ¬=¦  «        Z:dXd>„Z;dYd@„Z< e' e<dAdB¬C¦  «         e<dDdB¬C¦  «         e<dEdB¬C¦  «         e<dFdB¬C¦  «         e<d?¬G¦  «         e<dHdBdI¬J¦  «         e<dKdBdI¬J¦  «         e<dLdBdI¬J¦  «        dMœ¦  «        Z=e&dZde7fdN„¦   «         Z>e&dZde7fdO„¦   «         Z?e&dZde7fdP„¦   «         Z@e&dZde7fdQ„¦   «         ZAe&dZde7fdR„¦   «         ZBe&dZde7fdS„¦   «         ZCe&dZde7fdT„¦   «         ZDe&dZde7fdU„¦   «         ZEdS )[a   Multi-Scale Vision Transformer v2

@inproceedings{li2021improved,
  title={MViTv2: Improved multiscale vision transformers for classification and detection},
  author={Li, Yanghao and Wu, Chao-Yuan and Fan, Haoqi and Mangalam, Karttikeya and Xiong, Bo and Malik, Jitendra and Feichtenhofer, Christoph},
  booktitle={CVPR},
  year={2022}
}

Code adapted from original Apache 2.0 licensed impl at https://github.com/facebookresearch/mvit
Original copyright below.

Modifications and timm support by / Copyright 2022, Ross Wightman
é    N)ÚOrderedDict)Ú	dataclass)ÚpartialÚreduce)ÚUnionÚListÚTupleÚOptionalÚAnyÚType)Únn©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)ÚMlpÚDropPathÚcalculate_drop_path_ratesÚtrunc_normal_tf_Úget_norm_layerÚ	to_2tupleÚget_device_dtypeé   )Úbuild_model_with_cfg)Úfeature_take_indices)Úregister_notrace_function)Ú
checkpoint)Úregister_modelÚgenerate_default_cfgsÚMultiScaleVitÚMultiScaleVitCfgc                   ó”  — e Zd ZU dZeedf         ed<   dZeeeedf         f         ed<   dZ	eeeedf         f         ed<   dZ
eed	<   d
Zeed<   dZeed<   dZeed<   d
Zeed<   d
Zeed<   dZeed<   dZeed<   dZeeef         ed<   dZeeeeef                           ed<   dZeeeeef                           ed<   dZeeeef                  ed<   dZeeef         ed<   dZeeef         ed<   dZeeef         ed<   d Zeed!<   d"Zeed#<   d$Zeeeeef         f         ed%<   d&Z eeeeef         f         ed'<   d(Z!eed)<   d*„ Z"dS )+r    ©é   é   é   r$   .Údepthsé`   Ú	embed_dimr   Ú	num_headsç      @Ú	mlp_ratioFÚ
pool_firstTÚexpand_attnÚqkv_biasÚuse_cls_tokenÚuse_abs_posÚresidual_poolingÚconvÚmode©r$   r$   Ú
kernel_qkv)©r   r   ©r#   r#   r7   r7   Ústride_qNÚ	stride_kv©é   r;   Ústride_kv_adaptive©é   r>   Úpatch_kernelÚpatch_strideÚpatch_paddingÚmaxÚ	pool_typeÚspatialÚrel_pos_typeÚgeluÚ	act_layerÚ	layernormÚ
norm_layerg�íµ ÷Æ°>Únorm_epsc                 ó*  ‡ ‡‡— t          ‰ j        ¦  «        }t          ‰ j        t          t
          f¦  «        s-t	          ˆ fd„t          |¦  «        D ¦   «         ¦  «        ‰ _        t          ‰ j        ¦  «        |k    sJ ‚t          ‰ j        t          t
          f¦  «        s-t	          ˆ fd„t          |¦  «        D ¦   «         ¦  «        ‰ _        t          ‰ j        ¦  «        |k    sJ ‚‰ j        �¢‰ j	        €�‰ j        Šg }t          |¦  «        D ]lŠt          ‰ j        ‰         ¦  «        dk    r*ˆˆˆ fd„t          t          ‰¦  «        ¦  «        D ¦   «         Š|                     t	          ‰¦  «        ¦  «         Œmt	          |¦  «        ‰ _	        d S d S d S )Nc              3   ó2   •K  — | ]}‰j         d |z  z  V — ŒdS ©r#   N)r(   ©Ú.0ÚiÚselfs     €úb/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/models/mvitv2.pyú	<genexpr>z1MultiScaleVitCfg.__post_init__.<locals>.<genexpr>A   ó.   øè è € Ð"VÐ"V¸q 4¤>°A¸±FÑ#:Ð"VÐ"VÐ"VÐ"VÐ"VÐ"Vó    c              3   ó2   •K  — | ]}‰j         d |z  z  V — ŒdS rM   )r)   rN   s     €rR   rS   z1MultiScaleVitCfg.__post_init__.<locals>.<genexpr>E   rT   rU   r   c                 ód   •— g | ],}t          ‰|         ‰j        ‰         |         z  d ¦  «        ‘Œ-S ©r   )rB   r8   )rO   ÚdÚ
_stride_kvrP   rQ   s     €€€rR   ú
<listcomp>z2MultiScaleVitCfg.__post_init__.<locals>.<listcomp>M   sH   ø€ ð "ð "ð "àõ ˜J qœM¨T¬]¸1Ô-=¸aÔ-@Ñ@À!ÑDÔDð"ð "ð "rU   )Úlenr&   Ú
isinstancer(   ÚtupleÚlistÚranger)   r<   r9   Úminr8   Úappend)rQ   Ú
num_stagesÚpool_kv_striderZ   rP   s   `  @@rR   Ú__post_init__zMultiScaleVitCfg.__post_init__>   s¤  øøø€ Ý˜œÑ%Ô%ˆ
Ý˜$œ.­5µ$¨-Ñ8Ô8ð 	WÝ"Ð"VÐ"VÐ"VÐ"VÅEÈ*ÑDUÔDUÐ"VÑ"VÔ"VÑVÔVˆDŒNÝ�4”>Ñ"Ô" jÒ0Ð0Ð0Ð0å˜$œ.­5µ$¨-Ñ8Ô8ð 	WÝ"Ð"VÐ"VÐ"VÐ"VÅEÈ*ÑDUÔDUÐ"VÑ"VÔ"VÑVÔVˆDŒNÝ�4”>Ñ"Ô" jÒ0Ð0Ð0Ð0àÔ"Ð.°4´>Ð3IØÔ0ˆJØˆNÝ˜:Ñ&Ô&ð 9ð 9�Ý�t”} QÔ'Ñ(Ô(¨1Ò,Ð,ð"ð "ð "ð "ð "ð "å!&¥s¨:¡¤Ñ!7Ô!7ð"ñ "ô "�Jð ×%Ò%¥e¨JÑ&7Ô&7Ñ8Ô8Ð8Ð8Ý" >Ñ2Ô2ˆDŒNˆNˆNð /Ð.Ð3IÐ3IrU   )#Ú__name__Ú
__module__Ú__qualname__r&   r	   ÚintÚ__annotations__r(   r   r)   r+   Úfloatr,   Úboolr-   r.   r/   r0   r1   r3   Ústrr5   r8   r
   r9   r<   r?   r@   rA   rC   rE   rG   rI   rJ   re   © rU   rR   r    r    $   s8  € € € € € € à+€FˆE�#�s�(ŒOÐ+Ð+Ñ+Ø-/€Iˆu�S˜%  S œ/Ð)Ô*Ð/Ð/Ñ/Ø-.€Iˆu�S˜%  S œ/Ð)Ô*Ð.Ð.Ñ.Ø€IˆuÐÐÑØ€J�ÐÐÑØ€K�ÐÐÑØ€HˆdÐÐÑØ€M�4ÐÐÑØ€K�ÐÐÑØ!Ð�dÐ!Ð!Ñ!Ø€Dˆ#ÐÐÑØ"(€J��c˜3�h”Ð(Ð(Ñ(Ø1Q€Hˆh�u˜U 3¨ 8œ_Ô-Ô.ÐQÐQÑQØ26€Iˆx˜˜e C¨ HœoÔ.Ô/Ð6Ð6Ñ6Ø4:Ð˜  s¨C x¤Ô1Ð:Ð:Ñ:Ø$*€L�%˜˜S˜”/Ð*Ð*Ñ*Ø$*€L�%˜˜S˜”/Ð*Ð*Ñ*Ø%+€M�5˜˜c˜”?Ð+Ð+Ñ+Ø€IˆsÐÐÑØ!€L�#Ð!Ð!Ñ!Ø-3€Iˆu�S˜%  S œ/Ð)Ô*Ð3Ð3Ñ3Ø.9€J��c˜5  c œ?Ð*Ô+Ð9Ð9Ñ9Ø€HˆeÐÐÑð3ð 3ð 3ð 3ð 3rU   c                 ó8   — t          t          j        | d¦  «        S )Nr   )r   ÚoperatorÚmul)Úiterables    rR   Úprodrs   U   s   € Ý•(”, ¨!Ñ,Ô,Ð,rU   c                   óª   ‡ — e Zd ZdZ	 	 	 	 	 	 	 dded	ed
eeef         deeef         deeef         f
ˆ fd„Zdeej        e	e         f         fd„Z
ˆ xZS )Ú
PatchEmbedz
    PatchEmbed.
    r$   é   r=   r:   r4   NÚdim_inÚdim_outÚkernelÚstrideÚpaddingc                 óŠ   •— t          ¦   «                              ¦   «          ||dœ}t          j        ||f|||dœ|¤Ž| _        d S )N©ÚdeviceÚdtype)Úkernel_sizerz   r{   )ÚsuperÚ__init__r   ÚConv2dÚproj)
rQ   rw   rx   ry   rz   r{   r~   r   ÚddÚ	__class__s
            €rR   r‚   zPatchEmbed.__init__^   sd   ø€ õ 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆå”IØØð
ð ØØð
ð 
ð ð
ð 
ˆŒ	ˆ	ˆ	rU   Úreturnc                 óœ   — |                       |¦  «        }|                     d¦  «                             dd¦  «        |j        dd …         fS )Nr#   r   éþÿÿÿ)r„   ÚflattenÚ	transposeÚshape©rQ   Úxs     rR   ÚforwardzPatchEmbed.forwardt   s@   € Ø�IŠI�a‰LŒLˆà�yŠy˜‰|Œ|×%Ò% a¨Ñ+Ô+¨Q¬W°R°S°S¬\Ð9Ð9rU   )r$   rv   r=   r:   r4   NN)rf   rg   rh   Ú__doc__ri   r	   r‚   ÚtorchÚTensorr   r�   Ú__classcell__©r†   s   @rR   ru   ru   Y   sÖ   ø€ € € € € ðð ð ØØ&,Ø&,Ø'-ØØð
ð 
àð
ð ð
ð ˜#˜s˜(”Oð	
ð
 ˜#˜s˜(”Oð
ð ˜3 ˜8”_ð
ð 
ð 
ð 
ð 
ð 
ð,:˜E %¤,°°S´	Ð"9Ô:ð :ð :ð :ð :ð :ð :ð :ð :rU   ru   TÚ	feat_sizeÚhas_cls_tokenr‡   c                 ó  — |\  }}|r)| d d …d d …d d…d d …f         | d d …d d …dd …d d …f         } }nd }|                       d||| j        d         ¦  «                             dddd¦  «                             ¦   «         } | |fS )Nr   éÿÿÿÿr   r$   r#   )ÚreshaperŒ   ÚpermuteÚ
contiguous)rŽ   r•   r–   ÚHÚWÚcls_toks         rR   Úreshape_pre_poolrŸ   z   s¤   € ð �D€A€qØð Ø�q�q�q˜!˜!˜!˜R˜a˜R   �{”^ Q q q q¨!¨!¨!¨Q¨R¨R°°° {¤^�ˆˆàˆØ	�	Š	�"�a˜˜AœG BœKÑ(Ô(×0Ò0°°A°q¸!Ñ<Ô<×GÒGÑIÔI€AØˆgˆ:ÐrU   r)   rž   c                 ó  — | j         d         | j         d         g}| j         d         | j         d         z  }|                      d|| j         d         |¦  «                             dd¦  «        } |�t          j        || fd¬¦  «        } | |fS )Nr#   r$   r˜   r   ©Údim)rŒ   r™   r‹   r‘   Úcat)rŽ   r)   rž   r•   ÚL_pooleds        rR   Úreshape_post_poolr¥   ‰   s…   € ð ”˜”˜QœW QœZÐ(€IØŒw�qŒz˜AœG AœJÑ&€HØ	�	Š	�"�i ¤¨¤¨XÑ6Ô6×@Ò@ÀÀAÑFÔF€AØÐÝŒI�w �l¨Ð*Ñ*Ô*ˆØˆiˆ<ÐrU   ÚattnÚqÚq_sizeÚk_sizeÚ	rel_pos_hÚ	rel_pos_wc                 óü  — |rdnd}|\  }}	|\  }
}t          |
|z  d¦  «        }t          ||
z  d¦  «        }t          j        ||j        t          j        ¬¦  «                             d¦  «        |z  t          j        |
|j        t          j        ¬¦  «                             d¦  «        |z  z
  }||
dz
  |z  z  }t          ||	z  d¦  «        }t          |	|z  d¦  «        }t          j        |	|j        t          j        ¬¦  «                             d¦  «        |z  t          j        ||j        t          j        ¬¦  «                             d¦  «        |z  z
  }||dz
  |z  z  }||                     ¦   «                  }||                     ¦   «                  }|j        \  }}}}|dd…dd…|d…f                              ||||	|¦  «        }t          j        d||¦  «        }t          j        d||¦  «        }| dd…dd…|d…|d…f          	                    |d||	|
|¦  «        |                     d¦  «        z   |                     d	¦  «        z    	                    |d||	z  |
|z  ¦  «        | dd…dd…|d…|d…f<   | S )
z1
    Spatial Relative Positional Embeddings.
    r   r   g      ð?r}   r˜   Nzbyhwc,hkc->byhwkzbyhwc,wkc->byhwkr‰   )
rB   r‘   Úaranger~   ÚlongÚ	unsqueezerŒ   r™   ÚeinsumÚview)r¦   r§   r–   r¨   r©   rª   r«   Úsp_idxÚq_hÚq_wÚk_hÚk_wÚ	q_h_ratioÚ	k_h_ratioÚdist_hÚ	q_w_ratioÚ	k_w_ratioÚdist_wÚrel_hÚrel_wÚBÚn_headÚq_Nr¢   Úr_qs                            rR   Úcal_rel_pos_typerÃ   —   s–  € ð  Ð&ˆQˆQ Q€FØ�H€CˆØ�H€Cˆõ �C˜#‘I˜sÑ#Ô#€IÝ�C˜#‘I˜sÑ#Ô#€IåŒ�S ¤µ´Ð<Ñ<Ô<×FÒFÀrÑJÔJÈYÑVÝŒ�S ¤µ´Ð<Ñ<Ô<×FÒFÀqÑIÔIÈIÑUñ	Vð ð ˆs�Q‰w˜)Ñ#Ñ#€FÝ�C˜#‘I˜sÑ#Ô#€IÝ�C˜#‘I˜sÑ#Ô#€IåŒ�S ¤µ´Ð<Ñ<Ô<×FÒFÀrÑJÔJÈYÑVÝŒ�S ¤µ´Ð<Ñ<Ô<×FÒFÀqÑIÔIÈIÑUñ	Vð ð ˆs�Q‰w˜)Ñ#Ñ#€Fà�f—k’k‘m”mÔ$€EØ�f—k’k‘m”mÔ$€Eàœ'Ñ€A€vˆs�Cà
ˆAˆAˆAˆqˆqˆq�&�'�'ˆMÔ
×
"Ò
" 1 f¨c°3¸Ñ
<Ô
<€CÝŒLÐ+¨S°%Ñ8Ô8€EÝŒLÐ+¨S°%Ñ8Ô8€Eð 	ˆQˆQˆQ����6�7�7˜F˜G˜GÐ#Ô$×)Ò)¨!¨R°°c¸3ÀÑDÔDØ
�/Š/˜"Ñ
Ô
ñ	à
�/Š/˜"Ñ
Ô
ñ	÷ ‚dˆ1ˆb�#˜‘)˜S 3™YÑ'Ô'ð	 	ˆˆˆˆAˆAˆAˆvˆwˆw˜˜˜Ð	Ñ ð €KrU   c                   óü   ‡ — e Zd Zddddddddddej        ddfdeded	eeef         d
edededeeef         deeef         deeef         deeef         dededede	ej
                 fˆ fd„Zd	ee         fd„Zˆ xZS )ÚMultiScaleAttentionPoolFirsté   Tr2   r6   rD   Nr¢   rx   r•   r)   r.   r3   Úkernel_qÚ	kernel_kvr8   r9   r–   rE   r1   rI   c                 óB  •— ||dœ}t          ¦   «                              ¦   «          || _        || _        ||z  | _        | j        dz  | _        || _        t          d„ |D ¦   «         ¦  «        }t          d„ |D ¦   «         ¦  «        }t          j	        ||fd|i|¤Ž| _
        t          j	        ||fd|i|¤Ž| _        t          j	        ||fd|i|¤Ž| _        t          j	        ||fi |¤Ž| _        t          |¦  «        dk    rt          |	¦  «        dk    rd }t          |¦  «        dk    rt          |
¦  «        dk    rd }|| _        |dk    | _        d\  | _        | _        | _        d\  | _        | _        | _        |d	v rY|d
k    rt          j        nt          j        }|r |||	|¦  «        | _        |r$ |||
|¦  «        | _         |||
|¦  «        | _        n´|dk    s|dk    r–|dk    r||z  n|}|r,t          j        |||f|	||ddœ|¤Ž| _         ||fi |¤Ž| _        |rXt          j        |||f|
||ddœ|¤Ž| _         ||fi |¤Ž| _        t          j        |||f|
||ddœ|¤Ž| _         ||fi |¤Ž| _        nt5          d|› �¦  «        ‚|| _        | j        dk    rü|d         |d         k    sJ ‚|d         }t9          |	¦  «        dk    r||	d         z  n|}t9          |
¦  «        dk    r||
d         z  n|}dt;          ||¦  «        z  dz
  }t          j        t?          j         || j        fi |¤Ž¦  «        | _!        t          j        t?          j         || j        fi |¤Ž¦  «        | _"        tG          | j!        d¬¦  «         tG          | j"        d¬¦  «         || _$        d S )Nr}   ç      à¿c                 ó2   — g | ]}t          |d z  ¦  «        ‘ŒS ©r#   ©ri   ©rO   r§   s     rR   r[   z9MultiScaleAttentionPoolFirst.__init__.<locals>.<listcomp>å   ó"   € Ð9Ð9Ð9¨1�3˜q A™v™;œ;Ð9Ð9Ð9rU   c                 ó2   — g | ]}t          |d z  ¦  «        ‘ŒS rÌ   rÍ   ©rO   Úkvs     rR   r[   z9MultiScaleAttentionPoolFirst.__init__.<locals>.<listcomp>æ   ó"   € Ð=Ð=Ð=¨R�C  a¡™LœLÐ=Ð=Ð=rU   Úbiasr   Úconv_unshared©NNN©ÚavgrB   rB   r2   F©rz   r{   ÚgroupsrÔ   úUnsupported model rD   r   r#   ç{®Gáz”?©Ústd)%r�   r‚   r)   rx   Úhead_dimÚscaler–   r^   r   ÚLinearr§   ÚkÚvr„   rs   r3   ÚunsharedÚpool_qÚpool_kÚpool_vÚnorm_qÚnorm_kÚnorm_vÚ	MaxPool2dÚ	AvgPool2drƒ   ÚNotImplementedErrorrE   r\   rB   Ú	Parameterr‘   Úzerosrª   r«   r   r1   ©rQ   r¢   rx   r•   r)   r.   r3   rÇ   rÈ   r8   r9   r–   rE   r1   rI   r~   r   r…   Ú	padding_qÚ
padding_kvÚpool_opÚdim_convÚsizer¨   Úkv_sizeÚ
rel_sp_dimr†   s                             €rR   r‚   z%MultiScaleAttentionPoolFirst.__init__Ë   s€  ø€ ð& ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ"ˆŒØˆŒØ 9Ñ,ˆŒØ”] dÑ*ˆŒ
Ø*ˆÔÝÐ9Ð9°Ð9Ñ9Ô9Ñ:Ô:ˆ	ÝÐ=Ð=°9Ð=Ñ=Ô=Ñ>Ô>ˆ
å”˜3 Ð=Ð=¨hÐ=¸"Ð=Ð=ˆŒÝ”˜3 Ð=Ð=¨hÐ=¸"Ð=Ð=ˆŒÝ”˜3 Ð=Ð=¨hÐ=¸"Ð=Ð=ˆŒÝ”I˜g wÐ5Ð5°"Ð5Ð5ˆŒ	õ �‰>Œ>˜QÒÐ¥4¨¡>¤>°QÒ#6Ð#6ØˆHÝ�	‰?Œ?˜aÒÐ¥D¨¡O¤O°qÒ$8Ð$8ØˆIØˆŒ	Ø Ò/ˆŒØ0@Ñ-ˆŒ�T”[ $¤+Ø0@Ñ-ˆŒ�T”[ $¤+Ø�>Ð!Ð!Ø&*¨e¢m m•b”l�l½¼ˆGØð EØ%˜g h°¸)ÑDÔD�”Øð HØ%˜g i°¸JÑGÔG�”Ø%˜g i°¸JÑGÔG�”øØ�VŠ^ˆ^˜t Ò6Ð6Ø+/°6ª>¨>�s˜iÑ'Ð'¸sˆHØð 9Ý œiØØØð	ð $Ø%Ø#Øð	ð 	ð ð	ð 	�”ð )˜j¨Ð8Ð8°RÐ8Ð8�”Øð 9Ý œiØØØð	ð %Ø&Ø#Øð	ð 	ð ð	ð 	�”ð )˜j¨Ð8Ð8°RÐ8Ð8�”Ý œiØØØð	ð %Ø&Ø#Øð	ð 	ð ð	ð 	�”ð )˜j¨Ð8Ð8°RÐ8Ð8�”øå%Ð&A¸4Ð&AÐ&AÑBÔBÐBð )ˆÔØÔ 	Ò)Ð)Ø˜Q”< 9¨Q¤<Ò/Ð/Ð/Ð/Ø˜Q”<ˆDÝ,/°©M¬M¸AÒ,=Ð,=�T˜X aœ[Ñ(Ð(À4ˆFÝ.1°)©n¬n¸qÒ.@Ð.@�d˜i¨œlÑ*Ð*ÀdˆGØ�S ¨Ñ1Ô1Ñ1°AÑ5ˆJåœ\­%¬+°jÀ$Ä-Ð*VÐ*VÐSUÐ*VÐ*VÑWÔWˆDŒNÝœ\­%¬+°jÀ$Ä-Ð*VÐ*VÐSUÐ*VÐ*VÑWÔWˆDŒNÝ˜Tœ^°Ð6Ñ6Ô6Ð6Ý˜Tœ^°Ð6Ñ6Ô6Ð6à 0ˆÔÐÐrU   c           	      óÜ  — |j         \  }}}| j        rdn| j        }|                     |||d¦  «                             dddd¦  «        }|x}x}}	| j        �Ht          ||| j        ¦  «        \  }}
|                      |¦  «        }t          || j        |
¦  «        \  }}n|}| j	        �|  	                    |¦  «        }| j
        �Ht          ||| j        ¦  «        \  }}|  
                    |¦  «        }t          || j        |¦  «        \  }}n|}| j        �|                      |¦  «        }| j        �Ht          |	|| j        ¦  «        \  }	}|                      |	¦  «        }	t          |	| j        |¦  «        \  }	}n|}| j        �|                      |	¦  «        }	|d         |d         z  t          | j        ¦  «        z   }|                     dd¦  «                             ||d¦  «        }|                      |¦  «                             ||| j        d¦  «                             dd¦  «        }|d         |d         z  t          | j        ¦  «        z   }|                     dd¦  «                             ||d¦  «        }|                      |¦  «                             ||| j        d¦  «        }|d         |d         z  t          | j        ¦  «        z   }|	                     dd¦  «                             ||d¦  «        }	|                      |	¦  «                             ||| j        d¦  «                             dd¦  «        }	|| j        z  |z  }| j        dk    r$t+          ||| j        ||| j        | j        ¦  «        }|                     d¬¦  «        }||	z  }| j        r||z   }|                     dd¦  «                             |d| j        ¦  «        }|                      |¦  «        }||fS )Nr   r˜   r   r#   r$   rD   r¡   )rŒ   rä   r)   r™   rš   rå   rŸ   r–   r¥   rè   ræ   ré   rç   rê   ri   r‹   r§   râ   rã   rà   rE   rÃ   rª   r«   Úsoftmaxr1   rx   r„   )rQ   rŽ   r•   r¿   ÚNÚ_Úfold_dimr§   râ   rã   Úq_tokr¨   Úk_tokr©   Úv_tokÚv_sizerÁ   Úk_NÚv_Nr¦   s                       rR   r�   z$MultiScaleAttentionPoolFirst.forward5  s¤  € Ø”'‰ˆˆ1ˆaàœÐ9�1�1¨4¬>ˆØ�IŠI�a˜˜H bÑ)Ô)×1Ò1°!°Q¸¸1Ñ=Ô=ˆØˆˆˆˆA�àŒ;Ð"Ý'¨¨9°dÔ6HÑIÔI‰HˆAˆuØ—’˜A‘”ˆAÝ)¨!¨T¬^¸UÑCÔC‰IˆAˆvˆvàˆFØŒ;Ð"Ø—’˜A‘”ˆAàŒ;Ð"Ý'¨¨9°dÔ6HÑIÔI‰HˆAˆuØ—’˜A‘”ˆAÝ)¨!¨T¬^¸UÑCÔC‰IˆAˆvˆvàˆFØŒ;Ð"Ø—’˜A‘”ˆAàŒ;Ð"Ý'¨¨9°dÔ6HÑIÔI‰HˆAˆuØ—’˜A‘”ˆAÝ)¨!¨T¬^¸UÑCÔC‰IˆAˆvˆvàˆFØŒ;Ð"Ø—’˜A‘”ˆAà�QŒi˜& œ)Ñ#¥c¨$Ô*<Ñ&=Ô&=Ñ=ˆØ�KŠK˜˜1ÑÔ×%Ò% a¨¨bÑ1Ô1ˆØ�FŠF�1‰IŒI×Ò˜a  d¤n°bÑ9Ô9×CÒCÀAÀqÑIÔIˆà�QŒi˜& œ)Ñ#¥c¨$Ô*<Ñ&=Ô&=Ñ=ˆØ�KŠK˜˜1ÑÔ×%Ò% a¨¨bÑ1Ô1ˆØ�FŠF�1‰IŒI×Ò˜a  d¤n°bÑ9Ô9ˆà�QŒi˜& œ)Ñ#¥c¨$Ô*<Ñ&=Ô&=Ñ=ˆØ�KŠK˜˜1ÑÔ×%Ò% a¨¨bÑ1Ô1ˆØ�FŠF�1‰IŒI×Ò˜a  d¤n°bÑ9Ô9×CÒCÀAÀqÑIÔIˆà�D”J‘ !Ñ#ˆØÔ 	Ò)Ð)Ý#ØØØÔ"ØØØ”Ø”ñô ˆDð �|Š| ˆ|Ñ#Ô#ˆØ�1‰HˆàÔ ð 	Ø�A‘ˆAà�KŠK˜˜1ÑÔ×%Ò% a¨¨T¬\Ñ:Ô:ˆØ�IŠI�a‰LŒLˆà�&ˆyÐrU   ©rf   rg   rh   r   Ú	LayerNormri   r	   rl   rm   r   ÚModuler‚   r   r�   r“   r”   s   @rR   rÅ   rÅ   Ê   sc  ø€ € € € € ð Ø!ØØ(.Ø)/Ø(.Ø)/Ø"&Ø )Ø%)Ø*,¬,ØØð#h1ð h1àðh1ð ðh1ð ˜S #˜X”ð	h1ð
 ðh1ð ðh1ð ðh1ð ˜C ˜H”oðh1ð ˜S #˜X”ðh1ð ˜C ˜H”oðh1ð ˜S #˜X”ðh1ð  ðh1ð ðh1ð #ðh1ð ˜RœYœðh1ð h1ð h1ð h1ð h1ð h1ðTB D¨¤Ið Bð Bð Bð Bð Bð Bð Bð BrU   rÅ   c                   óü   ‡ — e Zd Zddddddddddej        ddfdeded	eeef         d
edededeeef         deeef         deeef         deeef         dededede	ej
                 fˆ fd„Zd	ee         fd„Zˆ xZS )ÚMultiScaleAttentionrÆ   Tr2   r6   rD   Nr¢   rx   r•   r)   r.   r3   rÇ   rÈ   r8   r9   r–   rE   r1   rI   c                 óà  •— ||dœ}t          ¦   «                              ¦   «          || _        || _        ||z  | _        | j        dz  | _        || _        t          d„ |D ¦   «         ¦  «        }t          d„ |D ¦   «         ¦  «        }t          j	        ||dz  fd|i|¤Ž| _
        t          j	        ||fi |¤Ž| _        t          |¦  «        dk    rt          |	¦  «        dk    rd }t          |¦  «        dk    rt          |
¦  «        dk    rd }|| _        |dk    | _        d	\  | _        | _        | _        d	\  | _        | _        | _        |d
v rY|dk    rt          j        nt          j        }|r |||	|¦  «        | _        |r$ |||
|¦  «        | _         |||
|¦  «        | _        n´|dk    s|dk    r–|dk    r||z  n|}|r,t          j        |||f|	||ddœ|¤Ž| _         ||fi |¤Ž| _        |rXt          j        |||f|
||ddœ|¤Ž| _         ||fi |¤Ž| _        t          j        |||f|
||ddœ|¤Ž| _         ||fi |¤Ž| _        nt1          d|› �¦  «        ‚|| _        | j        dk    rü|d         |d         k    sJ ‚|d         }t5          |	¦  «        dk    r||	d         z  n|}t5          |
¦  «        dk    r||
d         z  n|}dt7          ||¦  «        z  dz
  }t          j        t;          j        || j        fi |¤Ž¦  «        | _        t          j        t;          j        || j        fi |¤Ž¦  «        | _         tC          | j        d¬¦  «         tC          | j         d¬¦  «         || _"        d S )Nr}   rÊ   c                 ó2   — g | ]}t          |d z  ¦  «        ‘ŒS rÌ   rÍ   rÎ   s     rR   r[   z0MultiScaleAttention.__init__.<locals>.<listcomp>•  rÏ   rU   c                 ó2   — g | ]}t          |d z  ¦  «        ‘ŒS rÌ   rÍ   rÑ   s     rR   r[   z0MultiScaleAttention.__init__.<locals>.<listcomp>–  rÓ   rU   r$   rÔ   r   rÕ   rÖ   r×   rB   r2   FrÙ   rÛ   rD   r   r#   rÜ   rÝ   )#r�   r‚   r)   rx   rß   rà   r–   r^   r   rá   Úqkvr„   rs   r3   rä   rè   ré   rê   rå   ræ   rç   rë   rì   rƒ   rí   rE   r\   rB   rî   r‘   rï   rª   r«   r   r1   rð   s                             €rR   r‚   zMultiScaleAttention.__init__{  sD  ø€ ð& ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ"ˆŒØˆŒØ 9Ñ,ˆŒØ”] dÑ*ˆŒ
Ø*ˆÔÝÐ9Ð9°Ð9Ñ9Ô9Ñ:Ô:ˆ	ÝÐ=Ð=°9Ð=Ñ=Ô=Ñ>Ô>ˆ
å”9˜S '¨A¡+ÐCÐC°HÐCÀÐCÐCˆŒÝ”I˜g wÐ5Ð5°"Ð5Ð5ˆŒ	õ �‰>Œ>˜QÒÐ¥4¨¡>¤>°QÒ#6Ð#6ØˆHÝ�	‰?Œ?˜aÒÐ¥D¨¡O¤O°qÒ$8Ð$8ØˆIØˆŒ	Ø Ò/ˆŒØ0@Ñ-ˆŒ�T”[ $¤+Ø0@Ñ-ˆŒ�T”[ $¤+Ø�>Ð!Ð!Ø&*¨e¢m m•b”l�l½¼ˆGØð EØ%˜g h°¸)ÑDÔD�”Øð HØ%˜g i°¸JÑGÔG�”Ø%˜g i°¸JÑGÔG�”øØ�VŠ^ˆ^˜t Ò6Ð6Ø/3°vª~¨~�w )Ñ+Ð+À7ˆHØð 9Ý œiØØØð	ð $Ø%Ø#Øð	ð 	ð ð	ð 	�”ð )˜j¨Ð8Ð8°RÐ8Ð8�”Øð 9Ý œiØØØð	ð %Ø&Ø#Øð	ð 	ð ð	ð 	�”ð )˜j¨Ð8Ð8°RÐ8Ð8�”Ý œiØØØð	ð %Ø&Ø#Øð	ð 	ð ð	ð 	�”ð )˜j¨Ð8Ð8°RÐ8Ð8�”øå%Ð&A¸4Ð&AÐ&AÑBÔBÐBð )ˆÔØÔ 	Ò)Ð)Ø˜Q”< 9¨Q¤<Ò/Ð/Ð/Ð/Ø˜Q”<ˆDÝ,/°©M¬M¸AÒ,=Ð,=�T˜X aœ[Ñ(Ð(À4ˆFÝ.1°)©n¬n¸qÒ.@Ð.@�d˜i¨œlÑ*Ð*ÀdˆGØ�S ¨Ñ1Ô1Ñ1°AÑ5ˆJåœ\­%¬+°jÀ$Ä-Ð*VÐ*VÐSUÐ*VÐ*VÑWÔWˆDŒNÝœ\­%¬+°jÀ$Ä-Ð*VÐ*VÐSUÐ*VÐ*VÑWÔWˆDŒNÝ˜Tœ^°Ð6Ñ6Ô6Ð6Ý˜Tœ^°Ð6Ñ6Ô6Ð6à 0ˆÔÐÐrU   c           	      óä  — |j         \  }}}|                      |¦  «                             ||d| j        d¦  «                             ddddd¦  «        }|                     d¬¦  «        \  }}}	| j        �Ht          ||| j        ¦  «        \  }}
|                      |¦  «        }t          || j        |
¦  «        \  }}n|}| j
        �|  
                    |¦  «        }| j        �Ht          ||| j        ¦  «        \  }}|                      |¦  «        }t          || j        |¦  «        \  }}n|}| j        �|                      |¦  «        }| j        �Gt          |	|| j        ¦  «        \  }	}|                      |	¦  «        }	t          |	| j        |¦  «        \  }	}| j        �|                      |	¦  «        }	|| j        z  |                     dd¦  «        z  }| j        d	k    r$t%          ||| j        ||| j        | j        ¦  «        }|                     d¬¦  «        }||	z  }| j        r||z   }|                     dd¦  «                             |d| j        ¦  «        }|                      |¦  «        }||fS )
Nr$   r˜   r#   r   r   r;   r¡   r‰   rD   )rŒ   r  r™   r)   rš   Úunbindrå   rŸ   r–   r¥   rè   ræ   ré   rç   rê   rà   r‹   rE   rÃ   rª   r«   rù   r1   rx   r„   )rQ   rŽ   r•   r¿   rú   rû   r  r§   râ   rã   rý   r¨   rþ   r©   rÿ   r¦   s                   rR   r�   zMultiScaleAttention.forwardã  sU  € Ø”'‰ˆˆ1ˆaà�hŠh�q‰kŒk×!Ò! ! Q¨¨4¬>¸2Ñ>Ô>×FÒFÀqÈ!ÈQÐPQÐSTÑUÔUˆØ—*’* �*Ñ#Ô#‰ˆˆ1ˆaàŒ;Ð"Ý'¨¨9°dÔ6HÑIÔI‰HˆAˆuØ—’˜A‘”ˆAÝ)¨!¨T¬^¸UÑCÔC‰IˆAˆvˆvàˆFØŒ;Ð"Ø—’˜A‘”ˆAàŒ;Ð"Ý'¨¨9°dÔ6HÑIÔI‰HˆAˆuØ—’˜A‘”ˆAÝ)¨!¨T¬^¸UÑCÔC‰IˆAˆvˆvàˆFØŒ;Ð"Ø—’˜A‘”ˆAàŒ;Ð"Ý'¨¨9°dÔ6HÑIÔI‰HˆAˆuØ—’˜A‘”ˆAÝ$ Q¨¬¸Ñ>Ô>‰DˆAˆqØŒ;Ð"Ø—’˜A‘”ˆAà�D”J‘ !§+¢+¨b°"Ñ"5Ô"5Ñ5ˆØÔ 	Ò)Ð)Ý#ØØØÔ"ØØØ”Ø”ñô ˆDð �|Š| ˆ|Ñ#Ô#ˆØ�1‰HˆàÔ ð 	Ø�A‘ˆAà�KŠK˜˜1ÑÔ×%Ò% a¨¨T¬\Ñ:Ô:ˆØ�IŠI�a‰LŒLˆà�&ˆyÐrU   r  r”   s   @rR   r  r  z  sZ  ø€ € € € € ð Ø!ØØ(.Ø)/Ø(.Ø)/Ø"&Ø )Ø%)Ø*,¬,ØØð#f1ð f1àðf1ð ðf1ð ˜S #˜X”ð	f1ð
 ðf1ð ðf1ð ðf1ð ˜C ˜H”oðf1ð ˜S #˜X”ðf1ð ˜C ˜H”oðf1ð ˜S #˜X”ðf1ð  ðf1ð ðf1ð #ðf1ð ˜RœYœðf1ð f1ð f1ð f1ð f1ð f1ðP3 D¨¤Ið 3ð 3ð 3ð 3ð 3ð 3ð 3ð 3rU   r  c            %       ó*  ‡ — e Zd Zdddej        ddddddddddddfd	ed
ededeeef         dededede	ej
                 deeef         deeef         deeef         deeef         dedededededef$ˆ fd„Zdee         fd„Zdee         fd„Zˆ xZS )ÚMultiScaleBlockr*   Tç        r6   r2   FrD   Nr¢   rx   r)   r•   r+   r.   Ú	drop_pathrI   rÇ   rÈ   r8   r9   r3   r–   r-   r,   rE   r1   c                 óB  •— ||dœ}t          ¦   «                              ¦   «          ||k    }|| _        || _        || _         ||fi |¤Ž| _        |r|rt          j        ||fi |¤Žnd | _        |rIt          |¦  «        dk    r6d„ |D ¦   «         }|}d„ |D ¦   «         }t          j
        |||¦  «        | _        nd | _        |r|n|}|rt          nt          } |||f||||	|
|||||||dœ|¤Ž| _        |dk    rt          |¦  «        nt          j        ¦   «         | _         ||fi |¤Ž| _        |}|r|st          j        ||fi |¤Žnd | _        t)          d|t+          ||z  ¦  «        |dœ|¤Ž| _        |dk    rt          |¦  «        nt          j        ¦   «         | _        d S )	Nr}   r   c                 ó(   — g | ]}|d k    r|d z   n|‘ŒS rX   rn   )rO   Úss     rR   r[   z,MultiScaleBlock.__init__.<locals>.<listcomp><  s(   € ÐCÐCÐC°Q A¨¢E E˜1˜q™5˜5¨qÐCÐCÐCrU   c                 ó2   — g | ]}t          |d z  ¦  «        ‘ŒS rÌ   rÍ   )rO   Úskips     rR   r[   z,MultiScaleBlock.__init__.<locals>.<listcomp>>  s"   € ÐCÐCÐC¨t�C ¨¡	™NœNÐCÐCÐCrU   )r)   r•   r.   rÇ   rÈ   r8   r9   rI   r–   r3   rE   r1   r  )Úin_featuresÚhidden_featuresÚout_featuresrn   )r�   r‚   r¢   rx   r–   Únorm1r   rá   Úshortcut_proj_attnrs   rë   Úshortcut_pool_attnrÅ   r  r¦   r   ÚIdentityÚ
drop_path1Únorm2Úshortcut_proj_mlpr   ri   ÚmlpÚ
drop_path2)rQ   r¢   rx   r)   r•   r+   r.   r  rI   rÇ   rÈ   r8   r9   r3   r–   r-   r,   rE   r1   r~   r   r…   Úproj_neededÚkernel_skipÚstride_skipÚpadding_skipÚatt_dimÚ
attn_layerÚmlp_dim_outr†   s                                €rR   r‚   zMultiScaleBlock.__init__  s5  ø€ ð. ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ˜W’nˆØˆŒØˆŒØ*ˆÔà�Z Ð*Ð* rÐ*Ð*ˆŒ
àCNÐ"hÐS^Ð"h¥"¤)¨C°Ð"?Ð"?¸BÐ"?Ð"?Ð"?ÐdhˆÔØð 	+�˜X™œ¨Ò*Ð*ØCÐC¸(ÐCÑCÔCˆKØ"ˆKØCÐC°{ÐCÑCÔCˆLÝ&(¤l°;ÀÈ\Ñ&ZÔ&ZˆDÔ#Ð#à&*ˆDÔ#à(Ð1�'�'¨cˆØ5?ÐXÕ1Ð1ÕEXˆ
Ø�JØØð
ð  ØØØØØØØ!Ø'ØØ%Ø-ð
ð 
ð ð
ð 
ˆŒ	ð" 2;¸S²°�( 9Ñ-Ô-Ð-ÅbÄkÁmÄmˆŒà�Z Ð.Ð.¨2Ð.Ð.ˆŒ
ØˆØBMÐ!kÐVaÐ!k¥¤¨3°Ð!>Ð!>¸2Ð!>Ð!>Ð!>ÐgkˆÔÝð 
ØÝ ¨)Ñ 3Ñ4Ô4Ø$ð
ð 
ð ð	
ð 
ˆŒð 2;¸S²°�( 9Ñ-Ô-Ð-ÅbÄkÁmÄmˆŒˆˆrU   c                 óÄ  — | j         €|S | j        r#|d d …d d…d d …f         |d d …dd …d d …f         }}nd }|j        \  }}}|\  }}|                     ||||¦  «                             dddd¦  «                             ¦   «         }|                       |¦  «        }|                     ||d¦  «                             dd¦  «        }|�t          j        ||fd¬¦  «        }|S )Nr   r   r$   r#   r˜   r¡   )	r  r–   rŒ   r™   rš   r›   r‹   r‘   r£   )	rQ   rŽ   r•   rž   r¿   ÚLÚCrœ   r�   s	            rR   Ú_shortcut_poolzMultiScaleBlock._shortcut_poolc  sÿ   € ØÔ"Ð*ØˆHØÔð 	Ø˜1˜1˜1˜b˜q˜b ! ! !˜8œ a¨¨¨¨1¨2¨2¨q¨q¨q¨¤k�QˆGˆGàˆGØ”'‰ˆˆ1ˆaØ‰ˆˆ1Ø�IŠI�a˜˜A˜qÑ!Ô!×)Ò)¨!¨Q°°1Ñ5Ô5×@Ò@ÑBÔBˆØ×#Ò# AÑ&Ô&ˆØ�IŠI�a˜˜BÑÔ×)Ò)¨!¨QÑ/Ô/ˆØÐÝ”	˜7 A˜,¨AÐ.Ñ.Ô.ˆAØˆrU   c                 óº  — |                       |¦  «        }| j        €|n|                      |¦  «        }|                      ||¦  «        }|                      ||¦  «        \  }}||                      |¦  «        z   }|                      |¦  «        }| j        €|n|                      |¦  «        }||                      |                      |¦  «        ¦  «        z   }||fS ©N)	r  r  r-  r¦   r  r  r   r"  r!  )rQ   rŽ   r•   Úx_normÚ
x_shortcutÚfeat_size_news         rR   r�   zMultiScaleBlock.forwards  sÐ   € Ø—’˜A‘”ˆàÔ1Ð9�Q�Q¸t×?VÒ?VÐW]Ñ?^Ô?^ˆ
Ø×(Ò(¨°YÑ?Ô?ˆ
ØŸ9š9 V¨YÑ7Ô7Ñˆˆ=Ø˜Ÿš¨Ñ+Ô+Ñ+ˆà—’˜A‘”ˆØÔ0Ð8�Q�Q¸d×>TÒ>TÐU[Ñ>\Ô>\ˆ
Ø˜Ÿš¨¯ª°&Ñ)9Ô)9Ñ:Ô:Ñ:ˆØ�-ÐÐrU   )rf   rg   rh   r   r  ri   r	   rk   rl   r   r  rm   r‚   r   r-  r�   r“   r”   s   @rR   r  r    sÇ  ø€ € € € € ð  #Ø!Ø"Ø*,¬,Ø(.Ø)/Ø(.Ø)/ØØ"&Ø %Ø$Ø )Ø%)ØØð+GTð GTàðGTð ðGTð ð	GTð
 ˜S #˜X”ðGTð ðGTð ðGTð ðGTð ˜RœYœðGTð ˜C ˜H”oðGTð ˜S #˜X”ðGTð ˜C ˜H”oðGTð ˜S #˜X”ðGTð ðGTð  ðGTð  ð!GTð" ð#GTð$ ð%GTð& #ð'GTð GTð GTð GTð GTð GTðR¨4°¬9ð ð ð ð ð   D¨¤Ið  ð  ð  ð  ð  ð  ð  ð  rU   r  c            *       ó2  ‡ — e Zd Zddddddddddddej        dddfd	ed
edededeeef         dededeeef         deeef         deeef         deeef         de	dededede	dede
ej                 deeee         f         f&ˆ fd„Zdee         fd„Zˆ xZS )ÚMultiScaleVitStager*   Tr6   r2   FrD   r  Nr¢   rx   Údepthr)   r•   r+   r.   rÇ   rÈ   r8   r9   r3   r–   r-   r,   rE   r1   rI   r  c                 ót  •— ||dœ}t          ¦   «                              ¦   «          d| _        t          j        ¦   «         | _        |r|f|z  }n|f|dz
  z  |fz   }t          |¦  «        D ]Å}t          di d|“d||         “d|“d|“d|“d	|“d
|“d|	“d|dk    r|
nd“d|“d|“d|“d|“d|“d|“d|“d|“dt          |t          t          f¦  «        r||         n|“|¤Ž}||         }| j                             |¦  «         |dk    r't          d„ t          ||
¦  «        D ¦   «         ¦  «        }ŒÆ|| _        d S )Nr}   Fr   r¢   rx   r)   r•   r+   r.   rÇ   rÈ   r8   r   r6   r9   r3   r–   r,   rE   r1   r-   rI   r  c                 ó   — g | ]
\  }}||z  ‘ŒS rn   rn   )rO   rõ   rz   s      rR   r[   z/MultiScaleVitStage.__init__.<locals>.<listcomp>¾  s    € Ð"_Ð"_Ð"_±l°d¸F 4¨6¡>Ð"_Ð"_Ð"_rU   rn   )r�   r‚   Úgrad_checkpointingr   Ú
ModuleListÚblocksr`   r  r]   r_   r^   rb   Úzipr•   )rQ   r¢   rx   r5  r)   r•   r+   r.   rÇ   rÈ   r8   r9   r3   r–   r-   r,   rE   r1   rI   r  r~   r   r…   Úout_dimsrP   Úattention_blockr†   s                             €rR   r‚   zMultiScaleVitStage.__init__ƒ  sù  ø€ ð0 ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ"'ˆÔå”m‘o”oˆŒØð 	9Ø�z EÑ)ˆHˆHà�v ¨¡Ñ+¨w¨jÑ8ˆHå�u‘”ð 	að 	aˆAÝ-ð ð ð Ø�Cðà  œ˜ðð $˜)ðð $˜)ð	ð
 $˜)ðð "˜ðð "˜ðð $˜)ðð &'¨!¢V V˜˜°øðð $˜)ðð �Tðð ,˜mðð &˜:ðð *˜\ðð "2Ð!1ðð  (˜Kð!ð" &˜:ð#õ$ +5°YÅÅuÀÑ*NÔ*NÐ]˜) Aœ,˜,ÐT]øØð'ð ˆOð* ˜1”+ˆCØŒK×Ò˜Ñ/Ô/Ð/Ø�AŠvˆvÝ!Ð"_Ð"_ÅcÈ)ÐU]ÑF^ÔF^Ð"_Ñ"_Ô"_Ñ`Ô`�	øà"ˆŒˆˆrU   c                 ó°   — | j         D ]K}| j        r3t          j                             ¦   «         st          |||¦  «        \  }}Œ< |||¦  «        \  }}ŒL||fS r/  )r:  r8  r‘   ÚjitÚis_scriptingr   )rQ   rŽ   r•   Úblks       rR   r�   zMultiScaleVitStage.forwardÂ  sk   € Ø”;ð 	1ð 	1ˆCØÔ&ð 1­u¬y×/EÒ/EÑ/GÔ/Gð 1Ý)¨#¨q°)Ñ<Ô<‘��9�9à"˜s 1 iÑ0Ô0‘��9�9Ø�)ˆ|ÐrU   )rf   rg   rh   r   r  ri   r	   rk   rl   rm   r   r  r   r   r‚   r�   r“   r”   s   @rR   r4  r4  �  sŽ  ø€ € € € € ð  #Ø!Ø(.Ø)/Ø(.Ø)/ØØ"&Ø %Ø$Ø )Ø%)Ø*,¬,Ø36ØØð-=#ð =#àð=#ð ð=#ð ð	=#ð
 ð=#ð ˜S #˜X”ð=#ð ð=#ð ð=#ð ˜C ˜H”oð=#ð ˜S #˜X”ð=#ð ˜C ˜H”oð=#ð ˜S #˜X”ð=#ð ð=#ð  ð=#ð ð=#ð  ð!=#ð" ð#=#ð$ #ð%=#ð& ˜RœYœð'=#ð( ˜U D¨¤KÐ/Ô0ð)=#ð =#ð =#ð =#ð =#ð =#ð~ D¨¤Ið ð ð ð ð ð ð ð rU   r4  c                   óX  ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 d(dedeeef         d	ed
ee         dede	de	fˆ fd„Z
d„ Zej        j        d„ ¦   «         Zej        j        d)d„¦   «         Zej        j        d*d„¦   «         Zej        j        dej        fd„¦   «         Zd+ded
ee         fd„Z	 	 	 	 	 d,dej        deeeee         f                  dededededeeej                 eej        eej                 f         f         fd„Z	 	 	 d-deeee         f         d!ed"efd#„Zd$„ Zd)d%efd&„Zd'„ Zˆ xZS ).r   a¸  
    Improved Multiscale Vision Transformers for Classification and Detection
    Yanghao Li*, Chao-Yuan Wu*, Haoqi Fan, Karttikeya Mangalam, Bo Xiong, Jitendra Malik,
        Christoph Feichtenhofer*
    https://arxiv.org/abs/2112.01526

    Multiscale Vision Transformers
    Haoqi Fan*, Bo Xiong*, Karttikeya Mangalam*, Yanghao Li*, Zhicheng Yan, Jitendra Malik,
        Christoph Feichtenhofer*
    https://arxiv.org/abs/2104.11227
    ©éà   rD  r$   Néè  r  ÚcfgÚimg_sizeÚin_chansÚglobal_poolÚnum_classesÚdrop_path_rateÚ	drop_ratec
                 ó|  •— t          ¦   «                              ¦   «          ||	dœ}
t          |¦  «        }t          t	          |j        ¦  «        |j        ¬¦  «        }|| _        || _        || _	        |€|j
        rdnd}|| _        t          |j        ¦  «        | _        |j        | _        |j        d         }t!          d#|||j        |j        |j        dœ|
¤Ž| _        |d         |j        d         z  |d         |j        d         z  f}t+          |¦  «        }|j
        r8t-          j        t1          j        dd|fi |
¤Ž¦  «        | _        d| _        |dz   }nd| _        d | _        |}|j        r,t-          j        t1          j        d||fi |
¤Ž¦  «        | _        nd | _        t=          |j        ¦  «        }|}t?          |j        ¦  «        }tA          ||j        d¬	¦  «        }t-          j!        ¦   «         | _"        g | _#        tI          |¦  «        D �]F}|j        r|j        |         }n!|j        tK          |dz   |dz
  ¦  «                 }tM          d#i d
|“d|“d|j        |         “d|j'        |         “d|“d|j(        “d|j)        “d|j*        “d|j+        “d|j        “d|j,        “d|j,        “d|j-        |         “d|j.        |         “d|j
        “d|j/        “d|j0        “d|“d||         “|
¤Ž}|t?          |j-        |         ¦  «        z  }| xj#        tc          d|› �||¬¦  «        gz  c_#        |}|j2        }| j"         3                    |¦  «         �ŒH|x| _4        | _5         ||fi |
¤Ž| _6        t-          j7        tq          dt-          j9        | j	        ¦  «        fd |dk    rt-          j:        | j4        |fi |
¤Žnt-          j;        ¦   «         fg¦  «        ¦  «        | _<        | j        �t{          | j        d!¬"¦  «         | j        �t{          | j        d!¬"¦  «         |  >                    | j?        ¦  «         d S )$Nr}   )ÚepsÚtokenrØ   r   )rw   rx   ry   rz   r{   r   T)Ú	stagewiser¢   rx   r5  r)   r•   r+   r.   r3   r,   r-   rÇ   rÈ   r8   r9   r–   rE   r1   rI   r  zblock.)ÚmoduleÚnum_chsÚ	reductionÚdropÚfcrÜ   rÝ   rn   )@r�   r‚   r   r   r   rI   rJ   rJ  rH  rL  r/   rI  r^   r&   r-   r(   ru   r?   r@   rA   Úpatch_embedrs   r   rî   r‘   rï   Ú	cls_tokenÚnum_prefix_tokensr0   Ú	pos_embedr\   rB   r   r9  ÚstagesÚfeature_infor`   ra   r4  r)   r+   r.   r3   r,   r5   r8   r9   rE   r1   Údictr•   rb   Únum_featuresÚhead_hidden_sizeÚnormÚ
Sequentialr   ÚDropoutrá   r  Úheadr   ÚapplyÚ_init_weights)rQ   rF  rG  rH  rI  rJ  rK  rL  r~   r   r…   rI   r(   Ú
patch_dimsÚnum_patchesÚpos_embed_dimrc   r•   Úcurr_strideÚdprrP   rx   Ústager†   s                          €rR   r‚   zMultiScaleVit.__init__Ø  së  ø€ õ 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆÝ˜XÑ&Ô&ˆÝ�^¨C¬NÑ;Ô;ÀÄÐNÑNÔNˆ
Ø&ˆÔØ ˆŒØ"ˆŒØÐØ%(Ô%6ÐA˜'˜'¸EˆKØ&ˆÔÝ˜CœJÑ'Ô'ˆŒØœ?ˆÔà”M !Ô$ˆ	Ý%ð 
ØØØÔ#ØÔ#ØÔ%ð
ð 
ð ð
ð 
ˆÔð ˜q”k SÔ%5°aÔ%8Ñ8¸(À1¼+ÈÔIYÐZ[ÔI\Ñ:\Ð]ˆ
Ý˜:Ñ&Ô&ˆàÔð 	(Ýœ\­%¬+°a¸¸IÐ*LÐ*LÈÐ*LÐ*LÑMÔMˆDŒNØ%&ˆDÔ"Ø'¨!™OˆMˆMà%&ˆDÔ"Ø!ˆDŒNØ'ˆMàŒ?ð 	"Ýœ\­%¬+°a¸È	Ð*XÐ*XÐUWÐ*XÐ*XÑYÔYˆDŒNˆNà!ˆDŒNå˜œÑ'Ô'ˆ
Øˆ	Ý˜#Ô*Ñ+Ô+ˆÝ'¨¸¼
ÈdÐSÑSÔSˆÝ”m‘o”oˆŒØˆÔÝ�zÑ"Ô"ð 	&ñ 	&ˆAØŒð DØœ-¨Ô*��àœ-­¨A°©E°:À±>Ñ(BÔ(BÔC�Ý&ð ð ð Ø�Iðà˜ðð ”j ”m�mðð œ-¨Ô*Ð*ð	ð
 $˜)ðð œ-˜-ðð œ˜ðð ”X�Xðð œ>˜>ðð  œO˜Oðð œ˜ðð œ.˜.ðð œ aœ˜ðð œ-¨Ô*Ð*ðð "Ô/Ð/ðð  !Ô-Ð-ð!ð" "%Ô!5Ð!5ð#ð$ &˜:ð%ð& ˜aœ&˜&Øð)ð ˆEð, �3˜sœ|¨AœÑ/Ô/Ñ/ˆKØÐÔ¥$¨l°q¨l¨lÀGÐWbÐ"cÑ"cÔ"cÐ!dÑdÐÔØˆIØœˆIØŒK×Ò˜uÑ%Ô%Ð%Ñ%à4=Ð=ˆÔ˜DÔ1Ø�J˜yÐ/Ð/¨BÐ/Ð/ˆŒ	Ý”M¥+Ø•R”Z ¤Ñ/Ô/Ð0ØÀkÐTUÂoÀo•2”9˜TÔ.°ÐBÐB¸rÐBÐBÐBÕ[]Ô[fÑ[hÔ[hÐið/
ñ #ô #ñ ô ˆŒ	ð
 Œ>Ð%Ý˜Tœ^°Ð6Ñ6Ô6Ð6ØŒ>Ð%Ý˜Tœ^°Ð6Ñ6Ô6Ð6Ø�
Š
�4Ô%Ñ&Ô&Ð&Ð&Ð&rU   c                 óþ   — t          |t          j        ¦  «        r^t          |j        d¬¦  «         t          |t          j        ¦  «        r0|j        �+t          j                             |j        d¦  «         d S d S d S d S )NrÜ   rÝ   r  )r]   r   rá   r   ÚweightrÔ   ÚinitÚ	constant_)rQ   Úms     rR   rd  zMultiScaleVit._init_weights?  s€   € Ý�a�œÑ#Ô#ð 	/Ý˜QœX¨4Ð0Ñ0Ô0Ð0Ý˜!�RœYÑ'Ô'ð /¨A¬FÐ,>Ý”×!Ò! !¤&¨#Ñ.Ô.Ð.Ð.Ð.ð	/ð 	/ð/ð /Ð,>Ð,>rU   c                 ó>   — d„ |                       ¦   «         D ¦   «         S )Nc                 óP   ‡— h | ]"\  Š}t          ˆfd „dD ¦   «         ¦  «        ¯ ‰’Œ#S )c              3   ó    •K  — | ]}|‰v V — Œ	d S r/  rn   )rO   Únrâ   s     €rR   rS   z:MultiScaleVit.no_weight_decay.<locals>.<setcomp>.<genexpr>H  s'   øè è € Ð\Ð\ !�q˜A�vÐ\Ð\Ð\Ð\Ð\Ð\rU   )rY  rª   r«   rW  )Úany)rO   rû   râ   s     @rR   ú	<setcomp>z0MultiScaleVit.no_weight_decay.<locals>.<setcomp>G  sV   ø€ ð ^ð ^ð ^‘d�a˜ÝÐ\Ð\Ð\Ð\Ð'[Ð\Ñ\Ô\Ñ\Ô\ð^�ð ^ð ^ð ^rU   )Únamed_parameters©rQ   s    rR   Úno_weight_decayzMultiScaleVit.no_weight_decayE  s4   € ð^ð ^˜d×3Ò3Ñ5Ô5ð ^ñ ^ô ^ð 	^rU   Fc                 ó,   — t          dddg¬¦  «        }|S )Nz^patch_embed)z^stages\.(\d+)N)z^norm)iŸ† )Ústemr:  )r\  )rQ   ÚcoarseÚmatchers      rR   Úgroup_matcherzMultiScaleVit.group_matcherJ  s)   € åØ Ø-Ð/CÐDð
ñ 
ô 
ˆð ˆrU   Tc                 ó(   — | j         D ]	}||_        Œ
d S r/  )rZ  r8  )rQ   Úenabler  s      rR   Úset_grad_checkpointingz$MultiScaleVit.set_grad_checkpointingR  s(   € à”ð 	*ð 	*ˆAØ#)ˆAÔ Ð ð	*ð 	*rU   r‡   c                 ó   — | j         j        S r/  )rb  rU  rw  s    rR   Úget_classifierzMultiScaleVit.get_classifierW  s   € àŒyŒ|ÐrU   c                 óf  — t          | ¦  «        }|| _        |�|| _        t          j        t          dt          j        | j        ¦  «        fd|dk    rt          j        | j	        |fi |¤Žnt          j
        ¦   «         fg¦  «        ¦  «        | _        | j                             | j        ¦  «         d S )NrT  rU  r   )r   rJ  rI  r   r`  r   ra  rL  rá   r]  r  rb  ÚtrainÚtraining)rQ   rJ  rI  r…   s       rR   Úreset_classifierzMultiScaleVit.reset_classifier[  s²   € Ý˜dÑ#Ô#ˆØ&ˆÔØÐ"Ø*ˆDÔÝ”M¥+Ø•R”Z ¤Ñ/Ô/Ð0ØÀkÐTUÂoÀo•2”9˜TÔ.°ÐBÐB¸rÐBÐBÐBÕ[]Ô[fÑ[hÔ[hÐið/
ñ #ô #ñ ô ˆŒ	ð 	Œ	�Š˜œÑ&Ô&Ð&Ð&Ð&rU   ÚNCHWrŽ   Úindicesr_  Ú
stop_earlyÚ
output_fmtÚintermediates_onlyc                 ó4  — |dv s
J d¦   «         ‚|dk    }g }t          t          | j        ¦  «        |¦  «        \  }	}
|                      |¦  «        \  }}|j        d         }| j        �4| j                             |dd¦  «        }t          j        ||fd¬¦  «        }| j	        �
|| j	        z   }t          | j        ¦  «        dz
  }t          | j        ¦  «        D ]ž\  }} |||¦  «        \  }}||	v r†|r||k    r|                      |¦  «        }n|}|rO| j        �|dd…dd…f         }|                     ||d         |d         d¦  «                             dd	dd
¦  «        }|                     |¦  «         ŒŸ|r|S ||k    r|                      |¦  «        }||fS )aè   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to all intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )r‡  ÚNLCz!Output shape must be NCHW or NLC.r‡  r   Nr˜   r   r¡   r$   r#   )r   r\   rZ  rV  rŒ   rW  Úexpandr‘   r£   rY  Ú	enumerater_  r™   rš   rb   )rQ   rŽ   rˆ  r_  r‰  rŠ  r‹  r™   ÚintermediatesÚtake_indicesÚ	max_indexr•   r¿   Ú
cls_tokensÚlast_idxÚfeat_idxrj  Úx_inters                     rR   Úforward_intermediatesz#MultiScaleVit.forward_intermediatesf  sÚ  € ð* ˜_Ð,Ð,Ð,Ð.QÑ,Ô,Ð,Ø Ò&ˆØˆÝ"6µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�ið ×'Ò'¨Ñ*Ô*‰ˆˆ9ØŒG�AŒJˆØŒ>Ð%Øœ×.Ò.¨q°"°bÑ9Ô9ˆJÝ”	˜: q˜/¨qÐ1Ñ1Ô1ˆAØŒ>Ð%Ø�D”NÑ"ˆAå�t”{Ñ#Ô# aÑ'ˆÝ(¨¬Ñ5Ô5ð 	.ð 	.‰OˆH�eØ ˜5  IÑ.Ô.‰LˆAˆyØ˜<Ð'Ð'Øð  ˜H¨Ò0Ð0Ø"Ÿiši¨™lœl�G�Gà�GØð eØ”~Ð1à")¨!¨!¨!¨Q¨R¨R¨%¤.˜Ø%Ÿošo¨a°¸1´¸yÈ¼|ÈRÑPÔP×XÒXÐYZÐ\]Ð_`ÐbcÑdÔd�GØ×$Ò$ WÑ-Ô-Ð-øàð 	!Ø Ð à�xÒÐØ—	’	˜!‘”ˆAà�-ÐÐrU   r   Ú
prune_normÚ
prune_headc                 ó´   — t          t          | j        ¦  «        |¦  «        \  }}|rt          j        ¦   «         | _        |r|                      dd¦  «         |S )z@ Prune layers not required for specified intermediates.
        r   Ú )r   r\   rZ  r   r  r_  r†  )rQ   rˆ  r˜  r™  r‘  r’  s         rR   Úprune_intermediate_layersz'MultiScaleVit.prune_intermediate_layers¡  s]   € õ #7µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�ið ð 	&Ýœ™œˆDŒIØð 	)Ø×!Ò! ! RÑ(Ô(Ð(ØÐrU   c                 ó@  — |                       |¦  «        \  }}|j        \  }}}| j        �4| j                             |dd¦  «        }t	          j        ||fd¬¦  «        }| j        �
|| j        z   }| j        D ]} |||¦  «        \  }}Œ|                      |¦  «        }|S )Nr˜   r   r¡   )	rV  rŒ   rW  rŽ  r‘   r£   rY  rZ  r_  )rQ   rŽ   r•   r¿   rú   r,  r“  rj  s           rR   Úforward_featureszMultiScaleVit.forward_features²  s¯   € Ø×'Ò'¨Ñ*Ô*‰ˆˆ9Ø”'‰ˆˆ1ˆaàŒ>Ð%Øœ×.Ò.¨q°"°bÑ9Ô9ˆJÝ”	˜: q˜/¨qÐ1Ñ1Ô1ˆAàŒ>Ð%Ø�D”NÑ"ˆAà”[ð 	/ð 	/ˆEØ ˜5  IÑ.Ô.‰LˆAˆyˆyà�IŠI�a‰LŒLˆØˆrU   Ú
pre_logitsc                 ó¾   — | j         r>| j         dk    r'|d d …| j        d …f                              d¦  «        }n|d d …df         }|r|n|                      |¦  «        S )NrØ   r   r   )rI  rX  Úmeanrb  )rQ   rŽ   rŸ  s      rR   Úforward_headzMultiScaleVit.forward_headÃ  sq   € ØÔð 	ØÔ 5Ò(Ð(Ø�a�a�a˜Ô/Ð0Ð0Ð0Ô1×6Ò6°qÑ9Ô9��à�a�a�a˜�d”G�ØÐ0ˆqˆq D§I¢I¨a¡L¤LÐ0rU   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S r/  )rž  r¢  r�   s     rR   r�   zMultiScaleVit.forwardË  s-   € Ø×!Ò! !Ñ$Ô$ˆØ×Ò˜aÑ Ô ˆØˆrU   )rC  r$   NrE  r  r  NN©F©Tr/  )NFFr‡  F)r   FT) rf   rg   rh   r�   r    r	   ri   r
   rm   rk   r‚   rd  r‘   r?  Úignorerx  r}  r€  r   r  r‚  r†  r’   r   r   rl   r—  rœ  rž  r¢  r�   r“   r”   s   @rR   r   r   Ë  sÐ  ø€ € € € € ð
ð 
ð )3ØØ)-Ø#Ø$&Ø!ØØðe'ð e'à!ðe'ð ˜C ˜H”oðe'ð ð	e'ð
 " #œðe'ð ðe'ð "ðe'ð ðe'ð e'ð e'ð e'ð e'ð e'ðN/ð /ð /ð „YÔð^ð ^ñ Ôð^ð „YÔðð ð ñ Ôðð „YÔð*ð *ð *ñ Ôð*ð „YÔð ¤	ð ð ð ñ Ôðð	'ð 	'¨Cð 	'¸hÀs¼mð 	'ð 	'ð 	'ð 	'ð 8<ØØ$Ø$Ø',ð9 ð 9 àŒ|ð9 ð ˜e C¨¨c¬ NÔ3Ô4ð9 ð ð	9 ð
 ð9 ð ð9 ð !%ð9 ð 
ˆt�E”LÔ! 5¨¬°t¸E¼LÔ7IÐ)IÔ#JÐJÔ	Kð9 ð 9 ð 9 ð 9 ðz ./Ø$Ø#ð	ð à˜3  S¤	˜>Ô*ðð ðð ð	ð ð ð ð"ð ð ð"1ð 1¨$ð 1ð 1ð 1ð 1ðð ð ð ð ð ð rU   c           	      ó  ‡‡‡— d| v rð|                       ¦   «         D ]Ù}d|v rÓ| |         }|                     ¦   «         |         j        }|j        d         |d         k    r•t          j        j                             |                     d|j        d         d¦  «                             ddd¦  «        |d         d¬¦  «        }|                     d|d         ¦  «                             dd¦  «        | |<   ŒÚ| S dd l	}d	| v r| d	         } t          |d
d ¦  «        }t          |dd¦  «        }|€
J d¦   «         ‚i ŠdŠt          |¦  «        D ]=\  Š}	‰                     ˆˆfd„t          ‰‰|	z   ¦  «        D ¦   «         ¦  «         ‰|	z  ŠŒ>i }
|                      ¦   «         D ]o\  }}|                     dˆfd„|¦  «        }|r|                     dd|¦  «        }n|                     dd|¦  «        }d|v r|                     dd¦  «        }||
|<   Œp|
S )Nzstages.0.blocks.0.norm1.weightÚrel_posr   r   r˜   r#   Úlinear)rõ   r3   Úmodel_stater&   r-   Tz3model requires depth attribute to remap checkpointsc                 ó    •— i | ]
}|‰|‰z
  f“ŒS rn   rn   )rO   rP   Ú	block_idxÚ	stage_idxs     €€rR   ú
<dictcomp>z(checkpoint_filter_fn.<locals>.<dictcomp>ë  s$   ø€ ÐaÐaÐa¸A˜!˜i¨¨Y©Ð7ÐaÐaÐarU   zblocks\.(\d+)c           	      óÄ   •— d‰t          |                      d¦  «        ¦  «                 d         › d‰t          |                      d¦  «        ¦  «                 d         › �S )Nzstages.r   r   z.blocks.)ri   Úgroup)rŽ   Ú	depth_maps    €rR   ú<lambda>z&checkpoint_filter_fn.<locals>.<lambda>ò  sV   ø€ Ðf 	­#¨a¯gªg°a©j¬j©/¬/Ô :¸1Ô =ÐfÐfÀyÕQTÐUV×U\ÒU\Ð]^ÑU_ÔU_ÑQ`ÔQ`ÔGaÐbcÔGdÐfÐf€ rU   z stages\.(\d+).blocks\.(\d+).projz&stages.\1.blocks.\2.shortcut_proj_attnz%stages.\1.blocks.\2.shortcut_proj_mlprb  zhead.projectionúhead.fc)ÚkeysÚ
state_dictrŒ   r‘   r   Ú
functionalÚinterpolater™   rš   ÚreÚgetattrr�  Úupdater`   ÚitemsÚsubÚreplace)rµ  Úmodelrâ   r¨  Údest_rel_pos_shapeÚrel_pos_resizedr¸  r&   r-   rY   Úout_dictrã   r¬  r±  r­  s               @@@rR   Úcheckpoint_filter_fnrÂ  Ñ  sv  øøø€ Ø'¨:Ð5Ð5à—’Ñ"Ô"ð 
	eð 
	eˆAØ˜Aˆ~ˆ~Ø$ Qœ-�Ø%*×%5Ò%5Ñ%7Ô%7¸Ô%:Ô%@Ð"Ø”= Ô#Ð'9¸!Ô'<Ò<Ð<Ý&+¤hÔ&9×&EÒ&EØŸš¨¨7¬=¸Ô+;¸RÑ@Ô@×HÒHÈÈAÈqÑQÔQØ/°Ô2Ø%ð 'Fñ 'ô '�Oð
 %4×$;Ò$;¸BÐ@RÐSTÔ@UÑ$VÔ$V×$^Ò$^Ð_`ÐbcÑ$dÔ$d�J˜q‘MøØÐà€I€I€IØ˜
Ð"Ð"Ø Ô.ˆ
å�U˜H dÑ+Ô+€FÝ˜% °Ñ5Ô5€KØÐÐÐTÑÔÐØ€IØ€IÝ! &Ñ)Ô)ð ð ‰ˆ	�1Ø×ÒÐaÐaÐaÐaÐaÅÀyÐR[Ð^_ÑR_ÑA`ÔA`ÐaÑaÔaÑbÔbÐbØ�Q‰ˆ	ˆ	à€HØ× Ò Ñ"Ô"ð ð ‰ˆˆ1Ø�FŠFØØfÐfÐfÐfØñô ˆð
 ð 	kØ—’Ð:Ð<gÐijÑkÔkˆAˆAà—’Ð:Ð<fÐhiÑjÔjˆAØ�Qˆ;ˆ;Ø—	’	Ð+¨YÑ7Ô7ˆAØˆ�‰ˆà€OrU   )r   r#   é   r#   )r&   )r   r#   é   r#   r"   )r#   é   é$   r;   é�   r#   F)r&   r(   r)   r-   )r&   r/   )r&   r(   r)   r/   r-   )r;   rÆ   é<   rÆ   éÀ   r$   )Úmvitv2_tinyÚmvitv2_smallÚmvitv2_baseÚmvitv2_largeÚmvitv2_small_clsÚmvitv2_base_clsÚmvitv2_large_clsÚmvitv2_huge_clsc           	      óº   — |                      dd¦  «        }t          t          | |f|st          |          nt          |         t          t          |d¬¦  «        dœ|¤ŽS )NÚout_indicesr;   Úgetter)rÓ  Úfeature_cls)Ú	model_cfgÚpretrained_filter_fnÚfeature_cfg)Úpopr   r   Ú
model_cfgsrÂ  r\  )ÚvariantÚcfg_variantÚ
pretrainedÚkwargsrÓ  s        rR   Ú_create_mvitv2rß  *  sp   € Ø—*’*˜]¨AÑ.Ô.€KÝÝØØðð .9ÐU•*˜WÔ%Ð%½jÈÔ>UÝ1Ý [¸hÐGÑGÔGðð ð ðð ð rU   r›  c                 ó8   — | ddd ddt           t          ddddd	œ|¥S )
NrE  )r$   rD  rD  gÍÌÌÌÌÌì?Úbicubiczpatch_embed.projr³  Tz
apache-2.0)ÚurlrJ  Ú
input_sizeÚ	pool_sizeÚcrop_pctÚinterpolationr¡  rÞ   Ú
first_convÚ
classifierÚfixed_input_sizeÚlicenser   )râ  rÞ  s     rR   Ú_cfgrë  7  s;   € àØ¨=ÀtØ¨Ý%Õ.BØ(¸	Ø Øð	ð 	ð ð	ð 	rU   zDhttps://dl.fbaipublicfiles.com/mvit/mvitv2_models/MViTv2_T_in1k.pythztimm/)râ  Ú	hf_hub_idzDhttps://dl.fbaipublicfiles.com/mvit/mvitv2_models/MViTv2_S_in1k.pythzDhttps://dl.fbaipublicfiles.com/mvit/mvitv2_models/MViTv2_B_in1k.pythzDhttps://dl.fbaipublicfiles.com/mvit/mvitv2_models/MViTv2_L_in1k.pyth)râ  zEhttps://dl.fbaipublicfiles.com/mvit/mvitv2_models/MViTv2_B_in21k.pythiàJ  )râ  rì  rJ  zEhttps://dl.fbaipublicfiles.com/mvit/mvitv2_models/MViTv2_L_in21k.pythzEhttps://dl.fbaipublicfiles.com/mvit/mvitv2_models/MViTv2_H_in21k.pyth)zmvitv2_tiny.fb_in1kzmvitv2_small.fb_in1kzmvitv2_base.fb_in1kzmvitv2_large.fb_in1krÎ  zmvitv2_base_cls.fb_inw21kzmvitv2_large_cls.fb_inw21kzmvitv2_huge_cls.fb_inw21kc                 ó   — t          dd| i|¤ŽS )NrÊ  rÝ  )rÊ  ©rß  ©rÝ  rÞ  s     rR   rÊ  rÊ  _  ó   € åÐIÐI°JÐIÀ&ÐIÐIÐIrU   c                 ó   — t          dd| i|¤ŽS )NrË  rÝ  )rË  rî  rï  s     rR   rË  rË  d  ó   € åÐJÐJ°ZÐJÀ6ÐJÐJÐJrU   c                 ó   — t          dd| i|¤ŽS )NrÌ  rÝ  )rÌ  rî  rï  s     rR   rÌ  rÌ  i  rð  rU   c                 ó   — t          dd| i|¤ŽS )NrÍ  rÝ  )rÍ  rî  rï  s     rR   rÍ  rÍ  n  rò  rU   c                 ó   — t          dd| i|¤ŽS )NrÎ  rÝ  )rÎ  rî  rï  s     rR   rÎ  rÎ  s  ó   € åÐNÐN¸ÐNÀvÐNÐNÐNrU   c                 ó   — t          dd| i|¤ŽS )NrÏ  rÝ  )rÏ  rî  rï  s     rR   rÏ  rÏ  x  ó   € åÐMÐM¸
ÐMÀfÐMÐMÐMrU   c                 ó   — t          dd| i|¤ŽS )NrÐ  rÝ  )rÐ  rî  rï  s     rR   rÐ  rÐ  }  rö  rU   c                 ó   — t          dd| i|¤ŽS )NrÑ  rÝ  )rÑ  rî  rï  s     rR   rÑ  rÑ  ‚  rø  rU   r¥  r/  )NF)r›  r¤  )Fr�   rp   Úcollectionsr   Údataclassesr   Ú	functoolsr   r   Útypingr   r   r	   r
   r   r   r‘   r   Ú	timm.datar   r   Útimm.layersr   r   r   r   r   r   r   Ú_builderr   Ú	_featuresr   Ú_features_fxr   Ú_manipulater   Ú	_registryr   r   Ú__all__r    rs   r  ru   ri   rl   r’   rŸ   r¥   rÃ   rÅ   r  r  r4  r   rÂ  r\  rÚ  rß  rë  Údefault_cfgsrÊ  rË  rÌ  rÍ  rÎ  rÏ  rÐ  rÑ  rn   rU   rR   ú<module>r     sÙ  ððð ð €€€Ø #Ð #Ð #Ð #Ð #Ð #Ø !Ð !Ð !Ð !Ð !Ð !Ø %Ð %Ð %Ð %Ð %Ð %Ð %Ð %Ø :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :à €€€Ø Ð Ð Ð Ð Ð à AÐ AÐ AÐ AÐ AÐ AÐ AÐ AØ Ð Ð Ð Ð Ð Ð Ð Ð Ð Ð Ð Ð Ð Ð Ð Ð Ð Ø *Ð *Ð *Ð *Ð *Ð *Ø +Ð +Ð +Ð +Ð +Ð +Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø #Ð #Ð #Ð #Ð #Ð #Ø <Ð <Ð <Ð <Ð <Ð <Ð <Ð <àÐ.Ð
/€ð ð-3ð -3ð -3ð -3ð -3ñ -3ô -3ñ „ð-3ð`-ð -ð -ð:ð :ð :ð :ð :�”ñ :ô :ð :ðB ð #ðð à˜”9ðð ðð ˆ5Œ<˜ %¤,Ô/Ð/Ô0ð	ð ð ñ Ôðð ð +/ð
ð 
àð
ð ˜%œ,Ô'ð
ð ˆ5Œ<˜˜cœÐ"Ô#ð	
ð 
ð 
ñ Ôð
ð ð/ØŒlð/àŒ<ð/ð ð/ð �S”	ð	/ð
 �S”	ð/ð ”<ð/ð ”<ð/ð /ð /ñ Ôð/ðdmð mð mð mð m 2¤9ñ mô mð mð`\ð \ð \ð \ð \˜"œ)ñ \ô \ð \ð~e ð e ð e ð e ð e �b”iñ e ô e ð e ðPGð Gð Gð Gð G˜œñ Gô Gð GðTCð Cð Cð Cð C�B”Iñ Cô Cð CðL,ð ,ð ,ð^ ˆTØ Ð Øðñ ô ð "Ð!Øðñ ô ð !Ð Øðñ ô ð "Ð!ØØØØð	ñ ô ð &Ð%ØØðñ ô ð %Ð$ØØðñ ô ð &Ð%ØØØØØðñ ô ð %Ð$ØØØØØðñ ô ðA'ñ 'ô '€
ðT
ð 
ð 
ð 
ð
ð 
ð 
ð 
ð %Ð$Ø˜4ØRØðñ ô ð !˜DÐ%kØðñ ô à˜4Ð$jØðñ ô à ˜DÐ%kØðñ ô ð ˜ ˜™œØ!% ØSØØð"ñ "ô "ð #' $ØSØØð#ñ #ô #ð "& ØSØØð"ñ "ô "ð)&ð &ñ ô €ð6 ðJð J¨}ð Jð Jð Jñ „ðJð ðKð K°ð Kð Kð Kñ „ðKð ðJð J¨}ð Jð Jð Jñ „ðJð ðKð K°ð Kð Kð Kñ „ðKð ðOð O°Mð Oð Oð Oñ „ðOð ðNð N°=ð Nð Nð Nñ „ðNð ðOð O°Mð Oð Oð Oñ „ðOð ðNð N°=ð Nð Nð Nñ „ðNð Nð NrU   