§
    ·¯¸jh  ã                   ó´  — d Z ddlZddlmZmZmZmZ ddlZddlm	c m
Z ddlm	Z	 ddlmZmZmZmZmZmZ ddlmZ ddlmZ dd	lmZmZmZ dd
lmZmZmZmZm Z m!Z! ddl"m#Z$ g Z%did„Z&djd„Z'dkd„Z(dld„Z) e* e e&dddd¬¦  «         e&dddd¬¦  «         e'ddddddd¬¦  «         e'ddddddd¬¦  «         e'ddddddd¬¦  «        fddd d d!d"¬#¦  «         e e&dd$d¬%¦  «         e&ddd¬%¦  «         e'dddddd¬&¦  «         e'dddd'dd¬&¦  «         e'dddd(dd¬&¦  «        fddd d d!d)¬#¦  «         e e&dd$d¬%¦  «         e&ddd¬%¦  «         e'dddd(dd¬&¦  «         e'dd*dd+dd¬&¦  «         e'dd,dd-dd¬&¦  «        fddd d d!d.¬#¦  «         e e&dd$d¬%¦  «         e&ddd¬%¦  «         e'dddd(dd¬&¦  «         e'dd*dd+dd¬&¦  «         e'dd,dd-dd¬&¦  «        fddd d d/ e*d0¬1¦  «        d.¬2¦  «         e)d¦  «         e)d3¦  «         e)d4¦  «         e)d¦  «         e)d5¦  «         e)d6¦  «         e)d¦  «        ¬7¦  «        Z+e G d8„ d9e	j,        ¦  «        ¦   «         Z- G d:„ d;e	j,        ¦  «        Z. G d<„ d=e	j,        ¦  «        Z/e G d>„ d?e	j,        ¦  «        ¦   «         Z0 ed@e-¦  «          edAe0¦  «         dmdC„Z1dmdD„Z2dndE„Z3 ei dF e3dG¬H¦  «        “dI e3dG¬H¦  «        “dJ e3dG¬H¦  «        “dK e3dGdL¬M¦  «        “dN e3dGdL¬M¦  «        “dO e3dGdL¬M¦  «        “dP e3dGdL¬M¦  «        “dQ e3dGdL¬M¦  «        “dR e3dGdL¬M¦  «        “dS e3dGdL¬M¦  «        “dT e3dGdL¬M¦  «        “dU e3dGdL¬M¦  «        “dV e3dGdL¬M¦  «        “dW e3dGdXdYd¬Z¦  «        “d[ e3dGdXdYd¬Z¦  «        “d\ e3dGdXdYd¬Z¦  «        “¦  «        Z4edod]efd^„¦   «         Z5edod]efd_„¦   «         Z6edod]efd`„¦   «         Z7edod]efda„¦   «         Z8edod]efdb„¦   «         Z9edod]efdc„¦   «         Z:edod]efdd„¦   «         Z;edod]efde„¦   «         Z<edod]efdf„¦   «         Z=edod]efdg„¦   «         Z> ee?dTdUdVdWd[d\dhœ¦  «         dS )pa   MobileViT

Paper:
V1: `MobileViT: Light-weight, General-purpose, and Mobile-friendly Vision Transformer` - https://arxiv.org/abs/2110.02178
V2: `Separable Self-attention for Mobile Vision Transformers` - https://arxiv.org/abs/2206.02680

MobileVitBlock and checkpoints adapted from https://github.com/apple/ml-cvnets (original copyright below)
License: https://github.com/apple/ml-cvnets/blob/main/LICENSE (Apple open source)

Rest of code, ByobNet, and Transformer block hacked together by / Copyright 2022, Ross Wightman
é    N)ÚCallableÚTupleÚOptionalÚType)Únn)Ú	to_2tupleÚmake_divisibleÚ
GroupNorm1ÚConvMlpÚDropPathÚis_exportableé   )Úbuild_model_with_cfg)Úregister_notrace_module)Úregister_modelÚgenerate_default_cfgsÚregister_model_deprecations)Úregister_blockÚByoBlockCfgÚByoModelCfgÚByobNetÚLayerFnÚ
num_groups)ÚBlockç      @c                 óL   — t          d| ||d|t          dd¬¦  «        ¬¦  «        S )NÚbottler   T)Ú	bottle_inÚ
linear_out)ÚtypeÚdÚcÚsÚgsÚbrÚblock_kwargs)r   Údict©r!   r"   r#   r%   s       úe/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/models/mobilevit.pyÚ_inverted_residual_blockr*   !   s6   € åØ˜˜a 1¨¨rÝ D°TÐ:Ñ:Ô:ð<ñ <ô <ð <ó    é   c                 óp   — t          | |||¬¦  «        t          dd|dt          |||¬¦  «        ¬¦  «        fS )Nr(   Ú	mobilevitr   )Útransformer_dimÚtransformer_depthÚ
patch_size)r    r!   r"   r#   r&   ©r*   r   r'   ©r!   r"   r#   r/   r0   r1   r%   s          r)   Ú_mobilevit_blockr4   (   sX   € õ 	! 1¨¨Q°2Ð6Ñ6Ô6ÝØ  Q¨!ÝØ /Ø"3Ø%ð'ñ 'ô 'ð	
ñ 	
ô 	
ð	ð 	r+   é   ç       @ç      à?c                 ór   — t          | |||¬¦  «        t          dd|d|dt          ||¬¦  «        ¬¦  «        fS )Nr(   Ú
mobilevit2r   )r0   r1   )r    r!   r"   r#   r%   r$   r&   r2   )r!   r"   r#   r0   r1   r%   Útransformer_brs          r)   Ú_mobilevitv2_blockr;   6   sY   € õ 	! 1¨¨Q°2Ð6Ñ6Ô6ÝØ  a¨1°ÀAÝØ"3Ø%ð'ñ 'ô 'ð	
ñ 	
ô 	
ðð r+   ç      ð?c                 óˆ  ‡ — d}‰ dk    rt          ˆ fd„|D ¦   «         ¦  «        }t          t          d|d         dd¬¦  «        t          d|d         dd¬¦  «        t          d|d         dd¬	¦  «        t          d|d
         dd¬	¦  «        t          d|d         dd
¬	¦  «        ft	          d‰ z  ¦  «        dddd¬¦  «        }|S )N)é@   é€   é   é€  i   r<   c                 ó4   •— g | ]}t          |‰z  ¦  «        ‘ŒS © )Úint)Ú.0r"   Ú
multipliers     €r)   ú
<listcomp>z$_mobilevitv2_cfg.<locals>.<listcomp>F   s%   ø€ Ð6Ð6Ð6¨Q•S˜˜Z™Ñ(Ô(Ð6Ð6Ð6r+   r   r   r6   r(   r5   )r!   r"   r#   r0   é   r,   é    Ú3x3Ú Úsilu)ÚblocksÚstem_chsÚ	stem_typeÚ	stem_poolÚ
downsampleÚ	act_layer)Útupler   r*   r;   rD   )rF   ÚchsÚcfgs   `  r)   Ú_mobilevitv2_cfgrV   C   sê   ø€ Ø
"€CØ�SÒÐÝÐ6Ð6Ð6Ð6°#Ð6Ñ6Ô6Ñ7Ô7ˆÝ
å$ q¨C°¬F°a¸CÐ@Ñ@Ô@Ý$ q¨C°¬F°a¸CÐ@Ñ@Ô@Ý  c¨!¤f°ÀQÐGÑGÔGÝ  c¨!¤f°ÀQÐGÑGÔGÝ  c¨!¤f°ÀQÐGÑGÔGð
õ �R˜*‘_Ñ%Ô%ØØØØðñ ô €Cð €Jr+   é   r(   rH   é   é0   r>   r3   éP   é`   rJ   rK   rL   i@  )rM   rN   rO   rP   rQ   rR   Únum_featuresrI   )r!   r"   r#   )r!   r"   r#   r/   r0   r1   éx   é�   rA   r?   éÀ   é    éð   i€  Úseg      À?)Úrd_ratio)rM   rN   rO   rP   rQ   Ú
attn_layerÚattn_kwargsr\   g      è?g      ô?g      ø?g      ü?)Úmobilevit_xxsÚmobilevit_xsÚmobilevit_sÚsemobilevit_sÚmobilevitv2_050Úmobilevitv2_075Úmobilevitv2_125Úmobilevitv2_100Úmobilevitv2_150Úmobilevitv2_175Úmobilevitv2_200c            &       ó  ‡ — e Zd ZdZdddddddddd	d
dddddej        ddfdedee         dedededee         de	eef         dedee         dededededede
dededeej                 f$ˆ fd„Zd ej        d!ej        fd"„Zˆ xZS )#ÚMobileVitBlockzS MobileViT block
        Paper: https://arxiv.org/abs/2110.02178?context=cs.LG
    NrH   r   r<   ©r   r   r6   r5   é   r,   ç        FÚin_chsÚout_chsÚkernel_sizeÚstrideÚbottle_ratioÚ
group_sizeÚdilationÚ	mlp_ratior/   r0   r1   Ú	num_headsÚ	attn_dropÚdropÚ	no_fusionÚdrop_path_rateÚlayersÚtransformer_norm_layerc                 óª  •‡‡	‡‡‡‡‡‡‡— ||dœŠt          ¦   «                              ¦   «          ‰pt          ¦   «         Št          ||¦  «        }|p|}‰	pt	          ||z  ¦  «        Š	 ‰j        ||f||||d         dœ‰¤Ž| _        t          j        |‰	fdddœ‰¤Ž| _	        t          j
        ˆˆˆˆˆˆˆˆ	ˆf	d„t          |
¦  «        D ¦   «         Ž | _         ‰‰	fi ‰¤Ž| _         ‰j        ‰	|fdddœ‰¤Ž| _        |rd | _        n ‰j        ||z   |f|ddœ‰¤Ž| _        t!          |¦  «        | _        | j        d         | j        d         z  | _        d S )	N©ÚdeviceÚdtyper   ©rx   ry   Úgroupsr|   r   F©rx   Úbiasc                 óF   •	— g | ]}t          ‰	f‰‰d ‰‰‰‰j        ‰
dœ‰¤Ž‘ŒS )T)r}   r~   Úqkv_biasr   Ú	proj_dropÚ	drop_pathrR   Ú
norm_layer)ÚTransformerBlockÚact)rE   Ú_r   Úddr€   r‚   rƒ   r}   r~   r/   r„   s     €€€€€€€€€r)   rG   z+MobileVitBlock.__init__.<locals>.<listcomp>Ó   sf   ø€ ð +
ð +
ð +
ð õ Øðà#Ø#ØØ#ØØ(Ø œ*Ø1ðð ð ðð ð+
ð +
ð +
r+   ©rx   ry   )ÚsuperÚ__init__r   r   r	   Úconv_norm_actÚconv_kxkr   ÚConv2dÚconv_1x1Ú
SequentialÚrangeÚtransformerÚnormÚ	conv_projÚconv_fusionr   r1   Ú
patch_area)Úselfrv   rw   rx   ry   rz   r{   r|   r}   r/   r0   r1   r~   r   r€   r�   r‚   rƒ   r„   r‡   rˆ   ÚkwargsrŠ   r•   Ú	__class__s           ``  ``` ```    @€r)   r˜   zMobileVitBlock.__init__©   sñ  øøøøøøøøøø€ ð0 ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØÐ$�7™9œ9ˆÝ˜J¨Ñ/Ô/ˆØÐ#˜VˆØ)ÐR­^¸LÈ6Ñ<QÑ-RÔ-Rˆà,˜Ô,ØØð
ð $ØØØ˜a”[ð
ð 
ð ð
ð 
ˆŒõ œ	 &¨/Ð[ÀqÈuÐ[Ð[ÐXZÐ[Ð[ˆŒåœ=ð +
ð +
ð +
ð +
ð +
ð +
ð +
ð +
ð +
ð +
ð +
ð +
õ Ð,Ñ-Ô-ð+
ñ +
ô +
ð ˆÔð +Ð*¨?ÐAÐA¸bÐAÐAˆŒ	à-˜Ô-¨o¸wÐfÐTUÐ^_ÐfÐfÐceÐfÐfˆŒàð 	xØ#ˆDÔÐà3˜vÔ3°F¸WÑ4DÀgÐwÐ[fÐopÐwÐwÐtvÐwÐwˆDÔå# JÑ/Ô/ˆŒØœ/¨!Ô,¨t¬¸qÔ/AÑAˆŒˆˆr+   ÚxÚreturnc                 ó¤  — |}|                       |¦  «        }|                      |¦  «        }| j        \  }}|j        \  }}}}t	          j        ||z  ¦  «        |z  t	          j        ||z  ¦  «        |z  }
}	|	|z  |
|z  }}||z  }d}|	|k    s|
|k    rt          j        ||	|
fdd¬¦  «        }d}|                     ||z  |z  |||¦  «         	                    dd¦  «        }|                     |||| j
        ¦  «         	                    dd¦  «                             || j
        z  |d¦  «        }|                      |¦  «        }|                      |¦  «        }|                     ¦   «                              || j
        |d¦  «        }| 	                    dd¦  «                             ||z  |z  |||¦  «        }| 	                    dd¦  «                             ||||z  ||z  ¦  «        }|rt          j        |||fdd¬¦  «        }|                      |¦  «        }| j        �+|                      t#          j        ||fd¬	¦  «        ¦  «        }|S )
NFÚbilinear©ÚsizeÚmodeÚalign_cornersTr   r5   rH   éÿÿÿÿ©Údim)rš   rœ   r1   ÚshapeÚmathÚceilÚFÚinterpolateÚreshapeÚ	transposer£   rŸ   r    Ú
contiguousÚviewr¡   r¢   ÚtorchÚcat)r¤   r§   ÚshortcutÚpatch_hÚpatch_wÚBÚCÚHÚWÚnew_hÚnew_wÚnum_patch_hÚnum_patch_wÚnum_patchesr¶   s                  r)   ÚforwardzMobileVitBlock.forwardî   s^  € Øˆð �MŠM˜!ÑÔˆØ�MŠM˜!ÑÔˆð  œ?Ñˆ�Ø”W‰
ˆˆ1ˆa�Ý”y  W¡Ñ-Ô-°Ñ7½¼À1ÀwÁ;Ñ9OÔ9OÐRYÑ9YˆuˆØ#(¨GÑ#3°U¸gÑ5E�[ˆØ! KÑ/ˆØˆØ�AŠ:ˆ:˜ !š˜å”˜a u¨e n¸:ÐUZÐ[Ñ[Ô[ˆAØˆKð �IŠI�a˜!‘e˜kÑ)¨7°KÀÑIÔI×SÒSÐTUÐWXÑYÔYˆà�IŠI�a˜˜K¨¬Ñ9Ô9×CÒCÀAÀqÑIÔI×QÒQÐRSÐVZÔVeÑReÐgrÐtvÑwÔwˆð ×Ò˜QÑÔˆØ�IŠI�a‰LŒLˆð �LŠL‰NŒN×Ò  4¤?°KÀÑDÔDˆØ�KŠK˜˜1ÑÔ×%Ò% a¨!¡e¨kÑ&9¸;ÈÐQXÑYÔYˆà�KŠK˜˜1ÑÔ×%Ò% a¨¨K¸'Ñ,AÀ;ÐQXÑCXÑYÔYˆØð 	TÝ”˜a q¨! f°:ÈUÐSÑSÔSˆAà�NŠN˜1ÑÔˆØÔÐ'Ø× Ò ¥¤¨H°a¨=¸aÐ!@Ñ!@Ô!@ÑAÔAˆAØˆr+   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Ú	LayerNormrD   r   Úfloatr   Úboolr   r   ÚModuler˜   r»   ÚTensorrÉ   Ú__classcell__©r¦   s   @r)   rr   rr   ¤   s®  ø€ € € € € ðð ð &*Ø ØØ"%Ø(,Ø(.Ø"Ø-1Ø%&ØØØ!ØØ#Ø$&Ø"Ø68´lØØð+CBð CBàðCBð ˜c”]ðCBð ð	CBð
 ðCBð  ðCBð ! œðCBð ˜C ˜H”oðCBð ðCBð & cœ]ðCBð  #ðCBð ðCBð ðCBð ðCBð ðCBð  ð!CBð" "ð#CBð$ ð%CBð& %)¨¬¤Oð'CBð CBð CBð CBð CBð CBðJ(˜œð (¨%¬,ð (ð (ð (ð (ð (ð (ð (ð (r+   rr   c                   ó&  ‡ — e Zd ZdZ	 	 	 	 	 ddedededed	df
ˆ fd
„Zdej	        d	ej	        fd„Z
ej                             ¦   «         ddej	        deej	                 d	ej	        fd„¦   «         Zddej	        deej	                 d	ej	        fd„Zˆ xZS )ÚLinearSelfAttentiona·  
    This layer applies a self-attention with linear complexity, as described in `https://arxiv.org/abs/2206.02680`
    This layer can be used for self- as well as cross-attention.
    Args:
        embed_dim (int): :math:`C` from an expected input of size :math:`(N, C, H, W)`
        attn_drop (float): Dropout value for context scores. Default: 0.0
        bias (bool): Use bias in learnable layers. Default: True
    Shape:
        - Input: :math:`(N, C, P, N)` where :math:`N` is the batch size, :math:`C` is the input channels,
        :math:`P` is the number of pixels in the patch, and :math:`N` is the number of patches
        - Output: same as the input
    .. note::
        For MobileViTv2, we unfold the feature map [B, C, H, W] into [B, C, P, N] where P is the number of pixels
        in a patch and N is the number of patches. Because channel is the first dimension in this unfolded tensor,
        we use point-wise convolution (instead of a linear layer). This avoids a transpose operation (which may be
        expensive on resource-constrained devices) that may be required to convert the unfolded tensor from
        channel-first to channel-last format in case of a linear layer.
    ru   TNÚ	embed_dimr   r�   rŒ   r¨   c                 ó<  •— ||dœ}t          ¦   «                              ¦   «          || _        t          j        d|dd|z  z   |ddœ|¤Ž| _        t          j        |¦  «        | _        t          j        d|||ddœ|¤Ž| _        t          j        |¦  «        | _	        d S )Nr†   r   r5   )Úin_channelsÚout_channelsrŒ   rx   rC   )
r—   r˜   r×   r   r›   Úqkv_projÚDropoutr   Úout_projÚout_drop)	r¤   r×   r   r�   rŒ   r‡   rˆ   r•   r¦   s	           €r)   r˜   zLinearSelfAttention.__init__-  sÈ   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ"ˆŒåœ	ð 
Ø!Ø˜a )™mÑ,ØØð	
ð 
ð
 ð
ð 
ˆŒõ œ IÑ.Ô.ˆŒÝœ	ð 
Ø!Ø"ØØð	
ð 
ð
 ð
ð 
ˆŒõ œ
 9Ñ-Ô-ˆŒˆˆr+   r§   c                 ó²  — |                       |¦  «        }|                     d| j        | j        gd¬¦  «        \  }}}t          j        |d¬¦  «        }|                      |¦  «        }||z                       dd¬¦  «        }t          j        |¦  «        |                     |¦  «        z  }|  	                    |¦  «        }|  
                    |¦  «        }|S )Nr   r°   r¯   T©r±   Úkeepdim)rÛ   Úsplitr×   rµ   Úsoftmaxr   ÚsumÚreluÚ	expand_asrÝ   rÞ   )	r¤   r§   ÚqkvÚqueryÚkeyÚvalueÚcontext_scoresÚcontext_vectorÚouts	            r)   Ú_forward_self_attnz&LinearSelfAttention._forward_self_attnK  sË   € à�mŠm˜AÑÔˆð
  ŸIšI q¨$¬.¸$¼.Ð&IÈq˜IÑQÔQÑˆˆs�Eõ œ 5¨bÐ1Ñ1Ô1ˆØŸš¨Ñ7Ô7ˆð  Ñ.×3Ò3¸ÀDÐ3ÑIÔIˆõ Œf�U‰mŒm˜n×6Ò6°uÑ=Ô=Ñ=ˆØ�mŠm˜CÑ Ô ˆØ�mŠm˜CÑ Ô ˆØˆ
r+   Úx_prevc                 ó  — |j         \  }}}}|j         dd …         \  }}||k    s
J d¦   «         ‚t          j        || j        j        d | j        dz   …         | j        j        d | j        dz   …         ¬¦  «        }	|	                     d| j        gd¬¦  «        \  }
}t          j        || j        j        | j        dz            | j        j        �| j        j        | j        dz            nd ¬¦  «        }t          j        |
d¬¦  «        }|  	                    |¦  «        }||z   
                    dd¬¦  «        }t          j        |¦  «        |                     |¦  «        z  }|                      |¦  «        }|                      |¦  «        }|S )	NéþÿÿÿzJThe number of pixels in a patch for query and key_value should be the samer   )ÚweightrŒ   r°   r¯   Trà   )r²   rµ   Úconv2drÛ   rò   r×   rŒ   râ   rã   r   rä   rå   ræ   rÝ   rÞ   )r¤   r§   rï   Ú
batch_sizeÚin_dimÚkv_patch_areaÚkv_num_patchesÚq_patch_areaÚq_num_patchesÚqkrè   ré   rê   rë   rì   rí   s                   r)   Ú_forward_cross_attnz'LinearSelfAttention._forward_cross_attnc  s—  € ð =>¼GÑ9ˆ
�F˜M¨>Ø&'¤g¨b¨c¨c¤lÑ#ˆ�mð ˜\Ò)Ð)Ð)ØWñ *Ô)Ð)õ
 ŒXØØ”=Ô'Ð(;¨¬¸!Ñ);Ð(;Ô<Ø”Ô#Ð$7 T¤^°aÑ%7Ð$7Ô8ð
ñ 
ô 
ˆð —X’X˜q $¤.Ð1°q�XÑ9Ô9‰
ˆˆså”ØØ”=Ô'¨¬¸Ñ(:Ô;Ø;?¼=Ô;MÐ;Y�”Ô# D¤N°QÑ$6Ô7Ð7Ð_cð
ñ 
ô 
ˆõ œ 5¨bÐ1Ñ1Ô1ˆØŸš¨Ñ7Ô7ˆð  Ñ.×3Ò3¸ÀDÐ3ÑIÔIˆõ Œf�U‰mŒm˜n×6Ò6°uÑ=Ô=Ñ=ˆØ�mŠm˜CÑ Ô ˆØ�mŠm˜CÑ Ô ˆØˆ
r+   c                 ó^   — |€|                       |¦  «        S |                      ||¬¦  «        S )N)rï   )rî   rû   )r¤   r§   rï   s      r)   rÉ   zLinearSelfAttention.forwardŽ  s4   € Øˆ>Ø×*Ò*¨1Ñ-Ô-Ð-à×+Ò+¨A°fÐ+Ñ=Ô=Ð=r+   )ru   ru   TNN©N)rÊ   rË   rÌ   rÍ   rD   rÏ   rÐ   r˜   r»   rÒ   rî   ÚjitÚignorer   rû   rÉ   rÓ   rÔ   s   @r)   rÖ   rÖ     sE  ø€ € € € € ðð ð,  #Ø"ØØØð.ð .àð.ð ð.ð ð	.ð
 ð.ð 
ð.ð .ð .ð .ð .ð .ð< E¤Lð °U´\ð ð ð ð ð0 „Y×ÒÑÔð(ð ( U¤\ð (¸8ÀEÄLÔ;Qð (Ð]bÔ]ið (ð (ð (ñ Ôð(ðT>ð >˜œð >¨x¸¼Ô/Eð >ÐQVÔQ]ð >ð >ð >ð >ð >ð >ð >ð >r+   rÖ   c                   óÚ   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 ddedededed	ed
eeej	                          deeej	                          ddfˆ fd„Z
ddej        deej                 dej        fd„Zˆ xZS )ÚLinearTransformerBlockaF  
    This class defines the pre-norm transformer encoder with linear self-attention in `MobileViTv2 paper <>`_
    Args:
        embed_dim (int): :math:`C_{in}` from an expected input of size :math:`(B, C_{in}, P, N)`
        mlp_ratio (float): Inner dimension ratio of the FFN relative to embed_dim
        drop (float): Dropout rate. Default: 0.0
        attn_drop (float): Dropout rate for attention in multi-head attention. Default: 0.0
        drop_path (float): Stochastic depth rate Default: 0.0
        norm_layer (Callable): Normalization layer. Default: layer_norm_2d
    Shape:
        - Input: :math:`(B, C_{in}, P, N)` where :math:`B` is batch size, :math:`C_{in}` is input embedding dim,
            :math:`P` is number of pixels in a patch, and :math:`N` is number of patches,
        - Output: same shape as the input
    r6   ru   Nr×   r}   r€   r   r�   rR   r‘   r¨   c
                 ó~  •— ||	dœ}
t          ¦   «                              ¦   «          |pt          j        }|pt          } ||fi |
¤Ž| _        t          d|||dœ|
¤Ž| _        t          |¦  «        | _	         ||fi |
¤Ž| _
        t          d|t          ||z  ¦  «        ||dœ|
¤Ž| _        t          |¦  «        | _        d S )Nr†   )r×   r   r�   )Úin_featuresÚhidden_featuresrR   r€   rC   )r—   r˜   r   ÚSiLUr
   Únorm1rÖ   Úattnr   Ú
drop_path1Únorm2r   rD   ÚmlpÚ
drop_path2)r¤   r×   r}   r€   r   r�   rR   r‘   r‡   rˆ   r•   r¦   s              €r)   r˜   zLinearTransformerBlock.__init__¥  sò   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØÐ(¥¤ˆ	ØÐ-¥:ˆ
à�Z 	Ð0Ð0¨RÐ0Ð0ˆŒ
Ý'Ðg°)ÀyÐ\`ÐgÐgÐdfÐgÐgˆŒ	Ý" 9Ñ-Ô-ˆŒà�Z 	Ð0Ð0¨RÐ0Ð0ˆŒ
Ýð Ø!Ý 	¨IÑ 5Ñ6Ô6ØØð	ð ð
 ðð ˆŒõ # 9Ñ-Ô-ˆŒˆˆr+   r§   rï   c                 óŽ  — |€?||                       |                      |                      |¦  «        ¦  «        ¦  «        z   }nE|}|                      |¦  «        }|                      ||¦  «        }|                       |¦  «        |z   }||                      |                      |                      |¦  «        ¦  «        ¦  «        z   }|S rý   )r  r  r  r  r
  r	  )r¤   r§   rï   Úress       r)   rÉ   zLinearTransformerBlock.forwardÃ  s§   € Øˆ>à�D—O’O D§I¢I¨d¯jªj¸©m¬mÑ$<Ô$<Ñ=Ô=Ñ=ˆAˆAð ˆCØ—
’
˜1‘”ˆAØ—	’	˜!˜VÑ$Ô$ˆAØ—’ Ñ"Ô" SÑ(ˆAð �—’ §¢¨¯ª°A©¬Ñ 7Ô 7Ñ8Ô8Ñ8ˆØˆr+   )r6   ru   ru   ru   NNNNrý   )rÊ   rË   rÌ   rÍ   rD   rÏ   r   r   r   rÑ   r˜   r»   rÒ   rÉ   rÓ   rÔ   s   @r)   r  r  •  s  ø€ € € € € ðð ð$  #ØØ"Ø"Ø37Ø48ØØð.ð .àð.ð ð.ð ð	.ð
 ð.ð ð.ð    R¤Y¤Ô0ð.ð !  b¤i¤Ô1ð.ð 
ð.ð .ð .ð .ð .ð .ð<ð ˜œð ¨x¸¼Ô/Eð ÐQVÔQ]ð ð ð ð ð ð ð ð r+   r  c                    óð   ‡ — e Zd ZdZddddddddd	d
d
d
deddfdedee         dededee         deeef         dedee         dededededede	de
ej                 fˆ fd„Zdej        dej        fd„Zˆ xZS )ÚMobileVitV2Blockz8
    This class defines the `MobileViTv2 block <>`_
    NrH   r<   r   rs   r6   r5   rt   ru   rv   rw   rx   rz   r{   r|   r}   r/   r0   r1   r   r€   r‚   rƒ   r„   c           
      ó†  •‡‡‡‡‡‡‡‡— ||dœŠt          ¦   «                              ¦   «          ‰pt          ¦   «         Št          ||¦  «        }|p|}‰pt	          ||z  ¦  «        Š ‰j        ||f|d||d         dœ‰¤Ž| _        t          j        |‰fdddœ‰¤Ž| _	        t          j
        ˆˆˆˆˆˆˆˆfd„t          |	¦  «        D ¦   «         Ž | _         ‰‰fi ‰¤Ž| _         ‰j        ‰|fddddœ‰¤Ž| _        t          |
¦  «        | _        | j        d         | j        d         z  | _        t%          ¦   «         | _        d S )	Nr†   r   r   r‰   Fr‹   c                 óB   •— g | ]}t          ‰f‰‰‰‰‰j        ‰	d œ‰¤Ž‘ŒS ))r}   r   r€   r�   rR   r‘   )r  r“   )
rE   r”   r   r•   r€   r‚   rƒ   r}   r/   r„   s
     €€€€€€€€r)   rG   z-MobileVitV2Block.__init__.<locals>.<listcomp>   s`   ø€ ð +
ð +
ð +
ð õ #Øð	à#Ø#ØØ(Ø œ*Ø1ð	ð 	ð ð	ð 	ð+
ð +
ð +
r+   )rx   ry   Ú	apply_act)r—   r˜   r   r   r	   r™   rš   r   r›   rœ   r�   rž   rŸ   r    r¡   r   r1   r£   r   Úcoreml_exportable)r¤   rv   rw   rx   rz   r{   r|   r}   r/   r0   r1   r   r€   r‚   rƒ   r„   r‡   rˆ   r¥   rŠ   r•   r¦   s          ``  `````    @€r)   r˜   zMobileVitV2Block.__init__Ù  sº  øøøøøøøøø€ ð* ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØÐ$�7™9œ9ˆÝ˜J¨Ñ/Ô/ˆØÐ#˜VˆØ)ÐR­^¸LÈ6Ñ<QÑ-RÔ-Rˆà,˜Ô,ØØð
ð $ØØØ˜a”[ð
ð 
ð ð
ð 
ˆŒõ œ	 &¨/Ð[ÀqÈuÐ[Ð[ÐXZÐ[Ð[ˆŒåœ=ð +
ð +
ð +
ð +
ð +
ð +
ð +
ð +
ð +
ð +
ð +
õ Ð,Ñ-Ô-ð+
ñ +
ô +
ð ˆÔð +Ð*¨?ÐAÐA¸bÐAÐAˆŒ	à-˜Ô-¨o¸wÐwÐTUÐ^_ÐkpÐwÐwÐtvÐwÐwˆŒå# JÑ/Ô/ˆŒØœ/¨!Ô,¨t¬¸qÔ/AÑAˆŒÝ!.¡¤ˆÔÐÐr+   r§   r¨   c                 óâ  — |j         \  }}}}| j        \  }}t          j        ||z  ¦  «        |z  t          j        ||z  ¦  «        |z  }	}||z  |	|z  }}
|
|z  }||k    s|	|k    rt	          j        |||	fdd¬¦  «        }|                      |¦  «        }|                      |¦  «        }|j         d         }| j        rt	          j	        |||f||f¬¦  «        }n2| 
                    |||
|||¦  «                             ddddd	d
¦  «        }| 
                    ||d|¦  «        }|                      |¦  «        }|                      |¦  «        }| j        r5| 
                    |||z  |z  |
|¦  «        }t	          j        ||¬¦  «        }nP| 
                    |||||
|¦  «                             ddd
d	dd¦  «        }| 
                    |||
|z  ||z  ¦  «        }|                      |¦  «        }|S )Nrª   Tr«   r   r–   r   rH   é   r5   r,   r¯   )Úupscale_factor)r²   r1   r³   r´   rµ   r¶   rš   rœ   r  Úunfoldr·   ÚpermuterŸ   r    Úpixel_shuffler¡   )r¤   r§   rÀ   rÁ   rÂ   rÃ   r¾   r¿   rÄ   rÅ   rÆ   rÇ   rÈ   s                r)   rÉ   zMobileVitV2Block.forward  s  € Ø”W‰
ˆˆ1ˆa�Øœ?Ñˆ�Ý”y  W¡Ñ-Ô-°Ñ7½¼À1ÀwÁ;Ñ9OÔ9OÐRYÑ9YˆuˆØ#(¨GÑ#3°U¸gÑ5E�[ˆØ! KÑ/ˆØ�AŠ:ˆ:˜ !š˜Ý”˜a u¨e n¸:ÐUYÐZÑZÔZˆAð �MŠM˜!ÑÔˆØ�MŠM˜!ÑÔˆð ŒG�AŒJˆØÔ!ð 	fÝ”˜¨°'Ð(:ÀGÈWÐCUÐVÑVÔVˆAˆAà—	’	˜!˜Q ¨W°kÀ7ÑKÔK×SÒSÐTUÐWXÐZ[Ð]^Ð`aÐcdÑeÔeˆAØ�IŠI�a˜˜B Ñ,Ô,ˆð ×Ò˜QÑÔˆØ�IŠI�a‰LŒLˆð Ô!ð 	Nà—	’	˜!˜Q ™[¨7Ñ2°KÀÑMÔMˆAÝ” °'Ð:Ñ:Ô:ˆAˆAà—	’	˜!˜Q ¨°+¸{ÑKÔK×SÒSÐTUÐWXÐZ[Ð]^Ð`aÐcdÑeÔeˆAØ—	’	˜!˜Q ¨gÑ 5°{ÀWÑ7LÑMÔMˆAà�NŠN˜1ÑÔˆØˆr+   )rÊ   rË   rÌ   rÍ   r
   rD   r   rÏ   r   r   r   r   rÑ   r˜   r»   rÒ   rÉ   rÓ   rÔ   s   @r)   r  r  Ó  sS  ø€ € € € € ðð ð &*Ø Ø"%Ø()Ø(.Ø"Ø-1Ø%&ØØ!ØØ$&Ø"Ø6@ØØð%:1ð :1àð:1ð ˜c”]ð:1ð ð	:1ð
  ð:1ð ! œð:1ð ˜C ˜H”oð:1ð ð:1ð & cœ]ð:1ð  #ð:1ð ð:1ð ð:1ð ð:1ð "ð:1ð ð:1ð  %)¨¬¤Oð!:1ð :1ð :1ð :1ð :1ð :1ðx#˜œð #¨%¬,ð #ð #ð #ð #ð #ð #ð #ð #r+   r  r.   r9   Fc                 ó€   — t          t          | |f|st          |          nt          |         t          d¬¦  «        dœ|¤ŽS ©NT)Úflatten_sequential)Ú	model_cfgÚfeature_cfg©r   r   Ú
model_cfgsr'   ©ÚvariantÚcfg_variantÚ
pretrainedr¥   s       r)   Ú_create_mobilevitr%  ?  óT   € ÝÝ�˜*ðà-8ÐU•*˜WÔ%Ð%½jÈÔ>UÝ¨DÐ1Ñ1Ô1ðð ð ð	ð ð r+   c                 ó€   — t          t          | |f|st          |          nt          |         t          d¬¦  «        dœ|¤ŽS r  r  r!  s       r)   Ú_create_mobilevit2r(  G  r&  r+   c                 ó$   — | ddddddddd	d
ddœ|¥S )Niè  )rH   r@   r@   )rt   rt   gÍÌÌÌÌÌì?Úbicubic)ru   ru   ru   )r<   r<   r<   z	stem.convzhead.fcFzcvnets-license)ÚurlÚnum_classesÚ
input_sizeÚ	pool_sizeÚcrop_pctÚinterpolationÚmeanÚstdÚ
first_convÚ
classifierÚfixed_input_sizeÚlicenserC   )r+  r¥   s     r)   Ú_cfgr7  O  s:   € à 4°}ÐSYØ¨)Ø \Ø!°Ø!Ø#ðð ð ðð r+   zmobilevit_xxs.cvnets_in1kztimm/)Ú	hf_hub_idzmobilevit_xs.cvnets_in1kzmobilevit_s.cvnets_in1kzmobilevitv2_050.cvnets_in1kgÑ"Ûù~jì?)r8  r/  zmobilevitv2_075.cvnets_in1kzmobilevitv2_100.cvnets_in1kzmobilevitv2_125.cvnets_in1kzmobilevitv2_150.cvnets_in1kzmobilevitv2_175.cvnets_in1kzmobilevitv2_200.cvnets_in1kz$mobilevitv2_150.cvnets_in22k_ft_in1kz$mobilevitv2_175.cvnets_in22k_ft_in1kz$mobilevitv2_200.cvnets_in22k_ft_in1kz(mobilevitv2_150.cvnets_in22k_ft_in1k_384)rH   rA   rA   )é   r9  )r8  r-  r.  r/  z(mobilevitv2_175.cvnets_in22k_ft_in1k_384z(mobilevitv2_200.cvnets_in22k_ft_in1k_384r¨   c                 ó   — t          dd| i|¤ŽS )Nrf   r$  )rf   ©r%  ©r$  r¥   s     r)   rf   rf   Œ  s   € åÐNÐN¸ÐNÀvÐNÐNÐNr+   c                 ó   — t          dd| i|¤ŽS )Nrg   r$  )rg   r;  r<  s     r)   rg   rg   ‘  s   € åÐMÐM¸
ÐMÀfÐMÐMÐMr+   c                 ó   — t          dd| i|¤ŽS )Nrh   r$  )rh   r;  r<  s     r)   rh   rh   –  s   € åÐLÐL°zÐLÀVÐLÐLÐLr+   c                 ó   — t          dd| i|¤ŽS )Nrj   r$  )rj   r;  r<  s     r)   rj   rj   ›  ó   € åÐPÐP¸:ÐPÈÐPÐPÐPr+   c                 ó   — t          dd| i|¤ŽS )Nrk   r$  )rk   r;  r<  s     r)   rk   rk      r@  r+   c                 ó   — t          dd| i|¤ŽS )Nrm   r$  )rm   r;  r<  s     r)   rm   rm   ¥  r@  r+   c                 ó   — t          dd| i|¤ŽS )Nrl   r$  )rl   r;  r<  s     r)   rl   rl   ª  r@  r+   c                 ó   — t          dd| i|¤ŽS )Nrn   r$  )rn   r;  r<  s     r)   rn   rn   ¯  r@  r+   c                 ó   — t          dd| i|¤ŽS )Nro   r$  )ro   r;  r<  s     r)   ro   ro   ´  r@  r+   c                 ó   — t          dd| i|¤ŽS )Nrp   r$  )rp   r;  r<  s     r)   rp   rp   ¹  r@  r+   )Úmobilevitv2_150_in22ft1kÚmobilevitv2_175_in22ft1kÚmobilevitv2_200_in22ft1kÚmobilevitv2_150_384_in22ft1kÚmobilevitv2_175_384_in22ft1kÚmobilevitv2_200_384_in22ft1k)r   )r,   r   )r5   r6   r7   )r<   )NF)rK   )F)@rÍ   r³   Útypingr   r   r   r   r»   Útorch.nn.functionalr   Ú
functionalrµ   Útimm.layersr   r	   r
   r   r   r   Ú_builderr   Ú_features_fxr   Ú	_registryr   r   r   Úbyobnetr   r   r   r   r   r   Úvision_transformerr   r’   Ú__all__r*   r4   r;   rV   r'   r   rÑ   rr   rÖ   r  r  r%  r(  r7  Údefault_cfgsrf   rg   rh   rj   rk   rm   rl   rn   ro   rp   rÊ   rC   r+   r)   ú<module>rX     sš
  ðð
ð 
ð €€€Ø 2Ð 2Ð 2Ð 2Ð 2Ð 2Ð 2Ð 2Ð 2Ð 2Ð 2Ð 2à €€€Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à _Ð _Ð _Ð _Ð _Ð _Ð _Ð _Ð _Ð _Ð _Ð _Ð _Ð _Ð _Ð _Ø *Ð *Ð *Ð *Ð *Ð *Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YØ [Ð [Ð [Ð [Ð [Ð [Ð [Ð [Ð [Ð [Ð [Ð [Ð [Ð [Ð [Ð [Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9à
€ð<ð <ð <ð <ðð ð ð ð
ð 
ð 
ð 
ðð ð ð ð* ˆTØ�+à$Ð$ q¨B°!¸Ð<Ñ<Ô<Ø$Ð$ q¨B°!¸Ð<Ñ<Ô<ØÐ˜q B¨!¸RÐSTÐabÐgjÐkÑkÔkØÐ˜q B¨!¸RÐSTÐabÐgjÐkÑkÔkØÐ˜q B¨!¸RÐSTÐabÐgjÐkÑkÔkð
ð ØØØØØðñ ô ð  �à$Ð$ q¨B°!Ð4Ñ4Ô4Ø$Ð$ q¨B°!Ð4Ñ4Ô4ØÐ˜q B¨!¸RÐSTÐabÐcÑcÔcØÐ˜q B¨!¸SÐTUÐbcÐdÑdÔdØÐ˜q B¨!¸SÐTUÐbcÐdÑdÔdð
ð ØØØØØðñ ô ð  �à$Ð$ q¨B°!Ð4Ñ4Ô4Ø$Ð$ q¨B°!Ð4Ñ4Ô4ØÐ˜q B¨!¸SÐTUÐbcÐdÑdÔdØÐ˜q C¨1¸cÐUVÐcdÐeÑeÔeØÐ˜q C¨1¸cÐUVÐcdÐeÑeÔeð
ð ØØØØØðñ ô ð  �+à$Ð$ q¨B°!Ð4Ñ4Ô4Ø$Ð$ q¨B°!Ð4Ñ4Ô4ØÐ˜q B¨!¸SÐTUÐbcÐdÑdÔdØÐ˜q C¨1¸cÐUVÐcdÐeÑeÔeØÐ˜q C¨1¸cÐUVÐcdÐeÑeÔeð
ð ØØØØØ�D #Ð&Ñ&Ô&Øðñ ô ð" %Ð$ SÑ)Ô)Ø$Ð$ SÑ)Ô)Ø$Ð$ TÑ*Ô*Ø$Ð$ SÑ)Ô)Ø$Ð$ SÑ)Ô)Ø$Ð$ TÑ*Ô*Ø$Ð$ SÑ)Ô)ðQIñ Iô I€
ðX ðqð qð qð qð q�R”Yñ qô qñ Ôðqðhy>ð y>ð y>ð y>ð y>˜"œ)ñ y>ô y>ð y>ðx;ð ;ð ;ð ;ð ;˜RœYñ ;ô ;ð ;ð| ðdð dð dð dð d�r”yñ dô dñ ÔðdðN €ˆ{˜NÑ +Ô +Ð +Ø €ˆ|Ð-Ñ .Ô .Ð .ðð ð ð ðð ð ð ð	ð 	ð 	ð 	ð %Ð$ð .&Ø  °Ð!8Ñ!8Ô!8ð.&à  ¨wÐ 7Ñ 7Ô 7ð.&ð ˜t˜t¨gÐ6Ñ6Ô6ð.&ð
 " 4 4ØØð$ñ $ô $ð.&ð " 4 4ØØð$ñ $ô $ð.&ð " 4 4ØØð$ñ $ô $ð.&ð " 4 4ØØð$ñ $ô $ð.&ð" " 4 4ØØð$ñ $ô $ð#.&ð( " 4 4ØØð$ñ $ô $ð).&ð. " 4 4ØØð$ñ $ô $ð/.&ð6 +¨D¨DØØð-ñ -ô -ð7.&ð< +¨D¨DØØð-ñ -ô -ð=.&ðB +¨D¨DØØð-ñ -ô -ðC.&ðJ /°°ØØ ¨H¸sð1Dñ 1Dô 1DðK.&ðP /°°ØØ ¨H¸sð1Dñ 1Dô 1DðQ.&ðV /°°ØØ ¨H¸sð1Dñ 1Dô 1DðW.&ñ .ô .€ðb ðOð O°ð Oð Oð Oñ „ðOð ðNð N°ð Nð Nð Nñ „ðNð ðMð M¨wð Mð Mð Mñ „ðMð ðQð Q°7ð Qð Qð Qñ „ðQð ðQð Q°7ð Qð Qð Qñ „ðQð ðQð Q°7ð Qð Qð Qñ „ðQð ðQð Q°7ð Qð Qð Qñ „ðQð ðQð Q°7ð Qð Qð Qñ „ðQð ðQð Q°7ð Qð Qð Qñ „ðQð ðQð Q°7ð Qð Qð Qñ „ðQð Ð ˜HØ FØ FØ Fà$NØ$NØ$Nð'ð 'ñ ô ð ð ð r+   