§
    ·¯¸jSõ  ã                   ó¬  — d dl Z d dlmZ d dlmZmZmZmZmZ d dl	Z	d dl
mZ d dlmZmZmZmZ d dlmZmZmZmZmZmZmZmZmZmZ ddlmZ ddlmZ dd	l m!Z! dd
l"m#Z#m$Z$ dgZ%d„ Z& G d„ dej'        ¦  «        Z( G d„ dej'        ¦  «        Z)ej*        ddddfde+de+deej'                 de,de,dej-        fd„Z. G d„ dej'        ¦  «        Z/ G d„ dej'        ¦  «        Z0 G d„ dej'        ¦  «        Z1 G d „ d!ej'        ¦  «        Z2 G d"„ d#ej'        ¦  «        Z3 G d$„ d%ej'        ¦  «        Z4 G d&„ d'ej'        ¦  «        Z5 G d(„ d)ej'        ¦  «        Z6 G d*„ d+ej'        ¦  «        Z7 G d,„ dej'        ¦  «        Z8ddd.„Z9 e$i d/ e9d0¬1¦  «        “d2 e9d0¬1¦  «        “d3 e9d0¬1¦  «        “d4 e9d0¬1¦  «        “d5 e9d0¬1¦  «        “d6 e9d0¬1¦  «        “d7 e9d0d8¬9¦  «        “d: e9d0¬1¦  «        “d; e9d0¬1¦  «        “d< e9d0¬1¦  «        “d= e9d0¬1¦  «        “d> e9d0¬1¦  «        “d? e9d0¬1¦  «        “d@ e9d0d8¬9¦  «        “dA e9dBdCd8dDdEdFdG¬H¦  «        “dI e9dJdKd8dDdEdFdG¬H¦  «        “dL e9dMdNd8dDdEdFdG¬H¦  «        “ e9d0dOdDdEdFdG¬P¦  «         e9d0d8dDdEdFdG¬P¦  «         e9d0d8dQeedRdSdG¬T¦  «         e9d0d8dQeedRdSdG¬T¦  «        dUœ¥¦  «        Z:dV„ Z;dedW„Z<e#dedX„¦   «         Z=e#dedY„¦   «         Z>e#dedZ„¦   «         Z?e#ded[„¦   «         Z@e#ded\„¦   «         ZAe#ded]„¦   «         ZBe#ded^„¦   «         ZCe#ded_„¦   «         ZDe#ded`„¦   «         ZEe#deda„¦   «         ZFe#dedb„¦   «         ZGe#dedc„¦   «         ZHdS )fé    N)Úpartial)ÚListÚOptionalÚTupleÚTypeÚUnion)ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STDÚOPENAI_CLIP_MEANÚOPENAI_CLIP_STD)
Úget_device_dtypeÚDropPathÚcalculate_drop_path_ratesÚtrunc_normal_Úcreate_conv2dÚConvNormActÚSqueezeExciteÚuse_fused_attnÚClassifierHeadÚLayerNorm2dé   )Úbuild_model_with_cfg)Úfeature_take_indices)Úcheckpoint_seq)Úregister_modelÚgenerate_default_cfgsÚFastVitc                 ó*   — | sdS || z  dk    sJ ‚|| z  S )Nr   r   © )Ú
group_sizeÚchannelss     úc/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/models/fastvit.pyÚ
num_groupsr#   #   s2   € Øð &Øˆqð ˜*Ñ$¨Ò)Ð)Ð)Ð)Ø˜:Ñ%Ð%ó    c                   óB  ‡ — e Zd ZdZddddddddej        ddfdeded	ed
ededededededededeej	                 ddfˆ fd„Z
dej        dej        fd„Zd„ Zdeej        ej        f         fd„Zdeej        ej        f         deej        ej        f         fd„Zˆ xZS )ÚMobileOneBlocka#  MobileOne building block.

    This block has a multi-branched architecture at train-time
    and plain-CNN style architecture at inference time
    For more details, please refer to our paper:
    `An Improved One millisecond Mobile Backbone` -
    https://arxiv.org/pdf/2206.04040.pdf
    r   r   FTNÚin_chsÚout_chsÚkernel_sizeÚstrideÚdilationr    Úinference_modeÚuse_seÚuse_actÚuse_scale_branchÚnum_conv_branchesÚ	act_layerÚreturnc           	      ó  •‡ ‡‡— ||dœŠt          ¦   «                              ¦   «          |‰ _        t          ||¦  «        ‰ _        |‰ _        |‰ _        ‰‰ _        |‰ _        |‰ _	        |‰ _
        |rt          |fddi‰¤Žnt          j        ¦   «         ‰ _        |rt          ||f‰||‰ j        ddœ‰¤Ž‰ _        n¯d‰ _        ||k    r|dk    rt          j        dd|i‰¤Žnd‰ _        |dk    r:t          j        ˆˆˆ fd	„t)          ‰ j
        ¦  «        D ¦   «         ¦  «        ‰ _        nd‰ _        d‰ _        ‰dk    r.|
r,t/          ‰ j        ‰ j	        fd‰ j        ‰ j        d
dœ‰¤Ž‰ _        |	r
 |¦   «         nt          j        ¦   «         ‰ _        dS )a·  Construct a MobileOneBlock module.

        Args:
            in_chs: Number of channels in the input.
            out_chs: Number of channels produced by the block.
            kernel_size: Size of the convolution kernel.
            stride: Stride size.
            dilation: Kernel dilation factor.
            group_size: Convolution group size.
            inference_mode: If True, instantiates model in inference mode.
            use_se: Whether to use SE-ReLU activations.
            use_act: Whether to use activation. Default: ``True``
            use_scale_branch: Whether to use scale branch. Default: ``True``
            num_conv_branches: Number of linear conv branches.
        ©ÚdeviceÚdtypeÚ
rd_divisorr   T©r)   r*   r+   ÚgroupsÚbiasNÚnum_featuresr   c           
      ó^   •— g | ])}t          ‰j        ‰j        f‰‰j        ‰j        d dœ‰¤Ž‘Œ*S )F©r)   r*   r9   Ú	apply_act)r   r'   r(   r*   r9   )Ú.0Ú_Úddr)   Úselfs     €€€r"   ú
<listcomp>z+MobileOneBlock.__init__.<locals>.<listcomp>{   se   ø€ ð 
/ð 
/ð 
/ð õ  ØœØœðð %0Ø#œ{Ø#œ{Ø"'ðð ð ðð ð
/ð 
/ð 
/r$   Fr=   r   )ÚsuperÚ__init__r,   r#   r9   r*   r+   r)   r'   r(   r0   r   ÚnnÚIdentityÚser   Úreparam_convÚBatchNorm2dÚidentityÚ
ModuleListÚrangeÚconv_kxkÚ
conv_scaler   Úact)rB   r'   r(   r)   r*   r+   r    r,   r-   r.   r/   r0   r1   r5   r6   rA   Ú	__class__s   `  `           @€r"   rE   zMobileOneBlock.__init__6   s  øøøø€ ð@ ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ,ˆÔÝ  ¨VÑ4Ô4ˆŒØˆŒØ ˆŒØ&ˆÔØˆŒØˆŒØ!2ˆÔð AGÐY•- Ð<Ð<°AÐ<¸Ð<Ð<Ð<ÍBÌKÉMÌMˆŒàð 0	Ý -ØØð	!ð (ØØ!Ø”{Øð	!ð 	!ð ð	!ð 	!ˆDÔÐð !%ˆDÔð ˜fÒ$Ð$¨°1ª¨õ ”Ð9Ð9¨FÐ9°bÐ9Ð9Ð9àð ŒMð ! 1Ò$Ð$Ý "¤ð 
/ð 
/ð 
/ð 
/ð 
/ð 
/õ  % TÔ%;Ñ<Ô<ð
/ñ 
/ô 
/ñ 
!ô 
!�”�ð !%�”ð #ˆDŒOØ˜QŠˆÐ#3ˆÝ"-Ø”KØ”Lð#ð !"Øœ;Øœ;Ø#ð#ð #ð ð#ð #�”ð #*Ð<�9�9‘;”;�;­r¬{©}¬}ˆŒˆˆr$   Úxc                 ó–  — | j         �;|                      |                      |                       |¦  «        ¦  «        ¦  «        S d}| j        �|                      |¦  «        }d}| j        �|                      |¦  «        }||z   }| j        �| j        D ]}| ||¦  «        z  }Œ|                      |                      |¦  «        ¦  «        S )zApply forward pass.Nr   )rI   rP   rH   rK   rO   rN   )rB   rR   Úidentity_outÚ	scale_outÚoutÚrcs         r"   ÚforwardzMobileOneBlock.forward˜   sÍ   € ð ÔÐ(Ø—8’8˜DŸGšG D×$5Ò$5°aÑ$8Ô$8Ñ9Ô9Ñ:Ô:Ð:ð ˆØŒ=Ð$ØŸ=š=¨Ñ+Ô+ˆLð ˆ	ØŒ?Ð&ØŸš¨Ñ*Ô*ˆIð ˜,Ñ&ˆØŒ=Ð$Ø”mð ð �Ø�r�r˜!‘u”u‘��à�xŠx˜Ÿš ™œÑ%Ô%Ð%r$   c           	      ó  — | j         �dS |                      ¦   «         \  }}t          | j        | j        | j        | j        | j        | j        d¬¦  «        | _         || j         j	        _
        || j         j        _
        |                      ¦   «         D ]\  }}d|v rŒ
|                     ¦   «          Œ|                      d¦  «         |                      d¦  «         t          | d¦  «        r|                      d¦  «         d| _        dS )a  Following works like `RepVGG: Making VGG-style ConvNets Great Again` -
        https://arxiv.org/pdf/2101.03697.pdf. We re-parameterize multi-branched
        architecture used at training time to obtain a plain CNN-like structure
        for inference.
        NT)Úin_channelsÚout_channelsr)   r*   r+   r9   r:   rI   rN   rO   rK   )rI   Ú_get_kernel_biasr   r'   r(   r)   r*   r+   r9   ÚweightÚdatar:   Únamed_parametersÚdetach_Ú__delattr__Úhasattrr,   )rB   Úkernelr:   ÚnameÚparas        r"   ÚreparameterizezMobileOneBlock.reparameterize±   s  € ð ÔÐ(ØˆFà×,Ò,Ñ.Ô.‰ˆ�Ý)ØœØœØÔ(Ø”;Ø”]Ø”;Øð
ñ 
ô 
ˆÔð )/ˆÔÔ Ô%Ø&*ˆÔÔÔ#ð ×/Ò/Ñ1Ô1ð 	ð 	‰JˆD�$Ø Ð%Ð%ØØ�LŠL‰NŒNˆNˆNà×Ò˜Ñ$Ô$Ð$Ø×Ò˜Ñ&Ô&Ð&Ý�4˜Ñ$Ô$ð 	)Ø×Ò˜ZÑ(Ô(Ð(à"ˆÔÐÐr$   c                 óÎ  — d}d}| j         �P|                      | j         ¦  «        \  }}| j        dz  }t          j        j                             |||||g¦  «        }d}d}| j        �|                      | j        ¦  «        \  }}d}d}| j        �Dt          | j
        ¦  «        D ]/}|                      | j        |         ¦  «        \  }	}
||	z  }||
z  }Œ0||z   |z   }||z   |z   }||fS )z×Method to obtain re-parameterized kernel and bias.
        Reference: https://github.com/DingXiaoH/RepVGG/blob/main/repvgg.py#L83

        Returns:
            Tuple of (kernel, bias) after fusing branches.
        r   Né   )rO   Ú_fuse_bn_tensorr)   ÚtorchrF   Ú
functionalÚpadrK   rN   rM   r0   )rB   Úkernel_scaleÚ
bias_scalerl   Úkernel_identityÚbias_identityÚkernel_convÚ	bias_convÚixÚ_kernelÚ_biasÚkernel_finalÚ
bias_finals                r"   r\   zMobileOneBlock._get_kernel_biasÔ   s  € ð ˆØˆ
ØŒ?Ð&Ø'+×';Ò';¸D¼OÑ'LÔ'LÑ$ˆL˜*àÔ" aÑ'ˆCÝ œ8Ô.×2Ò2°<À#ÀsÈCÐQTÐAUÑVÔVˆLð ˆØˆØŒ=Ð$Ø-1×-AÒ-AÀ$Ä-Ñ-PÔ-PÑ*ˆO˜]ð ˆØˆ	ØŒ=Ð$Ý˜DÔ2Ñ3Ô3ð #ð #�Ø!%×!5Ò!5°d´mÀBÔ6GÑ!HÔ!H‘�˜Ø˜wÑ&�Ø˜UÑ"�	�	à" \Ñ1°OÑCˆØ Ñ+¨mÑ;ˆ
Ø˜ZÐ'Ð'r$   Úbranchc                 óü  — t          |t          ¦  «        rI|j        j        }|j        j        }|j        j        }|j        j        }|j        j        }|j        j        }nÝt          |t          j
        ¦  «        sJ ‚t          | d¦  «        s‡| j        | j        z  }t          j        | j        || j        | j        f|j        j        |j        j        ¬¦  «        }	t'          | j        ¦  «        D ]}
d|	|
|
|z  | j        dz  | j        dz  f<   Œ|	| _        | j        }|j        }|j        }|j        }|j        }|j        }||z                        ¦   «         }||z                       dddd¦  «        }||z  |||z  |z  z
  fS )a  Method to fuse batchnorm layer with preceding conv layer.
        Reference: https://github.com/DingXiaoH/RepVGG/blob/main/repvgg.py#L95

        Args:
            branch: Sequence of ops to be fused.

        Returns:
            Tuple of (kernel, bias) after fusing batchnorm.
        Ú	id_tensor©r6   r5   r   rh   éÿÿÿÿ)Ú
isinstancer   Úconvr]   ÚbnÚrunning_meanÚrunning_varr:   ÚepsrF   rJ   rb   r'   r9   rj   Úzerosr)   r6   r5   rM   rz   ÚsqrtÚreshape)rB   rx   rc   r€   r�   ÚgammaÚbetar‚   Ú	input_dimÚkernel_valueÚiÚstdÚts                r"   ri   zMobileOneBlock._fuse_bn_tensor÷   sš  € õ �f�kÑ*Ô*ð 	Ø”[Ô'ˆFØ!œ9Ô1ˆLØ œ)Ô/ˆKØ”IÔ$ˆEØ”9”>ˆDØ”)”-ˆCˆCå˜f¥b¤nÑ5Ô5Ð5Ð5Ð5Ý˜4 Ñ-Ô-ð .Ø œK¨4¬;Ñ6�	Ý$œ{Ø”[ )¨TÔ-=¸tÔ?OÐPØ œ-Ô-Ø!œ=Ô/ð ñ  ô  �õ
 ˜tœ{Ñ+Ô+ð ð �Að ð !Ø˜1˜y™=¨$Ô*:¸aÑ*?ÀÔAQÐUVÑAVÐVñð ð ".�”Ø”^ˆFØ!Ô.ˆLØ Ô,ˆKØ”MˆEØ”;ˆDØ”*ˆCØ˜SÑ ×&Ò&Ñ(Ô(ˆØ�S‰[×!Ò! " a¨¨AÑ.Ô.ˆØ˜‰z˜4 ,°Ñ"6¸Ñ"<Ñ<Ð<Ð<r$   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__rF   ÚGELUÚintÚboolr   ÚModulerE   rj   ÚTensorrX   rf   r   r\   r   Ú
SequentialrJ   ri   Ú__classcell__©rQ   s   @r"   r&   r&   ,   s®  ø€ € € € € ðð ð ØØØ#(Ø Ø Ø%)Ø%&Ø)+¬ØØð`=ð `=àð`=ð ð`=ð ð	`=ð
 ð`=ð ð`=ð ð`=ð !ð`=ð ð`=ð ð`=ð #ð`=ð  #ð`=ð ˜BœI”ð`=ð  
ð!`=ð `=ð `=ð `=ð `=ð `=ðD&˜œð &¨%¬,ð &ð &ð &ð &ð2!#ð !#ð !#ðF!( %¨¬°e´lÐ(BÔ"Cð !(ð !(ð !(ð !(ðF*=à˜"œ-¨¬Ð7Ô8ð*=ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð*=ð *=ð *=ð *=ð *=ð *=ð *=ð *=r$   r&   c                   ó4  ‡ — e Zd ZdZ	 	 	 	 	 	 ddededededed	ee         d
edeej                 deddfˆ fd„Z	de
j        de
j        fd„Zdee
j        e
j        f         fd„Zdd„Zedej        dej        dee
j        e
j        f         fd„¦   «         Zˆ xZS )ÚReparamLargeKernelConvzàBuilding Block of RepLKNet

    This class defines overparameterized large kernel conv block
    introduced in `RepLKNet <https://arxiv.org/abs/2203.06717>`_

    Reference: https://github.com/DingXiaoH/RepLKNet-pytorch
    NFr'   r(   r)   r*   r    Úsmall_kernelr-   r1   r,   r2   c           	      óX  •— |
|dœ}t          ¦   «                              ¦   «          || _        t          ||¦  «        | _        || _        || _        || _        || _        |	rt          ||f||d| j        ddœ|¤Ž| _
        n]d| _
        t          ||f|| j        | j        ddœ|¤Ž| _        |�2||k    s
J d¦   «         ‚t          ||f|| j        | j        ddœ|¤Ž| _        |rt          |fd	d
i|¤Žnt          j        ¦   «         | _        |�
 |¦   «         nt          j        ¦   «         | _        dS )a!  Construct a ReparamLargeKernelConv module.

        Args:
            in_chs: Number of input channels.
            out_chs: Number of output channels.
            kernel_size: Kernel size of the large kernel conv branch.
            stride: Stride size. Default: 1
            group_size: Group size. Default: 1
            small_kernel: Kernel size of small kernel conv branch.
            act_layer: Activation module. Default: ``nn.GELU``
            inference_mode: If True, instantiates model in inference mode. Default: ``False``
        r4   r   Tr8   NFr=   zDThe kernel size for re-param cannot be larger than the large kernel!Úrd_ratiog      Ð?)rD   rE   r*   r#   r9   r'   r(   r)   r›   r   rI   r   Ú
large_convÚ
small_convr   rF   rG   rH   rP   )rB   r'   r(   r)   r*   r    r›   r-   r1   r,   r5   r6   rA   rQ   s                €r"   rE   zReparamLargeKernelConv.__init__-  s   ø€ ð4 ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØˆŒÝ  ¨VÑ4Ô4ˆŒØˆŒØˆŒà&ˆÔØ(ˆÔØð "	Ý -ØØð	!ð (ØØØ”{Øð	!ð 	!ð ð	!ð 	!ˆDÔÐð !%ˆDÔÝ)ØØðð (Ø”{Ø”{Øðð ð ðð ˆDŒOð Ð'à  KÒ/Ð/Ð/ØYñ 0Ô/Ð/å"-ØØð#ð !-Øœ;Øœ;Ø#ð#ð #ð ð#ð #�”ð BHÐZ•- Ð=Ð=°$Ð=¸"Ð=Ð=Ð=ÍRÌ[É]Ì]ˆŒà"+Ð"7�9�9‘;”;�;½R¼[¹]¼]ˆŒˆˆr$   rR   c                 óü   — | j         �|                       |¦  «        }n4|                      |¦  «        }| j        �||                      |¦  «        z   }|                      |¦  «        }|                      |¦  «        }|S ©N)rI   rž   rŸ   rH   rP   )rB   rR   rV   s      r"   rX   zReparamLargeKernelConv.forwardw  sr   € ØÔÐ(Ø×#Ò# AÑ&Ô&ˆCˆCà—/’/ !Ñ$Ô$ˆCØŒÐ*Ø˜DŸOšO¨AÑ.Ô.Ñ.�Ø�gŠg�c‰lŒlˆØ�hŠh�s‰mŒmˆØˆ
r$   c                 óV  — |                       | j        j        | j        j        ¦  «        \  }}t	          | d¦  «        ri|                       | j        j        | j        j        ¦  «        \  }}||z  }|t          j                             || j	        | j
        z
  dz  gdz  ¦  «        z  }||fS )zÉMethod to obtain re-parameterized kernel and bias.
        Reference: https://github.com/DingXiaoH/RepLKNet-pytorch

        Returns:
            Tuple of (kernel, bias) after fusing branches.
        rŸ   rh   é   )Ú_fuse_bnrž   r~   r   rb   rŸ   rF   rk   rl   r)   r›   )rB   Úeq_kÚeq_bÚsmall_kÚsmall_bs        r"   Úget_kernel_biasz&ReparamLargeKernelConv.get_kernel_bias‚  s©   € ð —]’] 4¤?Ô#7¸¼Ô9KÑLÔL‰
ˆˆdÝ�4˜Ñ&Ô&ð 	Ø#Ÿ}š}¨T¬_Ô-AÀ4Ä?ÔCUÑVÔVÑˆG�WØ�G‰OˆDØ•B”M×%Ò%Ø˜4Ô+¨dÔ.?Ñ?ÀAÑEÐFÈÑJñô ñ ˆDð �TˆzÐr$   c                 óV  — |                       ¦   «         \  }}t          | j        | j        | j        | j        | j        d¬¦  «        | _        || j        j        _	        || j        j
        _	        |                      d¦  «         t          | d¦  «        r|                      d¦  «         dS dS )a  
        Following works like `RepVGG: Making VGG-style ConvNets Great Again` -
        https://arxiv.org/pdf/2101.03697.pdf. We re-parameterize multi-branched
        architecture used at training time to obtain a plain CNN-like structure
        for inference.
        T©r)   r*   r9   r:   rž   rŸ   N)r©   r   r'   r(   r)   r*   r9   rI   r]   r^   r:   ra   rb   )rB   r¥   r¦   s      r"   rf   z%ReparamLargeKernelConv.reparameterize’  sµ   € ð ×)Ò)Ñ+Ô+‰
ˆˆdÝ)ØŒKØŒLØÔ(Ø”;Ø”;Øð
ñ 
ô 
ˆÔð )-ˆÔÔ Ô%Ø&*ˆÔÔÔ#Ø×Ò˜Ñ&Ô&Ð&Ý�4˜Ñ&Ô&ð 	+Ø×Ò˜\Ñ*Ô*Ð*Ð*Ð*ð	+ð 	+r$   r~   r   c                 óÚ   — | j         }|j        }|j        }|j         }|j        }|j        }||z                        ¦   «         }||z                       dddd¦  «        }	||	z  |||z  |z  z
  fS )zéMethod to fuse batchnorm layer with conv layer.

        Args:
            conv: Convolutional kernel weights.
            bn: Batchnorm 2d layer.

        Returns:
            Tuple of (kernel, bias) after fusing batchnorm.
        r|   r   )r]   r€   r�   r:   r‚   r„   r…   )
r~   r   rc   r€   r�   r†   r‡   r‚   r‹   rŒ   s
             r"   r¤   zReparamLargeKernelConv._fuse_bn©  s   € ð ”ˆØ”ˆØ”nˆØ”	ˆØŒwˆØŒfˆØ˜SÑ ×&Ò&Ñ(Ô(ˆØ�S‰[×!Ò! " a¨¨AÑ.Ô.ˆØ˜‰z˜4 ,°Ñ"6¸Ñ"<Ñ<Ð<Ð<r$   )NFNFNN©r2   N)r�   rŽ   r�   r�   r’   r   r“   rF   r”   rE   rj   r•   rX   r   r©   rf   ÚstaticmethodÚConv2drJ   r¤   r—   r˜   s   @r"   rš   rš   $  sŸ  ø€ € € € € ðð ð +/Ø Ø-1Ø#(ØØðHKð HKàðHKð ðHKð ð	HKð
 ðHKð ðHKð # 3œ-ðHKð ðHKð   ¤	Ô*ðHKð !ðHKð 
ðHKð HKð HKð HKð HKð HKðT	˜œð 	¨%¬,ð 	ð 	ð 	ð 	ð  u¤|°U´\Ð'AÔ!Bð ð ð ð ð +ð +ð +ð +ð. ð=Ø”)ð=à”ð=ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð=ð =ð =ñ „\ð=ð =ð =ð =ð =r$   rš   FTr'   r(   r1   r,   r/   r2   c                 ó¦   — ||dœ}t          j        t          d| |dd|||dœ|¤Žt          d||ddd|||dœ|¤Žt          d||dd|||dœ|¤Ž¦  «        S )a,  Build convolutional stem with MobileOne blocks.

    Args:
        in_chs: Number of input channels.
        out_chs: Number of output channels.
        inference_mode: Flag to instantiate model in inference mode. Default: ``False``

    Returns:
        nn.Sequential object with stem elements.
    r4   é   rh   )r'   r(   r)   r*   r1   r,   r/   r   )r'   r(   r)   r*   r    r1   r,   r/   r   )rF   r–   r&   )r'   r(   r1   r,   r/   r5   r6   rA   s           r"   Úconvolutional_stemr²   Â  sÎ   € ð&  UÐ	+Ð	+€BÝŒ=Ýð 		
ØØØØØØ)Ø-ð		
ð 		
ð ð		
ð 		
õ 	ð 
	
ØØØØØØØ)Ø-ð
	
ð 
	
ð ð
	
ð 
	
õ 	ð 		
ØØØØØØ)Ø-ð		
ð 		
ð ð		
ð 		
ñ- ô  ð  r$   c                   óš   ‡ — e Zd ZU dZej        j        e         ed<   	 	 	 	 	 	 dde	de	d	ed
e
de
ddfˆ fd„Zdej        dej        fd„Zˆ xZS )Ú	Attentionz§Multi-headed Self Attention module.

    Source modified from:
    https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py
    Ú
fused_attné    Fç        NÚdimÚhead_dimÚqkv_biasÚ	attn_dropÚ	proj_dropr2   c                 ó¢  •— ||dœ}t          ¦   «                              ¦   «          ||z  dk    s
J d¦   «         ‚|| _        ||z  | _        |dz  | _        t          ¦   «         | _        t          j        ||dz  fd|i|¤Ž| _	        t          j
        |¦  «        | _        t          j        ||fi |¤Ž| _        t          j
        |¦  «        | _        dS )a}  Build MHSA module that can handle 3D or 4D input tensors.

        Args:
            dim: Number of embedding dimensions.
            head_dim: Number of hidden dimensions per head. Default: ``32``
            qkv_bias: Use bias or not. Default: ``False``
            attn_drop: Dropout rate for attention tensor.
            proj_drop: Dropout rate for projection tensor.
        r4   r   z#dim should be divisible by head_dimg      à¿r±   r:   N)rD   rE   r¹   Ú	num_headsÚscaler   rµ   rF   ÚLinearÚqkvÚDropoutr»   Úprojr¼   )
rB   r¸   r¹   rº   r»   r¼   r5   r6   rA   rQ   s
            €r"   rE   zAttention.__init__  sÖ   ø€ ð& ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ�X‰~ Ò"Ð"Ð"Ð$IÑ"Ô"Ð"Ø ˆŒØ ™ˆŒØ Ñ%ˆŒ
Ý(Ñ*Ô*ˆŒå”9˜S #¨¡'Ð?Ð?°Ð?¸BÐ?Ð?ˆŒÝœ IÑ.Ô.ˆŒÝ”I˜c 3Ð-Ð-¨"Ð-Ð-ˆŒ	Ýœ IÑ.Ô.ˆŒˆˆr$   rR   c                 ót  — |j         \  }}}}||z  }|                     d¦  «                             dd¦  «        }|                      |¦  «                             ||d| j        | j        ¦  «                             ddddd¦  «        }|                     d¦  «        \  }}	}
| j	        r<t          j        j                             ||	|
| j        r| j        j        nd¬	¦  «        }nS|| j        z  }||	                     dd¦  «        z  }|                     d¬
¦  «        }|                      |¦  «        }||
z  }|                     dd¦  «                             |||¦  «        }|                      |¦  «        }|                      |¦  «        }|                     dd¦  «                             ||||¦  «        }|S )Nrh   éþÿÿÿr|   r±   r   r   r£   r·   )Ú	dropout_p)r¸   )ÚshapeÚflattenÚ	transposerÁ   r…   r¾   r¹   ÚpermuteÚunbindrµ   rj   rF   rk   Úscaled_dot_product_attentionÚtrainingr»   Úpr¿   ÚsoftmaxrÃ   r¼   )rB   rR   ÚBÚCÚHÚWÚNrÁ   ÚqÚkÚvÚattns               r"   rX   zAttention.forward!  sš  € Ø”W‰
ˆˆ1ˆa�Ø�‰EˆØ�IŠI�a‰LŒL×"Ò" 2 rÑ*Ô*ˆà�HŠH�Q‰KŒKßŠW�Q˜˜1˜dœn¨d¬mÑ<Ô<ßŠW�Q˜˜1˜a Ñ#Ô#ð 	ð
 —*’*˜Q‘-”-‰ˆˆ1ˆaàŒ?ð 
	Ý”Ô#×@Ò@Ø�1�aØ.2¬mÐC˜$œ.Ô*Ð*Àð Añ ô ˆAˆAð
 �D”J‘ˆAØ�q—{’{ 2 rÑ*Ô*Ñ*ˆDØ—<’< B�<Ñ'Ô'ˆDØ—>’> $Ñ'Ô'ˆDØ�q‘ˆAà�KŠK˜˜1ÑÔ×%Ò% a¨¨AÑ.Ô.ˆØ�IŠI�a‰LŒLˆØ�NŠN˜1ÑÔˆØ�KŠK˜˜BÑÔ×'Ò'¨¨1¨a°Ñ3Ô3ˆàˆr$   )r¶   Fr·   r·   NN)r�   rŽ   r�   r�   rj   ÚjitÚFinalr“   Ú__annotations__r’   ÚfloatrE   r•   rX   r—   r˜   s   @r"   r´   r´   ù  sÚ   ø€ € € € € € ðð ð
 ”	” Ô%Ð%Ð%Ñ%ð
 Ø"Ø"Ø"ØØð/ð /àð/ð ð/ð ð	/ð
 ð/ð ð/ð 
ð/ð /ð /ð /ð /ð /ð@˜œð ¨%¬,ð ð ð ð ð ð ð ð r$   r´   c                   óš   ‡ — e Zd ZdZej        dddddfdededededeej                 d	e	d
e	de	ddfˆ fd„Z
dej        dej        fd„Zˆ xZS )Ú
PatchEmbedz$Convolutional patch embedding layer.FNÚ
patch_sizer*   r'   Ú	embed_dimr1   Úlkc_use_actr-   r,   r2   c                 óÚ   •— |	|
dœ}t          ¦   «                              ¦   «          t          j        t	          d||||dd||r|nd|dœ	|¤Žt          d||ddd||dœ|¤Ž¦  «        | _        dS )	a{  Build patch embedding layer.

        Args:
            patch_size: Patch size for embedding computation.
            stride: Stride for convolutional embedding layer.
            in_chs: Number of channels of input tensor.
            embed_dim: Number of embedding dimensions.
            inference_mode: Flag to instantiate model in inference mode. Default: ``False``
        r4   r   r±   N)	r'   r(   r)   r*   r    r›   r-   r1   r,   F)r'   r(   r)   r*   r-   r1   r,   r   )rD   rE   rF   r–   rš   r&   rÃ   )rB   rß   r*   r'   rà   r1   rá   r-   r,   r5   r6   rA   rQ   s               €r"   rE   zPatchEmbed.__init__C  s¼   ø€ ð, ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ”MÝ"ð ØØ!Ø&ØØØØØ'2Ð<˜)˜)¸Ø-ðð ð ðð õ ð 	Ø Ø!ØØØØ#Ø-ð	ð 	ð ð	ð 	ñ
ô 
ˆŒ	ˆ	ˆ	r$   rR   c                 ó0   — |                       |¦  «        }|S r¡   )rÃ   ©rB   rR   s     r"   rX   zPatchEmbed.forwardt  s   € Ø�IŠI�a‰LŒLˆØˆr$   )r�   rŽ   r�   r�   rF   r‘   r’   r   r”   r“   rE   rj   r•   rX   r—   r˜   s   @r"   rÞ   rÞ   @  sã   ø€ € € € € Ø.Ð.ð *,¬Ø %Ø Ø#(ØØð/
ð /
àð/
ð ð/
ð ð	/
ð
 ð/
ð ˜BœI”ð/
ð ð/
ð ð/
ð !ð/
ð 
ð/
ð /
ð /
ð /
ð /
ð /
ðb˜œð ¨%¬,ð ð ð ð ð ð ð ð r$   rÞ   c                   ó<   ‡ — e Zd Z	 	 	 	 d	dededefˆ fd„Zd„ Zˆ xZS )
ÚLayerScale2dçñhãˆµøä>FNr¸   Úinit_valuesÚinplacec           
      ó¼   •— t          ¦   «                              ¦   «          || _        t          j        |t          j        |dd||¬¦  «        z  ¦  «        | _        d S )Nr   r4   )rD   rE   ré   rF   Ú	Parameterrj   Úonesr†   )rB   r¸   rè   ré   r5   r6   rQ   s         €r"   rE   zLayerScale2d.__init__z  sR   ø€ õ 	‰Œ×ÒÑÔÐØˆŒÝ”\ +µ´
¸3ÀÀ1ÈVÐ[`Ð0aÑ0aÔ0aÑ"aÑbÔbˆŒ
ˆ
ˆ
r$   c                 óX   — | j         r|                     | j        ¦  «        n	|| j        z  S r¡   )ré   Úmul_r†   rä   s     r"   rX   zLayerScale2d.forward†  s(   € Ø%)¤\ÐEˆq�vŠv�d”jÑ!Ô!Ð!°q¸4¼:±~ÐEr$   )rç   FNN)	r�   rŽ   r�   r’   rÜ   r“   rE   rX   r—   r˜   s   @r"   ræ   ræ   y  sŒ   ø€ € € € € ð "&Ø!ØØð
cð 
càð
cð ð
cð ð	
cð 
cð 
cð 
cð 
cð 
cðFð Fð Fð Fð Fð Fð Fr$   ræ   c            	       óx   ‡ — e Zd ZdZ	 	 	 	 	 ddededee         d	efˆ fd
„Zde	j
        de	j
        fd„Zdd„Zˆ xZS )ÚRepMixerzÑReparameterizable token mixer.

    For more details, please refer to our paper:
    `FastViT: A Fast Hybrid Vision Transformer using Structural Reparameterization <https://arxiv.org/pdf/2303.14189.pdf>`_
    r±   rç   FNr¸   r)   Úlayer_scale_init_valuer,   c           	      óÄ  •— ||dœ}t          ¦   «                              ¦   «          || _        || _        || _        |r<t          j        | j        | j        f| j        d| j        dz  | j        ddœ|¤Ž| _        dS d| _        t          |||fddddd	œ|¤Ž| _	        t          |||fddd
œ|¤Ž| _
        |�t          ||fi |¤Ž| _        dS t          j        ¦   «         | _        dS )a˜  Build RepMixer Module.

        Args:
            dim: Input feature map dimension. :math:`C_{in}` from an expected input of size :math:`(B, C_{in}, H, W)`.
            kernel_size: Kernel size for spatial mixing. Default: 3
            layer_scale_init_value: Initial value for layer scale. Default: 1e-5
            inference_mode: If True, instantiates model in inference mode. Default: ``False``
        r4   r   rh   T©r)   r*   Úpaddingr9   r:   NFr   )r    r.   r/   r0   )r    r.   )rD   rE   r¸   r)   r,   rF   r¯   rI   r&   ÚnormÚmixerræ   Úlayer_scalerG   )	rB   r¸   r)   rñ   r,   r5   r6   rA   rQ   s	           €r"   rE   zRepMixer.__init__‘  sX  ø€ ð" ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØˆŒØ&ˆÔØ,ˆÔàð "	1Ý "¤	Ø”Ø”ð	!ð !Ô,ØØÔ(¨AÑ-Ø”xØð	!ð 	!ð ð	!ð 	!ˆDÔÐÐð !%ˆDÔÝ&ØØØð	ð ØØ!&Ø"#ð	ð 	ð ð	ð 	ˆDŒIõ (ØØØðð Øðð ð ðð ˆDŒJð &Ð1Ý#/°Ð5KÐ#RÐ#RÈrÐ#RÐ#R�Ô Ð Ð å#%¤;¡=¤=�Ô Ð Ð r$   rR   r2   c                 óÂ   — | j         �|                       |¦  «        }nA||                      |                      |¦  «        |                      |¦  «        z
  ¦  «        z   }|S r¡   )rI   r÷   rö   rõ   rä   s     r"   rX   zRepMixer.forwardÌ  sV   € ØÔÐ(Ø×!Ò! !Ñ$Ô$ˆAˆAà�D×$Ò$ T§Z¢Z°¡]¤]°T·Y²Y¸q±\´\Ñ%AÑBÔBÑBˆAØˆr$   c                 ó   — | j         rdS | j                             ¦   «          | j                             ¦   «          t	          | j        t          ¦  «        r“| j        j        | j        j         	                    d¦  «        | j        j
        j        | j        j
        j        z
  z  z   }t          j        | j        j        ¦  «        | j        j
        j        | j        j
        j        z
  z  }nS| j        j        | j        j
        j        z   | j        j
        j        z
  }| j        j
        j        | j        j
        j        z
  }t          | j        | j        | j        d| j        d¬¦  «        | _
        || j
        j        _        || j
        j        _        |                      ¦   «         D ]\  }}d|v rŒ
|                     ¦   «          Œ|                      d¦  «         |                      d¦  «         |                      d	¦  «         dS )
ziReparameterize mixer and norm into a single
        convolutional layer for efficient inference.
        Nr|   r   Tr«   rI   rö   rõ   r÷   )r,   rö   rf   rõ   r}   r÷   ræ   rz   r†   Ú	unsqueezerI   r]   rj   Úsqueezer:   r   r¸   r)   r^   r_   r`   ra   )rB   ÚwÚbrd   re   s        r"   rf   zRepMixer.reparameterizeÓ  sã  € ð Ôð 	ØˆFàŒ
×!Ò!Ñ#Ô#Ð#ØŒ	× Ò Ñ"Ô"Ð"å�dÔ&­Ñ5Ô5ð 	KØ”
Ô$ tÔ'7Ô'=×'GÒ'GÈÑ'KÔ'KØ”
Ô'Ô.°´Ô1GÔ1NÑNñ(ñ ˆAõ ”˜dÔ.Ô4Ñ5Ô5Ø”
Ô'Ô,¨t¬yÔ/EÔ/JÑJñˆAˆAð
 ”
Ô$Ø”*Ô)Ô0ñ1à”)Ô(Ô/ñ0ð ð
 ”
Ô'Ô,¨t¬yÔ/EÔ/JÑJˆAå)ØŒHØŒHØÔ(ØØ”8Øð
ñ 
ô 
ˆÔð )*ˆÔÔ Ô%Ø&'ˆÔÔÔ#à×/Ò/Ñ1Ô1ð 	ð 	‰JˆD�$Ø Ð%Ð%ØØ�LŠL‰NŒNˆNˆNØ×Ò˜Ñ!Ô!Ð!Ø×Ò˜Ñ Ô Ð Ø×Ò˜Ñ'Ô'Ð'Ð'Ð'r$   )r±   rç   FNNr­   )r�   rŽ   r�   r�   r’   r   rÜ   r“   rE   rj   r•   rX   rf   r—   r˜   s   @r"   rð   rð   Š  sÂ   ø€ € € € € ðð ð  !Ø6:Ø#(ØØð91ð 91àð91ð ð91ð %-¨U¤Oð	91ð
 !ð91ð 91ð 91ð 91ð 91ð 91ðv˜œð ¨%¬,ð ð ð ð ð*(ð *(ð *(ð *(ð *(ð *(ð *(ð *(r$   rð   c                   óÀ   ‡ — e Zd ZdZddej        dddfdedee         dee         deej	                 de
d	dfˆ fd
„Zdej	        d	dfd„Zdej        d	ej        fd„Zˆ xZS )ÚConvMlpzConvolutional FFN Module.Nr·   r'   Úhidden_channelsr(   r1   Údropr2   c                 ó|  •— ||dœ}t          ¦   «                              ¦   «          |p|}|p|}t          ||fd|ddœ|¤Ž| _        t	          j        ||fddi|¤Ž| _         |¦   «         | _        t	          j        ||fddi|¤Ž| _        t	          j	        |¦  «        | _
        |                      | j        ¦  «         dS )a_  Build convolutional FFN module.

        Args:
            in_chs: Number of input channels.
            hidden_channels: Number of channels after expansion. Default: None
            out_chs: Number of output channels. Default: None
            act_layer: Activation layer. Default: ``GELU``
            drop: Dropout rate. Default: ``0.0``.
        r4   é   F)r)   r9   r>   r)   r   N)rD   rE   r   r~   rF   r¯   Úfc1rP   Úfc2rÂ   r  ÚapplyÚ_init_weights)
rB   r'   r   r(   r1   r  r5   r6   rA   rQ   s
            €r"   rE   zConvMlp.__init__  sí   ø€ ð& ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØÐ#˜VˆØ)Ð3¨VˆÝØØð
ð ØØð
ð 
ð ð
ð 
ˆŒ	õ ”9˜V _ÐJÐJÀ!ÐJÀrÐJÐJˆŒØ�9‘;”;ˆŒÝ”9˜_¨gÐKÐKÀ1ÐKÈÐKÐKˆŒÝ”J˜tÑ$Ô$ˆŒ	Ø�
Š
�4Ô%Ñ&Ô&Ð&Ð&Ð&r$   Úmc                 óÆ   — t          |t          j        ¦  «        rDt          |j        d¬¦  «         |j        �)t          j                             |j        d¦  «         d S d S d S )Nç{®Gáz”?©r‹   r   )r}   rF   r¯   r   r]   r:   ÚinitÚ	constant_©rB   r  s     r"   r  zConvMlp._init_weights(  sc   € Ý�a�œÑ#Ô#ð 	-Ý˜!œ(¨Ð-Ñ-Ô-Ð-ØŒvÐ!Ý”×!Ò! !¤&¨!Ñ,Ô,Ð,Ð,Ð,ð	-ð 	-à!Ð!r$   rR   c                 ó  — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r¡   )r~   r  rP   r  r  rä   s     r"   rX   zConvMlp.forward.  sa   € Ø�IŠI�a‰LŒLˆØ�HŠH�Q‰KŒKˆØ�HŠH�Q‰KŒKˆØ�IŠI�a‰LŒLˆØ�HŠH�Q‰KŒKˆØ�IŠI�a‰LŒLˆØˆr$   )r�   rŽ   r�   r�   rF   r‘   r’   r   r   r”   rÜ   rE   r  rj   r•   rX   r—   r˜   s   @r"   rÿ   rÿ      sñ   ø€ € € € € Ø#Ð#ð
 .2Ø%)Ø)+¬ØØØð#'ð #'àð#'ð & cœ]ð#'ð ˜c”]ð	#'ð
 ˜BœI”ð#'ð ð#'ð 
ð#'ð #'ð #'ð #'ð #'ð #'ðJ-˜rœyð -¨Tð -ð -ð -ð -ð˜œð ¨%¬,ð ð ð ð ð ð ð ð r$   rÿ   c                   óœ   ‡ — e Zd ZdZ	 	 	 	 	 ddedee         deeeeef         f         ded	df
ˆ fd
„Z	de
j        d	e
j        fd„Zdd„Zˆ xZS )ÚRepConditionalPosEnca"  Implementation of conditional positional encoding.

    For more details refer to paper:
    `Conditional Positional Encodings for Vision Transformers <https://arxiv.org/pdf/2102.10882.pdf>`_

    In our implementation, we can reparameterize this module to eliminate a skip connection.
    N©r  r  Fr¸   Údim_outÚspatial_shaper,   r2   c           
      ó¬  •— ||dœ}t          ¦   «                              ¦   «          t          |t          ¦  «        rt	          |gdz  ¦  «        }t          |t
          ¦  «        sJ dt          |¦  «        › d�¦   «         ‚t          |¦  «        dk    sJ dt          |¦  «        › d�¦   «         ‚|| _        || _	        |p|| _
        || _        |r=t          j        | j	        | j
        f| j        d|d         dz  | j        dd	œ|¤Ž| _        d
S d
| _        t          j        | j	        | j
        |dt          |d         dz  ¦  «        f| j        ddœ|¤Ž| _        d
S )at  Build reparameterizable conditional positional encoding

        Args:
            dim: Number of input channels.
            dim_out: Number of embedding dimensions. Default: 768
            spatial_shape: Spatial shape of kernel for positional encoding. Default: (7, 7)
            inference_mode: Flag to instantiate block in inference mode. Default: ``False``
        r4   rh   z/"spatial_shape" must by a sequence or int, get z	 instead.z+Length of "spatial_shape" should be 2, got r   r   Tró   N)r9   r:   )rD   rE   r}   r’   Útupler   ÚtypeÚlenr  r¸   r  r9   rF   r¯   rI   Úpos_enc)	rB   r¸   r  r  r,   r5   r6   rA   rQ   s	           €r"   rE   zRepConditionalPosEnc.__init__A  sº  ø€ ð" ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ�m¥SÑ)Ô)ð 	7Ý! = /°AÑ"5Ñ6Ô6ˆMÝ˜-­Ñ/Ô/ð 	
ð 	
ð2Ý˜Ñ&Ô&ð2ð 2ð 2ñ	
ô 	
Ð/õ �=Ñ!Ô! QÒ&Ð&Ð&ð1Ý�}Ñ%Ô%ð1ð 1ð 1ñ 'Ô&Ð&ð
 +ˆÔØˆŒØ�~ #ˆŒØˆŒàð 	Ý "¤	Ø”Ø”ð	!ð !Ô.ØØ% aÔ(¨AÑ-Ø”{Øð	!ð 	!ð ð	!ð 	!ˆDÔÐÐð !%ˆDÔÝœ9Ø”Ø”ØØÝ�M !Ô$¨Ñ)Ñ*Ô*ð	ð ”{Øð	ð 	ð ð	ð 	ˆDŒLˆLˆLr$   rR   c                 óp   — | j         �|                       |¦  «        }n|                      |¦  «        |z   }|S r¡   )rI   r  rä   s     r"   rX   zRepConditionalPosEnc.forward|  s9   € ØÔÐ(Ø×!Ò! !Ñ$Ô$ˆAˆAà—’˜Q‘” !Ñ#ˆAØˆr$   c           
      óö  — | j         | j        z  }t          j        | j         || j        d         | j        d         f| j        j        j        | j        j        j        ¬¦  «        }t          | j         ¦  «        D ]*}d||||z  | j        d         dz  | j        d         dz  f<   Œ+|}|| j        j        z   }| j        j
        }t          j        | j         | j        | j        dt          | j        d         dz  ¦  «        | j        d¬¦  «        | _        || j        j        _        || j        j
        _        |                      ¦   «         D ]\  }}d|v rŒ
|                     ¦   «          Œ|                      d¦  «         d S )	Nr   r   r{   rh   Tró   rI   r  )r¸   r9   rj   rƒ   r  r  r]   r6   r5   rM   r:   rF   r¯   r  r’   rI   r^   r_   r`   ra   )	rB   rˆ   r‰   rŠ   rz   Úw_finalÚb_finalrd   re   s	            r"   rf   z#RepConditionalPosEnc.reparameterizeƒ  s¡  € à”H ¤Ñ+ˆ	Ý”{à”ØØÔ" 1Ô%ØÔ" 1Ô%ð	ð ”,Ô%Ô+Ø”<Ô&Ô-ð	
ñ 	
ô 	
ˆõ �t”x‘”ð 	ð 	ˆAð ð ØØ�I‘ØÔ" 1Ô%¨Ñ*ØÔ" 1Ô%¨Ñ*ð,ñð ð !ˆ	ð ˜dœlÔ1Ñ1ˆØ”,Ô#ˆõ œIØŒHØŒLØÔ*ØÝ˜Ô*¨1Ô-°Ñ2Ñ3Ô3Ø”;Øð
ñ 
ô 
ˆÔð )0ˆÔÔ Ô%Ø&-ˆÔÔÔ#à×/Ò/Ñ1Ô1ð 	ð 	‰JˆD�$Ø Ð%Ð%ØØ�LŠL‰NŒNˆNˆNØ×Ò˜Ñ#Ô#Ð#Ð#Ð#r$   )Nr  FNNr­   )r�   rŽ   r�   r�   r’   r   r   r   r“   rE   rj   r•   rX   rf   r—   r˜   s   @r"   r  r  8  sÞ   ø€ € € € € ðð ð &*Ø9?Ø#(ØØð9ð 9àð9ð ˜c”]ð9ð !  e¨C°¨H¤oÐ!5Ô6ð	9ð
 !ð9ð 
ð9ð 9ð 9ð 9ð 9ð 9ðv˜œð ¨%¬,ð ð ð ð ð+$ð +$ð +$ð +$ð +$ð +$ð +$ð +$r$   r  c                   ó~   ‡ — e Zd ZdZddej        ddddddf	ded	ed
edeej	                 dededede
fˆ fd„Zd„ Zˆ xZS )ÚRepMixerBlockzãImplementation of Metaformer block with RepMixer as token mixer.

    For more details on Metaformer structure, please refer to:
    `MetaFormer Is Actually What You Need for Vision <https://arxiv.org/pdf/2111.11418.pdf>`_
    r±   ç      @r·   rç   FNr¸   r)   Ú	mlp_ratior1   r¼   Ú	drop_pathrñ   r,   c                 ó€  •— |	|
dœ}t          ¦   «                              ¦   «          t          |f|||dœ|¤Ž| _        t	          d|t          ||z  ¦  «        ||dœ|¤Ž| _        |�t          ||fi |¤Ž| _        nt          j
        ¦   «         | _        |dk    rt          |¦  «        nt          j
        ¦   «         | _        dS )a,  Build RepMixer Block.

        Args:
            dim: Number of embedding dimensions.
            kernel_size: Kernel size for repmixer. Default: 3
            mlp_ratio: MLP expansion ratio. Default: 4.0
            act_layer: Activation layer. Default: ``nn.GELU``
            proj_drop: Dropout rate. Default: 0.0
            drop_path: Drop path rate. Default: 0.0
            layer_scale_init_value: Layer scale value at initialization. Default: 1e-5
            inference_mode: Flag to instantiate block in inference mode. Default: ``False``
        r4   )r)   rñ   r,   ©r'   r   r1   r  Nr·   r   )rD   rE   rð   Útoken_mixerrÿ   r’   Úmlpræ   r÷   rF   rG   r   r"  )rB   r¸   r)   r!  r1   r¼   r"  rñ   r,   r5   r6   rA   rQ   s               €r"   rE   zRepMixerBlock.__init__¸  só   ø€ ð2 ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐå#Øð
à#Ø#9Ø)ð	
ð 
ð
 ð
ð 
ˆÔõ ð 
ØÝ  i¡Ñ0Ô0ØØð	
ð 
ð
 ð
ð 
ˆŒð "Ð-Ý+¨CÐ1GÐNÐNÈ2ÐNÐNˆDÔÐå!œ{™}œ}ˆDÔØ09¸C²°� )Ñ,Ô,Ð,ÅRÄ[Á]Ä]ˆŒˆˆr$   c                 ó¬   — |                       |¦  «        }||                      |                      |                      |¦  «        ¦  «        ¦  «        z   }|S r¡   )r%  r"  r÷   r&  rä   s     r"   rX   zRepMixerBlock.forwardé  sH   € Ø×Ò˜QÑÔˆØ�—’˜t×/Ò/°·²¸±´Ñ<Ô<Ñ=Ô=Ñ=ˆØˆr$   )r�   rŽ   r�   r�   rF   r‘   r’   rÜ   r   r”   r“   rE   rX   r—   r˜   s   @r"   r  r  ±  sÞ   ø€ € € € € ðð ð  !Ø"Ø)+¬Ø"Ø"Ø,0Ø#(ØØð/Sð /Sàð/Sð ð/Sð ð	/Sð
 ˜BœI”ð/Sð ð/Sð ð/Sð %*ð/Sð !ð/Sð /Sð /Sð /Sð /Sð /Sðbð ð ð ð ð ð r$   r  c                   ó˜   ‡ — e Zd ZdZdej        ej        dddddfdedede	ej
                 d	e	ej
                 d
ededefˆ fd„Zd„ Zˆ xZS )ÚAttentionBlockzßImplementation of metaformer block with MHSA as token mixer.

    For more details on Metaformer structure, please refer to:
    `MetaFormer Is Actually What You Need for Vision <https://arxiv.org/pdf/2111.11418.pdf>`_
    r   r·   rç   Nr¸   r!  r1   Ú
norm_layerr¼   r"  rñ   c
                 óL  •— ||	dœ}
t          ¦   «                              ¦   «           ||fi |
¤Ž| _        t          dd|i|
¤Ž| _        |�t          ||fi |
¤Ž| _        nt          j        ¦   «         | _        |dk    rt          |¦  «        nt          j        ¦   «         | _
        t          d|t          ||z  ¦  «        ||dœ|
¤Ž| _        |�t          ||fi |
¤Ž| _        nt          j        ¦   «         | _        |dk    rt          |¦  «        nt          j        ¦   «         | _        dS )aÜ  Build Attention Block.

        Args:
            dim: Number of embedding dimensions.
            mlp_ratio: MLP expansion ratio. Default: 4.0
            act_layer: Activation layer. Default: ``nn.GELU``
            norm_layer: Normalization layer. Default: ``nn.BatchNorm2d``
            proj_drop: Dropout rate. Default: 0.0
            drop_path: Drop path rate. Default: 0.0
            layer_scale_init_value: Layer scale value at initialization. Default: 1e-5
        r4   r¸   Nr·   r$  r   )rD   rE   rõ   r´   r%  ræ   Úlayer_scale_1rF   rG   r   Ú
drop_path1rÿ   r’   r&  Úlayer_scale_2Ú
drop_path2)rB   r¸   r!  r1   r*  r¼   r"  rñ   r5   r6   rA   rQ   s              €r"   rE   zAttentionBlock.__init__ö  sG  ø€ ð. ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐà�J˜sÐ)Ð) bÐ)Ð)ˆŒ	Ý$Ð3Ð3¨Ð3°Ð3Ð3ˆÔØ!Ð-Ý!-¨cÐ3IÐ!PÐ!PÈRÐ!PÐ!PˆDÔÐå!#¤¡¤ˆDÔØ1:¸S²°�( 9Ñ-Ô-Ð-ÅbÄkÁmÄmˆŒåð 
ØÝ  i¡Ñ0Ô0ØØð	
ð 
ð
 ð
ð 
ˆŒð "Ð-Ý!-¨cÐ3IÐ!PÐ!PÈRÐ!PÐ!PˆDÔÐå!#¤¡¤ˆDÔØ1:¸S²°�( 9Ñ-Ô-Ð-ÅbÄkÁmÄmˆŒˆˆr$   c           
      ó$  — ||                       |                      |                      |                      |¦  «        ¦  «        ¦  «        ¦  «        z   }||                      |                      |                      |¦  «        ¦  «        ¦  «        z   }|S r¡   )r-  r,  r%  rõ   r/  r.  r&  rä   s     r"   rX   zAttentionBlock.forward%  sq   € Ø�—’ × 2Ò 2°4×3CÒ3CÀDÇIÂIÈaÁLÄLÑ3QÔ3QÑ RÔ RÑSÔSÑSˆØ�—’ × 2Ò 2°4·8²8¸A±;´;Ñ ?Ô ?Ñ@Ô@Ñ@ˆØˆr$   )r�   rŽ   r�   r�   rF   r‘   rJ   r’   rÜ   r   r”   rE   rX   r—   r˜   s   @r"   r)  r)  ï  sØ   ø€ € € € € ðð ð  #Ø)+¬Ø*,¬.Ø"Ø"Ø,0ØØð-Tð -Tàð-Tð ð-Tð ˜BœI”ð	-Tð
 ˜RœYœð-Tð ð-Tð ð-Tð %*ð-Tð -Tð -Tð -Tð -Tð -Tð^ð ð ð ð ð ð r$   r)  c            %       ó  ‡ — e Zd Zdddddddej        ej        ddd	ddddfd
edededededededede	ej
                 dededeej
                 deej
                 dedeee         ef         de	e         dedef$ˆ fd„Zd„ Zˆ xZS )ÚFastVitStageTFr  rh   Nr±   r   r·   rç   r¸   r  ÚdepthÚtoken_mixer_typeÚ
downsampleÚse_downsampleÚdown_patch_sizeÚdown_strideÚpos_emb_layerr)   r!  r1   r*  Úproj_drop_rateÚdrop_path_raterñ   rá   r,   c                 ó’  •— t          ¦   «                              ¦   «          ||dœ}d| _        |rt          d||||||||dœ|¤Ž| _        n ||k    sJ ‚t          j        ¦   «         | _        |	� |	|fd|i|¤Ž| _        nt          j        ¦   «         | _        g }t          |¦  «        D ]Œ}|dk    r/| 	                    t          |f|
|||||         ||dœ|¤Ž¦  «         Œ7|dk    r.| 	                    t          |f||||||         |d	œ|¤Ž¦  «         Œkt          d
                     |¦  «        ¦  «        ‚t          j        |Ž | _        dS )aQ  FastViT stage.

        Args:
            dim: Number of embedding dimensions.
            depth: Number of blocks in stage
            token_mixer_type: Token mixer type.
            kernel_size: Kernel size for repmixer.
            mlp_ratio: MLP expansion ratio.
            act_layer: Activation layer.
            norm_layer: Normalization layer.
            proj_drop_rate: Dropout rate.
            drop_path_rate: Drop path rate.
            layer_scale_init_value: Layer scale value at initialization.
            inference_mode: Flag to instantiate block in inference mode.
        r4   F)rß   r*   r'   rà   r-   r1   rá   r,   Nr,   Úrepmixer)r)   r!  r1   r¼   r"  rñ   r,   Ú	attention)r!  r1   r*  r¼   r"  rñ   z"Token mixer type: {} not supportedr   )rD   rE   Úgrad_checkpointingrÞ   r5  rF   rG   Úpos_embrM   Úappendr  r)  Ú
ValueErrorÚformatr–   Úblocks)rB   r¸   r  r3  r4  r5  r6  r7  r8  r9  r)   r!  r1   r*  r:  r;  rñ   rá   r,   r5   r6   rA   rD  Ú	block_idxrQ   s                           €r"   rE   zFastVitStage.__init__,  sá  ø€ õL 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆØ"'ˆÔàð 	,Ý(ð 
Ø*Ø"ØØ!Ø$Ø#Ø'Ø-ð
ð 
ð ð
ð 
ˆDŒOˆOð ˜'’>�>�>�>Ý œk™mœmˆDŒOàÐ$Ø(˜=¨ÐVÐVÀÐVÐSUÐVÐVˆDŒLˆLåœ;™=œ=ˆDŒLàˆÝ˜u™œð 	ð 	ˆIØ :Ò-Ð-Ø—’�mØð
à +Ø'Ø'Ø,Ø,¨YÔ7Ø+AØ#1ð
ð 
ð ð
ð 
ñ 
ô 
ð 
ð 
ð " [Ò0Ð0Ø—’�nØð	à'Ø'Ø)Ø,Ø,¨YÔ7Ø+Að	ð 	ð ð	ð 	ñ 	ô 	ð 	ð 	õ !Ø8×?Ò?Ð@PÑQÔQñô ð õ ”m VÐ,ˆŒˆˆr$   c                 óú   — |                       |¦  «        }|                      |¦  «        }| j        r4t          j                             ¦   «         st          | j        |¦  «        }n|                      |¦  «        }|S r¡   )r5  r@  r?  rj   rÙ   Úis_scriptingr   rD  rä   s     r"   rX   zFastVitStage.forwardŠ  si   € Ø�OŠO˜AÑÔˆØ�LŠL˜‰OŒOˆØÔ"ð 	­5¬9×+AÒ+AÑ+CÔ+Cð 	Ý˜tœ{¨AÑ.Ô.ˆAˆAà—’˜A‘”ˆAØˆr$   )r�   rŽ   r�   rF   r‘   rJ   r’   Ústrr“   r   r”   rÜ   r   r   r   rE   rX   r—   r˜   s   @r"   r2  r2  +  sx  ø€ € € € € ð  $Ø"'Ø#$Ø Ø15Ø Ø"Ø)+¬Ø*,¬.Ø$'Ø8;Ø6:Ø %Ø#(ØØð+\-ð \-àð\-ð ð\-ð ð	\-ð
 "ð\-ð ð\-ð  ð\-ð !ð\-ð ð\-ð $ B¤IÔ.ð\-ð ð\-ð ð\-ð ˜BœI”ð\-ð ˜RœYœð\-ð "ð\-ð  " $ u¤+¨uÐ"4Ô5ð!\-ð" %-¨U¤Oð#\-ð$ ð%\-ð& !ð'\-ð \-ð \-ð \-ð \-ð \-ð|ð ð ð ð ð ð r$   r2  c            3       óö  ‡ — e Zd ZU ej        j        e         ed<   	 ddddddddd	d
dddddddddddej	        ej
        dddfdedeedf         deedf         deedf         deedf         deedf         deedf         dededeeej                 df         ded ed!ed"ed#ed$ed%ed&eded'ed(ed)eej                 d*eej                 d+ed,df2ˆ fd-„Zd.ej        d,dfd/„Zej        j        d0„ ¦   «         Zej        j        dEd1„¦   «         Zej        j        dFd2„¦   «         Zej        j        d,ej        fd3„¦   «         ZdGded(ee         fd4„Z	 	 	 	 	 dHd6ej        d7eeeee         f                  d8ed9ed:ed;ed,eeej                 eej        eej                 f         f         fd<„Z	 	 	 dId7eeee         f         d>ed?efd@„Zd6ej        d,ej        fdA„ZdEd6ej        dBefdC„Z d6ej        d,ej        fdD„Z!ˆ xZ"S )Jr   Ú	fork_featr±   ©rh   rh   é   rh   ©r=  r=  r=  r=  ©é@   é€   é   é   ©r£   r£   r£   r£   )FTTT)FFFFéè  )NNNNr  rh   r·   rç   FTg       @ÚavgNÚin_chansÚlayers.Útoken_mixersÚ
embed_dimsÚ
mlp_ratiosÚdownsamplesÚse_downsamplesÚrepmixer_kernel_sizeÚnum_classesÚpos_embsr7  r8  Ú	drop_rater:  r;  rñ   rá   Ústem_use_scale_branchÚ	cls_ratioÚglobal_poolr*  r1   r,   r2   c                 ó^  •— t          ¦   «                              ¦   «          ||dœ}|rdn|	| _        || _        || _        g | _        t          ||d         ||fd|i|¤Ž| _        |d         }d}t          ||d¬¦  «        }g }t          t          |¦  «        ¦  «        D ]Í} ||          p|||          k    }!t          d$i d|“d||          “d	||          “d
|!“d||          “d|“d|“d|
|          “d||          “d|“d||          “d|“d|“d|“d||          “d|“d|“d|“|¤Ž}"|                     |"¦  «         ||          }|!r|dz  }| xj        t          |d|z  d| › �¬¦  «        gz  c_        ŒÎt          j        |Ž | _        t          | j        ¦  «        | _        |x| _        | _        | j        r‰g d¢| _        t+          | j        ¦  «        D ]j\  }#}$|#dk    r5t,          j                             dd ¦  «        r	 t          j        ¦   «         }%n |||#         fi |¤Ž}%d|$› �}&|                      |&|%¦  «         Œkn]t7          |d          |z  ¦  «        x| _        x| _        }'t9          d$|d          |'d!dd|d|dd"œ	|¤Ž| _        t=          |'|	f||d#œ|¤Ž| _        |                       | j!        ¦  «         d S )%Nr4   r   r/   r   T)Ú	stagewiser¸   r  r3  r5  r6  r7  r8  r9  r4  r)   r!  r1   r*  r:  r;  rñ   rá   r,   rh   r£   ústages.)Únum_chsÚ	reductionÚmodule©r   r   rh   r±   Ú
FORK_LAST3rõ   r|   r±   )	r'   r(   r)   r*   r    r,   r-   r1   r0   )Ú	pool_typer`  r   )"rD   rE   r^  rJ  rc  Úfeature_infor²   Ústemr   rM   r  r2  rA  ÚdictrF   r–   ÚstagesÚ
num_stagesr;   Úhead_hidden_sizeÚout_indicesÚ	enumerateÚosÚenvironÚgetrG   Ú
add_moduler’   r&   Ú
final_convr   Úheadr  r  ))rB   rV  rW  rX  rY  rZ  r[  r\  r]  r^  r_  r7  r8  r`  r:  r;  rñ   rá   ra  rJ  rb  rc  r*  r1   r,   r5   r6   rA   Úprev_dimr¿   Údprrp  rŠ   r5  ÚstageÚi_embÚi_layerÚlayerÚ
layer_nameÚfinal_featuresrQ   s)                                           €r"   rE   zFastVit.__init__›  sÿ  ø€ õ: 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆØ )Ð:˜1˜1¨{ˆÔØ"ˆŒØ&ˆÔØˆÔõ 'ØØ�qŒMØØð	
ð 
ð
 3ð
ð ð
ð 
ˆŒ	ð ˜a”=ˆØˆÝ'¨¸È$ÐOÑOÔOˆØˆÝ•s˜6‘{”{Ñ#Ô#ð 	eð 	eˆAØ$ QœÐD¨8°zÀ!´}Ò+DˆJÝ ð ð ð Ø�Hðà" 1œ˜ðð ˜Q”i�iðð &˜:ð	ð
 -¨QÔ/Ð/ðð !0 ðð (˜Kðð ' qœk˜kðð ".¨a¤ ðð 1Ð0ðð % Qœ-˜-ðð $˜)ðð &˜:ðð  .˜~ðð  # 1œv˜vðð  (>Ð'=ð!ð" (˜Kð#ð$  .˜~Øð'ð ˆEð* �MŠM˜%Ñ Ô Ð Ø! !”}ˆHØð Ø˜‘
�ØÐÔ¥$¨xÀ1ÀuÁ9ÐUbÐ_`ÐUbÐUbÐ"cÑ"cÔ"cÐ!dÑdÐÔÐÝ”m VÐ,ˆŒÝ˜dœkÑ*Ô*ˆŒØ4<Ð<ˆÔ˜DÔ1ð Œ>ð $	ð  ,˜|˜|ˆDÔÝ"+¨DÔ,<Ñ"=Ô"=ð 	3ð 	3‘��wØ˜A’:�:¥"¤*§.¢.°¸tÑ"DÔ"D�:ðõ œK™MœM�E�Eà&˜J z°%Ô'8Ð?Ð?¸BÐ?Ð?�EØ- GÐ-Ð-�
Ø—’ 
¨EÑ2Ô2Ð2Ð2ð	3õ JMÈZÐXZÌ^Ð^gÑMgÑIhÔIhÐhˆDÔÐh Ô 5¸Ý,ð Ø! "”~Ø&ØØØØ-ØØ#Ø"#ðð ð ðð ˆDŒOõ 'ØØðð &Ø#ð	ð ð
 ðð ˆDŒIð 	�
Š
�4Ô%Ñ&Ô&Ð&Ð&Ð&r$   r  c                 óþ   — t          |t          j        ¦  «        r^t          |j        d¬¦  «         t          |t          j        ¦  «        r0|j        �+t          j                             |j        d¦  «         dS dS dS dS )zInit. for classificationr
  r  Nr   )r}   rF   rÀ   r   r]   r:   r  r  r  s     r"   r  zFastVit._init_weights  s€   € å�a�œÑ#Ô#ð 	-Ý˜!œ(¨Ð-Ñ-Ô-Ð-Ý˜!�RœYÑ'Ô'ð -¨A¬FÐ,>Ý”×!Ò! !¤&¨!Ñ,Ô,Ð,Ð,Ð,ð	-ð 	-ð-ð -Ð,>Ð,>r$   c                 ó   — t          ¦   «         S r¡   )Úset©rB   s    r"   Úno_weight_decayzFastVit.no_weight_decay  s   € å‰uŒuˆr$   c                 ó0   — t          d|rdng d¢¬¦  «        S )Nz^stemz^stages\.(\d+)))z^stages\.(\d+).downsample©r   )z^stages\.(\d+).pos_embr‰  )z^stages\.(\d+)\.\w+\.(\d+)N)rn  rD  )ro  )rB   Úcoarses     r"   Úgroup_matcherzFastVit.group_matcher"  s9   € åØØ(.ð Ð$Ð$ð 5ð 5ð 5ð
ñ 
ô 
ð 	
r$   c                 ó(   — | j         D ]	}||_        Œ
d S r¡   )rp  r?  )rB   ÚenableÚss      r"   Úset_grad_checkpointingzFastVit.set_grad_checkpointing-  s(   € à”ð 	*ð 	*ˆAØ#)ˆAÔ Ð ð	*ð 	*r$   c                 ó   — | j         j        S r¡   )rz  Úfcr†  s    r"   Úget_classifierzFastVit.get_classifier2  s   € àŒyŒ|Ðr$   c                 óZ   — t          | ¦  «        }|| _         | j        j        ||fi |¤Ž d S r¡   )r   r^  rz  Úreset)rB   r^  rc  rA   s       r"   Úreset_classifierzFastVit.reset_classifier6  s;   € Ý˜dÑ#Ô#ˆØ&ˆÔØˆŒ	Œ˜ [Ð7Ð7°BÐ7Ð7Ð7Ð7Ð7r$   ÚNCHWrR   Úindicesrõ   Ú
stop_earlyÚ
output_fmtÚintermediates_onlyc                 óÚ  — |dv s
J d¦   «         ‚g }t          t          | j        ¦  «        |¦  «        \  }}	|                      |¦  «        }| j        dz
  }
t
          j                             ¦   «         s|s| j        }n| j        d|	dz   …         }d}t          |¦  «        D ])\  }} ||¦  «        }||v r| 	                    |¦  «         Œ*|r|S ||
k    r|  
                    |¦  «        }||fS )aï   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )r–  zOutput shape must be NCHW.r   Nr   )r   r  rp  rn  rq  rj   rÙ   rG  rt  rA  ry  )rB   rR   r—  rõ   r˜  r™  rš  ÚintermediatesÚtake_indicesÚ	max_indexÚlast_idxrp  Úfeat_idxr}  s                 r"   Úforward_intermediateszFastVit.forward_intermediates;  s  € ð* ˜YÐ&Ð&Ð&Ð(DÑ&Ô&Ð&ØˆÝ"6µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�ið �IŠI�a‰LŒLˆØ”? QÑ&ˆÝŒ9×!Ò!Ñ#Ô#ð 	1¨:ð 	1Ø”[ˆFˆFà”[  )¨a¡- Ô0ˆFØˆÝ(¨Ñ0Ô0ð 	(ð 	(‰OˆH�eØ��a‘”ˆAØ˜<Ð'Ð'Ø×$Ò$ QÑ'Ô'Ð'øàð 	!Ø Ð à�xÒÐØ—’ Ñ"Ô"ˆAà�-ÐÐr$   r   Ú
prune_normÚ
prune_headc                 ó®   — t          t          | j        ¦  «        |¦  «        \  }}| j        d|dz   …         | _        |r|                      dd¦  «         |S )z@ Prune layers not required for specified intermediates.
        Nr   r   Ú )r   r  rp  r•  )rB   r—  r¢  r£  r�  rž  s         r"   Úprune_intermediate_layersz!FastVit.prune_intermediate_layersi  s\   € õ #7µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�iØ”k . 9¨q¡= .Ô1ˆŒØð 	)Ø×!Ò! ! RÑ(Ô(Ð(ØÐr$   c                 ó@  — |                       |¦  «        }g }t          | j        ¦  «        D ]S\  }} ||¦  «        }| j        r<|| j        v r3t          | d|› �¦  «        } ||¦  «        }|                     |¦  «         ŒT| j        r|S |                      |¦  «        }|S )Nrõ   )rn  rt  rp  rJ  rs  ÚgetattrrA  ry  )rB   rR   ÚoutsÚidxÚblockr*  Úx_outs          r"   Úforward_featureszFastVit.forward_featuresw  s²   € à�IŠI�a‰LŒLˆØˆÝ# D¤KÑ0Ô0ð 	'ð 	'‰JˆC�Ø��a‘”ˆAØŒ~ð 'Ø˜$Ô*Ð*Ð*Ý!(¨¨|°c¨|¨|Ñ!<Ô!<�JØ&˜J q™MœM�EØ—K’K Ñ&Ô&Ð&øØŒ>ð 	àˆKØ�OŠO˜AÑÔˆØˆr$   Ú
pre_logitsc                 ó^   — |r|                       |d¬¦  «        n|                       |¦  «        S )NT)r®  )rz  )rB   rR   r®  s      r"   Úforward_headzFastVit.forward_headˆ  s,   € Ø0:ÐLˆt�yŠy˜ tˆyÑ,Ô,Ð,ÀÇ	Â	È!ÁÄÐLr$   c                 ól   — |                       |¦  «        }| j        r|S |                      |¦  «        }|S r¡   )r­  rJ  r°  rä   s     r"   rX   zFastVit.forward‹  s<   € Ø×!Ò! !Ñ$Ô$ˆØŒ>ð 	ØˆHØ×Ò˜aÑ Ô ˆØˆr$   ©F)Tr¡   )NFFr–  F)r   FT)#r�   rŽ   r�   rj   rÙ   rÚ   r“   rÛ   rF   rJ   r‘   r’   r   rH  rÜ   r   r”   r   rE   r  Úignorer‡  r‹  r�  r’  r•  r•   r   r   r¡  r¦  r­  r°  rX   r—   r˜   s   @r"   r   r   ”  sv  ø€ € € € € € ØŒyŒ˜tÔ$Ð$Ð$Ñ$ðð Ø&2Ø,\Ø*=Ø,4Ø,EØ/KØ()Ø#Ø8CØ#$Ø Ø"Ø$'Ø$'Ø,0Ø %Ø*.Ø#Ø"Ø$Ø*,¬.Ø)+¬Ø#(ØØð7z'ð z'àðz'ð ˜#˜s˜(”Oðz'ð    S œ/ð	z'ð
 ˜c 3˜hœðz'ð ˜e S˜jÔ)ðz'ð ˜t S˜yÔ)ðz'ð " $¨ )Ô,ðz'ð #&ðz'ð ðz'ð ˜H R¤YÔ/°Ð4Ô5ðz'ð !ðz'ð ðz'ð ðz'ð "ðz'ð  "ð!z'ð" %*ð#z'ð$ ð%z'ð& $(ð'z'ð( ð)z'ð* ð+z'ð, ð-z'ð. ˜RœYœð/z'ð0 ˜BœI”ð1z'ð2 !ð3z'ð8 
ð9z'ð z'ð z'ð z'ð z'ð z'ðx-˜rœyð -¨Tð -ð -ð -ð -ð „YÔðð ñ Ôðð „YÔð
ð 
ð 
ñ Ôð
ð „YÔð*ð *ð *ñ Ôð*ð „YÔð ¤	ð ð ð ñ Ôðð8ð 8¨Cð 8¸hÀs¼mð 8ð 8ð 8ð 8ð 8<ØØ$Ø$Ø',ð, ð , àŒ|ð, ð ˜e C¨¨c¬ NÔ3Ô4ð, ð ð	, ð
 ð, ð ð, ð !%ð, ð 
ˆt�E”LÔ! 5¨¬°t¸E¼LÔ7IÐ)IÔ#JÐJÔ	Kð, ð , ð , ð , ð` ./Ø$Ø#ð	ð à˜3  S¤	˜>Ô*ðð ðð ð	ð ð ð ð %¤,ð °5´<ð ð ð ð ð"Mð M˜eœlð M¸ð Mð Mð Mð Mð˜œð ¨%¬,ð ð ð ð ð ð ð ð r$   r¥  c                 ó6   — | dddddt           dt          ddd	œ|¥S )
NrT  )r±   rQ  rQ  )é   rµ  gÍÌÌÌÌÌì?Úbicubiczfastvit-license)ústem.0.conv_kxk.0.convzstem.0.conv_scale.convzhead.fc)Úurlr^  Ú
input_sizeÚ	pool_sizeÚcrop_pctÚinterpolationÚmeanÚlicenser‹   Ú
first_convÚ
classifier)r	   r
   )r¸  Úkwargss     r"   Ú_cfgrÂ  “  s<   € àØØ#ØØØ"Ý%Ø$Ý#ØJØðð ð ðð r$   zfastvit_t8.apple_in1kztimm/)Ú	hf_hub_idzfastvit_t12.apple_in1kzfastvit_s12.apple_in1kzfastvit_sa12.apple_in1kzfastvit_sa24.apple_in1kzfastvit_sa36.apple_in1kzfastvit_ma36.apple_in1kgffffffî?)rÃ  r»  zfastvit_t8.apple_dist_in1kzfastvit_t12.apple_dist_in1kzfastvit_s12.apple_dist_in1kzfastvit_sa12.apple_dist_in1kzfastvit_sa24.apple_dist_in1kzfastvit_sa36.apple_dist_in1kzfastvit_ma36.apple_dist_in1kzfastvit_mci0.apple_mclipzapple/mobileclip_s0_timmzXhttps://docs-assets.developer.apple.com/ml-research/datasets/mobileclip/mobileclip_s0.ptrR  )r·   r·   r·   )ç      ð?rÄ  rÄ  z
apple-amlr)rÃ  r¸  r»  r^  r½  r‹   r¾  zfastvit_mci1.apple_mclipzapple/mobileclip_s1_timmzXhttps://docs-assets.developer.apple.com/ml-research/datasets/mobileclip/mobileclip_s1.ptzfastvit_mci2.apple_mclipzapple/mobileclip_s2_timmzXhttps://docs-assets.developer.apple.com/ml-research/datasets/mobileclip/mobileclip_s2.ptrÄ  )rÃ  r»  r^  r½  r‹   r¾  é   )r£   r£   r·  )rÃ  r»  r^  r½  r‹   rº  r¿  r¾  )z!fastvit_mci0.apple_mclip2_dfndr2bz!fastvit_mci2.apple_mclip2_dfndr2bz!fastvit_mci3.apple_mclip2_dfndr2bz!fastvit_mci4.apple_mclip2_dfndr2bc                 ó\  — d| v r| S d| v rd„ |                       ¦   «         D ¦   «         S |                      d| ¦  «        } d| v rd}nd}dd	l}dd	l}g }|                       ¦   «         D ]R\  }}|                     d
|¦  «        }|r5|                     t          |                     d¦  «        ¦  «        ¦  «         ŒSt          t          t          |¦  «        ¦  «        ¦  «        }i }	|                       ¦   «         D �]²\  }}|r||vrŒ|                     |d¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd|¦  «        }|                     d ¦  «        r|                     d d!¦  «        }|                     d"d#¦  «        }|                     d$¦  «        r•|d%k    ryt          |j        d&¦  «        rdt#          |j        j        t&          j        ¦  «        r@|                     d%d'¦  «        }|j        }t-          j        |j        d         ¦  «        |	d(<   n|                     d$d)¦  «        }|                     d*|¦  «        }d+\  }
}|r8t          |                     d,¦  «        ¦  «        }|                     ||¦  «        }
|
�kd-|› �}d.|
› �}|d/z   |v r|                     |d/z   |d0z   ¦  «        }n=|d1z   |v r|                     |d1z   |d2z   ¦  «        }n|                     ||d3z   ¦  «        }||	|<   �Œ´|	S )4z$ Remap original checkpoints -> timm zstem.0.conv_kxk.0.conv.weightz1module.visual.trunk.stem.0.conv_kxk.0.conv.weightc                 ól   — i | ]1\  }}|                      d ¦  «        ¯|                     dd¦  «        |“Œ2S )zmodule.visual.trunkzmodule.visual.trunk.r¥  )Ú
startswithÚreplace)r?   rÖ   r×   s      r"   ú
<dictcomp>z(checkpoint_filter_fn.<locals>.<dictcomp>	  sB   € ÐÐÐ¹T¸QÀÐ[\×[gÒ[gÐh}Ñ[~Ô[~Ð�—	’	Ð0°"Ñ5Ô5°qÐÐÐr$   Ú
state_dictz8image_encoder.model.patch_embed.0.rbr_conv.0.conv.weightzimage_encoder.model.r¥  r   Nz^(.*?)network\.(\d+)\.proj.*rh   Úpatch_embedrn  Úrbr_convrN   Ú	rbr_scalerO   Úrbr_skiprK   Úconv_expry  Ú
lkb_originrž   Úconvffnr&  z	se.reducezse.fc1z	se.expandzse.fc2zlayer_scale_([0-9])zlayer_scale_\1.gammar÷   zlayer_scale.gammaÚ	dist_headÚ	head_distzhead.z	head.projr‘  zhead.fc.weightzhead.fc.biaszhead.fc.z^network\.(\d+))NNr   znetwork.rf  z.projz.downsample.projz.pez.pos_emb.pos_encz.blocks)Úitemsrw  ÚreÚbisectÚmatchrA  r’   ÚgroupÚlistÚsortedr…  rÉ  ÚsubÚendswithrÈ  rb   rz  r}   r‘  rF   rÀ   ÚTrj   rƒ   rÇ   Úbisect_right)rË  ÚmodelÚprefixrÖ  r×  Ú
stage_endsrÖ   r×   rØ  Úout_dictÚ	stage_idxÚnet_idxÚ
net_prefixÚstage_prefixs                 r"   Úcheckpoint_filter_fnrè    s×  € à&¨*Ð4Ð4ØÐà:¸jÐHÐHØÐÀZ×EUÒEUÑEWÔEWÐÑÔÐà—’ ¨jÑ9Ô9€JØAÀZÐOÐOà'ˆˆàˆà€I€I€IØ€M€M€Mð €JØ× Ò Ñ"Ô"ð 3ð 3‰ˆˆ1Ø—’Ð8¸!Ñ<Ô<ˆØð 	3Ø×Ò�c %§+¢+¨a¡.¤.Ñ1Ô1Ñ2Ô2Ð2øÝ•f�S ™_œ_Ñ-Ô-Ñ.Ô.€Jà€HØ× Ò Ñ"Ô"ð -ñ -‰ˆˆ1Øð 	&Ø˜QˆˆØØ—	’	˜& "Ñ%Ô%ˆAð �IŠI�m VÑ,Ô,ˆØ�IŠI�j *Ñ-Ô-ˆØ�IŠI�k <Ñ0Ô0ˆØ�IŠI�j *Ñ-Ô-ˆØ�IŠI�j ,Ñ/Ô/ˆØ�IŠI�l LÑ1Ô1ˆØ�IŠI�i Ñ'Ô'ˆØ�IŠI�k 8Ñ,Ô,ˆØ�IŠI�k 8Ñ,Ô,ˆØ�FŠFÐ)Ð+BÀAÑFÔFˆØ�:Š:�mÑ$Ô$ð 	>Ø—	’	˜-Ð)<Ñ=Ô=ˆAØ�IŠI�k ;Ñ/Ô/ˆØ�<Š<˜Ñ Ô ð 	3Ø�KÒÐ¥G¨E¬J¸Ñ$=Ô$=ÐÅ*ÈUÌZÌ]Õ\^Ô\eÑBfÔBfÐà—I’I˜kÐ+;Ñ<Ô<�Ø”C�Ý+0¬;°q´w¸q´zÑ+BÔ+B�˜Ñ(Ð(à—I’I˜g zÑ2Ô2�ð —’Ð+¨QÑ/Ô/ˆØ'Ñˆ	�7Øð 	AÝ˜%Ÿ+š+ a™.œ.Ñ)Ô)ˆGØ×+Ò+¨J¸Ñ@Ô@ˆIØÐ Ø- GÐ-Ð-ˆJØ0 YÐ0Ð0ˆLØ˜GÑ# qÐ(Ð(Ø—I’I˜j¨7Ñ2°LÐCUÑ4UÑVÔV��Ø˜eÑ# qÐ(Ð(Ø—I’I˜j¨5Ñ0°,ÐASÑ2SÑTÔT��à—I’I˜j¨,¸Ñ*BÑCÔC�àˆ�‰‰Ø€Or$   c                 óˆ   — |                      dd¦  «        }t          t          | |ft          t	          d|¬¦  «        dœ|¤Ž}|S )Nrs  rj  T)Úflatten_sequentialrs  )Úpretrained_filter_fnÚfeature_cfg)Úpopr   r   rè  ro  )ÚvariantÚ
pretrainedrÁ  rs  rà  s        r"   Ú_create_fastvitrð  O  s]   € Ø—*’*˜]¨LÑ9Ô9€KÝ ÝØØðõ 2Ý¨D¸kÐJÑJÔJðð ð ðð €Eð €Lr$   c           	      óZ   — t          dddd¬¦  «        }t          dd| it          |fi |¤Ž¤ŽS )	z%Instantiate FastViT-T8 model variant.)rh   rh   r£   rh   )é0   é`   éÀ   é€  ©r±   r±   r±   r±   rM  ©rW  rY  rZ  rX  Ú
fastvit_t8rï  )rø  ©ro  rð  ©rï  rÁ  Ú
model_argss      r"   rø  rø  \  sP   € õ ØØ%ØØEð	ñ ô €Jõ Ð]Ð]°JÐ]Å$ÀzÐB\ÐB\ÐU[ÐB\ÐB\Ð]Ð]Ð]r$   c           	      óZ   — t          dddd¬¦  «        }t          dd| it          |fi |¤Ž¤ŽS )	z&Instantiate FastViT-T12 model variant.rK  rN  rö  rM  r÷  Úfastvit_t12rï  )rý  rù  rú  s      r"   rý  rý  h  óP   € õ ØØ&ØØEð	ñ ô €Jõ Ð^Ð^°ZÐ^Å4È
ÐC]ÐC]ÐV\ÐC]ÐC]Ð^Ð^Ð^r$   c           	      óZ   — t          dddd¬¦  «        }t          dd| it          |fi |¤Ž¤ŽS )	z&Instantiate FastViT-S12 model variant.rK  rN  rS  rM  r÷  Úfastvit_s12rï  )r   rù  rú  s      r"   r   r   t  rþ  r$   c                 óŒ   — t          ddddddt          t          d¬¦  «        fd¬¦  «        }t          dd
| it          |fi |¤Ž¤ŽS )z'Instantiate FastViT-SA12 model variant.rK  rN  rS  Nr  ©r  ©r=  r=  r=  r>  ©rW  rY  rZ  r_  rX  Úfastvit_sa12rï  )r  ©ro  r   r  rð  rú  s      r"   r  r  €  sk   € õ ØØ&ØØ˜˜d¥GÕ,@ÐPVÐ$WÑ$WÔ$WÐXØFðñ ô €Jõ Ð_Ð_°jÐ_ÅDÈÐD^ÐD^ÐW]ÐD^ÐD^Ð_Ð_Ð_r$   c                 óŒ   — t          ddddddt          t          d¬¦  «        fd¬¦  «        }t          dd
| it          |fi |¤Ž¤ŽS )z'Instantiate FastViT-SA24 model variant.)r£   r£   é   r£   rN  rS  Nr  r  r  r  Úfastvit_sa24rï  )r	  r  rú  s      r"   r	  r	  �  ók   € õ ØØ&ØØ˜˜d¥GÕ,@ÐPVÐ$WÑ$WÔ$WÐXØFðñ ô €Jõ Ð_Ð_°jÐ_ÅDÈÐD^ÐD^ÐW]ÐD^ÐD^Ð_Ð_Ð_r$   c                 óŒ   — t          ddddddt          t          d¬¦  «        fd¬¦  «        }t          dd
| it          |fi |¤Ž¤ŽS )z'Instantiate FastViT-SA36 model variant.©rL  rL  é   rL  rN  rS  Nr  r  r  r  Úfastvit_sa36rï  )r  r  rú  s      r"   r  r  š  r
  r$   c                 óŒ   — t          ddddddt          t          d¬¦  «        fd¬¦  «        }t          dd
| it          |fi |¤Ž¤ŽS )z'Instantiate FastViT-MA36 model variant.r  )éL   é˜   i0  i`  rS  Nr  r  r  r  Úfastvit_ma36rï  )r  r  rú  s      r"   r  r  §  r
  r$   c                 ó�   — t          dddddddt          t          d¬¦  «        fdd	¬
¦  «        }t          dd| it          |fi |¤Ž¤ŽS )zInstantiate MCi0 model variant.)rh   rL  é
   rh   rN  rö  ©FFTTNr  r  r  T©rW  rY  rZ  r\  r_  rX  rá   Úfastvit_mci0rï  )r  r  rú  s      r"   r  r  ´  sq   € õ ØØ&ØØ1Ø˜˜d¥GÕ,@ÐPVÐ$WÑ$WÔ$WÐXØFØðñ ô €Jõ Ð_Ð_°jÐ_ÅDÈÐD^ÐD^ÐW]ÐD^ÐD^Ð_Ð_Ð_r$   c                 ó�   — t          dddddddt          t          d¬¦  «        fdd	¬
¦  «        }t          dd| it          |fi |¤Ž¤ŽS )zInstantiate MCi1 model variant.)r£   r  é   r£   rN  rö  r  Nr  r  r  Tr  Úfastvit_mci1rï  )r  r  rú  s      r"   r  r  Ã  óq   € õ ØØ&ØØ1Ø˜˜d¥GÕ,@ÐPVÐ$WÑ$WÔ$WÐXØFØðñ ô €Jõ Ð_Ð_°jÐ_ÅDÈÐD^ÐD^ÐW]ÐD^ÐD^Ð_Ð_Ð_r$   c                 ó�   — t          dddddddt          t          d¬¦  «        fdd	¬
¦  «        }t          dd| it          |fi |¤Ž¤ŽS )zInstantiate MCi2 model variant.)r£   r  é   r£   )éP   é    i@  i€  rö  r  Nr  r  r  Tr  Úfastvit_mci2rï  )r   r  rú  s      r"   r   r   Ò  r  r$   c                 óì   — t          ddddddddt          t          d¬¦  «        t          t          d¬¦  «        fd	d
t          t          d¬¦  «        d¬¦
  «
        }t	          dd| it          |fi |¤Ž¤Ž}|S )zInstantiate L model variant.)rh   r  r  r£   rh   )ró  rô  rõ  rÅ  i   ©r£   r£   r£   r£   r£   ©FFFFF©FTTTTNr  r  ©r=  r=  r=  r>  r>  Trç   ©r‚   F©
rW  rY  rZ  r\  r[  r_  rX  rá   r*  ra  Úfastvit_mci3rï  )r(  ©ro  r   r  r   rð  ©rï  rÁ  rû  rà  s       r"   r(  r(  á  s£   € õ Ø Ø,Ø"Ø:Ø3àØØÝÕ(¸Ð?Ñ?Ô?ÝÕ(¸Ð?Ñ?Ô?ð
ð TØÝ�;¨DÐ1Ñ1Ô1Ø#ð!ñ ô €Jõ$ Ð`Ð`°zÐ`ÅTÈ*ÐE_ÐE_ÐX^ÐE_ÐE_Ð`Ð`€EØ€Lr$   c                 óì   — t          ddddddddt          t          d¬¦  «        t          t          d¬¦  «        fd	d
t          t          d¬¦  «        d¬¦
  «
        }t	          dd| it          |fi |¤Ž¤Ž}|S )zInstantiate XL model variant.)rh   r  r  r£   r£   )rP  rQ  rR  i   i   r"  r#  r$  Nr  r  r%  Trç   r&  Fr'  Úfastvit_mci4rï  )r,  r)  r*  s       r"   r,  r,  ú  s£   € õ Ø Ø.Ø"Ø:Ø3àØØÝÕ(¸Ð?Ñ?Ô?ÝÕ(¸Ð?Ñ?Ô?ð
ð TØÝ�;¨DÐ1Ñ1Ô1Ø#ð!ñ ô €Jõ& Ð`Ð`°zÐ`ÅTÈ*ÐE_ÐE_ÐX^ÐE_ÐE_Ð`Ð`€EØ€Lr$   )r¥  r²  )Iru  Ú	functoolsr   Útypingr   r   r   r   r   rj   Útorch.nnrF   Ú	timm.datar	   r
   r   r   Útimm.layersr   r   r   r   r   r   r   r   r   r   Ú_builderr   Ú	_featuresr   Ú_manipulater   Ú	_registryr   r   Ú__all__r#   r”   r&   rš   r‘   r’   r“   r–   r²   r´   rÞ   ræ   rð   rÿ   r  r  r)  r2  r   rÂ  Údefault_cfgsrè  rð  rø  rý  r   r  r	  r  r  r  r  r   r(  r,  r   r$   r"   ú<module>r8     sd	  ðð 
€	€	€	Ø Ð Ð Ð Ð Ð Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5à €€€Ø Ð Ð Ð Ð Ð à dÐ dÐ dÐ dÐ dÐ dÐ dÐ dÐ dÐ dÐ dÐ dðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð +Ð *Ð *Ð *Ð *Ð *Ø +Ð +Ð +Ð +Ð +Ð +Ø 'Ð 'Ð 'Ð 'Ð 'Ð 'Ø <Ð <Ð <Ð <Ð <Ð <Ð <Ð <àˆ+€ð&ð &ð &ðu=ð u=ð u=ð u=ð u=�R”Yñ u=ô u=ð u=ðp[=ð [=ð [=ð [=ð [=˜RœYñ [=ô [=ð [=ðB &(¤WØ$Ø!%ØØð4ð 4Øð4àð4ð ˜œ	”?ð4ð ð	4ð
 ð4ð „]ð4ð 4ð 4ð 4ðnDð Dð Dð Dð D�”	ñ Dô Dð DðN6ð 6ð 6ð 6ð 6�”ñ 6ô 6ð 6ðrFð Fð Fð Fð F�2”9ñ Fô Fð Fð"s(ð s(ð s(ð s(ð s(ˆrŒyñ s(ô s(ð s(ðl5ð 5ð 5ð 5ð 5ˆbŒiñ 5ô 5ð 5ðpv$ð v$ð v$ð v$ð v$˜2œ9ñ v$ô v$ð v$ðr;ð ;ð ;ð ;ð ;�B”Iñ ;ô ;ð ;ð|9ð 9ð 9ð 9ð 9�R”Yñ 9ô 9ð 9ðxfð fð fð fð f�2”9ñ fô fð fðR|ð |ð |ð |ð |ˆbŒiñ |ô |ð |ð~ð ð ð ð" %Ð$ð \&Ø˜T˜TØðñ ô ð\&ð ˜d˜dØðñ ô ð\&ð ˜d˜dØðñ ô ð\&ð ˜t˜tØð ñ  ô  ð\&ð ˜t˜tØð ñ  ô  ð\&ð ˜t˜tØð ñ  ô  ð\&ð ˜t˜tØØð ñ  ô  ð\&ð& ! $ $Øð#ñ #ô #ð'\&ð* " 4 4Øð$ñ $ô $ð+\&ð0 " 4 4Øð$ñ $ô $ð1\&ð4 # D DØð%ñ %ô %ð5\&ð8 # D DØð%ñ %ô %ð9\&ð< # D DØð%ñ %ô %ð=\&ðB # D DØØð%ñ %ô %ðC\&ðL   Ø,ØfØØØ˜|°\ð!ñ !ô !ðM\&ðZ   Ø,ØfØØØ˜|°\ð!ñ !ô !ð[\&ðh   Ø,ØfØØØ˜|°\ð!ñ !ô !ði\&ðx *.¨ØØØØ˜|Øð*ñ *ô *ð *.¨ØØØØ˜|Øð*ñ *ô *ð *.¨ØØØØ ?ØØ+Øð*ñ *ô *ð *.¨ØØØØ ?ØØ+Øð*ñ *ô *ðg\&ð \&ð \&ñ \ô \€ð~Ið Ið IðX
ð 
ð 
ð 
ð ð^ð ^ð ^ñ „ð^ð ð_ð _ð _ñ „ð_ð ð_ð _ð _ñ „ð_ð ð	`ð 	`ð 	`ñ „ð	`ð ð	`ð 	`ð 	`ñ „ð	`ð ð	`ð 	`ð 	`ñ „ð	`ð ð	`ð 	`ð 	`ñ „ð	`ð ð`ð `ð `ñ „ð`ð ð`ð `ð `ñ „ð`ð ð`ð `ð `ñ „ð`ð ðð ð ñ „ðð0 ðð ð ñ „ðð ð r$   