§
    ·¯¸j¤j  ã                   óâ  — d Z ddlmZmZmZmZmZ ddlZddlm	Z	 ddlm
Z ddlmZmZmZmZmZmZmZmZmZmZmZmZ g d¢Zee	j                 Zdee         defd	„Z G d
„ de	j        ¦  «        Z G d„ de	j        ¦  «        Z G d„ de	j        ¦  «        Z  G d„ de	j        ¦  «        Z! G d„ de	j        ¦  «        Z" G d„ de	j        ¦  «        Z# G d„ de!¦  «        Z$ G d„ de	j        ¦  «        Z%dS )z[ EfficientNet, MobileNetV3, etc Blocks

Hacked together by / Copyright 2019, Ross Wightman
é    )ÚCallableÚDictÚOptionalÚTypeÚUnionN)Ú
functional)Úcreate_conv2dÚDropPathÚmake_divisibleÚcreate_act_layerÚ	create_aaÚ	to_2tupleÚ	LayerTypeÚConvNormActÚget_norm_act_layerÚMultiQueryAttention2dÚAttention2dÚLayerScale2d)ÚSqueezeExciteÚ	ConvBnActÚDepthwiseSeparableConvÚInvertedResidualÚCondConvResidualÚEdgeResidualÚUniversalInvertedResidualÚMobileAttentionÚ
group_sizeÚchannelsc                 ó*   — | sdS || z  dk    sJ ‚|| z  S )Né   r   © )r   r   s     úp/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/models/_efficientnet_blocks.pyÚ
num_groupsr#   "   s2   € Øð &Øˆqð ˜*Ñ$¨Ò)Ð)Ð)Ð)Ø˜:Ñ%Ð%ó    c                   ó�   ‡ — e Zd ZdZddej        ej        ddddfdedede	e         de
de
d	e	e
         d
e	e         fˆ fd„Zd„ Zˆ xZS )r   aç   Squeeze-and-Excitation w/ specific features for EfficientNet/MobileNet family

    Args:
        in_chs (int): input channels to layer
        rd_ratio (float): ratio of squeeze reduction
        act_layer (nn.Module): activation layer of containing block
        gate_layer (Callable): attention gate function
        force_act_layer (nn.Module): override block's activation fn if this is set/bound
        rd_round_fn (Callable): specify a fn to calculate rounding of reduced chs
    g      Ð?NÚin_chsÚrd_ratioÚrd_channelsÚ	act_layerÚ
gate_layerÚforce_act_layerÚrd_round_fnc
                 óL  •— ||	dœ}
t          ¦   «                              ¦   «          |€|pt          } |||z  ¦  «        }|p|}t          j        ||dfddi|
¤Ž| _        t          |d¬¦  «        | _        t          j        ||dfddi|
¤Ž| _        t          |¦  «        | _	        d S )N©ÚdeviceÚdtyper    ÚbiasT)Úinplace)
ÚsuperÚ__init__ÚroundÚnnÚConv2dÚconv_reducer   Úact1Úconv_expandÚgate)Úselfr&   r'   r(   r)   r*   r+   r,   r/   r0   ÚddÚ	__class__s              €r"   r4   zSqueezeExcite.__init__7   sÃ   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØÐØ%Ð.­ˆKØ%˜+ f¨xÑ&7Ñ8Ô8ˆKØ#Ð0 yˆ	Ýœ9 V¨[¸!ÐMÐMÀ$ÐMÈ"ÐMÐMˆÔÝ$ Y¸Ð=Ñ=Ô=ˆŒ	Ýœ9 [°&¸!ÐMÐMÀ$ÐMÈ"ÐMÐMˆÔÝ$ ZÑ0Ô0ˆŒ	ˆ	ˆ	r$   c                 óÞ   — |                      dd¬¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }||                      |¦  «        z  S )N)é   é   T)Úkeepdim)Úmeanr8   r9   r:   r;   )r<   ÚxÚx_ses      r"   ÚforwardzSqueezeExcite.forwardN   sa   € Ø�vŠv�f dˆvÑ+Ô+ˆØ×Ò Ñ%Ô%ˆØ�yŠy˜‰ŒˆØ×Ò Ñ%Ô%ˆØ�4—9’9˜T‘?”?Ñ"Ð"r$   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r6   ÚReLUÚSigmoidÚintÚfloatr   r   r   r4   rF   Ú__classcell__©r>   s   @r"   r   r   +   sË   ø€ € € € € ð	ð 	ð #Ø)-Ø#%¤7Ø$&¤JØ37Ø.2ØØð1ð 1àð1ð ð1ð " #œð	1ð
 !ð1ð "ð1ð & iÔ0ð1ð " (Ô+ð1ð 1ð 1ð 1ð 1ð 1ð.#ð #ð #ð #ð #ð #ð #r$   r   c                   ó´   ‡ — e Zd ZdZdddddej        ej        ddddfded	ed
ededededeee	f         de
dee         dedee         defˆ fd„Zd„ Zd„ Zˆ xZS )r   z@ Conv + Norm Layer + Activation w/ optional skip connection
    r    r   Ú FNç        r&   Úout_chsÚkernel_sizeÚstrideÚdilationr   Úpad_typeÚskipr)   Ú
norm_layerÚaa_layerÚdrop_path_ratec                 ó¤  •— ||dœ}t          ¦   «                              ¦   «          t          |
|	¦  «        }t          ||¦  «        }|o|dk    o||k    | _        |d uo|dk    }t          |||f|rdn||||dœ|¤Ž| _         ||fddi|¤Ž| _        t          |f|||dœ|¤Ž| _	        |rt          |¦  «        nt          j        ¦   «         | _        d S )Nr.   r    ©rV   rW   ÚgroupsÚpaddingr2   T©r   rV   Úenable)r3   r4   r   r#   Úhas_skipr	   ÚconvÚbn1r   Úaar
   r6   ÚIdentityÚ	drop_path)r<   r&   rT   rU   rV   rW   r   rX   rY   r)   rZ   r[   r\   r/   r0   r=   Únorm_act_layerr_   Úuse_aar>   s                      €r"   r4   zConvBnAct.__init__Y   s  ø€ ð" ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ+¨J¸	ÑBÔBˆÝ˜J¨Ñ/Ô/ˆØÐB ¨1¢ÐB°¸7Ò1BˆŒØ Ð%Ð4¨&°1ª*ˆå!ØØØð	
ð Ð*�1�1 FØØØð	
ð 	
ð ð	
ð 	
ˆŒ	ð "�> 'Ð>Ð>°4Ð>¸2Ð>Ð>ˆŒÝ˜HÐ[¨w¸vÈfÐ[Ð[ÐXZÐ[Ð[ˆŒØ5CÐV� .Ñ1Ô1Ð1ÍÌÉÌˆŒˆˆr$   c                 ó|   — |dk    rt          dd| j        j        ¬¦  «        S t          d| j        j        ¬¦  «        S )NÚ	expansionre   rF   ©ÚmoduleÚ	hook_typeÚnum_chsrR   ©rn   rp   )Údictrd   Úout_channels©r<   Úlocations     r"   Úfeature_infozConvBnAct.feature_info   s?   € Ø�{Ò"Ð"Ý˜u°	À4Ä9ÔCYÐZÑZÔZÐZå˜r¨4¬9Ô+AÐBÑBÔBÐBr$   c                 óÆ   — |}|                       |¦  «        }|                      |¦  «        }|                      |¦  «        }| j        r|                      |¦  «        |z   }|S ©N)rd   re   rf   rc   rh   ©r<   rD   Úshortcuts      r"   rF   zConvBnAct.forward…   sY   € ØˆØ�IŠI�a‰LŒLˆØ�HŠH�Q‰KŒKˆØ�GŠG�A‰JŒJˆØŒ=ð 	-Ø—’˜qÑ!Ô! HÑ,ˆAØˆr$   )rG   rH   rI   rJ   r6   rK   ÚBatchNorm2drM   r   ÚstrÚboolr   r   rN   r4   rv   rF   rO   rP   s   @r"   r   r   V   s0  ø€ € € € € ðð ð ØØØ(*ØØ-/¬WØ$&¤NØ,0Ø$&ØØð$Wð $Wàð$Wð ð$Wð ð	$Wð
 ð$Wð ð$Wð ð$Wð ˜C ˜H”oð$Wð ð$Wð   	Ô*ð$Wð "ð$Wð ˜yÔ)ð$Wð "ð$Wð $Wð $Wð $Wð $Wð $WðLCð Cð Cðð ð ð ð ð ð r$   r   c            !       ó¾   ‡ — e Zd ZdZdddddddddej        ej        dddddfd	ed
edededededede	dede	dede
de
dee
         dee         def ˆ fd„Zd„ Zd„ Zˆ xZS )r   zÕ Depthwise-separable block
    Used for DS convs in MobileNet-V1 and in the place of IR blocks that have no expansion
    (factor of 1.0). This is an alternative to having a IR with an optional first pw conv.
    rA   r    rR   Fr   NrS   r&   rT   Údw_kernel_sizerV   rW   r   rX   ÚnoskipÚpw_kernel_sizeÚpw_actÚs2dr)   rZ   r[   Úse_layerr\   c                 ó  •— ||dœ}t          ¦   «                              ¦   «          t          ||¦  «        }|dk    o||k    o| | _        |
| _        |d uo|dk    }|dk    rNt          |dz  ¦  «        }t          ||fddddœ|¤Ž| _         ||fi |¤Ž| _        |dz   dz  }|dk    rdn|}|}d}nd | _        d | _        |}t          ||¦  «        }t          |||f|rdn||||dœ|¤Ž| _
         ||fd	d
i|¤Ž| _        t          |f|||dœ|¤Ž| _        |r ||fd|i|¤Žnt          j        ¦   «         | _        t          |||	fd|i|¤Ž| _         ||fd
| j        dœ|¤Ž| _        |rt'          |¦  «        nt          j        ¦   «         | _        d S )Nr.   r    é   r@   Úsame©rU   rV   r`   F)rV   rW   r`   r_   r2   Tra   r)   r`   )r2   Ú	apply_act)r3   r4   r   rc   Ú
has_pw_actrM   r	   Úconv_s2dÚbn_s2dr#   Úconv_dwre   r   rf   r6   rg   ÚseÚconv_pwÚbn2r
   rh   )r<   r&   rT   r   rV   rW   r   rX   r€   r�   r‚   rƒ   r)   rZ   r[   r„   r\   r/   r0   r=   ri   rj   Úsd_chsÚdw_pad_typer_   r>   s                            €r"   r4   zDepthwiseSeparableConv.__init__”   s4  ø€ ð* ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ+¨J¸	ÑBÔBˆØ 1šÐ:¨°7Ò):ÐJÀFÀ
ˆŒØ ˆŒØ Ð%Ð4¨&°1ª*ˆð �!Š8ˆ8Ý˜ !™‘_”_ˆFÝ)¨&°&ÐhÀaÐPQÐ[aÐhÐhÐegÐhÐhˆDŒMØ(˜.¨Ð6Ð6°2Ð6Ð6ˆDŒKØ,¨qÑ0°QÑ6ˆNØ$2°aÒ$7Ð$7˜&˜&¸XˆKØˆFØˆFˆFà ˆDŒMØˆDŒKØ"ˆKå˜J¨Ñ/Ô/ˆå$ØØØð	
ð Ð*�1�1 FØØØð	
ð 	
ð ð	
ð 	
ˆŒð "�> &Ð=Ð=°$Ð=¸"Ð=Ð=ˆŒÝ˜HÐ[¨w¸vÈfÐ[Ð[ÐXZÐ[Ð[ˆŒð BJÐ\�(�(˜6Ð=Ð=¨YÐ=¸"Ð=Ð=Ð=ÍrÌ{É}Ì}ˆŒå$ V¨W°nÐ]Ð]ÈhÐ]ÐZ\Ð]Ð]ˆŒØ!�> 'ÐY°4À4Ä?ÐYÐYÐVXÐYÐYˆŒØ5CÐV� .Ñ1Ô1Ð1ÍÌÉÌˆŒˆˆr$   c                 ó|   — |dk    rt          dd| j        j        ¬¦  «        S t          d| j        j        ¬¦  «        S ©Nrl   r�   Úforward_prerm   rR   rq   ©rr   r�   Úin_channelsrs   rt   s     r"   rv   z#DepthwiseSeparableConv.feature_infoÔ   ó?   € Ø�{Ò"Ð"Ý˜y°MÈ4Ì<ÔKcÐdÑdÔdÐdå˜r¨4¬<Ô+DÐEÑEÔEÐEr$   c                 ó¦  — |}| j         �*|                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }| j        r|  	                    |¦  «        |z   }|S rx   )
r‹   rŒ   r�   re   rf   rŽ   r�   r�   rc   rh   ry   s      r"   rF   zDepthwiseSeparableConv.forwardÚ   s®   € ØˆØŒ=Ð$Ø—’˜aÑ Ô ˆAØ—’˜A‘”ˆAØ�LŠL˜‰OŒOˆØ�HŠH�Q‰KŒKˆØ�GŠG�A‰JŒJˆØ�GŠG�A‰JŒJˆØ�LŠL˜‰OŒOˆØ�HŠH�Q‰KŒKˆØŒ=ð 	-Ø—’˜qÑ!Ô! HÑ,ˆAØˆr$   )rG   rH   rI   rJ   r6   rK   r{   rM   r|   r}   r   r   Ú
ModuleTyperN   r4   rv   rF   rO   rP   s   @r"   r   r   �   sc  ø€ € € € € ðð ð #$ØØØØØ Ø"#Ø ØØ#%¤7Ø$&¤NØ,0Ø-1Ø$&ØØð'>Wð >Wàð>Wð ð>Wð  ð	>Wð
 ð>Wð ð>Wð ð>Wð ð>Wð ð>Wð  ð>Wð ð>Wð ð>Wð !ð>Wð "ð>Wð ˜yÔ)ð>Wð  ˜zÔ*ð!>Wð" "ð#>Wð >Wð >Wð >Wð >Wð >Wð@Fð Fð Fðð ð ð ð ð ð r$   r   c            %       óÖ   ‡ — e Zd ZdZddddddddddej        ej        dddd	ddfd
ededededededede	de
dedededededee         dee         dee         de
f$ˆ fd„Zd„ Zd„ Zˆ xZS )r   a„   Inverted residual block w/ optional SE

    Originally used in MobileNet-V2 - https://arxiv.org/abs/1801.04381v4, this layer is often
    referred to as 'MBConv' for (Mobile inverted bottleneck conv) and is also used in
      * MNasNet - https://arxiv.org/abs/1807.11626
      * EfficientNet - https://arxiv.org/abs/1905.11946
      * MobileNet-V3 - https://arxiv.org/abs/1905.02244
    rA   r    rR   Fç      ð?r   NrS   r&   rT   r   rV   rW   r   rX   r€   Ú	exp_ratioÚexp_kernel_sizer�   rƒ   r)   rZ   r[   r„   Úconv_kwargsr\   c                 óˆ  •— ||dœ}t          ¦   «                              ¦   «          t          ||¦  «        }|pi }||k    o|dk    o| | _        |d uo|dk    }|dk    rNt	          |dz  ¦  «        }t          ||fddddœ|¤Ž| _         ||fi |¤Ž| _        |dz   dz  }|dk    rdn|}|}d}nd | _        d | _        |}t          ||	z  ¦  «        }t          ||¦  «        }t          |||
fd|i|¤|¤Ž| _
         ||fd	d
i|¤Ž| _        t          |||f|rdn||||dœ|¤|¤Ž| _         ||fd	d
i|¤Ž| _        t          |f|||dœ|¤Ž| _        |r ||fd|i|¤Žnt!          j        ¦   «         | _        t          |||fd|i|¤|¤Ž| _         ||fddi|¤Ž| _        |rt+          |¦  «        nt!          j        ¦   «         | _        d S )Nr.   r    r†   r@   r‡   rˆ   Fr`   r2   Tr^   ra   r)   r‰   )r3   r4   r   rc   rM   r	   r‹   rŒ   r   r#   r�   re   r�   r�   r   rf   r6   rg   rŽ   Úconv_pwlÚbn3r
   rh   )r<   r&   rT   r   rV   rW   r   rX   r€   r�   rž   r�   rƒ   r)   rZ   r[   r„   rŸ   r\   r/   r0   r=   ri   rj   r‘   r’   Úmid_chsr_   r>   s                               €r"   r4   zInvertedResidual.__init__ô   s¤  ø€ ð. ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ+¨J¸	ÑBÔBˆØ!Ð' RˆØ 7Ò*Ð:¨v¸ª{ÐJÀFÀ
ˆŒØ Ð%Ð4¨&°1ª*ˆð �!Š8ˆ8Ý˜ !™‘_”_ˆFÝ)¨&°&ÐhÀaÐPQÐ[aÐhÐhÐegÐhÐhˆDŒMØ(˜.¨Ð6Ð6°2Ð6Ð6ˆDŒKØ,¨qÑ0°QÑ6ˆNØ$2°aÒ$7Ð$7˜&˜&¸XˆKØˆFØˆFˆFà ˆDŒMØˆDŒKØ"ˆKå  ¨)Ñ!3Ñ4Ô4ˆÝ˜J¨Ñ0Ô0ˆõ % V¨W°oÐmÐmÈxÐmÐ[fÐmÐjlÐmÐmˆŒØ!�> 'Ð>Ð>°4Ð>¸2Ð>Ð>ˆŒõ %ØØØð

ð Ð*�1�1 FØØØð

ð 

ð ð

ð ð

ð 

ˆŒð "�> 'Ð>Ð>°4Ð>¸2Ð>Ð>ˆŒÝ˜HÐ[¨w¸vÈfÐ[Ð[ÐXZÐ[Ð[ˆŒð CKÐ]�(�(˜7Ð>Ð>¨iÐ>¸2Ð>Ð>Ð>ÕPRÔP[ÑP]ÔP]ˆŒõ & g¨w¸ÐnÐnÐPXÐnÐ\gÐnÐkmÐnÐnˆŒØ!�> 'ÐAÐA°UÐA¸bÐAÐAˆŒØ5CÐV� .Ñ1Ô1Ð1ÍÌÉÌˆŒˆˆr$   c                 ó|   — |dk    rt          dd| j        j        ¬¦  «        S t          d| j        j        ¬¦  «        S ©Nrl   r¡   r•   rm   rR   rq   ©rr   r¡   r—   rs   rt   s     r"   rv   zInvertedResidual.feature_info>  ó?   € Ø�{Ò"Ð"Ý˜z°]ÈDÌMÔLeÐfÑfÔfÐfå˜r¨4¬=Ô+EÐFÑFÔFÐFr$   c                 óú  — |}| j         �*|                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|  	                    |¦  «        }| j
        r|                      |¦  «        |z   }|S rx   )r‹   rŒ   r�   re   r�   r�   rf   rŽ   r¡   r¢   rc   rh   ry   s      r"   rF   zInvertedResidual.forwardD  sÎ   € ØˆØŒ=Ð$Ø—’˜aÑ Ô ˆAØ—’˜A‘”ˆAØ�LŠL˜‰OŒOˆØ�HŠH�Q‰KŒKˆØ�LŠL˜‰OŒOˆØ�HŠH�Q‰KŒKˆØ�GŠG�A‰JŒJˆØ�GŠG�A‰JŒJˆØ�MŠM˜!ÑÔˆØ�HŠH�Q‰KŒKˆØŒ=ð 	-Ø—’˜qÑ!Ô! HÑ,ˆAØˆr$   ©rG   rH   rI   rJ   r6   rK   r{   rM   r|   r}   rN   r   r   rš   r   r4   rv   rF   rO   rP   s   @r"   r   r   ê   sœ  ø€ € € € € ðð ð #$ØØØØØ Ø"Ø#$Ø"#ØØ#%¤7Ø$&¤NØ,0Ø-1Ø*.Ø$&ØØð+HWð HWàðHWð ðHWð  ð	HWð
 ðHWð ðHWð ðHWð ðHWð ðHWð ðHWð !ðHWð  ðHWð ðHWð !ðHWð "ðHWð  ˜yÔ)ð!HWð" ˜zÔ*ð#HWð$ " $œð%HWð& "ð'HWð HWð HWð HWð HWð HWðTGð Gð Gðð ð ð ð ð ð r$   r   c            &       óâ   ‡ — e Zd ZdZdddddddddej        ej        dddd	d
ddfdededededededededede	de
dededee         dee         dee         de
dee
         f$ˆ fd„Zd„ Zd„ Zˆ xZS ) r   a   Universal Inverted Residual Block (aka Universal Inverted Bottleneck, UIB)

    For MobileNetV4 - https://arxiv.org/abs/, referenced from
    https://github.com/tensorflow/models/blob/d93c7e932de27522b2fa3b115f58d06d6f640537/official/vision/modeling/layers/nn_blocks.py#L778
    r   rA   r    rR   Frœ   NrS   çñhãˆµøä>r&   rT   Údw_kernel_size_startÚdw_kernel_size_midÚdw_kernel_size_endrV   rW   r   rX   r€   r�   r)   rZ   r[   r„   rŸ   r\   Úlayer_scale_init_valuec                 ó  •— ||dœ}t          ¦   «                              ¦   «          |pi }||k    o|dk    o|
 | _        |dk    r|s|s|sJ ‚|r6|s|nd}t          ||¦  «        }t	          |||f||||	d|||dœ|¤|¤Ž| _        nt          j        ¦   «         | _        t          ||z  ¦  «        }t	          ||df|	||dœ|¤|¤Ž| _	        |r/t          ||¦  «        }t	          |||f||||	|||dœ|¤|¤Ž| _
        nt          j        ¦   «         | _
        |r ||fd|i|¤Žnt          j        ¦   «         | _        t	          ||df|	d||dœ|¤|¤Ž| _        |rA|s|s|nd}t          ||¦  «        }|dk    r|rJ ‚t	          |||f||||	d||d	œ|¤|¤Ž| _        nt          j        ¦   «         | _        |�t          ||fi |¤Ž| _        nt          j        ¦   «         | _        |rt!          |¦  «        nt          j        ¦   «         | _        d S )
Nr.   r    F)rV   rW   r_   r`   r‰   r)   rZ   r[   )r`   r)   rZ   )rV   rW   r_   r`   r)   rZ   r[   r)   )r`   r‰   r)   rZ   )rV   rW   r_   r`   r‰   r)   rZ   )r3   r4   rc   r#   r   Údw_startr6   rg   r   Úpw_expÚdw_midrŽ   Úpw_projÚdw_endr   Úlayer_scaler
   rh   )r<   r&   rT   r¬   r­   r®   rV   rW   r   rX   r€   r�   r)   rZ   r[   r„   rŸ   r\   r¯   r/   r0   r=   Údw_start_strideÚdw_start_groupsr£   r_   Údw_end_strideÚdw_end_groupsr>   s                               €r"   r4   z"UniversalInvertedResidual.__init__]  s/  ø€ ð. ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ!Ð' RˆØ 7Ò*Ð:¨v¸ª{ÐJÀFÀ
ˆŒØ�AŠ:ˆ:Ø'ÐSÐ+=ÐSÐASÐSÐSÐSð  ð 	*Ø,>ÐE˜f˜fÀAˆOÝ(¨°VÑ<Ô<ˆOÝ'Ø˜Ð 4ðà&Ø!Ø&Ø ØØ#Ø%Ø!ðð ð ðð ðð ˆDŒMˆMõ œK™MœMˆDŒMõ ! ¨)Ñ!3Ñ4Ô4ˆÝ!Ø�G˜Qð
àØØ!ð	
ð 
ð
 ð
ð ð
ð 
ˆŒð ð 	(Ý 
¨GÑ4Ô4ˆFÝ%Ø˜Ð"4ðàØ!ØØ Ø#Ø%Ø!ðð ð ðð ðð ˆDŒKˆKõ œ+™-œ-ˆDŒKð CKÐ]�(�(˜7Ð>Ð>¨iÐ>¸2Ð>Ð>Ð>ÕPRÔP[ÑP]ÔP]ˆŒõ #Ø�W˜að
àØØØ!ð
ð 
ð ð
ð ð
ð 
ˆŒð ð 	(Ø*>Ð`ÐGYÐ`˜F˜FÐ_`ˆMÝ& z°7Ñ;Ô;ˆMØ˜qÒ Ð Ø#Ð#Ð#�|Ý%Ø˜Ð"4ðà$Ø!Ø$Ø ØØ#Ø%ðð ð ðð ðð ˆDŒKˆKõ œ+™-œ-ˆDŒKà!Ð-Ý+¨GÐ5KÐRÐRÈrÐRÐRˆDÔÐå!œ{™}œ}ˆDÔØ5CÐV� .Ñ1Ô1Ð1ÍÌÉÌˆŒˆˆr$   c                 ó�   — |dk    r!t          dd| j        j        j        ¬¦  «        S t          d| j        j        j        ¬¦  «        S )Nrl   zpw_proj.convr•   rm   rR   rq   )rr   r´   rd   r—   rs   rt   s     r"   rv   z&UniversalInvertedResidual.feature_infoÕ  sG   € Ø�{Ò"Ð"Ý˜~¸ÐPTÔP\ÔPaÔPmÐnÑnÔnÐnå˜r¨4¬<Ô+<Ô+IÐJÑJÔJÐJr$   c                 ón  — |}|                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }| j        r|                      |¦  «        |z   }|S rx   )	r±   r²   r³   rŽ   r´   rµ   r¶   rc   rh   ry   s      r"   rF   z!UniversalInvertedResidual.forwardÛ  s›   € ØˆØ�MŠM˜!ÑÔˆØ�KŠK˜‰NŒNˆØ�KŠK˜‰NŒNˆØ�GŠG�A‰JŒJˆØ�LŠL˜‰OŒOˆØ�KŠK˜‰NŒNˆØ×Ò˜QÑÔˆØŒ=ð 	-Ø—’˜qÑ!Ô! HÑ,ˆAØˆr$   r©   rP   s   @r"   r   r   V  s   ø€ € € € € ðð ð )*Ø&'Ø&'ØØØØØ Ø"Ø#%¤7Ø$&¤NØ,0Ø-1Ø*.Ø$&Ø6:ØØð+vWð vWàðvWð ðvWð #&ð	vWð
 !$ðvWð !$ðvWð ðvWð ðvWð ðvWð ðvWð ðvWð ðvWð !ðvWð "ðvWð ˜yÔ)ðvWð  ˜zÔ*ð!vWð" " $œð#vWð$ "ð%vWð& %-¨U¤Oð'vWð vWð vWð vWð vWð vWðpKð Kð Kðð ð ð ð ð ð r$   r   c            1       óî   ‡ — e Zd ZdZdddddddddddddej        ej        d	d
d
d
dddd	d	fdedededededededededede	dededede	de
de
dee
         deded ed!ee         d"e	d#e	f0ˆ fd$„Zd%„ Zd&„ Zˆ xZS )'r   zå Mobile Attention Block

    For MobileNetV4 - https://arxiv.org/abs/, referenced from
    https://github.com/tensorflow/models/blob/d93c7e932de27522b2fa3b115f58d06d6f640537/official/vision/modeling/layers/nn_blocks.py#L1504
    r    rA   rR   é   é@   F)r    r    NrS   r«   r&   rT   rV   r   rW   r   rX   Ú	num_headsÚkey_dimÚ	value_dimÚuse_multi_queryÚquery_stridesÚ	kv_strideÚcpe_dw_kernel_sizer€   r)   rZ   r[   r\   Ú	attn_dropÚ	proj_dropr¯   Úuse_biasÚuse_cpec                 óÀ  •— ||dœ}t          ¦   «                              ¦   «          t          ||¦  «        }|dk    o||k    o| | _        t	          |¦  «        | _        || _        t          d„ | j        D ¦   «         ¦  «        | _        |rt          ||f||dddœ|¤Ž| _
        nd | _
         ||fddi|¤Ž| _        |€||	z  dk    sJ ‚||	z  }|r t          |f|||	|
||||||||d	œ|¤Ž| _        nt          |f|||||d
œ|¤Ž| _        |�t          ||fi |¤Ž| _        nt#          j        ¦   «         | _        |rt'          |¦  «        nt#          j        ¦   «         | _        d S )Nr.   r    c                 ó   — g | ]}|d k    ‘Œ	S )r    r!   )Ú.0Úss     r"   ú
<listcomp>z,MobileAttention.__init__.<locals>.<listcomp>  s   € Ð$GÐ$GÐ$G¨q Q¨¢UÐ$GÐ$GÐ$Gr$   T)rU   rW   Ú	depthwiser1   r‰   Fr   )Údim_outrÀ   rÁ   rÂ   rÄ   rÅ   r   rW   r`   rÇ   rÈ   rZ   )rÑ   rÀ   rÇ   rÈ   r1   )r3   r4   r   rc   r   rÄ   rÅ   ÚanyÚhas_query_strider	   Úconv_cpe_dwÚnormr   Úattnr   r   r¶   r6   rg   r
   rh   )r<   r&   rT   rV   r   rW   r   rX   rÀ   rÁ   rÂ   rÃ   rÄ   rÅ   rÆ   r€   r)   rZ   r[   r\   rÇ   rÈ   r¯   rÉ   rÊ   r/   r0   r=   ri   r>   s                                €r"   r4   zMobileAttention.__init__ï  s  ø€ ð: ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ+¨J¸	ÑBÔBˆØ 1šÐ:¨°7Ò):ÐJÀFÀ
ˆŒÝ& }Ñ5Ô5ˆÔØ"ˆŒÝ #Ð$GÐ$G°DÔ4FÐ$GÑ$GÔ$GÑ HÔ HˆÔð ð 
	$Ý,Ø˜ð à.Ø!ØØð ð  ð ð ð  ˆDÔÐð  $ˆDÔà"�N 6ÐAÐA°UÐA¸bÐAÐAˆŒ	àÐØ˜GÑ# qÒ(Ð(Ð(Ð(Ø 'Ñ)ˆIàð 	Ý-ØðàØ#ØØ#Ø+Ø#Ø-Ø!Ø Ø#Ø#Ø%ðð ð ðð ˆDŒIˆIõ$ $ØðàØ#Ø#Ø#Øðð ð ðð ˆDŒIð "Ð-Ý+¨GÐ5KÐRÐRÈrÐRÐRˆDÔÐå!œ{™}œ}ˆDÔà5CÐV� .Ñ1Ô1Ð1ÍÌÉÌˆŒˆˆr$   c                 ó|   — |dk    rt          dd| j        j        ¬¦  «        S t          d| j        j        ¬¦  «        S r”   r–   rt   s     r"   rv   zMobileAttention.feature_infoO  r˜   r$   c                 ó  — | j         �|                       |¦  «        }||z   }|}|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }| j        r|                      |¦  «        |z   }|S rx   )rÔ   rÕ   rÖ   r¶   rc   rh   )r<   rD   Úx_cperz   s       r"   rF   zMobileAttention.forwardU  s‚   € ØÔÐ'Ø×$Ò$ QÑ'Ô'ˆEØ�E‘	ˆAàˆØ�IŠI�a‰LŒLˆØ�IŠI�a‰LŒLˆØ×Ò˜QÑÔˆØŒ=ð 	-Ø—’˜qÑ!Ô! HÑ,ˆAàˆr$   )rG   rH   rI   rJ   r6   rK   r{   rM   r|   r}   r   r   rN   r4   rv   rF   rO   rP   s   @r"   r   r   é  sñ  ø€ € € € € ðð ð Ø"#ØØØØØØØ$)Ø!'ØØ&'Ø Ø#%¤7Ø$&¤NØ,0Ø$&Ø"Ø"Ø6:Ø"Ø!ØØð7^Wð ^Wàð^Wð ð^Wð ð	^Wð
  ð^Wð ð^Wð ð^Wð ð^Wð ð^Wð ð^Wð ð^Wð "ð^Wð ð^Wð ð^Wð !$ð^Wð  ð!^Wð" !ð#^Wð$ "ð%^Wð& ˜yÔ)ð'^Wð( "ð)^Wð* ð+^Wð, ð-^Wð. %-¨U¤Oð/^Wð0 ð1^Wð2 ð3^Wð ^Wð ^Wð ^Wð ^Wð ^Wð@Fð Fð Fðð ð ð ð ð ð r$   r   c            #       ó¾   ‡ — e Zd ZdZdddddddddej        ej        dddd	ddfd
ededededededede	de
dededededee         dee         dede
f"ˆ fd„Zd„ Zˆ xZS )r   z, Inverted residual block w/ CondConv routingrA   r    rR   Frœ   Nr   rS   r&   rT   r   rV   rW   r   rX   r€   r�   rž   r�   r)   rZ   r[   r„   Únum_expertsr\   c                 óÚ   •— ||dœ}|| _         t          | j         ¬¦  «        } t          ¦   «         j        ||f|||||||	|
|||||||dœ|¤Ž t	          j        || j         fi |¤Ž| _        d S )Nr.   )rÛ   )r   rV   rW   r   rX   r€   r�   rž   r�   r)   rZ   r[   r„   rŸ   r\   )rÛ   rr   r3   r4   r6   ÚLinearÚ
routing_fn)r<   r&   rT   r   rV   rW   r   rX   r€   r�   rž   r�   r)   rZ   r[   r„   rÛ   r\   r/   r0   r=   rŸ   r>   s                         €r"   r4   zCondConvResidual.__init__g  sµ   ø€ ð, ¨Ð/Ð/ˆØ&ˆÔÝ tÔ'7Ð8Ñ8Ô8ˆØ�‰ŒÔØØð	
ð *ØØØ!ØØØØ+Ø)ØØ!ØØØ#Ø)ð#	
ð 	
ð$ ð%	
ð 	
ð 	
õ( œ) F¨DÔ,<ÐCÐCÀÐCÐCˆŒˆˆr$   c                 ó  — |}t          j        |d¦  «                             d¦  «        }t          j        |                      |¦  «        ¦  «        }|                      ||¦  «        }|                      |¦  «        }|                      ||¦  «        }|  	                    |¦  «        }|  
                    |¦  «        }|                      ||¦  «        }|                      |¦  «        }| j        r|                      |¦  «        |z   }|S )Nr    )ÚFÚadaptive_avg_pool2dÚflattenÚtorchÚsigmoidrÞ   r�   re   r�   r�   rŽ   r¡   r¢   rc   rh   )r<   rD   rz   Úpooled_inputsÚrouting_weightss        r"   rF   zCondConvResidual.forward–  sÜ   € ØˆÝÔ-¨a°Ñ3Ô3×;Ò;¸AÑ>Ô>ˆÝœ-¨¯ª¸Ñ(FÔ(FÑGÔGˆØ�LŠL˜˜OÑ,Ô,ˆØ�HŠH�Q‰KŒKˆØ�LŠL˜˜OÑ,Ô,ˆØ�HŠH�Q‰KŒKˆØ�GŠG�A‰JŒJˆØ�MŠM˜!˜_Ñ-Ô-ˆØ�HŠH�Q‰KŒKˆØŒ=ð 	-Ø—’˜qÑ!Ô! HÑ,ˆAØˆr$   )rG   rH   rI   rJ   r6   rK   r{   rM   r|   r}   rN   r   r   rš   r4   rF   rO   rP   s   @r"   r   r   d  s[  ø€ € € € € Ø6Ð6ð #$ØØØØØ Ø"Ø#$Ø"#Ø#%¤7Ø$&¤NØ,0Ø-1Ø Ø$&ØØð)-Dð -Dàð-Dð ð-Dð  ð	-Dð
 ð-Dð ð-Dð ð-Dð ð-Dð ð-Dð ð-Dð !ð-Dð  ð-Dð !ð-Dð "ð-Dð ˜yÔ)ð-Dð  ˜zÔ*ð!-Dð" ð#-Dð$ "ð%-Dð -Dð -Dð -Dð -Dð -Dð^ð ð ð ð ð ð r$   r   c            !       ó¾   ‡ — e Zd ZdZdddddddddej        ej        ddd	ddfd
edededededededede	de
dedededee         dee         de
f ˆ fd„Zd„ Zd„ Zˆ xZS )r   a(   Residual block with expansion convolution followed by pointwise-linear w/ stride

    Originally introduced in `EfficientNet-EdgeTPU: Creating Accelerator-Optimized Neural Networks with AutoML`
        - https://ai.googleblog.com/2019/08/efficientnet-edgetpu-creating.html

    This layer is also called FusedMBConv in the MobileDet, EfficientNet-X, and EfficientNet-V2 papers
      * MobileDet - https://arxiv.org/abs/2004.14525
      * EfficientNet-X - https://arxiv.org/abs/2102.05610
      * EfficientNet-V2 - https://arxiv.org/abs/2104.00298
    rA   r    r   rR   Frœ   NrS   r&   rT   rž   rV   rW   r   rX   Úforce_in_chsr€   r�   r�   r)   rZ   r[   r„   r\   c                 ó’  •— ||dœ}t          ¦   «                              ¦   «          t          ||¦  «        }|dk    rt          ||
z  ¦  «        }nt          ||
z  ¦  «        }t	          ||¦  «        }||k    o|dk    o|	 | _        |d uo|dk    }t          |||f|rdn||||dœ|¤Ž| _         ||fddi|¤Ž| _        t          |f|||dœ|¤Ž| _
        |r ||fd|i|¤Žnt          j        ¦   «         | _        t          |||fd	|i|¤Ž| _         ||fd
di|¤Ž| _        |rt!          |¦  «        nt          j        ¦   «         | _        d S )Nr.   r   r    r^   r2   Tra   r)   r`   r‰   F)r3   r4   r   r   r#   rc   r	   Úconv_expre   r   rf   r6   rg   rŽ   r¡   r�   r
   rh   )r<   r&   rT   rž   rV   rW   r   rX   rè   r€   r�   r�   r)   rZ   r[   r„   r\   r/   r0   r=   ri   r£   r_   rj   r>   s                           €r"   r4   zEdgeResidual.__init__²  sÉ  ø€ ð* ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ+¨J¸	ÑBÔBˆØ˜!ÒÐÝ$ \°IÑ%=Ñ>Ô>ˆGˆGå$ V¨iÑ%7Ñ8Ô8ˆGÝ˜J¨Ñ0Ô0ˆØ 7Ò*Ð:¨v¸ª{ÐJÀFÀ
ˆŒØ Ð%Ð4¨&°1ª*ˆõ &ØØØð	
ð Ð*�1�1 FØØØð	
ð 	
ð ð	
ð 	
ˆŒð "�> 'Ð>Ð>°4Ð>¸2Ð>Ð>ˆŒå˜HÐ[¨w¸vÈfÐ[Ð[ÐXZÐ[Ð[ˆŒð CKÐ]�(�(˜7Ð>Ð>¨iÐ>¸2Ð>Ð>Ð>ÕPRÔP[ÑP]ÔP]ˆŒõ & g¨w¸Ð_Ð_ÐPXÐ_Ð\^Ð_Ð_ˆŒØ!�> 'ÐAÐA°UÐA¸bÐAÐAˆŒØ5CÐV� .Ñ1Ô1Ð1ÍÌÉÌˆŒˆˆr$   c                 ó|   — |dk    rt          dd| j        j        ¬¦  «        S t          d| j        j        ¬¦  «        S r¥   r¦   rt   s     r"   rv   zEdgeResidual.feature_infoé  r§   r$   c                 óD  — |}|                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }| j        r|                      |¦  «        |z   }|S rx   )rê   re   rf   rŽ   r¡   r�   rc   rh   ry   s      r"   rF   zEdgeResidual.forwardï  sŠ   € ØˆØ�MŠM˜!ÑÔˆØ�HŠH�Q‰KŒKˆØ�GŠG�A‰JŒJˆØ�GŠG�A‰JŒJˆØ�MŠM˜!ÑÔˆØ�HŠH�Q‰KŒKˆØŒ=ð 	-Ø—’˜qÑ!Ô! HÑ,ˆAØˆr$   )rG   rH   rI   rJ   r6   rK   r{   rM   r|   r}   rN   r   r   rš   r4   rv   rF   rO   rP   s   @r"   r   r   ¦  sc  ø€ € € € € ð	ð 	ð $%ØØØØØ !Ø Ø"Ø#$Ø#%¤7Ø$&¤NØ,0Ø-1Ø$&ØØð'5Wð 5Wàð5Wð ð5Wð !ð	5Wð
 ð5Wð ð5Wð ð5Wð ð5Wð ð5Wð ð5Wð ð5Wð !ð5Wð !ð5Wð "ð5Wð ˜yÔ)ð5Wð  ˜zÔ*ð!5Wð" "ð#5Wð 5Wð 5Wð 5Wð 5Wð 5WðnGð Gð Gð
ð 
ð 
ð 
ð 
ð 
ð 
r$   r   )&rJ   Útypingr   r   r   r   r   rã   Útorch.nnr6   r   rà   Útimm.layersr	   r
   r   r   r   r   r   r   r   r   r   r   Ú__all__ÚModulerš   rM   r#   r   r   r   r   r   r   r   r   r!   r$   r"   ú<module>rò      sÒ  ððð ð 9Ð 8Ð 8Ð 8Ð 8Ð 8Ð 8Ð 8Ð 8Ð 8Ð 8Ð 8Ð 8Ð 8à €€€Ø Ð Ð Ð Ð Ð Ø $Ð $Ð $Ð $Ð $Ð $ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ðð ð €ð
 �"”)Œ_€
ð&˜8 Cœ=ð &°Cð &ð &ð &ð &ð(#ð (#ð (#ð (#ð (#�B”Iñ (#ô (#ð (#ðV6ð 6ð 6ð 6ð 6�”	ñ 6ô 6ð 6ðrXð Xð Xð Xð X˜RœYñ Xô Xð Xðvið ið ið ið i�r”yñ iô ið iðXPð Pð Pð Pð P ¤	ñ Pô Pð Pðfxð xð xð xð x�b”iñ xô xð xðv?ð ?ð ?ð ?ð ?Ð'ñ ?ô ?ð ?ðDSð Sð Sð Sð S�2”9ñ Sô Sð Sð Sð Sr$   