§
    ·¯¸jà!  ã                   ó  — d Z ddlmZmZmZ ddlZddlmZ ddlmc m	Z
 ddlmZ ddlmZmZmZ  G d„ dej        ¦  «        Ze G d	„ d
ej        ¦  «        ¦   «         Z G d„ dej        ¦  «        Ze G d„ dej        ¦  «        ¦   «         ZdS )a    Convolution with Weight Standardization (StdConv and ScaledStdConv)

StdConv:
@article{weightstandardization,
  author    = {Siyuan Qiao and Huiyu Wang and Chenxi Liu and Wei Shen and Alan Yuille},
  title     = {Weight Standardization},
  journal   = {arXiv preprint arXiv:1903.10520},
  year      = {2019},
}
Code: https://github.com/joe-siyuan-qiao/WeightStandardization

ScaledStdConv:
Paper: `Characterizing signal propagation to close the performance gap in unnormalized ResNets`
    - https://arxiv.org/abs/2101.08692
Official Deepmind JAX code: https://github.com/deepmind/deepmind-research/tree/master/nfnets

Hacked together by / copyright Ross Wightman, 2021.
é    )ÚOptionalÚTupleÚUnionNé   )Úregister_notrace_module)Úget_paddingÚget_padding_valueÚpad_samec                   óì   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 ddededeeeeef         f         d	eeeeef         f         d
eeeeeef         f                  deeeeef         f         dedede	fˆ fd„Z
d„ Zˆ xZS )Ú	StdConv2dzÓConv2d with Weight Standardization. Used for BiT ResNet-V2 models.

    Paper: `Micro-Batch Training with Batch-Channel Normalization and Weight Standardization` -
        https://arxiv.org/abs/1903.10520v2
    r   NFç�íµ ÷Æ°>Ú
in_channelÚout_channelsÚkernel_sizeÚstrideÚpaddingÚdilationÚgroupsÚbiasÚepsc                 ó’   •— |€t          |||¦  «        }t          ¦   «                              |||||||||
|¬¦
  «
         |	| _        d S )N©r   r   r   r   r   ÚdeviceÚdtype)r   ÚsuperÚ__init__r   )Úselfr   r   r   r   r   r   r   r   r   r   r   Ú	__class__s               €úd/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/layers/std_conv.pyr   zStdConv2d.__init__#   sd   ø€ ð ˆ?Ý! +¨v°xÑ@Ô@ˆGÝ‰Œ×ÒØ˜ k¸&Ø h°vÀDÐQWÐ_dð 	ñ 	fô 	fð 	fð ˆŒˆˆó    c           	      ó  — t          j        | j                             d| j        d¦  «        d d dd| j        ¬¦  «                             | j        ¦  «        }t          j        ||| j        | j	        | j
        | j        | j        ¦  «        }|S ©Nr   éÿÿÿÿTç        )ÚtrainingÚmomentumr   )ÚFÚ
batch_normÚweightÚreshaper   r   Ú
reshape_asÚconv2dr   r   r   r   r   ©r   Úxr)   s      r   ÚforwardzStdConv2d.forward8   s…   € Ý”ØŒK×Ò  4Ô#4°bÑ9Ô9ØØØØØ”ð
ñ 
ô 
÷ Š*�T”[Ñ
!Ô
!ð 	õ ŒH�Q˜ ¤	¨4¬;¸¼ÀdÄmÐUYÔU`ÑaÔaˆØˆr    )r   Nr   r   Fr   NN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Úintr   r   r   ÚboolÚfloatr   r/   Ú__classcell__©r   s   @r   r   r      s  ø€ € € € € ðð ð 34Ø=AØ45ØØØØØðð àðð ðð ˜s E¨#¨s¨(¤OÐ3Ô4ð	ð
 ˜#˜u S¨# XœÐ.Ô/ðð ˜e C¨¨s°C¨x¬Ð$8Ô9Ô:ðð ˜C  s¨C x¤Ð0Ô1ðð ðð ðð ðð ð ð ð ð ð*
ð 
ð 
ð 
ð 
ð 
ð 
r    r   c                   óÀ   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 ddeded	eeeeef         f         d
eeeeef         f         dedeeeeef         f         dedede	fˆ fd„Z
d„ Zˆ xZS )ÚStdConv2dSamezëConv2d with Weight Standardization. TF compatible SAME padding. Used for ViT Hybrid model.

    Paper: `Micro-Batch Training with Batch-Channel Normalization and Weight Standardization` -
        https://arxiv.org/abs/1903.10520v2
    r   ÚSAMEFr   Nr   r   r   r   r   r   r   r   r   c                 ó¦   •— t          ||||¬¦  «        \  }}t          ¦   «                              |||||||||
|¬¦
  «
         || _        |	| _        d S )N©r   r   r   )r	   r   r   Úsame_padr   )r   r   r   r   r   r   r   r   r   r   r   r   Ú
is_dynamicr   s                €r   r   zStdConv2dSame.__init__L   sp   ø€ õ 0°¸ÈVÐ^fÐgÑgÔgÑˆ�Ý‰Œ×ÒØ˜ k¸&È'Ð\dØ ¨V¸5ð 	ñ 	Bô 	Bð 	Bð #ˆŒØˆŒˆˆr    c           	      óh  — | j         r!t          || j        | j        | j        ¦  «        }t          j        | j                             d| j	        d¦  «        d d dd| j
        ¬¦  «                             | j        ¦  «        }t          j        ||| j        | j        | j        | j        | j        ¦  «        }|S r"   )r>   r
   r   r   r   r'   r(   r)   r*   r   r   r+   r,   r   r   r   r-   s      r   r/   zStdConv2dSame.forwarda   sª   € ØŒ=ð 	JÝ˜˜DÔ,¨d¬k¸4¼=ÑIÔIˆAÝ”ØŒK×Ò  4Ô#4°bÑ9Ô9ØØØØØ”ð
ñ 
ô 
÷ Š*�T”[Ñ
!Ô
!ð 	õ ŒH�Q˜ ¤	¨4¬;¸¼ÀdÄmÐUYÔU`ÑaÔaˆØˆr    )r   r;   r   r   Fr   NN)r0   r1   r2   r3   r4   r   r   Ústrr5   r6   r   r/   r7   r8   s   @r   r:   r:   E   s  ø€ € € € € ðð ð 34Ø!Ø45ØØØØØðð àðð ðð ˜s E¨#¨s¨(¤OÐ3Ô4ð	ð
 ˜#˜u S¨# XœÐ.Ô/ðð ðð ˜C  s¨C x¤Ð0Ô1ðð ðð ðð ðð ð ð ð ð ð*ð ð ð ð ð ð r    r:   c                   ó  ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 ddeded	eeeeef         f         d
eeeeef         f         deeeeeef         ef                  deeeeef         f         dede	de
de
de
fˆ fd„Zdˆ fd„Zd„ Zˆ xZS )ÚScaledStdConv2da4  Conv2d layer with Scaled Weight Standardization.

    Paper: `Characterizing signal propagation to close the performance gap in unnormalized ResNets` -
        https://arxiv.org/abs/2101.08692

    NOTE: the operations used in this impl differ slightly from the DeepMind Haiku impl. The impact is minor.
    r   NTç      ð?r   Úin_channelsr   r   r   r   r   r   r   Úgammar   Ú	gain_initc           	      óx  •— ||dœ}|€t          |||¦  «        } t          ¦   «         j        |||f|||||dœ|¤Ž |	| j        d                              ¦   «         dz  z  | _        |
| _        || _        t          j	        t          j        | j        dddffi |¤Ž¦  «        | _        |                      ¦   «          d S )N©r   r   ©r   r   r   r   r   r   ç      à¿r   )r   r   r   r)   ÚnumelÚscaler   rG   ÚnnÚ	ParameterÚtorchÚemptyr   ÚgainÚreset_parameters)r   rE   r   r   r   r   r   r   r   rF   r   rG   r   r   Úddr   s                  €r   r   zScaledStdConv2d.__init__y   sã   ø€ ð  ¨Ð/Ð/ˆØˆ?Ý! +¨v°xÑ@Ô@ˆGØ�‰ŒÔØ˜ {ð	,Ø;AÈ7Ð]eØ ð	,ð 	,à(*ð	,ð 	,ð 	,ð ˜Tœ[¨œ^×1Ò1Ñ3Ô3°tÑ;Ñ;ˆŒ
ØˆŒØ"ˆŒå”L¥¤¨dÔ.?ÀÀAÀqÐ-IÐ!PÐ!PÈRÐ!PÐ!PÑQÔQˆŒ	à×ÒÑÔÐÐÐr    Úreturnc                 óÊ   •— t          | d¦  «        rQt          j        j                             | j        | j        ¦  «         t          ¦   «                              ¦   «          d S d S ©NrR   ©	ÚhasattrrP   rN   ÚinitÚ	constant_rR   rG   r   rS   ©r   r   s    €r   rS   z ScaledStdConv2d.reset_parameters—   óY   ø€ å�4˜Ñ Ô ð 	'ÝŒHŒM×#Ò# D¤I¨t¬~Ñ>Ô>Ð>å‰GŒG×$Ò$Ñ&Ô&Ð&Ð&Ð&ð	'ð 	'r    c           	      óV  — t          j        | j                             d| j        d¦  «        d d | j        | j        z                       d¦  «        dd| j        ¬¦  «         	                    | j        ¦  «        }t          j
        ||| j        | j        | j        | j        | j        ¦  «        S ©Nr   r#   Tr$   )r)   r%   r&   r   )r'   r(   r)   r*   r   rR   rM   Úviewr   r+   r,   r   r   r   r   r   r-   s      r   r/   zScaledStdConv2d.forwardŸ   s›   € Ý”ØŒK×Ò  4Ô#4°bÑ9Ô9ØØØ”I ¤
Ñ*×0Ò0°Ñ4Ô4ØØØ”ð
ñ 
ô 
÷ Š*�T”[Ñ
!Ô
!ð 	õ Œx˜˜6 4¤9¨d¬k¸4¼<ÈÌÐX\ÔXcÑdÔdÐdr    )
r   Nr   r   TrD   r   rD   NN©rU   N)r0   r1   r2   r3   r4   r   r   r   rA   r5   r6   r   rS   r/   r7   r8   s   @r   rC   rC   p   s[  ø€ € € € € ðð ð 34ØBFØ45ØØØØØ"ØØð ð  àð ð ð ð ˜s E¨#¨s¨(¤OÐ3Ô4ð	 ð
 ˜#˜u S¨# XœÐ.Ô/ð ð ˜e C¨¨s°C¨x¬¸#Ð$=Ô>Ô?ð ð ˜C  s¨C x¤Ð0Ô1ð ð ð ð ð ð ð ð ð ð ð ð  ð  ð  ð  ð  ð<'ð 'ð 'ð 'ð 'ð 'ð
eð 
eð 
eð 
eð 
eð 
eð 
er    rC   c                   óØ   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 dded	ed
eeeeef         f         deeeeef         f         dedeeeeef         f         dedede	de	de	fˆ fd„Z
dˆ fd„Zd„ Zˆ xZS )ÚScaledStdConv2dSamea\  Conv2d layer with Scaled Weight Standardization and Tensorflow-like SAME padding support

    Paper: `Characterizing signal propagation to close the performance gap in unnormalized ResNets` -
        https://arxiv.org/abs/2101.08692

    NOTE: the operations used in this impl differ slightly from the DeepMind Haiku impl. The impact is minor.
    r   r;   TrD   r   NrE   r   r   r   r   r   r   r   rF   r   rG   c           	      óŒ  •— ||dœ}t          ||||¬¦  «        \  }} t          ¦   «         j        |||f|||||dœ|¤Ž |	| j        d                              ¦   «         dz  z  | _        || _        |
| _        || _        t          j
        t          j        | j        dddffi |¤Ž¦  «        | _        |                      ¦   «          d S )NrI   r=   rJ   r   rK   r   )r	   r   r   r)   rL   rM   r>   r   rG   rN   rO   rP   rQ   r   rR   rS   )r   rE   r   r   r   r   r   r   r   rF   r   rG   r   r   rT   r?   r   s                   €r   r   zScaledStdConv2dSame.__init__¶   sð   ø€ ð  ¨Ð/Ð/ˆÝ/°¸ÈVÐ^fÐgÑgÔgÑˆ�Ø�‰ŒÔØ˜ {ð	,Ø;AÈ7Ð]eØ ð	,ð 	,à(*ð	,ð 	,ð 	,ð ˜Tœ[¨œ^×1Ò1Ñ3Ô3°tÑ;Ñ;ˆŒ
Ø"ˆŒØˆŒØ"ˆŒå”L¥¤¨dÔ.?ÀÀAÀqÐ-IÐ!PÐ!PÈRÐ!PÐ!PÑQÔQˆŒ	à×ÒÑÔÐÐÐr    rU   c                 óÊ   •— t          | d¦  «        rQt          j        j                             | j        | j        ¦  «         t          ¦   «                              ¦   «          d S d S rW   rX   r\   s    €r   rS   z$ScaledStdConv2dSame.reset_parametersÔ   r]   r    c           	      ó¦  — | j         r!t          || j        | j        | j        ¦  «        }t          j        | j                             d| j	        d¦  «        d d | j
        | j        z                       d¦  «        dd| j        ¬¦  «                             | j        ¦  «        }t          j        ||| j        | j        | j        | j        | j        ¦  «        S r_   )r>   r
   r   r   r   r'   r(   r)   r*   r   rR   rM   r`   r   r+   r,   r   r   r   r-   s      r   r/   zScaledStdConv2dSame.forwardÜ   sÀ   € ØŒ=ð 	JÝ˜˜DÔ,¨d¬k¸4¼=ÑIÔIˆAÝ”ØŒK×Ò  4Ô#4°bÑ9Ô9ØØØ”I ¤
Ñ*×0Ò0°Ñ4Ô4ØØØ”ð
ñ 
ô 
÷ Š*�T”[Ñ
!Ô
!ð 	õ Œx˜˜6 4¤9¨d¬k¸4¼<ÈÌÐX\ÔXcÑdÔdÐdr    )
r   r;   r   r   TrD   r   rD   NNra   )r0   r1   r2   r3   r4   r   r   rA   r5   r6   r   rS   r/   r7   r8   s   @r   rc   rc   ¬   sB  ø€ € € € € ðð ð 34Ø!Ø45ØØØØØ"ØØð ð  àð ð ð ð ˜s E¨#¨s¨(¤OÐ3Ô4ð	 ð
 ˜#˜u S¨# XœÐ.Ô/ð ð ð ð ˜C  s¨C x¤Ð0Ô1ð ð ð ð ð ð ð ð ð ð ð ð  ð  ð  ð  ð  ð<'ð 'ð 'ð 'ð 'ð 'ðeð eð eð eð eð eð er    rc   )r3   Útypingr   r   r   rP   Útorch.nnrN   Útorch.nn.functionalÚ
functionalr'   Ú_fxr   r   r   r	   r
   ÚConv2dr   r:   rC   rc   © r    r   ú<module>rn      s~  ððð ð$ *Ð )Ð )Ð )Ð )Ð )Ð )Ð )Ð )Ð )à €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð à (Ð (Ð (Ð (Ð (Ð (Ø =Ð =Ð =Ð =Ð =Ð =Ð =Ð =Ð =Ð =ð%ð %ð %ð %ð %�”	ñ %ô %ð %ðP ð'ð 'ð 'ð 'ð '�B”Iñ 'ô 'ñ Ôð'ðT9eð 9eð 9eð 9eð 9e�b”iñ 9eô 9eð 9eðx ð;eð ;eð ;eð ;eð ;e˜"œ)ñ ;eô ;eñ Ôð;eð ;eð ;er    