§
    ·¯¸jpf  ã                   óZ  — d Z ddlmZmZmZmZmZmZ ddlZddlm	Z	 ddl
mZ ddlmZ ddlmZ dd	lmZ dd
lmZmZmZmZmZmZ ddlmZmZ ddlmZ ddlmZ 	 ddl m!Z! n# e"$ r	 ddlm!Z! Y nw xY w	 	 	 d8dedee#ef         dee$         de$de	j%        f
d„Z&e G d„ de	j'        ¦  «        ¦   «         Z(e G d„ de	j)        ¦  «        ¦   «         Z*d9d„Z+e G d„ dej	        j%        ¦  «        ¦   «         Z,d„ Z-d„ Z.d e/d!e/d"e/fd#„Z0 G d$„ d%e	j1        ¦  «        Z2 G d&„ d'e	j1        ¦  «        Z3 G d(„ d)e	j4        ¦  «        Z5 G d*„ d+e	j4        ¦  «        Z6 G d,„ d-e	j4        ¦  «        Z7 G d.„ d/e	j4        ¦  «        Z8 G d0„ d1e¦  «        Z9 G d2„ d3e¦  «        Z: G d4„ d5e¦  «        Z; G d6„ d7e¦  «        Z<dS ):aw   Normalization + Activation Layers

Provides Norm+Act fns for standard PyTorch norm layers such as
* BatchNorm
* GroupNorm
* LayerNorm

This allows swapping with alternative layers that are natively both norm + act such as
* EvoNorm (evo_norm.py)
* FilterResponseNorm (filter_response_norm.py)
* InplaceABN (inplace_abn.py)

Hacked together by / Copyright 2022 Ross Wightman
é    )ÚAnyÚDictÚListÚOptionalÚTypeÚUnionN)Únn)Ú
functional)ÚFrozenBatchNorm2dé   )Úregister_notrace_module)Úcreate_act_layer)Úis_fast_normÚfast_group_normÚfast_layer_normÚfast_rms_normÚ
rms_norm2dÚfast_rms_norm2d)ÚRmsNormÚ	RmsNorm2d)Ú_assert)Ú	LayerType)Úrms_normFTÚ	act_layerÚ
act_kwargsÚinplaceÚ	apply_actÚreturnc                 ó†   — |pi }|                      d|¦  «         d }|rt          | fi |¤Ž}|€t          j        ¦   «         n|S )Nr   )Ú
setdefaultr   r	   ÚIdentity)r   r   r   r   Úacts        úd/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/layers/norm_act.pyÚ_create_actr$   *   sZ   € ð Ð!˜r€JØ×Ò˜) WÑ-Ô-Ð-Ø
€CØð 8Ý˜yÐ7Ð7¨JÐ7Ð7ˆØ˜K�2Œ;‰=Œ=ˆ=¨SÐ0ó    c                   ó¦   ‡ — e Zd ZdZdddddej        dddddfdededed	ed
edede	de
eef         dedeeej                          fˆ fd„Zd„ Zˆ xZS )ÚBatchNormAct2dzúBatchNorm + Activation

    This module performs BatchNorm + Activation in a manner that will remain backwards
    compatible with weights trained with separate bn, act. This is why we inherit from BN
    instead of composing it as a .bn member.
    çñhãˆµøä>gš™™™™™¹?TNÚnum_featuresÚepsÚmomentumÚaffineÚtrack_running_statsr   r   r   r   Ú
drop_layerc                 ó8  •— 	 ||dœ} t          ¦   «         j        |f||||dœ|¤Ž n6# t          $ r) t          ¦   «                              |||||¬¦  «         Y nw xY w|
�
 |
¦   «         nt          j        ¦   «         | _        t          |||	|¬¦  «        | _        d S )N©ÚdeviceÚdtype)r*   r+   r,   r-   ©r   r   r   )ÚsuperÚ__init__Ú	TypeErrorr	   r!   Údropr$   r"   )Úselfr)   r*   r+   r,   r-   r   r   r   r   r.   r1   r2   Úfactory_kwargsÚ	__class__s                 €r#   r5   zBatchNormAct2d.__init__@   sê   ø€ ð	Ø(.¸Ð?Ð?ˆNØ�E‰GŒGÔØðàØ!ØØ$7ðð ð !ðð ð ð øõ ð 	ð 	ð 	å‰GŒG×ÒØØØ!ØØ$7ð ñ ô ð ð ð ð	øøøð %/Ð$:�J�J‘L”L�LÅÄÁÄˆŒ	Ý˜y°ZÈÐ\eÐfÑfÔfˆŒˆˆs   ƒ$( ¨0AÁAc           
      óN  — t          |j        dk    d|j        › d�¦  «         | j        €d}n| j        }| j        rN| j        rG| j        �@| j                             d¦  «         | j        €dt          | j        ¦  «        z  }n| j        }	 | j        rd}n| j        d u o| j	        d u }	 t          j        || j        r| j        r| j        nd | j        r| j        r| j	        nd | j        | j        ||| j        ¦  «        }|                      |¦  «        }|                      |¦  «        }|S )Né   zexpected 4D input (got zD input)g        r   g      ð?T)r   Úndimr+   Útrainingr-   Únum_batches_trackedÚadd_ÚfloatÚrunning_meanÚrunning_varÚFÚ
batch_normÚweightÚbiasr*   r7   r"   )r8   ÚxÚexponential_average_factorÚbn_trainings       r#   ÚforwardzBatchNormAct2d.forwarde   s\  € å�”˜!’ÐG°q´vÐGÐGÐGÑHÔHÐHð
 Œ=Ð Ø),Ð&Ð&à)-¬Ð&àŒ=ð 	?˜TÔ5ð 	?àÔ'Ð3ØÔ(×-Ò-¨aÑ0Ô0Ð0Ø”=Ð(Ø14µu¸TÔ=UÑ7VÔ7VÑ1VÐ.Ð.à15´Ð.ð	ð Œ=ð 	UØˆKˆKàÔ,°Ð4ÐT¸4Ô;KÈtÐ;SˆKð	õ
 ŒLØà%)¤]ÐX°dÔ6NÐXˆDÔÐÐTXØ$(¤MÐW°TÔ5MÐWˆDÔÐÐSWØŒKØŒIØØ&ØŒHñ

ô 

ˆð �IŠI�a‰LŒLˆØ�HŠH�Q‰KŒKˆØˆr%   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r	   ÚReLUÚintrA   Úboolr   r   Ústrr   r   r   ÚModuler5   rK   Ú__classcell__©r:   s   @r#   r'   r'   8   s  ø€ € € € € ðð ð Ø!ØØ(,Ø"Ø#%¤7Ø)-Ø Ø48ØØð#gð #gàð#gð ð#gð ð	#gð
 ð#gð "&ð#gð ð#gð !ð#gð ˜S #˜Xœð#gð ð#gð !  b¤i¤Ô1ð#gð #gð #gð #gð #gð #gðJ0ð 0ð 0ð 0ð 0ð 0ð 0r%   r'   c                   ó<   ‡ — e Zd Zdej        dej        fˆ fd„Zˆ xZS )ÚSyncBatchNormActrH   r   c                 óÞ   •— t          ¦   «                              |¦  «        }t          | d¦  «        r|                      |¦  «        }t          | d¦  «        r|                      |¦  «        }|S )Nr7   r"   )r4   rK   Úhasattrr7   r"   )r8   rH   r:   s     €r#   rK   zSyncBatchNormAct.forwardž   s_   ø€ Ý‰GŒG�OŠO˜AÑÔˆÝ�4˜Ñ Ô ð 	Ø—	’	˜!‘”ˆAÝ�4˜ÑÔð 	Ø—’˜‘”ˆAØˆr%   )rL   rM   rN   ÚtorchÚTensorrK   rU   rV   s   @r#   rX   rX   ˜   sO   ø€ € € € € ð˜œð ¨%¬,ð ð ð ð ð ð ð ð ð ð r%   rX   c                 ó:  — | }t          | t          j        j        j        j        ¦  «        �r/t          | t          ¦  «        rGt          | j        | j	        | j
        | j        | j        |¬¦  «        }| j        |_        | j        |_        n=t          j                             | j        | j	        | j
        | j        | j        |¦  «        }| j        rCt          j        ¦   «         5  | j        |_        | j        |_        d d d ¦  «         n# 1 swxY w Y   | j        |_        | j        |_        | j        |_        | j        |_        t/          | d¦  «        r| j        |_        |                      ¦   «         D ])\  }}|                     |t7          ||¦  «        ¦  «         Œ*~ |S )N)Úprocess_groupÚqconfig)Ú
isinstancer[   r	   ÚmodulesÚ	batchnormÚ
_BatchNormr'   rX   r)   r*   r+   r,   r-   r"   r7   ÚSyncBatchNormÚno_gradrF   rG   rB   rC   r?   r>   rZ   r_   Únamed_childrenÚ
add_moduleÚconvert_sync_batchnorm)Úmoduler^   Úmodule_outputÚnameÚchilds        r#   rh   rh   §   sÐ  € à€MÝ�&�%œ(Ô*Ô4Ô?Ñ@Ô@ñ !3Ý�f�nÑ-Ô-ð 	å,ØÔ#Ø”
Ø”Ø”ØÔ*Ø+ðñ ô ˆMð !'¤
ˆMÔØ!'¤ˆMÔÐõ "œH×2Ò2ØÔ#Ø”
Ø”Ø”ØÔ*Øñô ˆMð Œ=ð 	1Ý”‘”ð 1ð 1Ø'-¤}�Ô$Ø%+¤[�Ô"ð1ð 1ð 1ñ 1ô 1ð 1ð 1ð 1ð 1ð 1ð 1øøøð 1ð 1ð 1ð 1ð &,Ô%8ˆÔ"Ø$*Ô$6ˆÔ!Ø,2Ô,FˆÔ)Ø!'¤ˆÔÝ�6˜9Ñ%Ô%ð 	3Ø$*¤NˆMÔ!Ø×,Ò,Ñ.Ô.ð Uð U‰ˆˆeØ× Ò  Õ'=¸eÀ]Ñ'SÔ'SÑTÔTÐTÐTØØÐs   Ã DÄD	ÄD	c                   ó
  ‡ — e Zd ZdZddej        dddddfdededede	d	e
eef         d
edeeej                          fˆ fd„Zdededededee         dee         dee         fˆ fd„Zdej        dej        fd„Zdefd„Zˆ xZS )ÚFrozenBatchNormAct2da$  
    BatchNormAct2d where the batch statistics and the affine parameters are fixed

    Args:
        num_features (int): Number of features ``C`` from an expected input of size ``(N, C, H, W)``
        eps (float): a value added to the denominator for numerical stability. Default: 1e-5
    r(   TNr)   r*   r   r   r   r   r.   c
                 ó  •— ||	dœ}
t          ¦   «                              ¦   «          || _        |                      dt	          j        |fi |
¤Ž¦  «         |                      dt	          j        |fi |
¤Ž¦  «         |                      dt	          j        |fi |
¤Ž¦  «         |                      dt	          j        |fi |
¤Ž¦  «         |�
 |¦   «         nt          j        ¦   «         | _	        t          ||||¬¦  «        | _        d S )Nr0   rF   rG   rB   rC   r3   )r4   r5   r*   Úregister_bufferr[   ÚonesÚzerosr	   r!   r7   r$   r"   )r8   r)   r*   r   r   r   r   r.   r1   r2   Úddr:   s              €r#   r5   zFrozenBatchNormAct2d.__init__Ü   s  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØˆŒØ×Ò˜X¥u¤z°,Ð'EÐ'EÀ"Ð'EÐ'EÑFÔFÐFØ×Ò˜V¥U¤[°Ð%DÐ%DÀÐ%DÐ%DÑEÔEÐEØ×Ò˜^­U¬[¸Ð-LÐ-LÈÐ-LÐ-LÑMÔMÐMØ×Ò˜]­E¬J°|Ð,JÐ,JÀrÐ,JÐ,JÑKÔKÐKà$.Ð$:�J�J‘L”L�LÅÄÁÄˆŒ	Ý˜y°ZÈÐ\eÐfÑfÔfˆŒˆˆr%   Ú
state_dictÚprefixÚlocal_metadataÚstrictÚmissing_keysÚunexpected_keysÚ
error_msgsc           	      ón   •— |dz   }||v r||= t          ¦   «                              |||||||¦  «         d S )Nr?   )r4   Ú_load_from_state_dict)
r8   rt   ru   rv   rw   rx   ry   rz   Únum_batches_tracked_keyr:   s
            €r#   r|   z*FrozenBatchNormAct2d._load_from_state_dictó   s[   ø€ ð #)Ð+@Ñ"@ÐØ" jÐ0Ð0ØÐ2Ð3å‰Œ×%Ò%Ø˜ °¸ÀoÐWañ	
ô 	
ð 	
ð 	
ð 	
r%   rH   r   c                 óœ  — | j                              dddd¦  «        }| j                             dddd¦  «        }| j                             dddd¦  «        }| j                             dddd¦  «        }||| j        z                        ¦   «         z  }|||z  z
  }||z  |z   }|                      |                      |¦  «        ¦  «        }|S )Nr   éÿÿÿÿ)	rF   ÚreshaperG   rC   rB   r*   Úrsqrtr"   r7   )r8   rH   ÚwÚbÚrvÚrmÚscalerG   s           r#   rK   zFrozenBatchNormAct2d.forward  sÉ   € ð ŒK×Ò  2 q¨!Ñ,Ô,ˆØŒI×Ò˜a  Q¨Ñ*Ô*ˆØÔ×%Ò% a¨¨Q°Ñ2Ô2ˆØÔ×&Ò& q¨"¨a°Ñ3Ô3ˆØ�R˜$œ(‘]×)Ò)Ñ+Ô+Ñ+ˆØ�2˜‘:‰~ˆØ�‰I˜ÑˆØ�HŠH�T—Y’Y˜q‘\”\Ñ"Ô"ˆØˆr%   c                 óf   — | j         j        › d| j        j        d         › d| j        › d| j        › d�S )Nú(r   z, eps=z, act=ú))r:   rL   rF   Úshaper*   r"   )r8   s    r#   Ú__repr__zFrozenBatchNormAct2d.__repr__  s>   € Ø”.Ô)ÐcÐc¨D¬KÔ,=¸aÔ,@ÐcÐcÈÌÐcÐcÐX\ÔX`ÐcÐcÐcÐcr%   )rL   rM   rN   rO   r	   rP   rQ   rA   rR   r   r   rS   r   r   r   rT   r5   Údictr   r|   r[   r\   rK   r‹   rU   rV   s   @r#   rn   rn   Ò   s…  ø€ € € € € ðð ð Ø"Ø#%¤7Ø)-Ø Ø48ØØðgð gàðgð ðgð ð	gð
 !ðgð ˜S #˜Xœðgð ðgð !  b¤i¤Ô1ðgð gð gð gð gð gð.
àð
ð ð
ð ð	
ð
 ð
ð ˜3”ið
ð ˜cœð
ð ˜”Ið
ð 
ð 
ð 
ð 
ð 
ð$˜œð ¨%¬,ð ð ð ð ðd˜#ð dð dð dð dð dð dð dð dr%   rn   c                 ó  — | }t          | t          t          f¦  «        rùt          | j        ¦  «        }| j        |_        | j        |_        | j        rt| j        j                             ¦   «          	                    ¦   «         |j        _        | j
        j                             ¦   «          	                    ¦   «         |j
        _        | j        j        |j        _        | j        j        |j        _        | j        |_        | j        |_        | j        |_        �ngt          | t           j        j        j        j        t           j        j        j        j        f¦  «        ràt-          | j        ¦  «        }| j        |_        | j        |_        | j        rt| j        j                             ¦   «          	                    ¦   «         |j        _        | j
        j                             ¦   «          	                    ¦   «         |j
        _        | j        j        |j        _        | j        j        |j        _        | j        |_        nC|                      ¦   «         D ].\  }}t1          |¦  «        }||ur|                     ||¦  «         Œ/|S )aÂ  
    Converts all `BatchNorm2d` and `SyncBatchNorm` or `BatchNormAct2d` and `SyncBatchNormAct2d` layers
    of provided module into `FrozenBatchNorm2d` or `FrozenBatchNormAct2d` respectively.

    Args:
        module (torch.nn.Module): Any PyTorch module.

    Returns:
        torch.nn.Module: Resulting module

    Inspired by https://github.com/pytorch/pytorch/blob/a5895f85be0f10212791145bfedc0261d364f103/torch/nn/modules/batchnorm.py#L762
    )r`   r'   rX   rn   r)   r,   rF   ÚdataÚcloneÚdetachrG   rB   rC   r*   r7   r"   r[   r	   ra   rb   ÚBatchNorm2drd   r   rf   Úfreeze_batch_norm_2drg   ©ri   Úresrk   rl   Ú	new_childs        r#   r’   r’     sý  € ð €CÝ�&�>Õ+;Ð<Ñ=Ô=ð 0Ý" 6Ô#6Ñ7Ô7ˆØ!Ô.ˆÔØ”]ˆŒ
ØŒ=ð 	>Ø$œmÔ0×6Ò6Ñ8Ô8×?Ò?ÑAÔAˆCŒJŒOØ"œKÔ,×2Ò2Ñ4Ô4×;Ò;Ñ=Ô=ˆCŒHŒMØ &Ô 3Ô 8ˆÔÔØ%Ô1Ô6ˆŒÔØ”*ˆŒØ”;ˆŒØ”*ˆŒ‰Ý	�F�UœXÔ-Ô7ÔCÅUÄXÔEUÔE_ÔEmÐnÑ	oÔ	oð 0Ý Ô 3Ñ4Ô4ˆØ!Ô.ˆÔØ”]ˆŒ
ØŒ=ð 	>Ø$œmÔ0×6Ò6Ñ8Ô8×?Ò?ÑAÔAˆCŒJŒOØ"œKÔ,×2Ò2Ñ4Ô4×;Ò;Ñ=Ô=ˆCŒHŒMØ &Ô 3Ô 8ˆÔÔØ%Ô1Ô6ˆŒÔØ”*ˆŒˆà!×0Ò0Ñ2Ô2ð 	0ð 	0‰KˆD�%Ý,¨UÑ3Ô3ˆIØ Ð%Ð%Ø—’˜t YÑ/Ô/Ð/øØ€Jr%   c                 óV  — | }t          | t          ¦  «        rát          | j        ¦  «        }| j        rt| j        j                             ¦   «                              ¦   «         |j        _        | j	        j                             ¦   «                              ¦   «         |j	        _        | j
        j        |j
        _        | j        j        |j        _        | j        |_        | j        |_        | j        |_        �n0t          | t          ¦  «        rØt           j                             | j        ¦  «        }| j        rt| j        j                             ¦   «                              ¦   «         |j        _        | j	        j                             ¦   «                              ¦   «         |j	        _        | j
        j        |j
        _        | j        j        |j        _        | j        |_        nC|                      ¦   «         D ].\  }}t)          |¦  «        }||ur|                     ||¦  «         Œ/|S )a  
    Converts all `FrozenBatchNorm2d` layers of provided module into `BatchNorm2d`. If `module` is itself and instance
    of `FrozenBatchNorm2d`, it is converted into `BatchNorm2d` and returned. Otherwise, the module is walked
    recursively and submodules are converted in place.

    Args:
        module (torch.nn.Module): Any PyTorch module.

    Returns:
        torch.nn.Module: Resulting module

    Inspired by https://github.com/pytorch/pytorch/blob/a5895f85be0f10212791145bfedc0261d364f103/torch/nn/modules/batchnorm.py#L762
    )r`   rn   r'   r)   r,   rF   rŽ   r�   r�   rG   rB   rC   r*   r7   r"   r   r[   r	   r‘   rf   Úunfreeze_batch_norm_2drg   r“   s        r#   r—   r—   B  s¾  € ð €CÝ�&Õ.Ñ/Ô/ð 0Ý˜VÔ0Ñ1Ô1ˆØŒ=ð 	>Ø$œmÔ0×6Ò6Ñ8Ô8×?Ò?ÑAÔAˆCŒJŒOØ"œKÔ,×2Ò2Ñ4Ô4×;Ò;Ñ=Ô=ˆCŒHŒMØ &Ô 3Ô 8ˆÔÔØ%Ô1Ô6ˆŒÔØ”*ˆŒØ”;ˆŒØ”*ˆŒ‰Ý	�FÕ-Ñ	.Ô	.ð 0ÝŒh×"Ò" 6Ô#6Ñ7Ô7ˆØŒ=ð 	>Ø$œmÔ0×6Ò6Ñ8Ô8×?Ò?ÑAÔAˆCŒJŒOØ"œKÔ,×2Ò2Ñ4Ô4×;Ò;Ñ=Ô=ˆCŒHŒMØ &Ô 3Ô 8ˆÔÔØ%Ô1Ô6ˆŒÔØ”*ˆŒˆà!×0Ò0Ñ2Ô2ð 	0ð 	0‰KˆD�%Ý.¨uÑ5Ô5ˆIØ Ð%Ð%Ø—’˜t YÑ/Ô/Ð/øØ€Jr%   Únum_channelsÚ
num_groupsÚ
group_sizec                 ó*   — |r| |z  dk    sJ ‚| |z  S |S )Nr   © )r˜   r™   rš   s      r#   Ú_num_groupsr�   k  s1   € Øð *Ø˜jÑ(¨AÒ-Ð-Ð-Ð-Ø˜zÑ)Ð)ØÐr%   c                   óÚ   ‡ — e Zd ZU ej        j        e         ed<   dddddej	        dddddfde
de
ded	ed
ee
         dededeeef         dedeeej                          fˆ fd„Zd„ Zˆ xZS )ÚGroupNormActÚ
_fast_normé    r(   TNr˜   r™   r*   r,   rš   r   r   r   r   r.   c                 ó  •— t          ¦   «                              t          |||¦  «        |||||¬¦  «         |
�
 |
¦   «         nt          j        ¦   «         | _        t          |||	|¬¦  «        | _        t          ¦   «         | _	        d S )N©r*   r,   r1   r2   r3   )
r4   r5   r�   r	   r!   r7   r$   r"   r   r    )r8   r˜   r™   r*   r,   rš   r   r   r   r   r.   r1   r2   r:   s                €r#   r5   zGroupNormAct.__init__v  sŠ   ø€ õ 	‰Œ×ÒÝ˜ j°*Ñ=Ô=ØØØØØð 	ñ 	
ô 	
ð 	
ð %/Ð$:�J�J‘L”L�LÅÄÁÄˆŒ	Ý˜y°ZÈÐ\eÐfÑfÔfˆŒå&™.œ.ˆŒˆˆr%   c                 ó  — | j         r(t          || j        | j        | j        | j        ¦  «        }n,t          j        || j        | j        | j        | j        ¦  «        }|                      |¦  «        }|  	                    |¦  «        }|S ©N©
r    r   r™   rF   rG   r*   rD   Ú
group_normr7   r"   ©r8   rH   s     r#   rK   zGroupNormAct.forward’  ón   € ØŒ?ð 	SÝ  4¤?°D´KÀÄÈDÌHÑUÔUˆAˆAå”˜Q ¤°´¸d¼iÈÌÑRÔRˆAØ�IŠI�a‰LŒLˆØ�HŠH�Q‰KŒKˆØˆr%   )rL   rM   rN   r[   ÚjitÚFinalrR   Ú__annotations__r	   rP   rQ   rA   r   r   r   rS   r   r   rT   r5   rK   rU   rV   s   @r#   rŸ   rŸ   r  s  ø€ € € € € € Ø”	” Ô%Ð%Ð%Ñ%ð !ØØØ(,Ø"Ø#%¤7Ø)-Ø Ø48ØØð)ð )àð)ð ð)ð ð	)ð
 ð)ð ! œð)ð ð)ð !ð)ð ˜S #˜Xœð)ð ð)ð !  b¤i¤Ô1ð)ð )ð )ð )ð )ð )ð8ð ð ð ð ð ð r%   rŸ   c                   óÂ   ‡ — e Zd ZU ej        j        e         ed<   dddej	        dddddf	de
dededed	ed
eeef         dedeeej                          fˆ fd„Zd„ Zˆ xZS )ÚGroupNorm1Actr    r(   TNr˜   r*   r,   r   r   r   r   r.   c                 óô   •— t          ¦   «                              d||||	|
¬¦  «         |�
 |¦   «         nt          j        ¦   «         | _        t          ||||¬¦  «        | _        t          ¦   «         | _        d S )Nr   r£   r3   ©	r4   r5   r	   r!   r7   r$   r"   r   r    )r8   r˜   r*   r,   r   r   r   r   r.   r1   r2   r:   s              €r#   r5   zGroupNorm1Act.__init__Ÿ  so   ø€ õ 	‰Œ×Ò˜˜L¨c¸&ÈÐW\ÐÑ]Ô]Ð]Ø$.Ð$:�J�J‘L”L�LÅÄÁÄˆŒ	Ý˜y°ZÈÐ\eÐfÑfÔfˆŒå&™.œ.ˆŒˆˆr%   c                 ó  — | j         r(t          || j        | j        | j        | j        ¦  «        }n,t          j        || j        | j        | j        | j        ¦  «        }|                      |¦  «        }|  	                    |¦  «        }|S r¥   r¦   r¨   s     r#   rK   zGroupNorm1Act.forward²  r©   r%   ©rL   rM   rN   r[   rª   r«   rR   r¬   r	   rP   rQ   rA   r   r   rS   r   r   r   rT   r5   rK   rU   rV   s   @r#   r®   r®   œ  sè   ø€ € € € € € Ø”	” Ô%Ð%Ð%Ñ%ð
 ØØ"Ø#%¤7Ø)-Ø Ø48ØØð)ð )àð)ð ð)ð ð	)ð
 ð)ð !ð)ð ˜S #˜Xœð)ð ð)ð !  b¤i¤Ô1ð)ð )ð )ð )ð )ð )ð&ð ð ð ð ð ð r%   r®   c                   óæ   ‡ — e Zd ZU ej        j        e         ed<   dddej	        dddfde
eee         ej        f         dededed	ed
eeef         dedeeej                          fˆ fd„Zd„ Zˆ xZS )ÚLayerNormActr    r(   TNÚnormalization_shaper*   r,   r   r   r   r   r.   c	                 óà   •—  t          ¦   «         j        |f||dœ|	¤Ž |�
 |¦   «         nt          j        ¦   «         | _        t          ||||¬¦  «        | _        t          ¦   «         | _        d S ©N)r*   Úelementwise_affiner3   r°   ©r8   rµ   r*   r,   r   r   r   r   r.   Úkwargsr:   s             €r#   r5   zLayerNormAct.__init__¿  sq   ø€ ð 	�‰ŒÔÐ,Ð[°#È&Ð[Ð[ÐTZÐ[Ð[Ð[Ø$.Ð$:�J�J‘L”L�LÅÄÁÄˆŒ	Ý˜y°ZÈÐ\eÐfÑfÔfˆŒå&™.œ.ˆŒˆˆr%   c                 ó  — | j         r(t          || j        | j        | j        | j        ¦  «        }n,t          j        || j        | j        | j        | j        ¦  «        }|                      |¦  «        }|  	                    |¦  «        }|S r¥   )
r    r   Únormalized_shaperF   rG   r*   rD   Ú
layer_normr7   r"   r¨   s     r#   rK   zLayerNormAct.forwardÑ  sr   € ØŒ?ð 	YÝ  4Ô#8¸$¼+ÀtÄyÐRVÔRZÑ[Ô[ˆAˆAå”˜Q Ô 5°t´{ÀDÄIÈtÌxÑXÔXˆAØ�IŠI�a‰LŒLˆØ�HŠH�Q‰KŒKˆØˆr%   )rL   rM   rN   r[   rª   r«   rR   r¬   r	   rP   r   rQ   r   ÚSizerA   r   r   rS   r   r   r   rT   r5   rK   rU   rV   s   @r#   r´   r´   ¼  sô   ø€ € € € € € Ø”	” Ô%Ð%Ð%Ñ%ð
 ØØ"Ø#%¤7Ø)-Ø Ø48ð)ð )à!& s¨D°¬I°u´zÐ'AÔ!Bð)ð ð)ð ð	)ð
 ð)ð !ð)ð ˜S #˜Xœð)ð ð)ð !  b¤i¤Ô1ð)ð )ð )ð )ð )ð )ð$ð ð ð ð ð ð r%   r´   c                   óº   ‡ — e Zd Zdddej        dddfdeeee         ej	        f         de
dededed	eeef         d
edeeej                          fˆ fd„Zd„ Zˆ xZS )ÚLayerNormActFp32r(   TNrµ   r*   r,   r   r   r   r   r.   c	                 óº   •—  t          ¦   «         j        |f||dœ|	¤Ž |�
 |¦   «         nt          j        ¦   «         | _        t          ||||¬¦  «        | _        d S r·   ©r4   r5   r	   r!   r7   r$   r"   r¹   s             €r#   r5   zLayerNormActFp32.__init__Ý  sf   ø€ ð 	�‰ŒÔÐ,Ð[°#È&Ð[Ð[ÐTZÐ[Ð[Ð[Ø$.Ð$:�J�J‘L”L�LÅÄÁÄˆŒ	Ý˜y°ZÈÐ\eÐfÑfÔfˆŒˆˆr%   c                 óz  — | j         �| j                              ¦   «         nd }| j        �| j                             ¦   «         nd }t          j        |                     ¦   «         | j        ||| j        ¦  «                             |j        ¦  «        }|  	                    |¦  «        }|  
                    |¦  «        }|S r¥   )rF   rA   rG   rD   r½   r¼   r*   Útor2   r7   r"   ©r8   rH   rF   rG   s       r#   rK   zLayerNormActFp32.forwardí  s—   € Ø(,¬Ð(?�”×"Ò"Ñ$Ô$Ð$ÀTˆØ$(¤IÐ$9ˆtŒy�ŠÑ Ô Ð ¸tˆÝŒL˜Ÿš™œ DÔ$9¸6À4ÈÌÑRÔR×UÒUÐVWÔV]Ñ^Ô^ˆØ�IŠI�a‰LŒLˆØ�HŠH�Q‰KŒKˆØˆr%   )rL   rM   rN   r	   rP   r   rQ   r   r[   r¾   rA   rR   r   r   rS   r   r   r   rT   r5   rK   rU   rV   s   @r#   rÀ   rÀ   Û  sì   ø€ € € € € ð
 ØØ"Ø#%¤7Ø)-Ø Ø48ðgð gà!& s¨D°¬I°u´zÐ'AÔ!Bðgð ðgð ð	gð
 ðgð !ðgð ˜S #˜Xœðgð ðgð !  b¤i¤Ô1ðgð gð gð gð gð gð ð ð ð ð ð ð r%   rÀ   c                   ó¾   ‡ — e Zd ZU ej        j        e         ed<   dddej	        dddfde
dededed	ed
eeef         dedeeej                          fˆ fd„Zd„ Zˆ xZS )ÚLayerNormAct2dr    r(   TNr˜   r*   r,   r   r   r   r   r.   c	                 óà   •—  t          ¦   «         j        |f||dœ|	¤Ž |�
 |¦   «         nt          j        ¦   «         | _        t          ||||¬¦  «        | _        t          ¦   «         | _        d S r·   r°   ©r8   r˜   r*   r,   r   r   r   r   r.   rº   r:   s             €r#   r5   zLayerNormAct2d.__init__ù  so   ø€ ð 	�‰ŒÔ˜ÐT¨3À6ÐTÐTÈVÐTÐTÐTØ$.Ð$:�J�J‘L”L�LÅÄÁÄˆŒ	Ý˜y°ZÈÐ\eÐfÑfÔfˆŒÝ&™.œ.ˆŒˆˆr%   c                 óp  — |                      dddd¦  «        }| j        r(t          || j        | j        | j        | j        ¦  «        }n,t          j        || j        | j        | j        | j        ¦  «        }|                      dddd¦  «        }|  	                    |¦  «        }|  
                    |¦  «        }|S ©Nr   é   é   r   )Úpermuter    r   r¼   rF   rG   r*   rD   r½   r7   r"   r¨   s     r#   rK   zLayerNormAct2d.forward
  s    € Ø�IŠI�a˜˜A˜qÑ!Ô!ˆØŒ?ð 	YÝ  4Ô#8¸$¼+ÀtÄyÐRVÔRZÑ[Ô[ˆAˆAå”˜Q Ô 5°t´{ÀDÄIÈtÌxÑXÔXˆAØ�IŠI�a˜˜A˜qÑ!Ô!ˆØ�IŠI�a‰LŒLˆØ�HŠH�Q‰KŒKˆØˆr%   r²   rV   s   @r#   rÇ   rÇ   ö  sâ   ø€ € € € € € Ø”	” Ô%Ð%Ð%Ñ%ð
 ØØ"Ø#%¤7Ø)-Ø Ø48ð)ð )àð)ð ð)ð ð	)ð
 ð)ð !ð)ð ˜S #˜Xœð)ð ð)ð !  b¤i¤Ô1ð)ð )ð )ð )ð )ð )ð"	ð 	ð 	ð 	ð 	ð 	ð 	r%   rÇ   c                   ó’   ‡ — e Zd Zdddej        dddfdededededed	e	e
ef         d
edeeej                          fˆ fd„Zd„ Zˆ xZS )ÚLayerNormAct2dFp32r(   TNr˜   r*   r,   r   r   r   r   r.   c	                 óº   •—  t          ¦   «         j        |f||dœ|	¤Ž |�
 |¦   «         nt          j        ¦   «         | _        t          ||||¬¦  «        | _        d S r·   rÂ   rÉ   s             €r#   r5   zLayerNormAct2dFp32.__init__  sd   ø€ ð 	�‰ŒÔ˜ÐT¨3À6ÐTÐTÈVÐTÐTÐTØ$.Ð$:�J�J‘L”L�LÅÄÁÄˆŒ	Ý˜y°ZÈÐ\eÐfÑfÔfˆŒˆˆr%   c                 óÚ  — |                      dddd¦  «        }| j        �| j                             ¦   «         nd }| j        �| j                             ¦   «         nd }t	          j        |                     ¦   «         | j        ||| j        ¦  «                             |j	        ¦  «        }|                      dddd¦  «        }|  
                    |¦  «        }|                      |¦  «        }|S rË   )rÎ   rF   rA   rG   rD   r½   r¼   r*   rÄ   r2   r7   r"   rÅ   s       r#   rK   zLayerNormAct2dFp32.forward(  sÅ   € Ø�IŠI�a˜˜A˜qÑ!Ô!ˆØ(,¬Ð(?�”×"Ò"Ñ$Ô$Ð$ÀTˆØ$(¤IÐ$9ˆtŒy�ŠÑ Ô Ð ¸tˆÝŒL˜Ÿš™œ DÔ$9¸6À4ÈÌÑRÔR×UÒUÐVWÔV]Ñ^Ô^ˆØ�IŠI�a˜˜A˜qÑ!Ô!ˆØ�IŠI�a‰LŒLˆØ�HŠH�Q‰KŒKˆØˆr%   )rL   rM   rN   r	   rP   rQ   rA   rR   r   r   rS   r   r   r   rT   r5   rK   rU   rV   s   @r#   rÐ   rÐ     sÚ   ø€ € € € € ð
 ØØ"Ø#%¤7Ø)-Ø Ø48ðgð gàðgð ðgð ð	gð
 ðgð !ðgð ˜S #˜Xœðgð ðgð !  b¤i¤Ô1ðgð gð gð gð gð gð ð ð ð ð ð ð r%   rÐ   c                   ó´   ‡ — e Zd ZdZdddej        dddfdedededed	e	d
e
eef         dedeeej                          fˆ fd„Zdej        dej        fd„Zˆ xZS )Ú
RmsNormActá*   RMSNorm + Activation for '2D' NCHW tensors

    NOTE: It's currently (2025-05-10) faster to use an eager 2d kernel that does reduction
    on dim=1 than to permute and use internal PyTorch F.rms_norm, this may change if something
    like https://github.com/pytorch/pytorch/pull/150576 lands.
    ç�íµ ÷Æ°>TNr˜   r*   r,   r   r   r   r   r.   c	                 óà   •—  t          ¦   «         j        d|||dœ|	¤Ž |�
 |¦   «         nt          j        ¦   «         | _        t          ||||¬¦  «        | _        t          ¦   «         | _        d S ©N)Úchannelsr*   r,   r3   rœ   r°   rÉ   s             €r#   r5   zRmsNormAct.__init__:  óo   ø€ ð 	�‰ŒÔÐQ ,°CÀÐQÐQÈ&ÐQÐQÐQØ$.Ð$:�J�J‘L”L�LÅÄÁÄˆŒ	Ý˜y°ZÈÐ\eÐfÑfÔfˆŒÝ&™.œ.ˆŒˆˆr%   rH   r   c                 óî   — | j         r"t          || j        | j        | j        ¦  «        }n!t          || j        | j        | j        ¦  «        }|                      |¦  «        }|                      |¦  «        }|S r¥   )r    r   r¼   rF   r*   r   r7   r"   r¨   s     r#   rK   zRmsNormAct.forwardK  sf   € ØŒ?ð 	JÝ˜a Ô!6¸¼ÀTÄXÑNÔNˆAˆAå˜˜DÔ1°4´;ÀÄÑIÔIˆAØ�IŠI�a‰LŒLˆØ�HŠH�Q‰KŒKˆØˆr%   ©rL   rM   rN   rO   r	   rP   rQ   rA   rR   r   r   rS   r   r   r   rT   r5   r[   r\   rK   rU   rV   s   @r#   rÔ   rÔ   3  óì   ø€ € € € € ðð ð ØØ"Ø#%¤7Ø)-Ø Ø48ð)ð )àð)ð ð)ð ð	)ð
 ð)ð !ð)ð ˜S #˜Xœð)ð ð)ð !  b¤i¤Ô1ð)ð )ð )ð )ð )ð )ð"˜œð ¨%¬,ð ð ð ð ð ð ð ð r%   rÔ   c                   ó´   ‡ — e Zd ZdZdddej        dddfdedededed	e	d
e
eef         dedeeej                          fˆ fd„Zdej        dej        fd„Zˆ xZS )ÚRmsNormActFp32rÕ   rÖ   TNr˜   r*   r,   r   r   r   r   r.   c	                 óº   •—  t          ¦   «         j        d|||dœ|	¤Ž |�
 |¦   «         nt          j        ¦   «         | _        t          ||||¬¦  «        | _        d S rØ   rÂ   rÉ   s             €r#   r5   zRmsNormActFp32.__init__\  ód   ø€ ð 	�‰ŒÔÐQ ,°CÀÐQÐQÈ&ÐQÐQÐQØ$.Ð$:�J�J‘L”L�LÅÄÁÄˆŒ	Ý˜y°ZÈÐ\eÐfÑfÔfˆŒˆˆr%   rH   r   c                 ó*  — | j         �| j                              ¦   «         nd }t          |                     ¦   «         | j        || j        ¦  «                             |j        ¦  «        }|                      |¦  «        }|                      |¦  «        }|S r¥   )	rF   rA   r   r¼   r*   rÄ   r2   r7   r"   ©r8   rH   rF   s      r#   rK   zRmsNormActFp32.forwardl  st   € Ø(,¬Ð(?�”×"Ò"Ñ$Ô$Ð$ÀTˆÝ�Q—W’W‘Y”Y Ô 5°v¸t¼xÑHÔH×KÒKÈAÌGÑTÔTˆØ�IŠI�a‰LŒLˆØ�HŠH�Q‰KŒKˆØˆr%   rÜ   rV   s   @r#   rß   rß   U  óû   ø€ € € € € ðð ð ØØ"Ø#%¤7Ø)-Ø Ø48ðgð gàðgð ðgð ð	gð
 ðgð !ðgð ˜S #˜Xœðgð ðgð !  b¤i¤Ô1ðgð gð gð gð gð gð ˜œð ¨%¬,ð ð ð ð ð ð ð ð r%   rß   c                   ó´   ‡ — e Zd ZdZdddej        dddfdedededed	e	d
e
eef         dedeeej                          fˆ fd„Zdej        dej        fd„Zˆ xZS )ÚRmsNormAct2drÕ   rÖ   TNr˜   r*   r,   r   r   r   r   r.   c	                 óà   •—  t          ¦   «         j        d|||dœ|	¤Ž |�
 |¦   «         nt          j        ¦   «         | _        t          ||||¬¦  «        | _        t          ¦   «         | _        d S rØ   r°   rÉ   s             €r#   r5   zRmsNormAct2d.__init__{  rÚ   r%   rH   r   c                 óî   — | j         r"t          || j        | j        | j        ¦  «        }n!t          || j        | j        | j        ¦  «        }|                      |¦  «        }|                      |¦  «        }|S r¥   )r    r   r¼   rF   r*   r   r7   r"   r¨   s     r#   rK   zRmsNormAct2d.forwardŒ  sf   € ØŒ?ð 	LÝ  4Ô#8¸$¼+ÀtÄxÑPÔPˆAˆAå˜1˜dÔ3°T´[À$Ä(ÑKÔKˆAØ�IŠI�a‰LŒLˆØ�HŠH�Q‰KŒKˆØˆr%   rÜ   rV   s   @r#   ræ   ræ   t  rÝ   r%   ræ   c                   ó´   ‡ — e Zd ZdZdddej        dddfdedededed	e	d
e
eef         dedeeej                          fˆ fd„Zdej        dej        fd„Zˆ xZS )ÚRmsNormAct2dFp32rÕ   rÖ   TNr˜   r*   r,   r   r   r   r   r.   c	                 óº   •—  t          ¦   «         j        d|||dœ|	¤Ž |�
 |¦   «         nt          j        ¦   «         | _        t          ||||¬¦  «        | _        d S rØ   rÂ   rÉ   s             €r#   r5   zRmsNormAct2dFp32.__init__�  rá   r%   rH   r   c                 ó*  — | j         �| j                              ¦   «         nd }t          |                     ¦   «         | j        || j        ¦  «                             |j        ¦  «        }|                      |¦  «        }|                      |¦  «        }|S r¥   )	rF   rA   r   r¼   r*   rÄ   r2   r7   r"   rã   s      r#   rK   zRmsNormAct2dFp32.forward­  st   € Ø(,¬Ð(?�”×"Ò"Ñ$Ô$Ð$ÀTˆÝ�q—w’w‘y”y $Ô"7¸ÀÄÑJÔJ×MÒMÈaÌgÑVÔVˆØ�IŠI�a‰LŒLˆØ�HŠH�Q‰KŒKˆØˆr%   rÜ   rV   s   @r#   rê   rê   –  rä   r%   rê   )NFTr¥   )=rO   Útypingr   r   r   r   r   r   r[   r	   Útorch.nnr
   rD   Útorchvision.ops.miscr   Ú_fxr   Ú
create_actr   Ú	fast_normr   r   r   r   r   r   Únormr   r   Útrace_utilsr   r   Útorch.nn.functionalr   ÚImportErrorrS   rR   rT   r$   r‘   r'   rd   rX   rh   rn   r’   r—   rQ   r�   Ú	GroupNormrŸ   r®   Ú	LayerNormr´   rÀ   rÇ   rÐ   rÔ   rß   ræ   rê   rœ   r%   r#   ú<module>rù      sÄ  ððð ð :Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9à €€€Ø Ð Ð Ð Ð Ð Ø $Ð $Ð $Ð $Ð $Ð $Ø 2Ð 2Ð 2Ð 2Ð 2Ð 2à (Ð (Ð (Ð (Ð (Ð (Ø (Ð (Ð (Ð (Ð (Ð (ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð %Ð $Ð $Ð $Ð $Ð $Ð $Ð $Ø  Ð  Ð  Ð  Ð  Ð  Ø Ð Ð Ð Ð Ð ð$Ø,Ð,Ð,Ð,Ð,Ð,Ð,øØð $ð $ð $Ø#Ð#Ð#Ð#Ð#Ð#Ð#Ð#ð$øøøð &*Ø"'Øð	1ð 1Øð1à˜˜c˜”Nð1ð ˜$”ð1ð ð	1ð
 „Yð1ð 1ð 1ð 1ð ð\ð \ð \ð \ð \�R”^ñ \ô \ñ Ôð\ð~ ðð ð ð ð �rÔ'ñ ô ñ Ôðð(ð (ð (ð (ðV ð@dð @dð @dð @dð @d˜5œ8œ?ñ @dô @dñ Ôð@dðF)ð )ð )ðX&ð &ð &ðR˜cð ¨sð Àð ð ð ð ð'ð 'ð 'ð 'ð '�2”<ñ 'ô 'ð 'ðTð ð ð ð �B”Lñ ô ð ð@ð ð ð ð �2”<ñ ô ð ð>ð ð ð ð �r”|ñ ô ð ð6ð ð ð ð �R”\ñ ô ð ð@ð ð ð ð ˜œñ ô ð ð:ð ð ð ð �ñ ô ð ðDð ð ð ð �Wñ ô ð ð>ð ð ð ð �9ñ ô ð ðDð ð ð ð �yñ ô ð ð ð s   ÁA! Á!A/Á.A/