§
    ·¯¸j£)  ã                   ó\  — d Z ddlmZ ddlmZmZmZmZ ddlm	Z	 ddl
mZ ddlmZ  G d„ d	e	j        ¦  «        Z G d
„ de	j        ¦  «        Z eee	j        d¬¦  «        Z G d„ de	j        ¦  «        Z G d„ de	j        ¦  «        Z G d„ de	j        ¦  «        Z G d„ de	j        ¦  «        ZdS )zl MLP module w/ dropout and configurable activation layer

Hacked together by / Copyright 2020 Ross Wightman
é    )Úpartial)ÚOptionalÚTypeÚUnionÚTuple)Únné   )ÚGlobalResponseNorm)Ú	to_2tuplec                   óø   ‡ — e Zd ZdZddej        ddddddf	dedee         dee         d	eej	                 d
eeej	                          de
eeeef         f         de
eeeef         f         defˆ fd„Zd„ Zˆ xZS )ÚMlpz— MLP as used in Vision Transformer, MLP-Mixer and related networks

    NOTE: When use_conv=True, expects 2D NCHW tensors, otherwise N*C expected.
    NTç        FÚin_featuresÚhidden_featuresÚout_featuresÚ	act_layerÚ
norm_layerÚbiasÚdropÚuse_convc                 ó,  •— |	|
dœ}t          ¦   «                              ¦   «          |p|}|p|}t          |¦  «        }t          |¦  «        }|rt          t          j        d¬¦  «        nt          j        } |||fd|d         i|¤Ž| _         |¦   «         | _        t	          j	        |d         ¦  «        | _
        |�	 ||fi |¤Žnt	          j        ¦   «         | _         |||fd|d         i|¤Ž| _        t	          j	        |d         ¦  «        | _        d S )N©ÚdeviceÚdtyper	   ©Úkernel_sizer   r   )ÚsuperÚ__init__r   r   r   ÚConv2dÚLinearÚfc1ÚactÚDropoutÚdrop1ÚIdentityÚnormÚfc2Údrop2)Úselfr   r   r   r   r   r   r   r   r   r   ÚddÚ
drop_probsÚlinear_layerÚ	__class__s                 €ú_/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/layers/mlp.pyr   zMlp.__init__   s"  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ#Ð2 {ˆØ)Ð8¨[ˆÝ˜‰ŒˆÝ˜t‘_”_ˆ
Ø<DÐS•w�rœy°aÐ8Ñ8Ô8Ð8Í"Ì)ˆà�< ¨_ÐQÐQÀ4ÈÄ7ÐQÈbÐQÐQˆŒØ�9‘;”;ˆŒÝ”Z 
¨1¤Ñ.Ô.ˆŒ
Ø9CÐ9O�J�J˜Ð5Ð5°"Ð5Ð5Ð5ÕUWÔU`ÑUbÔUbˆŒ	Ø�< °ÐRÐRÀDÈÄGÐRÈrÐRÐRˆŒÝ”Z 
¨1¤Ñ.Ô.ˆŒ
ˆ
ˆ
ó    c                 ó  — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S ©N)r!   r"   r$   r&   r'   r(   ©r)   Úxs     r.   ÚforwardzMlp.forward/   sa   € Ø�HŠH�Q‰KŒKˆØ�HŠH�Q‰KŒKˆØ�JŠJ�q‰MŒMˆØ�IŠI�a‰LŒLˆØ�HŠH�Q‰KŒKˆØ�JŠJ�q‰MŒMˆØˆr/   ©Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚGELUÚintr   r   ÚModuler   Úboolr   Úfloatr   r4   Ú__classcell__©r-   s   @r.   r   r      s  ø€ € € € € ðð ð .2Ø*.Ø)+¬Ø48Ø37Ø68Ø"ØØð/ð /àð/ð & cœ]ð/ð # 3œ-ð	/ð
 ˜BœI”ð/ð !  b¤i¤Ô1ð/ð ˜˜e D¨$ JÔ/Ð/Ô0ð/ð ˜˜u U¨E \Ô2Ð2Ô3ð/ð ð/ð /ð /ð /ð /ð /ð8ð ð ð ð ð ð r/   r   c                   ó  ‡ — e Zd ZdZddej        dddddddf
dedee         dee         d	eej	                 d
eeej	                          de
eeeef         f         de
eeeef         f         dedefˆ fd„Zd„ Zd„ Zˆ xZS )ÚGluMlpz¹ MLP w/ GLU style gating
    See: https://arxiv.org/abs/1612.08083, https://arxiv.org/abs/2002.05202

    NOTE: When use_conv=True, expects 2D NCHW tensors, otherwise N*C expected.
    NTr   Fr   r   r   r   r   r   r   r   Ú	gate_lastc                 ór  •— |
|dœ}t          ¦   «                              ¦   «          |p|}|p|}|dz  dk    sJ ‚t          |¦  «        }t          |¦  «        }|rt          t          j        d¬¦  «        nt          j        }|rdnd| _        |	| _         |||fd|d         i|¤Ž| _	         |¦   «         | _
        t	          j        |d         ¦  «        | _        |� ||dz  fi |¤Žnt	          j        ¦   «         | _         ||dz  |fd|d         i|¤Ž| _        t	          j        |d         ¦  «        | _        d S )Nr   é   r   r	   r   éÿÿÿÿr   )r   r   r   r   r   r   r    Ú	chunk_dimrC   r!   r"   r#   r$   r%   r&   r'   r(   )r)   r   r   r   r   r   r   r   r   rC   r   r   r*   r+   r,   r-   s                  €r.   r   zGluMlp.__init__?   sZ  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ#Ð2 {ˆØ)Ð8¨[ˆØ Ñ" aÒ'Ð'Ð'Ð'Ý˜‰ŒˆÝ˜t‘_”_ˆ
Ø<DÐS•w�rœy°aÐ8Ñ8Ô8Ð8Í"Ì)ˆØ&Ð.˜˜¨BˆŒØ"ˆŒà�< ¨_ÐQÐQÀ4ÈÄ7ÐQÈbÐQÐQˆŒØ�9‘;”;ˆŒÝ”Z 
¨1¤Ñ.Ô.ˆŒ
Ø>HÐ>T�J�J˜°!Ñ3Ð:Ð:°rÐ:Ð:Ð:ÕZ\ÔZeÑZgÔZgˆŒ	Ø�< °1Ñ 4°lÐWÐWÈÈaÌÐWÐTVÐWÐWˆŒÝ”Z 
¨1¤Ñ.Ô.ˆŒ
ˆ
ˆ
r/   c                 óF  — | j         j        �It          j                             | j         j        | j         j        j        d         dz  d …         ¦  «         t          j                             | j         j        | j         j        j        d         dz  d …         d¬¦  «         d S )Nr   rE   ç�íµ ÷Æ°>©Ústd)r!   r   r   ÚinitÚones_ÚshapeÚnormal_Úweight©r)   s    r.   Úinit_weightszGluMlp.init_weights_   s~   € àŒ8Œ=Ð$ÝŒG�MŠM˜$œ(œ-¨¬¬Ô(;¸AÔ(>À!Ñ(CÐ(DÐ(DÔEÑFÔFÐFÝ
Œ�Š˜œœ¨¬¬Ô(=¸aÔ(@ÀAÑ(EÐ(FÐ(FÔGÈTˆÑRÔRÐRÐRÐRr/   c                 ó„  — |                       |¦  «        }|                     d| j        ¬¦  «        \  }}| j        r||                      |¦  «        z  n|                      |¦  «        |z  }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S )NrE   )Údim)	r!   ÚchunkrG   rC   r"   r$   r&   r'   r(   )r)   r3   Úx1Úx2s       r.   r4   zGluMlp.forwarde   sœ   € Ø�HŠH�Q‰KŒKˆØ—’˜ ¤�Ñ/Ô/‰ˆˆBØ!%¤ÐFˆB�—’˜"‘”ÑÐ°T·X²X¸b±\´\ÀBÑ5FˆØ�JŠJ�q‰MŒMˆØ�IŠI�a‰LŒLˆØ�HŠH�Q‰KŒKˆØ�JŠJ�q‰MŒMˆØˆr/   )r6   r7   r8   r9   r   ÚSigmoidr;   r   r   r<   r   r=   r   r>   r   rR   r4   r?   r@   s   @r.   rB   rB   9   s'  ø€ € € € € ðð ð .2Ø*.Ø)+¬Ø48Ø37Ø68Ø"Ø"ØØð/ð /àð/ð & cœ]ð/ð # 3œ-ð	/ð
 ˜BœI”ð/ð !  b¤i¤Ô1ð/ð ˜˜e D¨$ JÔ/Ð/Ô0ð/ð ˜˜u U¨E \Ô2Ð2Ô3ð/ð ð/ð ð/ð /ð /ð /ð /ð /ð@Sð Sð Sðð ð ð ð ð ð r/   rB   F)r   rC   c                   óþ   ‡ — e Zd ZdZddej        ddddddf	dedee         dee         d	eej	                 d
eeej	                          de
eeeef         f         de
eeeef         f         defˆ fd„Zd„ Zd„ Zˆ xZS )ÚSwiGLUz¬ SwiGLU
    NOTE: GluMLP above can implement SwiGLU, but this impl has split fc1 and
    better matches some other common impl which makes mapping checkpoints simpler.
    NTr   r   r   r   r   r   r   r   r   Úalign_toc                 óT  •— |	|
dœ}t          ¦   «                              ¦   «          |p|}|p|}t          |¦  «        }t          |¦  «        }|r	|| |z  z   }t          j        ||fd|d         i|¤Ž| _        t          j        ||fd|d         i|¤Ž| _         |¦   «         | _        t          j        |d         ¦  «        | _	        |�	 ||fi |¤Žnt          j
        ¦   «         | _        t          j        ||fd|d         i|¤Ž| _        t          j        |d         ¦  «        | _        d S )Nr   r   r   r	   )r   r   r   r   r    Úfc1_gÚfc1_xr"   r#   r$   r%   r&   r'   r(   )r)   r   r   r   r   r   r   r   r[   r   r   r*   r+   r-   s                €r.   r   zSwiGLU.__init__x   sA  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ#Ð2 {ˆØ)Ð8¨[ˆÝ˜‰ŒˆÝ˜t‘_”_ˆ
àð 	NØ-°/Ð1AÀHÑ1LÑMˆOå”Y˜{¨OÐPÐPÀ$ÀqÄ'ÐPÈRÐPÐPˆŒ
Ý”Y˜{¨OÐPÐPÀ$ÀqÄ'ÐPÈRÐPÐPˆŒ
Ø�9‘;”;ˆŒÝ”Z 
¨1¤Ñ.Ô.ˆŒ
Ø9CÐ9O�J�J˜Ð5Ð5°"Ð5Ð5Ð5ÕUWÔU`ÑUbÔUbˆŒ	Ý”9˜_¨lÐOÐOÀÀaÄÐOÈBÐOÐOˆŒÝ”Z 
¨1¤Ñ.Ô.ˆŒ
ˆ
ˆ
r/   c                 óÆ   — | j         j        �)t          j                             | j         j        ¦  «         t          j                             | j         j        d¬¦  «         d S )NrI   rJ   )r]   r   r   rL   rM   rO   rP   rQ   s    r.   rR   zSwiGLU.init_weights—   sG   € àŒ:Œ?Ð&ÝŒG�MŠM˜$œ*œ/Ñ*Ô*Ð*Ý
Œ�Š˜œ
Ô)¨tˆÑ4Ô4Ð4Ð4Ð4r/   c                 ó2  — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        |z  }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r1   )r]   r^   r"   r$   r&   r'   r(   )r)   r3   Úx_gates      r.   r4   zSwiGLU.forward�   sw   € Ø—’˜A‘”ˆØ�JŠJ�q‰MŒMˆØ�HŠH�VÑÔ˜qÑ ˆØ�JŠJ�q‰MŒMˆØ�IŠI�a‰LŒLˆØ�HŠH�Q‰KŒKˆØ�JŠJ�q‰MŒMˆØˆr/   )r6   r7   r8   r9   r   ÚSiLUr;   r   r   r<   r   r=   r   r>   r   rR   r4   r?   r@   s   @r.   rZ   rZ   s   s  ø€ € € € € ðð ð .2Ø*.Ø)+¬Ø48Ø37Ø68ØØØð/ð /àð/ð & cœ]ð/ð # 3œ-ð	/ð
 ˜BœI”ð/ð !  b¤i¤Ô1ð/ð ˜˜e D¨$ JÔ/Ð/Ô0ð/ð ˜˜u U¨E \Ô2Ð2Ô3ð/ð ð/ð /ð /ð /ð /ð /ð>5ð 5ð 5ðð ð ð ð ð ð r/   rZ   c                   ó  ‡ — e Zd ZdZddej        ddddddf	dedee         dee         deej	                 d	eeej	                          d
eeej	                          de
eeeef         f         de
eeeef         f         fˆ fd„Zd„ Zˆ xZS )ÚGatedMlpz MLP as used in gMLP
    NTr   r   r   r   r   r   Ú
gate_layerr   r   c                 óp  •— |	|
dœ}t          ¦   «                              ¦   «          |p|}|p|}t          |¦  «        }t          |¦  «        }t          j        ||fd|d         i|¤Ž| _         |¦   «         | _        t          j        |d         ¦  «        | _        |�|dz  dk    sJ ‚ ||fi |¤Ž| _	        |dz  }nt          j
        ¦   «         | _	        |�	 ||fi |¤Žnt          j
        ¦   «         | _        t          j        ||fd|d         i|¤Ž| _        t          j        |d         ¦  «        | _        d S )Nr   r   r   rE   r	   )r   r   r   r   r    r!   r"   r#   r$   Úgater%   r&   r'   r(   )r)   r   r   r   r   r   re   r   r   r   r   r*   r+   r-   s                €r.   r   zGatedMlp.__init__«   sP  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ#Ð2 {ˆØ)Ð8¨[ˆÝ˜‰ŒˆÝ˜t‘_”_ˆ
å”9˜[¨/ÐNÐNÀÀQÄÐNÈ2ÐNÐNˆŒØ�9‘;”;ˆŒÝ”Z 
¨1¤Ñ.Ô.ˆŒ
ØÐ!Ø" QÑ&¨!Ò+Ð+Ð+Ð+Ø"˜
 ?Ð9Ð9°bÐ9Ð9ˆDŒIØ-°Ñ2ˆOˆOåœ™œˆDŒIØ9CÐ9O�J�J˜Ð5Ð5°"Ð5Ð5Ð5ÕUWÔU`ÑUbÔUbˆŒ	Ý”9˜_¨lÐOÐOÀÀaÄÐOÈBÐOÐOˆŒÝ”Z 
¨1¤Ñ.Ô.ˆŒ
ˆ
ˆ
r/   c                 ó,  — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r1   )r!   r"   r$   rg   r&   r'   r(   r2   s     r.   r4   zGatedMlp.forwardÌ   sp   € Ø�HŠH�Q‰KŒKˆØ�HŠH�Q‰KŒKˆØ�JŠJ�q‰MŒMˆØ�IŠI�a‰LŒLˆØ�IŠI�a‰LŒLˆØ�HŠH�Q‰KŒKˆØ�JŠJ�q‰MŒMˆØˆr/   r5   r@   s   @r.   rd   rd   ¨   s  ø€ € € € € ðð ð
 .2Ø*.Ø)+¬Ø48Ø48Ø37Ø68ØØð/ð /àð/ð & cœ]ð/ð # 3œ-ð	/ð
 ˜BœI”ð/ð !  b¤i¤Ô1ð/ð !  b¤i¤Ô1ð/ð ˜˜e D¨$ JÔ/Ð/Ô0ð/ð ˜˜u U¨E \Ô2Ð2Ô3ð/ð /ð /ð /ð /ð /ðBð ð ð ð ð ð r/   rd   c                   óÒ   ‡ — e Zd ZdZddej        dddddfdedee         dee         deej	                 d	eeej	                          d
e
eeeef         f         defˆ fd„Zd„ Zˆ xZS )ÚConvMlpzG MLP using 1x1 convs that keeps spatial dims (for 2D NCHW tensors)
    NTr   r   r   r   r   r   r   r   c
                 óš  •— ||	dœ}
t          ¦   «                              ¦   «          |p|}|p|}t          |¦  «        }t          j        ||fd|d         dœ|
¤Ž| _        |r	 ||fi |
¤Žnt          j        ¦   «         | _         |¦   «         | _        t          j	        |¦  «        | _
        t          j        ||fd|d         dœ|
¤Ž| _        d S )Nr   r	   r   )r   r   )r   r   r   r   r   r!   r%   r&   r"   r#   r   r'   )r)   r   r   r   r   r   r   r   r   r   r*   r-   s              €r.   r   zConvMlp.__init__Ú   sä   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ#Ð2 {ˆØ)Ð8¨[ˆÝ˜‰Œˆå”9˜[¨/Ð]ÀqÈtÐTUÌwÐ]Ð]ÐZ\Ð]Ð]ˆŒØ9CÐV�J�J˜Ð5Ð5°"Ð5Ð5Ð5ÍÌÉÌˆŒ	Ø�9‘;”;ˆŒÝ”J˜tÑ$Ô$ˆŒ	Ý”9˜_¨lÐ^ÈÐPTÐUVÔPWÐ^Ð^Ð[]Ð^Ð^ˆŒˆˆr/   c                 óØ   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r1   )r!   r&   r"   r   r'   r2   s     r.   r4   zConvMlp.forwardò   sR   € Ø�HŠH�Q‰KŒKˆØ�IŠI�a‰LŒLˆØ�HŠH�Q‰KŒKˆØ�IŠI�a‰LŒLˆØ�HŠH�Q‰KŒKˆØˆr/   )r6   r7   r8   r9   r   ÚReLUr;   r   r   r<   r   r=   r   r>   r   r4   r?   r@   s   @r.   rj   rj   ×   sõ   ø€ € € € € ðð ð
 .2Ø*.Ø)+¬Ø48Ø37ØØØð_ð _àð_ð & cœ]ð_ð # 3œ-ð	_ð
 ˜BœI”ð_ð !  b¤i¤Ô1ð_ð ˜˜e D¨$ JÔ/Ð/Ô0ð_ð ð_ð _ð _ð _ð _ð _ð0ð ð ð ð ð ð r/   rj   c                   óÐ   ‡ — e Zd ZdZddej        dddddfdedee         dee         d	eej	                 d
e
eeeef         f         de
eeeef         f         defˆ fd„Zd„ Zˆ xZS )ÚGlobalResponseNormMlpz° MLP w/ Global Response Norm (see grn.py), nn.Linear or 1x1 Conv2d

    NOTE: Intended for '2D' NCHW (use_conv=True) or NHWC (use_conv=False, channels-last) tensor layouts
    NTr   Fr   r   r   r   r   r   r   c
                 ó  •— ||	dœ}
t          ¦   «                              ¦   «          |p|}|p|}t          |¦  «        }t          |¦  «        }|rt          t          j        d¬¦  «        nt          j        } |||fd|d         i|
¤Ž| _         |¦   «         | _        t	          j	        |d         ¦  «        | _
        t          |fd| i|
¤Ž| _         |||fd|d         i|
¤Ž| _        t	          j	        |d         ¦  «        | _        d S )Nr   r	   r   r   r   Úchannels_last)r   r   r   r   r   r   r    r!   r"   r#   r$   r
   Úgrnr'   r(   )r)   r   r   r   r   r   r   r   r   r   r*   r+   r,   r-   s                €r.   r   zGlobalResponseNormMlp.__init__   s  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ#Ð2 {ˆØ)Ð8¨[ˆÝ˜‰ŒˆÝ˜t‘_”_ˆ
Ø<DÐS•w�rœy°aÐ8Ñ8Ô8Ð8Í"Ì)ˆà�< ¨_ÐQÐQÀ4ÈÄ7ÐQÈbÐQÐQˆŒØ�9‘;”;ˆŒÝ”Z 
¨1¤Ñ.Ô.ˆŒ
Ý% oÐXÐXÈÀ\ÐXÐUWÐXÐXˆŒØ�< °ÐRÐRÀDÈÄGÐRÈrÐRÐRˆŒÝ”Z 
¨1¤Ñ.Ô.ˆŒ
ˆ
ˆ
r/   c                 ó  — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r1   )r!   r"   r$   rr   r'   r(   r2   s     r.   r4   zGlobalResponseNormMlp.forward  sa   € Ø�HŠH�Q‰KŒKˆØ�HŠH�Q‰KŒKˆØ�JŠJ�q‰MŒMˆØ�HŠH�Q‰KŒKˆØ�HŠH�Q‰KŒKˆØ�JŠJ�q‰MŒMˆØˆr/   r5   r@   s   @r.   ro   ro   û   sï   ø€ € € € € ðð ð .2Ø*.Ø)+¬Ø37Ø68Ø"ØØð/ð /àð/ð & cœ]ð/ð # 3œ-ð	/ð
 ˜BœI”ð/ð ˜˜e D¨$ JÔ/Ð/Ô0ð/ð ˜˜u U¨E \Ô2Ð2Ô3ð/ð ð/ð /ð /ð /ð /ð /ð6ð ð ð ð ð ð r/   ro   N)r9   Ú	functoolsr   Útypingr   r   r   r   Útorchr   rr   r
   Úhelpersr   r<   r   rB   rb   ÚSwiGLUPackedrZ   rd   rj   ro   © r/   r.   ú<module>rz      s®  ððð ð Ð Ð Ð Ð Ð Ø /Ð /Ð /Ð /Ð /Ð /Ð /Ð /Ð /Ð /Ð /Ð /à Ð Ð Ð Ð Ð à #Ð #Ð #Ð #Ð #Ð #Ø Ð Ð Ð Ð Ð ð(ð (ð (ð (ð (ˆ"Œ)ñ (ô (ð (ðV4ð 4ð 4ð 4ð 4ˆRŒYñ 4ô 4ð 4ðn ˆw�v¨¬¸EÐBÑBÔB€ð2ð 2ð 2ð 2ð 2ˆRŒYñ 2ô 2ð 2ðj,ð ,ð ,ð ,ð ,ˆrŒyñ ,ô ,ð ,ð^!ð !ð !ð !ð !ˆbŒiñ !ô !ð !ðH'ð 'ð 'ð 'ð '˜BœIñ 'ô 'ð 'ð 'ð 'r/   