§
    ·¯¸j/)  ã                   óè   — d Z ddlmZmZmZ ddlZddlmZ ddlmc m	Z
 ddlmZ  G d„ dej        ¦  «        Z G d„ d	ej        ¦  «        Z G d
„ dej        ¦  «        Z G d„ dej        ¦  «        ZdS )aH   Non-Local Attention Pooling Layers

A collection of global pooling layers that go beyond simple avg/max pooling.

LSEPool - LogSumExp pooling, a smooth approximation between avg and max pooling
SimPool - Attention-based pooling from 'Keep It SimPool' (ICCV 2023)

Based on implementations from:
* LSE Pooling: custom implementation by Bill Psomas
* SimPool: https://arxiv.org/abs/2309.06891 - 'Keep It SimPool: Who Said Supervised Transformers
    Suffer from Attention Deficit?' by Bill Psomas et al.

Hacked together by / Copyright 2024 Ross Wightman, original code by Bill Psomas
é    )ÚOptionalÚTypeÚUnionNé   )Úuse_fused_attnc                   ó`   ‡ — e Zd ZdZ	 	 	 	 	 ddededefˆ fd„Zd	ej        d
ej        fd„Z	ˆ xZ
S )Ú	LsePlus2da…  LogSumExp (LSE) Pooling for 2D inputs.

    A smooth approximation to max pooling that provides a learnable interpolation between
    average and max pooling. When r is large, LSE approaches max pooling; when r is small,
    it approaches average pooling.

    Implements: (1/r) * log((1/n) * sum(exp(r * (x - x_max)))) + x_max

    The x_max subtraction provides numerical stability.
    ç      $@TNÚrÚr_learnableÚflattenc                 ó  •— t          ¦   «                              ¦   «          |r/t          j        t	          j        |||¬¦  «        ¦  «        | _        n+|                      dt	          j        |||¬¦  «        ¦  «         || _        dS )zê
        Args:
            r: Initial value of the pooling parameter. Higher = closer to max pooling.
            r_learnable: If True, r is a learnable parameter.
            flatten: If True, flatten spatial dims in output.
        ©ÚdeviceÚdtyper   N)	ÚsuperÚ__init__ÚnnÚ	ParameterÚtorchÚtensorr   Úregister_bufferr   )Úselfr   r   r   r   r   Ú	__class__s         €úf/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/layers/other_pool.pyr   zLsePlus2d.__init__$   sz   ø€ õ 	‰Œ×ÒÑÔÐØð 	SÝ”\¥%¤,¨q¸ÀuÐ"MÑ"MÔ"MÑNÔNˆDŒFˆFà× Ò  ¥e¤l°1¸VÈ5Ð&QÑ&QÔ&QÑRÔRÐRØˆŒˆˆó    ÚxÚreturnc                 ó  — t          j        |d¦  «        }t          j        | j        ||z
  z  ¦  «        }|                     dd¬¦  «        }|d| j        z  t          j        |¦  «        z  z   }| j        r|                     d¦  «        }|S )Nr   )é   é   T©ÚdimÚkeepdimç      ð?)ÚFÚadaptive_max_pool2dr   Úexpr   ÚmeanÚlogr   ©r   r   Úx_maxÚexp_xÚsum_expÚouts         r   ÚforwardzLsePlus2d.forward9   s€   € ÝÔ% a¨Ñ+Ô+ˆÝ”	˜$œ& A¨¡IÑ.Ñ/Ô/ˆØ—*’* °�*Ñ6Ô6ˆØ�s˜TœV‘|¥u¤y°Ñ'9Ô'9Ñ9Ñ9ˆØŒ<ð 	!Ø—+’+˜a‘.”.ˆCØˆ
r   )r
   TTNN©Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚfloatÚboolr   r   ÚTensorr0   Ú__classcell__©r   s   @r   r	   r	      sŸ   ø€ € € € € ð	ð 	ð Ø $Ø ØØðð àðð ðð ð	ð ð ð ð ð ð*˜œð ¨%¬,ð ð ð ð ð ð ð ð r   r	   c                   óZ   ‡ — e Zd ZdZ	 	 	 	 ddedefˆ fd„Zdej        d	ej        fd
„Z	ˆ xZ
S )Ú	LsePlus1da  LogSumExp (LSE) Pooling for sequence (NLC) inputs.

    A smooth approximation to max pooling that provides a learnable interpolation between
    average and max pooling. When r is large, LSE approaches max pooling; when r is small,
    it approaches average pooling.
    r
   TNr   r   c                 ó  •— t          ¦   «                              ¦   «          |r0t          j        t	          j        |||¬¦  «        ¦  «        | _        dS |                      dt	          j        |||¬¦  «        ¦  «         dS )z¬
        Args:
            r: Initial value of the pooling parameter. Higher = closer to max pooling.
            r_learnable: If True, r is a learnable parameter.
        r   r   N)r   r   r   r   r   r   r   r   )r   r   r   r   r   r   s        €r   r   zLsePlus1d.__init__K   sw   ø€ õ 	‰Œ×ÒÑÔÐØð 	SÝ”\¥%¤,¨q¸ÀuÐ"MÑ"MÔ"MÑNÔNˆDŒFˆFˆFà× Ò  ¥e¤l°1¸VÈ5Ð&QÑ&QÔ&QÑRÔRÐRÐRÐRr   r   r   c                 ó  — |                      dd¬¦  «        j        }t          j        | j        ||z
  z  ¦  «        }|                     dd¬¦  «        }|d| j        z  t          j        |¦  «        z  z   }|                     d¦  «        S )Nr   Tr"   r%   )ÚmaxÚvaluesr   r(   r   r)   r*   Úsqueezer+   s         r   r0   zLsePlus1d.forward]   sx   € à—’˜! T�Ñ*Ô*Ô1ˆÝ”	˜$œ& A¨¡IÑ.Ñ/Ô/ˆØ—*’* ¨D�*Ñ1Ô1ˆØ�s˜TœV‘|¥u¤y°Ñ'9Ô'9Ñ9Ñ9ˆØ�{Š{˜1‰~Œ~Ðr   )r
   TNNr1   r:   s   @r   r<   r<   C   s›   ø€ € € € € ðð ð Ø $ØØðSð SàðSð ðSð Sð Sð Sð Sð Sð$˜œð ¨%¬,ð ð ð ð ð ð ð ð r   r<   c                   óÊ   ‡ — e Zd ZU dZej        j        e         ed<   	 	 	 	 	 	 	 dde	de	ded	ed
e
e         de
eej                          fˆ fd„Zdej        dej        fd„Zˆ xZS )Ú	SimPool2dab  SimPool: Simple Attention-Based Pooling for 2D (NCHW) inputs.

    From 'Keep It SimPool: Who Said Supervised Transformers Suffer from Attention Deficit?'
    https://arxiv.org/abs/2309.06891

    Uses GAP as query initialization and applies cross-attention between the GAP query
    and spatial features to produce a weighted pooled representation.
    Ú
fused_attnr   FNr#   Ú	num_headsÚqkv_biasÚqk_normÚgammaÚ
norm_layerc	                 ó@  •— t          ¦   «                              ¦   «          ||dœ}	||z  dk    s
J d¦   «         ‚|| _        ||z  | _        | j        dz  | _        || _        t          ¦   «         | _        |pt          j	        } ||fi |	¤Ž| _
        t          j        ||fd|i|	¤Ž| _        t          j        ||fd|i|	¤Ž| _        |r( || j        fi |	¤Ž| _         || j        fi |	¤Ž| _        dS t          j        ¦   «         | _        t          j        ¦   «         | _        dS )aõ  
        Args:
            dim: Input feature dimension (number of channels).
            num_heads: Number of attention heads.
            qkv_bias: If True, add bias to query and key projections.
            qk_norm: If True, apply normalization to queries and keys.
            gamma: If provided, apply power normalization to values with this exponent.
            norm_layer: Normalization layer for patches and optionally qk_norm.
            flatten: If True, flatten output to (B, C).
        r   r   ú"dim must be divisible by num_headsç      à¿ÚbiasN©r   r   rE   Úhead_dimÚscalerH   r   rD   r   Ú	LayerNormÚnormÚLinearÚqÚkÚq_normÚk_normÚIdentity©r   r#   rE   rF   rG   rH   rI   r   r   Úddr   s             €r   r   zSimPool2d.__init__q   s@  ø€ õ* 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆØ�Y‰ !Ò#Ð#Ð#Ð%IÑ#Ô#Ð#Ø"ˆŒØ˜yÑ(ˆŒØ”] dÑ*ˆŒ
ØˆŒ
Ý(Ñ*Ô*ˆŒàÐ/¥2¤<ˆ
Ø�J˜sÐ)Ð) bÐ)Ð)ˆŒ	Ý”˜3 Ð9Ð9¨(Ð9°bÐ9Ð9ˆŒÝ”˜3 Ð9Ð9¨(Ð9°bÐ9Ð9ˆŒØð 	(Ø$˜* T¤]Ð9Ð9°bÐ9Ð9ˆDŒKØ$˜* T¤]Ð9Ð9°bÐ9Ð9ˆDŒKˆKˆKåœ+™-œ-ˆDŒKÝœ+™-œ-ˆDŒKˆKˆKr   r   r   c                 óZ  — |j         \  }}}}||z  }|                     d¦  «                             dd¦  «        }|                     dd¬¦  «        }|                      |¦  «        }|                      |¦  «                             |d| j        | j        ¦  «                             dd¦  «        }|  	                    |¦  «                             ||| j        | j        ¦  «                             dd¦  «        }	|                     ||| j        | j        ¦  «                             dd¦  «        }
|  
                    |¦  «        |                      |	¦  «        }	}| j        �Ç|
                     dd¬¦  «        }|
|z
  dz   }| j        r/t          j        ||	|                     | j        ¦  «        ¦  «        }nT|| j        z  |	                     dd¦  «        z  }|                     d¬¦  «        }||                     | j        ¦  «        z  }|                     d	| j        z  ¦  «        }nZ| j        rt          j        ||	|
¦  «        }n<|| j        z  |	                     dd¦  «        z  }|                     d¬¦  «        }||
z  }|                     dd¦  «                             ||¦  «        }|S )
Nr    r   Tr"   éþÿÿÿç�íµ ÷Æ°>éÿÿÿÿ©r#   r%   )Úshaper   Ú	transposer)   rR   rT   ÚreshaperE   rO   rU   rV   rW   rH   ÚaminrD   r&   Úscaled_dot_product_attentionÚpowrP   Úsoftmax)r   r   ÚBÚCÚHÚWÚNrT   Úx_normrU   ÚvÚv_minÚ	v_shiftedÚattn_outÚattnr/   s                   r   r0   zSimPool2d.forwardš   s^  € Ø”W‰
ˆˆ1ˆa�Ø�‰Eˆð �IŠI�a‰LŒL×"Ò" 1 aÑ(Ô(ˆð �FŠF�q $ˆFÑ'Ô'ˆð —’˜1‘”ˆð �FŠF�1‰IŒI×Ò˜a  D¤N°D´MÑBÔB×LÒLÈQÐPQÑRÔRˆØ�FŠF�6‰NŒN×"Ò" 1 a¨¬¸¼ÑGÔG×QÒQÐRSÐUVÑWÔWˆØ�NŠN˜1˜a ¤°´Ñ?Ô?×IÒIÈ!ÈQÑOÔOˆà�{Š{˜1‰~Œ~˜tŸ{š{¨1™~œ~ˆ1ˆàŒ:Ð!à—F’F˜r¨4�FÑ0Ô0ˆEØ˜E™	 DÑ(ˆIØŒð <ÝÔ9¸!¸QÀ	ÇÂÈdÌjÑ@YÔ@YÑZÔZ��à˜DœJ™¨!¯+ª+°b¸"Ñ*=Ô*=Ñ=�Ø—|’|¨�|Ñ+Ô+�Ø )§-¢-°´
Ñ";Ô";Ñ;�Ø—,’,˜s T¤ZÑ/Ñ0Ô0ˆCˆCàŒð ÝÔ4°Q¸¸1Ñ=Ô=��à˜DœJ™¨!¯+ª+°b¸"Ñ*=Ô*=Ñ=�Ø—|’|¨�|Ñ+Ô+�Ø˜Q‘h�ð �mŠm˜A˜qÑ!Ô!×)Ò)¨!¨QÑ/Ô/ˆØˆ
r   ©r   FFNNNN©r2   r3   r4   r5   r   ÚjitÚFinalr7   Ú__annotations__Úintr   r6   r   r   ÚModuler   r8   r0   r9   r:   s   @r   rC   rC   f   sì   ø€ € € € € € ðð ð ”	” Ô%Ð%Ð%Ñ%ð
 Ø"Ø!Ø%)Ø48ØØð'(ð '(àð'(ð ð'(ð ð	'(ð
 ð'(ð ˜E”?ð'(ð !  b¤i¤Ô1ð'(ð '(ð '(ð '(ð '(ð '(ðR)˜œð )¨%¬,ð )ð )ð )ð )ð )ð )ð )ð )r   rC   c                   óÊ   ‡ — e Zd ZU dZej        j        e         ed<   	 	 	 	 	 	 	 dde	de	ded	ed
e
e         de
eej                          fˆ fd„Zdej        dej        fd„Zˆ xZS )Ú	SimPool1daf  SimPool: Simple Attention-Based Pooling for sequence (NLC) inputs.

    From 'Keep It SimPool: Who Said Supervised Transformers Suffer from Attention Deficit?'
    https://arxiv.org/abs/2309.06891

    Uses GAP as query initialization and applies cross-attention between the GAP query
    and sequence tokens to produce a weighted pooled representation.
    rD   r   FNr#   rE   rF   rG   rH   rI   c	                 ó@  •— t          ¦   «                              ¦   «          ||dœ}	||z  dk    s
J d¦   «         ‚|| _        ||z  | _        | j        dz  | _        || _        t          ¦   «         | _        |pt          j	        } ||fi |	¤Ž| _
        t          j        ||fd|i|	¤Ž| _        t          j        ||fd|i|	¤Ž| _        |r( || j        fi |	¤Ž| _         || j        fi |	¤Ž| _        dS t          j        ¦   «         | _        t          j        ¦   «         | _        dS )a§  
        Args:
            dim: Input feature dimension.
            num_heads: Number of attention heads.
            qkv_bias: If True, add bias to query and key projections.
            qk_norm: If True, apply normalization to queries and keys.
            gamma: If provided, apply power normalization to values with this exponent.
            norm_layer: Normalization layer for tokens and optionally qk_norm.
        r   r   rK   rL   rM   NrN   rY   s             €r   r   zSimPool1d.__init__Ñ   s@  ø€ õ( 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆØ�Y‰ !Ò#Ð#Ð#Ð%IÑ#Ô#Ð#Ø"ˆŒØ˜yÑ(ˆŒØ”] dÑ*ˆŒ
ØˆŒ
Ý(Ñ*Ô*ˆŒàÐ/¥2¤<ˆ
Ø�J˜sÐ)Ð) bÐ)Ð)ˆŒ	Ý”˜3 Ð9Ð9¨(Ð9°bÐ9Ð9ˆŒÝ”˜3 Ð9Ð9¨(Ð9°bÐ9Ð9ˆŒØð 	(Ø$˜* T¤]Ð9Ð9°bÐ9Ð9ˆDŒKØ$˜* T¤]Ð9Ð9°bÐ9Ð9ˆDŒKˆKˆKåœ+™-œ-ˆDŒKÝœ+™-œ-ˆDŒKˆKˆKr   r   r   c                 óü  — |j         \  }}}|                     dd¬¦  «        }|                      |¦  «        }|                      |¦  «                             |d| j        | j        ¦  «                             dd¦  «        }|                      |¦  «                             ||| j        | j        ¦  «                             dd¦  «        }|                     ||| j        | j        ¦  «                             dd¦  «        }|  	                    |¦  «        |  
                    |¦  «        }}| j        �Ç|                     dd¬¦  «        }	||	z
  dz   }
| j        r/t          j        |||
                     | j        ¦  «        ¦  «        }nT|| j        z  |                     dd¦  «        z  }|                     d¬¦  «        }||
                     | j        ¦  «        z  }|                     d	| j        z  ¦  «        }nZ| j        rt          j        |||¦  «        }n<|| j        z  |                     dd¦  «        z  }|                     d¬¦  «        }||z  }|                     dd¦  «                             ||¦  «        }|S )
Nr   Tr"   r    r\   r]   r^   r_   r%   )r`   r)   rR   rT   rb   rE   rO   ra   rU   rV   rW   rH   rc   rD   r&   rd   re   rP   rf   )r   r   rg   rk   rh   rT   rl   rU   rm   rn   ro   rp   rq   r/   s                 r   r0   zSimPool1d.forwardù   s2  € Ø”'‰ˆˆ1ˆað �FŠF�q $ˆFÑ'Ô'ˆð —’˜1‘”ˆð �FŠF�1‰IŒI×Ò˜a  D¤N°D´MÑBÔB×LÒLÈQÐPQÑRÔRˆØ�FŠF�6‰NŒN×"Ò" 1 a¨¬¸¼ÑGÔG×QÒQÐRSÐUVÑWÔWˆØ�NŠN˜1˜a ¤°´Ñ?Ô?×IÒIÈ!ÈQÑOÔOˆà�{Š{˜1‰~Œ~˜tŸ{š{¨1™~œ~ˆ1ˆàŒ:Ð!à—F’F˜r¨4�FÑ0Ô0ˆEØ˜E™	 DÑ(ˆIØŒð <ÝÔ9¸!¸QÀ	ÇÂÈdÌjÑ@YÔ@YÑZÔZ��à˜DœJ™¨!¯+ª+°b¸"Ñ*=Ô*=Ñ=�Ø—|’|¨�|Ñ+Ô+�Ø )§-¢-°´
Ñ";Ô";Ñ;�Ø—,’,˜s T¤ZÑ/Ñ0Ô0ˆCˆCàŒð ÝÔ4°Q¸¸1Ñ=Ô=��à˜DœJ™¨!¯+ª+°b¸"Ñ*=Ô*=Ñ=�Ø—|’|¨�|Ñ+Ô+�Ø˜Q‘h�ð �mŠm˜A˜qÑ!Ô!×)Ò)¨!¨QÑ/Ô/ˆØˆ
r   rr   rs   r:   s   @r   rz   rz   Æ   sì   ø€ € € € € € ðð ð ”	” Ô%Ð%Ð%Ñ%ð
 Ø"Ø!Ø%)Ø48ØØð&(ð &(àð&(ð ð&(ð ð	&(ð
 ð&(ð ˜E”?ð&(ð !  b¤i¤Ô1ð&(ð &(ð &(ð &(ð &(ð &(ðP%˜œð %¨%¬,ð %ð %ð %ð %ð %ð %ð %ð %r   rz   )r5   Útypingr   r   r   r   Útorch.nnr   Útorch.nn.functionalÚ
functionalr&   Úconfigr   rx   r	   r<   rC   rz   © r   r   ú<module>rƒ      sF  ððð ð )Ð (Ð (Ð (Ð (Ð (Ð (Ð (Ð (Ð (à €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð à "Ð "Ð "Ð "Ð "Ð "ð(ð (ð (ð (ð (�”	ñ (ô (ð (ðV ð  ð  ð  ð  �”	ñ  ô  ð  ðF]ð ]ð ]ð ]ð ]�”	ñ ]ô ]ð ]ð@Xð Xð Xð Xð X�”	ñ Xô Xð Xð Xð Xr   