§
    ·¯¸jÊ  ã                   óÐ   — d Z ddlmZmZmZ ddlZddlmZ ddlmc m	Z
 ddlmZmZ ddlmZ ddlmZ dee         fd	„Z G d
„ dej        ¦  «        Z G d„ dej        ¦  «        ZdS )aP   Bottleneck Self Attention (Bottleneck Transformers)

Paper: `Bottleneck Transformers for Visual Recognition` - https://arxiv.org/abs/2101.11605

@misc{2101.11605,
Author = {Aravind Srinivas and Tsung-Yi Lin and Niki Parmar and Jonathon Shlens and Pieter Abbeel and Ashish Vaswani},
Title = {Bottleneck Transformers for Visual Recognition},
Year = {2021},
}

Based on ref gist at: https://gist.github.com/aravindsrinivas/56359b79f0ce4449bcb04ab4b56a57a2

This impl is a WIP but given that it is based on the ref gist likely not too far off.

Hacked together by / Copyright 2021 Ross Wightman
é    )ÚListÚOptionalÚTupleNé   )Ú	to_2tupleÚmake_divisible)Útrunc_normal_)Ú_assertÚpermute_maskc                 ó   — | j         \  }}}}| |                     dd¦  «        z  }|                     d|d|z  dz
  ¦  «        }t          j        |ddg¦  «                             d¦  «        }t          j        |d|dz
  g¦  «        }|                     d|dz   d|z  dz
  ¦  «        }|dd…d|…|dz
  d…f         }|                     ||d||¦  «                             dd|dd¦  «        }|                     |¦  «        S )a„   Compute relative logits along one dimension

    As per: https://gist.github.com/aravindsrinivas/56359b79f0ce4449bcb04ab4b56a57a2
    Originally from: `Attention Augmented Convolutional Networks` - https://arxiv.org/abs/1904.09925

    Args:
        q: (batch, heads, height, width, dim)
        rel_k: (2 * width - 1, dim)
        permute_mask: permute output dim according to this
    éÿÿÿÿéþÿÿÿé   r   r   N)ÚshapeÚ	transposeÚreshapeÚFÚpadÚflattenÚexpandÚpermute)	ÚqÚrel_kr   ÚBÚHÚWÚdimÚxÚx_pads	            úk/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/layers/bottleneck_attn.pyÚrel_logits_1dr!      s  € ð ”7�L€A€qˆ!ˆSØ	
ˆU�_Š_˜R Ñ$Ô$Ñ	$€AØ	�	Š	�"�a˜˜Q™ ™Ñ"Ô"€Aõ ŒE�!�a˜�VÑÔ×$Ò$ QÑ'Ô'€EÝŒE�%˜!˜Q ™U˜Ñ$Ô$€Eð �MŠM˜"˜a !™e Q¨¡U¨Q¡YÑ/Ô/€EØˆaˆaˆa��!��Q˜‘U�V�VˆmÔ€Að 	
�	Š	�!�Q˜˜1˜aÑ Ô ×'Ò'¨¨B°°2°rÑ:Ô:€AØ�9Š9�\Ñ"Ô"Ð"ó    c                   óR   ‡ — e Zd ZdZ	 	 d	deeef         dedefˆ fd„Zd„ Zd„ Z	ˆ xZ
S )
ÚPosEmbedRelzÛ Relative Position Embedding
    As per: https://gist.github.com/aravindsrinivas/56359b79f0ce4449bcb04ab4b56a57a2
    Originally from: `Attention Augmented Convolutional Networks` - https://arxiv.org/abs/1904.09925
    NÚ	feat_sizeÚdim_headÚscalec                 ó¢  •— ||dœ}t          ¦   «                              ¦   «          t          |¦  «        \  | _        | _        || _        || _        t          j        t          j
        | j        dz  dz
  |fi |¤Ž¦  «        | _        t          j        t          j
        | j        dz  dz
  |fi |¤Ž¦  «        | _        |                      ¦   «          d S )N©ÚdeviceÚdtyper   r   )ÚsuperÚ__init__r   ÚheightÚwidthr&   r'   ÚnnÚ	ParameterÚtorchÚemptyÚ
height_relÚ	width_relÚreset_parameters)Úselfr%   r&   r'   r*   r+   ÚddÚ	__class__s          €r    r-   zPosEmbedRel.__init__=   sÆ   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ"+¨IÑ"6Ô"6ÑˆŒ�T”ZØ ˆŒØˆŒ
åœ,¥u¤{°4´;À±?ÀQÑ3FÈÐ'WÐ'WÐTVÐ'WÐ'WÑXÔXˆŒÝœ¥e¤k°$´*¸q±.À1Ñ2DÀhÐ&UÐ&UÐRTÐ&UÐ&UÑVÔVˆŒà×ÒÑÔÐÐÐr"   c                 óÆ   — t           j        j                             | j        | j        ¬¦  «         t           j        j                             | j        | j        ¬¦  «         d S )N©Ústd)r2   r0   ÚinitÚnormal_r4   r'   r5   ©r7   s    r    r6   zPosEmbedRel.reset_parametersP   sJ   € ÝŒŒ×Ò˜dœo°4´:ÐÑ>Ô>Ð>ÝŒŒ×Ò˜dœn°$´*ÐÑ=Ô=Ð=Ð=Ð=r"   c                 ó   — |j         \  }}}|                     || j        | j        d¦  «        }t	          || j        d¬¦  «        }|                     dd¦  «        }t	          || j        d¬¦  «        }||z   }|                     |||¦  «        }|S )Nr   )r   r   é   r   é   )r   r   r   )r   rA   r   rB   r   )r   r   r.   r/   r!   r5   r   r4   )r7   r   r   ÚHWÚ_Úrel_logits_wÚrel_logits_hÚ
rel_logitss           r    ÚforwardzPosEmbedRel.forwardT   s”   € Ø”7‰ˆˆ2ˆqð �IŠI�a˜œ d¤j°"Ñ5Ô5ˆÝ$ Q¨¬À_ÐUÑUÔUˆð �KŠK˜˜1ÑÔˆÝ$ Q¨¬ÀoÐVÑVÔVˆà! LÑ0ˆ
Ø×'Ò'¨¨2¨rÑ2Ô2ˆ
ØÐr"   )NN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚintÚfloatr-   r6   rH   Ú__classcell__©r9   s   @r    r$   r$   8   s–   ø€ € € € € ðð ð Øð ð  à˜S #˜X”ð ð ð ð ð	 ð  ð  ð  ð  ð  ð&>ð >ð >ðð ð ð ð ð ð r"   r$   c                   óž   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 ddedee         d	eeeef                  d
ededee         dededefˆ fd„Z	d„ Z
d„ Zˆ xZS )ÚBottleneckAttna†   Bottleneck Attention
    Paper: `Bottleneck Transformers for Visual Recognition` - https://arxiv.org/abs/2101.11605

    The internal dimensions of the attention module are controlled by the interaction of several arguments.
      * the output dimension of the module is specified by dim_out, which falls back to input dim if not set
      * the value (v) dimension is set to dim_out // num_heads, the v projection determines the output dim
      * the query and key (qk) dimensions are determined by
        * num_heads * dim_head if dim_head is not None
        * num_heads * (dim_out * attn_ratio // num_heads) if dim_head is None
      * as seen above, attn_ratio determines the ratio of q and k relative to the output if dim_head not used

    Args:
        dim (int): input dimension to the module
        dim_out (int): output dimension of the module, same as dim if not set
        stride (int): output stride of the module, avg pool used if stride == 2 (default: 1).
        num_heads (int): parallel attention heads (default: 4)
        dim_head (int): dimension of query and key heads, calculated from dim_out * attn_ratio // num_heads if not set
        qk_ratio (float): ratio of q and k dimensions to output dimension when dim_head not set. (default: 1.0)
        qkv_bias (bool): add bias to q, k, and v projections
        scale_pos_embed (bool): scale the position embedding as well as Q @ K
    Nr   rB   ç      ð?Fr   Údim_outr%   ÚstrideÚ	num_headsr&   Úqk_ratioÚqkv_biasÚscale_pos_embedc                 óz  •— |
|dœ}t          ¦   «                              ¦   «          |€
J d¦   «         ‚|p|}||z  dk    sJ ‚|| _        |pt          ||z  d¬¦  «        |z  | _        || j        z  | _        || j        z  | _        || j        z  | _        | j        dz  | _        |	| _	        t          j        || j        dz  | j        z   dfd	|i|¤Ž| _        t          |f| j        | j        d
œ|¤Ž| _        |dk    rt          j        dd¦  «        nt          j        ¦   «         | _        |                      ¦   «          d S )Nr)   zBA concrete feature size matching expected input (H, W) is requiredr   é   )Údivisorç      à¿r   r   Úbias)r&   r'   )r,   r-   rV   r   Údim_head_qkÚ
dim_head_vÚ
dim_out_qkÚ	dim_out_vr'   rY   r0   ÚConv2dÚqkvr$   Ú	pos_embedÚ	AvgPool2dÚIdentityÚpoolr6   )r7   r   rT   r%   rU   rV   r&   rW   rX   rY   r*   r+   r8   r9   s                €r    r-   zBottleneckAttn.__init__z   sc  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØÐ$Ð$Ð&jÑ$Ô$Ð$Ø�.˜SˆØ˜Ñ" aÒ'Ð'Ð'Ð'Ø"ˆŒØ#Ða¥~°gÀÑ6HÐRSÐ'TÑ'TÔ'TÐXaÑ'aˆÔØ! T¤^Ñ3ˆŒØ# dÔ&6Ñ6ˆŒØ" T¤_Ñ4ˆŒØÔ%¨Ñ-ˆŒ
Ø.ˆÔå”9˜S $¤/°AÑ"5¸¼Ñ"FÈÐ_Ð_ÐPXÐ_Ð\^Ð_Ð_ˆŒõ % YÐb¸Ô9IÐQUÔQ[ÐbÐbÐ_aÐbÐbˆŒà*0°Aª+¨+•B”L  AÑ&Ô&Ð&½2¼;¹=¼=ˆŒ	à×ÒÑÔÐÐÐr"   c                 óì   — t          | j        j        | j        j        j        d         dz  ¬¦  «         t          | j        j        | j        ¬¦  «         t          | j        j        | j        ¬¦  «         d S )Nr   r]   r;   )r	   rd   Úweightr   re   r4   r'   r5   r?   s    r    r6   zBottleneckAttn.reset_parametersž   sd   € Ý�d”h”o¨4¬8¬?Ô+@ÀÔ+CÀtÑ+KÐLÑLÔLÐLÝ�d”nÔ/°T´ZÐ@Ñ@Ô@Ð@Ý�d”nÔ.°D´JÐ?Ñ?Ô?Ð?Ð?Ð?r"   c                 óœ  — |j         \  }}}}t          || j        j        k    d¦  «         t          || j        j        k    d¦  «         |                      |¦  «        }t          j        || j        | j        | j	        gd¬¦  «        \  }}}| 
                    || j        z  | j        d¦  «                             dd¦  «        }| 
                    || j        z  | j        d¦  «        }| 
                    || j        z  | j        d¦  «                             dd¦  «        }| j        r$||z  |                      |¦  «        z   | j        z  }	n#||z  | j        z  |                      |¦  «        z   }	|	                     d¬¦  «        }	|	|z                       dd¦  «         
                    || j	        ||¦  «        }
|                      |
¦  «        }
|
S )NÚ r   )r   r   r   )r   r
   re   r.   r/   rd   r2   Úsplitra   rb   r   rV   r_   r   r`   rY   r'   Úsoftmaxrh   )r7   r   r   ÚCr   r   r   ÚkÚvÚattnÚouts              r    rH   zBottleneckAttn.forward£   s¡  € Ø”W‰
ˆˆ1ˆa�Ý��T”^Ô*Ò*¨BÑ/Ô/Ð/Ý��T”^Ô)Ò)¨2Ñ.Ô.Ð.à�HŠH�Q‰KŒKˆõ ”+˜a $¤/°4´?ÀDÄNÐ!SÐYZÐ[Ñ[Ô[‰ˆˆ1ˆaØ�IŠI�a˜$œ.Ñ(¨$Ô*:¸BÑ?Ô?×IÒIÈ"ÈbÑQÔQˆØ�IŠI�a˜$œ.Ñ(¨$Ô*:¸BÑ?Ô?ˆØ�IŠI�a˜$œ.Ñ(¨$¬/¸2Ñ>Ô>×HÒHÈÈRÑPÔPˆàÔð 	<Ø˜‘E˜DŸNšN¨1Ñ-Ô-Ñ-°´Ñ;ˆDˆDà˜‘E˜TœZÑ'¨$¯.ª.¸Ñ*;Ô*;Ñ;ˆDØ�|Š| ˆ|Ñ#Ô#ˆà�a‰x×"Ò" 2 rÑ*Ô*×2Ò2°1°d´nÀaÈÑKÔKˆØ�iŠi˜‰nŒnˆØˆ
r"   )
NNr   rB   NrS   FFNN)rI   rJ   rK   rL   rM   r   r   rN   Úboolr-   r6   rH   rO   rP   s   @r    rR   rR   d   sû   ø€ € € € € ðð ð0 &*Ø37ØØØ&*Ø!Ø"Ø$)ØØð" ð " àð" ð ˜c”]ð" ð    c¨3 h¤Ô0ð	" ð
 ð" ð ð" ð ˜s”mð" ð ð" ð ð" ð "ð" ð " ð " ð " ð " ð " ðH@ð @ð @ð
ð ð ð ð ð ð r"   rR   )rL   Útypingr   r   r   r2   Útorch.nnr0   Útorch.nn.functionalÚ
functionalr   Úhelpersr   r   Úweight_initr	   Útrace_utilsr
   rM   r!   ÚModuler$   rR   © r"   r    ú<module>r~      s-  ððð ð  )Ð (Ð (Ð (Ð (Ð (Ð (Ð (Ð (Ð (à €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð à .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø &Ð &Ð &Ð &Ð &Ð &Ø  Ð  Ð  Ð  Ð  Ð  ð#¨$¨s¬)ð #ð #ð #ð #ð8)ð )ð )ð )ð )�"”)ñ )ô )ð )ðXUð Uð Uð Uð U�R”Yñ Uô Uð Uð Uð Ur"   