§
    ·¯¸jí,  ã                   óÐ   — d Z ddlmZmZmZmZ ddlZddlmZ ddlmc m	Z
 ddlmZ ddlmZ ddlmZ d	ee         fd
„Z G d„ dej        ¦  «        Z G d„ dej        ¦  «        ZdS )a]   Halo Self Attention

Paper: `Scaling Local Self-Attention for Parameter Efficient Visual Backbones`
    - https://arxiv.org/abs/2103.12731

@misc{2103.12731,
Author = {Ashish Vaswani and Prajit Ramachandran and Aravind Srinivas and Niki Parmar and Blake Hechtman and
    Jonathon Shlens},
Title = {Scaling Local Self-Attention for Parameter Efficient Visual Backbones},
Year = {2021},
}

Status:
This impl is a WIP, there is no official ref impl and some details in paper weren't clear to me.
The attention mechanism works but it's slow as implemented.

Hacked together by / Copyright 2021 Ross Wightman
é    )ÚListÚOptionalÚTupleÚUnionN)Únné   )Úmake_divisible)Útrunc_normal_)Ú_assertÚpermute_maskc                 ó  — | j         \  }}}}|j         d         }|dz   dz  }| |                     dd¦  «        z  }	|	                     d||¦  «        }	t          j        |	ddg¦  «                             d¦  «        }
t          j        |
d||z
  g¦  «        }
|
                     d|dz   |¦  «        }
|
dd…d|…|dz
  d…f         }	|	                     ||d||¦  «                             dd|dd¦  «        }	|	                     |¦  «        S )a~   Compute relative logits along one dimension

    As per: https://gist.github.com/aravindsrinivas/56359b79f0ce4449bcb04ab4b56a57a2
    Originally from: `Attention Augmented Convolutional Networks` - https://arxiv.org/abs/1904.09925

    Args:
        q: (batch, height, width, dim)
        rel_k: (2 * window - 1, dim)
        permute_mask: permute output dim according to this
    r   r   é   éÿÿÿÿéþÿÿÿN)ÚshapeÚ	transposeÚreshapeÚFÚpadÚflattenÚexpandÚpermute)ÚqÚrel_kr   ÚBÚHÚWÚdimÚrel_sizeÚwin_sizeÚxÚx_pads              úe/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/layers/halo_attn.pyÚrel_logits_1dr$      s  € ð ”7�L€A€qˆ!ˆSØŒ{˜1Œ~€HØ˜1‘ Ñ"€Hà	
ˆU�_Š_˜R Ñ$Ô$Ñ	$€AØ	�	Š	�"�a˜Ñ"Ô"€Aõ ŒE�!�a˜�VÑÔ×$Ò$ QÑ'Ô'€EÝŒE�%˜!˜X¨™\Ð*Ñ+Ô+€Eð �MŠM˜"˜a !™e XÑ.Ô.€EØˆaˆaˆa��!��X ‘\�]�]Ð"Ô#€Að 	
�	Š	�!�Q˜˜1˜hÑ'Ô'×.Ò.¨r°2°xÀÀRÑHÔH€AØ�9Š9�\Ñ"Ô"Ð"ó    c            	       óF   ‡ — e Zd ZdZ	 	 d
dedededefˆ fd„Zd„ Zd	„ Zˆ xZ	S )ÚPosEmbedRelzÜ Relative Position Embedding
    As per: https://gist.github.com/aravindsrinivas/56359b79f0ce4449bcb04ab4b56a57a2
    Originally from: `Attention Augmented Convolutional Networks` - https://arxiv.org/abs/1904.09925

    NÚ
block_sizer    Údim_headÚscalec                 ód  •— ||dœ}t          ¦   «                              ¦   «          || _        || _        || _        t          j        t          j        |dz  dz
  |fi |¤Ž¦  «        | _	        t          j        t          j        |dz  dz
  |fi |¤Ž¦  «        | _
        |                      ¦   «          dS )z¾
        Args:
            block_size: block size
            win_size: neighbourhood window size
            dim_head: attention head dim
            scale: scale factor (for init)
        ©ÚdeviceÚdtyper   r   N)ÚsuperÚ__init__r(   r)   r*   r   Ú	ParameterÚtorchÚemptyÚ
height_relÚ	width_relÚreset_parameters)	Úselfr(   r    r)   r*   r-   r.   ÚddÚ	__class__s	           €r#   r0   zPosEmbedRel.__init__C   s³   ø€ ð  ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ$ˆŒØ ˆŒØˆŒ
åœ,¥u¤{°8¸a±<À!Ñ3CÀXÐ'TÐ'TÐQSÐ'TÐ'TÑUÔUˆŒÝœ¥e¤k°(¸Q±,ÀÑ2BÀHÐ&SÐ&SÐPRÐ&SÐ&SÑTÔTˆŒà×ÒÑÔÐÐÐr%   c                 óÆ   — t           j        j                             | j        | j        ¬¦  «         t           j        j                             | j        | j        ¬¦  «         d S )N©Ústd)r2   r   ÚinitÚnormal_r4   r*   r5   )r7   s    r#   r6   zPosEmbedRel.reset_parameters^   sJ   € ÝŒŒ×Ò˜dœo°4´:ÐÑ>Ô>Ð>ÝŒŒ×Ò˜dœn°$´*ÐÑ=Ô=Ð=Ð=Ð=r%   c                 ó.  — |j         \  }}}}|                     d| j        | j        | j        ¦  «        }t	          || j        d¬¦  «        }|                     dd¦  «        }t	          || j        d¬¦  «        }||z   }|                     |||d¦  «        }|S )Nr   )r   r   é   r   é   )r   r   r   )r   r@   r   rA   r   )r   r   r(   r)   r$   r5   r   r4   )	r7   r   r   ÚBBÚHWÚ_Úrel_logits_wÚrel_logits_hÚ
rel_logitss	            r#   ÚforwardzPosEmbedRel.forwardb   sš   € Ø”w‰ˆˆ2ˆr�1ð �IŠI�b˜$œ/¨4¬?¸D¼MÑJÔJˆÝ$ Q¨¬À_ÐUÑUÔUˆð �KŠK˜˜1ÑÔˆÝ$ Q¨¬ÀoÐVÑVÔVˆà! LÑ0ˆ
Ø×'Ò'¨¨2¨r°2Ñ6Ô6ˆ
ØÐr%   )NN)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚintÚfloatr0   r6   rH   Ú__classcell__©r9   s   @r#   r'   r'   =   s˜   ø€ € € € € ðð ð Øð ð  àð ð ð ð ð	 ð
 ð ð  ð  ð  ð  ð  ð6>ð >ð >ðð ð ð ð ð ð r%   r'   c                   ó°   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 dded	ee         d
eeeef                  dededee         dedededededefˆ fd„Z	d„ Z
d„ Zˆ xZS )ÚHaloAttnaÀ   Halo Attention

    Paper: `Scaling Local Self-Attention for Parameter Efficient Visual Backbones`
        - https://arxiv.org/abs/2103.12731

    The internal dimensions of the attention module are controlled by the interaction of several arguments.
      * the output dimension of the module is specified by dim_out, which falls back to input dim if not set
      * the value (v) dimension is set to dim_out // num_heads, the v projection determines the output dim
      * the query and key (qk) dimensions are determined by
        * num_heads * dim_head if dim_head is not None
        * num_heads * (dim_out * attn_ratio // num_heads) if dim_head is None
      * as seen above, attn_ratio determines the ratio of q and k relative to the output if dim_head not used

    Args:
        dim (int): input dimension to the module
        dim_out (int): output dimension of the module, same as dim if not set
        feat_size (Tuple[int, int]): size of input feature_map (not used, for arg compat with bottle/lambda)
        stride: output stride of the module, query downscaled if > 1 (default: 1).
        num_heads: parallel attention heads (default: 8).
        dim_head: dimension of query and key heads, calculated from dim_out * attn_ratio // num_heads if not set
        block_size (int): size of blocks. (default: 8)
        halo_size (int): size of halo overlap. (default: 3)
        qk_ratio (float): ratio of q and k dimensions to output dimension when dim_head not set. (default: 1.0)
        qkv_bias (bool) : add bias to q, k, and v projections
        avg_down (bool): use average pool downsample instead of strided query blocks
        scale_pos_embed (bool): scale the position embedding as well as Q @ K
    Nr   é   r@   ç      ð?Fr   Údim_outÚ	feat_sizeÚstrideÚ	num_headsr)   r(   Ú	halo_sizeÚqk_ratioÚqkv_biasÚavg_downÚscale_pos_embedc                 óx  •— ||dœ}t          ¦   «                              ¦   «          |p|}||z  dk    sJ ‚|dv sJ ‚|| _        |pt          ||	z  d¬¦  «        |z  | _        || j        z  | _        || j        z  | _        || j        z  | _        | j        dz  | _        || _	        |x| _
        | _        || _        ||dz  z   | _        d| _        d	}|dk    r*|p||z  dk    }|rdn|| _        | j
        | j        z  | _        t          j        || j        df| j        |
d
œ|¤Ž| _        t          j        || j        | j        z   dfd|
i|¤Ž| _        t'          d| j        | j        | j        | j        dœ|¤Ž| _        |rt          j        dd¦  «        nt          j        ¦   «         | _        |                      ¦   «          d S )Nr,   r   )r   r   rS   )Údivisorç      à¿r   r   F)rW   Úbiasra   )r(   r    r)   r*   © )r/   r0   rX   r	   Údim_head_qkÚ
dim_head_vÚ
dim_out_qkÚ	dim_out_vr*   r]   r(   Úblock_size_dsrY   r    Úblock_strider   ÚConv2dr   Úkvr'   Ú	pos_embedÚ	AvgPool2dÚIdentityÚpoolr6   )r7   r   rU   rV   rW   rX   r)   r(   rY   rZ   r[   r\   r]   r-   r.   r8   Úuse_avg_poolr9   s                    €r#   r0   zHaloAttn.__init__Ž   s  ø€ ð" ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ�.˜SˆØ˜Ñ" aÒ'Ð'Ð'Ð'Ø˜ÐÐÐÐØ"ˆŒØ#Ða¥~°gÀÑ6HÐRSÐ'TÑ'TÔ'TÐXaÑ'aˆÔØ! T¤^Ñ3ˆŒØ# dÔ&6Ñ6ˆŒØ" T¤_Ñ4ˆŒØÔ%¨Ñ-ˆŒ
Ø.ˆÔØ/9Ð9ˆŒ˜$Ô,Ø"ˆŒØ" Y°¡]Ñ2ˆŒØˆÔØˆØ�AŠ:ˆ:Ø#Ð? z°FÑ':¸aÒ'?ˆLØ%1Ð =  °vˆDÔØ!%¤°DÔ4EÑ!EˆDÔõ
 ”˜3 ¤°Ðb¸4Ô;LÐS[ÐbÐbÐ_aÐbÐbˆŒÝ”)˜C ¤°4´>Ñ!AÀ1ÐZÐZÈ8ÐZÐWYÐZÐZˆŒå$ð 
ØÔ)Ø”]ØÔ%Ø”*ð	
ð 
ð
 ð
ð 
ˆŒð +7ÐI•B”L  AÑ&Ô&Ð&½B¼K¹M¼MˆŒ	à×ÒÑÔÐÐÐr%   c                 ó&  — | j         j        j        d         dz  }t          | j         j        |¬¦  «         t          | j        j        |¬¦  «         t          | j        j        | j        ¬¦  «         t          | j        j        | j        ¬¦  «         d S )Nr   r`   r;   )	r   Úweightr   r
   rj   rk   r4   r*   r5   )r7   r<   s     r#   r6   zHaloAttn.reset_parametersÇ   s€   € ØŒfŒmÔ! !Ô$¨Ñ,ˆÝ�d”f”m¨Ð-Ñ-Ô-Ð-Ý�d”g”n¨#Ð.Ñ.Ô.Ð.Ý�d”nÔ/°T´ZÐ@Ñ@Ô@Ð@Ý�d”nÔ.°D´JÐ?Ñ?Ô?Ð?Ð?Ð?r%   c                 ó  — |j         \  }}}}t          || j        z  dk    d¦  «         t          || j        z  dk    d¦  «         || j        z  }|| j        z  }||z  }|                      |¦  «        }	|	                     d| j        || j        || j        ¦  «                             dddddd¦  «        }	|	                     || j        z  | j        d|¦  «         	                    dd¦  «        }	|  
                    |¦  «        }
t          j        |
| j        | j        | j        | j        g¦  «        }
|
                     d| j        | j        ¦  «                             d| j        | j        ¦  «                             || j        z  | j        | j        z   |d¦  «                             dddd¦  «        }
t#          j        |
| j        | j        gd¬	¦  «        \  }}| j        r8|	| 	                    dd
¦  «        z  |                      |	¦  «        z   | j        z  }n7|	| 	                    dd
¦  «        z  | j        z  |                      |	¦  «        z   }|                     d¬	¦  «        }||z   	                    dd¦  «        }|                     d| j        | j        ||¦  «        }|                     ddddd¦  «                             ¦   «                              || j        || j        z  || j        z  ¦  «        }|                      |¦  «        }|S )Nr   Ú r   r   r@   é   r   rA   )r   r   )r   r   r(   r   r   rc   rg   r   rX   r   rj   r   r   rY   Úunfoldr    rd   r2   Úsplitr]   rk   r*   ÚsoftmaxÚ
contiguousÚviewrf   rh   rn   )r7   r!   r   ÚCr   r   Únum_h_blocksÚnum_w_blocksÚ
num_blocksr   rj   ÚkÚvÚattnÚouts                  r#   rH   zHaloAttn.forwardÎ   sò  € Ø”W‰
ˆˆ1ˆa�Ý��D”OÑ# qÒ(¨"Ñ-Ô-Ð-Ý��D”OÑ# qÒ(¨"Ñ-Ô-Ð-Ø˜DœOÑ+ˆØ˜DœOÑ+ˆØ! LÑ0ˆ
à�FŠF�1‰IŒIˆà�IŠIØ�Ô Ø˜$Ô,¨l¸DÔ<NñPô PçPWÒPWÐXYÐ[\Ð^_ÐabÐdeÐghÑPiÔPið 	
ð �IŠI�a˜$œ.Ñ(¨$Ô*:¸BÀ
ÑKÔK×UÒUÐVWÐYZÑ[Ô[ˆð �WŠW�Q‰ZŒZˆõ ŒU�2˜œ¨¬¸¼ÈÌÐWÑXÔXˆØ�YŠY�q˜$œ-¨¬Ñ9Ô9×@Ò@ÀÀDÄMÐSWÔSbÑcÔc×kÒkØ�”Ñ Ô 0°4´?Ñ BÀJÐPRñTô TßT[ÒT[Ð\]Ð_`ÐbcÐefÑTgÔTgð 	åŒ{˜2 Ô 0°$´/ÐBÈÐKÑKÔK‰ˆˆ1ð Ôð 	NØ˜Ÿš B¨Ñ+Ô+Ñ+¨d¯nªn¸QÑ.?Ô.?Ñ?À4Ä:ÑMˆDˆDà˜Ÿš B¨Ñ+Ô+Ñ+¨t¬zÑ9¸D¿NºNÈ1Ñ<MÔ<MÑMˆDà�|Š| ˆ|Ñ#Ô#ˆà�a‰x×"Ò" 1 aÑ(Ô(ˆà�kŠk˜"˜dÔ0°$Ô2DÀlÐT`ÑaÔaˆØ�kŠk˜!˜Q  1 aÑ(Ô(×3Ò3Ñ5Ô5×:Ò:ØˆtŒ~˜q DÔ$5Ñ5°q¸DÔ<MÑ7MñOô Oˆð �iŠi˜‰nŒnˆØˆ
r%   )NNr   rS   NrS   r@   rT   FFFNN)rI   rJ   rK   rL   rM   r   r   rN   Úboolr0   r6   rH   rO   rP   s   @r#   rR   rR   r   s"  ø€ € € € € ðð ð< &*Ø37ØØØ&*ØØØ!Ø"Ø"Ø$)ØØð7 ð 7 àð7 ð ˜c”]ð7 ð    c¨3 h¤Ô0ð	7 ð
 ð7 ð ð7 ð ˜s”mð7 ð ð7 ð ð7 ð ð7 ð ð7 ð ð7 ð "ð7 ð 7 ð 7 ð 7 ð 7 ð 7 ðr@ð @ð @ð)ð )ð )ð )ð )ð )ð )r%   rR   )rL   Útypingr   r   r   r   r2   r   Útorch.nn.functionalÚ
functionalr   Úhelpersr	   Úweight_initr
   Útrace_utilsr   rM   r$   ÚModuler'   rR   rb   r%   r#   ú<module>rŠ      s,  ððð ð$ 0Ð /Ð /Ð /Ð /Ð /Ð /Ð /Ð /Ð /Ð /Ð /à €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð à #Ð #Ð #Ð #Ð #Ð #Ø &Ð &Ð &Ð &Ð &Ð &Ø  Ð  Ð  Ð  Ð  Ð  ð#¨$¨s¬)ð #ð #ð #ð #ð>2ð 2ð 2ð 2ð 2�"”)ñ 2ô 2ð 2ðjEð Eð Eð Eð EˆrŒyñ Eô Eð EðPð r%   