§
    ·¯¸j#4  ã                   ó¤  — d dl mZmZmZ d dlZd dlmZ d dlmZ ddl	m
Z
 ddlmZ ddlmZ g d	¢Zej        j        e
dd
ej        deej                 fd„¦   «         ¦   «         Zej        j        e
	 	 ddedej        deej                 dedeej                 f
d„¦   «         ¦   «         Z G d„ dej        ¦  «        Z G d„ dej        ¦  «        ZdS )é    )ÚFinalÚOptionalÚTypeN)Únn)Ú
functionalé   )Úregister_notrace_function)Úuse_fused_attn)Úapply_rot_embed_cat)Ú	AttentionÚAttentionRopeÚmaybe_add_maskÚresolve_self_attn_maskÚscoresÚ	attn_maskc                 ó   — |€| n| |z   S ©N© )r   r   s     úe/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/layers/attention.pyr   r      s   € ð Ð&ˆ6ˆ6¨F°YÑ,>Ð>ó    FÚseq_lenÚattnÚ	is_causalÚreturnc                 ó4  — |r9|                      | | ft          d¦  «        ¦  «                             d¦  «        }n\|€d }nW|j        t          j        k    r@t	          j        ||j        ¬¦  «        }|                     | t          d¦  «        ¦  «         n|}|S )Nz-infr   )Údtype)Únew_fullÚfloatÚtriu_r   ÚtorchÚboolÚ
zeros_likeÚmasked_fill_)r   r   r   r   Ú	attn_biass        r   r   r      s™   € ð ð Ø—M’M 7¨GÐ"4µe¸F±m´mÑDÔD×JÒJÈ1ÑMÔMˆ	ˆ	Ø	Ð	Øˆ	ˆ	Ø	Œ�EœJÒ	&Ð	&ÝÔ$ Y°d´jÐAÑAÔAˆ	Ø×Ò 	˜z­5°©=¬=Ñ9Ô9Ð9Ð9àˆ	ØÐr   c                   ó  ‡ — e Zd ZU dZee         ed<   	 	 	 	 	 	 	 	 	 	 	 	 	 dded	ed
ee         dee         dededededede	de	dee
ej                          ddfˆ fd„Z	 	 ddej        deej                 dedej        fd„Zˆ xZS )r   a£  Standard Multi-head Self Attention module with QKV projection.

    This module implements the standard multi-head attention mechanism used in transformers.
    It supports both the fused attention implementation (scaled_dot_product_attention) for
    efficiency when available, and a manual implementation otherwise. The module includes
    options for QK normalization, attention dropout, and projection dropout.
    Ú
fused_attné   NFTç        ÚdimÚ	num_headsÚattn_head_dimÚdim_outÚqkv_biasÚqk_normÚ
scale_normÚ	proj_biasÚgatedÚ	attn_dropÚ	proj_dropÚ
norm_layerr   c                 ó$  •— t          ¦   «                              ¦   «          ||dœ}|p|}|}|€||z  dk    s
J d¦   «         ‚||z  }|s|r|€
J d¦   «         ‚|| _        || _        ||z  | _        |dz  | _        t          ¦   «         | _        t          j	        || j        dz  fd|i|¤Ž| _
        |r	 ||fi |¤Žnt          j        ¦   «         | _        |r	 ||fi |¤Žnt          j        ¦   «         | _        t          j        |
¦  «        | _        |r || j        fi |¤Žnt          j        ¦   «         | _        |	rt          j	        || j        fd|i|¤Žnd| _        t          j	        | j        |fd|i|¤Ž| _        t          j        |¦  «        | _        dS )	aà  Initialize the Attention module.

        Args:
            dim: Input dimension of the token embeddings.
            num_heads: Number of attention heads.
            attn_head_dim: Dimension of each attention head. If None, computed as dim // num_heads.
            dim_out: Output dimension. If None, same as dim.
            qkv_bias: Whether to use bias in the query, key, value projections.
            qk_norm: Whether to apply normalization to query and key vectors.
            scale_norm: Whether to apply normalization to attention output before projection.
            proj_bias: Whether to use bias in the output projection.
            gated: Apply an elementwise sigmoid gate to the attention output (anti attention-sink, GenLIP-style).
            attn_drop: Dropout rate applied to the attention weights.
            proj_drop: Dropout rate applied after the output projection.
            norm_layer: Normalization layer constructor for QK normalization if enabled.
        ©Údevicer   Nr   ú$dim should be divisible by num_headsú<norm_layer must be provided if qk_norm or scale_norm is Trueç      à¿é   Úbias)ÚsuperÚ__init__r*   Úhead_dimÚattn_dimÚscaler
   r&   r   ÚLinearÚqkvÚIdentityÚq_normÚk_normÚDropoutr2   ÚnormÚgateÚprojr3   )Úselfr)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r7   r   Úddr?   Ú	__class__s                    €r   r>   zAttention.__init__5   s×  ø€ õB 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆØ�.˜SˆØ ˆØÐØ˜‘? aÒ'Ð'Ð'Ð)OÑ'Ô'Ð'Ø˜iÑ'ˆHØð 	j�jð 	jØÐ)Ð)Ð+iÑ)Ô)Ð)à"ˆŒØ ˆŒØ! HÑ,ˆŒØ Ñ%ˆŒ
Ý(Ñ*Ô*ˆŒå”9˜S $¤-°!Ñ"3ÐIÐI¸(ÐIÀbÐIÐIˆŒØ4;ÐN�j�j Ð0Ð0¨RÐ0Ð0Ð0ÅÄÁÄˆŒØ4;ÐN�j�j Ð0Ð0¨RÐ0Ð0Ð0ÅÄÁÄˆŒÝœ IÑ.Ô.ˆŒØ7AÐT�J�J˜tœ}Ð3Ð3°Ð3Ð3Ð3ÅrÄ{Á}Ä}ˆŒ	ØJOÐY•B”I˜c 4¤=ÐFÐF°xÐFÀ2ÐFÐFÐFÐUYˆŒ	Ý”I˜dœm¨WÐKÐK¸9ÐKÈÐKÐKˆŒ	Ýœ IÑ.Ô.ˆŒˆˆr   Úxr   r   c                 óÜ  — |j         \  }}}| j        �'|                      |¦  «                             ¦   «         nd }|                      |¦  «                             ||d| j        | j        ¦  «                             ddddd¦  «        }|                     d¦  «        \  }	}
}|  	                    |	¦  «        |  
                    |
¦  «        }
}	| j        r.t          j        |	|
||| j        r| j        j        nd|¬¦  «        }nu|	| j        z  }	|	|
                     dd	¦  «        z  }t'          ||||¦  «        }t)          ||¦  «        }|                     d	¬
¦  «        }|                      |¦  «        }||z  }|                     dd¦  «                             ||| j        ¦  «        }|                      |¦  «        }|�||z  }|                      |¦  «        }|                      |¦  «        }|S )Nr;   é   r   r   é   r(   ©r   Ú	dropout_pr   éþÿÿÿéÿÿÿÿ©r)   )ÚshaperI   ÚsigmoidrC   Úreshaper*   r?   ÚpermuteÚunbindrE   rF   r&   ÚFÚscaled_dot_product_attentionÚtrainingr2   ÚprA   Ú	transposer   r   Úsoftmaxr@   rH   rJ   r3   )rK   rN   r   r   ÚBÚNÚCrI   rC   ÚqÚkÚvr   r$   s                 r   ÚforwardzAttention.forwardo   sÐ  € ð ”'‰ˆˆ1ˆaØ)-¬Ð)>ˆt�yŠy˜‰|Œ|×#Ò#Ñ%Ô%Ð%ÀDˆØ�hŠh�q‰kŒk×!Ò! ! Q¨¨4¬>¸4¼=ÑIÔI×QÒQÐRSÐUVÐXYÐ[\Ð^_Ñ`Ô`ˆØ—*’*˜Q‘-”-‰ˆˆ1ˆaØ�{Š{˜1‰~Œ~˜tŸ{š{¨1™~œ~ˆ1ˆàŒ?ð 	ÝÔ.Ø�1�aØ#Ø.2¬mÐC˜$œ.Ô*Ð*ÀØ#ð	ñ ô ˆAˆAð �D”J‘ˆAØ�q—{’{ 2 rÑ*Ô*Ñ*ˆDÝ.¨q°$¸	À9ÑMÔMˆIÝ! $¨	Ñ2Ô2ˆDØ—<’< B�<Ñ'Ô'ˆDØ—>’> $Ñ'Ô'ˆDØ�q‘ˆAà�KŠK˜˜1ÑÔ×%Ò% a¨¨D¬MÑ:Ô:ˆØ�IŠI�a‰LŒLˆØÐØ�D‘ˆAØ�IŠI�a‰LŒLˆØ�NŠN˜1ÑÔˆØˆr   )r'   NNFFFTFr(   r(   NNN©NF)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r!   Ú__annotations__Úintr   r   r   r   ÚModuler>   r    ÚTensorrh   Ú__classcell__©rM   s   @r   r   r   +   so  ø€ € € € € € ðð ð �d”ÐÐÑð
 Ø+/Ø%)Ø"Ø!Ø$Ø"ØØ!Ø!Ø48ØØð8/ð 8/àð8/ð ð8/ð $ Cœ=ð	8/ð
 ˜c”]ð8/ð ð8/ð ð8/ð ð8/ð ð8/ð ð8/ð ð8/ð ð8/ð !  b¤i¤Ô1ð8/ð  
ð!8/ð 8/ð 8/ð 8/ð 8/ð 8/ðz 15Ø#ð	"ð "àŒ|ð"ð   ¤Ô-ð"ð ð	"ð
 
Œð"ð "ð "ð "ð "ð "ð "ð "r   r   c            "       ó6  ‡ — e Zd ZU dZej        j        e         ed<   	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd	e	d
e	de
e	         dedede	dedede
e	         deej                 dededededede
e	         f ˆ fd„Z	 	 	 dde
ej                 de
ej                 defd„Zˆ xZS ) r   a   A Self Attention module with ROPE support.

    Includes options for:
     * QK normalization option
     * Attention output (scale) normalization
     * Fused or unfused QKV projection support
     * Grouped-query attention (fewer K/V heads than Q heads) via `num_kv_heads`
    r&   r'   NTr   r(   Fr)   r*   r,   r-   Ú	qkv_fusedÚnum_prefix_tokensr2   r3   r+   r4   r.   r/   r0   Úrotate_halfr1   Únum_kv_headsc                 óÆ  •— t          ¦   «                              ¦   «          ||dœ}|p|}|	}|€||z  dk    s
J d¦   «         ‚||z  }|s|r|
€
J d¦   «         ‚|p|}||z  dk    s
J d¦   «         ‚|| _        || _        ||z  | _        || _        ||z  | _        ||z  | _        |dz  | _        || _	        t          ¦   «         | _        || _        |rM| j        dk    s
J d	¦   «         ‚t          j        || j        d
z  fd|i|¤Ž| _        dx| _        x| _        | _        ndd| _        t          j        || j        fd|i|¤Ž| _        t          j        || j        fd|i|¤Ž| _        t          j        || j        fd|i|¤Ž| _        |r	 |
|fi |¤Žnt          j        ¦   «         | _        |r	 |
|fi |¤Žnt          j        ¦   «         | _        t          j        |¦  «        | _        |r |
| j        fi |¤Žnt          j        ¦   «         | _        |rt          j        || j        fd|i|¤Žnd| _        t          j        | j        |fd|i|¤Ž| _        t          j        |¦  «        | _        dS )aÈ  Initialize the Attention module.

        Args:
            dim: Input dimension of the token embeddings
            num_heads: Number of attention heads
            dim_out: Output dimension. If None, same as dim.
            qkv_bias: Whether to add a bias term to the query, key, and value projections
            qkv_fused: Whether to use fused QKV projection (single linear) or separate projections
            num_prefix_tokens: Number of reg/cls tokens at the beginning of the sequence that
                should not have position embeddings applied
            attn_drop: Dropout rate for attention weights
            proj_drop: Dropout rate for the output projection
            attn_head_dim: Dimension of each attention head. If None, computed as dim // num_heads.
            norm_layer: Normalization layer constructor to use for QK and scale normalization
            qk_norm: Enable normalization of query (Q) and key (K) vectors with norm_layer
            scale_norm: Enable normalization (scaling) of attention output with norm_layer
            proj_bias: Whether to use bias in the output projection
            rotate_half: Use 'half' ROPE layout instead of default 'interleaved'
            gated: Add a sigmoid gate on the attention output (before projection)
            num_kv_heads: Number of key/value heads for grouped-query attention. None or equal to
                num_heads gives standard multi-head attention. Requires qkv_fused=False.
        r6   Nr   r8   r9   z+num_heads must be divisible by num_kv_headsr:   r   z=fused qkv projection does not support grouped-query attentionr;   r<   )r=   r>   r*   rx   Únum_kv_groupsr?   r@   Úkv_dimrA   rv   r
   r&   rw   r   rB   rC   Úq_projÚk_projÚv_projrD   rE   rF   rG   r2   rH   rI   rJ   r3   )rK   r)   r*   r,   r-   ru   rv   r2   r3   r+   r4   r.   r/   r0   rw   r1   rx   r7   r   rL   r?   rM   s                        €r   r>   zAttentionRope.__init__Ÿ   sÛ  ø€ õV 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆØ�.˜SˆØ ˆØÐØ˜‘? aÒ'Ð'Ð'Ð)OÑ'Ô'Ð'Ø˜iÑ'ˆHØð 	j˜ð 	jØÐ)Ð)Ð+iÑ)Ô)Ð)Ø#Ð0 yˆØ˜<Ñ'¨1Ò,Ð,Ð,Ð.[Ñ,Ô,Ð,à"ˆŒØ(ˆÔØ&¨,Ñ6ˆÔØ ˆŒØ  9Ñ,ˆŒØ Ñ-ˆŒØ Ñ%ˆŒ
Ø!2ˆÔÝ(Ñ*Ô*ˆŒØ&ˆÔàð 	KØÔ%¨Ò*Ð*Ð*Ð,kÑ*Ô*Ð*Ý”y  d¤m°aÑ&7ÐMÐM¸hÐMÈ"ÐMÐMˆDŒHØ6:Ð:ˆDŒKÐ:˜$œ+¨¬¨àˆDŒHÝœ) C¨¬ÐLÐL¸XÐLÈÐLÐLˆDŒKÝœ) C¨¬ÐJÐJ¸8ÐJÀrÐJÐJˆDŒKÝœ) C¨¬ÐJÐJ¸8ÐJÀrÐJÐJˆDŒKà4;ÐN�j�j Ð0Ð0¨RÐ0Ð0Ð0ÅÄÁÄˆŒØ4;ÐN�j�j Ð0Ð0¨RÐ0Ð0Ð0ÅÄÁÄˆŒÝœ IÑ.Ô.ˆŒØ7AÐT�J�J˜tœ}Ð3Ð3°Ð3Ð3Ð3ÅrÄ{Á}Ä}ˆŒ	ØJOÐY•B”I˜c 4¤=ÐFÐF°xÐFÀ2ÐFÐFÐFÐUYˆŒ	Ý”I˜dœm¨WÐKÐK¸9ÐKÈÐKÐKˆŒ	Ýœ IÑ.Ô.ˆŒˆˆr   Úroper   r   c                 óÜ  — |j         \  }}}| j        �'|                      |¦  «                             ¦   «         nd}| j        �i|                      |¦  «        }	|	                     ||d| j        | j        ¦  «                             ddddd¦  «        }	|	                     d¦  «        \  }
}}nÛ|  	                    |¦  «                             ||| j        | j        ¦  «         
                    dd¦  «        }
|                      |¦  «                             ||| j        | j        ¦  «         
                    dd¦  «        }|                      |¦  «                             ||| j        | j        ¦  «         
                    dd¦  «        }|                      |
¦  «        |                      |¦  «        }}
|�Ö| j        }t#          | dd¦  «        }t%          j        |
dd…dd…d|…dd…f         t)          |
dd…dd…|d…dd…f         ||¬	¦  «        gd¬
¦  «                             |¦  «        }
t%          j        |dd…dd…d|…dd…f         t)          |dd…dd…|d…dd…f         ||¬	¦  «        gd¬
¦  «                             |¦  «        }| j        dk    r8|                     | j        d¬
¦  «        }|                     | j        d¬
¦  «        }| j        r.t3          j        |
|||| j        r| j        j        nd|¬¦  «        }nu|
| j        z  }
|
| 
                    dd¦  «        z  }t?          ||||¦  «        }tA          ||¦  «        }| !                    d¬
¦  «        }|                      |¦  «        }||z  }| 
                    dd¦  «                             ||| j"        ¦  «        }|  #                    |¦  «        }|�||z  }|  $                    |¦  «        }|  %                    |¦  «        }|S )aÏ  Forward pass for the attention module.

        Args:
            x: Input tensor of shape (batch_size, sequence_length, embedding_dim)
            rope: Rotary position embeddings tensor for position-aware attention
            attn_mask: Optional attention mask to apply during attention computation
            is_causal: If True, use causal (autoregressive) masking

        Returns:
            Tensor of shape (batch_size, sequence_length, dim_out)
        Nr;   rP   r   r   rQ   rw   F)ÚhalfrV   r(   rR   rT   rU   )&rW   rI   rX   rC   rY   r*   r?   rZ   r[   r|   r`   r}   rx   r~   rE   rF   rv   Úgetattrr    Úcatr   Útype_asrz   Úrepeat_interleaver&   r\   r]   r^   r2   r_   rA   r   r   ra   r@   rH   rJ   r3   )rK   rN   r   r   r   rb   rc   rd   rI   rC   re   rf   rg   Únptr�   r   r$   s                    r   rh   zAttentionRope.forwardó   sØ  € ð$ ”'‰ˆˆ1ˆaØ)-¬Ð)>ˆt�yŠy˜‰|Œ|×#Ò#Ñ%Ô%Ð%ÀDˆàŒ8ÐØ—(’(˜1‘+”+ˆCØ—+’+˜a  A t¤~°t´}ÑEÔE×MÒMÈaÐQRÐTUÐWXÐZ[Ñ\Ô\ˆCØ—j’j ‘m”m‰GˆAˆq�!�!à—’˜A‘”×&Ò& q¨!¨T¬^¸T¼]ÑKÔK×UÒUÐVWÐYZÑ[Ô[ˆAØ—’˜A‘”×&Ò& q¨!¨TÔ->ÀÄÑNÔN×XÒXÐYZÐ\]Ñ^Ô^ˆAØ—’˜A‘”×&Ò& q¨!¨TÔ->ÀÄÑNÔN×XÒXÐYZÐ\]Ñ^Ô^ˆAà�{Š{˜1‰~Œ~˜tŸ{š{¨1™~œ~ˆ1ˆàÐØÔ(ˆCÝ˜4 °Ñ6Ô6ˆDÝ”	˜1˜Q˜Q˜Q    4 C 4¨¨¨˜]Ô+Õ-@ÀÀ1À1À1ÀaÀaÀaÈÈÈÈqÈqÈqÀ=ÔAQÐSWÐ^bÐ-cÑ-cÔ-cÐdÐjkÐlÑlÔl×tÒtÐuvÑwÔwˆAÝ”	˜1˜Q˜Q˜Q    4 C 4¨¨¨˜]Ô+Õ-@ÀÀ1À1À1ÀaÀaÀaÈÈÈÈqÈqÈqÀ=ÔAQÐSWÐ^bÐ-cÑ-cÔ-cÐdÐjkÐlÑlÔl×tÒtÐuvÑwÔwˆAàÔ Ò!Ð!ð ×#Ò# DÔ$6¸AÐ#Ñ>Ô>ˆAØ×#Ò# DÔ$6¸AÐ#Ñ>Ô>ˆAàŒ?ð 	ÝÔ.Ø�1�aØ#Ø.2¬mÐC˜$œ.Ô*Ð*ÀØ#ð	ñ ô ˆAˆAð �D”J‘ˆAØ�q—{’{ 2 rÑ*Ô*Ñ*ˆDÝ.¨q°$¸	À9ÑMÔMˆIÝ! $¨	Ñ2Ô2ˆDØ—<’< B�<Ñ'Ô'ˆDà—>’> $Ñ'Ô'ˆDØ�q‘ˆAà�KŠK˜˜1ÑÔ×%Ò% a¨¨D¬MÑ:Ô:ˆØ�IŠI�a‰LŒLˆØÐØ�D‘ˆAØ�IŠI�a‰LŒLˆØ�NŠN˜1ÑÔˆØˆr   )r'   NTTr   r(   r(   NNFFTFFNNN)NNF)rj   rk   rl   rm   r    Újitr   r!   rn   ro   r   r   r   r   rp   r>   rq   rh   rr   rs   s   @r   r   r   ”   s¾  ø€ € € € € € ðð ð ”	” Ô%Ð%Ð%Ñ%ð
 Ø%)Ø!Ø"Ø%&Ø!Ø!Ø+/Ø*.Ø!Ø$Ø"Ø %ØØ*.ØØð'R/ð R/àðR/ð ðR/ð ˜c”]ð	R/ð
 ðR/ð ðR/ð  #ðR/ð ðR/ð ðR/ð $ Cœ=ðR/ð ˜RœYœðR/ð ðR/ð ðR/ð ðR/ð ðR/ð  ð!R/ð" # 3œ-ð#R/ð R/ð R/ð R/ð R/ð R/ðn ,0Ø04Ø#ðDð Dð ˜5œ<Ô(ðDð   ¤Ô-ð	Dð
 ðDð Dð Dð Dð Dð Dð Dð Dr   r   r   ri   )Útypingr   r   r   r    r   Útorch.nnr   r\   Ú_fxr	   Úconfigr
   Úpos_embed_sincosr   Ú__all__ÚfxÚwraprq   r   ro   r!   r   rp   r   r   r   r   r   ú<module>r�      sÎ  ðØ (Ð (Ð (Ð (Ð (Ð (Ð (Ð (Ð (Ð (à €€€Ø Ð Ð Ð Ð Ð Ø $Ð $Ð $Ð $Ð $Ð $à *Ð *Ð *Ð *Ð *Ð *Ø "Ð "Ð "Ð "Ð "Ð "Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1ð UÐ
TÐ
T€ð „„Øð?ð ?˜5œ<ð ?°H¸U¼\Ô4Jð ?ð ?ð ?ñ Ôñ „ð?ð „„Øð -1Øð	ð ØðàŒlðð ˜EœLÔ)ðð ð	ð
 ˆeŒlÔðð ð ñ Ôñ „ðð(fð fð fð fð f�”	ñ fô fð fðRcð cð cð cð c�B”Iñ cô cð cð cð cr   