§
    ·¯¸jª  ã                   ó–   — d Z ddlZddlmZmZ ddlZddlmZ ddlmc m	Z
 ddlmZmZ ddlmZ ddlmZ  G d„ d	ej        ¦  «        ZdS )
zâDifferential Attention

Paper: 'Differential Transformer' - https://arxiv.org/abs/2410.05258

Reference impl: https://github.com/microsoft/unilm/tree/master/Diff-Transformer

Hacked together by / Copyright 2024, Ross Wightman
é    N)ÚOptionalÚTypeé   )Úmaybe_add_maskÚresolve_self_attn_mask)Úuse_fused_attn)ÚRmsNormc                   ó,  ‡ — e Zd ZU dZej        j        e         ed<   	 	 	 	 	 	 	 	 	 	 	 	 dd	e	d
e	dedededede
de
deeej                          de	deddfˆ fd„Zde	fd„Zd„ Zdej        fd„Z	 	 ddej        deej                 dedej        fd„Zˆ xZS )ÚDiffAttentiona0  Differential Attention module.

    Computes attention as the difference between two softmax attention maps, which helps
    cancel out noise and promotes sparse attention patterns. The module splits Q and K
    into two groups, computes separate attention maps, and subtracts one from the other
    scaled by a learnable lambda parameter.

    The attention output is computed as:
        Attn = softmax(Q1 @ K1^T) - lambda * softmax(Q2 @ K2^T)
        Output = Attn @ V

    Supports both fused (scaled_dot_product_attention) and manual implementations.
    Ú
fused_attné   FTç        Nr   ÚdimÚ	num_headsÚqkv_biasÚqk_normÚ
scale_normÚ	proj_biasÚ	attn_dropÚ	proj_dropÚ
norm_layerÚdepthÚdual_lambdaÚreturnc                 ó†  •— t          ¦   «                              ¦   «          ||dœ}||z  dk    s
J d¦   «         ‚|	€t          }	|| _        ||z  dz  | _        | j        dz  | _        t          ¦   «         | _        t          j	        ||dz  fd|i|¤Ž| _
        |r |	| j        fi |¤Žnt          j        ¦   «         | _        |r |	| j        fi |¤Žnt          j        ¦   «         | _        t          j        |¦  «        | _        || _        |r	 |	|fi |¤Žnt          j        ¦   «         | _        t          j	        ||fd|i|¤Ž| _        t          j        |¦  «        | _        || _        |rŽt          j        t-          j        d	t,          j        |¬
¦  «        ¦  «        | _        t          j        t-          j        d	t,          j        |¬
¦  «        ¦  «        | _        dx| _        x| _        x| _        | _        �ndx| _        | _        t          j        t-          j        | j        t,          j        |¬
¦  «        ¦  «        | _        t          j        t-          j        | j        t,          j        |¬
¦  «        ¦  «        | _        t          j        t-          j        | j        t,          j        |¬
¦  «        ¦  «        | _        t          j        t-          j        | j        t,          j        |¬
¦  «        ¦  «        | _        t          d| j        z  fddi|¤Ž| _        d| _         |  !                    |
¦  «         |  "                    ¦   «          dS )að  Initialize the DiffAttention module.

        Args:
            dim: Input dimension of the token embeddings.
            num_heads: Number of attention heads.
            qkv_bias: Whether to use bias in the query, key, value projections.
            qk_norm: Whether to apply normalization to query and key vectors.
            scale_norm: Whether to apply normalization before the output projection.
            proj_bias: Whether to use bias in the output projection.
            attn_drop: Dropout rate applied to the attention weights.
            proj_drop: Dropout rate applied after the output projection.
            norm_layer: Normalization layer constructor (defaults to RmsNorm).
            depth: Block depth index, used to compute depth-dependent lambda_init.
            dual_lambda: If True, use simplified dual scalar lambda parameterization
                (2 params). If False, use the paper's original formulation with
                lambda_q/k vectors (4 * head_dim params).
        )ÚdeviceÚdtyper   z$dim should be divisible by num_headsNé   g      à¿é   Úbias© )r   r   Úepsgñhãˆµøä>çš™™™™™é?)#ÚsuperÚ__init__r	   r   Úhead_dimÚscaler   r   ÚnnÚLinearÚqkvÚIdentityÚq_normÚk_normÚDropoutr   Úattn_drop_pÚnormÚprojr   r   Ú	ParameterÚtorchÚemptyÚfloat32Úlambda_aÚlambda_bÚ	lambda_q1Ú	lambda_k1Ú	lambda_q2Ú	lambda_k2Úsub_normÚlambda_initÚset_lambda_initÚreset_parameters)Úselfr   r   r   r   r   r   r   r   r   r   r   r   r   ÚddÚ	__class__s                  €új/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/layers/diff_attention.pyr%   zDiffAttention.__init__%   sì  ø€ õB 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆØ�Y‰ !Ò#Ð#Ð#Ð%KÑ#Ô#Ð#ØÐÝ ˆJØ"ˆŒØ˜yÑ(¨AÑ-ˆŒØ”] dÑ*ˆŒ
Ý(Ñ*Ô*ˆŒå”9˜S #¨¡'Ð?Ð?°Ð?¸BÐ?Ð?ˆŒØ9@ÐS�j�j ¤Ð5Ð5°"Ð5Ð5Ð5ÅbÄkÁmÄmˆŒØ9@ÐS�j�j ¤Ð5Ð5°"Ð5Ð5Ð5ÅbÄkÁmÄmˆŒÝœ IÑ.Ô.ˆŒØ$ˆÔØ-7ÐJ�J�J˜sÐ)Ð) bÐ)Ð)Ð)½R¼[¹]¼]ˆŒ	Ý”I˜c 3Ð=Ð=¨YÐ=¸"Ð=Ð=ˆŒ	Ýœ IÑ.Ô.ˆŒà&ˆÔØð 		jÝœL­¬°R½u¼}ÐU[Ð)\Ñ)\Ô)\Ñ]Ô]ˆDŒMÝœL­¬°R½u¼}ÐU[Ð)\Ñ)\Ô)\Ñ]Ô]ˆDŒMØPTÐTˆDŒNÐT˜Tœ^ÐT¨d¬n¸t¼~¹~à,0Ð0ˆDŒM˜DœMÝœ\­%¬+°d´mÍ5Ì=ÐagÐ*hÑ*hÔ*hÑiÔiˆDŒNÝœ\­%¬+°d´mÍ5Ì=ÐagÐ*hÑ*hÔ*hÑiÔiˆDŒNÝœ\­%¬+°d´mÍ5Ì=ÐagÐ*hÑ*hÔ*hÑiÔiˆDŒNÝœ\­%¬+°d´mÍ5Ì=ÐagÐ*hÑ*hÔ*hÑiÔiˆDŒNå  D¤MÑ 1ÐBÐB°tÐB¸rÐBÐBˆŒàˆÔØ×Ò˜UÑ#Ô#Ð#Ø×ÒÑÔÐÐÐó    c                 óJ   — ddt          j        d|z  ¦  «        z  z
  | _        d S )Nr#   g333333ã?g333333Ó¿)ÚmathÚexpr=   )r@   r   s     rC   r>   zDiffAttention.set_lambda_initk   s(   € Ø ¥t¤x°°u±Ñ'=Ô'=Ñ!=Ñ=ˆÔÐÐrD   c                 óà  — | j         rJt          j                             | j        ¦  «         t          j                             | j        ¦  «         d S t          j                             | j        dd¬¦  «         t          j                             | j        dd¬¦  «         t          j                             | j	        dd¬¦  «         t          j                             | j
        dd¬¦  «         d S )Nr   gš™™™™™¹?)ÚmeanÚstd)r   r(   ÚinitÚzeros_r6   r7   Únormal_r8   r9   r:   r;   )r@   s    rC   r?   zDiffAttention.reset_parametersn   sµ   € ØÔð 	=ÝŒG�NŠN˜4œ=Ñ)Ô)Ð)ÝŒG�NŠN˜4œ=Ñ)Ô)Ð)Ð)Ð)åŒG�OŠO˜DœN°¸ˆOÑ<Ô<Ð<ÝŒG�OŠO˜DœN°¸ˆOÑ<Ô<Ð<ÝŒG�OŠO˜DœN°¸ˆOÑ<Ô<Ð<ÝŒG�OŠO˜DœN°¸ˆOÑ<Ô<Ð<Ð<Ð<rD   c                 ó¬  — | j         �3t          j        | j         ¦  «        }t          j        | j        ¦  «        }nŽt          j        t          j        | j        | j        z  d¬¦  «                             ¦   «         ¦  «        }t          j        t          j        | j        | j	        z  d¬¦  «                             ¦   «         ¦  «        }||z
  | j
        z   S )Néÿÿÿÿ©r   )r6   r3   rG   r7   Úsumr8   r9   Úfloatr:   r;   r=   )r@   Úlambda_1Úlambda_2s      rC   Ú_compute_lambdazDiffAttention._compute_lambdax   s©   € ØŒ=Ð$Ý”y ¤Ñ/Ô/ˆHÝ”y ¤Ñ/Ô/ˆHˆHå”y¥¤¨4¬>¸D¼NÑ+JÐPRÐ!SÑ!SÔ!S×!YÒ!YÑ![Ô![Ñ\Ô\ˆHÝ”y¥¤¨4¬>¸D¼NÑ+JÐPRÐ!SÑ!SÔ!S×!YÒ!YÑ![Ô![Ñ\Ô\ˆHØ˜(Ñ" TÔ%5Ñ5Ð5rD   ÚxÚ	attn_maskÚ	is_causalc                 óˆ  — |j         \  }}}|                      |¦  «                             dd¬¦  «        \  }}}	|                     ||d| j        z  | j        ¦  «                             dd¦  «        }|                     ||d| j        z  | j        ¦  «                             dd¦  «        }|	                     ||| j        d| j        z  ¦  «                             dd¦  «        }	|                      |¦  «        |                      |¦  «        }}|  	                    ¦   «          
                    |¦  «        }
| j        rÃ|                     || j        d|| j        ¦  «        }|                     || j        d|| j        ¦  «        }|                     d¦  «        \  }}|                     d¦  «        \  }}| j        r| j        nd}t          j        |||	|||¬¦  «        }t          j        |||	|||¬¦  «        }||
|z  z
  }n¶|| j        z  }||                     dd¦  «        z  }t%          ||||¬	¦  «        }t'          ||¦  «        }|                     d¬¦  «        }|                      |¦  «        }|                     || j        d||¦  «        }|d d …d d …d
f         |
|d d …d d …df         z  z
  }||	z  }|                      |¦  «        }|d| j        z
  z  }|                     dd¦  «                             |||¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S )Nr   r   rP   r   r   )rW   Ú	dropout_prX   éþÿÿÿrO   )rX   r   )Úshaper*   ÚchunkÚreshaper   r&   Ú	transposer,   r-   rU   Útype_asr   ÚunbindÚtrainingr/   ÚFÚscaled_dot_product_attentionr'   r   r   Úsoftmaxr   Úviewr<   r=   r0   r1   r   )r@   rV   rW   rX   ÚBÚNÚCÚqÚkÚvÚlambda_fullÚq1Úq2Úk1Úk2rZ   Úattn1Úattn2ÚattnÚ	attn_biass                       rC   ÚforwardzDiffAttention.forward�   s#  € ð ”'‰ˆˆ1ˆaà—(’(˜1‘+”+×#Ò# A¨1Ð#Ñ-Ô-‰ˆˆ1ˆaØ�IŠI�a˜˜A ¤Ñ.°´Ñ>Ô>×HÒHÈÈAÑNÔNˆØ�IŠI�a˜˜A ¤Ñ.°´Ñ>Ô>×HÒHÈÈAÑNÔNˆØ�IŠI�a˜˜DœN¨A°´Ñ,=Ñ>Ô>×HÒHÈÈAÑNÔNˆà�{Š{˜1‰~Œ~˜tŸ{š{¨1™~œ~ˆ1ˆà×*Ò*Ñ,Ô,×4Ò4°QÑ7Ô7ˆàŒ?ð 	Ø—	’	˜!˜Tœ^¨Q°°4´=ÑAÔAˆAØ—	’	˜!˜Tœ^¨Q°°4´=ÑAÔAˆAØ—X’X˜a‘[”[‰FˆB�Ø—X’X˜a‘[”[‰FˆB�à,0¬MÐB˜Ô(Ð(¸sˆIÝÔ2Ø�B˜ Y¸)ÈyðZñ Zô ZˆEåÔ2Ø�B˜ Y¸)ÈyðZñ Zô ZˆEð ˜ eÑ+Ñ+ˆAˆAà�D”J‘ˆAØ�q—{’{ 2 rÑ*Ô*Ñ*ˆDÝ.¨q°$¸	ÈYÐWÑWÔWˆIÝ! $¨	Ñ2Ô2ˆDØ—<’< B�<Ñ'Ô'ˆDØ—>’> $Ñ'Ô'ˆDà—9’9˜Q ¤°°1°aÑ8Ô8ˆDØ˜˜˜˜1˜1˜1˜a˜”= ;°°a°a°a¸¸¸¸A°g´Ñ#>Ñ>ˆDØ�q‘ˆAà�MŠM˜!ÑÔˆØ��TÔ%Ñ%Ñ&ˆØ�KŠK˜˜1ÑÔ×%Ò% a¨¨AÑ.Ô.ˆà�IŠI�a‰LŒLˆØ�IŠI�a‰LŒLˆØ�NŠN˜1ÑÔˆàˆrD   )r   FFFTr   r   Nr   FNN)NF)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r3   ÚjitÚFinalÚboolÚ__annotations__ÚintrR   r   r   r(   ÚModuler%   r>   r?   ÚTensorrU   rv   Ú__classcell__)rB   s   @rC   r   r      s¹  ø€ € € € € € ðð ð ”	” Ô%Ð%Ð%Ñ%ð
 Ø"Ø!Ø$Ø"Ø!Ø!Ø48ØØ %ØØðD ð D àðD ð ðD ð ð	D ð
 ðD ð ðD ð ðD ð ðD ð ðD ð !  b¤i¤Ô1ðD ð ðD ð ðD ð 
ðD ð D ð D ð D ð D ð D ðL> Sð >ð >ð >ð >ð=ð =ð =ð6 ¤ð 6ð 6ð 6ð 6ð 15Ø#ð	2ð 2àŒ|ð2ð   ¤Ô-ð2ð ð	2ð
 
Œð2ð 2ð 2ð 2ð 2ð 2ð 2ð 2rD   r   )rz   rF   Útypingr   r   r3   Útorch.nnr(   Útorch.nn.functionalÚ
functionalrc   Ú	attentionr   r   Úconfigr   r0   r	   r€   r   r!   rD   rC   ú<module>r‰      sä   ððð ð €€€Ø !Ð !Ð !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð à =Ð =Ð =Ð =Ð =Ð =Ð =Ð =Ø "Ð "Ð "Ð "Ð "Ð "Ø Ð Ð Ð Ð Ð ð^ð ^ð ^ð ^ð ^�B”Iñ ^ô ^ð ^ð ^ð ^rD   