§
    l¬¸jç3  ã                  ó†   — d dl mZ d dlZd dlZd dlmZ ddlmZ  G d„ dej        ¦  «        Z G d„ d	ej        ¦  «        Z	dS )
é    )ÚannotationsN)Únné   )ÚRoPEAttentionc                  óZ   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 d!d"ˆ fd„Zd#d„Z	 d$d%d„Z	 	 	 d&d'd „Zˆ xZS )(ÚMemoryAttentionLayeraß  Implements a memory attention layer with self-attention and cross-attention mechanisms for neural networks.

    This class combines self-attention, cross-attention, and feedforward components to process input tensors and
    generate memory-based attention outputs.

    Attributes:
        d_model (int): Dimensionality of the model.
        dim_feedforward (int): Dimensionality of the feedforward network.
        dropout_value (float): Dropout rate for regularization.
        self_attn (RoPEAttention): Self-attention mechanism using RoPE (Rotary Position Embedding).
        cross_attn_image (RoPEAttention): Cross-attention mechanism for image processing.
        linear1 (nn.Linear): First linear layer of the feedforward network.
        linear2 (nn.Linear): Second linear layer of the feedforward network.
        norm1 (nn.LayerNorm): Layer normalization for self-attention output.
        norm2 (nn.LayerNorm): Layer normalization for cross-attention output.
        norm3 (nn.LayerNorm): Layer normalization for feedforward network output.
        dropout1 (nn.Dropout): Dropout layer after self-attention.
        dropout2 (nn.Dropout): Dropout layer after cross-attention.
        dropout3 (nn.Dropout): Dropout layer after feedforward network.
        activation (nn.ReLU): Activation function for the feedforward network.
        pos_enc_at_attn (bool): Flag to add positional encoding at attention.
        pos_enc_at_cross_attn_queries (bool): Flag to add positional encoding to cross-attention queries.
        pos_enc_at_cross_attn_keys (bool): Flag to add positional encoding to cross-attention keys.

    Methods:
        forward: Performs the full memory attention operation on input tensors.
        _forward_sa: Performs self-attention on input tensor.
        _forward_ca: Performs cross-attention between target and memory tensors.

    Examples:
        >>> layer = MemoryAttentionLayer(d_model=256, dim_feedforward=2048, dropout=0.1)
        >>> tgt = torch.randn(1, 100, 256)
        >>> memory = torch.randn(1, 100, 64)
        >>> pos = torch.randn(1, 100, 256)
        >>> query_pos = torch.randn(1, 100, 256)
        >>> output = layer(tgt, memory, pos, query_pos)
        >>> print(output.shape)
        torch.Size([1, 100, 256])
    é   é   çš™™™™™¹?FTNÚd_modelÚintÚdim_feedforwardÚdropoutÚfloatÚpos_enc_at_attnÚboolÚpos_enc_at_cross_attn_keysÚpos_enc_at_cross_attn_queriesÚ	self_attnúnn.Module | NoneÚ
cross_attnc	                óú  •— t          ¦   «                              ¦   «          || _        || _        || _        |pt          ddd¬¦  «        | _        |pt          ddddd¬¦  «        | _        t          j	        ||¦  «        | _
        t          j        |¦  «        | _        t          j	        ||¦  «        | _        t          j        |¦  «        | _        t          j        |¦  «        | _        t          j        |¦  «        | _        t          j        |¦  «        | _        t          j        |¦  «        | _        t          j        |¦  «        | _        t          j        ¦   «         | _        || _        || _        || _        dS )aX  Initialize a memory attention layer with self-attention, cross-attention, and feedforward components.

        Args:
            d_model (int): Dimensionality of the model.
            dim_feedforward (int): Dimensionality of the feedforward network.
            dropout (float): Dropout rate for regularization.
            pos_enc_at_attn (bool): Whether to add positional encoding at attention.
            pos_enc_at_cross_attn_keys (bool): Whether to add positional encoding to cross-attention keys.
            pos_enc_at_cross_attn_queries (bool): Whether to add positional encoding to cross-attention queries.
            self_attn (nn.Module | None): Custom self-attention module. If None, a default RoPEAttention is used.
            cross_attn (nn.Module | None): Custom cross-attention module. If None, a default RoPEAttention is used.
        r	   r   )Úembedding_dimÚ	num_headsÚdownsample_rateTé@   )Úrope_k_repeatr   r   r   Ú	kv_in_dimN)ÚsuperÚ__init__r   r   Údropout_valuer   r   Úcross_attn_imager   ÚLinearÚlinear1ÚDropoutr   Úlinear2Ú	LayerNormÚnorm1Únorm2Únorm3Údropout1Údropout2Údropout3ÚReLUÚ
activationr   r   r   )
Úselfr   r   r   r   r   r   r   r   Ú	__class__s
            €ú/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/ultralytics/models/sam/modules/memory_attention.pyr    zMemoryAttentionLayer.__init__6   sH  ø€ õ. 	‰Œ×ÒÑÔÐØˆŒØ.ˆÔØ$ˆÔØ"Ðf¥mÀ#ÐQRÐdeÐ&fÑ&fÔ&fˆŒØ *ð !
­mØØØØØð/
ñ /
ô /
ˆÔõ ”y ¨/Ñ:Ô:ˆŒÝ”z 'Ñ*Ô*ˆŒÝ”y °'Ñ:Ô:ˆŒå”\ 'Ñ*Ô*ˆŒ
Ý”\ 'Ñ*Ô*ˆŒ
Ý”\ 'Ñ*Ô*ˆŒ
Ýœ
 7Ñ+Ô+ˆŒÝœ
 7Ñ+Ô+ˆŒÝœ
 7Ñ+Ô+ˆŒåœ'™)œ)ˆŒð  /ˆÔØ-JˆÔ*Ø*DˆÔ'Ð'Ð'ó    Útgtútorch.TensorÚ	query_posútorch.Tensor | NoneÚreturnc                ó°   — |                       |¦  «        }| j        r||z   n|x}}|                      |||¬¦  «        }||                      |¦  «        z   }|S )z^Perform self-attention on input tensor using positional encoding and RoPE attention mechanism.)Úv)r(   r   r   r+   )r0   r4   r6   Útgt2ÚqÚks         r2   Ú_forward_saz MemoryAttentionLayer._forward_sam   s_   € à�zŠz˜#‰ŒˆØ$(Ô$8ÐB��yÑ Ð ¸dÐBˆˆAØ�~Š~˜a  dˆ~Ñ+Ô+ˆØ�D—M’M $Ñ'Ô'Ñ'ˆØˆ
r3   r   ÚmemoryÚposÚnum_k_exclude_ropec                ó  — i }|dk    r t          | j        t          ¦  «        sJ ‚d|i}|                      |¦  «        } | j        d| j        r||z   n|| j        r||z   n||dœ|¤Ž}||                      |¦  «        z   }|S )zXPerform cross-attention between target and memory tensors using RoPEAttention mechanism.r   rA   )r<   r=   r:   © )Ú
isinstancer"   r   r)   r   r   r,   )r0   r4   r?   r6   r@   rA   Úkwdsr;   s           r2   Ú_forward_caz MemoryAttentionLayer._forward_cau   s¹   € ð ˆØ Ò!Ð!Ý˜dÔ3µ]ÑCÔCÐCÐCÐCØ(Ð*<Ð=ˆDð �zŠz˜#‰ŒˆØ$ˆtÔ$ð 
Ø"&Ô"DÐNˆd�YÑÐÈ$Ø"Ô=ÐIˆf�s‰lˆlÀ6Øð
ð 
ð ð	
ð 
ˆð �D—M’M $Ñ'Ô'Ñ'ˆØˆ
r3   c           	     óZ  — |                       ||¦  «        }|                      |||||¦  «        }|                      |¦  «        }|                      |                      |                      |                      |¦  «        ¦  «        ¦  «        ¦  «        }||                      |¦  «        z   }|S )a¦  Process input tensors through self-attention, cross-attention, and feedforward network layers.

        Args:
            tgt (torch.Tensor): Target tensor for self-attention with shape (N, L, D).
            memory (torch.Tensor): Memory tensor for cross-attention with shape (N, S, D).
            pos (torch.Tensor | None): Positional encoding for memory tensor.
            query_pos (torch.Tensor | None): Positional encoding for target tensor.
            num_k_exclude_rope (int): Number of keys to exclude from rotary position embedding.

        Returns:
            (torch.Tensor): Processed tensor after attention and feedforward layers with shape (N, L, D).
        )r>   rF   r*   r&   r   r/   r$   r-   )r0   r4   r?   r@   r6   rA   r;   s          r2   ÚforwardzMemoryAttentionLayer.forwardŽ   s”   € ð( ×Ò˜s IÑ.Ô.ˆØ×Ò˜s F¨I°sÐ<NÑOÔOˆà�zŠz˜#‰ŒˆØ�|Š|˜DŸLšL¨¯ª¸¿ºÀdÑ9KÔ9KÑ)LÔ)LÑMÔMÑNÔNˆØ�D—M’M $Ñ'Ô'Ñ'ˆØˆ
r3   )r	   r
   r   FTFNN)r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   )r4   r5   r6   r7   r8   r5   )r   )r4   r5   r?   r5   r6   r7   r@   r7   rA   r   r8   r5   ©NNr   )r4   r5   r?   r5   r@   r7   r6   r7   rA   r   r8   r5   )	Ú__name__Ú
__module__Ú__qualname__Ú__doc__r    r>   rF   rH   Ú__classcell__©r1   s   @r2   r   r      sÅ   ø€ € € € € ð&ð &ðT Ø#ØØ %Ø+/Ø.3Ø&*Ø'+ð5Eð 5Eð 5Eð 5Eð 5Eð 5Eð 5Eðnð ð ð ð #$ðð ð ð ð ð: $(Ø)-Ø"#ðð ð ð ð ð ð ð ð r3   r   c                  ó8   ‡ — e Zd ZdZ	 ddˆ fd„Z	 	 	 ddd„Zˆ xZS )ÚMemoryAttentiona˜  Memory attention module for processing sequential data with self and cross-attention mechanisms.

    This class implements a multi-layer attention mechanism that combines self-attention and cross-attention for
    processing sequential data, particularly useful in transformer-like architectures.

    Attributes:
        d_model (int): The dimension of the model's hidden state.
        layers (nn.ModuleList): A list of MemoryAttentionLayer modules.
        num_layers (int): The number of attention layers.
        norm (nn.LayerNorm): Layer normalization applied to the output.
        pos_enc_at_input (bool): Whether to apply positional encoding at the input.
        batch_first (bool): Whether the input tensors are in batch-first format.

    Methods:
        forward: Processes input tensors through the attention layers.

    Examples:
        >>> d_model = 256
        >>> layer = MemoryAttentionLayer(d_model)
        >>> attention = MemoryAttention(d_model, pos_enc_at_input=True, layer=layer, num_layers=3)
        >>> curr = torch.randn(10, 32, d_model)  # (seq_len, batch_size, d_model)
        >>> memory = torch.randn(20, 32, d_model)  # (mem_len, batch_size, d_model)
        >>> curr_pos = torch.randn(10, 32, d_model)
        >>> memory_pos = torch.randn(20, 32, d_model)
        >>> output = attention(curr, memory, curr_pos, memory_pos)
        >>> print(output.shape)
        torch.Size([10, 32, 256])
    Tr   r   Úpos_enc_at_inputr   Úlayerú	nn.ModuleÚ
num_layersÚbatch_firstc                ó  •‡— t          ¦   «                              ¦   «          || _        t          j        ˆfd„t          |¦  «        D ¦   «         ¦  «        | _        || _        t          j        |¦  «        | _	        || _
        || _        dS )aÉ  Initialize MemoryAttention with specified layers and normalization for sequential data processing.

        This class implements a multi-layer attention mechanism that combines self-attention and cross-attention for
        processing sequential data, particularly useful in transformer-like architectures.

        Args:
            d_model (int): The dimension of the model's hidden state.
            pos_enc_at_input (bool): Whether to apply positional encoding at the input.
            layer (nn.Module): The attention layer to be used in the module.
            num_layers (int): The number of attention layers.
            batch_first (bool): Whether the input tensors are in batch-first format.
        c                ó8   •— g | ]}t          j        ‰¦  «        ‘ŒS rC   )ÚcopyÚdeepcopy)Ú.0Ú_rS   s     €r2   ú
<listcomp>z,MemoryAttention.__init__.<locals>.<listcomp>ß   s#   ø€ Ð$UÐ$UÐ$U¸a¥T¤]°5Ñ%9Ô%9Ð$UÐ$UÐ$Ur3   N)r   r    r   r   Ú
ModuleListÚrangeÚlayersrU   r'   ÚnormrR   rV   )r0   r   rR   rS   rU   rV   r1   s      `  €r2   r    zMemoryAttention.__init__É   s   øø€ õ( 	‰Œ×ÒÑÔÐØˆŒÝ”mÐ$UÐ$UÐ$UÐ$UÅ5ÈÑCTÔCTÐ$UÑ$UÔ$UÑVÔVˆŒØ$ˆŒÝ”L Ñ)Ô)ˆŒ	Ø 0ˆÔØ&ˆÔÐÐr3   Nr   Úcurrr5   r?   Úcurr_posr7   Ú
memory_posÚnum_obj_ptr_tokensr8   c           	     ó  — t          |t          ¦  «        rRt          |t          ¦  «        sJ ‚t          |¦  «        t          |¦  «        cxk    rdk    sn J ‚|d         |d         }}|j        d         |j        d         k    s
J d¦   «         ‚|}| j        r
|�|d|z  z   }| j        rX|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }| j        D ]/}i }t          |j        t          ¦  «        rd|i} |d||||dœ|¤Ž}Œ0|  
                    |¦  «        }	| j        r,|	                     dd¦  «        }	|                     dd¦  «        }|	S )	a^  Process inputs through attention layers, applying self and cross-attention with positional encoding.

        Args:
            curr (torch.Tensor): Self-attention input tensor, representing the current state.
            memory (torch.Tensor): Cross-attention input tensor, representing memory information.
            curr_pos (torch.Tensor | None): Positional encoding for self-attention inputs.
            memory_pos (torch.Tensor | None): Positional encoding for cross-attention inputs.
            num_obj_ptr_tokens (int): Number of object pointer tokens to exclude from rotary position embedding.

        Returns:
            (torch.Tensor): Processed output tensor after applying attention layers and normalization.

        Examples:
            >>> d_model = 256
            >>> layer = MemoryAttentionLayer(d_model)
            >>> attention = MemoryAttention(d_model, pos_enc_at_input=True, layer=layer, num_layers=3)
            >>> curr = torch.randn(10, 32, d_model)  # (seq_len, batch_size, d_model)
            >>> memory = torch.randn(20, 32, d_model)  # (mem_len, batch_size, d_model)
            >>> curr_pos = torch.randn(10, 32, d_model)
            >>> memory_pos = torch.randn(20, 32, d_model)
            >>> output = attention(curr, memory, curr_pos, memory_pos)
            >>> print(output.shape)
            torch.Size([10, 32, 256])
        r   r   z/Batch size must be the same for curr and memoryNr   rA   )r4   r?   r@   r6   rC   )rD   ÚlistÚlenÚshaperR   rV   Ú	transposer`   r"   r   ra   )
r0   rb   r?   rc   rd   re   ÚoutputrS   rE   Únormed_outputs
             r2   rH   zMemoryAttention.forwardå   sÏ  € õ@ �d�DÑ!Ô!ð 	2Ý˜h­Ñ-Ô-Ð-Ð-Ð-Ý�t‘9”9¥ H¡¤Ð2Ð2Ò2Ð2°Ò2Ð2Ð2Ð2Ð2Ð2Ø! !œW h¨q¤k�(ˆDàŒz˜!Œ} ¤¨Q¤Ò/Ð/Ð/Ð1bÑ/Ô/Ð/àˆØÔ ð 	- XÐ%9Ø˜c H™nÑ,ˆFàÔð 	4à×%Ò% a¨Ñ+Ô+ˆFØ×)Ò)¨!¨QÑ/Ô/ˆHØ×%Ò% a¨Ñ+Ô+ˆFØ#×-Ò-¨a°Ñ3Ô3ˆJà”[ð 	ð 	ˆEØˆDÝ˜%Ô0µ-Ñ@Ô@ð BØ,Ð.@ÐA�à�Uð ØØØØ"ð	ð ð
 ðð ˆFˆFð Ÿ	š	 &Ñ)Ô)ˆàÔð 	0à)×3Ò3°A°qÑ9Ô9ˆMØ×)Ò)¨!¨QÑ/Ô/ˆHàÐr3   )T)
r   r   rR   r   rS   rT   rU   r   rV   r   rI   )rb   r5   r?   r5   rc   r7   rd   r7   re   r   r8   r5   )rJ   rK   rL   rM   r    rH   rN   rO   s   @r2   rQ   rQ   «   s€   ø€ € € € € ðð ðF !ð'ð 'ð 'ð 'ð 'ð 'ð 'ð@ )-Ø*.Ø"#ðEð Eð Eð Eð Eð Eð Eð Eð Er3   rQ   )
Ú
__future__r   rY   Útorchr   Úblocksr   ÚModuler   rQ   rC   r3   r2   ú<module>rq      sÄ   ðð #Ð "Ð "Ð "Ð "Ð "à €€€à €€€Ø Ð Ð Ð Ð Ð à !Ð !Ð !Ð !Ð !Ð !ð[ð [ð [ð [ð [˜2œ9ñ [ô [ð [ð|ð ð ð ð �b”iñ ô ð ð ð r3   