§
    l¬¸j$:  ã                  ó°   — d dl mZ d dlZd dlZd dlmZmZ d dlmZ  G d„ dej        ¦  «        Z	 G d„ dej        ¦  «        Z
 G d	„ d
ej        ¦  «        ZdS )é    )ÚannotationsN)ÚTensorÚnn)ÚMLPBlockc                  ó<   ‡ — e Zd ZdZej        dfdˆ fd„Zdd„Zˆ xZS )ÚTwoWayTransformera  A Two-Way Transformer module for simultaneous attention to image and query points.

    This class implements a specialized transformer decoder that attends to an input image using queries with supplied
    positional embeddings. It's useful for tasks like object detection, image segmentation, and point cloud processing.

    Attributes:
        depth (int): Number of layers in the transformer.
        embedding_dim (int): Channel dimension for input embeddings.
        num_heads (int): Number of heads for multihead attention.
        mlp_dim (int): Internal channel dimension for the MLP block.
        layers (nn.ModuleList): List of TwoWayAttentionBlock layers composing the transformer.
        final_attn_token_to_image (Attention): Final attention layer from queries to image.
        norm_final_attn (nn.LayerNorm): Layer normalization applied to final queries.

    Methods:
        forward: Process image and point embeddings through the transformer.

    Examples:
        >>> transformer = TwoWayTransformer(depth=6, embedding_dim=256, num_heads=8, mlp_dim=2048)
        >>> image_embedding = torch.randn(1, 256, 32, 32)
        >>> image_pe = torch.randn(1, 256, 32, 32)
        >>> point_embedding = torch.randn(1, 100, 256)
        >>> output_queries, output_image = transformer(image_embedding, image_pe, point_embedding)
        >>> print(output_queries.shape, output_image.shape)
    é   ÚdepthÚintÚembedding_dimÚ	num_headsÚmlp_dimÚ
activationútype[nn.Module]Úattention_downsample_rateÚreturnÚNonec                ó–  •— t          ¦   «                              ¦   «          || _        || _        || _        || _        t          j        ¦   «         | _        t          |¦  «        D ]3}| j         
                    t          ||||||dk    ¬¦  «        ¦  «         Œ4t          |||¬¦  «        | _        t          j        |¦  «        | _        dS )ab  Initialize a Two-Way Transformer for simultaneous attention to image and query points.

        Args:
            depth (int): Number of layers in the transformer.
            embedding_dim (int): Channel dimension for input embeddings.
            num_heads (int): Number of heads for multihead attention. Must divide embedding_dim.
            mlp_dim (int): Internal channel dimension for the MLP block.
            activation (type[nn.Module], optional): Activation function to use in the MLP block.
            attention_downsample_rate (int, optional): Downsampling rate for attention mechanism.
        r   )r   r   r   r   r   Úskip_first_layer_pe©Údownsample_rateN)ÚsuperÚ__init__r
   r   r   r   r   Ú
ModuleListÚlayersÚrangeÚappendÚTwoWayAttentionBlockÚ	AttentionÚfinal_attn_token_to_imageÚ	LayerNormÚnorm_final_attn)	Úselfr
   r   r   r   r   r   ÚiÚ	__class__s	           €úz/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/ultralytics/models/sam/modules/transformer.pyr   zTwoWayTransformer.__init__(   sÓ   ø€ õ& 	‰Œ×ÒÑÔÐØˆŒ
Ø*ˆÔØ"ˆŒØˆŒÝ”m‘o”oˆŒå�u‘”ð 
	ð 
	ˆAØŒK×ÒÝ$Ø"/Ø'Ø#Ø)Ø.GØ)*¨aªðñ ô ñ	ô 	ð 	ð 	õ *3°=À)Ð]vÐ)wÑ)wÔ)wˆÔ&Ý!œ|¨MÑ:Ô:ˆÔÐÐó    Úimage_embeddingútorch.TensorÚimage_peÚpoint_embeddingú!tuple[torch.Tensor, torch.Tensor]c                ój  — |                      d¦  «                             ddd¦  «        }|                      d¦  «                             ddd¦  «        }|}|}| j        D ]} |||||¬¦  «        \  }}Œ||z   }||z   }|                      |||¬¦  «        }	||	z   }|                      |¦  «        }||fS )aƒ  Process image and point embeddings through the Two-Way Transformer.

        Args:
            image_embedding (torch.Tensor): Image to attend to, with shape (B, embedding_dim, H, W).
            image_pe (torch.Tensor): Positional encoding to add to the image, with same shape as image_embedding.
            point_embedding (torch.Tensor): Embedding to add to query points, with shape (B, N_points, embedding_dim).

        Returns:
            queries (torch.Tensor): Processed point embeddings with shape (B, N_points, embedding_dim).
            keys (torch.Tensor): Processed image embeddings with shape (B, H*W, embedding_dim).
        r	   r   é   )ÚqueriesÚkeysÚquery_peÚkey_pe©ÚqÚkÚv)ÚflattenÚpermuter   r    r"   )
r#   r(   r*   r+   r/   r0   Úlayerr4   r5   Úattn_outs
             r&   ÚforwardzTwoWayTransformer.forwardQ   së   € ð$ *×1Ò1°!Ñ4Ô4×<Ò<¸QÀÀ1ÑEÔEˆØ×#Ò# AÑ&Ô&×.Ò.¨q°!°QÑ7Ô7ˆð "ˆØˆð ”[ð 	ð 	ˆEØ!˜EØØØ(Øð	ñ ô ‰MˆG�T�Tð �oÑ%ˆØ�8‰OˆØ×1Ò1°A¸¸dÐ1ÑCÔCˆØ˜HÑ$ˆØ×&Ò& wÑ/Ô/ˆà˜ˆ}Ðr'   )r
   r   r   r   r   r   r   r   r   r   r   r   r   r   )r(   r)   r*   r)   r+   r)   r   r,   ©	Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚReLUr   r;   Ú__classcell__©r%   s   @r&   r   r      sl   ø€ € € € € ðð ð@ ')¤gØ)*ð';ð ';ð ';ð ';ð ';ð ';ð ';ðR)ð )ð )ð )ð )ð )ð )ð )r'   r   c                  ó@   ‡ — e Zd ZdZdej        ddfdˆ fd„Zdd„Zˆ xZS )r   aB  A two-way attention block for simultaneous attention to image and query points.

    This class implements a specialized transformer block with four main layers: self-attention on sparse inputs,
    cross-attention of sparse inputs to dense inputs, MLP block on sparse inputs, and cross-attention of dense inputs to
    sparse inputs.

    Attributes:
        self_attn (Attention): Self-attention layer for queries.
        norm1 (nn.LayerNorm): Layer normalization after self-attention.
        cross_attn_token_to_image (Attention): Cross-attention layer from queries to keys.
        norm2 (nn.LayerNorm): Layer normalization after token-to-image attention.
        mlp (MLPBlock): MLP block for transforming query embeddings.
        norm3 (nn.LayerNorm): Layer normalization after MLP block.
        norm4 (nn.LayerNorm): Layer normalization after image-to-token attention.
        cross_attn_image_to_token (Attention): Cross-attention layer from keys to queries.
        skip_first_layer_pe (bool): Whether to skip positional encoding in the first layer.

    Methods:
        forward: Apply self-attention and cross-attention to queries and keys.

    Examples:
        >>> embedding_dim, num_heads = 256, 8
        >>> block = TwoWayAttentionBlock(embedding_dim, num_heads)
        >>> queries = torch.randn(1, 100, embedding_dim)
        >>> keys = torch.randn(1, 1000, embedding_dim)
        >>> query_pe = torch.randn(1, 100, embedding_dim)
        >>> key_pe = torch.randn(1, 1000, embedding_dim)
        >>> processed_queries, processed_keys = block(queries, keys, query_pe, key_pe)
    i   r	   Fr   r   r   r   r   r   r   r   Úboolr   r   c                óÐ  •— t          ¦   «                              ¦   «          t          ||¦  «        | _        t	          j        |¦  «        | _        t          |||¬¦  «        | _        t	          j        |¦  «        | _        t          |||¦  «        | _
        t	          j        |¦  «        | _        t	          j        |¦  «        | _        t          |||¬¦  «        | _        || _        dS )a‡  Initialize a TwoWayAttentionBlock for simultaneous attention to image and query points.

        This block implements a specialized transformer layer with four main components: self-attention on sparse
        inputs, cross-attention of sparse inputs to dense inputs, MLP block on sparse inputs, and cross-attention of
        dense inputs to sparse inputs.

        Args:
            embedding_dim (int): Channel dimension of the embeddings.
            num_heads (int): Number of attention heads in the attention layers.
            mlp_dim (int, optional): Hidden dimension of the MLP block.
            activation (type[nn.Module], optional): Activation function for the MLP block.
            attention_downsample_rate (int, optional): Downsampling rate for the attention mechanism.
            skip_first_layer_pe (bool, optional): Whether to skip positional encoding in the first layer.
        r   N)r   r   r   Ú	self_attnr   r!   Únorm1Úcross_attn_token_to_imageÚnorm2r   ÚmlpÚnorm3Únorm4Úcross_attn_image_to_tokenr   )r#   r   r   r   r   r   r   r%   s          €r&   r   zTwoWayAttentionBlock.__init__œ   sÁ   ø€ õ. 	‰Œ×ÒÑÔÐÝ" =°)Ñ<Ô<ˆŒÝ”\ -Ñ0Ô0ˆŒ
å)2°=À)Ð]vÐ)wÑ)wÔ)wˆÔ&Ý”\ -Ñ0Ô0ˆŒ
å˜M¨7°JÑ?Ô?ˆŒÝ”\ -Ñ0Ô0ˆŒ
å”\ -Ñ0Ô0ˆŒ
Ý)2°=À)Ð]vÐ)wÑ)wÔ)wˆÔ&à#6ˆÔ Ð Ð r'   r/   r)   r0   r1   r2   r,   c                ó  — | j         r|                      |||¬¦  «        }n"||z   }|                      |||¬¦  «        }||z   }|                      |¦  «        }||z   }||z   }|                      |||¬¦  «        }||z   }|                      |¦  «        }|                      |¦  «        }||z   }|                      |¦  «        }||z   }||z   }|                      |||¬¦  «        }||z   }|                      |¦  «        }||fS )a¹  Apply two-way attention to process query and key embeddings in a transformer block.

        Args:
            queries (torch.Tensor): Query embeddings with shape (B, N_queries, embedding_dim).
            keys (torch.Tensor): Key embeddings with shape (B, N_keys, embedding_dim).
            query_pe (torch.Tensor): Positional encodings for queries with same shape as queries.
            key_pe (torch.Tensor): Positional encodings for keys with same shape as keys.

        Returns:
            queries (torch.Tensor): Processed query embeddings with shape (B, N_queries, embedding_dim).
            keys (torch.Tensor): Processed key embeddings with shape (B, N_keys, embedding_dim).
        r3   )	r   rG   rH   rI   rJ   rK   rL   rN   rM   )	r#   r/   r0   r1   r2   r4   r:   r5   Úmlp_outs	            r&   r;   zTwoWayAttentionBlock.forwardÂ   s/  € ð  Ô#ð 	)Ø—n’n w°'¸W�nÑEÔEˆGˆGà˜(Ñ"ˆAØ—~’~¨¨Q°'�~Ñ:Ô:ˆHØ Ñ(ˆGØ—*’*˜WÑ%Ô%ˆð �hÑˆØ�6‰MˆØ×1Ò1°A¸¸dÐ1ÑCÔCˆØ˜HÑ$ˆØ—*’*˜WÑ%Ô%ˆð —(’(˜7Ñ#Ô#ˆØ˜GÑ#ˆØ—*’*˜WÑ%Ô%ˆð �hÑˆØ�6‰MˆØ×1Ò1°A¸¸gÐ1ÑFÔFˆØ�h‰ˆØ�zŠz˜$ÑÔˆà˜ˆ}Ðr'   )r   r   r   r   r   r   r   r   r   r   r   rE   r   r   )
r/   r)   r0   r)   r1   r)   r2   r)   r   r,   r<   rC   s   @r&   r   r   }   sr   ø€ € € € € ðð ðD Ø&(¤gØ)*Ø$)ð$7ð $7ð $7ð $7ð $7ð $7ð $7ðL+ð +ð +ð +ð +ð +ð +ð +r'   r   c                  ób   ‡ — e Zd ZdZ	 	 ddˆ fd„Zedd„¦   «         Zedd„¦   «         Zdd„Zˆ xZ	S )r   aÜ  An attention layer with downscaling capability for embedding size after projection.

    This class implements a multi-head attention mechanism with the option to downsample the internal dimension of
    queries, keys, and values.

    Attributes:
        embedding_dim (int): Dimensionality of input embeddings.
        kv_in_dim (int): Dimensionality of key and value inputs.
        internal_dim (int): Internal dimension after downsampling.
        num_heads (int): Number of attention heads.
        q_proj (nn.Linear): Linear projection for queries.
        k_proj (nn.Linear): Linear projection for keys.
        v_proj (nn.Linear): Linear projection for values.
        out_proj (nn.Linear): Linear projection for output.

    Methods:
        _separate_heads: Separate input tensor into attention heads.
        _recombine_heads: Recombine separated attention heads.
        forward: Compute attention output for given query, key, and value tensors.

    Examples:
        >>> attn = Attention(embedding_dim=256, num_heads=8, downsample_rate=2)
        >>> q = torch.randn(1, 100, 256)
        >>> k = v = torch.randn(1, 50, 256)
        >>> output = attn(q, k, v)
        >>> print(output.shape)
        torch.Size([1, 100, 256])
    r.   Nr   r   r   r   Ú	kv_in_dimú
int | Noner   r   c                óÊ  •— t          ¦   «                              ¦   «          || _        |�|n|| _        ||z  | _        || _        | j        |z  dk    s
J d¦   «         ‚t          j        || j        ¦  «        | _        t          j        | j        | j        ¦  «        | _	        t          j        | j        | j        ¦  «        | _
        t          j        | j        |¦  «        | _        dS )a6  Initialize the Attention module with specified dimensions and settings.

        Args:
            embedding_dim (int): Dimensionality of input embeddings.
            num_heads (int): Number of attention heads.
            downsample_rate (int, optional): Factor by which internal dimensions are downsampled.
            kv_in_dim (int | None, optional): Dimensionality of key and value inputs. If None, uses embedding_dim.

        Raises:
            AssertionError: If num_heads does not evenly divide the internal dim (embedding_dim / downsample_rate).
        Nr   z$num_heads must divide embedding_dim.)r   r   r   rR   Úinternal_dimr   r   ÚLinearÚq_projÚk_projÚv_projÚout_proj)r#   r   r   r   rR   r%   s        €r&   r   zAttention.__init__  sÉ   ø€ õ$ 	‰Œ×ÒÑÔÐØ*ˆÔØ&/Ð&;˜˜ÀˆŒØ)¨_Ñ<ˆÔØ"ˆŒØÔ  9Ñ,°Ò1Ð1Ð1Ð3YÑ1Ô1Ð1å”i ¨tÔ/@ÑAÔAˆŒÝ”i ¤°Ô0AÑBÔBˆŒÝ”i ¤°Ô0AÑBÔBˆŒÝœ	 $Ô"3°]ÑCÔCˆŒˆˆr'   Úxr)   c                óz   — | j         \  }}}|                      |||||z  ¦  «        } |                      dd¦  «        S )zGSeparate the input tensor into the specified number of attention heads.r.   r	   )ÚshapeÚreshapeÚ	transpose)r[   r   ÚbÚnÚcs        r&   Ú_separate_headszAttention._separate_heads,  s@   € ð ”'‰ˆˆ1ˆaØ�IŠI�a˜˜I q¨I¡~Ñ6Ô6ˆØ�{Š{˜1˜aÑ Ô Ð r'   r   c                óz   — | j         \  }}}}|                      dd¦  «        } |                      ||||z  ¦  «        S )z9Recombine separated attention heads into a single tensor.r.   r	   )r]   r_   r^   )r[   r`   Ún_headsÚn_tokensÚ
c_per_heads        r&   Ú_recombine_headszAttention._recombine_heads3  sB   € ð ,-¬7Ñ(ˆˆ7�H˜jØ�KŠK˜˜1ÑÔˆØ�yŠy˜˜H g°
Ñ&:Ñ;Ô;Ð;r'   r4   r5   r6   c                ó(  — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      || j        ¦  «        }|                      || j        ¦  «        }|                      || j        ¦  «        }|j        \  }}}}||                     dddd¦  «        z  }|t          j        |¦  «        z  }t          j
        |d¬¦  «        }||z  }|                      |¦  «        }|                      |¦  «        S )aÂ  Apply multi-head attention to query, key, and value tensors with optional downsampling.

        Args:
            q (torch.Tensor): Query tensor with shape (B, N_q, embedding_dim).
            k (torch.Tensor): Key tensor with shape (B, N_k, kv_in_dim).
            v (torch.Tensor): Value tensor with shape (B, N_k, kv_in_dim).

        Returns:
            (torch.Tensor): Output tensor after attention with shape (B, N_q, embedding_dim).
        r   r.   é   r	   éÿÿÿÿ)Údim)rW   rX   rY   rc   r   r]   r8   ÚmathÚsqrtÚtorchÚsoftmaxrh   rZ   )r#   r4   r5   r6   Ú_rg   ÚattnÚouts           r&   r;   zAttention.forward:  sþ   € ð �KŠK˜‰NŒNˆØ�KŠK˜‰NŒNˆØ�KŠK˜‰NŒNˆð × Ò   D¤NÑ3Ô3ˆØ× Ò   D¤NÑ3Ô3ˆØ× Ò   D¤NÑ3Ô3ˆð  œgÑˆˆ1ˆa�Ø�1—9’9˜Q  1 aÑ(Ô(Ñ(ˆØ•d”i 
Ñ+Ô+Ñ+ˆÝŒ}˜T rÐ*Ñ*Ô*ˆð �Q‰hˆØ×#Ò# CÑ(Ô(ˆØ�}Š}˜SÑ!Ô!Ð!r'   )r.   N)
r   r   r   r   r   r   rR   rS   r   r   )r[   r)   r   r   r   r)   )r[   r   r   r   )r4   r)   r5   r)   r6   r)   r   r)   )
r=   r>   r?   r@   r   Ústaticmethodrc   rh   r;   rB   rC   s   @r&   r   r   ð   s°   ø€ € € € € ðð ðB  !Ø $ðDð Dð Dð Dð Dð Dð Dð< ð!ð !ð !ñ „\ð!ð ð<ð <ð <ñ „\ð<ð"ð "ð "ð "ð "ð "ð "ð "r'   r   )Ú
__future__r   rm   ro   r   r   Úultralytics.nn.modulesr   ÚModuler   r   r   © r'   r&   ú<module>ry      sÿ   ðð #Ð "Ð "Ð "Ð "Ð "à €€€à €€€Ø Ð Ð Ð Ð Ð Ð Ð à +Ð +Ð +Ð +Ð +Ð +ðmð mð mð mð m˜œ	ñ mô mð mð`pð pð pð pð p˜2œ9ñ pô pð pðfh"ð h"ð h"ð h"ð h"�”	ñ h"ô h"ð h"ð h"ð h"r'   