§
    l¬¸jÛ/  ã                  óú   — d dl mZ d dlmZ d dlmZ d dlZd dlmZ d dlm	Z	 ddl
mZ  G d	„ d
ej        ¦  «        Z G d„ dej        ¦  «        Z	 ddd„Z G d„ dej        ¦  «        Z G d„ dej        ¦  «        ZdS )é    )Úannotations)ÚOrderedDict)ÚCallableN)Únn)Ú
checkpointé   )Ú
LayerScalec                  óZ   ‡ — e Zd ZdZddej        ej        fdˆ fd„Z	 ddd„Z	 ddd„Z	ˆ xZ
S )ÚResidualAttentionBlockz_Transformer block with multi-head attention, layer normalization, and MLP feed-forward network.ç      @NÚd_modelÚintÚn_headÚ	mlp_ratioÚfloatÚls_init_valueúfloat | NoneÚ	act_layerúCallable[[], nn.Module]Ú
norm_layerúCallable[[int], nn.Module]c                óF  •— t          ¦   «                              ¦   «          t          j        ||d¬¦  «        | _         ||¦  «        | _         ||¦  «        | _        |�t          ||¦  «        nt          j        ¦   «         | _	        |�t          ||¦  «        nt          j        ¦   «         | _
        t          ||z  ¦  «        }t          j        t          dt          j        ||¦  «        fd |¦   «         fdt          j        ||¦  «        fg¦  «        ¦  «        | _        dS )zSInitialize residual attention block with configurable dimensions and normalization.T)Úbatch_firstNÚc_fcÚgeluÚc_proj)ÚsuperÚ__init__r   ÚMultiheadAttentionÚattnÚln_1Úln_2r	   ÚIdentityÚls_1Úls_2r   Ú
Sequentialr   ÚLinearÚmlp)	Úselfr   r   r   r   r   r   Ú	mlp_widthÚ	__class__s	           €ú{/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/ultralytics/models/sam/sam3/text_encoder_ve.pyr   zResidualAttentionBlock.__init__   s  ø€ õ 	‰Œ×ÒÑÔÐåÔ)¨'°6ÀtÐLÑLÔLˆŒ	ð �J˜wÑ'Ô'ˆŒ	Ø�J˜wÑ'Ô'ˆŒ	à:GÐ:S•J˜w¨Ñ6Ô6Ð6ÕY[ÔYdÑYfÔYfˆŒ	Ø:GÐ:S•J˜w¨Ñ6Ô6Ð6ÕY[ÔYdÑYfÔYfˆŒ	õ ˜ )Ñ+Ñ,Ô,ˆ	Ý”=Ýà�RœY w°	Ñ:Ô:Ð;Ø˜Y˜Y™[œ[Ð)Ø�rœy¨°GÑ<Ô<Ð=ðñô ñ
ô 
ˆŒˆˆó    Úq_xútorch.TensorÚk_xÚv_xÚ	attn_maskÚreturnc                ó¼   — |�|n|}|�|n|}|�/|j         t          j        k    r|                     |j         ¦  «        }|                      |||d|¬¦  «        d         S )zOCompute multi-head attention with optional cross-attention support and masking.NF)Úneed_weightsr2   r   )ÚdtypeÚtorchÚboolÚtor    )r)   r.   r0   r1   r2   s        r,   Ú	attentionz ResidualAttentionBlock.attention5   sg   € ð �_ˆcˆc¨#ˆØ�_ˆcˆc¨#ˆØÐ  Y¤_½¼
Ò%BÐ%BØ!Ÿš S¤YÑ/Ô/ˆIà�yŠy˜˜c 3°UÀiˆyÑPÔPÐQRÔSÐSr-   c           	     óª  — t          | d¦  «        r|�|                      |¦  «        nd}t          | d¦  «        r|�|                      |¦  «        nd}||                      |                      |                      |¦  «        |||¬¦  «        ¦  «        z   }||                      |                      |                      |¦  «        ¦  «        ¦  «        z   }|S )z_Apply residual attention with layer normalization and MLP, supporting optional cross-attention.Úln_1_kvN)r.   r0   r1   r2   )Úhasattrr<   r$   r:   r!   r%   r(   r"   )r)   r.   r0   r1   r2   Úxs         r,   ÚforwardzResidualAttentionBlock.forward@   s¾   € õ $+¨4°Ñ#;Ô#;ÐYÀÀˆd�lŠl˜3ÑÔÐÐUYˆÝ#*¨4°Ñ#;Ô#;ÐYÀÀˆd�lŠl˜3ÑÔÐÐUYˆØ�$—)’)˜DŸNšN¨t¯yªy¸©~¬~À3ÈCÐ[d˜NÑeÔeÑfÔfÑfˆØ�—	’	˜$Ÿ(š( 4§9¢9¨Q¡<¤<Ñ0Ô0Ñ1Ô1Ñ1ˆØˆr-   )r   r   r   r   r   r   r   r   r   r   r   r   )NNN)
r.   r/   r0   r/   r1   r/   r2   r/   r3   r/   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚGELUÚ	LayerNormr   r:   r?   Ú__classcell__©r+   s   @r,   r   r      s    ø€ € € € € ØiÐið Ø&*Ø-/¬WØ13´ð
ð 
ð 
ð 
ð 
ð 
ð 
ðD ptð	Tð 	Tð 	Tð 	Tð 	Tð ptðð ð ð ð ð ð ð ð r-   r   c                  óP   ‡ — e Zd ZdZddej        ej        ddfdˆ fd„Zddd„Zˆ xZ	S )ÚTransformerzfStack of residual attention blocks forming a transformer encoder with optional gradient checkpointing.r   NFÚwidthr   ÚlayersÚheadsr   r   r   r   r   r   r   r   Úcompile_modeú
str | NoneÚuse_act_checkpointr8   c
                ót  •‡‡‡‡‡‡— t          ¦   «                              ¦   «          ‰| _        || _        |	| _        t          j        ˆˆˆˆˆˆfd„t          |¦  «        D ¦   «         ¦  «        | _        |�@t          j
        | j        |d¬¦  «        | _        | j        rdt          j        j        _        dS dS dS )z^Initialize transformer with configurable depth, width, and optional compilation/checkpointing.c           
     ó:   •— g | ]}t          ‰‰‰‰‰‰¬ ¦  «        ‘ŒS ))r   r   r   )r   )Ú.0Ú_r   rL   r   r   r   rJ   s     €€€€€€r,   ú
<listcomp>z(Transformer.__init__.<locals>.<listcomp>`   sL   ø€ ð 
ð 
ð 
ð õ 'ØØØØ"/Ø'Ø)ðñ ô ð
ð 
ð 
r-   NT)ÚmodeÚ	fullgraphF)r   r   rJ   rK   Úgrad_checkpointingr   Ú
ModuleListÚrangeÚ	resblocksr7   Úcompiler?   Ú_dynamoÚconfigÚoptimize_ddp)r)   rJ   rK   rL   r   r   r   r   rM   rO   r+   s    ` `````  €r,   r   zTransformer.__init__N   sä   øøøøøøø€ õ 	‰Œ×ÒÑÔÐØˆŒ
ØˆŒØ"4ˆÔÝœð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
õ ˜v™œð
ñ 
ô 
ñ
ô 
ˆŒð Ð#Ý œ=¨¬¸LÐTXÐYÑYÔYˆDŒLØÔ&ð :Ø49•”Ô$Ô1Ð1Ð1ð $Ð#ð:ð :r-   r>   r/   r2   r3   c           	     óØ   — t          | j        ¦  «        D ]T\  }}| j        r;t          j                             ¦   «         s| j        rt          ||dd|d¬¦  «        }ŒG |||¬¦  «        }ŒU|S )zbProcess input through all transformer blocks with optional gradient checkpointing during training.NF)Úuse_reentrant©r2   )Ú	enumeraterZ   rW   r7   ÚjitÚis_scriptingÚtrainingr   )r)   r>   r2   rS   Úrs        r,   r?   zTransformer.forwardr   s   € å˜dœnÑ-Ô-ð 	.ð 	.‰DˆAˆqØÔ&ð .­u¬y×/EÒ/EÑ/GÔ/Gð .ÈDÌMð .Ý˜q ! T¨4°È%ÐPÑPÔP��à�A�a 9Ð-Ñ-Ô-��Øˆr-   )rJ   r   rK   r   rL   r   r   r   r   r   r   r   r   r   rM   rN   rO   r8   ©N)r>   r/   r2   r/   r3   r/   )
r@   rA   rB   rC   r   rD   rE   r   r?   rF   rG   s   @r,   rI   rI   K   sz   ø€ € € € € ØpÐpð Ø&*Ø-/¬WØ13´Ø#'Ø#(ð":ð ":ð ":ð ":ð ":ð ":ð ":ðHð ð ð ð ð ð ð ð r-   rI   Úargmaxr>   r/   ÚtextÚ	pool_typeÚstrr3   ú!tuple[torch.Tensor, torch.Tensor]c                ó&  — |dk    r| dd…df         | dd…dd…f         }}nm|dk    r| dd…df         | dd…dd…f         }}nL|dk    rB|€J ‚| t          j        | j        d         ¦  «        |                     d¬¦  «        f         | }}n| x}}||fS )	z�Extract pooled representation and tokens from text embeddings using specified pooling strategy
    (first/last/argmax/none).
    ÚfirstNr   r   Úlastéÿÿÿÿrh   )Údim)r7   ÚarangeÚshaperh   )r>   ri   rj   ÚpooledÚtokenss        r,   Útext_global_poolrv   |   sÆ   € ð �GÒÐØ˜1˜1˜1˜a˜4œ ! A A A q r r E¤(�ˆˆØ	�fÒ	Ð	Ø˜1˜1˜1˜b˜5œ 1 Q Q Q¨¨¨ V¤9�ˆˆØ	�hÒ	Ð	àÐÐÐØ�5œ<¨¬°¬
Ñ3Ô3°T·[²[ÀR°[Ñ5HÔ5HÐHÔIÈ1�ˆˆàÐˆ�Ø�6ˆ>Ðr-   c                  ól   ‡ — e Zd ZdZddddddddd	d
d	ej        ej        d	ddd	fd+ˆ fd$„Zd,d'„Zd-d*„Z	ˆ xZ
S ).ÚTextTransformerzMText transformer encoder with causal masking and flexible pooling strategies.éM   é Á  i   é   é   r   NFÚnoneTÚcontext_lengthr   Ú
vocab_sizerJ   rL   rK   r   r   r   r   Ú
output_dimÚno_causal_maskr8   rj   rk   Ú	proj_biasr   r   r   Úoutput_tokensÚuse_ln_postrM   rN   rO   c                óä  •— t          ¦   «                              ¦   «          |
dv sJ ‚|| _        |x| _        | _        || _        || _        || _        || _        |
| _	        t          j        | j        |¦  «        | _        t          j        t          j        | j        |¦  «        ¦  «        | _        t#          |||||||||¬¦	  «	        | _        |r ||¦  «        nt          j        ¦   «         | _        |	rd| _        n*|                      d|                      ¦   «         d¬¦  «         |rt          j        ||¦  «        | _        dS t          j        t          j        ||¦  «        ¦  «        | _        dS )z[Initialize text transformer with embedding layers, transformer blocks, and pooling options.)rn   ro   rh   r}   )	rJ   rK   rL   r   r   r   r   rM   rO   Nr2   F)Ú
persistent)r   r   rƒ   Únum_posr~   r   rJ   r€   rL   rj   r   Ú	EmbeddingÚtoken_embeddingÚ	Parameterr7   ÚemptyÚpositional_embeddingrI   Útransformerr#   Úln_finalr2   Úregister_bufferÚbuild_causal_maskr'   Útext_projection)r)   r~   r   rJ   rL   rK   r   r   r€   r�   rj   r‚   r   r   rƒ   r„   rM   rO   r+   s                     €r,   r   zTextTransformer.__init__’   ss  ø€ õ* 	‰Œ×ÒÑÔÐØÐ?Ð?Ð?Ð?Ð?Ø*ˆÔØ-;Ð;ˆŒ�tÔ*Ø$ˆŒØˆŒ
Ø$ˆŒØˆŒ
Ø"ˆŒå!œ|¨D¬O¸UÑCÔCˆÔÝ$&¤Lµ´¸T¼\È5Ñ1QÔ1QÑ$RÔ$RˆÔ!Ý&ØØØØØ'ØØ!Ø%Ø1ð

ñ 

ô 

ˆÔð .9ÐK˜
˜
 5Ñ)Ô)Ð)½b¼k¹m¼mˆŒØð 	ZØ!ˆDŒNˆNà× Ò  ¨d×.DÒ.DÑ.FÔ.FÐSXÐ ÑYÔYÐYØð 	PÝ#%¤9¨U°JÑ#?Ô#?ˆDÔ Ð Ð å#%¤<µ´¸EÀ:Ñ0NÔ0NÑ#OÔ#OˆDÔ Ð Ð r-   r3   r/   c                ó²   — t          j        | j        | j        ¦  «        }|                     t	          d¦  «        ¦  «         |                     d¦  «         |S )zECreate a causal attention mask to prevent attention to future tokens.z-infr   )r7   r‹   r‡   Úfill_r   Útriu_)r)   Úmasks     r,   r�   z!TextTransformer.build_causal_maskÈ   sE   € õ Œ{˜4œ<¨¬Ñ6Ô6ˆØ�
Š
•5˜‘=”=Ñ!Ô!Ð!Ø�
Š
�1‰ŒˆØˆr-   ri   ú0torch.Tensor | tuple[torch.Tensor, torch.Tensor]c                óÊ  — |j         d         }|                      |¦  «        }| j        }|�|d|…d|…f         }|| j        d|…         z   }|                      ||¬¦  «        }|                      |¦  «        }t          ||| j        ¬¦  «        \  }}| j        �?t          | j        t          j        ¦  «        r|                      |¦  «        }n
|| j        z  }| j        r||fS |S )zcForward pass through the text transformer, returning pooled output and optionally token embeddings.r   Nra   )rj   )rs   r‰   r2   rŒ   r�   rŽ   rv   rj   r‘   Ú
isinstancer   r'   rƒ   )r)   ri   Úseq_lenr>   r2   rt   ru   s          r,   r?   zTextTransformer.forwardÑ   sù   € à”*˜Q”-ˆØ× Ò  Ñ&Ô&ˆà”Nˆ	ØÐ Ø! ( 7 (¨H¨W¨HÐ"4Ô5ˆIà�Ô)¨(¨7¨(Ô3Ñ3ˆØ×Ò˜Q¨)ÐÑ4Ô4ˆà�MŠM˜!ÑÔˆÝ)¨!¨T¸T¼^ÐLÑLÔL‰ˆ�ØÔÐ+Ý˜$Ô.µ´	Ñ:Ô:ð 7Ø×-Ò-¨fÑ5Ô5��à $Ô"6Ñ6�ØÔð 	"Ø˜6�>Ð!Øˆr-   )"r~   r   r   r   rJ   r   rL   r   rK   r   r   r   r   r   r€   r   r�   r8   rj   rk   r‚   r8   r   r   r   r   rƒ   r8   r„   r8   rM   rN   rO   r8   )r3   r/   )ri   r/   r3   r–   )r@   rA   rB   rC   r   rD   rE   r   r�   r?   rF   rG   s   @r,   rx   rx   �   s±   ø€ € € € € ØWÐWð !ØØØØØØ&*ØØ$ØØØ œgØ!œ|Ø#Ø Ø#'Ø#(ð%4Pð 4Pð 4Pð 4Pð 4Pð 4Pð 4Pðlð ð ð ðð ð ð ð ð ð ð r-   rx   c                  óB   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 dd ˆ fd„Z	 d!d"d„Zˆ xZS )#ÚVETextEncoderz_Text encoder for Vision Encoder (VE) models, combining a text transformer and a linear resizer.é   é   é   é    rz   TNr   r   Ú	tokenizerr   rJ   rL   rK   r~   r   r„   r8   rM   rN   rO   c                óþ   •— t          ¦   «                              ¦   «          || _        || _        || _        t          | j        ||||d||	|
¬¦	  «	        | _        t          j        | j        j	        |¦  «        | _
        dS )zdInitialize VE text encoder with a text transformer and a linear resizer to match decoder dimensions.T)	r~   r   rJ   rL   rK   rƒ   r„   rM   rO   N)r   r   r~   r„   r    rx   Úencoderr   r'   rJ   Úresizer)r)   r   r    rJ   rL   rK   r~   r   r„   rM   rO   r+   s              €r,   r   zVETextEncoder.__init__ì   s„   ø€ õ 	‰Œ×ÒÑÔÐØ,ˆÔØ&ˆÔØ"ˆŒå&ØÔ.Ø!ØØØàØ#Ø%Ø1ð
ñ 
ô 
ˆŒõ ”y ¤Ô!3°WÑ=Ô=ˆŒˆˆr-   ri   ú3list[str] | tuple[torch.Tensor, torch.Tensor, dict]Úinput_boxesúlist | Noner3   ú/tuple[torch.Tensor, torch.Tensor, torch.Tensor]c                óÒ  — t          |d         t          ¦  «        �r|�t          |¦  «        dk    s
J d¦   «         ‚|                      || j        ¬¦  «                             | j        j        j        ¦  «        }|dk     	                    ¦   «         }| j
                             |¦  «        }|  
                    |¦  «        \  }}|j        d         |j        d         k    sJ ‚|                     d¦  «        }|                     dd¦  «        }|                      |¦  «        }n-|\  }}}|d         }|�t          |¦  «        dk    s
J d¦   «         ‚|||                     dd¦  «        fS )zeEncode text input, either raw strings or pre-encoded tensors, and resize to match decoder dimensions.r   Nznot supported)r~   r   Úinputs_embedsz3Can't replace boxes in text if it's already encoded)r˜   rk   Úlenr    r~   r9   r£   ÚweightÚdevicer8   r¢   r‰   rs   ÚneÚ	transpose)	r)   ri   r¥   Ú	tokenizedÚtext_attention_maskr©   rS   Útext_memoryÚtext_memory_resizeds	            r,   r?   zVETextEncoder.forward  s�  € õ �d˜1”g�sÑ#Ô#ñ 	wàÐ&­#¨kÑ*:Ô*:¸aÒ*?Ð*?Ð*?ÀÑ*?Ô*?Ð?ð Ÿš t¸DÔ<O˜ÑPÔP×SÒSØ”Ô#Ô*ñô ˆIð $-°¢>×"7Ò"7Ñ"9Ô"9Ðð !œL×8Ò8¸ÑCÔCˆMØ!Ÿ\š\¨)Ñ4Ô4‰NˆAˆ{àÔ$ QÔ'¨=Ô+>¸qÔ+AÒAÐAÐAÐAà"5×"8Ò"8¸Ñ";Ô";Ðà%×/Ò/°°1Ñ5Ô5ˆKà"&§,¢,¨{Ñ";Ô";ÐÐð CGÑ?ÐÐ!4°iØ% oÔ6ˆMØÐ&­#¨kÑ*:Ô*:¸aÒ*?Ð*?Ð*?ÐAvÑ*?Ô*?Ð?ð  ØØ×#Ò# A qÑ)Ô)ð
ð 	
r-   )rœ   r�   rž   rŸ   rz   TNT)r   r   r    r   rJ   r   rL   r   rK   r   r~   r   r   r   r„   r8   rM   rN   rO   r8   rg   )ri   r¤   r¥   r¦   r3   r§   )r@   rA   rB   rC   r   r?   rF   rG   s   @r,   r›   r›   é   sƒ   ø€ € € € € ØiÐið ØØØ ØØ Ø#'Ø#'ð>ð >ð >ð >ð >ð >ð >ðD eið$
ð $
ð $
ð $
ð $
ð $
ð $
ð $
ð $
r-   r›   )Nrh   )r>   r/   ri   r/   rj   rk   r3   rl   )Ú
__future__r   Úcollectionsr   Útypingr   r7   r   Útorch.utils.checkpointr   Ú
model_miscr	   ÚModuler   rI   rv   rx   r›   © r-   r,   ú<module>rº      sk  ðð
 #Ð "Ð "Ð "Ð "Ð "à #Ð #Ð #Ð #Ð #Ð #Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð Ø -Ð -Ð -Ð -Ð -Ð -à "Ð "Ð "Ð "Ð "Ð "ð7ð 7ð 7ð 7ð 7˜RœYñ 7ô 7ð 7ðt.ð .ð .ð .ð .�"”)ñ .ô .ð .ðd BJðð ð ð ð ð&Wð Wð Wð Wð W�b”iñ Wô Wð WðtH
ð H
ð H
ð H
ð H
�B”Iñ H
ô H
ð H
ð H
ð H
r-   