§
    l¬¸jWV  ã                  ó"  — d Z ddlmZ ddlZddlmZ ddlmZ ddlZddl	m
c mZ ddlmZm
Z
 ddlmZ ddlmZ dd	lmZmZmZmZmZmZ dd
lmZ ddlmZ  G d„ de
j        ¦  «        Z G d„ de
j        ¦  «        Z G d„ de
j        ¦  «        Z dS )a@  
ViTDet backbone adapted from Detectron2.
This module implements Vision Transformer (ViT) backbone for object detection.

Rope embedding code adopted from:
1. https://github.com/meta-llama/codellama/blob/main/llama/model.py
2. https://github.com/naver-ai/rope-vit
3. https://github.com/lucidrains/rotary-embedding-torch
é    )ÚannotationsN)Úpartial)ÚCallable)ÚTensorÚnn)Ú
checkpoint)Ú
PatchEmbed)Úapply_rotary_encÚcompute_axial_cisÚconcat_rel_posÚget_abs_posÚwindow_partitionÚwindow_unpartition)Úcheck_requirementsé   )Ú
LayerScalec                  ó^   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 d d!ˆ fd„Zd"d#d„Zd$d%d„Zd&d„Zd'd„Zˆ xZ	S )(Ú	AttentionzIMulti-head Attention block with relative position embeddings and 2d-rope.é   TFNç     ˆÃ@ÚdimÚintÚ	num_headsÚqkv_biasÚboolÚuse_rel_posÚrel_pos_zero_initÚ
input_sizeútuple[int, int] | NoneÚ	cls_tokenÚuse_ropeÚ
rope_thetaÚfloatÚrope_pt_sizeÚrope_interpc                ó²  •— t          ¦   «                              ¦   «          || _        ||z  | _        | j        dz  | _        || _        t          j        ||dz  |¬¦  «        | _        t          j        ||¦  «        | _	        || _
        || _        || _        |	| _        |
| _        || _        |                      ||¦  «         |                      |¦  «         dS )a	  
        Args:
            dim (int): Number of input channels.
            num_heads (int): Number of attention heads.
            qkv_bias (bool): If True, add a learnable bias to query, key, value.
            use_rel_pos (bool): If True, add relative positional embeddings to the attention map.
            rel_pos_zero_init (bool): If True, zero initialize relative positional parameters.
            input_size (tuple[int, int] or None): Input resolution for calculating the relative positional parameter
                size or rope size.
            cls_token (bool): Whether a cls_token is present.
            use_rope (bool): Whether to use rope 2d (independent of use_rel_pos, as it can be used together).
            rope_theta (float): Control frequencies of rope.
            rope_pt_size (tuple[int, int] or None): Size of rope in previous stage of training, needed for interpolation
                or tiling.
            rope_interp (bool): Whether to interpolate (or extrapolate) rope to match input size.
        ç      à¿é   )ÚbiasN)ÚsuperÚ__init__r   Úhead_dimÚscaler    r   ÚLinearÚqkvÚprojr   r   r!   r"   r$   r%   Ú_setup_rel_posÚ_setup_rope_freqs)Úselfr   r   r   r   r   r   r    r!   r"   r$   r%   Ú	__class__s               €úr/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/ultralytics/models/sam/sam3/vitdet.pyr+   zAttention.__init__+   sÔ   ø€ õ< 	‰Œ×ÒÑÔÐØ"ˆŒØ˜yÑ(ˆŒØ”] DÑ(ˆŒ
Ø"ˆŒå”9˜S #¨¡'°Ð9Ñ9Ô9ˆŒÝ”I˜c 3Ñ'Ô'ˆŒ	ð 'ˆÔØ$ˆŒà ˆŒØ$ˆŒØ(ˆÔØ&ˆÔð 	×ÒÐ-¨zÑ:Ô:Ð:Ø×Ò˜zÑ*Ô*Ð*Ð*Ð*ó    ÚreturnÚNonec                ó¼  — | j         sd| _        d| _        dS |€J ‚| j        du s
J d¦   «         ‚t	          j        t          j        d|d         z  dz
  | j        ¦  «        ¦  «        | _        t	          j        t          j        d|d         z  dz
  | j        ¦  «        ¦  «        | _        |sLt          j	         
                    | j        d¬¦  «         t          j	         
                    | j        d¬¦  «         |\  }}t          j        |¦  «        dd…df         }t          j        |¦  «        ddd…f         }||z
  |dz
  z   }|                     ¦   «         | _        dS )	z%Setup relative positional embeddings.NFznot supportedé   r   r   ç{®Gáz”?©Ústd)r   Ú	rel_pos_hÚ	rel_pos_wr    r   Ú	ParameterÚtorchÚzerosr,   ÚinitÚtrunc_normal_ÚarangeÚlongÚrelative_coords)r3   r   r   ÚHÚWÚq_coordsÚk_coordsrG   s           r5   r1   zAttention._setup_rel_pos_   sO  € àÔð 	Ø!ˆDŒNØ!ˆDŒNØˆFàÐ%Ð%Ð%ØŒ~ Ð&Ð&Ð&¨Ñ&Ô&Ð&åœ¥e¤k°!°jÀ´mÑ2CÀaÑ2GÈÌÑ&WÔ&WÑXÔXˆŒÝœ¥e¤k°!°jÀ´mÑ2CÀaÑ2GÈÌÑ&WÔ&WÑXÔXˆŒà ð 	<ÝŒG×!Ò! $¤.°dÐ!Ñ;Ô;Ð;ÝŒG×!Ò! $¤.°dÐ!Ñ;Ô;Ð;ð ‰ˆˆ1Ý”< ‘?”? 1 1 1 d 7Ô+ˆÝ”< ‘?”? 4¨¨¨ 7Ô+ˆØ# hÑ.°1°q±5Ñ9ˆØ.×3Ò3Ñ5Ô5ˆÔÐÐr6   c                ó&  — | j         s	d| _        dS |€J ‚| j        €|| _        t          t          | j        | j        ¬¦  «        | _        d}| j        r| j        d         |d         z  }|                      |d         |d         |¬¦  «        }| j	        rwt          j        | j        dz  t          j        |j        ¬¦  «        }t          j        t          j        |¦  «        |¦  «        ddd…f         }t          j        ||gd¬	¦  «        }|| _        dS )
zSetup 2d-rope frequencies.N)r   Úthetaç      ð?r   r   )Úend_xÚend_yÚ	scale_posr:   )ÚdtypeÚdevice©r   )r!   Ú	freqs_cisr$   r   r   r,   r"   Úcompute_cisr%   r    rA   rB   Úfloat32rS   ÚpolarÚ	ones_likeÚcat)r3   r   rQ   rU   ÚtÚcls_freqs_ciss         r5   r2   zAttention._setup_rope_freqsw   s4  € àŒ}ð 	Ø!ˆDŒNØˆFàÐ%Ð%Ð%àÔÐ$Ø *ˆDÔõ #ÝØ”Ø”/ð
ñ 
ô 
ˆÔð ˆ	ØÔð 	=ØÔ)¨!Ô,¨z¸!¬}Ñ<ˆIà×$Ò$Ø˜Q”-Ø˜Q”-Øð %ñ 
ô 
ˆ	ð
 Œ>ð 	EÝ”Ø” Ñ"Ý”mØ Ô'ðñ ô ˆAõ
 "œK­¬¸Ñ(:Ô(:¸AÑ>Ô>¸tÀQÀQÀQ¸wÔGˆMÝœ	 =°)Ð"<À!ÐDÑDÔDˆIà"ˆŒˆˆr6   útuple[Tensor, Tensor]c                óˆ   — | j         s||fS | j        €J ‚t          ||| j                             |j        ¦  «        ¬¦  «        S )zApply 2d-rope to q and k.N)rU   )r!   rU   r
   ÚtorS   )r3   ÚqÚks      r5   Ú_apply_ropezAttention._apply_ropež   sJ   € àŒ}ð 	Ø�a�4ˆKàŒ~Ð)Ð)Ð)Ý  1°´×0AÒ0AÀ!Ä(Ñ0KÔ0KÐLÑLÔLÐLr6   Úxr   c           
     óœ  — | j         rdnd}|j        dk    r|j        \  }}}}|dk    sJ ‚||z  }d}n3|j        dk    sJ ‚|j        \  }}}d}t          j        ||z
  ¦  «        x}}|                      |¦  «                             ||d| j        d¦  «        }	|	                     ddddd¦  «         	                    d¦  «        \  }
}}|  
                    |
|¦  «        \  }
}| j        r t          |
                     dd¦  «        |                     dd¦  «        ||f|j        dd…         | j        | j        d| j        ¬¦  «        \  }
}|
                     || j        ||z  d¦  «        }
|                     || j        ||z  d¦  «        }t#          j        |
||¦  «        }|dk    rL|                     || j        ||d¦  «                             ddddd¦  «                             |||d¦  «        }nH|                     || j        |d¦  «                             dddd¦  «                             ||d¦  «        }|                      |¦  «        }|S )	z Forward pass of attention block.r   r   é   r(   éÿÿÿÿr:   T)ÚrescalerG   )r    ÚndimÚshapeÚmathÚsqrtr/   Úreshaper   ÚpermuteÚunbindrb   r   r   Úflattenr>   r?   rG   ÚFÚscaled_dot_product_attentionÚviewr0   )r3   rc   ÚsÚBrH   rI   Ú_ÚLrh   r/   r`   ra   Úvs                r5   ÚforwardzAttention.forward¦   sT  € à”Ð&ˆAˆA QˆØŒ6�QŠ;ˆ;Øœ‰JˆAˆq�!�QØ˜’6�6�6�6Ø�A‘ˆAØˆDˆDà”6˜Q’;�;�;�;Ø”g‰GˆAˆq�!ØˆDÝ”I˜a !™eÑ$Ô$Ð$ˆA�ð �hŠh�q‰kŒk×!Ò! ! Q¨¨4¬>¸2Ñ>Ô>ˆà—+’+˜a  A q¨!Ñ,Ô,×3Ò3°AÑ6Ô6‰ˆˆ1ˆað ×Ò  1Ñ%Ô%‰ˆˆ1ØÔð 	8Ý!Ø—	’	˜!˜Q‘”Ø—	’	˜!˜Q‘”Ø�A�Ø”˜˜!˜”Ø”Ø”ØØ $Ô 4ð	ñ 	ô 	‰DˆAˆqð —	’	˜!˜Tœ^¨Q°©U°BÑ7Ô7ˆAØ—	’	˜!˜Tœ^¨Q°©U°BÑ7Ô7ˆAåÔ*¨1¨a°Ñ3Ô3ˆà�1Š9ˆ9Ø—’�q˜$œ.¨!¨Q°Ñ3Ô3×;Ò;¸A¸qÀ!ÀQÈÑJÔJ×RÒRÐSTÐVWÐYZÐ\^Ñ_Ô_ˆAˆAà—’�q˜$œ.¨!¨RÑ0Ô0×8Ò8¸¸A¸qÀ!ÑDÔD×LÒLÈQÐPQÐSUÑVÔVˆAà�IŠI�a‰LŒLˆàˆr6   )
r   TFTNFFr   NF)r   r   r   r   r   r   r   r   r   r   r   r   r    r   r!   r   r"   r#   r$   r   r%   r   )TN)r   r   r   r   r7   r8   ©N)r   r   r7   r8   )r7   r]   ©rc   r   r7   r   )
Ú__name__Ú
__module__Ú__qualname__Ú__doc__r+   r1   r2   rb   rx   Ú__classcell__©r4   s   @r5   r   r   (   sÈ   ø€ € € € € ØSÐSð
 ØØ!Ø"&Ø-1ØØØ#Ø/3Ø!ð2+ð 2+ð 2+ð 2+ð 2+ð 2+ð 2+ðh6ð 6ð 6ð 6ð 6ð0%#ð %#ð %#ð %#ð %#ðNMð Mð Mð Mð.ð .ð .ð .ð .ð .ð .ð .r6   r   c                  ó`   ‡ — e Zd ZdZdddej        ej        ddddddddddfd$ˆ fd„Zd%d#„Zˆ xZ	S )&ÚBlockz4Transformer blocks with support of window attention.ç      @Tç        Fr   Nr   r   r   Ú	mlp_ratior#   r   r   Ú	drop_pathÚ
norm_layerúCallable[..., nn.Module]Ú	act_layerr   r   Úwindow_sizer   r   r!   r$   r%   r    ÚdropoutÚinit_valuesúfloat | Nonec                ó�  •— t          ¦   «                              ¦   «          t          d¦  «         ddlm}m}  ||¦  «        | _        t          |||||	|
dk    r|n|
|
f||||¬¦
  «
        | _        |rt          ||¬¦  «        nt          j        ¦   «         | _        |dk    r ||¦  «        nt          j        ¦   «         | _         ||¦  «        | _         ||t          ||z  ¦  «        ||df¬¦  «        | _        |rt          ||¬¦  «        nt          j        ¦   «         | _        t          j        |¦  «        | _        |
| _        dS )	a  
        Args:
            dim (int): Number of input channels.
            num_heads (int): Number of attention heads in each ViT block.
            mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
            qkv_bias (bool): If True, add a learnable bias to query, key, value.
            drop_path (float): Stochastic depth rate.
            norm_layer (Callable): Normalization layer constructor.
            act_layer (Callable): Activation layer constructor.
            use_rel_pos (bool): If True, add relative positional embeddings to the attention map.
            rel_pos_zero_init (bool): If True, zero initialize relative positional parameters.
            window_size (int): Window size for window attention blocks. If it equals 0, then not use window attention.
            input_size (tuple[int, int] | None): Input resolution for calculating the relative positional parameter
                size.
            use_rope (bool): Whether to use rope 2d (independent of use_rel_pos, as it can be used together).
            rope_pt_size (tuple[int, int] | None): Size of rope in previous stage of training, needed for interpolation
                or tiling.
            rope_interp (bool): Whether to interpolate (or extrapolate) rope to match target input size, expected to
                specify source size as rope_pt_size.
            cls_token (bool): Whether a cls_token is present.
            dropout (float): Dropout rate.
            init_values (float | None): Layer scale init, None for no layer scale.
        Útimmr   )ÚDropPathÚMlp)	r   r   r   r   r   r!   r$   r%   r    )rŒ   r„   )Úin_featuresÚhidden_featuresr‰   ÚdropN)r*   r+   r   Útimm.layersr�   r‘   Únorm1r   Úattnr   r   ÚIdentityÚls1r†   Únorm2r   ÚmlpÚls2ÚDropoutr‹   rŠ   )r3   r   r   r…   r   r†   r‡   r‰   r   r   rŠ   r   r!   r$   r%   r    r‹   rŒ   r�   r‘   r4   s                       €r5   r+   zBlock.__init__Ú   sv  ø€ õV 	‰Œ×ÒÑÔÐå˜6Ñ"Ô"Ð"Ø-Ð-Ð-Ð-Ð-Ð-Ð-Ð-à�Z ‘_”_ˆŒ
ÝØØØØ#Ø/Ø%0°AÒ%5Ð%5�z�z¸KÈÐ;UØØ%Ø#Øð
ñ 
ô 
ˆŒ	ð @KÐ]•:˜c¨{Ð;Ñ;Ô;Ð;ÕPRÔP[ÑP]ÔP]ˆŒØ09¸C²°˜˜ )Ñ,Ô,Ð,ÅRÄ[Á]Ä]ˆŒà�Z ‘_”_ˆŒ
Ø�3ØÝ  i¡Ñ0Ô0ØØ˜3�ð	
ñ 
ô 
ˆŒð @KÐ]•:˜c¨{Ð;Ñ;Ô;Ð;ÕPRÔP[ÑP]ÔP]ˆŒÝ”z 'Ñ*Ô*ˆŒØ&ˆÔÐÐr6   rc   r   r7   c                ód  — |}|                       |¦  «        }| j        dk    r2|j        d         |j        d         }}t          || j        ¦  «        \  }}|                      |                      |¦  «        ¦  «        }| j        dk    rt          || j        |||f¦  «        }||                      |                      |¦  «        ¦  «        z   }||                      |                      |  	                    |  
                    |                      |¦  «        ¦  «        ¦  «        ¦  «        ¦  «        z   }|S )z&Forward pass of the transformer block.r   r   r:   )r–   rŠ   ri   r   r™   r—   r   r‹   r†   rœ   r›   rš   )r3   rc   ÚshortcutrH   rI   Úpad_hws         r5   rx   zBlock.forward%  sý   € àˆØ�JŠJ�q‰MŒMˆàÔ˜aÒÐØ”7˜1”:˜qœw qœzˆqˆAÝ(¨¨DÔ,<Ñ=Ô=‰IˆAˆvà�HŠH�T—Y’Y˜q‘\”\Ñ"Ô"ˆàÔ˜aÒÐÝ" 1 dÔ&6¸ÀÀAÀÑGÔGˆAà�t—|’| D§N¢N°1Ñ$5Ô$5Ñ6Ô6Ñ6ˆØ�—’˜TŸ^š^¨D¯HªH°T·X²X¸d¿jºjÈ¹m¼mÑ5LÔ5LÑ,MÔ,MÑNÔNÑOÔOÑOˆàˆr6   )"r   r   r   r   r…   r#   r   r   r†   r#   r‡   rˆ   r‰   rˆ   r   r   r   r   rŠ   r   r   r   r!   r   r$   r   r%   r   r    r   r‹   r#   rŒ   r�   rz   )
r{   r|   r}   r~   r   Ú	LayerNormÚGELUr+   rx   r   r€   s   @r5   r‚   r‚   ×   s—   ø€ € € € € Ø>Ð>ð ØØØ/1¬|Ø.0¬gØ!Ø"&ØØ-1ØØ/3Ø!ØØØ$(ð%I'ð I'ð I'ð I'ð I'ð I'ð I'ðVð ð ð ð ð ð ð r6   r‚   c                  ó˜   ‡ — e Zd ZdZddddddddd	d
ej        ddddddddddddd	dddddddfdFˆ fd9„ZedGd>„¦   «         ZdHdB„Z	dIdJdE„Z
ˆ xZS )KÚViTz¼This module implements Vision Transformer (ViT) backbone in :paper:`vitdet`. "Exploring Plain Vision Transformer
    Backbones for Object Detection", https://arxiv.org/abs/2203.16527.
    i   é   r(   i   é   rƒ   Tr„   r¡   )r:   é   r   é   é   FNéà   Úimg_sizer   Ú
patch_sizeÚin_chansÚ	embed_dimÚdepthr   r…   r#   r   r   Údrop_path_rater‡   úCallable[..., nn.Module] | strr‰   rˆ   Úuse_abs_posÚtile_abs_posÚrel_pos_blocksútuple[int, ...] | boolr   rŠ   Úglobal_att_blocksútuple[int, ...]r!   r$   ú
int | NoneÚuse_interp_ropeÚpretrain_img_sizeÚpretrain_use_cls_tokenÚretain_cls_tokenr‹   Úreturn_interm_layersrŒ   r�   Úln_preÚln_postÚbias_patch_embedÚcompile_modeú
str | NoneÚuse_act_checkpointc            	     ó¬  •‡— t          ¦   «                              ¦   «          || _        ˆfd„t          |¦  «        D ¦   «         } t	          ‰¦  «        | _        dg|z  | _        t          |t          ¦  «        r|rdg|z  | _        n|D ]}!d| j        |!<   Œ|| _	        | j	        rx|sJ ‚t          | ¦  «        dk    s
J d¦   «         ‚t          | j        ¦  «        dk    s
J d¦   «         ‚|dz  }"t          j        |"t          j        dd|¦  «        z  ¦  «        | _        t          |
t"          ¦  «        r$t%          t'          t          |
¦  «        d	¬
¦  «        }
t)          ||f||f|||¬¦  «        | _        || _        || _        | j        r	| j        sJ ‚| j        rB||z  ||z  z  }#|r|#dz   n|#}$t          j        t          j        d|$|¦  «        ¦  «        | _        nd| _        d„ t          j        d|	|¦  «        D ¦   «         }%|| _        || _        t          j        ¦   «         | _        d}&t          |¦  «        D ]•}!t?          d"i d|“d|“d|“d|“d|%|!         “d|
“d|“d| j        |!         “d|“d|!| v r|nd“d||z  ||z  f“d|“d|€||fn||f“d|“d| j	        “d|“d|“Ž}'|!| vr|&dz  }&|| _         | j         !                    |'¦  «         Œ–|| _"        |r|gt          | j        ¦  «        z  n|g| _#        | j        �&t          j$         %                    | j        d¬ ¦  «         |r |
|¦  «        nt          j&        ¦   «         | _'        |r |
|¦  «        nt          j&        ¦   «         | _(        |  )                    | j*        ¦  «         |�Gt          j+        | j,        |d¬!¦  «        | _,        | j         r!| j-        rdt          j.        j/        _0        dS dS dS dS )#ak
  
        Args:
            img_size (int): Input image size. Only relevant for rel pos or rope.
            patch_size (int): Patch size.
            in_chans (int): Number of input image channels.
            embed_dim (int): Patch embedding dimension.
            depth (int): Depth of ViT.
            num_heads (int): Number of attention heads in each ViT block.
            mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
            qkv_bias (bool): If True, add a learnable bias to query, key, value.
            drop_path_rate (float): Stochastic depth rate.
            norm_layer (Callable or str): Normalization layer constructor or name.
            act_layer (Callable): Activation layer constructor.
            use_abs_pos (bool): If True, use absolute positional embeddings.
            tile_abs_pos (bool): If True, tile absolute positional embeddings instead of interpolation.
            rel_pos_blocks (tuple[int, ...] | bool): Blocks which have rel pos embeddings.
            rel_pos_zero_init (bool): If True, zero initialize relative positional parameters.
            window_size (int): Window size for window attention blocks.
            global_att_blocks (tuple[int, ...]): Indexes for blocks using global attention (other blocks use window
                attention).
            use_rope (bool): Whether to use rope 2d (independent of rel_pos_blocks, as it can be used together).
            rope_pt_size (int | None): Size of rope in previous stage of training, needed for interpolation or tiling.
            use_interp_rope (bool): Whether to interpolate (or extrapolate) rope to match target input size, expected to
                specify source size as rope_pt_size.
            pretrain_img_size (int): Input image size for pretraining models.
            pretrain_use_cls_token (bool): If True, pretraining models use class token.
            retain_cls_token (bool): Whether cls_token should be retained.
            dropout (float): Dropout rate. Applied in residual blocks of attn, mlp and inside the mlp.
            return_interm_layers (bool): Whether to return intermediate layers (all global attention blocks).
            init_values (float | None): Layer scale init, None for no layer scale.
            ln_pre (bool): If True, apply layer norm before transformer blocks.
            ln_post (bool): If True, apply layer norm after transformer blocks.
            bias_patch_embed (bool): If True, use bias in conv for patch embed.
            compile_mode (str | None): Mode to compile the forward, or None to disable.
            use_act_checkpoint (bool): If True, use activation checkpointing.
        c                ó   •— g | ]}|‰v¯|‘Œ	S © rÆ   )Ú.0Úir¶   s     €r5   ú
<listcomp>z ViT.__init__.<locals>.<listcomp>‡  s$   ø€ ÐVÐVÐV a¸1ÐDUÐ;UÐ;U Ð;UÐ;UÐ;Ur6   FTr   z&windowing not supported with cls tokenz$rel pos not supported with cls tokenr'   r   gñhãˆµøä>)Úeps)Úkernel_sizeÚstrider­   r®   r)   Nc                ó6   — g | ]}|                      ¦   «         ‘ŒS rÆ   )Úitem)rÇ   rc   s     r5   rÉ   z ViT.__init__.<locals>.<listcomp>´  s    € ÐJÐJÐJ˜Aˆq�vŠv‰xŒxÐJÐJÐJr6   r   r   r…   r   r†   r‡   r‰   r   r   rŠ   r   r!   r$   r%   r    r‹   rŒ   r;   r<   )ÚmodeÚ	fullgraphrÆ   )1r*   r+   r»   ÚrangeÚlistÚfull_attn_idsr´   Ú
isinstancer   r¼   ÚlenÚsumr   r@   rA   ÚrandnÚclass_embeddingÚstrr   Úgetattrr	   Úpatch_embedr³   r²   rB   Ú	pos_embedÚlinspacer¬   rŠ   Ú
ModuleListÚblocksr‚   rÃ   Úappendr½   Úchannel_listrC   rD   r˜   r¾   r¿   ÚapplyÚ_init_weightsÚcompilerx   ÚtrainingÚ_dynamoÚconfigÚoptimize_ddp))r3   r«   r¬   r­   r®   r¯   r   r…   r   r°   r‡   r‰   r²   r³   r´   r   rŠ   r¶   r!   r$   r¹   rº   r»   r¼   r‹   r½   rŒ   r¾   r¿   rÀ   rÁ   rÃ   Úwindow_block_indexesrÈ   r-   Únum_patchesÚnum_positionsÚdprÚ	cur_stageÚblockr4   s)                    `                      €r5   r+   zViT.__init__>  sí  øø€ õL 	‰Œ×ÒÑÔÐØ&<ˆÔ#àVÐVÐVÐV­5°©<¬<ÐVÑVÔVÐÝ!Ð"3Ñ4Ô4ˆÔØ$˜g¨™oˆÔÝ�n¥dÑ+Ô+ð 	.°ð 	.Ø#' &¨5¡.ˆDÔÐà#ð .ð .�Ø)-�Ô# AÑ&Ð&à 0ˆÔØÔ ð 	VØ)Ð)Ð)Ð)ÝÐ+Ñ,Ô,°Ò1Ð1Ð1Ð3[Ñ1Ô1Ð1å�tÔ*Ñ+Ô+¨qÒ0Ð0Ð0Ð2XÑ0Ô0Ð0à˜t‘OˆEÝ#%¤<°½¼ÀAÀqÈ)Ñ8TÔ8TÑ0TÑ#UÔ#UˆDÔ å�j¥#Ñ&Ô&ð 	DÝ ¥­¨ZÑ!8Ô!8¸dÐCÑCÔCˆJå%Ø# ZÐ0Ø 
Ð+ØØØ!ð
ñ 
ô 
ˆÔð )ˆÔØ&ˆÔØÔð 	$ØÔ#Ð#Ð#Ð#àÔð 	"à,°
Ñ:Ð?PÐT^Ñ?^Ñ_ˆKØ1GÐX˜[¨1™_˜_È[ˆMÝœ\­%¬+°a¸È	Ñ*RÔ*RÑSÔSˆDŒNˆNà!ˆDŒNð KÐJ¥¤°°>À5Ñ!IÔ!IÐJÑJÔJˆà$ˆŒØ&ˆÔÝ”m‘o”oˆŒØˆ	Ý�u‘”ð 	&ð 	&ˆAÝð ð ð Ø�Iðà#˜)ðð $˜)ðð "˜ð	ð
 ˜aœ&˜&ðð &˜:ðð $˜)ðð !Ô/°Ô2Ð2ðð #4Ð"3ðð ,-Ð0DÐ+DÐ+D˜K˜KÈ!øðð %¨
Ñ2°HÀ
Ñ4JÐKÐKðð "˜ðð =IÐ<P˜{¨KÐ8Ð8ÐWcÐeqÐVrøðð ,˜Oðð Ô/Ð/ðð   ˜ð!ð" (˜Kð#ˆEð( Ð,Ð,Ð,Ø˜Q‘�	à&8ˆDÔ#àŒK×Ò˜uÑ%Ô%Ð%Ð%à$8ˆÔ!ØEYÐj˜Y˜K­#¨dÔ.@Ñ*AÔ*AÑAÐAÐ`iÐ_jˆÔàŒ>Ð%ÝŒG×!Ò! $¤.°dÐ!Ñ;Ô;Ð;à/5ÐH�j�j Ñ+Ô+Ð+½2¼;¹=¼=ˆŒØ07ÐJ�z�z )Ñ,Ô,Ð,½R¼[¹]¼]ˆŒà�
Š
�4Ô%Ñ&Ô&Ð&àÐ#Ý œ=¨¬¸LÐTXÐYÑYÔYˆDŒLØÔ&ð :¨4¬=ð :Ø49•”Ô$Ô1Ð1Ð1ð $Ð#ð:ð :ð :ð :r6   Úmú	nn.Moduler7   r8   c                óê  — t          | t          j        ¦  «        rrt          j                             | j        d¬¦  «         t          | t          j        ¦  «        r.| j        �)t          j                             | j        d¦  «         dS dS dS t          | t          j        ¦  «        rLt          j                             | j        d¦  «         t          j                             | j        d¦  «         dS dS )zInitialize the weights.r;   r<   Nr   rN   )	rÔ   r   r.   rC   rD   Úweightr)   Ú	constant_r¡   )rï   s    r5   rã   zViT._init_weightsæ  sØ   € õ �a�œÑ#Ô#ð 	-ÝŒG×!Ò! !¤(°Ð!Ñ5Ô5Ð5Ý˜!�RœYÑ'Ô'ð -¨A¬FÐ,>Ý”×!Ò! !¤&¨!Ñ,Ô,Ð,Ð,Ð,ð-ð -Ð,>Ð,>å˜�2œ<Ñ(Ô(ð 	-ÝŒG×Ò˜aœf aÑ(Ô(Ð(ÝŒG×Ò˜aœh¨Ñ,Ô,Ð,Ð,Ð,ð	-ð 	-r6   rc   útorch.Tensorúlist[torch.Tensor]c                ó  — |                       |¦  «        }|j        d         |j        d         }}d}| j        r3t          j        | j        |                     dd¦  «        gd¬¦  «        }d}| j        �-|t          | j        | j	        ||f| j        | j
        ¬¦  «        z   }|                      |¦  «        }g }t          | j        ¦  «        D �]7\  }}| j        r| j        rt!          j        ||d¬¦  «        }n ||¦  «        }|| j        d	         k    s| j        rè|| j        v rß|| j        d	         k    r|                      |¦  «        }|dd…|d…f         }|j        d
k    r|                     dddd¦  «        }nr|j        dk    sJ ‚t-          j        |j        d         ¦  «        x}}|                     |j        d         |||j        d	         ¦  «                             dddd¦  «        }|                     |¦  «         �Œ9|S )z&Vit forward path and get feature maps.r   r:   r   rT   N)ÚtilingF)Úuse_reentrantrf   re   r(   )rÛ   ri   r¼   rA   rZ   rØ   ro   rÜ   r   r»   r³   r¾   Ú	enumeraterß   rÃ   rå   r   rÓ   r½   r¿   rh   rm   rj   rk   rl   rà   )	r3   rc   ÚhÚwrs   ÚoutputsrÈ   ÚblkÚfeatss	            r5   rx   zViT.forwardñ  s  € à×Ò˜QÑÔˆØŒw�qŒz˜1œ7 1œ:ˆ1ˆàˆØÔ ð 	õ ”	˜4Ô/°·²¸1¸a±´ÐAÀqÐIÑIÔIˆAØˆAàŒ>Ð%Ø•KØ”ØÔ+Ø�A�ØÔ%ØÔ(ðñ ô ñ ˆAð �KŠK˜‰NŒNˆàˆÝ ¤Ñ,Ô,ð 	&ñ 	&‰FˆAˆsØÔ&ð ¨4¬=ð ÝÔ)¨#¨qÀÐFÑFÔF��à�C˜‘F”F�Ø�TÔ'¨Ô+Ò+Ð+°Ô1JÐ+ÈqÐTXÔTfÐOfÐOfØ˜Ô*¨2Ô.Ò.Ð.ØŸš Q™œ�Aà˜!˜!˜!˜Q˜R˜R˜%œ�Ø”: ’?�?Ø!ŸMšM¨!¨Q°°1Ñ5Ô5�E�Eà œ:¨š?˜?˜?˜?Ý œI e¤k°!¤nÑ5Ô5Ð5�A˜Ø!ŸMšM¨%¬+°a¬.¸!¸QÀÄÈBÄÑPÔP×XÒXÐYZÐ\]Ð_`ÐbcÑdÔd�Eà—’˜uÑ%Ô%Ð%ùàˆr6   Úimgszúlist[int] | Nonec                ó&  — |�|nddg}| j         D ]€}|j        dk    rŒ|j                             |d         | j        z  |d         | j        z  f¬¦  «         |j                             |d         | j        z  |d         | j        z  f¬¦  «         Œ�dS )zCSetup rel pos embeddings and rope freqs for a new input image size.Nið  r   r   )r   )rß   rŠ   r—   r1   r¬   r2   )r3   rÿ   rî   s      r5   Ú	set_imgszzViT.set_imgsz  s·   € àÐ*��°°t°ˆØ”[ð 	pð 	pˆEØÔ  AÒ%Ð%ØØŒJ×%Ò%°%¸´(¸d¼oÑ2MÈuÐUVÌxÐ[_Ô[jÑOjÐ1kÐ%ÑlÔlÐlØŒJ×(Ò(°U¸1´XÀÄÑ5PÐRWÐXYÔRZÐ^bÔ^mÑRmÐ4nÐ(ÑoÔoÐoÐoð		pð 	pr6   )>r«   r   r¬   r   r­   r   r®   r   r¯   r   r   r   r…   r#   r   r   r°   r#   r‡   r±   r‰   rˆ   r²   r   r³   r   r´   rµ   r   r   rŠ   r   r¶   r·   r!   r   r$   r¸   r¹   r   rº   r   r»   r   r¼   r   r‹   r#   r½   r   rŒ   r�   r¾   r   r¿   r   rÀ   r   rÁ   rÂ   rÃ   r   )rï   rð   r7   r8   )rc   rô   r7   rõ   ry   )rÿ   r   )r{   r|   r}   r~   r   r¢   r+   Ústaticmethodrã   rx   r  r   r€   s   @r5   r¤   r¤   9  s  ø€ € € € € ðð ð ØØØØØØØØ #Ø5@Ø.0¬gØ Ø!Ø1>Ø"&ØØ-:ØØ#'Ø %Ø!$Ø'+Ø!%ØØ%*Ø$(ØØØ!%Ø#'Ø#'ðAf:ð f:ð f:ð f:ð f:ð f:ð f:ðP ð-ð -ð -ñ „\ð-ð+ð +ð +ð +ðZpð pð pð pð pð pð pð pð pr6   r¤   )!r~   Ú
__future__r   rj   Ú	functoolsr   Útypingr   rA   Útorch.nn.functionalr   Ú
functionalrp   r   Útorch.utilsr   Ú%ultralytics.models.sam.modules.blocksr	   Ú$ultralytics.models.sam.modules.utilsr
   r   r   r   r   r   Úultralytics.utils.checksr   Ú
model_miscr   ÚModuler   r‚   r¤   rÆ   r6   r5   ú<module>r     sÚ  ðð
ð ð #Ð "Ð "Ð "Ð "Ð "à €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ø "Ð "Ð "Ð "Ð "Ð "à <Ð <Ð <Ð <Ð <Ð <ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð 8Ð 7Ð 7Ð 7Ð 7Ð 7à "Ð "Ð "Ð "Ð "Ð "ðlð lð lð lð l�”	ñ lô lð lð^_ð _ð _ð _ð _ˆBŒIñ _ô _ð _ðDlpð lpð lpð lpð lpˆ"Œ)ñ lpô lpð lpð lpð lpr6   