§
    l¬¸jâð  ã                  ó  — d dl mZ d dlZd dlmc mZ d dlmZ d dlmZ d dl	m
Z
 d dlmZ ddlmZmZ dd	lmZmZ dd
lmZmZ ddlmZmZ dZ G d„ dej        ¦  «        Z G d„ dej        j        ¦  «        Z G d„ de¦  «        ZdS )é    )ÚannotationsN)Únn)Útrunc_normal_)ÚMLP)ÚLOGGERé   )ÚSAM2TwoWayTransformerÚTwoWayTransformer)ÚMaskDecoderÚSAM2MaskDecoder)ÚImageEncoderViTÚPromptEncoder)Úget_1d_sine_peÚselect_closest_cond_framesg      �Àc                  ó@   ‡ — e Zd ZU dZdZded<   	 	 ddˆ fd„Zd„ Zˆ xZS )ÚSAMModelaÆ  Segment Anything Model (SAM) for object segmentation tasks.

    This class combines image encoders, prompt encoders, and mask decoders to predict object masks from images and input
    prompts.

    Attributes:
        mask_threshold (float): Threshold value for mask prediction.
        image_encoder (ImageEncoderViT): Backbone for encoding images into embeddings.
        prompt_encoder (PromptEncoder): Encoder for various types of input prompts.
        mask_decoder (MaskDecoder): Predicts object masks from image and prompt embeddings.
        pixel_mean (torch.Tensor): Mean values for normalizing pixels in the input image.
        pixel_std (torch.Tensor): Standard deviation values for normalizing pixels in the input image.

    Methods:
        set_imgsz: Set image size to make model compatible with different image sizes.

    Examples:
        >>> image_encoder = ImageEncoderViT(...)
        >>> prompt_encoder = PromptEncoder(...)
        >>> mask_decoder = MaskDecoder(...)
        >>> sam_model = SAMModel(image_encoder, prompt_encoder, mask_decoder)
        >>> # Further usage depends on SAMPredictor class

    Notes:
        All forward() operations are implemented in the SAMPredictor class.
    ç        ÚfloatÚmask_threshold©g33333ë^@gR¸…ë]@gR¸…ëáY@©gÃõ(\�2M@g�Âõ(\�L@g     °L@Úimage_encoderr   Úprompt_encoderr   Úmask_decoderr   Ú
pixel_meanúlist[float]Ú	pixel_stdÚreturnÚNonec                ój  •— t          ¦   «                              ¦   «          || _        || _        || _        |                      dt          j        |¦  «                             ddd¦  «        d¦  «         |                      dt          j        |¦  «                             ddd¦  «        d¦  «         dS )a¥  Initialize the SAMModel class to predict object masks from an image and input prompts.

        Args:
            image_encoder (ImageEncoderViT): The backbone used to encode the image into image embeddings.
            prompt_encoder (PromptEncoder): Encodes various types of input prompts.
            mask_decoder (MaskDecoder): Predicts masks from the image embeddings and encoded prompts.
            pixel_mean (list[float]): Mean values for normalizing pixels in the input image.
            pixel_std (list[float]): Standard deviation values for normalizing pixels in the input image.

        Notes:
            All forward() operations moved to SAMPredictor.
        r   éÿÿÿÿr   Fr   N)	ÚsuperÚ__init__r   r   r   Úregister_bufferÚtorchÚTensorÚview)Úselfr   r   r   r   r   Ú	__class__s         €úr/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/ultralytics/models/sam/modules/sam.pyr#   zSAMModel.__init__7   s¤   ø€ õ( 	‰Œ×ÒÑÔÐØ*ˆÔØ,ˆÔØ(ˆÔØ×Ò˜\­5¬<¸
Ñ+CÔ+C×+HÒ+HÈÈQÐPQÑ+RÔ+RÐTYÑZÔZÐZØ×Ò˜[­%¬,°yÑ*AÔ*A×*FÒ*FÀrÈ1ÈaÑ*PÔ*PÐRWÑXÔXÐXÐXÐXó    c                óÌ   — t          | j        d¦  «        r| j                             |¦  «         || j        _        d„ |D ¦   «         | j        _        |d         | j        _        dS )úCSet image size to make model compatible with different image sizes.Ú	set_imgszc                ó   — g | ]}|d z  ‘ŒS )é   © ©Ú.0Úxs     r*   ú
<listcomp>z&SAMModel.set_imgsz.<locals>.<listcomp>W   s   € Ð3KÐ3KÐ3KÀ°A¸±GÐ3KÐ3KÐ3Kr+   r   N)Úhasattrr   r.   r   Úinput_image_sizeÚimage_embedding_sizeÚimg_size©r(   Úimgszs     r*   r.   zSAMModel.set_imgszR   sg   € å�4Ô% {Ñ3Ô3ð 	0ØÔ×(Ò(¨Ñ/Ô/Ð/Ø/4ˆÔÔ,Ø3KÐ3KÀUÐ3KÑ3KÔ3KˆÔÔ0Ø&+¨A¤hˆÔÔ#Ð#Ð#r+   )r   r   )r   r   r   r   r   r   r   r   r   r   r   r   )	Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Ú__annotations__r#   r.   Ú__classcell__©r)   s   @r*   r   r      s   ø€ € € € € € ðð ð6  €NÐÐÐÑð #<Ø!8ðYð Yð Yð Yð Yð Yð Yð6/ð /ð /ð /ð /ð /ð /r+   r   c                  ó  ‡ — e Zd ZU dZdZded<   	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d*d+ˆ fd„Zed„ ¦   «         Zd„ Z	d„ Z
	 	 	 	 d,d„Zd-d„Zd.d„Zd/d „Z	 d0d!„Zd"„ Zd#„ Zd$„ Z	 	 	 d1d%„Zd&„ Zed'„ ¦   «         Zd0d(„Zd)„ Zˆ xZS )2Ú	SAM2Modelaò  SAM2Model class for Segment Anything Model 2 with memory-based video object segmentation capabilities.

    This class extends the functionality of SAM to handle video sequences, incorporating memory mechanisms for temporal
    consistency and efficient tracking of objects across frames.

    Attributes:
        mask_threshold (float): Threshold value for mask prediction.
        image_encoder (ImageEncoderViT): Visual encoder for extracting image features.
        memory_attention (nn.Module): Module for attending to memory features.
        memory_encoder (nn.Module): Encoder for generating memory representations.
        num_maskmem (int): Number of accessible memory frames.
        image_size (int): Size of input images.
        backbone_stride (int): Stride of the backbone network output.
        sam_prompt_embed_dim (int): Dimension of SAM prompt embeddings.
        sam_image_embedding_size (int): Size of SAM image embeddings.
        sam_prompt_encoder (PromptEncoder): Encoder for processing input prompts.
        sam_mask_decoder (SAM2MaskDecoder): Decoder for generating object masks.
        obj_ptr_proj (nn.Module): Projection layer for object pointers.
        obj_ptr_tpos_proj (nn.Module): Projection for temporal positional encoding in object pointers.
        hidden_dim (int): Hidden dimension of the model.
        mem_dim (int): Memory dimension for encoding features.
        use_high_res_features_in_sam (bool): Whether to use high-resolution feature maps in the SAM mask decoder.
        use_obj_ptrs_in_encoder (bool): Whether to cross-attend to object pointers from other frames in the encoder.
        max_obj_ptrs_in_encoder (int): Maximum number of object pointers from other frames in encoder cross-attention.
        add_tpos_enc_to_obj_ptrs (bool): Whether to add temporal positional encoding to object pointers.
        proj_tpos_enc_in_obj_ptrs (bool): Whether to add an extra linear projection layer for temporal positional
            encoding in object pointers.
        use_signed_tpos_enc_to_obj_ptrs (bool): Whether to use signed distance in temporal positional encoding.
        only_obj_ptrs_in_the_past_for_eval (bool): Whether to only attend to object pointers in the past during
            evaluation.
        pred_obj_scores (bool): Whether to predict if there is an object in the frame.
        pred_obj_scores_mlp (bool): Whether to use an MLP to predict object scores.
        fixed_no_obj_ptr (bool): Whether to have a fixed no-object pointer when there is no object present.
        soft_no_obj_ptr (bool): Whether to mix in no-object pointer softly for easier recovery and error mitigation.
        use_mlp_for_obj_ptr_proj (bool): Whether to use MLP for object pointer projection.
        no_obj_embed_spatial (torch.Tensor | None): No-object embedding for spatial frames.
        max_cond_frames_in_attn (int): Maximum number of conditioning frames to participate in memory attention.
        directly_add_no_mem_embed (bool): Whether to directly add no-memory embedding to image feature on the first
            frame.
        multimask_output_in_sam (bool): Whether to output multiple masks for the first click on initial conditioning
            frames.
        multimask_min_pt_num (int): Minimum number of clicks to use multimask output in SAM.
        multimask_max_pt_num (int): Maximum number of clicks to use multimask output in SAM.
        multimask_output_for_tracking (bool): Whether to use multimask output for tracking.
        use_multimask_token_for_obj_ptr (bool): Whether to use multimask tokens for object pointers.
        iou_prediction_use_sigmoid (bool): Whether to use sigmoid to restrict IoU prediction to [0-1].
        memory_temporal_stride_for_eval (int): Memory bank's temporal stride during evaluation.
        non_overlap_masks_for_mem_enc (bool): Whether to apply non-overlapping constraints on object masks in memory
            encoder during evaluation.
        sigmoid_scale_for_mem_enc (float): Scale factor for mask sigmoid probability.
        sigmoid_bias_for_mem_enc (float): Bias factor for mask sigmoid probability.
        binarize_mask_from_pts_for_mem_enc (bool): Whether to binarize sigmoid mask logits on interacted frames with
            clicks during evaluation.
        use_mask_input_as_output_without_sam (bool): Whether to directly output the input mask without using SAM prompt
            encoder and mask decoder on frames with mask input.

    Methods:
        forward_image: Process image batch through encoder to extract multi-level features.
        track_step: Perform a single tracking step, updating object masks and memory features.
        set_binarize: Set binarize for VideoPredictor.
        set_imgsz: Set image size to make model compatible with different image sizes.

    Examples:
        >>> model = SAM2Model(image_encoder, memory_attention, memory_encoder)
        >>> image_batch = torch.rand(1, 3, 512, 512)
        >>> features = model.forward_image(image_batch)
        >>> track_results = model.track_step(0, True, features, None, None, None, {})
    r   r   r   é   é   r0   ç      ð?Fr!   r   TNÚuse_multimask_token_for_obj_ptrÚboolÚpred_obj_scoresÚpred_obj_scores_mlpÚfixed_no_obj_ptrÚsoft_no_obj_ptrÚuse_mlp_for_obj_ptr_projÚno_obj_embed_spatialÚcompile_image_encoderc$                ó¶  •— t          ¦   «                              ¦   «          || _        || _        |rdnd| _        || _        || _        |r(t          j         	                    dddd¬¦  «        | _
        || _        |r|sJ ‚|| _        || _        || _        || _        |j        | _        || _        | j        | _        t)          | j        d¦  «        r;t)          | j        j        d¦  «        r!| j        j        j        j        d         | _        || _        t          j                             t          j        |dd| j        ¦  «        ¦  «        | _        t9          | j        d¬	¦  «         t          j                             t          j        dd| j        ¦  «        ¦  «        | _        t          j                             t          j        dd| j        ¦  «        ¦  «        | _        t9          | j        d¬	¦  «         t9          | j        d¬	¦  «         || _        || _         || _!        |	| _"        || _#        || _$        |
| _%        || _&        || _'        || _(        || _)        || _*        || _+        || _,        || _-        |"| _.        || _/        || _0        || _1        || _2        | j1        r| j/        sJ ‚| j        sJ ‚| j/        rY| j        rRt          j                             t          j        d| j        ¦  «        ¦  «        | _3        t9          | j3        d¬	¦  «         | | _4        d
| _5        |!rRt          j                             t          j        d| j        ¦  «        ¦  «        | _5        t9          | j5        d¬	¦  «         |  6                    ¦   «          || _7        d| _8        |#rBts          j:        d¦  «         t          j;        | j        j<        ddd¬¦  «        | j        _<        d
S d
S )aÕ  Initialize the SAM2Model for video object segmentation with memory-based tracking.

        Args:
            image_encoder (nn.Module): Visual encoder for extracting image features.
            memory_attention (nn.Module): Module for attending to memory features.
            memory_encoder (nn.Module): Encoder for generating memory representations.
            num_maskmem (int): Number of accessible memory frames.
            image_size (int): Size of input images.
            backbone_stride (int): Stride of the image backbone output.
            sigmoid_scale_for_mem_enc (float): Scale factor for mask sigmoid probability.
            sigmoid_bias_for_mem_enc (float): Bias factor for mask sigmoid probability.
            binarize_mask_from_pts_for_mem_enc (bool): Whether to binarize sigmoid mask logits on interacted frames with
                clicks during evaluation.
            use_mask_input_as_output_without_sam (bool): Whether to directly output the input mask without using SAM
                prompt encoder and mask decoder on frames with mask input.
            max_cond_frames_in_attn (int): Maximum number of conditioning frames to participate in memory attention.
            directly_add_no_mem_embed (bool): Whether to directly add no-memory embedding to image feature on the first
                frame.
            use_high_res_features_in_sam (bool): Whether to use high-resolution feature maps in the SAM mask decoder.
            multimask_output_in_sam (bool): Whether to output multiple masks for the first click on initial conditioning
                frames.
            multimask_min_pt_num (int): Minimum number of clicks to use multimask output in SAM.
            multimask_max_pt_num (int): Maximum number of clicks to use multimask output in SAM.
            multimask_output_for_tracking (bool): Whether to use multimask output for tracking.
            use_multimask_token_for_obj_ptr (bool): Whether to use multimask tokens for object pointers.
            iou_prediction_use_sigmoid (bool): Whether to use sigmoid to restrict IoU prediction to [0-1].
            memory_temporal_stride_for_eval (int): Memory bank's temporal stride during evaluation.
            non_overlap_masks_for_mem_enc (bool): Whether to apply non-overlapping constraints on object masks in memory
                encoder during evaluation.
            use_obj_ptrs_in_encoder (bool): Whether to cross-attend to object pointers from other frames in the encoder.
            max_obj_ptrs_in_encoder (int): Maximum number of object pointers from other frames in encoder
                cross-attention.
            add_tpos_enc_to_obj_ptrs (bool): Whether to add temporal positional encoding to object pointers in the
                encoder.
            proj_tpos_enc_in_obj_ptrs (bool): Whether to add an extra linear projection layer for temporal positional
                encoding in object pointers.
            use_signed_tpos_enc_to_obj_ptrs (bool): Whether to use signed distance in the temporal positional encoding
                in the object pointers.
            only_obj_ptrs_in_the_past_for_eval (bool): Whether to only attend to object pointers in the past during
                evaluation.
            pred_obj_scores (bool): Whether to predict if there is an object in the frame.
            pred_obj_scores_mlp (bool): Whether to use an MLP to predict object scores.
            fixed_no_obj_ptr (bool): Whether to have a fixed no-object pointer when there is no object present.
            soft_no_obj_ptr (bool): Whether to mix in no-object pointer softly for easier recovery and error mitigation.
            use_mlp_for_obj_ptr_proj (bool): Whether to use MLP for object pointer projection.
            no_obj_embed_spatial (bool): Whether to add no-object embedding to spatial frames.
            sam_mask_decoder_extra_args (dict | None): Extra arguments for constructing the SAM mask decoder.
            compile_image_encoder (bool): Whether to compile the image encoder for faster inference.
        é   r   é   )Úkernel_sizeÚstrideÚout_projÚweightr   g{®Gáz”?)ÚstdNTzFImage encoder compilation is enabled. First forward pass will be slow.zmax-autotuneF)ÚmodeÚ	fullgraphÚdynamic)=r"   r#   r   Úuse_high_res_features_in_samÚnum_feature_levelsÚuse_obj_ptrs_in_encoderÚmax_obj_ptrs_in_encoderr%   r   ÚConv2dÚmask_downsampleÚadd_tpos_enc_to_obj_ptrsÚproj_tpos_enc_in_obj_ptrsÚuse_signed_tpos_enc_to_obj_ptrsÚ"only_obj_ptrs_in_the_past_for_evalÚmemory_attentionÚd_modelÚ
hidden_dimÚmemory_encoderÚmem_dimr6   rV   rW   ÚshapeÚnum_maskmemÚ	ParameterÚzerosÚmaskmem_tpos_encr   Úno_mem_embedÚno_mem_pos_encÚdirectly_add_no_mem_embedÚsigmoid_scale_for_mem_encÚsigmoid_bias_for_mem_encÚ"binarize_mask_from_pts_for_mem_encÚnon_overlap_masks_for_mem_encÚmemory_temporal_stride_for_evalÚ$use_mask_input_as_output_without_samÚmultimask_output_in_samÚmultimask_min_pt_numÚmultimask_max_pt_numÚmultimask_output_for_trackingrH   Úiou_prediction_use_sigmoidÚ
image_sizeÚbackbone_strideÚsam_mask_decoder_extra_argsrJ   rK   rL   rM   Ú
no_obj_ptrrN   rO   Ú_build_sam_headsÚmax_cond_frames_in_attnÚ!add_all_frames_to_correct_as_condr   ÚinfoÚcompileÚforward©%r(   r   rf   ri   rl   r~   r   rs   rt   ru   rx   rƒ   rr   r\   ry   rz   r{   r|   rH   r}   rw   rv   r^   r_   rb   rc   rd   re   rJ   rK   rL   rM   rN   rO   r€   rP   r)   s%                                       €r*   r#   zSAM2Model.__init__£   sÎ  ø€ õn 	‰Œ×ÒÑÔÐð +ˆÔà,HˆÔ)Ø'CÐ"J ! !ÈˆÔØ'>ˆÔ$Ø'>ˆÔ$Ø"ð 	Rõ $)¤8§?¢?°1°aÀQÈq ?Ñ#QÔ#QˆDÔ Ø(@ˆÔ%Ø$ð 	,Ø+Ð+Ð+Ð+Ø)BˆÔ&Ø/NˆÔ,Ø2TˆÔ/ð !1ˆÔØ*Ô2ˆŒð -ˆÔØ”ˆŒÝ�4Ô&¨
Ñ3Ô3ð 	H½ÀÔ@SÔ@\Ð^fÑ8gÔ8gð 	HàÔ.Ô7Ô>ÔDÀQÔGˆDŒLØ&ˆÔå %¤× 2Ò 2µ5´;¸{ÈAÈqÐRVÔR^Ñ3_Ô3_Ñ `Ô `ˆÔÝ�dÔ+°Ð6Ñ6Ô6Ð6å!œH×.Ò.­u¬{¸1¸aÀÄÑ/QÔ/QÑRÔRˆÔÝ#œh×0Ò0µ´¸QÀÀ4Ä?Ñ1SÔ1SÑTÔTˆÔÝ�dÔ'¨TÐ2Ñ2Ô2Ð2Ý�dÔ)¨tÐ4Ñ4Ô4Ð4Ø)BˆÔ&ð *CˆÔ&Ø(@ˆÔ%Ø2TˆÔ/Ø-JˆÔ*Ø/NˆÔ,ð 5YˆÔ1Ø'>ˆÔ$Ø$8ˆÔ!Ø$8ˆÔ!Ø-JˆÔ*Ø/NˆÔ,Ø*DˆÔ'ð %ˆŒØ.ˆÔØ+FˆÔ(Ø.ˆÔØ#6ˆÔ Ø 0ˆÔØ.ˆÔØÔ ð 	0ØÔ'Ð'Ð'Ð'ØÔ/Ð/Ð/Ð/ØÔð 	5 DÔ$@ð 	5Ý#œh×0Ò0µ´¸QÀÄÑ1PÔ1PÑQÔQˆDŒOÝ˜$œ/¨tÐ4Ñ4Ô4Ð4Ø(@ˆÔ%Ø$(ˆÔ!Øð 	?Ý(-¬×(:Ò(:½5¼;ÀqÈ$Ì,Ñ;WÔ;WÑ(XÔ(XˆDÔ%Ý˜$Ô3¸Ð>Ñ>Ô>Ð>à×ÒÑÔÐØ'>ˆÔ$Ø15ˆÔ.ð !ð 	åŒKÐ`ÑaÔaÐaÝ).¬ØÔ"Ô*Ø#ØØð	*ñ *ô *ˆDÔÔ&Ð&Ð&ð	ð 	r+   c                óN   — t          |                      ¦   «         ¦  «        j        S )z=Return the device on which the model's parameters are stored.)ÚnextÚ
parametersÚdevice©r(   s    r*   rŒ   zSAM2Model.deviceY  s   € õ �D—O’OÑ%Ô%Ñ&Ô&Ô-Ð-r+   c                ó    — t          d¦  «        ‚)zWProcess image and prompt inputs to generate object masks and scores in video sequences.z„Please use the corresponding methods in SAM2VideoPredictor for inference.See notebooks/video_predictor_example.ipynb for an example.)ÚNotImplementedError)r(   ÚargsÚkwargss      r*   r‡   zSAM2Model.forward^  s   € å!ðJñ
ô 
ð 	
r+   c                ó"  — | j         | _        | j        | j        z  | _        t          | j        | j        | j        f| j        | j        fd¬¦  «        | _        t          ddt          d| j        dd¬¦  «        | j        dd| j	        | j
        | j        | j        | j        d	œ
| j        pi ¤Ž| _        | j        r]t"          j                             | j         | j         ¦  «        | _        | j        r&t-          | j         | j         | j         d¦  «        | _        n#t"          j                             ¦   «         | _        | j        r1t"          j                             | j         | j        ¦  «        | _        d
S t"          j                             ¦   «         | _        d
S )zMBuild SAM-style prompt encoder and mask decoder for image segmentation tasks.r0   )Ú	embed_dimr8   r7   Úmask_in_chansrR   é   é   é   ©ÚdepthÚembedding_dimÚmlp_dimÚ	num_headsé   ©
Únum_multimask_outputsÚtransformerÚtransformer_dimÚiou_head_depthÚiou_head_hidden_dimÚuse_high_res_featuresr}   rJ   rK   rH   Nr1   )rh   Úsam_prompt_embed_dimr~   r   Úsam_image_embedding_sizer   Úsam_prompt_encoderr   r	   r\   r}   rJ   rK   rH   r€   Úsam_mask_decoderr^   r%   r   ÚLinearÚobj_ptr_projrN   r   ÚIdentityrc   rj   Úobj_ptr_tpos_projr�   s    r*   r‚   zSAM2Model._build_sam_headse  s“  € à$(¤OˆÔ!Ø(,¬¸4Ô;OÑ(OˆÔ%õ #0ØÔ/àÔ-ØÔ-ð"ð #œo¨t¬Ð?Øð#
ñ #
ô #
ˆÔõ !0ð !
Ø"#Ý-ØØ"Ô7ØØð	ñ ô ð !Ô5ØØ #Ø"&Ô"CØ'+Ô'FØ Ô0Ø $Ô 8Ø,0Ô,Pð!
ð !
ð  Ô/Ð5°2ð!!
ð !
ˆÔð$ Ô'ð 	4å %¤§¢°´ÀÄÑ QÔ QˆDÔØÔ,ð ^Ý$'¨¬¸¼È$Ì/Ð[\Ñ$]Ô$]�Ô!øå %¤× 1Ò 1Ñ 3Ô 3ˆDÔØÔ)ð 	9õ &+¤X§_¢_°T´_ÀdÄlÑ%SÔ%SˆDÔ"Ð"Ð"å%*¤X×%6Ò%6Ñ%8Ô%8ˆDÔ"Ð"Ð"r+   c           	     óv  — |j         d         }|j        }|                     d¦  «        | j        k    sJ ‚|                     d¦  «        | j        k    sJ ‚|                     d¦  «        | j        k    sJ ‚|�5|d         }|d         }	|j         d         |k    r|	j         d         |k    sJ ‚nAt          j        |dd||j        ¬¦  «        }t          j        |dt
          j	        |¬	¦  «         }	|�‹t          |j         ¦  «        d
k    r|j         dd…         |dfk    sJ ‚|j         dd…         | j        j        k    r<t          j        |                     |j        ¦  «        | j        j        ddd¬¦  «        }
n|}
nd}
|                      ||	fd|
¬¦  «        \  }}|                      || j                             ¦   «         |||d|¬¦  «        \  }}}}| j        r,|dk    }t          j        |dd…ddf         |t(          ¦  «        }t          j        || j        | j        fdd¬¦  «        }|dd…df         }|rŠt          j        |d¬¦  «        }t          j        ||¬¦  «        }|||f                              d¦  «        }|||f                              d¦  «        }|                     d¦  «        dk    r
|||f         }n||}}|                      |¦  «        }| j        rR| j        r|                     ¦   «         }n|                     |j        ¦  «        }| j        r||z  }|d|z
  | j        z  z   }|||||||fS )a[
  Forward pass through SAM prompt encoders and mask heads.

        This method processes image features and optional point/mask inputs to generate object masks and scores.

        Args:
            backbone_features (torch.Tensor): Image features with shape (B, C, H, W).
            point_inputs (dict[str, torch.Tensor] | None): Dictionary containing point prompts with keys 'point_coords'
                (Tensor of shape (B, P, 2) with float32 dtype, containing absolute pixel-unit coordinates in (x, y)
                format for P input points) and 'point_labels' (Tensor of shape (B, P) with int32 dtype, where 1 means
                positive clicks, 0 means negative clicks, and -1 means padding).
            mask_inputs (torch.Tensor | None): Mask of shape (B, 1, H*16, W*16), float or bool, with the same spatial
                size as the image.
            high_res_features (list[torch.Tensor] | None): List of two feature maps with shapes (B, C, 4*H, 4*W) and (B,
                C, 2*H, 2*W) respectively, used as high-resolution feature maps for SAM decoder.
            multimask_output (bool): If True, output 3 candidate masks and their IoU estimates; if False, output only 1
                mask and its IoU estimate.

        Returns:
            low_res_multimasks (torch.Tensor): Tensor of shape (B, M, H*4, W*4) with SAM output mask logits.
            high_res_multimasks (torch.Tensor): Tensor of shape (B, M, H*16, W*16) with upsampled mask logits.
            ious (torch.Tensor): Tensor of shape (B, M) with estimated IoU for each output mask.
            low_res_masks (torch.Tensor): Tensor of shape (B, 1, H*4, W*4) with the best low-resolution mask.
            high_res_masks (torch.Tensor): Tensor of shape (B, 1, H*16, W*16) with the best high-resolution mask.
            obj_ptr (torch.Tensor): Tensor of shape (B, C) with object pointer vector for the output mask.
            object_score_logits (torch.Tensor): Tensor of shape (B, 1) with object score logits.

        Examples:
            >>> backbone_features = torch.rand(1, 256, 32, 32)
            >>> point_inputs = {"point_coords": torch.rand(1, 2, 2), "point_labels": torch.tensor([[1, 0]])}
            >>> mask_inputs = torch.rand(1, 1, 512, 512)
            >>> results = model._forward_sam_heads(backbone_features, point_inputs, mask_inputs)
            >>> (
            ...     low_res_multimasks,
            ...     high_res_multimasks,
            ...     ious,
            ...     low_res_masks,
            ...     high_res_masks,
            ...     obj_ptr,
            ...     object_score_logits,
            ... ) = results
        r   r   r•   rR   NÚpoint_coordsÚpoint_labels©rŒ   Údtype)r±   rŒ   rS   éþÿÿÿFÚbilinearT©ÚsizeÚalign_cornersrY   Ú	antialias)ÚpointsÚboxesÚmasks)Úimage_embeddingsÚimage_peÚsparse_prompt_embeddingsÚdense_prompt_embeddingsÚmultimask_outputÚrepeat_imageÚhigh_res_features)rµ   rY   r¶   r!   ©Údim©rŒ   )rk   rŒ   rµ   r¥   r¦   r%   rn   r±   ÚonesÚint32Úlenr§   Úmask_input_sizeÚFÚinterpolateÚtor¨   Úget_dense_perJ   ÚwhereÚNO_OBJ_SCOREr~   ÚargmaxÚarangeÚ	unsqueezerª   rM   ÚsigmoidrL   r�   )r(   Úbackbone_featuresÚpoint_inputsÚmask_inputsrÁ   r¿   ÚBrŒ   Úsam_point_coordsÚsam_point_labelsÚsam_mask_promptÚsparse_embeddingsÚdense_embeddingsÚlow_res_multimasksÚiousÚsam_output_tokensÚobject_score_logitsÚis_obj_appearingÚhigh_res_multimasksÚsam_output_tokenÚbest_iou_indsÚ
batch_indsÚlow_res_masksÚhigh_res_masksÚobj_ptrÚlambda_is_obj_appearings                             r*   Ú_forward_sam_headszSAM2Model._forward_sam_heads”  s  € ðb Ô# AÔ&ˆØ"Ô)ˆØ ×%Ò% aÑ(Ô(¨DÔ,EÒEÐEÐEÐEØ ×%Ò% aÑ(Ô(¨DÔ,IÒIÐIÐIÐIØ ×%Ò% aÑ(Ô(¨DÔ,IÒIÐIÐIÐIð Ð#Ø+¨NÔ;ÐØ+¨NÔ;ÐØ#Ô)¨!Ô,°Ò1Ð1Ð6FÔ6LÈQÔ6OÐSTÒ6TÐ6TÐ6TÐTÐ6Tõ  %œ{¨1¨a°¸6ÐIZÔI`ÐaÑaÔaÐÝ %¤
¨1¨aµu´{È6Ð RÑ RÔ RÐRÐð Ð"õ �{Ô(Ñ)Ô)¨QÒ.Ð.°;Ô3DÀRÀaÀRÔ3HÈQÐPQÈFÒ3RÐ3RÐ3RÐRØÔ    Ô%¨Ô)@Ô)PÒPÐPÝ"#¤-Ø—N’NÐ#4Ô#:Ñ;Ô;ØÔ0Ô@Ø"'Ø#Ø"ð#ñ #ô #��ð #.��ð #ˆOà.2×.EÒ.EØ$Ð&6Ð7ØØ!ð /Fñ /
ô /
Ñ+ÐÐ+ð
 LP×K`ÒK`Ø.ØÔ,×9Ò9Ñ;Ô;Ø%6Ø$4Ø-ØØ/ð Lañ L
ô L
ÑHÐ˜DÐ"3Ð5Hð Ôð 	pØ2°QÒ6Ðõ "'¤Ð-=¸a¸a¸aÀÀt¸mÔ-LÐN`ÕbnÑ!oÔ!oÐõ  œmØØ”/ 4¤?Ð3ØØð	
ñ 
ô 
Ðð -¨Q¨Q¨Q°¨TÔ2ÐØð 		Tå!œL¨°2Ð6Ñ6Ô6ˆMÝœ a°Ð7Ñ7Ô7ˆJØ.¨z¸=Ð/HÔI×SÒSÐTUÑVÔVˆMØ0°¸]Ð1JÔK×UÒUÐVWÑXÔXˆNØ ×%Ò% aÑ(Ô(¨1Ò,Ð,Ø#4°ZÀÐ5NÔ#OÐ øà,>Ð@S˜>ˆMð ×#Ò#Ð$4Ñ5Ô5ˆØÔð 		PàÔ#ð MØ*=×*EÒ*EÑ*GÔ*GÐ'Ð'à*:×*=Ò*=¸g¼mÑ*LÔ*LÐ'àÔ$ð <Ø1°GÑ;�Ø Ð%<Ñ!<ÀÄÑ OÑOˆGàØØØØØØð
ð 	
r+   c                ó\  — d\  }}|                      ¦   «         }||z  |z   }t          j        ||                     d¦  «        dz  |                     d¦  «        dz  fddd¬¦  «        }|                     |j        d	         d
¦  «                              ¦   «         }	| j        r|�|€-t          j        |j        d	         | j	        |j
        ¬¦  «        }
nK|                      ||                      |                     |j        ¦  «        ¦  «        |¬¦  «        \  }}}}}}
}t          j        |                     d
¦  «                              ¦   «         dk    d
¬¦  «        }|d         }|                      ¦   «         }||z  |z   }| j        r| j        r||
z  }
|
d
|z
  | j        z  z   }
|||	|||
|fS )zFProcess mask inputs directly as output, bypassing SAM encoder/decoder.)g      4@ç      $Àr²   rS   r!   Fr³   Tr´   r   r   NrÄ   )rÓ   rÕ   rÁ   r   rÂ   ).N)r   rÉ   rÊ   rµ   Únew_onesrk   r^   r%   rn   rh   rŒ   ré   ra   rË   r±   ÚanyÚflattenrJ   rL   r�   )r(   rÕ   rÓ   rÁ   Ú	out_scaleÚout_biasÚmask_inputs_floatræ   rå   rÝ   rç   Ú_rà   rè   rß   s                  r*   Ú_use_mask_as_outputzSAM2Model._use_mask_as_output(  sú  € ð *Ñˆ	�8Ø'×-Ò-Ñ/Ô/ÐØ*¨YÑ6¸ÑAˆÝœØØ ×%Ò% bÑ)Ô)¨QÑ.°×0CÒ0CÀBÑ0GÔ0GÈ1Ñ0LÐMØØØð
ñ 
ô 
ˆð ×#Ò# KÔ$5°aÔ$8¸!Ñ<Ô<×BÒBÑDÔDˆØÔ+ð 		Ð/@Ð/HÐL]ÐLeå”k +Ô"3°AÔ"6¸¼ÐP[ÔPbÐcÑcÔcˆGˆGð )-×(?Ò(?Ø"3Ø ×0Ò0Ð1B×1EÒ1EÐFWÔF]Ñ1^Ô1^Ñ_Ô_Ø"3ð )@ñ )ô )Ñ%ˆAˆq�!�Q˜˜7 Aõ !œ9 [×%8Ò%8¸Ñ%;Ô%;×%AÒ%AÑ%CÔ%CÀcÒ%IÈqÐQÑQÔQÐØ+¨IÔ6ÐØ"2×"8Ò"8Ñ":Ô":ÐØ'Ð*AÑAÀHÑLÐØÔð 	PØÔ$ð <Ø1°GÑ;�Ø Ð%<Ñ!<ÀÄÑ OÑOˆGð ØØØØØØð
ð 	
r+   Ú	img_batchútorch.Tensorc                óú   — |                       |¦  «        }| j        r^| j                             |d         d         ¦  «        |d         d<   | j                             |d         d         ¦  «        |d         d<   |S ©zRProcess image batch through encoder to extract multi-level features for SAM model.Úbackbone_fpnr   r   )r   r\   r¨   Úconv_s0Úconv_s1©r(   rô   Úbackbone_outs      r*   Úforward_imagezSAM2Model.forward_imageW  s�   € à×)Ò)¨)Ñ4Ô4ˆØÔ,ð 	mð /3Ô.C×.KÒ.KÈLÐYgÔLhÐijÔLkÑ.lÔ.lˆL˜Ô(¨Ñ+Ø.2Ô.C×.KÒ.KÈLÐYgÔLhÐijÔLkÑ.lÔ.lˆL˜Ô(¨Ñ+ØÐr+   c                ó²  ‡— ‰dk    r-i |¥ˆfd„|d         D ¦   «         ˆfd„|d         D ¦   «         dœ¥}t          |d         ¦  «        t          |d         ¦  «        k    sJ ‚t          |d         ¦  «        | j        k    sJ ‚|d         | j         d…         }|d         | j         d…         }d„ |D ¦   «         }d	„ |D ¦   «         }d
„ |D ¦   «         }||||fS )zZPrepare and flatten visual features from the image backbone output for further processing.r   c                ó@   •— g | ]}|                      ‰d d d ¦  «        ‘ŒS ©r!   ©Úexpand)r3   ÚfeatÚbatchs     €r*   r5   z8SAM2Model._prepare_backbone_features.<locals>.<listcomp>f  s+   ø€ Ð iÐ iÐ iÀD §¢¨U°B¸¸BÑ!?Ô!?Ð iÐ iÐ ir+   rø   c                ó@   •— g | ]}|                      ‰d d d ¦  «        ‘ŒS r   r  )r3   Úposr  s     €r*   r5   z8SAM2Model._prepare_backbone_features.<locals>.<listcomp>g  s+   ø€ Ð"kÐ"kÐ"kÀS 3§:¢:¨e°R¸¸RÑ#@Ô#@Ð"kÐ"kÐ"kr+   Úvision_pos_enc)rø   r  Nc                óB   — g | ]}|j         d          |j         d         f‘ŒS )r²   r!   )rk   r2   s     r*   r5   z8SAM2Model._prepare_backbone_features.<locals>.<listcomp>o  s)   € ÐLÐLÐL°Q�q”w˜r”{ A¤G¨B¤KÐ0ÐLÐLÐLr+   c                ób   — g | ],}|                      d ¦  «                             d dd¦  «        ‘Œ-S ©r•   r   r   ©rî   Úpermuter2   s     r*   r5   z8SAM2Model._prepare_backbone_features.<locals>.<listcomp>q  s4   € ÐLÐLÐL¸!˜Ÿ	š	 !™œ×,Ò,¨Q°°1Ñ5Ô5ÐLÐLÐLr+   c                ób   — g | ],}|                      d ¦  «                             d dd¦  «        ‘Œ-S r
  r  r2   s     r*   r5   z8SAM2Model._prepare_backbone_features.<locals>.<listcomp>r  s4   € ÐVÐVÐV¸q˜QŸYšY q™\œ\×1Ò1°!°Q¸Ñ:Ô:ÐVÐVÐVr+   )rÇ   r]   )r(   rü   r  Úfeature_mapsÚvision_pos_embedsÚ
feat_sizesÚvision_featss     `    r*   Ú_prepare_backbone_featuresz$SAM2Model._prepare_backbone_featuresa  s7  ø€ à�1Š9ˆ9ðØðà iÐ iÐ iÐ iÈLÐYgÔLhÐ iÑ iÔ iØ"kÐ"kÐ"kÐ"kÈLÐYiÔLjÐ"kÑ"kÔ"kðð ð ˆLõ
 �< Ô/Ñ0Ô0µC¸ÐEUÔ8VÑ4WÔ4WÒWÐWÐWÐWÝ�< Ô/Ñ0Ô0°DÔ4KÒKÐKÐKÐKà# NÔ3°TÔ5LÐ4LÐ4NÐ4NÔOˆØ(Ð)9Ô:¸DÔ<SÐ;SÐ;UÐ;UÔVÐàLÐLÐ:KÐLÑLÔLˆ
àLÐL¸|ÐLÑLÔLˆØVÐVÐDUÐVÑVÔVÐØ˜\Ð+<¸jÐHÐHr+   c	                ó4  ‡ ‡‡‡+— |d                               d¦  «        }	‰ j        }
|d         \  }}|d         j        }‰ j        dk    r3|d                              ddd¦  «                             |	|
||¦  «        S d}‰rdndŠ+|�s“g g }}t          |d         ¦  «        dk    sJ ‚|d         }t          ‰|‰ j        ¦  «        \  }}d„ | 	                    ¦   «         D ¦   «         }‰ j
        rdn‰ j        }t          d‰ j        ¦  «        D ]›}‰ j        |z
  }|dk    r‰r‰|z   n‰|z
  }n1‰s‰dz
  |z  |z  }||dz
  |z  z
  }n‰dz    |z   |z  }||dz
  |z  z   }|d                              |d¦  «        }|€|                     |d¦  «        }|                     ||f¦  «         Œœ|D ]ç\  }}|€Œ|d	                              ||j        d
k    ¬¦  «        }|                     |                     d¦  «                             ddd¦  «        ¦  «         |d         d                              |¬¦  «        }|                     d¦  «                             ddd¦  «        }|‰ j        ‰ j        |z
  dz
           z   }|                     |¦  «         Œè‰ j        �r�t)          |‰ j        ¦  «        }‰ j
        s)‰ j        r"ˆˆfd„|                     ¦   «         D ¦   «         }n|}ˆˆ ˆ+fd„|                     ¦   «         D ¦   «         }t          d|¦  «        D ]m} ‰r‰| z   n‰| z
  }!|!dk     s|�|!|k    r nP|d                              |!|                     |!d¦  «        ¦  «        }|�|                     | |d         f¦  «         Œn|�r•t1          |Ž \  }"}#t3          j        |#d¬¦  «        }$‰ j        r�|dz
  }%‰ j        r|
n‰ j        }&t3          j        |"||d         j        ¬¦  «        }'tA          |'|%z  |&¬¦  «        }'‰  !                    |'¦  «        }'|' "                    d¦  «         #                    d|	‰ j        ¦  «        }'n)|$ $                    t          |"¦  «        |	‰ j        ¦  «        }'‰ j        |
k     rp|$ %                    d|	|
‰ j        z  ‰ j        ¦  «        }$|$                     dddd¦  «                             dd¦  «        }$|' &                    |
‰ j        z  d¬¦  «        }'|                     |$¦  «         |                     |'¦  «         |$j'        d         }n�d}nŠ‰ j(        r?|d         ‰ j)        z   }(|(                     ddd¦  «                             |	|
||¦  «        }(|(S ‰ j)         #                    d|	‰ j        ¦  «        g}‰ j*         #                    d|	‰ j        ¦  «        g}t3          j+        |d¬¦  «        })t3          j+        |d¬¦  «        }*‰  ,                    |||)|*|¬¦  «        }(|(                     ddd¦  «                             |	|
||¦  «        }(|(S )zePrepare memory-conditioned features by fusing current frame's visual features with previous memories.r!   r   r   r•   Úcond_frame_outputsc                ó   — g | ]}d |f‘ŒS )r   r1   )r3   Úouts     r*   r5   zBSAM2Model._prepare_memory_conditioned_features.<locals>.<listcomp>—  s   € ÐRÐRÐR¨C  3˜xÐRÐRÐRr+   Únon_cond_frame_outputsNÚmaskmem_featuresÚcuda)rŒ   Únon_blockingÚmaskmem_pos_encrÄ   c                ó:   •— i | ]\  }}‰r|‰k    ¯n|‰k    ¯||“ŒS r1   r1   )r3   Útr  Ú	frame_idxÚtrack_in_reverses      €€r*   ú
<dictcomp>zBSAM2Model._prepare_memory_conditioned_features.<locals>.<dictcomp>É  sD   ø€ ð (ð (ð (á"˜A˜sØ.>ð(ð  šN˜N˜NÀAÈÂNÀNð ˜3àDRÀNÀNr+   c                óh   •— g | ].\  }}‰j         r‰|z
  ‰z  nt          ‰|z
  ¦  «        |d          f‘Œ/S )rç   )rd   Úabs)r3   r  r  r  r(   Útpos_sign_muls      €€€r*   r5   zBSAM2Model._prepare_memory_conditioned_features.<locals>.<listcomp>Ð  s`   ø€ ð  ð  ð  ñ ˜˜3ð  $ÔCð4˜Y¨™]¨mÑ;Ð;å!$ Y°¡]Ñ!3Ô!3à˜Iœðð ð  ð  r+   rç   rÂ   r°   rR   )ÚcurrÚcurr_posÚmemoryÚ
memory_posÚnum_obj_ptr_tokens)-rµ   rh   rŒ   rl   r  r'   rÇ   r   rƒ   ÚvaluesÚtrainingrw   ÚrangeÚgetÚappendrË   Útyperî   ro   r^   Úminr_   re   ÚitemsÚzipr%   Ústackrb   rc   rj   Útensorr±   r   r¬   rÑ   r  Ú	new_zerosÚreshapeÚrepeat_interleaverk   rr   rp   rq   Úcatrf   ),r(   r  Úis_init_cond_frameÚcurrent_vision_featsÚcurrent_vision_pos_embedsr  Úoutput_dictÚ
num_framesr  rÖ   ÚCÚHÚWrŒ   r(  Úto_cat_memoryÚto_cat_memory_pos_embedÚcond_outputsÚselected_cond_outputsÚunselected_cond_outputsÚt_pos_and_prevsÚrÚt_posÚt_relÚprev_frame_idxr  ÚprevÚfeatsÚmaskmem_encr_   Úptr_cond_outputsÚpos_and_ptrsÚt_diffr  Úpos_listÚ	ptrs_listÚobj_ptrsÚ
t_diff_maxÚtpos_dimÚobj_posÚpix_feat_with_memr&  Úmemory_pos_embedr#  s,   ``      `                                  @r*   Ú$_prepare_memory_conditioned_featuresz.SAM2Model._prepare_memory_conditioned_featuresu  sA  øøøø€ ð ! Ô$×)Ò)¨!Ñ,Ô,ˆØŒOˆØ˜"Œ~‰ˆˆ1Ø% bÔ)Ô0ˆð Ô˜qÒ Ð Ø'¨Ô+×3Ò3°A°q¸!Ñ<Ô<×AÒAÀ!ÀQÈÈ1ÑMÔMÐMØÐØ.Ð5˜˜°Aˆà!ñ |	Wà57¸Ð2ˆMõ �{Ð#7Ô8Ñ9Ô9¸AÒ=Ð=Ð=Ð=à&Ð';Ô<ˆLÝ=WØ˜<¨Ô)Eñ>ô >Ñ:Ð!Ð#:ð SÐRÐ3H×3OÒ3OÑ3QÔ3QÐRÑRÔRˆOð
 ”]ÐL��¨Ô(LˆAÝ˜q $Ô"2Ñ3Ô3ð 5ð 5�ØÔ(¨5Ñ0�Ø˜A’:�:à:JÐ%a Y°Ñ%6Ð%6ÐPYÐ\aÑPa�N�NØ)ð Fð (1°1¡}¸Ñ&:¸aÑ%?�Nà%3°u¸q±yÀA±oÑ%E�N�Nð *3°Q©Ð'7¸1Ñ'<Ð%=ÀÑ%A�Nà%3°u¸q±yÀA±oÑ%E�NØ!Ð":Ô;×?Ò?ÀÐPTÑUÔU�Ø�;ð 2×5Ò5°nÀdÑKÔK�CØ×&Ò&¨¨s |Ñ4Ô4Ð4Ð4à.ð <ð <‘��tØ�<Øð Ð/Ô0×3Ò3¸6ÐPVÔP[Ð_eÒPeÐ3ÑfÔf�Ø×$Ò$ U§]¢]°1Ñ%5Ô%5×%=Ò%=¸aÀÀAÑ%FÔ%FÑGÔGÐGà"Ð#4Ô5°bÔ9×<Ò<ÀFÐ<ÑKÔK�Ø)×1Ò1°!Ñ4Ô4×<Ò<¸QÀÀ1ÑEÔE�à)¨DÔ,AÀ$ÔBRÐUZÑBZÐ]^ÑB^Ô,_Ñ_�Ø'×.Ò.¨{Ñ;Ô;Ð;Ð;ð Ô+ñ 9+Ý*-¨j¸$Ô:VÑ*WÔ*WÐ'ð ”}ð =¨Ô)Pð =ð(ð (ð (ð (ð (à&;×&AÒ&AÑ&CÔ&Cð(ñ (ô (Ð$Ð$ð (=Ð$ð ð  ð  ð  ð  ð  ð #3×"8Ò"8Ñ":Ô":ð ñ  ô  �õ $ AÐ'>Ñ?Ô?ð Fð F�FØ.>ÐV˜	 FÑ*Ð*ÀIÐPVÑDV�AØ˜1’u�u Ð!7¸AÀºO¸OØ˜Ø%Ð&>Ô?×CÒCÀAÐG^×GbÒGbÐcdÐfjÑGkÔGkÑlÔl�CØ�Ø$×+Ò+¨V°S¸´^Ð,DÑEÔEÐEøàñ +Ý*-¨|Ð*<Ñ'�H˜iå$œ{¨9¸!Ð<Ñ<Ô<�Hð Ô4ð UØ%<¸qÑ%@˜
Ø(,Ô(FÐ#X 1 1ÈDÌL˜Ý"'¤,¨xÀÐNbÐceÔNfÔNlÐ"mÑ"mÔ"m˜Ý"0°¸:Ñ1EÈ8Ð"TÑ"TÔ"T˜Ø"&×"8Ò"8¸Ñ"AÔ"A˜Ø")×"3Ò"3°AÑ"6Ô"6×"=Ò"=¸bÀ!ÀTÄ\Ñ"RÔ"R˜˜à"*×"4Ò"4µS¸±]´]ÀAÀtÄ|Ñ"TÔ"T˜Ø”| aÒ'Ð'à#+×#3Ò#3°B¸¸1ÀÄÑ;LÈdÌlÑ#[Ô#[˜Ø#+×#3Ò#3°A°q¸!¸QÑ#?Ô#?×#GÒ#GÈÈ1Ñ#MÔ#M˜Ø")×";Ò";¸AÀÄÑ<MÐSTÐ";Ñ"UÔ"U˜Ø!×(Ò(¨Ñ2Ô2Ð2Ø+×2Ò2°7Ñ;Ô;Ð;Ø)1¬¸Ô):Ð&Ð&à)*Ð&øð Ô-ð )à$8¸Ô$<¸tÔ?PÑ$PÐ!Ø$5×$=Ò$=¸aÀÀAÑ$FÔ$F×$KÒ$KÈAÈqÐRSÐUVÑ$WÔ$WÐ!Ø(Ð(ð "Ô.×5Ò5°a¸¸D¼LÑIÔIÐJˆMØ'+Ô':×'AÒ'AÀ!ÀQÈÌÑ'UÔ'UÐ&VÐ#õ ”˜=¨aÐ0Ñ0Ô0ˆÝ œ9Ð%<À!ÐDÑDÔDÐà ×1Ò1Ø%Ø.ØØ'Ø1ð 2ñ 
ô 
Ðð .×5Ò5°a¸¸AÑ>Ô>×CÒCÀAÀqÈ!ÈQÑOÔOÐØ Ð r+   c                óÊ  — |d                               d¦  «        }| j        }|d         \  }}	|d                              ddd¦  «                             ||||	¦  «        }
| j        r| j        s|                      |¦  «        }| j        o|}|r&| j        s|dk                         |
j	        ¦  «        }nt          j        |¦  «        }| j        dk    r
|| j        z  }| j        dk    r
|| j        z   }|                      |
|d¬¦  «        }|d	         }| j        �A|dk                         ¦   «         }|d|d         z
   | j        d         j        |j        Ž z  z  }||d         fS )zXEncode frame features and masks into a new memory representation for video segmentation.r!   r   r•   r   rG   r   T)Úskip_mask_sigmoidÚvision_featuresN©.NNr  )rµ   rh   r  r'   rv   r*  Ú"_apply_non_overlapping_constraintsru   rË   r±   r%   rÒ   rs   rt   ri   rO   r   r  rk   )r(   r9  r  Úpred_masks_high_resrß   Úis_mask_from_ptsrÖ   r=  r>  r?  Úpix_featÚbinarizeÚmask_for_memÚmaskmem_outr  rà   s                   r*   Ú_encode_new_memoryzSAM2Model._encode_new_memory  s¬  € ð ! Ô$×)Ò)¨!Ñ,Ô,ˆØŒOˆØ˜"Œ~‰ˆˆ1à'¨Ô+×3Ò3°A°q¸!Ñ<Ô<×AÒAÀ!ÀQÈÈ1ÑMÔMˆØÔ-ð 	_°d´mð 	_ð #'×"IÒ"IÐJ]Ñ"^Ô"^ÐàÔ:ÐOÐ?OˆØð 	>˜DœMð 	>Ø/°!Ò3×7Ò7¸¼ÑGÔGˆLˆLõ !œ=Ð)<Ñ=Ô=ˆLàÔ)¨SÒ0Ð0Ø'¨$Ô*HÑHˆLØÔ(¨CÒ/Ð/Ø'¨$Ô*GÑGˆLØ×)Ò)¨(°LÐTXÐ)ÑYÔYˆØ&Ð'8Ô9Ðð Ô$Ð0Ø 3°aÒ 7×>Ò>Ñ@Ô@ÐØ Ð%5°oÔ%FÑ!Fð KÈ$ÔJcØôKäÐ&Ô,ðK.ñ !.ñ .Ðð   Ð-=Ô!>Ð>Ð>r+   c           
     ó  — t          |¦  «        dk    r+d„ t          |dd…         |dd…         ¦  «        D ¦   «         }nd}|�V| j        rO|d                              ddd¦  «        } |j        d| j        g|d         ¢R Ž }|                      |||¦  «        }no|                      |||dd…         |dd…         |dd…         ||	|
¬¦  «        }|�|�|�J ‚|}|                      ||¦  «        }|  	                    |||||¬¦  «        }|||fS )	úhPerform a single tracking step, updating object masks and memory features based on current frame inputs.r   c                ó¨   — g | ]O\  }} |                      d dd¦  «        j        |                     d ¦  «        |                     d¦  «        g|¢R Ž ‘ŒPS )r   r•   r   )r  r'   rµ   )r3   r4   Úss      r*   r5   z)SAM2Model._track_step.<locals>.<listcomp>U  sg   € ð !ð !ð !á�A�qð (�—	’	˜!˜Q Ñ"Ô"Ô'¨¯ª¨q©	¬	°1·6²6¸!±9´9ÐA¸qÐAÐAÐAð!ð !ð !r+   Nr!   r•   r   )r  r8  r9  r:  r  r;  r<  r  )rÓ   rÔ   rÕ   rÁ   r¿   )
rÇ   r1  rx   r  r'   rh   ró   rX  Ú_use_multimaskré   )r(   r  r8  r9  r:  r  rÔ   rÕ   r;  r<  r  Úprev_sam_mask_logitsrÁ   r`  Úsam_outputsr¿   s                   r*   Ú_track_stepzSAM2Model._track_stepD  s‚  € õ  Ð#Ñ$Ô$ qÒ(Ð(ð!ð !åÐ 4°S°b°SÔ 9¸:ÀcÀrÀc¼?ÑKÔKð!ñ !ô !ÐÐð
 !%ÐØÐ" tÔ'PÐ"ð ,¨BÔ/×7Ò7¸¸1¸aÑ@Ô@ˆHØ$�x”} R¨¬ÐJ¸:Àb¼>ÐJÐJÐJˆHØ×2Ò2°;ÀÐJ[Ñ\Ô\ˆKˆKð ×@Ò@Ø#Ø#5Ø%9¸"¸#¸#Ô%>Ø*CÀBÀCÀCÔ*HØ% b c cœ?Ø'Ø%Ø!1ð Añ 	ô 	ˆHð $Ð/Ø#Ð/°KÐ4GÐ4GÐGØ2�Ø#×2Ò2Ð3EÀ|ÑTÔTÐØ×1Ò1Ø"*Ø)Ø'Ø"3Ø!1ð 2ñ ô ˆKð Ð-¨xÐ7Ð7r+   c                óŠ   — |r6| j         dk    r+|                      |||||du¬¦  «        \  }}	||d<   |	|d<   dS d|d<   d|d<   dS )z^Run memory encoder on predicted mask to encode it into a new memory feature for future frames.r   N)r9  r  r^  rß   r_  r  r  )rl   rd  )
r(   r9  r  rÔ   Úrun_mem_encoderræ   rß   Úcurrent_outr  r  s
             r*   Ú_encode_memory_in_outputz"SAM2Model._encode_memory_in_output~  sˆ   € ð ð 	2˜tÔ/°!Ò3Ð3Ø04×0GÒ0GØ%9Ø%Ø$2Ø$7Ø".°dÐ":ð 1Hñ 1ô 1Ñ-Ð˜oð /?ˆKÐ*Ñ+Ø-<ˆKÐ)Ñ*Ð*Ð*à.2ˆKÐ*Ñ+Ø-1ˆKÐ)Ñ*Ð*Ð*r+   c                óº   — |                       |||||||||	|
|¦  «        \  }}}|\  }}}}}}}|||dœ}| j        s||d<   |                      |||||||¦  «         |S )rf  )Ú
pred_masksr^  rç   rß   )rl  r*  rp  )r(   r  r8  r9  r:  r  rÔ   rÕ   r;  r<  r  rn  rj  rk  rò   rå   ræ   rç   rß   ro  s                       r*   Ú
track_stepzSAM2Model.track_step—  sÃ   € ð, !×,Ò,ØØØ Ø%ØØØØØØØ ñ
ô 
Ñˆ�Q˜ð P[ÑLˆˆ1ˆa� °Ð9Lð (Ø#1Øð
ð 
ˆð
 Œ}ð 	Eð 2EˆKÐ-Ñ.ð 	×%Ò%Ø ØØØØØØñ	
ô 	
ð 	
ð Ðr+   c                ó”   — |€dn|d                               d¦  «        }| j        o"|p| j        o| j        |cxk    o
| j        k    nc S )zaDetermine whether to use multiple mask outputs in the SAM head based on configuration and inputs.Nr   r¯   r   )rµ   ry   r|   rz   r{   )r(   r8  rÔ   Únum_ptss       r*   ri  zSAM2Model._use_multimaskÓ  sk   € à#Ð+�!�!°¸nÔ1M×1RÒ1RÐSTÑ1UÔ1UˆàÔ(ð TØ#ÐI tÔ'IðTàÔ*¨gÐRÐRÒRÐR¸Ô9RÒRÐRÐRÐRð	
r+   c                ó  — | j         d         }|dk     r| S | j        }|                      dd¬¦  «        j        }t	          j        ||¬¦  «        dd…dddf         }||k    }t	          j        || t	          j        | d¬¦  «        ¦  «        } | S )	z\Apply non-overlapping constraints to masks, keeping the highest scoring object per location.r   r•   T)rÃ   ÚkeepdimrÄ   Nrë   ©Úmax)rk   rŒ   ry  Úindicesr%   rÐ   rÍ   Úclamp)rr  Ú
batch_sizerŒ   Úmax_obj_indsÚbatch_obj_indsÚkeeps         r*   r]  z,SAM2Model._apply_non_overlapping_constraintsÜ  s›   € ð  Ô% aÔ(ˆ
Ø˜Š>ˆ>ØÐàÔ"ˆà!—~’~¨!°T�~Ñ:Ô:ÔBˆåœ j¸Ð@Ñ@Ô@ÀÀÀÀDÈ$ÐPTÐATÔUˆØ˜~Ò-ˆõ ”[  zµ5´;¸zÈuÐ3UÑ3UÔ3UÑVÔVˆ
ØÐr+   c                ó   — || _         dS )z Set binarize for VideoPredictor.N)ru   )r(   ra  s     r*   Úset_binarizezSAM2Model.set_binarizeî  s   € à2:ˆÔ/Ð/Ð/r+   c                ó   ‡ — t          ‰ j        d¦  «        r‰ j                             |¦  «         |d         ‰ _        |‰ j        _        ˆ fd„|D ¦   «         ‰ j        _        ˆ fd„|D ¦   «         ‰ j        _        ‰ j        ‰ j        z  ‰ _	        dS )r-   r.   r   c                ó$   •— g | ]}|‰j         z  ‘ŒS r1   ©r   ©r3   r4   r(   s     €r*   r5   z'SAM2Model.set_imgsz.<locals>.<listcomp>ø  s.   ø€ ð 8
ð 8
ð 8
Ø*+ˆA�Ô%Ñ%ð8
ð 8
ð 8
r+   c                ó*   •— g | ]}|‰j         z  d z  ‘ŒS )rS   r„  r…  s     €r*   r5   z'SAM2Model.set_imgsz.<locals>.<listcomp>û  s3   ø€ ð 3
ð 3
ð 3
Ø./ˆA�Ô%Ñ%¨Ñ)ð3
ð 3
ð 3
r+   N)
r6   r   r.   r~   r§   r7   r8   rÈ   r   r¦   r:   s   ` r*   r.   zSAM2Model.set_imgszò  s»   ø€ å�4Ô% {Ñ3Ô3ð 	0ØÔ×(Ò(¨Ñ/Ô/Ð/Ø œ(ˆŒØ38ˆÔÔ0ð8
ð 8
ð 8
ð 8
Ø/4ð8
ñ 8
ô 8
ˆÔÔ4ð3
ð 3
ð 3
ð 3
Ø38ð3
ñ 3
ô 3
ˆÔÔ/ð )-¬¸4Ô;OÑ(OˆÔ%Ð%Ð%r+   ) rE   rF   r0   rG   r   FFr!   FFFr   r   FFFr   FFr0   TFFFFFFFFFNF©rH   rI   rJ   rI   rK   rI   rL   rI   rM   rI   rN   rI   rO   rI   rP   rI   )NNNF)NN©rô   rõ   )r   )F)FTN)r<   r=   r>   r?   r   r@   r#   ÚpropertyrŒ   r‡   r‚   ré   ró   rý   r  rX  rd  rl  rp  rs  ri  Ústaticmethodr]  r�  r.   rA   rB   s   @r*   rD   rD   [   s5  ø€ € € € € € ðCð CðJ  €NÐÐÐÑð ØØØ"%Ø!$Ø+0Ø-2Ø "Ø"'Ø%*Ø %ØØØ&+Ø05Ø#(Ø()Ø&+Ø %Ø "Ø!%Ø"'Ø(-Ø+0Ø %Ø$)Ø!&Ø %Ø).Ø%*Ø$(Ø&+ðItð tð tð tð tð tð tðl ð.ð .ñ „Xð.ð
ð 
ð 
ð-9ð -9ð -9ðd ØØØðR
ð R
ð R
ð R
ðh-
ð -
ð -
ð -
ð^ð ð ð ðIð Ið Ið Ið: ðb!ð b!ð b!ð b!ðH)?ð )?ð )?ðV88ð 88ð 88ðt2ð 2ð 2ðH ð à!ð':ð :ð :ð :ðx
ð 
ð 
ð ðð ñ „\ðð";ð ;ð ;ð ;ðPð Pð Pð Pð Pð Pð Pr+   rD   c                  ó˜   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd ˆ fd„Zd!d„Zd"ˆ fd„Zed#d„¦   «         Zd„ Z	ˆ xZ
S )$Ú	SAM3ModelúfSAM3Model class for Segment Anything Model 3 with memory-based video object segmentation capabilities.rE   éð  é   r   r   Fr!   r0   TNrH   rI   rJ   rK   rL   rM   rN   rO   rP   c$                óp  •—  t          ¦   «         j        g |‘|‘|‘|‘|‘|‘|‘|‘|	‘|
‘|‘|‘|‘|‘|‘|‘|‘|‘|‘|‘|‘|‘|‘|‘|‘|‘|‘|‘|‘|‘|‘| ‘|!‘|"‘|#‘R Ž  t          d	dt          d| j        dd¬¦  «        | j        dd| j        | j        | j        | j        | j	        dœ
| j
        pi ¤Ž| _        dS )
r�  rR   r•   r–   r—   r˜   r�   rž   Nr1   )r"   r#   r   r
   r¥   r\   r}   rJ   rK   rH   r€   r¨   rˆ   s%                                       €r*   r#   zSAM3Model.__init__  s  ø€ ðN 	�‰ŒÔð $	
Øð$	
àð$	
ð ð$	
ð ð	$	
ð
 ð$	
ð ð$	
ð &ð$	
ð %ð$	
ð /ð$	
ð 1ð$	
ð $ð$	
ð &ð$	
ð )ð$	
ð $ð$	
ð !ð$	
ð  !ð!$	
ð" *ð#$	
ð$ ,ð%$	
ð& 'ð'$	
ð( ,ð)$	
ð* *ð+$	
ð, $ð-$	
ð. $ð/$	
ð0 %ð1$	
ð2 &ð3$	
ð4 ,ð5$	
ð6 /ð7$	
ð8 ð9$	
ð:  ð;$	
ð< ð=$	
ð> ð?$	
ð@ %ðA$	
ðB !ðC$	
ðD (ðE$	
ðF "ðG$	
ð $	
ð $	
ð $	
õJ !0ð !
Ø"#Ý)ØØ"Ô7ØØð	ñ ô ð !Ô5ØØ #Ø"&Ô"CØ'+Ô'FØ Ô0Ø $Ô 8Ø,0Ô,Pð!
ð !
ð  Ô/Ð5°2ð!!
ð !
ˆÔÐÐr+   rô   rõ   c                ó  — | j                              |¦  «        }| j        r^| j                             |d         d         ¦  «        |d         d<   | j                             |d         d         ¦  «        |d         d<   |S r÷   )r   Úforward_image_sam2r\   r¨   rù   rú   rû   s      r*   rý   zSAM3Model.forward_imagec  s„   € àÔ)×<Ò<¸YÑGÔGˆØÔ,ð 	mð /3Ô.C×.KÒ.KÈLÐYgÔLhÐijÔLkÑ.lÔ.lˆL˜Ô(¨Ñ+Ø.2Ô.C×.KÒ.KÈLÐYgÔLhÐijÔLkÑ.lÔ.lˆL˜Ô(¨Ñ+ØÐr+   r;   útuple[int, int]c                ó€   •— t          ¦   «                              |¦  «         d„ |D ¦   «         | j        j        _        dS )z6Set the image size for the model and mask downsampler.c                ó   — g | ]
}|d z  dz  ‘ŒS )r�  r0   r1   )r3   rµ   s     r*   r5   z'SAM3Model.set_imgsz.<locals>.<listcomp>p  s!   € Ð=`Ð=`Ð=`ÐRV¸dÀb¹jÈ2¹oÐ=`Ð=`Ð=`r+   N)r"   r.   ri   Úmask_downsamplerÚinterpol_size)r(   r;   r)   s     €r*   r.   zSAM3Model.set_imgszm  s>   ø€ å‰Œ×Ò˜%Ñ Ô Ð Ø=`Ð=`ÐZ_Ð=`Ñ=`Ô=`ˆÔÔ,Ô:Ð:Ð:r+   ç333333Ó?c                ó:  — | dk                          d¬¦  «        }|dk                          d¬¦  «        }t          j        |d¬¦  «        }||z  }||k    }|d                              | ¦  «        }t          j        || t          j        | d¬¦  «        ¦  «        }|S )	zXSuppress masks that shrink in area after applying pixelwise non-overlapping constraints.r   )r!   r²   rÂ   rG   )r/  r\  rë   rx  )Úsumr%   r{  Ú	expand_asrÍ   )	rr  Únew_pred_masksÚshrink_thresholdÚarea_beforeÚ
area_afterÚ
area_ratior  Ú	keep_maskÚpred_masks_afters	            r*   Ú_suppress_shrinked_masksz"SAM3Model._suppress_shrinked_masksr  s¤   € ð " A’~×*Ò*¨xÐ*Ñ8Ô8ˆØ$ qÒ(×-Ò-°(Ð-Ñ;Ô;ˆ
Ý”k +°3Ð7Ñ7Ô7ˆØ +Ñ-ˆ
ØÐ-Ò-ˆØ˜Ô)×3Ò3°JÑ?Ô?ˆ	Ý œ; y°*½e¼kÈ*ÐZ_Ð>`Ñ>`Ô>`ÑaÔaÐØÐr+   c                ó\   — |                       |¦  «        }|                      ||¦  «        }|S )z®This function suppresses masks that shrink in area after applying pixelwise non-overlapping constraints. Note
        that the final output can still be overlapping.
        )r]  r£  )r(   rr  Ú!pixel_level_non_overlapping_maskss      r*   Ú"_suppress_object_pw_area_shrinkagez,SAM3Model._suppress_object_pw_area_shrinkage~  s7   € ð
 -1×,SÒ,SÐT^Ñ,_Ô,_Ð)ð ×2Ò2°:Ð?`ÑaÔaˆ
ØÐr+   ) rE   rŽ  r�  r   r   FFr!   FFFr   r   FFFr   FFr0   TFFFFFFFFFNFr‡  rˆ  )r;   r“  )r˜  )r<   r=   r>   r?   r#   rý   r.   rŠ  r£  r¦  rA   rB   s   @r*   rŒ  rŒ    s  ø€ € € € € ØpÐpð ØØØ"#Ø!"Ø+0Ø-2Ø "Ø"'Ø%*Ø %ØØØ&+Ø05Ø#(Ø()Ø&+Ø %Ø "Ø!%Ø"'Ø(-Ø+0Ø %Ø$)Ø!&Ø %Ø).Ø%*Ø$(Ø&+ðI]
ð ]
ð ]
ð ]
ð ]
ð ]
ð ]
ð~ð ð ð ðað að að að að að
 ð	 ð 	 ð 	 ñ „\ð	 ð	ð 	ð 	ð 	ð 	ð 	ð 	r+   rŒ  )Ú
__future__r   r%   Útorch.nn.functionalr   Ú
functionalrÉ   Útorch.nn.initr   Úultralytics.nn.modulesr   Úultralytics.utilsr   Úblocksr	   r
   Údecodersr   r   Úencodersr   r   Úutilsr   r   rÎ   ÚModuler   rD   rŒ  r1   r+   r*   ú<module>r²     s–  ðð #Ð "Ð "Ð "Ð "Ð "à €€€Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø 'Ð 'Ð 'Ð 'Ð 'Ð 'à &Ð &Ð &Ð &Ð &Ð &Ø $Ð $Ð $Ð $Ð $Ð $à <Ð <Ð <Ð <Ð <Ð <Ð <Ð <Ø 2Ð 2Ð 2Ð 2Ð 2Ð 2Ð 2Ð 2Ø 4Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4Ø =Ð =Ð =Ð =Ð =Ð =Ð =Ð =ð €ð?/ð ?/ð ?/ð ?/ð ?/ˆrŒyñ ?/ô ?/ð ?/ðDcPð cPð cPð cPð cP�””ñ cPô cPð cPðLFð Fð Fð Fð F�	ñ Fô Fð Fð Fð Fr+   