§
    l¬¸j†9  ã                  óš   — d dl mZ d dlmZ d dlZd dlmZ d dlmZ ddl	m
Z
 dd	lmZ dd
lmZ dd„Z G d„ dej        j        ¦  «        ZdS )é    )Úannotations)ÚdeepcopyN)Úinverse_sigmoid)Ú	xywh2xyxyé   )Ú	SAM2Modelé   )ÚPrompt)ÚSAM3VLBackboneTc                ó<  — |r|d         n|| |<   |r†|r†d| vr,d„ t          t          |¦  «        dz
  ¦  «        D ¦   «         | d<   t          | d         ¦  «        t          |¦  «        dz
  k    sJ ‚t          | d         |dd…         ¦  «        D ]\  }}|||<   ŒdS dS dS )zLHelper function to update output dictionary with main and auxiliary outputs.éÿÿÿÿÚaux_outputsc                ó   — g | ]}i ‘ŒS © r   )Ú.0Ú_s     úv/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/ultralytics/models/sam/sam3/sam3_image.pyú
<listcomp>z_update_out.<locals>.<listcomp>   s   € Ð!HÐ!HÐ!H¨ "Ð!HÐ!HÐ!Hó    r	   N)ÚrangeÚlenÚzip)ÚoutÚout_nameÚ	out_valueÚ	auxiliaryÚ
update_auxÚ
aux_outputÚ	aux_values          r   Ú_update_outr       sÜ   € à%.Ð=�I˜b”M�M°I€Cˆ�MØð -�Zð -Ø Ð#Ð#Ø!HÐ!H­eµC¸	±N´NÀQÑ4FÑ.GÔ.GÐ!HÑ!HÔ!HˆC�ÑÝ�3�}Ô%Ñ&Ô&­#¨i©.¬.¸1Ñ*<Ò<Ð<Ð<Ð<Ý%(¨¨]Ô);¸YÀsÈÀs¼^Ñ%LÔ%Lð 	-ð 	-Ñ!ˆJ˜	Ø#,ˆJ�xÑ Ð ð-ð -ð -ð -ð	-ð 	-r   c                  ó¬   ‡ — e Zd ZU dZdZded<   dZded<   	 	 	 	 	 	 	 	 	 	 	 	 	 d,d-ˆ fd„Z	 	 	 d.d„Z	 d/d0d„Z	d„ Z
	 	 d1d„Zd„ Z	 d/d2d%„Zd3d(„Zd4d+„Zˆ xZS )5ÚSAM3SemanticModelzCSAM3 model for semantic segmentation with vision-language backbone.g        ÚfloatÚmask_thresholdé   ÚintÚmax_text_batchNr	   TFr   Úbackboner   Úuse_instance_queryÚboolÚmultimask_outputÚuse_act_checkpoint_seg_headÚsupervise_joint_box_scoresÚdetach_presence_in_joint_scoreÚseparate_scorer_for_instanceÚnum_interactive_steps_valc                óì  •— t          ¦   «                              ¦   «          || _        || _        || _        |j        | _        || _        || _        || _	        || _
        |
| _        || _        || _        || _        | j        r)|€J ‚|| _
        d| _        |rt!          |¦  «        | _        nLt"          j                             | j        d¦  «        | _        d| _        |rt!          | j        ¦  «        | _        || _        || _        | j        j        j        }| j        j        j        }|| j        j        j        r|ndk    sJ ‚| j        j        j        | _        || _        |	| _        i | _        g | _        dS )z!Initialize the SAM3SemanticModel.Nr	   r   ) ÚsuperÚ__init__r(   Úgeometry_encoderÚtransformerÚd_modelÚ
hidden_dimÚnum_feature_levelsÚsegmentation_headÚo2m_mask_predictÚdot_prod_scoringr,   Úmatcherr0   Úuse_dot_prod_scoringÚinstance_dot_prod_scoringr   ÚtorchÚnnÚLinearÚclass_embedÚinstance_class_embedr-   r.   ÚdecoderÚnum_queriesÚnum_o2m_queriesÚdacr)   r+   Útext_embeddingsÚnames)Úselfr(   r5   Úinput_geometry_encoderr9   r8   r:   r;   r)   r+   r,   r<   r=   r-   r.   r/   r0   Únum_o2o_staticÚnum_o2m_staticÚ	__class__s                      €r   r3   zSAM3SemanticModel.__init__%   s�  ø€ õ( 	‰Œ×ÒÑÔÐØ ˆŒØ 6ˆÔØ&ˆÔØ%Ô-ˆŒØ"4ˆÔØ!2ˆÔà 0ˆÔà 0ˆÔØ+FˆÔ(ØˆŒà)BˆÔ&Ø$8ˆÔ!àÔ$ð 
	GØ#Ð/Ð/Ð/Ø$4ˆDÔ!Ø-1ˆDÔ*Ø+ð LÝ19Ð:JÑ1KÔ1K�Ô.øå$œxŸš¨t¬ÀÑBÔBˆDÔØ(,ˆDÔ%Ø+ð GÝ,4°TÔ5EÑ,FÔ,F�Ô)à*DˆÔ'Ø.LˆÔ+ð Ô)Ô1Ô=ˆØÔ)Ô1ÔAˆØ°DÔ4DÔ4LÔ4PÐ"W . .ÐVWÒXÐXÐXÐXØÔ#Ô+Ô/ˆŒà"4ˆÔØ 0ˆÔà!ˆÔØˆŒ
ˆ
ˆ
r   c                ó€  — |�|d         |z   g}|                       ||||¬¦  «        \  }}	|€_t          j        dg|j        dd…         ¢R |j        ¬¦  «        }t          j        g |	j        dd…         ¢d‘R |	j        |	j        ¬¦  «        }t          j        ||gd¬¦  «        }
t          j        |	|gd¬¦  «        }|
|fS )	z(Encode the geometric and visual prompts.Nr   )Ú
geo_promptÚ	img_featsÚ	img_sizesÚimg_pos_embedsr   r	   )Údevice)rT   Údtype©Údim)r4   r?   ÚzerosÚshaperT   rU   Úcat)rJ   rQ   rS   Úvis_feat_sizesÚgeometric_promptÚvisual_prompt_embedÚvisual_prompt_maskÚprev_mask_predÚ	geo_featsÚ	geo_masksÚpromptÚprompt_masks               r   Ú_encode_promptz SAM3SemanticModel._encode_prompte   sû   € ð Ð%Ø" 2œ¨Ñ7Ð8ˆIà#×4Ò4Ø'ØØ$Ø)ð	  5ñ  
ô  
Ñˆ	�9ð Ð&Ý"'¤+¨qÐ.G°9´?À1À2À2Ô3FÐ.GÐ.GÐPYÔP`Ð"aÑ"aÔ"aÐÝ!&¤Ø*�)”/ # 2 #Ô&Ð*¨Ð*Ð*Ø Ô'Ø”oð"ñ "ô "Ðõ
 ”˜IÐ':Ð;ÀÐCÑCÔCˆÝ”i Ð,>Ð ?ÀQÐGÑGÔGˆØ�{Ð"Ð"r   Úencoder_extra_kwargsúdict | Nonec           
     óþ   — | j                              |                     ¦   «         d|                     ¦   «         ||||¬¦  «        }|d         |d         |d         |d         |||                     d|¦  «        |dœ}|S )	zRun the transformer encoder.N)ÚsrcÚsrc_key_padding_maskÚsrc_posrb   Úprompt_key_padding_maskÚ
feat_sizesre   ÚmemoryÚ	pos_embedÚpadding_maskÚvalid_ratiosÚmemory_text)Úencoder_hidden_statesrn   ro   rp   r[   Úprompt_before_encÚprompt_after_encrc   )r5   ÚencoderÚcopyÚget)	rJ   rQ   rS   r[   rb   rc   re   rm   Úencoder_outs	            r   Ú_run_encoderzSAM3SemanticModel._run_encoder„   sœ   € ð Ô!×)Ò)Ø—’Ñ Ô Ø!%Ø"×'Ò'Ñ)Ô)ØØ$/Ø%Ø!5ð *ñ 
ô 
ˆð &,¨HÔ%5Ø Ô,Ø" >Ô2Ø" >Ô2Ø,à!'Ø &§
¢
¨=¸&Ñ AÔ AØ&ð
ð 
ˆð Ðr   c                óÎ  — |j         d         }| j        j        j        j        }	|	                     d¦  «                             d|d¦  «        }
| j                             |
|||d|d         |d         d||d¬¦  «        \  }}}}|                     dd¦  «        }|                     dd¦  «        }|�|                     dd¦  «        }|                      ||||||¬¦  «         ||fS )	zRun the transformer decoder.r	   Nr[   rp   F)Útgtrm   Úmemory_key_padding_maskÚposÚreference_boxesÚspatial_shapesrp   Útgt_maskrq   Útext_attention_maskÚ	apply_dacr   )Údec_presence_out)	rY   r5   rD   Úquery_embedÚweightÚ	unsqueezeÚrepeatÚ	transposeÚ_update_scores_and_boxes)rJ   rn   rm   Úsrc_maskr   rb   rc   rx   Úbsr„   r{   Úhsr~   rƒ   r   s                  r   Ú_run_decoderzSAM3SemanticModel._run_decoder§   s  € ð Œ\˜!Œ_ˆØÔ&Ô.Ô:ÔAˆØ×#Ò# AÑ&Ô&×-Ò-¨a°°QÑ7Ô7ˆà37Ô3C×3KÒ3KØØØ$,ØØ Ø&Ð'7Ô8Ø$ ^Ô4ØØØ +Øð 4Lñ 4
ô 4
Ñ0ˆˆOÐ-¨qð �\Š\˜!˜QÑÔˆØ)×3Ò3°A°qÑ9Ô9ˆØÐ'à/×9Ò9¸!¸QÑ?Ô?ÐØ×%Ò%ØØØØØØ-ð 	&ñ 	
ô 	
ð 	
ð �Bˆwˆr   c           	     óÜ  — |                      d¦  «        }| j        r%| j        }	|r| j        �| j        }	 |	|||¦  «        }
n"| j        }|r| j        �| j        } ||¦  «        }
| j        j        j        }|r"| j        j        j	        �| j        j        j	        } ||¦  «        }t          |¦  «        }||z                        ¦   «         }t          |¦  «        }|�t          |d|d¬¦  «         | j        r‘|€J ‚|                     ¦   «                              ¦   «         }| j        r|                     ¦   «         }t          |
                     ¦   «         |                     d¦  «        z  ¦  «                             dd¬¦  «        }
t          |d	|
dd…dd…d|…f         d¬¦  «         t          |d
|dd…dd…d|…f         d¬¦  «         t          |d|dd…dd…d|…f         d¬¦  «         dS )z9Update output dict with class scores and box predictions.r   NÚpresence_logit_decF)r   g      $Àg      $@)ÚminÚmaxÚpred_logitsÚ
pred_boxesÚpred_boxes_xyxy)Úsizer=   r;   r>   rB   rC   r5   rD   Ú
bbox_embedÚinstance_bbox_embedr   Úsigmoidr   r    r-   Úcloner.   Údetachr†   Úclamp)rJ   r   rŒ   r~   rb   rc   rƒ   Úis_instance_promptÚnum_o2oÚdot_prod_scoring_headÚoutputs_classÚclass_embed_headÚbox_headÚanchor_box_offsetsÚreference_boxes_inv_sigÚoutputs_coordÚoutputs_boxes_xyxyÚprob_dec_presence_outs                     r   r‰   z*SAM3SemanticModel._update_scores_and_boxesÒ   sR  € ð —'’'˜!‘*”*ˆàÔ$ð 		1Ø$(Ô$9Ð!Ø!ð G dÔ&DÐ&PØ(,Ô(FÐ%Ø1Ð1°"°f¸kÑJÔJˆMˆMà#Ô/ÐØ!ð = dÔ&?Ð&KØ#'Ô#<Ð Ø,Ð,¨RÑ0Ô0ˆMð Ô#Ô+Ô6ˆØð 	D $Ô"2Ô":Ô"NÐ"ZØÔ'Ô/ÔCˆHØ%˜X b™\œ\ÐÝ"1°/Ñ"BÔ"BÐØ0Ð3EÑE×NÒNÑPÔPˆÝ& }Ñ5Ô5ÐàÐ'Ý˜Ð1Ð3CÐPUÐVÑVÔVÐVàÔ*ð 	Ø#Ð/Ð/Ð/Ø$4×$:Ò$:Ñ$<Ô$<×$DÒ$DÑ$FÔ$FÐ!ØÔ2ð GØ(=×(DÒ(DÑ(FÔ(FÐ%å+¨M×,AÒ,AÑ,CÔ,CÐF[×FeÒFeÐfgÑFhÔFhÑ,hÑiÔi×oÒoØ˜tð pñ ô ˆMõ 	�C˜¨°a°a°a¸¸¸¸H¸W¸H°nÔ(EÐRWÐXÑXÔXÐXÝ�C˜ }°Q°Q°Q¸¸¸¸8¸G¸8°^Ô'DÐQVÐWÑWÔWÐWÝ�CÐ*Ð,>¸q¸q¸qÀ!À!À!ÀXÀgÀX¸~Ô,NÐ[`ÐaÑaÔaÐaÐaÐar   c           	     ó|  — | j         �ž|                     d¦  «        }| j        r|n|dd…dd…d|…f         }|                       |d         ||||¬¦  «        }	|	                     ¦   «         D ]8\  }
}|
| j         j        v r t          ||
|dd…d|…f         d¬¦  «         Œ3|||
<   Œ9dS |                     dd¦  «         dS )z%Run segmentation heads and get masks.Nr   Úbackbone_fpn)Úbackbone_featsÚobj_queriesrr   rb   rc   F)r   )r9   r•   r:   ÚitemsÚinstance_keysr    Úpop)rJ   r   Úbackbone_outrr   rb   rc   rŒ   r�   rª   Úseg_head_outputsÚkÚvs               r   Ú_run_segmentation_headsz)SAM3SemanticModel._run_segmentation_heads  s  € ð Ô!Ð-Ø—g’g˜a‘j”jˆGØ $Ô 5ÐM˜"˜"¸2¸a¸a¸aÀÀÀÀHÀWÀH¸nÔ;MˆKØ#×5Ò5Ø+¨NÔ;Ø'Ø&;ØØ'ð  6ñ  ô  Ðð )×.Ò.Ñ0Ô0ð ð ‘��1Ø˜Ô.Ô<Ð<Ð<Ý  Q¨¨!¨!¨!¨X¨g¨X¨+¬À%ÐHÑHÔHÐHÐHà�C˜‘F�Fð	ð ð ×Ò˜^¨TÑ2Ô2Ð2Ð2Ð2r   r®   údict[str, torch.Tensor]Útext_idsútorch.Tensorr\   r
   c           
     ój  ‡— t          |¦  «        | j        k    rý|�|j        j        d         dk    råg Št	          dt          |¦  «        | j        ¦  «        D ]™}d}|�Yt          |j        dd…||| j        z   …f         |j        ||| j        z   …         |j        dd…||| j        z   …f         ¬¦  «        }‰                     |  	                    ||||| j        z   …         |¦  «        ¦  «         Œšˆfd„‰d          
                    ¦   «         D ¦   «         S t          j        | |t          |¦  «        ¬¦  «        \  }}}}|                     d„ | j         
                    ¦   «         D ¦   «         ¦  «         |d         dd…|f         }	|d         |         }
|�ƒt          j                             d	¦  «        5  |                      ||||¦  «        \  }}ddd¦  «         n# 1 swxY w Y   t          j        |	|gd¬
¦  «        }t          j        |
|gd¬
¦  «        }n|	}|
}t          j                             d¦  «        5  |                      |||||¦  «        }ddd¦  «         n# 1 swxY w Y   d|i}t          j                             d¦  «        5  |                      |d         |d         |d         ||||¬¦  «        \  }}ddd¦  «         n# 1 swxY w Y   t          j                             d¦  «        5  |                      |||d         |||¬¦  «         ddd¦  «         n# 1 swxY w Y   |S )zRForward pass for grounding (detection + segmentation) given input images and text.Nr   )Úbox_embeddingsÚbox_maskÚ
box_labelsc                ó’   •‡— i | ]B\  Š}‰t          |t          j        ¦  «        r t          j        ˆfd „‰D ¦   «         ¦  «        n|“ŒCS )c                ó    •— g | ]
}|‰         ‘ŒS r   r   )r   Úor°   s     €r   r   zBSAM3SemanticModel.forward_grounding.<locals>.<dictcomp>.<listcomp>4  s   ø€ Ð1Ð1Ð1 q˜a œdÐ1Ð1Ð1r   )Ú
isinstancer?   ÚTensorrZ   )r   r±   r°   Úoutss     @€r   ú
<dictcomp>z7SAM3SemanticModel.forward_grounding.<locals>.<dictcomp>3  se   øø€ ð ð ð Ù]aÐ]^Ð`a�µjÀÅEÄLÑ6QÔ6QÐX•5”9Ð1Ð1Ð1Ð1¨DÐ1Ñ1Ô1Ñ2Ô2Ð2ÐWXðð ð r   )Úbatchc                ó   — i | ]\  }}||“Œ	S r   r   )r   r°   r±   s      r   rÀ   z7SAM3SemanticModel.forward_grounding.<locals>.<dictcomp>9  s   € ÐKÐKÐK¡d a¨˜Q ÐKÐKÐKr   Úlanguage_featuresÚlanguage_maskzSAM3Image._encode_promptrV   r	   zSAM3Image._run_encoderr®   zSAM3Image._run_decoderrr   rn   ro   )rm   rn   rŠ   r   rb   rc   rx   z!SAM3Image._run_segmentation_heads)r   r®   rr   rb   rc   rŒ   )r   r'   r·   rY   r   r
   r¸   r¹   ÚappendÚforward_groundingr«   r   Ú_prepare_backbone_featuresÚupdaterH   r?   ÚprofilerÚrecord_functionrd   rZ   ry   r�   r²   )rJ   r®   r´   r\   Úirb   rQ   rS   r[   Ú	txt_featsÚ	txt_masksrP   Úgeo_maskrc   rx   r   rŒ   r¿   s                    @r   rÆ   z#SAM3SemanticModel.forward_grounding   sÊ  ø€ õ ˆx‰=Œ=˜4Ô.Ò.Ð.ØÐ$Ð(8Ô(GÔ(MÈaÔ(PÐTUÒ(UÐ(Uð ˆDÝ˜1�c (™mœm¨TÔ-@ÑAÔAð qð q�Ø�Ø#Ð/Ý#Ø'7Ô'FÀqÀqÀqÈ!ÈaÐRVÔReÑNeÐJeÐGeÔ'fØ!1Ô!:¸1¸qÀ4ÔCVÑ?VÐ;VÔ!WØ#3Ô#>¸q¸q¸qÀ!ÀaÈ$ÔJ]ÑF]ÐB]Ð?]Ô#^ðñ ô �Fð
 —’˜D×2Ò2°<ÀÈ!ÈaÐRVÔReÑNeÐJeÔAfÐhnÑoÔoÑpÔpÐpÐpðð ð ð ØeiÐjkÔel×erÒerÑetÔetðñ ô ð õ CLÔBfØ�,¥c¨(¡m¤mðC
ñ C
ô C
Ñ?ˆ�i °ð 	×ÒÐKÐK¨dÔ.B×.HÒ.HÑ.JÔ.JÐKÑKÔKÑLÔLÐLð !Ð!4Ô5°a°a°a¸°kÔBˆ	Ø  Ô1°(Ô;ˆ	ØÐ'Ý”×/Ò/Ð0JÑKÔKð xð xØ'+×':Ò':¸9ÀnÐVdÐfvÑ'wÔ'wÑ$�
˜Hðxð xð xñ xô xð xð xð xð xð xð xøøøð xð xð xð xå”Y 	¨:Ð6¸AÐ>Ñ>Ô>ˆFÝœ) Y°Ð$9¸qÐAÑAÔAˆKˆKàˆFØ#ˆKõ Œ^×+Ò+Ð,DÑEÔEð 	lð 	lØ×+Ò+¨I°~À~ÐW]Ð_jÑkÔkˆKð	lð 	lð 	lñ 	lô 	lð 	lð 	lð 	lð 	lð 	lð 	løøøð 	lð 	lð 	lð 	là˜|Ð,ˆõ Œ^×+Ò+Ð,DÑEÔEð 		ð 		Ø×'Ò'Ø"Ð#:Ô;Ø% kÔ2Ø$ ^Ô4ØØØ'Ø'ð (ñ ô ‰GˆC�ð		ð 		ð 		ñ 		ô 		ð 		ð 		ð 		ð 		ð 		ð 		øøøð 		ð 		ð 		ð 		õ Œ^×+Ò+Ð,OÑPÔPð 	ð 	Ø×(Ò(ØØ)Ø&1Ð2IÔ&JØØ'Øð )ñ ô ð ð	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	ð ˆ
sH   Æ7GÇG#Ç&G#È>I$É$I(É+I(Ê2KËKËKË:"L(Ì(L,Ì/L,Útextú	list[str]c                óR   — | j                              |¦  «        | _        || _        dS )z2Set the text embeddings for the given class names.N)r(   Úforward_textrH   rI   )rJ   rÏ   s     r   Úset_classeszSAM3SemanticModel.set_classesd  s%   € à#œ}×9Ò9¸$Ñ?Ô?ˆÔØˆŒ
ˆ
ˆ
r   Úimgszútuple[int, int]c                ó:   — | j                              |¦  «         dS )z!Set the image size for the model.N)r(   Ú	set_imgsz)rJ   rÔ   s     r   r×   zSAM3SemanticModel.set_imgszi  s   € àŒ×Ò Ñ&Ô&Ð&Ð&Ð&r   )Nr	   TNTTTNTFFFr   )r(   r   r)   r*   r+   r*   r,   r*   r-   r*   r.   r*   r/   r*   r0   r&   )NNN)N)re   rf   )NF)r®   r³   r´   rµ   r\   r
   )rÏ   rÐ   )rÔ   rÕ   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r$   Ú__annotations__r'   r3   rd   ry   r�   r‰   r²   rÆ   rÓ   r×   Ú__classcell__)rN   s   @r   r"   r"      sc  ø€ € € € € € ØMÐMà€NÐÐÐÑà€NÐÐÐÑð ØØØØ#'Ø!%Ø,0ØØ!Ø+0Ø/4Ø-2Ø)*ð#>ð >ð >ð >ð >ð >ð >ðL !ØØð#ð #ð #ð #ðL -1ð!ð !ð !ð !ð !ðF)ð )ð )ðd Ø ð0bð 0bð 0bð 0bðd3ð 3ð 3ð: imðBð Bð Bð Bð BðHð ð ð ð
'ð 'ð 'ð 'ð 'ð 'ð 'ð 'r   r"   )TT)Ú
__future__r   rv   r   r?   Úultralytics.nn.modules.utilsr   Úultralytics.utils.opsr   Úmodules.samr   Úgeometry_encodersr
   Úvl_combinerr   r    r@   ÚModuler"   r   r   r   ú<module>rå      sä   ðð
 #Ð "Ð "Ð "Ð "Ð "à Ð Ð Ð Ð Ð à €€€à 8Ð 8Ð 8Ð 8Ð 8Ð 8Ø +Ð +Ð +Ð +Ð +Ð +à #Ð #Ð #Ð #Ð #Ð #Ø %Ð %Ð %Ð %Ð %Ð %Ø 'Ð 'Ð 'Ð 'Ð 'Ð 'ð-ð -ð -ð -ðM'ð M'ð M'ð M'ð M'˜œœñ M'ô M'ð M'ð M'ð M'r   