§
    9²¸jÎœ  ã                   óä  — d dl mZ d dlmZ d dlZd dlmZ d dlmZ ddl	m
Z ddlmZmZ ddlmZ dd	lmZmZmZmZmZmZmZ dd
lmZmZ ddlmZmZmZ ddl m!Z!m"Z" ddl#m$Z$m%Z% ddl&m'Z' ddl(m)Z)m*Z*m+Z+m,Z, ddl-m.Z. ddl/m0Z0 ddl1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z: ddl;m<Z<m=Z=m>Z>m?Z? ddl@mAZA  e,jB        eC¦  «        ZD e*d¬¦  «        e G d„ de0e¦  «        ¦   «         ¦   «         ZE e*d¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         ZF G d„ de?¦  «        ZG G d„ de<¦  «        ZH G d „ d!ejI        ¦  «        ZJ G d"„ d#e4¦  «        ZK G d$„ d%e7¦  «        ZL G d&„ d'e8¦  «        ZM G d(„ d)e2¦  «        ZN G d*„ d+e¦  «        ZOdZP G d,„ d-e6¦  «        ZQd.eRd/eeReReReRgeSf         fd0„ZT G d1„ d2e5¦  «        ZU G d3„ d4e3¦  «        ZV G d5„ d6ejW        ¦  «        ZXdLd7ejY        d8ejZ        dz  d/ejY        fd9„Z[d:ed;ejY        d<ejY        dz  d=edz  d>ejY        dz  d?ejY        d/e\fd@„Z] G dA„ dBe>¦  «        Z^ G dC„ dDe=¦  «        Z_ e*dE¬F¦  «         G dG„ dHeeQ¦  «        ¦   «         Z` G dI„ dJeeQ¦  «        Zag dK¢ZbdS )Mé    )ÚCallable)ÚAnyN)Ústricté   )Úinitialization)ÚCacheÚDynamicCache)ÚPreTrainedConfig)Ú_preprocess_mask_argumentsÚblockwise_overlayÚcreate_causal_maskÚcreate_masks_for_generateÚ!create_sliding_window_causal_maskÚmaybe_pad_block_sequence_idsÚsliding_window_overlay)Ú GenericForSequenceClassificationÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚBaseModelOutputWithPoolingÚ SequenceClassifierOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmaybe_autocasté   )ÚGemma2Config)	ÚGemma2AttentionÚGemma2ForCausalLMÚ	Gemma2MLPÚGemma2ModelÚGemma2PreTrainedModelÚGemma2RMSNormÚGemma2RotaryEmbeddingÚapply_rotary_pos_embÚeager_attention_forward)ÚPaliGemmaCausalLMOutputWithPastÚ!PaliGemmaForConditionalGenerationÚPaliGemmaModelÚPaligemmaModelOutputWithPast)ÚSiglipVisionConfigzgoogle/gemma-3-4b-it)Ú
checkpointc            
       óø   ‡ — e Zd ZU dZdZddddddddddœ	Zddd	œZd
Zee	d<   dZ
ee	d<   dZee         dz  e	d<   dZedz  e	d<   dZedz  e	d<   dZedz  e	d<   dZedz  e	d<   d„ Zdeeef         fˆ fd„Zd„ Zˆ xZS )ÚGemma3TextConfigaé  
    query_pre_attn_scalar (`float`, *optional*, defaults to 256):
        scaling factor used on the attention scores
    final_logit_softcapping (`float`, *optional*):
        Scaling factor when applying tanh softcapping on the logits.
    attn_logit_softcapping (`float`, *optional*):
        Scaling factor when applying tanh softcapping on the attention scores.
    use_bidirectional_attention (`bool`, *optional*, defaults to `False`):
        If True, the model will attend to all text tokens instead of using a causal mask. This does not change
        behavior for vision tokens.

    ```python
    >>> from transformers import Gemma3TextModel, Gemma3TextConfig
    >>> # Initializing a Gemma3Text gemma3_text-7b style configuration
    >>> configuration = Gemma3TextConfig()
    >>> # Initializing a model from the gemma3_text-7b style configuration
    >>> model = Gemma3TextModel(configuration)
    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```
    Úgemma3_textÚcolwiseÚreplicated_with_grad_allreduceÚrowwise)	zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.q_normzlayers.*.self_attn.k_normzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projg    €„.Ag     ˆÃ@)ÚglobalÚlocali@  Ú
vocab_sizei   Úmax_position_embeddingsNÚlayer_typesÚfinal_logit_softcappingÚattn_logit_softcappingÚrope_parametersFÚuse_bidirectional_attentionc                 óê   ‡ — ‰ j         r‰ j        dz  dz   ‰ _        |                     dd¦  «        ‰ _        ‰ j        €%ˆ fd„t          ‰ j        ¦  «        D ¦   «         ‰ _        t          j        di |¤Ž d S )Nr!   é   Úsliding_window_patterné   c                 óL   •— g | ] }t          |d z   ‰j        z  ¦  «        rdnd‘Œ!S )rB   Úsliding_attentionÚfull_attention)ÚboolÚ_sliding_window_pattern)Ú.0ÚiÚselfs     €úy/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/gemma3/modular_gemma3.pyú
<listcomp>z2Gemma3TextConfig.__post_init__.<locals>.<listcomp>{   sI   ø€ ð  ð  ð  àõ (,¨Q°©U°dÔ6RÑ,RÑ'SÔ'SÐiÐ#Ð#ÐYið ð  ð  ó    © )	r@   Úsliding_windowÚgetrI   r<   ÚrangeÚnum_hidden_layersr
   Ú__post_init__)rL   Úkwargss   ` rM   rU   zGemma3TextConfig.__post_init__s   s›   ø€ ØÔ+ð 	AØ#'Ô#6¸!Ñ#;¸qÑ"@ˆDÔð (.§z¢zÐ2JÈAÑ'NÔ'NˆÔ$àÔÐ#ð ð  ð  ð  å˜tÔ5Ñ6Ô6ð ñ  ô  ˆDÔõ
 	Ô&Ð0Ð0¨Ð0Ð0Ð0Ð0Ð0rO   Úreturnc                 óŠ   •— t          ¦   «                              ¦   «         }t          | dd¦  «        r| j        dz
  dz  |d<   |S )Nr@   FrB   r!   rQ   )ÚsuperÚto_dictÚgetattrrQ   )rL   ÚoutputÚ	__class__s     €rM   rZ   zGemma3TextConfig.to_dict‚   sK   ø€ Ý‘”—’Ñ"Ô"ˆõ �4Ð6¸Ñ>Ô>ð 	EØ(,Ô(;¸aÑ(?À1Ñ'DˆFÐ#Ñ$ØˆrO   c                 ór  — |                      dd ¦  «        }ddiddidœ}| j        �| j        n|| _        |� | j        d                              |¦  «         | j                             d¦  «        €ddi| j        d<   | j        d                              d|                      d| j        d         ¦  «        ¦  «         | j                             d¦  «        €ddi| j        d<   | j        d                              d|                      d	| j        d
         ¦  «        ¦  «         |                      ¦   «          |S )NÚrope_scalingÚ	rope_typeÚdefault)rF   rG   rG   Ú
rope_thetar8   rF   Úrope_local_base_freqr9   )Úpopr?   ÚupdaterR   Ú
setdefaultÚdefault_thetaÚstandardize_rope_params)rL   rV   r_   Údefault_rope_paramss       rM   Úconvert_rope_params_to_dictz,Gemma3TextConfig.convert_rope_params_to_dict‹   s_  € Ø—z’z .°$Ñ7Ô7ˆð
 #.¨yÐ!9Ø*¨IÐ6ð
ð 
Ðð 8<Ô7KÐ7W˜tÔ3Ð3Ð]pˆÔØÐ#ØÔ Ð!1Ô2×9Ò9¸,ÑGÔGÐGð Ô×#Ò#Ð$4Ñ5Ô5Ð=Ø6AÀ9Ð5MˆDÔ Ð!1Ñ2ØÔÐ-Ô.×9Ò9Ø˜&Ÿ*š* \°4Ô3EÀhÔ3OÑPÔPñ	
ô 	
ð 	
ð Ô×#Ò#Ð$7Ñ8Ô8Ð@Ø9DÀiÐ8PˆDÔ Ð!4Ñ5ØÔÐ0Ô1×<Ò<Ø˜&Ÿ*š*Ð%;¸TÔ=OÐPWÔ=XÑYÔYñ	
ô 	
ð 	
ð
 	×$Ò$Ñ&Ô&Ð&ØˆrO   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚbase_model_tp_planrg   r:   ÚintÚ__annotations__r;   r<   ÚlistÚstrr=   Úfloatr>   r?   Údictr@   rH   rU   r   rZ   rj   Ú__classcell__©r]   s   @rM   r3   r3   D   sA  ø€ € € € € € ðð ð, €Jà%.Ø%.Ø%.Ø%EØ%EØ%.Ø"+Ø )Ø"+ð
ð 
Ðð  +°XÐ>Ð>€Mà€J�ÐÐÑØ#*Ð˜SÐ*Ð*Ñ*Ø$(€K��c”˜TÑ!Ð(Ð(Ñ(Ø,0Ð˜U T™\Ð0Ð0Ñ0Ø+/Ð˜E D™LÐ/Ð/Ñ/Ø#'€O�T˜D‘[Ð'Ð'Ñ'Ø/4Ð ¨¡Ð4Ð4Ñ4ð1ð 1ð 1ð˜˜c 3˜hœð ð ð ð ð ð ðð ð ð ð ð ð rO   r3   c                   ó
  ‡ — e Zd ZU dZdZddddœZeedœZdZ	ee
eef         z  dz  ed	<   dZee
eef         z  dz  ed
<   dZedz  ed<   dZedz  ed<   dZedz  ed<   dZedz  ed<   dZedz  ed<   dZedz  ed<   ˆ fd„Zˆ xZS )ÚGemma3Configa  
    mm_tokens_per_image (`int`, *optional*, defaults to 256):
        The number of tokens per image embedding.
    boi_token_index (`int`, *optional*, defaults to 255999):
        The begin-of-image token index to wrap the image prompt.
    eoi_token_index (`int`, *optional*, defaults to 256000):
        The end-of-image token index to wrap the image prompt.

    Example:

    ```python
    >>> from transformers import Gemma3ForConditionalGeneration, Gemma3Config, SiglipVisionConfig, Gemma3TextConfig

    >>> # Initializing a Siglip-like vision config
    >>> vision_config = SiglipVisionConfig()

    >>> # Initializing a Gemma3 Text config
    >>> text_config = Gemma3TextConfig()

    >>> # Initializing a Gemma3 gemma-3-4b style configuration
    >>> configuration = Gemma3Config(vision_config, text_config)

    >>> # Initializing a model from the gemma-3-4b style configuration
    >>> model = Gemma3ForConditionalGeneration(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```Úgemma3Úimage_token_indexÚboi_token_indexÚeoi_token_index)Úimage_token_idÚboi_token_idÚeoi_token_id)Útext_configÚvision_configNr‚   rƒ   é   Úmm_tokens_per_imageiÿç i è i   g{®Gáz”?Úinitializer_rangeTÚtie_word_embeddingsc                 óÎ  •— | j         €.t          ¦   «         | _         t                               d¦  «         n0t	          | j         t
          ¦  «        rt          di | j         ¤Ž| _         t	          | j        t
          ¦  «        rt          di | j        ¤Ž| _        n4| j        €-t          ¦   «         | _        t                               d¦  «          t          ¦   «         j	        di |¤Ž d S )Nz@text_config is None, using default Gemma3TextConfig text config.zFvision_config is None, using default SiglipVisionConfig vision config.rP   )
r‚   r3   ÚloggerÚinfoÚ
isinstancerv   rƒ   r0   rY   rU   )rL   rV   r]   s     €rM   rU   zGemma3Config.__post_init__Ý   sÞ   ø€ ØÔÐ#Ý/Ñ1Ô1ˆDÔÝ�KŠKÐZÑ[Ô[Ð[Ð[Ý˜Ô(­$Ñ/Ô/ð 	DÝ/ÐCÐC°$Ô2BÐCÐCˆDÔå�dÔ(­$Ñ/Ô/ð 	bÝ!3Ð!IÐ!I°dÔ6HÐ!IÐ!IˆDÔÐØÔÐ'Ý!3Ñ!5Ô!5ˆDÔÝ�KŠKÐ`ÑaÔaÐaà�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'rO   )rk   rl   rm   rn   ro   Úattribute_mapr3   r0   Úsub_configsr‚   rv   rt   r   rr   rƒ   r…   rq   r}   r~   r|   r†   ru   r‡   rH   rU   rw   rx   s   @rM   rz   rz   ©   s9  ø€ € € € € € ðð ð: €Jà-Ø)Ø)ðð €Mð (Ø+ðð €Kð
 =A€KÐ! D¨¨c¨¤NÑ2°TÑ9Ð@Ð@Ñ@Ø@D€MÐ%¨¨S°#¨X¬Ñ6¸Ñ=ÐDÐDÑDØ&)Ð˜˜t™Ð)Ð)Ñ)Ø")€O�S˜4‘ZÐ)Ð)Ñ)Ø")€O�S˜4‘ZÐ)Ð)Ñ)Ø$+Ð�s˜T‘zÐ+Ð+Ñ+Ø&*Ð�u˜t‘|Ð*Ð*Ñ*Ø'+Ð˜ ™Ð+Ð+Ñ+ð(ð (ð (ð (ð (ð (ð (ð (ð (rO   rz   c                   ó   — e Zd ZdS )ÚGemma3ModelOutputWithPastN©rk   rl   rm   rP   rO   rM   r�   r�   í   ó   € € € € € Ø€DrO   r�   c                   ó   — e Zd ZdS )ÚGemma3CausalLMOutputWithPastNr�   rP   rO   rM   r“   r“   ñ   r‘   rO   r“   c            	       óP   ‡ — e Zd ZdZd
dedededefˆ fd„Zdej        fˆ fd	„Z	ˆ xZ
S )ÚGemma3TextScaledWordEmbeddingz\
    This module overrides nn.Embeddings' forward by multiplying with embeddings scale.
    ç      ð?Únum_embeddingsÚembedding_dimÚpadding_idxÚembed_scalec                 ó¶   •— t          ¦   «                              |||¦  «         || _        t          j        t          j        |¦  «        d¬¦  «        | _        d S )NF©Ú
persistent)rY   Ú__init__Úscalar_embed_scaleÚnnÚBufferÚtorchÚtensorrš   )rL   r—   r˜   r™   rš   r]   s        €rM   rž   z&Gemma3TextScaledWordEmbedding.__init__ú   sN   ø€ Ý‰Œ×Ò˜¨¸ÑDÔDÐDØ"-ˆÔÝœ9¥U¤\°+Ñ%>Ô%>È5ÐQÑQÔQˆÔÐÐrO   Ú	input_idsc                 ó�   •— t          ¦   «                              |¦  «        | j                             | j        j        ¦  «        z  S ©N)rY   Úforwardrš   ÚtoÚweightÚdtype)rL   r¤   r]   s     €rM   r§   z%Gemma3TextScaledWordEmbedding.forwardÿ   s4   ø€ Ý‰wŒw�Š˜yÑ)Ô)¨DÔ,<×,?Ò,?ÀÄÔ@QÑ,RÔ,RÑRÐRrO   )r–   )rk   rl   rm   rn   rq   ru   rž   r¢   ÚTensorr§   rw   rx   s   @rM   r•   r•   õ   s¦   ø€ € € € € ðð ðRð R sð R¸3ð RÈSð RÐ_dð Rð Rð Rð Rð Rð Rð
S ¤ð Sð Sð Sð Sð Sð Sð Sð Sð Sð SrO   r•   c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )Ú	Gemma3MLPÚconfigc                 óJ   •— t          ¦   «                              |¦  «         d S r¦   ©rY   rž   ©rL   r®   r]   s     €rM   rž   zGemma3MLP.__init__  s!   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ð Ð rO   )rk   rl   rm   r3   rž   rw   rx   s   @rM   r­   r­     sE   ø€ € € € € ð!Ð/ð !ð !ð !ð !ð !ð !ð !ð !ð !ð !rO   r­   c                   ó*   ‡ — e Zd Zddedefˆ fd„Zˆ xZS )ÚGemma3RMSNormç�íµ ÷Æ°>ÚdimÚepsc                 óN   •— t          ¦   «                              ||¬¦  «         d S )N©rµ   r¶   r°   )rL   rµ   r¶   r]   s      €rM   rž   zGemma3RMSNorm.__init__	  s&   ø€ Ý‰Œ×Ò˜S cÐÑ*Ô*Ð*Ð*Ð*rO   )r´   )rk   rl   rm   rq   ru   rž   rw   rx   s   @rM   r³   r³     sP   ø€ € € € € ð+ð +˜Cð + eð +ð +ð +ð +ð +ð +ð +ð +ð +ð +rO   r³   c            	       ó”   — e Zd Zddefd„Z	 d	dededz  deej        e	f         fd„Z
 ej        ¦   «         ed„ ¦   «         ¦   «         ZdS )
ÚGemma3RotaryEmbeddingNr®   c           	      ó²  — t           j                             | ¦  «         |j        | _        |j        | _        || _        t          t          |j	        ¦  «        ¦  «        | _	        i | _
        | j	        D ]ã}| j        j        |         }|€Œ|d         | j
        |<   | j        }| j
        |         dk    rt          | j
        |                  } || j        ||¬¦  «        \  }}t          | |› d�t          j        |d¬¦  «        ¦  «         t          | |› d�t          j        |                     ¦   «         d¬¦  «        ¦  «         t          | |› d�|¦  «         Œäd S )	Nr`   ra   ©Ú
layer_typeÚ	_inv_freqFrœ   Ú_original_inv_freqÚ_attention_scaling)r    ÚModulerž   r;   Úmax_seq_len_cachedÚoriginal_max_seq_lenr®   ÚsortedÚsetr<   r`   r?   Úcompute_default_rope_parametersr   Úsetattrr¡   Úclone)rL   r®   Údevicer½   Úrope_paramsÚrope_init_fnÚcurr_inv_freqÚcurr_attention_scalings           rM   rž   zGemma3RotaryEmbedding.__init__  sk  € Ý
Œ	×Ò˜4Ñ Ô Ð Ø"(Ô"@ˆÔØ$*Ô$BˆÔ!ØˆŒÝ!¥# fÔ&8Ñ"9Ô"9Ñ:Ô:ˆÔØˆŒØÔ*ð 	Uð 	UˆJØœ+Ô5°jÔAˆKØÐ"Øà)4°[Ô)AˆDŒN˜:Ñ&Ø%)Ô%IˆLØŒ~˜jÔ)¨YÒ6Ð6Ý2°4´>À*Ô3MÔN�Ø4@°LÀÄÈfÐakÐ4lÑ4lÔ4lÑ1ˆMÐ1Ý�D˜ZÐ2Ð2Ð2µB´I¸mÐX]Ð4^Ñ4^Ô4^Ñ_Ô_Ð_Ý�D˜ZÐ;Ð;Ð;½R¼YÀ}×GZÒGZÑG\ÔG\ÐinÐ=oÑ=oÔ=oÑpÔpÐpÝ�D˜ZÐ;Ð;Ð;Ð=SÑTÔTÐTÐTð	Uð 	UrO   r½   rW   c                 óð   — | j         |         d         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «        |z  z  z  }|                     |¦  «        |fS )a†  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            layer_type (`str`):
                The current layer type if the model has different RoPE parameters per type.
                Should not be used unless `config.layer_types is not None`

        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        rb   Úhead_dimNr–   r   r!   ©rª   )r?   r[   Úhidden_sizeÚnum_attention_headsr¢   Úarangeru   r¨   )r®   rÉ   r½   rV   Úbaserµ   Úattention_factorÚinv_freqs           rM   rÆ   z5Gemma3RotaryEmbedding.compute_default_rope_parameters#  s   € ð" Ô% jÔ1°,Ô?ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐà˜$¥5¤<°°3¸ÅÄÐ#MÑ#MÔ#MÐPSÑ#SÑTÑUˆØ�{Š{˜6Ñ"Ô"Ð$4Ð4Ð4rO   c                 ó(  — t          | |› d�¦  «        }t          | |› d�¦  «        }|d d d …d f                              |j        d         dd¦  «                             t          j        |j        ¬¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d	¬
¦  «        5  ||z                       dd¦  «        }	t	          j        |	|	fd¬¦  «        }
|
                     ¦   «         |z  }|
                     ¦   «         |z  }d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬¦  «        |                     |j        ¬¦  «        fS )Nr¾   rÀ   r   éÿÿÿÿrB   )rª   rÉ   ÚmpsÚcpuF)Údevice_typeÚenabledr!   ©rµ   rÐ   )r[   ÚexpandÚshaper¨   r¢   ru   rÉ   r‹   Útypert   r    Ú	transposeÚcatÚcosÚsinrª   )rL   ÚxÚposition_idsr½   rÖ   Úattention_scalingÚinv_freq_expandedÚposition_ids_expandedrÛ   ÚfreqsÚembrã   rä   s                rM   r§   zGemma3RotaryEmbedding.forward<  sÈ  € õ ˜4 JÐ!9Ð!9Ð!9Ñ:Ô:ˆÝ# D¨ZÐ*KÐ*KÐ*KÑLÔLÐð �T˜1˜1˜1˜d�]Ô#×*Ò*¨<Ô+=¸aÔ+@À"ÀaÑHÔH×KÒKÕRWÔR]ÐfgÔfnÐKÑoÔoð 	ð !-¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆå¨¸UÐCÑCÔCð 	0ð 	0Ø&Ð)>Ñ>×IÒIÈ!ÈQÑOÔOˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)Ð/Ñ/ˆCØ—'’'‘)”)Ð/Ñ/ˆCð		0ð 	0ð 	0ñ 	0ô 	0ð 	0ð 	0ð 	0ð 	0ð 	0ð 	0øøøð 	0ð 	0ð 	0ð 	0ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   Ã'A EÅEÅEr¦   )NN)rk   rl   rm   r3   rž   rt   Útupler¢   r«   ru   rÆ   Úno_gradr   r§   rP   rO   rM   rº   rº     s±   € € € € € ðUð UÐ/ð Uð Uð Uð Uð, IMð5ð 5Ø ð5Ø;>À¹:ð5à	ˆuŒ|˜UÐ"Ô	#ð5ð 5ð 5ð 5ð2 €U„]�_„_Øð<ð <ñ Ôñ „_ð<ð <ð <rO   rº   c                   óÎ   ‡ — e Zd Zdedefˆ fd„Z	 	 	 ddej        dej        dej        dz  dedz  d	e	e
         d
eej        ej        dz  eej                 dz  f         fd„Zˆ xZS )ÚGemma3Attentionr®   Ú	layer_idxc                 óB  •— t          ¦   «                              ||¦  «         | j        dk    r|j        nd | _        | j        dk    | _        | j        j         | _        t          |j	        |j
        ¬¦  «        | _        t          |j	        |j
        ¬¦  «        | _        d S )NrF   r¸   )rY   rž   r½   rQ   Ú
is_slidingr®   r@   Ú	is_causalr³   rÏ   Úrms_norm_epsÚq_normÚk_norm©rL   r®   rð   r]   s      €rM   rž   zGemma3Attention.__init__T  s�   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+Ø7;´ÐJ]Ò7]Ð7]˜fÔ3Ð3ÐcgˆÔØœ/Ð-@Ò@ˆŒØ!œ[ÔDÐDˆŒå#¨¬¸VÔ=PÐQÑQÔQˆŒÝ#¨¬¸VÔ=PÐQÑQÔQˆŒˆˆrO   NÚhidden_statesÚposition_embeddingsÚattention_maskÚpast_key_valuesrV   rW   c                 ó‚  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }	|                      |¦  «                             |¦  «                             dd¦  «        }
|                      |¦  «        }|                      |	¦  «        }	|\  }}t          ||	||¦  «        \  }}	|�| 
                    |	|
| j        ¦  «        \  }	}
t          j        | j        j        t           ¦  «        } || ||	|
|f| j        r| j        nd| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )NrØ   rB   r!   g        )ÚdropoutÚscalingrQ   )rß   rÏ   Úq_projÚviewrá   Úk_projÚv_projrõ   rö   r*   re   rð   r   Úget_interfacer®   Ú_attn_implementationr+   ÚtrainingÚattention_dropoutrþ   rQ   ÚreshapeÚ
contiguousÚo_proj)rL   rø   rù   rú   rû   rV   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesrã   rä   Úattention_interfaceÚattn_outputÚattn_weightss                   rM   r§   zGemma3Attention.forward]  sé  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆà—{’{ <Ñ0Ô0ˆØ—[’[ Ñ,Ô,ˆ
à&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð /3¬mÐD�DÔ*Ð*ÀØ”LØÔ.ð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(rO   )NNN)rk   rl   rm   r3   rq   rž   r¢   r«   r   r   r   rì   r§   rw   rx   s   @rM   rï   rï   S  sè   ø€ € € € € ðRÐ/ð R¸Cð Rð Rð Rð Rð Rð Rð -1Ø.2Ø(,ð*)ð *)à”|ð*)ð #œ\ð*)ð œ tÑ+ð	*)ð
  ™ð*)ð Ð+Ô,ð*)ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	Mð*)ð *)ð *)ð *)ð *)ð *)ð *)ð *)rO   rï   c                   óà   ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 ddej        dej        dej        dz  dej        dz  d	e	dz  d
e
e         deej        eej        ej        f         dz  f         fd„Zˆ xZS )ÚGemma3DecoderLayerr®   rð   c                 óÐ  •— t          ¦   «                              ¦   «          || _        |j        | _        || _        t          ||¬¦  «        | _        t          |¦  «        | _        t          | j        |j
        ¬¦  «        | _        t          | j        |j
        ¬¦  «        | _        t          | j        |j
        ¬¦  «        | _        t          | j        |j
        ¬¦  «        | _        d S )N)r®   rð   ©r¶   )rY   rž   r®   rÑ   rð   rï   Ú	self_attnr­   Úmlpr³   rô   Úinput_layernormÚpost_attention_layernormÚpre_feedforward_layernormÚpost_feedforward_layernormr÷   s      €rM   rž   zGemma3DecoderLayer.__init__‹  sÇ   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ"ˆŒÝ(°À)ÐLÑLÔLˆŒÝ˜VÑ$Ô$ˆŒÝ,¨TÔ-=À6ÔCVÐWÑWÔWˆÔÝ(5°dÔ6FÈFÔL_Ð(`Ñ(`Ô(`ˆÔ%Ý)6°tÔ7GÈVÔM`Ð)aÑ)aÔ)aˆÔ&Ý*7¸Ô8HÈfÔNaÐ*bÑ*bÔ*bˆÔ'Ð'Ð'rO   Nrø   rù   rú   ræ   rû   rV   rW   c           	      ó   — |}|                       |¦  «        } | j        d|||||dœ|¤Ž\  }}|                      |¦  «        }||z   }|}|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }||z   }|S )N)rø   rù   rú   ræ   rû   rP   )r  r  r  r  r  r  )	rL   rø   rù   rú   ræ   rû   rV   ÚresidualÚ_s	            rM   r§   zGemma3DecoderLayer.forward—  sÂ   € ð !ˆà×,Ò,¨]Ñ;Ô;ˆà)˜4œ>ð 
Ø'Ø 3Ø)Ø%Ø+ð
ð 
ð ð
ð 
Ñˆ�qð ×5Ò5°mÑDÔDˆØ  =Ñ0ˆà ˆØ×6Ò6°}ÑEÔEˆØŸš Ñ/Ô/ˆØ×7Ò7¸ÑFÔFˆØ  =Ñ0ˆàÐrO   )NNNN)rk   rl   rm   r3   rq   rž   r¢   r«   Ú
LongTensorr   r   r   rì   ÚFloatTensorr§   rw   rx   s   @rM   r  r  Š  sÿ   ø€ € € € € ð
cÐ/ð 
c¸Cð 
cð 
cð 
cð 
cð 
cð 
cð -1Ø.2Ø04Ø(,ðð à”|ðð #œ\ðð œ tÑ+ð	ð
 Ô&¨Ñ-ðð  ™ðð Ð+Ô,ðð 
ˆuÔ  %¨Ô(9¸5Ô;LÐ(LÔ"MÐPTÑ"TÐTÔ	Uðð ð ð ð ð ð ð rO   r  c                   óL   — e Zd ZdZdZdgZ ej        ¦   «         d„ ¦   «         ZdS )ÚGemma3PreTrainedModelÚmodel)ÚimageÚtextr  c                 ó²  — t          j        | |¦  «         t          |t          ¦  «        rt	          j        |j        ¦  «         d S d|j        j        v rt	          j        |j	        ¦  «         d S t          |t          ¦  «        r!t	          j        |j        |j        ¦  «         d S t          |t          ¦  «        r›|j        D ]•}|j        }|j        |         dk    rt$          |j        |                  } ||j        |¬¦  «        \  }}t	          j        t+          ||› d�¦  «        |¦  «         t	          j        t+          ||› d�¦  «        |¦  «         Œ”d S d S )NÚRMSNormra   r¼   r¾   r¿   )r   Ú_init_weightsr‹   ÚGemma3MultiModalProjectorÚinitÚzeros_Úmm_input_projection_weightr]   rk   r©   r•   Ú	constant_rš   rŸ   rº   r<   rÆ   r`   r   r®   Úcopy_r[   )rL   Úmoduler½   rË   rÌ   r  s         rM   r(  z#Gemma3PreTrainedModel._init_weightsÀ  sr  € åÔ% d¨FÑ3Ô3Ð3Ý�fÕ7Ñ8Ô8ð 	^ÝŒK˜Ô9Ñ:Ô:Ð:Ð:Ð:à˜&Ô*Ô3Ð3Ð3ÝŒK˜œÑ&Ô&Ð&Ð&Ð&Ý˜Õ =Ñ>Ô>ð 		^ÝŒN˜6Ô-¨vÔ/HÑIÔIÐIÐIÐIÝ˜Õ 5Ñ6Ô6ð 	^Ø$Ô0ð ^ð ^�
Ø%ÔE�ØÔ# JÔ/°9Ò<Ð<Ý#6°vÔ7GÈ
Ô7SÔ#T�LØ#/ <°´È*Ð#UÑ#UÔ#UÑ �˜qÝ”
�7 6¨jÐ+CÐ+CÐ+CÑDÔDÀmÑTÔTÐTÝ”
�7 6¨jÐ+LÐ+LÐ+LÑMÔMÈ}Ñ]Ô]Ð]Ð]ð	^ð 	^ð^ð ^rO   N)	rk   rl   rm   Úbase_model_prefixÚinput_modalitiesÚ_no_split_modulesr¢   rí   r(  rP   rO   rM   r"  r"  »  sO   € € € € € ØÐØ(ÐØ-Ð.Ðà€U„]�_„_ð^ð ^ñ „_ð^ð ^ð ^rO   r"  rQ   rW   c           
      óZ   ‡ — dt           dt           dt           dt           dt          f
ˆ fd„}|S )zA
    Enables a bidirectional mask within the sliding window.
    Ú	batch_idxÚhead_idxÚq_idxÚkv_idxrW   c                 ó0   •— t          ||z
  ¦  «        ‰k     S )z“A token can attend to any other token if their absolute distance is within
        the (exclusive) sliding window size (distance < sliding_window).)Úabs)r4  r5  r6  r7  rQ   s       €rM   Ú
inner_maskz1_bidirectional_window_overlay.<locals>.inner_maskÙ  s   ø€ õ �5˜6‘>Ñ"Ô" ^Ò3Ð3rO   )rq   rH   )rQ   r:  s   ` rM   Ú_bidirectional_window_overlayr;  Ô  sL   ø€ ð
4�cð 4­Sð 4½ð 4Åcð 4Ídð 4ð 4ð 4ð 4ð 4ð 4ð
 ÐrO   c                   óÂ   ‡ — e Zd ZU eed<   dZdefˆ fd„Z	 	 	 	 	 	 ddej        dz  dej	        dz  dej        dz  de
dz  d	ej        dz  d
edz  dee         defd„Zˆ xZS )ÚGemma3TextModelr®   ©r%  c                 ó²   •— t          ¦   «                              |¦  «         t          |j        |j        | j        | j        j        dz  ¬¦  «        | _        d S )Nç      à?)rš   )rY   rž   r•   r:   rÑ   r™   r®   Úembed_tokensr±   s     €rM   rž   zGemma3TextModel.__init__å  sY   ø€ Ý‰Œ×Ò˜Ñ Ô Ð õ :ØÔ˜vÔ1°4Ô3CÐQUÔQ\ÔQhÐjmÑQmð
ñ 
ô 
ˆÔÐÐrO   Nr¤   rú   ræ   rû   Úinputs_embedsÚ	use_cacherV   rW   c           	      óÌ  — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|r|€t          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   }| 	                    d¦  «        }t          |x}	t          ¦  «        sh| j        ||||dœ}
|
                     ¦   «         }| j        j        r"d„ |
d<   t          | j        j        ¦  «        |d<   t!          di |
¤Žt#          di |¤Žd	œ}	|}i }t%          | j        j        ¦  «        D ]}|                      |||¦  «        ||<   Œt+          | j        d | j        j        …         ¦  «        D ]?\  }} ||f|	| j        j        |                  || j        j        |                  ||d
œ|¤Ž}Œ@|                      |¦  «        }t3          ||¬¦  «        S )Nú:You must specify exactly one of input_ids or inputs_embeds)r®   r   rB   ©rÉ   ©r®   rB  rú   rû   ræ   c                  óB   — t          j        dt           j        ¬¦  «        S )NTrÐ   )r¢   r£   rH   )Úargss    rM   ú<lambda>z)Gemma3TextModel.forward.<locals>.<lambda>  s   € ÅÄÈTÕY^ÔYcÐ@dÑ@dÔ@d€ rO   Úor_mask_function©rG   rF   )rú   rù   ræ   rû   )Úlast_hidden_staterû   rP   )Ú
ValueErrorrA  r	   r®   Úget_seq_lengthr¢   rÓ   rß   rÉ   Ú	unsqueezer‹   rv   Úcopyr@   r;  rQ   r   r   rÅ   r<   Ú
rotary_embÚ	enumerateÚlayersrT   Únormr   )rL   r¤   rú   ræ   rû   rB  rC  rV   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsÚsliding_mask_kwargsrø   rù   r½   rK   Údecoder_layers                    rM   r§   zGemma3TextModel.forwardí  sq  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLõ °Ð?Ð-ÅÑFÔFð 	ð œ+Ø!.Ø"0Ø#2Ø ,ðð ˆKð #.×"2Ò"2Ñ"4Ô"4ÐàŒ{Ô6ð tØ2dÐ2d�Ð.Ñ/Ý:WÐX\ÔXcÔXrÑ:sÔ:sÐ#Ð$6Ñ7õ #5Ð"CÐ"C°{Ð"CÐ"CÝ%FÐ%]Ð%]ÐI\Ð%]Ð%]ð#ð #Ðð &ˆØ ÐÝ˜dœkÔ5Ñ6Ô6ð 	gð 	gˆJØ.2¯oªo¸mÈ\Ð[eÑ.fÔ.fÐ 
Ñ+Ð+å )¨$¬+Ð6U¸¼Ô8UÐ6UÔ*VÑ WÔ Wð 	ð 	ÑˆAˆ}Ø)˜MØðà2°4´;Ô3JÈ1Ô3MÔNØ$7¸¼Ô8OÐPQÔ8RÔ$SØ)Ø /ðð ð ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆå&Ø+Ø+ð
ñ 
ô 
ð 	
rO   )NNNNNN)rk   rl   rm   r3   rr   r1  rž   r¢   r  r«   r   r   rH   r   r   r   r§   rw   rx   s   @rM   r=  r=  á  s  ø€ € € € € € ØÐÐÑØ Ðð
Ð/ð 
ð 
ð 
ð 
ð 
ð 
ð .2Ø.2Ø04Ø(,Ø26Ø!%ðC
ð C
àÔ# dÑ*ðC
ð œ tÑ+ðC
ð Ô&¨Ñ-ð	C
ð
  ™ðC
ð Ô(¨4Ñ/ðC
ð ˜$‘;ðC
ð Ð+Ô,ðC
ð 
!ðC
ð C
ð C
ð C
ð C
ð C
ð C
ð C
rO   r=  c                   ó0   ‡ — e Zd ZU eed<   defˆ fd„Zˆ xZS )ÚGemma3ForCausalLMr®   c                 ór   •— t          ¦   «                              |¦  «         t          |¦  «        | _        d S r¦   )rY   rž   r=  r#  r±   s     €rM   rž   zGemma3ForCausalLM.__init__6  s.   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý$ VÑ,Ô,ˆŒ
ˆ
ˆ
rO   )rk   rl   rm   r3   rr   rž   rw   rx   s   @rM   r\  r\  3  sS   ø€ € € € € € ØÐÐÑð-Ð/ð -ð -ð -ð -ð -ð -ð -ð -ð -ð -rO   r\  c                   ó:   ‡ — e Zd Zdefˆ fd„Zdej        fd„Zˆ xZS )r)  r®   c                 ó  •— t          ¦   «                              ¦   «          t          j        t	          j        |j        j        |j        j        ¦  «        ¦  «        | _	        t          |j        j        |j        j        ¬¦  «        | _        t          |j        j        |j        j        z  ¦  «        | _        t          |j        dz  ¦  «        | _        | j        | j        z  | _        t          j        | j        | j        ¬¦  «        | _        d S )Nr  r@  )Úkernel_sizeÚstride)rY   rž   r    Ú	Parameterr¢   Úzerosrƒ   rÑ   r‚   r,  r³   Úlayer_norm_epsÚmm_soft_emb_normrq   Ú
image_sizeÚ
patch_sizeÚpatches_per_imager…   Útokens_per_sider`  Ú	AvgPool2dÚavg_poolr±   s     €rM   rž   z"Gemma3MultiModalProjector.__init__<  sà   ø€ Ý‰Œ×ÒÑÔÐå*,¬,ÝŒK˜Ô,Ô8¸&Ô:LÔ:XÑYÔYñ+
ô +
ˆÔ'õ !.ØÔ Ô,°&Ô2FÔ2Uð!
ñ !
ô !
ˆÔõ "% VÔ%9Ô%DÈÔH\ÔHgÑ%gÑ!hÔ!hˆÔÝ" 6Ô#=¸sÑ#BÑCÔCˆÔØÔ1°TÔ5IÑIˆÔÝœ°Ô1AÈ$ÔJZÐ[Ñ[Ô[ˆŒˆˆrO   Úvision_outputsc                 ó¸  — |j         \  }}}|                     dd¦  «        }|                     ||| j        | j        ¦  «        }|                     ¦   «         }|                      |¦  «        }|                     d¦  «        }|                     dd¦  «        }|                      |¦  «        }t          j	        || j
        ¦  «        }|                     |¦  «        S )NrB   r!   )rß   rá   r  rh  r  rk  Úflattenre  r¢   Úmatmulr,  Útype_as)	rL   rl  Ú
batch_sizer  rÑ   Úreshaped_vision_outputsÚpooled_vision_outputsÚnormed_vision_outputsÚprojected_vision_outputss	            rM   r§   z!Gemma3MultiModalProjector.forwardL  sÜ   € Ø%3Ô%9Ñ"ˆ
�A�{à"0×":Ò":¸1¸aÑ"@Ô"@ÐØ"9×"AÒ"AØ˜ TÔ%;¸TÔ=Sñ#
ô #
Ðð #:×"DÒ"DÑ"FÔ"FÐà $§¢Ð.EÑ FÔ FÐØ 5× =Ò =¸aÑ @Ô @ÐØ 5× ?Ò ?ÀÀ1Ñ EÔ EÐà $× 5Ò 5Ð6KÑ LÔ LÐå#(¤<Ð0EÀtÔGfÑ#gÔ#gÐ Ø'×/Ò/°Ñ?Ô?Ð?rO   )	rk   rl   rm   rz   rž   r¢   r«   r§   rw   rx   s   @rM   r)  r)  ;  sq   ø€ € € € € ð\˜|ð \ð \ð \ð \ð \ð \ð @ e¤lð @ð @ð @ð @ð @ð @ð @ð @rO   r)  Útoken_type_idsrÉ   c                 ó$  — | dk                          |¬¦  «        }t          j                             |dd¬¦  «        d d …d d…f         }|| z  }t	          j        |                     ¦   «         d¬¦  «        dz
  }t	          j        ||d¦  «        }|S )NrB   rF  )rB   r   r   )ÚvaluerØ   rÝ   )r¨   r    Ú
functionalÚpadr¢   Úcumsumrq   Úwhere)rv  rÉ   Úis_imageÚis_previous_imageÚnew_image_startÚ	group_idsÚblock_sequence_idss          rM   Úget_block_sequence_ids_for_maskr‚  _  sš   € ð  !Ò#×'Ò'¨vÐ'Ñ6Ô6€HÝœ×)Ò)¨(°FÀ!Ð)ÑDÔDÀQÀQÀQÈÈÈÀVÔLÐØÐ"3Ð!3Ñ3€OÝ”˜_×0Ò0Ñ2Ô2¸Ð:Ñ:Ô:¸QÑ>€IÝœ X¨y¸"Ñ=Ô=ÐØÐrO   r®   rB  rú   rû   ræ   r�  c                 ój  — | ||||dœ}t          d
i |¤d|i¤Ž}t          |dd¦  «        r$d|j        v r|j                             d¦  «        }nd}t	          d
i |¤d|i¤Ž\  }	}
}
}
}}
}|	r|}nt          ||||¦  «        }t          d
i |¤t          |¦  «        t          | j        ¦  «        |dœ¤Ž}||d	œS )zãCreate full_attention and sliding_attention masks with correct composition.

    For global (full attention) layers:  OR(causal, blockwise)
    For local (sliding window) layers:  AND(sliding_window, OR(causal, blockwise))
    rG  r�  rò   NTr   rð   )rK  Úand_mask_functionrð   rL  rP   )	r   r[   rò   Úindexr   r   r   r   rQ   )r®   rB  rú   rû   ræ   r�  rX  Ú	full_maskÚsliding_layer_idxÚ
early_exitr  Ú	kv_lengthÚ	kv_offsetÚpadded_block_sequence_idsÚsliding_masks                  rM   Úcreate_masks_for_vision_modelr�  j  sP  € ð Ø&Ø(Ø*Ø$ðð €Kõ #ÐXÐX [ÐXÐXÐEWÐXÐXÐX€Iõ ˆ ¨dÑ3Ô3ð ¸ÀÔ@ZÐ8ZÐ8ZØ+Ô6×<Ò<¸TÑBÔBÐÐàÐõ 4Nð 4ð 4Ø
ð4ð 4à#ð4ð 4ð 4Ñ0€J��1�a˜ A yð ð 
Ø$6Ð!Ð!å$@Ø °	¸9ñ%
ô %
Ð!õ &ð ð Ø
ðå*Ð+DÑEÔEÝ0°Ô1FÑGÔGØ#ð	ð ð ð €Lð $Ø)ðð ð rO   c                   ó‚  ‡ — e Zd ZdZdefˆ fd„Ze ed¬¦  «        dej	        de
e         deez  fd	„¦   «         ¦   «         Zee	 	 	 	 	 	 	 	 	 ddej        d
z  dej	        d
z  dej        d
z  dej        d
z  ded
z  dej        d
z  dej	        d
z  dej        d
z  ded
z  de
e         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚGemma3ModelFr®   c                 óN   •— t          ¦   «                              |¦  «         | `d S r¦   )rY   rž   Útext_config_dtyper±   s     €rM   rž   zGemma3Model.__init__ª  s'   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØÐ"Ð"Ð"rO   zOProjects the last hidden state from the vision model into language model space.©Úcustom_introÚpixel_valuesrV   rW   c                 óh   —  | j         d|ddœ|¤Ž}|j        }|                      |¦  «        |_        |S )NT)r”  Úreturn_dictrP   )Úvision_towerrM  Úmulti_modal_projectorÚpooler_output)rL   r”  rV   rl  rM  s        rM   Úget_image_featureszGemma3Model.get_image_features®  sK   € ð
 +˜Ô*Ða¸ÐRVÐaÐaÐZ`ÐaÐaˆØ*Ô<ÐØ'+×'AÒ'AÐBSÑ'TÔ'TˆÔ$àÐrO   Nr¤   rú   ræ   rû   rv  rB  ÚlabelsrC  Ú	lm_kwargsc
           
      ó   — |d u |d uz  rt          d¦  «        ‚|�?| j        j        | j        k    r*|| j        j        k    }|                     ¦   «         }d||<   n|}|€ |                      ¦   «         |¦  «        }|�j|                      |d¬¦  «        j        }|                     |j	        |j
        ¦  «        }|                      |||¬¦  «        }|                     ||¦  «        }t          |x}t          ¦  «        sR| j                             ¦   «         ||||dœ}|�%t!          ||j	        ¬¦  «        }t#          dd|i|¤Ž}nt%          di |¤Ž} | j        d|||||	dd	œ|
¤Ž}t)          |j        |j        |j        |j        |�|nd ¬
¦  «        S )NrE  r   T)r–  )rB  Úimage_featuresrG  rF  r�  )rú   ræ   rû   rB  rC  r–  )rM  rû   rø   Ú
attentionsÚimage_hidden_statesrP   )rN  r®   r   r:   rÈ   Úget_input_embeddingsrš  r™  r¨   rÉ   rª   Úget_placeholder_maskÚmasked_scatterr‹   rv   Úget_text_configr‚  r�  r   Úlanguage_modelr�   rM  rû   rø   rŸ  )rL   r¤   r”  rú   ræ   rû   rv  rB  r›  rC  rœ  Úspecial_image_maskÚllm_input_idsrž  rW  rX  r�  Úoutputss                     rM   r§   zGemma3Model.forward¹  s)  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZð Ð  T¤[Ô%?À4Ä?Ò%RÐ%RØ!*¨d¬kÔ.HÒ!HÐØ%ŸOšOÑ-Ô-ˆMØ01ˆMÐ,Ñ-Ð-à%ˆMàÐ Ø7˜D×5Ò5Ñ7Ô7¸ÑFÔFˆMð Ð#Ø!×4Ò4°\ÈtÐ4ÑTÔTÔbˆNØ+×.Ò.¨}Ô/CÀ]ÔEXÑYÔYˆNØ!%×!:Ò!:Ø¨À~ð ";ñ "ô "Ðð *×8Ò8Ð9KÈ^Ñ\Ô\ˆMõ °Ð?Ð-ÅÑFÔFð 	Oàœ+×5Ò5Ñ7Ô7Ø!.Ø"0Ø#2Ø ,ðð ˆKð Ð)Ý%DÀ^Ð\iÔ\pÐ%qÑ%qÔ%qÐ"Ý&Cð 'ð 'Ø'9ð'à!ð'ð 'Ð#Ð#õ
 '@Ð&NÐ&NÀ+Ð&NÐ&NÐ#à%�$Ô%ð 
Ø.Ø%Ø+Ø'ØØð
ð 
ð ð
ð 
ˆõ )Ø%Ô7Ø#Ô3Ø!Ô/ØÔ)Ø2>Ð2J  ÐPTð
ñ 
ô 
ð 	
rO   )	NNNNNNNNN)rk   rl   rm   Úaccepts_loss_kwargsrz   rž   r   r   r¢   r   r   r   rì   r   rš  r  r«   r   rH   r�   r§   rw   rx   s   @rM   r�  r�  ¦  sÄ  ø€ € € € € àÐð#˜|ð #ð #ð #ð #ð #ð #ð Ø€^Ð!rÐsÑsÔsðØ!Ô-ðØ9?Ð@RÔ9Sðà	Ð+Ñ	+ðð ð ñ tÔsñ Ôðð Øð .2Ø15Ø.2Ø04Ø(,Ø26Ø26Ø*.Ø!%ðG
ð G
àÔ# dÑ*ðG
ð Ô'¨$Ñ.ðG
ð œ tÑ+ð	G
ð
 Ô&¨Ñ-ðG
ð  ™ðG
ð Ô(¨4Ñ/ðG
ð Ô(¨4Ñ/ðG
ð Ô  4Ñ'ðG
ð ˜$‘;ðG
ð Ð.Ô/ðG
ð 
Ð*Ñ	*ðG
ð G
ð G
ñ „^ñ ÔðG
ð G
ð G
ð G
ð G
rO   r�  c                   ó   ‡ — e Zd Zee	 	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  de	dz  dej        dz  d	ej        dz  d
ej        dz  de
dz  deej        z  dee         deez  fd„¦   «         ¦   «         Zdˆ fd„	Z	 	 dded	ej        dej        dz  de	dz  dej        dz  dej        dz  de
dz  defd„Zˆ xZS )ÚGemma3ForConditionalGenerationNr   r¤   r”  rú   ræ   rû   rv  rB  r›  rC  Úlogits_to_keeprœ  rW   c                 óp  —  | j         d||||||||	|ddœ
|¤Ž}|d         }t          |
t          ¦  «        rt          |
 d¦  «        n|
}|                      |dd…|dd…f         ¦  «        }d}|�  | j        d||| j        j        j        dœ|¤Ž}t          |||j
        |j        |j        |j        ¬¦  «        S )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`.

        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, Gemma3ForConditionalGeneration

        >>> model = Gemma3ForConditionalGeneration.from_pretrained("google/gemma-3-4b-it")
        >>> processor = AutoProcessor.from_pretrained("google/gemma-3-4b-it")

        >>> messages = [
        ...     {
        ...         "role": "system",
        ...         "content": [
        ...             {"type": "text", "text": "You are a helpful assistant."}
        ...         ]
        ...     },
        ...     {
        ...         "role": "user", "content": [
        ...             {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"},
        ...             {"type": "text", "text": "Where is the cat standing?"},
        ...         ]
        ...     },
        ... ]

        >>> inputs = processor.apply_chat_template(
        ...     messages,
        ...     tokenize=True,
        ...     return_dict=True,
        ...     return_tensors="pt",
        ...     add_generation_prompt=True
        ... )
        >>> # Generate
        >>> generate_ids = model.generate(**inputs)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "user\nYou are a helpful assistant.\n\n\n\n\n\nWhere is the cat standing?\nmodel\nBased on the image, the cat is standing in a snowy area, likely outdoors. It appears to"
        ```
        T)
r¤   r”  rv  rú   ræ   rû   rB  rC  r›  r–  r   N)Úlogitsr›  r:   )Úlossr®  rû   rø   rŸ  r   rP   )r#  r‹   rq   ÚsliceÚlm_headÚloss_functionr®   r‚   r:   r“   rû   rø   rŸ  r   )rL   r¤   r”  rú   ræ   rû   rv  rB  r›  rC  r¬  rœ  r¨  rø   Úslice_indicesr®  r¯  s                    rM   r§   z&Gemma3ForConditionalGeneration.forward  s  € ðz �$”*ð 
ØØ%Ø)Ø)Ø%Ø+Ø'ØØØð
ð 
ð ð
ð 
ˆð   œ
ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%ð Ø f¸¼Ô9PÔ9[ðð Ø_hðð ˆDõ ,ØØØ#Ô3Ø!Ô/ØÔ)Ø 'Ô ;ð
ñ 
ô 
ð 	
rO   TFc                 óT   •—  t          ¦   «         j        |f||dœ|¤Ž}|s|rd |d<   |S )N)rC  Úis_first_iterationrv  )rY   Úprepare_inputs_for_generation)rL   r¤   rC  rµ  rV   Úmodel_inputsr]   s         €rM   r¶  z<Gemma3ForConditionalGeneration.prepare_inputs_for_generatione  sX   ø€ Ø<•u‘w”wÔ<Øð
Ø!*Ð?Qð
ð 
ØU[ð
ð 
ˆð #ð 	2¨)ð 	2à-1ˆLÐ)Ñ*àÐrO   r®   rµ  c                 óš   — |                       ¦   «         ||||dœ}|�$t          ||j        ¬¦  «        }	t          dd|	i|¤ŽS t	          di |¤ŽS )NrG  rF  r�  rP   )r¤  r‚  rÉ   r�  r   )
r®   rB  rú   rû   ræ   rv  rµ  rV   rX  r�  s
             rM   r   z8Gemma3ForConditionalGeneration.create_masks_for_generateo  s‡   € ð ×,Ò,Ñ.Ô.Ø*Ø,Ø.Ø(ð
ð 
ˆð Ð%Ý!@ÀÐXeÔXlÐ!mÑ!mÔ!mÐÝ0ð ð Ø#5ðàðð ð õ
 )Ð7Ð7¨;Ð7Ð7Ð7rO   )
NNNNNNNNNr   )TF)NF)rk   rl   rm   r   r   r¢   r  r   r«   r   rH   rq   r   r   rì   r“   r§   r¶  r
   rv   r   rw   rx   s   @rM   r«  r«    sñ  ø€ € € € € ØØð .2Ø15Ø.2Ø04Ø(,Ø26Ø26Ø*.Ø!%Ø-.ð[
ð [
àÔ# dÑ*ð[
ð Ô'¨$Ñ.ð[
ð œ tÑ+ð	[
ð
 Ô&¨Ñ-ð[
ð  ™ð[
ð Ô(¨4Ñ/ð[
ð Ô(¨4Ñ/ð[
ð Ô  4Ñ'ð[
ð ˜$‘;ð[
ð ˜eœlÑ*ð[
ð Ð.Ô/ð[
ð 
Ð-Ñ	-ð[
ð [
ð [
ñ „^ñ Ôð[
ðzð ð ð ð ð ð  /3Ø*/ð8ð 8Ø ð8à”|ð8ð œ tÑ+ð8ð  ™ð	8ð
 ”l TÑ)ð8ð œ tÑ+ð8ð ! 4™Kð8ð 
ð8ð 8ð 8ð 8ð 8ð 8ð 8ð 8rO   r«  zÎ
Gemma3TextForSequenceClassification is a text-only sequence classification model that works with Gemma3TextConfig.
It uses the generic sequence classification implementation for efficiency and consistency.r’  c                   ó   — e Zd ZU eed<   dZdS )Ú#Gemma3TextForSequenceClassificationr®   r>  N)rk   rl   rm   r3   rr   r1  rP   rO   rM   rº  rº  ‹  s&   € € € € € € ð ÐÐÑØ ÐÐÐrO   rº  c                   óÜ   ‡ — e Zd Z	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dedz  dej        dz  dej        dz  d	ej        dz  d
ee	         de
fˆ fd„Zˆ xZS )ÚGemma3ForSequenceClassificationNr¤   r”  rú   ræ   rû   rv  rB  r›  rV   rW   c	                 óH   •—  t          ¦   «         j        d||||||||dœ|	¤ŽS )N)r¤   rú   ræ   rû   rB  r”  rv  r›  rP   )rY   r§   )rL   r¤   r”  rú   ræ   rû   rv  rB  r›  rV   r]   s             €rM   r§   z'Gemma3ForSequenceClassification.forward–  sK   ø€ ð �u‰wŒwŒð 

ØØ)Ø%Ø+Ø'Ø%Ø)Øð

ð 

ð ð

ð 

ð 
	
rO   )NNNNNNNN)rk   rl   rm   r¢   r  r   r«   r   r   r   r   r§   rw   rx   s   @rM   r¼  r¼  •  sÿ   ø€ € € € € ð .2Ø15Ø.2Ø04Ø(,Ø26Ø26Ø*.ð
ð 
àÔ# dÑ*ð
ð Ô'¨$Ñ.ð
ð œ tÑ+ð	
ð
 Ô&¨Ñ-ð
ð  ™ð
ð Ô(¨4Ñ/ð
ð Ô(¨4Ñ/ð
ð Ô  4Ñ'ð
ð Ð+Ô,ð
ð 
*ð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
rO   r¼  )	rz   r3   r"  r=  r\  r«  r�  r¼  rº  r¦   )cÚcollections.abcr   Útypingr   r¢   Útorch.nnr    Úhuggingface_hub.dataclassesr   Ú r   r*  Úcache_utilsr   r	   Úconfiguration_utilsr
   Úmasking_utilsr   r   r   r   r   r   r   Úmodeling_layersr   r   Úmodeling_outputsr   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr    Úgemma2.configuration_gemma2r"   Úgemma2.modeling_gemma2r#   r$   r%   r&   r'   r(   r)   r*   r+   Úpaligemma.modeling_paligemmar,   r-   r.   r/   Úsiglipr0   Ú
get_loggerrk   r‰   r3   rz   r�   r“   Ú	Embeddingr•   r­   r³   rº   rï   r  ÚGEMMA3_START_DOCSTRINGr"  rq   rH   r;  r=  r\  rÁ   r)  r«   rÉ   r‚  rv   r�  r�  r«  rº  r¼  Ú__all__rP   rO   rM   ú<module>rÕ     sW  ðð %Ð $Ð $Ð $Ð $Ð $Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð Ø .Ð .Ð .Ð .Ð .Ð .à &Ð &Ð &Ð &Ð &Ð &Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð \Ð [Ð [Ð [Ð [Ð [Ð [Ð [Ø uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uðð ð ð ð ð ð ð ð GÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RØ +Ð +Ð +Ð +Ð +Ð +Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6ð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ðð ð ð ð ð ð ð ð ð ð ð ð (Ð 'Ð 'Ð 'Ð 'Ð 'ð 
ˆÔ	˜HÑ	%Ô	%€ð €Ð1Ð2Ñ2Ô2Øð`ð `ð `ð `ð `�|Ð%5ñ `ô `ñ „ñ 3Ô2ð`ðF €Ð1Ð2Ñ2Ô2Øð?(ð ?(ð ?(ð ?(ð ?(Ð#ñ ?(ô ?(ñ „ñ 3Ô2ð?(ðD	ð 	ð 	ð 	ð 	Ð <ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð#Bñ 	ô 	ð 	ðSð Sð Sð Sð S B¤Lñ Sô Sð Sð!ð !ð !ð !ð !�	ñ !ô !ð !ð
+ð +ð +ð +ð +�Mñ +ô +ð +ð
B<ð B<ð B<ð B<ð B<Ð1ñ B<ô B<ð B<ðL4)ð 4)ð 4)ð 4)ð 4)�oñ 4)ô 4)ð 4)ðn+ð +ð +ð +ð +Ð3ñ +ô +ð +ð\ Ð ð^ð ^ð ^ð ^ð ^Ð1ñ ^ô ^ð ^ð2
°#ð 
¸(ÀCÈÈcÐSVÐCWÐY]ÐC]Ô:^ð 
ð 
ð 
ð 
ðO
ð O
ð O
ð O
ð O
�kñ O
ô O
ð O
ðd-ð -ð -ð -ð -Ð)ñ -ô -ð -ð!@ð !@ð !@ð !@ð !@ ¤	ñ !@ô !@ð !@ðHð °E´Lð È%Ì,ÐY]ÑJ]ð ÐinÔiuð ð ð ð ð9Øð9à”<ð9ð ”L 4Ñ'ð9ð ˜T‘\ð	9ð
 ”, Ñ%ð9ð œð9ð 
ð9ð 9ð 9ð 9ðx\
ð \
ð \
ð \
ð \
�.ñ \
ô \
ð \
ð~C8ð C8ð C8ð C8ð C8Ð%Fñ C8ô C8ð C8ðL €ð^ðñ ô ð
!ð !ð !ð !ð !Ð*JÐLañ !ô !ñô ð
!ð

ð 
ð 
ð 
ð 
Ð&FÐH]ñ 
ô 
ð 
ð4
ð 
ð 
€€€rO   