§
    9²¸j…À  ã                   óF  — d dl mZ d dlmZ d dlZd dlmZ ddlmZ	 ddl
mZ ddlmZmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZmZmZmZmZmZmZ ddlmZm Z  ddl!m"Z"m#Z#m$Z$m%Z% ddl&m'Z'm(Z( ddl)m*Z*m+Z+ ddl,m-Z- ddl.m/Z/m0Z0m1Z1m2Z2m3Z3 ddl4m5Z5 ddl6m7Z7m8Z8 ddl9m:Z: ddl;m<Z< ddl=m>Z>m?Z?  e1d¬¦  «        e G d„ de"¦  «        ¦   «         ¦   «         Z@ e1d¬¦  «        e G d„ de/¦  «        ¦   «         ¦   «         ZA G d „ d!ejB        ¦  «        ZC G d"„ d#ejD        ¦  «        ZE G d$„ d%ejD        ¦  «        ZF G d&„ d'ejD        ¦  «        ZGd(„ ZH ed)¦  «        d[d*„¦   «         ZId+ejJ        d,eKd-ejJ        fd.„ZL	 	 	 d\d0ejD        d1ejJ        d2ejJ        d3ejJ        d4ejJ        dz  d5eMeKz  d6eMdz  d7eMdz  d-eNejJ        ejJ        f         fd8„ZO eeI¦  «         G d9„ d:ejD        ¦  «        ¦   «         ZP G d;„ d<e ¦  «        ZQe1 G d=„ d>e+¦  «        ¦   «         ZRd?eKd-eeKeKeKeKgeSf         fd@„ZTe1 G dA„ dBeR¦  «        ¦   «         ZUe1 G dC„ dDeRe¦  «        ¦   «         ZV G dE„ dFejD        ¦  «        ZWd]dGejJ        dHejX        dz  d-ejJ        fdI„ZYdJedKejJ        d4ejJ        dz  dLedz  dMejJ        dz  dNejJ        d-eZfdO„Z[ e1dP¬¦  «         G dQ„ dReR¦  «        ¦   «         Z\ e1dP¬¦  «         G dS„ dTeRe¦  «        ¦   «         Z] e1dU¬¦  «         G dV„ dWeeR¦  «        ¦   «         Z^ G dX„ dYeeR¦  «        Z_g dZ¢Z`dS )^é    )ÚCallable)Ú	dataclassNé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚPreTrainedConfig)ÚGenerationMixin)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Ú_preprocess_mask_argumentsÚblockwise_overlayÚcreate_causal_maskÚcreate_masks_for_generateÚ!create_sliding_window_causal_maskÚmaybe_pad_block_sequence_idsÚsliding_window_overlay)Ú GenericForSequenceClassificationÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚBaseModelOutputWithPoolingÚCausalLMOutputWithPastÚ SequenceClassifierOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚModelOutputÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚtorch_compilable_check)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )Ú	AutoModelé   )ÚGemma3ConfigÚGemma3TextConfigzK
    Base class for Gemma3 outputs, with hidden states and attentions.
    ©Úcustom_introc                   ó2   — e Zd ZU dZdZej        dz  ed<   dS )ÚGemma3ModelOutputWithPasta  
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
    NÚimage_hidden_states)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r2   ÚtorchÚFloatTensorÚ__annotations__© ó    úz/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/gemma3/modeling_gemma3.pyr1   r1   <   s7   € € € € € € ðð ð 59Ð˜Ô*¨TÑ1Ð8Ð8Ñ8Ð8Ð8r;   r1   zR
    Base class for Gemma3 causal language model (or autoregressive) outputs.
    c                   óÖ   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dZej        dz  ed<   dS )	ÚGemma3CausalLMOutputWithPasta8  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.text_config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder after projecting last hidden state.
    NÚlossÚlogitsÚpast_key_valuesÚhidden_statesÚ
attentionsr2   )r3   r4   r5   r6   r?   r7   r8   r9   r@   rA   r   rB   ÚtuplerC   r2   r:   r;   r<   r>   r>   L   sµ   € € € € € € ðð ð &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø$(€O�U˜T‘\Ð(Ð(Ñ(Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ø48Ð˜Ô*¨TÑ1Ð8Ð8Ñ8Ð8Ð8r;   r>   c            	       óP   ‡ — e Zd ZdZd
dedededefˆ fd„Zdej        fˆ fd	„Z	ˆ xZ
S )ÚGemma3TextScaledWordEmbeddingz\
    This module overrides nn.Embeddings' forward by multiplying with embeddings scale.
    ç      ð?Únum_embeddingsÚembedding_dimÚpadding_idxÚembed_scalec                 ó¶   •— t          ¦   «                              |||¦  «         || _        t          j        t          j        |¦  «        d¬¦  «        | _        d S )NF©Ú
persistent)ÚsuperÚ__init__Úscalar_embed_scaleÚnnÚBufferr7   ÚtensorrK   )ÚselfrH   rI   rJ   rK   Ú	__class__s        €r<   rP   z&Gemma3TextScaledWordEmbedding.__init__o   sN   ø€ Ý‰Œ×Ò˜¨¸ÑDÔDÐDØ"-ˆÔÝœ9¥U¤\°+Ñ%>Ô%>È5ÐQÑQÔQˆÔÐÐr;   Ú	input_idsc                 ó�   •— t          ¦   «                              |¦  «        | j                             | j        j        ¦  «        z  S ©N)rO   ÚforwardrK   ÚtoÚweightÚdtype)rU   rW   rV   s     €r<   rZ   z%Gemma3TextScaledWordEmbedding.forwardt   s4   ø€ Ý‰wŒw�Š˜yÑ)Ô)¨DÔ,<×,?Ò,?ÀÄÔ@QÑ,RÔ,RÑRÐRr;   )rG   )r3   r4   r5   r6   ÚintÚfloatrP   r7   ÚTensorrZ   Ú__classcell__©rV   s   @r<   rF   rF   j   s¦   ø€ € € € € ðð ðRð R sð R¸3ð RÈSð RÐ_dð Rð Rð Rð Rð Rð Rð
S ¤ð Sð Sð Sð Sð Sð Sð Sð Sð Sð Sr;   rF   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )Ú	Gemma3MLPÚconfigc                 ó˜  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _	        t          |j                 | _        d S ©NF©Úbias)rO   rP   re   Úhidden_sizeÚintermediate_sizerR   ÚLinearÚ	gate_projÚup_projÚ	down_projr   Úhidden_activationÚact_fn©rU   re   rV   s     €r<   rP   zGemma3MLP.__init__y   s¦   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ!'Ô!9ˆÔÝœ 4Ô#3°TÔ5KÐRWÐXÑXÔXˆŒÝ”y Ô!1°4Ô3IÐPUÐVÑVÔVˆŒÝœ 4Ô#9¸4Ô;KÐRWÐXÑXÔXˆŒÝ˜VÔ5Ô6ˆŒˆˆr;   c                 ó¨   — |                       |                      |                      |¦  «        ¦  «        |                      |¦  «        z  ¦  «        }|S rY   )ro   rq   rm   rn   )rU   Úxro   s      r<   rZ   zGemma3MLP.forwardƒ   sA   € Ø—N’N 4§;¢;¨t¯~ª~¸aÑ/@Ô/@Ñ#AÔ#AÀDÇLÂLÐQRÁOÄOÑ#SÑTÔTˆ	ØÐr;   )r3   r4   r5   r-   rP   rZ   ra   rb   s   @r<   rd   rd   x   sT   ø€ € € € € ð7Ð/ð 7ð 7ð 7ð 7ð 7ð 7ðð ð ð ð ð ð r;   rd   c                   ó<   ‡ — e Zd Zddedefˆ fd„Zd„ Zd„ Zd„ Zˆ xZ	S )	ÚGemma3RMSNormç�íµ ÷Æ°>ÚdimÚepsc                 ó¬   •— t          ¦   «                              ¦   «          || _        t          j        t          j        |¦  «        ¦  «        | _        d S rY   )rO   rP   ry   rR   Ú	Parameterr7   Úzerosr\   )rU   rx   ry   rV   s      €r<   rP   zGemma3RMSNorm.__init__‰   s?   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”l¥5¤;¨sÑ#3Ô#3Ñ4Ô4ˆŒˆˆr;   c                 ó�   — |t          j        |                     d¦  «                             dd¬¦  «        | j        z   ¦  «        z  S )Nr)   éÿÿÿÿT)Úkeepdim)r7   ÚrsqrtÚpowÚmeanry   )rU   rt   s     r<   Ú_normzGemma3RMSNorm._normŽ   s8   € Ø•5”;˜qŸušu Q™xœxŸ}š}¨R¸˜}Ñ>Ô>ÀÄÑIÑJÔJÑJÐJr;   c                 ó¸   — |                       |                     ¦   «         ¦  «        }|d| j                             ¦   «         z   z  }|                     |¦  «        S )NrG   )rƒ   r_   r\   Útype_as)rU   rt   Úoutputs      r<   rZ   zGemma3RMSNorm.forward‘   sL   € Ø—’˜AŸGšG™IœIÑ&Ô&ˆð ˜3 ¤×!2Ò!2Ñ!4Ô!4Ñ4Ñ5ˆØ�~Š~˜aÑ Ô Ð r;   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)rD   r\   Úshapery   )rU   s    r<   Ú
extra_reprzGemma3RMSNorm.extra_repr˜   s%   € Ý˜œÔ)Ñ*Ô*Ð<Ð<°$´(Ð<Ð<Ð<r;   )rw   )
r3   r4   r5   r^   r_   rP   rƒ   rZ   r‰   ra   rb   s   @r<   rv   rv   ˆ   s€   ø€ € € € € ð5ð 5˜Cð 5 eð 5ð 5ð 5ð 5ð 5ð 5ð
Kð Kð Kð!ð !ð !ð=ð =ð =ð =ð =ð =ð =r;   rv   c                   óú   ‡ — e Zd Z edd¬¦  «        ddefˆ fd„¦   «         Ze edd¬¦  «        	 ddededz  dee	j
        ef         fd	„¦   «         ¦   «         Z e	j        ¦   «         ed
„ ¦   «         ¦   «         Zˆ xZS )ÚGemma3RotaryEmbeddingÚdevicez5.18)ÚversionNre   c           	      ó¶  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        t          t          |j        ¦  «        ¦  «        | _        i | _	        | j        D ]ã}| j        j
        |         }|€Œ|d         | j	        |<   | j        }| j	        |         dk    rt          | j	        |                  } || j        ||¬¦  «        \  }}t          | |› d�t          j        |d¬¦  «        ¦  «         t          | |› d�t          j        |                     ¦   «         d¬¦  «        ¦  «         t          | |› d�|¦  «         Œäd S )	NÚ	rope_typeÚdefault©Ú
layer_typeÚ	_inv_freqFrM   Ú_original_inv_freqÚ_attention_scaling)rO   rP   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenre   ÚsortedÚsetÚlayer_typesr�   Úrope_parametersÚcompute_default_rope_parametersr   ÚsetattrrR   rS   Úclone)	rU   re   rŒ   r’   Úrope_paramsÚrope_init_fnÚcurr_inv_freqÚcurr_attention_scalingrV   s	           €r<   rP   zGemma3RotaryEmbedding.__init__�   sl  ø€ å‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!ØˆŒÝ!¥# fÔ&8Ñ"9Ô"9Ñ:Ô:ˆÔØˆŒØÔ*ð 	Uð 	UˆJØœ+Ô5°jÔAˆKØÐ"Øà)4°[Ô)AˆDŒN˜:Ñ&Ø%)Ô%IˆLØŒ~˜jÔ)¨YÒ6Ð6Ý2°4´>À*Ô3MÔN�Ø4@°LÀÄÈfÐakÐ4lÑ4lÔ4lÑ1ˆMÐ1Ý�D˜ZÐ2Ð2Ð2µB´I¸mÐX]Ð4^Ñ4^Ô4^Ñ_Ô_Ð_Ý�D˜ZÐ;Ð;Ð;½R¼YÀ}×GZÒGZÑG\ÔG\ÐinÐ=oÑ=oÔ=oÑpÔpÐpÝ�D˜ZÐ;Ð;Ð;Ð=SÑTÔTÐTÐTð	Uð 	Ur;   r’   Úreturnc                 óð   — | j         |         d         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «        |z  z  z  }|                     |¦  «        |fS )a†  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            layer_type (`str`):
                The current layer type if the model has different RoPE parameters per type.
                Should not be used unless `config.layer_types is not None`

        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNrG   r   r)   ©r]   )rœ   Úgetattrrj   Únum_attention_headsr7   Úaranger_   r[   )re   rŒ   r’   ÚkwargsÚbaserx   Úattention_factorÚinv_freqs           r<   r�   z5Gemma3RotaryEmbedding.compute_default_rope_parameters³   s   € ð& Ô% jÔ1°,Ô?ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐà˜$¥5¤<°°3¸ÅÄÐ#MÑ#MÔ#MÐPSÑ#SÑTÑUˆØ�{Š{˜6Ñ"Ô"Ð$4Ð4Ð4r;   c                 ó(  — t          | |› d�¦  «        }t          | |› d�¦  «        }|d d d …d f                              |j        d         dd¦  «                             t          j        |j        ¬¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d	¬
¦  «        5  ||z                       dd¦  «        }	t	          j        |	|	fd¬¦  «        }
|
                     ¦   «         |z  }|
                     ¦   «         |z  }d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬¦  «        |                     |j        ¬¦  «        fS )Nr“   r•   r   r~   r+   ©r]   rŒ   ÚmpsÚcpuF)Údevice_typeÚenabledr)   ©rx   r¨   )r©   Úexpandrˆ   r[   r7   r_   rŒ   Ú
isinstanceÚtypeÚstrr&   Ú	transposeÚcatÚcosÚsinr]   )rU   rt   Úposition_idsr’   r¯   Úattention_scalingÚinv_freq_expandedÚposition_ids_expandedr´   ÚfreqsÚembr½   r¾   s                r<   rZ   zGemma3RotaryEmbedding.forwardÎ   sÈ  € õ ˜4 JÐ!9Ð!9Ð!9Ñ:Ô:ˆÝ# D¨ZÐ*KÐ*KÐ*KÑLÔLÐð �T˜1˜1˜1˜d�]Ô#×*Ò*¨<Ô+=¸aÔ+@À"ÀaÑHÔH×KÒKÕRWÔR]ÐfgÔfnÐKÑoÔoð 	ð !-¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆå¨¸UÐCÑCÔCð 	0ð 	0Ø&Ð)>Ñ>×IÒIÈ!ÈQÑOÔOˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)Ð/Ñ/ˆCØ—'’'‘)”)Ð/Ñ/ˆCð		0ð 	0ð 	0ñ 	0ô 	0ð 	0ð 	0ð 	0ð 	0ð 	0ð 	0øøøð 	0ð 	0ð 	0ð 	0ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   Ã'A EÅEÅErY   )NN)r3   r4   r5   r%   r-   rP   Ústaticmethodrº   rD   r7   r`   r_   r�   Úno_gradr   rZ   ra   rb   s   @r<   r‹   r‹   œ   s  ø€ € € € € Ø€_�X vÐ.Ñ.Ô.ðUð UÐ/ð Uð Uð Uð Uð Uñ /Ô.ðUð* Ø€_�X vÐ.Ñ.Ô.àHLð5ð 5Ø ð5Ø;>À¹:ð5à	ˆuŒ|˜UÐ"Ô	#ð5ð 5ð 5ñ /Ô.ñ „\ð5ð2 €U„]�_„_Øð<ð <ñ Ôñ „_ð<ð <ð <ð <ð <r;   r‹   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..Nr~   r)   r¶   )rˆ   r7   r¼   )rt   Úx1Úx2s      r<   Úrotate_halfrÊ   ä   s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'r;   Úrotary_pos_embc                 ó¾   — |                      |¦  «        }|                      |¦  «        }| |z  t          | ¦  «        |z  z   }||z  t          |¦  «        |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezerÊ   )ÚqÚkr½   r¾   Úunsqueeze_dimÚq_embedÚk_embeds          r<   Úapply_rotary_pos_embrÓ   ë   sc   € ð& �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�GÐÐr;   rB   Ún_repr¤   c                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r+   N)rˆ   r·   Úreshape)rB   rÔ   ÚbatchÚnum_key_value_headsÚslenr§   s         r<   Ú	repeat_kvrÚ     s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTr;   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚdropoutÚscalingÚsoftcapc                 ól  — |€
| j         dz  }t          || j        ¦  «        }	t          || j        ¦  «        }
t          j        ||	                     dd¦  «        ¦  «        |z  }|�||z  }t          j        |¦  «        }||z  }|�||z   }t          j         	                    |dt          j
        ¬¦  «                             |j        ¦  «        }t          j                             ||| j        ¬¦  «        }t          j        ||
¦  «        }|                     dd¦  «                             ¦   «         }||fS )Nç      à¿r)   r   r~   )rx   r]   )ÚpÚtrainingr+   )r§   rÚ   Únum_key_value_groupsr7   Úmatmulr»   ÚtanhrR   Ú
functionalÚsoftmaxÚfloat32r[   r]   rá   rç   Ú
contiguous)rÜ   rÝ   rÞ   rß   rà   rá   râ   rã   r¬   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs                r<   Úeager_attention_forwardró     s%  € ð €Ø”/ 4Ñ'ˆå˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LàÐØ# gÑ-ˆÝ”z ,Ñ/Ô/ˆØ# gÑ-ˆØÐ!Ø# nÑ4ˆõ ”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€KØ˜Ð$Ð$r;   c                   óÒ   ‡ — e Zd ZdZdedefˆ fd„Z	 	 	 ddej        dej        dej        dz  d	e	dz  d
e
e         deej        ej        dz  eej                 dz  f         fd„Zˆ xZS )ÚGemma3Attentionz=Multi-headed attention from 'Attention Is All You Need' paperre   Ú	layer_idxc                 ó
  •— t          ¦   «                              ¦   «          t          |d¦  «        r|j        |         nd | _        || _        || _        t          |d|j        |j	        z  ¦  «        | _
        |j	        |j        z  | _        |j        dz  | _        | j        j        | _        | j        j         | _        t%          j        |j        |j	        | j
        z  |j        ¬¦  «        | _        t%          j        |j        |j        | j
        z  |j        ¬¦  «        | _        t%          j        |j        |j        | j
        z  |j        ¬¦  «        | _        t%          j        |j	        | j
        z  |j        |j        ¬¦  «        | _        | j        j        | _        | j        dk    r|j        nd | _        | j        dk    | _        t9          |j
        |j        ¬¦  «        | _        t9          |j
        |j        ¬¦  «        | _        d S )Nr›   r§   rå   rh   Úsliding_attention)rx   ry   ) rO   rP   Úhasattrr›   r’   re   rö   r©   rj   rª   r§   rØ   rè   Úquery_pre_attn_scalarrâ   Úattention_dropoutÚuse_bidirectional_attentionÚ	is_causalrR   rl   Úattention_biasÚq_projÚk_projÚv_projÚo_projÚattn_logit_softcappingÚsliding_windowÚ
is_slidingrv   Úrms_norm_epsÚq_normÚk_norm©rU   re   rö   rV   s      €r<   rP   zGemma3Attention.__init__7  sß  ø€ Ý‰Œ×ÒÑÔÐÝ;BÀ6È=Ñ;YÔ;YÐc˜&Ô,¨YÔ7Ð7Ð_cˆŒØˆŒØ"ˆŒÝ ¨
°FÔ4FÈ&ÔJdÑ4dÑeÔeˆŒØ$*Ô$>À&ÔB\Ñ$\ˆÔ!ØÔ3°TÑ9ˆŒØ!%¤Ô!>ˆÔØ!œ[ÔDÐDˆŒå”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ&¨¬Ñ6¸Ô8JÐQWÔQfð
ñ 
ô 
ˆŒð '+¤kÔ&HˆÔ#Ø7;´ÐJ]Ò7]Ð7]˜fÔ3Ð3ÐcgˆÔØœ/Ð-@Ò@ˆŒå#¨¬¸VÔ=PÐQÑQÔQˆŒÝ#¨¬¸VÔ=PÐQÑQÔQˆŒˆˆr;   NrB   Úposition_embeddingsrà   rA   r¬   r¤   c                 ó‚  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }	|                      |¦  «                             |¦  «                             dd¦  «        }
|                      |¦  «        }|                      |	¦  «        }	|\  }}t          ||	||¦  «        \  }}	|�| 
                    |	|
| j        ¦  «        \  }	}
t          j        | j        j        t           ¦  «        } || ||	|
|f| j        r| j        nd| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Nr~   r+   r)   rÛ   )rá   râ   r  )rˆ   r§   rÿ   Úviewr»   r   r  r  r  rÓ   Úupdaterö   r   Úget_interfacere   Ú_attn_implementationró   rç   rû   râ   r  rÖ   rî   r  )rU   rB   r
  rà   rA   r¬   Úinput_shapeÚhidden_shapeÚquery_statesrï   rð   r½   r¾   Úattention_interfacerò   rñ   s                   r<   rZ   zGemma3Attention.forwardU  sé  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆà—{’{ <Ñ0Ô0ˆØ—[’[ Ñ,Ô,ˆ
à&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð /3¬mÐD�DÔ*Ð*ÀØ”LØÔ.ð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r;   )NNN)r3   r4   r5   r6   r-   r^   rP   r7   r`   r   r   r!   rD   rZ   ra   rb   s   @r<   rõ   rõ   3  sï   ø€ € € € € àGÐGðRÐ/ð R¸Cð Rð Rð Rð Rð Rð RðB -1Ø.2Ø(,ð*)ð *)à”|ð*)ð #œ\ð*)ð œ tÑ+ð	*)ð
  ™ð*)ð Ð+Ô,ð*)ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	Mð*)ð *)ð *)ð *)ð *)ð *)ð *)ð *)r;   rõ   c                   óà   ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 ddej        dej        dej        dz  dej        dz  d	e	dz  d
e
e         deej        eej        ej        f         dz  f         fd„Zˆ xZS )ÚGemma3DecoderLayerre   rö   c                 óÐ  •— t          ¦   «                              ¦   «          || _        |j        | _        || _        t          ||¬¦  «        | _        t          |¦  «        | _        t          | j        |j
        ¬¦  «        | _        t          | j        |j
        ¬¦  «        | _        t          | j        |j
        ¬¦  «        | _        t          | j        |j
        ¬¦  «        | _        d S )N)re   rö   ©ry   )rO   rP   re   rj   rö   rõ   Ú	self_attnrd   Úmlprv   r  Úinput_layernormÚpost_attention_layernormÚpre_feedforward_layernormÚpost_feedforward_layernormr	  s      €r<   rP   zGemma3DecoderLayer.__init__ƒ  sÇ   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ"ˆŒÝ(°À)ÐLÑLÔLˆŒÝ˜VÑ$Ô$ˆŒÝ,¨TÔ-=À6ÔCVÐWÑWÔWˆÔÝ(5°dÔ6FÈFÔL_Ð(`Ñ(`Ô(`ˆÔ%Ý)6°tÔ7GÈVÔM`Ð)aÑ)aÔ)aˆÔ&Ý*7¸Ô8HÈfÔNaÐ*bÑ*bÔ*bˆÔ'Ð'Ð'r;   NrB   r
  rà   r¿   rA   r¬   r¤   c           	      ó   — |}|                       |¦  «        } | j        d|||||dœ|¤Ž\  }}|                      |¦  «        }||z   }|}|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }||z   }|S )N)rB   r
  rà   r¿   rA   r:   )r  r  r  r  r  r  )	rU   rB   r
  rà   r¿   rA   r¬   ÚresidualÚ_s	            r<   rZ   zGemma3DecoderLayer.forward�  sÂ   € ð !ˆà×,Ò,¨]Ñ;Ô;ˆà)˜4œ>ð 
Ø'Ø 3Ø)Ø%Ø+ð
ð 
ð ð
ð 
Ñˆ�qð ×5Ò5°mÑDÔDˆØ  =Ñ0ˆà ˆØ×6Ò6°}ÑEÔEˆØŸš Ñ/Ô/ˆØ×7Ò7¸ÑFÔFˆØ  =Ñ0ˆàÐr;   )NNNN)r3   r4   r5   r-   r^   rP   r7   r`   Ú
LongTensorr   r   r!   rD   r8   rZ   ra   rb   s   @r<   r  r  ‚  sÿ   ø€ € € € € ð
cÐ/ð 
c¸Cð 
cð 
cð 
cð 
cð 
cð 
cð -1Ø.2Ø04Ø(,ðð à”|ðð #œ\ðð œ tÑ+ð	ð
 Ô&¨Ñ-ðð  ™ðð Ð+Ô,ðð 
ˆuÔ  %¨Ô(9¸5Ô;LÐ(LÔ"MÐPTÑ"TÐTÔ	Uðð ð ð ð ð ð ð r;   r  c                   óŠ   ‡ — e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZdZeedœZdZ ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )	ÚGemma3PreTrainedModelre   ÚmodelTr  rA   )rB   rC   )ÚimageÚtextc                 óÌ  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rt	          j        |j        ¦  «         d S d|j        j        v rt	          j        |j	        ¦  «         d S t          |t          ¦  «        r!t	          j        |j        |j        ¦  «         d S t          |t          ¦  «        r›|j        D ]•}|j        }|j        |         dk    rt$          |j        |                  } ||j        |¬¦  «        \  }}t	          j        t+          ||› d�¦  «        |¦  «         t	          j        t+          ||› d�¦  «        |¦  «         Œ”d S d S )NÚRMSNormr�   r‘   r“   r”   )rO   Ú_init_weightsr¸   ÚGemma3MultiModalProjectorÚinitÚzeros_Úmm_input_projection_weightrV   r3   r\   rF   Ú	constant_rK   rQ   r‹   r›   r�   r�   r   re   Úcopy_r©   )rU   rÜ   r’   r¡   r¢   r   rV   s         €r<   r)  z#Gemma3PreTrainedModel._init_weightsÃ  sx  ø€ å‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ7Ñ8Ô8ð 	^ÝŒK˜Ô9Ñ:Ô:Ð:Ð:Ð:à˜&Ô*Ô3Ð3Ð3ÝŒK˜œÑ&Ô&Ð&Ð&Ð&Ý˜Õ =Ñ>Ô>ð 		^ÝŒN˜6Ô-¨vÔ/HÑIÔIÐIÐIÐIÝ˜Õ 5Ñ6Ô6ð 	^Ø$Ô0ð ^ð ^�
Ø%ÔE�ØÔ# JÔ/°9Ò<Ð<Ý#6°vÔ7GÈ
Ô7SÔ#T�LØ#/ <°´È*Ð#UÑ#UÔ#UÑ �˜qÝ”
�7 6¨jÐ+CÐ+CÐ+CÑDÔDÀmÑTÔTÐTÝ”
�7 6¨jÐ+LÐ+LÐ+LÑMÔMÈ}Ñ]Ô]Ð]Ð]ð	^ð 	^ð^ð ^r;   )r3   r4   r5   r,   r9   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr  rõ   Ú_can_record_outputsÚinput_modalitiesr7   rÆ   r)  ra   rb   s   @r<   r#  r#  °  s·   ø€ € € € € € àÐÐÑØÐØ&*Ð#Ø-Ð.ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&Ðà+Ø%ðð Ðð )Ðà€U„]�_„_ð^ð ^ð ^ð ^ñ „_ð^ð ^ð ^ð ^ð ^r;   r#  r  c           
      óZ   ‡ — dt           dt           dt           dt           dt          f
ˆ fd„}|S )zA
    Enables a bidirectional mask within the sliding window.
    Ú	batch_idxÚhead_idxÚq_idxÚkv_idxr¤   c                 ó0   •— t          ||z
  ¦  «        ‰k     S )z“A token can attend to any other token if their absolute distance is within
        the (exclusive) sliding window size (distance < sliding_window).)Úabs)r<  r=  r>  r?  r  s       €r<   Ú
inner_maskz1_bidirectional_window_overlay.<locals>.inner_maskÜ  s   ø€ õ �5˜6‘>Ñ"Ô" ^Ò3Ð3r;   )r^   Úbool)r  rB  s   ` r<   Ú_bidirectional_window_overlayrD  ×  sL   ø€ ð
4�cð 4­Sð 4½ð 4Åcð 4Ídð 4ð 4ð 4ð 4ð 4ð 4ð
 Ðr;   c                   óò   ‡ — e Zd ZU eed<   dZdefˆ fd„Zeee		 	 	 	 	 	 dde
j        dz  de
j        dz  de
j        dz  dedz  d	e
j        dz  d
edz  dee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚGemma3TextModelre   ©r&  c                 óð  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          ‰j        ‰j        | j        | j        j        dz  ¬¦  «        | _        t          j
        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        t#          ‰¦  «        | _        d| _        |                      ¦   «          d S )Nç      à?)rK   c                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r:   )r  )Ú.0rö   re   s     €r<   ú
<listcomp>z,Gemma3TextModel.__init__.<locals>.<listcomp>ó  s$   ø€ ÐdÐdÐd°yÕ ¨	Ñ2Ô2ÐdÐdÐdr;   r  F)rO   rP   Úpad_token_idrJ   Ú
vocab_sizerF   rj   re   Úembed_tokensrR   Ú
ModuleListÚrangeÚnum_hidden_layersÚlayersrv   r  Únormr‹   Ú
rotary_embÚgradient_checkpointingÚ	post_initrr   s    `€r<   rP   zGemma3TextModel.__init__é  së   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒõ :ØÔ˜vÔ1°4Ô3CÐQUÔQ\ÔQhÐjmÑQmð
ñ 
ô 
ˆÔõ ”mØdÐdÐdÐdÅEÈ&ÔJbÑDcÔDcÐdÑdÔdñ
ô 
ˆŒõ " &Ô"4¸&Ô:MÐNÑNÔNˆŒ	Ý/°Ñ7Ô7ˆŒØ&+ˆÔ#ð 	�ŠÑÔÐÐÐr;   NrW   rà   r¿   rA   Úinputs_embedsÚ	use_cacher¬   r¤   c           	      óÌ  — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|r|€t          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   }| 	                    d¦  «        }t          |x}	t          ¦  «        sh| j        ||||dœ}
|
                     ¦   «         }| j        j        r"d„ |
d<   t          | j        j        ¦  «        |d<   t!          di |
¤Žt#          di |¤Žd	œ}	|}i }t%          | j        j        ¦  «        D ]}|                      |||¦  «        ||<   Œt+          | j        d | j        j        …         ¦  «        D ]?\  }} ||f|	| j        j        |                  || j        j        |                  ||d
œ|¤Ž}Œ@|                      |¦  «        }t3          ||¬¦  «        S )Nú:You must specify exactly one of input_ids or inputs_embeds©re   r   r+   ©rŒ   ©re   rX  rà   rA   r¿   c                  óB   — t          j        dt           j        ¬¦  «        S )NTr¨   )r7   rT   rC  )Úargss    r<   ú<lambda>z)Gemma3TextModel.forward.<locals>.<lambda>$  s   € ÅÄÈTÕY^ÔYcÐ@dÑ@dÔ@d€ r;   Úor_mask_function©Úfull_attentionrø   )rà   r
  r¿   rA   )Úlast_hidden_staterA   r:   )Ú
ValueErrorrO  r	   re   Úget_seq_lengthr7   r«   rˆ   rŒ   rÍ   r¸   ÚdictÚcopyrü   rD  r  r   r   rš   r›   rU  Ú	enumeraterS  rR  rT  r   )rU   rW   rà   r¿   rA   rX  rY  r¬   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsÚsliding_mask_kwargsrB   r
  r’   ÚiÚdecoder_layers                    r<   rZ   zGemma3TextModel.forwardü  sq  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLõ °Ð?Ð-ÅÑFÔFð 	ð œ+Ø!.Ø"0Ø#2Ø ,ðð ˆKð #.×"2Ò"2Ñ"4Ô"4ÐàŒ{Ô6ð tØ2dÐ2d�Ð.Ñ/Ý:WÐX\ÔXcÔXrÑ:sÔ:sÐ#Ð$6Ñ7õ #5Ð"CÐ"C°{Ð"CÐ"CÝ%FÐ%]Ð%]ÐI\Ð%]Ð%]ð#ð #Ðð &ˆØ ÐÝ˜dœkÔ5Ñ6Ô6ð 	gð 	gˆJØ.2¯oªo¸mÈ\Ð[eÑ.fÔ.fÐ 
Ñ+Ð+å )¨$¬+Ð6U¸¼Ô8UÐ6UÔ*VÑ WÔ Wð 	ð 	ÑˆAˆ}Ø)˜MØðà2°4´;Ô3JÈ1Ô3MÔNØ$7¸¼Ô8OÐPQÔ8RÔ$SØ)Ø /ðð ð ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆå&Ø+Ø+ð
ñ 
ô 
ð 	
r;   )NNNNNN)r3   r4   r5   r-   r9   r:  rP   r'   r(   r"   r7   r!  r`   r   r8   rC  r   r!   r   rZ   ra   rb   s   @r<   rF  rF  ä  s3  ø€ € € € € € àÐÐÑØ ÐðÐ/ð ð ð ð ð ð ð&  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ðC
ð C
àÔ# dÑ*ðC
ð œ tÑ+ðC
ð Ô&¨Ñ-ð	C
ð
  ™ðC
ð Ô(¨4Ñ/ðC
ð ˜$‘;ðC
ð Ð+Ô,ðC
ð 
!ðC
ð C
ð C
ñ „^ñ „_ñ  ÔðC
ð C
ð C
ð C
ð C
r;   rF  c                   ó2  ‡ — e Zd ZU ddiZddiZddgdgfiZddiZeed<   defˆ fd	„Z	e
e	 	 	 	 	 	 	 	 ddej        d
z  dej        d
z  dej        d
z  ded
z  dej        d
z  dej        d
z  ded
z  deej        z  dee         defd„¦   «         ¦   «         Zˆ xZS )ÚGemma3ForCausalLMúlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputrB   r@   Úkeep_full_weightre   c                 óþ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        | _        t          j        |j        |j        d¬¦  «        | _        |  	                    ¦   «          d S rg   )
rO   rP   rF  r$  rN  rR   rl   rj   rt  rW  rr   s     €r<   rP   zGemma3ForCausalLM.__init__M  sj   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý$ VÑ,Ô,ˆŒ
Ø Ô+ˆŒÝ”y Ô!3°VÔ5FÈUÐSÑSÔSˆŒð 	�ŠÑÔÐÐÐr;   Nr   rW   rà   r¿   rA   rX  ÚlabelsrY  Úlogits_to_keepr¬   r¤   c	           
      óÀ  —  | j         d||||||dœ|	¤Ž}
|
j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }| j        j        �2|| j        j        z  }t          j	        |¦  «        }|| j        j        z  }d}|� | j
        ||| j        fi |	¤Ž}t          |||
j        |
j        |
j        ¬¦  «        S )a‚  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, Gemma3ForCausalLM

        >>> model = Gemma3ForCausalLM.from_pretrained("google/gemma-2-9b")
        >>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-2-9b")

        >>> prompt = "What is your favorite condiment?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "What is your favorite condiment?"
        ```)rW   rà   r¿   rA   rX  rY  N)r?   r@   rA   rB   rC   r:   )r$  re  r¸   r^   Úslicert  re   Úfinal_logit_softcappingr7   rê   Úloss_functionrN  r   rA   rB   rC   )rU   rW   rà   r¿   rA   rX  rx  rY  ry  r¬   ÚoutputsrB   Úslice_indicesr@   r?   s                  r<   rZ   zGemma3ForCausalLM.forwardV  s&  € ð@ ,6¨4¬:ð ,
ØØ)Ø%Ø+Ø'Øð,
ð ,
ð ð,
ð ,
ˆð  Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆØŒ;Ô.Ð:Ø˜dœkÔAÑAˆFÝ”Z Ñ'Ô'ˆFØ˜dœkÔAÑAˆFàˆØÐØ%�4Ô% f¨f°d´oÐPÐPÈÐPÐPˆDå%ØØØ#Ô3Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r;   )NNNNNNNr   )r3   r4   r5   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr-   r9   rP   r#   r"   r7   r!  r`   r   r8   rC  r^   r   r!   r   rZ   ra   rb   s   @r<   rr  rr  E  sq  ø€ € € € € € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€JØÐÐÑðÐ/ð ð ð ð ð ð ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ð;
ð ;
àÔ# dÑ*ð;
ð œ tÑ+ð;
ð Ô&¨Ñ-ð	;
ð
  ™ð;
ð Ô(¨4Ñ/ð;
ð Ô  4Ñ'ð;
ð ˜$‘;ð;
ð ˜eœlÑ*ð;
ð Ð+Ô,ð;
ð 
 ð;
ð ;
ð ;
ñ „^ñ Ôð;
ð ;
ð ;
ð ;
ð ;
r;   rr  c                   ó:   ‡ — e Zd Zdefˆ fd„Zdej        fd„Zˆ xZS )r*  re   c                 ó  •— t          ¦   «                              ¦   «          t          j        t	          j        |j        j        |j        j        ¦  «        ¦  «        | _	        t          |j        j        |j        j        ¬¦  «        | _        t          |j        j        |j        j        z  ¦  «        | _        t          |j        dz  ¦  «        | _        | j        | j        z  | _        t          j        | j        | j        ¬¦  «        | _        d S )Nr  rI  )Úkernel_sizeÚstride)rO   rP   rR   r{   r7   r|   Úvision_configrj   Útext_configr-  rv   Úlayer_norm_epsÚmm_soft_emb_normr^   Ú
image_sizeÚ
patch_sizeÚpatches_per_imageÚmm_tokens_per_imageÚtokens_per_sider†  Ú	AvgPool2dÚavg_poolrr   s     €r<   rP   z"Gemma3MultiModalProjector.__init__—  sà   ø€ Ý‰Œ×ÒÑÔÐå*,¬,ÝŒK˜Ô,Ô8¸&Ô:LÔ:XÑYÔYñ+
ô +
ˆÔ'õ !.ØÔ Ô,°&Ô2FÔ2Uð!
ñ !
ô !
ˆÔõ "% VÔ%9Ô%DÈÔH\ÔHgÑ%gÑ!hÔ!hˆÔÝ" 6Ô#=¸sÑ#BÑCÔCˆÔØÔ1°TÔ5IÑIˆÔÝœ°Ô1AÈ$ÔJZÐ[Ñ[Ô[ˆŒˆˆr;   Úvision_outputsc                 ó¸  — |j         \  }}}|                     dd¦  «        }|                     ||| j        | j        ¦  «        }|                     ¦   «         }|                      |¦  «        }|                     d¦  «        }|                     dd¦  «        }|                      |¦  «        }t          j	        || j
        ¦  «        }|                     |¦  «        S )Nr+   r)   )rˆ   r»   rÖ   rŽ  rî   r’  Úflattenr‹  r7   ré   r-  r…   )	rU   r“  Ú
batch_sizer   rj   Úreshaped_vision_outputsÚpooled_vision_outputsÚnormed_vision_outputsÚprojected_vision_outputss	            r<   rZ   z!Gemma3MultiModalProjector.forward§  sÜ   € Ø%3Ô%9Ñ"ˆ
�A�{à"0×":Ò":¸1¸aÑ"@Ô"@ÐØ"9×"AÒ"AØ˜ TÔ%;¸TÔ=Sñ#
ô #
Ðð #:×"DÒ"DÑ"FÔ"FÐà $§¢Ð.EÑ FÔ FÐØ 5× =Ò =¸aÑ @Ô @ÐØ 5× ?Ò ?ÀÀ1Ñ EÔ EÐà $× 5Ò 5Ð6KÑ LÔ LÐå#(¤<Ð0EÀtÔGfÑ#gÔ#gÐ Ø'×/Ò/°Ñ?Ô?Ð?r;   )	r3   r4   r5   r,   rP   r7   r`   rZ   ra   rb   s   @r<   r*  r*  –  sq   ø€ € € € € ð\˜|ð \ð \ð \ð \ð \ð \ð @ e¤lð @ð @ð @ð @ð @ð @ð @ð @r;   r*  Útoken_type_idsrŒ   c                 ó$  — | dk                          |¬¦  «        }t          j                             |dd¬¦  «        d d …d d…f         }|| z  }t	          j        |                     ¦   «         d¬¦  «        dz
  }t	          j        ||d¦  «        }|S )Nr+   r]  )r+   r   r   )rß   r~   r¶   )r[   rR   rë   Úpadr7   Úcumsumr^   Úwhere)r›  rŒ   Úis_imageÚis_previous_imageÚnew_image_startÚ	group_idsÚblock_sequence_idss          r<   Úget_block_sequence_ids_for_maskr¥  º  sš   € ð  !Ò#×'Ò'¨vÐ'Ñ6Ô6€HÝœ×)Ò)¨(°FÀ!Ð)ÑDÔDÀQÀQÀQÈÈÈÀVÔLÐØÐ"3Ð!3Ñ3€OÝ”˜_×0Ò0Ñ2Ô2¸Ð:Ñ:Ô:¸QÑ>€IÝœ X¨y¸"Ñ=Ô=ÐØÐr;   re   rX  rA   r¿   r¤  c                 ój  — | ||||dœ}t          d
i |¤d|i¤Ž}t          |dd¦  «        r$d|j        v r|j                             d¦  «        }nd}t	          d
i |¤d|i¤Ž\  }	}
}
}
}}
}|	r|}nt          ||||¦  «        }t          d
i |¤t          |¦  «        t          | j        ¦  «        |dœ¤Ž}||d	œS )zãCreate full_attention and sliding_attention masks with correct composition.

    For global (full attention) layers:  OR(causal, blockwise)
    For local (sliding window) layers:  AND(sliding_window, OR(causal, blockwise))
    r^  r¤  r  NTr   rö   )rb  Úand_mask_functionrö   rc  r:   )	r   r©   r  Úindexr   r   r   r   r  )re   rX  rà   rA   r¿   r¤  rm  Ú	full_maskÚsliding_layer_idxÚ
early_exitr   Ú	kv_lengthÚ	kv_offsetÚpadded_block_sequence_idsÚsliding_masks                  r<   Úcreate_masks_for_vision_modelr°  Å  sP  € ð Ø&Ø(Ø*Ø$ðð €Kõ #ÐXÐX [ÐXÐXÐEWÐXÐXÐX€Iõ ˆ ¨dÑ3Ô3ð ¸ÀÔ@ZÐ8ZÐ8ZØ+Ô6×<Ò<¸TÑBÔBÐÐàÐõ 4Nð 4ð 4Ø
ð4ð 4à#ð4ð 4ð 4Ñ0€J��1�a˜ A yð ð 
Ø$6Ð!Ð!å$@Ø °	¸9ñ%
ô %
Ð!õ &ð ð Ø
ðå*Ð+DÑEÔEÝ0°Ô1FÑGÔGØ#ð	ð ð ð €Lð $Ø)ðð ð r;   zy
    The Base Gemma3 model which consists of a vision backbone and a language model without language modeling head.,
    c                   ó´  ‡ — e Zd ZdZdefˆ fd„Ze ed¬¦  «        dej	        de
e         deez  fd	„¦   «         ¦   «         Zd
ej        dej	        dej	        fd„Zee	 	 	 	 	 	 	 	 	 dd
ej        dz  dej	        dz  dej        dz  dej        dz  dedz  dej        dz  dej	        dz  dej        dz  dedz  de
e         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚGemma3ModelFre   c                 ó<  •— t          ¦   «                              |¦  «         t          j        |j        ¬¦  «        | _        t          |¦  «        | _        |j        j	        | _	        t          j        |j        ¬¦  «        }|| _
        |                      ¦   «          d S )Nr\  )rO   rP   r*   Úfrom_configrˆ  Úvision_towerr*  Úmulti_modal_projectorr‰  rN  Úlanguage_modelrW  )rU   re   r·  rV   s      €r<   rP   zGemma3Model.__init__
  s…   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý%Ô1¸Ô9MÐNÑNÔNˆÔÝ%>¸vÑ%FÔ%FˆÔ"Ø Ô,Ô7ˆŒå"Ô.°fÔ6HÐIÑIÔIˆØ,ˆÔØ�ŠÑÔÐÐÐr;   zOProjects the last hidden state from the vision model into language model space.r.   Úpixel_valuesr¬   r¤   c                 óh   —  | j         d|ddœ|¤Ž}|j        }|                      |¦  «        |_        |S )NT)r¸  Úreturn_dictr:   )rµ  re  r¶  Úpooler_output)rU   r¸  r¬   r“  re  s        r<   Úget_image_featureszGemma3Model.get_image_features  sK   € ð
 +˜Ô*Ða¸ÐRVÐaÐaÐZ`ÐaÐaˆØ*Ô<ÐØ'+×'AÒ'AÐBSÑ'TÔ'TˆÔ$àÐr;   rW   rX  Úimage_featuresc           	      ó"  — |€f| |                       ¦   «         t          j        d| j        j        t          j        |j        ¬¦  «        ¦  «        k    }|                     d¦  «        }n|| j        j        k    }|                     ¦   «         }|j	        d         |j	        d         z  }| 
                    d¦  «                             |j        ¦  «        }t          ||j	        d         z  |                     ¦   «         k    d|› d|› �¦  «         |S )	zï
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        Nr:   r±   r~   r   r+   z6Image features and image tokens do not match, tokens: z, features: )Úget_input_embeddingsr7   Úfullre   Úimage_token_idÚlongrŒ   ÚallÚsumrˆ   rÍ   r[   r$   Únumel)rU   rW   rX  r½  Úspecial_image_maskÚn_image_tokensÚn_image_featuress          r<   Úget_placeholder_maskz Gemma3Model.get_placeholder_mask  s  € ð ÐØ!.Ð2M°$×2KÒ2KÑ2MÔ2MÝ”
˜2˜tœ{Ô9ÅÄÐTaÔThÐiÑiÔiñ3ô 3ò "Ðð "4×!7Ò!7¸Ñ!;Ô!;ÐÐà!*¨d¬kÔ.HÒ!HÐà+×/Ò/Ñ1Ô1ˆØ)Ô/°Ô2°^Ô5IÈ!Ô5LÑLÐØ/×9Ò9¸"Ñ=Ô=×@Ò@ÀÔAUÑVÔVÐÝØ˜]Ô0°Ô4Ñ4¸×8LÒ8LÑ8NÔ8NÒNØsÀ^ÐsÐsÐaqÐsÐsñ	
ô 	
ð 	
ð "Ð!r;   Nrà   r¿   rA   r›  rx  rY  Ú	lm_kwargsc
           
      ó   — |du |duz  rt          d¦  «        ‚|�?| j        j        | j        k    r*|| j        j        k    }|                     ¦   «         }d||<   n|}|€ |                      ¦   «         |¦  «        }|�j|                      |d¬¦  «        j        }|                     |j	        |j
        ¦  «        }|                      |||¬¦  «        }|                     ||¦  «        }t          |x}t          ¦  «        sR| j                             ¦   «         ||||dœ}|�%t!          ||j	        ¬¦  «        }t#          dd	|i|¤Ž}nt%          di |¤Ž} | j        d|||||	dd
œ|
¤Ž}t)          |j        |j        |j        |j        |�|nd¬¦  «        S )a®  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`.

        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, Gemma3ForConditionalGeneration

        >>> model = Gemma3ForConditionalGeneration.from_pretrained("google/gemma32-3b-mix-224")
        >>> processor = AutoProcessor.from_pretrained("google/gemma32-3b-mix-224")

        >>> prompt = "Where is the cat standing?"
        >>> url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> inputs = processor(images=image, text=prompt,  return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(**inputs,)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Where is the cat standing?\nsnow"
        ```Nr[  r   T)rº  )rX  r½  r^  r]  r¤  )rà   r¿   rA   rX  rY  rº  )re  rA   rB   rC   r2   r:   )rf  re   rÁ  rN  rŸ   r¿  r¼  r»  r[   rŒ   r]   rÉ  Úmasked_scatterr¸   rh  Úget_text_configr¥  r°  r   r·  r1   re  rA   rB   rC   )rU   rW   r¸  rà   r¿   rA   r›  rX  rx  rY  rÊ  rÆ  Úllm_input_idsr½  rl  rm  r¤  r~  s                     r<   rZ   zGemma3Model.forward7  s*  € ðX ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZð Ð  T¤[Ô%?À4Ä?Ò%RÐ%RØ!*¨d¬kÔ.HÒ!HÐØ%ŸOšOÑ-Ô-ˆMØ01ˆMÐ,Ñ-Ð-à%ˆMàÐ Ø7˜D×5Ò5Ñ7Ô7¸ÑFÔFˆMð Ð#Ø!×4Ò4°\ÈtÐ4ÑTÔTÔbˆNØ+×.Ò.¨}Ô/CÀ]ÔEXÑYÔYˆNØ!%×!:Ò!:Ø¨À~ð ";ñ "ô "Ðð *×8Ò8Ð9KÈ^Ñ\Ô\ˆMõ °Ð?Ð-ÅÑFÔFð 	Oàœ+×5Ò5Ñ7Ô7Ø!.Ø"0Ø#2Ø ,ðð ˆKð Ð)Ý%DÀ^Ð\iÔ\pÐ%qÑ%qÔ%qÐ"Ý&Cð 'ð 'Ø'9ð'à!ð'ð 'Ð#Ð#õ
 '@Ð&NÐ&NÀ+Ð&NÐ&NÐ#à%�$Ô%ð 
Ø.Ø%Ø+Ø'ØØð
ð 
ð ð
ð 
ˆõ )Ø%Ô7Ø#Ô3Ø!Ô/ØÔ)Ø2>Ð2J  ÐPTð
ñ 
ô 
ð 	
r;   )	NNNNNNNNN)r3   r4   r5   Úaccepts_loss_kwargsr,   rP   r#   r"   r7   r8   r   r!   rD   r   r¼  r!  rÉ  r`   r   rC  r1   rZ   ra   rb   s   @r<   r²  r²    sû  ø€ € € € € ð  Ðð˜|ð ð ð ð ð ð ð Ø€^Ð!rÐsÑsÔsðØ!Ô-ðØ9?Ð@RÔ9Sðà	Ð+Ñ	+ðð ð ñ tÔsñ Ôðð"ØÔ)ð"Ø:?Ô:Kð"Ø]bÔ]nð"ð "ð "ð "ð0 Øð .2Ø15Ø.2Ø04Ø(,Ø26Ø26Ø*.Ø!%ðd
ð d
àÔ# dÑ*ðd
ð Ô'¨$Ñ.ðd
ð œ tÑ+ð	d
ð
 Ô&¨Ñ-ðd
ð  ™ðd
ð Ô(¨4Ñ/ðd
ð Ô(¨4Ñ/ðd
ð Ô  4Ñ'ðd
ð ˜$‘;ðd
ð Ð.Ô/ðd
ð 
Ð*Ñ	*ðd
ð d
ð d
ñ „^ñ Ôðd
ð d
ð d
ð d
ð d
r;   r²  c                   óþ  ‡ — e Zd ZddiZdefˆ fd„Zedej        de	e
         fd„¦   «         Zee	 	 	 	 	 	 	 	 	 	 dd
ej        dz  dej        dz  dej        dz  dej        dz  dedz  dej        dz  dej        dz  dej        dz  dedz  deej        z  de	e
         deez  fd„¦   «         ¦   «         Zdˆ fd„	Ze	 	 ddedej        dej        dz  dedz  dej        dz  dej        dz  dedz  defd„¦   «         Zˆ xZS )ÚGemma3ForConditionalGenerationrs  z(model.language_model.embed_tokens.weightre   c                 óú   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        j        |j        j        d¬¦  «        | _	        |  
                    ¦   «          d S rg   )rO   rP   r²  r$  rR   rl   r‰  rj   rN  rt  rW  rr   s     €r<   rP   z'Gemma3ForConditionalGeneration.__init__¨  se   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý  Ñ(Ô(ˆŒ
Ý”y Ô!3Ô!?ÀÔASÔA^ÐejÐkÑkÔkˆŒØ�ŠÑÔÐÐÐr;   r¸  r¬   c                 ó(   —  | j         j        |fi |¤ŽS rY   )r$  r¼  )rU   r¸  r¬   s      r<   r¼  z1Gemma3ForConditionalGeneration.get_image_features®  s   € à,ˆtŒzÔ,¨\ÐDÐD¸VÐDÐDÐDr;   Nr   rW   rà   r¿   rA   r›  rX  rx  rY  ry  rÊ  r¤   c                 óp  —  | j         d||||||||	|ddœ
|¤Ž}|d         }t          |
t          ¦  «        rt          |
 d¦  «        n|
}|                      |dd…|dd…f         ¦  «        }d}|�  | j        d||| j        j        j        dœ|¤Ž}t          |||j
        |j        |j        |j        ¬¦  «        S )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`.

        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, Gemma3ForConditionalGeneration

        >>> model = Gemma3ForConditionalGeneration.from_pretrained("google/gemma-3-4b-it")
        >>> processor = AutoProcessor.from_pretrained("google/gemma-3-4b-it")

        >>> messages = [
        ...     {
        ...         "role": "system",
        ...         "content": [
        ...             {"type": "text", "text": "You are a helpful assistant."}
        ...         ]
        ...     },
        ...     {
        ...         "role": "user", "content": [
        ...             {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"},
        ...             {"type": "text", "text": "Where is the cat standing?"},
        ...         ]
        ...     },
        ... ]

        >>> inputs = processor.apply_chat_template(
        ...     messages,
        ...     tokenize=True,
        ...     return_dict=True,
        ...     return_tensors="pt",
        ...     add_generation_prompt=True
        ... )
        >>> # Generate
        >>> generate_ids = model.generate(**inputs)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "user\nYou are a helpful assistant.\n\n\n\n\n\nWhere is the cat standing?\nmodel\nBased on the image, the cat is standing in a snowy area, likely outdoors. It appears to"
        ```
        T)
rW   r¸  r›  rà   r¿   rA   rX  rY  rx  rº  r   N)r@   rx  rN  )r?   r@   rA   rB   rC   r2   r:   )r$  r¸   r^   r{  rt  r}  re   r‰  rN  r>   rA   rB   rC   r2   )rU   rW   r¸  rà   r¿   rA   r›  rX  rx  rY  ry  rÊ  r~  rB   r  r@   r?   s                    r<   rZ   z&Gemma3ForConditionalGeneration.forward²  s  € ðz �$”*ð 
ØØ%Ø)Ø)Ø%Ø+Ø'ØØØð
ð 
ð ð
ð 
ˆð   œ
ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%ð Ø f¸¼Ô9PÔ9[ðð Ø_hðð ˆDõ ,ØØØ#Ô3Ø!Ô/ØÔ)Ø 'Ô ;ð
ñ 
ô 
ð 	
r;   TFc                 óT   •—  t          ¦   «         j        |f||dœ|¤Ž}|s|rd |d<   |S )N)rY  Úis_first_iterationr›  )rO   Úprepare_inputs_for_generation)rU   rW   rY  rÖ  r¬   Úmodel_inputsrV   s         €r<   r×  z<Gemma3ForConditionalGeneration.prepare_inputs_for_generation  sX   ø€ Ø<•u‘w”wÔ<Øð
Ø!*Ð?Qð
ð 
ØU[ð
ð 
ˆð #ð 	2¨)ð 	2à-1ˆLÐ)Ñ*àÐr;   rÖ  c                 óš   — |                       ¦   «         ||||dœ}|�$t          ||j        ¬¦  «        }	t          dd|	i|¤ŽS t	          di |¤ŽS )Nr^  r]  r¤  r:   )rÍ  r¥  rŒ   r°  r   )
re   rX  rà   rA   r¿   r›  rÖ  r¬   rm  r¤  s
             r<   r   z8Gemma3ForConditionalGeneration.create_masks_for_generate  s‡   € ð ×,Ò,Ñ.Ô.Ø*Ø,Ø.Ø(ð
ð 
ˆð Ð%Ý!@ÀÐXeÔXlÐ!mÑ!mÔ!mÐÝ0ð ð Ø#5ðàðð ð õ
 )Ð7Ð7¨;Ð7Ð7Ð7r;   )
NNNNNNNNNr   )TF)NF)r3   r4   r5   r€  r,   rP   r"   r7   r8   r   r!   r¼  r#   r!  r`   r   rC  r^   rD   r>   rZ   r×  rÅ   r
   rh  r   ra   rb   s   @r<   rÑ  rÑ     so  ø€ € € € € ð +Ð,VÐWÐð˜|ð ð ð ð ð ð ð ðE¨uÔ/@ð EÈFÐSeÔLfð Eð Eð Eñ „^ðEð Øð .2Ø15Ø.2Ø04Ø(,Ø26Ø26Ø*.Ø!%Ø-.ð[
ð [
àÔ# dÑ*ð[
ð Ô'¨$Ñ.ð[
ð œ tÑ+ð	[
ð
 Ô&¨Ñ-ð[
ð  ™ð[
ð Ô(¨4Ñ/ð[
ð Ô(¨4Ñ/ð[
ð Ô  4Ñ'ð[
ð ˜$‘;ð[
ð ˜eœlÑ*ð[
ð Ð.Ô/ð[
ð 
Ð-Ñ	-ð[
ð [
ð [
ñ „^ñ Ôð[
ðzð ð ð ð ð ð ð /3Ø*/ð8ð 8Ø ð8à”|ð8ð œ tÑ+ð8ð  ™ð	8ð
 ”l TÑ)ð8ð œ tÑ+ð8ð ! 4™Kð8ð 
ð8ð 8ð 8ñ „\ð8ð 8ð 8ð 8ð 8r;   rÑ  zÎ
Gemma3TextForSequenceClassification is a text-only sequence classification model that works with Gemma3TextConfig.
It uses the generic sequence classification implementation for efficiency and consistency.c                   ó   — e Zd ZU eed<   dZdS )Ú#Gemma3TextForSequenceClassificationre   rG  N)r3   r4   r5   r-   r9   r:  r:   r;   r<   rÛ  rÛ  8  s&   € € € € € € ð ÐÐÑØ ÐÐÐr;   rÛ  c                   óÜ   ‡ — e Zd Z	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dedz  dej        dz  dej        dz  d	ej        dz  d
ee	         de
fˆ fd„Zˆ xZS )ÚGemma3ForSequenceClassificationNrW   r¸  rà   r¿   rA   r›  rX  rx  r¬   r¤   c	                 óH   •—  t          ¦   «         j        d||||||||dœ|	¤ŽS )N)rW   rà   r¿   rA   rX  r¸  r›  rx  r:   )rO   rZ   )rU   rW   r¸  rà   r¿   rA   r›  rX  rx  r¬   rV   s             €r<   rZ   z'Gemma3ForSequenceClassification.forwardC  sK   ø€ ð �u‰wŒwŒð 

ØØ)Ø%Ø+Ø'Ø%Ø)Øð

ð 

ð ð

ð 

ð 
	
r;   )NNNNNNNN)r3   r4   r5   r7   r!  r8   r`   r   r   r!   r   rZ   ra   rb   s   @r<   rÝ  rÝ  B  sÿ   ø€ € € € € ð .2Ø15Ø.2Ø04Ø(,Ø26Ø26Ø*.ð
ð 
àÔ# dÑ*ð
ð Ô'¨$Ñ.ð
ð œ tÑ+ð	
ð
 Ô&¨Ñ-ð
ð  ™ð
ð Ô(¨4Ñ/ð
ð Ô(¨4Ñ/ð
ð Ô  4Ñ'ð
ð Ð+Ô,ð
ð 
*ð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r;   rÝ  )r#  rF  rr  rÑ  r²  rÝ  rÛ  )r+   )rÛ   NNrY   )aÚcollections.abcr   Údataclassesr   r7   Útorch.nnrR   Ú r   r+  Úactivationsr   Úcache_utilsr   r	   Úconfiguration_utilsr
   Ú
generationr   Úintegrationsr   r   Úmasking_utilsr   r   r   r   r   r   r   Úmodeling_layersr   r   Úmodeling_outputsr   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr    r!   r"   r#   r$   Úutils.deprecationr%   Úutils.genericr&   r'   Úutils.output_capturingr(   Úautor*   Úconfiguration_gemma3r,   r-   r1   r>   Ú	EmbeddingrF   ÚModulerd   rv   r‹   rÊ   rÓ   r`   r^   rÚ   r_   rD   ró   rõ   r  r#  rC  rD  rF  rr  r*  rŒ   r¥  rh  r°  r²  rÑ  rÛ  rÝ  Ú__all__r:   r;   r<   ú<module>r÷     sk  ðð* %Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø )Ð )Ð )Ð )Ð )Ð )Ø LÐ LÐ LÐ LÐ LÐ LÐ LÐ Lðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð \Ð [Ð [Ð [Ð [Ð [Ð [Ð [ðð ð ð ð ð ð ð ð ð ð ð ð LÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø nÐ nÐ nÐ nÐ nÐ nÐ nÐ nÐ nÐ nÐ nÐ nÐ nÐ nØ 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø Ð Ð Ð Ð Ð Ø @Ð @Ð @Ð @Ð @Ð @Ð @Ð @ð €ððñ ô ð
 ð9ð 9ð 9ð 9ð 9Ð 7ñ 9ô 9ñ „ñô ð9ð €ððñ ô ð
 ð9ð 9ð 9ð 9ð 9 ;ñ 9ô 9ñ „ñô ð9ð0Sð Sð Sð Sð S B¤Lñ Sô Sð Sðð ð ð ð �”	ñ ô ð ð =ð =ð =ð =ð =�B”Iñ =ô =ð =ð(E<ð E<ð E<ð E<ð E<˜BœIñ E<ô E<ð E<ðP(ð (ð (ð ÐÐ-Ñ.Ô.ðð ð ñ /Ô.ðð2	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð$ Ø Ø ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð �S‰[ð%ð �T‰\ð%ð �T‰\ð%ð ˆ5Œ<˜œÐ%Ô&ð%ð %ð %ð %ðD ÐÐ)Ñ*Ô*ðK)ð K)ð K)ð K)ð K)�b”iñ K)ô K)ñ +Ô*ðK)ð\+ð +ð +ð +ð +Ð3ñ +ô +ð +ð\ ð#^ð #^ð #^ð #^ð #^˜Oñ #^ô #^ñ „ð#^ðL
°#ð 
¸(ÀCÈÈcÐSVÐCWÐY]ÐC]Ô:^ð 
ð 
ð 
ð 
ð ð]
ð ]
ð ]
ð ]
ð ]
Ð+ñ ]
ô ]
ñ „ð]
ð@ ðM
ð M
ð M
ð M
ð M
Ð-¨ñ M
ô M
ñ „ðM
ð`!@ð !@ð !@ð !@ð !@ ¤	ñ !@ô !@ð !@ðHð °E´Lð È%Ì,ÐY]ÑJ]ð ÐinÔiuð ð ð ð ð9Øð9à”<ð9ð ”L 4Ñ'ð9ð ˜T‘\ð	9ð
 ”, Ñ%ð9ð œð9ð 
ð9ð 9ð 9ð 9ðx €ððñ ô ð
W
ð W
ð W
ð W
ð W
Ð'ñ W
ô W
ñô ð
W
ðt €ððñ ô ð
P8ð P8ð P8ð P8ð P8Ð%:¸Oñ P8ô P8ñô ð
P8ðf €ð^ðñ ô ð
!ð !ð !ð !ð !Ð*JÐLañ !ô !ñô ð
!ð

ð 
ð 
ð 
ð 
Ð&FÐH]ñ 
ô 
ð 
ð4ð ð €€€r;   