§
    9²¸jW¢ ã                   ój  — d dl Z d dlmZ d dlmZ d dlmZ d dlmZ d dl	Z	d dl	m
Z
 d dlmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZ ddlmZmZmZmZmZmZmZm Z  ddl!m"Z" ddl#m$Z$m%Z% ddl&m'Z' ddl(m)Z)m*Z* ddl+m,Z, ddl-m.Z.m/Z/m0Z0m1Z1m2Z2 ddl3m4Z4m5Z5m6Z6 ddl7m8Z8m9Z9 ddl:m;Z; ddl<m=Z=m>Z>m?Z?m@Z@mAZAmBZBmCZC ddlDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZLmMZM ddlNmOZO ddlPmQZQ ddlRmSZS ddlTmUZUmVZVmWZWmXZX  e1jY        eZ¦  «        Z[ded e	j\        d!e	j\        dz  d"edz  d#e	j\        dz  d$e	j\        d%e]fd&„Z^ G d'„ d(eH¦  «        Z_ G d)„ d*eE¦  «        Z`e G d+„ d,e$¦  «        ¦   «         Zae/e G d-„ d.e%¦  «        ¦   «         ¦   «         Zb G d/„ d0e
jc        ¦  «        Zd G d1„ d2eK¦  «        Ze G d3„ d4e
jc        ¦  «        Zf G d5„ d6e
jc        ¦  «        Zg G d7„ d8e
jc        ¦  «        Zh G d9„ d:e
jc        ¦  «        Zi G d;„ d<e
jc        ¦  «        Zj G d=„ d>e
jk        ¦  «        Zl G d?„ d@e
jc        ¦  «        Zm G dA„ dBe
jc        ¦  «        Zn G dC„ dDe
jc        ¦  «        Zo G dE„ dFe
jc        ¦  «        Zp G dG„ dHe@¦  «        Zq	 d€dIe	j\        dJe	j\        dKe	j\        d#e	j\        dLerd%e	j\        fdM„Zs G dN„ dOeS¦  «        Zte6 G dP„ dQe=¦  «        ¦   «         Zu G dR„ dSe>¦  «        Zv G dT„ dUe
jc        ¦  «        Zw G dV„ dWe@¦  «        Zx G dX„ dYeA¦  «        Zy G dZ„ d[e
jc        ¦  «        Zz G d\„ d]eO¦  «        Z{ G d^„ d_e
jc        ¦  «        Z| G d`„ dae>¦  «        Z} G db„ dceC¦  «        Z~ G dd„ deeJ¦  «        Z e/df¬g¦  «         G dh„ dieB¦  «        ¦   «         Z€ e/dj¬g¦  «         G dk„ dle?¦  «        ¦   «         Z� G dm„ dne¦  «        Z‚ G do„ dpe¦  «        Zƒ G dq„ dreI¦  «        Z„dse	j\        dz  dte	j\        dz  d%edz  fdu„Z…dve	j\        dwe	j†        d%e	j\        fdx„Z‡ e/dy¬g¦  «         G dz„ d{eG¦  «        ¦   «         Zˆ e/d|¬g¦  «         G d}„ d~eF¦  «        ¦   «         Z‰g d¢ZŠdS )�é    N)ÚUserDict)ÚCallable)Ú	dataclass)Úcached_property)Únn)Ú
functionalé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚPreTrainedConfig)Ú_preprocess_mask_argumentsÚblockwise_overlayÚcreate_bidirectional_maskÚcreate_causal_maskÚcreate_masks_for_generateÚ!create_sliding_window_causal_maskÚmaybe_pad_block_sequence_idsÚsliding_window_overlay)ÚFlashAttentionKwargs)ÚBaseModelOutputWithPastÚBaseModelOutputWithPooling)ÚROPE_INIT_FUNCTIONS)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚloggingÚtorch_compilable_check)Úmaybe_autocastÚmerge_with_config_defaultsÚno_inherit_decorator)ÚOutputRecorderÚcapture_outputsé   )Ú	AutoModel)ÚGemma3AttentionÚGemma3DecoderLayerÚGemma3ForCausalLMÚ	Gemma3MLPÚGemma3RotaryEmbeddingÚGemma3TextModelÚGemma3TextScaledWordEmbedding)	ÚGemma3nCausalLMOutputWithPastÚGemma3nForConditionalGenerationÚGemma3nModelÚGemma3nModelOutputWithPastÚGemma3nMultimodalEmbedderÚGemma3nPreTrainedModelÚGemma3nRMSNormÚapply_rotary_pos_embÚeager_attention_forward)ÚMixtralExperts)Úsliding_window_mask_function)ÚSam3ViTRotaryEmbeddingé   )ÚGemma4AudioConfigÚGemma4ConfigÚGemma4TextConfigÚGemma4VisionConfigÚconfigÚinputs_embedsÚattention_maskÚpast_key_valuesÚposition_idsÚblock_sequence_idsÚreturnc                 ób  — | ||||dœ}t          d	i |¤Ž}t          |dd¦  «        r$d|j        v r|j                             d¦  «        }nd}t	          d	i |¤d|i¤Ž\  }	}
}
}
}}
}|	r|}nt          ||||¦  «        }t          d	i |¤t          |¦  «        t          | j        ¦  «        |dœ¤Ž}||dœS )
a‹  Create full_attention and sliding_attention masks with correct composition.

    For global (full attention) layers:  causal only (no bidirectional)
    For local (sliding window) layers:  AND(sliding_window, OR(causal, blockwise))

    Unlike Gemma 3 (which applies bidirectional attention on all layers), Gemma 4
    explicitly disables bidirectional attention on global attention layers.
    ©rB   rC   rD   rE   rF   Ú
is_slidingNTr   Ú	layer_idx)Úor_mask_functionÚand_mask_functionrL   ©Úfull_attentionÚsliding_attention© )	r   ÚgetattrrK   Úindexr   r   r   r   Úsliding_window)rB   rC   rD   rE   rF   rG   Úmask_kwargsÚ	full_maskÚsliding_layer_idxÚ
early_exitÚ_Ú	kv_lengthÚ	kv_offsetÚpadded_block_sequence_idsÚsliding_masks                  úy/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/gemma4/modular_gemma4.pyÚcreate_masks_for_vision_modelr`   S   sD  € ð" Ø&Ø(Ø*Ø$ðð €Kõ #Ð1Ð1 [Ð1Ð1€Iõ ˆ ¨dÑ3Ô3ð ¸ÀÔ@ZÐ8ZÐ8ZØ+Ô6×<Ò<¸TÑBÔBÐÐàÐõ 4Nð 4ð 4Ø
ð4ð 4à#ð4ð 4ð 4Ñ0€J��1�a˜ A yð ð 
Ø$6Ð!Ð!å$@Ø °	¸9ñ%
ô %
Ð!õ &ð ð Ø
ðå*Ð+DÑEÔEÝ0°Ô1FÑGÔGØ#ð	ð ð ð €Lð $Ø)ðð ð ó    c                   ó\   — e Zd ZU dZdZeeeej	        ej	        f         f         dz  e
d<   dS )ÚGemma4ModelOutputWithPasta¯  
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
    audio_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        audio_hidden_states of the model produced by the audio encoder and after projecting the last hidden state.
    shared_kv_states (`dict`, *optional*):
        Dictionary mapping layer type strings to tuples of (key_states, value_states) tensors.
        Used to pass shared KV states between layers during KV sharing.
    NÚshared_kv_states©Ú__name__Ú
__module__Ú__qualname__Ú__doc__rd   ÚdictÚstrÚtupleÚtorchÚTensorÚ__annotations__rR   ra   r_   rc   rc   ’   sN   € € € € € € ðð ð" MQÐ�d˜3  e¤l°E´LÐ&@Ô AÐAÔBÀTÑIÐPÐPÑPÐPÐPra   rc   c                   ó\   — e Zd ZU dZdZeeeej	        ej	        f         f         dz  e
d<   dS )ÚGemma4CausalLMOutputWithPasta  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.text_config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder after projecting last hidden state.
    audio_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        audio_hidden_states of the model produced by the audio encoder and after projecting the last hidden state.
    shared_kv_states (`dict`, *optional*):
        Dictionary mapping layer type strings to tuples of (key_states, value_states) tensors.
        Used to pass shared KV states between layers during KV sharing.
    Nrd   re   rR   ra   r_   rq   rq   §   sN   € € € € € € ðð ð* MQÐ�d˜3  e¤l°E´LÐ&@Ô AÐAÔBÀTÑIÐPÐPÑPÐPÐPra   rq   c                   ó\   — e Zd ZU dZdZeeeej	        ej	        f         f         dz  e
d<   dS )ÚGemma4TextModelOutputWithPasta9  
    BaseModelOutputWithPast extended with shared_kv_states for KV sharing.

    Args:
        shared_kv_states (`dict`, *optional*):
            Dictionary mapping layer type strings to tuples of (key_states, value_states) tensors.
            Used to pass shared KV states between layers during KV sharing.
    Nrd   re   rR   ra   r_   rs   rs   À   sN   € € € € € € ðð ð MQÐ�d˜3  e¤l°E´LÐ&@Ô AÐAÔBÀTÑIÐPÐPÑPÐPÐPra   rs   c                   ó2   — e Zd ZU dZdZej        dz  ed<   dS )ÚGemma4AudioModelOutputz§
    attention_mask (`torch.BoolTensor`, *optional*):
        A torch.BoolTensor of shape `(batch_size, num_frames)`. True for valid positions, False for padding.
    NrD   )rf   rg   rh   ri   rD   rm   Ú
BoolTensorro   rR   ra   r_   ru   ru   Î   s6   € € € € € € ðð ð
 /3€N�EÔ$ tÑ+Ð2Ð2Ñ2Ð2Ð2ra   ru   c                   óZ   ‡ — e Zd Zdeez  dededdfˆ fd„Zdej        dej        fd„Z	ˆ xZ
S )	ÚGemma4ClippableLinearrB   Úin_featuresÚout_featuresrH   Nc                 ón  •— t          ¦   «                              ¦   «          |j        | _        t          j        ||d¬¦  «        | _        | j        rät          j        t          j        t          d¦  «         ¦  «        ¦  «        | _
        t          j        t          j        t          d¦  «        ¦  «        ¦  «        | _        t          j        t          j        t          d¦  «         ¦  «        ¦  «        | _        t          j        t          j        t          d¦  «        ¦  «        ¦  «        | _        d S d S )NF©ÚbiasÚinf)ÚsuperÚ__init__Úuse_clipped_linearsr   ÚLinearÚlinearÚBufferrm   ÚtensorÚfloatÚ	input_minÚ	input_maxÚ
output_minÚ
output_max)ÚselfrB   ry   rz   Ú	__class__s       €r_   r€   zGemma4ClippableLinear.__init__Ú   sâ   ø€ õ 	‰Œ×ÒÑÔÐØ#)Ô#=ˆÔ Ý”i ¨\ÀÐFÑFÔFˆŒàÔ#ð 	DÝœY¥u¤|µU¸5±\´\°MÑ'BÔ'BÑCÔCˆDŒNÝœY¥u¤|µE¸%±L´LÑ'AÔ'AÑBÔBˆDŒNÝ œi­¬µe¸E±l´l°]Ñ(CÔ(CÑDÔDˆDŒOÝ œi­¬µU¸5±\´\Ñ(BÔ(BÑCÔCˆDŒOˆOˆOð		Dð 	Dra   Úhidden_statesc                 óÌ   — | j         r t          j        || j        | j        ¦  «        }|                      |¦  «        }| j         r t          j        || j        | j        ¦  «        }|S ©N)r�   rm   Úclampr‡   rˆ   rƒ   r‰   rŠ   )r‹   r�   s     r_   ÚforwardzGemma4ClippableLinear.forwardê   s_   € ØÔ#ð 	WÝ!œK¨°t´~ÀtÄ~ÑVÔVˆMàŸš MÑ2Ô2ˆàÔ#ð 	YÝ!œK¨°t´ÈÌÑXÔXˆMàÐra   )rf   rg   rh   rA   r>   Úintr€   rm   rn   r‘   Ú__classcell__©rŒ   s   @r_   rx   rx   Ù   s™   ø€ € € € € ðDà"Ð%6Ñ6ðDð ðDð ð	Dð
 
ðDð Dð Dð Dð Dð Dð 	 U¤\ð 	°e´lð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ra   rx   c                   ó   — e Zd ZdS )ÚGemma4RMSNormN©rf   rg   rh   rR   ra   r_   r–   r–   ö   ó   € € € € € Ø€Dra   r–   c                   óŒ   ‡ — e Zd ZU dZej        ed<   defˆ fd„Z ej	        ¦   «         dej        dej        fd„¦   «         Z
ˆ xZS )Ú Gemma4AudioRelPositionalEncodingzíSinusoidal relative positional encoding for the audio encoder.

    Produces position embeddings of shape [1, context_size // 2 + 1, hidden_size] with
    concatenated [sin..., cos...] layout matching the original Gemma4 convention.
    Úinv_timescalesrB   c                 óì  •— t          ¦   «                              ¦   «          |j        | _        |j        |j        z   dz
  |j        z   | _        d}d}| j        dz  }t          j        ||z  ¦  «        t          |dz
  d¦  «        z  }|t          j        t          j        |¦  «        | z  ¦  «        z  }t          j        |                     d¦  «                             d¦  «        d¬¦  «        | _        d S )Nr=   ç      ð?ç     ˆÃ@r(   r   F©Ú
persistent)r   r€   Úhidden_sizeÚattention_chunk_sizeÚattention_context_leftÚattention_context_rightÚcontext_sizeÚmathÚlogÚmaxrm   ÚexpÚaranger   r„   Ú	unsqueezer›   )r‹   rB   Úmin_timescaleÚmax_timescaleÚnum_timescalesÚlog_timescale_incrementr›   rŒ   s          €r_   r€   z)Gemma4AudioRelPositionalEncoding.__init__  sí   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔàÔ'¨&Ô*GÑGÈ!ÑKÈfÔNlÑlð 	Ôð ˆØˆØÔ)¨QÑ.ˆÝ"&¤(¨=¸=Ñ+HÑ"IÔ"IÍCÐP^ÐabÑPbÐdeÑLfÔLfÑ"fÐØ&­¬µ5´<ÀÑ3OÔ3OÐSjÐRjÑ3jÑ)kÔ)kÑkˆÝ œi¨×(@Ò(@ÀÑ(CÔ(C×(MÒ(MÈaÑ(PÔ(PÐ]bÐcÑcÔcˆÔÐÐra   r�   rH   c                 óP  — t          j        | j        dz  dd|j        ¬¦  «        }|d         }|| j                             |j        ¬¦  «        z  }t          j        t          j        |¦  «        t          j        |¦  «        gd¬¦  «        }|                     |j	        ¬¦  «        S )Nr(   éÿÿÿÿ©Údevice©.N©Údim©Údtype)
rm   rª   r¥   r³   r›   ÚtoÚcatÚsinÚcosr¸   )r‹   r�   rF   Úscaled_timeÚ	pos_embeds        r_   r‘   z(Gemma4AudioRelPositionalEncoding.forward  s•   € å”| DÔ$5¸Ñ$:¸BÀÈ=ÔK_Ð`Ñ`Ô`ˆØ# IÔ.ˆØ" TÔ%8×%;Ò%;À=ÔCWÐ%;Ñ%XÔ%XÑXˆÝ”I�uœy¨Ñ5Ô5µu´yÀÑ7MÔ7MÐNÐTVÐWÑWÔWˆ	Ø�|Š| -Ô"5ˆ|Ñ6Ô6Ð6ra   )rf   rg   rh   ri   rm   rn   ro   r>   r€   Úno_gradr‘   r“   r”   s   @r_   rš   rš   ú   s¢   ø€ € € € € € ðð ð ”LÐ Ð Ñ ðdÐ0ð dð dð dð dð dð dð €U„]�_„_ð7 U¤\ð 7°e´lð 7ð 7ð 7ñ „_ð7ð 7ð 7ð 7ð 7ra   rš   c                   óò   ‡ — e Zd ZdZdedefˆ fd„Zdej        dej        fd„Z	dej        dej        fd„Z
d	ej        dej        fd
„Z	 ddej        dej        dej        dz  deej        df         fd„Zˆ xZS )ÚGemma4AudioAttentionz3Chunked local attention with relative position biasrB   rL   c                 óP  •— t          ¦   «                              ¦   «          || _        || _        |j        | _        |j        |j        z  | _        |j        | _	        | j        dz  t          j        d¦  «        z  | _        t          j        dt          j        z   ¦  «        t          j        d¦  «        z  | _        |j        | _        |j        dz
  | _        |j        | _        | j        | j        z   | j        z   | _        t-          ||j        | j	        | j        z  ¦  «        | _        t-          ||j        | j	        | j        z  ¦  «        | _        t-          ||j        | j	        | j        z  ¦  «        | _        t-          ||j        |j        ¦  «        | _        t7          j        |j        | j	        | j        z  d¬¦  «        | _        t7          j        t?          j         | j        ¦  «        ¦  «        | _!        t7          j"        t?          j#        | j        ¦  «        d¬¦  «        | _$        d S )Nç      à¿r(   r=   Fr|   rŸ   )%r   r€   rB   rL   Úattention_logit_capÚattention_logits_soft_capr¡   Únum_attention_headsÚhead_dimÚ	num_headsr¦   r§   Úq_scaleÚeÚk_scaler¢   Ú
chunk_sizer£   Úmax_past_horizonr¤   Úmax_future_horizonr¥   rx   Úq_projÚk_projÚv_projÚpostr   r‚   Úrelative_k_projÚ	Parameterrm   ÚzerosÚper_dim_scaler„   r…   Úsoftcap©r‹   rB   rL   rŒ   s      €r_   r€   zGemma4AudioAttention.__init__  s´  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒØ)/Ô)CˆÔ&ØÔ*¨fÔ.HÑHˆŒØÔ3ˆŒàœ tÑ+­t¬x¸©{¬{Ñ:ˆŒÝ”x ¥D¤F¡
Ñ+Ô+­d¬h°q©k¬kÑ9ˆŒà Ô5ˆŒØ &Ô =ÀÑ AˆÔØ"(Ô"@ˆÔØ œO¨dÔ.CÑCÀdÔF]Ñ]ˆÔå+¨F°FÔ4FÈÌÐY]ÔYfÑHfÑgÔgˆŒÝ+¨F°FÔ4FÈÌÐY]ÔYfÑHfÑgÔgˆŒÝ+¨F°FÔ4FÈÌÐY]ÔYfÑHfÑgÔgˆŒÝ)¨&°&Ô2DÀfÔFXÑYÔYˆŒ	å!œy¨Ô);¸T¼^ÈdÌmÑ=[ÐbgÐhÑhÔhˆÔÝœ\­%¬+°d´mÑ*DÔ*DÑEÔEˆÔå”y¥¤¨dÔ.LÑ!MÔ!MÐZ_Ð`Ñ`Ô`ˆŒˆˆra   r�   rH   c           	      óô   — |j         \  }}}}|| j        z   dz
  | j        z  }|| j        z  |z
  }t          j        |ddddd|f¦  «        }|                     ||| j        ||¦  «                             ¦   «         S )z€Splits a `(batch_size, seq_len, num_heads, head_dim)` tensor into non-overlapping blocks of `chunk_size` along the sequence dim.r=   r   )ÚshaperÌ   ÚFÚpadÚreshapeÚ
contiguous)r‹   r�   Ú
batch_sizeÚseq_lenrÈ   rÇ   Ú
num_blocksrÜ   s           r_   Ú_convert_to_blockz&Gemma4AudioAttention._convert_to_block6  s…   € à3@Ô3FÑ0ˆ
�G˜Y¨Ø ¤Ñ/°!Ñ3¸¼ÑGˆ
Ø˜4œ?Ñ*¨WÑ4ˆÝœ˜m¨a°°A°q¸!¸SÐ-AÑBÔBˆØ×$Ò$ Z°¸T¼_ÈiÐYaÑbÔb×mÒmÑoÔoÐora   c           
      ó  — |j         \  }}}}t          j        |dddd| j        | j        | j        z   dz
  f¦  «        }|                     d| j        | j        ¦  «        }t          j	        |dd¦  «        }| 
                    ¦   «         S )z`Extracts overlapping context windows of `context_size` for every block, strided by `chunk_size`.r   r=   r±   r(   )rÚ   rÛ   rÜ   rÍ   rÎ   rÌ   Úunfoldr¥   rm   ÚmovedimrÞ   )r‹   r�   rß   rà   rÈ   rÇ   s         r_   Ú_extract_block_contextz+Gemma4AudioAttention._extract_block_context>  sŽ   € à3@Ô3FÑ0ˆ
�G˜Y¨ÝœØ˜A˜q ! Q¨Ô(=¸tÔ?VÐY]ÔYhÑ?hÐklÑ?lÐmñ
ô 
ˆð &×,Ò,¨Q°Ô0AÀ4Ä?ÑSÔSˆÝœ m°R¸Ñ;Ô;ˆØ×'Ò'Ñ)Ô)Ð)ra   Úxc                 óð   — |j         \  }}}}}| j        }t          j        |d|dz   |z
  f¦  «        }|                     |||||dz   z  ¦  «        }|dd||z  …f         }|                     |||||¦  «        S )zjRelative position shift for blocked attention. See appendix B of https://huggingface.co/papers/1901.02860.r   r=   .N)rÚ   r¥   rÛ   rÜ   Úview)r‹   rç   rß   rÈ   rá   Ú
block_sizeÚposition_lengthr¥   s           r_   Ú
_rel_shiftzGemma4AudioAttention._rel_shiftH  s’   € àIJÌÑFˆ
�I˜z¨:°ØÔ(ˆÝŒE�!�a˜¨Ñ)¨OÑ;Ð<Ñ=Ô=ˆØ�FŠF�:˜y¨*°jÀLÐSTÑDTÑ6UÑVÔVˆØˆcÐ.�Z ,Ñ.Ð.Ð.Ô/ˆØ�vŠv�j )¨Z¸À\ÑRÔRÐRra   NÚposition_embeddingsrD   c                 óö  — |j         \  }}}||| j        | j        f}|                      |¦  «                             ¦   «                              |¦  «        }|                      |¦  «                             ¦   «                              |¦  «        }	|                      |¦  «                             ¦   «                              |¦  «        }
|| j        z  t          j
        | j        ¦  «        z  }|	| j        z  }	|                      |¦  «        }|                      |	¦  «        }	|                      |
¦  «        }
|j         d         }|                      |¦  «        }|                     d| j        | j        ¦  «        }|                     |j        ¬¦  «        }|                     ddddd¦  «        }||	                     ddddd¦  «        z  }|                     || j        d| j        ¦  «        }||                     ddd¦  «        z  }|                     || j        || j        d¦  «        }|                      |¦  «        }||z   }|| j        z  }t/          j        |¦  «        }|| j        z  }|�2|                     |                     ¦   «         | j        j        ¦  «        }t          j        |dt.          j        ¬¦  «                             |
j        ¦  «        }||
                     ddddd¦  «        z  }|                     ddddd¦  «                             ||| j        z  d¦  «        }|d d …d |…f                              ¦   «         }|                       |                     |j        ¦  «        ¦  «        }||fS )	Nr=   r±   r·   r   r	   r(   é   ©r¶   r¸   )!rÚ   rÈ   rÇ   rÏ   r†   ré   rÐ   rÑ   rÉ   rÛ   ÚsoftplusrÖ   rË   râ   ræ   rÓ   r¹   r¸   ÚpermuterÝ   rÌ   rì   r×   rm   ÚtanhÚmasked_fillÚlogical_notrB   Úattention_invalid_logits_valueÚsoftmaxÚfloat32rÞ   rÒ   )r‹   r�   rí   rD   rß   Ú
seq_lengthrZ   Úhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesrá   Úrelative_key_statesÚqueriesÚ	matrix_acÚqueries_flatÚ	matrix_bdÚattn_weightsÚattn_outputs                      r_   r‘   zGemma4AudioAttention.forwardQ  s6  € ð %2Ô$7Ñ!ˆ
�J Ø" J°´ÀÄÐNˆà—{’{ =Ñ1Ô1×7Ò7Ñ9Ô9×>Ò>¸|ÑLÔLˆØ—[’[ Ñ/Ô/×5Ò5Ñ7Ô7×<Ò<¸\ÑJÔJˆ
Ø—{’{ =Ñ1Ô1×7Ò7Ñ9Ô9×>Ò>¸|ÑLÔLˆà# d¤lÑ2µQ´ZÀÔ@RÑ5SÔ5SÑSˆØ $¤,Ñ.ˆ
à×-Ò-¨lÑ;Ô;ˆØ×0Ò0°Ñ<Ô<ˆ
Ø×2Ò2°<Ñ@Ô@ˆØ!Ô'¨Ô*ˆ
à"×2Ò2Ð3FÑGÔGÐØ1×6Ò6°r¸4¼>È4Ì=ÑYÔYÐØ1×4Ò4¸<Ô;MÐ4ÑNÔNÐà×&Ò& q¨!¨Q°°1Ñ5Ô5ˆØ˜j×0Ò0°°A°q¸!¸QÑ?Ô?Ñ?ˆ	à—’ z°4´>À2ÀtÄ}ÑUÔUˆØ Ð#6×#>Ò#>¸qÀ!ÀQÑ#GÔ#GÑGˆ	Ø×%Ò% j°$´.À*ÈdÌoÐ_aÑbÔbˆ	Ø—O’O IÑ.Ô.ˆ	à  9Ñ,ˆØ# d¤lÑ2ˆÝ”z ,Ñ/Ô/ˆØ# d¤lÑ2ˆàÐ%Ø'×3Ò3Ø×*Ò*Ñ,Ô,¨d¬kÔ.Xñô ˆLõ ”y °2½U¼]ÐKÑKÔK×NÒNÈ|ÔOaÑbÔbˆØ" \×%9Ò%9¸!¸QÀÀ1ÀaÑ%HÔ%HÑHˆØ!×)Ò)¨!¨Q°°1°aÑ8Ô8×@Ò@ÀÈZÐZ^ÔZiÑMiÐkmÑnÔnˆØ! ! ! ! [ j [ .Ô1×<Ò<Ñ>Ô>ˆØ—i’i §¢¨}Ô/BÑ CÔ CÑDÔDˆà˜LÐ(Ð(ra   r�   )rf   rg   rh   ri   r>   r’   r€   rm   rn   râ   ræ   rì   rv   rl   r‘   r“   r”   s   @r_   rÁ   rÁ     s4  ø€ € € € € Ø=Ð=ðaÐ0ð a¸Sð að að að að að að4p¨u¬|ð pÀÄð pð pð pð pð*°E´Lð *ÀUÄ\ð *ð *ð *ð *ðS˜EœLð S¨U¬\ð Sð Sð Sð Sð 37ð	1)ð 1)à”|ð1)ð #œ\ð1)ð Ô(¨4Ñ/ð	1)ð
 
ˆuŒ|˜TÐ!Ô	"ð1)ð 1)ð 1)ð 1)ð 1)ð 1)ð 1)ð 1)ra   rÁ   c                   óJ   ‡ — e Zd Zˆ fd„Zddej        dej        dz  fd„Zˆ xZS )Ú'Gemma4AudioSubSampleConvProjectionLayerc                 óð   •— t          ¦   «                              ¦   «          t          j        ||dddd¬¦  «        | _        t          j        ||dd¬¦  «        | _        t          j        ¦   «         | _        d S )N)r	   r	   )r(   r(   r=   F)Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚpaddingr}   T)ÚepsÚelementwise_affiner}   )	r   r€   r   ÚConv2dÚconvÚ	LayerNormÚnormÚReLUÚact)r‹   r  r	  Únorm_epsrŒ   s       €r_   r€   z0Gemma4AudioSubSampleConvProjectionLayer.__init__†  sr   ø€ Ý‰Œ×ÒÑÔÐÝ”IØ#Ø%ØØØØð
ñ 
ô 
ˆŒ	õ ”L °8ÐPTÐ[`ÐaÑaÔaˆŒ	Ý”7‘9”9ˆŒˆˆra   Nr�   Úmaskc           
      óÆ  — |�.|                      |j        ¬¦  «        }||d d …d d d …d f         z  }|                      |                      | j        j        j        ¦  «        ¦  «        }|                      |                      |                     dddd¦  «        ¦  «                             dddd¦  «                             ¦   «         ¦  «        }|�|d d …d d d…f         }||fS )Nr²   r   r(   r	   r=   )	r¹   r³   r  Úweightr¸   r  r  rò   rÞ   )r‹   r�   r  s      r_   r‘   z/Gemma4AudioSubSampleConvProjectionLayer.forward“  sâ   € ØÐØ—7’7 -Ô"6�7Ñ7Ô7ˆDØ)¨D°°°°D¸!¸!¸!¸TÐ1AÔ,BÑBˆMàŸ	š	 -×"2Ò"2°4´9Ô3CÔ3IÑ"JÔ"JÑKÔKˆØŸš §¢¨=×+@Ò+@ÀÀAÀqÈ!Ñ+LÔ+LÑ!MÔ!M×!UÒ!UÐVWÐYZÐ\]Ð_`Ñ!aÔ!a×!lÒ!lÑ!nÔ!nÑoÔoˆàÐØ˜˜˜˜3˜3˜Q˜3˜”<ˆDà˜dÐ"Ð"ra   r�   )rf   rg   rh   r€   rm   rn   r‘   r“   r”   s   @r_   r  r  …  sh   ø€ € € € € ðð ð ð ð ð#ð # U¤\ð #¸¼ÈÑ9Lð #ð #ð #ð #ð #ð #ð #ð #ra   r  c            	       óz   ‡ — e Zd Zdefˆ fd„Z	 ddej        dej        dz  deej        ej        f         fd„Zˆ xZ	S )	Ú"Gemma4AudioSubSampleConvProjectionrB   c                 óx  •— t          ¦   «                              ¦   «          t          d|j        d         |j        ¬¦  «        | _        t          |j        d         |j        d         |j        ¬¦  «        | _        |j        d         dz  |j        d         z  }t          j        ||j	        d¬¦  «        | _
        d S )Nr=   r   )r  r	  r  rï   Fr|   )r   r€   r  Úsubsampling_conv_channelsÚrms_norm_epsÚlayer0Úlayer1r   r‚   r¡   Úinput_proj_linear)r‹   rB   Úproj_input_dimrŒ   s      €r_   r€   z+Gemma4AudioSubSampleConvProjection.__init__¢  s¼   ø€ Ý‰Œ×ÒÑÔÐÝ=ØØÔ9¸!Ô<ØÔ(ð
ñ 
ô 
ˆŒõ
 >ØÔ8¸Ô;ØÔ9¸!Ô<ØÔ(ð
ñ 
ô 
ˆŒð
 !Ô:¸1Ô=ÀÑBÀfÔFfÐghÔFiÑiˆÝ!#¤¨>¸6Ô;MÐTYÐ!ZÑ!ZÔ!ZˆÔÐÐra   NÚinput_featuresÚinput_features_maskrH   c                 óT  — |                      d¦  «        }|                      ||¦  «        \  }}|                      ||¦  «        \  }}|j        \  }}}}|                     dddd¦  «                             ¦   «                              ||d¦  «        }|                      |¦  «        |fS )Nr=   r   r(   r	   r±   )r«   r  r  rÚ   rò   rÞ   rÝ   r   )r‹   r"  r#  r�   r  rß   rZ   rà   s           r_   r‘   z*Gemma4AudioSubSampleConvProjection.forward±  s«   € ð
 '×0Ò0°Ñ3Ô3ˆØ"Ÿkšk¨-Ð9LÑMÔMÑˆ�tØ"Ÿkšk¨-¸Ñ>Ô>Ñˆ�tà$1Ô$7Ñ!ˆ
�A�w Ø%×-Ò-¨a°°A°qÑ9Ô9×DÒDÑFÔF×NÒNÈzÐ[bÐdfÑgÔgˆØ×%Ò% mÑ4Ô4°dÐ:Ð:ra   r�   )
rf   rg   rh   r>   r€   rm   rn   rl   r‘   r“   r”   s   @r_   r  r  ¡  s�   ø€ € € € € ð[Ð0ð [ð [ð [ð [ð [ð [ð$ 48ð;ð ;àœð;ð #œ\¨DÑ0ð;ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð	;ð ;ð ;ð ;ð ;ð ;ð ;ð ;ra   r  c                   óH   ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Zˆ xZS )ÚGemma4AudioFeedForwardrB   c                 ó¤  •— t          ¦   «                              ¦   «          || _        t          ||j        |j        dz  ¦  «        | _        t          ||j        dz  |j        ¦  «        | _        t          |j        ¦  «        | _        t          |j        ¦  «        | _	        t          |j                 | _        |j        | _        |j        | _        d S )Nrï   )r   r€   rB   rx   r¡   Úffw_layer_1Úffw_layer_2r–   Úpre_layer_normÚpost_layer_normr   Ú
hidden_actÚact_fnÚgradient_clippingÚresidual_weightÚpost_layer_scale©r‹   rB   rŒ   s     €r_   r€   zGemma4AudioFeedForward.__init__À  s±   ø€ Ý‰Œ×ÒÑÔÐØˆŒå0°¸Ô9KÈVÔM_ÐbcÑMcÑdÔdˆÔÝ0°¸Ô9KÈaÑ9OÐQWÔQcÑdÔdˆÔå+¨FÔ,>Ñ?Ô?ˆÔÝ,¨VÔ-?Ñ@Ô@ˆÔÝ˜VÔ.Ô/ˆŒà!'Ô!9ˆÔØ &Ô 6ˆÔÐÐra   r�   rH   c                 ó¸  — t          | j        t          j        |j        ¦  «        j        ¦  «        }|}t          j        || |¦  «        }|                      |¦  «        }|                      |¦  «        }|  	                    |¦  «        }|  
                    |¦  «        }t          j        || |¦  «        }|                      |¦  «        }|| j        z  }||z  }|S r�   )Úminr.  rm   Úfinfor¸   r¨   r�   r*  r(  r-  r)  r+  r0  )r‹   r�   r.  Úresiduals       r_   r‘   zGemma4AudioFeedForward.forwardÎ  sÔ   € å Ô 6½¼ÀMÔDWÑ8XÔ8XÔ8\Ñ]Ô]Ðà ˆÝœ MÐ4EÐ3EÐGXÑYÔYˆØ×+Ò+¨MÑ:Ô:ˆà×(Ò(¨Ñ7Ô7ˆØŸš MÑ2Ô2ˆØ×(Ò(¨Ñ7Ô7ˆåœ MÐ4EÐ3EÐGXÑYÔYˆØ×,Ò,¨]Ñ;Ô;ˆØ˜Ô.Ñ.ˆØ˜Ñ!ˆàÐra   ©	rf   rg   rh   r>   r€   rm   rn   r‘   r“   r”   s   @r_   r&  r&  ¿  sk   ø€ € € € € ð7Ð0ð 7ð 7ð 7ð 7ð 7ð 7ð U¤\ð °e´lð ð ð ð ð ð ð ð ra   r&  c                   óR   ‡ — e Zd Zed„ ¦   «         Zdej        dej        fˆ fd„Zˆ xZS )ÚGemma4AudioCausalConv1dc                 ód   — | j         d         dz
  | j        d         z  dz   }|| j        d         z
  S )Nr   r=   )r
  Údilationr  )r‹   Úeffective_kernel_sizes     r_   Úleft_padz Gemma4AudioCausalConv1d.left_padñ  s7   € à!%Ô!1°!Ô!4°qÑ!8¸D¼MÈ!Ô<LÑ LÈqÑ PÐØ$ t¤{°1¤~Ñ5Ð5ra   rç   rH   c                 ó”   •— t           j                             || j        df¦  «        }t	          ¦   «                              |¦  «        S )Nr   )r   r   rÜ   r<  r   r‘   )r‹   rç   rŒ   s     €r_   r‘   zGemma4AudioCausalConv1d.forwardö  s9   ø€ õ ŒM×Ò˜a $¤-°Ð!3Ñ4Ô4ˆå‰wŒw�Š˜qÑ!Ô!Ð!ra   )	rf   rg   rh   r   r<  rm   rn   r‘   r“   r”   s   @r_   r8  r8  ã  sn   ø€ € € € € ð ð6ð 6ñ „_ð6ð"àŒ<ð"ð 
Œð	"ð "ð "ð "ð "ð "ð "ð "ð "ð "ra   r8  c                   óH   ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Zˆ xZS )ÚGemma4AudioLightConv1drB   c                 ó   •— t          ¦   «                              ¦   «          || _        t          ||j        |j        dz  ¦  «        | _        t          ||j        |j        ¦  «        | _        t          |j        |j        |j        |j        d¬¦  «        | _	        t          |j        |j        d¬¦  «        | _        t          |j        |j        d¬¦  «        | _        t          |j                 | _        |j        | _        d S )Nr(   F)r  r	  r
  Úgroupsr}   T©r  Ú
with_scale)r   r€   rB   rx   r¡   Úlinear_startÚ
linear_endr8  Úconv_kernel_sizeÚdepthwise_conv1dr–   r  r*  Ú	conv_normr   r,  r-  r.  r1  s     €r_   r€   zGemma4AudioLightConv1d.__init__  sè   ø€ Ý‰Œ×ÒÑÔÐØˆŒå1°&¸&Ô:LÈfÔN`ÐcdÑNdÑeÔeˆÔÝ/°¸Ô8JÈFÔL^Ñ_Ô_ˆŒÝ 7ØÔ*ØÔ+ØÔ/ØÔ%Øð!
ñ !
ô !
ˆÔõ ,¨FÔ,>ÀFÔDWÐdhÐiÑiÔiˆÔÝ& vÔ'9¸vÔ?RÐ_cÐdÑdÔdˆŒÝ˜VÔ.Ô/ˆŒà!'Ô!9ˆÔÐÐra   r�   rH   c                 ó2  — |}|                       |¦  «        }|                      |¦  «        }t          j                             |d¬¦  «        }|                      |                     dd¦  «        ¦  «                             dd¦  «        }t          | j        t          j
        |j        ¦  «        j        ¦  «        }t          j        || |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }||z  }|S )Nr±   rµ   r=   r(   )r*  rD  r   r   ÚglurG  Ú	transposer3  r.  rm   r4  r¸   r¨   r�   rH  r-  rE  )r‹   r�   r5  r.  s       r_   r‘   zGemma4AudioLightConv1d.forward  sü   € Ø ˆà×+Ò+¨MÑ:Ô:ˆØ×)Ò)¨-Ñ8Ô8ˆÝœ×)Ò)¨-¸RÐ)Ñ@Ô@ˆà×-Ò-¨m×.EÒ.EÀaÈÑ.KÔ.KÑLÔL×VÒVÐWXÐZ[Ñ\Ô\ˆõ   Ô 6½¼ÀMÔDWÑ8XÔ8XÔ8\Ñ]Ô]ÐÝœ MÐ4EÐ3EÐGXÑYÔYˆØŸš }Ñ5Ô5ˆàŸš MÑ2Ô2ˆØŸš¨Ñ6Ô6ˆØ˜Ñ!ˆØÐra   r6  r”   s   @r_   r?  r?    sk   ø€ € € € € ð:Ð0ð :ð :ð :ð :ð :ð :ð( U¤\ð °e´lð ð ð ð ð ð ð ð ra   r?  c            
       ó~   ‡ — e Zd Zdedefˆ fd„Zdej        dej        dz  dej        de	e
         d	ej        f
d
„Zˆ xZS )ÚGemma4AudioLayerrB   rL   c                 ó¦  •— t          ¦   «                              ¦   «          || _        t          |¦  «        | _        t          |¦  «        | _        t          ||¦  «        | _        t          |¦  «        | _	        t          |j        ¦  «        | _        t          |j        ¦  «        | _        t          |j        ¦  «        | _        |j        | _        d S r�   )r   r€   rB   r&  Úfeed_forward1Úfeed_forward2rÁ   Ú	self_attnr?  Úlconv1dr–   r¡   Únorm_pre_attnÚnorm_post_attnÚnorm_outr.  rØ   s      €r_   r€   zGemma4AudioLayer.__init__.  s©   ø€ Ý‰Œ×ÒÑÔÐØˆŒå3°FÑ;Ô;ˆÔÝ3°FÑ;Ô;ˆÔÝ-¨f°iÑ@Ô@ˆŒÝ-¨fÑ5Ô5ˆŒå*¨6Ô+=Ñ>Ô>ˆÔÝ+¨FÔ,>Ñ?Ô?ˆÔÝ% fÔ&8Ñ9Ô9ˆŒà!'Ô!9ˆÔÐÐra   r�   rD   Nrí   ÚkwargsrH   c                 óF  — t          | j        t          j        | j        j        j        ¦  «        j        ¦  «        }|                      |¦  «        }|}t          j	        || |¦  «        }|                      |¦  «        }|  
                    |||¬¦  «        \  }}t          j	        || |¦  «        }|                      |¦  «        }||z  }|                      |¦  «        }|                      |¦  «        }t          j	        || |¦  «        }|                      |¦  «        }|S )N)r�   rí   rD   )r3  r.  rm   r4  rS  r  r¸   r¨   rO  r�   rQ  rT  rR  rP  rU  )r‹   r�   rD   rí   rV  r.  r5  rZ   s           r_   r‘   zGemma4AudioLayer.forward=  s   € õ   Ô 6½¼ÀDÔDVÔD]ÔDcÑ8dÔ8dÔ8hÑiÔiÐà×*Ò*¨=Ñ9Ô9ˆØ ˆåœ MÐ4EÐ3EÐGXÑYÔYˆØ×*Ò*¨=Ñ9Ô9ˆàŸ>š>Ø'Ø 3Ø)ð *ñ 
ô 
Ñˆ�qõ œ MÐ4EÐ3EÐGXÑYÔYˆØ×+Ò+¨MÑ:Ô:ˆØ˜Ñ!ˆàŸš ]Ñ3Ô3ˆØ×*Ò*¨=Ñ9Ô9ˆåœ MÐ4EÐ3EÐGXÑYÔYˆØŸš mÑ4Ô4ˆàÐra   )rf   rg   rh   r>   r’   r€   rm   rn   rv   r   r   r‘   r“   r”   s   @r_   rM  rM  -  s¤   ø€ € € € € ð:Ð0ð :¸Sð :ð :ð :ð :ð :ð :ð à”|ð ð Ô(¨4Ñ/ð ð #œ\ð	 ð
 Ð+Ô,ð ð 
Œð ð  ð  ð  ð  ð  ð  ð  ra   rM  c                   ó–   ‡ — e Zd Zdefˆ fd„Zdej        dej        dej        fd„Zdej        dej        dej        dej        fd„Zˆ xZ	S )	ÚGemma4VisionPatchEmbedderrB   c                 ód  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        |j        | _        t          j        d| j        dz  z  | j        d¬¦  «        | _        t          j	        t          j        d| j        | j        ¦  «        ¦  «        | _        d S )Nr	   r(   Fr|   )r   r€   rB   r¡   Ú
patch_sizeÚposition_embedding_sizer   r‚   Ú
input_projrÔ   rm   ÚonesÚposition_embedding_tabler1  s     €r_   r€   z"Gemma4VisionPatchEmbedder.__init__d  s•   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ Ô+ˆŒØ'-Ô'EˆÔ$åœ) A¨¬¸Ñ(:Ñ$:¸DÔ<LÐSXÐYÑYÔYˆŒÝ(*¬µU´ZÀÀ4ÔC_ÐaeÔaqÑ5rÔ5rÑ(sÔ(sˆÔ%Ð%Ð%ra   Úpixel_position_idsÚpadding_positionsrH   c                 ó&  — |                      d¬¦  «        }t          j        |d         | j        d         ¦  «        }t          j        |d         | j        d         ¦  «        }||z   }t	          j        |                     d¦  «        d|¦  «        }|S )ak  Compute 2-D patch position embeddings via embedding lookup.

        ``pixel_position_ids`` has shape ``(batch, num_patches, 2)`` where the
        last dimension holds (x, y) indices into ``position_embedding_table``
        (shape ``(2, position_embedding_size, hidden_size)``).  The result is the
        sum of the x- and y-embeddings for each patch.
        r   ©r3  ©.r   ©.r=   r=   r±   ç        )r�   rÛ   Ú	embeddingr_  rm   Úwherer«   )r‹   r`  ra  Úclamped_positionsÚx_embÚy_embrí   s          r_   Ú_position_embeddingsz.Gemma4VisionPatchEmbedder._position_embeddingsn  s‘   € ð /×4Ò4¸Ð4Ñ;Ô;Ðõ ”Ð-¨fÔ5°tÔ7TÐUVÔ7WÑXÔXˆÝ”Ð-¨fÔ5°tÔ7TÐUVÔ7WÑXÔXˆØ# e™mÐÝ#œkÐ*;×*EÒ*EÀbÑ*IÔ*IÈ3ÐPcÑdÔdÐØ"Ð"ra   Úpixel_valuesc                 óÌ   — d|dz
  z  }| j         j        j        x}j        r|                     |¦  «        }|                       |¦  «        }|                      ||¦  «        }||z   S )Nr(   ç      à?)r]  r  r¸   Úis_floating_pointr¹   rl  )r‹   rm  r`  ra  Útarget_dtyper�   rí   s          r_   r‘   z!Gemma4VisionPatchEmbedder.forward„  sq   € ð ˜L¨3Ñ.Ñ/ˆØ œOÔ2Ô8Ð8ˆLÔKð 	9Ø'Ÿ?š?¨<Ñ8Ô8ˆLØŸš¨Ñ5Ô5ˆØ"×7Ò7Ð8JÐL]Ñ^Ô^ÐØÐ2Ñ2Ð2ra   )
rf   rg   rh   rA   r€   rm   rn   rl  r‘   r“   r”   s   @r_   rY  rY  c  s¼   ø€ € € € € ðtÐ1ð tð tð tð tð tð tð#°u´|ð #ÐX]ÔXdð #ÐinÔiuð #ð #ð #ð #ð,	3Ø!œLð	3Ø>C¼lð	3Ø_dÔ_kð	3à	Œð	3ð 	3ð 	3ð 	3ð 	3ð 	3ð 	3ð 	3ra   rY  c                   óà   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        dede	ej        ej        f         fd„Z
	 ddej        dej        d
ej        ded	z  de	ej        ej        f         f
d„Zˆ xZS )ÚGemma4VisionPoolera[  Spatial pooling and ``sqrt(hidden_size)`` scaling for vision encodings.

    The scaling expands the activation magnitude, which can exceed the float16 range, so it is
    computed in float32 and the pooled features are returned in float32. The caller
    (``Gemma4VisionModel.forward``) standardizes them and casts back to the working dtype.
    rB   c                 ó~   •— t          ¦   «                              ¦   «          |j        | _        | j        dz  | _        d S )Nro  )r   r€   r¡   Úroot_hidden_sizer1  s     €r_   r€   zGemma4VisionPooler.__init__˜  s:   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔØ $Ô 0°#Ñ 5ˆÔÐÐra   r�   r`  ÚlengthrH   c                 óÜ  — |j         d         }t          ||z  dz  ¦  «        }|dz  }||z  |k    r$t          d|j         › d|› d|›d|›d|› d	�¦  «        ‚|                     d
¬¦  «        }|d                              dd¬¦  «        d
         dz   }t          j        ||d¬¦  «        }	|	d         ||z  |	d         z  z   }	t          j        |	 	                    ¦   «         |¦  «         
                    ¦   «         |z  }
|
                     dd¦  «        | 
                    ¦   «         z  }t          j        |
d
k                         d¬¦  «        ¦  «        }|                     |j        ¦  «        |fS )zÙ
        2D spatial pooling according to patch positions.
        Pools the input tokens by averaging patches within a `k^2` grid, where `k` is determined by the ratio between
        input and output lengths
        r=   ro  r(   zCannot pool z to z: k=z^2 times length=z	 must be ú.r   rc  rd  r±   T©r¶   ÚkeepdimÚfloor)Úrounding_modere  rµ   )rÚ   r’   Ú
ValueErrorr�   r¨   rm   ÚdivrÛ   Úone_hotÚlongr†   rK  rõ   Úallr¹   r¸   )r‹   r�   r`  rv  Úinput_seq_lenÚkÚ	k_squaredri  Úmax_xÚkernel_idxsÚweightsÚoutputr  s                r_   Ú_avg_pool_by_positionsz)Gemma4VisionPooler._avg_pool_by_positions�  sŒ  € ð &Ô+¨AÔ.ˆÝ� &Ñ(¨SÑ0Ñ1Ô1ˆØ�q‘Dˆ	Ø�vÑ Ò.Ð.ÝØq˜}Ô2ÐqÐq¸ÐqÐqÀ!ÐqÐqÈvÐqÐqÐanÐqÐqÐqñô ð ð /×4Ò4¸Ð4Ñ;Ô;ÐØ! &Ô)×-Ò-°"¸dÐ-ÑCÔCÀAÔFÈÑJˆÝ”iÐ 1°1ÀGÐLÑLÔLˆØ! &Ô)¨U°a©Z¸;ÀvÔ;NÑ,NÑNˆÝ”)˜K×,Ò,Ñ.Ô.°Ñ7Ô7×=Ò=Ñ?Ô?À)ÑKˆØ×"Ò" 1 aÑ(Ô(¨=×+>Ò+>Ñ+@Ô+@Ñ@ˆÝÔ  '¨Q¢,×!3Ò!3¸Ð!3Ñ!:Ô!:Ñ;Ô;ˆØ�yŠy˜Ô,Ñ-Ô-¨tÐ3Ð3ra   Nra  Úoutput_lengthc                 óN  — ||j         d         k    r!t          d|› d|j         d         › d�¦  «        ‚|                     |                     d¦  «        d¦  «        }|j         d         |k    r|                      |||¦  «        \  }}|                     ¦   «         | j        z  }||fS )Nr=   z*Cannot output more soft tokens (requested z) than there are patches (z9). Change the value of `num_soft_tokens` when processing.r±   rf  )rÚ   r}  rô   r«   r‰  r†   ru  )r‹   r�   r`  ra  rŠ  s        r_   r‘   zGemma4VisionPooler.forward¸  sÞ   € ð ˜=Ô.¨qÔ1Ò1Ð1Ýðg¸]ð gð gØ"Ô(¨Ô+ðgð gð gñô ð ð
 &×1Ò1Ð2C×2MÒ2MÈbÑ2QÔ2QÐSVÑWÔWˆàÔ˜qÔ! ]Ò2Ð2Ø/3×/JÒ/JØÐ1°=ñ0ô 0Ñ,ˆMÐ,ð &×+Ò+Ñ-Ô-°Ô0EÑEˆØÐ/Ð/Ð/ra   r�   )rf   rg   rh   ri   rA   r€   rm   rn   r’   rl   r‰  r‘   r“   r”   s   @r_   rs  rs  �  sü   ø€ € € € € ðð ð6Ð1ð 6ð 6ð 6ð 6ð 6ð 6ð
4Ø"œ\ð4Ø?D¼|ð4ØUXð4à	ˆuŒ|˜Uœ\Ð)Ô	*ð4ð 4ð 4ð 4ð@ %)ð0ð 0à”|ð0ð "œLð0ð !œ<ð	0ð
 ˜T‘zð0ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð0ð 0ð 0ð 0ð 0ð 0ð 0ð 0ra   rs  c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚGemma4VisionMLPrB   c                 ó  •— t          ¦   «                              | |¦  «         t          || j        | j        ¦  «        | _        t          || j        | j        ¦  «        | _        t          || j        | j        ¦  «        | _        d S r�   )r   r€   rx   r¡   Úintermediate_sizeÚ	gate_projÚup_projÚ	down_projr1  s     €r_   r€   zGemma4VisionMLP.__init__Ô  sl   ø€ Ý‰Œ×Ò˜˜vÑ&Ô&Ð&Ý.¨v°tÔ7GÈÔI_Ñ`Ô`ˆŒÝ,¨V°TÔ5EÀtÔG]Ñ^Ô^ˆŒÝ.¨v°tÔ7MÈtÔO_Ñ`Ô`ˆŒˆˆra   )rf   rg   rh   rA   r€   r“   r”   s   @r_   r�  r�  Ó  sP   ø€ € € € € ðaÐ1ð að að að að að að að að að ara   r�  rç   r¼   r»   Úunsqueeze_dimc           	      ó�  ‡‡
‡‡— |j         d         }| j         d         }d|d|z  z  z  }|dk    rt          d|› d|› d|› d�¦  «        ‚|g|z  }t          j        | |d¬¦  «        Št          j        ||d¬¦  «        Š
t          j        ||d¬¦  «        Šˆ
ˆˆˆfd	„t	          |¦  «        D ¦   «         }	t          j        |	d¬¦  «        S )
ak  Applies multidimensional RoPE to inputs.

    Args:
        x (`torch.Tensor`): The tensor to embed.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`, *optional*):
            If position_ids.ndim + 2 == x.ndim, then this function passes through to `apply_rotary_pos_emb()`.
            Otherwise, position_ids is used to split the inputs, x, into multiple pieces, where each piece is fed to
            `apply_rotary_pos_emb()`, and then concatenated back together.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.

    Returns:
      Tensor of shape [B, L, N, H] with RoPE applied.
    r±   r(   r   zEInvalid configuration: num_rotated_channels_per_dim must be > 0, got z (num_input_channels=z, ndim=ú)rµ   c                 óZ   •— g | ]'}t          ‰|         ‰|         ‰|         ‰¬ ¦  «        ‘Œ(S ))rç   r¼   r»   r“  )r8   )Ú.0rƒ  Ú	cos_partsÚ	sin_partsr“  Úx_partss     €€€€r_   ú
<listcomp>z/apply_multidimensional_rope.<locals>.<listcomp>  sR   ø€ ð ð ð ð õ 	Ø�aŒjØ˜!”Ø˜!”Ø'ð		
ñ 	
ô 	
ðð ð ra   )rÚ   r}  rm   ÚsplitÚrangerº   )rç   r¼   r»   rF   r“  ÚndimÚnum_input_channelsÚnum_rotated_channels_per_dimÚsplit_sizesÚy_partsr˜  r™  rš  s       `     @@@r_   Úapply_multidimensional_roper£  Û  s+  øøøø€ ð8 Ô˜bÔ!€DØœ œÐØ#$Ð(:¸qÀ4¹xÑ(HÑ#IÐ à# qÒ(Ð(ÝðØ,ðð ØCUðð àðð ð ñ
ô 
ð 	
ð 0Ð0°4Ñ7€KÝŒk˜!˜[¨bÐ1Ñ1Ô1€GÝ”˜C °"Ð5Ñ5Ô5€IÝ”˜C °"Ð5Ñ5Ô5€Iðð ð ð ð ð ð õ �t‘”ðñ ô €Gõ Œ9�W "Ð%Ñ%Ô%Ð%ra   c                   ób   ‡ — e Zd Zddefˆ fd„Zddedeej        ef         fˆ fd„Z	d„ Z
d„ Zˆ xZS )	ÚGemma4VisionRotaryEmbeddingNrB   c                 óL   •— t          ¦   «                              ||¦  «         d S r�   )r   r€   )r‹   rB   r³   rŒ   s      €r_   r€   z$Gemma4VisionRotaryEmbedding.__init__  s#   ø€ Ý‰Œ×Ò˜ Ñ(Ô(Ð(Ð(Ð(ra   rH   c                 ó:   •—  t          ¦   «         j        | |fi |¤ŽS r�   )r   Úcompute_axial_rope_parameters)rB   r³   rV  rŒ   s      €r_   r¨  z9Gemma4VisionRotaryEmbedding.compute_axial_rope_parameters  s$   ø€ Ø4�u‰wŒwÔ4°V¸VÐNÐNÀvÐNÐNÐNra   c                 óv  — | j         d                              ¦   «         }|d                              ¦   «         }t          |j        j        t
          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  ||z  }|                     ¦   «         | j        z  }| 	                    ¦   «         | j        z  }d d d ¦  «         n# 1 swxY w Y   |  
                    |¦  «        }|  
                    |¦  «        }|                     |j        ¦  «        |                     |j        ¦  «        fS )N)N.r´   ÚmpsÚcpuF)Údevice_typeÚenabled)Úinv_freqr†   Ú
isinstancer³   Útyperk   r#   r¼   Úattention_scalingr»   Úrecomposition_frequenciesr¹   r¸   )	r‹   rç   rF   Úinv_freq_expandedÚposition_ids_expandedr¬  Úfreqsr¼   r»   s	            r_   r‘   z#Gemma4VisionRotaryEmbedding.forward  sJ  € Ø œM¨)Ô4×:Ò:Ñ<Ô<ÐØ ,¨YÔ 7× =Ò =Ñ ?Ô ?Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆÝ¨¸UÐCÑCÔCð 	7ð 	7Ø)Ð,=Ñ=ˆEØ—)’)‘+”+ Ô 6Ñ6ˆCØ—)’)‘+”+ Ô 6Ñ6ˆCð	7ð 	7ð 	7ñ 	7ô 	7ð 	7ð 	7ð 	7ð 	7ð 	7ð 	7øøøð 	7ð 	7ð 	7ð 	7ð
 ×,Ò,¨SÑ1Ô1ˆØ×,Ò,¨SÑ1Ô1ˆØ�vŠv�a”g‰Œ §¢ q¤w¡¤Ð/Ð/s   Â>CÃCÃCc                 ór   — |dd…dd…df         |dd…dd…df         }}t          j        ||||gd¬¦  «        S )z]
        Recompose the frequencies into the final spatial layout used per each grid.
        Nr   r=   r±   rµ   )rm   rº   )r‹   ÚfreqÚfreq_hÚfreq_ws       r_   r²  z5Gemma4VisionRotaryEmbedding.recomposition_frequencies)  sO   € ð
 ˜a˜a˜a    A˜gœ¨¨Q¨Q¨Q°°°°1¨W¬�ˆÝŒy˜& &¨&°&Ð9¸rÐBÑBÔBÐBra   r�   )rf   rg   rh   rA   r€   rl   rm   rn   r†   r¨  r‘   r²  r“   r”   s   @r_   r¥  r¥    sº   ø€ € € € € ð)ð )Ð1ð )ð )ð )ð )ð )ð )ðOð OÐ.@ð OÐ\aÐbgÔbnÐpuÐbuÔ\vð Oð Oð Oð Oð Oð Oð0ð 0ð 0ðCð Cð Cð Cð Cð Cð Cra   r¥  c                   óØ   ‡ — e Zd Zdedefˆ fd„Z	 	 	 ddej        dej        dej        dz  dej        dz  d	e	e
         d
eej        ej        dz  eej                 dz  f         fd„Zˆ xZS )ÚGemma4VisionAttentionrB   rL   c                 óø  •— t          ¦   «                              | ||¦  «         | `| `| `d| _        d| _        t          ||j        |j	        | j
        z  ¦  «        | _        t          ||j        |j        | j
        z  ¦  «        | _        t          ||j        |j	        | j
        z  ¦  «        | _        t          ||j        | j
        z  |j        ¦  «        | _        t!          | j
        |j        d¬¦  «        | _        d S )Nr�   FrB  )r   r€   Úattn_logit_softcappingrU   rK   ÚscalingÚ	is_causalrx   r¡   Únum_key_value_headsrÇ   rÐ   rÆ   rÏ   rÑ   Úo_projr–   r  Úv_normrØ   s      €r_   r€   zGemma4VisionAttention.__init__4  sç   ø€ Ý‰Œ×Ò˜˜v yÑ1Ô1Ð1ØÐ'ØÐØˆOØˆŒØˆŒÝ+¨F°FÔ4FÈÔHbÐeiÔerÑHrÑsÔsˆŒÝ+¨F°FÔ4FÈÔHbÐeiÔerÑHrÑsÔsˆŒÝ+¨F°FÔ4FÈÔHbÐeiÔerÑHrÑsÔsˆŒÝ+¨F°FÔ4NÐQUÔQ^Ñ4^Ð`fÔ`rÑsÔsˆŒÝ# D¤M°vÔ7JÐW\Ð]Ñ]Ô]ˆŒˆˆra   Nr�   rí   rD   rF   rV  rH   c                 óˆ  — |j         d d…         }g |¢d‘| j        ‘R }|\  }}	|                      |¦  «                             |¦  «        }
|                      |
¦  «        }
t          |
||	|¦  «        }
|
                     dd¦  «        }
|                      |¦  «                             |¦  «        }|                      |¦  «        }t          |||	|¦  «        }|                     dd¦  «        }|  	                    |¦  «                             |¦  «        }|  
                    |¦  «        }|                     dd¦  «        }t          j        | j        j        t          ¦  «        } || |
|||f| j        r| j        nd| j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Nr±   r=   r(   rf  )Údropoutr¾  )rÚ   rÇ   rÏ   ré   Úq_normr£  rK  rÐ   Úk_normrÑ   rÂ  r   Úget_interfacerB   Ú_attn_implementationr9   ÚtrainingÚattention_dropoutr¾  rÝ   rÞ   rÁ  )r‹   r�   rí   rD   rF   rV  Úinput_shaperú   r¼   r»   rû   rü   rý   Úattention_interfacer  r  s                   r_   r‘   zGemma4VisionAttention.forwardA  sæ  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà&‰ˆˆSà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔDˆØ—{’{ <Ñ0Ô0ˆÝ2°<ÀÀcÈ<ÑXÔXˆØ#×-Ò-¨a°Ñ3Ô3ˆà—[’[ Ñ/Ô/×4Ò4°\ÑBÔBˆ
Ø—[’[ Ñ,Ô,ˆ
Ý0°¸SÀ#À|ÑTÔTˆ
Ø×)Ò)¨!¨QÑ/Ô/ˆ
à—{’{ =Ñ1Ô1×6Ò6°|ÑDÔDˆØ—{’{ <Ñ0Ô0ˆØ#×-Ò-¨a°Ñ3Ô3ˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð /3¬mÐD�DÔ*Ð*ÀØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(ra   ©NNN)rf   rg   rh   rA   r’   r€   rm   rn   Ú
LongTensorr   r   rl   r‘   r“   r”   s   @r_   r»  r»  2  sì   ø€ € € € € ð^Ð1ð ^¸cð ^ð ^ð ^ð ^ð ^ð ^ð  -1Ø.2Ø04ð,)ð ,)à”|ð,)ð #œ\ð,)ð œ tÑ+ð	,)ð
 Ô&¨Ñ-ð,)ð Ð+Ô,ð,)ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	Mð,)ð ,)ð ,)ð ,)ð ,)ð ,)ð ,)ð ,)ra   r»  c                   óÔ   ‡ — e Zd Zdedefˆ fd„Z	 	 	 ddej        dej        dej        dz  dej        dz  d	e	e
         d
eej        eej        ej        f         dz  f         fd„Zˆ xZS )ÚGemma4VisionEncoderLayerrB   rL   c                 ó¢   •— t          ¦   «                              | ||¦  «         t          ||¬¦  «        | _        t	          |¦  «        | _        d S )N©rB   rL   )r   r€   r»  rQ  r�  ÚmlprØ   s      €r_   r€   z!Gemma4VisionEncoderLayer.__init__r  sF   ø€ Ý‰Œ×Ò˜˜v yÑ1Ô1Ð1Ý.°fÈ	ÐRÑRÔRˆŒÝ" 6Ñ*Ô*ˆŒˆˆra   Nr�   rí   rD   rF   rV  rH   c                 ó  — |}|                       |¦  «        } | j        d||||dœ|¤Ž\  }}|                      |¦  «        }||z   }|}|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }||z   }|S )N)r�   rí   rD   rF   rR   )Úinput_layernormrQ  Úpost_attention_layernormÚpre_feedforward_layernormrÓ  Úpost_feedforward_layernorm)r‹   r�   rí   rD   rF   rV  r5  rZ   s           r_   r‘   z Gemma4VisionEncoderLayer.forwardw  s¿   € ð !ˆà×,Ò,¨]Ñ;Ô;ˆà)˜4œ>ð 
Ø'Ø 3Ø)Ø%ð	
ð 
ð
 ð
ð 
Ñˆ�qð ×5Ò5°mÑDÔDˆØ  =Ñ0ˆà ˆØ×6Ò6°}ÑEÔEˆØŸš Ñ/Ô/ˆØ×7Ò7¸ÑFÔFˆØ  =Ñ0ˆàÐra   rÍ  )rf   rg   rh   rA   r’   r€   rm   rn   rÎ  r   r   rl   ÚFloatTensorr‘   r“   r”   s   @r_   rÐ  rÐ  q  sæ   ø€ € € € € ð+Ð1ð +¸cð +ð +ð +ð +ð +ð +ð -1Ø.2Ø04ðð à”|ðð #œ\ðð œ tÑ+ð	ð
 Ô&¨Ñ-ðð Ð+Ô,ðð 
ˆuÔ  %¨Ô(9¸5Ô;LÐ(LÔ"MÐPTÑ"TÐTÔ	Uðð ð ð ð ð ð ð ra   rÐ  c                   ót   ‡ — e Zd Zdefˆ fd„Z	 d
dej        dej        dej        dz  dee	         de
f
d	„Zˆ xZS )ÚGemma4VisionEncoderrB   c                 ó  •‡— t          ¦   «                              ¦   «          ‰| _        ‰j        | _        t          ‰¦  «        | _        t          j        ˆfd„t          | j        ¦  «        D ¦   «         ¦  «        | _
        d S )Nc                 ó2   •— g | ]}t          ‰|¬ ¦  «        ‘ŒS )rÒ  )rÐ  )r—  ÚirB   s     €r_   r›  z0Gemma4VisionEncoder.__init__.<locals>.<listcomp>�  s'   ø€ ÐbÐbÐbÀaÕ%¨V¸qÐAÑAÔAÐbÐbÐbra   )r   r€   rB   Únum_hidden_layersÚ
num_layersr¥  Ú
rotary_embr   Ú
ModuleListr�  Úlayersr1  s    `€r_   r€   zGemma4VisionEncoder.__init__—  st   øø€ Ý‰Œ×ÒÑÔÐØˆŒØ Ô2ˆŒÝ5°fÑ=Ô=ˆŒÝ”mØbÐbÐbÐbÍ5ÐQUÔQ`ÑKaÔKaÐbÑbÔbñ
ô 
ˆŒˆˆra   NrC   rD   r`  rV  rH   c                 óÒ   — t          | j        ||¬¦  «        }|}|                      ||¦  «        }| j        d| j        j        …         D ]} ||f|||dœ|¤Ž}Œt          |¬¦  «        S )z�
        pixel_position_ids (torch.Tensor):
            Patch positions as (x, y) coordinates in the image as [batch, num_patches, 2].
        )rB   rC   rD   N)rD   rí   rF   ©Úlast_hidden_state)r   rB   rá  rã  rß  r   )r‹   rC   rD   r`  rV  r�   rí   Údecoder_layers           r_   r‘   zGemma4VisionEncoder.forward   s¨   € õ 3Ø”;Ø'Ø)ð
ñ 
ô 
ˆð &ˆØ"Ÿošo¨mÐ=OÑPÔPÐð "œ[Ð)H¨4¬;Ô+HÐ)HÔIð 	ð 	ˆMØ)˜MØðà-Ø$7Ø/ð	ð ð
 ðð ˆMˆMõ '¸ÐGÑGÔGÐGra   r�   )rf   rg   rh   rA   r€   rm   rn   rÎ  r   r   r   r‘   r“   r”   s   @r_   rÛ  rÛ  –  s³   ø€ € € € € ð
Ð1ð 
ð 
ð 
ð 
ð 
ð 
ð 7;ð	Hð Hà”|ðHð œðHð "Ô,¨tÑ3ð	Hð
 Ð+Ô,ðHð 
!ðHð Hð Hð Hð Hð Hð Hð Hra   rÛ  c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚGemma4TextMLPrB   rL   c                 ó¾   •— |j         |j        z
  }||cxk    odk    nc }|j        o|}t          ¦   «                              ¦   «          |j        |rdndz  | _        d S )Nr   r(   r=   )rß  Únum_kv_shared_layersÚuse_double_wide_mlpr   r€   r�  )r‹   rB   rL   Úfirst_kv_shared_layer_idxÚis_kv_shared_layerrì  rŒ   s         €r_   r€   zGemma4TextMLP.__init__Æ  s}   ø€ Ø$*Ô$<¸vÔ?ZÑ$ZÐ!Ø&Ð*CÐGÐGÒGÐGÀaÒGÐGÐGÐGÐØ$Ô8ÐOÐ=OÐÝ‰Œ×ÒÑÔÐØ!'Ô!9ÐBUÐ=\¸Q¸QÐ[\Ñ!]ˆÔÐÐra   )rf   rg   rh   r@   r’   r€   r“   r”   s   @r_   ré  ré  Å  sX   ø€ € € € € ð^Ð/ð ^¸Cð ^ð ^ð ^ð ^ð ^ð ^ð ^ð ^ð ^ð ^ra   ré  c                   ó   — e Zd Zddefd„ZdS )ÚGemma4TextRotaryEmbeddingNrB   c           	      ó¾  — t           j                             | ¦  «         |j        | _        |j        | _        || _        t          t          |j	        ¦  «        ¦  «        | _	        i | _
        i | _        | j	        D ]â}| j        j        |         }|€Œ|d         x}dk    rt          |         }n| j        }|| j
        |<   || j        |<   |j        |         } ||||¬¦  «        \  }}	t!          | |› d�t          j        |d¬¦  «        ¦  «         t!          | |› d�t          j        |                     ¦   «         d¬¦  «        ¦  «         t!          | |› d�|	¦  «         Œãd S )	NÚ	rope_typeÚdefault©Ú
layer_typeÚ	_inv_freqFrŸ   Ú_original_inv_freqÚ_attention_scaling)r   ÚModuler€   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrB   ÚsortedÚsetÚlayer_typesÚrope_init_fnsrò  Úrope_parametersr   Úcompute_default_rope_parametersÚper_layer_configÚsetattrr„   Úclone)
r‹   rB   r³   rõ  Úrope_paramsrò  Úrope_init_fnÚrope_configÚcurr_inv_freqÚcurr_attention_scalings
             r_   r€   z"Gemma4TextRotaryEmbedding.__init__Ï  s†  € Ý
Œ	×Ò˜4Ñ Ô Ð Ø"(Ô"@ˆÔØ$*Ô$BˆÔ!àˆŒÝ!¥# fÔ&8Ñ"9Ô"9Ñ:Ô:ˆÔØSUˆÔØ)+ˆŒàÔ*ð 	Uð 	UˆJØœ+Ô5°jÔAˆKØÐ"Øà(¨Ô5Ð5�	¸)ÒCÐCÝ2°9Ô=��à#ÔC�à-9ˆDÔ˜zÑ*Ø)2ˆDŒN˜:Ñ&ð !Ô1°*Ô=ˆKØ4@°LÀÈfÐakÐ4lÑ4lÔ4lÑ1ˆMÐ1Ý�D˜ZÐ2Ð2Ð2µB´I¸mÐX]Ð4^Ñ4^Ô4^Ñ_Ô_Ð_Ý�D˜ZÐ;Ð;Ð;½R¼YÀ}×GZÒGZÑG\ÔG\ÐinÐ=oÑ=oÔ=oÑpÔpÐpÝ�D˜ZÐ;Ð;Ð;Ð=SÑTÔTÐTÐTð'	Uð 	Ura   r�   )rf   rg   rh   r@   r€   rR   ra   r_   rð  rð  Î  s=   € € € € € ðUð UÐ/ð Uð Uð Uð Uð Uð Ura   rð  c                   óè   ‡ — e Zd ZdZdedefˆ fd„Z	 ddej        dej        dej        dz  d	e	e
eej        ej        f         f         d
edz  dee         deej        ej        dz  f         fd„Zˆ xZS )ÚGemma4TextAttentionz=Multi-headed attention from 'Attention Is All You Need' paperrB   rL   c                 ój  •— t          ¦   «                              ¦   «          t          |d¦  «        r|j        |         nd | _        || _        || _        | j        dk    | _        | j        r|j        nd | _        |j	        |         }|j
        | _
        |j        o| j         | _        |j        |j        z  | _        d| _        | j        j        | _        |j        dk    | _        | j        j        t+          | j        dd¦  «        z
  }||cxk    odk    nc | _        |j        d |…         }| j         o?|t/          |¦  «        dz
  |d d d…                              |j        |         ¦  «        z
  k    | _        t5          j        |j        |j        | j
        z  |j        ¬	¦  «        | _        t?          | j
        |j         ¬
¦  «        | _!        | j        s°t?          | j
        |j         ¬
¦  «        | _"        t?          | j
        |j         d¬¦  «        | _#        t5          j        |j        |j        | j
        z  |j        ¬	¦  «        | _$        | j        s.t5          j        |j        |j        | j
        z  |j        ¬	¦  «        nd | _%        t5          j        |j        | j
        z  |j        |j        ¬	¦  «        | _&        d S )Nrÿ  rQ   r�   r�  rë  r   r=   r±   r|   )r¶   r  FrB  )'r   r€   Úhasattrrÿ  rõ  rB   rL   rK   rU   r  rÇ   Úattention_k_eq_vÚuse_alternative_attentionrÆ   rÀ  Únum_key_value_groupsr¾  rÊ  Úuse_bidirectional_attentionr¿  rß  rS   rî  ÚlenrT   Ústore_full_length_kvr   r‚   r¡   Úattention_biasrÏ   r–   r  rÅ  rÆ  rÂ  rÐ   rÑ   rÁ  )r‹   rB   rL   Úlayer_configrí  Úprev_layersrŒ   s         €r_   r€   zGemma4TextAttention.__init__ò  sÆ  ø€ Ý‰Œ×ÒÑÔÐÝ;BÀ6È=Ñ;YÔ;YÐc˜&Ô,¨YÔ7Ð7Ð_cˆŒØˆŒØ"ˆŒØœ/Ð-@Ò@ˆŒØ7;´ÐP˜fÔ3Ð3ÈDˆÔàÔ.¨yÔ9ˆØ$Ô-ˆŒØ)/Ô)@Ð)XÈÌÐEXˆÔ&Ø$*Ô$>À,ÔBbÑ$bˆÔ!ØˆŒØ!%¤Ô!>ˆÔØÔ;¸uÒDˆŒð %)¤KÔ$AÅGÈDÌKÐYoÐqrÑDsÔDsÑ$sÐ!Ø"+Ð/HÐ"MÐ"MÒ"MÐ"MÈAÒ"MÐ"MÐ"MÐ"MˆÔØÔ(Ð)CÐ*CÐ)CÔDˆØ(,Ô(?Ð$?ð %/ÀIÕQTÐU`ÑQaÔQaÐdeÑQeÐhsØˆDˆbˆDôi
ç
Š%�Ô" 9Ô-Ñ
.Ô
.ñR/ò E/ˆÔ!õ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ $¨¬¸6Ô;NÐOÑOÔOˆŒð Ô&ð 	Ý'¨D¬M¸vÔ?RÐSÑSÔSˆDŒKÝ'¨¬¸6Ô;NÐ[`ÐaÑaÔaˆDŒKåœ)ØÔ" LÔ$DÀtÄ}Ñ$TÐ[aÔ[pðñ ô ˆDŒKð Ô5ð•”	ØÔ&¨Ô(HÈ4Ì=Ñ(XÐ_eÔ_tðñ ô ð ð ð ŒKõ ”iØÔ&¨¬Ñ6¸Ô8JÐQWÔQfð
ñ 
ô 
ˆŒˆˆra   Nr�   rí   rD   rd   rE   rV  rH   c                 ó¸  — |j         d d…         }g |¢d‘| j        ‘R }|\  }	}
|                      |¦  «                             |¦  «        }|                      |¦  «        }t          ||	|
d¬¦  «        }|                     dd¦  «        }| j        rE|| j                 \  }}| 	                    |j
        ¦  «        }| 	                    |j
        ¦  «        }nÂ|                      |¦  «                             |¦  «        }| j        �(|                      |¦  «                             |¦  «        n|}|                      |¦  «        }t          ||	|
d¬¦  «        }|                     dd¦  «        }|                      |¦  «        }|                     dd¦  «        }|�&| j        s|                     ||| j        ¦  «        \  }}| j        r||f|| j        <   t%          j        | j        j        t,          ¦  «        } || ||||f| j        r| j        nd| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Nr±   r(   )r“  r=   rf  )rÄ  r¾  rU   )rÚ   rÇ   rÏ   ré   rÅ  r8   rK  rî  rõ  r¹   r³   rÐ   rÑ   rÆ  rÂ  ÚupdaterL   r  r   rÇ  rB   rÈ  r9   rÉ  rÊ  r¾  rU   rÝ   rÞ   rÁ  )r‹   r�   rí   rD   rd   rE   rV  rË  rú   r¼   r»   rû   rü   rý   rÌ  r  r  s                    r_   r‘   zGemma4TextAttention.forward#  s•  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà&‰ˆˆSà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔDˆØ—{’{ <Ñ0Ô0ˆÝ+¨L¸#¸sÐRSÐTÑTÔTˆØ#×-Ò-¨a°Ñ3Ô3ˆð
 Ô"ð 	8Ø'7¸¼Ô'HÑ$ˆJ˜à#Ÿš |Ô':Ñ;Ô;ˆJØ'Ÿ?š?¨<Ô+>Ñ?Ô?ˆLˆLàŸš ]Ñ3Ô3×8Ò8¸ÑFÔFˆJØLPÌKÐLc˜4Ÿ;š; }Ñ5Ô5×:Ò:¸<ÑHÔHÐHÐisˆLàŸš ZÑ0Ô0ˆJÝ-¨j¸#¸sÐRSÐTÑTÔTˆJØ#×-Ò-¨a°Ñ3Ô3ˆJàŸ;š; |Ñ4Ô4ˆLØ'×1Ò1°!°QÑ7Ô7ˆLàÐ&¨tÔ/FÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜ØÔ$ð 	IØ0:¸LÐ0HÐ˜Tœ_Ñ-å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð /3¬mÐD�DÔ*Ð*ÀØ”LØÔ.ð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(ra   r�   )rf   rg   rh   ri   r@   r’   r€   rm   rn   rj   rk   rl   r   r   r   r‘   r“   r”   s   @r_   r  r  ï  sõ   ø€ € € € € ØGÐGð/
Ð/ð /
¸Cð /
ð /
ð /
ð /
ð /
ð /
ðn )-ð=)ð =)à”|ð=)ð #œ\ð=)ð œ tÑ+ð	=)ð
 ˜s E¨%¬,¸¼Ð*DÔ$EÐEÔFð=)ð  ™ð=)ð Ð-Ô.ð=)ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð=)ð =)ð =)ð =)ð =)ð =)ð =)ð =)ra   r  c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚGemma4TextExpertsrB   c                 ó¦   •— t          ¦   «                              ¦   «          |j        | _        |j        | _        t
          |j                 | _        d S r�   )r   r€   Únum_expertsÚmoe_intermediate_sizeÚintermediate_dimr   Úhidden_activationr-  r1  s     €r_   r€   zGemma4TextExperts.__init__d  sB   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔØ &Ô <ˆÔÝ˜VÔ5Ô6ˆŒˆˆra   )rf   rg   rh   r@   r€   r“   r”   s   @r_   r  r  c  sE   ø€ € € € € ð7Ð/ð 7ð 7ð 7ð 7ð 7ð 7ð 7ð 7ð 7ð 7ra   r  c                   ób   ‡ — e Zd Zdefˆ fd„Zdej        deej        ej        f         fd„Zˆ xZ	S )ÚGemma4TextRouterrB   c                 óò  •— t          ¦   «                              ¦   «          || _        |j        | _        | j        dz  | _        |j        | _        t          | j        | j        d¬¦  «        | _        t          j
        |j        |j        d¬¦  «        | _        t          j        t          j        | j        ¦  «        ¦  «        | _        t          j        t          j        |j        ¦  «        ¦  «        | _        d S )NrÃ   FrB  r|   )r   r€   rB   r¡   Úscalar_root_sizer  r  r–   r  r   r‚   r  ÚprojrÔ   rm   r^  ÚscaleÚper_expert_scaler1  s     €r_   r€   zGemma4TextRouter.__init__l  sÁ   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ $Ô 0°$Ñ 6ˆÔØÔ&ˆŒå! $Ô"2¸¼ÈUÐSÑSÔSˆŒ	Ý”I˜fÔ0°&Ô2DÈ5ÐQÑQÔQˆŒ	Ý”\¥%¤*¨TÔ-=Ñ">Ô">Ñ?Ô?ˆŒ
Ý "¤­U¬Z¸Ô8JÑ-KÔ-KÑ LÔ LˆÔÐÐra   r�   rH   c                 óx  — |                       |¦  «        }|| j        z  | j        z  }|                      |¦  «        }t          j                             |dt          j        ¬¦  «        }t          j	        || j
        j        d¬¦  «        \  }}||                     dd¬¦  «        z  }|| j        |         z  }|||fS )Nr±   rð   )rƒ  r¶   Try  )r  r&  r$  r%  r   r   r÷   rm   rø   ÚtopkrB   Útop_k_expertsÚsumr'  )r‹   r�   Úexpert_scoresÚrouter_probabilitiesÚtop_k_weightsÚtop_k_indexs         r_   r‘   zGemma4TextRouter.forwardx  sÄ   € ØŸ	š	 -Ñ0Ô0ˆØ%¨¬
Ñ2°TÔ5JÑJˆàŸ	š	 -Ñ0Ô0ˆå!œ}×4Ò4°]ÈÕRWÔR_Ð4Ñ`Ô`Ðõ &+¤ZØ ØŒkÔ'Øð&
ñ &
ô &
Ñ"ˆ�{ð 	˜×*Ò*¨r¸4Ð*Ñ@Ô@Ñ@ˆð &¨Ô(=¸kÔ(JÑJˆà# ]°KÐ?Ð?ra   )
rf   rg   rh   r@   r€   rm   rn   rl   r‘   r“   r”   s   @r_   r"  r"  k  sˆ   ø€ € € € € ð
MÐ/ð 
Mð 
Mð 
Mð 
Mð 
Mð 
Mð@ U¤\ð @°e¸E¼LÈ%Ì,Ð<VÔ6Wð @ð @ð @ð @ð @ð @ð @ð @ra   r"  c                   óì   ‡ — e Zd Zdeez  defˆ fd„Z	 	 	 	 	 	 ddej        dej        de	e
eej        ej        f         f         dz  dej        d	ej        dz  d
ej        dz  dedz  dej        fd„Zˆ xZS )ÚGemma4TextDecoderLayerrB   rL   c                 ó^  •— t          ¦   «                              ||¦  «         t          ||¬¦  «        | _        t	          ||¦  «        | _        t          j        t          j	        d¦  «        ¦  «        | _
        |j        | _        | j        rƒt          |j                 | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _        t'          | j        |j        ¬¦  «        | _        |j        | _        | j        rŠt/          |¦  «        | _        t3          |¦  «        | _        t'          | j        |j        ¬¦  «        | _        t'          | j        |j        ¬¦  «        | _        t'          | j        |j        ¬¦  «        | _        d S d S )NrÒ  r=   Fr|   ©r  )r   r€   r  rQ  ré  rÓ  r   r„   rm   r^  Úlayer_scalarÚhidden_size_per_layer_inputr   r   r-  r‚   r¡   Úper_layer_input_gateÚper_layer_projectionr–   r  Úpost_per_layer_input_normÚenable_moe_blockr"  Úrouterr  ÚexpertsÚpost_feedforward_layernorm_1Úpost_feedforward_layernorm_2Úpre_feedforward_layernorm_2rØ   s      €r_   r€   zGemma4TextDecoderLayer.__init__‘  sz  ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+Ý,°FÀiÐPÑPÔPˆŒÝ  ¨Ñ3Ô3ˆŒÝœI¥e¤j°¡m¤mÑ4Ô4ˆÔà+1Ô+MˆÔ(ØÔ+ð 	fÝ  Ô!9Ô:ˆDŒKÝ(*¬	°$Ô2BÀDÔDdÐkpÐ(qÑ(qÔ(qˆDÔ%Ý(*¬	°$Ô2RÐTXÔTdÐkpÐ(qÑ(qÔ(qˆDÔ%Ý-:¸4Ô;KÐQWÔQdÐ-eÑ-eÔ-eˆDÔ*à &Ô 7ˆÔØÔ ð 	hÝ*¨6Ñ2Ô2ˆDŒKÝ,¨VÑ4Ô4ˆDŒLÝ0=¸dÔ>NÐTZÔTgÐ0hÑ0hÔ0hˆDÔ-Ý0=¸dÔ>NÐTZÔTgÐ0hÑ0hÔ0hˆDÔ-Ý/<¸TÔ=MÐSYÔSfÐ/gÑ/gÔ/gˆDÔ,Ð,Ð,ð	hð 	hra   Nr�   Úper_layer_inputrd   rí   rD   rF   rE   rH   c           
      óp  — |}	|                       |¦  «        } | j        d||||||dœ|¤Ž\  }}
|                      |¦  «        }|	|z   }|}	|                      |¦  «        }|                      |¦  «        }| j        r¯|                      |¦  «        }|	                     d|	j        d         ¦  «        }|  	                    |¦  «        \  }
}}|  
                    |¦  «        }|                      |||¦  «        }|                     |	j        ¦  «        }|                      |¦  «        }||z   }|                      |¦  «        }|	|z   }| j        r`|}	|                      |¦  «        }|                      |¦  «        }||z  }|                      |¦  «        }|                      |¦  «        }|	|z   }|| j        z  }|S )N)r�   rí   rD   rd   rF   rE   r±   rR   )rÕ  rQ  rÖ  r×  rÓ  r9  r<  rÝ   rÚ   r:  r>  r;  r=  rØ  r5  r6  r-  r7  r8  r4  )r‹   r�   r?  rd   rí   rD   rF   rE   rV  r5  rZ   Úhidden_states_1Úhidden_states_flatr.  r/  Úhidden_states_2s                   r_   r‘   zGemma4TextDecoderLayer.forward¦  só  € ð !ˆà×,Ò,¨]Ñ;Ô;ˆØ)˜4œ>ð 
Ø'Ø 3Ø)Ø-Ø%Ø+ð
ð 
ð ð
ð 
Ñˆ�qð ×5Ò5°mÑDÔDˆØ  =Ñ0ˆà ˆØ×6Ò6°}ÑEÔEˆØŸš Ñ/Ô/ˆàÔ ð 	>Ø"×?Ò?ÀÑNÔNˆOð "*×!1Ò!1°"°h´nÀRÔ6HÑ!IÔ!IÐØ,0¯KªKÐ8JÑ,KÔ,KÑ)ˆAˆ}˜kØ"×>Ò>Ð?QÑRÔRˆOØ"Ÿlšl¨?¸KÈÑWÔWˆOØ-×5Ò5°h´nÑEÔEˆOØ"×?Ò?ÀÑPÔPˆOð ,¨oÑ=ˆMà×7Ò7¸ÑFÔFˆØ  =Ñ0ˆàÔ+ð 	5Ø$ˆHØ ×5Ò5°mÑDÔDˆMØ ŸKšK¨Ñ6Ô6ˆMØ)¨OÑ;ˆMØ ×5Ò5°mÑDÔDˆMØ ×:Ò:¸=ÑIÔIˆMØ$ }Ñ4ˆMà˜Ô*Ñ*ˆØÐra   )NNNNNN)rf   rg   rh   r@   rA   r’   r€   rm   rn   rj   rk   rl   rÎ  r   r‘   r“   r”   s   @r_   r1  r1  �  s  ø€ € € € € ðhÐ/Ð2DÑDð hÐQTð hð hð hð hð hð hð0 )-ØPTØ,0Ø.2Ø04Ø(,ð9ð 9à”|ð9ð œð9ð ˜s E¨%¬,¸¼Ð*DÔ$EÐEÔFÈÑMð	9ð
 #œ\ð9ð œ tÑ+ð9ð Ô&¨Ñ-ð9ð  ™ð9ð 
Œð9ð 9ð 9ð 9ð 9ð 9ð 9ð 9ra   r1  c                   ó   — e Zd ZdS )ÚGemma4TextScaledWordEmbeddingNr—   rR   ra   r_   rE  rE  â  r˜   ra   rE  c                   óN   — e Zd Zg d¢ZdZdZ ej        ¦   «         d„ ¦   «         ZdS )ÚGemma4PreTrainedModel)r1  rÐ  rY  rM  )ÚimageÚtextÚvideoÚaudioNc                 ó(  — t          j        |¦  «         t          |t          ¦  «        rt	          j        |j        ¦  «         d S t          |t          ¦  «        r¨d}d}|j        dz  }t          j
        ||z  ¦  «        t          |dz
  d¦  «        z  }|t          j        t          j        |¦  «        | z  ¦  «        z  }t	          j        |j        |                     d¦  «                             d¦  «        ¦  «         d S t          |t$          ¦  «        r:t	          j        |j        |j        ¦  «         t	          j        |j        ¦  «         d S t          |t0          ¦  «        r�|j                             ¦   «         D ]s\  }}|j        j        |         }	 ||	|¬¦  «        \  }
}t	          j        t;          ||› d�¦  «        |
¦  «         t	          j        t;          ||› d�¦  «        |
¦  «         Œtd S t          |t<          ¦  «        r!t	          j        |j        |j         ¦  «         d S t          |tB          ¦  «        r4t	          j        |j"        ¦  «         t	          j        |j#        ¦  «         d S t          |tH          ¦  «        rF| j        j%        }t	          j&        |j'        d	|¬
¦  «         t	          j&        |j(        d	|¬
¦  «         d S t          |tR          ¦  «        rt	          j        |j*        ¦  «         d S t          |tV          ¦  «        r§|j,        r t	          j        |j-        t]          d¦  «         ¦  «         t	          j        |j/        t]          d¦  «        ¦  «         t	          j        |j0        t]          d¦  «         ¦  «         t	          j        |j1        t]          d¦  «        ¦  «         d S t          |td          ¦  «        r@|j        j3        r6t	          j        |j4        ¦  «         t	          j        |j5        ¦  «         d S d S d S )Nr�   rž   r(   r=   r   rô  rö  r÷  rf  )ÚmeanÚstdr~   )6r   Ú_init_weightsr¯  rY  ÚinitÚones_r_  rš   r¡   r¦   r§   r¨   rm   r©   rª   Úcopy_r›   r«   rÁ   Ú	constant_r×   rÅ   Úzeros_rÖ   rð  r   ÚitemsrB   r  rS   rE  Úembed_scaleÚscalar_embed_scaler"  r&  r'  r  Úinitializer_rangeÚnormal_Úgate_up_projr’  r1  r4  rx   r�   r‡   r†   rˆ   r‰   rŠ   ÚGemma4VisionModelÚstandardizeÚstd_biasÚ	std_scale)r‹   Úmoduler¬   r­   r®   r¯   r›   rõ  r  r  r	  rZ   rN  s                r_   rO  z#Gemma4PreTrainedModel._init_weightsó  sÖ  € åÔ% fÑ-Ô-Ð-Ý�fÕ7Ñ8Ô8ð $	)ÝŒJ�vÔ6Ñ7Ô7Ð7Ð7Ð7Ý˜Õ @ÑAÔAð "	)ØˆMØ#ˆMØ#Ô/°1Ñ4ˆNÝ&*¤h¨}¸}Ñ/LÑ&MÔ&MÕPSÐTbÐefÑTfÐhiÑPjÔPjÑ&jÐ#Ø*­U¬Yµu´|ÀNÑ7SÔ7SÐWnÐVnÑ7nÑ-oÔ-oÑoˆNÝŒJ�vÔ,¨n×.FÒ.FÀqÑ.IÔ.I×.SÒ.SÐTUÑ.VÔ.VÑWÔWÐWÐWÐWÝ˜Õ 4Ñ5Ô5ð 	)ÝŒN˜6œ>¨6Ô+KÑLÔLÐLÝŒK˜Ô,Ñ-Ô-Ð-Ð-Ð-Ý˜Õ 9Ñ:Ô:ð 	)Ø,2Ô,@×,FÒ,FÑ,HÔ,Hð ^ð ^Ñ(�
˜LØ$œmÔ<¸ZÔH�Ø#/ <°È
Ð#SÑ#SÔ#SÑ �˜qÝ”
�7 6¨jÐ+CÐ+CÐ+CÑDÔDÀmÑTÔTÐTÝ”
�7 6¨jÐ+LÐ+LÐ+LÑMÔMÈ}Ñ]Ô]Ð]Ð]ð	^ð ^õ
 ˜Õ =Ñ>Ô>ð 	)ÝŒN˜6Ô-¨vÔ/HÑIÔIÐIÐIÐIÝ˜Õ 0Ñ1Ô1ð 	)ÝŒJ�v”|Ñ$Ô$Ð$ÝŒJ�vÔ.Ñ/Ô/Ð/Ð/Ð/Ý˜Õ 1Ñ2Ô2ð 	)Ø”+Ô/ˆCÝŒL˜Ô,°3¸CÐ@Ñ@Ô@Ð@ÝŒL˜Ô)°¸Ð=Ñ=Ô=Ð=Ð=Ð=Ý˜Õ 6Ñ7Ô7ð 		)ÝŒJ�vÔ*Ñ+Ô+Ð+Ð+Ð+Ý˜Õ 5Ñ6Ô6ð 	)¸6Ô;Uð 	)ÝŒN˜6Ô+­e°E©l¬l¨]Ñ;Ô;Ð;ÝŒN˜6Ô+­U°5©\¬\Ñ:Ô:Ð:ÝŒN˜6Ô,­u°U©|¬|¨mÑ<Ô<Ð<ÝŒN˜6Ô,­e°E©l¬lÑ;Ô;Ð;Ð;Ð;Ý˜Õ 1Ñ2Ô2ð 	)°v´}Ô7Pð 	)ÝŒK˜œÑ(Ô(Ð(ÝŒJ�vÔ'Ñ(Ô(Ð(Ð(Ð(ð	)ð 	)ð 	)ð 	)ra   )	rf   rg   rh   Ú_no_split_modulesÚinput_modalitiesÚ_can_record_outputsrm   r¿   rO  rR   ra   r_   rG  rG  é  sU   € € € € € ðð ð Ðð ;ÐØÐà€U„]�_„_ð&)ð &)ñ „_ð&)ð &)ð &)ra   rG  zAThe base Gemma 4 language model without a language modeling head.©Úcustom_introc                   ó   ‡ — e Zd ZU eed<    eed¬¦  «        eedœZ	defˆ fd„Z
dej        dz  dej        dz  d	ej        fd
„Z	 ddej        dej        dz  d	ej        fd„Zeee	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dedz  dej        dz  dej        dz  dedz  dee         d	efd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚGemma4TextModelrB   r   )rT   )Úrouter_logitsr�   Ú
attentionsc                 ó"  •‡‡— t          ¦   «                              ‰¦  «         t          j        ˆfd„t	          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰¦  «        | _        t          | j
        j        ¦  «        | _        ‰j        | _        | j        r›t          ‰j        ‰j        ‰j        z  | j        ‰j        dz  ¬¦  «        | _        d| _        t          j        ‰j        ‰j        ‰j        z  d¬¦  «        | _        ‰j        dz  | _        t/          ‰j        ‰j        ¬¦  «        | _        g | _        t7          | j        ¦  «        D ]7\  Š}|j        j        r&| j                             ˆfd	„d
D ¦   «         ¦  «         Œ8d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS rR   )r1  ©r—  rL   rB   s     €r_   r›  z,Gemma4TextModel.__init__.<locals>.<listcomp>)  s$   ø€ ÐhÐhÐh¸9Õ# F¨IÑ6Ô6ÐhÐhÐhra   ro  )rV  gÍ;fž æ?Fr|   rÃ   r3  c                 ó    •— g | ]
}d ‰› d|› �‘ŒS )zlayers.z.self_attn.rR   )r—  ÚnamerÞ  s     €r_   r›  z,Gemma4TextModel.__init__.<locals>.<listcomp>G  s*   ø€ ÐiÐiÐi¸Ð3˜qÐ3Ð3¨TÐ3Ð3ÐiÐiÐira   )rÐ   rÑ   rÆ  rÂ  )r   r€   r   râ  r�  rß  rã  rð  rá  rþ  rB   rÿ  Úunique_layer_typesr5  rE  Úvocab_size_per_layer_inputÚpadding_idxÚembed_tokens_per_layerÚper_layer_input_scaler‚   r¡   Úper_layer_model_projectionÚ per_layer_model_projection_scaler–   r  Úper_layer_projection_normÚ"_keys_to_ignore_on_load_unexpectedÚ	enumeraterQ  rî  Úextend)r‹   rB   ÚlayerrÞ  rŒ   s    ` @€r_   r€   zGemma4TextModel.__init__&  s§  øøø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý”mØhÐhÐhÐhÍÈfÔNfÑHgÔHgÐhÑhÔhñ
ô 
ˆŒõ 4°FÑ;Ô;ˆŒÝ"% d¤kÔ&=Ñ">Ô">ˆÔð
 ,2Ô+MˆÔ(ØÔ+ð 	xÝ*GØÔ1ØÔ(¨6Ô+MÑMØÔ Ø"Ô>ÀÑCð	+ñ +ô +ˆDÔ'ð *3ˆDÔ&Ý.0¬iØÔ"ØÔ(¨6Ô+MÑMØð/ñ /ô /ˆDÔ+ð
 5;Ô4FÈÑ4LˆDÔ1Ý-:¸6Ô;]ÐciÔcvÐ-wÑ-wÔ-wˆDÔ*ð 35ˆÔ/Ý! $¤+Ñ.Ô.ð 	ð 	‰HˆAˆuØŒÔ1ð ØÔ7×>Ò>ØiÐiÐiÐiÐ@hÐiÑiÔiñô ð øð	ð 	ra   Ú	input_idsNrC   rH   c                 óh  — | j         st          d| j        › �¦  «        ‚|€Ût          j        ¦   «         5  |dd…dd…ddd…f         | j        j        dddd…dd…f         | j        j        dz  z  k                         d¬¦  «         	                    ¦   «         dd…df         }	 | 
                    |j        dd…         ¦  «        }n# t          $ r t          d¦  «        ‚w xY w	 ddd¦  «         n# 1 swxY w Y    |                      |¦  «        j        g |j        ¢| j        j        ‘| j         ‘R Ž S )a  Compute the token-identity component of Per-Layer Embeddings (PLE).

        Looks up `input_ids` in `embed_tokens_per_layer` (a scaled embedding that multiplies
        by `sqrt(hidden_size_per_layer_input)`) and reshapes the packed output from
        `[batch, seq, num_hidden_layers * hidden_size_per_layer_input]` to
        `[batch, seq, num_hidden_layers, hidden_size_per_layer_input]`.

        If only `inputs_embeds` is provided (no `input_ids`), reverses the main embedding
        to recover `input_ids` for the PLE lookup.
        z}Attempting to call get_per_layer_inputs() from a model initialized with a config that does not support per-layer embeddings. Nro  r	   rµ   r(   a)  It seems like you tried to call `forward` from `inputs_embeds` without providing `input_ids`, and that the `inputs_embeds` you provided do not exactly match the embedding weights. Since Gemma4 needs to reverse the embedding to compute another embedding, make sure you provide exact `inputs_embeds`)r5  ÚRuntimeErrorrB   rm   r¿   Úembed_tokensr  r¡   r�  Únonzeroré   rÚ   rq  rÝ   rß  )r‹   rz  rC   s      r_   Úget_per_layer_inputsz$Gemma4TextModel.get_per_layer_inputsJ  sÉ  € ð Ô/ð 	Ýð8Ø*.¬+ð8ð 8ñô ð ð ÐÝ”‘”ð ð ð & a a a¨¨¨¨D°!°!°! mÔ4ØÔ,Ô3°D¸$ÀÀÀÀ1À1À1Ð4DÔEÈÌÔH_ÐadÑHdÑdòe÷ ’S˜Q�S‘Z”Zß’W‘Y”Y˜q˜q˜q !˜tô%ð ðØ )§¢¨}Ô/BÀ2ÀAÀ2Ô/FÑ GÔ G�I�IøÝ#ð ð ð Ý&ðrñô ð ðøøøð ðð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð ð$ >ˆt×*Ò*¨9Ñ5Ô5Ô=ð 
ØŒ_ð
àŒKÔ)ð
ð Ô,ð
ð 
ð 
ð 	
s*   ´A/C1Â$"CÃC1ÃC!Ã!C1Ã1C5Ã8C5Úper_layer_inputsc                 ó  — | j         st          d| j        › �¦  «        ‚|                      |¦  «        | j        z  } |j        g |j        dd…         ¢| j        j        ‘| j         ‘R Ž }|                      |¦  «        }|€|S ||z   | j	        z  S )aŒ  Compute the context-aware component of PLE and combine with token-identity.

        Projects `inputs_embeds` through `per_layer_model_projection` (Linear), scales by
        `1/sqrt(hidden_size)`, reshapes to `[batch, seq, num_layers, ple_dim]`, and normalizes
        with `per_layer_projection_norm` (RMSNorm).

        If `per_layer_inputs` (the token-identity component from `get_per_layer_inputs()`)
        is provided, combines both: `(context_projection + token_identity) * (1/sqrt(2))`.
        If `per_layer_inputs` is None (e.g. for multimodal inputs where input_ids are not
        available), returns just the context projection.
        z�Attempting to call project_per_layer_inputs() from a model initialized with a config that does not support per-layer embeddings. Nr±   )
r5  r|  rB   rs  rt  rÝ   rÚ   rß  ru  rr  )r‹   rC   r€  r7  s       r_   Úproject_per_layer_inputsz(Gemma4TextModel.project_per_layer_inputsv  s×   € ð  Ô/ð 	Ýð@Ø26´+ð@ð @ñô ð ð
  $×>Ò>¸}ÑMÔMÐPTÔPuÑuÐØ;Ð3Ô;ð  
ØÔ   " Ô%ð 
àŒKÔ)ð 
ð Ô,ð 
ð  
ð  
Ðð
  $×=Ò=Ð>RÑSÔSÐàÐ#Ø'Ð'à$Ð'7Ñ7¸4Ô;UÑUÐUra   rD   rF   rE   Ú	use_cacherV  c           
      óZ  — |du |duz  rt          d¦  «        ‚|�|�t          d¦  «        ‚|�|                      |¦  «        }| j        r.|€|                      ||¦  «        }|                      ||¦  «        }|r|€t          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}	t          j	        |j
        d         |j        ¬¦  «        |	z   }|                     d¦  «        }t          |x}
t          ¦  «        s&| j        ||||dœ}t          di |¤Žt!          di |¤Žd	œ}
|}i }| j        D ]}|                      |||¦  «        ||<   Œ|                     d
t)          ¦   «         ¦  «        }t+          | j        d| j        j        …         ¦  «        D ]W\  }}|�|dd…dd…|dd…f         nd} |||f||| j        j        |                  |
| j        j        |                  ||dœ|¤Ž}ŒX|                      |¦  «        }t5          |||                     dd¦  «        r|nd¬¦  «        S )ak  
        per_layer_inputs (`torch.Tensor`, *optional*):
            Pre-computed per-layer input text embeddings of shape `(batch_size, sequence_length, num_hidden_layers,
            hidden_size_per_layer_input)`. When provided, these are used directly instead of being computed from `input_ids`
            via `get_per_layer_inputs()` in the text model. If calling the `forward` with `inputs_embeds` instead of `input_ids`,
            you should probably precompute them and forward them along `inputs_embeds`, otherwise recomputing them needs
            to reverse the main embedding, which is expensive.
        Nú:You must specify exactly one of input_ids or inputs_embedsú<You cannot specify per_layer_inputs if input_ids is provided)rB   r   r=   r²   rJ   rO   rd   )rd   rí   rD   rF   rE   Úreturn_shared_kv_statesF)ræ  rE   rd   rR   )r}  r}  r5  r  r‚  r   rB   Úget_seq_lengthrm   rª   rÚ   r³   r«   r¯  rj   r   r   rn  rá  Úpopr   rw  rã  rß  rÿ  r  rs   Úget)r‹   rz  rD   rF   rE   rC   r€  rƒ  rV  Úpast_seen_tokensÚcausal_mask_mappingrV   r�   rí   rõ  rd   rÞ  rç  r?  s                      r_   r‘   zGemma4TextModel.forward™  sã  € ð, ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ð%5Ð%AÝÐ[Ñ\Ô\Ð\àÐ Ø ×-Ò-¨iÑ8Ô8ˆMàÔ+ð 	^ØÐ'Ø#'×#<Ò#<¸YÈÑ#VÔ#VÐ Ø#×<Ò<¸]ÐL\Ñ]Ô]Ðàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLõ °Ð?Ð-ÅÑFÔFð 	ð œ+Ø!.Ø"0Ø#2Ø ,ðð ˆKõ #5Ð"CÐ"C°{Ð"CÐ"CÝ%FÐ%UÐ%UÈÐ%UÐ%Uð#ð #Ðð &ˆØ ÐØÔ1ð 	gð 	gˆJØ.2¯oªo¸mÈ\Ð[eÑ.fÔ.fÐ 
Ñ+Ð+ð
 "Ÿ:š:Ð&8½(¹*¼*ÑEÔEÐõ !*¨$¬+Ð6U¸¼Ô8UÐ6UÔ*VÑ WÔ Wð 	ð 	ÑˆAˆ}Ø>NÐ>ZÐ.¨q¨q¨q°!°!°!°Q¸¸¸¨zÔ:Ð:Ð`dˆOà)˜MØØð	ð "2Ø$7¸¼Ô8OÐPQÔ8RÔ$SØ2°4´;Ô3JÈ1Ô3MÔNØ)Ø /ð	ð 	ð ð	ð 	ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆå,Ø+Ø+Ø17·²Ð<UÐW\Ñ1]Ô1]ÐgÐ-Ð-Ðcgð
ñ 
ô 
ð 	
ra   r�   )NNNNNNN)rf   rg   rh   r@   ro   r&   r"  r1  r  rb  r€   rm   rn   r  r‚  r$   r'   r   rÎ  r   rÙ  Úboolr   r   rs   r‘   r“   r”   s   @r_   rf  rf    sö  ø€ € € € € € àÐÐÑà'˜Ð(8ÀÐBÑBÔBØ/Ø)ðð Ðð"Ð/ð "ð "ð "ð "ð "ð "ðH*
¨e¬l¸TÑ.Að *
ÐRWÔR^ÐaeÑReð *
ÐjoÔjvð *
ð *
ð *
ð *
ð^ 15ð!Vð !Và”|ð!Vð  œ,¨Ñ-ð!Vð 
Œð	!Vð !Vð !Vð !VðF  ØØð .2Ø.2Ø04Ø(,Ø26Ø04Ø!%ðY
ð Y
àÔ# dÑ*ðY
ð œ tÑ+ðY
ð Ô&¨Ñ-ð	Y
ð
  ™ðY
ð Ô(¨4Ñ/ðY
ð  œ,¨Ñ-ðY
ð ˜$‘;ðY
ð Ð+Ô,ðY
ð 
'ðY
ð Y
ð Y
ñ „^ñ „_ñ  ÔðY
ð Y
ð Y
ð Y
ð Y
ra   rf  z>The base Gemma 4 language model with a language modeling head.c                   ó  — e Zd ZdZee	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  de	dz  dej
        dz  d	ej        dz  d
edz  deej        z  dej        dz  dee         defd„¦   «         ¦   «         ZdS )ÚGemma4ForCausalLMÚmodelNr   rz  rD   rF   rE   rC   Úlabelsrƒ  Úlogits_to_keepr€  rV  rH   c
                 óÎ  —  | j         d||||||	|dœ|
¤Ž}|j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }| j        j        �2|| j        j        z  }t          j	        |¦  «        }|| j        j        z  }d}|� | j
        ||| j        fi |
¤Ž}t          |||j        |j        |j        |j        ¬¦  «        S )aí  
        per_layer_inputs (`torch.Tensor`, *optional*):
            Pre-computed per-layer input text embeddings of shape `(batch_size, sequence_length, num_hidden_layers,
            hidden_size_per_layer_input)`. When provided, these are used directly instead of being computed from `input_ids`
            via `get_per_layer_inputs()` in the text model. If calling the `forward` with `inputs_embeds` instead of `input_ids`,
            you should probably precompute them and forward them along `inputs_embeds`, otherwise recomputing them needs
            to reverse the main embedding, which is expensive.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, Gemma4ForCausalLM

        >>> model = Gemma4ForCausalLM.from_pretrained("google/gemma-4-E2B-it")
        >>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-4-E2B-it")

        >>> prompt = "What is your favorite condiment?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "What is your favorite condiment?"
        ```)rz  rD   rF   rE   rC   r€  rƒ  N)ÚlossÚlogitsrE   r�   rh  rd   rR   )r�  ræ  r¯  r’   ÚsliceÚlm_headrB   Úfinal_logit_softcappingrm   ró   Úloss_functionÚ
vocab_sizerq   rE   r�   rh  rd   )r‹   rz  rD   rF   rE   rC   r‘  rƒ  r’  r€  rV  Úoutputsr�   Úslice_indicesr•  r”  s                   r_   r‘   zGemma4ForCausalLM.forwardü  s/  € ðP 2<°´ð 	2
ØØ)Ø%Ø+Ø'Ø-Øð	2
ð 	2
ð ð	2
ð 	2
ˆð  Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆØŒ;Ô.Ð:Ø˜dœkÔAÑAˆFÝ”Z Ñ'Ô'ˆFØ˜dœkÔAÑAˆFàˆØÐØ%�4Ô% f¨f°d´oÐPÐPÈÐPÐPˆDå+ØØØ#Ô3Ø!Ô/ØÔ)Ø$Ô5ð
ñ 
ô 
ð 	
ra   )	NNNNNNNr   N)rf   rg   rh   Úbase_model_prefixr    r   rm   rÎ  rn   r   rÙ  r�  r’   r   r   rq   r‘   rR   ra   r_   r�  r�  ø  s&  € € € € € àÐàØð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.Ø04ðE
ð E
àÔ# dÑ*ðE
ð œ tÑ+ðE
ð Ô&¨Ñ-ð	E
ð
  ™ðE
ð Ô(¨4Ñ/ðE
ð Ô  4Ñ'ðE
ð ˜$‘;ðE
ð ˜eœlÑ*ðE
ð  œ,¨Ñ-ðE
ð Ð+Ô,ðE
ð 
&ðE
ð E
ð E
ñ „^ñ ÔðE
ð E
ð E
ra   r�  c                   ó  ‡ — e Zd ZU dZeed<   dZdZee	dœZ
defˆ fd„Zdej        dej        fd	„Zee ed
¬¦  «        	 ddej        dej        dz  dee         deej        ej        f         fd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚGemma4AudioModelznAn audio encoder based on the [Universal Speech Model](https://huggingface.co/papers/2303.01037) architecture.rB   r"  zmodel.audio_tower©r�   rh  c                 óŒ  •‡— t          ¦   «                              ‰¦  «         ‰| _        t          ‰¦  «        | _        t          ‰¦  «        | _        t          j        ˆfd„t          ‰j
        ¦  «        D ¦   «         ¦  «        | _        t          j        ‰j        ‰j        d¬¦  «        | _        |                      ¦   «          d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS rR   )rM  rk  s     €r_   r›  z-Gemma4AudioModel.__init__.<locals>.<listcomp>X  s$   ø€ ÐbÐbÐb°YÕ˜f iÑ0Ô0ÐbÐbÐbra   Tr|   )r   r€   rB   r  Úsubsample_conv_projectionrš   Úrel_pos_encr   râ  r�  rß  rã  r‚   r¡   Úoutput_proj_dimsÚoutput_projÚ	post_initr1  s    `€r_   r€   zGemma4AudioModel.__init__Q  s°   øø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒå)KÈFÑ)SÔ)SˆÔ&Ý;¸FÑCÔCˆÔÝ”mØbÐbÐbÐbÅ%ÈÔH`ÑBaÔBaÐbÑbÔbñ
ô 
ˆŒõ œ9 VÔ%7¸Ô9PÐW[Ð\Ñ\Ô\ˆÔà�ŠÑÔÐÐÐra   Úmask_4drH   c                 óR  — |j         \  }}}}|j        }| j        j        }| j        j        dz
  }| j        j        }||z   dz
  |z  }	|	|z  }
|
|z
  }t          j        |d|d|fd¬¦  «        }|                     |d|	||
¦  «        }t          j        |||fd¬¦  «        }t          j
        |	|¬¦  «        |z  }t          j
        ||z   |z   |¬¦  «        }|dd…df         |ddd…f         z   }|dddd…ddd…f                              |dd|d¦  «        }|                     d|¦  «        S )zÜ
        Convert a standard 4D attention mask `[batch_size, 1, seq_len, seq_len]` to the 5D blocked format
        `[batch_size, 1, num_blocks, chunk_size, context_size]` expected by the chunked local attention,
        r=   r   F)Úvaluer²   Nr±   )rÚ   r³   rB   r¢   r£   r¤   rÛ   rÜ   rÝ   rm   rª   ÚexpandÚgather)r‹   r¨  rß   rZ   rà   r³   rÌ   rÍ   rÎ   rá   Úpadded_seq_lenÚ
pad_amountÚmask_5dÚblock_startsÚoffsetsÚ
kv_indicess                   r_   Ú_convert_4d_mask_to_blocked_5dz/Gemma4AudioModel._convert_4d_mask_to_blocked_5d^  sd  € ð
 %,¤MÑ!ˆ
�A�w Ø”ˆà”[Ô5ˆ
Øœ;Ô=ÀÑAÐØ!œ[Ô@Ðà 
Ñ*¨QÑ.°:Ñ=ˆ
Ø# jÑ0ˆØ# gÑ-ˆ
å”%˜ ! Z°°JÐ!?ÀuÐMÑMÔMˆØ—/’/ *¨a°¸ZÈÑXÔXˆÝ”%˜Ð"2Ð4FÐ!GÈuÐUÑUÔUˆå”| J°vÐ>Ñ>Ô>ÀÑKˆÝ”,˜zÐ,<Ñ<Ð?QÑQÐZ`ÐaÑaÔaˆØ! ! ! ! T 'Ô*¨W°T¸1¸1¸1°WÔ-=Ñ=ˆ
Ø  d¨A¨A¨A¨t°Q°Q°QÐ 6Ô7×>Ò>¸zÈ1ÈbÐR\Ð^`ÑaÔaˆ
à�~Š~˜b *Ñ-Ô-Ð-ra   z&Encodes audio features to soft tokens.rc  NrD   rV  c           	      ó¤  — |                       ||¦  «        \  }}|                      |¦  «        }t          | j        ||t	          | j        j        dz
  | j        j        f¦  «        ¬¦  «        }|�|                      |¦  «        }| j        d | j        j	        …         D ]} ||f||dœ|¤Ž}Œ|  
                    |¦  «        }t          ||¬¦  «        S )Nr=   )rB   rC   rD   rN   )rD   rí   )ræ  rD   )r£  r¤  r   rB   r;   r£   r¤   r³  rã  rß  r¦  ru   )r‹   r"  rD   rV  r�   Úoutput_maskrí   Úencoder_layers           r_   r‘   zGemma4AudioModel.forwardy  s  € ð &*×%CÒ%CÀNÐTbÑ%cÔ%cÑ"ˆ�{Ø"×.Ò.¨}Ñ=Ô=Ðå2Ø”;Ø'Ø&Ý:Ø”Ô3°aÑ7¸¼Ô9\Ð]ñô ð	
ñ 
ô 
ˆð Ð%Ø!×@Ò@ÀÑPÔPˆNà!œ[Ð)H¨4¬;Ô+HÐ)HÔIð 	ð 	ˆMØ)˜MØðà-Ø$7ðð ð ð	ð ˆMˆMð ×(Ò(¨Ñ7Ô7ˆÝ%¸ÐVaÐbÑbÔbÐbra   r�   )rf   rg   rh   ri   r>   ro   Úmain_input_namer�  rM  rÁ   rb  r€   rm   rn   r³  r$   r'   r   r   r   rl   rv   r‘   r“   r”   s   @r_   rŸ  rŸ  F  s=  ø€ € € € € € ØxÐxàÐÐÑØ&€OØ+Ðà)Ø*ðð Ðð
Ð0ð ð ð ð ð ð ð.°e´lð .ÀuÄ|ð .ð .ð .ð .ð6  ØØ€^Ð!IÐJÑJÔJð /3ðcð càœðcð œ tÑ+ðcð Ð+Ô,ð	cð
 
ˆuŒ|˜UÔ-Ð-Ô	.ðcð cð cñ KÔJñ „_ñ  Ôðcð cð cð cð cra   rŸ  c                   óº   ‡ — e Zd ZU dZeed<   eedœZdefˆ fd„Z	e
e ed¬¦  «        dej        dej        d	ee         d
efd„¦   «         ¦   «         ¦   «         Zˆ xZS )r[  zThe Gemma 4 Vision Encoder.rB   r   c                 óÖ  •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          |¦  «        | _        t          |¦  «        | _        | j        j	        rjt          j        t          j        | j        j        ¦  «        ¦  «        | _        t          j        t          j        | j        j        ¦  «        ¦  «        | _        |                      ¦   «          d S r�   )r   r€   rY  Úpatch_embedderrÛ  Úencoderrs  ÚpoolerrB   r\  r   r„   rm   Úemptyr¡   r]  r^  r§  r1  s     €r_   r€   zGemma4VisionModel.__init__¥  s­   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý7¸Ñ?Ô?ˆÔÝ*¨6Ñ2Ô2ˆŒÝ(¨Ñ0Ô0ˆŒàŒ;Ô"ð 	MÝœI¥e¤k°$´+Ô2IÑ&JÔ&JÑKÔKˆDŒMÝœY¥u¤{°4´;Ô3JÑ'KÔ'KÑLÔLˆDŒNà�ŠÑÔÐÐÐra   z1Encodes image pixels to soft tokens from patches.rc  rm  r`  rV  rH   c                 óð  — | j         j        }|j        d         ||z  z  }|dk                         d¬¦  «        }|                      |||¦  «        } | j        d|| |dœ|¤Ž}|                      |j        |||¬¦  «        \  }	}
|	|
         }	| j         j        r6|	| j	         
                    ¦   «         z
  | j         
                    ¦   «         z  }	|	                     |j        ¦  «        }	t          |	¬¦  «        S )aà  
        pixel_values (`torch.FloatTensor` or `list[torch.FloatTensor]`):
            The images to encode. Either a single `[batch, channels, height, width]` tensor
            (all images same size) or a list of `[1, channels, height, width]` tensors (different sizes).
        pixel_position_ids (`torch.LongTensor` of shape `(batch_size, max_patches, 2)`):
            The patch positions as (x, y) coordinates in the image. Padding patches are indicated by (-1, -1).
        éþÿÿÿr±   rµ   )rC   rD   r`  )r�   r`  ra  rŠ  rå  rR   )rB   Úpooling_kernel_sizerÚ   r�  rº  r»  r¼  ræ  r\  r]  r†   r^  r¹   r¸   r   )r‹   rm  r`  rV  rÀ  rŠ  ra  rC   rˆ  r�   Úpooler_masks              r_   r‘   zGemma4VisionModel.forward±  s,  € ð  #œkÔ=ÐØ$Ô*¨2Ô.Ð3FÐI\Ñ3\Ñ]ˆà/°2Ò5×:Ò:¸rÐ:ÑBÔBÐØ×+Ò+¨LÐ:LÐN_Ñ`Ô`ˆØ�”ð 
Ø'Ø-Ð-Ø1ð
ð 
ð ð	
ð 
ˆð &*§[¢[Ø Ô2Ø1Ø/Ø'ð	 &1ñ &
ô &
Ñ"ˆ�{ð & kÔ2ˆð Œ;Ô"ð 	]Ø*¨T¬]×-@Ò-@Ñ-BÔ-BÑBÀdÄn×FZÒFZÑF\ÔF\Ñ\ˆMØ%×(Ò(¨Ô)<Ñ=Ô=ˆå&¸ÐGÑGÔGÐGra   )rf   rg   rh   ri   rA   ro   rÐ  r»  rb  r€   r$   r'   r   rm   rÙ  rÎ  r   r   r   r‘   r“   r”   s   @r_   r[  r[  œ  sï   ø€ € € € € € Ø%Ð%àÐÐÑà1Ø+ðð Ðð

Ð1ð 
ð 
ð 
ð 
ð 
ð 
ð  ØØ€^Ð!TÐUÑUÔUð)HàÔ'ð)Hð "Ô,ð)Hð Ð+Ô,ð	)Hð
 
!ð)Hð )Hð )Hñ VÔUñ „_ñ  Ôð)Hð )Hð )Hð )Hð )Hra   r[  c                   óR   ‡ — e Zd Zdeez  defˆ fd„Zdej        dej        fd„Z	ˆ xZ
S )ÚGemma4MultimodalEmbedderÚmultimodal_configÚtext_configc                 óÜ   •— t          ¦   «                              ||¦  «         | `| `| `| `| `| `t          |d|j	        ¦  «        | _
        t          | j
        | j        d¬¦  «        | _        d S )Nr¥  FrB  )r   r€   rg  Úhard_embedding_normÚsoft_embedding_normÚvocab_offsetrš  Úembedding_post_projection_normrS   r¡   Úmultimodal_hidden_sizer–   r  Úembedding_pre_projection_norm)r‹   rÄ  rÅ  rŒ   s      €r_   r€   z!Gemma4MultimodalEmbedder.__init__á  s�   ø€ õ 	‰Œ×ÒÐ*¨KÑ8Ô8Ð8ØˆNØÐ$ØÐ$ØÐØˆOØÐ/å&-Ð.?ÐASÐUfÔUrÑ&sÔ&sˆÔ#Ý-:¸4Ô;VÐ\`Ô\dÐqvÐ-wÑ-wÔ-wˆÔ*Ð*Ð*ra   rC   rH   c                 óV   — |                       |¦  «        }|                      |¦  «        S )a:  Embeds token ids or soft tokens for multimodal content into language model space.
        Args:
            inputs_embeds: A torch.Tensor containing the soft tokens to embed.
        Returns:
            A torch.Tensor of embeddings with shape `[batch_size, seq_len, self.config.text_config.hidden_size]`.
        )rÌ  Úembedding_projection)r‹   rC   Úembs_normeds      r_   r‘   z Gemma4MultimodalEmbedder.forwardó  s+   € ð ×8Ò8¸ÑGÔGˆØ×(Ò(¨Ñ5Ô5Ð5ra   )rf   rg   rh   r>   rA   r@   r€   rm   rn   r‘   r“   r”   s   @r_   rÃ  rÃ  à  sƒ   ø€ € € € € ðxà,Ð/AÑAðxð &ðxð xð xð xð xð xð$6 U¤\ð 6°e´lð 6ð 6ð 6ð 6ð 6ð 6ð 6ð 6ra   rÃ  Útoken_type_idsÚimage_group_idsc           
      ób   ‡— | €dS dt           dt           dt           dt           dt          f
ˆfd„}|S )z•
    This function adds the correct offsets to the `q_idx` and `kv_idx` as the torch API can only accept lengths,
    not start and end indices.
    NÚ	batch_idxÚhead_idxÚq_idxÚkv_idxrH   c                 ó,  •— ‰	j         d         }|                     |dz
  ¬¦  «        }|                     |dz
  ¬¦  «        }‰	| |f         }‰	| |f         }t          j        ||k     |d¦  «        }t          j        ||k     |d¦  «        }||k    |dk    z  S )Nr±   r=   )r¨   r   )rÚ   r�   rm   rh  )
rÓ  rÔ  rÕ  rÖ  rù   Úq_idx_clampedÚkv_idx_clampedÚq_groupÚkv_grouprÑ  s
            €r_   Ú
inner_maskz0token_type_ids_mask_function.<locals>.inner_mask  s¤   ø€ Ø$Ô*¨2Ô.ˆ
ð Ÿš¨
°Q©˜Ñ7Ô7ˆØŸš¨*°q©.˜Ñ9Ô9ˆð " )¨]Ð":Ô;ˆØ" 9¨nÐ#<Ô=ˆÝ”+˜e jÒ0°'¸2Ñ>Ô>ˆÝ”;˜v¨
Ò2°H¸bÑAÔAˆØ˜8Ò#¨°1ªÑ5Ð5ra   )r’   r�  )rÐ  rÑ  rÜ  s    ` r_   Útoken_type_ids_mask_functionrÝ  ÿ  sY   ø€ ð ÐØˆtð6�cð 6­Sð 6½ð 6Åcð 6Ídð 6ð 6ð 6ð 6ð 6ð 6ð Ðra   Úmm_token_type_idsr³   c                 ó  — |                       |¦  «        } | dk    | dk    z  }t          j        |dd¬¦  «        }d|d<   || z  }t          j        |                     ¦   «         d¬¦  «        dz
  }t          j        ||d¦  «        }|S )Nr=   r(   r±   )ÚshiftsÚdimsFrd  rµ   )r¹   rm   ÚrollÚcumsumr’   rh  )rÞ  r³   Ú	is_visionÚis_prev_visionÚnew_vision_startsÚvision_group_idsrG   s          r_   Úget_block_sequence_ids_for_maskrè    sš   € Ø)×,Ò,¨VÑ4Ô4Ðà" aÒ'Ð,=ÀÒ,BÑC€IÝ”Z 	°!¸"Ð=Ñ=Ô=€NØ"€N�6ÑØ! ^ OÑ3ÐÝ”|Ð$5×$9Ò$9Ñ$;Ô$;ÀÐCÑCÔCÀaÑGÐÝœ YÐ0@À"ÑEÔEÐØÐra   zŒ
    The base Gemma 4 model comprising a vision backbone, an audio backbone, and a language model without a
    language modeling head.
    c            $       óf  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Ze ed¬¦  «        	 dde	j
        d	e	j        dz  d
ee         defd„¦   «         ¦   «         Ze ed¬¦  «        	 dde	j
        de	j        dz  d
ee         defd„¦   «         ¦   «         Z	 	 d de	j        dz  de	j
        dz  dee	j        e	j        e	j        f         fd„Zeee	 	 	 	 	 	 	 	 	 	 	 	 	 	 d!de	j        dz  de	j
        dz  de	j
        dz  de	j
        dz  de	j        dz  de	j        dz  de	j        dz  dedz  de	j        dz  de	j
        dz  dedz  d	e	j        dz  de	j        dz  de	j        dz  d
ee         def d„¦   «         ¦   «         ¦   «         Ze ed¬¦  «        de	j        de	j        d
ee         deez  fd„¦   «         ¦   «         Zˆ xZS )"ÚGemma4ModelrB   c                 ó†  •— t          ¦   «                              |¦  «         |j        �t          j        |j        ¦  «        nd | _        |j        �t          |j        |j        ¦  «        nd | _        |j	        �t          j        |j	        ¦  «        nd | _
        |j	        �t          |j	        |j        ¦  «        nd | _        d S r�   )r   r€   Úvision_configr)   Úfrom_configÚvision_towerrÃ  rÅ  Úembed_visionÚaudio_configÚaudio_towerÚembed_audior1  s     €r_   r€   zGemma4Model.__init__/  sÉ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØKQÔK_ÐKk�IÔ1°&Ô2FÑGÔGÐGÐquˆÔð Ô#Ð/õ % VÔ%9¸6Ô;MÑNÔNÐNàð 	Ôð
 JPÔI\ÐIh�9Ô0°Ô1DÑEÔEÐEÐnrˆÔð Ô"Ð.õ % VÔ%8¸&Ô:LÑMÔMÐMàð 	ÔÐÐra   c                 ó   — | j         j        S r�   ©Úlanguage_modelrq  ©r‹   s    r_   Úget_per_layer_input_embeddingsz*Gemma4Model.get_per_layer_input_embeddings>  s   € ØÔ"Ô9Ð9ra   c                 ó   — || j         _        d S r�   rô  ©r‹   rª  s     r_   Úset_per_layer_input_embeddingsz*Gemma4Model.set_per_layer_input_embeddingsA  s   € Ø5:ˆÔÔ2Ð2Ð2ra   zOProjects the last hidden state from the vision model into language model space.rc  Nrm  Úimage_position_idsrV  rH   c                 ó¨  —  | j         d	||dœ|¤Ž}|j        }|                      |¬¦  «        }|j        d         | j        j        j        dz  z  }t          |j        d         |z  dz  ¦  «        dz  }|dk                         d¬¦  «        }	|	 	                    d¬¦  «        |z   
                    ¦   «         }
t          j        ||
¦  «        |_        |S )
zÝ
        image_position_ids (`torch.LongTensor` of shape `(batch_size, max_patches, 2)`, *optional*):
            The patch positions as (x, y) coordinates in the image. Padding patches are indicated by (-1, -1).
        ©rm  r`  ©rC   r¿  r(   r=   ro  r±   rµ   rR   )rî  ræ  rï  rÚ   rB   rì  rÀ  r’   r�  r+  Útolistrm   rœ  Úpooler_output)r‹   rm  rû  rV  Úvision_outputsræ  r   rŠ  r„  Únon_pad_maskr¡  s              r_   Úget_image_featureszGemma4Model.get_image_featuresD  sö   € ð +˜Ô*ð 
Ø%Ø1ð
ð 
ð ð
ð 
ˆð
 +Ô<ÐØ×)Ò)Ð8IÐ)ÑJÔJˆà$Ô*¨2Ô.°4´;Ô3LÔ3`ÐbcÑ3cÑdˆÝÐ+Ô1°!Ô4¸ÑEÈ#ÑMÑNÔNÐRSÑSˆ	Ø*¨bÒ0×5Ò5¸"Ð5Ñ=Ô=ˆØ#×'Ò'¨BÐ'Ñ/Ô/°9Ñ<×DÒDÑFÔFˆÝ',¤{°=À+Ñ'NÔ'NˆÔ$ØÐra   zQProjects the last hidden state from the vision encoder into language model space.Úpixel_values_videosÚvideo_position_idsc                 óü  — |                      dd¦  «        } | j        d||                      dd¦  «        dœ|¤Ž}|j        }|                      |¬¦  «        }|j        d         | j        j        j        dz  z  }t          |j        d         |z  dz  ¦  «        dz  }|dk     	                    d¬	¦  «        }	|	 
                    d
¬	¦  «        |z                       ¦   «         }
t          j        ||
¦  «        |_        |S )a9  
        video_position_ids (`torch.LongTensor` of shape `(num_videos, num_frames, max_patches, 2)`, *optional*):
            2D patch position coordinates from the video processor, with `(-1, -1)` indicating padding.
            Passed through to the vision encoder for positional embedding computation.
        r   r=   rý  rþ  r¿  r(   ro  r±   rµ   )r¿  r±   rR   )Úflattenrî  ræ  rï  rÚ   rB   rì  rÀ  r’   r�  r+  rÿ  rm   rœ  r   )r‹   r  r  rV  r  ræ  r   rŠ  r„  r  r¡  s              r_   Úget_video_featureszGemma4Model.get_video_features_  s  € ð 2×9Ò9¸!¸QÑ?Ô?ÐØ*˜Ô*ð 
Ø,Ø1×9Ò9¸!¸QÑ?Ô?ð
ð 
ð ð
ð 
ˆð
 +Ô<ÐØ×)Ò)Ð8IÐ)ÑJÔJˆà+Ô1°"Ô5¸$¼+Ô:SÔ:gÐijÑ:jÑkˆÝÐ,Ô2°2Ô6¸-ÑGÈCÑOÑPÔPÐTUÑUˆ	Ø*¨bÒ0×5Ò5¸"Ð5Ñ=Ô=ˆØ#×'Ò'¨HÐ'Ñ5Ô5¸ÑB×JÒJÑLÔLˆå',¤{°=À+Ñ'NÔ'NˆÔ$ØÐra   rz  rC   c           	      óÆ  — |�2|| j         j        k    }|| j         j        k    }|| j         j        k    }�n)| |                      ¦   «         t          j        d| j         j        t
          j        |j        ¬¦  «        ¦  «        k     	                    d¦  «        }| |                      ¦   «         t          j        d| j         j        t
          j        |j        ¬¦  «        ¦  «        k     	                    d¦  «        }| |                      ¦   «         t          j        d| j         j        t
          j        |j        ¬¦  «        ¦  «        k     	                    d¦  «        }|||fS )a‹  
        Obtains mask for multimodal placeholders (replaced by soft tokens) and hard text tokens.

        Masks will be obtained from `mm_token_type_ids`, `input_ids`, or `inputs_embeds` as available and in that
        precedence order. If passing `input_ids` or `inputs_embeds`, the image mask will be derived using
        `config.image_token_id`. Same goes for audio and video masks

        Args:
            input_ids: A tensor containing the hard token IDs from the text tokenizer.
            inputs_embeds: A tensor containing the embeddings for all hard text tokens.

        Returns:
            image_mask, video_mask, audio_mask
        NrR   )r¸   r³   r±   )
rB   Úimage_token_idÚvideo_token_idÚaudio_token_idÚget_input_embeddingsrm   Úfullr€  r³   r�  )r‹   rz  rC   Úspecial_image_maskÚspecial_video_maskÚspecial_audio_masks         r_   Úget_placeholder_maskz Gemma4Model.get_placeholder_mask}  sV  € ð& Ð Ø!*¨d¬kÔ.HÒ!HÐØ!*¨d¬kÔ.HÒ!HÐØ!*¨d¬kÔ.HÒ!HÐÑð Ø.�4×,Ò,Ñ.Ô.Ý”J˜r 4¤;Ô#=ÅUÄZÐXeÔXlÐmÑmÔmñô ò÷ Šc�"‰gŒgð ð Ø.�4×,Ò,Ñ.Ô.Ý”J˜r 4¤;Ô#=ÅUÄZÐXeÔXlÐmÑmÔmñô ò÷ Šc�"‰gŒgð ð Ø.�4×,Ò,Ñ.Ô.Ý”J˜r 4¤;Ô#=ÅUÄZÐXeÔXlÐmÑmÔmñô ò÷ Šc�"‰gŒgð ð "Ð#5Ð7IÐIÐIra   r"  rD   r#  rF   rE   rÞ  rƒ  r€  c                 óœ  — |du |
duz  rt          d¦  «        ‚|�|�t          d¦  «        ‚|                      ||
¦  «        \  }}}||z  |z  }d}|
€V|                     ¦   «         }t          j        || j        j        j        |¦  «        } |                      ¦   «         |¦  «        }
|€®| j         	                    ¦   «         j
        r�| j        j        j        | j        j        j        dd…f         }|                     |
j        ¦  «        }t          j        |d         |                     ddd¦  «        |
¦  «        }| j                             ||¦  «        }|��:|                      ||d¬¦  «        j        }t          j        |d	¬
¦  «                             |
j        |
j        ¦  «        }|                     ¦   «         }|                     d¦  «                             |
¦  «                             |
j        ¦  «        }t3          |
|                              ¦   «         |                     ¦   «         k    d|› d|j        d	         › �¦  «         |
                     |                     |
j        ¦  «        |                     |
j        ¦  «        ¦  «        }
|��:|                      ||d¬¦  «        j        }t          j        |d	¬
¦  «                             |
j        |
j        ¦  «        }|                     ¦   «         }|                     d¦  «                             |
¦  «                             |
j        ¦  «        }t3          |
|                              ¦   «         |                     ¦   «         k    d|› d|j        d	         › �¦  «         |
                     |                     |
j        ¦  «        |                     |
j        ¦  «        ¦  «        }
|��F|��C|                      ||d¬¦  «        }|j        }|j        }||                     |j        ¦  «                 }|                     ¦   «         }|                     d¦  «                             |
¦  «                             |
j        ¦  «        }t3          |
|                              ¦   «         |                     ¦   «         k    d|› d|j        d	         |j        d         z  › �¦  «         |
                     |                     |
j        ¦  «        |                     |
j        |
j        ¦  «        ¦  «        }
|€V|�|                      ¦   «         nd	}t          j!        |
j        d         |
j        ¬¦  «        |z   }|                     d	¦  «        }tE          |x} tF          ¦  «        sx| j         	                    ¦   «         |
|||dœ}!| j         	                    ¦   «         }"|"j$        dk    }#|#r'|	�%tK          |	|
j        ¬¦  «        }$tM          dd|$i|!¤Ž} ntO          di |!¤Ž}  | j        d|| |||
|ddœ|¤Ž}%tQ          |%j)        |%j*        |%j+        |%j,        |�|nd|�|nd|%j-        ¬¦  «        S )áJ  
        input_features_mask (`torch.FloatTensor` of shape `(num_images, seq_length)`):
            The attention mask for the input audio.
        image_position_ids (`torch.LongTensor` of shape `(batch_size, max_patches, 2)`, *optional*):
            2D patch position coordinates from the image processor, with `(-1, -1)` indicating padding.
            Passed through to the vision encoder for positional embedding computation.
        video_position_ids (`torch.LongTensor` of shape `(num_videos, num_frames, max_patches, 2)`, *optional*):
            2D patch position coordinates from the video processor, with `(-1, -1)` indicating padding.
            Passed through to the vision encoder for positional embedding computation.
        per_layer_inputs (`torch.Tensor`, *optional*):
            Pre-computed per-layer input text embeddings of shape `(batch_size, sequence_length, num_hidden_layers,
            hidden_size_per_layer_input)`. When provided, these are used directly instead of being computed from `input_ids`
            via `get_per_layer_inputs()` in the text model. If calling the `forward` with `inputs_embeds` instead of `input_ids`,
            you should probably precompute them and forward them along `inputs_embeds`, otherwise recomputing them needs
            to reverse the main embedding, which is expensive.
        Nr…  r†  r´   r=   r±   T)Úreturn_dictr   rµ   z6Image features and image tokens do not match, tokens: z, features: z6Video features and video tokens do not match, tokens: z6Audio features and audio tokens do not match, tokens: r²   rJ   ÚvisionrG   )r€  rD   rF   rE   rC   rƒ  r  )ræ  rE   r�   rh  Úimage_hidden_statesÚaudio_hidden_statesrd   rR   ).r}  r  r  rm   rh  rB   rÅ  Úpad_token_idr  Úget_text_configr5  rõ  r}  r  r¹   r³   ré   r  r  r   rº   r¸   r+  r«   Ú	expand_asr"   ÚnumelrÚ   Úmasked_scatterr  Úget_audio_featuresrD   rˆ  rª   r¯  rj   r  rè  r`   r   rc   ræ  rE   r�   rh  rd   )&r‹   rz  rm  r  r"  rD   r#  rF   rE   rÞ  rC   rƒ  rû  r  r€  rV  Ú
image_maskÚ
video_maskÚ
audio_maskÚmultimodal_maskÚllm_input_idsÚpad_embeddingÚllm_inputs_embedsÚimage_featuresÚn_image_tokensÚvideo_featuresÚn_video_tokensÚaudio_outputÚaudio_featuresÚaudio_mask_from_encoderÚn_audio_tokensr‹  rŒ  rV   rÅ  Ú	use_bidirrG   r›  s&                                         r_   r‘   zGemma4Model.forwardª  sÏ  € ðJ ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ð%5Ð%AÝÐ[Ñ\Ô\Ð\à-1×-FÒ-FÀyÐR_Ñ-`Ô-`Ñ*ˆ
�J 
Ø$ zÑ1°JÑ>ˆð ˆØÐ Ø%ŸOšOÑ-Ô-ˆMÝ!œK¨¸¼Ô9PÔ9]Ð_lÑmÔmˆMØ7˜D×5Ò5Ñ7Ô7¸ÑFÔFˆMàÐ#¨¬×(CÒ(CÑ(EÔ(EÔ(aÐ#Ø Ô/Ô<ÔCÀDÄKÔD[ÔDhÐjkÐjkÐjkÐDkÔlˆMØ-×0Ò0°Ô1EÑFÔFˆOÝ %¤¨O¸IÔ,FÈ×HZÒHZÐ[\Ð^_ÐacÑHdÔHdÐfsÑ tÔ tÐØ#Ô2×GÒGÈÐWhÑiÔiÐð Ñ#Ø!×4Ò4°\ÐCUÐcgÐ4ÑhÔhÔvˆNÝ"œY ~¸1Ð=Ñ=Ô=×@Ò@ÀÔAUÐWdÔWjÑkÔkˆNð (Ÿ^š^Ñ-Ô-ˆNØ#×-Ò-¨bÑ1Ô1×;Ò;¸MÑJÔJ×MÒMÈmÔNbÑcÔcˆJÝ"Ø˜jÔ)×/Ò/Ñ1Ô1°^×5IÒ5IÑ5KÔ5KÒKð.Èð .ð .Ø"Ô(¨Ô+ð.ð .ñô ð ð *×8Ò8Ø—’˜mÔ2Ñ3Ô3°^×5FÒ5FÀ}ÔG[Ñ5\Ô5\ñô ˆMð Ñ*Ø!×4Ò4Ø#Ð%7ÀTð 5ñ ô äð õ #œY ~¸1Ð=Ñ=Ô=×@Ò@ÀÔAUÐWdÔWjÑkÔkˆNð (Ÿ^š^Ñ-Ô-ˆNØ#×-Ò-¨bÑ1Ô1×;Ò;¸MÑJÔJ×MÒMÈmÔNbÑcÔcˆJÝ"Ø˜jÔ)×/Ò/Ñ1Ô1°^×5IÒ5IÑ5KÔ5KÒKð.Èð .ð .Ø"Ô(¨Ô+ð.ð .ñô ð ð *×8Ò8Ø—’˜mÔ2Ñ3Ô3°^×5FÒ5FÀ}ÔG[Ñ5\Ô5\ñô ˆMð
 Ñ%Ð*=Ñ*IØ×2Ò2°>ÐCVÐdhÐ2ÑiÔiˆLØ)Ô7ˆNØ&2Ô&AÐ#ð
 ,Ð,C×,FÒ,FÀ~ÔG\Ñ,]Ô,]Ô^ˆNà'Ÿ^š^Ñ-Ô-ˆNØ#×-Ò-¨bÑ1Ô1×;Ò;¸MÑJÔJ×MÒMÈmÔNbÑcÔcˆJÝ"Ø˜jÔ)×/Ò/Ñ1Ô1°^×5IÒ5IÑ5KÔ5KÒKðHÈð Hð HØ"Ô(¨Ô+¨nÔ.BÀ1Ô.EÑEðHð Hñô ð ð *×8Ò8Ø—’˜mÔ2Ñ3Ô3°^×5FÒ5FÀ}ÔG[Ð]jÔ]pÑ5qÔ5qñô ˆMð
 ÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå°Ð?Ð-ÅÑFÔFð 	Oàœ+×5Ò5Ñ7Ô7Ø!.Ø"0Ø#2Ø ,ðð ˆKð œ+×5Ò5Ñ7Ô7ˆKØ#Ô?À8ÒKˆIàð 	OÐ.Ð:Ý%DÐEVÐ_lÔ_sÐ%tÑ%tÔ%tÐ"Ý&Cð 'ð 'Ø'9ð'à!ð'ð 'Ð#Ð#õ '@Ð&NÐ&NÀ+Ð&NÐ&NÐ#à%�$Ô%ð 	
Ø-Ø.Ø%Ø+Ø'ØØð	
ð 	
ð ð	
ð 	
ˆõ )Ø%Ô7Ø#Ô3Ø!Ô/ØÔ)Ø2>Ð2J  ÐPTØ2@Ð2L  ÐRVØ$Ô5ð
ñ 
ô 
ð 	
ra   zPProjects the last hidden state from the audio encoder into language model space.c                 ó”   — | j         €t          d¦  «        ‚ | j         ||fddi|¤Ž}|                      |j        ¬¦  «        |_        |S )a.  
        input_features (`torch.FloatTensor` of shape `(num_images, seq_length, num_features)`):
            The tensors corresponding to the input audio.
        input_features_mask (`torch.FloatTensor` of shape `(num_images, seq_length)`):
            The attention mask for the input audio.
        Nz•Audio features were requested, but the model was initialized without an audio_config. Cannot process audio without an audio tower and audio embedder.r  Trþ  )rñ  r}  rò  ræ  r   )r‹   r"  r#  rV  Úaudio_outputss        r_   r  zGemma4Model.get_audio_featuresS  sm   € ð ÔÐ#ÝðRñô ð ð
 )˜Ô(¨Ð9LÐiÐiÐZ^ÐiÐbhÐiÐiˆØ&*×&6Ò&6À]ÔEdÐ&6Ñ&eÔ&eˆÔ#àÐra   r�   )NN)NNNNNNNNNNNNNN)rf   rg   rh   r?   r€   r÷  rú  r    r   rm   rÙ  rÎ  r   r   r   r  r  rl   rv   r  r$   rn   r   r�  rc   r‘   ru   r  r“   r”   s   @r_   rê  rê  (  s¼  ø€ € € € € ð
˜|ð 
ð 
ð 
ð 
ð 
ð 
ð:ð :ð :ð;ð ;ð ;ð Ø€^Ð!rÐsÑsÔsð 7;ðð àÔ'ðð "Ô,¨tÑ3ðð Ð+Ô,ð	ð
 
$ðð ð ñ tÔsñ Ôðð2 Ø€^Ð!tÐuÑuÔuð 7;ðð à"Ô.ðð "Ô,¨tÑ3ðð Ð+Ô,ð	ð
 
$ðð ð ñ vÔuñ Ôðð< .2Ø26ð+Jð +JàÔ# dÑ*ð+Jð Ô(¨4Ñ/ð+Jð 
ˆuÔ Ô!1°5Ô3CÐCÔ	Dð	+Jð +Jð +Jð +JðZ  ØØð .2Ø15Ø8<Ø37Ø.2Ø37Ø04Ø(,Ø59Ø26Ø!%Ø6:Ø6:Ø04ðd
ð d
àÔ# dÑ*ðd
ð Ô'¨$Ñ.ðd
ð #Ô.°Ñ5ð	d
ð
 Ô)¨DÑ0ðd
ð œ tÑ+ðd
ð #œ\¨DÑ0ðd
ð Ô&¨Ñ-ðd
ð  ™ðd
ð !Ô+¨dÑ2ðd
ð Ô(¨4Ñ/ðd
ð ˜$‘;ðd
ð "Ô,¨tÑ3ðd
ð "Ô,¨tÑ3ðd
ð  œ,¨Ñ-ðd
ð  Ð+Ô,ð!d
ð" 
#ð#d
ð d
ð d
ñ „^ñ Ôñ  Ôðd
ðL Ø€^Ð!sÐtÑtÔtðàœðð #œ\ðð Ð+Ô,ð	ð
 
Ð'Ñ	'ðð ð ñ uÔtñ Ôðð ð ð ð ra   rê  z…
    The base Gemma 4 model comprising a vision backbone, an audio backbone, a language model, and a language modeling
    head.
    c            %       ól  ‡ — e Zd ZdZd„ Zd„ Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d dej        dz  dej        dz  dej        dz  d	ej        dz  d
ej	        dz  dej	        dz  dej        dz  dej        dz  dej        dz  de
dz  dej        dz  dej        dz  dej        dz  dedz  deej	        z  dej	        dz  dee         def$d„Ze	 d!dej        dej        dz  dee         fd„¦   «         Ze	 	 d"dedej	        d
ej	        dz  de
dz  dej	        dz  dej	        dz  dedz  defd„¦   «         Zd#ˆ fd„	Zˆ xZS )$ÚGemma4ForConditionalGenerationr�  c                 ó4   — | j                              ¦   «         S r�   )r�  r÷  rö  s    r_   r÷  z=Gemma4ForConditionalGeneration.get_per_layer_input_embeddingsv  s   € ØŒz×8Ò8Ñ:Ô:Ð:ra   c                 ó:   — | j                              |¦  «         d S r�   )r�  rú  rù  s     r_   rú  z=Gemma4ForConditionalGeneration.set_per_layer_input_embeddingsy  s   € ØŒ
×1Ò1°%Ñ8Ô8Ð8Ð8Ð8ra   Nr   rz  rm  r  r"  rD   r#  rF   rû  r  rE   rÞ  rC   r‘  rƒ  r’  r€  rV  rH   c           
      ód  —  | j         di d|“d|“d|“d|“d|“d|“d|“d|
“d	|“d
|“d|“d|“d|“d|“d|	“dd“|¤Ž}|j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }| j                             ¦   «         j        x}�||z  }t          j
        |¦  «        }||z  }d}|�, | j        ||| j                             ¦   «         j        fi |¤Ž}t          |||j        |j        |j        |j        |j        |j        ¬¦  «        S )r  rz  rm  r  r"  rD   r#  rF   rE   rÞ  rC   r€  r‘  rƒ  rû  r  r  TN)r”  r•  rE   r�   rh  r  r  rd   rR   )r�  ræ  r¯  r’   r–  r—  rB   r  r˜  rm   ró   r™  rš  rq   rE   r�   rh  r  r  rd   )r‹   rz  rm  r  r"  rD   r#  rF   rû  r  rE   rÞ  rC   r‘  rƒ  r’  r€  rV  r›  r�   rœ  r•  r˜  r”  s                           r_   r‘   z&Gemma4ForConditionalGeneration.forward|  sû  € ðH �$”*ð 
ð 
ð 
Ø�ið
à%˜ð
ð !4Ð 3ð
ð *˜>ð	
ð
 *˜>ð
ð !4Ð 3ð
ð &˜ð
ð ,˜Oð
ð 0Ð/ð
ð (˜-ð
ð .Ð-ð
ð �6ð
ð  �ið
ð  2Ð1ð
ð  2Ð1ð
ð  ˜Øð#
ð 
ˆð(  Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆØ'+¤{×'BÒ'BÑ'DÔ'DÔ'\Ð\Ð#ÐiØÐ5Ñ5ˆFÝ”Z Ñ'Ô'ˆFØÐ5Ñ5ˆFàˆØÐØ%�4Ô% f¨f°d´k×6QÒ6QÑ6SÔ6SÔ6^ÐiÐiÐbhÐiÐiˆDå+ØØØ#Ô3Ø!Ô/ØÔ)Ø 'Ô ;Ø 'Ô ;Ø$Ô5ð	
ñ 	
ô 	
ð 		
ra   c                 ó*   —  | j         j        ||fi |¤ŽS )a-  
        image_position_ids (`torch.LongTensor` of shape `(batch_size, max_patches, 2)`, *optional*):
            2D patch position coordinates from the image processor, with `(-1, -1)` indicating padding.
            Passed through to the vision encoder for positional embedding computation.
        )r�  r  )r‹   rm  rû  rV  s       r_   r  z1Gemma4ForConditionalGeneration.get_image_featuresÌ  s%   € ð -ˆtŒzÔ,¨\Ð;MÐXÐXÐQWÐXÐXÐXra   FrB   Úis_first_iterationc                 óð   — |                       ¦   «         ||||dœ}|                       ¦   «         }	t          |	dd ¦  «        dk    }
|
r&|�$t          ||j        ¬¦  «        }t	          dd|i|¤ŽS t          di |¤ŽS )NrJ   r  r  r²   rG   rR   )r  rS   rè  r³   r`   r   )rB   rC   rD   rE   rF   rÞ  r7  rV  rV   rÅ  r.  rG   s               r_   r   z8Gemma4ForConditionalGeneration.create_masks_for_generateÚ  s¹   € ð ×,Ò,Ñ.Ô.Ø*Ø,Ø.Ø(ð
ð 
ˆð ×,Ò,Ñ.Ô.ˆÝ˜KÐ)FÈÑMÔMÐQYÒYˆ	àð 	Ð*Ð6Ý!@ÐARÐ[hÔ[oÐ!pÑ!pÔ!pÐÝ0ð ð Ø#5ðàðð ð õ
 )Ð7Ð7¨;Ð7Ð7Ð7ra   Tc                 ó„   •—  t          ¦   «         j        |f||dœ|¤Ž}|s|rd |d<   |s|                     dd ¦  «        }|S )N)rƒ  r7  rÞ  r€  )r   Úprepare_inputs_for_generationr‰  )r‹   rz  rƒ  r7  rV  Úmodel_inputsrZ   rŒ   s          €r_   r:  z<Gemma4ForConditionalGeneration.prepare_inputs_for_generationù  sv   ø€ Ø<•u‘w”wÔ<Øð
Ø!*Ð?Qð
ð 
ØU[ð
ð 
ˆð #ð 	5¨)ð 	5à04ˆLÐ,Ñ-à!ð 	;Ø× Ò Ð!3°TÑ:Ô:ˆAàÐra   )NNNNNNNNNNNNNNr   Nr�   )NF)TF)rf   rg   rh   r�  r÷  rú  rm   rÎ  rÙ  rn   r   r�  r’   r   r   rq   r‘   r   r  Ústaticmethodr   rj   r   r:  r“   r”   s   @r_   r2  r2  m  sé  ø€ € € € € ð  Ðð;ð ;ð ;ð9ð 9ð 9ð
 .2Ø15Ø8<Ø37Ø.2Ø37Ø04Ø6:Ø6:Ø(,Ø59Ø26Ø*.Ø!%Ø-.Ø04ð#N
ð N
àÔ# dÑ*ðN
ð Ô'¨$Ñ.ðN
ð #Ô.°Ñ5ð	N
ð
 Ô)¨DÑ0ðN
ð œ tÑ+ðN
ð #œ\¨DÑ0ðN
ð Ô&¨Ñ-ðN
ð "Ô,¨tÑ3ðN
ð "Ô,¨tÑ3ðN
ð  ™ðN
ð !Ô+¨dÑ2ðN
ð Ô(¨4Ñ/ðN
ð Ô  4Ñ'ðN
ð ˜$‘;ðN
ð  ˜eœlÑ*ð!N
ð"  œ,¨Ñ-ð#N
ð$ Ð+Ô,ð%N
ð& 
&ð'N
ð N
ð N
ð N
ð` ð 7;ðYð YàÔ'ðYð "Ô,¨tÑ3ðYð Ð+Ô,ð	Yð Yð Yñ „^ðYð ð 26Ø*/ð8ð 8Ø ð8à”|ð8ð œ tÑ+ð8ð  ™ð	8ð
 ”l TÑ)ð8ð !œ<¨$Ñ.ð8ð ! 4™Kð8ð 
ð8ð 8ð 8ñ „\ð8ð<ð ð ð ð ð ð ð ð ð ra   r2  )rŸ  r�  r2  rê  rG  rf  r[  )r(   )‹r¦   Úcollectionsr   Úcollections.abcr   Údataclassesr   Ú	functoolsr   rm   r   Útorch.nnr   rÛ   Ú r
   rP  Úactivationsr   Úcache_utilsr   r   Úconfiguration_utilsr   Úmasking_utilsr   r   r   r   r   r   r   r   Úmodeling_flash_attention_utilsr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r    r!   r"   Úutils.genericr#   r$   r%   Úutils.output_capturingr&   r'   Úauto.modeling_autor)   Úgemma3.modeling_gemma3r*   r+   r,   r-   r.   r/   r0   Úgemma3n.modeling_gemma3nr1   r2   r3   r4   r5   r6   r7   r8   r9   Úmixtral.modeling_mixtralr:   Ú0moonshine_streaming.modeling_moonshine_streamingr;   Úsam3.modeling_sam3r<   Úconfiguration_gemma4r>   r?   r@   rA   Ú
get_loggerrf   Úloggerrn   rj   r`   rc   rq   rs   ru   rù  rx   r–   rš   rÁ   r  r  r&  ÚConv1dr8  r?  rM  rY  rs  r�  r’   r£  r¥  r»  rÐ  rÛ  ré  rð  r  r  r"  r1  rE  rG  rf  r�  rŸ  r[  rÃ  rÝ  r³   rè  rê  r2  Ú__all__rR   ra   r_   ú<module>rZ     s>  ðð €€€Ø  Ð  Ð  Ð  Ð  Ð  Ø $Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !Ø %Ð %Ð %Ð %Ð %Ð %à €€€Ø Ð Ð Ð Ð Ð Ø $Ð $Ð $Ð $Ð $Ð $à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3ð	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð CÐ BÐ BÐ BÐ BÐ BØ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SØ 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ^Ð ]Ð ]Ð ]Ð ]Ð ]Ð ]Ð ]Ð ]Ð ]Ø EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ *Ð *Ð *Ð *Ð *Ð *ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 6Ð 5Ð 5Ð 5Ð 5Ð 5Ø [Ð [Ð [Ð [Ð [Ð [Ø 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø gÐ gÐ gÐ gÐ gÐ gÐ gÐ gÐ gÐ gÐ gÐ gð 
ˆÔ	˜HÑ	%Ô	%€ð<Øð<à”<ð<ð ”L 4Ñ'ð<ð ˜T‘\ð	<ð
 ”, Ñ%ð<ð œð<ð 
ð<ð <ð <ð <ð~Qð Qð Qð Qð QÐ :ñ Qô Qð Qð*Qð Qð Qð Qð QÐ#@ñ Qô Qð Qð2 ð
Qð 
Qð 
Qð 
Qð 
QÐ$;ñ 
Qô 
Qñ „ð
Qð Ø
ð3ð 3ð 3ð 3ð 3Ð7ñ 3ô 3ñ „ñ „ð3ðð ð ð ð ˜BœIñ ô ð ð:	ð 	ð 	ð 	ð 	�Nñ 	ô 	ð 	ð7ð 7ð 7ð 7ð 7 r¤yñ 7ô 7ð 7ð>i)ð i)ð i)ð i)ð i)˜2œ9ñ i)ô i)ð i)ðX#ð #ð #ð #ð #¨b¬iñ #ô #ð #ð8;ð ;ð ;ð ;ð ;¨¬ñ ;ô ;ð ;ð< ð  ð  ð  ð  ˜RœYñ  ô  ð  ðH"ð "ð "ð "ð "˜bœiñ "ô "ð "ðB&ð &ð &ð &ð &˜RœYñ &ô &ð &ðR0ð 0ð 0ð 0ð 0�r”yñ 0ô 0ð 0ðl*3ð *3ð *3ð *3ð *3 ¤	ñ *3ô *3ð *3ðZ@0ð @0ð @0ð @0ð @0˜œñ @0ô @0ð @0ðFað að að að a�iñ aô að að ð5&ð 5&Ø„|ð5&à	Œð5&ð 
Œð5&ð ”,ð	5&ð
 ð5&ð „\ð5&ð 5&ð 5&ð 5&ðrCð Cð Cð Cð CÐ"8ñ Cô Cð Cð< ð:)ð :)ð :)ð :)ð :)˜Oñ :)ô :)ñ Ôð:)ð|"ð "ð "ð "ð "Ð1ñ "ô "ð "ðJ)Hð )Hð )Hð )Hð )H˜"œ)ñ )Hô )Hð )Hð^^ð ^ð ^ð ^ð ^�Iñ ^ô ^ð ^ðUð Uð Uð Uð UÐ 5ñ Uô Uð UðBq)ð q)ð q)ð q)ð q)˜"œ)ñ q)ô q)ð q)ðh7ð 7ð 7ð 7ð 7˜ñ 7ô 7ð 7ð"@ð "@ð "@ð "@ð "@�r”yñ "@ô "@ð "@ðJOð Oð Oð Oð OÐ/ñ Oô Oð Oðd	ð 	ð 	ð 	ð 	Ð$Añ 	ô 	ð 	ð1)ð 1)ð 1)ð 1)ð 1)Ð2ñ 1)ô 1)ð 1)ðh €Ð`ÐaÑaÔaðW
ð W
ð W
ð W
ð W
�oñ W
ô W
ñ bÔaðW
ðt €Ð]Ð^Ñ^Ô^ðJ
ð J
ð J
ð J
ð J
Ð)ñ J
ô J
ñ _Ô^ðJ
ðZScð Scð Scð Scð ScÐ,ñ Scô Scð ScðlAHð AHð AHð AHð AHÐ-ñ AHô AHð AHðH6ð 6ð 6ð 6ð 6Ð8ñ 6ô 6ð 6ð>Ø”L 4Ñ'ðà”\ DÑ(ðð ��_ðð ð ð ð:	°u´|ð 	ÈUÌ\ð 	Ð^cÔ^jð 	ð 	ð 	ð 	ð €ððñ ô ð|ð |ð |ð |ð |�,ñ |ô |ñô ð|ð~	 €ððñ ô ðQð Qð Qð Qð QÐ%Dñ Qô Qñô ðQðhð ð €€€ra   