§
    9²¸j×  ã                   óf  — d dl Z d dlmZ d dlmZ d dlZd dlmZ d dlm	Z	 ddl
mZ ddlmZmZmZ ddlmZ dd	lmZmZmZ dd
lmZ ddlmZ ddlmZmZmZmZm Z m!Z!m"Z" ddl#m$Z$ ddl%m&Z&m'Z' ddl(m)Z) ddl*m+Z+m,Z,m-Z-m.Z.m/Z/ ddl0m1Z1 ddl2m3Z3m4Z4 ddl5m6Z6 ddl7m8Z8m9Z9 ddl:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZE ddlFmGZG ddlHmIZImJZJmKZK  e.jL        eM¦  «        ZN e,d¬¦  «        e	 G d„ de9e¦  «        ¦   «         ¦   «         ZO e,d¬¦  «        e	 G d„ de8¦  «        ¦   «         ¦   «         ZP e,d¬¦  «        e	 G d„ d e9e¦  «        ¦   «         ¦   «         ZQ e,d¬¦  «        e	 G d!„ d"e¦  «        ¦   «         ¦   «         ZR G d#„ d$e?¦  «        ZS G d%„ d&e<¦  «        ZT G d'„ d(e@¦  «        ZU G d)„ d*e;¦  «        ZV G d+„ d,e;¦  «        ZWdNd.eXd/efd0„ZY G d1„ d2eJ¦  «        ZZ G d3„ d4eJ¦  «        Z[ G d5„ d6eK¦  «        Z\ G d7„ d8eI¦  «        Z] G d9„ d:e=¦  «        Z^ G d;„ d<eA¦  «        Z_e, G d=„ d>e>¦  «        ¦   «         Z` G d?„ d@e`¦  «        Za G dA„ dBe`¦  «        Zb G dC„ dDe`¦  «        Zce, G dE„ dFe`¦  «        ¦   «         Zd G dG„ dHe`e¦  «        Zee, G dI„ dJe`¦  «        ¦   «         Zfe, G dK„ dLe`¦  «        ¦   «         Zgg dM¢ZhdS )Oé    N)ÚCallable)ÚAny)Ústricté   )Úinitialization)ÚDynamicCacheÚEncoderDecoderCacheÚStaticCache)ÚPreTrainedConfig)ÚGenerationConfigÚGenerationMixinÚGenerationMode)Úcreate_bidirectional_mask)ÚFlashAttentionKwargs)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚBaseModelOutputWithPoolingÚSeq2SeqLMOutputÚSeq2SeqModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚROPE_INIT_FUNCTIONS)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚloggingÚtorch_compilable_check)Úmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )Ú	AutoModel)ÚGemma3ConfigÚGemma3TextConfig)ÚGemma3AttentionÚ	Gemma3MLPÚGemma3MultiModalProjectorÚGemma3PreTrainedModelÚGemma3RMSNormÚGemma3RotaryEmbeddingÚGemma3TextScaledWordEmbeddingÚapply_rotary_pos_embÚcreate_causal_maskÚ!create_sliding_window_causal_maskÚeager_attention_forward)ÚSiglipVisionConfig)ÚT5GemmaClassificationHeadÚT5GemmaEncoderLayerÚT5GemmaLMHeadzgoogle/t5gemma-2-270m-270m)Ú
checkpointc                   ó0   — e Zd ZdZdZ e¦   «         Zd„ ZdS )ÚT5Gemma2TextConfigát  
    query_pre_attn_scalar (`float`, *optional*, defaults to 256):
        Scaling factor used on the attention scores
    final_logit_softcapping (`float`, *optional*):
        Scaling factor when applying tanh softcapping on the logits.
    attn_logit_softcapping (`float`, *optional*):
        Scaling factor when applying tanh softcapping on the attention scores.
    Út5gemma2_textc                 ó®   ‡— |                      dd¦  «        Š| j        €%ˆfd„t          | j        ¦  «        D ¦   «         | _        t	          j        di |¤Ž d S )NÚsliding_window_patterné   c                 óB   •— g | ]}t          |d z   ‰z  ¦  «        rdnd‘ŒS ©é   Úsliding_attentionÚfull_attention©Úbool©Ú.0ÚiÚ_sliding_window_patterns     €ú}/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/t5gemma2/modular_t5gemma2.pyú
<listcomp>z4T5Gemma2TextConfig.__post_init__.<locals>.<listcomp>_   óG   ø€ ð  ð  ð  àõ (,¨Q°©UÐ6MÑ,MÑ'NÔ'NÐdÐ#Ð#ÐTdð ð  ð  ó    © ©ÚpopÚlayer_typesÚrangeÚnum_hidden_layersr   Ú__post_init__©ÚselfÚkwargsrI   s     @rJ   rT   z T5Gemma2TextConfig.__post_init__[   óv   ø€ à"(§*¢*Ð-EÀqÑ"IÔ"IÐØÔÐ#ð ð  ð  ð  å˜tÔ5Ñ6Ô6ð ñ  ô  ˆDÔõ
 	Ô&Ð0Ð0¨Ð0Ð0Ð0Ð0Ð0rM   N©Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚAttributeErrorÚuse_bidirectional_attentionrT   rN   rM   rJ   r9   r9   L   sB   € € € € € ðð ð !€JØ"0 .Ñ"2Ô"2Ðð	1ð 	1ð 	1ð 	1ð 	1rM   r9   c                   ó   — e Zd ZdZeedœZdS )ÚT5Gemma2EncoderConfigÚt5gemma2_encoder)Útext_configÚvision_configN)rZ   r[   r\   r^   r9   r3   Úsub_configsrN   rM   rJ   rb   rb   g   s)   € € € € € ð $€Jð *Ø+ðð €K€K€KrM   rb   c                   ó0   — e Zd ZdZdZ e¦   «         Zd„ ZdS )ÚT5Gemma2DecoderConfigr:   Út5gemma2_decoderc                 ó®   ‡— |                      dd¦  «        Š| j        €%ˆfd„t          | j        ¦  «        D ¦   «         | _        t	          j        di |¤Ž d S )Nr=   r>   c                 óB   •— g | ]}t          |d z   ‰z  ¦  «        rdnd‘ŒS r@   rD   rF   s     €rJ   rK   z7T5Gemma2DecoderConfig.__post_init__.<locals>.<listcomp>…   rL   rM   rN   rO   rU   s     @rJ   rT   z#T5Gemma2DecoderConfig.__post_init__�   rX   rM   NrY   rN   rM   rJ   rh   rh   r   sB   € € € € € ðð ð $€JØ"0 .Ñ"2Ô"2Ðð	1ð 	1ð 	1ð 	1ð 	1rM   rh   c                   ó$  ‡ — e Zd ZU dZdZdgZeedœZdddœZ	dZ
eeeef         z  dz  ed	<   dZeeeef         z  dz  ed
<   dZeed<   dZeez  ed<   dZeez  ed<   dZeez  ed<   dZeed<   dZeed<   dZedz  ed<   dZeed<   ˆ fd„Zd„ Zˆ xZS )ÚT5Gemma2Configa`  
    encoder (`Union[T5Gemma2EncoderConfig, dict]`, optional, *optional*):
        Configuration for the encoder.
    decoder (`Union[T5Gemma2DecoderConfig, dict]`, optional, *optional*):
        Configuration for the decoder.
    eoi_token_index (`int`, *optional*):
        The end-of-image token index to wrap the image prompt. Will be same as
        `self.encoder.eoi_token_index`

    ```python
    >>> from transformers import T5Gemma2Config, T5Gemma2Model
    >>> t5gemma2_config = T5Gemma2Config.from_pretrained("google/t5gemma-270m-270m")
    >>> model = T5Gemma2Model(t5gemma2_config)
    ```
    Út5gemma2Úpast_key_values)ÚencoderÚdecoderÚimage_token_indexÚeoi_token_index)Úimage_token_idÚeoi_token_idNrp   rq   TÚis_encoder_decoderç        Údropout_rateÚattention_dropoutÚclassifier_dropout_rateg{®Gáz”?Úinitializer_rangeiè Útie_word_embeddingsc                 ó  •— t          | j        t          ¦  «        rt          di | j        ¤Ž| _        n4| j        €-t          ¦   «         | _        t                               d¦  «         t          | j        t          ¦  «        rt          di | j        ¤Ž| _        n4| j        €-t          ¦   «         | _        t                               d¦  «         | j        | j        j	        _        | j
        | j        j	        _
        | j
        | j        j        _
        | j        | j        _        | j        | j        _        | j
        | j        _
        | j        j        | _        dD ]}||vrt          | j        |¦  «        ||<   Œ t          ¦   «         j        di |¤Ž d S )NzDencoder is None, using default T5Gemma2EncoderConfig encoder config.zDdecoder is None, using default T5Gemma2DecoderConfig decoder config.)Úbos_token_idÚpad_token_idÚeos_token_idÚ
vocab_sizerN   )Ú
isinstancerp   Údictrb   ÚloggerÚinforq   rh   rx   rd   ry   re   rr   rs   ÚgetattrÚsuperrT   )rV   rW   Úspecial_token_keyÚ	__class__s      €rJ   rT   zT5Gemma2Config.__post_init__¸   sl  ø€ Ý�d”l¥DÑ)Ô)ð 	`Ý0Ð@Ð@°4´<Ð@Ð@ˆDŒLˆLØŒ\Ð!Ý0Ñ2Ô2ˆDŒLÝ�KŠKÐ^Ñ_Ô_Ð_å�d”l¥DÑ)Ô)ð 	`Ý0Ð@Ð@°4´<Ð@Ð@ˆDŒLˆLØŒ\Ð!Ý0Ñ2Ô2ˆDŒLÝ�KŠKÐ^Ñ_Ô_Ð_à04Ô0AˆŒÔ Ô-Ø59Ô5KˆŒÔ Ô2Ø7;Ô7MˆŒÔ"Ô4Ø)-Ô)?ˆŒÔ&à$(Ô$5ˆŒÔ!Ø)-Ô)?ˆŒÔ&Ø#œ|Ô;ˆÔà!_ð 	Uð 	UÐØ ¨Ð.Ð.Ý,3°D´LÐBSÑ,TÔ,T�Ð(Ñ)øà�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'rM   c                 ój  — | j         j        j        | j        j        k    r/t	          d| j         j        j        › d| j        j        › d�¦  «        ‚| j        st	          d¦  «        ‚| j         j        j        | j        j        k    r/t	          d| j         j        j        › d| j        j        › d�¦  «        ‚dS )zOPart of `@strict`-powered validation. Validates the architecture of the config.zBImbalanced encoder-decoder is not supported in T5Gemma2: encoder (z) vs decoder (z).z4T5Gemma2Model only support encoder-decoder modeling.zRImbalanced encoder-decoder vocabulary size is not supported in T5Gemma2: encoder (N)rp   rd   Úhidden_sizerq   Ú
ValueErrorrv   r�   ©rV   s    rJ   Úvalidate_architecturez$T5Gemma2Config.validate_architectureÔ   së   € àŒ<Ô#Ô/°4´<Ô3KÒKÐKÝðmØ œLÔ4Ô@ðmð mØPTÔP\ÔPhðmð mð mñô ð ð
 Ô&ð 	UÝÐSÑTÔTÐTàŒ<Ô#Ô.°$´,Ô2IÒIÐIÝðkØ œLÔ4Ô?ðkð kØOSÌ|ÔOfðkð kð kñô ð ð JÐIrM   )rZ   r[   r\   r]   r^   Úkeys_to_ignore_at_inferencerb   rh   rf   Úattribute_maprp   rƒ   Ústrr   Ú__annotations__rq   rv   rE   rx   ÚfloatÚintry   rz   r{   rr   rs   r|   rT   rŽ   Ú__classcell__©r‰   s   @rJ   rm   rm   �   sj  ø€ € € € € € ðð ð  €JØ#4Ð"5Ðð )Ø(ðð €Kð .Ø)ðð €Mð
 >B€GÐ" T¨#¨s¨(¤^Ñ3°dÑ:ÐAÐAÑAØ=A€GÐ" T¨#¨s¨(¤^Ñ3°dÑ:ÐAÐAÑAØ#Ð˜Ð#Ð#Ñ#Ø #€L�%˜#‘+Ð#Ð#Ñ#Ø%(Ð�u˜s‘{Ð(Ð(Ñ(Ø+.Ð˜U S™[Ð.Ð.Ñ.Ø#Ð�uÐ#Ð#Ñ#Ø$Ð�sÐ$Ð$Ñ$Ø"&€O�S˜4‘ZÐ&Ð&Ñ&Ø $Ð˜Ð$Ð$Ñ$ð(ð (ð (ð (ð (ð8ð ð ð ð ð ð rM   rm   c                   ó   — e Zd ZdS )ÚT5Gemma2RMSNormN©rZ   r[   r\   rN   rM   rJ   r˜   r˜   æ   ó   € € € € € Ø€DrM   r˜   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚT5Gemma2MLPÚconfigc                 ó†   •— t          ¦   «                              |¦  «         t          j        |j        ¦  «        | _        d S ©N)r‡   Ú__init__ÚnnÚDropoutrx   Údropout©rV   r�   r‰   s     €rJ   r    zT5Gemma2MLP.__init__ë   s3   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý”z &Ô"5Ñ6Ô6ˆŒˆˆrM   c                 óÖ   — |                       |                      |¦  «        ¦  «        |                      |¦  «        z  }|                      |¦  «        }|                      |¦  «        }|S rŸ   )Úact_fnÚ	gate_projÚup_projr£   Ú	down_proj)rV   ÚxÚhidden_statesr©   s       rJ   ÚforwardzT5Gemma2MLP.forwardï   sV   € ØŸš D§N¢N°1Ñ$5Ô$5Ñ6Ô6¸¿ºÀa¹¼ÑHˆØŸš ]Ñ3Ô3ˆØ—N’N =Ñ1Ô1ˆ	ØÐrM   )rZ   r[   r\   r9   r    r¬   r•   r–   s   @rJ   rœ   rœ   ê   sT   ø€ € € € € ð7Ð1ð 7ð 7ð 7ð 7ð 7ð 7ðð ð ð ð ð ð rM   rœ   c                   ó   — e Zd ZdS )ÚT5Gemma2RotaryEmbeddingNr™   rN   rM   rJ   r®   r®   ö   rš   rM   r®   c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚT5Gemma2SelfAttentionr�   Ú	layer_idxc                 óZ   •— t          ¦   «                              ||¦  «         d| _        d S ©NF©r‡   r    Ú	is_causal©rV   r�   r±   r‰   s      €rJ   r    zT5Gemma2SelfAttention.__init__û   ó(   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+ØˆŒˆˆrM   )rZ   r[   r\   r9   r”   r    r•   r–   s   @rJ   r°   r°   ú   sL   ø€ € € € € ðÐ1ð ¸cð ð ð ð ð ð ð ð ð ð rM   r°   c                   óö   ‡ — e Zd ZdZdedefˆ fd„Z	 ddej        de	ej        ej        f         dej        dz  d	ej        d
e
dz  dee         de	ej        ej        dz  e	ej                 dz  f         fd„Zˆ xZS )ÚT5Gemma2MergedAttentionz6Merged self-attention and cross-attention for decoder.r�   r±   c                 óZ   •— t          ¦   «                              ||¦  «         d| _        d S r³   r´   r¶   s      €rJ   r    z T5Gemma2MergedAttention.__init__  r·   rM   Nr«   Úposition_embeddingsÚmerged_attention_maskÚencoder_hidden_statesro   rW   Úreturnc                 ó¤  — |j         d d…         }g |¢d‘| j        ‘R }|j         d d…         }	g |	¢d‘| j        ‘R }
|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «        }|                      |¦  «        }|\  }}t          ||||¦  «        \  }}|�L|j
        }|                     ||| j        ¦  «        \  }}|j                             | j        ¦  «        }|j        }|�|s¾|                      |¦  «                             |
¦  «                             dd¦  «        }|                      |¦  «                             |
¦  «                             dd¦  «        }|                      |¦  «        }|�.|                     ||| j        ¦  «        \  }}d|j        | j        <   n.|j        | j                 j        }|j        | j                 j        }|}|	d         }t'          j        ||gd¬¦  «        }t'          j        ||gd¬¦  «        }t+          j        | j        j        t2          ¦  «        } || ||||f| j        r| j        nd| j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }|�|dd | …f         }|d| d …f         }nd	\  }}|||fS )
NéÿÿÿÿrA   r$   T©Údimrw   )r£   Úscaling.)NN) ÚshapeÚhead_dimÚq_projÚviewÚ	transposeÚk_projÚv_projÚq_normÚk_normr/   Úself_attention_cacheÚupdater±   Ú
is_updatedÚgetÚcross_attention_cacheÚlayersÚkeysÚvaluesÚtorchÚcatr   Úget_interfacer�   Ú_attn_implementationr2   Útrainingry   rÃ   ÚreshapeÚ
contiguousÚo_proj)rV   r«   r»   r¼   r½   ro   rW   Úinput_shapeÚhidden_shapeÚcross_input_shapeÚcross_hidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinrÍ   rÏ   rÑ   Úcross_key_statesÚcross_value_statesÚcross_key_sizeÚattention_interfaceÚattn_outputÚattn_weightsÚself_attn_weightsÚcross_attn_weightss                              rJ   r¬   zT5Gemma2MergedAttention.forward  s¹  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆØ1Ô7¸¸¸Ô<ÐØDÐ0ÐD°"ÐD°d´mÐDÐDÐð —{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆà—{’{ <Ñ0Ô0ˆØ—[’[ Ñ,Ô,ˆ
à&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&à#2Ô#GÐ Ø';×'BÒ'BÀ:È|Ð]aÔ]kÑ'lÔ'lÑ$ˆJ˜ð )Ô3×7Ò7¸¼ÑGÔGˆJØ$3Ô$IÐ!àÐ"¨*Ð"Ø#Ÿ{š{Ð+@ÑAÔA×FÒFÐGYÑZÔZ×dÒdÐefÐhiÑjÔjÐØ!%§¢Ð-BÑ!CÔ!C×!HÒ!HÐI[Ñ!\Ô!\×!fÒ!fÐghÐjkÑ!lÔ!lÐà#Ÿ{š{Ð+;Ñ<Ô<ÐàÐ*Ø7L×7SÒ7SØ$Ð&8¸$¼.ñ8ô 8Ñ4Ð Ð"4ð >B�Ô*¨4¬>Ñ:øà4Ô;¸D¼NÔKÔPÐØ!6Ô!=¸d¼nÔ!MÔ!TÐð $ˆØ*¨1Ô-ˆÝ”Y 
Ð,<Ð=À1ÐEÑEÔEˆ
Ý”y ,Ð0BÐ!CÈÐKÑKÔKˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØ!ð	%
ð /3¬mÐD�DÔ*Ð*ÀØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆð Ð#Ø ,¨SÐ2B°N°?Ð2BÐ-BÔ CÐØ!-¨c°N°?Ð3CÐ3CÐ.CÔ!DÐÐà4>Ñ1ÐÐ1ØÐ-Ð/AÐAÐArM   rŸ   )rZ   r[   r\   r]   r9   r”   r    rÕ   ÚTensorÚtupler	   r   r   r¬   r•   r–   s   @rJ   r¹   r¹      s  ø€ € € € € Ø@Ð@ðÐ1ð ¸cð ð ð ð ð ð ð 7;ðTBð TBð ”|ðTBð # 5¤<°´Ð#=Ô>ð	TBð
  %œ|¨dÑ2ðTBð  %œ|ðTBð -¨tÑ3ðTBð Ð-Ô.ðTBð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	MðTBð TBð TBð TBð TBð TBð TBð TBrM   r¹   TÚsliding_windowr¾   c           
      ó^   ‡ ‡— dt           dt           dt           dt           dt          f
ˆˆ fd„}|S )zL
    This creates uni/bidirectional attention mask with sliding window.
    Ú	batch_idxÚhead_idxÚq_idxÚkv_idxr¾   c                 ó|   •— ‰	r‰
d}}n‰
dz   dz  ‰
dz  dz   }}||z
  }|dk    ||k     z  }|dk     | |k     z  }||z  S )Nr   rA   r$   rN   )rò   ró   rô   rõ   Úleft_window_sizeÚright_window_sizeÚdistÚ	left_maskÚ
right_maskrµ   rð   s            €€rJ   Ú
inner_maskz0sliding_window_mask_function.<locals>.inner_maskc  sy   ø€ Øð 	iØ2@À!Ð/ÐÐà4BÀQÑ4FÈ1Ñ3LÈ~ÐbcÑNcÐfgÑNgÐ/Ðà�v‰~ˆØ˜Q’Y 4Ð*:Ò#:Ñ;ˆ	Ø˜Q’h D 5Ð+<Ò#<Ñ=ˆ
Ø˜:Ñ%Ð%rM   )r”   rE   )rð   rµ   rü   s   `` rJ   Úsliding_window_mask_functionrý   ^  sR   øø€ ð
	&�cð 	&­Sð 	&½ð 	&Åcð 	&Ídð 	&ð 	&ð 	&ð 	&ð 	&ð 	&ð 	&ð ÐrM   c                   ó   — e Zd ZdS )ÚT5Gemma2EncoderLayerNr™   rN   rM   rJ   rÿ   rÿ   q  rš   rM   rÿ   c                   óÐ   ‡ — e Zd ZdZdefˆ fd„Z	 	 	 	 	 ddej        deej        ej        f         dej        dz  d	ej	        dz  d
e
dz  dedz  dej        dz  dej        fd„Zˆ xZS )ÚT5Gemma2DecoderLayerzFDecoder sub-layer: merged attention instead of vanilla self-attention.r±   c                 óx   •— t          ¦   «                              ||¦  «         t          ||¬¦  «        | _        d S )N)r�   r±   )r‡   r    r¹   Ú	self_attnr¶   s      €rJ   r    zT5Gemma2DecoderLayer.__init__x  s?   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+õ 1ØØð
ñ 
ô 
ˆŒˆˆrM   NFr«   r»   r¼   Úposition_idsro   Ú	use_cacher½   r¾   c                 ór  — |}	|                       |¦  «        } | j        d|||||||dœ|¤Ž\  }}
}
|                      |¦  «        }|	|                      |¦  «        z   }|}	|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|	|                      |¦  «        z   }|S )N)r«   r»   r¼   r  ro   r  r½   rN   )Úpre_self_attn_layernormr  Úpost_self_attn_layernormr£   Úpre_feedforward_layernormÚmlpÚpost_feedforward_layernorm)rV   r«   r»   r¼   r  ro   r  r½   rW   ÚresidualÚ_s              rJ   r¬   zT5Gemma2DecoderLayer.forward�  sâ   € ð !ˆØ×4Ò4°]ÑCÔCˆà,˜dœnð 	
Ø'Ø 3Ø"7Ø%Ø+ØØ"7ð	
ð 	
ð ð	
ð 	
Ñˆ�q˜!ð ×5Ò5°mÑDÔDˆØ  4§<¢<°Ñ#>Ô#>Ñ>ˆà ˆØ×6Ò6°}ÑEÔEˆØŸš Ñ/Ô/ˆØ×7Ò7¸ÑFÔFˆØ  4§<¢<°Ñ#>Ô#>Ñ>ˆØÐrM   )NNNFN)rZ   r[   r\   r]   r”   r    rÕ   rî   rï   Ú
LongTensorr	   rE   ÚFloatTensorr¬   r•   r–   s   @rJ   r  r  u  sô   ø€ € € € € ØPÐPð
¨#ð 
ð 
ð 
ð 
ð 
ð 
ð 6:Ø04Ø6:Ø!&Ø59ð ð  à”|ð ð # 5¤<°´Ð#=Ô>ð ð  %œ|¨dÑ2ð	 ð
 Ô&¨Ñ-ð ð -¨tÑ3ð ð ˜$‘;ð ð  %œ|¨dÑ2ð ð 
Ô	ð ð  ð  ð  ð  ð  ð  ð  rM   r  c                   ó   — e Zd ZdS )ÚT5Gemma2LMHeadNr™   rN   rM   rJ   r  r  ¤  rš   rM   r  c                   ó   — e Zd ZdS )ÚT5Gemma2ClassificationHeadNr™   rN   rM   rJ   r  r  ¨  rš   rM   r  c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚT5Gemma2MultiModalProjectorr�   c                 óJ   •— t          ¦   «                              |¦  «         d S rŸ   )r‡   r    r¤   s     €rJ   r    z$T5Gemma2MultiModalProjector.__init__­  s!   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ð Ð rM   )rZ   r[   r\   rb   r    r•   r–   s   @rJ   r  r  ¬  sE   ø€ € € € € ð!Ð4ð !ð !ð !ð !ð !ð !ð !ð !ð !ð !rM   r  c                   óX   ‡ — e Zd ZdZ	 	 ddededededef
ˆ fd	„Zd
ej        fˆ fd„Z	ˆ xZ
S )ÚT5Gemma2TextScaledWordEmbeddingzCT5Gemma2 Embedding: override to add eoi token embedding separately.ç      ð?é è Únum_embeddingsÚembedding_dimÚpadding_idxÚembed_scalers   c                 ó¾   •— t          ¦   «                              ||||¦  «         || _        t          j        t          j        | j        ¦  «        ¦  «        | _        d S rŸ   )	r‡   r    rs   r¡   Ú	ParameterrÕ   Úzerosr  Úeoi_embedding)rV   r  r  r  r  rs   r‰   s         €rJ   r    z(T5Gemma2TextScaledWordEmbedding.__init__´  sP   ø€ õ 	‰Œ×Ò˜¨¸À[ÑQÔQÐQØ.ˆÔÝœ\­%¬+°dÔ6HÑ*IÔ*IÑJÔJˆÔÐÐrM   Ú	input_idsc                 óê   •— t          ¦   «                              |¦  «        | j                             | j        j        ¦  «        z  }| j                             |j        ¦  «        ||| j        k    <   |S rŸ   )r‡   r¬   r  ÚtoÚweightÚdtyper"  rs   )rV   r#  Úinput_embeddingsr‰   s      €rJ   r¬   z'T5Gemma2TextScaledWordEmbedding.forwardÀ  sb   ø€ Ý ™7œ7Ÿ?š?¨9Ñ5Ô5¸Ô8H×8KÒ8KÈDÌKÔL]Ñ8^Ô8^Ñ^ÐØ>BÔ>P×>SÒ>SÐTdÔTjÑ>kÔ>kÐ˜ dÔ&:Ò:Ñ;ØÐrM   )r  r  )rZ   r[   r\   r]   r”   r“   r    rÕ   rî   r¬   r•   r–   s   @rJ   r  r  ±  s³   ø€ € € € € ØMÐMð !Ø&ð
Kð 
Kàð
Kð ð
Kð ð	
Kð
 ð
Kð ð
Kð 
Kð 
Kð 
Kð 
Kð 
Kð  ¤ð  ð  ð  ð  ð  ð  ð  ð  ð  ð  rM   r  c                   óR   — e Zd ZU eed<   dZdZdZdZddgZ	dZ
d„ Zd	ej        fd
„ZdS )ÚT5Gemma2PreTrainedModelr�   ÚmodelTFrÿ   r  Nc                 ó(  — t          j        | |¦  «         t          |t          ¦  «        rt	          j        |j        ¦  «         d S t          |t          ¦  «        r:t	          j        |j        ¦  «         t	          j	        |j
        |j        ¦  «         d S t          |t          ¦  «        r�|j        j        j        d         dz  }t	          j        |j        j        d| j        j        |z  ¬¦  «         t'          |j        d¦  «        r,|j        j        �"t	          j        |j        j        ¦  «         d S d S d S d|j        j        v rt	          j        |j        ¦  «         d S t          |t.          ¦  «        r›|j        D ]•}|j        }|j        |         dk    rt6          |j        |                  } ||j        |¬¦  «        \  }}t	          j        t;          ||› d	�¦  «        |¦  «         t	          j        t;          ||› d
�¦  «        |¦  «         Œ”d S d S )Nr   g      à¿rw   )ÚmeanÚstdÚbiasÚRMSNormÚdefault)Ú
layer_typeÚ	_inv_freqÚ_original_inv_freq)r   Ú_init_weightsr‚   r  ÚinitÚzeros_Úmm_input_projection_weightr  r"  Ú	constant_r  Úscalar_embed_scaler  Úout_projr&  rÄ   Únormal_r�   r{   Úhasattrr/  r‰   rZ   r®   rQ   Úcompute_default_rope_parametersÚ	rope_typer   Úcopy_r†   )rV   ÚmoduleÚscaler2  Úrope_init_fnÚcurr_inv_freqr  s          rJ   r5  z%T5Gemma2PreTrainedModel._init_weightsÙ  s"  € ÝÔ% d¨FÑ3Ô3Ð3Ý�fÕ9Ñ:Ô:ð 	^ÝŒK˜Ô9Ñ:Ô:Ð:Ð:Ð:Ý˜Õ ?Ñ@Ô@ð 	^ÝŒK˜Ô,Ñ-Ô-Ð-ÝŒN˜6Ô-¨vÔ/HÑIÔIÐIÐIÐIÝ˜Õ :Ñ;Ô;ð 	^Ø”OÔ*Ô0°Ô3°tÑ;ˆEÝŒL˜œÔ/°c¸t¼{Ô?\Ð_dÑ?dÐeÑeÔeÐeÝ�v”¨Ñ/Ô/ð 2°F´OÔ4HÐ4TÝ”˜FœOÔ0Ñ1Ô1Ð1Ð1Ð1ð2ð 2Ð4TÐ4Tð ˜&Ô*Ô3Ð3Ð3ÝŒK˜œÑ&Ô&Ð&Ð&Ð&Ý˜Õ 7Ñ8Ô8ð 	^Ø$Ô0ð ^ð ^�
Ø%ÔE�ØÔ# JÔ/°9Ò<Ð<Ý#6°vÔ7GÈ
Ô7SÔ#T�LØ#/ <°´È*Ð#UÑ#UÔ#UÑ �˜qÝ”
�7 6¨jÐ+CÐ+CÐ+CÑDÔDÀmÑTÔTÐTÝ”
�7 6¨jÐ+LÐ+LÐ+LÑMÔMÈ}Ñ]Ô]Ð]Ð]ð	^ð 	^ð^ð ^rM   Úlabelsc                 ó:  — | j         j        }|j        }|j        }|€t	          d¦  «        ‚|                     |j        ¦  «        }|ddd…f                              ¦   «         |ddd…f<   ||d<   |€t	          d¦  «        ‚|                     |dk    |¦  «         |S )	zú
        Shifts input_ids to the right, prepends the decoder_start_token_id, and handles
        pad_token_id replacement for labels that were -100.
        This is a common preparation step for decoder inputs in sequence-to-sequence models.
        Nz:self.model.config.decoder.bos_token_id has to be defined. .rÀ   rA   ).r   z9self.model.config.decoder.pad_token_id has to be defined.iœÿÿÿ)	r�   rq   r~   r   rŒ   Ú	new_zerosrÄ   ÚcloneÚmasked_fill_)rV   rE  Údecoder_configÚdecoder_start_token_idr   Úshifted_input_idss         rJ   Ú%prepare_decoder_input_ids_from_labelsz=T5Gemma2PreTrainedModel.prepare_decoder_input_ids_from_labelsñ  s¼   € ð œÔ,ˆØ!/Ô!<ÐØ%Ô2ˆà!Ð)ÝÐYÑZÔZÐZð #×,Ò,¨V¬\Ñ:Ô:ÐØ%+¨C°°"°¨HÔ%5×%;Ò%;Ñ%=Ô%=Ð˜#˜q˜r˜r˜'Ñ"Ø$:Ð˜&Ñ!àÐÝÐXÑYÔYÐYð 	×&Ò&Ð'8¸DÒ'@À,ÑOÔOÐOà Ð rM   )rZ   r[   r\   rm   r’   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_flex_attnÚ_no_split_modulesÚ_can_record_outputsr5  rÕ   rî   rM  rN   rM   rJ   r*  r*  Æ  sƒ   € € € € € € àÐÐÑØÐØ&*Ð#ð !ÐàÐð 	ØðÐð
 Ðð^ð ^ð ^ð0!¸E¼Lð !ð !ð !ð !ð !ð !rM   r*  c                   óþ   ‡ — e Zd ZU eed<   eedœZ	 ddedefˆ fd„Z	e
ee	 	 	 	 	 ddej        dz  dej        dz  d	ej        dz  d
ej        dz  dej        dz  dee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚT5Gemma2TextEncoderr�   )Ú
attentionsr«   r  rs   c                 ó$  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          ‰j        ‰j        | j        ‰j        dz  |¬¦  «        | _        t          ‰j        ‰j	        ¬¦  «        | _
        d| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          j        ‰j        ¦  «        | _        t)          ‰¦  «        | _        |                      ¦   «          d S )Nç      à?©r  rs   ©ÚepsFc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS rN   )rÿ   ©rG   r±   r�   s     €rJ   rK   z0T5Gemma2TextEncoder.__init__.<locals>.<listcomp>(  ó$   ø€ ÐfÐfÐf¸Õ! &¨)Ñ4Ô4ÐfÐfÐfrM   ©r‡   r    r   r  r�   r  r‹   Úembed_tokensr˜   Úrms_norm_epsÚnormÚgradient_checkpointingr¡   Ú
ModuleListrR   rS   rÒ   r¢   rx   r£   r®   Ú
rotary_embÚ	post_init©rV   r�   rs   r‰   s    ` €rJ   r    zT5Gemma2TextEncoder.__init__  s   øø€ õ
 	‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒå;ØÔØÔØÔØÔ*¨CÑ/Ø+ð
ñ 
ô 
ˆÔõ $ FÔ$6¸FÔ<OÐPÑPÔPˆŒ	Ø&+ˆÔ#å”mØfÐfÐfÐfÅeÈFÔLdÑFeÔFeÐfÑfÔfñ
ô 
ˆŒõ ”z &Ô"5Ñ6Ô6ˆŒÝ1°&Ñ9Ô9ˆŒð 	�ŠÑÔÐÐÐrM   Nr#  Úattention_maskr  Úinputs_embedsÚtoken_type_idsrW   r¾   c           
      óŽ  — |d u |d uz  rt          d¦  «        ‚|                     dd ¦  «         |€|                      |¦  «        }|€:t          j        d|j        d         |j        ¬¦  «                             d¦  «        }t          |x}t          ¦  «        sA| j
        ||dœ}t          di |¤Žt          di |¤dt          | j
        j        d¬	¦  «        i¤Žd
œ}|}	i }
t          | j
        j        ¦  «        D ]}|                      |	||¦  «        |
|<   Œ|                      |	¦  «        }	t%          | j        d | j
        j        …         ¦  «        D ]=\  }} ||	|
| j
        j        |                  || j
        j        |                  |fi |¤Ž}	Œ>|                      |	¦  «        }	|                      |	¦  «        }	t-          |	¬¦  «        S )Nú:You must specify exactly one of input_ids or inputs_embedsro   r   rA   ©Údevice)r�   ri  rh  Úand_mask_functionF)rµ   ©rC   rB   )Úlast_hidden_staterN   )rŒ   rP   r`  rÕ   ÚarangerÄ   rn  Ú	unsqueezer‚   rƒ   r�   r   rý   rð   ÚsetrQ   re  r£   Ú	enumeraterÒ   rS   rb  r   )rV   r#  rh  r  ri  rj  rW   Úself_attn_mask_mappingÚmask_kwargsr«   r»   r2  rH   Úlayer_modules                 rJ   r¬   zT5Gemma2TextEncoder.forward0  s5  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZð 	�
Š
Ð$ dÑ+Ô+Ð+àÐ Ø ×-Ò-¨iÑ8Ô8ˆMàÐÝ œ<¨¨=Ô+>¸qÔ+AÈ-ÔJ^Ð_Ñ_Ô_×iÒiÐjkÑlÔlˆLå°NÐBÐ0ÅDÑIÔIð 	àœ+Ø!.Ø"0ðð ˆKõ #<Ð"JÐ"J¸kÐ"JÐ"JÝ%>ð &ð &Ø!ð&ð &å&BÀ4Ä;ÔC]ÐinÐ&oÑ&oÔ&oð&ð &ð &ð&ð &Ð"ð &ˆð !ÐÝ˜dœkÔ5Ñ6Ô6ð 	gð 	gˆJØ.2¯oªo¸mÈ\Ð[eÑ.fÔ.fÐ 
Ñ+Ð+ð Ÿš ]Ñ3Ô3ˆå(¨¬Ð5T°t´{Ô7TÐ5TÔ)UÑVÔVð 	ð 	‰OˆAˆ|Ø(˜LØØ# D¤KÔ$;¸AÔ$>Ô?Ø& t¤{Ô'>¸qÔ'AÔBØð	ð ð
 ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆØŸš ]Ñ3Ô3ˆÝØ+ð
ñ 
ô 
ð 	
rM   ©r  )NNNNN)rZ   r[   r\   r9   r’   r°   rÿ   rS  r”   r    r!   r#   r   rÕ   r  rî   r  r   r   r   r¬   r•   r–   s   @rJ   rU  rU    s5  ø€ € € € € € ØÐÐÑà+Ø-ðð Ðð  'ðð à"ðð ðð ð ð ð ð ð8  ØØð .2Ø.2Ø04Ø26à.2ð<
ð <
àÔ# dÑ*ð<
ð œ tÑ+ð<
ð Ô&¨Ñ-ð	<
ð
 Ô(¨4Ñ/ð<
ð œ tÑ+ð<
ð Ð+Ô,ð<
ð 
ð<
ð <
ð <
ñ „^ñ „_ñ  Ôð<
ð <
ð <
ð <
ð <
rM   rU  c                   ó„  ‡ — e Zd ZU eed<   	 ddedefˆ fd„Zd„ Zd„ Ze	e
dej        dee         d	eez  fd
„¦   «         ¦   «         Zdej        dz  dej        dz  dej        fd„Ze
	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dee         d	efd„¦   «         Zˆ xZS )ÚT5Gemma2Encoderr�   r  rs   c                 ó$  •— t          ¦   «                              |¦  «         t                               |j        |¬¦  «        | _        t          j        |j        ¬¦  «        | _	        t          |¦  «        | _        |                      ¦   «          d S )N)rs   ©r�   )r‡   r    rU  Ú_from_configrd   Ú
text_modelr%   Úfrom_configre   Úvision_towerr  Úmulti_modal_projectorrf  rg  s      €rJ   r    zT5Gemma2Encoder.__init__u  s|   ø€ õ
 	‰Œ×Ò˜Ñ Ô Ð å-×:Ò:¸6Ô;MÐ_nÐ:ÑoÔoˆŒÝ%Ô1¸Ô9MÐNÑNÔNˆÔÝ%@ÀÑ%HÔ%HˆÔ"ð 	�ŠÑÔÐÐÐrM   c                 ó4   — | j                              ¦   «         S rŸ   )r  Úget_input_embeddingsr�   s    rJ   r„  z$T5Gemma2Encoder.get_input_embeddingsƒ  s   € ØŒ×3Ò3Ñ5Ô5Ð5rM   c                 ó6   — | j                              |¦  «        S rŸ   )r  Úset_input_embeddings©rV   Únew_embeddingss     rJ   r†  z$T5Gemma2Encoder.set_input_embeddings†  s   € ØŒ×3Ò3°NÑCÔCÐCrM   Úpixel_valuesrW   r¾   c                 ól   —  | j         d|ddœ|¤Ž}|j        }|                      |¦  «        }||_        |S )NT)r‰  Úreturn_dictrN   )r�  rq  r‚  Úpooler_output)rV   r‰  rW   Úvision_outputsrq  Úimage_featuress         rJ   Úget_image_featuresz"T5Gemma2Encoder.get_image_features‰  sP   € ð +˜Ô*Ða¸ÐRVÐaÐaÐZ`ÐaÐaˆØ*Ô<ÐØ×3Ò3Ð4EÑFÔFˆØ'5ˆÔ$àÐrM   r#  Nri  rŽ  c           	      ó4  — | j         j        }|€m|€t          d¦  «        ‚| |                      ¦   «         t	          j        d|t          j        |j        ¬¦  «        ¦  «        k    }|                     d¦  «        }n||k    }| 	                    ¦   «         }| 
                    d¦  «                             |j        ¦  «        }|j        d         |j        d         z  }t          ||j        d         z  |                     ¦   «         k    d|› d	|› �¦  «         |S )
zï
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        Nz9Either `input_ids` or `inputs_embeds` has to be provided.rN   )r'  rn  rÀ   r   rA   z6Image features and image tokens do not match: tokens: z, features )r�   rt   rŒ   r„  rÕ   ÚfullÚlongrn  ÚallÚsumrs  r%  rÄ   r    Únumel)rV   r#  ri  rŽ  rt   Úspecial_image_maskÚn_image_tokensÚn_image_featuress           rJ   Úget_image_placeholder_maskz*T5Gemma2Encoder.get_image_placeholder_mask—  s.  € ð œÔ3ˆØÐØÐ$Ý Ð!\Ñ]Ô]Ð]Ø!.Ð2M°$×2KÒ2KÑ2MÔ2MÝ”
˜2˜~µU´ZÈÔH\Ð]Ñ]Ô]ñ3ô 3ò "Ðð "4×!7Ò!7¸Ñ!;Ô!;ÐÐà!*¨nÒ!<Ðà+×/Ò/Ñ1Ô1ˆØ/×9Ò9¸"Ñ=Ô=×@Ò@ÀÔAUÑVÔVÐØ)Ô/°Ô2°^Ô5IÈ!Ô5LÑLÐÝØ˜]Ô0°Ô4Ñ4¸×8LÒ8LÑ8NÔ8NÒNØrÀ^ÐrÐrÐ`pÐrÐrñ	
ô 	
ð 	
ð "Ð!rM   rh  r  rj  c                 óh  — |d u |d uz  rt          d¦  «        ‚|€| j                             |¦  «        }|�j|                      |d¬¦  «        j        }|                     |j        |j        ¦  «        }|                      |||¬¦  «        }	| 	                    |	|¦  «        } | j        d|||dœ|¤Ž}
|
S )Nrl  T)r‹  )ri  rŽ  )ri  rh  r  rN   )
rŒ   r  r`  r�  rŒ  r%  rn  r'  r™  Úmasked_scatter)rV   r#  rh  r  ri  r‰  rj  rW   rŽ  Ú
image_maskÚoutputss              rJ   r¬   zT5Gemma2Encoder.forwardµ  sñ   € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø œO×8Ò8¸ÑCÔCˆMàÐ#Ø!×4Ò4°\ÈtÐ4ÑTÔTÔbˆNØ+×.Ò.¨}Ô/CÀ]ÔEXÑYÔYˆNà×8Ò8Ø¨À~ð 9ñ ô ˆJð *×8Ò8¸À^ÑTÔTˆMà!�$”/ð 
Ø'Ø)Ø%ð
ð 
ð ð	
ð 
ˆð ˆrM   ry  )NNNNNN)rZ   r[   r\   rb   r’   r”   r    r„  r†  r   r   rÕ   rî   r   r   rï   r   r�  r  r  r™  r   r¬   r•   r–   s   @rJ   r{  r{  r  s×  ø€ € € € € € Ø!Ð!Ð!Ñ!ð
  'ðð à%ðð ðð ð ð ð ð ð6ð 6ð 6ðDð Dð Dð Øð
Ø!œLð
Ø4:Ð;MÔ4Nð
à	Ð+Ñ	+ð
ð 
ð 
ñ „^ñ Ôð
ð"àÔ# dÑ*ð"ð Ô(¨4Ñ/ð"ð Ô)ð	"ð "ð "ð "ð< ð .2Ø.2Ø04Ø26Ø15à.2ð!ð !àÔ# dÑ*ð!ð œ tÑ+ð!ð Ô&¨Ñ-ð	!ð
 Ô(¨4Ñ/ð!ð Ô'¨$Ñ.ð!ð œ tÑ+ð!ð Ð+Ô,ð!ð 
ð!ð !ð !ñ „^ð!ð !ð !ð !ð !rM   r{  c                   óX  ‡ — e Zd ZU eed<    eed¬¦  «         eed¬¦  «        edœZddede	fˆ fd„Z
eee	 	 	 	 	 	 	 	 dd
ej        d	z  dej        d	z  dej        d	z  ded	z  dej        d	z  ded	z  dej        d	z  dej        d	z  dee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚT5Gemma2Decoderr�   rA   )Úindexr$   )rV  Úcross_attentionsr«   r  rs   c                 ó$  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          ‰j        ‰j        ‰j        ‰j        dz  |¬¦  «        | _        t          ‰j        ‰j	        ¬¦  «        | _
        d| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          j        ‰j        ¦  «        | _        t)          ‰¦  «        | _        |                      ¦   «          d S )NrX  rY  rZ  Fc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS rN   )r  r]  s     €rJ   rK   z,T5Gemma2Decoder.__init__.<locals>.<listcomp>ò  r^  rM   r_  rg  s    ` €rJ   r    zT5Gemma2Decoder.__init__â  sü   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒå;ØÔØÔØÔØÔ*¨CÑ/Ø+ð
ñ 
ô 
ˆÔõ $ FÔ$6¸FÔ<OÐPÑPÔPˆŒ	Ø&+ˆÔ#å”mØfÐfÐfÐfÅeÈFÔLdÑFeÔFeÐfÑfÔfñ
ô 
ˆŒõ ”z &Ô"5Ñ6Ô6ˆŒÝ1°&Ñ9Ô9ˆŒØ�ŠÑÔÐÐÐrM   Nr#  rh  r  ro   ri  r  r½   Úencoder_attention_maskrW   r¾   c	           
      ó  — |d u |d uz  rt          d¦  «        ‚|€t          d¦  «        ‚|€|                      |¦  «        }| j        s3|r1|€/t          t	          | j        ¬¦  «        t	          ¦   «         ¦  «        }|€V|�|                     ¦   «         nd}
t          j        |j	        d         |j
        ¬¦  «        |
z   }|                     d¦  «        }t          |x}t          ¦  «        s0| j        |||�|j        nd |ddœ}t          di |¤Žt!          di |¤Žd	œ}t          |x}t          ¦  «        sd
t#          | j        |||d¬¦  «        i}t          j        |d
         |d
         gd¬¦  «        t          j        |d         |d
         gd¬¦  «        d	œ}|}i }t'          | j        j        ¦  «        D ]}|                      |||¦  «        ||<   Œ|                      |¦  «        }t/          | j        d | j        j        …         ¦  «        D ]@\  }} |||| j        j        |                  || j        j        |                  ||||fi |	¤Ž}ŒA|                      |¦  «        }|                      |¦  «        }t7          ||¬¦  «        S )Nrl  z0`encoder_hidden_states` must be given in decoderr}  r   rA   rm  F)r�   ri  rh  ro   r  Úallow_is_causal_skiprp  rC   )r�   ri  rh  r½   Úallow_is_bidirectional_skiprÀ   rÁ   rB   )rq  ro   rN   )rŒ   r`  rÙ   r	   r   r�   Úget_seq_lengthrÕ   rr  rÄ   rn  rs  r‚   rƒ   rÍ   r0   r1   r   rÖ   rt  rQ   re  r£   ru  rÒ   rS   rb  r   )rV   r#  rh  r  ro   ri  r  r½   r¤  rW   Úpast_seen_tokensrv  rw  Úcross_attn_mask_mappingÚmerged_attn_mask_mappingr«   r»   r2  rH   rx  s                       rJ   r¬   zT5Gemma2Decoder.forwardø  s<  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZØ Ð(ÝÐOÑPÔPÐPàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàŒ}ð 	d ð 	d¨Ð/FÝ1µ,ÀdÄkÐ2RÑ2RÔ2RÕT`ÑTbÔTbÑcÔcˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå°NÐBÐ0ÅDÑIÔIð 	ð œ+Ø!.Ø"0ØKZÐKf ?Ô#GÐ#GÐlpØ ,Ø(-ðð ˆKõ #5Ð"CÐ"C°{Ð"CÐ"CÝ%FÐ%UÐ%UÈÐ%UÐ%Uð&ð &Ð"õ
 Ð5KÐKÐ1ÍTÑRÔRð 		à Õ";Øœ;Ø"/Ø#9Ø*?Ø05ð#ñ #ô #ð'Ð#õ $œiØ'Ð(8Ô9Ð;RÐScÔ;dÐeÐkmðñ ô õ "'¤Ø'Ð(;Ô<Ð>UÐVfÔ>gÐhÐnpð"ñ "ô "ð	$
ð $
Ð ð &ˆð !ÐÝ˜dœkÔ5Ñ6Ô6ð 	gð 	gˆJØ.2¯oªo¸mÈ\Ð[eÑ.fÔ.fÐ 
Ñ+Ð+ð Ÿš ]Ñ3Ô3ˆå(¨¬Ð5T°t´{Ô7TÐ5TÔ)UÑVÔVð 
	ð 
	‰OˆAˆ|Ø(˜LØØ# D¤KÔ$;¸AÔ$>Ô?Ø(¨¬Ô)@ÀÔ)CÔDØØØØ%ð	ð 	ð ð	ð 	ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆØŸš ]Ñ3Ô3ˆÝ8Ø+Ø+ð
ñ 
ô 
ð 	
rM   ry  )NNNNNNNN)rZ   r[   r\   rh   r’   r"   r¹   r  rS  r”   r    r!   r#   r   rÕ   r  rî   r	   r  rE   r   r   r   r¬   r•   r–   s   @rJ   rŸ  rŸ  Ú  sš  ø€ € € € € € Ø!Ð!Ð!Ñ!à$�nÐ%<ÀAÐFÑFÔFØ*˜NÐ+BÈ!ÐLÑLÔLØ-ðð Ððð Ð4ð Àsð ð ð ð ð ð ð,  ØØð .2Ø.2Ø04Ø6:Ø26Ø!%Ø59Ø6:ð\
ð \
àÔ# dÑ*ð\
ð œ tÑ+ð\
ð Ô&¨Ñ-ð	\
ð
 -¨tÑ3ð\
ð Ô(¨4Ñ/ð\
ð ˜$‘;ð\
ð  %œ|¨dÑ2ð\
ð !&¤¨tÑ 3ð\
ð Ð+Ô,ð\
ð 
3ð\
ð \
ð \
ñ „^ñ „_ñ  Ôð\
ð \
ð \
ð \
ð \
rM   rŸ  c                   ó|  ‡ — e Zd ZdddœZdefˆ fd„Zddedz  fd„Zd	„ Zd
„ Z	d„ Z
ee	 	 	 	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dedz  dedz  dej        dz  dej        dz  dedz  dee         defd„¦   «         ¦   «         Zˆ xZS )ÚT5Gemma2Modelz&encoder.text_model.embed_tokens.weightz-encoder.text_model.embed_tokens.eoi_embedding)zdecoder.embed_tokens.weightz"decoder.embed_tokens.eoi_embeddingr�   c                 óî   •— t          ¦   «                              |¦  «         t          |j        |j        ¦  «        | _        t          |j        |j        ¦  «        | _        |                      ¦   «          d S rŸ   )r‡   r    r{  rp   rs   rŸ  rq   rf  r¤   s     €rJ   r    zT5Gemma2Model.__init__a  s_   ø€ Ý‰Œ×Ò˜Ñ Ô Ð õ ' v¤~°vÔ7MÑNÔNˆŒÝ& v¤~°vÔ7MÑNÔNˆŒà�ŠÑÔÐÐÐrM   NÚmodalityc                 ó   — | j         S rŸ   )rp   ©rV   r¯  s     rJ   Úget_encoderzT5Gemma2Model.get_encoderj  ó
   € ØŒ|ÐrM   c                 ó   — | j         S rŸ   ©rq   r�   s    rJ   Úget_decoderzT5Gemma2Model.get_decoderm  r³  rM   c                 ó4   — | j                              ¦   «         S rŸ   )rp   r„  r�   s    rJ   r„  z"T5Gemma2Model.get_input_embeddingsp  s   € ØŒ|×0Ò0Ñ2Ô2Ð2rM   c                 ó6   — | j                              |¦  «        S rŸ   )rp   r†  r‡  s     rJ   r†  z"T5Gemma2Model.set_input_embeddingss  s   € ØŒ|×0Ò0°Ñ@Ô@Ð@rM   r#  r‰  rh  r  Údecoder_input_idsÚdecoder_attention_maskÚdecoder_position_idsÚencoder_outputsro   ri  Údecoder_inputs_embedsr  rW   r¾   c                 óè   — |€ | j         d||||
|ddœ|¤Ž}|j        } | j        d|||||	|||ddœ	|¤Ž}t          |j        |j        |j        |j        |j        |j        |j        |j        ¬¦  «        S )aX  
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
            Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
            config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
        NT)r#  rh  r  ri  r‰  r‹  )	r#  rh  r  ri  ro   r½   r¤  r  r‹  )rq  ro   Údecoder_hidden_statesÚdecoder_attentionsr¡  Úencoder_last_hidden_stater½   Úencoder_attentionsrN   )rp   rq  rq   r   ro   r«   rV  r¡  )rV   r#  r‰  rh  r  r¹  rº  r»  r¼  ro   ri  r½  r  rW   r½   Údecoder_outputss                   rJ   r¬   zT5Gemma2Model.forwardv  sÛ   € ð6 Ð"Ø*˜dœlð Ø#Ø-Ø)Ø+Ø)Ø ðð ð ðð ˆOð !0Ô AÐð '˜$œ,ð 
Ø'Ø1Ø-Ø/Ø+Ø"7Ø#1ØØð
ð 
ð ð
ð 
ˆõ "Ø-Ô?Ø+Ô;Ø"1Ô"?Ø.Ô9Ø,Ô=Ø&5Ô&GØ"1Ô"?Ø.Ô9ð	
ñ 	
ô 	
ð 		
rM   rŸ   )NNNNNNNNNNNN)rZ   r[   r\   Ú_tied_weights_keysrm   r    r‘   r²  r¶  r„  r†  r   r   rÕ   r  r  Ú
BoolTensorr   r	   rî   rE   r   r   r   r¬   r•   r–   s   @rJ   r­  r­  Z  sê  ø€ € € € € ð (PØ.]ðð Ðð
˜~ð ð ð ð ð ð ðð  C¨$¡Jð ð ð ð ðð ð ð3ð 3ð 3ðAð Að Að Øð .2Ø15Ø37Ø04à59Ø:>Ø8<à26Ø6:Ø-1Ø59Ø!%ð!=
ð =
ð Ô# dÑ*ð=
ð Ô'¨$Ñ.ð	=
ð
 Ô)¨DÑ0ð=
ð Ô&¨Ñ-ð=
ð !Ô+¨dÑ2ð=
ð !&Ô 0°4Ñ 7ð=
ð $Ô.°Ñ5ð=
ð )¨4Ñ/ð=
ð -¨tÑ3ð=
ð ”| dÑ*ð=
ð  %œ|¨dÑ2ð=
ð  ˜$‘;ð!=
ð" Ð+Ô,ð#=
ð$ 
ð%=
ð =
ð =
ñ „^ñ Ôð=
ð =
ð =
ð =
ð =
rM   r­  c            $       óp  ‡ — e Zd ZddiZddiZddgdgfiZdefˆ fd„Zd	„ Zd
„ Z	d„ Z
d„ Zd+dedz  fd„Zd„ Zeedej        dee         deez  fd„¦   «         ¦   «         Zed„ ¦   «         Zee	 	 	 	 	 	 	 	 	 	 	 	 	 	 d,dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dedz  dedz  dej        dz  d ej        dz  d!ej        dz  d"edz  d#e ej        z  dee         deej                 e!z  f d$„¦   «         ¦   «         Z"d%e#d&e$d'e%d(e d)e defˆ fd*„Z&ˆ xZ'S )-Ú T5Gemma2ForConditionalGenerationzlm_head.out_proj.weightz,model.encoder.text_model.embed_tokens.weightzlm_head.out_projÚcolwise_gather_outputr«   Úlogitsr�   c                 ó  •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        j        | _        t          |j        j        | j        ¦  «        | _        d| _	        |  
                    ¦   «          d S )NÚForMaskedLM)r‡   r    r­  r+  rq   r�   r  r‹   Úlm_headÚ	loss_typerf  r¤   s     €rJ   r    z)T5Gemma2ForConditionalGeneration.__init__¿  sk   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å" 6Ñ*Ô*ˆŒ
Ø œ.Ô3ˆŒÝ% f¤nÔ&@À$Ä/ÑRÔRˆŒØ&ˆŒà�ŠÑÔÐÐÐrM   c                 ó   — || j         _        d S rŸ   ©rÌ  r;  r‡  s     rJ   Úset_output_embeddingsz6T5Gemma2ForConditionalGeneration.set_output_embeddingsÉ  s   € Ø .ˆŒÔÐÐrM   c                 ó   — | j         j        S rŸ   rÏ  r�   s    rJ   Úget_output_embeddingsz6T5Gemma2ForConditionalGeneration.get_output_embeddingsÌ  s   € ØŒ|Ô$Ð$rM   c                 ó4   — | j                              ¦   «         S rŸ   ©r+  r„  r�   s    rJ   r„  z5T5Gemma2ForConditionalGeneration.get_input_embeddingsÏ  ó   € ØŒz×.Ò.Ñ0Ô0Ð0rM   c                 ó:   — | j                              |¦  «         d S rŸ   ©r+  r†  ©rV   Úvalues     rJ   r†  z5T5Gemma2ForConditionalGeneration.set_input_embeddingsÒ  ó   € ØŒ
×'Ò'¨Ñ.Ô.Ð.Ð.Ð.rM   Nr¯  c                 ó8   — | j                              |¬¦  «        S )N)r¯  )r+  r²  r±  s     rJ   r²  z,T5Gemma2ForConditionalGeneration.get_encoderÕ  s   € ØŒz×%Ò%¨xÐ%Ñ8Ô8Ð8rM   c                 ó4   — | j                              ¦   «         S rŸ   )r+  r¶  r�   s    rJ   r¶  z,T5Gemma2ForConditionalGeneration.get_decoderØ  s   € ØŒz×%Ò%Ñ'Ô'Ð'rM   r‰  rW   r¾   c                 óB   —  |                       ¦   «         j        |fi |¤ŽS rŸ   )r²  r�  )rV   r‰  rW   s      rJ   r�  z3T5Gemma2ForConditionalGeneration.get_image_featuresÛ  s+   € ð
 5ˆt×ÒÑ!Ô!Ô4°\ÐLÐLÀVÐLÐLÐLrM   c                 ó4   — |                       ¦   «         j        S rŸ   )r²  r�  r�   s    rJ   r�  z-T5Gemma2ForConditionalGeneration.vision_towerâ  s   € à×ÒÑ!Ô!Ô.Ð.rM   r   r#  rh  r  r¹  rº  r»  r¼  ro   ri  r½  rE  r  Úlogits_to_keepc                 ó,  — |�|€|€|                       |¦  «        } | j        d|||||||||	|
||dœ|¤Ž}|j        }t          |t          ¦  «        rt          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }| j        j        }|j	        �(||j	        z  }t          j        |¦  «        }||j	        z  }d}|� | j        ||| j        fi |¤Ž}t          |||j        |j        |j        |j        |j        |j        |j        ¬¦	  «	        S )aü  
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
            Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
            config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        N)r#  r‰  rh  r  r¹  rº  r»  r¼  ro   ri  r½  r  )	ÚlossrÉ  ro   r¿  rÀ  r¡  rÁ  r½   rÂ  rN   )rM  r+  rq  r‚   r”   ÚslicerÌ  r�   rq   Úfinal_logit_softcappingrÕ   ÚtanhÚloss_functionr�   r   ro   r¿  rÀ  r¡  rÁ  r½   rÂ  )rV   r#  r‰  rh  r  r¹  rº  r»  r¼  ro   ri  r½  rE  r  rß  rW   rÃ  r«   Úslice_indicesrÉ  rJ  rá  s                         rJ   r¬   z(T5Gemma2ForConditionalGeneration.forwardæ  sz  € ðB ÐÐ"3Ð";Ð@UÐ@]à $× JÒ JÈ6Ñ RÔ RÐà.8¨d¬jð /
ØØ%Ø)Ø%Ø/Ø#9Ø!5Ø+Ø+Ø'Ø"7Øð/
ð /
ð ð/
ð /
ˆð  (Ô9ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàœÔ,ˆØÔ1Ð=Ø˜nÔDÑDˆFÝ”Z Ñ'Ô'ˆFØ˜nÔDÑDˆFàˆØÐà%�4Ô% f¨f°d´oÐPÐPÈÐPÐPˆDåØØØ+Ô;Ø"1Ô"GØ.ÔAØ,Ô=Ø&5Ô&OØ"1Ô"GØ.ÔAð

ñ 

ô 

ð 
	
rM   Úgeneration_configÚmodel_kwargsÚgeneration_modeÚ
batch_sizeÚmax_cache_lengthc           	      óÀ  •— t          ¦   «                              |||||¦  «         |j        du rdS |j        }|€d}n	d|j        v }t	          j        | j                             d¬¦  «        ¦  «        }d|_        dg|j	        z  |_
        ||dœ}	|                     d¦  «        }
|
�¡t          |
t          ¦  «        st          d	¦  «        ‚t          |
j        ¦  «        d
k    r|
j                             d
¦  «        rdS t#          |
j        ¦  «        }|t&          k    r|d         d
         j        d         |	d<    |di |	¤Ž|
_        nEt          t+          di | j                             d¬¦  «        |dœ¤Žt+          ¦   «         ¦  «        |d<   t-          | d¦  «        r?| j        �:t          | j        t          ¦  «        st          d¦  «        ‚|d         | _        dS dS dS )zMOverride cache preparation to support T5Gemma2-specific EncoderDecoder Cache.FNÚ	offloadedTrµ  rC   )r�   Ú
offloadingro   zaThe `past_key_values` in `model_kwargs` must be of type `EncoderDecoderCache` for T5Gemma2 model.r   r¼  rA   Úmax_cache_lenÚ_cachezLThe internal cache must be of type `EncoderDecoderCache` for T5Gemma2 model.rN   )r‡   Ú_prepare_cache_for_generationr  Úcache_implementationÚcopyÚdeepcopyr�   Úget_text_configrð   rS   rQ   rÐ   r‚   r	   rŒ   ÚlenrÏ   ÚtyperÑ   r
   rÄ   r   r=  rð  )rV   rç  rè  ré  rê  rë  rò  Úoffload_cacheÚcross_attn_configÚcross_attn_cache_kwargsro   Úcross_attn_clsr‰   s               €rJ   rñ  z>T5Gemma2ForConditionalGeneration._prepare_cache_for_generation7  sN  ø€ õ 	‰Œ×-Ò-ØØØØØñ	
ô 	
ð 	
ð Ô&¨%Ð/Ð/ØˆFà0ÔEÐØÐ'Ø!ˆMˆMà'Ð+<Ô+QÐQˆMõ !œM¨$¬+×*EÒ*EÈdÐ*EÑ*SÔ*SÑTÔTÐð ,0ÐÔ(Ø)9Ð(:Ð=NÔ=`Ñ(`ÐÔ%ð (Ø'ð#
ð #
Ðð
 '×*Ò*Ð+<Ñ=Ô=ˆØÐ&Ý˜oÕ/BÑCÔCð Ý Øwñô ð õ
 �?Ô-Ñ.Ô.°Ò2Ð2°Ô7Q×7UÒ7UÐVWÑ7XÔ7XÐ2Ø�å! /Ô"GÑHÔHˆNØ¥Ò,Ð,Ø;GÐHYÔ;ZÐ[\Ô;]Ô;cÐdeÔ;fÐ'¨Ñ8à4B°NÐ4]Ð4]ÐE\Ð4]Ð4]ˆOÔ1Ð1õ /BÝð ð à"&¤+×"=Ò"=ÀdÐ"=Ñ"KÔ"KØ&3ðð ðð õ ‘”ñ/ô /ˆLÐ*Ñ+õ �4˜Ñ"Ô"ð 	: t¤{Ð'>Ý˜dœkÕ+>Ñ?Ô?ð qÝ Ð!oÑpÔpÐpà&Ð'8Ô9ˆDŒKˆKˆKð		:ð 	:Ð'>Ð'>rM   rŸ   )NNNNNNNNNNNNNr   )(rZ   r[   r\   rÄ  Ú_tp_planÚ_pp_planrm   r    rÐ  rÒ  r„  r†  r‘   r²  r¶  r   r   rÕ   rî   r   r   rï   r   r�  Úpropertyr�  r  r  rÅ  r   r	   rE   r”   r   r¬   r   rƒ   r   rñ  r•   r–   s   @rJ   rÇ  rÇ  ¸  sG  ø€ € € € € à!Ð#QðÐð #Ð$;Ð<€HØ" oÐ%6¸¸
Ð$CÐD€Hð˜~ð ð ð ð ð ð ð/ð /ð /ð%ð %ð %ð1ð 1ð 1ð/ð /ð /ð9ð 9 C¨$¡Jð 9ð 9ð 9ð 9ð(ð (ð (ð ØðMØ!œLðMØ4:Ð;MÔ4NðMà	Ð+Ñ	+ðMð Mð Mñ „^ñ ÔðMð
 ð/ð /ñ „Xð/ð Øð .2Ø15Ø37Ø04à59Ø:>Ø8<à26Ø6:Ø26Ø:>Ø*.Ø!%Ø-.ð%M
ð M
ð Ô# dÑ*ðM
ð Ô'¨$Ñ.ð	M
ð
 Ô)¨DÑ0ðM
ð Ô&¨Ñ-ðM
ð !Ô+¨dÑ2ðM
ð !&Ô 0°4Ñ 7ðM
ð $Ô.°Ñ5ðM
ð )¨4Ñ/ðM
ð -¨tÑ3ðM
ð Ô(¨4Ñ/ðM
ð  %Ô0°4Ñ7ðM
ð  Ô  4Ñ'ð!M
ð" ˜$‘;ð#M
ð$ ˜eœlÑ*ð%M
ð& Ð+Ô,ð'M
ð( 
ˆuÔ Ô	! OÑ	3ð)M
ð M
ð M
ñ „^ñ ÔðM
ð^I:à+ðI:ð ðI:ð (ð	I:ð
 ðI:ð ðI:ð 
ðI:ð I:ð I:ð I:ð I:ð I:ð I:ð I:ð I:ð I:rM   rÇ  c                   óV  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 	 	 	 dde	j
        dz  de	j        dz  de	j        dz  d	e	j
        dz  d
e	j
        dz  de	j        dz  de	j
        dz  dedz  de	j        dz  de	j        dz  de	j
        dz  dee         defd„¦   «         ¦   «         Zˆ xZS )Ú!T5Gemma2ForSequenceClassificationr�   c                 ó6  •— t          ¦   «                              |¦  «         |j        | _        |j        j        | _        t          |¦  «        | _        t          |dd¦  «        }t          | j        | j        |¦  «        | _	        |  
                    ¦   «          d S ©Nrz   gš™™™™™¹?©r‡   r    Ú
num_labelsrq   r‹   r­  r+  r†   r  Úscorerf  ©rV   r�   Úclassifier_dropoutr‰   s      €rJ   r    z*T5Gemma2ForSequenceClassification.__init__…  sƒ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒØ!œ>Ô5ˆÔå" 6Ñ*Ô*ˆŒ
å$ VÐ-FÈÑLÔLÐÝ/°Ô0@À$Ä/ÐSeÑfÔfˆŒ
Ø�ŠÑÔÐÐÐrM   c                 ó4   — | j                              ¦   «         S rŸ   rÔ  r�   s    rJ   r„  z6T5Gemma2ForSequenceClassification.get_input_embeddings�  rÕ  rM   c                 ó:   — | j                              |¦  «         d S rŸ   r×  rØ  s     rJ   r†  z6T5Gemma2ForSequenceClassification.set_input_embeddings“  rÚ  rM   Nr#  r‰  rh  r  r¹  rº  r»  r¼  ri  r½  rE  rW   r¾   c                 ó$  — |	€|
�t          d| j        j        › d�¦  «        ‚|€t          d¦  «        ‚|€|                      |¦  «        } | j        |f||||||||	|
ddœ
|¤Ž}|j        }|j        }|j        }|  	                    |¦  «        }|j
        d         }|| j        j        k                         |j        t          j        ¦  «        }t          j        |j
        d         |j        t          j        ¬	¦  «        }||z                       d¦  «        }t          j        ||j
        d         d
z
  ¬¦  «        }|t          j        ||j        ¬¦  «        |f         }d}|�|                      |||| j        ¬¦  «        }t+          ||||¬¦  «        S )áÜ  
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
            Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
            config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nú8Passing input embeddings is currently not supported for ú.úYou have to specify input_idsF©
r‰  rh  r  r¹  rº  r»  r¼  ri  r½  r  r   rÀ   )rn  r'  rA   )Úmaxrm  )rÉ  rE  Úpooled_logitsr�   ©rá  rÉ  r«   rV  )ÚNotImplementedErrorr‰   rZ   rŒ   rM  r+  rq  r¿  rÀ  r  rÄ   r�   r   r%  rn  rÕ   Úint32rr  ÚargmaxÚclamprå  r   )rV   r#  r‰  rh  r  r¹  rº  r»  r¼  ri  r½  rE  rW   r�  rq  r«   rV  rÉ  rê  Únon_pad_maskÚtoken_indicesÚlast_non_pad_tokenr  rá  s                           rJ   r¬   z)T5Gemma2ForSequenceClassification.forward–  sà  € ð4 Ð$Ð(=Ð(IÝ%ØeÈ4Ì>ÔKbÐeÐeÐeñô ð ð ÐÝÐ<Ñ=Ô=Ð=àÐ$Ø $× JÒ JÈ9Ñ UÔ UÐà&0 d¤jØð'
à%Ø)Ø%Ø/Ø#9Ø!5Ø+Ø'Ø"7Øð'
ð '
ð ð'
ð '
ˆð $Ô5ÐØÔ5ˆØÔ/ˆ
à—’Ð-Ñ.Ô.ˆà”_ QÔ'ˆ
à)¨T¬[Ô-EÒE×IÒIÈ&Ì-ÕY^ÔYdÑeÔeˆÝœÐ%6Ô%<¸RÔ%@ÈÌÕ^cÔ^iÐjÑjÔjˆØ+¨lÑ:×BÒBÀ2ÑFÔFÐÝ"œ[Ð);ÐARÔAXÐY[ÔA\Ð_`ÑA`ÐaÑaÔaÐà�uœ|¨J¸v¼}ÐMÑMÔMÐOaÐaÔbˆàˆØÐØ×%Ò%¨V¸FÐR_ÐhlÔhsÐ%ÑtÔtˆDå'ØØ Ø'Ø!ð	
ñ 
ô 
ð 	
rM   ©NNNNNNNNNNN)rZ   r[   r\   rm   r    r„  r†  r   r   rÕ   r  r  rî   r   r   r   r   r¬   r•   r–   s   @rJ   r   r   ƒ  s¤  ø€ € € € € ð	˜~ð 	ð 	ð 	ð 	ð 	ð 	ð1ð 1ð 1ð/ð /ð /ð Øð .2Ø15Ø.2Ø04Ø59Ø6:Ø8<Ø26Ø26Ø:>Ø*.ðJ
ð J
àÔ# dÑ*ðJ
ð Ô'¨$Ñ.ðJ
ð œ tÑ+ð	J
ð
 Ô&¨Ñ-ðJ
ð !Ô+¨dÑ2ðJ
ð !&¤¨tÑ 3ðJ
ð $Ô.°Ñ5ðJ
ð )¨4Ñ/ðJ
ð Ô(¨4Ñ/ðJ
ð  %Ô0°4Ñ7ðJ
ð Ô  4Ñ'ðJ
ð Ð+Ô,ðJ
ð 
"ðJ
ð J
ð J
ñ „^ñ ÔðJ
ð J
ð J
ð J
ð J
rM   r   c                   óV  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 	 	 	 dde	j
        dz  de	j        dz  de	j        dz  d	e	j
        dz  d
e	j
        dz  de	j        dz  de	j
        dz  dedz  de	j        dz  de	j        dz  de	j
        dz  dee         defd„¦   «         ¦   «         Zˆ xZS )ÚT5Gemma2ForTokenClassificationr�   c                 ó6  •— t          ¦   «                              |¦  «         |j        | _        |j        j        | _        t          |¦  «        | _        t          |dd¦  «        }t          | j        | j        |¦  «        | _	        |  
                    ¦   «          d S r  r  r  s      €rJ   r    z'T5Gemma2ForTokenClassification.__init__ç  sƒ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒØ!œ>Ô5ˆÔå" 6Ñ*Ô*ˆŒ
å$ VÐ-FÈÑLÔLÐÝ/°Ô0@À$Ä/ÐSeÑfÔfˆŒ
à�ŠÑÔÐÐÐrM   c                 ó4   — | j                              ¦   «         S rŸ   rÔ  r�   s    rJ   r„  z3T5Gemma2ForTokenClassification.get_input_embeddingsó  rÕ  rM   c                 ó:   — | j                              |¦  «         d S rŸ   r×  rØ  s     rJ   r†  z3T5Gemma2ForTokenClassification.set_input_embeddingsö  rÚ  rM   Nr#  r‰  rh  r  r¹  rº  r»  r¼  ri  r½  rE  rW   r¾   c                 ó€  — |	€|
�t          d| j        j        › d�¦  «        ‚|€t          d¦  «        ‚|€|                      |¦  «        } | j        |f||||||||	|
ddœ
|¤Ž}|j        }|j        }|j        }|  	                    |¦  «        }d}|�|  
                    ||| j        ¦  «        }t          ||||¬¦  «        S )r  Nr  r  r  Fr  r  )r  r‰   rZ   rŒ   rM  r+  rq  r¿  rÀ  r  rå  r�   r   )rV   r#  r‰  rh  r  r¹  rº  r»  r¼  ri  r½  rE  rW   r�  rq  r«   rV  rÉ  rá  s                      rJ   r¬   z&T5Gemma2ForTokenClassification.forwardù  s"  € ð4 Ð$Ð(=Ð(IÝ%ØeÈ4Ì>ÔKbÐeÐeÐeñô ð ð ÐÝÐ<Ñ=Ô=Ð=àÐ$Ø $× JÒ JÈ9Ñ UÔ UÐà&0 d¤jØð'
à%Ø)Ø%Ø/Ø#9Ø!5Ø+Ø'Ø"7Øð'
ð '
ð ð'
ð '
ˆð $Ô5ÐØÔ5ˆØÔ/ˆ
à—’Ð-Ñ.Ô.ˆàˆØÐØ×%Ò% f¨f°d´kÑBÔBˆDå$ØØØ'Ø!ð	
ñ 
ô 
ð 	
rM   r  )rZ   r[   r\   rm   r    r„  r†  r   r   rÕ   r  r  rî   r   r   r   r   r¬   r•   r–   s   @rJ   r  r  å  s¤  ø€ € € € € ð
˜~ð 
ð 
ð 
ð 
ð 
ð 
ð1ð 1ð 1ð/ð /ð /ð Øð .2Ø15Ø.2Ø04Ø59Ø6:Ø8<Ø26Ø26Ø:>Ø*.ð@
ð @
àÔ# dÑ*ð@
ð Ô'¨$Ñ.ð@
ð œ tÑ+ð	@
ð
 Ô&¨Ñ-ð@
ð !Ô+¨dÑ2ð@
ð !&¤¨tÑ 3ð@
ð $Ô.°Ñ5ð@
ð )¨4Ñ/ð@
ð Ô(¨4Ñ/ð@
ð  %Ô0°4Ñ7ð@
ð Ô  4Ñ'ð@
ð Ð+Ô,ð@
ð 
ð@
ð @
ð @
ñ „^ñ Ôð@
ð @
ð @
ð @
ð @
rM   r  )
rm   r9   rb   rh   rÇ  r­  r{  r*  r   r  )T)iró  Úcollections.abcr   Útypingr   rÕ   Útorch.nnr¡   Úhuggingface_hub.dataclassesr   Ú r   r6  Úcache_utilsr   r	   r
   Úconfiguration_utilsr   Ú
generationr   r   r   Úmasking_utilsr   Úmodeling_flash_attention_utilsr   Úmodeling_outputsr   r   r   r   r   r   r   Úmodeling_rope_utilsr   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r    Úutils.genericr!   Úutils.output_capturingr"   r#   Úautor%   Úgemma3.configuration_gemma3r&   r'   Úgemma3.modeling_gemma3r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   Úsiglipr3   Út5gemma.modeling_t5gemmar4   r5   r6   Ú
get_loggerrZ   r„   r9   rb   rh   rm   r˜   rœ   r®   r°   r¹   r”   rý   rÿ   r  r  r  r  r  r*  rU  r{  rŸ  r­  rÇ  r   r  Ú__all__rN   rM   rJ   ú<module>r9     s7  ðð €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð Ø .Ð .Ð .Ð .Ð .Ð .à &Ð &Ð &Ð &Ð &Ð &Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø KÐ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø BÐ BÐ BÐ BÐ BÐ Bðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð 7Ð 6Ð 6Ð 6Ð 6Ð 6Ø FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &ðð ð ð ð ð ð ð ð ð ð ð ð ð ð 8Ð 7Ð 7Ð 7Ð 7Ð 7Ø EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ Ð Ð Ð Ð Ð Ø HÐ HÐ HÐ HÐ HÐ HÐ HÐ Hðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð (Ð 'Ð 'Ð 'Ð 'Ð 'ðð ð ð ð ð ð ð ð ð ð 
ˆÔ	˜HÑ	%Ô	%€ð €Ð7Ð8Ñ8Ô8Øð1ð 1ð 1ð 1ð 1Ð)Ð+;ñ 1ô 1ñ „ñ 9Ô8ð1ð2 €Ð7Ð8Ñ8Ô8Øðð ð ð ð ˜Lñ ô ñ „ñ 9Ô8ðð €Ð7Ð8Ñ8Ô8Øð1ð 1ð 1ð 1ð 1Ð,Ð.>ñ 1ô 1ñ „ñ 9Ô8ð1ð2 €Ð7Ð8Ñ8Ô8ØðTð Tð Tð Tð TÐ%ñ Tô Tñ „ñ 9Ô8ðTðn	ð 	ð 	ð 	ð 	�mñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	�)ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð3ñ 	ô 	ð 	ðð ð ð ð ˜Oñ ô ð ð[Bð [Bð [Bð [Bð [B˜oñ [Bô [Bð [Bð|ð °ð Èð ð ð ð ð&	ð 	ð 	ð 	ð 	Ð.ñ 	ô 	ð 	ð,ð ,ð ,ð ,ð ,Ð.ñ ,ô ,ð ,ð^	ð 	ð 	ð 	ð 	�]ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð!:ñ 	ô 	ð 	ð!ð !ð !ð !ð !Ð";ñ !ô !ð !ð
 ð  ð  ð  ð  Ð&Cñ  ô  ð  ð* ðC!ð C!ð C!ð C!ð C!Ð3ñ C!ô C!ñ „ðC!ðLb
ð b
ð b
ð b
ð b
Ð1ñ b
ô b
ð b
ðJeð eð eð eð eÐ-ñ eô eð eðP}
ð }
ð }
ð }
ð }
Ð-ñ }
ô }
ð }
ð@ ðZ
ð Z
ð Z
ð Z
ð Z
Ð+ñ Z
ô Z
ñ „ðZ
ðzH:ð H:ð H:ð H:ð H:Ð'>Àñ H:ô H:ð H:ðV ð^
ð ^
ð ^
ð ^
ð ^
Ð(?ñ ^
ô ^
ñ „ð^
ðB ðU
ð U
ð U
ð U
ð U
Ð%<ñ U
ô U
ñ „ðU
ðpð ð €€€rM   