§
    9²¸jN ã                   ón  — d dl Z d dlmZ d dlZd dlmZ ddlmZ ddl	m
Z
 ddlmZmZmZmZ ddlmZmZmZ ddlmZmZ dd	lmZmZmZ dd
lmZ ddlmZ ddlm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z& ddl'm(Z(m)Z) ddl*m+Z+m,Z, ddl-m.Z. ddl/m0Z0m1Z1m2Z2m3Z3 ddl4m5Z5 ddl6m7Z7m8Z8 ddl9m:Z:m;Z; ddl<m=Z= ddl>m?Z?m@Z@mAZAmBZB  G d„ dejC        ¦  «        ZD G d„ dejC        ¦  «        ZE G d„ dejC        ¦  «        ZFd„ ZG ed¦  «        dSd „¦   «         ZHd!ejI        d"eJd#ejI        fd$„ZK	 	 	 dTd&ejC        d'ejI        d(ejI        d)ejI        d*ejI        dz  d+eLeJz  d,eLdz  d-eLdz  d#eMejI        ejI        f         fd.„ZN eeH¦  «         G d/„ d0ejC        ¦  «        ¦   «         ZO eeH¦  «         G d1„ d2ejC        ¦  «        ¦   «         ZP G d3„ d4e¦  «        ZQ G d5„ d6e¦  «        ZR G d7„ d8ejC        ¦  «        ZS G d9„ d:ejC        ¦  «        ZT G d;„ d<ejC        ¦  «        ZU G d=„ d>ejV        ¦  «        ZWe1 G d?„ d@e,¦  «        ¦   «         ZXdUdBeJd#efdC„ZY G dD„ dEeX¦  «        ZZ G dF„ dGeX¦  «        Z[ G dH„ dIeX¦  «        Z\e1 G dJ„ dKeX¦  «        ¦   «         Z] G dL„ dMeXe¦  «        Z^e1 G dN„ dOeX¦  «        ¦   «         Z_e1 G dP„ dQeX¦  «        ¦   «         Z`g dR¢ZadS )Vé    N)ÚCallableé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCacheÚStaticCache)ÚGenerationConfigÚGenerationMixinÚGenerationMode)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_bidirectional_maskÚcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚBaseModelOutputWithPoolingÚSeq2SeqLMOutputÚSeq2SeqModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚtorch_compilable_check)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )Ú	AutoModelé   )ÚT5Gemma2ConfigÚT5Gemma2DecoderConfigÚT5Gemma2EncoderConfigÚT5Gemma2TextConfigc                   ó<   ‡ — e Zd Zddedefˆ fd„Zd„ Zd„ Zd„ Zˆ xZ	S )	ÚT5Gemma2RMSNormç�íµ ÷Æ°>ÚdimÚepsc                 ó¬   •— t          ¦   «                              ¦   «          || _        t          j        t          j        |¦  «        ¦  «        | _        d S ©N)ÚsuperÚ__init__r5   ÚnnÚ	ParameterÚtorchÚzerosÚweight)Úselfr4   r5   Ú	__class__s      €ú~/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/t5gemma2/modeling_t5gemma2.pyr9   zT5Gemma2RMSNorm.__init__8   s?   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”l¥5¤;¨sÑ#3Ô#3Ñ4Ô4ˆŒˆˆó    c                 ó�   — |t          j        |                     d¦  «                             dd¬¦  «        | j        z   ¦  «        z  S )Nr*   éÿÿÿÿT)Úkeepdim)r<   ÚrsqrtÚpowÚmeanr5   )r?   Úxs     rA   Ú_normzT5Gemma2RMSNorm._norm=   s8   € Ø•5”;˜qŸušu Q™xœxŸ}š}¨R¸˜}Ñ>Ô>ÀÄÑIÑJÔJÑJÐJrB   c                 ó¸   — |                       |                     ¦   «         ¦  «        }|d| j                             ¦   «         z   z  }|                     |¦  «        S )Nç      ð?)rJ   Úfloatr>   Útype_as)r?   rI   Úoutputs      rA   ÚforwardzT5Gemma2RMSNorm.forward@   sL   € Ø—’˜AŸGšG™IœIÑ&Ô&ˆð ˜3 ¤×!2Ò!2Ñ!4Ô!4Ñ4Ñ5ˆØ�~Š~˜aÑ Ô Ð rB   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)Útupler>   Úshaper5   ©r?   s    rA   Ú
extra_reprzT5Gemma2RMSNorm.extra_reprG   s%   € Ý˜œÔ)Ñ*Ô*Ð<Ð<°$´(Ð<Ð<Ð<rB   )r3   )
Ú__name__Ú
__module__Ú__qualname__ÚintrM   r9   rJ   rP   rU   Ú__classcell__©r@   s   @rA   r2   r2   7   s€   ø€ € € € € ð5ð 5˜Cð 5 eð 5ð 5ð 5ð 5ð 5ð 5ð
Kð Kð Kð!ð !ð !ð=ð =ð =ð =ð =ð =ð =rB   r2   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚT5Gemma2MLPÚconfigc                 óÔ  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _	        t          |j                 | _        t          j        |j        ¦  «        | _        d S )NF©Úbias)r8   r9   r^   Úhidden_sizeÚintermediate_sizer:   ÚLinearÚ	gate_projÚup_projÚ	down_projr   Úhidden_activationÚact_fnÚDropoutÚdropout_rateÚdropout©r?   r^   r@   s     €rA   r9   zT5Gemma2MLP.__init__L   sº   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ!'Ô!9ˆÔÝœ 4Ô#3°TÔ5KÐRWÐXÑXÔXˆŒÝ”y Ô!1°4Ô3IÐPUÐVÑVÔVˆŒÝœ 4Ô#9¸4Ô;KÐRWÐXÑXÔXˆŒÝ˜VÔ5Ô6ˆŒÝ”z &Ô"5Ñ6Ô6ˆŒˆˆrB   c                 óÖ   — |                       |                      |¦  «        ¦  «        |                      |¦  «        z  }|                      |¦  «        }|                      |¦  «        }|S r7   )ri   re   rf   rl   rg   )r?   rI   Úhidden_statesrg   s       rA   rP   zT5Gemma2MLP.forwardW   sV   € ØŸš D§N¢N°1Ñ$5Ô$5Ñ6Ô6¸¿ºÀa¹¼ÑHˆØŸš ]Ñ3Ô3ˆØ—N’N =Ñ1Ô1ˆ	ØÐrB   )rV   rW   rX   r0   r9   rP   rZ   r[   s   @rA   r]   r]   K   sT   ø€ € € € € ð	7Ð1ð 	7ð 	7ð 	7ð 	7ð 	7ð 	7ðð ð ð ð ð ð rB   r]   c                   óú   ‡ — e Zd Z edd¬¦  «        ddefˆ fd„¦   «         Ze edd¬¦  «        	 ddededz  dee	j
        ef         fd	„¦   «         ¦   «         Z e	j        ¦   «         ed
„ ¦   «         ¦   «         Zˆ xZS )ÚT5Gemma2RotaryEmbeddingÚdevicez5.18)ÚversionNr^   c           	      ó¶  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        t          t          |j        ¦  «        ¦  «        | _        i | _	        | j        D ]ã}| j        j
        |         }|€Œ|d         | j	        |<   | j        }| j	        |         dk    rt          | j	        |                  } || j        ||¬¦  «        \  }}t          | |› d�t          j        |d¬¦  «        ¦  «         t          | |› d�t          j        |                     ¦   «         d¬¦  «        ¦  «         t          | |› d�|¦  «         Œäd S )	NÚ	rope_typeÚdefault©Ú
layer_typeÚ	_inv_freqF©Ú
persistentÚ_original_inv_freqÚ_attention_scaling)r8   r9   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr^   ÚsortedÚsetÚlayer_typesru   Úrope_parametersÚcompute_default_rope_parametersr   Úsetattrr:   ÚBufferÚclone)	r?   r^   rr   rx   Úrope_paramsÚrope_init_fnÚcurr_inv_freqÚcurr_attention_scalingr@   s	           €rA   r9   z T5Gemma2RotaryEmbedding.__init___   sl  ø€ å‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!ØˆŒÝ!¥# fÔ&8Ñ"9Ô"9Ñ:Ô:ˆÔØˆŒØÔ*ð 	Uð 	UˆJØœ+Ô5°jÔAˆKØÐ"Øà)4°[Ô)AˆDŒN˜:Ñ&Ø%)Ô%IˆLØŒ~˜jÔ)¨YÒ6Ð6Ý2°4´>À*Ô3MÔN�Ø4@°LÀÄÈfÐakÐ4lÑ4lÔ4lÑ1ˆMÐ1Ý�D˜ZÐ2Ð2Ð2µB´I¸mÐX]Ð4^Ñ4^Ô4^Ñ_Ô_Ð_Ý�D˜ZÐ;Ð;Ð;½R¼YÀ}×GZÒGZÑG\ÔG\ÐinÐ=oÑ=oÔ=oÑpÔpÐpÝ�D˜ZÐ;Ð;Ð;Ð=SÑTÔTÐTÐTð	Uð 	UrB   rx   Úreturnc                 óð   — | j         |         d         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «        |z  z  z  }|                     |¦  «        |fS )a†  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            layer_type (`str`):
                The current layer type if the model has different RoPE parameters per type.
                Should not be used unless `config.layer_types is not None`

        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNrL   r   r*   ©Údtype)r„   Úgetattrrb   Únum_attention_headsr<   ÚarangerM   Úto)r^   rr   rx   ÚkwargsÚbaser4   Úattention_factorÚinv_freqs           rA   r…   z7T5Gemma2RotaryEmbedding.compute_default_rope_parametersu   s   € ð& Ô% jÔ1°,Ô?ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐà˜$¥5¤<°°3¸ÅÄÐ#MÑ#MÔ#MÐPSÑ#SÑTÑUˆØ�{Š{˜6Ñ"Ô"Ð$4Ð4Ð4rB   c                 ó(  — t          | |› d�¦  «        }t          | |› d�¦  «        }|d d d …d f                              |j        d         dd¦  «                             t          j        |j        ¬¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d	¬
¦  «        5  ||z                       dd¦  «        }	t	          j        |	|	fd¬¦  «        }
|
                     ¦   «         |z  }|
                     ¦   «         |z  }d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬¦  «        |                     |j        ¬¦  «        fS )Nry   r}   r   rD   r,   ©r’   rr   ÚmpsÚcpuF)Údevice_typeÚenabledr*   ©r4   r‘   )r“   ÚexpandrS   r–   r<   rM   rr   Ú
isinstanceÚtypeÚstrr&   Ú	transposeÚcatÚcosÚsinr’   )r?   rI   Úposition_idsrx   rš   Úattention_scalingÚinv_freq_expandedÚposition_ids_expandedrŸ   ÚfreqsÚembr¨   r©   s                rA   rP   zT5Gemma2RotaryEmbedding.forward�   sÈ  € õ ˜4 JÐ!9Ð!9Ð!9Ñ:Ô:ˆÝ# D¨ZÐ*KÐ*KÐ*KÑLÔLÐð �T˜1˜1˜1˜d�]Ô#×*Ò*¨<Ô+=¸aÔ+@À"ÀaÑHÔH×KÒKÕRWÔR]ÐfgÔfnÐKÑoÔoð 	ð !-¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆå¨¸UÐCÑCÔCð 	0ð 	0Ø&Ð)>Ñ>×IÒIÈ!ÈQÑOÔOˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)Ð/Ñ/ˆCØ—'’'‘)”)Ð/Ñ/ˆCð		0ð 	0ð 	0ñ 	0ô 	0ð 	0ð 	0ð 	0ð 	0ð 	0ð 	0øøøð 	0ð 	0ð 	0ð 	0ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   Ã'A EÅEÅEr7   ©NN)rV   rW   rX   r%   r0   r9   Ústaticmethodr¥   rR   r<   ÚTensorrM   r…   Úno_gradr   rP   rZ   r[   s   @rA   rq   rq   ^   s  ø€ € € € € Ø€_�X vÐ.Ñ.Ô.ðUð UÐ1ð Uð Uð Uð Uð Uñ /Ô.ðUð* Ø€_�X vÐ.Ñ.Ô.àJNð5ð 5Ø"ð5Ø=@À4¹Zð5à	ˆuŒ|˜UÐ"Ô	#ð5ð 5ð 5ñ /Ô.ñ „\ð5ð2 €U„]�_„_Øð<ð <ñ Ôñ „_ð<ð <ð <ð <ð <rB   rq   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..NrD   r*   r¡   )rS   r<   r§   )rI   Úx1Úx2s      rA   Úrotate_halfr·   ¦   s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'rB   Úrotary_pos_embc                 ó¾   — |                      |¦  «        }|                      |¦  «        }| |z  t          | ¦  «        |z  z   }||z  t          |¦  «        |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezer·   )ÚqÚkr¨   r©   Úunsqueeze_dimÚq_embedÚk_embeds          rA   Úapply_rotary_pos_embrÀ   ­   sc   € ð& �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�GÐÐrB   ro   Ún_repr�   c                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r,   N)rS   r¢   Úreshape)ro   rÁ   ÚbatchÚnum_key_value_headsÚslenr�   s         rA   Ú	repeat_kvrÇ   Ç   s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTrB   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskrl   ÚscalingÚsoftcapc                 ól  — |€
| j         dz  }t          || j        ¦  «        }	t          || j        ¦  «        }
t          j        ||	                     dd¦  «        ¦  «        |z  }|�||z  }t          j        |¦  «        }||z  }|�||z   }t          j         	                    |dt          j
        ¬¦  «                             |j        ¦  «        }t          j                             ||| j        ¬¦  «        }t          j        ||
¦  «        }|                     dd¦  «                             ¦   «         }||fS )Nç      à¿r*   r   rD   )r4   r’   )ÚpÚtrainingr,   )r�   rÇ   Únum_key_value_groupsr<   Úmatmulr¦   Útanhr:   Ú
functionalÚsoftmaxÚfloat32r–   r’   rl   rÓ   Ú
contiguous)rÉ   rÊ   rË   rÌ   rÍ   rl   rÎ   rÏ   r—   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs                rA   Úeager_attention_forwardrß   Ó   s%  € ð €Ø”/ 4Ñ'ˆå˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LàÐØ# gÑ-ˆÝ”z ,Ñ/Ô/ˆØ# gÑ-ˆØÐ!Ø# nÑ4ˆõ ”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€KØ˜Ð$Ð$rB   c                   óÒ   ‡ — e Zd ZdZdedefˆ fd„Z	 	 	 ddej        dej        dej        dz  d	e	dz  d
e
e         deej        ej        dz  eej                 dz  f         fd„Zˆ xZS )ÚT5Gemma2SelfAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr^   Ú	layer_idxc                 óô  •— t          ¦   «                              ¦   «          t          |d¦  «        r|j        |         nd | _        || _        || _        t          |d|j        |j	        z  ¦  «        | _
        |j	        |j        z  | _        |j        dz  | _        | j        j        | _        d| _        t#          j        |j        |j	        | j
        z  |j        ¬¦  «        | _        t#          j        |j        |j        | j
        z  |j        ¬¦  «        | _        t#          j        |j        |j        | j
        z  |j        ¬¦  «        | _        t#          j        |j	        | j
        z  |j        |j        ¬¦  «        | _        | j        j        | _        | j        dk    r|j        nd | _        | j        dk    | _        t7          |j
        |j        ¬¦  «        | _        t7          |j
        |j        ¬¦  «        | _        d S ©Nrƒ   r�   rÑ   Fr`   Úsliding_attention)r4   r5   ©r8   r9   Úhasattrrƒ   rx   r^   râ   r“   rb   r”   r�   rÅ   rÔ   Úquery_pre_attn_scalarrÎ   Úattention_dropoutÚ	is_causalr:   rd   Úattention_biasÚq_projÚk_projÚv_projÚo_projÚattn_logit_softcappingÚsliding_windowÚ
is_slidingr2   Úrms_norm_epsÚq_normÚk_norm©r?   r^   râ   r@   s      €rA   r9   zT5Gemma2SelfAttention.__init__ù   ó×  ø€ Ý‰Œ×ÒÑÔÐÝ;BÀ6È=Ñ;YÔ;YÐc˜&Ô,¨YÔ7Ð7Ð_cˆŒØˆŒØ"ˆŒÝ ¨
°FÔ4FÈ&ÔJdÑ4dÑeÔeˆŒØ$*Ô$>À&ÔB\Ñ$\ˆÔ!ØÔ3°TÑ9ˆŒØ!%¤Ô!>ˆÔØˆŒå”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ&¨¬Ñ6¸Ô8JÐQWÔQfð
ñ 
ô 
ˆŒð '+¤kÔ&HˆÔ#Ø7;´ÐJ]Ò7]Ð7]˜fÔ3Ð3ÐcgˆÔØœ/Ð-@Ò@ˆŒå%¨&¬/¸vÔ?RÐSÑSÔSˆŒÝ%¨&¬/¸vÔ?RÐSÑSÔSˆŒˆˆrB   Nro   Úposition_embeddingsrÍ   Úpast_key_valuesr—   r�   c                 ó‚  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }	|                      |¦  «                             |¦  «                             dd¦  «        }
|                      |¦  «        }|                      |	¦  «        }	|\  }}t          ||	||¦  «        \  }}	|�| 
                    |	|
| j        ¦  «        \  }	}
t          j        | j        j        t           ¦  «        } || ||	|
|f| j        r| j        nd| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )NrD   r,   r*   rÈ   )rl   rÎ   rñ   )rS   r�   rì   Úviewr¦   rí   rî   rô   rõ   rÀ   Úupdaterâ   r   Úget_interfacer^   Ú_attn_implementationrß   rÓ   ré   rÎ   rñ   rÃ   rÚ   rï   )r?   ro   rø   rÍ   rù   r—   Úinput_shapeÚhidden_shapeÚquery_statesrÛ   rÜ   r¨   r©   Úattention_interfacerÞ   rÝ   s                   rA   rP   zT5Gemma2SelfAttention.forward  sé  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆà—{’{ <Ñ0Ô0ˆØ—[’[ Ñ,Ô,ˆ
à&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð /3¬mÐD�DÔ*Ð*ÀØ”LØÔ.ð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(rB   ©NNN)rV   rW   rX   Ú__doc__r0   rY   r9   r<   r²   r   r    r!   rR   rP   rZ   r[   s   @rA   rá   rá   õ   sï   ø€ € € € € àGÐGðTÐ1ð T¸cð Tð Tð Tð Tð Tð TðB -1Ø.2Ø(,ð*)ð *)à”|ð*)ð #œ\ð*)ð œ tÑ+ð	*)ð
  ™ð*)ð Ð+Ô,ð*)ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	Mð*)ð *)ð *)ð *)ð *)ð *)ð *)ð *)rB   rá   c                   óö   ‡ — e Zd ZdZdedefˆ fd„Z	 ddej        de	ej        ej        f         dej        dz  d	ej        d
e
dz  dee         de	ej        ej        dz  e	ej                 dz  f         fd„Zˆ xZS )ÚT5Gemma2MergedAttentionz6Merged self-attention and cross-attention for decoder.r^   râ   c                 óô  •— t          ¦   «                              ¦   «          t          |d¦  «        r|j        |         nd | _        || _        || _        t          |d|j        |j	        z  ¦  «        | _
        |j	        |j        z  | _        |j        dz  | _        | j        j        | _        d| _        t#          j        |j        |j	        | j
        z  |j        ¬¦  «        | _        t#          j        |j        |j        | j
        z  |j        ¬¦  «        | _        t#          j        |j        |j        | j
        z  |j        ¬¦  «        | _        t#          j        |j	        | j
        z  |j        |j        ¬¦  «        | _        | j        j        | _        | j        dk    r|j        nd | _        | j        dk    | _        t7          |j
        |j        ¬¦  «        | _        t7          |j
        |j        ¬¦  «        | _        d S rä   ræ   rö   s      €rA   r9   z T5Gemma2MergedAttention.__init__H  r÷   rB   Nro   rø   Úmerged_attention_maskÚencoder_hidden_statesrù   r—   r�   c                 ó¤  — |j         d d…         }g |¢d‘| j        ‘R }|j         d d…         }	g |	¢d‘| j        ‘R }
|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «        }|                      |¦  «        }|\  }}t          ||||¦  «        \  }}|�L|j
        }|                     ||| j        ¦  «        \  }}|j                             | j        ¦  «        }|j        }|�|s¾|                      |¦  «                             |
¦  «                             dd¦  «        }|                      |¦  «                             |
¦  «                             dd¦  «        }|                      |¦  «        }|�.|                     ||| j        ¦  «        \  }}d|j        | j        <   n.|j        | j                 j        }|j        | j                 j        }|}|	d         }t'          j        ||gd¬¦  «        }t'          j        ||gd¬¦  «        }t+          j        | j        j        t2          ¦  «        } || ||||f| j        r| j        nd| j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }|�|dd | …f         }|d| d …f         }nd	\  }}|||fS )
NrD   r,   r*   Tr¡   rÈ   )rl   rÎ   .r°   ) rS   r�   rì   rû   r¦   rí   rî   rô   rõ   rÀ   Úself_attention_cacherü   râ   Ú
is_updatedÚgetÚcross_attention_cacheÚlayersÚkeysÚvaluesr<   r§   r   rý   r^   rþ   rß   rÓ   ré   rÎ   rÃ   rÚ   rï   )r?   ro   rø   r  r	  rù   r—   rÿ   r   Úcross_input_shapeÚcross_hidden_shaper  rÛ   rÜ   r¨   r©   r  r  r  Úcross_key_statesÚcross_value_statesÚcross_key_sizer  rÞ   rÝ   Úself_attn_weightsÚcross_attn_weightss                              rA   rP   zT5Gemma2MergedAttention.forwardf  s¹  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆØ1Ô7¸¸¸Ô<ÐØDÐ0ÐD°"ÐD°d´mÐDÐDÐð —{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆà—{’{ <Ñ0Ô0ˆØ—[’[ Ñ,Ô,ˆ
à&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&à#2Ô#GÐ Ø';×'BÒ'BÀ:È|Ð]aÔ]kÑ'lÔ'lÑ$ˆJ˜ð )Ô3×7Ò7¸¼ÑGÔGˆJØ$3Ô$IÐ!àÐ"¨*Ð"Ø#Ÿ{š{Ð+@ÑAÔA×FÒFÐGYÑZÔZ×dÒdÐefÐhiÑjÔjÐØ!%§¢Ð-BÑ!CÔ!C×!HÒ!HÐI[Ñ!\Ô!\×!fÒ!fÐghÐjkÑ!lÔ!lÐà#Ÿ{š{Ð+;Ñ<Ô<ÐàÐ*Ø7L×7SÒ7SØ$Ð&8¸$¼.ñ8ô 8Ñ4Ð Ð"4ð >B�Ô*¨4¬>Ñ:øà4Ô;¸D¼NÔKÔPÐØ!6Ô!=¸d¼nÔ!MÔ!TÐð $ˆØ*¨1Ô-ˆÝ”Y 
Ð,<Ð=À1ÐEÑEÔEˆ
Ý”y ,Ð0BÐ!CÈÐKÑKÔKˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØ!ð	%
ð /3¬mÐD�DÔ*Ð*ÀØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆð Ð#Ø ,¨SÐ2B°N°?Ð2BÐ-BÔ CÐØ!-¨c°N°?Ð3CÐ3CÐ.CÔ!DÐÐà4>Ñ1ÐÐ1ØÐ-Ð/AÐAÐArB   r7   )rV   rW   rX   r  r0   rY   r9   r<   r²   rR   r	   r    r   rP   rZ   r[   s   @rA   r  r  D  s$  ø€ € € € € à@Ð@ðTÐ1ð T¸cð Tð Tð Tð Tð Tð TðN 7;ðTBð TBð ”|ðTBð # 5¤<°´Ð#=Ô>ð	TBð
  %œ|¨dÑ2ðTBð  %œ|ðTBð -¨tÑ3ðTBð Ð-Ô.ðTBð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	MðTBð TBð TBð TBð TBð TBð TBð TBrB   r  c                   ó¸   ‡ — e Zd ZdZdefˆ fd„Z	 	 	 ddej        deej        ej        f         dz  dej        dz  dej	        dz  d	eej
        f         f
d
„Zˆ xZS )ÚT5Gemma2EncoderLayerzEncoder sub-layer.râ   c                 ó0  •— t          ¦   «                              ¦   «          |j        | _        || _        || _        |j        |         | _        t          ||¬¦  «        | _        t          |j        |j
        ¬¦  «        | _        t          |j        |j
        ¬¦  «        | _        t          |¦  «        | _        t          |j        |j
        ¬¦  «        | _        t          |j        |j
        ¬¦  «        | _        t#          j        |j        ¦  «        | _        d S ©N)r^   râ   ©r5   )r8   r9   rb   r^   râ   rƒ   Úattention_typerá   Ú	self_attnr2   ró   Úpre_self_attn_layernormÚpost_self_attn_layernormr]   ÚmlpÚpre_feedforward_layernormÚpost_feedforward_layernormr:   rj   rk   rl   rö   s      €rA   r9   zT5Gemma2EncoderLayer.__init__À  sö   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔØˆŒØ"ˆŒØ$Ô0°Ô;ˆÔå.ØØð
ñ 
ô 
ˆŒõ (7°vÔ7IÈvÔObÐ'cÑ'cÔ'cˆÔ$Ý(7¸Ô8JÐPVÔPcÐ(dÑ(dÔ(dˆÔ%å˜vÑ&Ô&ˆŒÝ)8¸Ô9KÐQWÔQdÐ)eÑ)eÔ)eˆÔ&Ý*9¸&Ô:LÐRXÔReÐ*fÑ*fÔ*fˆÔ'å”z &Ô"5Ñ6Ô6ˆŒˆˆrB   Nro   rø   rÍ   rª   r�   c           	      ól  — |}|                       |¦  «        } | j        d||||d dœ|¤Ž\  }}|                      |¦  «        }||                      |¦  «        z   }|}|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }||                      |¦  «        z   }|S )N)ro   rø   rÍ   rª   rù   © ©r   r  r!  rl   r#  r"  r$  )r?   ro   rø   rÍ   rª   r—   ÚresidualÚ_s           rA   rP   zT5Gemma2EncoderLayer.forwardÔ  sÚ   € ð !ˆØ×4Ò4°]ÑCÔCˆØ)˜4œ>ð 
Ø'Ø 3Ø)Ø%Ø ð
ð 
ð ð
ð 
Ñˆ�qð ×5Ò5°mÑDÔDˆØ  4§<¢<°Ñ#>Ô#>Ñ>ˆà ˆØ×6Ò6°}ÑEÔEˆØŸš Ñ/Ô/ˆØ×7Ò7¸ÑFÔFˆØ  4§<¢<°Ñ#>Ô#>Ñ>ˆØÐrB   r  )rV   rW   rX   r  rY   r9   r<   r²   rR   Ú
LongTensorÚFloatTensorrP   rZ   r[   s   @rA   r  r  ½  sÏ   ø€ € € € € ØÐð7¨#ð 7ð 7ð 7ð 7ð 7ð 7ð. IMØ.2Ø04ðð à”|ðð # 5¤<°´Ð#=Ô>ÀÑEðð œ tÑ+ð	ð
 Ô&¨Ñ-ðð 
ˆuÔ Ð!Ô	"ðð ð ð ð ð ð ð rB   r  c                   óÐ   ‡ — e Zd ZdZdefˆ fd„Z	 	 	 	 	 ddej        deej        ej        f         dej        dz  d	ej	        dz  d
e
dz  dedz  dej        dz  dej        fd„Zˆ xZS )ÚT5Gemma2DecoderLayerzFDecoder sub-layer: merged attention instead of vanilla self-attention.râ   c                 ó0  •— t          ¦   «                              ¦   «          |j        | _        || _        || _        |j        |         | _        t          ||¬¦  «        | _        t          |j        |j
        ¬¦  «        | _        t          |j        |j
        ¬¦  «        | _        t          |¦  «        | _        t          |j        |j
        ¬¦  «        | _        t          |j        |j
        ¬¦  «        | _        t#          j        |j        ¦  «        | _        d S r  )r8   r9   rb   r^   râ   rƒ   r  r  r  r2   ró   r   r!  r]   r"  r#  r$  r:   rj   rk   rl   rö   s      €rA   r9   zT5Gemma2DecoderLayer.__init__ô  sø   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔØˆŒØ"ˆŒØ$Ô0°Ô;ˆÔõ 1ØØð
ñ 
ô 
ˆŒõ (7°vÔ7IÈvÔObÐ'cÑ'cÔ'cˆÔ$Ý(7¸Ô8JÐPVÔPcÐ(dÑ(dÔ(dˆÔ%å˜vÑ&Ô&ˆŒÝ)8¸Ô9KÐQWÔQdÐ)eÑ)eÔ)eˆÔ&Ý*9¸&Ô:LÐRXÔReÐ*fÑ*fÔ*fˆÔ'å”z &Ô"5Ñ6Ô6ˆŒˆˆrB   NFro   rø   r  rª   rù   Ú	use_cacher	  r�   c                 ór  — |}	|                       |¦  «        } | j        d|||||||dœ|¤Ž\  }}
}
|                      |¦  «        }|	|                      |¦  «        z   }|}	|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|	|                      |¦  «        z   }|S )N)ro   rø   r  rª   rù   r/  r	  r&  r'  )r?   ro   rø   r  rª   rù   r/  r	  r—   r(  r)  s              rA   rP   zT5Gemma2DecoderLayer.forward	  sâ   € ð !ˆØ×4Ò4°]ÑCÔCˆà,˜dœnð 	
Ø'Ø 3Ø"7Ø%Ø+ØØ"7ð	
ð 	
ð ð	
ð 	
Ñˆ�q˜!ð ×5Ò5°mÑDÔDˆØ  4§<¢<°Ñ#>Ô#>Ñ>ˆà ˆØ×6Ò6°}ÑEÔEˆØŸš Ñ/Ô/ˆØ×7Ò7¸ÑFÔFˆØ  4§<¢<°Ñ#>Ô#>Ñ>ˆØÐrB   )NNNFN)rV   rW   rX   r  rY   r9   r<   r²   rR   r*  r	   Úboolr+  rP   rZ   r[   s   @rA   r-  r-  ñ  sô   ø€ € € € € ØPÐPð7¨#ð 7ð 7ð 7ð 7ð 7ð 7ð2 6:Ø04Ø6:Ø!&Ø59ð ð  à”|ð ð # 5¤<°´Ð#=Ô>ð ð  %œ|¨dÑ2ð	 ð
 Ô&¨Ñ-ð ð -¨tÑ3ð ð ˜$‘;ð ð  %œ|¨dÑ2ð ð 
Ô	ð ð  ð  ð  ð  ð  ð  ð  rB   r-  c                   óV   ‡ — e Zd ZdZd
dededefˆ fd„Zdej        dej        fd	„Z	ˆ xZ
S )ÚT5Gemma2LMHeadz.Head for language modeling (generation) tasks.Frb   Ú
vocab_sizera   c                 ó€   •— t          ¦   «                              ¦   «          t          j        |||¬¦  «        | _        d S )Nr`   )r8   r9   r:   rd   Úout_proj)r?   rb   r4  ra   r@   s       €rA   r9   zT5Gemma2LMHead.__init__/  s5   ø€ Ý‰Œ×ÒÑÔÐÝœ	 +¨zÀÐEÑEÔEˆŒˆˆrB   ro   r�   c                 ó0   — |                       |¦  «        }|S r7   )r6  )r?   ro   Úlogitss      rA   rP   zT5Gemma2LMHead.forward3  s   € Ø—’˜}Ñ-Ô-ˆØˆrB   )F)rV   rW   rX   r  rY   r1  r9   r<   r²   rP   rZ   r[   s   @rA   r3  r3  ,  s�   ø€ € € € € Ø8Ð8ðFð F Cð F°Sð FÀð Fð Fð Fð Fð Fð Fð U¤\ð °e´lð ð ð ð ð ð ð ð rB   r3  c                   óV   ‡ — e Zd ZdZd
dededefˆ fd„Zdej        dej        fd	„Z	ˆ xZ
S )ÚT5Gemma2ClassificationHeadz-Head for sentence-level classification tasks.rÈ   rb   Ú
num_labelsÚclassifier_dropout_ratec                 ó°   •— t          ¦   «                              ¦   «          t          j        |¬¦  «        | _        t          j        ||¦  «        | _        d S )N)rÒ   )r8   r9   r:   rj   rl   rd   r6  )r?   rb   r;  r<  r@   s       €rA   r9   z#T5Gemma2ClassificationHead.__init__;  sE   ø€ Ý‰Œ×ÒÑÔÐÝ”zÐ$;Ð<Ñ<Ô<ˆŒÝœ	 +¨zÑ:Ô:ˆŒˆˆrB   ro   r�   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S r7   )rl   r6  )r?   ro   s     rA   rP   z"T5Gemma2ClassificationHead.forward@  s*   € ØŸš ]Ñ3Ô3ˆØŸš mÑ4Ô4ˆØÐrB   )rÈ   ©rV   rW   rX   r  rY   rM   r9   r<   r²   rP   rZ   r[   s   @rA   r:  r:  8  s„   ø€ € € € € Ø7Ð7ð;ð ; Cð ;°Sð ;ÐSXð ;ð ;ð ;ð ;ð ;ð ;ð
 U¤\ð °e´lð ð ð ð ð ð ð ð rB   r:  c                   ó:   ‡ — e Zd Zdefˆ fd„Zdej        fd„Zˆ xZS )ÚT5Gemma2MultiModalProjectorr^   c                 ó  •— t          ¦   «                              ¦   «          t          j        t	          j        |j        j        |j        j        ¦  «        ¦  «        | _	        t          |j        j        |j        j        ¬¦  «        | _        t          |j        j        |j        j        z  ¦  «        | _        t          |j        dz  ¦  «        | _        | j        | j        z  | _        t          j        | j        | j        ¬¦  «        | _        d S )Nr  ç      à?)Úkernel_sizeÚstride)r8   r9   r:   r;   r<   r=   Úvision_configrb   Útext_configÚmm_input_projection_weightr2   Úlayer_norm_epsÚmm_soft_emb_normrY   Ú
image_sizeÚ
patch_sizeÚpatches_per_imageÚmm_tokens_per_imageÚtokens_per_siderD  Ú	AvgPool2dÚavg_poolrm   s     €rA   r9   z$T5Gemma2MultiModalProjector.__init__G  sà   ø€ Ý‰Œ×ÒÑÔÐå*,¬,ÝŒK˜Ô,Ô8¸&Ô:LÔ:XÑYÔYñ+
ô +
ˆÔ'õ !0ØÔ Ô,°&Ô2FÔ2Uð!
ñ !
ô !
ˆÔõ "% VÔ%9Ô%DÈÔH\ÔHgÑ%gÑ!hÔ!hˆÔÝ" 6Ô#=¸sÑ#BÑCÔCˆÔØÔ1°TÔ5IÑIˆÔÝœ°Ô1AÈ$ÔJZÐ[Ñ[Ô[ˆŒˆˆrB   Úvision_outputsc                 ó¸  — |j         \  }}}|                     dd¦  «        }|                     ||| j        | j        ¦  «        }|                     ¦   «         }|                      |¦  «        }|                     d¦  «        }|                     dd¦  «        }|                      |¦  «        }t          j	        || j
        ¦  «        }|                     |¦  «        S )Nr,   r*   )rS   r¦   rÃ   rM  rÚ   rQ  ÚflattenrJ  r<   rÕ   rH  rN   )	r?   rR  Ú
batch_sizer)  rb   Úreshaped_vision_outputsÚpooled_vision_outputsÚnormed_vision_outputsÚprojected_vision_outputss	            rA   rP   z#T5Gemma2MultiModalProjector.forwardW  sÜ   € Ø%3Ô%9Ñ"ˆ
�A�{à"0×":Ò":¸1¸aÑ"@Ô"@ÐØ"9×"AÒ"AØ˜ TÔ%;¸TÔ=Sñ#
ô #
Ðð #:×"DÒ"DÑ"FÔ"FÐà $§¢Ð.EÑ FÔ FÐØ 5× =Ò =¸aÑ @Ô @ÐØ 5× ?Ò ?ÀÀ1Ñ EÔ EÐà $× 5Ò 5Ð6KÑ LÔ LÐå#(¤<Ð0EÀtÔGfÑ#gÔ#gÐ Ø'×/Ò/°Ñ?Ô?Ð?rB   )	rV   rW   rX   r/   r9   r<   r²   rP   rZ   r[   s   @rA   rA  rA  F  sr   ø€ € € € € ð\Ð4ð \ð \ð \ð \ð \ð \ð @ e¤lð @ð @ð @ð @ð @ð @ð @ð @rB   rA  c                   óX   ‡ — e Zd ZdZ	 	 ddededededef
ˆ fd	„Zd
ej        fˆ fd„Z	ˆ xZ
S )ÚT5Gemma2TextScaledWordEmbeddingzCT5Gemma2 Embedding: override to add eoi token embedding separately.rL   é è Únum_embeddingsÚembedding_dimÚpadding_idxÚembed_scaleÚeoi_token_indexc                 ó$  •— t          ¦   «                              |||¦  «         || _        t          j        t          j        |¦  «        d¬¦  «        | _        || _        t          j	        t          j
        | j        ¦  «        ¦  «        | _        d S )NFrz   )r8   r9   Úscalar_embed_scaler:   r‡   r<   Útensorr`  ra  r;   r=   r^  Úeoi_embedding)r?   r]  r^  r_  r`  ra  r@   s         €rA   r9   z(T5Gemma2TextScaledWordEmbedding.__init__m  sw   ø€ õ 	‰Œ×Ò˜¨¸ÑDÔDÐDØ"-ˆÔÝœ9¥U¤\°+Ñ%>Ô%>È5ÐQÑQÔQˆÔØ.ˆÔÝœ\­%¬+°dÔ6HÑ*IÔ*IÑJÔJˆÔÐÐrB   Ú	input_idsc                 óê   •— t          ¦   «                              |¦  «        | j                             | j        j        ¦  «        z  }| j                             |j        ¦  «        ||| j        k    <   |S r7   )r8   rP   r`  r–   r>   r’   re  ra  )r?   rf  Úinput_embeddingsr@   s      €rA   rP   z'T5Gemma2TextScaledWordEmbedding.forward{  sb   ø€ Ý ™7œ7Ÿ?š?¨9Ñ5Ô5¸Ô8H×8KÒ8KÈDÌKÔL]Ñ8^Ô8^Ñ^ÐØ>BÔ>P×>SÒ>SÐTdÔTjÑ>kÔ>kÐ˜ dÔ&:Ò:Ñ;ØÐrB   )rL   r\  r?  r[   s   @rA   r[  r[  j  s³   ø€ € € € € ØMÐMð !Ø&ðKð KàðKð ðKð ð	Kð
 ðKð ðKð Kð Kð Kð Kð Kð  ¤ð  ð  ð  ð  ð  ð  ð  ð  ð  ð  rB   r[  c                   óœ   ‡ — e Zd ZU eed<   dZdZddgZdgZdZ	dZ
dZdZdZdZd	Z ej        ¦   «         ˆ fd
„¦   «         Zdej        fd„Zˆ xZS )ÚT5Gemma2PreTrainedModelr^   ÚmodelTr  r-  rù   FN)ÚimageÚtextc                 óB  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rt	          j        |j        ¦  «         d S t          |t          ¦  «        r:t	          j        |j        ¦  «         t	          j	        |j
        |j        ¦  «         d S t          |t          ¦  «        r�|j        j        j        d         dz  }t	          j        |j        j        d| j        j        |z  ¬¦  «         t'          |j        d¦  «        r,|j        j        �"t	          j        |j        j        ¦  «         d S d S d S d|j        j        v rt	          j        |j        ¦  «         d S t          |t.          ¦  «        r›|j        D ]•}|j        }|j        |         dk    rt6          |j        |                  } ||j        |¬¦  «        \  }}t	          j        t;          ||› d	�¦  «        |¦  «         t	          j        t;          ||› d
�¦  «        |¦  «         Œ”d S d S )Nr   rÑ   rÈ   )rH   Ústdra   ÚRMSNormrv   rw   ry   r|   )r8   Ú_init_weightsr£   rA  ÚinitÚzeros_rH  r[  re  Ú	constant_r`  rc  r:  r6  r>   rS   Únormal_r^   Úinitializer_rangerç   ra   r@   rV   rq   rƒ   r…   ru   r   Úcopy_r“   )r?   rÉ   Úscalerx   rŠ   r‹   r)  r@   s          €rA   rq  z%T5Gemma2PreTrainedModel._init_weightsš  s(  ø€ å‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ9Ñ:Ô:ð 	^ÝŒK˜Ô9Ñ:Ô:Ð:Ð:Ð:Ý˜Õ ?Ñ@Ô@ð 	^ÝŒK˜Ô,Ñ-Ô-Ð-ÝŒN˜6Ô-¨vÔ/HÑIÔIÐIÐIÐIÝ˜Õ :Ñ;Ô;ð 	^Ø”OÔ*Ô0°Ô3°tÑ;ˆEÝŒL˜œÔ/°c¸t¼{Ô?\Ð_dÑ?dÐeÑeÔeÐeÝ�v”¨Ñ/Ô/ð 2°F´OÔ4HÐ4TÝ”˜FœOÔ0Ñ1Ô1Ð1Ð1Ð1ð2ð 2Ð4TÐ4Tð ˜&Ô*Ô3Ð3Ð3ÝŒK˜œÑ&Ô&Ð&Ð&Ð&Ý˜Õ 7Ñ8Ô8ð 	^Ø$Ô0ð ^ð ^�
Ø%ÔE�ØÔ# JÔ/°9Ò<Ð<Ý#6°vÔ7GÈ
Ô7SÔ#T�LØ#/ <°´È*Ð#UÑ#UÔ#UÑ �˜qÝ”
�7 6¨jÐ+CÐ+CÐ+CÑDÔDÀmÑTÔTÐTÝ”
�7 6¨jÐ+LÐ+LÐ+LÑMÔMÈ}Ñ]Ô]Ð]Ð]ð	^ð 	^ð^ð ^rB   Úlabelsc                 ó:  — | j         j        }|j        }|j        }|€t	          d¦  «        ‚|                     |j        ¦  «        }|ddd…f                              ¦   «         |ddd…f<   ||d<   |€t	          d¦  «        ‚|                     |dk    |¦  «         |S )	zú
        Shifts input_ids to the right, prepends the decoder_start_token_id, and handles
        pad_token_id replacement for labels that were -100.
        This is a common preparation step for decoder inputs in sequence-to-sequence models.
        Nz:self.model.config.decoder.bos_token_id has to be defined. .rD   r,   ).r   z9self.model.config.decoder.pad_token_id has to be defined.iœÿÿÿ)	r^   ÚdecoderÚbos_token_idÚpad_token_idÚ
ValueErrorÚ	new_zerosrS   rˆ   Úmasked_fill_)r?   ry  Údecoder_configÚdecoder_start_token_idr}  Úshifted_input_idss         rA   Ú%prepare_decoder_input_ids_from_labelsz=T5Gemma2PreTrainedModel.prepare_decoder_input_ids_from_labels³  s¼   € ð œÔ,ˆØ!/Ô!<ÐØ%Ô2ˆà!Ð)ÝÐYÑZÔZÐZð #×,Ò,¨V¬\Ñ:Ô:ÐØ%+¨C°°"°¨HÔ%5×%;Ò%;Ñ%=Ô%=Ð˜#˜q˜r˜r˜'Ñ"Ø$:Ð˜&Ñ!àÐÝÐXÑYÔYÐYð 	×&Ò&Ð'8¸DÒ'@À,ÑOÔOÐOà Ð rB   )rV   rW   rX   r-   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendÚ_can_record_outputsÚinput_modalitiesr<   r³   rq  r²   r„  rZ   r[   s   @rA   rj  rj  �  sÌ   ø€ € € € € € àÐÐÑØÐØ&*Ð#ð 	ØðÐð $5Ð"5Ðð !ÐØ€NàÐà!ÐØ"&ÐàÐØ(Ðà€U„]�_„_ð^ð ^ð ^ð ^ñ „_ð^ð0!¸E¼Lð !ð !ð !ð !ð !ð !ð !ð !rB   rj  Trñ   c           
      ó^   ‡ ‡— dt           dt           dt           dt           dt          f
ˆˆ fd„}|S )zL
    This creates uni/bidirectional attention mask with sliding window.
    Ú	batch_idxÚhead_idxÚq_idxÚkv_idxr�   c                 ó|   •— ‰	r‰
d}}n‰
dz   dz  ‰
dz  dz   }}||z
  }|dk    ||k     z  }|dk     | |k     z  }||z  S )Nr   r,   r*   r&  )r’  r“  r”  r•  Úleft_window_sizeÚright_window_sizeÚdistÚ	left_maskÚ
right_maskrê   rñ   s            €€rA   Ú
inner_maskz0sliding_window_mask_function.<locals>.inner_maskÔ  sy   ø€ Øð 	iØ2@À!Ð/ÐÐà4BÀQÑ4FÈ1Ñ3LÈ~ÐbcÑNcÐfgÑNgÐ/Ðà�v‰~ˆØ˜Q’Y 4Ð*:Ò#:Ñ;ˆ	Ø˜Q’h D 5Ð+<Ò#<Ñ=ˆ
Ø˜:Ñ%Ð%rB   )rY   r1  )rñ   rê   rœ  s   `` rA   Úsliding_window_mask_functionr�  Ï  sR   øø€ ð
	&�cð 	&­Sð 	&½ð 	&Åcð 	&Ídð 	&ð 	&ð 	&ð 	&ð 	&ð 	&ð 	&ð ÐrB   c                   óþ   ‡ — e Zd ZU eed<   eedœZ	 ddedefˆ fd„Z	e
ee	 	 	 	 	 ddej        dz  dej        dz  d	ej        dz  d
ej        dz  dej        dz  dee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚT5Gemma2TextEncoderr^   )Ú
attentionsro   r\  ra  c                 ó$  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          ‰j        ‰j        | j        ‰j        dz  |¬¦  «        | _        t          ‰j        ‰j	        ¬¦  «        | _
        d| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          j        ‰j        ¦  «        | _        t)          ‰¦  «        | _        |                      ¦   «          d S )NrC  ©r`  ra  r  Fc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r&  )r  ©Ú.0râ   r^   s     €rA   ú
<listcomp>z0T5Gemma2TextEncoder.__init__.<locals>.<listcomp>ý  ó$   ø€ ÐfÐfÐf¸Õ! &¨)Ñ4Ô4ÐfÐfÐfrB   ©r8   r9   r}  r_  r4  r[  rb   Úembed_tokensr2   ró   ÚnormÚgradient_checkpointingr:   Ú
ModuleListÚrangeÚnum_hidden_layersr  rj   rk   rl   rq   Ú
rotary_embÚ	post_init©r?   r^   ra  r@   s    ` €rA   r9   zT5Gemma2TextEncoder.__init__é  s   øø€ õ
 	‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒå;ØÔØÔØÔØÔ*¨CÑ/Ø+ð
ñ 
ô 
ˆÔõ $ FÔ$6¸FÔ<OÐPÑPÔPˆŒ	Ø&+ˆÔ#å”mØfÐfÐfÐfÅeÈFÔLdÑFeÔFeÐfÑfÔfñ
ô 
ˆŒõ ”z &Ô"5Ñ6Ô6ˆŒÝ1°&Ñ9Ô9ˆŒð 	�ŠÑÔÐÐÐrB   Nrf  rÍ   rª   Úinputs_embedsÚtoken_type_idsr—   r�   c           
      óŽ  — |d u |d uz  rt          d¦  «        ‚|                     dd ¦  «         |€|                      |¦  «        }|€:t          j        d|j        d         |j        ¬¦  «                             d¦  «        }t          |x}t          ¦  «        sA| j
        ||dœ}t          di |¤Žt          di |¤dt          | j
        j        d¬	¦  «        i¤Žd
œ}|}	i }
t          | j
        j        ¦  «        D ]}|                      |	||¦  «        |
|<   Œ|                      |	¦  «        }	t%          | j        d | j
        j        …         ¦  «        D ]=\  }} ||	|
| j
        j        |                  || j
        j        |                  |fi |¤Ž}	Œ>|                      |	¦  «        }	|                      |	¦  «        }	t-          |	¬¦  «        S )Nú:You must specify exactly one of input_ids or inputs_embedsrù   r   r,   ©rr   )r^   r²  rÍ   Úand_mask_functionF)rê   ©Úfull_attentionrå   )Úlast_hidden_stater&  )r~  Úpopr©  r<   r•   rS   rr   rº   r£   Údictr^   r   r�  rñ   r‚   rƒ   r¯  rl   Ú	enumerater  r®  rª  r   )r?   rf  rÍ   rª   r²  r³  r—   Úself_attn_mask_mappingÚmask_kwargsro   rø   rx   ÚiÚlayer_modules                 rA   rP   zT5Gemma2TextEncoder.forward  s5  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZð 	�
Š
Ð$ dÑ+Ô+Ð+àÐ Ø ×-Ò-¨iÑ8Ô8ˆMàÐÝ œ<¨¨=Ô+>¸qÔ+AÈ-ÔJ^Ð_Ñ_Ô_×iÒiÐjkÑlÔlˆLå°NÐBÐ0ÅDÑIÔIð 	àœ+Ø!.Ø"0ðð ˆKõ #<Ð"JÐ"J¸kÐ"JÐ"JÝ%>ð &ð &Ø!ð&ð &å&BÀ4Ä;ÔC]ÐinÐ&oÑ&oÔ&oð&ð &ð &ð&ð &Ð"ð &ˆð !ÐÝ˜dœkÔ5Ñ6Ô6ð 	gð 	gˆJØ.2¯oªo¸mÈ\Ð[eÑ.fÔ.fÐ 
Ñ+Ð+ð Ÿš ]Ñ3Ô3ˆå(¨¬Ð5T°t´{Ô7TÐ5TÔ)UÑVÔVð 	ð 	‰OˆAˆ|Ø(˜LØØ# D¤KÔ$;¸AÔ$>Ô?Ø& t¤{Ô'>¸qÔ'AÔBØð	ð ð
 ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆØŸš ]Ñ3Ô3ˆÝØ+ð
ñ 
ô 
ð 	
rB   ©r\  )NNNNN)rV   rW   rX   r0   r…  rá   r  r�  rY   r9   r'   r)   r"   r<   r*  r²   r+  r    r!   r   rP   rZ   r[   s   @rA   rŸ  rŸ  â  s5  ø€ € € € € € ØÐÐÑà+Ø-ðð Ðð  'ðð à"ðð ðð ð ð ð ð ð8  ØØð .2Ø.2Ø04Ø26à.2ð<
ð <
àÔ# dÑ*ð<
ð œ tÑ+ð<
ð Ô&¨Ñ-ð	<
ð
 Ô(¨4Ñ/ð<
ð œ tÑ+ð<
ð Ð+Ô,ð<
ð 
ð<
ð <
ð <
ñ „^ñ „_ñ  Ôð<
ð <
ð <
ð <
ð <
rB   rŸ  c                   ó„  ‡ — e Zd ZU eed<   	 ddedefˆ fd„Zd„ Zd„ Ze	e
dej        dee         d	eez  fd
„¦   «         ¦   «         Zdej        dz  dej        dz  dej        fd„Ze
	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dee         d	efd„¦   «         Zˆ xZS )ÚT5Gemma2Encoderr^   r\  ra  c                 ó$  •— t          ¦   «                              |¦  «         t                               |j        |¬¦  «        | _        t          j        |j        ¬¦  «        | _	        t          |¦  «        | _        |                      ¦   «          d S )N)ra  ©r^   )r8   r9   rŸ  Ú_from_configrG  Ú
text_modelr+   Úfrom_configrF  Úvision_towerrA  Úmulti_modal_projectorr°  r±  s      €rA   r9   zT5Gemma2Encoder.__init__J  s|   ø€ õ
 	‰Œ×Ò˜Ñ Ô Ð å-×:Ò:¸6Ô;MÐ_nÐ:ÑoÔoˆŒÝ%Ô1¸Ô9MÐNÑNÔNˆÔÝ%@ÀÑ%HÔ%HˆÔ"ð 	�ŠÑÔÐÐÐrB   c                 ó4   — | j                              ¦   «         S r7   )rÈ  Úget_input_embeddingsrT   s    rA   rÍ  z$T5Gemma2Encoder.get_input_embeddingsX  s   € ØŒ×3Ò3Ñ5Ô5Ð5rB   c                 ó6   — | j                              |¦  «        S r7   )rÈ  Úset_input_embeddings©r?   Únew_embeddingss     rA   rÏ  z$T5Gemma2Encoder.set_input_embeddings[  s   € ØŒ×3Ò3°NÑCÔCÐCrB   Úpixel_valuesr—   r�   c                 ól   —  | j         d|ddœ|¤Ž}|j        }|                      |¦  «        }||_        |S )NT)rÒ  Úreturn_dictr&  )rÊ  rº  rË  Úpooler_output)r?   rÒ  r—   rR  rº  Úimage_featuress         rA   Úget_image_featuresz"T5Gemma2Encoder.get_image_features^  sP   € ð +˜Ô*Ða¸ÐRVÐaÐaÐZ`ÐaÐaˆØ*Ô<ÐØ×3Ò3Ð4EÑFÔFˆØ'5ˆÔ$àÐrB   rf  Nr²  rÖ  c           	      ó4  — | j         j        }|€m|€t          d¦  «        ‚| |                      ¦   «         t	          j        d|t          j        |j        ¬¦  «        ¦  «        k    }|                     d¦  «        }n||k    }| 	                    ¦   «         }| 
                    d¦  «                             |j        ¦  «        }|j        d         |j        d         z  }t          ||j        d         z  |                     ¦   «         k    d|› d	|› �¦  «         |S )
zï
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        Nz9Either `input_ids` or `inputs_embeds` has to be provided.r&  rœ   rD   r   r,   z6Image features and image tokens do not match: tokens: z, features )r^   Úimage_token_idr~  rÍ  r<   ÚfullÚlongrr   ÚallÚsumrº   r–   rS   r$   Únumel)r?   rf  r²  rÖ  rÙ  Úspecial_image_maskÚn_image_tokensÚn_image_featuress           rA   Úget_image_placeholder_maskz*T5Gemma2Encoder.get_image_placeholder_maskl  s.  € ð œÔ3ˆØÐØÐ$Ý Ð!\Ñ]Ô]Ð]Ø!.Ð2M°$×2KÒ2KÑ2MÔ2MÝ”
˜2˜~µU´ZÈÔH\Ð]Ñ]Ô]ñ3ô 3ò "Ðð "4×!7Ò!7¸Ñ!;Ô!;ÐÐà!*¨nÒ!<Ðà+×/Ò/Ñ1Ô1ˆØ/×9Ò9¸"Ñ=Ô=×@Ò@ÀÔAUÑVÔVÐØ)Ô/°Ô2°^Ô5IÈ!Ô5LÑLÐÝØ˜]Ô0°Ô4Ñ4¸×8LÒ8LÑ8NÔ8NÒNØrÀ^ÐrÐrÐ`pÐrÐrñ	
ô 	
ð 	
ð "Ð!rB   rÍ   rª   r³  c                 óh  — |d u |d uz  rt          d¦  «        ‚|€| j                             |¦  «        }|�j|                      |d¬¦  «        j        }|                     |j        |j        ¦  «        }|                      |||¬¦  «        }	| 	                    |	|¦  «        } | j        d|||dœ|¤Ž}
|
S )Nrµ  T)rÔ  )r²  rÖ  )r²  rÍ   rª   r&  )
r~  rÈ  r©  r×  rÕ  r–   rr   r’   râ  Úmasked_scatter)r?   rf  rÍ   rª   r²  rÒ  r³  r—   rÖ  Ú
image_maskÚoutputss              rA   rP   zT5Gemma2Encoder.forwardŠ  sñ   € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø œO×8Ò8¸ÑCÔCˆMàÐ#Ø!×4Ò4°\ÈtÐ4ÑTÔTÔbˆNØ+×.Ò.¨}Ô/CÀ]ÔEXÑYÔYˆNà×8Ò8Ø¨À~ð 9ñ ô ˆJð *×8Ò8¸À^ÑTÔTˆMà!�$”/ð 
Ø'Ø)Ø%ð
ð 
ð ð	
ð 
ˆð ˆrB   rÂ  )NNNNNN)rV   rW   rX   r/   r…  rY   r9   rÍ  rÏ  r#   r"   r<   r²   r    r!   rR   r   r×  r*  r+  râ  r   rP   rZ   r[   s   @rA   rÄ  rÄ  G  s×  ø€ € € € € € Ø!Ð!Ð!Ñ!ð
  'ðð à%ðð ðð ð ð ð ð ð6ð 6ð 6ðDð Dð Dð Øð
Ø!œLð
Ø4:Ð;MÔ4Nð
à	Ð+Ñ	+ð
ð 
ð 
ñ „^ñ Ôð
ð"àÔ# dÑ*ð"ð Ô(¨4Ñ/ð"ð Ô)ð	"ð "ð "ð "ð< ð .2Ø.2Ø04Ø26Ø15à.2ð!ð !àÔ# dÑ*ð!ð œ tÑ+ð!ð Ô&¨Ñ-ð	!ð
 Ô(¨4Ñ/ð!ð Ô'¨$Ñ.ð!ð œ tÑ+ð!ð Ð+Ô,ð!ð 
ð!ð !ð !ñ „^ð!ð !ð !ð !ð !rB   rÄ  c                   óX  ‡ — e Zd ZU eed<    eed¬¦  «         eed¬¦  «        edœZddede	fˆ fd„Z
eee	 	 	 	 	 	 	 	 dd
ej        d	z  dej        d	z  dej        d	z  ded	z  dej        d	z  ded	z  dej        d	z  dej        d	z  dee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚT5Gemma2Decoderr^   r,   )Úindexr*   )r   Úcross_attentionsro   r\  ra  c                 ó$  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          ‰j        ‰j        ‰j        ‰j        dz  |¬¦  «        | _        t          ‰j        ‰j	        ¬¦  «        | _
        d| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          j        ‰j        ¦  «        | _        t)          ‰¦  «        | _        |                      ¦   «          d S )NrC  r¢  r  Fc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r&  )r-  r¤  s     €rA   r¦  z,T5Gemma2Decoder.__init__.<locals>.<listcomp>Ç  r§  rB   r¨  r±  s    ` €rA   r9   zT5Gemma2Decoder.__init__·  sü   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒå;ØÔØÔØÔØÔ*¨CÑ/Ø+ð
ñ 
ô 
ˆÔõ $ FÔ$6¸FÔ<OÐPÑPÔPˆŒ	Ø&+ˆÔ#å”mØfÐfÐfÐfÅeÈFÔLdÑFeÔFeÐfÑfÔfñ
ô 
ˆŒõ ”z &Ô"5Ñ6Ô6ˆŒÝ1°&Ñ9Ô9ˆŒØ�ŠÑÔÐÐÐrB   Nrf  rÍ   rª   rù   r²  r/  r	  Úencoder_attention_maskr—   r�   c	           
      ó  — |d u |d uz  rt          d¦  «        ‚|€t          d¦  «        ‚|€|                      |¦  «        }| j        s3|r1|€/t          t	          | j        ¬¦  «        t	          ¦   «         ¦  «        }|€V|�|                     ¦   «         nd}
t          j        |j	        d         |j
        ¬¦  «        |
z   }|                     d¦  «        }t          |x}t          ¦  «        s0| j        |||�|j        nd |ddœ}t          di |¤Žt!          di |¤Žd	œ}t          |x}t          ¦  «        sd
t#          | j        |||d¬¦  «        i}t          j        |d
         |d
         gd¬¦  «        t          j        |d         |d
         gd¬¦  «        d	œ}|}i }t'          | j        j        ¦  «        D ]}|                      |||¦  «        ||<   Œ|                      |¦  «        }t/          | j        d | j        j        …         ¦  «        D ]@\  }} |||| j        j        |                  || j        j        |                  ||||fi |	¤Ž}ŒA|                      |¦  «        }|                      |¦  «        }t7          ||¬¦  «        S )Nrµ  z0`encoder_hidden_states` must be given in decoderrÆ  r   r,   r¶  F)r^   r²  rÍ   rù   rª   Úallow_is_causal_skipr¸  r¹  )r^   r²  rÍ   r	  Úallow_is_bidirectional_skiprD   r¡   rå   )rº  rù   r&  )r~  r©  rÓ   r	   r   r^   Úget_seq_lengthr<   r•   rS   rr   rº   r£   r¼  r  r   r   r   r§   r‚   rƒ   r¯  rl   r½  r  r®  rª  r   )r?   rf  rÍ   rª   rù   r²  r/  r	  rí  r—   Úpast_seen_tokensr¾  r¿  Úcross_attn_mask_mappingÚmerged_attn_mask_mappingro   rø   rx   rÀ  rÁ  s                       rA   rP   zT5Gemma2Decoder.forwardÍ  s<  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZØ Ð(ÝÐOÑPÔPÐPàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàŒ}ð 	d ð 	d¨Ð/FÝ1µ,ÀdÄkÐ2RÑ2RÔ2RÕT`ÑTbÔTbÑcÔcˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå°NÐBÐ0ÅDÑIÔIð 	ð œ+Ø!.Ø"0ØKZÐKf ?Ô#GÐ#GÐlpØ ,Ø(-ðð ˆKõ #5Ð"CÐ"C°{Ð"CÐ"CÝ%FÐ%UÐ%UÈÐ%UÐ%Uð&ð &Ð"õ
 Ð5KÐKÐ1ÍTÑRÔRð 		à Õ";Øœ;Ø"/Ø#9Ø*?Ø05ð#ñ #ô #ð'Ð#õ $œiØ'Ð(8Ô9Ð;RÐScÔ;dÐeÐkmðñ ô õ "'¤Ø'Ð(;Ô<Ð>UÐVfÔ>gÐhÐnpð"ñ "ô "ð	$
ð $
Ð ð &ˆð !ÐÝ˜dœkÔ5Ñ6Ô6ð 	gð 	gˆJØ.2¯oªo¸mÈ\Ð[eÑ.fÔ.fÐ 
Ñ+Ð+ð Ÿš ]Ñ3Ô3ˆå(¨¬Ð5T°t´{Ô7TÐ5TÔ)UÑVÔVð 
	ð 
	‰OˆAˆ|Ø(˜LØØ# D¤KÔ$;¸AÔ$>Ô?Ø(¨¬Ô)@ÀÔ)CÔDØØØØ%ð	ð 	ð ð	ð 	ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆØŸš ]Ñ3Ô3ˆÝ8Ø+Ø+ð
ñ 
ô 
ð 	
rB   rÂ  )NNNNNNNN)rV   rW   rX   r.   r…  r(   r  r-  r�  rY   r9   r'   r)   r"   r<   r*  r²   r	   r+  r1  r    r!   r   rP   rZ   r[   s   @rA   rè  rè  ¯  sš  ø€ € € € € € Ø!Ð!Ð!Ñ!à$�nÐ%<ÀAÐFÑFÔFØ*˜NÐ+BÈ!ÐLÑLÔLØ-ðð Ððð Ð4ð Àsð ð ð ð ð ð ð,  ØØð .2Ø.2Ø04Ø6:Ø26Ø!%Ø59Ø6:ð\
ð \
àÔ# dÑ*ð\
ð œ tÑ+ð\
ð Ô&¨Ñ-ð	\
ð
 -¨tÑ3ð\
ð Ô(¨4Ñ/ð\
ð ˜$‘;ð\
ð  %œ|¨dÑ2ð\
ð !&¤¨tÑ 3ð\
ð Ð+Ô,ð\
ð 
3ð\
ð \
ð \
ñ „^ñ „_ñ  Ôð\
ð \
ð \
ð \
ð \
rB   rè  c                   ó|  ‡ — e Zd ZdddœZdefˆ fd„Zddedz  fd„Zd	„ Zd
„ Z	d„ Z
ee	 	 	 	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dedz  dedz  dej        dz  dej        dz  dedz  dee         defd„¦   «         ¦   «         Zˆ xZS )ÚT5Gemma2Modelz&encoder.text_model.embed_tokens.weightz-encoder.text_model.embed_tokens.eoi_embedding)zdecoder.embed_tokens.weightz"decoder.embed_tokens.eoi_embeddingr^   c                 óî   •— t          ¦   «                              |¦  «         t          |j        |j        ¦  «        | _        t          |j        |j        ¦  «        | _        |                      ¦   «          d S r7   )r8   r9   rÄ  Úencoderra  rè  r{  r°  rm   s     €rA   r9   zT5Gemma2Model.__init__6  s_   ø€ Ý‰Œ×Ò˜Ñ Ô Ð õ ' v¤~°vÔ7MÑNÔNˆŒÝ& v¤~°vÔ7MÑNÔNˆŒà�ŠÑÔÐÐÐrB   NÚmodalityc                 ó   — | j         S r7   )rø  ©r?   rù  s     rA   Úget_encoderzT5Gemma2Model.get_encoder?  ó
   € ØŒ|ÐrB   c                 ó   — | j         S r7   ©r{  rT   s    rA   Úget_decoderzT5Gemma2Model.get_decoderB  rý  rB   c                 ó4   — | j                              ¦   «         S r7   )rø  rÍ  rT   s    rA   rÍ  z"T5Gemma2Model.get_input_embeddingsE  s   € ØŒ|×0Ò0Ñ2Ô2Ð2rB   c                 ó6   — | j                              |¦  «        S r7   )rø  rÏ  rÐ  s     rA   rÏ  z"T5Gemma2Model.set_input_embeddingsH  s   € ØŒ|×0Ò0°Ñ@Ô@Ð@rB   rf  rÒ  rÍ   rª   Údecoder_input_idsÚdecoder_attention_maskÚdecoder_position_idsÚencoder_outputsrù   r²  Údecoder_inputs_embedsr/  r—   r�   c                 óè   — |€ | j         d||||
|ddœ|¤Ž}|j        } | j        d|||||	|||ddœ	|¤Ž}t          |j        |j        |j        |j        |j        |j        |j        |j        ¬¦  «        S )aX  
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
            Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
            config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
        NT)rf  rÍ   rª   r²  rÒ  rÔ  )	rf  rÍ   rª   r²  rù   r	  rí  r/  rÔ  )rº  rù   Údecoder_hidden_statesÚdecoder_attentionsrê  Úencoder_last_hidden_stater	  Úencoder_attentionsr&  )rø  rº  r{  r   rù   ro   r   rê  )r?   rf  rÒ  rÍ   rª   r  r  r  r  rù   r²  r  r/  r—   r	  Údecoder_outputss                   rA   rP   zT5Gemma2Model.forwardK  sÛ   € ð6 Ð"Ø*˜dœlð Ø#Ø-Ø)Ø+Ø)Ø ðð ð ðð ˆOð !0Ô AÐð '˜$œ,ð 
Ø'Ø1Ø-Ø/Ø+Ø"7Ø#1ØØð
ð 
ð ð
ð 
ˆõ "Ø-Ô?Ø+Ô;Ø"1Ô"?Ø.Ô9Ø,Ô=Ø&5Ô&GØ"1Ô"?Ø.Ô9ð	
ñ 	
ô 	
ð 		
rB   r7   )NNNNNNNNNNNN)rV   rW   rX   Ú_tied_weights_keysr-   r9   r¥   rü  r   rÍ  rÏ  r#   r"   r<   r*  r+  Ú
BoolTensorr   r	   r²   r1  r    r!   r   rP   rZ   r[   s   @rA   rö  rö  /  sê  ø€ € € € € ð (PØ.]ðð Ðð
˜~ð ð ð ð ð ð ðð  C¨$¡Jð ð ð ð ðð ð ð3ð 3ð 3ðAð Að Að Øð .2Ø15Ø37Ø04à59Ø:>Ø8<à26Ø6:Ø-1Ø59Ø!%ð!=
ð =
ð Ô# dÑ*ð=
ð Ô'¨$Ñ.ð	=
ð
 Ô)¨DÑ0ð=
ð Ô&¨Ñ-ð=
ð !Ô+¨dÑ2ð=
ð !&Ô 0°4Ñ 7ð=
ð $Ô.°Ñ5ð=
ð )¨4Ñ/ð=
ð -¨tÑ3ð=
ð ”| dÑ*ð=
ð  %œ|¨dÑ2ð=
ð  ˜$‘;ð!=
ð" Ð+Ô,ð#=
ð$ 
ð%=
ð =
ð =
ñ „^ñ Ôð=
ð =
ð =
ð =
ð =
rB   rö  c            $       óp  ‡ — e Zd ZddiZddiZddgdgfiZdefˆ fd„Zd	„ Zd
„ Z	d„ Z
d„ Zd+dedz  fd„Zd„ Zeedej        dee         deez  fd„¦   «         ¦   «         Zed„ ¦   «         Zee	 	 	 	 	 	 	 	 	 	 	 	 	 	 d,dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dedz  dedz  dej        dz  d ej        dz  d!ej        dz  d"edz  d#e ej        z  dee         deej                 e!z  f d$„¦   «         ¦   «         Z"d%e#d&e$d'e%d(e d)e defˆ fd*„Z&ˆ xZ'S )-Ú T5Gemma2ForConditionalGenerationzlm_head.out_proj.weightz,model.encoder.text_model.embed_tokens.weightzlm_head.out_projÚcolwise_gather_outputro   r8  r^   c                 ó  •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        j        | _        t          |j        j        | j        ¦  «        | _        d| _	        |  
                    ¦   «          d S )NÚForMaskedLM)r8   r9   rö  rk  r{  r4  r3  rb   Úlm_headÚ	loss_typer°  rm   s     €rA   r9   z)T5Gemma2ForConditionalGeneration.__init__”  sk   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å" 6Ñ*Ô*ˆŒ
Ø œ.Ô3ˆŒÝ% f¤nÔ&@À$Ä/ÑRÔRˆŒØ&ˆŒà�ŠÑÔÐÐÐrB   c                 ó   — || j         _        d S r7   ©r  r6  rÐ  s     rA   Úset_output_embeddingsz6T5Gemma2ForConditionalGeneration.set_output_embeddingsž  s   € Ø .ˆŒÔÐÐrB   c                 ó   — | j         j        S r7   r  rT   s    rA   Úget_output_embeddingsz6T5Gemma2ForConditionalGeneration.get_output_embeddings¡  s   € ØŒ|Ô$Ð$rB   c                 ó4   — | j                              ¦   «         S r7   ©rk  rÍ  rT   s    rA   rÍ  z5T5Gemma2ForConditionalGeneration.get_input_embeddings¤  ó   € ØŒz×.Ò.Ñ0Ô0Ð0rB   c                 ó:   — | j                              |¦  «         d S r7   ©rk  rÏ  ©r?   rÌ   s     rA   rÏ  z5T5Gemma2ForConditionalGeneration.set_input_embeddings§  ó   € ØŒ
×'Ò'¨Ñ.Ô.Ð.Ð.Ð.rB   Nrù  c                 ó8   — | j                              |¬¦  «        S )N)rù  )rk  rü  rû  s     rA   rü  z,T5Gemma2ForConditionalGeneration.get_encoderª  s   € ØŒz×%Ò%¨xÐ%Ñ8Ô8Ð8rB   c                 ó4   — | j                              ¦   «         S r7   )rk  r   rT   s    rA   r   z,T5Gemma2ForConditionalGeneration.get_decoder­  s   € ØŒz×%Ò%Ñ'Ô'Ð'rB   rÒ  r—   r�   c                 óB   —  |                       ¦   «         j        |fi |¤ŽS r7   )rü  r×  )r?   rÒ  r—   s      rA   r×  z3T5Gemma2ForConditionalGeneration.get_image_features°  s+   € ð
 5ˆt×ÒÑ!Ô!Ô4°\ÐLÐLÀVÐLÐLÐLrB   c                 ó4   — |                       ¦   «         j        S r7   )rü  rÊ  rT   s    rA   rÊ  z-T5Gemma2ForConditionalGeneration.vision_tower·  s   € à×ÒÑ!Ô!Ô.Ð.rB   r   rf  rÍ   rª   r  r  r  r  rù   r²  r  ry  r/  Úlogits_to_keepc                 ó,  — |�|€|€|                       |¦  «        } | j        d|||||||||	|
||dœ|¤Ž}|j        }t          |t          ¦  «        rt          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }| j        j        }|j	        �(||j	        z  }t          j        |¦  «        }||j	        z  }d}|� | j        ||| j        fi |¤Ž}t          |||j        |j        |j        |j        |j        |j        |j        ¬¦	  «	        S )aü  
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
            Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
            config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        N)rf  rÒ  rÍ   rª   r  r  r  r  rù   r²  r  r/  )	Úlossr8  rù   r	  r
  rê  r  r	  r  r&  )r„  rk  rº  r£   rY   Úslicer  r^   r{  Úfinal_logit_softcappingr<   rÖ   Úloss_functionr4  r   rù   r	  r
  rê  r  r	  r  )r?   rf  rÒ  rÍ   rª   r  r  r  r  rù   r²  r  ry  r/  r'  r—   r  ro   Úslice_indicesr8  r�  r)  s                         rA   rP   z(T5Gemma2ForConditionalGeneration.forward»  sz  € ðB ÐÐ"3Ð";Ð@UÐ@]à $× JÒ JÈ6Ñ RÔ RÐà.8¨d¬jð /
ØØ%Ø)Ø%Ø/Ø#9Ø!5Ø+Ø+Ø'Ø"7Øð/
ð /
ð ð/
ð /
ˆð  (Ô9ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàœÔ,ˆØÔ1Ð=Ø˜nÔDÑDˆFÝ”Z Ñ'Ô'ˆFØ˜nÔDÑDˆFàˆØÐà%�4Ô% f¨f°d´oÐPÐPÈÐPÐPˆDåØØØ+Ô;Ø"1Ô"GØ.ÔAØ,Ô=Ø&5Ô&OØ"1Ô"GØ.ÔAð

ñ 

ô 

ð 
	
rB   Úgeneration_configÚmodel_kwargsÚgeneration_moderU  Úmax_cache_lengthc           	      óÀ  •— t          ¦   «                              |||||¦  «         |j        du rdS |j        }|€d}n	d|j        v }t	          j        | j                             d¬¦  «        ¦  «        }d|_        dg|j	        z  |_
        ||dœ}	|                     d¦  «        }
|
�¡t          |
t          ¦  «        st          d	¦  «        ‚t          |
j        ¦  «        d
k    r|
j                             d
¦  «        rdS t#          |
j        ¦  «        }|t&          k    r|d         d
         j        d         |	d<    |di |	¤Ž|
_        nEt          t+          di | j                             d¬¦  «        |dœ¤Žt+          ¦   «         ¦  «        |d<   t-          | d¦  «        r?| j        �:t          | j        t          ¦  «        st          d¦  «        ‚|d         | _        dS dS dS )zMOverride cache preparation to support T5Gemma2-specific EncoderDecoder Cache.FNÚ	offloadedTrÿ  r¹  )r^   Ú
offloadingrù   zaThe `past_key_values` in `model_kwargs` must be of type `EncoderDecoderCache` for T5Gemma2 model.r   r  r,   Úmax_cache_lenÚ_cachezLThe internal cache must be of type `EncoderDecoderCache` for T5Gemma2 model.r&  )r8   Ú_prepare_cache_for_generationr/  Úcache_implementationÚcopyÚdeepcopyr^   Úget_text_configrñ   r®  rƒ   r  r£   r	   r~  Úlenr  r¤   r  r
   rS   r   rç   r6  )r?   r.  r/  r0  rU  r1  r8  Úoffload_cacheÚcross_attn_configÚcross_attn_cache_kwargsrù   Úcross_attn_clsr@   s               €rA   r7  z>T5Gemma2ForConditionalGeneration._prepare_cache_for_generation  sN  ø€ õ 	‰Œ×-Ò-ØØØØØñ	
ô 	
ð 	
ð Ô&¨%Ð/Ð/ØˆFà0ÔEÐØÐ'Ø!ˆMˆMà'Ð+<Ô+QÐQˆMõ !œM¨$¬+×*EÒ*EÈdÐ*EÑ*SÔ*SÑTÔTÐð ,0ÐÔ(Ø)9Ð(:Ð=NÔ=`Ñ(`ÐÔ%ð (Ø'ð#
ð #
Ðð
 '×*Ò*Ð+<Ñ=Ô=ˆØÐ&Ý˜oÕ/BÑCÔCð Ý Øwñô ð õ
 �?Ô-Ñ.Ô.°Ò2Ð2°Ô7Q×7UÒ7UÐVWÑ7XÔ7XÐ2Ø�å! /Ô"GÑHÔHˆNØ¥Ò,Ð,Ø;GÐHYÔ;ZÐ[\Ô;]Ô;cÐdeÔ;fÐ'¨Ñ8à4B°NÐ4]Ð4]ÐE\Ð4]Ð4]ˆOÔ1Ð1õ /BÝð ð à"&¤+×"=Ò"=ÀdÐ"=Ñ"KÔ"KØ&3ðð ðð õ ‘”ñ/ô /ˆLÐ*Ñ+õ �4˜Ñ"Ô"ð 	: t¤{Ð'>Ý˜dœkÕ+>Ñ?Ô?ð qÝ Ð!oÑpÔpÐpà&Ð'8Ô9ˆDŒKˆKˆKð		:ð 	:Ð'>Ð'>rB   r7   )NNNNNNNNNNNNNr   )(rV   rW   rX   r  Ú_tp_planÚ_pp_planr-   r9   r  r  rÍ  rÏ  r¥   rü  r   r#   r"   r<   r²   r    r!   rR   r   r×  ÚpropertyrÊ  r*  r+  r  r   r	   r1  rY   r   rP   r   r¼  r   r7  rZ   r[   s   @rA   r  r  �  sG  ø€ € € € € à!Ð#QðÐð #Ð$;Ð<€HØ" oÐ%6¸¸
Ð$CÐD€Hð˜~ð ð ð ð ð ð ð/ð /ð /ð%ð %ð %ð1ð 1ð 1ð/ð /ð /ð9ð 9 C¨$¡Jð 9ð 9ð 9ð 9ð(ð (ð (ð ØðMØ!œLðMØ4:Ð;MÔ4NðMà	Ð+Ñ	+ðMð Mð Mñ „^ñ ÔðMð
 ð/ð /ñ „Xð/ð Øð .2Ø15Ø37Ø04à59Ø:>Ø8<à26Ø6:Ø26Ø:>Ø*.Ø!%Ø-.ð%M
ð M
ð Ô# dÑ*ðM
ð Ô'¨$Ñ.ð	M
ð
 Ô)¨DÑ0ðM
ð Ô&¨Ñ-ðM
ð !Ô+¨dÑ2ðM
ð !&Ô 0°4Ñ 7ðM
ð $Ô.°Ñ5ðM
ð )¨4Ñ/ðM
ð -¨tÑ3ðM
ð Ô(¨4Ñ/ðM
ð  %Ô0°4Ñ7ðM
ð  Ô  4Ñ'ð!M
ð" ˜$‘;ð#M
ð$ ˜eœlÑ*ð%M
ð& Ð+Ô,ð'M
ð( 
ˆuÔ Ô	! OÑ	3ð)M
ð M
ð M
ñ „^ñ ÔðM
ð^I:à+ðI:ð ðI:ð (ð	I:ð
 ðI:ð ðI:ð 
ðI:ð I:ð I:ð I:ð I:ð I:ð I:ð I:ð I:ð I:rB   r  c                   óV  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 	 	 	 dde	j
        dz  de	j        dz  de	j        dz  d	e	j
        dz  d
e	j
        dz  de	j        dz  de	j
        dz  dedz  de	j        dz  de	j        dz  de	j
        dz  dee         defd„¦   «         ¦   «         Zˆ xZS )Ú!T5Gemma2ForSequenceClassificationr^   c                 ó6  •— t          ¦   «                              |¦  «         |j        | _        |j        j        | _        t          |¦  «        | _        t          |dd¦  «        }t          | j        | j        |¦  «        | _	        |  
                    ¦   «          d S ©Nr<  gš™™™™™¹?©r8   r9   r;  r{  rb   rö  rk  r“   r:  Úscorer°  ©r?   r^   Úclassifier_dropoutr@   s      €rA   r9   z*T5Gemma2ForSequenceClassification.__init__Z  sƒ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒØ!œ>Ô5ˆÔå" 6Ñ*Ô*ˆŒ
å$ VÐ-FÈÑLÔLÐÝ/°Ô0@À$Ä/ÐSeÑfÔfˆŒ
Ø�ŠÑÔÐÐÐrB   c                 ó4   — | j                              ¦   «         S r7   r  rT   s    rA   rÍ  z6T5Gemma2ForSequenceClassification.get_input_embeddingse  r  rB   c                 ó:   — | j                              |¦  «         d S r7   r   r!  s     rA   rÏ  z6T5Gemma2ForSequenceClassification.set_input_embeddingsh  r"  rB   Nrf  rÒ  rÍ   rª   r  r  r  r  r²  r  ry  r—   r�   c                 ó$  — |	€|
�t          d| j        j        › d�¦  «        ‚|€t          d¦  «        ‚|€|                      |¦  «        } | j        |f||||||||	|
ddœ
|¤Ž}|j        }|j        }|j        }|  	                    |¦  «        }|j
        d         }|| j        j        k                         |j        t          j        ¦  «        }t          j        |j
        d         |j        t          j        ¬	¦  «        }||z                       d¦  «        }t          j        ||j
        d         d
z
  ¬¦  «        }|t          j        ||j        ¬¦  «        |f         }d}|�|                      |||| j        ¬¦  «        }t+          ||||¬¦  «        S )áÜ  
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
            Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
            config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nú8Passing input embeddings is currently not supported for ú.úYou have to specify input_idsF©
rÒ  rÍ   rª   r  r  r  r  r²  r  r/  r   rD   )rr   r’   r,   )Úmaxr¶  )r8  ry  Úpooled_logitsr^   ©r)  r8  ro   r   )ÚNotImplementedErrorr@   rV   r~  r„  rk  rº  r	  r
  rI  rS   r^   r}  r–   rr   r<   Úint32r•   ÚargmaxÚclampr,  r   )r?   rf  rÒ  rÍ   rª   r  r  r  r  r²  r  ry  r—   ræ  rº  ro   r   r8  rU  Únon_pad_maskÚtoken_indicesÚlast_non_pad_tokenrU  r)  s                           rA   rP   z)T5Gemma2ForSequenceClassification.forwardk  sà  € ð4 Ð$Ð(=Ð(IÝ%ØeÈ4Ì>ÔKbÐeÐeÐeñô ð ð ÐÝÐ<Ñ=Ô=Ð=àÐ$Ø $× JÒ JÈ9Ñ UÔ UÐà&0 d¤jØð'
à%Ø)Ø%Ø/Ø#9Ø!5Ø+Ø'Ø"7Øð'
ð '
ð ð'
ð '
ˆð $Ô5ÐØÔ5ˆØÔ/ˆ
à—’Ð-Ñ.Ô.ˆà”_ QÔ'ˆ
à)¨T¬[Ô-EÒE×IÒIÈ&Ì-ÕY^ÔYdÑeÔeˆÝœÐ%6Ô%<¸RÔ%@ÈÌÕ^cÔ^iÐjÑjÔjˆØ+¨lÑ:×BÒBÀ2ÑFÔFÐÝ"œ[Ð);ÐARÔAXÐY[ÔA\Ð_`ÑA`ÐaÑaÔaÐà�uœ|¨J¸v¼}ÐMÑMÔMÐOaÐaÔbˆàˆØÐØ×%Ò%¨V¸FÐR_ÐhlÔhsÐ%ÑtÔtˆDå'ØØ Ø'Ø!ð	
ñ 
ô 
ð 	
rB   ©NNNNNNNNNNN)rV   rW   rX   r-   r9   rÍ  rÏ  r#   r"   r<   r*  r+  r²   r   r    r!   r   rP   rZ   r[   s   @rA   rE  rE  X  s¤  ø€ € € € € ð	˜~ð 	ð 	ð 	ð 	ð 	ð 	ð1ð 1ð 1ð/ð /ð /ð Øð .2Ø15Ø.2Ø04Ø59Ø6:Ø8<Ø26Ø26Ø:>Ø*.ðJ
ð J
àÔ# dÑ*ðJ
ð Ô'¨$Ñ.ðJ
ð œ tÑ+ð	J
ð
 Ô&¨Ñ-ðJ
ð !Ô+¨dÑ2ðJ
ð !&¤¨tÑ 3ðJ
ð $Ô.°Ñ5ðJ
ð )¨4Ñ/ðJ
ð Ô(¨4Ñ/ðJ
ð  %Ô0°4Ñ7ðJ
ð Ô  4Ñ'ðJ
ð Ð+Ô,ðJ
ð 
"ðJ
ð J
ð J
ñ „^ñ ÔðJ
ð J
ð J
ð J
ð J
rB   rE  c                   óV  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 	 	 	 dde	j
        dz  de	j        dz  de	j        dz  d	e	j
        dz  d
e	j
        dz  de	j        dz  de	j
        dz  dedz  de	j        dz  de	j        dz  de	j
        dz  dee         defd„¦   «         ¦   «         Zˆ xZS )ÚT5Gemma2ForTokenClassificationr^   c                 ó6  •— t          ¦   «                              |¦  «         |j        | _        |j        j        | _        t          |¦  «        | _        t          |dd¦  «        }t          | j        | j        |¦  «        | _	        |  
                    ¦   «          d S rG  rH  rJ  s      €rA   r9   z'T5Gemma2ForTokenClassification.__init__¼  sƒ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒØ!œ>Ô5ˆÔå" 6Ñ*Ô*ˆŒ
å$ VÐ-FÈÑLÔLÐÝ/°Ô0@À$Ä/ÐSeÑfÔfˆŒ
à�ŠÑÔÐÐÐrB   c                 ó4   — | j                              ¦   «         S r7   r  rT   s    rA   rÍ  z3T5Gemma2ForTokenClassification.get_input_embeddingsÈ  r  rB   c                 ó:   — | j                              |¦  «         d S r7   r   r!  s     rA   rÏ  z3T5Gemma2ForTokenClassification.set_input_embeddingsË  r"  rB   Nrf  rÒ  rÍ   rª   r  r  r  r  r²  r  ry  r—   r�   c                 ó€  — |	€|
�t          d| j        j        › d�¦  «        ‚|€t          d¦  «        ‚|€|                      |¦  «        } | j        |f||||||||	|
ddœ
|¤Ž}|j        }|j        }|j        }|  	                    |¦  «        }d}|�|  
                    ||| j        ¦  «        }t          ||||¬¦  «        S )rO  NrP  rQ  rR  FrS  rV  )rW  r@   rV   r~  r„  rk  rº  r	  r
  rI  r,  r^   r   )r?   rf  rÒ  rÍ   rª   r  r  r  r  r²  r  ry  r—   ræ  rº  ro   r   r8  r)  s                      rA   rP   z&T5Gemma2ForTokenClassification.forwardÎ  s"  € ð4 Ð$Ð(=Ð(IÝ%ØeÈ4Ì>ÔKbÐeÐeÐeñô ð ð ÐÝÐ<Ñ=Ô=Ð=àÐ$Ø $× JÒ JÈ9Ñ UÔ UÐà&0 d¤jØð'
à%Ø)Ø%Ø/Ø#9Ø!5Ø+Ø'Ø"7Øð'
ð '
ð ð'
ð '
ˆð $Ô5ÐØÔ5ˆØÔ/ˆ
à—’Ð-Ñ.Ô.ˆàˆØÐØ×%Ò% f¨f°d´kÑBÔBˆDå$ØØØ'Ø!ð	
ñ 
ô 
ð 	
rB   r^  )rV   rW   rX   r-   r9   rÍ  rÏ  r#   r"   r<   r*  r+  r²   r   r    r!   r   rP   rZ   r[   s   @rA   r`  r`  º  s¤  ø€ € € € € ð
˜~ð 
ð 
ð 
ð 
ð 
ð 
ð1ð 1ð 1ð/ð /ð /ð Øð .2Ø15Ø.2Ø04Ø59Ø6:Ø8<Ø26Ø26Ø:>Ø*.ð@
ð @
àÔ# dÑ*ð@
ð Ô'¨$Ñ.ð@
ð œ tÑ+ð	@
ð
 Ô&¨Ñ-ð@
ð !Ô+¨dÑ2ð@
ð !&¤¨tÑ 3ð@
ð $Ô.°Ñ5ð@
ð )¨4Ñ/ð@
ð Ô(¨4Ñ/ð@
ð  %Ô0°4Ñ7ð@
ð Ô  4Ñ'ð@
ð Ð+Ô,ð@
ð 
ð@
ð @
ð @
ñ „^ñ Ôð@
ð @
ð @
ð @
ð @
rB   r`  )r  rö  rÄ  rj  rE  r`  )r,   )rÈ   NN)T)br9  Úcollections.abcr   r<   Útorch.nnr:   Ú r   rr  Úactivationsr   Úcache_utilsr   r   r	   r
   Ú
generationr   r   r   Úintegrationsr   r   Úmasking_utilsr   r   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr    Úutilsr!   r"   r#   r$   Úutils.deprecationr%   Úutils.genericr&   r'   Úutils.output_capturingr(   r)   Úautor+   Úconfiguration_t5gemma2r-   r.   r/   r0   ÚModuler2   r]   rq   r·   rÀ   r²   rY   rÇ   rM   rR   rß   rá   r  r  r-  r3  r:  rA  Ú	Embeddingr[  rj  r�  rŸ  rÄ  rè  rö  r  rE  r`  Ú__all__r&  rB   rA   ú<module>r|     s›  ðð* €€€Ø $Ð $Ð $Ð $Ð $Ð $à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø PÐ PÐ PÐ PÐ PÐ PÐ PÐ PÐ PÐ PÐ PÐ PØ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ LÐ LÐ LÐ LÐ LÐ LÐ LÐ LØ mÐ mÐ mÐ mÐ mÐ mÐ mÐ mÐ mÐ mØ BÐ BÐ BÐ BÐ BÐ BØ 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð LÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aØ 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ Ð Ð Ð Ð Ð Ø tÐ tÐ tÐ tÐ tÐ tÐ tÐ tÐ tÐ tÐ tÐ tð=ð =ð =ð =ð =�b”iñ =ô =ð =ð(ð ð ð ð �"”)ñ ô ð ð&E<ð E<ð E<ð E<ð E<˜bœiñ E<ô E<ð E<ðP(ð (ð (ð ÐÐ-Ñ.Ô.ðð ð ñ /Ô.ðð2	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð$ Ø Ø ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð �S‰[ð%ð �T‰\ð%ð �T‰\ð%ð ˆ5Œ<˜œÐ%Ô&ð%ð %ð %ð %ðD ÐÐ)Ñ*Ô*ðK)ð K)ð K)ð K)ð K)˜BœIñ K)ô K)ñ +Ô*ðK)ð\ ÐÐ)Ñ*Ô*ðuBð uBð uBð uBð uB˜bœiñ uBô uBñ +Ô*ðuBðp1ð 1ð 1ð 1ð 1Ð5ñ 1ô 1ð 1ðh8ð 8ð 8ð 8ð 8Ð5ñ 8ô 8ð 8ðv	ð 	ð 	ð 	ð 	�R”Yñ 	ô 	ð 	ðð ð ð ð  ¤ñ ô ð ð!@ð !@ð !@ð !@ð !@ "¤)ñ !@ô !@ð !@ðH ð  ð  ð  ð   b¤lñ  ô  ð  ð. ðJ!ð J!ð J!ð J!ð J!˜oñ J!ô J!ñ „ðJ!ðZð °ð Èð ð ð ð ð&b
ð b
ð b
ð b
ð b
Ð1ñ b
ô b
ð b
ðJeð eð eð eð eÐ-ñ eô eð eðP}
ð }
ð }
ð }
ð }
Ð-ñ }
ô }
ð }
ð@ ðZ
ð Z
ð Z
ð Z
ð Z
Ð+ñ Z
ô Z
ñ „ðZ
ðzH:ð H:ð H:ð H:ð H:Ð'>Àñ H:ô H:ð H:ðV ð^
ð ^
ð ^
ð ^
ð ^
Ð(?ñ ^
ô ^
ñ „ð^
ðB ðU
ð U
ð U
ð U
ð U
Ð%<ñ U
ô U
ñ „ðU
ðpð ð €€€rB   