§
    9²¸jo ã                   óª  — d Z ddlZddlZddlmZ ddlZddlmZ ddlmZm	Z	m
Z
 ddlmZ ddlmZ dd	lmZmZmZ dd
lmZ ddlmZmZ ddlmZ ddlmZmZmZmZm Z m!Z!m"Z" ddl#m$Z$m%Z% ddl&m'Z' ddl(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z. ddl/m0Z0m1Z1 ddl2m3Z3m4Z4 ddl5m6Z6  e-j7        e8¦  «        Z9	 	 	 dFdej:        dej;        dej;        dej;        dej;        dz  de<dz  de<dej;        dz  de'e+         fd„Z= G d „ d!ej:        ¦  «        Z> G d"„ d#ej:        ¦  «        Z? G d$„ d%ej:        ¦  «        Z@ G d&„ d'ej:        ¦  «        ZA G d(„ d)ej:        ¦  «        ZB G d*„ d+ej:        ¦  «        ZC G d,„ d-ej:        ¦  «        ZD G d.„ d/e¦  «        ZE G d0„ d1ej:        ¦  «        ZFe, G d2„ d3e%¦  «        ¦   «         ZG G d4„ d5eG¦  «        ZHe, G d6„ d7eG¦  «        ¦   «         ZI e,d8¬9¦  «         G d:„ d;eGe¦  «        ¦   «         ZJe, G d<„ d=eG¦  «        ¦   «         ZK e,d>¬9¦  «         G d?„ d@eG¦  «        ¦   «         ZLe, G dA„ dBeG¦  «        ¦   «         ZMe, G dC„ dDeG¦  «        ¦   «         ZNg dE¢ZOdS )GzPyTorch mT5 model.é    N)ÚCallable)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutputÚ#Seq2SeqQuestionAnsweringModelOutputÚSeq2SeqSequenceClassifierOutputÚTokenClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚDUMMY_INPUTSÚ
DUMMY_MASKÚTransformersKwargsÚauto_docstringÚloggingÚtorch_compilable_check)Úcan_return_tupleÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )Ú	MT5Configç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚposition_biasÚkwargsc                 ó¼  — |€|                      d¦  «        dz  }t          j        ||                     dd¦  «        ¦  «        |z  }	|�|	|z   }	|�|	|z   }	t          j                             |	d¬¦  «        }	t          j                             |	|| j        ¬¦  «        }	t          j        |	|¦  «        }
|
                     dd¦  «         	                    ¦   «         }
|
|	fS )Néÿÿÿÿç      à¿é   r   ©Údim)ÚpÚtrainingr&   )
ÚsizeÚtorchÚmatmulÚ	transposer   Ú
functionalÚsoftmaxr/   r9   Ú
contiguous)r)   r*   r+   r,   r-   r.   r/   r0   r1   Úattn_weightsÚattn_outputs              út/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/mt5/modeling_mt5.pyÚeager_attention_forwardrD   3   sØ   € ð €Ø—*’*˜R‘.”. DÑ(ˆõ ”<  s§}¢}°Q¸Ñ':Ô':Ñ;Ô;¸gÑE€LàÐ Ø# mÑ3ˆàÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2Ð(Ñ>Ô>€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$ó    c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚMT5LayerNormç�íµ ÷Æ°>c                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )zd
        Construct a layernorm module in the MT5 style. No bias and no subtraction of mean.
        N)ÚsuperÚ__init__r   Ú	Parameterr;   ÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizeÚepsÚ	__class__s      €rC   rK   zMT5LayerNorm.__init__U   sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐrE   c                 óh  — |                      t          j        ¦  «                             d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        j        t          j	        t          j
        fv r|                      | j        j        ¦  «        }| j        |z  S )Nr5   r3   T)Úkeepdim)Útor;   Úfloat32ÚpowÚmeanÚrsqrtrO   rN   ÚdtypeÚfloat16Úbfloat16)rP   Úhidden_statesÚvariances      rC   ÚforwardzMT5LayerNorm.forward]   s–   € ð !×#Ò#¥E¤MÑ2Ô2×6Ò6°qÑ9Ô9×>Ò>¸rÈ4Ð>ÑPÔPˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆð Œ;Ô¥¤µ´Ð ?Ð?Ð?Ø)×,Ò,¨T¬[Ô->Ñ?Ô?ˆMàŒ{˜]Ñ*Ð*rE   )rH   )Ú__name__Ú
__module__Ú__qualname__rK   r`   Ú__classcell__©rS   s   @rC   rG   rG   T   sL   ø€ € € € € ð$ð $ð $ð $ð $ð $ð+ð +ð +ð +ð +ð +ð +rE   rG   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚMT5DenseActDenseÚconfigc                 óJ  •— t          ¦   «                              ¦   «          t          j        |j        |j        d¬¦  «        | _        t          j        |j        |j        d¬¦  «        | _        t          j        |j	        ¦  «        | _
        t          |j                 | _        d S ©NF©Úbias)rJ   rK   r   ÚLinearÚd_modelÚd_ffÚwiÚwoÚDropoutÚdropout_rater/   r
   Údense_act_fnÚact©rP   rh   rS   s     €rC   rK   zMT5DenseActDense.__init__o   sx   ø€ Ý‰Œ×ÒÑÔÐÝ”)˜FœN¨F¬K¸eÐDÑDÔDˆŒÝ”)˜FœK¨¬¸eÐDÑDÔDˆŒÝ”z &Ô"5Ñ6Ô6ˆŒÝ˜&Ô-Ô.ˆŒˆˆrE   c                 ó°  — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }t          | j        j        t          j        ¦  «        r]|j        | j        j        j        k    rC| j        j        j        t          j	        k    r$| 
                    | j        j        j        ¦  «        }|                      |¦  «        }|S ©N)rp   ru   r/   Ú
isinstancerq   rN   r;   ÚTensorr[   Úint8rV   ©rP   r^   s     rC   r`   zMT5DenseActDense.forwardv   s¨   € ØŸš Ñ.Ô.ˆØŸš Ñ/Ô/ˆØŸš ]Ñ3Ô3ˆå�t”w”~¥u¤|Ñ4Ô4ð	CàÔ# t¤w¤~Ô';Ò;Ð;Ø””Ô$­¬
Ò2Ð2à)×,Ò,¨T¬W¬^Ô-AÑBÔBˆMØŸš Ñ.Ô.ˆØÐrE   ©ra   rb   rc   r'   rK   r`   rd   re   s   @rC   rg   rg   n   sS   ø€ € € € € ð/˜yð /ð /ð /ð /ð /ð /ðð ð ð ð ð ð rE   rg   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚMT5DenseGatedActDenserh   c                 ó–  •— t          ¦   «                              ¦   «          t          j        |j        |j        d¬¦  «        | _        t          j        |j        |j        d¬¦  «        | _        t          j        |j        |j        d¬¦  «        | _        t          j	        |j
        ¦  «        | _        t          |j                 | _        d S rj   )rJ   rK   r   rm   rn   ro   Úwi_0Úwi_1rq   rr   rs   r/   r
   rt   ru   rv   s     €rC   rK   zMT5DenseGatedActDense.__init__†   s”   ø€ Ý‰Œ×ÒÑÔÐÝ”I˜fœn¨f¬kÀÐFÑFÔFˆŒ	Ý”I˜fœn¨f¬kÀÐFÑFÔFˆŒ	Ý”)˜FœK¨¬¸eÐDÑDÔDˆŒÝ”z &Ô"5Ñ6Ô6ˆŒÝ˜&Ô-Ô.ˆŒˆˆrE   c                 óà  — |                       |                      |¦  «        ¦  «        }|                      |¦  «        }||z  }|                      |¦  «        }t	          | j        j        t          j        ¦  «        r]|j	        | j        j        j	        k    rC| j        j        j	        t          j
        k    r$|                     | j        j        j	        ¦  «        }|                      |¦  «        }|S rx   )ru   r�   r‚   r/   ry   rq   rN   r;   rz   r[   r{   rV   )rP   r^   Úhidden_geluÚhidden_linears       rC   r`   zMT5DenseGatedActDense.forwardŽ   sÀ   € Ø—h’h˜tŸyšy¨Ñ7Ô7Ñ8Ô8ˆØŸ	š	 -Ñ0Ô0ˆØ# mÑ3ˆØŸš ]Ñ3Ô3ˆõ �t”w”~¥u¤|Ñ4Ô4ð	CàÔ# t¤w¤~Ô';Ò;Ð;Ø””Ô$­¬
Ò2Ð2à)×,Ò,¨T¬W¬^Ô-AÑBÔBˆMàŸš Ñ.Ô.ˆØÐrE   r}   re   s   @rC   r   r   …   sS   ø€ € € € € ð/˜yð /ð /ð /ð /ð /ð /ðð ð ð ð ð ð rE   r   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )Ú
MT5LayerFFrh   c                 ó$  •— t          ¦   «                              ¦   «          |j        rt          |¦  «        | _        nt          |¦  «        | _        t          |j        |j        ¬¦  «        | _	        t          j        |j        ¦  «        | _        d S )N©rR   )rJ   rK   Úis_gated_actr   ÚDenseReluDenserg   rG   rn   Úlayer_norm_epsilonÚ
layer_normr   rr   rs   r/   rv   s     €rC   rK   zMT5LayerFF.__init__¤   sx   ø€ Ý‰Œ×ÒÑÔÐØÔð 	;Ý"7¸Ñ"?Ô"?ˆDÔÐå"2°6Ñ":Ô":ˆDÔå& v¤~¸6Ô;TÐUÑUÔUˆŒÝ”z &Ô"5Ñ6Ô6ˆŒˆˆrE   c                 óŠ   — |                       |¦  «        }|                      |¦  «        }||                      |¦  «        z   }|S rx   )r�   r‹   r/   )rP   r^   Úforwarded_statess      rC   r`   zMT5LayerFF.forward®   sF   € ØŸ?š?¨=Ñ9Ô9ÐØ×.Ò.Ð/?Ñ@Ô@ÐØ%¨¯ªÐ5EÑ(FÔ(FÑFˆØÐrE   r}   re   s   @rC   r‡   r‡   £   sS   ø€ € € € € ð7˜yð 7ð 7ð 7ð 7ð 7ð 7ðð ð ð ð ð ð rE   r‡   c                   ó|   ‡ — e Zd Z	 	 	 ddededz  defˆ fd„Zedd
„¦   «         Zdd„Z		 	 	 	 dde
e         fd„Zˆ xZS )ÚMT5AttentionFNrh   Ú	layer_idxÚ	is_causalc                 óT  •— t          ¦   «                              ¦   «          || _        |j        | _        || _        || _        |j        | _        |j        | _        |j        | _        |j	        | _
        |j        | _        |j        | _        | j        | j
        z  | _        d| _        || _        |€/| j        r(t$                               d| j        j        › d�¦  «         t-          j        | j        | j        d¬¦  «        | _        t-          j        | j        | j        d¬¦  «        | _        t-          j        | j        | j        d¬¦  «        | _        t-          j        | j        | j        d¬¦  «        | _        | j        r$t-          j        | j        | j        ¦  «        | _        d| _        d S )Nç      ð?zInstantiating a decoder z³ without passing `layer_idx` is not recommended and will to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` when creating this class.Frk   )rJ   rK   rh   Ú
is_decoderr“   Úhas_relative_attention_biasÚrelative_attention_num_bucketsÚrelative_attention_max_distancern   Úd_kvÚkey_value_proj_dimÚ	num_headsÚn_headsrs   r/   Ú	inner_dimr.   r’   ÚloggerÚwarning_oncerS   ra   r   rm   ÚqÚkÚvÚoÚ	EmbeddingÚrelative_attention_biasÚgradient_checkpointing)rP   rh   r—   r’   r“   rS   s        €rC   rK   zMT5Attention.__init__·   s~  ø€ õ 	‰Œ×ÒÑÔÐØˆŒØ Ô+ˆŒØ"ˆŒØ+FˆÔ(Ø.4Ô.SˆÔ+Ø/5Ô/UˆÔ,Ø”~ˆŒØ"(¤+ˆÔØÔ'ˆŒØÔ*ˆŒØœ¨Ô(?Ñ?ˆŒàˆŒØ"ˆŒØÐ ¤ÐÝ×Òð,¨4¬>Ô+Bð ,ð ,ð ,ñô ð õ ”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ>¨4¬<¸eÐDÑDÔDˆŒàÔ+ð 	kÝ+-¬<¸Ô8[Ð]aÔ]iÑ+jÔ+jˆDÔ(à&+ˆÔ#Ð#Ð#rE   Té    é€   c                 óP  — d}|rC|dz  }|| dk                          t          j        ¦  «        |z  z  }t          j        | ¦  «        } n(t          j        | t          j        | ¦  «        ¦  «         } |dz  }| |k     }|t          j        |                      ¦   «         |z  ¦  «        t          j        ||z  ¦  «        z  ||z
  z                        t          j        ¦  «        z   }t          j        |t          j	        ||dz
  ¦  «        ¦  «        }|t          j
        || |¦  «        z  }|S )aÒ  
        Adapted from Mesh Tensorflow:
        https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593

        Translate relative position to a bucket number for relative attention. The relative position is defined as
        memory_position - query_position, i.e. the distance in tokens from the attending position to the attended-to
        position. If bidirectional=False, then positive relative positions are invalid. We use smaller buckets for
        small absolute relative_position and larger buckets for larger absolute relative_positions. All relative
        positions >=max_distance map to the same bucket. All relative positions <=-max_distance map to the same bucket.
        This should allow for more graceful generalization to longer sequences than the model has been trained on

        Args:
            relative_position: an int32 Tensor
            bidirectional: a boolean - whether the attention is bidirectional
            num_buckets: an integer
            max_distance: an integer

        Returns:
            a Tensor with the same shape as relative_position, containing int32 values in the range [0, num_buckets)
        r   r5   r&   )rV   r;   ÚlongÚabsÚminÚ
zeros_likeÚlogÚfloatÚmathÚ	full_likeÚwhere)Úrelative_positionÚbidirectionalÚnum_bucketsÚmax_distanceÚrelative_bucketsÚ	max_exactÚis_smallÚrelative_position_if_larges           rC   Ú_relative_position_bucketz&MT5Attention._relative_position_bucketÞ   s>  € ð, ÐØð 	cØ˜AÑˆKØÐ!2°QÒ!6× :Ò :½5¼:Ñ FÔ FÈÑ TÑTÐÝ %¤	Ð*;Ñ <Ô <ÐÐå!&¤Ð+<½eÔ>NÐO`Ñ>aÔ>aÑ!bÔ!bÐ bÐð   1Ñ$ˆ	Ø$ yÒ0ˆð &/ÝŒIÐ'×-Ò-Ñ/Ô/°)Ñ;Ñ<Ô<ÝŒh�| iÑ/Ñ0Ô0ñ1à˜YÑ&ñ(÷ Š"�UŒZ‰.Œ.ñ	&Ð"õ
 &+¤YØ&­¬Ð8RÐT_ÐbcÑTcÑ(dÔ(dñ&
ô &
Ð"ð 	�EœK¨Ð2CÐE_Ñ`Ô`Ñ`ÐØÐrE   r   c                 ó¸  — |€| j         j        j        }t          j        |t          j        |¬¦  «        dd…df         |z   }t          j        |t          j        |¬¦  «        ddd…f         }||z
  }|                      || j         | j        | j	        ¬¦  «        }|                       |¦  «        }	|	 
                    g d¢¦  «                             d¦  «        }	|	S )z%Compute binned relative position biasN)r[   Údevice)rµ   r¶   r·   )r5   r   r&   r   )r¦   rN   r¾   r;   Úaranger«   r¼   r–   r˜   r™   ÚpermuteÚ	unsqueeze)
rP   Úquery_lengthÚ
key_lengthr¾   Úpast_seen_tokensÚcontext_positionÚmemory_positionr´   Úrelative_position_bucketÚvaluess
             rC   Úcompute_biaszMT5Attention.compute_bias  sí   € àˆ>ØÔ1Ô8Ô?ˆFÝ œ<¨½E¼JÈvÐVÑVÔVÐWXÐWXÐWXÐZ^ÐW^Ô_ÐbrÑrÐÝœ, z½¼ÈFÐSÑSÔSÐTXÐZ[ÐZ[ÐZ[ÐT[Ô\ˆØ+Ð.>Ñ>ÐØ#'×#AÒ#AØØ#œÐ.ØÔ;ØÔ=ð	 $Bñ $
ô $
Ð ð ×-Ò-Ð.FÑGÔGˆØ—’ 	 	 	Ñ*Ô*×4Ò4°QÑ7Ô7ˆØˆrE   r1   c                 ó"  — |j         dd…         }g |¢d‘| j        ‘R }|�|                     | j        ¦  «        nd}	t	          |	t
          j        ¦  «        r|	                     ¦   «         n|	}	|du}
|                      |¦  «         	                    |¦  «         
                    dd¦  «        }d}t	          |t          ¦  «        r1|j                             | j        ¦  «        }|
r|j        }n
|j        }n|}|
r|n|}|
r3|�1|r/|j        | j                 j        }|j        | j                 j        }nÚg |j         dd…         ¢d‘| j        ‘R }|                      |¦  «         	                    |¦  «         
                    dd¦  «        }|                      |¦  «         	                    |¦  «         
                    dd¦  «        }|�E|                     ||| j        ¦  «        \  }}|
r$t	          |t          ¦  «        rd|j        | j        <   |€~|j         d         }| j        sFt          j        d| j        |d         |f|j        |j        ¬	¦  «        }| j        r| j        rd|_        n$|                      |d         ||j        |	¬
¦  «        }t?          j         | j!        j"        tF          ¦  «        } || ||||f| j        sdn| j$        | j%        |dœ|¤Ž\  }} |j&        g |¢d‘R Ž  '                    ¦   «         }|  (                    |¦  «        }|||fS )z€
        Self-attention (if key_value_states is None) or attention over source sentence (provided by key_value_states).
        Nr3   r   r&   r5   FTéþÿÿÿ)r¾   r[   )r¾   rÄ   r(   )r/   r.   r0   ))Úshaper›   Úget_seq_lengthr’   ry   r;   rz   Úcloner¡   Úviewr=   r   Ú
is_updatedÚgetÚcross_attention_cacheÚself_attention_cacheÚlayersÚkeysrÈ   r¢   r£   Úupdater—   Úzerosr�   r¾   r[   r§   r9   Úrequires_gradrÉ   r   Úget_interfacerh   Ú_attn_implementationrD   r/   r.   Úreshaper@   r¤   )rP   r^   ÚmaskÚkey_value_statesr0   Úpast_key_valuesr1   Úinput_shapeÚhidden_shaperÄ   Úis_cross_attentionÚquery_statesrÐ   Úcurr_past_key_valuesÚcurrent_statesÚ
key_statesÚvalue_statesÚkv_shaperÃ   Úattention_interfacerB   rA   s                         rC   r`   zMT5Attention.forward  s�  € ð $Ô)¨#¨2¨#Ô.ˆØB˜ÐB bÐB¨$Ô*AÐBÐBˆØM\ÐMh˜?×9Ò9¸$¼.ÑIÔIÐIÐnoÐå7AÐBRÕTYÔT`Ñ7aÔ7aÐwÐ+×1Ò1Ñ3Ô3Ð3ÐgwÐð .°TÐ9Ðà—v’v˜mÑ,Ô,×1Ò1°,Ñ?Ô?×IÒIÈ!ÈQÑOÔOˆð ˆ
Ý�oÕ':Ñ;Ô;ð 	3Ø(Ô3×7Ò7¸¼ÑGÔGˆJØ!ð Là'6Ô'LÐ$Ð$à'6Ô'KÐ$Ð$à#2Ð à-?ÐRÐ)Ð)À]ˆØð 	F /Ð"=À*Ð"=à-Ô4°T´^ÔDÔIˆJØ/Ô6°t´~ÔFÔMˆLˆLàP˜Ô-¨c¨r¨cÔ2ÐP°BÐP¸Ô8OÐPÐPˆHØŸš Ñ/Ô/×4Ò4°XÑ>Ô>×HÒHÈÈAÑNÔNˆJØŸ6š6 .Ñ1Ô1×6Ò6°xÑ@Ô@×JÒJÈ1ÈaÑPÔPˆLàÐ*Ø+?×+FÒ+FÀzÐS_ÐaeÔaoÑ+pÔ+pÑ(�
˜Là%ð F­*°_ÕFYÑ*ZÔ*Zð FØAE�OÔ.¨t¬~Ñ>ð Ð Ø#Ô)¨"Ô-ˆJØÔ3ð Ý %¤Ø˜œ k°!¤n°jÐAØ'Ô.Ø&Ô,ð!ñ !ô !�ð
 Ô.ð 7°4´=ð 7Ø26�MÔ/øà $× 1Ò 1Ø ”N J°|Ô7JÐ]mð !2ñ !ô !�õ )@Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð  $œ}Ð>�C�C°$´,Ø”LØ'ð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—f’f˜[Ñ)Ô)ˆà˜M¨<Ð7Ð7rE   )FNF)Tr¨   r©   )Nr   ©NNNN)ra   rb   rc   r'   ÚintÚboolrK   Ústaticmethodr¼   rÉ   r   r   r`   rd   re   s   @rC   r‘   r‘   ¶   sä   ø€ € € € € ð %*Ø $Øð%,ð %,àð%,ð ˜‘:ð	%,ð
 ð%,ð %,ð %,ð %,ð %,ð %,ðN ð- ð - ð - ñ „\ð- ð^ð ð ð ð( ØØØðX8ð X8ð Ð+Ô,ðX8ð X8ð X8ð X8ð X8ð X8ð X8ð X8rE   r‘   c                   óL   ‡ — e Zd Zddedz  fˆ fd„Z	 	 	 ddee         fd„Zˆ xZS )	ÚMT5LayerSelfAttentionFNr’   c                 óþ   •— t          ¦   «                              ¦   «          t          ||||j        ¬¦  «        | _        t          |j        |j        ¬¦  «        | _        t          j
        |j        ¦  «        | _        d S )N©r—   r’   r“   r‰   )rJ   rK   r‘   r–   ÚSelfAttentionrG   rn   rŒ   r�   r   rr   rs   r/   ©rP   rh   r—   r’   rS   s       €rC   rK   zMT5LayerSelfAttention.__init__|  sr   ø€ Ý‰Œ×ÒÑÔÐÝ)ØØ(CØØÔ'ð	
ñ 
ô 
ˆÔõ ' v¤~¸6Ô;TÐUÑUÔUˆŒÝ”z &Ô"5Ñ6Ô6ˆŒˆˆrE   r1   c                 ó’   — |                       |¦  «        } | j        |f|||dœ|¤Ž\  }}}||                      |¦  «        z   }|||fS )N)rÜ   r0   rÞ   )r�   rñ   r/   )	rP   r^   r-   r0   rÞ   r1   Únormed_hidden_statesÚattention_outputrA   s	            rC   r`   zMT5LayerSelfAttention.forward‡  sy   € ð  $Ÿš¨}Ñ=Ô=ÐØ8J¸Ô8JØ ð9
àØ'Ø+ð	9
ð 9
ð
 ð9
ð 9
Ñ5Ð˜-¨ð )¨4¯<ª<Ð8HÑ+IÔ+IÑIÐØ °Ð<Ð<rE   ©FN©NNN©	ra   rb   rc   rê   rK   r   r   r`   rd   re   s   @rC   rî   rî   {  s‚   ø€ € € € € ð	7ð 	7ÈSÐSWÉZð 	7ð 	7ð 	7ð 	7ð 	7ð 	7ð ØØð=ð =ð Ð+Ô,ð=ð =ð =ð =ð =ð =ð =ð =rE   rî   c                   óL   ‡ — e Zd Zddedz  fˆ fd„Z	 	 	 ddee         fd„Zˆ xZS )ÚMT5LayerCrossAttentionNr’   c                 óô   •— t          ¦   «                              ¦   «          t          |d|d¬¦  «        | _        t	          |j        |j        ¬¦  «        | _        t          j	        |j
        ¦  «        | _        d S )NFrð   r‰   )rJ   rK   r‘   ÚEncDecAttentionrG   rn   rŒ   r�   r   rr   rs   r/   )rP   rh   r’   rS   s      €rC   rK   zMT5LayerCrossAttention.__init__�  sm   ø€ Ý‰Œ×ÒÑÔÐÝ+Ø°ÀÐV[ð 
ñ  
ô  
ˆÔõ ' v¤~¸6Ô;TÐUÑUÔUˆŒÝ”z &Ô"5Ñ6Ô6ˆŒˆˆrE   r1   c                 ó”   — |                       |¦  «        } | j        |f||||dœ|¤Ž\  }}}	||                      |¦  «        z   }
|
||	fS )N)rÜ   rÝ   r0   rÞ   )r�   rü   r/   )rP   r^   rÝ   r-   r0   rÞ   r1   rô   rõ   rA   Úlayer_outputs              rC   r`   zMT5LayerCrossAttention.forward¥  s{   € ð  $Ÿš¨}Ñ=Ô=ÐØ8L¸Ô8LØ ð9
àØ-Ø'Ø+ð9
ð 9
ð ð9
ð 9
Ñ5Ð˜-¨ð % t§|¢|Ð4DÑ'EÔ'EÑEˆØ˜]¨LÐ8Ð8rE   rx   r÷   rø   re   s   @rC   rú   rú   œ  s�   ø€ € € € € ð7ð 7¨#°©*ð 7ð 7ð 7ð 7ð 7ð 7ð ØØð9ð 9ð Ð+Ô,ð9ð 9ð 9ð 9ð 9ð 9ð 9ð 9rE   rú   c                   óR   ‡ — e Zd Zddedz  fˆ fd„Z	 	 	 	 	 	 ddee         fd„Zˆ xZS )	ÚMT5BlockFNr’   c                 ó’  •— t          ¦   «                              ¦   «          |j        | _        t          j        ¦   «         | _        | j                             t          |||¬¦  «        ¦  «         | j        r)| j                             t          ||¬¦  «        ¦  «         | j                             t          |¦  «        ¦  «         d S )N©r—   r’   )r’   )
rJ   rK   r–   r   Ú
ModuleListÚlayerÚappendrî   rú   r‡   rò   s       €rC   rK   zMT5Block.__init__½  s³   ø€ Ý‰Œ×ÒÑÔÐØ Ô+ˆŒÝ”]‘_”_ˆŒ
ØŒ
×ÒÝ! &ÐFaÐmvÐwÑwÔwñ	
ô 	
ð 	
ð Œ?ð 	SØŒJ×ÒÕ4°VÀyÐQÑQÔQÑRÔRÐRàŒ
×Ò�* VÑ,Ô,Ñ-Ô-Ð-Ð-Ð-rE   r1   c                 ó”  —  | j         d         |f|||dœ|¤Ž\  }}	}
|j        t          j        k    r�t          j        t          j        |¦  «                             ¦   «         t          j        |j        ¦  «        j        dz
  t          j        |j        ¦  «        j        ¦  «        }t          j	        || |¬¦  «        }d }| j
        o|d u}|r¿ | j         d         |f||||dœ|¤Ž\  }}}
|j        t          j        k    r�t          j        t          j        |¦  «                             ¦   «         t          j        |j        ¦  «        j        dz
  t          j        |j        ¦  «        j        ¦  «        }t          j	        || |¬¦  «        } | j         d         |¦  «        }|j        t          j        k    r�t          j        t          j        |¦  «                             ¦   «         t          j        |j        ¦  «        j        dz
  t          j        |j        ¦  «        j        ¦  «        }t          j	        || |¬¦  «        }||	|fS )Nr   )r-   r0   rÞ   iè  )r­   Úmaxr&   )rÝ   r-   r0   rÞ   r3   )r  r[   r;   r\   r³   ÚisinfÚanyÚfinfor  Úclampr–   )rP   r^   r-   r0   Úencoder_hidden_statesÚencoder_attention_maskÚencoder_decoder_position_biasrÞ   r1   Úself_attn_position_biasÚ_Úclamp_valueÚcross_attn_position_biasÚdo_cross_attentions                 rC   r`   zMT5Block.forwardÉ  sR  € ð 5B°D´J¸q´MØð5
à)Ø'Ø+ð	5
ð 5
ð
 ð5
ð 5
Ñ1ˆÐ.°ð Ô¥%¤-Ò/Ð/Ýœ+Ý”˜MÑ*Ô*×.Ò.Ñ0Ô0Ý”˜MÔ/Ñ0Ô0Ô4°tÑ;Ý”˜MÔ/Ñ0Ô0Ô4ñô ˆKõ
 "œK¨¸K¸<È[ÐYÑYÔYˆMà#'Ð Ø!œ_ÐRÐ1FÈdÐ1RÐØð 	^Ø9F¸¼ÀA¼Øð:à!6Ø5Ø;Ø /ð:ð :ð ð:ð :Ñ6ˆMÐ3°Qð Ô"¥e¤mÒ3Ð3Ý#œkÝ”K Ñ.Ô.×2Ò2Ñ4Ô4Ý”K Ô 3Ñ4Ô4Ô8¸4Ñ?Ý”K Ô 3Ñ4Ô4Ô8ñô �õ
 !&¤¨MÀ¸|ÐQ\Ð ]Ñ ]Ô ]�ð '˜œ
 2œ }Ñ5Ô5ˆð Ô¥%¤-Ò/Ð/Ýœ+Ý”˜MÑ*Ô*×.Ò.Ñ0Ô0Ý”˜MÔ/Ñ0Ô0Ô4°tÑ;Ý”˜MÔ/Ñ0Ô0Ô4ñô ˆKõ
 "œK¨¸K¸<È[ÐYÑYÔYˆMàÐ5Ð7OÐOÐOrE   rö   )NNNNNNrø   re   s   @rC   r   r   ¼  s•   ø€ € € € € ð
.ð 
.ÈSÐSWÉZð 
.ð 
.ð 
.ð 
.ð 
.ð 
.ð ØØ"Ø#Ø&*Øð=Pð =Pð Ð+Ô,ð=Pð =Pð =Pð =Pð =Pð =Pð =Pð =PrE   r   c                   óL   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚMT5ClassificationHeadz-Head for sentence-level classification tasks.rh   c                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        ¬¦  «        | _        t          j        |j        |j	        ¦  «        | _
        d S )N)r8   )rJ   rK   r   rm   rn   Údenserr   Úclassifier_dropoutr/   Ú
num_labelsÚout_projrv   s     €rC   rK   zMT5ClassificationHead.__init__  sc   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vœ~¨v¬~Ñ>Ô>ˆŒ
Ý”z FÔ$=Ð>Ñ>Ô>ˆŒÝœ	 &¤.°&Ô2CÑDÔDˆŒˆˆrE   r^   Úreturnc                 óÖ   — |                       |¦  «        }|                      |¦  «        }t          j        |¦  «        }|                       |¦  «        }|                      |¦  «        }|S rx   )r/   r  r;   Útanhr  r|   s     rC   r`   zMT5ClassificationHead.forward  s[   € ØŸš ]Ñ3Ô3ˆØŸ
š
 =Ñ1Ô1ˆÝœ
 =Ñ1Ô1ˆØŸš ]Ñ3Ô3ˆØŸš mÑ4Ô4ˆØÐrE   )
ra   rb   rc   Ú__doc__r'   rK   r;   rz   r`   rd   re   s   @rC   r  r  
  sw   ø€ € € € € Ø7Ð7ðE˜yð Eð Eð Eð Eð Eð Eð U¤\ð °e´lð ð ð ð ð ð ð ð rE   r  c                   óâ   ‡ — e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZ eed¬¦  «         eed	¬¦  «         eed	¬¦  «        d
œZed„ ¦   «         Z ej        ¦   «         ˆ fd„¦   «         Zd„ Zˆ xZS )ÚMT5PreTrainedModelrh   ÚtransformerTr   rq   Fr   )Úindexr3   )r^   Ú
attentionsÚcross_attentionsc                 óv   — t          j        t          ¦  «        }t          j        t          ¦  «        }|||dœ}|S )N)Údecoder_input_idsÚ	input_idsÚdecoder_attention_mask)r;   Útensorr   r   )rP   r'  Ú
input_maskÚdummy_inputss       rC   r+  zMT5PreTrainedModel.dummy_inputs2  s=   € å”L¥Ñ.Ô.ˆ	Ý”\¥*Ñ-Ô-ˆ
à!*Ø"Ø&0ð
ð 
ˆð
 ÐrE   c                 ó\  •— t          ¦   «                              |¦  «         | j        j        }t	          |t
          ¦  «        rt          j        |j        |dz  ¦  «         d	S t	          |t          t          t          t          f¦  «        rÇt          j        |j        j        d|dz  ¬¦  «         t          |d¦  «        r0| j        j        s$t          j        |j        j        d|dz  ¬¦  «         t          |d¦  «        rQt          j        |j        j        d|| j        j        dz  z  ¬¦  «         t          j        |j        j        ¦  «         d	S d	S t	          |t,          ¦  «        rVt          |d¦  «        rDt          j        |j        j        d|dz  ¬¦  «         t          j        |j        j        ¦  «         d	S d	S t	          |t0          ¦  «        ræt          j        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r*|j        j        �t          j        |j        j        ¦  «         t          j        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r,|j        j        �"t          j        |j        j        ¦  «         d	S d	S d	S t	          |t6          ¦  «        ræt          j        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r*|j        j        �t          j        |j        j        ¦  «         t          j        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r,|j        j        �"t          j        |j        j        ¦  «         d	S d	S d	S t	          |t>          ¦  «        �rVt          j        |j         j        d|| j        j        dz  z  ¬¦  «         t          |j         d¦  «        r*|j         j        �t          j        |j         j        ¦  «         t          j        |j!        j        d|| j        j        dz  z  ¬¦  «         t          |j!        d¦  «        r*|j!        j        �t          j        |j!        j        ¦  «         t          j        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r,|j        j        �"t          j        |j        j        ¦  «         d	S d	S d	S t	          |tD          ¦  «        rö| j        j        }| j        j#        }| j        j$        }t          j        |j%        j        d|||z  dz  z  ¬¦  «         t          j        |j&        j        d||dz  z  ¬¦  «         t          j        |j'        j        d||dz  z  ¬¦  «         t          j        |j(        j        d|||z  dz  z  ¬¦  «         |j)        r+t          j        |j*        j        d||dz  z  ¬¦  «         d	S d	S d	S )
zInitialize the weightsr•   r(   )rY   ÚstdÚlm_headÚ
qa_outputsr4   Ú
classifierrl   N)+rJ   Ú_init_weightsrh   Úinitializer_factorry   rG   ÚinitÚ	constant_rN   ÚMT5ModelÚMT5ForConditionalGenerationÚMT5EncoderModelÚMT5ForQuestionAnsweringÚnormal_ÚsharedÚhasattrÚtie_word_embeddingsr.  r/  rn   Úzeros_rl   ÚMT5ForTokenClassificationr0  r  r  r  rg   rp   rq   ro   r   r�   r‚   r‘   rš   rœ   r¡   r¢   r£   r¤   r—   r¦   )rP   r)   Úfactorrn   r›   r�   rS   s         €rC   r1  z MT5PreTrainedModel._init_weights=  sí  ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ø”Ô/ˆÝ�f�lÑ+Ô+ð 1	pÝŒN˜6œ=¨&°3©,Ñ7Ô7Ð7Ð7Ð7ÝØÝÕ2µOÕE\Ð]ñ
ô 
ð /	põ ŒL˜œÔ-°C¸VÀc¹\ÐJÑJÔJÐJÝ�v˜yÑ)Ô)ð P°$´+Ô2Qð PÝ”˜Vœ^Ô2¸À&È3Á,ÐOÑOÔOÐOÝ�v˜|Ñ,Ô,ð 4Ý”˜VÔ.Ô5¸CÀVÐPTÔP[ÔPcÐhlÑOlÑEmÐnÑnÔnÐnÝ”˜FÔ-Ô2Ñ3Ô3Ð3Ð3Ð3ð4ð 4õ ˜Õ 9Ñ:Ô:ð %	pÝ�v˜|Ñ,Ô,ð 4Ý”˜VÔ.Ô5¸CÀVÈcÁ\ÐRÑRÔRÐRÝ”˜FÔ-Ô2Ñ3Ô3Ð3Ð3Ð3ð4ð 4õ ˜Õ 5Ñ6Ô6ð !	pÝŒL˜œÔ,°3¸FÀtÄ{ÔGZÐ_cÑFcÑ<dÐeÑeÔeÐeÝ�v”| VÑ,Ô,ð /°´Ô1BÐ1NÝ”˜FœLÔ-Ñ.Ô.Ð.ÝŒL˜œÔ/°c¸vÈ$Ì+ÔJ]ÐbfÑIfÑ?gÐhÑhÔhÐhÝ�v”¨Ñ/Ô/ð 2°F´OÔ4HÐ4TÝ”˜FœOÔ0Ñ1Ô1Ð1Ð1Ð1ð2ð 2Ð4TÐ4Tå˜Õ 0Ñ1Ô1ð 	pÝŒL˜œÔ)°¸ÀDÄKÔDWÐ\`ÑC`Ñ9aÐbÑbÔbÐbÝ�v”y &Ñ)Ô)ð ,¨f¬i¬nÐ.HÝ”˜FœIœNÑ+Ô+Ð+ÝŒL˜œÔ)°¸ÀDÄKÔDTÐY]ÑC]Ñ9^Ð_Ñ_Ô_Ð_Ý�v”y &Ñ)Ô)ð ,¨f¬i¬nÐ.HÝ”˜FœIœNÑ+Ô+Ð+Ð+Ð+ð,ð ,Ð.HÐ.Hå˜Õ 5Ñ6Ô6ñ 	pÝŒL˜œÔ+°#¸6ÀdÄkÔFYÐ^bÑEbÑ;cÐdÑdÔdÐdÝ�v”{ FÑ+Ô+ð .°´Ô0@Ð0LÝ”˜FœKÔ,Ñ-Ô-Ð-ÝŒL˜œÔ+°#¸6ÀdÄkÔFYÐ^bÑEbÑ;cÐdÑdÔdÐdÝ�v”{ FÑ+Ô+ð .°´Ô0@Ð0LÝ”˜FœKÔ,Ñ-Ô-Ð-ÝŒL˜œÔ)°¸ÀDÄKÔDTÐY]ÑC]Ñ9^Ð_Ñ_Ô_Ð_Ý�v”y &Ñ)Ô)ð ,¨f¬i¬nÐ.HÝ”˜FœIœNÑ+Ô+Ð+Ð+Ð+ð,ð ,Ð.HÐ.Hå˜¥Ñ-Ô-ð 		pØ”kÔ)ˆGØ!%¤Ô!1ÐØ”kÔ+ˆGÝŒL˜œœ¨s¸À7ÐM_ÑC_ÐdhÑBhÑ8iÐjÑjÔjÐjÝŒL˜œœ¨s¸À'È4Á-Ñ8PÐQÑQÔQÐQÝŒL˜œœ¨s¸À'È4Á-Ñ8PÐQÑQÔQÐQÝŒL˜œœ¨s¸À7ÐM_ÑC_ÐdhÑBhÑ8iÐjÑjÔjÐjØÔ1ð pÝ”˜VÔ;ÔBÈÐRXÐ]dÐimÑ\mÑRnÐoÑoÔoÐoÐoÐoð		pð 		pðpð prE   c                 ó6  — | j         j        }| j         j        }|€t          d¦  «        ‚|                     |j        ¦  «        }|dd d…f                              ¦   «         |ddd …f<   ||d<   |€t          d¦  «        ‚|                     |dk    |¦  «         |S )NzŒself.model.config.decoder_start_token_id has to be defined. In MT5 it is usually set to the pad_token_id. See MT5 docs for more information..r3   r&   ).r   z1self.model.config.pad_token_id has to be defined.éœÿÿÿ)rh   Údecoder_start_token_idÚpad_token_idÚ
ValueErrorÚ	new_zerosrÌ   rÎ   Úmasked_fill_)rP   r'  rB  rC  Úshifted_input_idss        rC   Ú_shift_rightzMT5PreTrainedModel._shift_rightu  sº   € Ø!%¤Ô!CÐØ”{Ô/ˆà!Ð)Ýð5ñô ð ð
 &×/Ò/°	´Ñ@Ô@ÐØ%.¨s°C°R°C¨xÔ%8×%>Ò%>Ñ%@Ô%@Ð˜#˜q˜r˜r˜'Ñ"Ø$:Ð˜&Ñ!àÐÝÐPÑQÔQÐQà×&Ò&Ð'8¸DÒ'@À,ÑOÔOÐOà Ð rE   )ra   rb   rc   r'   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_keep_in_fp32_modulesÚ_supports_flash_attnÚ_supports_flex_attnÚ_supports_sdpaÚ_can_compile_fullgraphr$   r   rî   rú   Ú_can_record_outputsÚpropertyr+  r;   Úno_gradr1  rH  rd   re   s   @rC   r   r     s  ø€ € € € € € ð ÐÐÑØ%ÐØ&*Ð#à#˜ÐØ!˜FÐð !ÐØÐØ€NØ!Ðð (˜¨¸Ð:Ñ:Ô:Ø$�nÐ%:À"ÐEÑEÔEØ*˜NÐ+AÈÐLÑLÔLðð Ðð ðð ñ „Xðð €U„]�_„_ð5pð 5pð 5pð 5pñ „_ð5pðn!ð !ð !ð !ð !ð !ð !rE   r   c                   ó|   ‡ — e Zd Zˆ fd„Zd„ Zeee	 	 	 	 	 	 	 ddee	         fd„¦   «         ¦   «         ¦   «         Z
ˆ xZS )ÚMT5Stackc                 óÌ  •‡— t          ¦   «                              ‰¦  «         t          j        ‰j        ‰j        ¦  «        | _        ‰j        | _        t          j        ˆfd„t          ‰j
        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        t          j        ‰j        ¦  «        | _        |                      ¦   «          d| _        d S )Nc           	      óV   •— g | ]%}t          ‰t          |d k    ¦  «        |¬¦  «        ‘Œ&S )r   r  )r   rë   )Ú.0Úirh   s     €rC   ú
<listcomp>z%MT5Stack.__init__.<locals>.<listcomp>”  s4   ø€ ÐwÐwÐwÐYZ�X�f½$¸qÀAºv¹,¼,ÐRSÐTÑTÔTÐwÐwÐwrE   r‰   F)rJ   rK   r   r¥   Ú
vocab_sizern   Úembed_tokensr–   r  ÚrangeÚ
num_layersÚblockrG   rŒ   Úfinal_layer_normrr   rs   r/   Ú	post_initr§   rv   s    `€rC   rK   zMT5Stack.__init__�  sÆ   øø€ Ý‰Œ×Ò˜Ñ Ô Ð åœL¨Ô):¸F¼NÑKÔKˆÔØ Ô+ˆŒå”]ØwÐwÐwÐwÕ^cÐdjÔduÑ^vÔ^vÐwÑwÔwñ
ô 
ˆŒ
õ !-¨V¬^ÀÔAZÐ [Ñ [Ô [ˆÔÝ”z &Ô"5Ñ6Ô6ˆŒð 	�ŠÑÔÐØ&+ˆÔ#Ð#Ð#rE   c                 ó   — || _         d S rx   )r]  ©rP   Únew_embeddingss     rC   Úset_input_embeddingszMT5Stack.set_input_embeddings�  s   € Ø*ˆÔÐÐrE   Nr1   c           	      ój  — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|du r| j        st          d| › d�¦  «        ‚| j        r]|rZ|€X| j        j        r7t          t          | j        ¬¦  «        t          | j        ¬¦  «        ¦  «        }nt          | j        ¬¦  «        }n	| j        sd }| j        j        r5t          | j        ||t          |t
          ¦  «        r|j	        n|¬¦  «        }nt          | j        ||¬¦  «        }d }	| j        r|�t          | j        |||¬¦  «        }	d }
d }|                      |¦  «        }| j        D ]#} ||||
||	|fd	|i|¤Ž\  }}}|}
| j        r|�|}Œ$|                      |¦  «        }|                      |¦  «        }t          ||¬
¦  «        S )Nz:You must specify exactly one of input_ids or inputs_embedsTz)`use_cache` can only be set to `True` if z is used as a decoder)rh   )rh   Úinputs_embedsr-   rÞ   )rh   rh  r-   )rh   rh  r-   r  rÞ   )Úlast_hidden_staterÞ   )rD  r]  r–   rh   Úis_encoder_decoderr   r   r   ry   rÓ   r   r/   r`  ra  r   )rP   r'  r-   r  r  rh  rÞ   Ú	use_cacher1   Úencoder_extended_attention_maskr0   r  r^   Úlayer_moduleÚself_attention_position_biasÚcross_attention_position_biass                   rC   r`   zMT5Stack.forward   st  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMà˜ÐÐØ”?ð jÝ Ð!hÈTÐ!hÐ!hÐ!hÑiÔiÐiàŒ?ð 	#Øð G˜_Ð4Ø”;Ô1ð GÝ&9Ý$¨D¬KÐ8Ñ8Ô8½,ÈdÌkÐ:ZÑ:ZÔ:Zñ'ô '�O�Oõ '3¸$¼+Ð&FÑ&FÔ&F�OøØ”ð 	#ð #ˆOàŒ;Ô!ð 	Ý/Ø”{Ø+Ø-å˜oÕ/BÑCÔCð!% Ô DÐ Dà$ðñ ô ˆNˆNõ 7Ø”{Ø+Ø-ðñ ô ˆNð +/Ð'ØŒ?ð 	Ð4Ð@Ý.GØ”{Ø+Ø5Ø&;ð	/ñ /ô /Ð+ð ˆØ(,Ð%àŸš ]Ñ3Ô3ˆà œJð 	Nð 	NˆLØYeÐYeØØØØ%Ø/Ø-ð	Zð 	Zð !0ð	Zð ð	Zð 	ZÑVˆMÐ7Ð9Vð 9ˆMØŒð NÐ#8Ð#DØ0MÐ-øà×-Ò-¨mÑ<Ô<ˆØŸš ]Ñ3Ô3ˆå8Ø+Ø+ð
ñ 
ô 
ð 	
rE   )NNNNNNN)ra   rb   rc   rK   rf  r#   r%   r   r   r   r`   rd   re   s   @rC   rV  rV  Œ  s²   ø€ € € € € ð,ð ,ð ,ð ,ð ,ð +ð +ð +ð  ØØð ØØ"Ø#ØØØðW
ð W
ð Ð+Ô,ðW
ð W
ð W
ñ „^ñ „_ñ  ÔðW
ð W
ð W
ð W
ð W
rE   rV  c                   ó\  ‡ — e Zd ZU dZdZeed<   dgZdddœZdefˆ fd„Z	d„ Z
d	„ Zee	 	 	 	 	 	 	 	 	 ddej        d
z  dej        d
z  dej        d
z  dej        d
z  deeej                          d
z  ded
z  dej        d
z  dej        d
z  ded
z  dee         defd„¦   «         ¦   «         Zˆ xZS )r5  aw  
    Examples:

    ```python
    >>> from transformers import MT5Model, AutoTokenizer

    >>> model = MT5Model.from_pretrained("google/mt5-small")
    >>> tokenizer = AutoTokenizer.from_pretrained("google/mt5-small")
    >>> article = "UN Offizier sagt, dass weiter verhandelt werden muss in Syrien."
    >>> summary = "Weiter Verhandlung in Syrien."
    >>> inputs = tokenizer(article, return_tensors="pt")
    >>> labels = tokenizer(text_target=summary, return_tensors="pt")

    >>> outputs = model(input_ids=inputs["input_ids"], decoder_input_ids=labels["input_ids"])
    >>> hidden_states = outputs.last_hidden_state
    ```Úmt5rh   úFdecoder.block.0.layer.1.EncDecAttention.relative_attention_bias.weightúshared.weight©úencoder.embed_tokens.weightúdecoder.embed_tokens.weightc                 óœ  •— t          ¦   «                              |¦  «         t          j        |j        |j        ¦  «        | _        t          j        |¦  «        }d|_	        d|_
        t          |¦  «        | _        t          j        |¦  «        }d|_	        |j        |_        t          |¦  «        | _        |                      ¦   «          d S ©NFT)rJ   rK   r   r¥   r\  rn   r:  ÚcopyÚdeepcopyr–   rk  rV  ÚencoderÚnum_decoder_layersr_  Údecoderrb  ©rP   rh   Úencoder_configÚdecoder_configrS   s       €rC   rK   zMT5Model.__init__  sª   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý”l 6Ô#4°f´nÑEÔEˆŒåœ vÑ.Ô.ˆØ$)ˆÔ!Ø#(ˆÔ Ý Ñ/Ô/ˆŒåœ vÑ.Ô.ˆØ$(ˆÔ!Ø$*Ô$=ˆÔ!Ý Ñ/Ô/ˆŒð 	�ŠÑÔÐÐÐrE   c                 ó   — | j         S rx   ©r:  ©rP   s    rC   Úget_input_embeddingszMT5Model.get_input_embeddings+  ó
   € ØŒ{ÐrE   c                 ó|   — || _         | j                             |¦  «         | j                             |¦  «         d S rx   ©r:  r{  rf  r}  rd  s     rC   rf  zMT5Model.set_input_embeddings/  ó;   € Ø$ˆŒØŒ×)Ò)¨.Ñ9Ô9Ð9ØŒ×)Ò)¨.Ñ9Ô9Ð9Ð9Ð9rE   Nr'  r-   r&  r(  Úencoder_outputsrÞ   rh  Údecoder_inputs_embedsrk  r1   r  c
                 ó¨  — |€ | j         d	|||dœ|
¤Ž}nct          |t          ¦  «        sNt          |d         t          |¦  «        dk    r|d         ndt          |¦  «        dk    r|d         nd¬¦  «        }|d         } | j        d	|||||||	dœ|
¤Ž}t          |j        |j        |j        |j	        |j
        |j        |j        |j	        ¬¦  «        S )
a\
  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. MT5 is a model with relative position embeddings so you
            should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            [What are input IDs?](../glossary#input-ids)

            To know more on how to prepare `input_ids` for pretraining take a look a [MT5 Training](./mt5#training).
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            MT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
            is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).

            To know more on how to prepare `decoder_input_ids` for pretraining take a look at [MT5
            Training](./mt5#training).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, MT5Model

        >>> tokenizer = AutoTokenizer.from_pretrained("google/mt5-small")
        >>> model = MT5Model.from_pretrained("google/mt5-small")

        >>> input_ids = tokenizer(
        ...     "Studies have been shown that owning a dog is good for you", return_tensors="pt"
        ... ).input_ids  # Batch size 1
        >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="pt").input_ids  # Batch size 1

        >>> # preprocess: Prepend decoder_input_ids with start token which is pad token for MT5Model.
        >>> # This is not needed for torch's MT5ForConditionalGeneration as it does this internally using labels arg.
        >>> decoder_input_ids = model._shift_right(decoder_input_ids)

        >>> # forward pass
        >>> outputs = model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)
        >>> last_hidden_states = outputs.last_hidden_state
        ```N©r'  r-   rh  r   r&   r5   ©ri  r^   r#  ©r'  r-   rh  rÞ   r  r  rk  )ri  rÞ   Údecoder_hidden_statesÚdecoder_attentionsr$  Úencoder_last_hidden_stater  Úencoder_attentions© )r{  ry   r   Úlenr}  r   ri  rÞ   r^   r#  r$  )rP   r'  r-   r&  r(  r‰  rÞ   rh  rŠ  rk  r1   r^   Údecoder_outputss                rC   r`   zMT5Model.forward4  s?  € ðD Ð"Ø*˜dœlð Ø#Ø-Ø+ðð ð ð	ð ˆOˆOõ ˜O­_Ñ=Ô=ð 	Ý-Ø"1°!Ô"4Ý47¸Ñ4HÔ4HÈ1Ò4LÐ4L˜o¨aÔ0Ð0ÐRVÝ14°_Ñ1EÔ1EÈÒ1IÐ1I˜?¨1Ô-Ð-Ètðñ ô ˆOð (¨Ô*ˆð '˜$œ,ð 	
Ø'Ø1Ø/Ø+Ø"/Ø#1Øð	
ð 	
ð ð	
ð 	
ˆõ "Ø-Ô?Ø+Ô;Ø"1Ô"?Ø.Ô9Ø,Ô=Ø&5Ô&GØ"1Ô"?Ø.Ô9ð	
ñ 	
ô 	
ð 		
rE   ©	NNNNNNNNN)ra   rb   rc   r  Ú
model_typer'   rI  Ú"_keys_to_ignore_on_load_unexpectedÚ_tied_weights_keysrK   r„  rf  r"   r   r;   Ú
LongTensorÚFloatTensorÚ
BoolTensorÚtupler   rz   rë   r   r   r   r`   rd   re   s   @rC   r5  r5  ý  s¸  ø€ € € € € € ðð ð" €JØÐÐÑØ*rÐ)sÐ&à'6Ø'6ðð Ðð˜yð ð ð ð ð ð ð$ð ð ð:ð :ð :ð
 Øð .2Ø37Ø59Ø:>ØBFØ(,Ø-1Ø59Ø!%ðd
ð d
àÔ# dÑ*ðd
ð Ô)¨DÑ0ðd
ð !Ô+¨dÑ2ð	d
ð
 !&Ô 0°4Ñ 7ðd
ð ˜u UÔ%6Ô7Ô8¸4Ñ?ðd
ð  ™ðd
ð ”| dÑ*ðd
ð  %œ|¨dÑ2ðd
ð ˜$‘;ðd
ð Ð+Ô,ðd
ð 
ðd
ð d
ð d
ñ „^ñ Ôðd
ð d
ð d
ð d
ð d
rE   r5  z;
    MT5 Model with a `language modeling` head on top.
    )Úcustom_introc                   óŠ  ‡ — e Zd ZU dZdZeed<   dgZddddœZdefˆ fd„Z	d„ Z
d	„ Zee	 	 	 	 	 	 	 	 	 	 ddej        d
z  dej        d
z  dej        d
z  dej        d
z  deeej                          d
z  ded
z  dej        d
z  dej        d
z  dej        d
z  ded
z  dee         defd„¦   «         ¦   «         Zdej        fd„Zˆ xZS )r6  a  
    Examples:

    ```python
    >>> from transformers import MT5ForConditionalGeneration, AutoTokenizer

    >>> model = MT5ForConditionalGeneration.from_pretrained("google/mt5-small")
    >>> tokenizer = AutoTokenizer.from_pretrained("google/mt5-small")
    >>> article = "UN Offizier sagt, dass weiter verhandelt werden muss in Syrien."
    >>> summary = "Weiter Verhandlung in Syrien."
    >>> inputs = tokenizer(article, text_target=summary, return_tensors="pt")

    >>> outputs = model(**inputs)
    >>> loss = outputs.loss
    ```rq  rh   rr  rs  )ru  rv  zlm_head.weightc                 ó   •— t          ¦   «                              |¦  «         |j        | _        t	          j        |j        |j        ¦  «        | _        t          j	        |¦  «        }d|_
        d|_        t          |¦  «        | _        t          j	        |¦  «        }d|_
        |j        |_        t          |¦  «        | _        t	          j        |j        |j        d¬¦  «        | _        |                      ¦   «          d S )NFTrk   )rJ   rK   rn   Ú	model_dimr   r¥   r\  r:  ry  rz  r–   rk  rV  r{  r|  r_  r}  rm   r.  rb  r~  s       €rC   rK   z$MT5ForConditionalGeneration.__init__¾  sÐ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØœˆŒå”l 6Ô#4°f´nÑEÔEˆŒåœ vÑ.Ô.ˆØ$)ˆÔ!Ø#(ˆÔ Ý Ñ/Ô/ˆŒåœ vÑ.Ô.ˆØ$(ˆÔ!Ø$*Ô$=ˆÔ!Ý Ñ/Ô/ˆŒå”y ¤°Ô1BÈÐOÑOÔOˆŒð 	�ŠÑÔÐÐÐrE   c                 ó   — | j         S rx   r‚  rƒ  s    rC   r„  z0MT5ForConditionalGeneration.get_input_embeddingsÓ  r…  rE   c                 ó|   — || _         | j                             |¦  «         | j                             |¦  «         d S rx   r‡  rd  s     rC   rf  z0MT5ForConditionalGeneration.set_input_embeddingsÖ  rˆ  rE   Nr'  r-   r&  r(  r‰  rÞ   rh  rŠ  Úlabelsrk  r1   r  c                 óø  — |€ | j         d|||dœ|¤Ž}nct          |t          ¦  «        sNt          |d         t          |¦  «        dk    r|d         ndt          |¦  «        dk    r|d         nd¬¦  «        }|d         }|	�|€|€|                      |	¦  «        } | j        d|||||||
dœ|¤Ž}|d         }|                      |¦  «        }d}|	�pt          d¬	¦  «        }|	                     |j	        ¦  «        }	 || 
                    d
|                     d
¦  «        ¦  «        |	 
                    d
¦  «        ¦  «        }t          |||j        |j        |j        |j        |j        |j        |j        ¬¦	  «	        S )a  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. MT5 is a model with relative position embeddings so you
            should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            [What are input IDs?](../glossary#input-ids)

            To know more on how to prepare `input_ids` for pretraining take a look a [MT5 Training](./mt5#training).
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            MT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
            is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).

            To know more on how to prepare `decoder_input_ids` for pretraining take a look at [MT5
            Training](./mt5#training).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ...,
            config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for
            labels in `[0, ..., config.vocab_size]`

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, MT5ForConditionalGeneration

        >>> tokenizer = AutoTokenizer.from_pretrained("google/mt5-small")
        >>> model = MT5ForConditionalGeneration.from_pretrained("google/mt5-small")

        >>> # training
        >>> input_ids = tokenizer("The <extra_id_0> walks in <extra_id_1> park", return_tensors="pt").input_ids
        >>> labels = tokenizer("<extra_id_0> cute dog <extra_id_1> the <extra_id_2>", return_tensors="pt").input_ids
        >>> outputs = model(input_ids=input_ids, labels=labels)
        >>> loss = outputs.loss
        >>> logits = outputs.logits

        >>> # inference
        >>> input_ids = tokenizer(
        ...     "summarize: studies have shown that owning a dog is good for you", return_tensors="pt"
        ... ).input_ids  # Batch size 1
        >>> outputs = model.generate(input_ids)
        >>> print(tokenizer.decode(outputs[0], skip_special_tokens=True))
        >>> # studies have shown that owning a dog is good for you.
        ```NrŒ  r   r&   r5   r�  rŽ  rA  ©Úignore_indexr3   ©	ÚlossÚlogitsrÞ   r�  r�  r$  r‘  r  r’  r“  )r{  ry   r   r”  rH  r}  r.  r   rV   r¾   rÏ   r:   r   rÞ   r^   r#  r$  ri  )rP   r'  r-   r&  r(  r‰  rÞ   rh  rŠ  r¤  rk  r1   r^   r•  Úsequence_outputÚ	lm_logitsr©  Úloss_fcts                     rC   r`   z#MT5ForConditionalGeneration.forwardÛ  så  € ðR Ð"à*˜dœlð Ø#Ø-Ø+ðð ð ð	ð ˆOˆOõ ˜O­_Ñ=Ô=ð 	Ý-Ø"1°!Ô"4Ý47¸Ñ4HÔ4HÈ1Ò4LÐ4L˜o¨aÔ0Ð0ÐRVÝ14°_Ñ1EÔ1EÈÒ1IÐ1I˜?¨1Ô-Ð-Ètðñ ô ˆOð (¨Ô*ˆàÐÐ"3Ð";Ð@UÐ@]à $× 1Ò 1°&Ñ 9Ô 9Ðð '˜$œ,ð 	
Ø'Ø1Ø/Ø+Ø"/Ø#1Øð	
ð 	
ð ð	
ð 	
ˆð *¨!Ô,ˆà—L’L Ñ1Ô1ˆ	àˆØÐÝ'°TÐ:Ñ:Ô:ˆHà—Y’Y˜yÔ/Ñ0Ô0ˆFØ�8˜IŸNšN¨2¨y¯~ª~¸bÑ/AÔ/AÑBÔBÀFÇKÂKÐPRÁOÄOÑTÔTˆDåØØØ+Ô;Ø"1Ô"?Ø.Ô9Ø,Ô=Ø&5Ô&GØ"1Ô"?Ø.Ô9ð

ñ 

ô 

ð 
	
rE   c                 ó,   — |                       |¦  «        S rx   )rH  )rP   r¤  s     rC   Ú%prepare_decoder_input_ids_from_labelszAMT5ForConditionalGeneration.prepare_decoder_input_ids_from_labels]  s   € Ø× Ò  Ñ(Ô(Ð(rE   ©
NNNNNNNNNN)ra   rb   rc   r  r—  r'   rI  r˜  r™  rK   r„  rf  r"   r   r;   rš  r›  rœ  r�  rz   r   rë   r   r   r   r`   r¯  rd   re   s   @rC   r6  r6  ž  sì  ø€ € € € € € ðð ð  €JØÐÐÑØ*rÐ)sÐ&à'6Ø'6Ø)ðð Ðð˜yð ð ð ð ð ð ð*ð ð ð:ð :ð :ð
 Øð .2Ø37Ø59Ø:>Ø=AØ(,Ø26Ø:>Ø*.Ø!%ð}
ð }
àÔ# dÑ*ð}
ð Ô)¨DÑ0ð}
ð !Ô+¨dÑ2ð	}
ð
 !&Ô 0°4Ñ 7ð}
ð ˜u U¤\Ô2Ô3°dÑ:ð}
ð  ™ð}
ð Ô(¨4Ñ/ð}
ð  %Ô0°4Ñ7ð}
ð Ô  4Ñ'ð}
ð ˜$‘;ð}
ð Ð+Ô,ð}
ð 
ð}
ð }
ð }
ñ „^ñ Ôð}
ð@)¸E¼Lð )ð )ð )ð )ð )ð )ð )ð )rE   r6  c                   óÌ   ‡ — e Zd ZU dZdZeed<   ddiZdefˆ fd„Zd„ Z	d„ Z
ee	 	 	 dd
ej        d	z  dej        d	z  dej        d	z  dee         def
d„¦   «         ¦   «         Zˆ xZS )r7  aÞ  
    Examples:

    ```python
    >>> from transformers import MT5EncoderModel, AutoTokenizer

    >>> model = MT5EncoderModel.from_pretrained("google/mt5-small")
    >>> tokenizer = AutoTokenizer.from_pretrained("google/mt5-small")
    >>> article = "UN Offizier sagt, dass weiter verhandelt werden muss in Syrien."
    >>> input_ids = tokenizer(article, return_tensors="pt").input_ids
    >>> outputs = model(input_ids)
    >>> hidden_state = outputs.last_hidden_state
    ```rq  rh   ru  rs  c                 ó  •— t          ¦   «                              |¦  «         t          j        |j        |j        ¦  «        | _        |}d|_        d|_        t          |¦  «        | _
        |                      ¦   «          d S )NF)rJ   rK   r   r¥   r\  rn   r:  rk  rj  rV  r{  rb  )rP   rh   r  rS   s      €rC   rK   zMT5EncoderModel.__init__x  so   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý”l 6Ô#4°f´nÑEÔEˆŒàˆØ#(ˆÔ Ø,1ˆÔ)Ý Ñ/Ô/ˆŒð 	�ŠÑÔÐÐÐrE   c                 ó   — | j         S rx   r‚  rƒ  s    rC   r„  z$MT5EncoderModel.get_input_embeddings…  r…  rE   c                 óH   — || _         | j                             |¦  «         d S rx   )r:  r{  rf  rd  s     rC   rf  z$MT5EncoderModel.set_input_embeddings‰  s%   € Ø$ˆŒØŒ×)Ò)¨.Ñ9Ô9Ð9Ð9Ð9rE   Nr'  r-   rh  r1   r  c                 ó(   —  | j         d|||dœ|¤Ž}|S )aJ  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. MT5 is a model with relative position embeddings so you
            should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            To know more on how to prepare `input_ids` for pretraining take a look a [MT5 Training](./mt5#training).

        Example:

        ```python
        >>> from transformers import AutoTokenizer, MT5EncoderModel

        >>> tokenizer = AutoTokenizer.from_pretrained("google/mt5-small")
        >>> model = MT5EncoderModel.from_pretrained("google/mt5-small")
        >>> input_ids = tokenizer(
        ...     "Studies have been shown that owning a dog is good for you", return_tensors="pt"
        ... ).input_ids  # Batch size 1
        >>> outputs = model(input_ids=input_ids)
        >>> last_hidden_states = outputs.last_hidden_state
        ```rŒ  r“  )r{  )rP   r'  r-   rh  r1   r‰  s         rC   r`   zMT5EncoderModel.forward�  s=   € ðB ,8¨4¬<ð ,
ØØ)Ø'ð,
ð ,
ð ð	,
ð ,
ˆð ÐrE   r÷   )ra   rb   rc   r  r—  r'   rI  r™  rK   r„  rf  r"   r   r;   rš  r›  r   r   r   r`   rd   re   s   @rC   r7  r7  a  s  ø€ € € € € € ðð ð €JØÐÐÑà% ðÐð

˜yð 
ð 
ð 
ð 
ð 
ð 
ðð ð ð:ð :ð :ð Øð .2Ø37Ø26ð	%ð %àÔ# dÑ*ð%ð Ô)¨DÑ0ð%ð Ô(¨4Ñ/ð	%ð
 Ð+Ô,ð%ð 
ð%ð %ð %ñ „^ñ Ôð%ð %ð %ð %ð %rE   r7  z„
    MT5 model with a sequence classification/head on top (a linear layer on top of the pooled output) e.g. for GLUE
    tasks.
    c                   ó0  ‡ — e Zd ZdgZdefˆ fd„Zee	 	 	 	 	 	 	 	 	 ddej	        dz  dej
        dz  dej	        dz  dej	        dz  d	eej                 dz  d
ej        dz  dej        dz  dej	        dz  dedz  dee         defd„¦   «         ¦   «         Zˆ xZS )ÚMT5ForSequenceClassificationrr  rh   c                 óÂ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          |¦  «        | _        |                      ¦   «          d S rx   )rJ   rK   r5  r!  r  Úclassification_headrb  rv   s     €rC   rK   z%MT5ForSequenceClassification.__init__Â  sS   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý# FÑ+Ô+ˆÔÝ#8¸Ñ#@Ô#@ˆÔ ð 	�ŠÑÔÐÐÐrE   Nr'  r-   r&  r(  r‰  rh  rŠ  r¤  rk  r1   r  c
                 óR  — |�d}	|€|�t          d| j        j        › �¦  «        ‚|€(|€&|€t          d¦  «        ‚|                      |¦  «        } | j        |f|||||||	dœ|
¤Ž}|d         }|                     | j        j        ¦  «         	                    |j
        ¦  «        }t          t          j        |                     d¦  «        ¦  «                             ¦   «         dk    d¦  «         |j        \  }}}||dd…f         }t          |j        d         |z  dk    d	¦  «         |                     |d
|¦  «        dd…d
dd…f         }|                      |¦  «        }d}|��ˆ| 	                    |j
        ¦  «        }| j        j        €p| j        j        dk    rd| j        _        nS| j        j        dk    r7|j        t          j        k    s|j        t          j        k    rd| j        _        nd| j        _        | j        j        dk    r\t1          ¦   «         }| j        j        dk    r1 ||                     ¦   «         |                     ¦   «         ¦  «        }n“ |||¦  «        }n†| j        j        dk    rLt5          ¦   «         } ||                     d
| j        j        ¦  «        |                     d
¦  «        ¦  «        }n*| j        j        dk    rt7          ¦   «         } |||¦  «        }t9          |||j        |j        |j        |j         |j!        |j"        |j#        ¬¦	  «	        S )aª  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. MT5 is a model with relative position embeddings so you
            should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            [What are input IDs?](../glossary#input-ids)

            To know more on how to prepare `input_ids` for pretraining take a look a [MT5 Training](./mt5#training).
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            MT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
            is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).

            To know more on how to prepare `decoder_input_ids` for pretraining take a look at [MT5
            Training](./mt5#training).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        NFz8Passing input embeddings is currently not supported for ú°If no `decoder_input_ids` or `decoder_inputs_embeds` are passed, `input_ids` cannot be `None`. Please pass either `input_ids` or `decoder_input_ids` or `decoder_inputs_embeds`.)r-   r&  r(  r‰  rh  rŠ  rk  r   r&   z7All examples must have the same number of <eos> tokens.z3Each example must contain at least one <eos> token.r3   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr¨  )$ÚNotImplementedErrorrS   ra   rD  rH  r!  Úeqrh   Úeos_token_idrV   r¾   r!   r;   Úunique_consecutiveÚsumÚnumelrÌ   rÏ   r¹  Úproblem_typer  r[   r«   rê   r   Úsqueezer   r   r   rÞ   r�  r�  r$  r‘  r  r’  )rP   r'  r-   r&  r(  r‰  rh  rŠ  r¤  rk  r1   Úoutputsr«  Úeos_maskÚ
batch_sizer  rQ   ÚselectedÚsentence_representationrª  r©  r­  s                         rC   r`   z$MT5ForSequenceClassification.forwardÊ  so  € ð^ ÐØˆIàÐ Ð!:Ý%ØdÈ4Ì>ÔKbÐdÐdñô ð ð Ð$Ð)>Ð)FØÐ Ý ðUñô ð ð
 !%× 1Ò 1°)Ñ <Ô <Ðà&6 dÔ&6Øð
'
à)Ø/Ø#9Ø+Ø'Ø"7Øð
'
ð 
'
ð ð
'
ð 
'
ˆð " !œ*ˆà—<’< ¤Ô 8Ñ9Ô9×<Ò<¸_Ô=SÑTÔTˆåÝÔ$ X§\¢\°!¡_¤_Ñ5Ô5×;Ò;Ñ=Ô=ÀÒBØEñ	
ô 	
ð 	
ð &5Ô%:Ñ"ˆ
�A�{Ø" 8¨Q¨Q¨Q ;Ô/ˆÝØŒN˜1Ô Ñ+¨qÒ0ØAñ	
ô 	
ð 	
ð #+§-¢-°
¸BÀÑ"LÔ"LÈQÈQÈQÐPRÐTUÐTUÐTUÈXÔ"VÐØ×)Ò)Ð*AÑBÔBˆàˆØÑØ—Y’Y˜vœ}Ñ-Ô-ˆFØŒ{Ô'Ð/Ø”;Ô)¨QÒ.Ð.Ø/;�D”KÔ,Ð,Ø”[Ô+¨aÒ/Ð/°V´\ÅUÄZÒ5OÐ5OÐSYÔS_ÕchÔclÒSlÐSlØ/L�D”KÔ,Ð,à/K�D”KÔ,àŒ{Ô'¨<Ò7Ð7Ý"™9œ9�Ø”;Ô)¨QÒ.Ð.Ø#˜8 F§N¢NÑ$4Ô$4°f·n²nÑ6FÔ6FÑGÔG�D�Dà#˜8 F¨FÑ3Ô3�D�DØ”Ô)Ð-JÒJÐJÝ+Ñ-Ô-�Ø�x §¢¨B°´Ô0FÑ GÔ GÈÏÊÐUWÉÌÑYÔY��Ø”Ô)Ð-IÒIÐIÝ,Ñ.Ô.�Ø�x ¨Ñ/Ô/�å.ØØØ#Ô3Ø")Ô"?Ø&Ô9Ø$Ô5Ø&-Ô&GØ")Ô"?Ø&Ô9ð

ñ 

ô 

ð 
	
rE   r–  )ra   rb   rc   r˜  r'   rK   r"   r   r;   rš  rz   Úlistr›  rë   r   r   r   r`   rd   re   s   @rC   r·  r·  ¸  sk  ø€ € € € € ð +sÐ)sÐ&ð˜yð ð ð ð ð ð ð Øð .2Ø.2Ø59Ø:>Ø:>Ø26Ø:>Ø*.Ø!%ð}
ð }
àÔ# dÑ*ð}
ð œ tÑ+ð}
ð !Ô+¨dÑ2ð	}
ð
 !&Ô 0°4Ñ 7ð}
ð ˜eÔ/Ô0°4Ñ7ð}
ð Ô(¨4Ñ/ð}
ð  %Ô0°4Ñ7ð}
ð Ô  4Ñ'ð}
ð ˜$‘;ð}
ð Ð+Ô,ð}
ð 
)ð}
ð }
ð }
ñ „^ñ Ôð}
ð }
ð }
ð }
ð }
rE   r·  c                   óº   ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  de	e
         d	efd
„¦   «         ¦   «         Zˆ xZS )r>  rh   c                 ó6  •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          j        |j        ¦  «        | _        t          j	        |j
        |j        ¦  «        | _        |                      ¦   «          d S rx   )rJ   rK   r  r7  r!  r   rr   r  r/   rm   rQ   r0  rb  rv   s     €rC   rK   z"MT5ForTokenClassification.__init__P  sz   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒå*¨6Ñ2Ô2ˆÔÝ”z &Ô";Ñ<Ô<ˆŒÝœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐrE   Nr'  r-   rh  r¤  r1   r  c                 óV  —  | j         |f||dœ|¤Ž}|d         }|                      |¦  «        }|                      |¦  «        }d}	|�Ft          ¦   «         }
 |
|                     d| j        ¦  «        |                     d¦  «        ¦  «        }	t          |	||j        |j        ¬¦  «        S )a>  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. MT5 is a model with relative position embeddings so you
            should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            [What are input IDs?](../glossary#input-ids)

            To know more on how to prepare `input_ids` for pretraining take a look a [MT5 Training](./t5#training).
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        )r-   rh  r   Nr3   )r©  rª  r^   r#  )	r!  r/   r0  r   rÏ   r  r   r^   r#  )rP   r'  r-   rh  r¤  r1   rÇ  r^   rª  r©  r­  s              rC   r`   z!MT5ForTokenClassification.forward[  sÌ   € ð2 $4 4Ô#3Øð$
à)Ø'ð$
ð $
ð ð	$
ð $
ˆð   œ
ˆØŸš ]Ñ3Ô3ˆØ—’ Ñ/Ô/ˆàˆØÐÝ'Ñ)Ô)ˆHØ�8˜FŸKšK¨¨D¬OÑ<Ô<¸f¿kºkÈ"¹o¼oÑNÔNˆDå$ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
rE   ré   )ra   rb   rc   r'   rK   r"   r   r;   rz   r   r   r   r`   rd   re   s   @rC   r>  r>  M  sÞ   ø€ € € € € ð	˜yð 	ð 	ð 	ð 	ð 	ð 	ð Øð *.Ø.2Ø-1Ø&*ð+
ð +
à”< $Ñ&ð+
ð œ tÑ+ð+
ð ”| dÑ*ð	+
ð
 ”˜tÑ#ð+
ð Ð+Ô,ð+
ð 
ð+
ð +
ð +
ñ „^ñ Ôð+
ð +
ð +
ð +
ð +
rE   r>  c                   óh  ‡ — e Zd ZdgZdddœZdefˆ fd„Zd„ Zd„ Ze	e
	 	 	 	 	 	 	 	 	 	 dd	ej        dz  d
ej        dz  dej        dz  dej        dz  deeej                          dz  dej        dz  dej        dz  dej        dz  dej        dz  dedz  dee         defd„¦   «         ¦   «         Zˆ xZS )r8  rr  rs  rt  rh   c                 ó  •— t          ¦   «                              |¦  «         |j        | _        t	          j        |j        |j        ¦  «        | _        t          j	        |¦  «        }d|_
        d|_        t          |¦  «        | _        t          j	        |¦  «        }d|_
        |j        |_        t          |¦  «        | _        |j        | _        t	          j        |j        |j        ¦  «        | _        |                      ¦   «          d S rx  )rJ   rK   rn   r¡  r   r¥   r\  r:  ry  rz  r–   rk  rV  r{  r|  r_  r}  r  rm   rQ   r/  rb  r~  s       €rC   rK   z MT5ForQuestionAnswering.__init__•  sÖ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØœˆŒå”l 6Ô#4°f´nÑEÔEˆŒåœ vÑ.Ô.ˆØ$)ˆÔ!Ø#(ˆÔ Ý Ñ/Ô/ˆŒåœ vÑ.Ô.ˆØ$(ˆÔ!Ø$*Ô$=ˆÔ!Ý Ñ/Ô/ˆŒà Ô+ˆŒÝœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐrE   c                 ó   — | j         S rx   r‚  rƒ  s    rC   r„  z,MT5ForQuestionAnswering.get_input_embeddings¬  r…  rE   c                 ó|   — || _         | j                             |¦  «         | j                             |¦  «         d S rx   r‡  rd  s     rC   rf  z,MT5ForQuestionAnswering.set_input_embeddings°  rˆ  rE   Nr'  r-   r&  r(  r‰  Ústart_positionsÚend_positionsrh  rŠ  rk  r1   r  c                 óF  — |�|�d}
|€(|	€&|€t          d¦  «        ‚|                      |¦  «        }|€ | j        d|||dœ|¤Ž}nct          |t          ¦  «        sNt	          |d         t          |¦  «        dk    r|d         ndt          |¦  «        dk    r|d         nd¬¦  «        }|d         } | j        d|||	d|||
d	œ|¤Ž}|d         }|                      |¦  «        }|                     dd
¬¦  «        \  }}| 	                    d
¦  «         
                    ¦   «         }| 	                    d
¦  «         
                    ¦   «         }d}|��|��t          |                     ¦   «         ¦  «        dk    r-| 	                    d
¦  «                             |j        ¦  «        }t          |                     ¦   «         ¦  «        dk    r-| 	                    d
¦  «                             |j        ¦  «        }|                     d¦  «        }|                     d|¦  «        }|                     d|¦  «        }t          |¬¦  «        } |||¦  «        } |||¦  «        }||z   dz  }t!          ||||j        |j        |j        |j        |j        |j        |j        ¬¦
  «
        S )az  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you
            should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            [What are input IDs?](../glossary#input-ids)

            To know more on how to prepare `input_ids` for pretraining take a look a [T5 Training](./t5#training).
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            T5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
            is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).

            To know more on how to prepare `decoder_input_ids` for pretraining take a look at [T5
            Training](./t5#training).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.
        NFr»  rŒ  r   r&   r5   r�  rŽ  r3   r6   r¦  )
r©  Ústart_logitsÚ
end_logitsrÞ   r�  r�  r$  r‘  r  r’  r“  )rD  rH  r{  ry   r   r”  r}  r/  ÚsplitrÆ  r@   r:   rV   r¾   r  r   r   rÞ   r^   r#  r$  ri  )rP   r'  r-   r&  r(  r‰  rÔ  rÕ  rh  rŠ  rk  r1   r^   r•  r«  rª  r×  rØ  Ú
total_lossÚignored_indexr­  Ú
start_lossÚend_losss                          rC   r`   zMT5ForQuestionAnswering.forwardµ  s  € ðZ Ð&¨=Ð+DØˆIð
 Ð$Ð)>Ð)FØÐ Ý ðUñô ð ð
 !%× 1Ò 1°)Ñ <Ô <Ðð Ð"Ø*˜dœlð Ø#Ø-Ø+ðð ð ð	ð ˆOˆOõ ˜O­_Ñ=Ô=ð 	Ý-Ø"1°!Ô"4Ý47¸Ñ4HÔ4HÈ1Ò4LÐ4L˜o¨aÔ0Ð0ÐRVÝ14°_Ñ1EÔ1EÈÒ1IÐ1I˜?¨1Ô-Ð-Ètðñ ô ˆOð (¨Ô*ˆð '˜$œ,ð 	
Ø'Ø1Ø/Ø Ø"/Ø#1Øð	
ð 	
ð ð	
ð 	
ˆð *¨!Ô,ˆà—’ Ñ1Ô1ˆØ#)§<¢<°°r <Ñ#:Ô#:Ñ ˆ�jØ#×+Ò+¨BÑ/Ô/×:Ò:Ñ<Ô<ˆØ×'Ò'¨Ñ+Ô+×6Ò6Ñ8Ô8ˆ
àˆ
ØÑ&¨=Ñ+Då�?×'Ò'Ñ)Ô)Ñ*Ô*¨QÒ.Ð.Ø"1×"9Ò"9¸"Ñ"=Ô"=×"@Ò"@ÀÔATÑ"UÔ"U�Ý�=×%Ò%Ñ'Ô'Ñ(Ô(¨1Ò,Ð,Ø -× 5Ò 5°bÑ 9Ô 9× <Ò <¸ZÔ=NÑ OÔ O�à(×-Ò-¨aÑ0Ô0ˆMØ-×3Ò3°A°}ÑEÔEˆOØ)×/Ò/°°=ÑAÔAˆMå'°]ÐCÑCÔCˆHØ!˜ ,°Ñ@Ô@ˆJØ�x 
¨MÑ:Ô:ˆHØ$ xÑ/°1Ñ4ˆJå2ØØ%Ø!Ø+Ô;Ø"1Ô"?Ø.Ô9Ø,Ô=Ø&5Ô&GØ"1Ô"?Ø.Ô9ð
ñ 
ô 
ð 	
rE   r°  )ra   rb   rc   r˜  r™  r'   rK   r„  rf  r"   r   r;   rš  r›  rœ  r�  rz   rë   r   r   r   r`   rd   re   s   @rC   r8  r8  Œ  s´  ø€ € € € € à*rÐ)sÐ&à'6Ø'6ðð Ðð˜yð ð ð ð ð ð ð.ð ð ð:ð :ð :ð
 Øð .2Ø37Ø59Ø:>Ø=AØ37Ø15Ø26Ø:>Ø!%ðy
ð y
àÔ# dÑ*ðy
ð Ô)¨DÑ0ðy
ð !Ô+¨dÑ2ð	y
ð
 !&Ô 0°4Ñ 7ðy
ð ˜u U¤\Ô2Ô3°dÑ:ðy
ð Ô)¨DÑ0ðy
ð Ô'¨$Ñ.ðy
ð Ô(¨4Ñ/ðy
ð  %Ô0°4Ñ7ðy
ð ˜$‘;ðy
ð Ð+Ô,ðy
ð 
-ðy
ð y
ð y
ñ „^ñ Ôðy
ð y
ð y
ð y
ð y
rE   r8  )r7  r6  r8  r·  r>  r5  r   )Nr(   N)Pr  ry  r±   Úcollections.abcr   r;   r   Útorch.nnr   r   r   Ú r	   r3  Úactivationsr
   Úcache_utilsr   r   r   Ú
generationr   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r    r!   Úutils.genericr"   r#   Úutils.output_capturingr$   r%   Úconfiguration_mt5r'   Ú
get_loggerra   rŸ   ÚModulerz   r°   rD   rG   rg   r   r‡   r‘   rî   rú   r   r  r   rV  r5  r6  r7  r·  r>  r8  Ú__all__r“  rE   rC   ú<module>rð     sk  ðð Ð à €€€Ø €€€Ø $Ð $Ð $Ð $Ð $Ð $à €€€Ø Ð Ð Ð Ð Ð Ø AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aà &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CØ )Ð )Ð )Ð )Ð )Ð )Ø JÐ JÐ JÐ JÐ JÐ JÐ JÐ JØ 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð GÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø rÐ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rØ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ (Ð (Ð (Ð (Ð (Ð (ð 
ˆÔ	˜HÑ	%Ô	%€ð !ØØ)-ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð �T‰\ð%ð ð%ð ”< $Ñ&ð%ð Ð'Ô(ð%ð %ð %ð %ðB+ð +ð +ð +ð +�2”9ñ +ô +ð +ð4ð ð ð ð �r”yñ ô ð ð.ð ð ð ð ˜BœIñ ô ð ð<ð ð ð ð �”ñ ô ð ð&A8ð A8ð A8ð A8ð A8�2”9ñ A8ô A8ð A8ðJ=ð =ð =ð =ð =˜BœIñ =ô =ð =ðB9ð 9ð 9ð 9ð 9˜RœYñ 9ô 9ð 9ð@JPð JPð JPð JPð JPÐ)ñ JPô JPð JPð\ð ð ð ð ˜BœIñ ô ð ð$ ðj!ð j!ð j!ð j!ð j!˜ñ j!ô j!ñ „ðj!ð\n
ð n
ð n
ð n
ð n
Ð!ñ n
ô n
ð n
ðb ð]
ð ]
ð ]
ð ]
ð ]
Ð!ñ ]
ô ]
ñ „ð]
ð@ €ððñ ô ð
{)ð {)ð {)ð {)ð {)Ð"4°oñ {)ô {)ñô ð
{)ð| ðSð Sð Sð Sð SÐ(ñ Sô Sñ „ðSðl €ððñ ô ðL
ð L
ð L
ð L
ð L
Ð#5ñ L
ô L
ñô ðL
ð^ ð;
ð ;
ð ;
ð ;
ð ;
Ð 2ñ ;
ô ;
ñ „ð;
ð| ðd
ð d
ð d
ð d
ð d
Ð0ñ d
ô d
ñ „ðd
ðNð ð €€€rE   