§
    9²¸j ã                   óª  — d Z ddlZddlZddlmZ ddlZddlmZ ddlmZm	Z	m
Z
 ddlmZ ddlmZ dd	lmZmZmZ dd
lmZ ddlmZmZ ddlmZ ddlmZmZmZmZm Z m!Z!m"Z" ddl#m$Z$m%Z% ddl&m'Z' ddl(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z. ddl/m0Z0m1Z1 ddl2m3Z3m4Z4 ddl5m6Z6  e-j7        e8¦  «        Z9 G d„ dej:        ¦  «        Z; G d„ dej:        ¦  «        Z< G d„ dej:        ¦  «        Z= G d„ dej:        ¦  «        Z>	 	 	 dFdej:        dej?        d ej?        d!ej?        d"ej?        dz  d#e@dz  d$e@d%ej?        dz  d&e'e+         fd'„ZA G d(„ d)ej:        ¦  «        ZB G d*„ d+ej:        ¦  «        ZC G d,„ d-ej:        ¦  «        ZD G d.„ d/e¦  «        ZE G d0„ d1ej:        ¦  «        ZFe, G d2„ d3e%¦  «        ¦   «         ZG G d4„ d5eG¦  «        ZHe, G d6„ d7eG¦  «        ¦   «         ZI e,d8¬9¦  «         G d:„ d;eGe¦  «        ¦   «         ZJe, G d<„ d=eG¦  «        ¦   «         ZK e,d>¬9¦  «         G d?„ d@eG¦  «        ¦   «         ZLe, G dA„ dBeG¦  «        ¦   «         ZMe, G dC„ dDeG¦  «        ¦   «         ZNg dE¢ZOdS )GzPyTorch T5 model.é    N)ÚCallable)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutputÚ#Seq2SeqQuestionAnsweringModelOutputÚSeq2SeqSequenceClassifierOutputÚTokenClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚDUMMY_INPUTSÚ
DUMMY_MASKÚTransformersKwargsÚauto_docstringÚloggingÚtorch_compilable_check)Úcan_return_tupleÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚT5Configc                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚT5LayerNormç�íµ ÷Æ°>c                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )zc
        Construct a layernorm module in the T5 style. No bias and no subtraction of mean.
        N)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizeÚepsÚ	__class__s      €úr/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/t5/modeling_t5.pyr-   zT5LayerNorm.__init__3   sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐó    c                 óh  — |                      t          j        ¦  «                             d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        j        t          j	        t          j
        fv r|                      | j        j        ¦  «        }| j        |z  S )Né   éÿÿÿÿT)Úkeepdim)Útor/   Úfloat32ÚpowÚmeanÚrsqrtr2   r1   ÚdtypeÚfloat16Úbfloat16)r3   Úhidden_statesÚvariances      r7   ÚforwardzT5LayerNorm.forward;   s–   € ð !×#Ò#¥E¤MÑ2Ô2×6Ò6°qÑ9Ô9×>Ò>¸rÈ4Ð>ÑPÔPˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆð Œ;Ô¥¤µ´Ð ?Ð?Ð?Ø)×,Ò,¨T¬[Ô->Ñ?Ô?ˆMàŒ{˜]Ñ*Ð*r8   )r*   )Ú__name__Ú
__module__Ú__qualname__r-   rG   Ú__classcell__©r6   s   @r7   r)   r)   2   sL   ø€ € € € € ð$ð $ð $ð $ð $ð $ð+ð +ð +ð +ð +ð +ð +r8   r)   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚT5DenseActDenseÚconfigc                 óJ  •— t          ¦   «                              ¦   «          t          j        |j        |j        d¬¦  «        | _        t          j        |j        |j        d¬¦  «        | _        t          j        |j	        ¦  «        | _
        t          |j                 | _        d S ©NF©Úbias)r,   r-   r   ÚLinearÚd_modelÚd_ffÚwiÚwoÚDropoutÚdropout_rateÚdropoutr
   Údense_act_fnÚact©r3   rO   r6   s     €r7   r-   zT5DenseActDense.__init__L   sx   ø€ Ý‰Œ×ÒÑÔÐÝ”)˜FœN¨F¬K¸eÐDÑDÔDˆŒÝ”)˜FœK¨¬¸eÐDÑDÔDˆŒÝ”z &Ô"5Ñ6Ô6ˆŒÝ˜&Ô-Ô.ˆŒˆˆr8   c                 ó°  — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }t          | j        j        t          j        ¦  «        r]|j        | j        j        j        k    rC| j        j        j        t          j	        k    r$| 
                    | j        j        j        ¦  «        }|                      |¦  «        }|S ©N)rW   r]   r[   Ú
isinstancerX   r1   r/   ÚTensorrB   Úint8r=   ©r3   rE   s     r7   rG   zT5DenseActDense.forwardS   s¨   € ØŸš Ñ.Ô.ˆØŸš Ñ/Ô/ˆØŸš ]Ñ3Ô3ˆå�t”w”~¥u¤|Ñ4Ô4ð	CàÔ# t¤w¤~Ô';Ò;Ð;Ø””Ô$­¬
Ò2Ð2à)×,Ò,¨T¬W¬^Ô-AÑBÔBˆMØŸš Ñ.Ô.ˆØÐr8   ©rH   rI   rJ   r'   r-   rG   rK   rL   s   @r7   rN   rN   K   sS   ø€ € € € € ð/˜xð /ð /ð /ð /ð /ð /ðð ð ð ð ð ð r8   rN   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚT5DenseGatedActDenserO   c                 ó–  •— t          ¦   «                              ¦   «          t          j        |j        |j        d¬¦  «        | _        t          j        |j        |j        d¬¦  «        | _        t          j        |j        |j        d¬¦  «        | _        t          j	        |j
        ¦  «        | _        t          |j                 | _        d S rQ   )r,   r-   r   rT   rU   rV   Úwi_0Úwi_1rX   rY   rZ   r[   r
   r\   r]   r^   s     €r7   r-   zT5DenseGatedActDense.__init__b   s”   ø€ Ý‰Œ×ÒÑÔÐÝ”I˜fœn¨f¬kÀÐFÑFÔFˆŒ	Ý”I˜fœn¨f¬kÀÐFÑFÔFˆŒ	Ý”)˜FœK¨¬¸eÐDÑDÔDˆŒÝ”z &Ô"5Ñ6Ô6ˆŒÝ˜&Ô-Ô.ˆŒˆˆr8   c                 óà  — |                       |                      |¦  «        ¦  «        }|                      |¦  «        }||z  }|                      |¦  «        }t	          | j        j        t          j        ¦  «        r]|j	        | j        j        j	        k    rC| j        j        j	        t          j
        k    r$|                     | j        j        j	        ¦  «        }|                      |¦  «        }|S r`   )r]   ri   rj   r[   ra   rX   r1   r/   rb   rB   rc   r=   )r3   rE   Úhidden_geluÚhidden_linears       r7   rG   zT5DenseGatedActDense.forwardj   sÀ   € Ø—h’h˜tŸyšy¨Ñ7Ô7Ñ8Ô8ˆØŸ	š	 -Ñ0Ô0ˆØ# mÑ3ˆØŸš ]Ñ3Ô3ˆõ �t”w”~¥u¤|Ñ4Ô4ð	CàÔ# t¤w¤~Ô';Ò;Ð;Ø””Ô$­¬
Ò2Ð2à)×,Ò,¨T¬W¬^Ô-AÑBÔBˆMàŸš Ñ.Ô.ˆØÐr8   re   rL   s   @r7   rg   rg   a   sS   ø€ € € € € ð/˜xð /ð /ð /ð /ð /ð /ðð ð ð ð ð ð r8   rg   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )Ú	T5LayerFFrO   c                 ó$  •— t          ¦   «                              ¦   «          |j        rt          |¦  «        | _        nt          |¦  «        | _        t          |j        |j        ¬¦  «        | _	        t          j        |j        ¦  «        | _        d S )N©r5   )r,   r-   Úis_gated_actrg   ÚDenseReluDenserN   r)   rU   Úlayer_norm_epsilonÚ
layer_normr   rY   rZ   r[   r^   s     €r7   r-   zT5LayerFF.__init__   sx   ø€ Ý‰Œ×ÒÑÔÐØÔð 	:Ý"6°vÑ">Ô">ˆDÔÐå"1°&Ñ"9Ô"9ˆDÔå% f¤n¸&Ô:SÐTÑTÔTˆŒÝ”z &Ô"5Ñ6Ô6ˆŒˆˆr8   c                 óŠ   — |                       |¦  «        }|                      |¦  «        }||                      |¦  «        z   }|S r`   )ru   rs   r[   )r3   rE   Úforwarded_statess      r7   rG   zT5LayerFF.forward‰   sF   € ØŸ?š?¨=Ñ9Ô9ÐØ×.Ò.Ð/?Ñ@Ô@ÐØ%¨¯ªÐ5EÑ(FÔ(FÑFˆØÐr8   re   rL   s   @r7   ro   ro   ~   sS   ø€ € € € € ð7˜xð 7ð 7ð 7ð 7ð 7ð 7ðð ð ð ð ð ð r8   ro   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingr[   Úposition_biasÚkwargsc                 ó¼  — |€|                      d¦  «        dz  }t          j        ||                     dd¦  «        ¦  «        |z  }	|�|	|z   }	|�|	|z   }	t          j                             |	d¬¦  «        }	t          j                             |	|| j        ¬¦  «        }	t          j        |	|¦  «        }
|
                     dd¦  «         	                    ¦   «         }
|
|	fS )Nr;   ç      à¿r:   r   ©Údim)ÚpÚtrainingr&   )
Úsizer/   ÚmatmulÚ	transposer   Ú
functionalÚsoftmaxr[   r†   Ú
contiguous)ry   rz   r{   r|   r}   r~   r[   r   r€   Úattn_weightsÚattn_outputs              r7   Úeager_attention_forwardr�   �   sØ   € ð €Ø—*’*˜R‘.”. DÑ(ˆõ ”<  s§}¢}°Q¸Ñ':Ô':Ñ;Ô;¸gÑE€LàÐ Ø# mÑ3ˆàÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2Ð(Ñ>Ô>€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r8   c                   ó|   ‡ — e Zd Z	 	 	 ddededz  defˆ fd„Zedd
„¦   «         Zdd„Z		 	 	 	 dde
e         fd„Zˆ xZS )ÚT5AttentionFNrO   Ú	layer_idxÚ	is_causalc                 óT  •— t          ¦   «                              ¦   «          || _        |j        | _        || _        || _        |j        | _        |j        | _        |j        | _        |j	        | _
        |j        | _        |j        | _        | j        | j
        z  | _        d| _        || _        |€/| j        r(t$                               d| j        j        › d�¦  «         t-          j        | j        | j        d¬¦  «        | _        t-          j        | j        | j        d¬¦  «        | _        t-          j        | j        | j        d¬¦  «        | _        t-          j        | j        | j        d¬¦  «        | _        | j        r$t-          j        | j        | j        ¦  «        | _        d| _        d S )Nç      ð?zInstantiating a decoder z³ without passing `layer_idx` is not recommended and will to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` when creating this class.FrR   )r,   r-   rO   Ú
is_decoderr“   Úhas_relative_attention_biasÚrelative_attention_num_bucketsÚrelative_attention_max_distancerU   Úd_kvÚkey_value_proj_dimÚ	num_headsÚn_headsrZ   r[   Ú	inner_dimr~   r’   ÚloggerÚwarning_oncer6   rH   r   rT   ÚqÚkÚvÚoÚ	EmbeddingÚrelative_attention_biasÚgradient_checkpointing)r3   rO   r—   r’   r“   r6   s        €r7   r-   zT5Attention.__init__±   s~  ø€ õ 	‰Œ×ÒÑÔÐØˆŒØ Ô+ˆŒØ"ˆŒØ+FˆÔ(Ø.4Ô.SˆÔ+Ø/5Ô/UˆÔ,Ø”~ˆŒØ"(¤+ˆÔØÔ'ˆŒØÔ*ˆŒØœ¨Ô(?Ñ?ˆŒàˆŒØ"ˆŒØÐ ¤ÐÝ×Òð,¨4¬>Ô+Bð ,ð ,ð ,ñô ð õ ”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ>¨4¬<¸eÐDÑDÔDˆŒàÔ+ð 	kÝ+-¬<¸Ô8[Ð]aÔ]iÑ+jÔ+jˆDÔ(à&+ˆÔ#Ð#Ð#r8   Té    é€   c                 óP  — d}|rC|dz  }|| dk                          t          j        ¦  «        |z  z  }t          j        | ¦  «        } n(t          j        | t          j        | ¦  «        ¦  «         } |dz  }| |k     }|t          j        |                      ¦   «         |z  ¦  «        t          j        ||z  ¦  «        z  ||z
  z                        t          j        ¦  «        z   }t          j        |t          j	        ||dz
  ¦  «        ¦  «        }|t          j
        || |¦  «        z  }|S )aÒ  
        Adapted from Mesh Tensorflow:
        https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593

        Translate relative position to a bucket number for relative attention. The relative position is defined as
        memory_position - query_position, i.e. the distance in tokens from the attending position to the attended-to
        position. If bidirectional=False, then positive relative positions are invalid. We use smaller buckets for
        small absolute relative_position and larger buckets for larger absolute relative_positions. All relative
        positions >=max_distance map to the same bucket. All relative positions <=-max_distance map to the same bucket.
        This should allow for more graceful generalization to longer sequences than the model has been trained on

        Args:
            relative_position: an int32 Tensor
            bidirectional: a boolean - whether the attention is bidirectional
            num_buckets: an integer
            max_distance: an integer

        Returns:
            a Tensor with the same shape as relative_position, containing int32 values in the range [0, num_buckets)
        r   r:   r&   )r=   r/   ÚlongÚabsÚminÚ
zeros_likeÚlogÚfloatÚmathÚ	full_likeÚwhere)Úrelative_positionÚbidirectionalÚnum_bucketsÚmax_distanceÚrelative_bucketsÚ	max_exactÚis_smallÚrelative_position_if_larges           r7   Ú_relative_position_bucketz%T5Attention._relative_position_bucketØ   s>  € ð, ÐØð 	cØ˜AÑˆKØÐ!2°QÒ!6× :Ò :½5¼:Ñ FÔ FÈÑ TÑTÐÝ %¤	Ð*;Ñ <Ô <ÐÐå!&¤Ð+<½eÔ>NÐO`Ñ>aÔ>aÑ!bÔ!bÐ bÐð   1Ñ$ˆ	Ø$ yÒ0ˆð &/ÝŒIÐ'×-Ò-Ñ/Ô/°)Ñ;Ñ<Ô<ÝŒh�| iÑ/Ñ0Ô0ñ1à˜YÑ&ñ(÷ Š"�UŒZ‰.Œ.ñ	&Ð"õ
 &+¤YØ&­¬Ð8RÐT_ÐbcÑTcÑ(dÔ(dñ&
ô &
Ð"ð 	�EœK¨Ð2CÐE_Ñ`Ô`Ñ`ÐØÐr8   r   c                 ó¸  — |€| j         j        j        }t          j        |t          j        |¬¦  «        dd…df         |z   }t          j        |t          j        |¬¦  «        ddd…f         }||z
  }|                      || j         | j        | j	        ¬¦  «        }|                       |¦  «        }	|	 
                    g d¢¦  «                             d¦  «        }	|	S )z%Compute binned relative position biasN)rB   Údevice)rµ   r¶   r·   )r:   r   r&   r   )r¦   r1   r¾   r/   Úaranger«   r¼   r–   r˜   r™   ÚpermuteÚ	unsqueeze)
r3   Úquery_lengthÚ
key_lengthr¾   Úpast_seen_tokensÚcontext_positionÚmemory_positionr´   Úrelative_position_bucketÚvaluess
             r7   Úcompute_biaszT5Attention.compute_bias  sí   € àˆ>ØÔ1Ô8Ô?ˆFÝ œ<¨½E¼JÈvÐVÑVÔVÐWXÐWXÐWXÐZ^ÐW^Ô_ÐbrÑrÐÝœ, z½¼ÈFÐSÑSÔSÐTXÐZ[ÐZ[ÐZ[ÐT[Ô\ˆØ+Ð.>Ñ>ÐØ#'×#AÒ#AØØ#œÐ.ØÔ;ØÔ=ð	 $Bñ $
ô $
Ð ð ×-Ò-Ð.FÑGÔGˆØ—’ 	 	 	Ñ*Ô*×4Ò4°QÑ7Ô7ˆØˆr8   r€   c                 ó"  — |j         dd…         }g |¢d‘| j        ‘R }|�|                     | j        ¦  «        nd}	t	          |	t
          j        ¦  «        r|	                     ¦   «         n|	}	|du}
|                      |¦  «         	                    |¦  «         
                    dd¦  «        }d}t	          |t          ¦  «        r1|j                             | j        ¦  «        }|
r|j        }n
|j        }n|}|
r|n|}|
r3|�1|r/|j        | j                 j        }|j        | j                 j        }nÚg |j         dd…         ¢d‘| j        ‘R }|                      |¦  «         	                    |¦  «         
                    dd¦  «        }|                      |¦  «         	                    |¦  «         
                    dd¦  «        }|�E|                     ||| j        ¦  «        \  }}|
r$t	          |t          ¦  «        rd|j        | j        <   |€~|j         d         }| j        sFt          j        d| j        |d         |f|j        |j        ¬	¦  «        }| j        r| j        rd|_        n$|                      |d         ||j        |	¬
¦  «        }t?          j         | j!        j"        tF          ¦  «        } || ||||f| j        sdn| j$        | j%        |dœ|¤Ž\  }} |j&        g |¢d‘R Ž  '                    ¦   «         }|  (                    |¦  «        }|||fS )z€
        Self-attention (if key_value_states is None) or attention over source sentence (provided by key_value_states).
        Nr;   r   r&   r:   FTéþÿÿÿ)r¾   rB   )r¾   rÄ   rx   )r[   r~   r   ))Úshaper›   Úget_seq_lengthr’   ra   r/   rb   Úcloner¡   Úviewr‰   r   Ú
is_updatedÚgetÚcross_attention_cacheÚself_attention_cacheÚlayersÚkeysrÈ   r¢   r£   Úupdater—   Úzerosr�   r¾   rB   r§   r†   Úrequires_gradrÉ   r   Úget_interfacerO   Ú_attn_implementationr�   r[   r~   ÚreshaperŒ   r¤   )r3   rE   ÚmaskÚkey_value_statesr   Úpast_key_valuesr€   Úinput_shapeÚhidden_shaperÄ   Úis_cross_attentionÚquery_statesrÐ   Úcurr_past_key_valuesÚcurrent_statesÚ
key_statesÚvalue_statesÚkv_shaperÃ   Úattention_interfacerŽ   r�   s                         r7   rG   zT5Attention.forward  s�  € ð $Ô)¨#¨2¨#Ô.ˆØB˜ÐB bÐB¨$Ô*AÐBÐBˆØM\ÐMh˜?×9Ò9¸$¼.ÑIÔIÐIÐnoÐå7AÐBRÕTYÔT`Ñ7aÔ7aÐwÐ+×1Ò1Ñ3Ô3Ð3ÐgwÐð .°TÐ9Ðà—v’v˜mÑ,Ô,×1Ò1°,Ñ?Ô?×IÒIÈ!ÈQÑOÔOˆð ˆ
Ý�oÕ':Ñ;Ô;ð 	3Ø(Ô3×7Ò7¸¼ÑGÔGˆJØ!ð Là'6Ô'LÐ$Ð$à'6Ô'KÐ$Ð$à#2Ð à-?ÐRÐ)Ð)À]ˆØð 	F /Ð"=À*Ð"=à-Ô4°T´^ÔDÔIˆJØ/Ô6°t´~ÔFÔMˆLˆLàP˜Ô-¨c¨r¨cÔ2ÐP°BÐP¸Ô8OÐPÐPˆHØŸš Ñ/Ô/×4Ò4°XÑ>Ô>×HÒHÈÈAÑNÔNˆJØŸ6š6 .Ñ1Ô1×6Ò6°xÑ@Ô@×JÒJÈ1ÈaÑPÔPˆLàÐ*Ø+?×+FÒ+FÀzÐS_ÐaeÔaoÑ+pÔ+pÑ(�
˜Là%ð F­*°_ÕFYÑ*ZÔ*Zð FØAE�OÔ.¨t¬~Ñ>ð Ð Ø#Ô)¨"Ô-ˆJØÔ3ð Ý %¤Ø˜œ k°!¤n°jÐAØ'Ô.Ø&Ô,ð!ñ !ô !�ð
 Ô.ð 7°4´=ð 7Ø26�MÔ/øà $× 1Ò 1Ø ”N J°|Ô7JÐ]mð !2ñ !ô !�õ )@Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð  $œ}Ð>�C�C°$´,Ø”LØ'ð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—f’f˜[Ñ)Ô)ˆà˜M¨<Ð7Ð7r8   )FNF)Tr¨   r©   )Nr   ©NNNN)rH   rI   rJ   r'   ÚintÚboolr-   Ústaticmethodr¼   rÉ   r   r   rG   rK   rL   s   @r7   r‘   r‘   °   sä   ø€ € € € € ð %*Ø $Øð%,ð %,àð%,ð ˜‘:ð	%,ð
 ð%,ð %,ð %,ð %,ð %,ð %,ðN ð- ð - ð - ñ „\ð- ð^ð ð ð ð( ØØØðX8ð X8ð Ð+Ô,ðX8ð X8ð X8ð X8ð X8ð X8ð X8ð X8r8   r‘   c                   óL   ‡ — e Zd Zddedz  fˆ fd„Z	 	 	 ddee         fd„Zˆ xZS )	ÚT5LayerSelfAttentionFNr’   c                 óþ   •— t          ¦   «                              ¦   «          t          ||||j        ¬¦  «        | _        t          |j        |j        ¬¦  «        | _        t          j
        |j        ¦  «        | _        d S )N©r—   r’   r“   rq   )r,   r-   r‘   r–   ÚSelfAttentionr)   rU   rt   ru   r   rY   rZ   r[   ©r3   rO   r—   r’   r6   s       €r7   r-   zT5LayerSelfAttention.__init__u  sr   ø€ Ý‰Œ×ÒÑÔÐÝ(ØØ(CØØÔ'ð	
ñ 
ô 
ˆÔõ & f¤n¸&Ô:SÐTÑTÔTˆŒÝ”z &Ô"5Ñ6Ô6ˆŒˆˆr8   r€   c                 ó’   — |                       |¦  «        } | j        |f|||dœ|¤Ž\  }}}||                      |¦  «        z   }|||fS )N)rÜ   r   rÞ   )ru   rñ   r[   )	r3   rE   r}   r   rÞ   r€   Únormed_hidden_statesÚattention_outputr�   s	            r7   rG   zT5LayerSelfAttention.forward€  sy   € ð  $Ÿš¨}Ñ=Ô=ÐØ8J¸Ô8JØ ð9
àØ'Ø+ð	9
ð 9
ð
 ð9
ð 9
Ñ5Ð˜-¨ð )¨4¯<ª<Ð8HÑ+IÔ+IÑIÐØ °Ð<Ð<r8   ©FN©NNN©	rH   rI   rJ   rê   r-   r   r   rG   rK   rL   s   @r7   rî   rî   t  s‚   ø€ € € € € ð	7ð 	7ÈSÐSWÉZð 	7ð 	7ð 	7ð 	7ð 	7ð 	7ð ØØð=ð =ð Ð+Ô,ð=ð =ð =ð =ð =ð =ð =ð =r8   rî   c                   óL   ‡ — e Zd Zddedz  fˆ fd„Z	 	 	 ddee         fd„Zˆ xZS )ÚT5LayerCrossAttentionNr’   c                 óô   •— t          ¦   «                              ¦   «          t          |d|d¬¦  «        | _        t	          |j        |j        ¬¦  «        | _        t          j	        |j
        ¦  «        | _        d S )NFrð   rq   )r,   r-   r‘   ÚEncDecAttentionr)   rU   rt   ru   r   rY   rZ   r[   )r3   rO   r’   r6   s      €r7   r-   zT5LayerCrossAttention.__init__•  sm   ø€ Ý‰Œ×ÒÑÔÐÝ*Ø°ÀÐV[ð 
ñ  
ô  
ˆÔõ & f¤n¸&Ô:SÐTÑTÔTˆŒÝ”z &Ô"5Ñ6Ô6ˆŒˆˆr8   r€   c                 ó”   — |                       |¦  «        } | j        |f||||dœ|¤Ž\  }}}	||                      |¦  «        z   }
|
||	fS )N)rÜ   rÝ   r   rÞ   )ru   rü   r[   )r3   rE   rÝ   r}   r   rÞ   r€   rô   rõ   r�   Úlayer_outputs              r7   rG   zT5LayerCrossAttention.forward�  s{   € ð  $Ÿš¨}Ñ=Ô=ÐØ8L¸Ô8LØ ð9
àØ-Ø'Ø+ð9
ð 9
ð ð9
ð 9
Ñ5Ð˜-¨ð % t§|¢|Ð4DÑ'EÔ'EÑEˆØ˜]¨LÐ8Ð8r8   r`   r÷   rø   rL   s   @r7   rú   rú   ”  s�   ø€ € € € € ð7ð 7¨#°©*ð 7ð 7ð 7ð 7ð 7ð 7ð ØØð9ð 9ð Ð+Ô,ð9ð 9ð 9ð 9ð 9ð 9ð 9ð 9r8   rú   c                   óR   ‡ — e Zd Zddedz  fˆ fd„Z	 	 	 	 	 	 ddee         fd„Zˆ xZS )	ÚT5BlockFNr’   c                 ó’  •— t          ¦   «                              ¦   «          |j        | _        t          j        ¦   «         | _        | j                             t          |||¬¦  «        ¦  «         | j        r)| j                             t          ||¬¦  «        ¦  «         | j                             t          |¦  «        ¦  «         d S )N©r—   r’   )r’   )
r,   r-   r–   r   Ú
ModuleListÚlayerÚappendrî   rú   ro   rò   s       €r7   r-   zT5Block.__init__´  s³   ø€ Ý‰Œ×ÒÑÔÐØ Ô+ˆŒÝ”]‘_”_ˆŒ
ØŒ
×ÒÝ  ÐE`ÐluÐvÑvÔvñ	
ô 	
ð 	
ð Œ?ð 	RØŒJ×ÒÕ3°FÀiÐPÑPÔPÑQÔQÐQàŒ
×Ò�) FÑ+Ô+Ñ,Ô,Ð,Ð,Ð,r8   r€   c                 ó”  —  | j         d         |f|||dœ|¤Ž\  }}	}
|j        t          j        k    r�t          j        t          j        |¦  «                             ¦   «         t          j        |j        ¦  «        j        dz
  t          j        |j        ¦  «        j        ¦  «        }t          j	        || |¬¦  «        }d }| j
        o|d u}|r¿ | j         d         |f||||dœ|¤Ž\  }}}
|j        t          j        k    r�t          j        t          j        |¦  «                             ¦   «         t          j        |j        ¦  «        j        dz
  t          j        |j        ¦  «        j        ¦  «        }t          j	        || |¬¦  «        } | j         d         |¦  «        }|j        t          j        k    r�t          j        t          j        |¦  «                             ¦   «         t          j        |j        ¦  «        j        dz
  t          j        |j        ¦  «        j        ¦  «        }t          j	        || |¬¦  «        }||	|fS )Nr   )r}   r   rÞ   iè  )r­   Úmaxr&   )rÝ   r}   r   rÞ   r;   )r  rB   r/   rC   r³   ÚisinfÚanyÚfinfor  Úclampr–   )r3   rE   r}   r   Úencoder_hidden_statesÚencoder_attention_maskÚencoder_decoder_position_biasrÞ   r€   Úself_attn_position_biasÚ_Úclamp_valueÚcross_attn_position_biasÚdo_cross_attentions                 r7   rG   zT5Block.forwardÀ  sR  € ð 5B°D´J¸q´MØð5
à)Ø'Ø+ð	5
ð 5
ð
 ð5
ð 5
Ñ1ˆÐ.°ð Ô¥%¤-Ò/Ð/Ýœ+Ý”˜MÑ*Ô*×.Ò.Ñ0Ô0Ý”˜MÔ/Ñ0Ô0Ô4°tÑ;Ý”˜MÔ/Ñ0Ô0Ô4ñô ˆKõ
 "œK¨¸K¸<È[ÐYÑYÔYˆMà#'Ð Ø!œ_ÐRÐ1FÈdÐ1RÐØð 	^Ø9F¸¼ÀA¼Øð:à!6Ø5Ø;Ø /ð:ð :ð ð:ð :Ñ6ˆMÐ3°Qð Ô"¥e¤mÒ3Ð3Ý#œkÝ”K Ñ.Ô.×2Ò2Ñ4Ô4Ý”K Ô 3Ñ4Ô4Ô8¸4Ñ?Ý”K Ô 3Ñ4Ô4Ô8ñô �õ
 !&¤¨MÀ¸|ÐQ\Ð ]Ñ ]Ô ]�ð '˜œ
 2œ }Ñ5Ô5ˆð Ô¥%¤-Ò/Ð/Ýœ+Ý”˜MÑ*Ô*×.Ò.Ñ0Ô0Ý”˜MÔ/Ñ0Ô0Ô4°tÑ;Ý”˜MÔ/Ñ0Ô0Ô4ñô ˆKõ
 "œK¨¸K¸<È[ÐYÑYÔYˆMàÐ5Ð7OÐOÐOr8   rö   )NNNNNNrø   rL   s   @r7   r   r   ³  s•   ø€ € € € € ð
-ð 
-ÈSÐSWÉZð 
-ð 
-ð 
-ð 
-ð 
-ð 
-ð ØØ"Ø#Ø&*Øð=Pð =Pð Ð+Ô,ð=Pð =Pð =Pð =Pð =Pð =Pð =Pð =Pr8   r   c                   óL   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚT5ClassificationHeadz-Head for sentence-level classification tasks.rO   c                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        ¬¦  «        | _        t          j        |j        |j	        ¦  «        | _
        d S )N)r…   )r,   r-   r   rT   rU   ÚdenserY   Úclassifier_dropoutr[   Ú
num_labelsÚout_projr^   s     €r7   r-   zT5ClassificationHead.__init__  sc   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vœ~¨v¬~Ñ>Ô>ˆŒ
Ý”z FÔ$=Ð>Ñ>Ô>ˆŒÝœ	 &¤.°&Ô2CÑDÔDˆŒˆˆr8   rE   Úreturnc                 óÖ   — |                       |¦  «        }|                      |¦  «        }t          j        |¦  «        }|                       |¦  «        }|                      |¦  «        }|S r`   )r[   r  r/   Útanhr  rd   s     r7   rG   zT5ClassificationHead.forward	  s[   € ØŸš ]Ñ3Ô3ˆØŸ
š
 =Ñ1Ô1ˆÝœ
 =Ñ1Ô1ˆØŸš ]Ñ3Ô3ˆØŸš mÑ4Ô4ˆØÐr8   )
rH   rI   rJ   Ú__doc__r'   r-   r/   rb   rG   rK   rL   s   @r7   r  r     sw   ø€ € € € € Ø7Ð7ðE˜xð Eð Eð Eð Eð Eð Eð U¤\ð °e´lð ð ð ð ð ð ð ð r8   r  c                   óâ   ‡ — e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZ eed¬¦  «         eed	¬¦  «         eed	¬¦  «        d
œZed„ ¦   «         Z ej        ¦   «         ˆ fd„¦   «         Zd„ Zˆ xZS )ÚT5PreTrainedModelrO   ÚtransformerTr   rX   Fr   )Úindexr;   )rE   Ú
attentionsÚcross_attentionsc                 óv   — t          j        t          ¦  «        }t          j        t          ¦  «        }|||dœ}|S )N)Údecoder_input_idsÚ	input_idsÚdecoder_attention_mask)r/   Útensorr   r   )r3   r'  Ú
input_maskÚdummy_inputss       r7   r+  zT5PreTrainedModel.dummy_inputs'  s=   € å”L¥Ñ.Ô.ˆ	Ý”\¥*Ñ-Ô-ˆ
à!*Ø"Ø&0ð
ð 
ˆð
 Ðr8   c                 ó\  •— t          ¦   «                              |¦  «         | j        j        }t	          |t
          ¦  «        rt          j        |j        |dz  ¦  «         d	S t	          |t          t          t          t          f¦  «        rÇt          j        |j        j        d|dz  ¬¦  «         t          |d¦  «        r0| j        j        s$t          j        |j        j        d|dz  ¬¦  «         t          |d¦  «        rQt          j        |j        j        d|| j        j        dz  z  ¬¦  «         t          j        |j        j        ¦  «         d	S d	S t	          |t,          ¦  «        rVt          |d¦  «        rDt          j        |j        j        d|dz  ¬¦  «         t          j        |j        j        ¦  «         d	S d	S t	          |t0          ¦  «        ræt          j        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r*|j        j        �t          j        |j        j        ¦  «         t          j        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r,|j        j        �"t          j        |j        j        ¦  «         d	S d	S d	S t	          |t6          ¦  «        ræt          j        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r*|j        j        �t          j        |j        j        ¦  «         t          j        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r,|j        j        �"t          j        |j        j        ¦  «         d	S d	S d	S t	          |t>          ¦  «        �rVt          j        |j         j        d|| j        j        dz  z  ¬¦  «         t          |j         d¦  «        r*|j         j        �t          j        |j         j        ¦  «         t          j        |j!        j        d|| j        j        dz  z  ¬¦  «         t          |j!        d¦  «        r*|j!        j        �t          j        |j!        j        ¦  «         t          j        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r,|j        j        �"t          j        |j        j        ¦  «         d	S d	S d	S t	          |tD          ¦  «        rö| j        j        }| j        j#        }| j        j$        }t          j        |j%        j        d|||z  dz  z  ¬¦  «         t          j        |j&        j        d||dz  z  ¬¦  «         t          j        |j'        j        d||dz  z  ¬¦  «         t          j        |j(        j        d|||z  dz  z  ¬¦  «         |j)        r+t          j        |j*        j        d||dz  z  ¬¦  «         d	S d	S d	S )
zInitialize the weightsr•   rx   )r@   ÚstdÚlm_headÚ
qa_outputsr‚   Ú
classifierrS   N)+r,   Ú_init_weightsrO   Úinitializer_factorra   r)   ÚinitÚ	constant_r1   ÚT5ModelÚT5ForConditionalGenerationÚT5EncoderModelÚT5ForQuestionAnsweringÚnormal_ÚsharedÚhasattrÚtie_word_embeddingsr.  r/  rU   Úzeros_rS   ÚT5ForTokenClassificationr0  r  r  r  rN   rW   rX   rV   rg   ri   rj   r‘   rš   rœ   r¡   r¢   r£   r¤   r—   r¦   )r3   ry   ÚfactorrU   r›   r�   r6   s         €r7   r1  zT5PreTrainedModel._init_weights2  sì  ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ø”Ô/ˆÝ�f�kÑ*Ô*ð 1	pÝŒN˜6œ=¨&°3©,Ñ7Ô7Ð7Ð7Ð7ÝØÝÕ0µ.ÕBXÐYñ
ô 
ð /	põ ŒL˜œÔ-°C¸VÀc¹\ÐJÑJÔJÐJÝ�v˜yÑ)Ô)ð P°$´+Ô2Qð PÝ”˜Vœ^Ô2¸À&È3Á,ÐOÑOÔOÐOÝ�v˜|Ñ,Ô,ð 4Ý”˜VÔ.Ô5¸CÀVÐPTÔP[ÔPcÐhlÑOlÑEmÐnÑnÔnÐnÝ”˜FÔ-Ô2Ñ3Ô3Ð3Ð3Ð3ð4ð 4õ ˜Õ 8Ñ9Ô9ð %	pÝ�v˜|Ñ,Ô,ð 4Ý”˜VÔ.Ô5¸CÀVÈcÁ\ÐRÑRÔRÐRÝ”˜FÔ-Ô2Ñ3Ô3Ð3Ð3Ð3ð4ð 4õ ˜Õ 4Ñ5Ô5ð !	pÝŒL˜œÔ,°3¸FÀtÄ{ÔGZÐ_cÑFcÑ<dÐeÑeÔeÐeÝ�v”| VÑ,Ô,ð /°´Ô1BÐ1NÝ”˜FœLÔ-Ñ.Ô.Ð.ÝŒL˜œÔ/°c¸vÈ$Ì+ÔJ]ÐbfÑIfÑ?gÐhÑhÔhÐhÝ�v”¨Ñ/Ô/ð 2°F´OÔ4HÐ4TÝ”˜FœOÔ0Ñ1Ô1Ð1Ð1Ð1ð2ð 2Ð4TÐ4Tå˜¥Ñ0Ô0ð 	pÝŒL˜œÔ)°¸ÀDÄKÔDWÐ\`ÑC`Ñ9aÐbÑbÔbÐbÝ�v”y &Ñ)Ô)ð ,¨f¬i¬nÐ.HÝ”˜FœIœNÑ+Ô+Ð+ÝŒL˜œÔ)°¸ÀDÄKÔDTÐY]ÑC]Ñ9^Ð_Ñ_Ô_Ð_Ý�v”y &Ñ)Ô)ð ,¨f¬i¬nÐ.HÝ”˜FœIœNÑ+Ô+Ð+Ð+Ð+ð,ð ,Ð.HÐ.Hå˜Õ 4Ñ5Ô5ñ 	pÝŒL˜œÔ+°#¸6ÀdÄkÔFYÐ^bÑEbÑ;cÐdÑdÔdÐdÝ�v”{ FÑ+Ô+ð .°´Ô0@Ð0LÝ”˜FœKÔ,Ñ-Ô-Ð-ÝŒL˜œÔ+°#¸6ÀdÄkÔFYÐ^bÑEbÑ;cÐdÑdÔdÐdÝ�v”{ FÑ+Ô+ð .°´Ô0@Ð0LÝ”˜FœKÔ,Ñ-Ô-Ð-ÝŒL˜œÔ)°¸ÀDÄKÔDTÐY]ÑC]Ñ9^Ð_Ñ_Ô_Ð_Ý�v”y &Ñ)Ô)ð ,¨f¬i¬nÐ.HÝ”˜FœIœNÑ+Ô+Ð+Ð+Ð+ð,ð ,Ð.HÐ.Hå˜¥Ñ,Ô,ð 		pØ”kÔ)ˆGØ!%¤Ô!1ÐØ”kÔ+ˆGÝŒL˜œœ¨s¸À7ÐM_ÑC_ÐdhÑBhÑ8iÐjÑjÔjÐjÝŒL˜œœ¨s¸À'È4Á-Ñ8PÐQÑQÔQÐQÝŒL˜œœ¨s¸À'È4Á-Ñ8PÐQÑQÔQÐQÝŒL˜œœ¨s¸À7ÐM_ÑC_ÐdhÑBhÑ8iÐjÑjÔjÐjØÔ1ð pÝ”˜VÔ;ÔBÈÐRXÐ]dÐimÑ\mÑRnÐoÑoÔoÐoÐoÐoð		pð 		pðpð pr8   c                 ó6  — | j         j        }| j         j        }|€t          d¦  «        ‚|                     |j        ¦  «        }|dd d…f                              ¦   «         |ddd …f<   ||d<   |€t          d¦  «        ‚|                     |dk    |¦  «         |S )NzŠself.model.config.decoder_start_token_id has to be defined. In T5 it is usually set to the pad_token_id. See T5 docs for more information..r;   r&   ).r   z1self.model.config.pad_token_id has to be defined.éœÿÿÿ)rO   Údecoder_start_token_idÚpad_token_idÚ
ValueErrorÚ	new_zerosrÌ   rÎ   Úmasked_fill_)r3   r'  rB  rC  Úshifted_input_idss        r7   Ú_shift_rightzT5PreTrainedModel._shift_rightj  sº   € Ø!%¤Ô!CÐØ”{Ô/ˆà!Ð)Ýð4ñô ð ð
 &×/Ò/°	´Ñ@Ô@ÐØ%.¨s°C°R°C¨xÔ%8×%>Ò%>Ñ%@Ô%@Ð˜#˜q˜r˜r˜'Ñ"Ø$:Ð˜&Ñ!àÐÝÐPÑQÔQÐQà×&Ò&Ð'8¸DÒ'@À,ÑOÔOÐOà Ð r8   )rH   rI   rJ   r'   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_keep_in_fp32_modulesÚ_supports_flash_attnÚ_supports_flex_attnÚ_supports_sdpaÚ_can_compile_fullgraphr$   r   rî   rú   Ú_can_record_outputsÚpropertyr+  r/   Úno_gradr1  rH  rK   rL   s   @r7   r   r     s  ø€ € € € € € àÐÐÑØ%ÐØ&*Ð#à"˜ÐØ!˜FÐð !ÐØÐØ€NØ!Ðð (˜¨°qÐ9Ñ9Ô9Ø$�nÐ%9ÀÐDÑDÔDØ*˜NÐ+@ÈÐKÑKÔKðð Ðð ðð ñ „Xðð €U„]�_„_ð5pð 5pð 5pð 5pñ „_ð5pðn!ð !ð !ð !ð !ð !ð !r8   r   c                   ó|   ‡ — e Zd Zˆ fd„Zd„ Zeee	 	 	 	 	 	 	 ddee	         fd„¦   «         ¦   «         ¦   «         Z
ˆ xZS )ÚT5Stackc                 óÌ  •‡— t          ¦   «                              ‰¦  «         t          j        ‰j        ‰j        ¦  «        | _        ‰j        | _        t          j        ˆfd„t          ‰j
        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        t          j        ‰j        ¦  «        | _        |                      ¦   «          d| _        d S )Nc           	      óV   •— g | ]%}t          ‰t          |d k    ¦  «        |¬¦  «        ‘Œ&S )r   r  )r   rë   )Ú.0ÚirO   s     €r7   ú
<listcomp>z$T5Stack.__init__.<locals>.<listcomp>ˆ  s4   ø€ ÐvÐvÐvÐXY�W�V½¸aÀ1ºf¹¼ÐQRÐSÑSÔSÐvÐvÐvr8   rq   F)r,   r-   r   r¥   Ú
vocab_sizerU   Úembed_tokensr–   r  ÚrangeÚ
num_layersÚblockr)   rt   Úfinal_layer_normrY   rZ   r[   Ú	post_initr§   r^   s    `€r7   r-   zT5Stack.__init__�  sÆ   øø€ Ý‰Œ×Ò˜Ñ Ô Ð åœL¨Ô):¸F¼NÑKÔKˆÔØ Ô+ˆŒå”]ØvÐvÐvÐvÕ]bÐciÔctÑ]uÔ]uÐvÑvÔvñ
ô 
ˆŒ
õ !,¨F¬NÀÔ@YÐ ZÑ ZÔ ZˆÔÝ”z &Ô"5Ñ6Ô6ˆŒð 	�ŠÑÔÐØ&+ˆÔ#Ð#Ð#r8   c                 ó   — || _         d S r`   )r]  ©r3   Únew_embeddingss     r7   Úset_input_embeddingszT5Stack.set_input_embeddings‘  s   € Ø*ˆÔÐÐr8   Nr€   c           	      ój  — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|du r| j        st          d| › d�¦  «        ‚| j        r]|rZ|€X| j        j        r7t          t          | j        ¬¦  «        t          | j        ¬¦  «        ¦  «        }nt          | j        ¬¦  «        }n	| j        sd }| j        j        r5t          | j        ||t          |t
          ¦  «        r|j	        n|¬¦  «        }nt          | j        ||¬¦  «        }d }	| j        r|�t          | j        |||¬¦  «        }	d }
d }|                      |¦  «        }| j        D ]#} ||||
||	|fd	|i|¤Ž\  }}}|}
| j        r|�|}Œ$|                      |¦  «        }|                      |¦  «        }t          ||¬
¦  «        S )Nz:You must specify exactly one of input_ids or inputs_embedsTz)`use_cache` can only be set to `True` if z is used as a decoder)rO   )rO   Úinputs_embedsr}   rÞ   )rO   rh  r}   )rO   rh  r}   r  rÞ   )Úlast_hidden_staterÞ   )rD  r]  r–   rO   Úis_encoder_decoderr   r   r   ra   rÓ   r   r[   r`  ra  r   )r3   r'  r}   r  r  rh  rÞ   Ú	use_cacher€   Úencoder_extended_attention_maskr   r  rE   Úlayer_moduleÚself_attention_position_biasÚcross_attention_position_biass                   r7   rG   zT5Stack.forward”  st  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMà˜ÐÐØ”?ð jÝ Ð!hÈTÐ!hÐ!hÐ!hÑiÔiÐiàŒ?ð 	#Øð G˜_Ð4Ø”;Ô1ð GÝ&9Ý$¨D¬KÐ8Ñ8Ô8½,ÈdÌkÐ:ZÑ:ZÔ:Zñ'ô '�O�Oõ '3¸$¼+Ð&FÑ&FÔ&F�OøØ”ð 	#ð #ˆOàŒ;Ô!ð 	Ý/Ø”{Ø+Ø-å˜oÕ/BÑCÔCð!% Ô DÐ Dà$ðñ ô ˆNˆNõ 7Ø”{Ø+Ø-ðñ ô ˆNð +/Ð'ØŒ?ð 	Ð4Ð@Ý.GØ”{Ø+Ø5Ø&;ð	/ñ /ô /Ð+ð ˆØ(,Ð%àŸš ]Ñ3Ô3ˆà œJð 	Nð 	NˆLØYeÐYeØØØØ%Ø/Ø-ð	Zð 	Zð !0ð	Zð ð	Zð 	ZÑVˆMÐ7Ð9Vð 9ˆMØŒð NÐ#8Ð#DØ0MÐ-øà×-Ò-¨mÑ<Ô<ˆØŸš ]Ñ3Ô3ˆå8Ø+Ø+ð
ñ 
ô 
ð 	
r8   )NNNNNNN)rH   rI   rJ   r-   rf  r#   r%   r   r   r   rG   rK   rL   s   @r7   rV  rV  €  s²   ø€ € € € € ð,ð ,ð ,ð ,ð ,ð +ð +ð +ð  ØØð ØØ"Ø#ØØØðW
ð W
ð Ð+Ô,ðW
ð W
ð W
ñ „^ñ „_ñ  ÔðW
ð W
ð W
ð W
ð W
r8   rV  c                   óH  ‡ — e Zd ZdgZdddœZdefˆ fd„Zd„ Zd„ Ze	e
	 	 	 	 	 	 	 	 	 dd	ej        dz  d
ej        dz  dej        dz  dej        dz  deeej                          dz  dedz  dej        dz  dej        dz  dedz  dee         defd„¦   «         ¦   «         Zˆ xZS )r5  úFdecoder.block.0.layer.1.EncDecAttention.relative_attention_bias.weightúshared.weight©úencoder.embed_tokens.weightúdecoder.embed_tokens.weightrO   c                 óœ  •— t          ¦   «                              |¦  «         t          j        |j        |j        ¦  «        | _        t          j        |¦  «        }d|_	        d|_
        t          |¦  «        | _        t          j        |¦  «        }d|_	        |j        |_        t          |¦  «        | _        |                      ¦   «          d S ©NFT)r,   r-   r   r¥   r\  rU   r:  ÚcopyÚdeepcopyr–   rk  rV  ÚencoderÚnum_decoder_layersr_  Údecoderrb  ©r3   rO   Úencoder_configÚdecoder_configr6   s       €r7   r-   zT5Model.__init__û  sª   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý”l 6Ô#4°f´nÑEÔEˆŒåœ vÑ.Ô.ˆØ$)ˆÔ!Ø#(ˆÔ Ý˜~Ñ.Ô.ˆŒåœ vÑ.Ô.ˆØ$(ˆÔ!Ø$*Ô$=ˆÔ!Ý˜~Ñ.Ô.ˆŒð 	�ŠÑÔÐÐÐr8   c                 ó   — | j         S r`   ©r:  ©r3   s    r7   Úget_input_embeddingszT5Model.get_input_embeddings  ó
   € ØŒ{Ðr8   c                 ó|   — || _         | j                             |¦  «         | j                             |¦  «         d S r`   ©r:  rz  rf  r|  rd  s     r7   rf  zT5Model.set_input_embeddings  ó;   € Ø$ˆŒØŒ×)Ò)¨.Ñ9Ô9Ð9ØŒ×)Ò)¨.Ñ9Ô9Ð9Ð9Ð9r8   Nr'  r}   r&  r(  Úencoder_outputsrÞ   rh  Údecoder_inputs_embedsrk  r€   r  c
                 ó¨  — |€ | j         d	|||dœ|
¤Ž}nct          |t          ¦  «        sNt          |d         t          |¦  «        dk    r|d         ndt          |¦  «        dk    r|d         nd¬¦  «        }|d         } | j        d	|||||||	dœ|
¤Ž}t          |j        |j        |j        |j	        |j
        |j        |j        |j	        ¬¦  «        S )
aV
  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you
            should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            [What are input IDs?](../glossary#input-ids)

            To know more on how to prepare `input_ids` for pretraining take a look a [T5 Training](./t5#training).
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            T5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
            is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).

            To know more on how to prepare `decoder_input_ids` for pretraining take a look at [T5
            Training](./t5#training).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, T5Model

        >>> tokenizer = AutoTokenizer.from_pretrained("google-t5/t5-small")
        >>> model = T5Model.from_pretrained("google-t5/t5-small")

        >>> input_ids = tokenizer(
        ...     "Studies have been shown that owning a dog is good for you", return_tensors="pt"
        ... ).input_ids  # Batch size 1
        >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="pt").input_ids  # Batch size 1

        >>> # preprocess: Prepend decoder_input_ids with start token which is pad token for T5Model.
        >>> # This is not needed for torch's T5ForConditionalGeneration as it does this internally using labels arg.
        >>> decoder_input_ids = model._shift_right(decoder_input_ids)

        >>> # forward pass
        >>> outputs = model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)
        >>> last_hidden_states = outputs.last_hidden_state
        ```N©r'  r}   rh  r   r&   r:   ©ri  rE   r#  ©r'  r}   rh  rÞ   r  r  rk  )ri  rÞ   Údecoder_hidden_statesÚdecoder_attentionsr$  Úencoder_last_hidden_stater  Úencoder_attentions© )rz  ra   r   Úlenr|  r   ri  rÞ   rE   r#  r$  )r3   r'  r}   r&  r(  rˆ  rÞ   rh  r‰  rk  r€   rE   Údecoder_outputss                r7   rG   zT5Model.forward  s?  € ðB Ð"Ø*˜dœlð Ø#Ø-Ø+ðð ð ð	ð ˆOˆOõ ˜O­_Ñ=Ô=ð 	Ý-Ø"1°!Ô"4Ý47¸Ñ4HÔ4HÈ1Ò4LÐ4L˜o¨aÔ0Ð0ÐRVÝ14°_Ñ1EÔ1EÈÒ1IÐ1I˜?¨1Ô-Ð-Ètðñ ô ˆOð (¨Ô*ˆð '˜$œ,ð 	
Ø'Ø1Ø/Ø+Ø"/Ø#1Øð	
ð 	
ð ð	
ð 	
ˆõ "Ø-Ô?Ø+Ô;Ø"1Ô"?Ø.Ô9Ø,Ô=Ø&5Ô&GØ"1Ô"?Ø.Ô9ð	
ñ 	
ô 	
ð 		
r8   ©	NNNNNNNNN)rH   rI   rJ   Ú"_keys_to_ignore_on_load_unexpectedÚ_tied_weights_keysr'   r-   rƒ  rf  r"   r   r/   Ú
LongTensorÚFloatTensorÚ
BoolTensorÚtupler   rb   rë   r   r   r   rG   rK   rL   s   @r7   r5  r5  ñ  s   ø€ € € € € ð 	Qð*Ð&ð (7Ø'6ðð Ðð
˜xð ð ð ð ð ð ð"ð ð ð:ð :ð :ð
 Øð .2Ø37Ø59Ø:>ØBFØ(,Ø-1Ø59Ø!%ðd
ð d
àÔ# dÑ*ðd
ð Ô)¨DÑ0ðd
ð !Ô+¨dÑ2ð	d
ð
 !&Ô 0°4Ñ 7ðd
ð ˜u UÔ%6Ô7Ô8¸4Ñ?ðd
ð  ™ðd
ð ”| dÑ*ðd
ð  %œ|¨dÑ2ðd
ð ˜$‘;ðd
ð Ð+Ô,ðd
ð 
ðd
ð d
ð d
ñ „^ñ Ôðd
ð d
ð d
ð d
ð d
r8   r5  z:
    T5 Model with a `language modeling` head on top.
    )Úcustom_introc                   óv  ‡ — e Zd ZdgZddddœZdefˆ fd„Zd„ Zd„ Ze	e
	 	 	 	 	 	 	 	 	 	 dd	ej        dz  d
ej        dz  dej        dz  dej        dz  deeej                          dz  dedz  dej        dz  dej        dz  dej        dz  dedz  dee         defd„¦   «         ¦   «         Zdej        fd„Zˆ xZS )r6  rq  rr  )zlm_head.weightrt  ru  rO   c                 ó   •— t          ¦   «                              |¦  «         |j        | _        t	          j        |j        |j        ¦  «        | _        t          j	        |¦  «        }d|_
        d|_        t          |¦  «        | _        t          j	        |¦  «        }d|_
        |j        |_        t          |¦  «        | _        t	          j        |j        |j        d¬¦  «        | _        |                      ¦   «          d S )NFTrR   )r,   r-   rU   Ú	model_dimr   r¥   r\  r:  rx  ry  r–   rk  rV  rz  r{  r_  r|  rT   r.  rb  r}  s       €r7   r-   z#T5ForConditionalGeneration.__init__Œ  sÐ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØœˆŒå”l 6Ô#4°f´nÑEÔEˆŒåœ vÑ.Ô.ˆØ$)ˆÔ!Ø#(ˆÔ Ý˜~Ñ.Ô.ˆŒåœ vÑ.Ô.ˆØ$(ˆÔ!Ø$*Ô$=ˆÔ!Ý˜~Ñ.Ô.ˆŒå”y ¤°Ô1BÈÐOÑOÔOˆŒð 	�ŠÑÔÐÐÐr8   c                 ó   — | j         S r`   r�  r‚  s    r7   rƒ  z/T5ForConditionalGeneration.get_input_embeddings¡  r„  r8   c                 ó|   — || _         | j                             |¦  «         | j                             |¦  «         d S r`   r†  rd  s     r7   rf  z/T5ForConditionalGeneration.set_input_embeddings¤  r‡  r8   Nr'  r}   r&  r(  rˆ  rÞ   rh  r‰  Úlabelsrk  r€   r  c                 ó*  — |€ | j         d|||dœ|¤Ž}nct          |t          ¦  «        sNt          |d         t          |¦  «        dk    r|d         ndt          |¦  «        dk    r|d         nd¬¦  «        }|d         }|	�|€|€|                      |	¦  «        } | j        d|||||||
dœ|¤Ž}|d         }| j        j        r|| j        dz  z  }|  	                    |¦  «        }d}|	�pt          d	¬
¦  «        }|	                     |j        ¦  «        }	 ||                     d|                     d¦  «        ¦  «        |	                     d¦  «        ¦  «        }t          |||j        |j        |j        |j        |j        |j        |j        ¬¦	  «	        S )a  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you
            should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            [What are input IDs?](../glossary#input-ids)

            To know more on how to prepare `input_ids` for pretraining take a look a [T5 Training](./t5#training).
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            T5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
            is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).

            To know more on how to prepare `decoder_input_ids` for pretraining take a look at [T5
            Training](./t5#training).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ...,
            config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for
            labels in `[0, ..., config.vocab_size]`

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, T5ForConditionalGeneration

        >>> tokenizer = AutoTokenizer.from_pretrained("google-t5/t5-small")
        >>> model = T5ForConditionalGeneration.from_pretrained("google-t5/t5-small")

        >>> # training
        >>> input_ids = tokenizer("The <extra_id_0> walks in <extra_id_1> park", return_tensors="pt").input_ids
        >>> labels = tokenizer("<extra_id_0> cute dog <extra_id_1> the <extra_id_2>", return_tensors="pt").input_ids
        >>> outputs = model(input_ids=input_ids, labels=labels)
        >>> loss = outputs.loss
        >>> logits = outputs.logits

        >>> # inference
        >>> input_ids = tokenizer(
        ...     "summarize: studies have shown that owning a dog is good for you", return_tensors="pt"
        ... ).input_ids  # Batch size 1
        >>> outputs = model.generate(input_ids)
        >>> print(tokenizer.decode(outputs[0], skip_special_tokens=True))
        >>> # studies have shown that owning a dog is good for you.
        ```Nr‹  r   r&   r:   rŒ  r�  r‚   rA  ©Úignore_indexr;   ©	ÚlossÚlogitsrÞ   rŽ  r�  r$  r�  r  r‘  r’  )rz  ra   r   r“  rH  r|  rO   Úscale_decoder_outputsrŸ  r.  r   r=   r¾   rÏ   r‡   r   rÞ   rE   r#  r$  ri  )r3   r'  r}   r&  r(  rˆ  rÞ   rh  r‰  r¢  rk  r€   rE   r”  Úsequence_outputÚ	lm_logitsr§  Úloss_fcts                     r7   rG   z"T5ForConditionalGeneration.forward©  s  € ðP Ð"à*˜dœlð Ø#Ø-Ø+ðð ð ð	ð ˆOˆOõ ˜O­_Ñ=Ô=ð 	Ý-Ø"1°!Ô"4Ý47¸Ñ4HÔ4HÈ1Ò4LÐ4L˜o¨aÔ0Ð0ÐRVÝ14°_Ñ1EÔ1EÈÒ1IÐ1I˜?¨1Ô-Ð-Ètðñ ô ˆOð (¨Ô*ˆàÐÐ"3Ð";Ð@UÐ@]à $× 1Ò 1°&Ñ 9Ô 9Ðð '˜$œ,ð 	
Ø'Ø1Ø/Ø+Ø"/Ø#1Øð	
ð 	
ð ð	
ð 	
ˆð *¨!Ô,ˆàŒ;Ô,ð 	GØ-°´ÀÑ1EÑFˆOà—L’L Ñ1Ô1ˆ	àˆØÐÝ'°TÐ:Ñ:Ô:ˆHà—Y’Y˜yÔ/Ñ0Ô0ˆFØ�8˜IŸNšN¨2¨y¯~ª~¸bÑ/AÔ/AÑBÔBÀFÇKÂKÐPRÁOÄOÑTÔTˆDåØØØ+Ô;Ø"1Ô"?Ø.Ô9Ø,Ô=Ø&5Ô&GØ"1Ô"?Ø.Ô9ð

ñ 

ô 

ð 
	
r8   c                 ó,   — |                       |¦  «        S r`   )rH  )r3   r¢  s     r7   Ú%prepare_decoder_input_ids_from_labelsz@T5ForConditionalGeneration.prepare_decoder_input_ids_from_labels,  s   € Ø× Ò  Ñ(Ô(Ð(r8   ©
NNNNNNNNNN)rH   rI   rJ   r–  r—  r'   r-   rƒ  rf  r"   r   r/   r˜  r™  rš  r›  rb   r   rë   r   r   r   rG   r®  rK   rL   s   @r7   r6  r6  }  sÔ  ø€ € € € € ð 	Qð*Ð&ð *Ø'6Ø'6ðð Ðð˜xð ð ð ð ð ð ð*ð ð ð:ð :ð :ð
 Øð .2Ø37Ø59Ø:>Ø=AØ(,Ø26Ø:>Ø*.Ø!%ð
ð 
àÔ# dÑ*ð
ð Ô)¨DÑ0ð
ð !Ô+¨dÑ2ð	
ð
 !&Ô 0°4Ñ 7ð
ð ˜u U¤\Ô2Ô3°dÑ:ð
ð  ™ð
ð Ô(¨4Ñ/ð
ð  %Ô0°4Ñ7ð
ð Ô  4Ñ'ð
ð ˜$‘;ð
ð Ð+Ô,ð
ð 
ð
ð 
ð 
ñ „^ñ Ôð
ðB)¸E¼Lð )ð )ð )ð )ð )ð )ð )ð )r8   r6  c                   ó¾   ‡ — e Zd ZddiZdgZdefˆ fd„Zd„ Zd„ Ze	e
	 	 	 dd	ej        dz  d
ej        dz  dej        dz  dee         def
d„¦   «         ¦   «         Zˆ xZS )r7  rt  rr  r|  rO   c                 ó  •— t          ¦   «                              |¦  «         t          j        |j        |j        ¦  «        | _        |}d|_        d|_        t          |¦  «        | _
        |                      ¦   «          d S )NF)r,   r-   r   r¥   r\  rU   r:  rk  rj  rV  rz  rb  )r3   rO   r~  r6   s      €r7   r-   zT5EncoderModel.__init__5  so   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý”l 6Ô#4°f´nÑEÔEˆŒàˆØ#(ˆÔ Ø,1ˆÔ)Ý˜~Ñ.Ô.ˆŒð 	�ŠÑÔÐÐÐr8   c                 ó   — | j         S r`   r�  r‚  s    r7   rƒ  z#T5EncoderModel.get_input_embeddingsA  r„  r8   c                 óH   — || _         | j                             |¦  «         d S r`   )r:  rz  rf  rd  s     r7   rf  z#T5EncoderModel.set_input_embeddingsD  s%   € Ø$ˆŒØŒ×)Ò)¨.Ñ9Ô9Ð9Ð9Ð9r8   Nr'  r}   rh  r€   r  c                 ó(   —  | j         d|||dœ|¤Ž}|S )aI  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you
            should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            To know more on how to prepare `input_ids` for pretraining take a look a [T5 Training](./t5#training).

        Example:

        ```python
        >>> from transformers import AutoTokenizer, T5EncoderModel

        >>> tokenizer = AutoTokenizer.from_pretrained("google-t5/t5-small")
        >>> model = T5EncoderModel.from_pretrained("google-t5/t5-small")
        >>> input_ids = tokenizer(
        ...     "Studies have been shown that owning a dog is good for you", return_tensors="pt"
        ... ).input_ids  # Batch size 1
        >>> outputs = model(input_ids=input_ids)
        >>> last_hidden_states = outputs.last_hidden_state
        ```r‹  r’  )rz  )r3   r'  r}   rh  r€   rˆ  s         r7   rG   zT5EncoderModel.forwardH  s=   € ð@ ,8¨4¬<ð ,
ØØ)Ø'ð,
ð ,
ð ð	,
ð ,
ˆð Ðr8   r÷   )rH   rI   rJ   r—  r–  r'   r-   rƒ  rf  r"   r   r/   r˜  r™  r   r   r   rG   rK   rL   s   @r7   r7  r7  0  sþ   ø€ € € € € à7¸ÐIÐØ*4¨Ð&ð
˜xð 
ð 
ð 
ð 
ð 
ð 
ðð ð ð:ð :ð :ð Øð .2Ø37Ø26ð	%ð %àÔ# dÑ*ð%ð Ô)¨DÑ0ð%ð Ô(¨4Ñ/ð	%ð
 Ð+Ô,ð%ð 
ð%ð %ð %ñ „^ñ Ôð%ð %ð %ð %ð %r8   r7  zƒ
    T5 model with a sequence classification/head on top (a linear layer on top of the pooled output) e.g. for GLUE
    tasks.
    c                   ó0  ‡ — e Zd ZdgZdefˆ fd„Zee	 	 	 	 	 	 	 	 	 ddej	        dz  dej
        dz  dej	        dz  dej	        dz  d	eej                 dz  d
ej        dz  dej        dz  dej	        dz  dedz  dee         defd„¦   «         ¦   «         Zˆ xZS )ÚT5ForSequenceClassificationrq  rO   c                 óÂ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          |¦  «        | _        |                      ¦   «          d S r`   )r,   r-   r5  r!  r  Úclassification_headrb  r^   s     €r7   r-   z$T5ForSequenceClassification.__init__{  sQ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý" 6™?œ?ˆÔÝ#7¸Ñ#?Ô#?ˆÔ ð 	�ŠÑÔÐÐÐr8   Nr'  r}   r&  r(  rˆ  rh  r‰  r¢  rk  r€   r  c
                 óR  — |�d}	|€|�t          d| j        j        › �¦  «        ‚|€(|€&|€t          d¦  «        ‚|                      |¦  «        } | j        |f|||||||	dœ|
¤Ž}|d         }|                     | j        j        ¦  «         	                    |j
        ¦  «        }t          t          j        |                     d¦  «        ¦  «                             ¦   «         dk    d¦  «         |j        \  }}}||dd…f         }t          |j        d         |z  dk    d	¦  «         |                     |d
|¦  «        dd…d
dd…f         }|                      |¦  «        }d}|��ˆ| 	                    |j
        ¦  «        }| j        j        €p| j        j        dk    rd| j        _        nS| j        j        dk    r7|j        t          j        k    s|j        t          j        k    rd| j        _        nd| j        _        | j        j        dk    r\t1          ¦   «         }| j        j        dk    r1 ||                     ¦   «         |                     ¦   «         ¦  «        }n“ |||¦  «        }n†| j        j        dk    rLt5          ¦   «         } ||                     d
| j        j        ¦  «        |                     d
¦  «        ¦  «        }n*| j        j        dk    rt7          ¦   «         } |||¦  «        }t9          |||j        |j        |j        |j         |j!        |j"        |j#        ¬¦	  «	        S )a¤  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you
            should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            [What are input IDs?](../glossary#input-ids)

            To know more on how to prepare `input_ids` for pretraining take a look a [T5 Training](./t5#training).
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            T5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
            is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).

            To know more on how to prepare `decoder_input_ids` for pretraining take a look at [T5
            Training](./t5#training).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        NFz8Passing input embeddings is currently not supported for ú°If no `decoder_input_ids` or `decoder_inputs_embeds` are passed, `input_ids` cannot be `None`. Please pass either `input_ids` or `decoder_input_ids` or `decoder_inputs_embeds`.)r}   r&  r(  rˆ  rh  r‰  rk  r   r&   z7All examples must have the same number of <eos> tokens.z3Each example must contain at least one <eos> token.r;   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr¦  )$ÚNotImplementedErrorr6   rH   rD  rH  r!  ÚeqrO   Úeos_token_idr=   r¾   r!   r/   Úunique_consecutiveÚsumÚnumelrÌ   rÏ   r¸  Úproblem_typer  rB   r«   rê   r   Úsqueezer   r   r   rÞ   rŽ  r�  r$  r�  r  r‘  )r3   r'  r}   r&  r(  rˆ  rh  r‰  r¢  rk  r€   Úoutputsrª  Úeos_maskÚ
batch_sizer  r4   ÚselectedÚsentence_representationr¨  r§  r¬  s                         r7   rG   z#T5ForSequenceClassification.forwardƒ  so  € ð\ ÐØˆIàÐ Ð!:Ý%ØdÈ4Ì>ÔKbÐdÐdñô ð ð Ð$Ð)>Ð)FØÐ Ý ðUñô ð ð
 !%× 1Ò 1°)Ñ <Ô <Ðà&6 dÔ&6Øð
'
à)Ø/Ø#9Ø+Ø'Ø"7Øð
'
ð 
'
ð ð
'
ð 
'
ˆð " !œ*ˆà—<’< ¤Ô 8Ñ9Ô9×<Ò<¸_Ô=SÑTÔTˆåÝÔ$ X§\¢\°!¡_¤_Ñ5Ô5×;Ò;Ñ=Ô=ÀÒBØEñ	
ô 	
ð 	
ð &5Ô%:Ñ"ˆ
�A�{Ø" 8¨Q¨Q¨Q ;Ô/ˆÝØŒN˜1Ô Ñ+¨qÒ0ØAñ	
ô 	
ð 	
ð #+§-¢-°
¸BÀÑ"LÔ"LÈQÈQÈQÐPRÐTUÐTUÐTUÈXÔ"VÐØ×)Ò)Ð*AÑBÔBˆàˆØÑØ—Y’Y˜vœ}Ñ-Ô-ˆFØŒ{Ô'Ð/Ø”;Ô)¨QÒ.Ð.Ø/;�D”KÔ,Ð,Ø”[Ô+¨aÒ/Ð/°V´\ÅUÄZÒ5OÐ5OÐSYÔS_ÕchÔclÒSlÐSlØ/L�D”KÔ,Ð,à/K�D”KÔ,àŒ{Ô'¨<Ò7Ð7Ý"™9œ9�Ø”;Ô)¨QÒ.Ð.Ø#˜8 F§N¢NÑ$4Ô$4°f·n²nÑ6FÔ6FÑGÔG�D�Dà#˜8 F¨FÑ3Ô3�D�DØ”Ô)Ð-JÒJÐJÝ+Ñ-Ô-�Ø�x §¢¨B°´Ô0FÑ GÔ GÈÏÊÐUWÉÌÑYÔY��Ø”Ô)Ð-IÒIÐIÝ,Ñ.Ô.�Ø�x ¨Ñ/Ô/�å.ØØØ#Ô3Ø")Ô"?Ø&Ô9Ø$Ô5Ø&-Ô&GØ")Ô"?Ø&Ô9ð

ñ 

ô 

ð 
	
r8   r•  )rH   rI   rJ   r–  r'   r-   r"   r   r/   r˜  rb   Úlistr™  rë   r   r   r   rG   rK   rL   s   @r7   r¶  r¶  r  sk  ø€ € € € € ð +sÐ)sÐ&ð˜xð ð ð ð ð ð ð Øð .2Ø.2Ø59Ø:>Ø:>Ø26Ø:>Ø*.Ø!%ð}
ð }
àÔ# dÑ*ð}
ð œ tÑ+ð}
ð !Ô+¨dÑ2ð	}
ð
 !&Ô 0°4Ñ 7ð}
ð ˜eÔ/Ô0°4Ñ7ð}
ð Ô(¨4Ñ/ð}
ð  %Ô0°4Ñ7ð}
ð Ô  4Ñ'ð}
ð ˜$‘;ð}
ð Ð+Ô,ð}
ð 
)ð}
ð }
ð }
ñ „^ñ Ôð}
ð }
ð }
ð }
ð }
r8   r¶  c                   óº   ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  de	e
         d	efd
„¦   «         ¦   «         Zˆ xZS )r>  rO   c                 ó6  •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          j        |j        ¦  «        | _        t          j	        |j
        |j        ¦  «        | _        |                      ¦   «          d S r`   )r,   r-   r  r7  r!  r   rY   r  r[   rT   r4   r0  rb  r^   s     €r7   r-   z!T5ForTokenClassification.__init__  sz   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒå)¨&Ñ1Ô1ˆÔÝ”z &Ô";Ñ<Ô<ˆŒÝœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐr8   Nr'  r}   rh  r¢  r€   r  c                 óV  —  | j         |f||dœ|¤Ž}|d         }|                      |¦  «        }|                      |¦  «        }d}	|�Ft          ¦   «         }
 |
|                     d| j        ¦  «        |                     d¦  «        ¦  «        }	t          |	||j        |j        ¬¦  «        S )a<  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you
            should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            [What are input IDs?](../glossary#input-ids)

            To know more on how to prepare `input_ids` for pretraining take a look a [T5 Training](./t5#training).
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        )r}   rh  r   Nr;   )r§  r¨  rE   r#  )	r!  r[   r0  r   rÏ   r  r   rE   r#  )r3   r'  r}   rh  r¢  r€   rÆ  rE   r¨  r§  r¬  s              r7   rG   z T5ForTokenClassification.forward  sÌ   € ð0 $4 4Ô#3Øð$
à)Ø'ð$
ð $
ð ð	$
ð $
ˆð   œ
ˆØŸš ]Ñ3Ô3ˆØ—’ Ñ/Ô/ˆàˆØÐÝ'Ñ)Ô)ˆHØ�8˜FŸKšK¨¨D¬OÑ<Ô<¸f¿kºkÈ"¹o¼oÑNÔNˆDå$ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r8   ré   )rH   rI   rJ   r'   r-   r"   r   r/   rb   r   r   r   rG   rK   rL   s   @r7   r>  r>    sÞ   ø€ € € € € ð	˜xð 	ð 	ð 	ð 	ð 	ð 	ð Øð *.Ø.2Ø-1Ø&*ð+
ð +
à”< $Ñ&ð+
ð œ tÑ+ð+
ð ”| dÑ*ð	+
ð
 ”˜tÑ#ð+
ð Ð+Ô,ð+
ð 
ð+
ð +
ð +
ñ „^ñ Ôð+
ð +
ð +
ð +
ð +
r8   r>  c                   óh  ‡ — e Zd ZdgZdddœZdefˆ fd„Zd„ Zd„ Ze	e
	 	 	 	 	 	 	 	 	 	 dd	ej        dz  d
ej        dz  dej        dz  dej        dz  deeej                          dz  dej        dz  dej        dz  dej        dz  dej        dz  dedz  dee         defd„¦   «         ¦   «         Zˆ xZS )r8  rq  rr  rs  rO   c                 ó  •— t          ¦   «                              |¦  «         |j        | _        t	          j        |j        |j        ¦  «        | _        t          j	        |¦  «        }d|_
        d|_        t          |¦  «        | _        t          j	        |¦  «        }d|_
        |j        |_        t          |¦  «        | _        |j        | _        t	          j        |j        |j        ¦  «        | _        |                      ¦   «          d S rw  )r,   r-   rU   rŸ  r   r¥   r\  r:  rx  ry  r–   rk  rV  rz  r{  r_  r|  r  rT   r4   r/  rb  r}  s       €r7   r-   zT5ForQuestionAnswering.__init__J  sÖ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØœˆŒå”l 6Ô#4°f´nÑEÔEˆŒåœ vÑ.Ô.ˆØ$)ˆÔ!Ø#(ˆÔ Ý˜~Ñ.Ô.ˆŒåœ vÑ.Ô.ˆØ$(ˆÔ!Ø$*Ô$=ˆÔ!Ý˜~Ñ.Ô.ˆŒà Ô+ˆŒÝœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐr8   c                 ó   — | j         S r`   r�  r‚  s    r7   rƒ  z+T5ForQuestionAnswering.get_input_embeddings`  r„  r8   c                 ó|   — || _         | j                             |¦  «         | j                             |¦  «         d S r`   r†  rd  s     r7   rf  z+T5ForQuestionAnswering.set_input_embeddingsc  r‡  r8   Nr'  r}   r&  r(  rˆ  Ústart_positionsÚend_positionsrh  r‰  rk  r€   r  c                 óF  — |�|�d}
|€(|	€&|€t          d¦  «        ‚|                      |¦  «        }|€ | j        d|||dœ|¤Ž}nct          |t          ¦  «        sNt	          |d         t          |¦  «        dk    r|d         ndt          |¦  «        dk    r|d         nd¬¦  «        }|d         } | j        d|||	d|||
d	œ|¤Ž}|d         }|                      |¦  «        }|                     dd
¬¦  «        \  }}| 	                    d
¦  «         
                    ¦   «         }| 	                    d
¦  «         
                    ¦   «         }d}|��|��t          |                     ¦   «         ¦  «        dk    r-| 	                    d
¦  «                             |j        ¦  «        }t          |                     ¦   «         ¦  «        dk    r-| 	                    d
¦  «                             |j        ¦  «        }|                     d¦  «        }|                     d|¦  «        }|                     d|¦  «        }t          |¬¦  «        } |||¦  «        } |||¦  «        }||z   dz  }t!          ||||j        |j        |j        |j        |j        |j        |j        ¬¦
  «
        S )az  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you
            should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            [What are input IDs?](../glossary#input-ids)

            To know more on how to prepare `input_ids` for pretraining take a look a [T5 Training](./t5#training).
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            T5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
            is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).

            To know more on how to prepare `decoder_input_ids` for pretraining take a look at [T5
            Training](./t5#training).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.
        NFrº  r‹  r   r&   r:   rŒ  r�  r;   rƒ   r¤  )
r§  Ústart_logitsÚ
end_logitsrÞ   rŽ  r�  r$  r�  r  r‘  r’  )rD  rH  rz  ra   r   r“  r|  r/  ÚsplitrÅ  rŒ   r‡   r=   r¾   r  r   r   rÞ   rE   r#  r$  ri  )r3   r'  r}   r&  r(  rˆ  rÓ  rÔ  rh  r‰  rk  r€   rE   r”  rª  r¨  rÖ  r×  Ú
total_lossÚignored_indexr¬  Ú
start_lossÚend_losss                          r7   rG   zT5ForQuestionAnswering.forwardh  s  € ðX Ð&¨=Ð+DØˆIð
 Ð$Ð)>Ð)FØÐ Ý ðUñô ð ð
 !%× 1Ò 1°)Ñ <Ô <Ðð Ð"Ø*˜dœlð Ø#Ø-Ø+ðð ð ð	ð ˆOˆOõ ˜O­_Ñ=Ô=ð 	Ý-Ø"1°!Ô"4Ý47¸Ñ4HÔ4HÈ1Ò4LÐ4L˜o¨aÔ0Ð0ÐRVÝ14°_Ñ1EÔ1EÈÒ1IÐ1I˜?¨1Ô-Ð-Ètðñ ô ˆOð (¨Ô*ˆð '˜$œ,ð 	
Ø'Ø1Ø/Ø Ø"/Ø#1Øð	
ð 	
ð ð	
ð 	
ˆð *¨!Ô,ˆà—’ Ñ1Ô1ˆØ#)§<¢<°°r <Ñ#:Ô#:Ñ ˆ�jØ#×+Ò+¨BÑ/Ô/×:Ò:Ñ<Ô<ˆØ×'Ò'¨Ñ+Ô+×6Ò6Ñ8Ô8ˆ
àˆ
ØÑ&¨=Ñ+Då�?×'Ò'Ñ)Ô)Ñ*Ô*¨QÒ.Ð.Ø"1×"9Ò"9¸"Ñ"=Ô"=×"@Ò"@ÀÔATÑ"UÔ"U�Ý�=×%Ò%Ñ'Ô'Ñ(Ô(¨1Ò,Ð,Ø -× 5Ò 5°bÑ 9Ô 9× <Ò <¸ZÔ=NÑ OÔ O�à(×-Ò-¨aÑ0Ô0ˆMØ-×3Ò3°A°}ÑEÔEˆOØ)×/Ò/°°=ÑAÔAˆMå'°]ÐCÑCÔCˆHØ!˜ ,°Ñ@Ô@ˆJØ�x 
¨MÑ:Ô:ˆHØ$ xÑ/°1Ñ4ˆJå2ØØ%Ø!Ø+Ô;Ø"1Ô"?Ø.Ô9Ø,Ô=Ø&5Ô&GØ"1Ô"?Ø.Ô9ð
ñ 
ô 
ð 	
r8   r¯  )rH   rI   rJ   r–  r—  r'   r-   rƒ  rf  r"   r   r/   r˜  r™  rš  r›  rb   rë   r   r   r   rG   rK   rL   s   @r7   r8  r8  B  s´  ø€ € € € € à*rÐ)sÐ&à'6Ø'6ðð Ðð
˜xð ð ð ð ð ð ð,ð ð ð:ð :ð :ð
 Øð .2Ø37Ø59Ø:>Ø=AØ37Ø15Ø26Ø:>Ø!%ðy
ð y
àÔ# dÑ*ðy
ð Ô)¨DÑ0ðy
ð !Ô+¨dÑ2ð	y
ð
 !&Ô 0°4Ñ 7ðy
ð ˜u U¤\Ô2Ô3°dÑ:ðy
ð Ô)¨DÑ0ðy
ð Ô'¨$Ñ.ðy
ð Ô(¨4Ñ/ðy
ð  %Ô0°4Ñ7ðy
ð ˜$‘;ðy
ð Ð+Ô,ðy
ð 
-ðy
ð y
ð y
ñ „^ñ Ôðy
ð y
ð y
ð y
ð y
r8   r8  )r7  r6  r5  r   r8  r¶  r>  )Nrx   N)Pr  rx  r±   Úcollections.abcr   r/   r   Útorch.nnr   r   r   Ú r	   r3  Úactivationsr
   Úcache_utilsr   r   r   Ú
generationr   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r    r!   Úutils.genericr"   r#   Úutils.output_capturingr$   r%   Úconfiguration_t5r'   Ú
get_loggerrH   rŸ   ÚModuler)   rN   rg   ro   rb   r°   r�   r‘   rî   rú   r   r  r   rV  r5  r6  r7  r¶  r>  r8  Ú__all__r’  r8   r7   ú<module>rï     sb  ðð Ð à €€€Ø €€€Ø $Ð $Ð $Ð $Ð $Ð $à €€€Ø Ð Ð Ð Ð Ð Ø AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aà &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CØ )Ð )Ð )Ð )Ð )Ð )Ø JÐ JÐ JÐ JÐ JÐ JÐ JÐ JØ 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð GÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø rÐ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rØ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ &Ð &Ð &Ð &Ð &Ð &ð 
ˆÔ	˜HÑ	%Ô	%€ð+ð +ð +ð +ð +�"”)ñ +ô +ð +ð2ð ð ð ð �b”iñ ô ð ð,ð ð ð ð ˜2œ9ñ ô ð ð:ð ð ð ð �”	ñ ô ð ð0 !ØØ)-ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð �T‰\ð%ð ð%ð ”< $Ñ&ð%ð Ð'Ô(ð%ð %ð %ð %ð@A8ð A8ð A8ð A8ð A8�"”)ñ A8ô A8ð A8ðH=ð =ð =ð =ð =˜2œ9ñ =ô =ð =ð@9ð 9ð 9ð 9ð 9˜BœIñ 9ô 9ð 9ð>JPð JPð JPð JPð JPÐ(ñ JPô JPð JPðZð ð ð ð ˜2œ9ñ ô ð ð$ ðj!ð j!ð j!ð j!ð j!˜ñ j!ô j!ñ „ðj!ðZn
ð n
ð n
ð n
ð n
Ðñ n
ô n
ð n
ðb ðH
ð H
ð H
ð H
ð H
Ðñ H
ô H
ñ „ðH
ðV €ððñ ô ð
k)ð k)ð k)ð k)ð k)Ð!2°Oñ k)ô k)ñô ð
k)ð\ ð>ð >ð >ð >ð >Ð&ñ >ô >ñ „ð>ðB €ððñ ô ðJ
ð J
ð J
ð J
ð J
Ð"3ñ J
ô J
ñô ðJ
ðZ ð9
ð 9
ð 9
ð 9
ð 9
Ð0ñ 9
ô 9
ñ „ð9
ðx ð`
ð `
ð `
ð `
ð `
Ð.ñ `
ô `
ñ „ð`
ðFð ð €€€r8   