§
    9²¸j� ã                   óª  — d Z ddlZddlZddlmZ ddlZddlmZ ddlmZm	Z	m
Z
 ddlmZ ddlmZ dd	lmZmZmZ dd
lmZ ddlmZmZ ddlmZ ddlmZmZmZmZm Z m!Z!m"Z" ddl#m$Z$m%Z% ddl&m'Z' ddl(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z. ddl/m0Z0m1Z1 ddl2m3Z3m4Z4 ddl5m6Z6  e-j7        e8¦  «        Z9 G d„ dej:        ¦  «        Z; G d„ dej:        ¦  «        Z< G d„ dej:        ¦  «        Z= G d„ dej:        ¦  «        Z>	 	 	 dFdej:        dej?        d ej?        d!ej?        d"ej?        dz  d#e@dz  d$e@d%ej?        dz  d&e'e+         fd'„ZA G d(„ d)ej:        ¦  «        ZB G d*„ d+ej:        ¦  «        ZC G d,„ d-ej:        ¦  «        ZD G d.„ d/e¦  «        ZE G d0„ d1ej:        ¦  «        ZFe, G d2„ d3e%¦  «        ¦   «         ZG G d4„ d5eG¦  «        ZHe, G d6„ d7eG¦  «        ¦   «         ZI e,d8¬9¦  «         G d:„ d;eGe¦  «        ¦   «         ZJe, G d<„ d=eG¦  «        ¦   «         ZK e,d>¬9¦  «         G d?„ d@eG¦  «        ¦   «         ZLe, G dA„ dBeG¦  «        ¦   «         ZMe, G dC„ dDeG¦  «        ¦   «         ZNg dE¢ZOdS )GzPyTorch UMT5 model.é    N)ÚCallable)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutputÚ#Seq2SeqQuestionAnsweringModelOutputÚSeq2SeqSequenceClassifierOutputÚTokenClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚDUMMY_INPUTSÚ
DUMMY_MASKÚTransformersKwargsÚauto_docstringÚloggingÚtorch_compilable_check)Úcan_return_tupleÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )Ú
UMT5Configc                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚUMT5LayerNormç�íµ ÷Æ°>c                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )ze
        Construct a layernorm module in the UMT5 style. No bias and no subtraction of mean.
        N)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizeÚepsÚ	__class__s      €úv/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/umt5/modeling_umt5.pyr-   zUMT5LayerNorm.__init__;   sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐó    c                 óh  — |                      t          j        ¦  «                             d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        j        t          j	        t          j
        fv r|                      | j        j        ¦  «        }| j        |z  S )Né   éÿÿÿÿT)Úkeepdim)Útor/   Úfloat32ÚpowÚmeanÚrsqrtr2   r1   ÚdtypeÚfloat16Úbfloat16)r3   Úhidden_statesÚvariances      r7   ÚforwardzUMT5LayerNorm.forwardC   s–   € ð !×#Ò#¥E¤MÑ2Ô2×6Ò6°qÑ9Ô9×>Ò>¸rÈ4Ð>ÑPÔPˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆð Œ;Ô¥¤µ´Ð ?Ð?Ð?Ø)×,Ò,¨T¬[Ô->Ñ?Ô?ˆMàŒ{˜]Ñ*Ð*r8   )r*   )Ú__name__Ú
__module__Ú__qualname__r-   rG   Ú__classcell__©r6   s   @r7   r)   r)   :   sL   ø€ € € € € ð$ð $ð $ð $ð $ð $ð+ð +ð +ð +ð +ð +ð +r8   r)   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚUMT5DenseActDenseÚconfigc                 óJ  •— t          ¦   «                              ¦   «          t          j        |j        |j        d¬¦  «        | _        t          j        |j        |j        d¬¦  «        | _        t          j        |j	        ¦  «        | _
        t          |j                 | _        d S ©NF©Úbias)r,   r-   r   ÚLinearÚd_modelÚd_ffÚwiÚwoÚDropoutÚdropout_rateÚdropoutr
   Údense_act_fnÚact©r3   rO   r6   s     €r7   r-   zUMT5DenseActDense.__init__U   sx   ø€ Ý‰Œ×ÒÑÔÐÝ”)˜FœN¨F¬K¸eÐDÑDÔDˆŒÝ”)˜FœK¨¬¸eÐDÑDÔDˆŒÝ”z &Ô"5Ñ6Ô6ˆŒÝ˜&Ô-Ô.ˆŒˆˆr8   c                 ó°  — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }t          | j        j        t          j        ¦  «        r]|j        | j        j        j        k    rC| j        j        j        t          j	        k    r$| 
                    | j        j        j        ¦  «        }|                      |¦  «        }|S ©N)rW   r]   r[   Ú
isinstancerX   r1   r/   ÚTensorrB   Úint8r=   ©r3   rE   s     r7   rG   zUMT5DenseActDense.forward\   s¨   € ØŸš Ñ.Ô.ˆØŸš Ñ/Ô/ˆØŸš ]Ñ3Ô3ˆå�t”w”~¥u¤|Ñ4Ô4ð	CàÔ# t¤w¤~Ô';Ò;Ð;Ø””Ô$­¬
Ò2Ð2à)×,Ò,¨T¬W¬^Ô-AÑBÔBˆMØŸš Ñ.Ô.ˆØÐr8   ©rH   rI   rJ   r'   r-   rG   rK   rL   s   @r7   rN   rN   T   sS   ø€ € € € € ð/˜zð /ð /ð /ð /ð /ð /ðð ð ð ð ð ð r8   rN   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚUMT5DenseGatedActDenserO   c                 ó–  •— t          ¦   «                              ¦   «          t          j        |j        |j        d¬¦  «        | _        t          j        |j        |j        d¬¦  «        | _        t          j        |j        |j        d¬¦  «        | _        t          j	        |j
        ¦  «        | _        t          |j                 | _        d S rQ   )r,   r-   r   rT   rU   rV   Úwi_0Úwi_1rX   rY   rZ   r[   r
   r\   r]   r^   s     €r7   r-   zUMT5DenseGatedActDense.__init__l   s”   ø€ Ý‰Œ×ÒÑÔÐÝ”I˜fœn¨f¬kÀÐFÑFÔFˆŒ	Ý”I˜fœn¨f¬kÀÐFÑFÔFˆŒ	Ý”)˜FœK¨¬¸eÐDÑDÔDˆŒÝ”z &Ô"5Ñ6Ô6ˆŒÝ˜&Ô-Ô.ˆŒˆˆr8   c                 óà  — |                       |                      |¦  «        ¦  «        }|                      |¦  «        }||z  }|                      |¦  «        }t	          | j        j        t          j        ¦  «        r]|j	        | j        j        j	        k    rC| j        j        j	        t          j
        k    r$|                     | j        j        j	        ¦  «        }|                      |¦  «        }|S r`   )r]   ri   rj   r[   ra   rX   r1   r/   rb   rB   rc   r=   )r3   rE   Úhidden_geluÚhidden_linears       r7   rG   zUMT5DenseGatedActDense.forwardt   sÀ   € Ø—h’h˜tŸyšy¨Ñ7Ô7Ñ8Ô8ˆØŸ	š	 -Ñ0Ô0ˆØ# mÑ3ˆØŸš ]Ñ3Ô3ˆõ �t”w”~¥u¤|Ñ4Ô4ð	CàÔ# t¤w¤~Ô';Ò;Ð;Ø””Ô$­¬
Ò2Ð2à)×,Ò,¨T¬W¬^Ô-AÑBÔBˆMàŸš Ñ.Ô.ˆØÐr8   re   rL   s   @r7   rg   rg   k   sS   ø€ € € € € ð/˜zð /ð /ð /ð /ð /ð /ðð ð ð ð ð ð r8   rg   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚUMT5LayerFFrO   c                 ó$  •— t          ¦   «                              ¦   «          |j        rt          |¦  «        | _        nt          |¦  «        | _        t          |j        |j        ¬¦  «        | _	        t          j        |j        ¦  «        | _        d S )N©r5   )r,   r-   Úis_gated_actrg   ÚDenseReluDenserN   r)   rU   Úlayer_norm_epsilonÚ
layer_normr   rY   rZ   r[   r^   s     €r7   r-   zUMT5LayerFF.__init__Š   sx   ø€ Ý‰Œ×ÒÑÔÐØÔð 	<Ý"8¸Ñ"@Ô"@ˆDÔÐå"3°FÑ";Ô";ˆDÔå'¨¬¸FÔ<UÐVÑVÔVˆŒÝ”z &Ô"5Ñ6Ô6ˆŒˆˆr8   c                 óŠ   — |                       |¦  «        }|                      |¦  «        }||                      |¦  «        z   }|S r`   )ru   rs   r[   )r3   rE   Úforwarded_statess      r7   rG   zUMT5LayerFF.forward”   sF   € ØŸ?š?¨=Ñ9Ô9ÐØ×.Ò.Ð/?Ñ@Ô@ÐØ%¨¯ªÐ5EÑ(FÔ(FÑFˆØÐr8   re   rL   s   @r7   ro   ro   ‰   sS   ø€ € € € € ð7˜zð 7ð 7ð 7ð 7ð 7ð 7ðð ð ð ð ð ð r8   ro   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingr[   Úposition_biasÚkwargsc                 ó¼  — |€|                      d¦  «        dz  }t          j        ||                     dd¦  «        ¦  «        |z  }	|�|	|z   }	|�|	|z   }	t          j                             |	d¬¦  «        }	t          j                             |	|| j        ¬¦  «        }	t          j        |	|¦  «        }
|
                     dd¦  «         	                    ¦   «         }
|
|	fS )Nr;   ç      à¿r:   r   ©Údim)ÚpÚtrainingr&   )
Úsizer/   ÚmatmulÚ	transposer   Ú
functionalÚsoftmaxr[   r†   Ú
contiguous)ry   rz   r{   r|   r}   r~   r[   r   r€   Úattn_weightsÚattn_outputs              r7   Úeager_attention_forwardr�   œ   sØ   € ð €Ø—*’*˜R‘.”. DÑ(ˆõ ”<  s§}¢}°Q¸Ñ':Ô':Ñ;Ô;¸gÑE€LàÐ Ø# mÑ3ˆàÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2Ð(Ñ>Ô>€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r8   c            
       ó¸   ‡ — e Zd ZdZ	 ddedz  defˆ fd„Zdej        dej        fd	„Z	d
„ Z
dd„Z	 	 	 ddej        dej        dz  dedz  dej        dz  fd„Zˆ xZS )ÚUMT5Attentionz7
    T5's attention using relative_attention_bias.
    FNÚ	layer_idxÚ	is_causalc                 óJ  •— t          ¦   «                              ¦   «          || _        |j        | _        || _        || _        |j        | _        |j        | _        |j        | _        |j	        | _
        |j        | _        |j        | _        | j        | j
        z  | _        d| _        || _        |€/| j        r(t$                               d| j        j        › d�¦  «         t-          j        | j        | j        d¬¦  «        | _        t-          j        | j        | j        d¬¦  «        | _        t-          j        | j        | j        d¬¦  «        | _        t-          j        | j        | j        d¬¦  «        | _        | j        r&t-          j        | j        | j        ¦  «        | _        d S d S )Nç      ð?zInstantiating a decoder z³ without passing `layer_idx` is not recommended and will to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` when creating this class.FrR   )r,   r-   rO   Ú
is_decoderr“   Úhas_relative_attention_biasÚrelative_attention_num_bucketsÚrelative_attention_max_distancerU   Úd_kvÚkey_value_proj_dimÚ	num_headsÚn_headsrZ   r[   Ú	inner_dimr~   r’   ÚloggerÚwarning_oncer6   rH   r   rT   ÚqÚkÚvÚoÚ	EmbeddingÚrelative_attention_bias)r3   rO   r—   r’   r“   r6   s        €r7   r-   zUMT5Attention.__init__Á   s‚  ø€ õ 	‰Œ×ÒÑÔÐØˆŒØ Ô+ˆŒØ"ˆŒØ+FˆÔ(Ø.4Ô.SˆÔ+Ø/5Ô/UˆÔ,Ø”~ˆŒØ"(¤+ˆÔØÔ'ˆŒØÔ*ˆŒØœ¨Ô(?Ñ?ˆŒàˆŒØ"ˆŒØÐ ¤ÐÝ×Òð,¨4¬>Ô+Bð ,ð ,ð ,ñô ð õ ”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ>¨4¬<¸eÐDÑDÔDˆŒàÔ+ð 	kÝ+-¬<¸Ô8[Ð]aÔ]iÑ+jÔ+jˆDÔ(Ð(Ð(ð	kð 	kr8   Ú
projectionÚreturnc                 ó²   — |                      ¦   «         d d…         | j        | j        fz   }|                     |¦  «                             dddd¦  «        }|S )Nr;   r   r:   r&   r   )r‡   r�   r›   ÚviewÚpermute)r3   r§   Únew_projection_shapeÚnew_projections       r7   Ú_shapezUMT5Attention._shapeâ   sW   € Ø)ŸšÑ0Ô0°°"°Ô5¸¼ÀtÔG^Ð8_Ñ_Ðà#ŸšÐ)=Ñ>Ô>×FÒFÀqÈ!ÈQÐPQÑRÔRˆØÐr8   c                 ó~  — d}| j         }| j        }| j        sC|dz  }||dk                         t          j        ¦  «        |z  z  }t	          j        |¦  «        }n(t	          j        |t	          j        |¦  «        ¦  «         }|dz  }||k     }t	          j	        | 
                    ¦   «         |z  ¦  «        t          j	        ||z  ¦  «        z  }|||z
  z  }||                     t          j        ¦  «        z   }t	          j        |t	          j        ||dz
  ¦  «        ¦  «        }|t	          j        |||¦  «        z  }|S )aÒ  
        Adapted from Mesh Tensorflow:
        https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593

        Translate relative position to a bucket number for relative attention. The relative position is defined as
        memory_position - query_position, i.e. the distance in tokens from the attending position to the attended-to
        position. If bidirectional=False, then positive relative positions are invalid. We use smaller buckets for
        small absolute relative_position and larger buckets for larger absolute relative_positions. All relative
        positions >=max_distance map to the same bucket. All relative positions <=-max_distance map to the same bucket.
        This should allow for more graceful generalization to longer sequences than the model has been trained on

        Args:
            relative_position: an int32 Tensor
            bidirectional: a boolean - whether the attention is bidirectional
            num_buckets: an integer
            max_distance: an integer

        Returns:
            a Tensor with the same shape as relative_position, containing int32 values in the range [0, num_buckets)
        r   r:   r&   )r˜   r™   r–   r=   r/   ÚlongÚabsÚminÚ
zeros_likeÚlogÚfloatÚmathÚ	full_likeÚwhere)	r3   Úrelative_positionÚrelative_bucketsÚnum_bucketsÚmax_distanceÚ	max_exactÚis_smallÚ	log_ratioÚrelative_position_if_larges	            r7   Ú_relative_position_bucketz'UMT5Attention._relative_position_bucketè   sP  € ð* ÐØÔ9ˆØÔ;ˆØŒð 	cØ˜AÑˆKØÐ!2°QÒ!6× :Ò :½5¼:Ñ FÔ FÈÑ TÑTÐÝ %¤	Ð*;Ñ <Ô <ÐÐå!&¤Ð+<½eÔ>NÐO`Ñ>aÔ>aÑ!bÔ!bÐ bÐð   1Ñ$ˆ	Ø$ yÒ0ˆõ ”IÐ/×5Ò5Ñ7Ô7¸)ÑCÑDÔDÅtÄxÐP\Ð_hÑPhÑGiÔGiÑiˆ	Ø ¨yÑ!8Ñ9ˆ	Ø%.°·²½e¼jÑ1IÔ1IÑ%IÐ"Ý%*¤YØ&­¬Ð8RÐT_ÐbcÑTcÑ(dÔ(dñ&
ô &
Ð"ð 	�EœK¨Ð2CÐE_Ñ`Ô`Ñ`ÐØÐr8   r   c                 ó�  — |€| j         j        j        }t          j        |t          j        |¬¦  «        dd…df         |z   }t          j        |t          j        |¬¦  «        ddd…f         }||z
  }|                      |¦  «        }|                       |¦  «        }	|	                     g d¢¦  «                             d¦  «        }	|	S )z%Compute binned relative position biasN)rB   Údevice)r:   r   r&   r   )	r¦   r1   rÃ   r/   Úaranger°   rÁ   r«   Ú	unsqueeze)
r3   Úquery_lengthÚ
key_lengthrÃ   Úpast_seen_tokensÚcontext_positionÚmemory_positionr¹   Úrelative_position_bucketÚvaluess
             r7   Úcompute_biaszUMT5Attention.compute_bias  sÍ   € àˆ>ØÔ1Ô8Ô?ˆFÝ œ<¨½E¼JÈvÐVÑVÔVÐWXÐWXÐWXÐZ^ÐW^Ô_ÐbrÑrÐÝœ, z½¼ÈFÐSÑSÔSÐTXÐZ[ÐZ[ÐZ[ÐT[Ô\ˆØ+Ð.>Ñ>ÐØ#'×#AÒ#AÐBSÑ#TÔ#TÐ Ø×-Ò-Ð.FÑGÔGˆØ—’ 	 	 	Ñ*Ô*×4Ò4°QÑ7Ô7ˆØˆr8   rE   Úencoder_hidden_statesÚpast_key_valuesr}   c                 óü  — |j         d d…         \  }}|�|                     | j        ¦  «        nd}t          |t          j        ¦  «        r|                     ¦   «         n|}|d u}	|                      |¦  «        }
|
                     |d| j	        | j
        ¦  «                             dd¦  «        }
d}|�Ft          |t          ¦  «        r1|j                             | j        ¦  «        }|	r|j        }n
|j        }n|}|	r|n|}|	r3|�1|r/|j        | j                 j        }|j        | j                 j        }nÝ|                      |¦  «        }|                      |¦  «        }|                     |d| j	        | j
        ¦  «                             dd¦  «        }|                     |d| j	        | j
        ¦  «                             dd¦  «        }|�E|                     ||| j        ¦  «        \  }}|	r$t          |t          ¦  «        rd|j        | j        <   |j         d         }| j        s+t	          j        d| j	        ||f|
j        |
j        ¬¦  «        }n|                      |||
j        |¬	¦  «        }t9          j        | j        j        t@          ¦  «        } || |
|||f| j!        sd
n| j"        | j#        |dœ|¤Ž\  }}| $                    ||d¦  «         %                    ¦   «         }|  &                    |¦  «        }||fS )Nr:   r   r;   r&   FTéþÿÿÿ)rÃ   rB   )rÃ   rÈ   rx   )r[   r~   r   )'ÚshapeÚget_seq_lengthr’   ra   r/   rb   Úcloner¡   rª   r�   r›   r‰   r   Ú
is_updatedÚgetÚcross_attention_cacheÚself_attention_cacheÚlayersÚkeysrÌ   r¢   r£   Úupdater—   ÚzerosrÃ   rB   rÍ   r   Úget_interfacerO   Ú_attn_implementationr�   r†   r[   r~   ÚreshaperŒ   r¤   )r3   rE   rÎ   rÏ   r}   r€   Ú
batch_sizeÚ
seq_lengthrÈ   Úis_cross_attentionÚquery_statesrÕ   Úcurr_past_key_valuesÚcurrent_statesÚ
key_statesÚvalue_statesrÇ   r   Úattention_interfacerŽ   r�   s                        r7   rG   zUMT5Attention.forward#  sY  € ð "/Ô!4°R°a°RÔ!8Ñˆ
�JØM\ÐMh˜?×9Ò9¸$¼.ÑIÔIÐIÐnoÐå7AÐBRÕTYÔT`Ñ7aÔ7aÐwÐ+×1Ò1Ñ3Ô3Ð3ÐgwÐð 3¸$Ð>Ðà—v’v˜mÑ,Ô,ˆØ#×(Ò(¨°R¸¼ÀtÔG^Ñ_Ô_×iÒiÐjkÐmnÑoÔoˆð ˆ
ØÐ&­:°oÕGZÑ+[Ô+[Ð&Ø(Ô3×7Ò7¸¼ÑGÔGˆJØ!ð Là'6Ô'LÐ$Ð$à'6Ô'KÐ$Ð$à#2Ð à2DÐWÐ.Ð.È-ˆØð 	F /Ð"=À*Ð"=à-Ô4°T´^ÔDÔIˆJØ/Ô6°t´~ÔFÔMˆLˆLàŸš Ñ/Ô/ˆJØŸ6š6 .Ñ1Ô1ˆLØ#Ÿš¨°R¸¼ÀtÔG^Ñ_Ô_×iÒiÐjkÐmnÑoÔoˆJØ'×,Ò,¨Z¸¸T¼\È4ÔKbÑcÔc×mÒmÐnoÐqrÑsÔsˆLàÐ*à+?×+FÒ+FÀzÐS_ÐaeÔaoÑ+pÔ+pÑ(�
˜Là%ð F­*°_ÕFYÑ*ZÔ*Zð FØAE�OÔ.¨t¬~Ñ>ð  Ô% bÔ)ˆ
ØÔ/ð 	Ý!œKØ�D”L *¨jÐ9À,ÔBUÐ]iÔ]oðñ ô ˆMˆMð !×-Ò-Ø˜J¨|Ô/BÐUeð .ñ ô ˆMõ )@Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð  $œ}Ð>�C�C°$´,Ø”LØ'ð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð "×)Ò)¨*°jÀ"ÑEÔE×PÒPÑRÔRˆØ—f’f˜[Ñ)Ô)ˆØ˜LÐ(Ð(r8   )FNF)Nr   ©NNN)rH   rI   rJ   Ú__doc__ÚintÚboolr-   r/   rb   r®   rÁ   rÍ   r   rG   rK   rL   s   @r7   r‘   r‘   ¼   s&  ø€ € € € € ðð ð
 joðkð kØDGÈ$ÁJðkØbfðkð kð kð kð kð kðB ¤ð °%´,ð ð ð ð ð- ð - ð - ð^
ð 
ð 
ð 
ð 6:Ø(,Ø.2ðN)ð N)à”|ðN)ð  %œ|¨dÑ2ðN)ð  ™ð	N)ð
 œ tÑ+ðN)ð N)ð N)ð N)ð N)ð N)ð N)ð N)r8   r‘   c                   ó8   ‡ — e Zd Zddedz  fˆ fd„Z	 	 dd„Zˆ xZS )ÚUMT5LayerSelfAttentionNr’   c                 óò   •— t          ¦   «                              ¦   «          t          |d|¬¦  «        | _        t	          |j        |j        ¬¦  «        | _        t          j	        |j
        ¦  «        | _        d S )NT©r—   r’   rq   )r,   r-   r‘   ÚSelfAttentionr)   rU   rt   ru   r   rY   rZ   r[   ©r3   rO   r’   r6   s      €r7   r-   zUMT5LayerSelfAttention.__init__u  sb   ø€ Ý‰Œ×ÒÑÔÐÝ*¨6ÈtÐ_hÐiÑiÔiˆÔÝ'¨¬¸FÔ<UÐVÑVÔVˆŒÝ”z &Ô"5Ñ6Ô6ˆŒˆˆr8   c                 óª   — |                       |¦  «        } | j        |f||dœ|¤Ž}||                      |d         ¦  «        z   }|f|dd …         z   }|S )N©r}   rÏ   r   r&   )ru   rñ   r[   )r3   rE   r}   rÏ   r€   Únormed_hidden_statesÚattention_outputÚoutputss           r7   rG   zUMT5LayerSelfAttention.forward{  s‚   € ð  $Ÿš¨}Ñ=Ô=ÐØ-˜4Ô-Ø ð
à)Ø+ð
ð 
ð ð	
ð 
Ðð &¨¯ªÐ5EÀaÔ5HÑ(IÔ(IÑIˆØ Ð"Ð%5°a°b°bÔ%9Ñ9ˆØˆr8   r`   )NN©rH   rI   rJ   rë   r-   rG   rK   rL   s   @r7   rî   rî   t  si   ø€ € € € € ð7ð 7¨#°©*ð 7ð 7ð 7ð 7ð 7ð 7ð Øð	ð ð ð ð ð ð ð r8   rî   c                   ó:   ‡ — e Zd Zddedz  fˆ fd„Z	 	 	 dd„Zˆ xZS )ÚUMT5LayerCrossAttentionNr’   c                 óò   •— t          ¦   «                              ¦   «          t          |d|¬¦  «        | _        t	          |j        |j        ¬¦  «        | _        t          j	        |j
        ¦  «        | _        d S )NFrð   rq   )r,   r-   r‘   ÚEncDecAttentionr)   rU   rt   ru   r   rY   rZ   r[   rò   s      €r7   r-   z UMT5LayerCrossAttention.__init__�  sc   ø€ Ý‰Œ×ÒÑÔÐÝ,¨VÐQVÐbkÐlÑlÔlˆÔÝ'¨¬¸FÔ<UÐVÑVÔVˆŒÝ”z &Ô"5Ñ6Ô6ˆŒˆˆr8   c                 ó¬   — |                       |¦  «        } | j        |f|||dœ|¤Ž}||                      |d         ¦  «        z   }|f|dd …         z   }	|	S )N©rÎ   r}   rÏ   r   r&   )ru   rü   r[   )
r3   rE   rÎ   r}   rÏ   r€   rõ   rö   Úlayer_outputr÷   s
             r7   rG   zUMT5LayerCrossAttention.forward•  s„   € ð  $Ÿš¨}Ñ=Ô=ÐØ/˜4Ô/Ø ð
à"7Ø)Ø+ð	
ð 
ð
 ð
ð 
Ðð % t§|¢|Ð4DÀQÔ4GÑ'HÔ'HÑHˆØ�/Ð$4°Q°R°RÔ$8Ñ8ˆØˆr8   r`   ré   rø   rL   s   @r7   rú   rú   Ž  sl   ø€ € € € € ð7ð 7¨#°©*ð 7ð 7ð 7ð 7ð 7ð 7ð #ØØðð ð ð ð ð ð ð r8   rú   c                   ó<   ‡ — e Zd Zddedz  fˆ fd„Z	 	 	 	 dd„Zˆ xZS )Ú	UMT5BlockNr’   c                 ó�  •— t          ¦   «                              ¦   «          |j        | _        t          j        ¦   «         | _        | j                             t          ||¬¦  «        ¦  «         | j        r)| j                             t          ||¬¦  «        ¦  «         | j                             t          |¦  «        ¦  «         d S )N©r’   )
r,   r-   r–   r   Ú
ModuleListÚlayerÚappendrî   rú   ro   rò   s      €r7   r-   zUMT5Block.__init__«  s§   ø€ Ý‰Œ×ÒÑÔÐØ Ô+ˆŒÝ”]‘_”_ˆŒ
ØŒ
×ÒÕ0°À9ÐMÑMÔMÑNÔNÐNØŒ?ð 	TØŒJ×ÒÕ5°fÈ	ÐRÑRÔRÑSÔSÐSàŒ
×Ò�+ fÑ-Ô-Ñ.Ô.Ð.Ð.Ð.r8   c                 óæ  —  | j         d         |f||dœ|¤Ž\  }}|j        t          j        k    rst          j        |j        ¦  «        j        }t          j        t          j        |¦  «                             ¦   «         |dz
  |¦  «        }	t          j	        ||	 |	¬¦  «        }| j
        o|d u}
|
r£ | j         d         |f|||dœ|¤Ž\  }}|j        t          j        k    rst          j        |j        ¦  «        j        }t          j        t          j        |¦  «                             ¦   «         |dz
  |¦  «        }	t          j	        ||	 |	¬¦  «        } | j         d         |¦  «        }|j        t          j        k    rst          j        |j        ¦  «        j        }t          j        t          j        |¦  «                             ¦   «         |dz
  |¦  «        }	t          j	        ||	 |	¬¦  «        }|S )Nr   rô   iè  )r²   Úmaxr&   rþ   r;   )r  rB   r/   rC   Úfinfor  r¸   ÚisinfÚanyÚclampr–   )r3   rE   r}   rÎ   Úencoder_attention_maskrÏ   r€   Ú_Ú	max_dtypeÚclamp_valueÚdo_cross_attentions              r7   rG   zUMT5Block.forwardµ  s  € ð )˜4œ: aœ=Øð
à)Ø+ð
ð 
ð ð	
ð 
Ñˆ�qð Ô¥%¤-Ò/Ð/Ýœ MÔ$7Ñ8Ô8Ô<ˆIÝœ+¥e¤k°-Ñ&@Ô&@×&DÒ&DÑ&FÔ&FÈ	ÐTXÑHXÐZcÑdÔdˆKÝ!œK¨¸K¸<È[ÐYÑYÔYˆMð "œ_ÐRÐ1FÈdÐ1RÐØð 	^Ø,˜tœz¨!œ}Øð à&;Ø5Ø /ð	 ð  ð
 ð ð  ÑˆM˜1ð Ô"¥e¤mÒ3Ð3Ý!œK¨Ô(;Ñ<Ô<Ô@�	Ý#œk­%¬+°mÑ*DÔ*D×*HÒ*HÑ*JÔ*JÈIÐX\ÑL\Ð^gÑhÔh�Ý %¤¨MÀ¸|ÐQ\Ð ]Ñ ]Ô ]�ð '˜œ
 2œ }Ñ5Ô5ˆð Ô¥%¤-Ò/Ð/Ýœ MÔ$7Ñ8Ô8Ô<ˆIÝœ+¥e¤k°-Ñ&@Ô&@×&DÒ&DÑ&FÔ&FÈ	ÐTXÑHXÐZcÑdÔdˆKÝ!œK¨¸K¸<È[ÐYÑYÔYˆMàÐr8   r`   ©NNNNrø   rL   s   @r7   r  r  ª  so   ø€ € € € € ð/ð /¨#°©*ð /ð /ð /ð /ð /ð /ð Ø"Ø#Øð/ð /ð /ð /ð /ð /ð /ð /r8   r  c                   óL   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚUMT5ClassificationHeadz-Head for sentence-level classification tasks.rO   c                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        ¬¦  «        | _        t          j        |j        |j	        ¦  «        | _
        d S )N)r…   )r,   r-   r   rT   rU   ÚdenserY   Úclassifier_dropoutr[   Ú
num_labelsÚout_projr^   s     €r7   r-   zUMT5ClassificationHead.__init__ë  sc   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vœ~¨v¬~Ñ>Ô>ˆŒ
Ý”z FÔ$=Ð>Ñ>Ô>ˆŒÝœ	 &¤.°&Ô2CÑDÔDˆŒˆˆr8   rE   r¨   c                 óÖ   — |                       |¦  «        }|                      |¦  «        }t          j        |¦  «        }|                       |¦  «        }|                      |¦  «        }|S r`   )r[   r  r/   Útanhr  rd   s     r7   rG   zUMT5ClassificationHead.forwardñ  s[   € ØŸš ]Ñ3Ô3ˆØŸ
š
 =Ñ1Ô1ˆÝœ
 =Ñ1Ô1ˆØŸš ]Ñ3Ô3ˆØŸš mÑ4Ô4ˆØÐr8   )
rH   rI   rJ   rê   r'   r-   r/   rb   rG   rK   rL   s   @r7   r  r  è  sw   ø€ € € € € Ø7Ð7ðE˜zð Eð Eð Eð Eð Eð Eð U¤\ð °e´lð ð ð ð ð ð ð ð r8   r  c                   óâ   ‡ — e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZ eed¬¦  «         eed	¬¦  «         eed	¬¦  «        d
œZed„ ¦   «         Z ej        ¦   «         ˆ fd„¦   «         Zd„ Zˆ xZS )ÚUMT5PreTrainedModelrO   ÚtransformerTr  rX   Fr   )Úindexr;   )rE   Ú
attentionsÚcross_attentionsc                 óv   — t          j        t          ¦  «        }t          j        t          ¦  «        }|||dœ}|S )N)Údecoder_input_idsÚ	input_idsÚdecoder_attention_mask)r/   Útensorr   r   )r3   r$  Ú
input_maskÚdummy_inputss       r7   r(  z UMT5PreTrainedModel.dummy_inputs  s=   € å”L¥Ñ.Ô.ˆ	Ý”\¥*Ñ-Ô-ˆ
à!*Ø"Ø&0ð
ð 
ˆð
 Ðr8   c                 ó\  •— t          ¦   «                              |¦  «         | j        j        }t	          |t
          ¦  «        rt          j        |j        |dz  ¦  «         d	S t	          |t          t          t          t          f¦  «        rÇt          j        |j        j        d|dz  ¬¦  «         t          |d¦  «        r0| j        j        s$t          j        |j        j        d|dz  ¬¦  «         t          |d¦  «        rQt          j        |j        j        d|| j        j        dz  z  ¬¦  «         t          j        |j        j        ¦  «         d	S d	S t	          |t,          ¦  «        rVt          |d¦  «        rDt          j        |j        j        d|dz  ¬¦  «         t          j        |j        j        ¦  «         d	S d	S t	          |t0          ¦  «        ræt          j        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r*|j        j        �t          j        |j        j        ¦  «         t          j        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r,|j        j        �"t          j        |j        j        ¦  «         d	S d	S d	S t	          |t6          ¦  «        ræt          j        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r*|j        j        �t          j        |j        j        ¦  «         t          j        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r,|j        j        �"t          j        |j        j        ¦  «         d	S d	S d	S t	          |t>          ¦  «        �rVt          j        |j         j        d|| j        j        dz  z  ¬¦  «         t          |j         d¦  «        r*|j         j        �t          j        |j         j        ¦  «         t          j        |j!        j        d|| j        j        dz  z  ¬¦  «         t          |j!        d¦  «        r*|j!        j        �t          j        |j!        j        ¦  «         t          j        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r,|j        j        �"t          j        |j        j        ¦  «         d	S d	S d	S t	          |tD          ¦  «        rö| j        j        }| j        j#        }| j        j$        }t          j        |j%        j        d|||z  dz  z  ¬¦  «         t          j        |j&        j        d||dz  z  ¬¦  «         t          j        |j'        j        d||dz  z  ¬¦  «         t          j        |j(        j        d|||z  dz  z  ¬¦  «         |j)        r+t          j        |j*        j        d||dz  z  ¬¦  «         d	S d	S d	S )
zInitialize the weightsr•   rx   )r@   ÚstdÚlm_headÚ
qa_outputsr‚   Ú
classifierrS   N)+r,   Ú_init_weightsrO   Úinitializer_factorra   r)   ÚinitÚ	constant_r1   Ú	UMT5ModelÚUMT5ForConditionalGenerationÚUMT5EncoderModelÚUMT5ForQuestionAnsweringÚnormal_ÚsharedÚhasattrÚtie_word_embeddingsr+  r,  rU   Úzeros_rS   ÚUMT5ForTokenClassificationr-  r  r  r  rN   rW   rX   rV   rg   ri   rj   r‘   rš   rœ   r¡   r¢   r£   r¤   r—   r¦   )r3   ry   ÚfactorrU   r›   r�   r6   s         €r7   r.  z!UMT5PreTrainedModel._init_weights  sô  ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ø”Ô/ˆÝ�f�mÑ,Ô,ð =	pÝŒN˜6œ=¨&°3©,Ñ7Ô7Ð7Ð7Ð7ÝØåÝ,Ý Ý(ð	ñ
ô 
ð ;	põ ŒL˜œÔ-°C¸VÀc¹\ÐJÑJÔJÐJÝ�v˜yÑ)Ô)ð P°$´+Ô2Qð PÝ”˜Vœ^Ô2¸À&È3Á,ÐOÑOÔOÐOÝ�v˜|Ñ,Ô,ð 4Ý”˜VÔ.Ô5¸CÀVÐPTÔP[ÔPcÐhlÑOlÑEmÐnÑnÔnÐnÝ”˜FÔ-Ô2Ñ3Ô3Ð3Ð3Ð3ð4ð 4õ ˜Õ :Ñ;Ô;ð *	pÝ�v˜|Ñ,Ô,ð 4Ý”˜VÔ.Ô5¸CÀVÈcÁ\ÐRÑRÔRÐRÝ”˜FÔ-Ô2Ñ3Ô3Ð3Ð3Ð3ð4ð 4õ ˜Õ 6Ñ7Ô7ð &	pÝŒL˜œÔ,°3¸FÀtÄ{ÔGZÐ_cÑFcÑ<dÐeÑeÔeÐeÝ�v”| VÑ,Ô,ð /°´Ô1BÐ1NÝ”˜FœLÔ-Ñ.Ô.Ð.ÝŒL˜œÔ/°c¸vÈ$Ì+ÔJ]ÐbfÑIfÑ?gÐhÑhÔhÐhÝ�v”¨Ñ/Ô/ð 2°F´OÔ4HÐ4TÝ”˜FœOÔ0Ñ1Ô1Ð1Ð1Ð1ð2ð 2Ð4TÐ4Tå˜Õ 1Ñ2Ô2ð 	põ ŒL˜œÔ)°¸ÀDÄKÔDWÐ\`ÑC`Ñ9aÐbÑbÔbÐbÝ�v”y &Ñ)Ô)ð ,¨f¬i¬nÐ.HÝ”˜FœIœNÑ+Ô+Ð+ÝŒL˜œÔ)°¸ÀDÄKÔDTÐY]ÑC]Ñ9^Ð_Ñ_Ô_Ð_Ý�v”y &Ñ)Ô)ð ,¨f¬i¬nÐ.HÝ”˜FœIœNÑ+Ô+Ð+Ð+Ð+ð,ð ,Ð.HÐ.Hå˜Õ 6Ñ7Ô7ñ 	pÝŒL˜œÔ+°#¸6ÀdÄkÔFYÐ^bÑEbÑ;cÐdÑdÔdÐdÝ�v”{ FÑ+Ô+ð .°´Ô0@Ð0LÝ”˜FœKÔ,Ñ-Ô-Ð-ÝŒL˜œÔ+°#¸6ÀdÄkÔFYÐ^bÑEbÑ;cÐdÑdÔdÐdÝ�v”{ FÑ+Ô+ð .°´Ô0@Ð0LÝ”˜FœKÔ,Ñ-Ô-Ð-ÝŒL˜œÔ)°¸ÀDÄKÔDTÐY]ÑC]Ñ9^Ð_Ñ_Ô_Ð_Ý�v”y &Ñ)Ô)ð ,¨f¬i¬nÐ.HÝ”˜FœIœNÑ+Ô+Ð+Ð+Ð+ð,ð ,Ð.HÐ.Hå˜¥Ñ.Ô.ð 	pð ”kÔ)ˆGØ!%¤Ô!1ÐØ”kÔ+ˆGÝŒL˜œœ¨s¸À7ÐM_ÑC_ÐdhÑBhÑ8iÐjÑjÔjÐjÝŒL˜œœ¨s¸À'È4Á-Ñ8PÐQÑQÔQÐQÝŒL˜œœ¨s¸À'È4Á-Ñ8PÐQÑQÔQÐQÝŒL˜œœ¨s¸À7ÐM_ÑC_ÐdhÑBhÑ8iÐjÑjÔjÐjØÔ1ð pÝ”˜VÔ;ÔBÈÐRXÐ]dÐimÑ\mÑRnÐoÑoÔoÐoÐoÐoð	pð 	pðpð pr8   c                 ó6  — | j         j        }| j         j        }|€t          d¦  «        ‚|                     |j        ¦  «        }|dd d…f                              ¦   «         |ddd …f<   ||d<   |€t          d¦  «        ‚|                     |dk    |¦  «         |S )NzŽself.model.config.decoder_start_token_id has to be defined. In UMT5 it is usually set to the pad_token_id. See UMT5 docs for more information..r;   r&   ).r   z1self.model.config.pad_token_id has to be defined.éœÿÿÿ)rO   Údecoder_start_token_idÚpad_token_idÚ
ValueErrorÚ	new_zerosrÒ   rÔ   Úmasked_fill_)r3   r$  r?  r@  Úshifted_input_idss        r7   Ú_shift_rightz UMT5PreTrainedModel._shift_right]  sº   € Ø!%¤Ô!CÐØ”{Ô/ˆà!Ð)Ýð6ñô ð ð
 &×/Ò/°	´Ñ@Ô@ÐØ%.¨s°C°R°C¨xÔ%8×%>Ò%>Ñ%@Ô%@Ð˜#˜q˜r˜r˜'Ñ"Ø$:Ð˜&Ñ!àÐÝÐPÑQÔQÐQà×&Ò&Ð'8¸DÒ'@À,ÑOÔOÐOà Ð r8   )rH   rI   rJ   r'   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_keep_in_fp32_modulesÚ_supports_flash_attnÚ_supports_flex_attnÚ_supports_sdpaÚ_can_compile_fullgraphr$   r  rî   rú   Ú_can_record_outputsÚpropertyr(  r/   Úno_gradr.  rE  rK   rL   s   @r7   r  r  ú  s	  ø€ € € € € € àÐÐÑØ%ÐØ&*Ð#à$˜ÐØ!˜FÐà ÐØÐØ€NØ!Ðð (˜¨	¸Ð;Ñ;Ô;Ø$�nÐ%;À2ÐFÑFÔFØ*˜NÐ+BÈ"ÐMÑMÔMðð Ðð ðð ñ „Xðð €U„]�_„_ðApð Apð Apð Apñ „_ðApðF!ð !ð !ð !ð !ð !ð !r8   r  c                   ó€   ‡ — e Zd Zˆ fd„Zd„ Zeee	 	 	 	 	 	 	 ddee	         de
fd„¦   «         ¦   «         ¦   «         Zˆ xZS )Ú	UMT5Stackc                 óÌ  •‡— t          ¦   «                              ‰¦  «         t          j        ‰j        ‰j        ¦  «        | _        ‰j        | _        t          j        ˆfd„t          ‰j
        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        t          j        ‰j        ¦  «        | _        d| _        |                      ¦   «          d S )Nc                 ó2   •— g | ]}t          ‰|¬ ¦  «        ‘ŒS )r  )r  )Ú.0ÚirO   s     €r7   ú
<listcomp>z&UMT5Stack.__init__.<locals>.<listcomp>x  s&   ø€ Ð#eÐ#eÐ#eÀq¥I¨fÀÐ$BÑ$BÔ$BÐ#eÐ#eÐ#er8   rq   F)r,   r-   r   r¥   Ú
vocab_sizerU   Úembed_tokensr–   r  ÚrangeÚ
num_layersÚblockr)   rt   Úfinal_layer_normrY   rZ   r[   Úgradient_checkpointingÚ	post_initr^   s    `€r7   r-   zUMT5Stack.__init__t  s¿   øø€ Ý‰Œ×Ò˜Ñ Ô Ð ÝœL¨Ô):¸F¼NÑKÔKˆÔØ Ô+ˆŒÝ”]Ð#eÐ#eÐ#eÐ#eÍEÐRXÔRcÑLdÔLdÐ#eÑ#eÔ#eÑfÔfˆŒ
Ý -¨f¬nÀ&ÔB[Ð \Ñ \Ô \ˆÔÝ”z &Ô"5Ñ6Ô6ˆŒð ',ˆÔ#Ø�ŠÑÔÐÐÐr8   c                 ó   — || _         d S r`   )rZ  ©r3   Únew_embeddingss     r7   Úset_input_embeddingszUMT5Stack.set_input_embeddings€  s   € Ø*ˆÔÐÐr8   Nr€   r¨   c           	      ó4  — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|du r| j        st          d| › d�¦  «        ‚| j        r]|rZ|€X| j        j        r7t          t          | j        ¬¦  «        t          | j        ¬¦  «        ¦  «        }nt          | j        ¬¦  «        }n	| j        sd }| j        r5t          | j        ||t          |t
          ¦  «        r|j	        n|¬¦  «        }	nt          | j        ||¬¦  «        }	| j        r|�t          | j        |||¬¦  «        }
nd }
|                      |¦  «        }| j        D ]} |||	|f|
|d	œ|¤Ž}Œ|                      |¦  «        }|                      |¦  «        }t          ||¬
¦  «        S )Nz:You must specify exactly one of input_ids or inputs_embedsTz)`use_cache` can only be set to `True` if z is used as a decoder)rO   )rO   Úinputs_embedsr}   rÏ   )rO   rf  r}   )rO   rf  r}   rÎ   )r  rÏ   )Úlast_hidden_staterÏ   )rA  rZ  r–   rO   Úis_encoder_decoderr   r   r   ra   rØ   r   r[   r]  r^  r   )r3   r$  r}   rÎ   r  rf  rÏ   Ú	use_cacher€   Úcausal_maskÚencoder_extended_attention_maskrE   Úlayer_modules                r7   rG   zUMT5Stack.forwardƒ  s:  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMà˜ÐÐØ”?ð jÝ Ð!hÈTÐ!hÐ!hÐ!hÑiÔiÐið Œ?ð 	#Øð G˜_Ð4Ø”;Ô1ð GÝ&9Ý$¨D¬KÐ8Ñ8Ô8½,ÈdÌkÐ:ZÑ:ZÔ:Zñ'ô '�O�Oõ '3¸$¼+Ð&FÑ&FÔ&F�OøØ”ð 	#ð #ˆOàŒ?ð 	Ý,Ø”{Ø+Ø-å˜oÕ/BÑCÔCð!% Ô DÐ Dà$ðñ ô ˆKˆKõ 4Ø”{Ø+Ø-ðñ ô ˆKð Œ?ð 	3Ð4Ð@Ý.GØ”{Ø+Ø5Ø&;ð	/ñ /ô /Ð+Ð+ð /3Ð+àŸš ]Ñ3Ô3ˆà œJð 	ð 	ˆLØ(˜LØØØ%ðð (GØ /ðð ð ðð ˆMˆMð ×-Ò-¨mÑ<Ô<ˆØŸš ]Ñ3Ô3ˆå8Ø+Ø+ð
ñ 
ô 
ð 	
r8   )NNNNNNN)rH   rI   rJ   r-   rd  r#   r%   r   r   r   r   rG   rK   rL   s   @r7   rS  rS  s  s½   ø€ € € € € ð
ð 
ð 
ð 
ð 
ð+ð +ð +ð  ØØð ØØ"Ø#ØØØðO
ð O
ð Ð+Ô,ðO
ð 
3ðO
ð O
ð O
ñ „^ñ „_ñ  ÔðO
ð O
ð O
ð O
ð O
r8   rS  c                   óP  ‡ — e Zd ZU dZdZeed<   dddœZˆ fd„Zd„ Z	d„ Z
ee	 	 	 	 	 	 	 	 	 dd
ej        d	z  dej        d	z  dej        d	z  dej        d	z  deeej                          d	z  ded	z  dej        d	z  dej        d	z  ded	z  dee         defd„¦   «         ¦   «         Zˆ xZS )r2  ao  
    Examples:

    ```python
    >>> from transformers import UMT5Model, AutoTokenizer

    >>> model = UMT5Model.from_pretrained("google/umt5-small")
    >>> tokenizer = AutoTokenizer.from_pretrained("google/umt5-small")
    >>> noisy_text = "UN Offizier sagt, dass weiter <extra_id_0> werden muss in Syrien."
    >>> label = "<extra_id_0> verhandelt"
    >>> inputs = tokenizer(inputs, return_tensors="pt")
    >>> labels = tokenizer(label=label, return_tensors="pt")

    >>> outputs = model(input_ids=inputs["input_ids"], decoder_input_ids=labels["input_ids"])
    >>> hidden_states = outputs.last_hidden_state
    ```Úumt5rO   úshared.weight©úencoder.embed_tokens.weightúdecoder.embed_tokens.weightc                 óœ  •— t          ¦   «                              |¦  «         t          j        |j        |j        ¦  «        | _        t          j        |¦  «        }d|_	        d|_
        t          |¦  «        | _        t          j        |¦  «        }d|_	        |j        |_        t          |¦  «        | _        |                      ¦   «          d S ©NFT)r,   r-   r   r¥   rY  rU   r7  ÚcopyÚdeepcopyr–   ri  rS  ÚencoderÚnum_decoder_layersr\  Údecoderr`  ©r3   rO   Úencoder_configÚdecoder_configr6   s       €r7   r-   zUMT5Model.__init__ò  sª   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý”l 6Ô#4°f´nÑEÔEˆŒåœ vÑ.Ô.ˆØ$)ˆÔ!Ø#(ˆÔ Ý  Ñ0Ô0ˆŒåœ vÑ.Ô.ˆØ$(ˆÔ!Ø$*Ô$=ˆÔ!Ý  Ñ0Ô0ˆŒð 	�ŠÑÔÐÐÐr8   c                 ó   — | j         S r`   ©r7  ©r3   s    r7   Úget_input_embeddingszUMT5Model.get_input_embeddings  ó
   € ØŒ{Ðr8   c                 ó|   — || _         | j                             |¦  «         | j                             |¦  «         d S r`   ©r7  rw  rd  ry  rb  s     r7   rd  zUMT5Model.set_input_embeddings  ó;   € Ø$ˆŒØŒ×)Ò)¨.Ñ9Ô9Ð9ØŒ×)Ò)¨.Ñ9Ô9Ð9Ð9Ð9r8   Nr$  r}   r#  r%  Úencoder_outputsrÏ   rf  Údecoder_inputs_embedsri  r€   r¨   c
                 ó¨  — |€ | j         d	|||dœ|
¤Ž}nct          |t          ¦  «        sNt          |d         t          |¦  «        dk    r|d         ndt          |¦  «        dk    r|d         nd¬¦  «        }|d         } | j        d	|||||||	dœ|
¤Ž}t          |j        |j        |j        |j	        |j
        |j        |j        |j	        ¬¦  «        S )
ah
  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. UMT5 is a model with relative position embeddings so
            you should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            [What are input IDs?](../glossary#input-ids)

            To know more on how to prepare `input_ids` for pretraining take a look a [UMT5 Training](./umt5#training).
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            UMT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
            is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).

            To know more on how to prepare `decoder_input_ids` for pretraining take a look at [UMT5
            Training](./umt5#training).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, UMT5Model

        >>> tokenizer = AutoTokenizer.from_pretrained("google/umt5-small")
        >>> model = UMT5Model.from_pretrained("google/umt5-small")

        >>> input_ids = tokenizer(
        ...     "Studies have been shown that owning a dog is good for you", return_tensors="pt"
        ... ).input_ids  # Batch size 1
        >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="pt").input_ids  # Batch size 1

        >>> # preprocess: Prepend decoder_input_ids with start token which is pad token for UMT5Model.
        >>> # This is not needed for torch's UMT5ForConditionalGeneration as it does this internally using labels arg.
        >>> decoder_input_ids = model._shift_right(decoder_input_ids)

        >>> # forward pass
        >>> outputs = model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)
        >>> last_hidden_states = outputs.last_hidden_state
        ```N©r$  r}   rf  r   r&   r:   ©rg  rE   r   ©r$  r}   rf  rÏ   rÎ   r  ri  )rg  rÏ   Údecoder_hidden_statesÚdecoder_attentionsr!  Úencoder_last_hidden_staterÎ   Úencoder_attentions© )rw  ra   r   Úlenry  r   rg  rÏ   rE   r   r!  )r3   r$  r}   r#  r%  r…  rÏ   rf  r†  ri  r€   rE   Údecoder_outputss                r7   rG   zUMT5Model.forward  s?  € ðD Ð"Ø*˜dœlð Ø#Ø-Ø+ðð ð ð	ð ˆOˆOõ ˜O­_Ñ=Ô=ð 	Ý-Ø"1°!Ô"4Ý47¸Ñ4HÔ4HÈ1Ò4LÐ4L˜o¨aÔ0Ð0ÐRVÝ14°_Ñ1EÔ1EÈÒ1IÐ1I˜?¨1Ô-Ð-Ètðñ ô ˆOð (¨Ô*ˆð '˜$œ,ð 	
Ø'Ø1Ø/Ø+Ø"/Ø#1Øð	
ð 	
ð ð	
ð 	
ˆõ "Ø-Ô?Ø+Ô;Ø"1Ô"?Ø.Ô9Ø,Ô=Ø&5Ô&GØ"1Ô"?Ø.Ô9ð	
ñ 	
ô 	
ð 		
r8   ©	NNNNNNNNN)rH   rI   rJ   rê   Ú
model_typer'   rF  Ú_tied_weights_keysr-   r€  rd  r"   r   r/   Ú
LongTensorÚFloatTensorÚ
BoolTensorÚtupler   rb   rì   r   r   r   rG   rK   rL   s   @r7   r2  r2  Ø  s£  ø€ € € € € € ðð ð" €JØÐÐÑà'6Ø'6ðð Ðð
ð ð ð ð ð$ð ð ð:ð :ð :ð
 Øð .2Ø37Ø59Ø:>ØBFØ(,Ø-1Ø59Ø!%ðe
ð e
àÔ# dÑ*ðe
ð Ô)¨DÑ0ðe
ð !Ô+¨dÑ2ð	e
ð
 !&Ô 0°4Ñ 7ðe
ð ˜u UÔ%6Ô7Ô8¸4Ñ?ðe
ð  ™ðe
ð ”| dÑ*ðe
ð  %œ|¨dÑ2ðe
ð ˜$‘;ðe
ð Ð+Ô,ðe
ð 
ðe
ð e
ð e
ñ „^ñ Ôðe
ð e
ð e
ð e
ð e
r8   r2  z<
    UMT5 Model with a `language modeling` head on top.
    )Úcustom_introc                   ór  ‡ — e Zd ZdZdZddddœZˆ fd„Zd„ Zd„ Ze	e
	 	 	 	 	 	 	 	 	 	 dd	ej        dz  d
ej        dz  dej        dz  dej        dz  deeej                          dz  dedz  dej        dz  dej        dz  dej        dz  dedz  dee         defd„¦   «         ¦   «         Zdej        fd„Zˆ xZS )r3  a  
    Examples:

    ```python
    >>> from transformers import UMT5ForConditionalGeneration, AutoTokenizer

    >>> model = UMT5ForConditionalGeneration.from_pretrained("google/umt5-small")
    >>> tokenizer = AutoTokenizer.from_pretrained("google/umt5-small")
    >>> article = "UN Offizier sagt, dass weiter verhandelt werden muss in Syrien."
    >>> summary = "Weiter Verhandlung in Syrien."
    >>> inputs = tokenizer(article, text_target=summary, return_tensors="pt")

    >>> outputs = model(**inputs)
    >>> loss = outputs.loss
    ```rn  ro  )rq  rr  zlm_head.weightc                 ó   •— t          ¦   «                              |¦  «         |j        | _        t	          j        |j        |j        ¦  «        | _        t          j	        |¦  «        }d|_
        d|_        t          |¦  «        | _        t          j	        |¦  «        }d|_
        |j        |_        t          |¦  «        | _        t	          j        |j        |j        d¬¦  «        | _        |                      ¦   «          d S )NFTrR   )r,   r-   rU   Ú	model_dimr   r¥   rY  r7  ru  rv  r–   ri  rS  rw  rx  r\  ry  rT   r+  r`  rz  s       €r7   r-   z%UMT5ForConditionalGeneration.__init__”  sÐ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØœˆŒå”l 6Ô#4°f´nÑEÔEˆŒåœ vÑ.Ô.ˆØ$)ˆÔ!Ø#(ˆÔ Ý  Ñ0Ô0ˆŒåœ vÑ.Ô.ˆØ$(ˆÔ!Ø$*Ô$=ˆÔ!Ý  Ñ0Ô0ˆŒå”y ¤°Ô1BÈÐOÑOÔOˆŒð 	�ŠÑÔÐÐÐr8   c                 ó   — | j         S r`   r~  r  s    r7   r€  z1UMT5ForConditionalGeneration.get_input_embeddingsª  r�  r8   c                 ó|   — || _         | j                             |¦  «         | j                             |¦  «         d S r`   rƒ  rb  s     r7   rd  z1UMT5ForConditionalGeneration.set_input_embeddings®  r„  r8   Nr$  r}   r#  r%  r…  rÏ   rf  r†  Úlabelsri  r€   r¨   c                 ó*  — |€ | j         d|||dœ|¤Ž}nct          |t          ¦  «        sNt          |d         t          |¦  «        dk    r|d         ndt          |¦  «        dk    r|d         nd¬¦  «        }|d         }|	�|€|€|                      |	¦  «        } | j        d|||||||
dœ|¤Ž}|d         }| j        j        r|| j        dz  z  }|  	                    |¦  «        }d}|	�pt          d	¬
¦  «        }|	                     |j        ¦  «        }	 ||                     d|                     d¦  «        ¦  «        |	                     d¦  «        ¦  «        }t          |||j        |j        |j        |j        |j        |j        |j        ¬¦	  «	        S )aˆ  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. UMT5 is a model with relative position embeddings so
            you should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            [What are input IDs?](../glossary#input-ids)

            To know more on how to prepare `input_ids` for pretraining take a look a [UMT5 Training](./umt5#training).
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            UMT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
            is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).

            To know more on how to prepare `decoder_input_ids` for pretraining take a look at [UMT5
            Training](./umt5#training).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ...,
            config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for
            labels in `[0, ..., config.vocab_size]`

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, UMT5ForConditionalGeneration

        >>> tokenizer = AutoTokenizer.from_pretrained("google/umt5-small")
        >>> model = UMT5ForConditionalGeneration.from_pretrained("google/umt5-small")

        >>> # training
        >>> input_ids = tokenizer("The <extra_id_0> walks in <extra_id_1> park", return_tensors="pt").input_ids
        >>> labels = tokenizer("<extra_id_0> cute dog <extra_id_1> the <extra_id_2>", return_tensors="pt").input_ids
        >>> outputs = model(input_ids=input_ids, labels=labels)
        >>> loss = outputs.loss
        >>> logits = outputs.logits

        >>> # inference
        >>> input_ids = tokenizer("Studies have shown that <extra_id_0> good for you", return_tensors="pt").input_ids
        >>> outputs = model.generate(input_ids)
        >>> tokenizer.decode(outputs[0], skip_special_tokens=True)
        ```Nrˆ  r   r&   r:   r‰  rŠ  r‚   r>  ©Úignore_indexr;   ©	ÚlossÚlogitsrÏ   r‹  rŒ  r!  r�  rÎ   rŽ  r�  )rw  ra   r   r�  rE  ry  rO   r9  rœ  r+  r   r=   rÃ   rª   r‡   r   rÏ   rE   r   r!  rg  )r3   r$  r}   r#  r%  r…  rÏ   rf  r†  rŸ  ri  r€   rE   r‘  Úsequence_outputÚ	lm_logitsr¤  Úloss_fcts                     r7   rG   z$UMT5ForConditionalGeneration.forward³  s  € ðL Ð"à*˜dœlð Ø#Ø-Ø+ðð ð ð	ð ˆOˆOõ ˜O­_Ñ=Ô=ð 	Ý-Ø"1°!Ô"4Ý47¸Ñ4HÔ4HÈ1Ò4LÐ4L˜o¨aÔ0Ð0ÐRVÝ14°_Ñ1EÔ1EÈÒ1IÐ1I˜?¨1Ô-Ð-Ètðñ ô ˆOð (¨Ô*ˆàÐÐ"3Ð";Ð@UÐ@]à $× 1Ò 1°&Ñ 9Ô 9Ðð '˜$œ,ð 	
Ø'Ø1Ø/Ø+Ø"/Ø#1Øð	
ð 	
ð ð	
ð 	
ˆð *¨!Ô,ˆàŒ;Ô*ð 	Gð .°´ÀÑ1EÑFˆOà—L’L Ñ1Ô1ˆ	àˆØÐÝ'°TÐ:Ñ:Ô:ˆHà—Y’Y˜yÔ/Ñ0Ô0ˆFØ�8˜IŸNšN¨2¨y¯~ª~¸bÑ/AÔ/AÑBÔBÀFÇKÂKÐPRÁOÄOÑTÔTˆDåØØØ+Ô;Ø"1Ô"?Ø.Ô9Ø,Ô=Ø&5Ô&GØ"1Ô"?Ø.Ô9ð

ñ 

ô 

ð 
	
r8   c                 ó,   — |                       |¦  «        S r`   )rE  )r3   rŸ  s     r7   Ú%prepare_decoder_input_ids_from_labelszBUMT5ForConditionalGeneration.prepare_decoder_input_ids_from_labels7  s   € Ø× Ò  Ñ(Ô(Ð(r8   ©
NNNNNNNNNN)rH   rI   rJ   rê   r“  r”  r-   r€  rd  r"   r   r/   r•  r–  r—  r˜  rb   r   rì   r   r   r   rG   rª  rK   rL   s   @r7   r3  r3  w  sÉ  ø€ € € € € ðð ð  €Jà'6Ø'6Ø)ðð Ððð ð ð ð ð,ð ð ð:ð :ð :ð
 Øð .2Ø37Ø59Ø:>Ø=AØ(,Ø26Ø:>Ø*.Ø!%ð
ð 
àÔ# dÑ*ð
ð Ô)¨DÑ0ð
ð !Ô+¨dÑ2ð	
ð
 !&Ô 0°4Ñ 7ð
ð ˜u U¤\Ô2Ô3°dÑ:ð
ð  ™ð
ð Ô(¨4Ñ/ð
ð  %Ô0°4Ñ7ð
ð Ô  4Ñ'ð
ð ˜$‘;ð
ð Ð+Ô,ð
ð 
ð
ð 
ð 
ñ „^ñ Ôð
ðD)¸E¼Lð )ð )ð )ð )ð )ð )ð )ð )r8   r3  c                   óº   ‡ — e Zd ZdZdZddiZˆ fd„Zd„ Zd„ Ze	e
	 	 	 dd	ej        dz  d
ej        dz  dej        dz  dee         def
d„¦   «         ¦   «         Zˆ xZS )r4  aâ  
    Examples:

    ```python
    >>> from transformers import UMT5EncoderModel, AutoTokenizer

    >>> model = UMT5EncoderModel.from_pretrained("google/umt5-small")
    >>> tokenizer = AutoTokenizer.from_pretrained("google/umt5-small")
    >>> article = "UN Offizier sagt, dass weiter verhandelt werden muss in Syrien."
    >>> input_ids = tokenizer(article, return_tensors="pt").input_ids
    >>> outputs = model(input_ids)
    >>> hidden_state = outputs.last_hidden_state
    ```rn  rq  ro  c                 ó&  •— t          ¦   «                              |¦  «         t          j        |j        |j        ¦  «        | _        t          j        |¦  «        }d|_	        d|_
        t          |¦  «        | _        |                      ¦   «          d S )NF)r,   r-   r   r¥   rY  rU   r7  ru  rv  ri  rh  rS  rw  r`  )r3   rO   r{  r6   s      €r7   r-   zUMT5EncoderModel.__init__Q  sy   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý”l 6Ô#4°f´nÑEÔEˆŒåœ vÑ.Ô.ˆØ#(ˆÔ Ø,1ˆÔ)Ý  Ñ0Ô0ˆŒð 	�ŠÑÔÐÐÐr8   c                 ó   — | j         S r`   r~  r  s    r7   r€  z%UMT5EncoderModel.get_input_embeddings^  r�  r8   c                 óH   — || _         | j                             |¦  «         d S r`   )r7  rw  rd  rb  s     r7   rd  z%UMT5EncoderModel.set_input_embeddingsb  s%   € Ø$ˆŒØŒ×)Ò)¨.Ñ9Ô9Ð9Ð9Ð9r8   Nr$  r}   rf  r€   r¨   c                 ó(   —  | j         d|||dœ|¤Ž}|S )aQ  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. UMT5 is a model with relative position embeddings so you
            should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            To know more on how to prepare `input_ids` for pretraining take a look a [UMT5 Training](./umt5#training).

        Example:

        ```python
        >>> from transformers import AutoTokenizer, UMT5EncoderModel

        >>> tokenizer = AutoTokenizer.from_pretrained("google/umt5-small")
        >>> model = UMT5EncoderModel.from_pretrained("google/umt5-small")
        >>> input_ids = tokenizer(
        ...     "Studies have been shown that owning a dog is good for you", return_tensors="pt"
        ... ).input_ids  # Batch size 1
        >>> outputs = model(input_ids=input_ids)
        >>> last_hidden_states = outputs.last_hidden_state
        ```rˆ  r�  )rw  )r3   r$  r}   rf  r€   r…  s         r7   rG   zUMT5EncoderModel.forwardf  s=   € ðB ,8¨4¬<ð ,
ØØ)Ø'ð,
ð ,
ð ð	,
ð ,
ˆð Ðr8   ré   )rH   rI   rJ   rê   r“  r”  r-   r€  rd  r"   r   r/   r•  r–  r   r   r   rG   rK   rL   s   @r7   r4  r4  ;  sÿ   ø€ € € € € ðð ð €Jð 	& ðÐð
ð 
ð 
ð 
ð 
ðð ð ð:ð :ð :ð Øð .2Ø37Ø26ð	%ð %àÔ# dÑ*ð%ð Ô)¨DÑ0ð%ð Ô(¨4Ñ/ð	%ð
 Ð+Ô,ð%ð 
ð%ð %ð %ñ „^ñ Ôð%ð %ð %ð %ð %r8   r4  z…
    UMT5 model with a sequence classification/head on top (a linear layer on top of the pooled output) e.g. for GLUE
    tasks.
    c                   ó0  ‡ — e Zd ZdgZdefˆ fd„Zee	 	 	 	 	 	 	 	 	 ddej	        dz  dej
        dz  dej	        dz  dej	        dz  d	eej                 dz  d
ej        dz  dej        dz  dej	        dz  dedz  dee         defd„¦   «         ¦   «         Zˆ xZS )ÚUMT5ForSequenceClassificationúFdecoder.block.0.layer.1.EncDecAttention.relative_attention_bias.weightrO   c                 óÂ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          |¦  «        | _        |                      ¦   «          d S r`   )r,   r-   r2  r  r  Úclassification_headr`  r^   s     €r7   r-   z&UMT5ForSequenceClassification.__init__›  sS   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý$ VÑ,Ô,ˆÔÝ#9¸&Ñ#AÔ#AˆÔ ð 	�ŠÑÔÐÐÐr8   Nr$  r}   r#  r%  r…  rf  r†  rŸ  ri  r€   r¨   c
                 óR  — |�d}	|€|�t          d| j        j        › �¦  «        ‚|€(|€&|€t          d¦  «        ‚|                      |¦  «        } | j        |f|||||||	dœ|
¤Ž}|d         }|                     | j        j        ¦  «         	                    |j
        ¦  «        }t          t          j        |                     d¦  «        ¦  «                             ¦   «         dk    d¦  «         |j        \  }}}||dd…f         }t          |j        d         |z  dk    d	¦  «         |                     |d
|¦  «        dd…d
dd…f         }|                      |¦  «        }d}|��ˆ| 	                    |j
        ¦  «        }| j        j        €p| j        j        dk    rd| j        _        nS| j        j        dk    r7|j        t          j        k    s|j        t          j        k    rd| j        _        nd| j        _        | j        j        dk    r\t1          ¦   «         }| j        j        dk    r1 ||                     ¦   «         |                     ¦   «         ¦  «        }n“ |||¦  «        }n†| j        j        dk    rLt5          ¦   «         } ||                     d
| j        j        ¦  «        |                     d
¦  «        ¦  «        }n*| j        j        dk    rt7          ¦   «         } |||¦  «        }t9          |||j        |j        |j        |j         |j!        |j"        |j#        ¬¦	  «	        S )a°  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. UMT5 is a model with relative position embeddings so
            you should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            [What are input IDs?](../glossary#input-ids)

            To know more on how to prepare `input_ids` for pretraining take a look a [UMT5 Training](./umt5#training).
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            UMT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
            is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).

            To know more on how to prepare `decoder_input_ids` for pretraining take a look at [UMT5
            Training](./umt5#training).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        NFz8Passing input embeddings is currently not supported for ú°If no `decoder_input_ids` or `decoder_inputs_embeds` are passed, `input_ids` cannot be `None`. Please pass either `input_ids` or `decoder_input_ids` or `decoder_inputs_embeds`.)r}   r#  r%  r…  rf  r†  ri  r   r&   z7All examples must have the same number of <eos> tokens.z3Each example must contain at least one <eos> token.r;   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr£  )$ÚNotImplementedErrorr6   rH   rA  rE  r  ÚeqrO   Úeos_token_idr=   rÃ   r!   r/   Úunique_consecutiveÚsumÚnumelrÒ   rª   rµ  Úproblem_typer  rB   r°   rë   r   Úsqueezer   r   r   rÏ   r‹  rŒ  r!  r�  rÎ   rŽ  )r3   r$  r}   r#  r%  r…  rf  r†  rŸ  ri  r€   r÷   r¦  Úeos_maskrà   r  r4   ÚselectedÚsentence_representationr¥  r¤  r¨  s                         r7   rG   z%UMT5ForSequenceClassification.forward£  so  € ð\ ÐØˆIàÐ Ð!:Ý%ØdÈ4Ì>ÔKbÐdÐdñô ð ð Ð$Ð)>Ð)FØÐ Ý ðUñô ð ð
 !%× 1Ò 1°)Ñ <Ô <Ðà"�$Ô"Øð

à)Ø/Ø#9Ø+Ø'Ø"7Øð

ð 

ð ð

ð 

ˆð " !œ*ˆà—<’< ¤Ô 8Ñ9Ô9×<Ò<¸_Ô=SÑTÔTˆåÝÔ$ X§\¢\°!¡_¤_Ñ5Ô5×;Ò;Ñ=Ô=ÀÒBØEñ	
ô 	
ð 	
ð &5Ô%:Ñ"ˆ
�A�{Ø" 8¨Q¨Q¨Q ;Ô/ˆÝØŒN˜1Ô Ñ+¨qÒ0ØAñ	
ô 	
ð 	
ð #+§-¢-°
¸BÀÑ"LÔ"LÈQÈQÈQÐPRÐTUÐTUÐTUÈXÔ"VÐØ×)Ò)Ð*AÑBÔBˆàˆØÑØ—Y’Y˜vœ}Ñ-Ô-ˆFØŒ{Ô'Ð/Ø”;Ô)¨QÒ.Ð.Ø/;�D”KÔ,Ð,Ø”[Ô+¨aÒ/Ð/°V´\ÅUÄZÒ5OÐ5OÐSYÔS_ÕchÔclÒSlÐSlØ/L�D”KÔ,Ð,à/K�D”KÔ,àŒ{Ô'¨<Ò7Ð7Ý"™9œ9�Ø”;Ô)¨QÒ.Ð.Ø#˜8 F§N¢NÑ$4Ô$4°f·n²nÑ6FÔ6FÑGÔG�D�Dà#˜8 F¨FÑ3Ô3�D�DØ”Ô)Ð-JÒJÐJÝ+Ñ-Ô-�Ø�x §¢¨B°´Ô0FÑ GÔ GÈÏÊÐUWÉÌÑYÔY��Ø”Ô)Ð-IÒIÐIÝ,Ñ.Ô.�Ø�x ¨Ñ/Ô/�å.ØØØ#Ô3Ø")Ô"?Ø&Ô9Ø$Ô5Ø&-Ô&GØ")Ô"?Ø&Ô9ð

ñ 

ô 

ð 
	
r8   r’  )rH   rI   rJ   Ú"_keys_to_ignore_on_load_unexpectedr'   r-   r"   r   r/   r•  rb   Úlistr–  rì   r   r   r   rG   rK   rL   s   @r7   r²  r²  ‘  sk  ø€ € € € € ð +sÐ)sÐ&ð˜zð ð ð ð ð ð ð Øð .2Ø.2Ø59Ø:>Ø:>Ø26Ø:>Ø*.Ø!%ð}
ð }
àÔ# dÑ*ð}
ð œ tÑ+ð}
ð !Ô+¨dÑ2ð	}
ð
 !&Ô 0°4Ñ 7ð}
ð ˜eÔ/Ô0°4Ñ7ð}
ð Ô(¨4Ñ/ð}
ð  %Ô0°4Ñ7ð}
ð Ô  4Ñ'ð}
ð ˜$‘;ð}
ð Ð+Ô,ð}
ð 
)ð}
ð }
ð }
ñ „^ñ Ôð}
ð }
ð }
ð }
ð }
r8   r²  c                   óÀ   ‡ — e Zd ZdgZdefˆ fd„Zee	 	 	 	 ddej	        dz  dej	        dz  dej	        dz  dej	        dz  d	e
e         d
efd„¦   «         ¦   «         Zˆ xZS )r;  r³  rO   c                 ó6  •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          j        |j        ¦  «        | _        t          j	        |j
        |j        ¦  «        | _        |                      ¦   «          d S r`   )r,   r-   r  r4  r  r   rY   r  r[   rT   r4   r-  r`  r^   s     €r7   r-   z#UMT5ForTokenClassification.__init__*  sz   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒå+¨FÑ3Ô3ˆÔÝ”z &Ô";Ñ<Ô<ˆŒÝœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐr8   Nr$  r}   rf  rŸ  r€   r¨   c                 óV  —  | j         |f||dœ|¤Ž}|d         }|                      |¦  «        }|                      |¦  «        }d}	|�Ft          ¦   «         }
 |
|                     d| j        ¦  «        |                     d¦  «        ¦  «        }	t          |	||j        |j        ¬¦  «        S )aB  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. UMT5 is a model with relative position embeddings so you
            should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            [What are input IDs?](../glossary#input-ids)

            To know more on how to prepare `input_ids` for pretraining take a look a [UMT5 Training](./umt5#training).
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        )r}   rf  r   Nr;   )r¤  r¥  rE   r   )	r  r[   r-  r   rª   r  r   rE   r   )r3   r$  r}   rf  rŸ  r€   r÷   rE   r¥  r¤  r¨  s              r7   rG   z"UMT5ForTokenClassification.forward5  sÌ   € ð2 $4 4Ô#3Øð$
à)Ø'ð$
ð $
ð ð	$
ð $
ˆð   œ
ˆØŸš ]Ñ3Ô3ˆØ—’ Ñ/Ô/ˆàˆØÐÝ'Ñ)Ô)ˆHØ�8˜FŸKšK¨¨D¬OÑ<Ô<¸f¿kºkÈ"¹o¼oÑNÔNˆDå$ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r8   r  )rH   rI   rJ   rÆ  r'   r-   r"   r   r/   rb   r   r   r   rG   rK   rL   s   @r7   r;  r;  %  sç   ø€ € € € € à*rÐ)sÐ&ð	˜zð 	ð 	ð 	ð 	ð 	ð 	ð Øð *.Ø.2Ø-1Ø&*ð+
ð +
à”< $Ñ&ð+
ð œ tÑ+ð+
ð ”| dÑ*ð	+
ð
 ”˜tÑ#ð+
ð Ð+Ô,ð+
ð 
ð+
ð +
ð +
ñ „^ñ Ôð+
ð +
ð +
ð +
ð +
r8   r;  c                   ó\  ‡ — e Zd ZdddœZˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 	 	 dde	j
        dz  de	j        dz  d	e	j
        dz  d
e	j        dz  deee	j                          dz  de	j
        dz  de	j
        dz  de	j        dz  de	j        dz  dedz  dee         defd„¦   «         ¦   «         Zˆ xZS )r5  ro  rp  c                 ó  •— t          ¦   «                              |¦  «         |j        | _        t	          j        |j        |j        ¦  «        | _        t          j	        |¦  «        }d|_
        d|_        t          |¦  «        | _        t          j	        |¦  «        }d|_
        |j        |_        t          |¦  «        | _        |j        | _        t	          j        |j        |j        ¦  «        | _        |                      ¦   «          d S rt  )r,   r-   rU   rœ  r   r¥   rY  r7  ru  rv  r–   ri  rS  rw  rx  r\  ry  r  rT   r,  r`  rz  s       €r7   r-   z!UMT5ForQuestionAnswering.__init__m  sÕ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØœˆŒå”l 6Ô#4°f´nÑEÔEˆŒåœ vÑ.Ô.ˆØ$)ˆÔ!Ø#(ˆÔ Ý  Ñ0Ô0ˆŒåœ vÑ.Ô.ˆØ$(ˆÔ!Ø$*Ô$=ˆÔ!Ý  Ñ0Ô0ˆŒà Ô+ˆŒÝœ) F¤N°FÔ4EÑFÔFˆŒð 	�ŠÑÔÐÐÐr8   c                 ó   — | j         S r`   r~  r  s    r7   r€  z-UMT5ForQuestionAnswering.get_input_embeddings„  r�  r8   c                 ó|   — || _         | j                             |¦  «         | j                             |¦  «         d S r`   rƒ  rb  s     r7   rd  z-UMT5ForQuestionAnswering.set_input_embeddingsˆ  r„  r8   Nr$  r}   r#  r%  r…  Ústart_positionsÚend_positionsrf  r†  ri  r€   r¨   c                 óF  — |�|�d}
|€(|	€&|€t          d¦  «        ‚|                      |¦  «        }|€ | j        d|||dœ|¤Ž}nct          |t          ¦  «        sNt	          |d         t          |¦  «        dk    r|d         ndt          |¦  «        dk    r|d         nd¬¦  «        }|d         } | j        d|||	d|||
d	œ|¤Ž}|d         }|                      |¦  «        }|                     dd
¬¦  «        \  }}| 	                    d
¦  «         
                    ¦   «         }| 	                    d
¦  «         
                    ¦   «         }d}|��|��t          |                     ¦   «         ¦  «        dk    r-| 	                    d
¦  «                             |j        ¦  «        }t          |                     ¦   «         ¦  «        dk    r-| 	                    d
¦  «                             |j        ¦  «        }|                     d¦  «        }|                     d|¦  «        }|                     d|¦  «        }t          |¬¦  «        } |||¦  «        } |||¦  «        }||z   dz  }t!          ||||j        |j        |j        |j        |j        |j        |j        ¬¦
  «
        S )a†  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. UMT5 is a model with relative position embeddings so
            you should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            [What are input IDs?](../glossary#input-ids)

            To know more on how to prepare `input_ids` for pretraining take a look a [UMT5 Training](./umt5#training).
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            UMT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
            is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).

            To know more on how to prepare `decoder_input_ids` for pretraining take a look at [UMT5
            Training](./umt5#training).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.
        NFr·  rˆ  r   r&   r:   r‰  rŠ  r;   rƒ   r¡  )
r¤  Ústart_logitsÚ
end_logitsrÏ   r‹  rŒ  r!  r�  rÎ   rŽ  r�  )rA  rE  rw  ra   r   r�  ry  r,  ÚsplitrÂ  rŒ   r‡   r=   rÃ   r  r   r   rÏ   rE   r   r!  rg  )r3   r$  r}   r#  r%  r…  rÏ  rÐ  rf  r†  ri  r€   rE   r‘  r¦  r¥  rÒ  rÓ  Ú
total_lossÚignored_indexr¨  Ú
start_lossÚend_losss                          r7   rG   z UMT5ForQuestionAnswering.forward�  s  € ðX Ð&¨=Ð+DØˆIð
 Ð$Ð)>Ð)FØÐ Ý ðUñô ð ð
 !%× 1Ò 1°)Ñ <Ô <Ðð Ð"Ø*˜dœlð Ø#Ø-Ø+ðð ð ð	ð ˆOˆOõ ˜O­_Ñ=Ô=ð 	Ý-Ø"1°!Ô"4Ý47¸Ñ4HÔ4HÈ1Ò4LÐ4L˜o¨aÔ0Ð0ÐRVÝ14°_Ñ1EÔ1EÈÒ1IÐ1I˜?¨1Ô-Ð-Ètðñ ô ˆOð (¨Ô*ˆð '˜$œ,ð 	
Ø'Ø1Ø/Ø Ø"/Ø#1Øð	
ð 	
ð ð	
ð 	
ˆð *¨!Ô,ˆà—’ Ñ1Ô1ˆØ#)§<¢<°°r <Ñ#:Ô#:Ñ ˆ�jØ#×+Ò+¨BÑ/Ô/×:Ò:Ñ<Ô<ˆØ×'Ò'¨Ñ+Ô+×6Ò6Ñ8Ô8ˆ
àˆ
ØÑ&¨=Ñ+Då�?×'Ò'Ñ)Ô)Ñ*Ô*¨QÒ.Ð.Ø"1×"9Ò"9¸"Ñ"=Ô"=×"@Ò"@ÀÔATÑ"UÔ"U�Ý�=×%Ò%Ñ'Ô'Ñ(Ô(¨1Ò,Ð,Ø -× 5Ò 5°bÑ 9Ô 9× <Ò <¸ZÔ=NÑ OÔ O�à(×-Ò-¨aÑ0Ô0ˆMØ-×3Ò3°A°}ÑEÔEˆOØ)×/Ò/°°=ÑAÔAˆMå'°]ÐCÑCÔCˆHØ!˜ ,°Ñ@Ô@ˆJØ�x 
¨MÑ:Ô:ˆHØ$ xÑ/°1Ñ4ˆJå2ØØ%Ø!Ø+Ô;Ø"1Ô"?Ø.Ô9Ø,Ô=Ø&5Ô&GØ"1Ô"?Ø.Ô9ð
ñ 
ô 
ð 	
r8   r«  )rH   rI   rJ   r”  r-   r€  rd  r"   r   r/   r•  r–  r—  r˜  rb   rì   r   r   r   rG   rK   rL   s   @r7   r5  r5  f  s¡  ø€ € € € € ð (7Ø'6ðð Ðð
ð ð ð ð ð.ð ð ð:ð :ð :ð
 Øð .2Ø37Ø59Ø:>Ø=AØ37Ø15Ø26Ø:>Ø!%ðy
ð y
àÔ# dÑ*ðy
ð Ô)¨DÑ0ðy
ð !Ô+¨dÑ2ð	y
ð
 !&Ô 0°4Ñ 7ðy
ð ˜u U¤\Ô2Ô3°dÑ:ðy
ð Ô)¨DÑ0ðy
ð Ô'¨$Ñ.ðy
ð Ô(¨4Ñ/ðy
ð  %Ô0°4Ñ7ðy
ð ˜$‘;ðy
ð Ð+Ô,ðy
ð 
-ðy
ð y
ð y
ñ „^ñ Ôðy
ð y
ð y
ð y
ð y
r8   r5  )r4  r3  r5  r²  r;  r2  r  )Nrx   N)Prê   ru  r¶   Úcollections.abcr   r/   r   Útorch.nnr   r   r   Ú r	   r0  Úactivationsr
   Úcache_utilsr   r   r   Ú
generationr   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r    r!   Úutils.genericr"   r#   Úutils.output_capturingr$   r%   Úconfiguration_umt5r'   Ú
get_loggerrH   rŸ   ÚModuler)   rN   rg   ro   rb   rµ   r�   r‘   rî   rú   r  r  r  rS  r2  r3  r4  r²  r;  r5  Ú__all__r�  r8   r7   ú<module>rë     s|  ðð Ð à €€€Ø €€€Ø $Ð $Ð $Ð $Ð $Ð $à €€€Ø Ð Ð Ð Ð Ð Ø AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aà &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CØ )Ð )Ð )Ð )Ð )Ð )Ø JÐ JÐ JÐ JÐ JÐ JÐ JÐ JØ 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð GÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð JÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ *Ð *Ð *Ð *Ð *Ð *ð 
ˆÔ	˜HÑ	%Ô	%€ð+ð +ð +ð +ð +�B”Iñ +ô +ð +ð4ð ð ð ð ˜œ	ñ ô ð ð.ð ð ð ð ˜RœYñ ô ð ð<ð ð ð ð �"”)ñ ô ð ð2 !ØØ)-ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð �T‰\ð%ð ð%ð ”< $Ñ&ð%ð Ð'Ô(ð%ð %ð %ð %ð@u)ð u)ð u)ð u)ð u)�B”Iñ u)ô u)ð u)ðpð ð ð ð ˜RœYñ ô ð ð4ð ð ð ð ˜bœiñ ô ð ð8:ð :ð :ð :ð :Ð*ñ :ô :ð :ð|ð ð ð ð ˜RœYñ ô ð ð$ ðu!ð u!ð u!ð u!ð u!˜/ñ u!ô u!ñ „ðu!ðpb
ð b
ð b
ð b
ð b
Ð#ñ b
ô b
ð b
ðJ ð[
ð [
ð [
ð [
ð [
Ð#ñ [
ô [
ñ „ð[
ð| €ððñ ô ð
|)ð |)ð |)ð |)ð |)Ð#6¸ñ |)ô |)ñô ð
|)ð~ ðRð Rð Rð Rð RÐ*ñ Rô Rñ „ðRðj €ððñ ô ðK
ð K
ð K
ð K
ð K
Ð$7ñ K
ô K
ñô ðK
ð\ ð=
ð =
ð =
ð =
ð =
Ð!4ñ =
ô =
ñ „ð=
ð@ ða
ð a
ð a
ð a
ð a
Ð2ñ a
ô a
ñ „ða
ðHð ð €€€r8   