§
    9²¸j¿L ã                   ó  — d Z ddlZddlZddlmZ ddlmZ ddlZddlmZ ddl	m
Z
 ddlmZ dd	lmZ dd
lmZmZmZ ddlmZ ddlmZmZ ddlmZ ddlmZmZmZmZ ddl m!Z!m"Z" ddl#m$Z$ ddl%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+ ddl,m-Z-  e+j.        e/¦  «        Z0	 	 	 dZdej1        dej2        dej2        dej2        dej2        dz  de3dz  de3dej2        dz  de$e(         fd„Z4d[dej2        d e5d!e5d"e5d#ej2        f
d$„Z6dej2        d e5d!e5d#ej2        fd%„Z7d[dej2        d&e5d'e5d"e5d#ej2        f
d(„Z8d e5d#ej2        fd)„Z9d*ej2        d e5d#ej2        fd+„Z:dej2        d e5d,ej;        d#ej2        fd-„Z<dej2        d.e5d#e=ej2        ej2        f         fd/„Z>dej2        d.e5d#ej2        fd0„Z?d1ej2        d2ej2        d3e5d#ej2        fd4„Z@ G d5„ d6ej1        ¦  «        ZA G d7„ d8ej1        ¦  «        ZB G d9„ d:ej1        ¦  «        ZC G d;„ d<ej1        ¦  «        ZD G d=„ d>ej1        ¦  «        ZE G d?„ d@ej1        ¦  «        ZF G dA„ dBej1        ¦  «        ZG G dC„ dDej1        ¦  «        ZH G dE„ dFej1        ¦  «        ZI G dG„ dHej1        ¦  «        ZJ G dI„ dJej1        ¦  «        ZK G dK„ dLe¦  «        ZLe) G dM„ dNe"¦  «        ¦   «         ZM G dO„ dPeM¦  «        ZNe) G dQ„ dReM¦  «        ¦   «         ZO e)dS¬T¦  «         G dU„ dVeMe¦  «        ¦   «         ZPe) G dW„ dXeM¦  «        ¦   «         ZQg dY¢ZRdS )\zPyTorch LongT5 model.é    N)ÚCallable)ÚAny)Únn)ÚCrossEntropyLossé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚDUMMY_INPUTSÚ
DUMMY_MASKÚTransformersKwargsÚauto_docstringÚis_torchdynamo_compilingÚloggingé   )ÚLongT5Configç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚposition_biasÚkwargsc                 ó¼  — |€|                      d¦  «        dz  }t          j        ||                     dd¦  «        ¦  «        |z  }	|�|	|z   }	|�|	|z   }	t          j                             |	d¬¦  «        }	t          j                             |	|| j        ¬¦  «        }	t          j        |	|¦  «        }
|
                     dd¦  «         	                    ¦   «         }
|
|	fS )Néÿÿÿÿç      à¿é   r   ©Údim©ÚpÚtrainingr   )
ÚsizeÚtorchÚmatmulÚ	transposer   Ú
functionalÚsoftmaxr'   r2   Ú
contiguous)r!   r"   r#   r$   r%   r&   r'   r(   r)   Úattn_weightsÚattn_outputs              úz/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/longt5/modeling_longt5.pyÚeager_attention_forwardr=   6   sØ   € ð €Ø—*’*˜R‘.”. DÑ(ˆõ ”<  s§}¢}°Q¸Ñ':Ô':Ñ;Ô;¸gÑE€LàÐ Ø# mÑ3ˆàÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2Ð(Ñ>Ô>€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$ó    ÚxÚ	block_lenr/   Ú	pad_valueÚreturnc                 ój  — | j         |          |z  }t          | j         ¦  «        s?t          | j         ¦  «        }||xx         |z  cc<   t          j        || j        ¬¦  «        S dg| j        z  }d|f||<   t          |ddd…         d¦  «        }t          j	         
                    | |d|¬¦  «        } | S )	zHPad a tensor so that a sequence length will be a multiple of `block_len`©Údtype©r   r   r   Nr+   © Úconstant©ÚpadÚmoder$   )ÚshapeÚallÚlistr4   ÚzerosrE   ÚndimÚsumr   r7   rJ   )r?   r@   r/   rA   Úpad_lenÚ	new_shaperJ   s          r<   Ú_pad_to_multiplerT   Y   sµ   € àŒw�sŒ|ˆm˜iÑ'€GåˆqŒw‰<Œ<ð 5Ý˜œ‘M”Mˆ	Ø�#ˆˆŒ˜'Ñ!ˆˆ‰ÝŒ{˜9¨A¬GÐ4Ñ4Ô4Ð4àˆ(�Q”VÑ
€CØ�7ˆ|€Cˆ�HÝ
ˆc�$�$�B�$Œi˜Ñ
Ô
€CÝ
Œ×Ò˜! ¨:¸YÐÑGÔG€AØ€Hr>   c                 ó2  — | j         |         |z  dk    rt          | ||d¬¦  «        } | j         |         |z  }| j         d|…         ||fz   | j         |dz   d…         z   }d|v r!t          j        || j        | j        ¬¦  «        S |                      |¦  «        S )zÅSplit an input tensor into blocks of a given `block_len` along the given `dim`. If the dimension length
    is not a multiple of `block_len`, it will be padded first with selected `pad_value`.
    r   )rA   Nr   ©rE   Údevice)rL   rT   r4   ÚemptyrE   rW   Úreshape)r?   r@   r/   Ú
num_blocksÚoutput_shapes        r<   Ú_split_into_blocksr\   i   s¥   € ð
 	„wˆs„|�iÑ 1Ò$Ð$Ý˜Q 	¨3¸!Ð<Ñ<Ô<ˆØ”˜” Ñ*€JØ”7˜4˜C˜4”= J°	Ð#:Ñ:¸Q¼WÀcÈAÁgÀ[À[Ô=QÑQ€LàˆLÐÐÝŒ{˜<¨q¬w¸q¼xÐHÑHÔHÐHØ�9Š9�\Ñ"Ô"Ð"r>   Ú	block_dimÚsequence_dimc                 óº  — | j         |         }dg| j        z  }d||<   t          |ddd…         d¦  «        }t          j                             | |d|¬¦  «        } g }t          d¦  «        D ][}t          d	d¦  «        g| j        z  }t          |||z   ¦  «        ||<   t          |¦  «        }| 	                    | |         ¦  «         Œ\t          j        ||¬
¦  «        S )z�Concatenate three consecutive blocks for each input block for local attention.

    For more information, see: https://huggingface.co/papers/2112.07916.
    rF   )r   r   Nr+   rG   rH   rI   r   r   r.   )rL   rP   rQ   r   r7   rJ   ÚrangeÚsliceÚtupleÚappendr4   Úcat)	r?   r]   r^   rA   rZ   rJ   Úblocks_listÚiÚindicess	            r<   Ú_concatenate_3_blocksrh   x   sä   € ð
 ”˜Ô#€Jàˆ(�Q”VÑ
€CØ€Cˆ	�NÝ
ˆc�$�$�B�$Œi˜Ñ
Ô
€Cå
Œ×Ò˜! ¨:¸YÐÑGÔG€Aà&(€KÝ�1‰XŒXð 'ð 'ˆõ ˜˜D‘>”>Ð" Q¤VÑ+ˆÝ" 1 a¨*¡nÑ5Ô5ˆ�	ÑÝ˜‘.”.ˆØ×Ò˜1˜Wœ:Ñ&Ô&Ð&Ð&åŒ9�[ lÐ3Ñ3Ô3Ð3r>   c                 ó¸   — t          j        d| z  t           j        ¬¦  «        }|| |  …         }|                     d¦  «        |                     d¦  «        z
  }|S )z:Makes 3-blocked relative position ids for local attention.r   rD   r   r   )r4   ÚarangeÚint32Ú	unsqueeze)r@   Úposition_idsÚcenter_position_idsÚrelative_position_idss       r<   Ú"_make_3block_relative_position_idsrp   ‘   s]   € å”<  I¡µU´[ÐAÑAÔA€LØ& y°)°Ð';Ô<Ðà(×2Ò2°1Ñ5Ô5Ð8K×8UÒ8UÐVWÑ8XÔ8XÑXÐØ Ð r>   Úlocal_attention_maskc                 óÎ   — t          |¦  «        }t          j        |¦  «        |k     }|dddd…dd…f         }|                     | j        ¦  «        }t          j        | |¦  «        S )znMask local attention mask to enforce that tokens are not allowed to attend tokens farther than ``local_radius.N)rp   r4   ÚabsÚtorW   Úlogical_and)rq   r@   ro   Úlocality_masks       r<   Ú_mask_local_attention_maskrw   š   sk   € å>¸yÑIÔIÐÝ”IÐ3Ñ4Ô4°yÒ@€MØ! $¨¨a¨a¨a°°°Ð"2Ô3€MØ!×$Ò$Ð%9Ô%@ÑAÔA€MÝÔÐ1°=ÑAÔAÐAr>   rW   c                 ó8  — t          | |d¬¦  «        }t          |dd¬¦  «        }|                     d¦  «        }|                     d¦  «        }t          j        ||¦  «        }t          ||¦  «        }|                     d¦  «                             |¦  «        S )z;Prepare attention mask to be applied for a local attention.r   r.   r-   ©r]   r^   r+   éþÿÿÿ)r\   rh   rl   r4   ru   rw   rt   )r%   r@   rW   Ú_blocked_attention_maskÚ_3blocked_attention_maskrq   s         r<   Ú_get_local_attention_maskr}   £   s¢   € õ 1°ÀÐPQÐRÑRÔRÐå4Ð5LÐXYÐhiÐjÑjÔjÐà5×?Ò?ÀÑCÔCÐØ7×AÒAÀ"ÑEÔEÐå Ô,Ð-DÐF^Ñ_Ô_ÐÝ5Ð6JÈIÑVÔVÐà×)Ò)¨!Ñ,Ô,×/Ò/°Ñ7Ô7Ð7r>   Úglobal_block_sizec                 ó`  ‡‡— | j         dd…         \  }Šdt          j        dt          j        fˆˆfd„}t          j        | | j        ¬¦  «        ‰z  }t          j        |d¬¦  «        |z
  }t          j        | d	k    d
d¦  «                             | j        ¦  «        }t          j	        ||z   d
z
  ¦  «                             | j        ¦  «        }t          j
        dd|j        |j        ¬¦  «        }t          j        ||k    ||¦  «        }|| z  | dz
  z   } ||¦  «        }‰‰z  }|dk    rDt          j        |d¬¦  «        j                             |d¦  «                             dd¦  «        }	n"t          j        |d|j        |j        ¬¦  «        }	t          j        t          j        ||¦  «        d¬¦  «        dz
  }
|
                     | j        ¦  «        }
t          j        |
|	k    dd¦  «        }
|                     t          j        ¦  «        |
                     t          j        ¦  «        fS )a  Obtain the "fixed block" global id corresponding to each input token.

    This implementation is a simplified version of the original Flaxformr implementation adopted from:
    https://github.com/google/flaxformer/blob/main/flaxformer/architectures/longt5/long_attention.py.

    In our scenario, as we use this strategy only for a decoder, orphan tokens, i.e. those tokens which do not make for
    the whole fixed block, are assigned to the preceding block.

    Padding tokens from the original sequence are represented by -1.
    Nr-   Ú	block_idsrB   c                 ód  •— t          j        ‰¦  «        ‰z  ‰dz
  k    }|                     | j        ¦  «        }t          j        || dk    ¦  «        }|                     d¦  «                             d¦  «                             | j        ¦  «        dz
  }t          j	        | |k     | |¦  «        } | S )Nr   r   r+   )
r4   rj   rt   rW   ru   rQ   rl   ÚtyperE   Úwhere)r€   Ú
block_endsÚtrue_block_endsÚfull_blocksr~   Úseq_lens       €€r<   Úhandle_orphan_tokensz:_make_global_fixed_block_ids.<locals>.handle_orphan_tokensÂ   s£   ø€ Ý”l 7Ñ+Ô+Ð.?Ñ?ÐDUÐXYÑDYÒYˆ
Ø—]’] 9Ô#3Ñ4Ô4ˆ
ÝÔ+¨J¸	ÀQºÑGÔGˆØ%×)Ò)¨"Ñ-Ô-×7Ò7¸Ñ;Ô;×@Ò@ÀÄÑQÔQÐTUÑUˆÝ”K 	¨KÒ 7¸ÀKÑPÔPˆ	ØÐr>   ©rW   r   )Úaxisr    ç      ð?g     @�ÀrG   r+   rV   r   r.   )rL   r4   ÚTensorÚ	ones_likerW   Úcumsumrƒ   r‚   rE   ÚfloorÚfullÚmaxÚvaluesÚrepeatr6   rO   Úonesrt   Úint)r%   r~   Ú
batch_sizerˆ   Úfixed_block_maskÚmaskÚglobal_block_idsÚ_global_block_ids_lower_boundÚnum_globalsÚ_sequence_block_ids_maxÚglobal_segment_idsr‡   s    `         @r<   Ú_make_global_fixed_block_idsrž   ³   sR  øø€ ð )Ô.¨r°¨rÔ2Ñ€J�ð­¬ð ½¼ð ð ð ð ð ð ð õ ” ~¸nÔ>SÐTÑTÔTÐWhÑhÐÝ”|Ð$4¸1Ð=Ñ=Ô=Ð@PÑPÐÝŒ;�~¨Ò,¨c°7Ñ;Ô;×@Ò@ÀÔAUÑVÔV€DÝ”{ 4Ð*:Ñ#:¸SÑ#@ÑAÔA×FÒFÀ~ÔG[Ñ\Ô\ÐÝ$)¤J¨r°2Ð=MÔ=SÐ\lÔ\sÐ$tÑ$tÔ$tÐ!Ý”{ØÐ8Ò8Ð:JÐLiñô Ðð )¨>Ñ9¸nÈqÑ>PÑQÐà+Ð+Ð,<Ñ=Ô=ÐØÐ.Ñ.€Kà�Q‚€Ý"'¤)Ð,<À"Ð"EÑ"EÔ"EÔ"L×"SÒ"SÐT_ÐabÑ"cÔ"c×"mÒ"mÐnoÐqrÑ"sÔ"sÐÐå"'¤+Ø˜Ð!1Ô!7Ð@PÔ@Wð#
ñ #
ô #
Ðõ œ¥e¤j°¸[Ñ&IÔ&IÈrÐRÑRÔRÐUVÑVÐØ+×.Ò.¨~Ô/DÑEÔEÐÝœÐ%7Ð;RÒ%RÐTUÐWXÑYÔYÐØ× Ò ¥¤Ñ+Ô+Ð-?×-DÒ-DÅUÄYÑ-OÔ-OÐOÐOr>   c                 óÌ   — t          | |¦  «        \  }}|j        d         }t          j        ||j        ¬¦  «        }||d         z
  }|                     t          j        ¦  «        S )zBCreate the relative position tensor for local -> global attention.r+   r‰   ©.N)rž   rL   r4   rj   rW   r‚   Úint64)r%   r~   r€   r�   Úglobal_seq_lenÚglobal_positionsÚside_relative_positions          r<   Ú _make_side_relative_position_idsr¥   ä   sc   € å$@ÀÐQbÑ$cÔ$cÑ!€IÐ!Ø'Ô-¨bÔ1€NÝ”| N¸9Ô;KÐLÑLÔLÐØ-°	¸)Ô0DÑDÐØ!×&Ò&¥u¤{Ñ3Ô3Ð3r>   Úhidden_statesr€   r¢   c                 ón  — |                      |dk    t          j        ||j        |j        ¬¦  «        ¦  «        }t
          j                             |                     t          j	        ¦  «        |dz   ¦  «        dd…dd…dd…f         }t          j
        d| |                     | j        ¦  «        ¦  «        S )zFCompute individual block aggregates by summing over individual blocks.r   rV   r   Nr+   z...nd,...ng->...gd)rƒ   r4   ÚtensorrE   rW   r   r7   Úone_hotr‚   r¡   Úeinsum)r¦   r€   r¢   Úone_hot_block_idss       r<   Ú_create_global_aggregatesr¬   í   s®   € ð
 —’Ø�QŠ�œ ^¸9¼?ÐS\ÔScÐdÑdÔdñô €Iõ œ×-Ò-¨i¯nªn½U¼[Ñ.IÔ.IÈ>Ð\]ÑK]Ñ^Ô^Ð_`Ð_`Ð_`ÐbcÐbcÐbcÐehÐfhÐehÐ_hÔiÐÝŒ<Ð,¨mÐ=N×=SÒ=SÐTaÔTgÑ=hÔ=hÑiÔiÐir>   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚLongT5LayerNormç�íµ ÷Æ°>c                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )zg
        Construct a layernorm module in the LongT5 style. No bias and no subtraction of mean.
        N)ÚsuperÚ__init__r   Ú	Parameterr4   r”   ÚweightÚvariance_epsilon)ÚselfÚhidden_sizeÚepsÚ	__class__s      €r<   r²   zLongT5LayerNorm.__init__û   sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐr>   c                 óh  — |                      t          j        ¦  «                             d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        j        t          j	        t          j
        fv r|                      | j        j        ¦  «        }| j        |z  S )Nr-   r+   T)Úkeepdim)rt   r4   Úfloat32ÚpowÚmeanÚrsqrtrµ   r´   rE   Úfloat16Úbfloat16)r¶   r¦   Úvariances      r<   ÚforwardzLongT5LayerNorm.forward  s–   € ð !×#Ò#¥E¤MÑ2Ô2×6Ò6°qÑ9Ô9×>Ò>¸rÈ4Ð>ÑPÔPˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆð Œ;Ô¥¤µ´Ð ?Ð?Ð?Ø)×,Ò,¨T¬[Ô->Ñ?Ô?ˆMàŒ{˜]Ñ*Ð*r>   )r¯   )Ú__name__Ú
__module__Ú__qualname__r²   rÃ   Ú__classcell__©r¹   s   @r<   r®   r®   ú   sL   ø€ € € € € ð$ð $ð $ð $ð $ð $ð+ð +ð +ð +ð +ð +ð +r>   r®   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚLongT5DenseActDenseÚconfigc                 óJ  •— t          ¦   «                              ¦   «          t          j        |j        |j        d¬¦  «        | _        t          j        |j        |j        d¬¦  «        | _        t          j        |j	        ¦  «        | _
        t          |j                 | _        d S ©NF©Úbias)r±   r²   r   ÚLinearÚd_modelÚd_ffÚwiÚwoÚDropoutÚdropout_rater'   r	   Údense_act_fnÚact©r¶   rË   r¹   s     €r<   r²   zLongT5DenseActDense.__init__  sx   ø€ Ý‰Œ×ÒÑÔÐÝ”)˜FœN¨F¬K¸eÐDÑDÔDˆŒÝ”)˜FœK¨¬¸eÐDÑDÔDˆŒÝ”z &Ô"5Ñ6Ô6ˆŒÝ˜&Ô-Ô.ˆŒˆˆr>   c                 ó°  — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }t          | j        j        t          j        ¦  «        r]|j        | j        j        j        k    rC| j        j        j        t          j	        k    r$| 
                    | j        j        j        ¦  «        }|                      |¦  «        }|S ©N)rÓ   rØ   r'   Ú
isinstancerÔ   r´   r4   rŒ   rE   Úint8rt   )r¶   r¦   s     r<   rÃ   zLongT5DenseActDense.forward  s¨   € ØŸš Ñ.Ô.ˆØŸš Ñ/Ô/ˆØŸš ]Ñ3Ô3ˆå�t”w”~¥u¤|Ñ4Ô4ð	CàÔ# t¤w¤~Ô';Ò;Ð;Ø””Ô$­¬
Ò2Ð2à)×,Ò,¨T¬W¬^Ô-AÑBÔBˆMØŸš Ñ.Ô.ˆØÐr>   ©rÄ   rÅ   rÆ   r   r²   rÃ   rÇ   rÈ   s   @r<   rÊ   rÊ     sS   ø€ € € € € ð/˜|ð /ð /ð /ð /ð /ð /ðð ð ð ð ð ð r>   rÊ   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚLongT5DenseGatedActDenserË   c                 ó–  •— t          ¦   «                              ¦   «          t          j        |j        |j        d¬¦  «        | _        t          j        |j        |j        d¬¦  «        | _        t          j        |j        |j        d¬¦  «        | _        t          j	        |j
        ¦  «        | _        t          |j                 | _        d S rÍ   )r±   r²   r   rÐ   rÑ   rÒ   Úwi_0Úwi_1rÔ   rÕ   rÖ   r'   r	   r×   rØ   rÙ   s     €r<   r²   z!LongT5DenseGatedActDense.__init__+  s”   ø€ Ý‰Œ×ÒÑÔÐÝ”I˜fœn¨f¬kÀÐFÑFÔFˆŒ	Ý”I˜fœn¨f¬kÀÐFÑFÔFˆŒ	Ý”)˜FœK¨¬¸eÐDÑDÔDˆŒÝ”z &Ô"5Ñ6Ô6ˆŒÝ˜&Ô-Ô.ˆŒˆˆr>   c                 óÞ   — |                       |                      |¦  «        ¦  «        }|                      |¦  «        }||z  }|                      |¦  «        }|                      |¦  «        }|S rÛ   )rØ   râ   rã   r'   rÔ   )r¶   r¦   Úhidden_geluÚhidden_linears       r<   rÃ   z LongT5DenseGatedActDense.forward3  sb   € Ø—h’h˜tŸyšy¨Ñ7Ô7Ñ8Ô8ˆØŸ	š	 -Ñ0Ô0ˆØ# mÑ3ˆØŸš ]Ñ3Ô3ˆØŸš Ñ.Ô.ˆØÐr>   rÞ   rÈ   s   @r<   rà   rà   *  sS   ø€ € € € € ð/˜|ð /ð /ð /ð /ð /ð /ðð ð ð ð ð ð r>   rà   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚLongT5LayerFFrË   c                 ó$  •— t          ¦   «                              ¦   «          |j        rt          |¦  «        | _        nt          |¦  «        | _        t          |j        |j        ¬¦  «        | _	        t          j        |j        ¦  «        | _        d S )N©r¸   )r±   r²   Úis_gated_actrà   ÚDenseReluDenserÊ   r®   rÑ   Úlayer_norm_epsilonÚ
layer_normr   rÕ   rÖ   r'   rÙ   s     €r<   r²   zLongT5LayerFF.__init__>  sx   ø€ Ý‰Œ×ÒÑÔÐØÔð 	>Ý":¸6Ñ"BÔ"BˆDÔÐå"5°fÑ"=Ô"=ˆDÔå)¨&¬.¸fÔ>WÐXÑXÔXˆŒÝ”z &Ô"5Ñ6Ô6ˆŒˆˆr>   c                 óŠ   — |                       |¦  «        }|                      |¦  «        }||                      |¦  «        z   }|S rÛ   )rî   rì   r'   )r¶   r¦   Úforwarded_statess      r<   rÃ   zLongT5LayerFF.forwardH  sF   € ØŸ?š?¨=Ñ9Ô9ÐØ×.Ò.Ð/?Ñ@Ô@ÐØ%¨¯ªÐ5EÑ(FÔ(FÑFˆØÐr>   rÞ   rÈ   s   @r<   rè   rè   =  sS   ø€ € € € € ð7˜|ð 7ð 7ð 7ð 7ð 7ð 7ðð ð ð ð ð ð r>   rè   c                   ó|   ‡ — e Zd Z	 	 	 ddededz  defˆ fd„Zedd
„¦   «         Zdd„Z		 	 	 	 dde
e         fd„Zˆ xZS )ÚLongT5AttentionFNrË   Ú	layer_idxÚ	is_causalc                 óT  •— t          ¦   «                              ¦   «          || _        |j        | _        || _        || _        |j        | _        |j        | _        |j        | _        |j	        | _
        |j        | _        |j        | _        | j        | j
        z  | _        d| _        || _        |€/| j        r(t$                               d| j        j        › d�¦  «         t-          j        | j        | j        d¬¦  «        | _        t-          j        | j        | j        d¬¦  «        | _        t-          j        | j        | j        d¬¦  «        | _        t-          j        | j        | j        d¬¦  «        | _        | j        r$t-          j        | j        | j        ¦  «        | _        d| _        d S )Nr‹   zInstantiating a decoder z³ without passing `layer_idx` is not recommended and will to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` when creating this class.FrÎ   )r±   r²   rË   Ú
is_decoderrô   Úhas_relative_attention_biasÚrelative_attention_num_bucketsÚrelative_attention_max_distancerÑ   Úd_kvÚkey_value_proj_dimÚ	num_headsÚn_headsrÖ   r'   Ú	inner_dimr&   ró   ÚloggerÚwarning_oncer¹   rÄ   r   rÐ   ÚqÚkÚvÚoÚ	EmbeddingÚrelative_attention_biasÚgradient_checkpointing)r¶   rË   r÷   ró   rô   r¹   s        €r<   r²   zLongT5Attention.__init__Q  s~  ø€ õ 	‰Œ×ÒÑÔÐØˆŒØ Ô+ˆŒØ"ˆŒØ+FˆÔ(Ø.4Ô.SˆÔ+Ø/5Ô/UˆÔ,Ø”~ˆŒØ"(¤+ˆÔØÔ'ˆŒØÔ*ˆŒØœ¨Ô(?Ñ?ˆŒàˆŒØ"ˆŒØÐ ¤ÐÝ×Òð,¨4¬>Ô+Bð ,ð ,ð ,ñô ð õ ”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ>¨4¬<¸eÐDÑDÔDˆŒàÔ+ð 	kÝ+-¬<¸Ô8[Ð]aÔ]iÑ+jÔ+jˆDÔ(à&+ˆÔ#Ð#Ð#r>   Té    é€   c                 óP  — d}|rC|dz  }|| dk                          t          j        ¦  «        |z  z  }t          j        | ¦  «        } n(t          j        | t          j        | ¦  «        ¦  «         } |dz  }| |k     }|t          j        |                      ¦   «         |z  ¦  «        t          j        ||z  ¦  «        z  ||z
  z                        t          j        ¦  «        z   }t          j        |t          j	        ||dz
  ¦  «        ¦  «        }|t          j
        || |¦  «        z  }|S ©aÒ  
        Adapted from Mesh Tensorflow:
        https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593

        Translate relative position to a bucket number for relative attention. The relative position is defined as
        memory_position - query_position, i.e. the distance in tokens from the attending position to the attended-to
        position. If bidirectional=False, then positive relative positions are invalid. We use smaller buckets for
        small absolute relative_position and larger buckets for larger absolute relative_positions. All relative
        positions >=max_distance map to the same bucket. All relative positions <=-max_distance map to the same bucket.
        This should allow for more graceful generalization to longer sequences than the model has been trained on

        Args:
            relative_position: an int32 Tensor
            bidirectional: a boolean - whether the attention is bidirectional
            num_buckets: an integer
            max_distance: an integer

        Returns:
            a Tensor with the same shape as relative_position, containing int32 values in the range [0, num_buckets)
        r   r-   r   ©rt   r4   Úlongrs   ÚminÚ
zeros_likeÚlogÚfloatÚmathÚ	full_likerƒ   ©Úrelative_positionÚbidirectionalÚnum_bucketsÚmax_distanceÚrelative_bucketsÚ	max_exactÚis_smallÚrelative_position_if_larges           r<   Ú_relative_position_bucketz)LongT5Attention._relative_position_bucketx  s>  € ð, ÐØð 	cØ˜AÑˆKØÐ!2°QÒ!6× :Ò :½5¼:Ñ FÔ FÈÑ TÑTÐÝ %¤	Ð*;Ñ <Ô <ÐÐå!&¤Ð+<½eÔ>NÐO`Ñ>aÔ>aÑ!bÔ!bÐ bÐð   1Ñ$ˆ	Ø$ yÒ0ˆð &/ÝŒIÐ'×-Ò-Ñ/Ô/°)Ñ;Ñ<Ô<ÝŒh�| iÑ/Ñ0Ô0ñ1à˜YÑ&ñ(÷ Š"�UŒZ‰.Œ.ñ	&Ð"õ
 &+¤YØ&­¬Ð8RÐT_ÐbcÑTcÑ(dÔ(dñ&
ô &
Ð"ð 	�EœK¨Ð2CÐE_Ñ`Ô`Ñ`ÐØÐr>   r   c                 ó¸  — |€| j         j        j        }t          j        |t          j        |¬¦  «        dd…df         |z   }t          j        |t          j        |¬¦  «        ddd…f         }||z
  }|                      || j         | j        | j	        ¬¦  «        }|                       |¦  «        }	|	 
                    g d¢¦  «                             d¦  «        }	|	S )ú%Compute binned relative position biasNrV   ©r  r  r  ©r-   r   r   r   )r  r´   rW   r4   rj   r  r  rö   rø   rù   Úpermuterl   )
r¶   Úquery_lengthÚ
key_lengthrW   Úpast_seen_tokensÚcontext_positionÚmemory_positionr  Úrelative_position_bucketr’   s
             r<   Úcompute_biaszLongT5Attention.compute_bias¨  sí   € àˆ>ØÔ1Ô8Ô?ˆFÝ œ<¨½E¼JÈvÐVÑVÔVÐWXÐWXÐWXÐZ^ÐW^Ô_ÐbrÑrÐÝœ, z½¼ÈFÐSÑSÔSÐTXÐZ[ÐZ[ÐZ[ÐT[Ô\ˆØ+Ð.>Ñ>ÐØ#'×#AÒ#AØØ#œÐ.ØÔ;ØÔ=ð	 $Bñ $
ô $
Ð ð ×-Ò-Ð.FÑGÔGˆØ—’ 	 	 	Ñ*Ô*×4Ò4°QÑ7Ô7ˆØˆr>   r)   c                 ó"  — |j         dd…         }g |¢d‘| j        ‘R }|�|                     | j        ¦  «        nd}	t	          |	t
          j        ¦  «        r|	                     ¦   «         n|	}	|du}
|                      |¦  «         	                    |¦  «         
                    dd¦  «        }d}t	          |t          ¦  «        r1|j                             | j        ¦  «        }|
r|j        }n
|j        }n|}|
r|n|}|
r3|�1|r/|j        | j                 j        }|j        | j                 j        }nÚg |j         dd…         ¢d‘| j        ‘R }|                      |¦  «         	                    |¦  «         
                    dd¦  «        }|                      |¦  «         	                    |¦  «         
                    dd¦  «        }|�E|                     ||| j        ¦  «        \  }}|
r$t	          |t          ¦  «        rd|j        | j        <   |€~|j         d         }| j        sFt          j        d| j        |d         |f|j        |j        ¬	¦  «        }| j        r| j        rd|_        n$|                      |d         ||j        |	¬
¦  «        }t?          j         | j!        j"        tF          ¦  «        } || ||||f| j        sdn| j$        | j%        |dœ|¤Ž\  }} |j&        g |¢d‘R Ž  '                    ¦   «         }|  (                    |¦  «        }|||fS )z€
        Self-attention (if key_value_states is None) or attention over source sentence (provided by key_value_states).
        Nr+   r   r   r-   FTrz   ©rW   rE   )rW   r%  r    )r'   r&   r(   ))rL   rû   Úget_seq_lengthró   rÜ   r4   rŒ   Úcloner  Úviewr6   r   Ú
is_updatedÚgetÚcross_attention_cacheÚself_attention_cacheÚlayersÚkeysr’   r  r  Úupdater÷   rO   rý   rW   rE   r  r2   Úrequires_gradr)  r   Úget_interfacerË   Ú_attn_implementationr=   r'   r&   rY   r9   r  )r¶   r¦   r˜   Úkey_value_statesr(   Úpast_key_valuesr)   Úinput_shapeÚhidden_shaper%  Úis_cross_attentionÚquery_statesr/  Úcurr_past_key_valuesÚcurrent_statesÚ
key_statesÚvalue_statesÚkv_shaper$  Úattention_interfacer;   r:   s                         r<   rÃ   zLongT5Attention.forward¹  s�  € ð $Ô)¨#¨2¨#Ô.ˆØB˜ÐB bÐB¨$Ô*AÐBÐBˆØM\ÐMh˜?×9Ò9¸$¼.ÑIÔIÐIÐnoÐå7AÐBRÕTYÔT`Ñ7aÔ7aÐwÐ+×1Ò1Ñ3Ô3Ð3ÐgwÐð .°TÐ9Ðà—v’v˜mÑ,Ô,×1Ò1°,Ñ?Ô?×IÒIÈ!ÈQÑOÔOˆð ˆ
Ý�oÕ':Ñ;Ô;ð 	3Ø(Ô3×7Ò7¸¼ÑGÔGˆJØ!ð Là'6Ô'LÐ$Ð$à'6Ô'KÐ$Ð$à#2Ð à-?ÐRÐ)Ð)À]ˆØð 	F /Ð"=À*Ð"=à-Ô4°T´^ÔDÔIˆJØ/Ô6°t´~ÔFÔMˆLˆLàP˜Ô-¨c¨r¨cÔ2ÐP°BÐP¸Ô8OÐPÐPˆHØŸš Ñ/Ô/×4Ò4°XÑ>Ô>×HÒHÈÈAÑNÔNˆJØŸ6š6 .Ñ1Ô1×6Ò6°xÑ@Ô@×JÒJÈ1ÈaÑPÔPˆLàÐ*Ø+?×+FÒ+FÀzÐS_ÐaeÔaoÑ+pÔ+pÑ(�
˜Là%ð F­*°_ÕFYÑ*ZÔ*Zð FØAE�OÔ.¨t¬~Ñ>ð Ð Ø#Ô)¨"Ô-ˆJØÔ3ð Ý %¤Ø˜œ k°!¤n°jÐAØ'Ô.Ø&Ô,ð!ñ !ô !�ð
 Ô.ð 7°4´=ð 7Ø26�MÔ/øà $× 1Ò 1Ø ”N J°|Ô7JÐ]mð !2ñ !ô !�õ )@Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð  $œ}Ð>�C�C°$´,Ø”LØ'ð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—f’f˜[Ñ)Ô)ˆà˜M¨<Ð7Ð7r>   )FNF©Tr  r	  )Nr   )NNNN)rÄ   rÅ   rÆ   r   r•   Úboolr²   Ústaticmethodr  r)  r   r   rÃ   rÇ   rÈ   s   @r<   rò   rò   P  sä   ø€ € € € € ð %*Ø $Øð%,ð %,àð%,ð ˜‘:ð	%,ð
 ð%,ð %,ð %,ð %,ð %,ð %,ðN ð- ð - ð - ñ „\ð- ð^ð ð ð ð( ØØØðX8ð X8ð Ð+Ô,ðX8ð X8ð X8ð X8ð X8ð X8ð X8ð X8r>   rò   c                   ó`   ‡ — e Zd Zddededdfˆ fd„Zedd
„¦   «         Zdefd„Z		 	 	 dd„Z
ˆ xZS )ÚLongT5LocalAttentionFrË   r÷   rB   Nc                 óð  •— t          ¦   «                              ¦   «          |j        | _        || _        |j        | _        |j        | _        |j        | _        |j        | _        |j	        | _
        |j        | _        | j        dz   | _        |j        | _        | j
        | j        z  | _        t!          j        | j        | j        d¬¦  «        | _        t!          j        | j        | j        d¬¦  «        | _        t!          j        | j        | j        d¬¦  «        | _        t!          j        | j        | j        d¬¦  «        | _        | j        r$t!          j        | j        | j
        ¦  «        | _        d| _        d S )Nr   FrÎ   )r±   r²   rö   r÷   rø   rù   rÑ   rú   rû   rü   rý   Úlocal_radiusr@   rÖ   r'   rþ   r   rÐ   r  r  r  r  r  r  r  ©r¶   rË   r÷   r¹   s      €r<   r²   zLongT5LocalAttention.__init__  s8  ø€ Ý‰Œ×ÒÑÔÐØ Ô+ˆŒØ+FˆÔ(Ø.4Ô.SˆÔ+Ø/5Ô/UˆÔ,Ø”~ˆŒØ"(¤+ˆÔØÔ'ˆŒØ"Ô/ˆÔØÔ*¨QÑ.ˆŒØÔ*ˆŒØœ¨Ô(?Ñ?ˆŒå”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ>¨4¬<¸eÐDÑDÔDˆŒàÔ+ð 	kÝ+-¬<¸Ô8[Ð]aÔ]iÑ+jÔ+jˆDÔ(à&+ˆÔ#Ð#Ð#r>   Tr  r	  c                 óP  — d}|rC|dz  }|| dk                          t          j        ¦  «        |z  z  }t          j        | ¦  «        } n(t          j        | t          j        | ¦  «        ¦  «         } |dz  }| |k     }|t          j        |                      ¦   «         |z  ¦  «        t          j        ||z  ¦  «        z  ||z
  z                        t          j        ¦  «        z   }t          j        |t          j	        ||dz
  ¦  «        ¦  «        }|t          j
        || |¦  «        z  }|S r  r  r  s           r<   r  z.LongT5LocalAttention._relative_position_bucket-  ó>  € ð. ÐØð 	cØ˜AÑˆKØÐ!2°QÒ!6× :Ò :½5¼:Ñ FÔ FÈÑ TÑTÐÝ %¤	Ð*;Ñ <Ô <ÐÐå!&¤Ð+<½eÔ>NÐO`Ñ>aÔ>aÑ!bÔ!bÐ bÐð   1Ñ$ˆ	Ø$ yÒ0ˆð &/ÝŒIÐ'×-Ò-Ñ/Ô/°)Ñ;Ñ<Ô<ÝŒh�| iÑ/Ñ0Ô0ñ1à˜YÑ&ñ(÷ Š"�UŒZ‰.Œ.ñ	&Ð"õ
 &+¤YØ&­¬Ð8RÐT_ÐbcÑTcÑ(dÔ(dñ&
ô &
Ð"ð 	�EœK¨Ð2CÐE_Ñ`Ô`Ñ`ÐØÐr>   Úblock_lengthc                 óæ  — | j         j        j        j        dk    r| j         j        j        nd}t	          j        d|z  t          j        |¬¦  «        }||| …         }|ddd…f         |dd…df         z
  }|                      || j         | j	        | j
        ¬¦  «        }|                       |¦  «        }|                     g d¢¦  «                             d¦  «                             d¦  «        }|S ©r  ÚmetaNr   rV   r   r!  r   ©r  r´   rW   r‚   r4   rj   r  r  rö   rø   rù   r"  rl   ©r¶   rO  Útarget_devicer'  r&  r  r(  r’   s           r<   r)  z!LongT5LocalAttention.compute_bias^  ó  € ð Ô+Ô2Ô9Ô>À&ÒHÐHð Ô(Ô/Ô6Ð6àð 	õ
  œ, q¨<Ñ'7½u¼zÐR_Ð`Ñ`Ô`ˆØ*¨<¸¸Ð+EÔFÐð ,¨D°!°!°!¨GÔ4Ð7GÈÈÈÈ4ÈÔ7PÑPÐØ#'×#AÒ#AØØ#œÐ.ØÔ;ØÔ=ð	 $Bñ $
ô $
Ð ð ×-Ò-Ð.FÑGÔGˆà—’ 	 	 	Ñ*Ô*×4Ò4°QÑ7Ô7×AÒAÀ!ÑDÔDˆØˆr>   c                 ó  ‡ ‡— |j         d d…         \  Š}ˆˆ fd„}ˆˆ fd„} |‰                      |¦  «        ¦  «        } |‰                      |¦  «        ¦  «        }	 |‰                      |¦  «        ¦  «        }
t	          |‰ j        d¬¦  «        }t	          |	‰ j        d¬¦  «        }	t	          |
‰ j        d¬¦  «        }
t          |	dd¬¦  «        }	t          |
dd¬¦  «        }
t          j        d||	¦  «        }|€¤‰ j	        sNt          j
        dd‰ j        ‰ j        d‰ j        z  f|j        |j        ¬	¦  «        }‰ j        r‰ j        rd
|_        n‰                      ‰ j        ¦  «        }|�3t          j        |dk    dd¦  «        }||                     dd¦  «        z   }||z  }t(          j                             |                     ¦   «         d¬¦  «                             |¦  «        }t(          j                             |‰ j        ‰ j        ¬¦  «        }|                     |
j        ¦  «        } |t          j        d||
¦  «        ¦  «        }|d d …d |…d d …f         }‰                      |¦  «        }||f}|r||fz   }|S )Nr-   c                 óH   •— |                       ‰d‰j        ‰j        ¦  «        S ©Ú
projectionr+   ©r.  rý   rû   ©Ústatesr–   r¶   s    €€r<   rL   z+LongT5LocalAttention.forward.<locals>.shape  ó    ø€ à—;’;˜z¨2¨t¬|¸TÔ=TÑUÔUÐUr>   c                 ó`   •— |                       ¦   «                              ‰d‰j        ¦  «        S ©rY   r+   ©r9   r.  rþ   r\  s    €€r<   Úunshapez-LongT5LocalAttention.forward.<locals>.unshapeƒ  ó)   ø€ à×$Ò$Ñ&Ô&×+Ò+¨J¸¸D¼NÑKÔKÐKr>   r   r.   ry   ú...qhd,...khd->...hqkr   r+  Tr   r    ç    _ Âr+   r0   ú...hqk,...khd->...qhd)rL   r  r  r  r\   r@   rh   r4   rª   r÷   rO   rý   rW   rE   r  r2   r6  r)  rƒ   r6   r   r7   r8   r  Útype_asr'   r‚   r  )r¶   r¦   r˜   r(   Úoutput_attentionsÚ
seq_lengthrL   rb  r>  rA  rB  Úscoresr:   r;   Úoutputsr–   s   `              @r<   rÃ   zLongT5LocalAttention.forwardv  sâ  øø€ ð "/Ô!4°R°a°RÔ!8Ñˆ
�Jð	Vð 	Vð 	Vð 	Vð 	Vð 	Vð	Lð 	Lð 	Lð 	Lð 	Lð 	Lð
 �u˜TŸVšV MÑ2Ô2Ñ3Ô3ˆØ�U˜4Ÿ6š6 -Ñ0Ô0Ñ1Ô1ˆ
Ø�u˜TŸVšV MÑ2Ô2Ñ3Ô3ˆõ *¨,¸¼ÈAÐNÑNÔNˆÝ'¨
°D´NÈÐJÑJÔJˆ
Ý)¨,¸¼ÈAÐNÑNÔNˆõ +¨:ÀÐQRÐSÑSÔSˆ
Ý,¨\ÀQÐUVÐWÑWÔWˆõ ”Ø# \°:ñ
ô 
ˆð Ð àÔ3ð BÝ %¤Ø˜˜4œ<¨¬¸¸T¼^Ñ9KÐLÐU[ÔUbÐjpÔjvð!ñ !ô !�ð Ô.ð 7°4´=ð 7Ø26�MÔ/øà $× 1Ò 1°$´.Ñ AÔ A�àÐå”{ 4¨!¢8¨S°%Ñ8Ô8�à -°·²¸qÀ!Ñ0DÔ0DÑ D�à�-Ñˆå”}×,Ò,¨V¯\ª\©^¬^ÀÐ,ÑDÔD×LÒLÈVÑTÔTˆå”}×,Ò,¨\¸T¼\ÐTXÔTaÐ,ÑbÔbˆà#×(Ò(¨Ô);Ñ<Ô<ˆØ�g�eœlÐ+BÀLÐR^Ñ_Ô_Ñ`Ô`ˆØ! ! ! ! [ j [°!°!°!Ð"3Ô4ˆØ—f’f˜[Ñ)Ô)ˆð Øð
ˆð
 ð 	0Ø  Ñ/ˆGØˆr>   ©FrE  ©NNF)rÄ   rÅ   rÆ   r   rF  r²   rG  r  r•   r)  rÃ   rÇ   rÈ   s   @r<   rI  rI    s»   ø€ € € € € ð,ð ,˜|ð ,È$ð ,Ð[_ð ,ð ,ð ,ð ,ð ,ð ,ð0 ð- ð - ð - ñ „\ð- ð^¨ð ð ð ð ð6 ØØðGð Gð Gð Gð Gð Gð Gð Gr>   rI  c                   ó’   ‡ — e Zd Zddededdfˆ fd„Zedd
„¦   «         Zdefd„Z	de
j        de
j        de
j        fd„Z	 	 	 dd„Zˆ xZS )ÚLongT5TransientGlobalAttentionFrË   r÷   rB   Nc                 ó�  •— t          ¦   «                              ¦   «          |j        | _        || _        |j        | _        |j        | _        |j        | _        |j        | _        |j	        | _
        |j        | _        | j        dz   | _        |j        | _        |j        | _        | j
        | j        z  | _        t#          j        | j        | j        d¬¦  «        | _        t#          j        | j        | j        d¬¦  «        | _        t#          j        | j        | j        d¬¦  «        | _        t#          j        | j        | j        d¬¦  «        | _        | j        r$t#          j        | j        | j
        ¦  «        | _        | j        r$t#          j        | j        | j
        ¦  «        | _        t5          |j        |j        ¬¦  «        | _        d S )Nr   FrÎ   rê   )r±   r²   rö   r÷   rø   rù   rÑ   rú   rû   rü   rý   rK  r@   r~   rÖ   r'   rþ   r   rÐ   r  r  r  r  r  r  Úglobal_relative_attention_biasr®   rí   Úglobal_input_layer_normrL  s      €r<   r²   z'LongT5TransientGlobalAttention.__init__Á  s~  ø€ Ý‰Œ×ÒÑÔÐØ Ô+ˆŒØ+FˆÔ(Ø.4Ô.SˆÔ+Ø/5Ô/UˆÔ,Ø”~ˆŒØ"(¤+ˆÔØÔ'ˆŒØ"Ô/ˆÔØÔ*¨QÑ.ˆŒØ!'Ô!9ˆÔØÔ*ˆŒØœ¨Ô(?Ñ?ˆŒå”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ>¨4¬<¸eÐDÑDÔDˆŒàÔ+ð 	kÝ+-¬<¸Ô8[Ð]aÔ]iÑ+jÔ+jˆDÔ(ð Ô+ð 	rÝ24´,¸tÔ?bÐdhÔdpÑ2qÔ2qˆDÔ/Ý'6°v´~È6ÔKdÐ'eÑ'eÔ'eˆÔ$Ð$Ð$r>   Tr  r	  c                 óP  — d}|rC|dz  }|| dk                          t          j        ¦  «        |z  z  }t          j        | ¦  «        } n(t          j        | t          j        | ¦  «        ¦  «         } |dz  }| |k     }|t          j        |                      ¦   «         |z  ¦  «        t          j        ||z  ¦  «        z  ||z
  z                        t          j        ¦  «        z   }t          j        |t          j	        ||dz
  ¦  «        ¦  «        }|t          j
        || |¦  «        z  }|S r  r  r  s           r<   r  z8LongT5TransientGlobalAttention._relative_position_bucketÝ  rN  r>   rO  c                 óæ  — | j         j        j        j        dk    r| j         j        j        nd}t	          j        d|z  t          j        |¬¦  «        }||| …         }|ddd…f         |dd…df         z
  }|                      || j         | j	        | j
        ¬¦  «        }|                       |¦  «        }|                     g d¢¦  «                             d¦  «                             d¦  «        }|S rQ  rS  rT  s           r<   r)  z+LongT5TransientGlobalAttention.compute_bias  rV  r>   r˜   r�   c                 ó~  — t          j        |d         |d d …d d d …f         ¦  «        d d …d df         }t          j        |dk    dd¦  «        }t          || j        ¦  «        }|                      || j         | j        | j        ¬¦  «        }|  	                    |¦  «        }| 
                    g d¢¦  «        }||z   }|S )Nr    .r   r    re  r   )r   r   r   r-   )r4   Úeqrƒ   r¥   r~   r  rö   rø   rù   rq  r"  )r¶   r˜   r�   Úside_attention_maskÚattention_side_biasr¤   Úside_relative_position_bucketÚ	side_biass           r<   Úcompute_side_biasz0LongT5TransientGlobalAttention.compute_side_bias&  sä   € å#œh t¨I¤Ð8JÈ1È1È1ÈdÐTUÐTUÐTUÈ:Ô8VÑWÔWÐXYÐXYÐXYÐ[_ÐadÐXdÔeÐÝ#œkÐ*=ÀÒ*AÀ3ÈÑNÔNÐå!AÀ$ÈÔH^Ñ!_Ô!_ÐØ(,×(FÒ(FØ"Ø#œÐ.ØÔ;ØÔ=ð	 )Gñ )
ô )
Ð%ð ×7Ò7Ð8UÑVÔVˆ	ð ×%Ò% l l lÑ3Ô3ˆ	à1°IÑ=ÐØ"Ð"r>   c                 óR	  ‡ ‡— |j         d d…         \  Š}ˆˆ fd„}ˆˆ fd„}t          |�|n t          j        |j         d d…         ¦  «        ‰ j        ¦  «        \  }}	|	j         d         }
t          |||
¦  «        }‰                      |¦  «        } |‰                      |¦  «        ¦  «        } |‰                      |¦  «        ¦  «        } |‰  	                    |¦  «        ¦  «        } |‰                      |¦  «        ¦  «        } |‰  	                    |¦  «        ¦  «        }t          |‰ j        d¬¦  «        }t          |‰ j        d¬¦  «        }t          |‰ j        d¬¦  «        }t          |dd¬¦  «        }t          |dd¬¦  «        }dg|j        dz   z  }|j         d         |d<   |                     d¦  «                             |¦  «        }|                     d¦  «                             |¦  «        }t          j        ||gd¬¦  «        }t          j        ||gd¬¦  «        }t          j        d||¦  «        }|�6t%          |‰ j        |j        ¦  «        }t          j        |d	k    d
d¦  «        }nd }|�€F‰ j        sNt          j        dd‰ j        ‰ j        d‰ j        z  f|j        |j        ¬¦  «        }‰ j        r‰ j        rd|_        n‰                      ‰ j        ¦  «        }|�||                     dd¦  «        z   }|                     |j        ¦  «        }|€t          j        ‰|¦  «        }‰                      ||	¦  «        }t          |‰ j        d¬¦  «                             dd¦  «        }|                     |j        ¦  «                              |j        ¦  «        }t          j        ||gd¬¦  «        }||z  }tB          j"         #                    | $                    ¦   «         d¬¦  «         %                    |¦  «        }tB          j"         &                    |‰ j&        ‰ j        ¬¦  «        }|                     |j        ¦  «        } |t          j        d||¦  «        ¦  «        }|d d …d |…d d …f         }‰  '                    |¦  «        }||f}|r||fz   }|S )Nr-   c                 óH   •— |                       ‰d‰j        ‰j        ¦  «        S rY  r[  r\  s    €€r<   rL   z5LongT5TransientGlobalAttention.forward.<locals>.shapeD  r^  r>   c                 ó`   •— |                       ¦   «                              ‰d‰j        ¦  «        S r`  ra  r\  s    €€r<   rb  z7LongT5TransientGlobalAttention.forward.<locals>.unshapeH  rc  r>   r+   r   r.   ry   rd  r   r    re  r   r+  Trz   r0   rf  )(rL   rž   r4   r”   r~   r¬   rr  r  r  r  r\   r@   rh   rP   rl   r“   rd   rª   r}   rW   rƒ   r÷   rO   rý   rE   r  r2   r6  r)  r6   r‚   r{  rt   r   r7   r8   r  rg  r'   r  )r¶   r¦   r˜   r(   rh  ri  rL   rb  r€   r�   Ú_global_seq_lenÚglobal_inputsr>  rA  rB  Úside_key_statesÚside_value_statesÚrepsrj  rq   Úside_position_biasr:   r;   rk  r–   s   `                       @r<   rÃ   z&LongT5TransientGlobalAttention.forward;  sù  øø€ ð "/Ô!4°R°a°RÔ!8Ñˆ
�Jð	Vð 	Vð 	Vð 	Vð 	Vð 	Vð	Lð 	Lð 	Lð 	Lð 	Lð 	Lõ )EØÐ$ˆDˆD­%¬*°]Ô5HÈÈ"ÈÔ5MÑ*NÔ*NØÔ"ñ)
ô )
Ñ%ˆ	Ð%ð
 -Ô2°2Ô6ˆÝ1°-ÀÈOÑ\Ô\ˆØ×4Ò4°]ÑCÔCˆð �u˜TŸVšV MÑ2Ô2Ñ3Ô3ˆØ�U˜4Ÿ6š6 -Ñ0Ô0Ñ1Ô1ˆ
Ø�u˜TŸVšV MÑ2Ô2Ñ3Ô3ˆà˜% §¢ }Ñ 5Ô 5Ñ6Ô6ˆØ!˜E $§&¢&¨Ñ"7Ô"7Ñ8Ô8Ðõ *¨,¸¼ÈAÐNÑNÔNˆÝ'¨
°D´NÈÐJÑJÔJˆ
Ý)¨,¸¼ÈAÐNÑNÔNˆõ +¨:ÀÐQRÐSÑSÔSˆ
Ý,¨\ÀQÐUVÐWÑWÔWˆð ˆs�oÔ*¨QÑ.Ñ/ˆØÔ" 1Ô%ˆˆQ‰Ø)×3Ò3°AÑ6Ô6×=Ò=¸dÑCÔCˆØ-×7Ò7¸Ñ:Ô:×AÒAÀ$ÑGÔGÐõ ”Y 
¨OÐ<À!ÐDÑDÔDˆ
Ý”y ,Ð0AÐ!BÈÐJÑJÔJˆõ ”Ð5°|ÀZÑPÔPˆàÐå#<¸TÀ4Ä>ÐS`ÔSgÑ#hÔ#hÐ å#(¤;Ð/CÀaÒ/GÈÈeÑ#TÔ#TÐ Ð à#'Ð àÑ àÔ3ð 	BÝ %¤Ø˜˜4œ<¨¬¸¸T¼^Ñ9KÐLØ!œ=Ø œ,ð!ñ !ô !�ð
 Ô.ð 7°4´=ð 7Ø26�MÔ/øà $× 1Ò 1°$´.Ñ AÔ A�à#Ð/à -Ð0D×0NÒ0NÈqÐRSÑ0TÔ0TÑ T�Ø)×.Ò.¨v¬|Ñ<Ô<ˆMð ˆ|Ý”z *¨jÑ9Ô9�à!%×!7Ò!7¸Ð>PÑ!QÔ!QÐå!3Ð4FÈÌÐ\^Ð!_Ñ!_Ô!_×!iÒ!iÐjkÐmnÑ!oÔ!oÐØ!3×!8Ò!8¸¼Ñ!FÔ!F×!IÒ!IÈ&Ì-Ñ!XÔ!XÐå!œI }Ð6HÐ&IÈrÐRÑRÔRˆMà�-Ñˆå”}×,Ò,¨V¯\ª\©^¬^ÀÐ,ÑDÔD×LÒLÈVÑTÔTˆÝ”}×,Ò,¨\¸T¼\ÐTXÔTaÐ,ÑbÔbˆà#×(Ò(¨Ô);Ñ<Ô<ˆØ�g�eœlÐ+BÀLÐR^Ñ_Ô_Ñ`Ô`ˆØ! ! ! ! [ j [°!°!°!Ð"3Ô4ˆØ—f’f˜[Ñ)Ô)ˆà Ð.ˆàð 	0Ø  Ñ/ˆGØˆr>   rl  rE  rm  )rÄ   rÅ   rÆ   r   rF  r²   rG  r  r•   r)  r4   rŒ   r{  rÃ   rÇ   rÈ   s   @r<   ro  ro  À  sö   ø€ € € € € ðfð f˜|ð fÈ$ð fÐ[_ð fð fð fð fð fð fð8 ð- ð - ð - ñ „\ð- ð^¨ð ð ð ð ð0# e¤lð #ÈÌð #ÐY^ÔYeð #ð #ð #ð #ð0 ØØðqð qð qð qð qð qð qð qr>   ro  c                   óL   ‡ — e Zd Zddedz  fˆ fd„Z	 	 	 ddee         fd„Zˆ xZS )	ÚLongT5LayerSelfAttentionFNró   c                 óþ   •— t          ¦   «                              ¦   «          t          ||||j        ¬¦  «        | _        t          |j        |j        ¬¦  «        | _        t          j
        |j        ¦  «        | _        d S )N©r÷   ró   rô   rê   )r±   r²   rò   rö   ÚSelfAttentionr®   rÑ   rí   rî   r   rÕ   rÖ   r'   ©r¶   rË   r÷   ró   r¹   s       €r<   r²   z!LongT5LayerSelfAttention.__init__±  sr   ø€ Ý‰Œ×ÒÑÔÐÝ,ØØ(CØØÔ'ð	
ñ 
ô 
ˆÔõ *¨&¬.¸fÔ>WÐXÑXÔXˆŒÝ”z &Ô"5Ñ6Ô6ˆŒˆˆr>   r)   c                 ó’   — |                       |¦  «        } | j        |f|||dœ|¤Ž\  }}}||                      |¦  «        z   }|||fS )N)r˜   r(   r:  )rî   r‰  r'   )	r¶   r¦   r%   r(   r:  r)   Únormed_hidden_statesÚattention_outputr:   s	            r<   rÃ   z LongT5LayerSelfAttention.forward¼  sy   € ð  $Ÿš¨}Ñ=Ô=ÐØ8J¸Ô8JØ ð9
àØ'Ø+ð	9
ð 9
ð
 ð9
ð 9
Ñ5Ð˜-¨ð )¨4¯<ª<Ð8HÑ+IÔ+IÑIÐØ °Ð<Ð<r>   ©FN©NNN©	rÄ   rÅ   rÆ   r•   r²   r   r   rÃ   rÇ   rÈ   s   @r<   r†  r†  °  s‚   ø€ € € € € ð	7ð 	7ÈSÐSWÉZð 	7ð 	7ð 	7ð 	7ð 	7ð 	7ð ØØð=ð =ð Ð+Ô,ð=ð =ð =ð =ð =ð =ð =ð =r>   r†  c                   óD   ‡ — e Zd ZdZddedz  fˆ fd„Z	 	 	 d	defd„Zˆ xZS )
ÚLongT5LayerLocalSelfAttentionz$Local self attention used in encoderFNró   c                 óð   •— t          ¦   «                              ¦   «          t          ||¬¦  «        | _        t	          |j        |j        ¬¦  «        | _        t          j	        |j
        ¦  «        | _        d S ©N)r÷   rê   )r±   r²   rI  ÚLocalSelfAttentionr®   rÑ   rí   rî   r   rÕ   rÖ   r'   rŠ  s       €r<   r²   z&LongT5LayerLocalSelfAttention.__init__Ó  s`   ø€ Ý‰Œ×ÒÑÔÐÝ"6°vÐ[vÐ"wÑ"wÔ"wˆÔÝ)¨&¬.¸fÔ>WÐXÑXÔXˆŒÝ”z &Ô"5Ñ6Ô6ˆŒˆˆr>   r)   c                 óº   — |                       |¦  «        }|                      ||||¬¦  «        }||                      |d         ¦  «        z   }|f|dd …         z   }|S ©N)r˜   r(   rh  r   r   )rî   r•  r'   ©	r¶   r¦   r%   r(   rh  r)   rŒ  r�  rk  s	            r<   rÃ   z%LongT5LayerLocalSelfAttention.forwardÙ  sw   € ð  $Ÿš¨}Ñ=Ô=ÐØ×2Ò2Ø ØØ'Ø/ð	 3ñ 
ô 
Ðð &¨¯ªÐ5EÀaÔ5HÑ(IÔ(IÑIˆØ Ð"Ð%5°a°b°bÔ%9Ñ9ˆØˆr>   rŽ  rm  ©	rÄ   rÅ   rÆ   Ú__doc__r•   r²   r   rÃ   rÇ   rÈ   s   @r<   r’  r’  Ð  s‚   ø€ € € € € Ø.Ð.ð7ð 7ÈSÐSWÉZð 7ð 7ð 7ð 7ð 7ð 7ð ØØðð ð ðð ð ð ð ð ð ð r>   r’  c                   óD   ‡ — e Zd ZdZddedz  fˆ fd„Z	 	 	 d	defd„Zˆ xZS )
Ú'LongT5LayerTransientGlobalSelfAttentionz/Transient-Global self attention used in encoderFNró   c                 óð   •— t          ¦   «                              ¦   «          t          ||¬¦  «        | _        t	          |j        |j        ¬¦  «        | _        t          j	        |j
        ¦  «        | _        d S r”  )r±   r²   ro  ÚTransientGlobalSelfAttentionr®   rÑ   rí   rî   r   rÕ   rÖ   r'   rŠ  s       €r<   r²   z0LongT5LayerTransientGlobalSelfAttention.__init__ð  si   ø€ Ý‰Œ×ÒÑÔÐÝ,JØÐ0Kð-
ñ -
ô -
ˆÔ)õ *¨&¬.¸fÔ>WÐXÑXÔXˆŒÝ”z &Ô"5Ñ6Ô6ˆŒˆˆr>   r)   c                 óº   — |                       |¦  «        }|                      ||||¬¦  «        }||                      |d         ¦  «        z   }|f|dd …         z   }|S r—  )rî   rž  r'   r˜  s	            r<   rÃ   z/LongT5LayerTransientGlobalSelfAttention.forwardø  sw   € ð  $Ÿš¨}Ñ=Ô=ÐØ×<Ò<Ø ØØ'Ø/ð	 =ñ 
ô 
Ðð &¨¯ªÐ5EÀaÔ5HÑ(IÔ(IÑIˆØ Ð"Ð%5°a°b°bÔ%9Ñ9ˆØˆr>   rŽ  rm  r™  rÈ   s   @r<   rœ  rœ  í  s‚   ø€ € € € € Ø9Ð9ð7ð 7ÈSÐSWÉZð 7ð 7ð 7ð 7ð 7ð 7ð ØØðð ð ðð ð ð ð ð ð ð r>   rœ  c                   óL   ‡ — e Zd Zddedz  fˆ fd„Z	 	 	 ddee         fd„Zˆ xZS )ÚLongT5LayerCrossAttentionNró   c                 óô   •— t          ¦   «                              ¦   «          t          |d|d¬¦  «        | _        t	          |j        |j        ¬¦  «        | _        t          j	        |j
        ¦  «        | _        d S )NFrˆ  rê   )r±   r²   rò   ÚEncDecAttentionr®   rÑ   rí   rî   r   rÕ   rÖ   r'   )r¶   rË   ró   r¹   s      €r<   r²   z"LongT5LayerCrossAttention.__init__  sm   ø€ Ý‰Œ×ÒÑÔÐÝ.Ø°ÀÐV[ð 
ñ  
ô  
ˆÔõ *¨&¬.¸fÔ>WÐXÑXÔXˆŒÝ”z &Ô"5Ñ6Ô6ˆŒˆˆr>   r)   c                 ó”   — |                       |¦  «        } | j        |f||||dœ|¤Ž\  }}}	||                      |¦  «        z   }
|
||	fS )N)r˜   r9  r(   r:  )rî   r£  r'   )r¶   r¦   r9  r%   r(   r:  r)   rŒ  r�  r:   Úlayer_outputs              r<   rÃ   z!LongT5LayerCrossAttention.forward  s{   € ð  $Ÿš¨}Ñ=Ô=ÐØ8L¸Ô8LØ ð9
àØ-Ø'Ø+ð9
ð 9
ð ð9
ð 9
Ñ5Ð˜-¨ð % t§|¢|Ð4DÑ'EÔ'EÑEˆØ˜]¨LÐ8Ð8r>   rÛ   r�  r�  rÈ   s   @r<   r¡  r¡    s�   ø€ € € € € ð7ð 7¨#°©*ð 7ð 7ð 7ð 7ð 7ð 7ð ØØð9ð 9ð Ð+Ô,ð9ð 9ð 9ð 9ð 9ð 9ð 9ð 9r>   r¡  c                   óF   ‡ — e Zd Zddedz  fˆ fd„Z	 	 	 	 	 	 	 	 	 dd„Zˆ xZS )	ÚLongT5BlockFNró   c                 ó$  •— t          ¦   «                              ¦   «          |j        | _        |j        rt          }n>|j        dk    rt
          }n+|j        dk    rt          }nt          d|j        › d�¦  «        ‚t          j	        ¦   «         | _
        | j
                              ||||¬¦  «        ¦  «         | j        r)| j
                             t          ||¬¦  «        ¦  «         | j
                             t          |¦  «        ¦  «         d S )NÚlocalztransient-globalzjFor encoder attention mechanism, either `local` or `transient-global` attention type is expected, but got ú.©r÷   ró   )ró   )r±   r²   rö   r†  Úencoder_attention_typer’  rœ  Ú
ValueErrorr   Ú
ModuleListÚlayerrc   r¡  rè   )r¶   rË   r÷   ró   Úattention_layerr¹   s        €r<   r²   zLongT5Block.__init__-  s   ø€ Ý‰Œ×ÒÑÔÐØ Ô+ˆŒØÔð 
	Ý6ˆOˆOØÔ*¨gÒ5Ð5Ý;ˆOˆOØÔ*Ð.@Ò@Ð@ÝEˆOˆOåð<Ø!Ô8ð<ð <ð <ñô ð õ ”]‘_”_ˆŒ
ØŒ
×ÒØˆO˜FÐ@[ÐgpÐqÑqÔqñ	
ô 	
ð 	
ð Œ?ð 	VØŒJ×ÒÕ7¸È)ÐTÑTÔTÑUÔUÐUàŒ
×Ò�-¨Ñ/Ô/Ñ0Ô0Ð0Ð0Ð0r>   Tc                 óÊ  —  | j         d         ||||||	¬¦  «        }|d         }|dd …         }|j        t          j        k    r_t          j        |¦  «                             ¦   «         r9t          j        |j        ¦  «        j        dz
  }t          j        || |¬¦  «        }| j	        o|d u}|r¥ | j         d         ||||||	¬¦  «        }|d         }|j        t          j        k    r_t          j        |¦  «                             ¦   «         r9t          j        |j        ¦  «        j        dz
  }t          j        || |¬¦  «        }||dd …         z   } | j         d         |¦  «        }|j        t          j        k    r_t          j        |¦  «                             ¦   «         r9t          j        |j        ¦  «        j        dz
  }t          j        || |¬¦  «        }|f|z   S )Nr   )r%   r(   r:  Ú	use_cacherh  r   iè  )r  r‘   )r9  r%   r(   r:  rh  r+   )
r¯  rE   r4   rÀ   ÚisinfÚanyÚfinfor‘   Úclamprö   )r¶   r¦   r%   r(   Úencoder_hidden_statesÚencoder_attention_maskÚencoder_decoder_position_biasr:  r²  rh  Úreturn_dictr)   Úself_attention_outputsÚattention_outputsÚclamp_valueÚdo_cross_attentionÚcross_attention_outputss                    r<   rÃ   zLongT5Block.forwardD  s  € ð "/ ¤¨A¤ØØ)Ø'Ø+ØØ/ð"
ñ "
ô "
Ðð /¨qÔ1ˆØ2°1°2°2Ô6Ðð Ô¥%¤-Ò/Ð/µE´KÀÑ4NÔ4N×4RÒ4RÑ4TÔ4TÐ/Ýœ+ mÔ&9Ñ:Ô:Ô>ÀÑEˆKÝ!œK¨¸K¸<È[ÐYÑYÔYˆMà!œ_ÐRÐ1FÈdÐ1RÐØð 	PØ&3 d¤j°¤mØØ!6Ø5Ø;Ø /Ø"3ð'ñ 'ô 'Ð#ð 4°AÔ6ˆMð Ô"¥e¤mÒ3Ð3½¼ÀMÑ8RÔ8R×8VÒ8VÑ8XÔ8XÐ3Ý#œk¨-Ô*=Ñ>Ô>ÔBÀTÑI�Ý %¤¨MÀ¸|ÐQ\Ð ]Ñ ]Ô ]�ð !2Ð4KÈAÈBÈBÔ4OÑ OÐð '˜œ
 2œ }Ñ5Ô5ˆð Ô¥%¤-Ò/Ð/µE´KÀÑ4NÔ4N×4RÒ4RÑ4TÔ4TÐ/Ýœ+ mÔ&9Ñ:Ô:Ô>ÀÑEˆKÝ!œK¨¸K¸<È[ÐYÑYÔYˆMð ÐÐ0Ñ0ð	
r>   rŽ  )	NNNNNNFFT)rÄ   rÅ   rÆ   r•   r²   rÃ   rÇ   rÈ   s   @r<   r§  r§  ,  s   ø€ € € € € ð1ð 1ÈSÐSWÉZð 1ð 1ð 1ð 1ð 1ð 1ð4 ØØ"Ø#Ø&*ØØØØð<
ð <
ð <
ð <
ð <
ð <
ð <
ð <
r>   r§  c                   ó‚   ‡ — e Zd ZU eed<   dZdZdgZdZe	d„ ¦   «         Z
 ej        ¦   «         ˆ fd„¦   «         Zd„ Zˆ xZS )	ÚLongT5PreTrainedModelrË   ÚtransformerTr§  Fc                 óv   — t          j        t          ¦  «        }t          j        t          ¦  «        }|||dœ}|S )N)Údecoder_input_idsÚ	input_idsÚdecoder_attention_mask)r4   r¨   r   r   )r¶   rÅ  Ú
input_maskÚdummy_inputss       r<   rÈ  z"LongT5PreTrainedModel.dummy_inputsŒ  s?   € õ ”L¥Ñ.Ô.ˆ	Ý”\¥*Ñ-Ô-ˆ
à!*Ø"Ø&0ð
ð 
ˆð
 Ðr>   c                 ób	  •— t          ¦   «                              |¦  «         | j        j        }t	          |t
          ¦  «        rt          j        |j        |dz  ¦  «         dS t	          |t          t          t          f¦  «        rjt          j        |j        j        d|dz  ¬¦  «         t          |d¦  «        r2| j        j        s(t          j        |j        j        d|dz  ¬¦  «         dS dS dS t	          |t"          ¦  «        ræt          j        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r*|j        j        �t          j        |j        j        ¦  «         t          j        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r,|j        j        �"t          j        |j        j        ¦  «         dS dS dS t	          |t0          ¦  «        �rVt          j        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r*|j        j        �t          j        |j        j        ¦  «         t          j        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r*|j        j        �t          j        |j        j        ¦  «         t          j        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r,|j        j        �"t          j        |j        j        ¦  «         dS dS dS t	          |t6          t8          t:          f¦  «        �r2| j        j        }| j        j        }| j        j        }t          j        |j         j        d|||z  dz  z  ¬¦  «         t          j        |j!        j        d||dz  z  ¬¦  «         t          j        |j"        j        d||dz  z  ¬¦  «         t          j        |j#        j        d|||z  dz  z  ¬¦  «         |j$        rgt          j        |j%        j        d||dz  z  ¬¦  «         t	          |t:          ¦  «        r-t          j        |j&        j        d||dz  z  ¬¦  «         dS dS dS dS )zInitialize the weightsr‹   r    )r¾   ÚstdÚlm_headr,   rÏ   N)'r±   Ú_init_weightsrË   Úinitializer_factorrÜ   r®   ÚinitÚ	constant_r´   ÚLongT5ModelÚLongT5ForConditionalGenerationÚLongT5EncoderModelÚnormal_ÚsharedÚhasattrÚtie_word_embeddingsrË  rÊ   rÓ   rÑ   rÏ   Úzeros_rÔ   rÒ   rà   râ   rã   rò   rI  ro  rú   rü   r  r  r  r  r÷   r  rq  )r¶   r!   ÚfactorrÑ   rû   rý   r¹   s         €r<   rÌ  z#LongT5PreTrainedModel._init_weights˜  s”  ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ø”Ô/ˆÝ�f�oÑ.Ô.ð $	ÝŒN˜6œ=¨&°3©,Ñ7Ô7Ð7Ð7Ð7Ý˜¥Õ.LÕN`Ð aÑbÔbð "	ÝŒL˜œÔ-°C¸VÀc¹\ÐJÑJÔJÐJÝ�v˜yÑ)Ô)ð P°$´+Ô2Qð PÝ”˜Vœ^Ô2¸À&È3Á,ÐOÑOÔOÐOÐOÐOðPð Pð Pð På˜Õ 3Ñ4Ô4ð 	ÝŒL˜œÔ)°¸ÀDÄKÔDWÐ\`ÑC`Ñ9aÐbÑbÔbÐbÝ�v”y &Ñ)Ô)ð ,¨f¬i¬nÐ.HÝ”˜FœIœNÑ+Ô+Ð+ÝŒL˜œÔ)°¸ÀDÄKÔDTÐY]ÑC]Ñ9^Ð_Ñ_Ô_Ð_Ý�v”y &Ñ)Ô)ð ,¨f¬i¬nÐ.HÝ”˜FœIœNÑ+Ô+Ð+Ð+Ð+ð,ð ,Ð.HÐ.Hå˜Õ 8Ñ9Ô9ñ 	ÝŒL˜œÔ+°#¸6ÀdÄkÔFYÐ^bÑEbÑ;cÐdÑdÔdÐdÝ�v”{ FÑ+Ô+ð .°´Ô0@Ð0LÝ”˜FœKÔ,Ñ-Ô-Ð-ÝŒL˜œÔ+°#¸6ÀdÄkÔFYÐ^bÑEbÑ;cÐdÑdÔdÐdÝ�v”{ FÑ+Ô+ð .°´Ô0@Ð0LÝ”˜FœKÔ,Ñ-Ô-Ð-ÝŒL˜œÔ)°¸ÀDÄKÔDTÐY]ÑC]Ñ9^Ð_Ñ_Ô_Ð_Ý�v”y &Ñ)Ô)ð ,¨f¬i¬nÐ.HÝ”˜FœIœNÑ+Ô+Ð+Ð+Ð+ð,ð ,Ð.HÐ.Hå˜¥Õ2FÕHfÐ gÑhÔhñ 	Ø”kÔ)ˆGØ!%¤Ô!1ÐØ”kÔ+ˆGÝŒL˜œœ¨s¸À7ÐM_ÑC_ÐdhÑBhÑ8iÐjÑjÔjÐjÝŒL˜œœ¨s¸À'È4Á-Ñ8PÐQÑQÔQÐQÝŒL˜œœ¨s¸À'È4Á-Ñ8PÐQÑQÔQÐQÝŒL˜œœ¨s¸À7ÐM_ÑC_ÐdhÑBhÑ8iÐjÑjÔjÐjØÔ1ð Ý”˜VÔ;ÔBÈÐRXÐ]dÐimÑ\mÑRnÐoÑoÔoÐoÝ˜fÕ&DÑEÔEð Ý”LØÔ=ÔDÈ3ÐTZÐ_fÐkoÑ^oÑTpðñ ô ð ð ð ð	ð 	ðð ðð r>   c                 ó6  — | j         j        }| j         j        }|€t          d¦  «        ‚|                     |j        ¦  «        }|dd d…f                              ¦   «         |ddd …f<   ||d<   |€t          d¦  «        ‚|                     |dk    |¦  «         |S )Nz’self.model.config.decoder_start_token_id has to be defined. In LongT5 it is usually set to the pad_token_id. See LongT5 docs for more information..r+   r   ).r   z1self.model.config.pad_token_id has to be defined.éœÿÿÿ)rË   Údecoder_start_token_idÚpad_token_idr­  Ú	new_zerosrL   r-  Úmasked_fill_)r¶   rÅ  rÛ  rÜ  Úshifted_input_idss        r<   Ú_shift_rightz"LongT5PreTrainedModel._shift_rightÄ  sº   € Ø!%¤Ô!CÐØ”{Ô/ˆà!Ð)Ýð8ñô ð ð
 &×/Ò/°	´Ñ@Ô@ÐØ%.¨s°C°R°C¨xÔ%8×%>Ò%>Ñ%@Ô%@Ð˜#˜q˜r˜r˜'Ñ"Ø$:Ð˜&Ñ!àÐÝÐPÑQÔQÐQà×&Ò&Ð'8¸DÒ'@À,ÑOÔOÐOà Ð r>   )rÄ   rÅ   rÆ   r   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_can_compile_fullgraphÚpropertyrÈ  r4   Úno_gradrÌ  rà  rÇ   rÈ   s   @r<   rÁ  rÁ  ƒ  s�   ø€ € € € € € àÐÐÑØ%ÐØ&*Ð#Ø&˜Ðà"Ðàðð ñ „Xðð €U„]�_„_ð(ð (ð (ð (ñ „_ð(ðV!ð !ð !ð !ð !ð !ð !r>   rÁ  c                   ó@   ‡ — e Zd Zˆ fd„Zd„ Z	 	 	 	 	 	 	 	 	 	 dd„Zˆ xZS )ÚLongT5Stackc                 ó  •‡— t          ¦   «                              ‰¦  «         t          j        ‰j        ‰j        ¦  «        | _        ‰j        | _        ‰j        | _        | j        dz   | _	        t          j
        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        t          j        ‰j        ¦  «        | _        d| _        |                      ¦   «          d S )Nr   c           	      óV   •— g | ]%}t          ‰t          |d k    ¦  «        |¬¦  «        ‘Œ&S )r   r«  )r§  rF  )Ú.0rf   rË   s     €r<   ú
<listcomp>z(LongT5Stack.__init__.<locals>.<listcomp>å  sC   ø€ ð ð ð àõ ˜FÅÀQÈ!ÂVÁÄÐXYÐZÑZÔZðð ð r>   rê   F)r±   r²   r   r  Ú
vocab_sizerÑ   Úembed_tokensrö   rK  r@   r®  r`   Ú
num_layersÚblockr®   rí   Úfinal_layer_normrÕ   rÖ   r'   r  Ú	post_initrÙ   s    `€r<   r²   zLongT5Stack.__init__Û  sí   øø€ Ý‰Œ×Ò˜Ñ Ô Ð åœL¨Ô):¸F¼NÑKÔKˆÔØ Ô+ˆŒà"Ô/ˆÔØÔ*¨QÑ.ˆŒå”]ðð ð ð å˜vÔ0Ñ1Ô1ðñ ô ñ
ô 
ˆŒ
õ !0°´ÀFÔD]Ð ^Ñ ^Ô ^ˆÔÝ”z &Ô"5Ñ6Ô6ˆŒà&+ˆÔ#ð 	�ŠÑÔÐÐÐr>   c                 ó   — || _         d S rÛ   )rï  ©r¶   Únew_embeddingss     r<   Úset_input_embeddingsz LongT5Stack.set_input_embeddingsó  s   € Ø*ˆÔÐÐr>   Nc                 óÂ  — |�|n| j         j        }|�|n| j         j        }|	�|	n| j         j        }	|
�|
n| j         j        }
|�#|�!| j        rdnd}t          d|› d|› d�¦  «        ‚|�1|                     ¦   «         }|                     d|d         ¦  «        }n@|�|                     ¦   «         d d…         }n!| j        rdnd}t          d|› d|› d	�¦  «        ‚| j	        r%| j
        r|rt                               d
¦  «         d}|€&| j        €
J d¦   «         ‚|                      |¦  «        }|\  }}| j        r]|rZ|€X| j         j        r7t          t!          | j         ¬¦  «        t!          | j         ¬¦  «        ¦  «        }nt!          | j         ¬¦  «        }n	| j        sd }|�|                     ¦   «         nd}|€/t%          ¦   «         s!||z   }t'          j        |||j        ¬¦  «        }| j        rt-          | j         |||¬¦  «        }n.| j         j        dk    rt1          || j        |j        ¦  «        }n|}|�t5          | j         |||¬¦  «        }|	rdnd }|rdnd }|r	| j        rdnd }d }d }|                      |¦  «        }t9          | j        ¦  «        D ]d\  }}|	r||fz   } |||||||||||
¬¦
  «
        }|d         }|d         }| j        r
|�|d         }|r||d         fz   }| j        r||d         fz   }Œe|                      |¦  «        }|                      |¦  «        }|	r||fz   }|
st?          d„ |||||fD ¦   «         ¦  «        S tA          |||||¬¦  «        S )NÚdecoder_Ú zYou cannot specify both zinput_ids and zinputs_embeds at the same timer+   zYou have to specify either zinput_ids or Úinputs_embedszZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fz<You have to initialize the model with valid token embeddings)rË   r   r‰   )rË   rû  r%   r:  r©  )rË   rû  r%   r·  rG   )r:  r²  rh  rº  r   r   r-   é   c              3   ó   K  — | ]}|®|V — Œ	d S rÛ   rG   )rì  r  s     r<   ú	<genexpr>z&LongT5Stack.forward.<locals>.<genexpr>�  s4   è è € ð 
ð 
àð �=ð ð !�=�=�=ð
ð 
r>   )Úlast_hidden_stater:  r¦   Ú
attentionsÚcross_attentions)!rË   r²  rh  Úoutput_hidden_statesrº  rö   r­  r3   r.  r  r2   rÿ   r   rï  Úis_encoder_decoderr   r   r,  r   r4   r”   rW   r   r¬  r}   r@   r   r'   Ú	enumeraterñ  rò  rb   r   )r¶   rÅ  r%   r·  r¸  rû  r:  r²  rh  r  rº  r)   Úerr_msg_prefixr;  r–   ri  Úpast_key_values_lengthÚmask_seq_lengthÚcausal_maskÚall_hidden_statesÚall_attentionsÚall_cross_attentionsr(   r¹  r¦   rf   Úlayer_moduleÚlayer_outputss                               r<   rÃ   zLongT5Stack.forwardö  sË  € ð "+Ð!6�I�I¸D¼KÔ<Qˆ	Ø1BÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆàÐ  ]Ð%>Ø+/¬?ÐB˜Z˜ZÀˆNÝØw¨>ÐwÐwÈÐwÐwÐwñô ð ð Ð"Ø#Ÿ.š.Ñ*Ô*ˆKØ!Ÿš r¨;°r¬?Ñ;Ô;ˆIˆIØÐ&Ø'×,Ò,Ñ.Ô.¨s°¨sÔ3ˆKˆKà+/¬?ÐB˜Z˜ZÀˆNÝÐu¸>ÐuÐuÐXfÐuÐuÐuÑvÔvÐvàÔ&ð 	"¨4¬=ð 	"Øð "Ý×#Ò#Øpñô ð ð "�	àÐ ØÔ$Ð0Ð0Ð2pÑ0Ô0Ð0Ø ×-Ò-¨iÑ8Ô8ˆMà!,Ñˆ
�JàŒ?ð 	#Øð G˜_Ð4Ø”;Ô1ð GÝ&9Ý$¨D¬KÐ8Ñ8Ô8½,ÈdÌkÐ:ZÑ:ZÔ:Zñ'ô '�O�Oõ '3¸$¼+Ð&FÑ&FÔ&F�OøØ”ð 	#ð #ˆOàETÐE` ×!?Ò!?Ñ!AÔ!AÐ!AÐfgÐØÐ!Õ*BÑ*DÔ*DÐ!à4°zÑAˆOÝ"œZ¨
°OÈMÔL`ÐaÑaÔaˆNàŒ?ð 	)Ý,Ø”{Ø+Ø-Ø /ð	ñ ô ˆKˆKð Œ[Ô/°7Ò:Ð:Ý3°NÀDÄNÐTaÔThÑiÔiˆKˆKà(ˆKà!Ð-Ý%>Ø”{Ø+Ø5Ø&;ð	&ñ &ô &Ð"ð #7Ð@˜B˜B¸DÐØ0Ð:˜˜°dˆØ&7ÐV¸D¼OÐV˜r˜rÐRVÐØˆØ(,Ð%àŸš ]Ñ3Ô3ˆå(¨¬Ñ4Ô4ð !	Vð !	V‰OˆAˆ|Ø#ð IØ$5¸Ð8HÑ$HÐ!à(˜LØØØØ%Ø&Ø-Ø /Ø#Ø"3Ø'ðñ ô ˆMð  *¨!Ô,ˆMð *¨!Ô,ˆMØŒð AÐ#8Ð#DØ0=¸aÔ0@Ð-à ð VØ!/°=ÀÔ3CÐ2EÑ!E�Ø”?ð VØ+?À=ÐQRÔCSÐBUÑ+UÐ(øà×-Ò-¨mÑ<Ô<ˆØŸš ]Ñ3Ô3ˆð  ð 	EØ 1°]Ð4DÑ DÐàð 	Ýð 
ð 
ð "Ø#Ø%Ø"Ø(ðð
ñ 
ô 
ñ 
ô 
ð 
õ 9Ø+Ø+Ø+Ø%Ø1ð
ñ 
ô 
ð 	
r>   )
NNNNNNNNNN)rÄ   rÅ   rÆ   r²   r÷  rÃ   rÇ   rÈ   s   @r<   ré  ré  Ú  sƒ   ø€ € € € € ðð ð ð ð ð0+ð +ð +ð
 ØØ"Ø#ØØØØØ!Øð\
ð \
ð \
ð \
ð \
ð \
ð \
ð \
r>   ré  c                   óh  ‡ — e Zd ZdgZdddœZdefˆ fd„Zd„ Zd„ Ze		 	 	 	 	 	 	 	 	 	 	 	 dd	e
j        dz  d
e
j        dz  de
j        dz  de
j        dz  deee
j                          dz  dedz  de
j        dz  de
j        dz  dedz  dedz  dedz  dedz  dee
j                 ez  fd„¦   «         Zˆ xZS )rÐ  úFdecoder.block.0.layer.1.EncDecAttention.relative_attention_bias.weightúshared.weight)úencoder.embed_tokens.weightúdecoder.embed_tokens.weightrË   c                 óœ  •— t          ¦   «                              |¦  «         t          j        |j        |j        ¦  «        | _        t          j        |¦  «        }d|_	        d|_
        t          |¦  «        | _        t          j        |¦  «        }d|_	        |j        |_        t          |¦  «        | _        |                      ¦   «          d S )NFT)r±   r²   r   r  rî  rÑ   rÔ  ÚcopyÚdeepcopyrö   r²  ré  ÚencoderÚnum_decoder_layersrð  Údecoderró  ©r¶   rË   Úencoder_configÚdecoder_configr¹   s       €r<   r²   zLongT5Model.__init__Ÿ  sª   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý”l 6Ô#4°f´nÑEÔEˆŒåœ vÑ.Ô.ˆØ$)ˆÔ!Ø#(ˆÔ Ý" >Ñ2Ô2ˆŒåœ vÑ.Ô.ˆØ$(ˆÔ!Ø$*Ô$=ˆÔ!Ý" >Ñ2Ô2ˆŒð 	�ŠÑÔÐÐÐr>   c                 ó   — | j         S rÛ   ©rÔ  ©r¶   s    r<   Úget_input_embeddingsz LongT5Model.get_input_embeddings°  ó
   € ØŒ{Ðr>   c                 ó|   — || _         | j                             |¦  «         | j                             |¦  «         d S rÛ   ©rÔ  r  r÷  r  rõ  s     r<   r÷  z LongT5Model.set_input_embeddings³  ó;   € Ø$ˆŒØŒ×)Ò)¨.Ñ9Ô9Ð9ØŒ×)Ò)¨.Ñ9Ô9Ð9Ð9Ð9r>   NrÅ  r%   rÄ  rÆ  Úencoder_outputsr:  rû  Údecoder_inputs_embedsr²  rh  r  rº  rB   c                 ó"  — |	�|	n| j         j        }	|�|n| j         j        }|€|                      ||||
||¬¦  «        }ne|rct	          |t
          ¦  «        sNt          |d         t          |¦  «        dk    r|d         ndt          |¦  «        dk    r|d         nd¬¦  «        }|d         }|                      |||||||	|
||¬¦
  «
        }|s||z   S t          |j	        |j
        |j        |j        |j        |j	        |j        |j        ¬¦  «        S )	aµ	  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. LongT5 is a model with relative position embeddings so
            you should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            [What are input IDs?](../glossary#input-ids)

            To know more on how to prepare `input_ids` for pretraining take a look a [LONGT5
            Training](./longt5#training).
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            LONGT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If
            `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
            `past_key_values`).

            To know more on how to prepare `decoder_input_ids` for pretraining take a look at [LONGT5
            Training](./longt5#training).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, LongT5Model

        >>> tokenizer = AutoTokenizer.from_pretrained("google/long-t5-local-base")
        >>> model = LongT5Model.from_pretrained("google/long-t5-local-base")

        >>> # Let's try a very long encoder input.
        >>> input_ids = tokenizer(
        ...     100 * "Studies have been shown that owning a dog is good for you", return_tensors="pt"
        ... ).input_ids  # Batch size 1

        >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="pt").input_ids  # Batch size 1

        >>> # forward pass
        >>> outputs = model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)
        >>> last_hidden_states = outputs.last_hidden_state
        ```N©rÅ  r%   rû  rh  r  rº  r   r   r-   ©rÿ  r¦   r   ©
rÅ  r%   rû  r:  r·  r¸  r²  rh  r  rº  )rÿ  r:  Údecoder_hidden_statesÚdecoder_attentionsr  Úencoder_last_hidden_stater·  Úencoder_attentions)rË   r²  rº  r  rÜ   r   Úlenr  r   rÿ  r:  r¦   r   r  )r¶   rÅ  r%   rÄ  rÆ  r$  r:  rû  r%  r²  rh  r  rº  r)   r¦   Údecoder_outputss                   r<   rÃ   zLongT5Model.forward¸  su  € ðD "+Ð!6�I�I¸D¼KÔ<Qˆ	Ø%0Ð%<�k�kÀ$Ä+ÔBYˆð Ð"Ø"ŸlšlØ#Ø-Ø+Ø"3Ø%9Ø'ð +ñ ô ˆOˆOð ð 	¥¨O½_Ñ!MÔ!Mð 	Ý-Ø"1°!Ô"4Ý47¸Ñ4HÔ4HÈ1Ò4LÐ4L˜o¨aÔ0Ð0ÐRVÝ14°_Ñ1EÔ1EÈÒ1IÐ1I˜?¨1Ô-Ð-Ètðñ ô ˆOð (¨Ô*ˆð Ÿ,š,Ø'Ø1Ø/Ø+Ø"/Ø#1ØØ/Ø!5Ø#ð 'ñ 
ô 
ˆð ð 	5Ø" _Ñ4Ð4å!Ø-Ô?Ø+Ô;Ø"1Ô"?Ø.Ô9Ø,Ô=Ø&5Ô&GØ"1Ô"?Ø.Ô9ð	
ñ 	
ô 	
ð 		
r>   )NNNNNNNNNNNN)rÄ   rÅ   rÆ   Ú"_keys_to_ignore_on_load_unexpectedÚ_tied_weights_keysr   r²   r  r÷  r   r4   Ú
LongTensorÚFloatTensorÚ
BoolTensorrb   r
   rŒ   rF  r   rÃ   rÇ   rÈ   s   @r<   rÐ  rÐ  •  sÈ  ø€ € € € € ð 	Rð*Ð&ð (7Ø'6ðð Ðð
˜|ð ð ð ð ð ð ð"ð ð ð:ð :ð :ð
 ð .2Ø37Ø59Ø:>ØBFØ(,Ø-1Ø59Ø!%Ø)-Ø,0Ø#'ðq
ð q
àÔ# dÑ*ðq
ð Ô)¨DÑ0ðq
ð !Ô+¨dÑ2ð	q
ð
 !&Ô 0°4Ñ 7ðq
ð ˜u UÔ%6Ô7Ô8¸4Ñ?ðq
ð  ™ðq
ð ”| dÑ*ðq
ð  %œ|¨dÑ2ðq
ð ˜$‘;ðq
ð   $™;ðq
ð # T™kðq
ð ˜D‘[ðq
ð 
ˆuÔ Ô	!Ð$6Ñ	6ðq
ð q
ð q
ñ „^ðq
ð q
ð q
ð q
ð q
r>   rÐ  z>
    LONGT5 Model with a `language modeling` head on top.
    )Úcustom_introc                   ó–  ‡ — e Zd ZdgZddddœZdefˆ fd„Zd„ Zd„ Ze		 	 	 	 	 	 	 	 	 	 	 	 	 dd	e
j        dz  d
e
j        dz  de
j        dz  de
j        dz  deee
j                          dz  dedz  de
j        dz  de
j        dz  de
j        dz  dedz  dedz  dedz  dedz  dee
j                 ez  fd„¦   «         Zde
j        fd„Zˆ xZS )rÑ  r  r  )r  r  zlm_head.weightrË   c                 ó   •— t          ¦   «                              |¦  «         |j        | _        t	          j        |j        |j        ¦  «        | _        t          j	        |¦  «        }d|_
        d|_        t          |¦  «        | _        t          j	        |¦  «        }d|_
        |j        |_        t          |¦  «        | _        t	          j        |j        |j        d¬¦  «        | _        |                      ¦   «          d S )NFTrÎ   )r±   r²   rÑ   Ú	model_dimr   r  rî  rÔ  r  r  rö   r²  ré  r  r  rð  r  rÐ   rË  ró  r  s       €r<   r²   z'LongT5ForConditionalGeneration.__init__<  sÐ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØœˆŒå”l 6Ô#4°f´nÑEÔEˆŒåœ vÑ.Ô.ˆØ$)ˆÔ!Ø#(ˆÔ Ý" >Ñ2Ô2ˆŒåœ vÑ.Ô.ˆØ$(ˆÔ!Ø$*Ô$=ˆÔ!Ý" >Ñ2Ô2ˆŒå”y ¤°Ô1BÈÐOÑOÔOˆŒð 	�ŠÑÔÐÐÐr>   c                 ó   — | j         S rÛ   r  r  s    r<   r  z3LongT5ForConditionalGeneration.get_input_embeddingsQ  r   r>   c                 ó|   — || _         | j                             |¦  «         | j                             |¦  «         d S rÛ   r"  rõ  s     r<   r÷  z3LongT5ForConditionalGeneration.set_input_embeddingsT  r#  r>   NrÅ  r%   rÄ  rÆ  r$  r:  rû  r%  Úlabelsr²  rh  r  rº  rB   c                 óÐ  — |
�|
n| j         j        }
|�|n| j         j        }|€|                      ||||||¬¦  «        }ne|rct	          |t
          ¦  «        sNt          |d         t          |¦  «        dk    r|d         ndt          |¦  «        dk    r|d         nd¬¦  «        }|d         }|	�|€|€|                      |	¦  «        }|                      |||||||
|||¬¦
  «
        }|d         }| j         j	        r|| j
        dz  z  }|                      |¦  «        }d}|	�pt          d	¬
¦  «        }|	                     |j        ¦  «        }	 ||                     d|                     d¦  «        ¦  «        |	                     d¦  «        ¦  «        }|s|f|dd…         z   |z   }|�|f|z   n|S t#          |||j        |j        |j        |j        |j        |j        |j        ¬¦	  «	        S )a7  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. LongT5 is a model with relative position embeddings so
            you should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            [What are input IDs?](../glossary#input-ids)

            To know more on how to prepare `input_ids` for pretraining take a look a [LONGT5
            Training](./longt5#training).
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            LONGT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If
            `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
            `past_key_values`).

            To know more on how to prepare `decoder_input_ids` for pretraining take a look at [LONGT5
            Training](./longt5#training).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ...,
            config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for
            labels in `[0, ..., config.vocab_size]`

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, LongT5ForConditionalGeneration

        >>> tokenizer = AutoTokenizer.from_pretrained("Stancld/longt5-tglobal-large-16384-pubmed-3k_steps")
        >>> model = LongT5ForConditionalGeneration.from_pretrained(
        ...     "Stancld/longt5-tglobal-large-16384-pubmed-3k_steps"
        ... )

        >>> # Let's try a very long input.
        >>> inputs = tokenizer(100 * "studies have shown that owning a dog is good for you ", return_tensors="pt")
        >>> input_ids = inputs.input_ids

        >>> outputs = model.generate(input_ids)
        >>> print(tokenizer.decode(outputs[0], skip_special_tokens=True))
        abstractthe aim of this article is to provide an overview of the literature on the role of dog
        ```Nr'  r   r   r-   r(  r)  r,   rÚ  )Úignore_indexr+   )	ÚlossÚlogitsr:  r*  r+  r  r,  r·  r-  )rË   r²  rº  r  rÜ   r   r.  rà  r  Úscale_decoder_outputsr8  rË  r   rt   rW   r.  r3   r   r:  r¦   r   r  rÿ  )r¶   rÅ  r%   rÄ  rÆ  r$  r:  rû  r%  r;  r²  rh  r  rº  r)   r¦   r/  Úsequence_outputÚ	lm_logitsr>  Úloss_fctÚoutputs                         r<   rÃ   z&LongT5ForConditionalGeneration.forwardY  sa  € ðL "+Ð!6�I�I¸D¼KÔ<Qˆ	Ø%0Ð%<�k�kÀ$Ä+ÔBYˆð Ð"à"ŸlšlØ#Ø-Ø+Ø"3Ø%9Ø'ð +ñ ô ˆOˆOð ð 	¥¨O½_Ñ!MÔ!Mð 	Ý-Ø"1°!Ô"4Ý47¸Ñ4HÔ4HÈ1Ò4LÐ4L˜o¨aÔ0Ð0ÐRVÝ14°_Ñ1EÔ1EÈÒ1IÐ1I˜?¨1Ô-Ð-Ètðñ ô ˆOð (¨Ô*ˆàÐÐ"3Ð";Ð@UÐ@]à $× 1Ò 1°&Ñ 9Ô 9Ðð Ÿ,š,Ø'Ø1Ø/Ø+Ø"/Ø#1ØØ/Ø!5Ø#ð 'ñ 
ô 
ˆð *¨!Ô,ˆàŒ;Ô,ð 	GØ-°´ÀÑ1EÑFˆOà—L’L Ñ1Ô1ˆ	àˆØÐÝ'°TÐ:Ñ:Ô:ˆHà—Y’Y˜yÔ/Ñ0Ô0ˆFØ�8˜IŸNšN¨2¨y¯~ª~¸bÑ/AÔ/AÑBÔBÀFÇKÂKÐPRÁOÄOÑTÔTˆDàð 	FØ�\ O°A°B°BÔ$7Ñ7¸/ÑIˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEåØØØ+Ô;Ø"1Ô"?Ø.Ô9Ø,Ô=Ø&5Ô&GØ"1Ô"?Ø.Ô9ð

ñ 

ô 

ð 
	
r>   c                 ó,   — |                       |¦  «        S rÛ   )rà  )r¶   r;  s     r<   Ú%prepare_decoder_input_ids_from_labelszDLongT5ForConditionalGeneration.prepare_decoder_input_ids_from_labelsæ  s   € Ø× Ò  Ñ(Ô(Ð(r>   )NNNNNNNNNNNNN)rÄ   rÅ   rÆ   r0  r1  r   r²   r  r÷  r   r4   r2  r3  r4  rb   rŒ   r
   rF  r   rÃ   rF  rÇ   rÈ   s   @r<   rÑ  rÑ  -  sû  ø€ € € € € ð 	Rð*Ð&ð (7Ø'6Ø)ðð Ðð˜|ð ð ð ð ð ð ð*ð ð ð:ð :ð :ð
 ð .2Ø37Ø59Ø:>Ø=AØ(,Ø26Ø:>Ø*.Ø!%Ø)-Ø,0Ø#'ðJ
ð J
àÔ# dÑ*ðJ
ð Ô)¨DÑ0ðJ
ð !Ô+¨dÑ2ð	J
ð
 !&Ô 0°4Ñ 7ðJ
ð ˜u U¤\Ô2Ô3°dÑ:ðJ
ð  ™ðJ
ð Ô(¨4Ñ/ðJ
ð  %Ô0°4Ñ7ðJ
ð Ô  4Ñ'ðJ
ð ˜$‘;ðJ
ð   $™;ðJ
ð # T™kðJ
ð ˜D‘[ðJ
ð  
ˆuÔ Ô	! OÑ	3ð!J
ð J
ð J
ñ „^ðJ
ðX)¸E¼Lð )ð )ð )ð )ð )ð )ð )ð )r>   rÑ  c                   óÞ   ‡ — e Zd ZddiZdgZdefˆ fd„Zd„ Zd„ Ze		 	 	 	 	 	 dd	e
j        dz  d
e
j        dz  de
j        dz  dedz  dedz  dedz  dee
j                 ez  fd„¦   «         Zˆ xZS )rÒ  r  r  r  rË   c                 ó  •— t          ¦   «                              |¦  «         t          j        |j        |j        ¦  «        | _        t          j        |¦  «        }d|_	        t          |¦  «        | _        |                      ¦   «          d S )NF)r±   r²   r   r  rî  rÑ   rÔ  r  r  r²  ré  r  ró  )r¶   rË   r  r¹   s      €r<   r²   zLongT5EncoderModel.__init__ñ  sq   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý”l 6Ô#4°f´nÑEÔEˆŒåœ vÑ.Ô.ˆØ#(ˆÔ Ý" >Ñ2Ô2ˆŒð 	�ŠÑÔÐÐÐr>   c                 ó   — | j         S rÛ   r  r  s    r<   r  z'LongT5EncoderModel.get_input_embeddingsü  r   r>   c                 óH   — || _         | j                             |¦  «         d S rÛ   )rÔ  r  r÷  rõ  s     r<   r÷  z'LongT5EncoderModel.set_input_embeddingsÿ  s%   € Ø$ˆŒØŒ×)Ò)¨.Ñ9Ô9Ð9Ð9Ð9r>   NrÅ  r%   rû  rh  r  rº  rB   c                 ó\   — |�|n| j         j        }|                      ||||||¬¦  «        }|S )aŠ  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. LongT5 is a model with relative position embeddings so
            you should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            To know more on how to prepare `input_ids` for pretraining take a look a [LONGT5
            Training](./longt5#training).

        Example:

        ```python
        >>> from transformers import AutoTokenizer, LongT5ForConditionalGeneration

        >>> tokenizer = AutoTokenizer.from_pretrained("google/long-t5-local-base")
        >>> model = LongT5EncoderModel.from_pretrained("google/long-t5-local-base")
        >>> input_ids = tokenizer(
        ...     100 * "Studies have been shown that owning a dog is good for you ", return_tensors="pt"
        ... ).input_ids  # Batch size 1
        >>> outputs = model(input_ids=input_ids)
        >>> last_hidden_states = outputs.last_hidden_state
        ```Nr'  )rË   rº  r  )	r¶   rÅ  r%   rû  rh  r  rº  r)   r$  s	            r<   rÃ   zLongT5EncoderModel.forward  sJ   € ðF &1Ð%<�k�kÀ$Ä+ÔBYˆàŸ,š,ØØ)Ø'Ø/Ø!5Ø#ð 'ñ 
ô 
ˆð Ðr>   )NNNNNN)rÄ   rÅ   rÆ   r1  r0  r   r²   r  r÷  r   r4   r2  r3  rF  rb   r   rÃ   rÇ   rÈ   s   @r<   rÒ  rÒ  ê  s)  ø€ € € € € ð 	& ðÐð +5¨Ð&ð	˜|ð 	ð 	ð 	ð 	ð 	ð 	ðð ð ð:ð :ð :ð ð .2Ø37Ø26Ø)-Ø,0Ø#'ð-ð -àÔ# dÑ*ð-ð Ô)¨DÑ0ð-ð Ô(¨4Ñ/ð	-ð
   $™;ð-ð # T™kð-ð ˜D‘[ð-ð 
ˆuÔ Ô	! OÑ	3ð-ð -ð -ñ „^ð-ð -ð -ð -ð -r>   rÒ  )rÒ  rÑ  rÐ  rÁ  )Nr    N)r   )Srš  r  r  Úcollections.abcr   Útypingr   r4   r   Útorch.nnr   rú  r   rÎ  Úactivationsr	   Úcache_utilsr
   r   r   Ú
generationr   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   r   Úconfiguration_longt5r   Ú
get_loggerrÄ   rÿ   ÚModulerŒ   r  r=   r•   rT   r\   rh   rp   rw   rW   r}   rb   rž   r¥   r¬   r®   rÊ   rà   rè   rò   rI  ro  r†  r’  rœ  r¡  r§  rÁ  ré  rÐ  rÑ  rÒ  Ú__all__rG   r>   r<   ú<module>r\     s  ðð Ð à €€€Ø €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð Ø %Ð %Ð %Ð %Ð %Ð %à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CØ )Ð )Ð )Ð )Ð )Ð )Ø JÐ JÐ JÐ JÐ JÐ JÐ JÐ JØ 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð GÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð /Ð .Ð .Ð .Ð .Ð .ð 
ˆÔ	˜HÑ	%Ô	%€ð !ØØ)-ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð �T‰\ð%ð ð%ð ”< $Ñ&ð%ð Ð'Ô(ð%ð %ð %ð %ðFð ˜œð °ð ¸3ð È3ð ÐW\ÔWcð ð ð ð ð #˜%œ,ð #°3ð #¸Sð #ÀUÄ\ð #ð #ð #ð #ð4ð 4˜Uœ\ð 4°cð 4Èð 4ÐY\ð 4ÐejÔeqð 4ð 4ð 4ð 4ð2!°#ð !¸%¼,ð !ð !ð !ð !ðB°U´\ð BÈcð BÐV[ÔVbð Bð Bð Bð Bð8¨e¬lð 8Àsð 8ÐTYÔT`ð 8ÐejÔeqð 8ð 8ð 8ð 8ð .PØ”Lð.PØ58ð.Pà
ˆ5Œ<˜œÐ%Ô&ð.Pð .Pð .Pð .Pðb4°U´\ð 4ÐVYð 4Ð^cÔ^jð 4ð 4ð 4ð 4ð	jØ”<ð	jØ,1¬Lð	jØJMð	jà
„\ð	jð 	jð 	jð 	jð+ð +ð +ð +ð +�b”iñ +ô +ð +ð4ð ð ð ð ˜"œ)ñ ô ð ð,ð ð ð ð ˜rœyñ ô ð ð&ð ð ð ð �B”Iñ ô ð ð&A8ð A8ð A8ð A8ð A8�b”iñ A8ô A8ð A8ðHið ið ið ið i˜2œ9ñ iô ið iðXlð lð lð lð l R¤Yñ lô lð lð`=ð =ð =ð =ð =˜rœyñ =ô =ð =ð@ð ð ð ð  B¤Iñ ô ð ð:ð ð ð ð ¨b¬iñ ô ð ð@9ð 9ð 9ð 9ð 9 ¤	ñ 9ô 9ð 9ð>T
ð T
ð T
ð T
ð T
Ð,ñ T
ô T
ð T
ðn ðS!ð S!ð S!ð S!ð S!˜Oñ S!ô S!ñ „ðS!ðlx
ð x
ð x
ð x
ð x
Ð'ñ x
ô x
ð x
ðv ðT
ð T
ð T
ð T
ð T
Ð'ñ T
ô T
ñ „ðT
ðn €ððñ ô ð
u)ð u)ð u)ð u)ð u)Ð%:¸Oñ u)ô u)ñô ð
u)ðp ðFð Fð Fð Fð FÐ.ñ Fô Fñ „ðFðR kÐ
jÐ
j€€€r>   