§
    9²¸jq% ã                   ó®  — d Z ddlZddlZddlZddlZddlmZmZ ddlm	Z	m
Z
 ddlmZ ddlmZ ddlmZ ddlZddlmZmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZ ddlmZ ddl m!Z!m"Z"m#Z# ddl$m%Z% ddl&m'Z'm(Z( ddl)m*Z* ddl+m,Z,m-Z- ddl.m/Z/ ddl0m1Z1m2Z2m3Z3m4Z4 ddl5m6Z6m7Z7 ddl8m9Z9m:Z:  ej;        e<¦  «        Z=	 	 	 d\dej>        dej        dej        dej        dej        dz  de?dz  de?d ej        dz  d!e/e2         fd"„Z@ e3d#¬$¦  «        e G d%„ d&e1¦  «        ¦   «         ¦   «         ZAd]d)„ZBd^d*„ZC	 	 	 	 	 d_d,„ZD G d-„ d.ej>        ¦  «        ZE G d/„ d0ej>        ¦  «        ZF G d1„ d2ej>        ¦  «        ZG G d3„ d4ej>        ¦  «        ZH G d5„ d6ej>        ¦  «        ZI G d7„ d8ej>        ¦  «        ZJ G d9„ d:ej>        ¦  «        ZK G d;„ d<ej>        ¦  «        ZL G d=„ d>e*¦  «        ZMe3 G d?„ d@e-¦  «        ¦   «         ZN G dA„ dBej>        ¦  «        ZOeJjP        ZQdCZR G dD„ dEej>        e¦  «        ZS G dF„ dGeS¦  «        ZT G dH„ dIeS¦  «        ZU G dJ„ dKeS¦  «        ZV G dL„ dMej>        ¦  «        ZWeTeUeVdNœZXdOedPe
eS         fdQ„ZY G dR„ dSeN¦  «        ZZe3 G dT„ dUeN¦  «        ¦   «         Z[ e3dV¬$¦  «         G dW„ dXeNe%¦  «        ¦   «         Z\e3 G dY„ dZeN¦  «        ¦   «         Z]g d[¢Z^dS )`zPyTorch UDOP model.é    N)ÚABCÚabstractmethod)ÚCallableÚSequence)Údeepcopy)Ú	dataclass)ÚAny)ÚTensorÚnn)ÚCrossEntropyLoss)Ú
UdopConfig)ÚSeq2SeqLMOutputÚSeq2SeqModelOutputé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚModelOutputÚTransformersKwargsÚauto_docstringÚis_torchdynamo_compiling)Úcan_return_tupleÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚposition_biasÚkwargsc                 ó¼  — |€|                      d¦  «        dz  }t          j        ||                     dd¦  «        ¦  «        |z  }	|�|	|z   }	|�|	|z   }	t          j                             |	d¬¦  «        }	t          j                             |	|| j        ¬¦  «        }	t          j        |	|¦  «        }
|
                     dd¦  «         	                    ¦   «         }
|
|	fS )Néÿÿÿÿç      à¿é   r   ©Údim)ÚpÚtrainingé   )
ÚsizeÚtorchÚmatmulÚ	transposer   Ú
functionalÚsoftmaxr,   r6   Ú
contiguous)r&   r'   r(   r)   r*   r+   r,   r-   r.   Úattn_weightsÚattn_outputs              úv/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/udop/modeling_udop.pyÚeager_attention_forwardrB   :   sØ   € ð €Ø—*’*˜R‘.”. DÑ(ˆõ ”<  s§}¢}°Q¸Ñ':Ô':Ñ;Ô;¸gÑE€LàÐ Ø# mÑ3ˆàÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2Ð(Ñ>Ô>€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$ó    z�
    Class for the model's outputs that may also contain a past key/values (to speed up sequential decoding). Includes
    an additional attention mask.
    )Úcustom_introc                   óâ   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dS )	Ú BaseModelOutputWithAttentionMaska 
  
    last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
        Sequence of hidden-states at the output of the last layer of the model. If `past_key_values` is used only
        the last hidden-state of the sequences of shape `(batch_size, 1, hidden_size)` is output.
    attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
        Attention mask used in the model's forward pass to avoid performing attention on padding token indices.
        Mask values selected in `[0, 1]`:
        - 1 for tokens that are **not masked**,
        - 0 for tokens that are **masked**.
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the
        self-attention blocks and optionally if `config.is_encoder_decoder=True` in the cross-attention blocks)
        that can be used (see `past_key_values` input) to speed up sequential decoding.
    hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
        one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of
        the model at the output of each layer plus the optional initial embedding outputs.
    attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
        the self-attention heads.
    cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` and `config.add_cross_attention=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`. Attentions weights of the decoder's cross-attention layer, after the attention softmax,
        used to compute the weighted average in the cross-attention heads.
    NÚlast_hidden_stater*   Úpast_key_valuesÚhidden_statesÚ
attentionsÚcross_attentions)Ú__name__Ú
__module__Ú__qualname__Ú__doc__rG   r9   ÚFloatTensorÚ__annotations__r*   rH   r   rI   ÚtuplerJ   rK   © rC   rA   rF   rF   Z   s»   € € € € € € ðð ð: 37Ð�uÔ(¨4Ñ/Ð6Ð6Ñ6Ø/3€N�EÔ%¨Ñ,Ð3Ð3Ñ3Ø$(€O�U˜T‘\Ð(Ð(Ñ(Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ø8<Ð�e˜EÔ-Ô.°Ñ5Ð<Ð<Ñ<Ð<Ð<rC   rF   éà   é   c           	      óŽ  — | |z  | |z  g}t          j        dd|d         dz   z  d¦  «        }||d         z  }t          j        dd|d         dz   z  d¦  «        }||d         z  }t          j        |d d…                              |d         d¦  «        |d d…                              |d         d¦  «                             dd¦  «        |dd …                              |d         d¦  «        |dd …                              |d         d¦  «                             dd¦  «        gd¬¦  «        }|                     dd¦  «        }|S )Nr   ç      ð?r7   r0   r3   é   )r9   ÚarangeÚstackÚrepeatr;   Úview)Ú
image_sizeÚ
patch_sizeÚimage_feature_pool_shapeÚvisual_bbox_xÚvisual_bbox_yÚvisual_bbox_inputs         rA   Úget_visual_bboxrc   ‡   sg  € Ø *¨jÑ 8¸*È
Ñ:RÐSÐÝ”L  CÐ+CÀAÔ+FÈÑ+JÑ$KÈSÑQÔQ€MØÐ-¨aÔ0Ñ0€Må”L  CÐ+CÀAÔ+FÈÑ+JÑ$KÈSÑQÔQ€MØÐ-¨aÔ0Ñ0€Måœà˜#˜2˜#Ô×%Ò%Ð&>¸qÔ&AÀ1ÑEÔEØ˜#˜2˜#Ô×%Ò%Ð&>¸qÔ&AÀ1ÑEÔE×OÒOÐPQÐSTÑUÔUØ˜!˜"˜"Ô×$Ò$Ð%=¸aÔ%@À!ÑDÔDØ˜!˜"˜"Ô×$Ò$Ð%=¸aÔ%@À!ÑDÔD×NÒNÈqÐRSÑTÔTð		
ð ðñ ô Ðð *×.Ò.¨r°1Ñ5Ô5ÐàÐrC   c                 óH  — t          | t          j        ¦  «        r| j        d         }n#t	          | ¦  «        }t          j        | ¦  «        } ||z
  }|dk    rCt          j        |g|z  ¦  «                             | ¦  «        }t          j        | |gd¬¦  «        } | d |…         S )Nr   r3   )	Ú
isinstancer9   r
   ÚshapeÚlenÚtensorrZ   ÚtoÚcat)ÚseqÚ
target_lenÚ	pad_valueÚnÚmÚrets         rA   Úpad_sequencerq   ž   s™   € Ý�#•u”|Ñ$Ô$ð  ØŒI�aŒLˆˆå�‰HŒHˆÝŒl˜3ÑÔˆØ�Q‰€AØˆ1‚u€uÝŒk˜9˜+¨™/Ñ*Ô*×-Ò-¨cÑ2Ô2ˆÝŒi˜˜c˜
¨Ð*Ñ*Ô*ˆØˆ{�
ˆ{ÔÐrC   é   c	                 óB  ‡ ‡‡‡‡‡— |}	t          j        t          j        ‰dd…dd…df         ‰dd…dd…df         z   dz  |	z  ¦  «                             ¦   «         d|	dz
  ¦  «        }
t          j        t          j        ‰dd…dd…df         ‰dd…dd…df         z   dz  |	z  ¦  «                             ¦   «         d|	dz
  ¦  «        |	z  }|
|z   }‰                     t           j        ¦  «        Š‰                     d¦  «        dk    ‰                     d¦  «        d	k    z  }t          j        ‰ d|                     d¦  «         	                    dd‰  
                    d¦  «        ¦  «        ¦  «        }d||<   ||z  }t          j        ‰ dd…dd…df         d
¦  «                             ¦   «         Št          j        t          j        t          |¦  «        ¦  «        dd…df          	                    d| 
                    d¦  «        ¦  «        dd…dd…df                              |¦  «        |dd…dd…df         gd¬¦  «        }|                     dd¦  «        }t#          |Ž \  }}d‰||f<   ˆ ˆfd„t%          t          ‰¦  «        ¦  «        D ¦   «         }‰€ht'          ||¬¦  «        Š‰                     d¦  «         	                    ‰  
                    d¦  «        dd¦  «        Š‰                     ‰ j        ¦  «        Šˆˆfd„t%          t          ‰¦  «        ¦  «        D ¦   «         Š‰�ˆfd„‰D ¦   «         }‰dk    r‰  
                    d¦  «        Šn‰| 
                    d¦  «        z
  Št          j        ˆ ˆfd„|D ¦   «         ¦  «        }t          j        ˆˆfd„‰D ¦   «         ¦  «        Š‰�!t          j        ˆˆfd„|D ¦   «         ¦  «        }t          j        ||gd¦  «        }t          j        ‰‰gd¦  «        Š‰�t          j        ‰|gd¦  «        Š|‰‰fS )a»  
    Combine the image and text embeddings for the input to the encoder/decoder of UDOP.

    First, the image embeddings are created by checking for each visual patch if it is inside the bounding box of a
    token. If it is, the visual patch is combined with the token embedding. Then, the visual bounding boxes are combined
    with the text bounding boxes. Finally, the visual bounding boxes are combined with the text attention mask.
    Nr   r2   g       @r7   r   r0   r%   rW   Tr3   Fc                 ó8   •— g | ]}‰|         ‰|                  ‘ŒS rS   rS   )Ú.0ÚiÚimage_embeddingsÚ
patch_indss     €€rA   ú
<listcomp>z1combine_image_text_embeddings.<locals>.<listcomp>Ü   s(   ø€ Ð_Ð_Ð_À1Ð,¨QÔ/°
¸1´Ô>Ð_Ð_Ð_rC   )r]   r^   c                 ó8   •— g | ]}‰|         ‰|                  ‘ŒS rS   rS   )ru   rv   rx   Úvisual_bboxs     €€rA   ry   z1combine_image_text_embeddings.<locals>.<listcomp>ã   s&   ø€ ÐQÐQÐQ°Q�;˜q”> *¨Q¤-Ô0ÐQÐQÐQrC   c                 óx   •— g | ]6}t          j        |                     d ¦  «        ‰j        ‰j        ¬¦  «        ‘Œ7S )r   ©ÚdtypeÚdevice)r9   Úonesr8   r~   r   )ru   Úitemr*   s     €rA   ry   z1combine_image_text_embeddings.<locals>.<listcomp>æ   sG   ø€ ð !
ð !
ð !
Øcg�EŒJ�t—y’y ‘|”|¨>Ô+?ÈÔH]Ð^Ñ^Ô^ð!
ð !
ð !
rC   c           
      ób   •— g | ]+}t          |‰t          j        ‰d          ¦  «        ¦  «        ‘Œ,S ©)r   r   ©rq   r9   Ú
zeros_like)ru   r�   rw   Úmax_lens     €€rA   ry   z1combine_image_text_embeddings.<locals>.<listcomp>ï   s7   ø€ ÐpÐpÐpÐSW��d˜G¥UÔ%5Ð6FÀtÔ6LÑ%MÔ%MÑ	NÔ	NÐpÐpÐprC   c           
      ób   •— g | ]+}t          |‰t          j        ‰d          ¦  «        ¦  «        ‘Œ,S rƒ   r„   )ru   r�   Úbboxr†   s     €€rA   ry   z1combine_image_text_embeddings.<locals>.<listcomp>ñ   s6   ø€ ÐqÐqÐqÐ]a�|¨D°'½5Ô;KÈDÐQUÌJÑ;WÔ;WÑXÔXÐqÐqÐqrC   c           
      ób   •— g | ]+}t          |‰t          j        ‰d          ¦  «        ¦  «        ‘Œ,S rƒ   r„   )ru   r�   r*   r†   s     €€rA   ry   z1combine_image_text_embeddings.<locals>.<listcomp>ô   s6   ø€ ÐsÐsÐsÐUY�\˜$ ­Ô)9¸.ÈÔ:NÑ)OÔ)OÑPÔPÐsÐsÐsrC   )r9   ÚclipÚfloorÚlongri   Úfloat64ÚmeanÚgatherÚ	unsqueezer[   r8   Ú	full_likeÚboolrj   rY   rg   ÚflattenÚzipÚrangerc   r   rZ   )rw   Úinputs_embedsrˆ   r{   r*   Únum_patchesr†   r]   r^   Úsequence_lengthÚocr_points_xÚocr_points_yÚ
ocr_pointsÚ
target_segÚrepeated_vision_embedsÚindÚrowsÚcolsÚinput_vision_patchesÚvisual_attention_maskÚinputs_vision_patchesrx   s   ` ``` `              @rA   Úcombine_image_text_embeddingsr¤   «   s‘  øøøøøø€ ð& "€OÝ”:ÝŒ�T˜!˜!˜!˜Q˜Q˜Q ˜'”] T¨!¨!¨!¨Q¨Q¨Q°¨'¤]Ñ2°cÑ9¸OÑKÑLÔL×QÒQÑSÔSÐUVÐXgÐjkÑXkñô €Lõ 	Œ
•5”;  Q Q Q¨¨¨¨1 W¤°°Q°Q°Q¸¸¸¸1°W´Ñ =ÀÑDÀÑVÑWÔW×\Ò\Ñ^Ô^Ð`aÐcrÐuvÑcvÑwÔwØ
ñ	ð ð  Ñ,€Jà�7Š7•5”=Ñ!Ô!€DØ—)’)˜B‘-”- 3Ò&¨4¯9ª9°R©=¬=¸CÒ+?Ñ@€JÝ"œ\Ø˜!˜Z×1Ò1°"Ñ5Ô5×<Ò<¸QÀÐCS×CXÒCXÐY[ÑC\ÔC\Ñ]Ô]ñô Ðð *-Ð˜:Ñ&ØÐ+Ñ+€Må”Ð!1°!°!°!°Q°Q°Q¸°'Ô!:¸DÑAÔA×FÒFÑHÔH€JÝ
Œ)åŒL�˜Z™œÑ)Ô)¨!¨!¨!¨T¨'Ô2×9Ò9¸!¸Z¿_º_ÈRÑ=PÔ=PÑQÔQÐRSÐRSÐRSÐUVÐUVÐUVÐX\ÐR\Ô]×`Ò`ÐakÑlÔlØ�q�q�q˜!˜!˜!˜T�zÔ"ð	
ð ðñ ô €Cð �+Š+�a˜Ñ
Ô
€CÝ�c��J€Dˆ$Ø"€Jˆt�TˆzÑà_Ð_Ð_Ð_Ð_ÍÍcÐR\ÉoÌoÑH^ÔH^Ð_Ñ_Ô_ÐàÐÝ%°È
ÐSÑSÔSˆØ!×+Ò+¨AÑ.Ô.×5Ò5Ð6F×6KÒ6KÈAÑ6NÔ6NÐPQÐSTÑUÔUˆØ!—n’nÐ%5Ô%<Ñ=Ô=ˆàQÐQÐQÐQÐQ½%ÅÀJÁÄÑ:PÔ:PÐQÑQÔQ€KàÐ!ð!
ð !
ð !
ð !
Økvð!
ñ !
ô !
Ðð �!‚|€|Ø"×'Ò'¨Ñ*Ô*ˆˆà˜M×.Ò.¨qÑ1Ô1Ñ1ˆÝ!œKØpÐpÐpÐpÐpÐ[oÐpÑpÔpñô Ðõ ”+ÐqÐqÐqÐqÐqÐepÐqÑqÔqÑrÔr€KØÐ!Ý %¤ØsÐsÐsÐsÐsÐ]rÐsÑsÔsñ!
ô !
Ðõ ”I˜}Ð.CÐDÀaÑHÔH€MÝŒ9�d˜KÐ(¨!Ñ,Ô,€DØÐ!Ýœ NÐ4IÐ#JÈAÑNÔNˆØ˜$ Ð.Ð.rC   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚUdopPatchEmbeddingsz2D Image to Patch Embeddingsc                 óÌ  •— t          ¦   «                              ¦   «          |j        |j        }}|j        |j        }}t          |t          j        j	        ¦  «        r|n||f}t          |t          j        j	        ¦  «        r|n||f}|d         |d         z  |d         |d         z  z  }|| _        || _        || _        || _
        t          j        ||||¬¦  «        | _        d S )Nr7   r   )Úkernel_sizeÚstride)ÚsuperÚ__init__r]   r^   Únum_channelsÚhidden_sizere   ÚcollectionsÚabcÚIterabler—   r   ÚConv2dÚproj)ÚselfÚconfigr]   r^   r¬   r­   r—   Ú	__class__s          €rA   r«   zUdopPatchEmbeddings.__init__  sá   ø€ Ý‰Œ×ÒÑÔÐØ!'Ô!2°FÔ4E�Jˆ
Ø$*Ô$7¸Ô9K�kˆå#-¨j½+¼/Ô:RÑ#SÔ#SÐq�Z�ZÐZdÐfpÐYqˆ
Ý#-¨j½+¼/Ô:RÑ#SÔ#SÐq�Z�ZÐZdÐfpÐYqˆ
Ø! !”}¨
°1¬Ñ5¸*ÀQ¼-È:ÐVWÌ=Ñ:XÑYˆØ$ˆŒØ$ˆŒØ(ˆÔØ&ˆÔå”I˜l¨KÀZÐXbÐcÑcÔcˆŒ	ˆ	ˆ	rC   c                 óB  — |j         \  }}}}|| j        d         k    s|| j        d         k    r2t          d|› d|› d| j        d         › d| j        d         › d�	¦  «        ‚|                      |¦  «        }|                     d¦  «                             dd¦  «        }|S )Nr   r7   zInput image size (Ú*z) doesn't match model (z).r2   )rf   r]   Ú
ValueErrorr²   r“   r;   )r³   Úpixel_valuesÚ
batch_sizer¬   ÚheightÚwidthÚ
embeddingss          rA   ÚforwardzUdopPatchEmbeddings.forward  s¼   € Ø2>Ô2DÑ/ˆ
�L &¨%Ø�T”_ QÔ'Ò'Ð'¨5°D´OÀAÔ4FÒ+FÐ+FÝØw VÐwÐw¨eÐwÐwÈDÌOÐ\]ÔL^ÐwÐwÐaeÔapÐqrÔasÐwÐwÐwñô ð ð —Y’Y˜|Ñ,Ô,ˆ
Ø×'Ò'¨Ñ*Ô*×4Ò4°Q¸Ñ:Ô:ˆ
ØÐrC   )rL   rM   rN   rO   r«   r¾   Ú__classcell__©rµ   s   @rA   r¦   r¦   þ   sR   ø€ € € € € Ø&Ð&ðdð dð dð dð dðð ð ð ð ð ð rC   r¦   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚUdopLayerNormç�íµ ÷Æ°>c                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )ze
        Construct a layernorm module in the Udop style. No bias and no subtraction of mean.
        N)rª   r«   r   Ú	Parameterr9   r€   ÚweightÚvariance_epsilon)r³   r­   Úepsrµ   s      €rA   r«   zUdopLayerNorm.__init__  sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐrC   c                 óh  — |                      t          j        ¦  «                             d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        j        t          j	        t          j
        fv r|                      | j        j        ¦  «        }| j        |z  S )Nr2   r0   T)Úkeepdim)ri   r9   Úfloat32ÚpowrŽ   ÚrsqrtrÇ   rÆ   r~   Úfloat16Úbfloat16)r³   rI   Úvariances      rA   r¾   zUdopLayerNorm.forward%  s–   € ð !×#Ò#¥E¤MÑ2Ô2×6Ò6°qÑ9Ô9×>Ò>¸rÈ4Ð>ÑPÔPˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆð Œ;Ô¥¤µ´Ð ?Ð?Ð?Ø)×,Ò,¨T¬[Ô->Ñ?Ô?ˆMàŒ{˜]Ñ*Ð*rC   )rÃ   ©rL   rM   rN   r«   r¾   r¿   rÀ   s   @rA   rÂ   rÂ     sL   ø€ € € € € ð$ð $ð $ð $ð $ð $ð+ð +ð +ð +ð +ð +ð +rC   rÂ   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚUdopDenseActDenser´   c                 óJ  •— t          ¦   «                              ¦   «          t          j        |j        |j        d¬¦  «        | _        t          j        |j        |j        d¬¦  «        | _        t          j        |j	        ¦  «        | _
        t          |j                 | _        d S ©NF©Úbias)rª   r«   r   ÚLinearÚd_modelÚd_ffÚwiÚwoÚDropoutÚdropout_rater,   r   Údense_act_fnÚact©r³   r´   rµ   s     €rA   r«   zUdopDenseActDense.__init__7  sx   ø€ Ý‰Œ×ÒÑÔÐÝ”)˜FœN¨F¬K¸eÐDÑDÔDˆŒÝ”)˜FœK¨¬¸eÐDÑDÔDˆŒÝ”z &Ô"5Ñ6Ô6ˆŒÝ˜&Ô-Ô.ˆŒˆˆrC   c                 ó°  — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }t          | j        j        t          j        ¦  «        r]|j        | j        j        j        k    rC| j        j        j        t          j	        k    r$| 
                    | j        j        j        ¦  «        }|                      |¦  «        }|S ©N)rÛ   rà   r,   re   rÜ   rÆ   r9   r
   r~   Úint8ri   )r³   rI   s     rA   r¾   zUdopDenseActDense.forward>  s¨   € ØŸš Ñ.Ô.ˆØŸš Ñ/Ô/ˆØŸš ]Ñ3Ô3ˆå�t”w”~¥u¤|Ñ4Ô4ð	CàÔ# t¤w¤~Ô';Ò;Ð;Ø””Ô$­¬
Ò2Ð2à)×,Ò,¨T¬W¬^Ô-AÑBÔBˆMØŸš Ñ.Ô.ˆØÐrC   ©rL   rM   rN   r   r«   r¾   r¿   rÀ   s   @rA   rÓ   rÓ   6  sS   ø€ € € € € ð/˜zð /ð /ð /ð /ð /ð /ðð ð ð ð ð ð rC   rÓ   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚUdopDenseGatedActDenser´   c                 ó–  •— t          ¦   «                              ¦   «          t          j        |j        |j        d¬¦  «        | _        t          j        |j        |j        d¬¦  «        | _        t          j        |j        |j        d¬¦  «        | _        t          j	        |j
        ¦  «        | _        t          |j                 | _        d S rÕ   )rª   r«   r   rØ   rÙ   rÚ   Úwi_0Úwi_1rÜ   rÝ   rÞ   r,   r   rß   rà   rá   s     €rA   r«   zUdopDenseGatedActDense.__init__N  s”   ø€ Ý‰Œ×ÒÑÔÐÝ”I˜fœn¨f¬kÀÐFÑFÔFˆŒ	Ý”I˜fœn¨f¬kÀÐFÑFÔFˆŒ	Ý”)˜FœK¨¬¸eÐDÑDÔDˆŒÝ”z &Ô"5Ñ6Ô6ˆŒÝ˜&Ô-Ô.ˆŒˆˆrC   c                 óà  — |                       |                      |¦  «        ¦  «        }|                      |¦  «        }||z  }|                      |¦  «        }t	          | j        j        t          j        ¦  «        r]|j	        | j        j        j	        k    rC| j        j        j	        t          j
        k    r$|                     | j        j        j	        ¦  «        }|                      |¦  «        }|S rã   )rà   ré   rê   r,   re   rÜ   rÆ   r9   r
   r~   rä   ri   )r³   rI   Úhidden_geluÚhidden_linears       rA   r¾   zUdopDenseGatedActDense.forwardV  sÀ   € Ø—h’h˜tŸyšy¨Ñ7Ô7Ñ8Ô8ˆØŸ	š	 -Ñ0Ô0ˆØ# mÑ3ˆØŸš ]Ñ3Ô3ˆõ �t”w”~¥u¤|Ñ4Ô4ð	CàÔ# t¤w¤~Ô';Ò;Ð;Ø””Ô$­¬
Ò2Ð2à)×,Ò,¨T¬W¬^Ô-AÑBÔBˆMàŸš Ñ.Ô.ˆØÐrC   rå   rÀ   s   @rA   rç   rç   M  sS   ø€ € € € € ð/˜zð /ð /ð /ð /ð /ð /ðð ð ð ð ð ð rC   rç   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚUdopLayerFFr´   c                 ó$  •— t          ¦   «                              ¦   «          |j        rt          |¦  «        | _        nt          |¦  «        | _        t          |j        |j        ¬¦  «        | _	        t          j        |j        ¦  «        | _        d S )N©rÈ   )rª   r«   Úis_gated_actrç   ÚDenseReluDenserÓ   rÂ   rÙ   Úlayer_norm_epsilonÚ
layer_normr   rÝ   rÞ   r,   rá   s     €rA   r«   zUdopLayerFF.__init__l  sx   ø€ Ý‰Œ×ÒÑÔÐØÔð 	<Ý"8¸Ñ"@Ô"@ˆDÔÐå"3°FÑ";Ô";ˆDÔå'¨¬¸FÔ<UÐVÑVÔVˆŒÝ”z &Ô"5Ñ6Ô6ˆŒˆˆrC   c                 óŠ   — |                       |¦  «        }|                      |¦  «        }||                      |¦  «        z   }|S rã   )rõ   ró   r,   )r³   rI   Úforwarded_statess      rA   r¾   zUdopLayerFF.forwardv  sF   € ØŸ?š?¨=Ñ9Ô9ÐØ×.Ò.Ð/?Ñ@Ô@ÐØ%¨¯ªÐ5EÑ(FÔ(FÑFˆØÐrC   rå   rÀ   s   @rA   rï   rï   k  sS   ø€ € € € € ð7˜zð 7ð 7ð 7ð 7ð 7ð 7ðð ð ð ð ð ð rC   rï   c                   ó|   ‡ — e Zd Z	 	 	 ddededz  defˆ fd„Zedd
„¦   «         Zdd„Z		 	 	 	 dde
e         fd„Zˆ xZS )ÚUdopAttentionFNr´   Ú	layer_idxÚ	is_causalc                 óT  •— t          ¦   «                              ¦   «          || _        |j        | _        || _        || _        |j        | _        |j        | _        |j        | _        |j	        | _
        |j        | _        |j        | _        | j        | j
        z  | _        d| _        || _        |€/| j        r(t$                               d| j        j        › d�¦  «         t-          j        | j        | j        d¬¦  «        | _        t-          j        | j        | j        d¬¦  «        | _        t-          j        | j        | j        d¬¦  «        | _        t-          j        | j        | j        d¬¦  «        | _        | j        r$t-          j        | j        | j        ¦  «        | _        d| _        d S )NrW   zInstantiating a decoder z³ without passing `layer_idx` is not recommended and will to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` when creating this class.FrÖ   )rª   r«   r´   Ú
is_decoderrû   Úhas_relative_attention_biasÚrelative_attention_num_bucketsÚrelative_attention_max_distancerÙ   Úd_kvÚkey_value_proj_dimÚ	num_headsÚn_headsrÞ   r,   Ú	inner_dimr+   rú   ÚloggerÚwarning_oncerµ   rL   r   rØ   ÚqÚkÚvÚoÚ	EmbeddingÚrelative_attention_biasÚgradient_checkpointing)r³   r´   rþ   rú   rû   rµ   s        €rA   r«   zUdopAttention.__init__  s~  ø€ õ 	‰Œ×ÒÑÔÐØˆŒØ Ô+ˆŒØ"ˆŒØ+FˆÔ(Ø.4Ô.SˆÔ+Ø/5Ô/UˆÔ,Ø”~ˆŒØ"(¤+ˆÔØÔ'ˆŒØÔ*ˆŒØœ¨Ô(?Ñ?ˆŒàˆŒØ"ˆŒØÐ ¤ÐÝ×Òð,¨4¬>Ô+Bð ,ð ,ð ,ñô ð õ ”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ>¨4¬<¸eÐDÑDÔDˆŒàÔ+ð 	kÝ+-¬<¸Ô8[Ð]aÔ]iÑ+jÔ+jˆDÔ(à&+ˆÔ#Ð#Ð#rC   Té    é€   c                 óP  — d}|rC|dz  }|| dk                          t          j        ¦  «        |z  z  }t          j        | ¦  «        } n(t          j        | t          j        | ¦  «        ¦  «         } |dz  }| |k     }|t          j        |                      ¦   «         |z  ¦  «        t          j        ||z  ¦  «        z  ||z
  z                        t          j        ¦  «        z   }t          j        |t          j	        ||dz
  ¦  «        ¦  «        }|t          j
        || |¦  «        z  }|S )aÒ  
        Adapted from Mesh Tensorflow:
        https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593

        Translate relative position to a bucket number for relative attention. The relative position is defined as
        memory_position - query_position, i.e. the distance in tokens from the attending position to the attended-to
        position. If bidirectional=False, then positive relative positions are invalid. We use smaller buckets for
        small absolute relative_position and larger buckets for larger absolute relative_positions. All relative
        positions >=max_distance map to the same bucket. All relative positions <=-max_distance map to the same bucket.
        This should allow for more graceful generalization to longer sequences than the model has been trained on

        Args:
            relative_position: an int32 Tensor
            bidirectional: a boolean - whether the attention is bidirectional
            num_buckets: an integer
            max_distance: an integer

        Returns:
            a Tensor with the same shape as relative_position, containing int32 values in the range [0, num_buckets)
        r   r2   r7   )ri   r9   rŒ   ÚabsÚminr…   ÚlogÚfloatÚmathr‘   Úwhere)Úrelative_positionÚbidirectionalÚnum_bucketsÚmax_distanceÚrelative_bucketsÚ	max_exactÚis_smallÚrelative_position_if_larges           rA   Ú_relative_position_bucketz'UdopAttention._relative_position_bucket¦  s>  € ð, ÐØð 	cØ˜AÑˆKØÐ!2°QÒ!6× :Ò :½5¼:Ñ FÔ FÈÑ TÑTÐÝ %¤	Ð*;Ñ <Ô <ÐÐå!&¤Ð+<½eÔ>NÐO`Ñ>aÔ>aÑ!bÔ!bÐ bÐð   1Ñ$ˆ	Ø$ yÒ0ˆð &/ÝŒIÐ'×-Ò-Ñ/Ô/°)Ñ;Ñ<Ô<ÝŒh�| iÑ/Ñ0Ô0ñ1à˜YÑ&ñ(÷ Š"�UŒZ‰.Œ.ñ	&Ð"õ
 &+¤YØ&­¬Ð8RÐT_ÐbcÑTcÑ(dÔ(dñ&
ô &
Ð"ð 	�EœK¨Ð2CÐE_Ñ`Ô`Ñ`ÐØÐrC   r   c                 ó¸  — |€| j         j        j        }t          j        |t          j        |¬¦  «        dd…df         |z   }t          j        |t          j        |¬¦  «        ddd…f         }||z
  }|                      || j         | j        | j	        ¬¦  «        }|                       |¦  «        }	|	 
                    g d¢¦  «                             d¦  «        }	|	S )z%Compute binned relative position biasNr}   ©r  r  r  )r2   r   r7   r   )r  rÆ   r   r9   rY   rŒ   r   rý   rÿ   r   Úpermuter�   )
r³   Úquery_lengthÚ
key_lengthr   Úpast_seen_tokensÚcontext_positionÚmemory_positionr  Úrelative_position_bucketÚvaluess
             rA   Úcompute_biaszUdopAttention.compute_biasÖ  sí   € àˆ>ØÔ1Ô8Ô?ˆFÝ œ<¨½E¼JÈvÐVÑVÔVÐWXÐWXÐWXÐZ^ÐW^Ô_ÐbrÑrÐÝœ, z½¼ÈFÐSÑSÔSÐTXÐZ[ÐZ[ÐZ[ÐT[Ô\ˆØ+Ð.>Ñ>ÐØ#'×#AÒ#AØØ#œÐ.ØÔ;ØÔ=ð	 $Bñ $
ô $
Ð ð ×-Ò-Ð.FÑGÔGˆØ—’ 	 	 	Ñ*Ô*×4Ò4°QÑ7Ô7ˆØˆrC   r.   c                 ó"  — |j         dd…         }g |¢d‘| j        ‘R }|�|                     | j        ¦  «        nd}	t	          |	t
          j        ¦  «        r|	                     ¦   «         n|	}	|du}
|                      |¦  «         	                    |¦  «         
                    dd¦  «        }d}t	          |t          ¦  «        r1|j                             | j        ¦  «        }|
r|j        }n
|j        }n|}|
r|n|}|
r3|�1|r/|j        | j                 j        }|j        | j                 j        }nÚg |j         dd…         ¢d‘| j        ‘R }|                      |¦  «         	                    |¦  «         
                    dd¦  «        }|                      |¦  «         	                    |¦  «         
                    dd¦  «        }|�E|                     ||| j        ¦  «        \  }}|
r$t	          |t          ¦  «        rd|j        | j        <   |€~|j         d         }| j        sFt          j        d| j        |d         |f|j        |j        ¬	¦  «        }| j        r| j        rd|_        n$|                      |d         ||j        |	¬
¦  «        }t?          j         | j!        j"        tF          ¦  «        } || ||||f| j        sdn| j$        | j%        |dœ|¤Ž\  }} |j&        g |¢d‘R Ž  '                    ¦   «         }|  (                    |¦  «        }|||fS )z€
        Self-attention (if key_value_states is None) or attention over source sentence (provided by key_value_states).
        Nr0   r   r7   r2   FTéþÿÿÿ©r   r~   )r   r&  r%   )r,   r+   r-   ))rf   r  Úget_seq_lengthrú   re   r9   r
   Úcloner  r\   r;   r   Ú
is_updatedÚgetÚcross_attention_cacheÚself_attention_cacheÚlayersÚkeysr*  r	  r
  Úupdaterþ   Úzerosr  r   r~   r  r6   Úrequires_gradr+  r   Úget_interfacer´   Ú_attn_implementationrB   r,   r+   Úreshaper>   r  )r³   rI   ÚmaskÚkey_value_statesr-   rH   r.   Úinput_shapeÚhidden_shaper&  Úis_cross_attentionÚquery_statesr1  Úcurr_past_key_valuesÚcurrent_statesÚ
key_statesÚvalue_statesÚkv_shaper%  Úattention_interfacer@   r?   s                         rA   r¾   zUdopAttention.forwardç  s�  € ð $Ô)¨#¨2¨#Ô.ˆØB˜ÐB bÐB¨$Ô*AÐBÐBˆØM\ÐMh˜?×9Ò9¸$¼.ÑIÔIÐIÐnoÐå7AÐBRÕTYÔT`Ñ7aÔ7aÐwÐ+×1Ò1Ñ3Ô3Ð3ÐgwÐð .°TÐ9Ðà—v’v˜mÑ,Ô,×1Ò1°,Ñ?Ô?×IÒIÈ!ÈQÑOÔOˆð ˆ
Ý�oÕ':Ñ;Ô;ð 	3Ø(Ô3×7Ò7¸¼ÑGÔGˆJØ!ð Là'6Ô'LÐ$Ð$à'6Ô'KÐ$Ð$à#2Ð à-?ÐRÐ)Ð)À]ˆØð 	F /Ð"=À*Ð"=à-Ô4°T´^ÔDÔIˆJØ/Ô6°t´~ÔFÔMˆLˆLàP˜Ô-¨c¨r¨cÔ2ÐP°BÐP¸Ô8OÐPÐPˆHØŸš Ñ/Ô/×4Ò4°XÑ>Ô>×HÒHÈÈAÑNÔNˆJØŸ6š6 .Ñ1Ô1×6Ò6°xÑ@Ô@×JÒJÈ1ÈaÑPÔPˆLàÐ*Ø+?×+FÒ+FÀzÐS_ÐaeÔaoÑ+pÔ+pÑ(�
˜Là%ð F­*°_ÕFYÑ*ZÔ*Zð FØAE�OÔ.¨t¬~Ñ>ð Ð Ø#Ô)¨"Ô-ˆJØÔ3ð Ý %¤Ø˜œ k°!¤n°jÐAØ'Ô.Ø&Ô,ð!ñ !ô !�ð
 Ô.ð 7°4´=ð 7Ø26�MÔ/øà $× 1Ò 1Ø ”N J°|Ô7JÐ]mð !2ñ !ô !�õ )@Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð  $œ}Ð>�C�C°$´,Ø”LØ'ð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—f’f˜[Ñ)Ô)ˆà˜M¨<Ð7Ð7rC   )FNF)Tr  r  )Nr   )NNNN)rL   rM   rN   r   Úintr’   r«   Ústaticmethodr   r+  r   r   r¾   r¿   rÀ   s   @rA   rù   rù   ~  sä   ø€ € € € € ð %*Ø $Øð%,ð %,àð%,ð ˜‘:ð	%,ð
 ð%,ð %,ð %,ð %,ð %,ð %,ðN ð- ð - ð - ñ „\ð- ð^ð ð ð ð( ØØØðX8ð X8ð Ð+Ô,ðX8ð X8ð X8ð X8ð X8ð X8ð X8ð X8rC   rù   c                   óL   ‡ — e Zd Zddedz  fˆ fd„Z	 	 	 ddee         fd„Zˆ xZS )	ÚUdopLayerSelfAttentionFNrú   c                 óþ   •— t          ¦   «                              ¦   «          t          ||||j        ¬¦  «        | _        t          |j        |j        ¬¦  «        | _        t          j
        |j        ¦  «        | _        d S )N©rþ   rú   rû   rñ   )rª   r«   rù   rý   ÚSelfAttentionrÂ   rÙ   rô   rõ   r   rÝ   rÞ   r,   ©r³   r´   rþ   rú   rµ   s       €rA   r«   zUdopLayerSelfAttention.__init__D  sr   ø€ Ý‰Œ×ÒÑÔÐÝ*ØØ(CØØÔ'ð	
ñ 
ô 
ˆÔõ (¨¬¸FÔ<UÐVÑVÔVˆŒÝ”z &Ô"5Ñ6Ô6ˆŒˆˆrC   r.   c                 ó’   — |                       |¦  «        } | j        |f|||dœ|¤Ž\  }}}||                      |¦  «        z   }|||fS )N)r=  r-   rH   )rõ   rO  r,   )	r³   rI   r*   r-   rH   r.   Únormed_hidden_statesÚattention_outputr?   s	            rA   r¾   zUdopLayerSelfAttention.forwardO  sy   € ð  $Ÿš¨}Ñ=Ô=ÐØ8J¸Ô8JØ ð9
àØ'Ø+ð	9
ð 9
ð
 ð9
ð 9
Ñ5Ð˜-¨ð )¨4¯<ª<Ð8HÑ+IÔ+IÑIÐØ °Ð<Ð<rC   ©FN©NNN©	rL   rM   rN   rI  r«   r   r   r¾   r¿   rÀ   s   @rA   rL  rL  C  s‚   ø€ € € € € ð	7ð 	7ÈSÐSWÉZð 	7ð 	7ð 	7ð 	7ð 	7ð 	7ð ØØð=ð =ð Ð+Ô,ð=ð =ð =ð =ð =ð =ð =ð =rC   rL  c                   óL   ‡ — e Zd Zddedz  fˆ fd„Z	 	 	 ddee         fd„Zˆ xZS )ÚUdopLayerCrossAttentionNrú   c                 óô   •— t          ¦   «                              ¦   «          t          |d|d¬¦  «        | _        t	          |j        |j        ¬¦  «        | _        t          j	        |j
        ¦  «        | _        d S )NFrN  rñ   )rª   r«   rù   ÚEncDecAttentionrÂ   rÙ   rô   rõ   r   rÝ   rÞ   r,   )r³   r´   rú   rµ   s      €rA   r«   z UdopLayerCrossAttention.__init__e  sm   ø€ Ý‰Œ×ÒÑÔÐÝ,Ø°ÀÐV[ð 
ñ  
ô  
ˆÔõ (¨¬¸FÔ<UÐVÑVÔVˆŒÝ”z &Ô"5Ñ6Ô6ˆŒˆˆrC   r.   c                 ó”   — |                       |¦  «        } | j        |f||||dœ|¤Ž\  }}}	||                      |¦  «        z   }
|
||	fS )N)r=  r>  r-   rH   )rõ   rZ  r,   )r³   rI   r>  r*   r-   rH   r.   rR  rS  r?   Úlayer_outputs              rA   r¾   zUdopLayerCrossAttention.forwardm  s{   € ð  $Ÿš¨}Ñ=Ô=ÐØ8L¸Ô8LØ ð9
àØ-Ø'Ø+ð9
ð 9
ð ð9
ð 9
Ñ5Ð˜-¨ð % t§|¢|Ð4DÑ'EÔ'EÑEˆØ˜]¨LÐ8Ð8rC   rã   rU  rV  rÀ   s   @rA   rX  rX  d  s�   ø€ € € € € ð7ð 7¨#°©*ð 7ð 7ð 7ð 7ð 7ð 7ð ØØð9ð 9ð Ð+Ô,ð9ð 9ð 9ð 9ð 9ð 9ð 9ð 9rC   rX  c                   óR   ‡ — e Zd Zddedz  fˆ fd„Z	 	 	 	 	 	 ddee         fd„Zˆ xZS )	Ú	UdopBlockFNrú   c                 ó’  •— t          ¦   «                              ¦   «          |j        | _        t          j        ¦   «         | _        | j                             t          |||¬¦  «        ¦  «         | j        r)| j                             t          ||¬¦  «        ¦  «         | j                             t          |¦  «        ¦  «         d S )N©rþ   rú   )rú   )
rª   r«   rý   r   Ú
ModuleListÚlayerÚappendrL  rX  rï   rP  s       €rA   r«   zUdopBlock.__init__…  sº   ø€ Ý‰Œ×ÒÑÔÐØ Ô+ˆŒÝ”]‘_”_ˆŒ
ØŒ
×ÒÝ"ØÐ4OÐ[dðñ ô ñ	
ô 	
ð 	
ð
 Œ?ð 	TØŒJ×ÒÕ5°fÈ	ÐRÑRÔRÑSÔSÐSàŒ
×Ò�+ fÑ-Ô-Ñ.Ô.Ð.Ð.Ð.rC   r.   c                 ó”  —  | j         d         |f|||dœ|¤Ž\  }}	}
|j        t          j        k    r�t          j        t          j        |¦  «                             ¦   «         t          j        |j        ¦  «        j        dz
  t          j        |j        ¦  «        j        ¦  «        }t          j	        || |¬¦  «        }d }| j
        o|d u}|r¿ | j         d         |f||||dœ|¤Ž\  }}}
|j        t          j        k    r�t          j        t          j        |¦  «                             ¦   «         t          j        |j        ¦  «        j        dz
  t          j        |j        ¦  «        j        ¦  «        }t          j	        || |¬¦  «        } | j         d         |¦  «        }|j        t          j        k    r�t          j        t          j        |¦  «                             ¦   «         t          j        |j        ¦  «        j        dz
  t          j        |j        ¦  «        j        ¦  «        }t          j	        || |¬¦  «        }||	|fS )Nr   )r*   r-   rH   iè  )r  Úmaxr7   )r>  r*   r-   rH   r0   )rb  r~   r9   rÎ   r  ÚisinfÚanyÚfinfore  Úclamprý   )r³   rI   r*   r-   Úencoder_hidden_statesÚencoder_attention_maskÚencoder_decoder_position_biasrH   r.   Úself_attn_position_biasÚ_Úclamp_valueÚcross_attn_position_biasÚdo_cross_attentions                 rA   r¾   zUdopBlock.forward“  sR  € ð 5B°D´J¸q´MØð5
à)Ø'Ø+ð	5
ð 5
ð
 ð5
ð 5
Ñ1ˆÐ.°ð Ô¥%¤-Ò/Ð/Ýœ+Ý”˜MÑ*Ô*×.Ò.Ñ0Ô0Ý”˜MÔ/Ñ0Ô0Ô4°tÑ;Ý”˜MÔ/Ñ0Ô0Ô4ñô ˆKõ
 "œK¨¸K¸<È[ÐYÑYÔYˆMà#'Ð Ø!œ_ÐRÐ1FÈdÐ1RÐØð 	^Ø9F¸¼ÀA¼Øð:à!6Ø5Ø;Ø /ð:ð :ð ð:ð :Ñ6ˆMÐ3°Qð Ô"¥e¤mÒ3Ð3Ý#œkÝ”K Ñ.Ô.×2Ò2Ñ4Ô4Ý”K Ô 3Ñ4Ô4Ô8¸4Ñ?Ý”K Ô 3Ñ4Ô4Ô8ñô �õ
 !&¤¨MÀ¸|ÐQ\Ð ]Ñ ]Ô ]�ð '˜œ
 2œ }Ñ5Ô5ˆð Ô¥%¤-Ò/Ð/Ýœ+Ý”˜MÑ*Ô*×.Ò.Ñ0Ô0Ý”˜MÔ/Ñ0Ô0Ô4°tÑ;Ý”˜MÔ/Ñ0Ô0Ô4ñô ˆKõ
 "œK¨¸K¸<È[ÐYÑYÔYˆMàÐ5Ð7OÐOÐOrC   rT  ©NNNNNNrV  rÀ   s   @rA   r^  r^  „  s•   ø€ € € € € ð/ð /ÈSÐSWÉZð /ð /ð /ð /ð /ð /ð" ØØ"Ø#Ø&*Øð=Pð =Pð Ð+Ô,ð=Pð =Pð =Pð =Pð =Pð =Pð =Pð =PrC   r^  c                   óÊ   ‡ — e Zd ZU eed<   dZdZdZdZdgZ	dZ
dZdZ eed¬¦  «         eed	¬¦  «         eed	¬¦  «        d
œZ ej        ¦   «         ˆ fd„¦   «         Zd„ Zˆ xZS )ÚUdopPreTrainedModelr´   Útransformer)ÚimageÚtextTFrÜ   r   )Úindexr0   )rI   rJ   rK   c                 ó>
  •— t          ¦   «                              |¦  «         | j        j        }t	          |t
          ¦  «        rt          j        |j        |dz  ¦  «         dS t	          |t          j
        ¦  «        r@t          j        |j        d|¬¦  «         |j        �t          j        |j        ¦  «         dS dS t	          |t          ¦  «        rA| j        j        }| j        j        }t          j        |j        j        d||dz  z  ¬¦  «         dS t	          |t$          ¦  «        r&t          j        |j        j        d|dz  ¬¦  «         dS t	          |t(          ¦  «        rFt+          |d¦  «        r2| j        j        s(t          j        |j        j        d|dz  ¬¦  «         dS dS dS t	          |t0          ¦  «        ræt          j        |j        j        d|| j        j        dz  z  ¬¦  «         t+          |j        d¦  «        r*|j        j        �t          j        |j        j        ¦  «         t          j        |j        j        d|| j        j        dz  z  ¬¦  «         t+          |j        d¦  «        r,|j        j        �"t          j        |j        j        ¦  «         dS dS dS t	          |t8          ¦  «        �rVt          j        |j        j        d|| j        j        dz  z  ¬¦  «         t+          |j        d¦  «        r*|j        j        �t          j        |j        j        ¦  «         t          j        |j        j        d|| j        j        dz  z  ¬¦  «         t+          |j        d¦  «        r*|j        j        �t          j        |j        j        ¦  «         t          j        |j        j        d|| j        j        dz  z  ¬¦  «         t+          |j        d¦  «        r,|j        j        �"t          j        |j        j        ¦  «         dS dS dS t	          |t>          ¦  «        rö| j        j        }| j        j         }| j        j!        }t          j        |j"        j        d|||z  dz  z  ¬¦  «         t          j        |j#        j        d||dz  z  ¬¦  «         t          j        |j$        j        d||dz  z  ¬¦  «         t          j        |j%        j        d|||z  dz  z  ¬¦  «         |j&        r+t          j        |j        j        d||dz  z  ¬¦  «         dS dS dS )zInitialize the weightsrW   r%   )rŽ   ÚstdNr1   Úlm_headr×   )'rª   Ú_init_weightsr´   Úinitializer_factorre   rÂ   ÚinitÚ	constant_rÆ   r   r±   Útrunc_normal_r×   Úzeros_ÚRelativePositionBiasBaserÙ   Únormal_r  Ú	UdopModelÚsharedÚUdopForConditionalGenerationÚhasattrÚtie_word_embeddingsr{  rÓ   rÛ   rÜ   rÚ   rç   ré   rê   rù   r  r  r  r	  r
  r  rþ   )r³   r&   ÚfactorrÙ   r  r  rµ   s         €rA   r|  z!UdopPreTrainedModel._init_weightsæ  s   ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ø”Ô/ˆÝ�f�mÑ,Ô,ð )	pÝŒN˜6œ=¨&°3©,Ñ7Ô7Ð7Ð7Ð7Ý˜¥¤	Ñ*Ô*ð '	pÝÔ˜vœ}°3¸FÐCÑCÔCÐCØŒ{Ð&Ý”˜FœKÑ(Ô(Ð(Ð(Ð(ð 'Ð&å˜Õ 8Ñ9Ô9ð #	pØ”[Ô3ˆFØ”kÔ)ˆGÝŒL˜Ô7Ô>ÀSÈfÐY`ÐeiÑXiÑNjÐkÑkÔkÐkÐkÐkÝ˜¥	Ñ*Ô*ð 	pÝŒL˜œÔ-°C¸VÀc¹\ÐJÑJÔJÐJÐJÐJÝ˜Õ <Ñ=Ô=ð 	pÝ�v˜yÑ)Ô)ð P°$´+Ô2Qð PÝ”˜Vœ^Ô2¸À&È3Á,ÐOÑOÔOÐOÐOÐOðPð Pð Pð På˜Õ 1Ñ2Ô2ð 	pÝŒL˜œÔ)°¸ÀDÄKÔDWÐ\`ÑC`Ñ9aÐbÑbÔbÐbÝ�v”y &Ñ)Ô)ð ,¨f¬i¬nÐ.HÝ”˜FœIœNÑ+Ô+Ð+ÝŒL˜œÔ)°¸ÀDÄKÔDTÐY]ÑC]Ñ9^Ð_Ñ_Ô_Ð_Ý�v”y &Ñ)Ô)ð ,¨f¬i¬nÐ.HÝ”˜FœIœNÑ+Ô+Ð+Ð+Ð+ð,ð ,Ð.HÐ.Hå˜Õ 6Ñ7Ô7ñ 	pÝŒL˜œÔ+°#¸6ÀdÄkÔFYÐ^bÑEbÑ;cÐdÑdÔdÐdÝ�v”{ FÑ+Ô+ð .°´Ô0@Ð0LÝ”˜FœKÔ,Ñ-Ô-Ð-ÝŒL˜œÔ+°#¸6ÀdÄkÔFYÐ^bÑEbÑ;cÐdÑdÔdÐdÝ�v”{ FÑ+Ô+ð .°´Ô0@Ð0LÝ”˜FœKÔ,Ñ-Ô-Ð-ÝŒL˜œÔ)°¸ÀDÄKÔDTÐY]ÑC]Ñ9^Ð_Ñ_Ô_Ð_Ý�v”y &Ñ)Ô)ð ,¨f¬i¬nÐ.HÝ”˜FœIœNÑ+Ô+Ð+Ð+Ð+ð,ð ,Ð.HÐ.Hå˜¥Ñ.Ô.ð 		pØ”kÔ)ˆGØ!%¤Ô!1ÐØ”kÔ+ˆGÝŒL˜œœ¨s¸À7ÐM_ÑC_ÐdhÑBhÑ8iÐjÑjÔjÐjÝŒL˜œœ¨s¸À'È4Á-Ñ8PÐQÑQÔQÐQÝŒL˜œœ¨s¸À'È4Á-Ñ8PÐQÑQÔQÐQÝŒL˜œœ¨s¸À7ÐM_ÑC_ÐdhÑBhÑ8iÐjÑjÔjÐjØÔ1ð pÝ”˜VÔ;ÔBÈÐRXÐ]dÐimÑ\mÑRnÐoÑoÔoÐoÐoÐoð		pð 		pðpð prC   c                 óŠ  — | j         j        }| j         j        }|€
J d¦   «         ‚|                     |j        ¦  «        }|dd d…f                              ¦   «         |ddd …f<   ||d<   |€
J d¦   «         ‚|                     |dk    |¦  «         t          j        |dk    ¦  «         	                    ¦   «         s
J d	¦   «         ‚|S )
Nz�self.model.config.decoder_start_token_id has to be defined. In Udop it is usually set to the pad_token_id. See Udop docs for more information.r0   r7   ).r   z1self.model.config.pad_token_id has to be defined.éœÿÿÿr   z8Verify that `shifted_input_ids` has only positive values)
r´   Údecoder_start_token_idÚpad_token_idÚ	new_zerosrf   r0  Úmasked_fill_r9   Úallr�   )r³   Ú	input_idsrŒ  r�  Úshifted_input_idss        rA   Ú_shift_rightz UdopPreTrainedModel._shift_right  sè   € Ø!%¤Ô!CÐØ”{Ô/ˆà%Ð1Ð1ð@ñ 2Ô1Ð1ð &×/Ò/°	´Ñ@Ô@ÐØ%.¨s°C°R°C¨xÔ%8×%>Ò%>Ñ%@Ô%@Ð˜#˜q˜r˜r˜'Ñ"Ø$:Ð˜&Ñ!àÐ'Ð'Ð)\Ñ'Ô'Ð'à×&Ò&Ð'8¸DÒ'@À,ÑOÔOÐOåŒyÐ*¨aÒ/Ñ0Ô0×5Ò5Ñ7Ô7ÐsÐsÐ9sÑsÔsÐ7à Ð rC   )rL   rM   rN   r   rQ   Úbase_model_prefixÚinput_modalitiesÚsupports_gradient_checkpointingÚ_can_compile_fullgraphÚ_keep_in_fp32_modulesÚ_supports_flash_attnÚ_supports_flex_attnÚ_supports_sdpar#   r^  rL  rX  Ú_can_record_outputsr9   Úno_gradr|  r“  r¿   rÀ   s   @rA   rt  rt  Ó  sç   ø€ € € € € € àÐÐÑØ%ÐØ(ÐØ&*Ð#à"ÐØ!˜FÐØ ÐØÐØ€Nð (˜¨	¸Ð;Ñ;Ô;Ø$�nÐ%;À2ÐFÑFÔFØ*˜NÐ+BÈ"ÐMÑMÔMðð Ðð €U„]�_„_ð-pð -pð -pð -pñ „_ð-pð`!ð !ð !ð !ð !ð !ð !rC   rt  c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚUdopCellEmbeddingséõ  é   c                 ó¾   •— t          ¦   «                              ¦   «          || _        t          j        ||¦  «        | _        t          j        ||¦  «        | _        d S rã   )rª   r«   Úmax_2d_position_embeddingsr   r  Úx_position_embeddingsÚy_position_embeddings)r³   r£  r­   rµ   s      €rA   r«   zUdopCellEmbeddings.__init__/  sQ   ø€ Ý‰Œ×ÒÑÔÐØ*DˆÔ'å%'¤\Ð2LÈkÑ%ZÔ%ZˆÔ"Ý%'¤\Ð2LÈkÑ%ZÔ%ZˆÔ"Ð"Ð"rC   c                 óÒ  — t          j        |dd¦  «        }|| j        dz
  z                       ¦   «         }|                     d| j        dz
  ¦  «        }|                      |d d …d d …df         ¦  «        }|                      |d d …d d …df         ¦  «        }|                      |d d …d d …df         ¦  «        }|                      |d d …d d …df         ¦  «        }||z   |z   |z   }|S )Nr%   rW   r7   r   r2   r   )r9   rŠ   r£  rŒ   ri  r¤  r¥  )r³   rˆ   Úleft_position_embeddingsÚupper_position_embeddingsÚright_position_embeddingsÚlower_position_embeddingsr½   s          rA   r¾   zUdopCellEmbeddings.forward6  s  € ÝŒz˜$  SÑ)Ô)ˆØ˜Ô7¸!Ñ;Ñ<×BÒBÑDÔDˆð �zŠz˜!˜TÔ<¸qÑ@ÑAÔAˆØ#'×#=Ò#=¸dÀ1À1À1ÀaÀaÀaÈÀ7¼mÑ#LÔ#LÐ Ø$(×$>Ò$>¸tÀAÀAÀAÀqÀqÀqÈ!ÀG¼}Ñ$MÔ$MÐ!Ø$(×$>Ò$>¸tÀAÀAÀAÀqÀqÀqÈ!ÀG¼}Ñ$MÔ$MÐ!Ø$(×$>Ò$>¸tÀAÀAÀAÀqÀqÀqÈ!ÀG¼}Ñ$MÔ$MÐ!ð %Ø'ñ(à'ñ(ð (ñ(ð 	ð ÐrC   )r   r¡  rÑ   rÀ   s   @rA   rŸ  rŸ  .  sR   ø€ € € € € ð[ð [ð [ð [ð [ð [ðð ð ð ð ð ð rC   rŸ  )gš™™™™™é?g      ô?c                   óæ   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 dˆ fd	„	Ze	 	 dd
edz  deee	f         dz  defd„¦   «         Z
dd
edz  deee	f         dz  defd„Zd„ Zdd
edz  deee	f         dz  defd„Zˆ xZS )r‚  a×  
    Base class of relative biases.

    Args:
        num_heads (`int`):
            Number of attention heads in the model, it will create embeddings of size `num_heads`, which will be added to the scores of each token pair.
        relative_attention_num_buckets (`int`, *optional*, defaults to 32):
            Pair token metric (distance in the sequence, distance in pixels etc.) will be bucketed, parameter is defining number of such
            buckets.
        bidirectional (`bool`, *optional*, defaults to `True`):
            Whether the distance should be bidirectional for a pair of tokens. If `False`, then distance(tok1, tok2) == distance(tok2, tok1).
        scaling_factor (`int`, *optional*, defaults to 1):
            Defining factor which will be used to scale relative distance.
        max_distance (`int`, *optional*, defaults to 128):
            All distances above this value will end up in the one/same bucket.
        augmentation (`bool`, *optional*, defaults to `False`):
            Whether to multiply relative distances by a random scalar.
        expand (`bool`, *optional*, defaults to `False`):
            Whether to expand an existing pretrained model with subsequent additions of prefix_bucket.
    Nr  Tr7   r  ÚtokensFc
                 ó.  •— t          ¦   «                              ¦   «          || _        || _        || _        || _        || _        || _        || _        |	| _	        || _
        |r	| j	        sdnd}
t          j        | j
        |
z   | j        ¦  «        | _        d S )Nr2   r   )rª   r«   Úprefix_bucketÚaugmentationÚlevelr  Úscaling_factorr  r  Úexpandrÿ   r   r  r  )r³   r  rÿ   r  r±  r  r°  r¯  r®  r²  Ú
extra_headrµ   s              €rA   r«   z!RelativePositionBiasBase.__init__g  sœ   ø€ õ 	‰Œ×ÒÑÔÐØ*ˆÔØ(ˆÔØˆŒ
Ø(ˆÔØ,ˆÔØ*ˆÔØ"ˆŒØˆŒØ.LˆÔ+Ø'ÐB°´ÐB�Q�QÀˆ
Ý')¤|°DÔ4WÐZdÑ4dÐfjÔftÑ'uÔ'uˆÔ$Ð$Ð$rC   r*   rˆ   Úreturnc                 ó   — d S rã   rS   )r³   r*   rˆ   s      rA   Úprepare_inputz&RelativePositionBiasBase.prepare_input€  s	   € ð 	ˆrC   c                 óv   — |                       ||¦  «        }t          || j        | j        | j        ¬¦  «        }|S )Nr"  )r¶  Úget_relative_position_bucketr  rÿ   r  )r³   r*   rˆ   r  Ú	rp_buckets        rA   Ú
get_bucketz#RelativePositionBiasBase.get_bucketˆ  sI   € Ø ×.Ò.¨~¸tÑDÔDÐÝ8ØØÔ,ØÔ;ØÔ*ð	
ñ 
ô 
ˆ	ð ÐrC   c                 ó  — |d d …d d …d f         }|d d …d d d …f         }||z
                        ¦   «         }| j        r| j        r|t          j        t
          Ž z  }|| j        z  }|                     t          j	        ¦  «        S rã   )
r  r¯  r6   ÚrandomÚuniformÚAUGMENTATION_RANGEr±  ri   r9   rŒ   )r³   Ú	positionsr'  r(  r  s        rA   Úget_relative_positionz.RelativePositionBiasBase.get_relative_position’  s˜   € Ø$ Q Q Q¨¨¨¨4 ZÔ0ÐØ# A A A t¨Q¨Q¨Q JÔ/ˆà,Ð/?Ñ?×FÒFÑHÔHÐØÔð 	E ¤ð 	EØ¥¤Õ1CÐ!DÑDÐØ˜TÔ0Ñ0Ðà ×#Ò#¥E¤JÑ/Ô/Ð/rC   c                 óâ  — | j         rr| j        rkt          j        | j        dz   | j        ¦  «        }| j        j        j        |j        j        d | j        …<   d|j        j        | j        d …<   || _        d| _         |  	                    ||¦  «        }| j        râ| 
                    d¦  «        dk    rC| 
                    d¦  «        dk    r*|                     | 
                    d¦  «        dd¦  «        }|d d …d d …df         dk     }|                     d¦  «        }t          |                     ¦   «                              ¦   «         ¦  «        D ]*\  }}| j        ||d |…|d …f<   | j        dz   |||d …d |…f<   Œ+|                     d| j        j        dz
  ¦  «        }|                      |¦  «        }	|	                     ¦   «         dk    rt'          d¦  «        ‚|	                     g d	¢¦  «        }	|	S )
Nr2   gš™™™™™¹?Fr   r7   r0   rX   z Wrong dimension of values tensor)r   r   r7   r2   )r²  r®  r   r  rÿ   r  r  rÆ   Údatarº  r8   r[   ÚsumÚ	enumerateÚcpuÚnumpyri  Únum_embeddingsr4   r¸   r#  )
r³   r*   rˆ   Únew_biasr¹  Ú	is_prefixÚ
num_prefixÚidxÚnum_prefix_rowr*  s
             rA   r¾   z RelativePositionBiasBase.forward�  sþ  € àŒ;ð 	 ˜4Ô-ð 	 Ý”| DÔ$GÈ!Ñ$KÈTÌ^Ñ\Ô\ˆHØJNÔJfÔJmÔJrˆHŒOÔ Ð!F 4Ô#FÐ!FÑGØJMˆHŒOÔ  Ô!DÐ!FÐ!FÑGØ+3ˆDÔ(ØˆDŒKà—O’O N°DÑ9Ô9ˆ	àÔð 	kØ�~Š~˜aÑ Ô  AÒ%Ð%¨.×*=Ò*=¸aÑ*@Ô*@À1Ò*DÐ*DØ%×,Ò,¨^×-@Ò-@ÀÑ-CÔ-CÀQÈÑJÔJ�	à˜Q˜Q˜Q    1˜Wœ¨Ò)ˆIØ"Ÿš rÑ*Ô*ˆJÝ'0°·²Ñ1AÔ1A×1GÒ1GÑ1IÔ1IÑ'JÔ'Jð kð kÑ#��^ØCGÔCf�	˜#˜ ˜°°°Ð?Ñ@ØCGÔCfÐijÑCj�	˜#˜~˜˜°°°Ð?Ñ@Ð@ð —O’O A tÔ'CÔ'RÐUVÑ'VÑWÔWˆ	Ø×5Ò5°iÑ@Ô@ˆØ�:Š:‰<Œ<˜1ÒÐÝÐ?Ñ@Ô@Ð@Ø—’   Ñ-Ô-ˆàˆrC   )	Nr  Tr7   r  r¬  FFF©NN)rL   rM   rN   rO   r«   r   r
   ÚdictÚstrr	   r¶  rº  rÀ  r¾   r¿   rÀ   s   @rA   r‚  r‚  Q  sU  ø€ € € € € ðð ð. Ø')ØØØØØØØðvð vð vð vð vð vð2 ð )-Ø&*ðð à ™ðð �3˜�8Œn˜tÑ#ðð 
ð	ð ð ñ „^ððð ¨°$©ð ÀTÈ#ÈsÈ(Ä^ÐVZÑEZð Ðflð ð ð ð ð	0ð 	0ð 	0ðð  f¨t¡mð À$ÀsÈCÀxÄ.ÐSWÑBWð Ðcið ð ð ð ð ð ð ð rC   r‚  c                   óR   ‡ — e Zd Zd	ˆ fd„	Zd
dedz  deeef         dz  defd„Zˆ xZ	S )ÚRelativePositionBias1Dr7   r  c                 ó@   •—  t          ¦   «         j        d||dœ|¤Ž dS )z²
        Reimplementation of T5 relative position bias. Distance between given tokens is their distance in the sequence.
        Parameters are the same as in base class
        ©r±  r  NrS   ©rª   r«   ©r³   r±  r  r.   rµ   s       €rA   r«   zRelativePositionBias1D.__init__½  ó0   ø€ ð
 	�‰ŒÔÐ\¨À\Ð\Ð\ÐU[Ð\Ð\Ð\Ð\Ð\rC   Nr*   rˆ   r´  c                 óæ   — | j         dk    rt          d¦  «        ‚|                      t          j        |                     d¦  «        t          j        |j        ¬¦  «        d d d …f         ¦  «        }|S )Nr7   zNo need to scale 1d featuresr}   )r±  r¸   rÀ  r9   rY   r8   rŒ   r   )r³   r*   rˆ   r  s       rA   r¶  z$RelativePositionBias1D.prepare_inputÄ  sv   € ØÔ !Ò#Ð#ÝÐ;Ñ<Ô<Ð<Ø ×6Ò6ÝŒL˜×,Ò,¨QÑ/Ô/µu´zÈ.ÔJ_Ð`Ñ`Ô`ÐaeÐghÐghÐghÐahÔiñ
ô 
Ðð !Ð rC   )r7   r  rÍ  ©
rL   rM   rN   r«   r
   rÎ  rÏ  r	   r¶  r¿   rÀ   s   @rA   rÑ  rÑ  ¼  s…   ø€ € € € € ð]ð ]ð ]ð ]ð ]ð ]ð!ð !¨F°T©Mð !ÈÈSÐRUÈXÌÐY]ÑH]ð !Ðioð !ð !ð !ð !ð !ð !ð !ð !rC   rÑ  c                   óR   ‡ — e Zd Zdˆ fd„	Zd	dedz  deeef         dz  defd„Zˆ xZ	S )
ÚRelativePositionBiasHorizontaléd   c                 ó@   •—  t          ¦   «         j        d||dœ|¤Ž dS )zŽ
        Represents in the bucket embeddings horizontal distance between two tokens. Parameters are the same as in base
        class
        rÓ  NrS   rÔ  rÕ  s       €rA   r«   z'RelativePositionBiasHorizontal.__init__Ï  rÖ  rC   Nr*   rˆ   r´  c                 óÌ   — | j         dk    st          d¦  «        ‚|€t          d¦  «        ‚|d d …d d …ddgf                              d¬¦  «        }|                      |¦  «        S )NrW   úENeed to scale the values of bboxes, as there are in small (0,1) rangez6Bbox is required for horizontal relative position biasr   r2   r0   r3   ©r±  r¸   rŽ   rÀ  )r³   r*   rˆ   Úhorizontal_positions       rA   r¶  z,RelativePositionBiasHorizontal.prepare_inputÖ  sv   € ØÔ" SÒ(Ð(ÝÐdÑeÔeÐeØˆ<ÝÐUÑVÔVÐVà&*¨1¨1¨1¨a¨a¨a°!°Q°¨<Ô&8×&=Ò&=À"Ð&=Ñ&EÔ&EÐà×)Ò)Ð*=Ñ>Ô>Ð>rC   ©rÛ  rÛ  rÍ  rØ  rÀ   s   @rA   rÚ  rÚ  Î  s…   ø€ € € € € ð]ð ]ð ]ð ]ð ]ð ]ð?ð ?¨F°T©Mð ?ÈÈSÐRUÈXÌÐY]ÑH]ð ?Ðioð ?ð ?ð ?ð ?ð ?ð ?ð ?ð ?rC   rÚ  c                   óR   ‡ — e Zd Zdˆ fd„	Zd	dedz  deeef         dz  defd„Zˆ xZ	S )
ÚRelativePositionBiasVerticalrÛ  c                 ó@   •—  t          ¦   «         j        d||dœ|¤Ž dS )zŒ
        Represents in the bucket embeddings vertical distance between two tokens. Parameters are the same as in base
        class
        rÓ  NrS   rÔ  rÕ  s       €rA   r«   z%RelativePositionBiasVertical.__init__â  rÖ  rC   Nr*   rˆ   r´  c                 óÌ   — | j         dk    st          d¦  «        ‚|€t          d¦  «        ‚|d d …d d …ddgf                              d¬¦  «        }|                      |¦  «        S )NrW   rÞ  z4Bbox is required for vertical relative position biasr7   r   r0   r3   rß  )r³   r*   rˆ   Úvertical_positions       rA   r¶  z*RelativePositionBiasVertical.prepare_inputé  sv   € ØÔ" SÒ(Ð(ÝÐdÑeÔeÐeØˆ<ÝÐSÑTÔTÐTà$(¨¨¨¨A¨A¨A°°1¨v¨Ô$6×$;Ò$;ÀÐ$;Ñ$CÔ$CÐà×)Ò)Ð*;Ñ<Ô<Ð<rC   rá  rÍ  rØ  rÀ   s   @rA   rã  rã  á  s…   ø€ € € € € ð]ð ]ð ]ð ]ð ]ð ]ð=ð =¨F°T©Mð =ÈÈSÐRUÈXÌÐY]ÑH]ð =Ðioð =ð =ð =ð =ð =ð =ð =ð =rC   rã  c                   óh   ‡ — e Zd Zdee         fˆ fd„Zddedz  deee	f         dz  de
ez  fd„Zˆ xZS )	ÚRelativePositionBiasAggregatedÚmodulesc                 óz   •— t          ¦   «                              ¦   «          t          j        |¦  «        | _        dS )z¶
        Class which sums up various computed biases.

        Args:
            modules (Sequence[RelativePositionBiasBase]):
                List of relative bias modules.
        N)rª   r«   r   ra  Úbiases)r³   ré  rµ   s     €rA   r«   z'RelativePositionBiasAggregated.__init__õ  s0   ø€ õ 	‰Œ×ÒÑÔÐÝ”m GÑ,Ô,ˆŒˆˆrC   Nr*   rˆ   r´  c                 ó<   — d}| j         D ]} |||¦  «        |z   }Œ|S )Nr%   )rë  )r³   r*   rˆ   Úoutputr×   s        rA   r¾   z&RelativePositionBiasAggregated.forward   s5   € ØˆØ”Kð 	9ð 	9ˆDØ�T˜.¨$Ñ/Ô/°&Ñ8ˆFˆFàˆrC   rÍ  )rL   rM   rN   r   r‚  r«   r
   rÎ  rÏ  r	   r  r¾   r¿   rÀ   s   @rA   rè  rè  ô  s‘   ø€ € € € € ð	- Ð)AÔ Bð 	-ð 	-ð 	-ð 	-ð 	-ð 	-ðð  f¨t¡mð À$ÀsÈCÀxÄ.ÐSWÑBWð ÐchÐkqÑcqð ð ð ð ð ð ð ð rC   rè  )Ú1dÚ
horizontalÚverticalr´   r´  c                 óV  — g }t          | d¦  «        r–| j        D ]Ž}t          |¦  «        }|                     d¦  «        }t          | d¦  «        r| j        n| j        }d|v r|d         |k    rt          d¦  «        ‚n||d<   |                     t          |         di |¤Ž¦  «         Œ�|S )z”
    Creates empty list or one/multiple relative biases.

    :param config: Model's configuration :return: Sequence with created bias modules.
    Úrelative_bias_argsÚtyper  z4Number of heads must match num of heads in the modelrS   )	r‡  rò  r   Úpopr  Únum_attention_headsr¸   rc  ÚBIAS_CLASSES)r´   Ú	bias_listÚbias_kwargs_orgÚbias_kwargsÚ	bias_typeÚmodel_num_headss         rA   Úcreate_relative_biasrü    sÚ   € ð €IÝˆvÐ+Ñ,Ô,ð 
EØ%Ô8ð 		Eð 		EˆOÝ" ?Ñ3Ô3ˆKØ#Ÿš¨Ñ/Ô/ˆIÝ29¸&À+Ñ2NÔ2NÐn˜fÔ.Ð.ÐTZÔTnˆOØ˜kÐ)Ð)Ø˜{Ô+¨Ò>Ð>Ý$Ð%[Ñ\Ô\Ð\ð ?ð ,;�˜KÑ(Ø×Ò�\¨)Ô4ÐCÐC°{ÐCÐCÑDÔDÐDÐDàÐrC   c                   ó¤   ‡ — e Zd ZdZˆ fd„Zededefd„¦   «         Zd„ Z	d„ Z
ee	 	 	 	 	 	 	 	 	 	 	 	 dd	ee         defd
„¦   «         ¦   «         Zˆ xZS )Ú	UdopStackz‡
    This class is based on `T5Stack`, but modified to take into account the image modality as well as 2D position
    embeddings.
    c                 ó~  •‡— t          ¦   «                              ‰¦  «         t          j        ‰j        ‰j        ¦  «        | _        t          ‰¦  «        | _        ‰j	        | _	        ‰j
        | _
        t          j        ˆfd„t          | j
        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        t          j        ‰j        ¦  «        | _        | j	        st)          ‰j        ‰j        ¦  «        | _        |                      ‰¦  «        | _        |                      ¦   «          d S )Nc           	      óV   •— g | ]%}t          ‰t          |d k    ¦  «        |¬¦  «        ‘Œ&S )r   r`  )r^  r’   )ru   rv   r´   s     €rA   ry   z&UdopStack.__init__.<locals>.<listcomp>4  s4   ø€ ÐvÐvÐvÐZ[�Y�v½4ÀÀQÂ¹<¼<ÐSTÐUÑUÔUÐvÐvÐvrC   rñ   )rª   r«   r   r  Ú
vocab_sizerÙ   Úembed_tokensr¦   Úembed_patchesrý   Ú
num_layersra  r•   ÚblockrÂ   rô   Úfinal_layer_normrÝ   rÞ   r,   rŸ  r£  r­   Úcell_2d_embeddingÚ_get_relative_biasÚrelative_biasÚ	post_initrá   s    `€rA   r«   zUdopStack.__init__+  s  øø€ Ý‰Œ×Ò˜Ñ Ô Ð åœL¨Ô):¸F¼NÑKÔKˆÔÝ0°Ñ8Ô8ˆÔØ Ô+ˆŒØ Ô+ˆŒå”]ØvÐvÐvÐvÕ_dÐeiÔetÑ_uÔ_uÐvÑvÔvñ
ô 
ˆŒ
õ !.¨f¬nÀ&ÔB[Ð \Ñ \Ô \ˆÔå”z &Ô"5Ñ6Ô6ˆŒàŒð 	oÝ%7¸Ô8YÐ[aÔ[mÑ%nÔ%nˆDÔ"ð "×4Ò4°VÑ<Ô<ˆÔØ�ŠÑÔÐÐÐrC   r´   r´  c                 ó>   — t          | ¦  «        }t          |¦  «        S rã   )rü  rè  )r´   Úrelative_bias_lists     rA   r  zUdopStack._get_relative_biasA  s   € å1°&Ñ9Ô9ÐÝ-Ð.@ÑAÔAÐArC   c                 ó   — | j         S rã   ©r  ©r³   s    rA   Úget_output_embeddingszUdopStack.get_output_embeddingsF  s   € ØÔ Ð rC   c                 ó   — || _         d S rã   r  ©r³   Únew_embeddingss     rA   Úset_input_embeddingszUdopStack.set_input_embeddingsI  s   € Ø*ˆÔÐÐrC   Nr.   c                 óÀ  — |d u |d uz  r!| j         rdnd}t          d|› d|› d�¦  «        ‚|�Út          j        |¦  «        dk    rÂt          j        d| j        j        |j        |j        ¬¦  «        }t          j	        d|j        |j        ¬¦  «        }t          j	        d	|j        |j        ¬¦  «        }t          j
        |d d …d d d d …f         ¦  «                             | j        ¦  «        }
t                               d
¦  «         |�1|                     ¦   «         }|                     d|d         ¦  «        }n|                     ¦   «         d d…         }|€|                      |¦  «        }|�|                      |¦  «        }	|	�d| j        j        | j        j        z  }t)          |	|||||d| j        j        | j        j        ¦	  «	        \  }}}|                     ¦   «         d d…         }| j         s|�||                      |¦  «        z  }|\  }}|du r| j         sJ d| › d�¦   «         ‚| j         r]|rZ|€X| j        j        r7t/          t1          | j        ¬¦  «        t1          | j        ¬¦  «        ¦  «        }nt1          | j        ¬¦  «        }n	| j         sd }|�|                     ¦   «         nd}|€/t5          ¦   «         s!||z   }t          j        |||j        ¬¦  «        }| j        j         rt9          | j        |||¬¦  «        }nt;          | j        ||¬¦  «        }| j         r|�t;          | j        |||¬¦  «        }nd }| j         rd }
n|                      ||¬¦  «        }
d }|}|                      |¦  «        }| j         D ]#} ||||
|||fd|i|¤Ž\  }}}|}
| j         r|�|}Œ$|  !                    |¦  «        }|                      |¦  «        }tE          |||¬¦  «        S )NÚdecoder_Ú z You must specify exactly one of zinput_ids or r–   r   )rX   r¡  r.  )rX   r¡  rX   zEmpty batchr0   Tz)`use_cache` can only be set to `True` if z is used as a decoder)r´   )r   )r´   r–   r*   rH   )r´   r–   r*   )r´   r–   r*   rj  )r*   rˆ   rH   )rG   r*   rH   )#rý   r¸   r9   ÚnumelÚfullr´   r�  r   r~   r8  r…   ri   r  Úwarningr8   r\   r  r  r]   r^   r¤   r  Úis_encoder_decoderr   r   r/  r    r€   r   r   r	  r,   r  r  rF   )r³   r‘  r*   rˆ   rj  rk  r–   r¹   r{   rw   r-   rH   Ú	use_cacher.   Úerr_msg_prefixr?  r—   rº   Ú
seq_lengthÚpast_key_values_lengthÚmask_seq_lengthÚcausal_maskÚencoder_extended_attention_maskrl  rI   Úlayer_moduleÚself_attention_position_biasÚcross_attention_position_biass                               rA   r¾   zUdopStack.forwardL  sÈ  € ð( ˜Ð -°tÐ";Ñ<ð 	Ø+/¬?ÐB˜Z˜ZÀˆNÝØm°>ÐmÐmÐP^ÐmÐmÐmñô ð ð Ð ¥U¤[°Ñ%;Ô%;¸qÒ%@Ð%@Ýœ
 9¨d¬kÔ.FÈyÔO_ÐgpÔgvÐwÑwÔwˆIÝ"œ[¨¸9Ô;KÐS\ÔSbÐcÑcÔcˆNÝ”;˜|°IÔ4DÈIÌOÐ\Ñ\Ô\ˆDÝ!Ô,¨^¸A¸A¸A¸tÀTÈ1È1È1Ð<LÔ-MÑNÔN×QÒQÐRVÔR\Ñ]Ô]ˆMÝ�NŠN˜=Ñ)Ô)Ð)àÐ Ø#Ÿ.š.Ñ*Ô*ˆKØ!Ÿš r¨;°r¬?Ñ;Ô;ˆIˆIà'×,Ò,Ñ.Ô.¨s°¨sÔ3ˆKàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàÐ#Ø#×1Ò1°,Ñ?Ô?ÐàÐ'àœ+Ô0°D´KÔ4JÑJˆKÝ2OØ ØØØØØØØ”Ô&Ø”Ô&ñ
3ô 
3Ñ/ˆM˜4 ð (×,Ò,Ñ.Ô.¨s°¨sÔ3ˆKàŒð 	: 4Ð#3Ø˜T×3Ò3°DÑ9Ô9Ñ9ˆMà!,Ñˆ
�Jà˜ÐÐØ”?ÐkÐkÐ$kÐPTÐ$kÐ$kÐ$kÑkÔk�?àŒ?ð 	#Øð G˜_Ð4Ø”;Ô1ð GÝ&9Ý$¨D¬KÐ8Ñ8Ô8½,ÈdÌkÐ:ZÑ:ZÔ:Zñ'ô '�O�Oõ '3¸$¼+Ð&FÑ&FÔ&F�OøØ”ð 	#ð #ˆOàETÐE` ×!?Ò!?Ñ!AÔ!AÐ!AÐfgÐØÐ!Õ*BÑ*DÔ*DÐ!à4°zÑAˆOÝ"œZ¨
°OÈMÔL`ÐaÑaÔaˆNàŒ;Ô!ð 	Ý,Ø”{Ø+Ø-Ø /ð	ñ ô ˆKˆKõ 4Ø”{Ø+Ø-ðñ ô ˆKð Œ?ð 	3Ð5ÐAÝ.GØ”{Ø+Ø5Ø&;ð	/ñ /ô /Ð+Ð+ð /3Ð+àŒ?ð 	YØ ˆMˆMà ×.Ò.¸nÐSWÐ.ÑXÔXˆMØ(,Ð%à%ˆàŸš ]Ñ3Ô3ˆà œJð 	Nð 	NˆLØYeÐYeØØØØ%Ø/Ø-ð	Zð 	Zð !0ð	Zð ð	Zð 	ZÑVˆMÐ7Ð9Vð 9ˆMØŒð NÐ#8Ð#DØ0MÐ-øà×-Ò-¨mÑ<Ô<ˆØŸš ]Ñ3Ô3ˆå/Ø+Ø)Ø+ð
ñ 
ô 
ð 	
rC   ©NNNNNNNNNNNN)rL   rM   rN   rO   r«   rJ  r   rè  r  r  r  r"   r$   r   r   rF   r¾   r¿   rÀ   s   @rA   rþ  rþ  %  s  ø€ € € € € ðð ð
ð ð ð ð ð, ðB :ð BÐ2Pð Bð Bð Bñ „\ðBð!ð !ð !ð+ð +ð +ð  Øð ØØØ"Ø#ØØØØØØØðP
ð P
ð Ð+Ô,ðP
ð 
*ðP
ð P
ð P
ñ „_ñ  ÔðP
ð P
ð P
ð P
ð P
rC   rþ  c                   ó&  ‡ — e Zd ZdddddœZˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 	 	 	 	 dd	e	dz  d
e	dz  de
eef         dz  de	dz  de
eef         dz  de	dz  de	dz  de	dz  de	dz  dedz  de	dz  dedz  dee         defd„¦   «         ¦   «         Zˆ xZS )r„  úshared.weightúpatch_embed.proj.weightúpatch_embed.proj.bias)úencoder.embed_tokens.weightúdecoder.embed_tokens.weightú!encoder.embed_patches.proj.weightúencoder.embed_patches.proj.biasc                 ó°  •— t          ¦   «                              |¦  «         t          j        |j        |j        ¦  «        | _        t          |¦  «        | _        t          |¦  «        }d|_
        d|_        t          |¦  «        | _        t          |¦  «        }d|_
        |j        |_        t          |¦  «        | _        |                      ¦   «          d S )NFT)rª   r«   r   r  r  rÙ   r…  r¦   Úpatch_embedr   rý   r  rþ  ÚencoderÚnum_decoder_layersr  Údecoderr
  ©r³   r´   Úencoder_configÚdecoder_configrµ   s       €rA   r«   zUdopModel.__init__ê  s¸   ø€ Ý‰Œ×Ò˜Ñ Ô Ð õ ”l 6Ô#4°f´nÑEÔEˆŒÝ.¨vÑ6Ô6ˆÔå! &Ñ)Ô)ˆØ$)ˆÔ!Ø#(ˆÔ Ý  Ñ0Ô0ˆŒå! &Ñ)Ô)ˆØ$(ˆÔ!Ø$*Ô$=ˆÔ!Ý  Ñ0Ô0ˆŒð 	�ŠÑÔÐÐÐrC   c                 ó   — | j         S rã   ©r…  r  s    rA   Úget_input_embeddingszUdopModel.get_input_embeddingsþ  ó
   € ØŒ{ÐrC   c                 ó|   — || _         | j                             |¦  «         | j                             |¦  «         d S rã   ©r…  r1  r  r3  r  s     rA   r  zUdopModel.set_input_embeddings  ó;   € Ø$ˆŒØŒ×)Ò)¨.Ñ9Ô9Ð9ØŒ×)Ò)¨.Ñ9Ô9Ð9Ð9Ð9rC   Nr‘  r*   rˆ   r¹   r{   Údecoder_input_idsÚdecoder_attention_maskr–   Úencoder_outputsrH   Údecoder_inputs_embedsr  r.   r´  c                 óô   — |	€ | j         d||||||dœ|¤Ž}	|	d         }|	j        } | j        d||||
|||dœ|¤Ž}t          |j        |j        |j        |j        |j        |	j        |	j        |	j        ¬¦  «        S )aA  
        bbox (`torch.LongTensor` of shape `({0}, 4)`, *optional*):
            Bounding boxes of each input sequence tokens. Selected in the range `[0,
            config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1)
            format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1,
            y1) represents the position of the lower right corner.

            Note that `sequence_length = token_sequence_length + patch_sequence_length + 1` where `1` is for [CLS]
            token. See `pixel_values` for `patch_sequence_length`.
        visual_bbox (`torch.LongTensor` of shape `(batch_size, patch_sequence_length, 4)`, *optional*):
            Bounding boxes of each patch in the image. If not provided, bounding boxes are created in the model.
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary. Indices can be obtained using
            [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details.
            [What are decoder input IDs?](../glossary#decoder-input-ids) T5 uses the `pad_token_id` as the starting
            token for `decoder_input_ids` generation. If `past_key_values` is used, optionally only the last
            `decoder_input_ids` have to be input (see `past_key_values`). To know more on how to prepare
            `decoder_input_ids` for pretraining take a look at [T5 Training](./t5#training).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.

        Example:

        ```python
        >>> from transformers import AutoProcessor, AutoModel
        >>> from datasets import load_dataset
        >>> import torch

        >>> # load model and processor
        >>> # in this case, we already have performed OCR ourselves
        >>> # so we initialize the processor with `apply_ocr=False`
        >>> processor = AutoProcessor.from_pretrained("microsoft/udop-large", apply_ocr=False)
        >>> model = AutoModel.from_pretrained("microsoft/udop-large")

        >>> # load an example image, along with the words and coordinates
        >>> # which were extracted using an OCR engine
        >>> dataset = load_dataset("nielsr/funsd-layoutlmv3", split="train")
        >>> example = dataset[0]
        >>> image = example["image"]
        >>> words = example["tokens"]
        >>> boxes = example["bboxes"]
        >>> inputs = processor(image, words, boxes=boxes, return_tensors="pt")

        >>> decoder_input_ids = torch.tensor([[model.config.decoder_start_token_id]])

        >>> # forward pass
        >>> outputs = model(**inputs, decoder_input_ids=decoder_input_ids)
        >>> last_hidden_states = outputs.last_hidden_state
        >>> list(last_hidden_states.shape)
        [1, 1, 1024]
        ```N)r‘  r*   rˆ   r¹   r{   r–   r   ©r‘  r*   r–   rH   rj  rk  r  )rG   rH   Údecoder_hidden_statesÚdecoder_attentionsrK   Úencoder_last_hidden_staterj  Úencoder_attentionsrS   )	r1  r*   r3  r   rG   rH   rI   rJ   rK   )r³   r‘  r*   rˆ   r¹   r{   r>  r?  r–   r@  rH   rA  r  r.   rI   rk  Údecoder_outputss                    rA   r¾   zUdopModel.forward  sà   € ðP Ð"Ø*˜dœlð Ø#Ø-ØØ)Ø'Ø+ðð ð ðð ˆOð (¨Ô*ˆØ!0Ô!?Ðð '˜$œ,ð 	
Ø'Ø1Ø/Ø+Ø"/Ø#9Øð	
ð 	
ð ð	
ð 	
ˆõ "Ø-Ô?Ø+Ô;Ø"1Ô"?Ø.Ô9Ø,Ô=Ø&5Ô&GØ"1Ô"?Ø.Ô9ð	
ñ 	
ô 	
ð 		
rC   r&  )rL   rM   rN   Ú_tied_weights_keysr«   r9  r  r!   r   r
   rÎ  rÏ  r	   r   r’   r   r   r   r¾   r¿   rÀ   s   @rA   r„  r„  á  s°  ø€ € € € € ð (7Ø'6Ø-FØ+Bð	ð Ððð ð ð ð ð(ð ð ð:ð :ð :ð
 Øð $(Ø(,Ø&*Ø&*Ø-1Ø+/Ø04Ø'+Ø)-Ø(,Ø/3Ø!%ði
ð i
à˜D‘=ði
ð  ™ði
ð �3˜�8Œn˜tÑ#ð	i
ð
 ˜t‘mði
ð ˜#˜s˜(”^ dÑ*ði
ð " D™=ði
ð !'¨¡ði
ð  ‘}ði
ð   $™ði
ð  ™ði
ð  &¨™}ði
ð ˜$‘;ði
ð Ð+Ô,ði
ð 
ði
ð i
ð i
ñ „^ñ Ôði
ð i
ð i
ð i
ð i
rC   r„  a  
    The UDOP encoder-decoder Transformer with a language modeling head on top, enabling to generate text given document
    images and an optional prompt.

    This class is based on [`T5ForConditionalGeneration`], extended to deal with images and layout (2D) data.
    c            !       ó8  ‡ — e Zd ZddddddddœZˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 	 	 	 	 	 dd
e	d	z  de	d	z  de
eef         d	z  de	d	z  de
eef         d	z  de	d	z  de	d	z  de	d	z  de	d	z  ded	z  de	d	z  ded	z  de	d	z  dee         defd„¦   «         ¦   «         Zˆ xZS )r†  r(  r)  r*  úDencoder.block.0.layer.0.SelfAttention.relative_attention_bias.weight)r+  r,  r-  r.  ú=encoder.relative_bias.biases.0.relative_attention_bias.weightz=decoder.relative_bias.biases.0.relative_attention_bias.weightzlm_head.weightc                 óü  •— t          ¦   «                              |¦  «         t          j        |j        |j        ¦  «        | _        t          |¦  «        | _        t          |¦  «        }d|_
        d|_        t          |¦  «        | _        t          |¦  «        }d|_
        |j        |_        t          |¦  «        | _        t          j        |j        |j        d¬¦  «        | _        |                      ¦   «          d S )NFTrÖ   )rª   r«   r   r  r  rÙ   r…  r¦   r0  r   rý   r  rþ  r1  r2  r  r3  rØ   r{  r
  r4  s       €rA   r«   z%UdopForConditionalGeneration.__init__‡  s×   ø€ Ý‰Œ×Ò˜Ñ Ô Ð õ ”l 6Ô#4°f´nÑEÔEˆŒÝ.¨vÑ6Ô6ˆÔå! &Ñ)Ô)ˆØ$)ˆÔ!Ø#(ˆÔ Ý  Ñ0Ô0ˆŒå! &Ñ)Ô)ˆØ$(ˆÔ!Ø$*Ô$=ˆÔ!Ý  Ñ0Ô0ˆŒõ ”y ¤°Ô1BÈÐOÑOÔOˆŒð 	�ŠÑÔÐÐÐrC   c                 ó   — | j         S rã   r8  r  s    rA   r9  z1UdopForConditionalGeneration.get_input_embeddingsž  r:  rC   c                 ó|   — || _         | j                             |¦  «         | j                             |¦  «         d S rã   r<  r  s     rA   r  z1UdopForConditionalGeneration.set_input_embeddings¡  r=  rC   Nr‘  r*   rˆ   r¹   r{   r>  r?  r–   r@  rH   rA  r  Úlabelsr.   r´  c                 óH  — |€|�|                       |¦  «        }|	€ | j        d
||||||dœ|¤Ž}	|	d         }|	j        } | j        d
||||
|||dœ|¤Ž}|d         }| j        j        r|| j        j        dz  z  }|                      |¦  «        }d}|�Vt          d¬¦  «        } || 	                    d| 
                    d¦  «        ¦  «        | 	                    d¦  «        ¦  «        }t          |||j        |j        |j        |j        |	j        |	j        |	j        ¬	¦	  «	        S )a0  
        bbox (`torch.LongTensor` of shape `({0}, 4)`, *optional*):
            Bounding boxes of each input sequence tokens. Selected in the range `[0,
            config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1)
            format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1,
            y1) represents the position of the lower right corner.

            Note that `sequence_length = token_sequence_length + patch_sequence_length + 1` where `1` is for [CLS]
            token. See `pixel_values` for `patch_sequence_length`.
        visual_bbox (`torch.LongTensor` of shape `(batch_size, patch_sequence_length, 4)`, *optional*):
            Bounding boxes of each patch in the image. If not provided, bounding boxes are created in the model.
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary. Indices can be obtained using
            [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details.
            [What are decoder input IDs?](../glossary#decoder-input-ids) T5 uses the `pad_token_id` as the starting
            token for `decoder_input_ids` generation. If `past_key_values` is used, optionally only the last
            `decoder_input_ids` have to be input (see `past_key_values`). To know more on how to prepare
            `decoder_input_ids` for pretraining take a look at [T5 Training](./t5#training).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size -
            1]`. All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ...,
            config.vocab_size]`.

        Examples:

        ```python
        >>> from transformers import AutoProcessor, UdopForConditionalGeneration
        >>> from datasets import load_dataset

        >>> # load model and processor
        >>> # in this case, we already have performed OCR ourselves
        >>> # so we initialize the processor with `apply_ocr=False`
        >>> processor = AutoProcessor.from_pretrained("microsoft/udop-large", apply_ocr=False)
        >>> model = UdopForConditionalGeneration.from_pretrained("microsoft/udop-large")

        >>> # load an example image, along with the words and coordinates
        >>> # which were extracted using an OCR engine
        >>> dataset = load_dataset("nielsr/funsd-layoutlmv3", split="train")
        >>> example = dataset[0]
        >>> image = example["image"]
        >>> words = example["tokens"]
        >>> boxes = example["bboxes"]

        >>> # one can use the various task prefixes (prompts) used during pre-training
        >>> # e.g. the task prefix for DocVQA is "Question answering. "
        >>> question = "Question answering. What is the date on the form?"
        >>> encoding = processor(image, question, text_pair=words, boxes=boxes, return_tensors="pt")

        >>> # autoregressive generation
        >>> predicted_ids = model.generate(**encoding)
        >>> print(processor.batch_decode(predicted_ids, skip_special_tokens=True)[0])
        9/30/92
        ```N©r‘  rˆ   r{   r¹   r*   r–   r   rC  r1   r‹  )Úignore_indexr0   )	ÚlossÚlogitsrH   rD  rE  rK   rF  rj  rG  rS   )r“  r1  r*   r3  r´   rˆ  rÙ   r{  r   r\   r8   r   rH   rI   rJ   rK   rG   )r³   r‘  r*   rˆ   r¹   r{   r>  r?  r–   r@  rH   rA  r  rP  r.   rI   rk  rH  Úsequence_outputÚ	lm_logitsrT  Úloss_fcts                         rA   r¾   z$UdopForConditionalGeneration.forward¦  s�  € ðX Ð$¨Ð);Ø $× 1Ò 1°&Ñ 9Ô 9Ðð Ð"Ø*˜dœlð Ø#ØØ'Ø)Ø-Ø+ðð ð ðð ˆOð (¨Ô*ˆØ!0Ô!?Ðð '˜$œ,ð 	
Ø'Ø1Ø/Ø+Ø"/Ø#9Øð	
ð 	
ð ð	
ð 	
ˆð *¨!Ô,ˆàŒ;Ô*ð 	LØ-°´Ô1DÀdÑ1JÑKˆOà—L’L Ñ1Ô1ˆ	àˆØÐÝ'°TÐ:Ñ:Ô:ˆHØ�8˜IŸNšN¨2¨y¯~ª~¸bÑ/AÔ/AÑBÔBÀFÇKÂKÐPRÁOÄOÑTÔTˆDåØØØ+Ô;Ø"1Ô"?Ø.Ô9Ø,Ô=Ø&5Ô&GØ"1Ô"?Ø.Ô9ð

ñ 

ô 

ð 
	
rC   )NNNNNNNNNNNNN)rL   rM   rN   rI  r«   r9  r  r!   r   r
   rÎ  rÏ  r	   r   r’   r   r   r   r¾   r¿   rÀ   s   @rA   r†  r†  t  sÓ  ø€ € € € € ð (7Ø'6Ø-FØ+Bð JPð JPØ)ðð Ððð ð ð ð ð.ð ð ð:ð :ð :ð
 Øð $(Ø(,Ø&*Ø&*Ø-1Ø+/Ø04Ø'+Ø)-Ø(,Ø/3Ø!%Ø $ð~
ð ~
à˜D‘=ð~
ð  ™ð~
ð �3˜�8Œn˜tÑ#ð	~
ð
 ˜t‘mð~
ð ˜#˜s˜(”^ dÑ*ð~
ð " D™=ð~
ð !'¨¡ð~
ð  ‘}ð~
ð   $™ð~
ð  ™ð~
ð  &¨™}ð~
ð ˜$‘;ð~
ð ˜‘ð~
ð Ð+Ô,ð~
ð  
ð!~
ð ~
ð ~
ñ „^ñ Ôð~
ð ~
ð ~
ð ~
ð ~
rC   r†  c                   óä   ‡ — e Zd ZdddddœZdefˆ fd„Zd„ Zd	„ Zee		 	 	 	 	 	 dde
d
z  deeef         d
z  de
d
z  de
d
z  deeef         d
z  de
d
z  dee         defd„¦   «         ¦   «         Zˆ xZS )ÚUdopEncoderModelr(  r)  r*  rK  )r+  r-  r.  rL  r´   c                 óR  •— t          ¦   «                              |¦  «         t          j        |j        |j        ¦  «        | _        t          |¦  «        | _        t          |¦  «        }d|_
        d|_        d|_        t          |¦  «        | _        |                      ¦   «          d S )NF)rª   r«   r   r  r  rÙ   r…  r¦   r0  r   rý   r  r  rþ  r1  r
  )r³   r´   r5  rµ   s      €rA   r«   zUdopEncoderModel.__init__2  s‘   ø€ Ý‰Œ×Ò˜Ñ Ô Ð õ ”l 6Ô#4°f´nÑEÔEˆŒÝ.¨vÑ6Ô6ˆÔå! &Ñ)Ô)ˆØ$)ˆÔ!Ø#(ˆÔ Ø,1ˆÔ)Ý  Ñ0Ô0ˆŒð 	�ŠÑÔÐÐÐrC   c                 ó   — | j         S rã   r8  r  s    rA   r9  z%UdopEncoderModel.get_input_embeddingsB  r:  rC   c                 óH   — || _         | j                             |¦  «         d S rã   )r…  r1  r  r  s     rA   r  z%UdopEncoderModel.set_input_embeddingsE  s%   € Ø$ˆŒØŒ×)Ò)¨.Ñ9Ô9Ð9Ð9Ð9rC   Nr‘  rˆ   r*   r¹   r{   r–   r.   r´  c           
      ó.   —  | j         d||||||dœ|¤Ž}|S )aÎ	  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you
            should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            To know more on how to prepare `input_ids` for pretraining take a look a [T5 Training](./t5#training).
        bbox (`torch.LongTensor` of shape `({0}, 4)`, *optional*):
            Bounding boxes of each input sequence tokens. Selected in the range `[0,
            config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1)
            format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1,
            y1) represents the position of the lower right corner.

            Note that `sequence_length = token_sequence_length + patch_sequence_length + 1` where `1` is for [CLS]
            token. See `pixel_values` for `patch_sequence_length`.
        visual_bbox (`torch.LongTensor` of shape `(batch_size, patch_sequence_length, 4)`, *optional*):
            Bounding boxes of each patch in the image. If not provided, bounding boxes are created in the model.

        Example:

        ```python
        >>> from transformers import AutoProcessor, UdopEncoderModel
        >>> from huggingface_hub import hf_hub_download
        >>> from datasets import load_dataset

        >>> # load model and processor
        >>> # in this case, we already have performed OCR ourselves
        >>> # so we initialize the processor with `apply_ocr=False`
        >>> processor = AutoProcessor.from_pretrained("microsoft/udop-large", apply_ocr=False)
        >>> model = UdopEncoderModel.from_pretrained("microsoft/udop-large")

        >>> # load an example image, along with the words and coordinates
        >>> # which were extracted using an OCR engine
        >>> dataset = load_dataset("nielsr/funsd-layoutlmv3", split="train")
        >>> example = dataset[0]
        >>> image = example["image"]
        >>> words = example["tokens"]
        >>> boxes = example["bboxes"]
        >>> encoding = processor(image, words, boxes=boxes, return_tensors="pt")

        >>> outputs = model(**encoding)
        >>> last_hidden_states = outputs.last_hidden_state
        ```rR  rS   )r1  )	r³   r‘  rˆ   r*   r¹   r{   r–   r.   r@  s	            rA   r¾   zUdopEncoderModel.forwardI  sF   € ðr '˜$œ,ð 
ØØØ#Ø%Ø)Ø'ð
ð 
ð ð
ð 
ˆð ÐrC   rr  )rL   rM   rN   rI  r   r«   r9  r  r!   r   r
   rÎ  rÏ  r	   r   r   rF   r¾   r¿   rÀ   s   @rA   rZ  rZ  )  sT  ø€ € € € € ð (7Ø-FØ+Bð JPð	ð Ðð˜zð ð ð ð ð ð ð ð ð ð:ð :ð :ð Øð $(Ø&*Ø(,Ø&*Ø-1Ø'+ðAð Aà˜D‘=ðAð �3˜�8Œn˜tÑ#ðAð  ™ð	Að
 ˜t‘mðAð ˜#˜s˜(”^ dÑ*ðAð  ‘}ðAð Ð+Ô,ðAð 
*ðAð Að Añ „^ñ ÔðAð Að Að Að ArC   rZ  )r†  rt  r„  rZ  )Nr%   N)rT   rU   )r   )Nrr   r   rT   rU   )_rO   r®   Úloggingr  r¼  r¯   r   r   Úcollections.abcr   r   Úcopyr   Údataclassesr   Útypingr	   r9   r
   r   Útorch.nnr   Útransformersr   Útransformers.modeling_outputsr   r   r  r   r~  Úactivationsr   Úcache_utilsr   r   r   Ú
generationr   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r    Úutils.genericr!   r"   Úutils.output_capturingr#   r$   Ú	getLoggerrL   r  ÚModuler  rB   rF   rc   rq   r¤   r¦   rÂ   rÓ   rç   rï   rù   rL  rX  r^  rt  rŸ  r   r¸  r¾  r‚  rÑ  rÚ  rã  rè  rö  rü  rþ  r„  r†  rZ  Ú__all__rS   rC   rA   ú<module>rt     sä  ðð Ð à Ð Ð Ð Ø €€€Ø €€€Ø €€€Ø #Ð #Ð #Ð #Ð #Ð #Ð #Ð #Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø Ð Ð Ð Ð Ð Ø !Ð !Ð !Ð !Ð !Ð !Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð Ð Ð Ø %Ð %Ð %Ð %Ð %Ð %à #Ð #Ð #Ð #Ð #Ð #ðð ð ð ð ð ð ð ð
 'Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CØ )Ð )Ð )Ð )Ð )Ð )Ø JÐ JÐ JÐ JÐ JÐ JÐ JÐ JØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &ðð ð ð ð ð ð ð ð ð ð ð ð JÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ EÐ EÐ EÐ EÐ EÐ EÐ EÐ Eð 
ˆÔ	˜8Ñ	$Ô	$€ð !ØØ)-ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð �T‰\ð%ð ð%ð ”< $Ñ&ð%ð Ð'Ô(ð%ð %ð %ð %ð@ €ððñ ô ð ð#=ð #=ð #=ð #=ð #= {ñ #=ô #=ñ „ñô ð#=ðLð ð ð ð.
ð 
ð 
ð 
ð$ ØØØØðP/ð P/ð P/ð P/ðfð ð ð ð ˜"œ)ñ ô ð ð<+ð +ð +ð +ð +�B”Iñ +ô +ð +ð4ð ð ð ð ˜œ	ñ ô ð ð.ð ð ð ð ˜RœYñ ô ð ð<ð ð ð ð �"”)ñ ô ð ð&A8ð A8ð A8ð A8ð A8�B”Iñ A8ô A8ð A8ðJ=ð =ð =ð =ð =˜RœYñ =ô =ð =ðB9ð 9ð 9ð 9ð 9˜bœiñ 9ô 9ð 9ð@LPð LPð LPð LPð LPÐ*ñ LPô LPð LPð^ ðW!ð W!ð W!ð W!ð W!˜/ñ W!ô W!ñ „ðW!ðtð ð ð ð ˜œñ ô ð ð>  -ÔFÐ Ø!Ð ðhð hð hð hð h˜rœy¨#ñ hô hð hðV!ð !ð !ð !ð !Ð5ñ !ô !ð !ð$?ð ?ð ?ð ?ð ?Ð%=ñ ?ô ?ð ?ð&=ð =ð =ð =ð =Ð#;ñ =ô =ð =ð&ð ð ð ð  R¤Yñ ô ð ð* !Ø0Ø,ðð €ð ð °Ð9QÔ0Rð ð ð ð ð,y
ð y
ð y
ð y
ð y
Ð#ñ y
ô y
ð y
ðx ðO
ð O
ð O
ð O
ð O
Ð#ñ O
ô O
ñ „ðO
ðd €ððñ ô ðj
ð j
ð j
ð j
ð j
Ð#6¸ñ j
ô j
ñô ðj
ðZ ðbð bð bð bð bÐ*ñ bô bñ „ðbðJ cÐ
bÐ
b€€€rC   