§
    9²¸jáY  ã                   ó0  — d Z ddlZddlmZ ddlmZmZmZ ddlmZ	 ddl
mZ ddlmZmZmZmZmZmZmZ ddlmZ dd	lmZ dd
lmZmZmZ ddlmZ ddlmZm Z m!Z!m"Z"m#Z#m$Z$m%Z% ddl&m'Z'  ej(        e)¦  «        Z* G d„ de!¦  «        Z+ G d„ de%¦  «        Z, G d„ de ¦  «        Z- G d„ de"¦  «        Z.e G d„ de¦  «        ¦   «         Z/e G d„ de$¦  «        ¦   «         Z0 G d„ de#¦  «        Z1 G d„ de¦  «        Z2 ed ¬!¦  «         G d"„ d#e/e¦  «        ¦   «         Z3e G d$„ d%e/¦  «        ¦   «         Z4 ed&¬!¦  «         G d'„ d(e/¦  «        ¦   «         Z5e G d)„ d*e/¦  «        ¦   «         Z6e G d+„ d,e/¦  «        ¦   «         Z7e G d-„ d.e/¦  «        ¦   «         Z8g d/¢Z9dS )0zPyTorch Data2VecText model.é    N)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚGenerationMixin)Ú,BaseModelOutputWithPoolingAndCrossAttentionsÚ!CausalLMOutputWithCrossAttentionsÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)Úcan_return_tupleé   )ÚRobertaClassificationHeadÚRobertaCrossAttentionÚRobertaEmbeddingsÚRobertaLayerÚRobertaLMHeadÚRobertaModelÚRobertaSelfAttentioné   )ÚData2VecTextConfigc                   ó   — e Zd ZdS )ÚData2VecTextEmbeddingsN©Ú__name__Ú
__module__Ú__qualname__© ó    ú‚/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/data2vec/modular_data2vec_text.pyr!   r!   2   ó   € € € € € Ø€Dr'   r!   c                   ó   — e Zd ZdS )ÚData2VecTextSelfAttentionNr"   r&   r'   r(   r+   r+   6   r)   r'   r+   c                   ó   — e Zd ZdS )ÚData2VecTextCrossAttentionNr"   r&   r'   r(   r-   r-   :   r)   r'   r-   c                   ó   — e Zd ZdS )ÚData2VecTextLayerNr"   r&   r'   r(   r/   r/   >   r)   r'   r/   c                   óN   ‡ — e Zd ZeZdZdZddgZdZdZ	dZ
dZeeedœZˆ fd„Zˆ xZS )ÚData2VecTextPreTrainedModelÚdata2vec_textTr!   r/   )Úhidden_statesÚ
attentionsÚcross_attentionsc                 óH  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rjt	          j        |j        t          j        |j        j	        d         ¦  «         
                    d¦  «        ¦  «         t	          j        |j        ¦  «         d S d S )Néÿÿÿÿ)r   r7   )ÚsuperÚ_init_weightsÚ
isinstancer!   ÚinitÚcopy_Úposition_idsÚtorchÚarangeÚshapeÚexpandÚzeros_Útoken_type_ids)ÚselfÚmoduleÚ	__class__s     €r(   r9   z)Data2VecTextPreTrainedModel._init_weightsR   s‹   ø€ Ý‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ4Ñ5Ô5ð 	/ÝŒJ�vÔ*­E¬L¸Ô9LÔ9RÐSUÔ9VÑ,WÔ,W×,^Ò,^Ð_fÑ,gÔ,gÑhÔhÐhÝŒK˜Ô-Ñ.Ô.Ð.Ð.Ð.ð	/ð 	/r'   )r#   r$   r%   r   Úconfig_classÚbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendr/   r+   r-   Ú_can_record_outputsr9   Ú__classcell__©rF   s   @r(   r1   r1   B   s‚   ø€ € € € € à%€LØ'ÐØ&*Ð#Ø1Ð3FÐGÐØÐØ€NØÐØ"&Ðà*Ø/Ø6ðð Ðð/ð /ð /ð /ð /ð /ð /ð /ð /r'   r1   c                   ó   — e Zd ZdS )ÚData2VecTextModelNr"   r&   r'   r(   rS   rS   Y   s   € € € € € à€Dr'   rS   c                   ó   — e Zd ZdS )ÚData2VecTextLMHeadNr"   r&   r'   r(   rU   rU   ^   r)   r'   rU   c                   ó   — e Zd ZdS )ÚData2VecTextClassificationHeadNr"   r&   r'   r(   rW   rW   b   r)   r'   rW   zX
    Data2VecText Model with a `language modeling` head on top for CLM fine-tuning.
    )Úcustom_introc                   óx  ‡ — e Zd ZdddœZˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 	 	 	 dd	e	j
        dz  d
e	j        dz  de	j
        dz  de	j
        dz  de	j        dz  de	j        dz  de	j        dz  de	j
        dz  deee	j                          dz  dedz  dee	j        z  dee         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚData2VecTextForCausalLMú/data2vec_text.embeddings.word_embeddings.weightúlm_head.bias©zlm_head.decoder.weightzlm_head.decoder.biasc                 ó  •— t          ¦   «                              |¦  «         |j        st                               d¦  «         t          |d¬¦  «        | _        t          |¦  «        | _        |  	                    ¦   «          d S )NzTIf you want to use `Data2VecTextLMHeadModel` as a standalone, add `is_decoder=True.`F©Úadd_pooling_layer©
r8   Ú__init__Ú
is_decoderÚloggerÚwarningrS   r2   rU   Úlm_headÚ	post_init©rD   ÚconfigrF   s     €r(   rb   z Data2VecTextForCausalLM.__init__q   sv   ø€ Ý‰Œ×Ò˜Ñ Ô Ð àÔ ð 	sÝ�NŠNÐqÑrÔrÐrå.¨vÈÐOÑOÔOˆÔÝ)¨&Ñ1Ô1ˆŒð 	�ŠÑÔÐÐÐr'   c                 ó   — | j         j        S ©N©rf   Údecoder©rD   s    r(   Úget_output_embeddingsz-Data2VecTextForCausalLM.get_output_embeddings}   ó   € ØŒ|Ô#Ð#r'   c                 ó   — || j         _        d S rk   rl   ©rD   Únew_embeddingss     r(   Úset_output_embeddingsz-Data2VecTextForCausalLM.set_output_embeddings€   ó   € Ø-ˆŒÔÐÐr'   Nr   Ú	input_idsÚattention_maskrC   r=   Úinputs_embedsÚencoder_hidden_statesÚencoder_attention_maskÚlabelsÚpast_key_valuesÚ	use_cacheÚlogits_to_keepÚkwargsÚreturnc                 ól  — |�d}
 | j         |f|||||||	|
ddœ	|¤Ž}|j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }d}|� | j        d||| j        j        dœ|¤Ž}t          |||j
        |j        |j        |j        ¬¦  «        S )aA  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
            `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
            ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`

        Example:

        ```python
        >>> from transformers import AutoTokenizer, Data2VecTextForCausalLM, Data2VecTextConfig
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("facebook/data2vec-text-base")
        >>> config = Data2VecTextConfig.from_pretrained("facebook/data2vec-text-base")
        >>> config.is_decoder = True
        >>> model = Data2VecTextForCausalLM.from_pretrained("facebook/data2vec-text-base", config=config)

        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
        >>> outputs = model(**inputs)

        >>> prediction_logits = outputs.logits
        ```NFT)	rw   rC   r=   rx   ry   rz   r|   r}   Úreturn_dict)Úlogitsr{   Ú
vocab_size)Úlossrƒ   r|   r3   r4   r5   r&   )r2   Úlast_hidden_stater:   ÚintÚslicerf   Úloss_functionri   r„   r
   r|   r3   r4   r5   )rD   rv   rw   rC   r=   rx   ry   rz   r{   r|   r}   r~   r   Úoutputsr3   Úslice_indicesrƒ   r…   s                     r(   ÚforwardzData2VecTextForCausalLM.forwardƒ   s  € ðN ÐØˆIà@RÀÔ@RØðA
à)Ø)Ø%Ø'Ø"7Ø#9Ø+ØØðA
ð A
ð ðA
ð A
ˆð  Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%Ðp¨V¸FÈtÌ{ÔOeÐpÐpÐioÐpÐpˆDå0ØØØ#Ô3Ø!Ô/ØÔ)Ø$Ô5ð
ñ 
ô 
ð 	
r'   )NNNNNNNNNNr   )r#   r$   r%   Ú_tied_weights_keysrb   ro   rt   r   r   r>   Ú
LongTensorÚFloatTensorÚtupleÚboolr‡   ÚTensorr   r   r
   rŒ   rP   rQ   s   @r(   rZ   rZ   f   s¾  ø€ € € € € ð #TØ .ðð Ðð

ð 
ð 
ð 
ð 
ð$ð $ð $ð.ð .ð .ð Øð .2Ø37Ø26Ø04Ø26Ø:>Ø;?Ø*.ØBFØ!%Ø-.ðF
ð F
àÔ# dÑ*ðF
ð Ô)¨DÑ0ðF
ð Ô(¨4Ñ/ð	F
ð
 Ô&¨Ñ-ðF
ð Ô(¨4Ñ/ðF
ð  %Ô0°4Ñ7ðF
ð !&Ô 1°DÑ 8ðF
ð Ô  4Ñ'ðF
ð ˜u UÔ%6Ô7Ô8¸4Ñ?ðF
ð ˜$‘;ðF
ð ˜eœlÑ*ðF
ð Ð+Ô,ðF
ð 
Ð2Ñ	2ðF
ð F
ð F
ñ „^ñ ÔðF
ð F
ð F
ð F
ð F
r'   rZ   c                   ó(  ‡ — e Zd ZdddœZˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 dde	j
        dz  d	e	j        dz  d
e	j
        dz  de	j
        dz  de	j        dz  de	j        dz  de	j        dz  de	j
        dz  dee         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚData2VecTextForMaskedLMr[   r\   r]   c                 ó  •— t          ¦   «                              |¦  «         |j        rt                               d¦  «         t          |d¬¦  «        | _        t          |¦  «        | _        |  	                    ¦   «          d S )NzsIf you want to use `Data2VecTextForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention.Fr_   ra   rh   s     €r(   rb   z Data2VecTextForMaskedLM.__init__Õ   s   ø€ Ý‰Œ×Ò˜Ñ Ô Ð àÔð 	Ý�NŠNð1ñô ð õ
 /¨vÈÐOÑOÔOˆÔÝ)¨&Ñ1Ô1ˆŒð 	�ŠÑÔÐÐÐr'   c                 ó   — | j         j        S rk   rl   rn   s    r(   ro   z-Data2VecTextForMaskedLM.get_output_embeddingsä   rp   r'   c                 ó   — || j         _        d S rk   rl   rr   s     r(   rt   z-Data2VecTextForMaskedLM.set_output_embeddingsç   ru   r'   Nrv   rw   rC   r=   rx   ry   rz   r{   r   r€   c	                 ót  —  | j         |f||||||ddœ|	¤Ž}
|
d         }|                      |¦  «        }d}|�et          ¦   «         }|                     |j        ¦  «        } ||                     d| j        j        ¦  «        |                     d¦  «        ¦  «        }t          |||
j	        |
j
        ¬¦  «        S )a¢  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        T)rw   rC   r=   rx   ry   rz   r‚   r   Nr7   ©r…   rƒ   r3   r4   )r2   rf   r   ÚtoÚdeviceÚviewri   r„   r   r3   r4   )rD   rv   rw   rC   r=   rx   ry   rz   r{   r   rŠ   Úsequence_outputÚprediction_scoresÚmasked_lm_lossÚloss_fcts                  r(   rŒ   zData2VecTextForMaskedLM.forwardê   sì   € ð( %�$Ô$Øð

à)Ø)Ø%Ø'Ø"7Ø#9Øð

ð 

ð ð

ð 

ˆð " !œ*ˆØ ŸLšL¨Ñ9Ô9ÐàˆØÐÝ'Ñ)Ô)ˆHà—Y’YÐ0Ô7Ñ8Ô8ˆFØ%˜XÐ&7×&<Ò&<¸RÀÄÔAWÑ&XÔ&XÐZ`×ZeÒZeÐfhÑZiÔZiÑjÔjˆNåØØ$Ø!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r'   )NNNNNNNN)r#   r$   r%   r�   rb   ro   rt   r   r   r>   rŽ   r�   r   r   r�   r   rŒ   rP   rQ   s   @r(   r”   r”   Î   sc  ø€ € € € € ð #TØ .ðð Ðð
ð ð ð ð ð$ð $ð $ð.ð .ð .ð Øð .2Ø37Ø26Ø04Ø26Ø:>Ø;?Ø*.ð,
ð ,
àÔ# dÑ*ð,
ð Ô)¨DÑ0ð,
ð Ô(¨4Ñ/ð	,
ð
 Ô&¨Ñ-ð,
ð Ô(¨4Ñ/ð,
ð  %Ô0°4Ñ7ð,
ð !&Ô 1°DÑ 8ð,
ð Ô  4Ñ'ð,
ð Ð+Ô,ð,
ð 
�Ñ	ð,
ð ,
ð ,
ñ „^ñ Ôð,
ð ,
ð ,
ð ,
ð ,
r'   r”   z¤
    Data2VecText Model transformer with a sequence classification/regression head on top (a linear layer on top of the
    pooled output) e.g. for GLUE tasks.
    c                   óæ   ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	e	e
         d
eez  fd„¦   «         ¦   «         Zˆ xZS )Ú%Data2VecTextForSequenceClassificationc                 óì   •— t          ¦   «                              |¦  «         |j        | _        || _        t	          |d¬¦  «        | _        t          |¦  «        | _        |                      ¦   «          d S ©NFr_   )	r8   rb   Ú
num_labelsri   rS   r2   rW   Ú
classifierrg   rh   s     €r(   rb   z.Data2VecTextForSequenceClassification.__init__"  sh   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒØˆŒå.¨vÈÐOÑOÔOˆÔÝ8¸Ñ@Ô@ˆŒð 	�ŠÑÔÐÐÐr'   Nrv   rw   rC   r=   rx   r{   r   r€   c           	      ó�  —  | j         |f||||ddœ|¤Ž}|d         }	|                      |	¦  «        }
d}|��t|                     |
j        ¦  «        }| j        j        €f| j        dk    rd| j        _        nN| j        dk    r7|j        t          j	        k    s|j        t          j
        k    rd| j        _        nd| j        _        | j        j        dk    rWt          ¦   «         }| j        dk    r1 ||
                     ¦   «         |                     ¦   «         ¦  «        }nŽ ||
|¦  «        }n�| j        j        dk    rGt          ¦   «         } ||
                     d	| j        ¦  «        |                     d	¦  «        ¦  «        }n*| j        j        dk    rt          ¦   «         } ||
|¦  «        }t!          ||
|j        |j        ¬
¦  «        S )a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        T©rw   rC   r=   rx   r‚   r   Nr   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr7   r™   )r2   r¦   rš   r›   ri   Úproblem_typer¥   Údtyper>   Úlongr‡   r   Úsqueezer   rœ   r   r   r3   r4   ©rD   rv   rw   rC   r=   rx   r{   r   rŠ   r�   rƒ   r…   r    s                r(   rŒ   z-Data2VecTextForSequenceClassification.forward-  sÞ  € ð$ %�$Ô$Øð
à)Ø)Ø%Ø'Øð
ð 
ð ð
ð 
ˆð " !œ*ˆØ—’ Ñ1Ô1ˆàˆØÑØ—Y’Y˜vœ}Ñ-Ô-ˆFàŒ{Ô'Ð/Ø”? aÒ'Ð'Ø/;�D”KÔ,Ð,Ø”_ qÒ(Ð(¨f¬l½e¼jÒ.HÐ.HÈFÌLÕ\aÔ\eÒLeÐLeØ/L�D”KÔ,Ð,à/K�D”KÔ,àŒ{Ô'¨<Ò7Ð7Ý"™9œ9�Ø”? aÒ'Ð'Ø#˜8 F§N¢NÑ$4Ô$4°f·n²nÑ6FÔ6FÑGÔG�D�Dà#˜8 F¨FÑ3Ô3�D�DØ”Ô)Ð-JÒJÐJÝ+Ñ-Ô-�Ø�x §¢¨B°´Ñ @Ô @À&Ç+Â+ÈbÁ/Ä/ÑRÔR��Ø”Ô)Ð-IÒIÐIÝ,Ñ.Ô.�Ø�x ¨Ñ/Ô/�å'ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r'   ©NNNNNN)r#   r$   r%   rb   r   r   r>   rŽ   r�   r   r   r�   r   rŒ   rP   rQ   s   @r(   r¢   r¢     s  ø€ € € € € ð	ð 	ð 	ð 	ð 	ð Øð .2Ø37Ø26Ø04Ø26Ø*.ð:
ð :
àÔ# dÑ*ð:
ð Ô)¨DÑ0ð:
ð Ô(¨4Ñ/ð	:
ð
 Ô&¨Ñ-ð:
ð Ô(¨4Ñ/ð:
ð Ô  4Ñ'ð:
ð Ð+Ô,ð:
ð 
Ð)Ñ	)ð:
ð :
ð :
ñ „^ñ Ôð:
ð :
ð :
ð :
ð :
r'   r¢   c                   óæ   ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	e	e
         d
eez  fd„¦   «         ¦   «         Zˆ xZS )ÚData2VecTextForMultipleChoicec                 ó  •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        ¦  «        | _        t	          j        |j	        d¦  «        | _
        |                      ¦   «          d S )Nr   )r8   rb   rS   r2   ÚnnÚDropoutÚhidden_dropout_probÚdropoutÚLinearÚhidden_sizer¦   rg   rh   s     €r(   rb   z&Data2VecTextForMultipleChoice.__init__n  sm   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å.¨vÑ6Ô6ˆÔÝ”z &Ô"<Ñ=Ô=ˆŒÝœ) FÔ$6¸Ñ:Ô:ˆŒð 	�ŠÑÔÐÐÐr'   Nrv   rC   rw   r{   r=   rx   r   r€   c           	      ó†  — |�|j         d         n|j         d         }|�)|                     d|                     d¦  «        ¦  «        nd}	|�)|                     d|                     d¦  «        ¦  «        nd}
|�)|                     d|                     d¦  «        ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�=|                     d|                     d¦  «        |                     d¦  «        ¦  «        nd} | j        |	f|
|||ddœ|¤Ž}|d         }|                      |¦  «        }|                      |¦  «        }|                     d|¦  «        }d}|�4t          ¦   «         }|                     |j        ¦  «        } |||¦  «        }t          |||j
        |j        ¬¦  «        S )a[  
        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        Nr   r7   éþÿÿÿT)r=   rC   rw   rx   r‚   r™   )r@   rœ   Úsizer2   r¸   r¦   r   rš   r›   r   r3   r4   )rD   rv   rC   rw   r{   r=   rx   r   Únum_choicesÚflat_input_idsÚflat_position_idsÚflat_token_type_idsÚflat_attention_maskÚflat_inputs_embedsrŠ   Úpooled_outputrƒ   Úreshaped_logitsr…   r    s                       r(   rŒ   z%Data2VecTextForMultipleChoice.forwardx  s  € ðT -6Ð,A�i”o aÔ(Ð(À}ÔGZÐ[\ÔG]ˆàCLÐCX˜Ÿš¨¨I¯NªN¸2Ñ,>Ô,>Ñ?Ô?Ð?Ð^bˆØLXÐLd˜L×-Ò-¨b°,×2CÒ2CÀBÑ2GÔ2GÑHÔHÐHÐjnÐØR`ÐRl˜n×1Ò1°"°n×6IÒ6IÈ"Ñ6MÔ6MÑNÔNÐNÐrvÐØR`ÐRl˜n×1Ò1°"°n×6IÒ6IÈ"Ñ6MÔ6MÑNÔNÐNÐrvÐð Ð(ð ×Ò˜r =×#5Ò#5°bÑ#9Ô#9¸=×;MÒ;MÈbÑ;QÔ;QÑRÔRÐRàð 	ð %�$Ô$Øð
à*Ø.Ø.Ø,Øð
ð 
ð ð
ð 
ˆð   œ
ˆàŸš ]Ñ3Ô3ˆØ—’ Ñ/Ô/ˆØ Ÿ+š+ b¨+Ñ6Ô6ˆàˆØÐÝ'Ñ)Ô)ˆHà—Y’Y˜Ô5Ñ6Ô6ˆFØ�8˜O¨VÑ4Ô4ˆDå(ØØ"Ø!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r'   r±   )r#   r$   r%   rb   r   r   r>   rŽ   r�   r   r   r�   r   rŒ   rP   rQ   s   @r(   r³   r³   l  s  ø€ € € € € ðð ð ð ð ð Øð .2Ø26Ø37Ø*.Ø04Ø26ðO
ð O
àÔ# dÑ*ðO
ð Ô(¨4Ñ/ðO
ð Ô)¨DÑ0ð	O
ð
 Ô  4Ñ'ðO
ð Ô&¨Ñ-ðO
ð Ô(¨4Ñ/ðO
ð Ð+Ô,ðO
ð 
Ð*Ñ	*ðO
ð O
ð O
ñ „^ñ ÔðO
ð O
ð O
ð O
ð O
r'   r³   c                   óæ   ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	e	e
         d
eez  fd„¦   «         ¦   «         Zˆ xZS )Ú"Data2VecTextForTokenClassificationc                 óZ  •— t          ¦   «                              |¦  «         |j        | _        t          |d¬¦  «        | _        |j        �|j        n|j        }t          j        |¦  «        | _	        t          j
        |j        |j        ¦  «        | _        |                      ¦   «          d S r¤   )r8   rb   r¥   rS   r2   Úclassifier_dropoutr·   rµ   r¶   r¸   r¹   rº   r¦   rg   )rD   ri   rÉ   rF   s      €r(   rb   z+Data2VecTextForTokenClassification.__init__Î  s›   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒå.¨vÈÐOÑOÔOˆÔà)/Ô)BÐ)NˆFÔ%Ð%ÐTZÔTnð 	õ ”zÐ"4Ñ5Ô5ˆŒÝœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐr'   Nrv   rw   rC   r=   rx   r{   r   r€   c           	      ó�  —  | j         |f||||ddœ|¤Ž}|d         }	|                      |	¦  «        }	|                      |	¦  «        }
d}|�`t          ¦   «         }|                     |
j        ¦  «        } ||
                     d| j        ¦  «        |                     d¦  «        ¦  «        }t          ||
|j	        |j
        ¬¦  «        S )zÛ
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        Tr¨   r   Nr7   r™   )r2   r¸   r¦   r   rš   r›   rœ   r¥   r   r3   r4   r°   s                r(   rŒ   z*Data2VecTextForTokenClassification.forwardÜ  sè   € ð  %�$Ô$Øð
à)Ø)Ø%Ø'Øð
ð 
ð ð
ð 
ˆð " !œ*ˆàŸ,š, Ñ7Ô7ˆØ—’ Ñ1Ô1ˆàˆØÐÝ'Ñ)Ô)ˆHà—Y’Y˜vœ}Ñ-Ô-ˆFØ�8˜FŸKšK¨¨D¬OÑ<Ô<¸f¿kºkÈ"¹o¼oÑNÔNˆDå$ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r'   r±   )r#   r$   r%   rb   r   r   r>   rŽ   r�   r   r   r�   r   rŒ   rP   rQ   s   @r(   rÇ   rÇ   Ì  s  ø€ € € € € ðð ð ð ð ð Øð .2Ø37Ø26Ø04Ø26Ø*.ð)
ð )
àÔ# dÑ*ð)
ð Ô)¨DÑ0ð)
ð Ô(¨4Ñ/ð	)
ð
 Ô&¨Ñ-ð)
ð Ô(¨4Ñ/ð)
ð Ô  4Ñ'ð)
ð Ð+Ô,ð)
ð 
Ð&Ñ	&ð)
ð )
ð )
ñ „^ñ Ôð)
ð )
ð )
ð )
ð )
r'   rÇ   c                   óü   ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	ej        dz  d
e	e
         deez  fd„¦   «         ¦   «         Zˆ xZS )Ú Data2VecTextForQuestionAnsweringc                 óþ   •— t          ¦   «                              |¦  «         |j        | _        t          |d¬¦  «        | _        t          j        |j        |j        ¦  «        | _        |  	                    ¦   «          d S r¤   )
r8   rb   r¥   rS   r2   rµ   r¹   rº   Ú
qa_outputsrg   rh   s     €r(   rb   z)Data2VecTextForQuestionAnswering.__init__  sk   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒå.¨vÈÐOÑOÔOˆÔÝœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐr'   Nrv   rw   rC   r=   rx   Ústart_positionsÚend_positionsr   r€   c           	      óF  —  | j         |f||||ddœ|¤Ž}	|	d         }
|                      |
¦  «        }|                     dd¬¦  «        \  }}|                     d¦  «                             ¦   «         }|                     d¦  «                             ¦   «         }d }|�ç|�åt          |                     ¦   «         ¦  «        dk    r|                     d¦  «        }t          |                     ¦   «         ¦  «        dk    r|                     d¦  «        }|                     d¦  «        }|                     d|¦  «        }|                     d|¦  «        }t          |¬¦  «        } |||¦  «        } |||¦  «        }||z   dz  }t          ||||	j
        |	j        ¬	¦  «        S )
NTr¨   r   r   r7   )Údim)Úignore_indexr   )r…   Ústart_logitsÚ
end_logitsr3   r4   )r2   rÎ   Úsplitr¯   Ú
contiguousÚlenr½   Úclampr   r   r3   r4   )rD   rv   rw   rC   r=   rx   rÏ   rÐ   r   rŠ   r�   rƒ   rÔ   rÕ   Ú
total_lossÚignored_indexr    Ú
start_lossÚend_losss                      r(   rŒ   z(Data2VecTextForQuestionAnswering.forward  sÐ  € ð %�$Ô$Øð
à)Ø)Ø%Ø'Øð
ð 
ð ð
ð 
ˆð " !œ*ˆà—’ Ñ1Ô1ˆØ#)§<¢<°°r <Ñ#:Ô#:Ñ ˆ�jØ#×+Ò+¨BÑ/Ô/×:Ò:Ñ<Ô<ˆØ×'Ò'¨Ñ+Ô+×6Ò6Ñ8Ô8ˆ
àˆ
ØÐ&¨=Ð+Då�?×'Ò'Ñ)Ô)Ñ*Ô*¨QÒ.Ð.Ø"1×"9Ò"9¸"Ñ"=Ô"=�Ý�=×%Ò%Ñ'Ô'Ñ(Ô(¨1Ò,Ð,Ø -× 5Ò 5°bÑ 9Ô 9�à(×-Ò-¨aÑ0Ô0ˆMØ-×3Ò3°A°}ÑEÔEˆOØ)×/Ò/°°=ÑAÔAˆMå'°]ÐCÑCÔCˆHØ!˜ ,°Ñ@Ô@ˆJØ�x 
¨MÑ:Ô:ˆHØ$ xÑ/°1Ñ4ˆJå+ØØ%Ø!Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r'   )NNNNNNN)r#   r$   r%   rb   r   r   r>   rŽ   r�   r   r   r�   r   rŒ   rP   rQ   s   @r(   rÌ   rÌ   
  s  ø€ € € € € ðð ð ð ð ð Øð .2Ø37Ø26Ø04Ø26Ø37Ø15ð3
ð 3
àÔ# dÑ*ð3
ð Ô)¨DÑ0ð3
ð Ô(¨4Ñ/ð	3
ð
 Ô&¨Ñ-ð3
ð Ô(¨4Ñ/ð3
ð Ô)¨DÑ0ð3
ð Ô'¨$Ñ.ð3
ð Ð+Ô,ð3
ð 
Ð-Ñ	-ð3
ð 3
ð 3
ñ „^ñ Ôð3
ð 3
ð 3
ð 3
ð 3
r'   rÌ   )rZ   r”   r³   rÌ   r¢   rÇ   rS   r1   ):Ú__doc__r>   Útorch.nnrµ   r   r   r   Ú r   r;   Ú
generationr   Úmodeling_outputsr	   r
   r   r   r   r   r   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   Úroberta.modeling_robertar   r   r   r   r   r   r   Úconfiguration_data2vec_textr   Ú
get_loggerr#   rd   r!   r+   r-   r/   r1   rS   rU   rW   rZ   r”   r¢   r³   rÇ   rÌ   Ú__all__r&   r'   r(   ú<module>rë      sÉ  ðð "Ð !à €€€Ø Ð Ð Ð Ð Ð Ø AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aà &Ð &Ð &Ð &Ð &Ð &Ø )Ð )Ð )Ð )Ð )Ð )ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð .Ð -Ð -Ð -Ð -Ð -Ø &Ð &Ð &Ð &Ð &Ð &Ø @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ø -Ð -Ð -Ð -Ð -Ð -ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð <Ð ;Ð ;Ð ;Ð ;Ð ;ð 
ˆÔ	˜HÑ	%Ô	%€ð	ð 	ð 	ð 	ð 	Ð.ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð 4ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð!6ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	˜ñ 	ô 	ð 	ð ð/ð /ð /ð /ð / /ñ /ô /ñ „ð/ð, ð	ð 	ð 	ð 	ð 	˜ñ 	ô 	ñ „ð	ð	ð 	ð 	ð 	ð 	˜ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð%>ñ 	ô 	ð 	ð €ððñ ô ð
`
ð `
ð `
ð `
ð `
Ð9¸?ñ `
ô `
ñô ð
`
ðF ðI
ð I
ð I
ð I
ð I
Ð9ñ I
ô I
ñ „ðI
ðX €ððñ ô ðH
ð H
ð H
ð H
ð H
Ð,Gñ H
ô H
ñô ðH
ðV ð\
ð \
ð \
ð \
ð \
Ð$?ñ \
ô \
ñ „ð\
ð~ ð:
ð :
ð :
ð :
ð :
Ð)Dñ :
ô :
ñ „ð:
ðz ð@
ð @
ð @
ð @
ð @
Ð'Bñ @
ô @
ñ „ð@
ðF	ð 	ð 	€€€r'   