§
    9²¸j“  ã                   óV  — d dl Z d dlmZ d dlmZ d dlZd dlmZ d dlmZ d dl	m
Z
mZmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZmZ ddlmZ ddlmZmZmZm Z m!Z!m"Z" ddl#m$Z$ ddl%m&Z&m'Z' ddl(m)Z) ddl*m+Z+m,Z,m-Z- ddl.m/Z/m0Z0 ddl1m2Z2 ddl3m4Z4 ddl5m6Z6m7Z7  e-j8        e9¦  «        Z: e,d¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z; G d„ dej<        ¦  «        Z= G d„ dej<        ¦  «        Z> G d „ d!e6¦  «        Z? ed"¦  «        d?d$„¦   «         Z@ ee@¦  «         G d%„ d&ej<        ¦  «        ¦   «         ZA G d'„ d(e¦  «        ZBe, G d)„ d*e'¦  «        ¦   «         ZCe, G d+„ d,eC¦  «        ¦   «         ZD G d-„ d.ej<        ¦  «        ZE e,d/¬0¦  «         G d1„ d2eC¦  «        ¦   «         ZF e,d3¬0¦  «         G d4„ d5eC¦  «        ¦   «         ZG e,d6¬0¦  «         G d7„ d8eC¦  «        ¦   «         ZHe, G d9„ d:eC¦  «        ¦   «         ZI e,d;¬0¦  «         G d<„ d=eC¦  «        ¦   «         ZJg d>¢ZKdS )@é    N)ÚCallable)ÚLiteral)Ústrict)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FN)ÚPreTrainedConfig)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_bidirectional_maskÚ(create_bidirectional_sliding_window_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚROPE_INIT_FUNCTIONS)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)Úcan_return_tupleÚmerge_with_config_defaults)Úcapture_outputsé   )Úeager_attention_forward)ÚGemma3RotaryEmbeddingÚrotate_halfzanswerdotai/ModernBERT-base)Ú
checkpointc                   óÔ  ‡ — e Zd ZU dZdZdgZdddœZdZee	d<   d	Z
ee	d
<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZedz  e	d<   d Zeee         z  dz  e	d!<   d"Zedz  e	d#<   d"Zedz  e	d$<   d Zedz  e	d%<   dZee	d&<   d'Zeez  e	d(<   dZee         dz  e	d)<   dZ e!e"d*         e!f         dz  e	d+<   d,Z#ee	d-<   d'Z$eez  e	d.<   dZ%ee	d/<   d'Z&eez  e	d0<   d1Z'ee	d2<   d3Z(e"d4         e	d5<   d'Z)eez  e	d6<   dZ*ee	d7<   dZ+ee	d8<   dZ,ee	d9<   dZ-ee	d:<   d;Z.ee	d<<   d1Z/ee	d=<   ˆ fd>„Z0d?„ Z1ˆ fd@„Z2e3dA„ ¦   «         Z4e4j5        dB„ ¦   «         Z4ˆ xZ6S )CÚModernBertConfiga+  
    initializer_cutoff_factor (`float`, *optional*, defaults to 2.0):
        The cutoff factor for the truncated_normal_initializer for initializing all weight matrices.
    norm_eps (`float`, *optional*, defaults to 1e-05):
        The epsilon used by the rms normalization layers.
    norm_bias (`bool`, *optional*, defaults to `False`):
        Whether to use bias in the normalization layers.
    local_attention (`int`, *optional*, defaults to 128):
        The window size for local attention.
    mlp_dropout (`float`, *optional*, defaults to 0.0):
        The dropout ratio for the MLP layers.
    decoder_bias (`bool`, *optional*, defaults to `True`):
        Whether to use bias in the decoder layers.
    classifier_pooling (`str`, *optional*, defaults to `"cls"`):
        The pooling method for the classifier. Should be either `"cls"` or `"mean"`. In local attention layers, the
        CLS token doesn't attend to all tokens on long sequences.
    classifier_bias (`bool`, *optional*, defaults to `False`):
        Whether to use bias in the classifier.
    classifier_activation (`str`, *optional*, defaults to `"gelu"`):
        The activation function for the classifier.
    deterministic_flash_attn (`bool`, *optional*, defaults to `False`):
        Whether to use deterministic flash attention. If `False`, inference will be faster but not deterministic.
    sparse_prediction (`bool`, *optional*, defaults to `False`):
        Whether to use sparse prediction for the masked language model instead of returning the full dense logits.
    sparse_pred_ignore_index (`int`, *optional*, defaults to -100):
        The index to ignore for the sparse prediction.

    Examples:

    ```python
    >>> from transformers import ModernBertModel, ModernBertConfig

    >>> # Initializing a ModernBert style configuration
    >>> configuration = ModernBertConfig()

    >>> # Initializing a model from the modernbert-base style configuration
    >>> model = ModernBertModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```Ú
modernbertÚpast_key_valuesg     ˆAg     ˆÃ@)ÚglobalÚlocaliÀÄ  Ú
vocab_sizei   Úhidden_sizei€  Úintermediate_sizeé   Únum_hidden_layersé   Únum_attention_headsÚgeluÚhidden_activationi    Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangeç       @Úinitializer_cutoff_factorgñhãˆµøä>Únorm_epsFÚ	norm_biasikÄ  NÚpad_token_idijÄ  Úeos_token_idiiÄ  Úbos_token_idÚcls_token_idÚsep_token_idÚattention_biasç        Úattention_dropoutÚlayer_types©Úfull_attentionÚsliding_attentionÚrope_parametersé€   Úlocal_attentionÚembedding_dropoutÚmlp_biasÚmlp_dropoutTÚdecoder_biasÚcls)rP   ÚmeanÚclassifier_poolingÚclassifier_dropoutÚclassifier_biasÚclassifier_activationÚdeterministic_flash_attnÚsparse_predictioniœÿÿÿÚsparse_pred_ignore_indexÚtie_word_embeddingsc                 óÀ   •‡— |                      dd¦  «        Š| j        €%ˆfd„t          | j        ¦  «        D ¦   «         | _         t	          ¦   «         j        di |¤Ž d S )NÚglobal_attn_every_n_layersr
   c                 ó<   •— g | ]}t          |‰z  ¦  «        rd nd‘ŒS ©rH   rG   )Úbool)Ú.0Úir[   s     €ú�/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/modernbert/modular_modernbert.pyú
<listcomp>z2ModernBertConfig.__post_init__.<locals>.<listcomp>‹   sC   ø€ ð  ð  ð  àõ (,¨AÐ0JÑ,JÑ'KÔ'KÐaÐ#Ð#ÐQað ð  ð  ó    © )ÚgetrE   Úranger2   ÚsuperÚ__post_init__)ÚselfÚkwargsr[   Ú	__class__s     @€ra   rh   zModernBertConfig.__post_init__‡   s}   øø€ à%+§Z¢ZÐ0LÈaÑ%PÔ%PÐ"ØÔÐ#ð ð  ð  ð  å˜tÔ5Ñ6Ô6ð ñ  ô  ˆDÔð
 	�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'rc   c                 ó²  — |                      dd ¦  «        }ddiddidœ}| j        �| j        n|| _        |�@| j        d                              |¦  «         | j        d                              |¦  «         | j                             d¦  «        €ddi| j        d<   | j        d                              d|                      d| j        d	         ¦  «        ¦  «         | j                             d¦  «        €ddi| j        d<   | j        d                              d|                      d
| j        d         ¦  «        ¦  «         |                      ¦   «          |S )NÚrope_scalingÚ	rope_typeÚdefaultr]   rG   rH   Ú
rope_thetaÚglobal_rope_thetar,   Úlocal_rope_thetar-   )ÚpoprI   Úupdatere   Ú
setdefaultÚdefault_thetaÚstandardize_rope_params)ri   rj   rm   Údefault_rope_paramss       ra   Úconvert_rope_params_to_dictz,ModernBertConfig.convert_rope_params_to_dict’   s|  € Ø—z’z .°$Ñ7Ô7ˆð
 #.¨yÐ!9Ø*¨IÐ6ð
ð 
Ðð 8<Ô7KÐ7W˜tÔ3Ð3Ð]pˆÔØÐ#ØÔ Ð!1Ô2×9Ò9¸,ÑGÔGÐGØÔ Ð!4Ô5×<Ò<¸\ÑJÔJÐJð Ô×#Ò#Ð$4Ñ5Ô5Ð=Ø6AÀ9Ð5MˆDÔ Ð!1Ñ2ØÔÐ-Ô.×9Ò9Ø˜&Ÿ*š*Ð%8¸$Ô:LÈXÔ:VÑWÔWñ	
ô 	
ð 	
ð Ô×#Ò#Ð$7Ñ8Ô8Ð@Ø9DÀiÐ8PˆDÔ Ð!4Ñ5ØÔÐ0Ô1×<Ò<Ø˜&Ÿ*š*Ð%7¸Ô9KÈGÔ9TÑUÔUñ	
ô 	
ð 	
ð
 	×$Ò$Ñ&Ô&Ð&Øˆrc   c                 ót   •— t          ¦   «                              ¦   «         }|                     dd ¦  «         |S )NÚreference_compile)rg   Úto_dictrs   )ri   Úoutputrk   s     €ra   r|   zModernBertConfig.to_dict°   s0   ø€ Ý‘”—’Ñ"Ô"ˆØ�
Š
Ð&¨Ñ-Ô-Ð-Øˆrc   c                 ó   — | j         dz  S )zKHalf-window size: `local_attention` is the total window, so we divide by 2.r#   ©rK   ©ri   s    ra   Úsliding_windowzModernBertConfig.sliding_windowµ   s   € ð Ô# qÑ(Ð(rc   c                 ó   — |dz  | _         dS )z<Set sliding_window by updating local_attention to 2 * value.r#   Nr   ©ri   Úvalues     ra   r�   zModernBertConfig.sliding_windowº   s   € ð  % q™yˆÔÐÐrc   )7Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferencerv   r.   ÚintÚ__annotations__r/   r0   r2   r4   r6   Ústrr7   r8   Úfloatr:   r;   r<   r^   r=   r>   Úlistr?   r@   rA   rB   rD   rE   rI   Údictr   rK   rL   rM   rN   rO   rR   rS   rT   rU   rV   rW   rX   rY   rh   ry   r|   Úpropertyr�   ÚsetterÚ__classcell__©rk   s   @ra   r)   r)   4   s5  ø€ € € € € € ð(ð (ðT €JØ#4Ð"5ÐØ(°8Ð<Ð<€Mà€J�ÐÐÑØ€K�ÐÐÑØ!Ð�sÐ!Ð!Ñ!ØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø#Ð�sÐ#Ð#Ñ#Ø#'Ð˜SÐ'Ð'Ñ'Ø#Ð�uÐ#Ð#Ñ#Ø'*Ð˜uÐ*Ð*Ñ*Ø€HˆeÐÐÑØ€IˆtÐÐÑØ$€L�#˜‘*Ð$Ð$Ñ$Ø+0€L�#˜˜Sœ	‘/ DÑ(Ð0Ð0Ñ0Ø$€L�#˜‘*Ð$Ð$Ñ$Ø$€L�#˜‘*Ð$Ð$Ñ$Ø$€L�#˜‘*Ð$Ð$Ñ$Ø €N�DÐ Ð Ñ Ø%(Ð�u˜s‘{Ð(Ð(Ñ(Ø$(€K��c”˜TÑ!Ð(Ð(Ñ(ØY]€O�T˜'Ð"GÔHÈ$ÐNÔOÐRVÑVÐ]Ð]Ñ]Ø€O�SÐÐÑØ%(Ð�u˜s‘{Ð(Ð(Ñ(Ø€HˆdÐÐÑØ"€K�˜‘Ð"Ð"Ñ"Ø€L�$ÐÐÑØ16Ð˜ Ô.Ð6Ð6Ñ6Ø&)Ð˜ ™Ð)Ð)Ñ)Ø!€O�TÐ!Ð!Ñ!Ø!'Ð˜3Ð'Ð'Ñ'Ø%*Ð˜dÐ*Ð*Ñ*Ø#Ð�tÐ#Ð#Ñ#Ø$(Ð˜cÐ(Ð(Ñ(Ø $Ð˜Ð$Ð$Ñ$ð	(ð 	(ð 	(ð 	(ð 	(ðð ð ð<ð ð ð ð ð
 ð)ð )ñ „Xð)ð Ôð)ð )ñ Ôð)ð )ð )ð )ð )rc   r)   c                   ój   ‡ — e Zd ZdZdefˆ fd„Z	 d	dej        dz  dej        dz  dej        fd„Z	ˆ xZ
S )
ÚModernBertEmbeddingszV
    Same as BertEmbeddings with a tiny tweak for positional embeddings indexing.
    Úconfigc                 ó>  •— t          ¦   «                              ¦   «          || _        t          j        |j        |j        |j        ¬¦  «        | _        t          j	        |j        |j
        |j        ¬¦  «        | _        t          j        |j        ¦  «        | _        d S )N)Úpadding_idx©ÚepsÚbias)rg   Ú__init__r—   r   Ú	Embeddingr.   r/   r=   Útok_embeddingsÚ	LayerNormr;   r<   ÚnormÚDropoutrL   Údrop©ri   r—   rk   s     €ra   r�   zModernBertEmbeddings.__init__Å   s{   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ œl¨6Ô+<¸fÔ>PÐ^dÔ^qÐrÑrÔrˆÔÝ”L Ô!3¸¼ÈvÔO_Ð`Ñ`Ô`ˆŒ	Ý”J˜vÔ7Ñ8Ô8ˆŒ	ˆ	ˆ	rc   NÚ	input_idsÚinputs_embedsÚreturnc                 óÒ   — |�)|                       |                      |¦  «        ¦  «        }n;|                       |                      |                      |¦  «        ¦  «        ¦  «        }|S ©N)r£   r¡   rŸ   )ri   r¥   r¦   Úhidden_statess       ra   ÚforwardzModernBertEmbeddings.forwardÌ   sZ   € ð Ð$Ø ŸIšI d§i¢i°Ñ&>Ô&>Ñ?Ô?ˆMˆMà ŸIšI d§i¢i°×0CÒ0CÀIÑ0NÔ0NÑ&OÔ&OÑPÔPˆMØÐrc   ©NN)r…   r†   r‡   rˆ   r)   r�   ÚtorchÚ
LongTensorÚTensorr«   r“   r”   s   @ra   r–   r–   À   s™   ø€ € € € € ðð ð9Ð/ð 9ð 9ð 9ð 9ð 9ð 9ð _cðð ØÔ)¨DÑ0ðØHMÌÐW[ÑH[ðà	Œðð ð ð ð ð ð ð rc   r–   c                   óL   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚModernBertMLPa6  Applies the GLU at the end of each ModernBERT layer.

    Compared to the default BERT architecture, this block replaces :class:`~transformers.model.bert.modeling_bert.BertIntermediate`
    and :class:`~transformers.model.bert.modeling_bert.SelfOutput` with a single module that has similar functionality.
    r—   c                 óŒ  •— t          ¦   «                              ¦   «          || _        t          j        |j        t          |j        ¦  «        dz  |j        ¬¦  «        | _	        t          |j                 | _        t          j        |j        ¦  «        | _        t          j        |j        |j        |j        ¬¦  «        | _        d S )Nr#   ©rœ   )rg   r�   r—   r   ÚLinearr/   r‹   r0   rM   ÚWir   r6   Úactr¢   rN   r£   ÚWor¤   s     €ra   r�   zModernBertMLP.__init__Ý   s—   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”)˜FÔ.µ°FÔ4LÑ0MÔ0MÐPQÑ0QÐX^ÔXgÐhÑhÔhˆŒÝ˜&Ô2Ô3ˆŒÝ”J˜vÔ1Ñ2Ô2ˆŒ	Ý”)˜FÔ4°fÔ6HÈvÌÐ_Ñ_Ô_ˆŒˆˆrc   rª   r§   c                 óØ   — |                       |¦  «                             dd¬¦  «        \  }}|                      |                      |                      |¦  «        |z  ¦  «        ¦  «        S )Nr#   éÿÿÿÿ©Údim)rµ   Úchunkr·   r£   r¶   )ri   rª   ÚinputÚgates       ra   r«   zModernBertMLP.forwardå   sW   € Ø—g’g˜mÑ,Ô,×2Ò2°1¸"Ð2Ñ=Ô=‰ˆˆtØ�wŠw�t—y’y §¢¨%¡¤°4Ñ!7Ñ8Ô8Ñ9Ô9Ð9rc   )
r…   r†   r‡   rˆ   r)   r�   r­   r¯   r«   r“   r”   s   @ra   r±   r±   Ö   s|   ø€ € € € € ðð ð`Ð/ð `ð `ð `ð `ð `ð `ð: U¤\ð :°e´lð :ð :ð :ð :ð :ð :ð :ð :rc   r±   c            	       ób   ‡ — e Zd Zddefˆ fd„Z	 ddededz  deej        e	f         fˆ fd„Z
ˆ xZS )	ÚModernBertRotaryEmbeddingNr—   c                 óJ   •— t          ¦   «                              |¦  «         d S r©   )rg   r�   )ri   r—   Údevicerk   s      €ra   r�   z"ModernBertRotaryEmbedding.__init__ë   s!   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ð Ð rc   Ú
layer_typer§   c                 óH   •— t          ¦   «                              | |¦  «        S r©   )rg   Úcompute_default_rope_parameters)r—   rÂ   rÃ   rj   rk   s       €ra   rÅ   z9ModernBertRotaryEmbedding.compute_default_rope_parametersî   s   ø€ õ ‰wŒw×6Ò6°v¸zÑJÔJÐJrc   r©   r¬   )r…   r†   r‡   r)   r�   r�   Útupler­   r¯   rŽ   rÅ   r“   r”   s   @ra   rÀ   rÀ   ê   sª   ø€ € € € € ð!ð !Ð/ð !ð !ð !ð !ð !ð !ð IMðKð KØ ðKØ;>À¹:ðKà	ˆuŒ|˜UÐ"Ô	#ðKð Kð Kð Kð Kð Kð Kð Kð Kð Krc   rÀ   Úrotary_pos_embé   c                 ó¨  — | j         }|                     |¦  «        }|                     |¦  «        }|                      ¦   «         |z  t          |                      ¦   «         ¦  «        |z  z   }|                     ¦   «         |z  t          |                     ¦   «         ¦  «        |z  z   }|                     |¦  «        |                     |¦  «        fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )ÚdtypeÚ	unsqueezerŽ   r&   Úto)ÚqÚkÚcosÚsinÚunsqueeze_dimÚoriginal_dtypeÚq_embedÚk_embeds           ra   Úapply_rotary_pos_embrÕ   ô   s¦   € ð& ”W€NØ
�-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�wŠw‰yŒy˜3‰¥;¨q¯wªw©y¬yÑ#9Ô#9¸CÑ#?Ñ@€GØ�wŠw‰yŒy˜3‰¥;¨q¯wªw©y¬yÑ#9Ô#9¸CÑ#?Ñ@€GØ�:Š:�nÑ%Ô% w§z¢z°.Ñ'AÔ'AÐAÐArc   c                   óÐ   ‡ — e Zd ZdZddededz  fˆ fd„Z	 	 ddej        de	ej        ej        f         dz  dej        dz  d	e
e         d
e	ej        ej        dz  f         f
d„Zˆ xZS )ÚModernBertAttentiona‚  Performs multi-headed self attention on a batch of unpadded sequences.

    If Flash Attention 2 is installed, this module uses Flash Attention to improve throughput.
    If Flash Attention 2 is not installed, the implementation will use PyTorch's SDPA kernel,
    which requires padding and unpadding inputs, adding some overhead.

    See `forward` method for additional details.
    Nr—   Ú	layer_idxc                 óº  •— t          ¦   «                              ¦   «          || _        || _        |j        |j        z  dk    r t          d|j        › d|j        › d�¦  «        ‚|j        | _        |j        | _        |j        |j        z  | _	        t          j        |j        d| j	        z  |j        z  |j        ¬¦  «        | _        |j        |         dk    r|j        dz   | _        nd | _        d	| _        t          j        |j        |j        |j        ¬¦  «        | _        |j        d
k    rt          j        |j        ¦  «        nt          j        ¦   «         | _        d S )Nr   zThe hidden size (z6) is not a multiple of the number of attention heads (ú)r
   r³   rH   rÈ   FrC   )rg   r�   r—   rØ   r/   r4   Ú
ValueErrorrD   rV   Úhead_dimr   r´   rB   ÚWqkvrE   r�   Ú	is_causalr·   r¢   ÚIdentityÚout_drop©ri   r—   rØ   rk   s      €ra   r�   zModernBertAttention.__init__  sq  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒàÔ Ô :Ñ:¸aÒ?Ð?Ýð L FÔ$6ð  Lð  LÐntô  oIð  Lð  Lð  Lñô ð ð "(Ô!9ˆÔØ(.Ô(GˆÔ%ØÔ*¨fÔ.HÑHˆŒÝ”IØÔ  D¤MÑ 1°FÔ4NÑ NÐU[ÔUjð
ñ 
ô 
ˆŒ	ð Ô˜iÔ(Ð,?Ò?Ð?ð #)Ô"7¸!Ñ";ˆDÔÐà"&ˆDÔàˆŒå”)˜FÔ.°Ô0BÈÔI^Ð_Ñ_Ô_ˆŒØ@FÔ@XÐ[^Ò@^Ð@^�œ
 6Ô#;Ñ<Ô<Ð<ÕdfÔdoÑdqÔdqˆŒˆˆrc   rª   Úposition_embeddingsÚattention_maskrj   r§   c                 ó¶  — |j         d d…         }|                      |¦  «        } |j        g |¢d‘d‘| j        ‘R Ž }|                     d¬¦  «        \  }}}	|                     dd¦  «        }|                     dd¦  «        }|	                     dd¦  «        }	|\  }
}t          |||
|d¬¦  «        \  }}t          j        | j	        j
        t          ¦  «        } || |||	|f| j        r| j        nd| j        d	z  | j        | j        d
œ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |                      |¦  «        ¦  «        }||fS )Nr¹   r
   éýÿÿÿrº   rÈ   r#   )rÑ   rC   ç      à¿)ÚdropoutÚscalingr�   Údeterministic)ÚshaperÝ   ÚviewrÜ   ÚunbindÚ	transposerÕ   r   Úget_interfacer—   Ú_attn_implementationr$   ÚtrainingrD   r�   rV   ÚreshapeÚ
contiguousrà   r·   )ri   rª   râ   rã   rj   Úinput_shapeÚqkvÚquery_statesÚ
key_statesÚvalue_statesrÏ   rÐ   Úattention_interfaceÚattn_outputÚattn_weightss                  ra   r«   zModernBertAttention.forward7  s§  € ð $Ô)¨#¨2¨#Ô.ˆà�iŠi˜Ñ&Ô&ˆØˆcŒhÐ:˜Ð: QÐ:¨Ð:¨D¬MÐ:Ð:Ð:ˆØ14·²À°Ñ1CÔ1CÑ.ˆ�j ,à#×-Ò-¨a°Ñ3Ô3ˆØ×)Ò)¨!¨QÑ/Ô/ˆ
Ø#×-Ò-¨a°Ñ3Ô3ˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÐjkÐ#lÑ#lÔ#lÑ ˆ�jå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð%
ð /3¬mÐD�DÔ*Ð*ÀØ”M 4Ñ'ØÔ.ØÔ7ð%
ð %
ð ð%
ð %
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—m’m D§G¢G¨KÑ$8Ô$8Ñ9Ô9ˆØ˜LÐ(Ð(rc   r©   r¬   )r…   r†   r‡   rˆ   r)   r‹   r�   r­   r¯   rÆ   r   r   r«   r“   r”   s   @ra   r×   r×     sñ   ø€ € € € € ðð ðrð rÐ/ð r¸CÀ$¹Jð rð rð rð rð rð rð@ IMØ.2ð	')ð ')à”|ð')ð # 5¤<°´Ð#=Ô>ÀÑEð')ð œ tÑ+ð	')ð
 Ð+Ô,ð')ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð')ð ')ð ')ð ')ð ')ð ')ð ')ð ')rc   r×   c                   ó’   ‡ — e Zd Zddededz  fˆ fd„Z	 	 ddej        dej        dz  dej        dz  dee	         d	ej        f
d
„Z
ˆ xZS )ÚModernBertEncoderLayerNr—   rØ   c                 óÆ  •— t          ¦   «                              ¦   «          || _        || _        |dk    rt	          j        ¦   «         | _        n+t	          j        |j        |j	        |j
        ¬¦  «        | _        t          ||¬¦  «        | _        t	          j        |j        |j	        |j
        ¬¦  «        | _        t          |¦  «        | _        |j        |         | _        d S )Nr   rš   )r—   rØ   )rg   r�   r—   rØ   r   rß   Ú	attn_normr    r/   r;   r<   r×   ÚattnÚmlp_normr±   ÚmlprE   Úattention_typerá   s      €ra   r�   zModernBertEncoderLayer.__init__b  sº   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒØ˜Š>ˆ>Ýœ[™]œ]ˆDŒNˆNåœ\¨&Ô*<À&Ä/ÐX^ÔXhÐiÑiÔiˆDŒNÝ'¨vÀÐKÑKÔKˆŒ	Ýœ VÔ%7¸V¼_ÐSYÔScÐdÑdÔdˆŒÝ  Ñ(Ô(ˆŒØ$Ô0°Ô;ˆÔÐÐrc   rª   rã   râ   rj   r§   c                 ó´   —  | j         |                      |¦  «        f||dœ|¤Ž\  }}||z   }||                      |                      |¦  «        ¦  «        z   }|S )N)râ   rã   )rÿ   rþ   r  r   )ri   rª   rã   râ   rj   rù   Ú_s          ra   r«   zModernBertEncoderLayer.forwardo  sx   € ð #˜œØ�NŠN˜=Ñ)Ô)ð
à 3Ø)ð
ð 
ð ð	
ð 
‰ˆ�Qð &¨Ñ3ˆØ%¨¯ª°·²¸}Ñ1MÔ1MÑ(NÔ(NÑNˆØÐrc   r©   r¬   )r…   r†   r‡   r)   r‹   r�   r­   r¯   r   r   r«   r“   r”   s   @ra   rü   rü   a  s¾   ø€ € € € € ð<ð <Ð/ð <¸CÀ$¹Jð <ð <ð <ð <ð <ð <ð  /3Ø37ð	ð à”|ðð œ tÑ+ðð #œ\¨DÑ0ð	ð
 Ð+Ô,ðð 
Œðð ð ð ð ð ð ð rc   rü   c                   óŽ   ‡ — e Zd ZU eed<   dZdZddgZdZdZ	dZ
dZeedœZ ej        ¦   «         dej        fˆ fd„¦   «         Zˆ xZS )	ÚModernBertPreTrainedModelr—   ÚmodelTr–   rü   )rª   Ú
attentionsÚmodulec                 óT  •‡— t          ¦   «                              |¦  «         | j        j        Š‰€dŠdt          j        dt          fˆfd„}| j        j        | j        j        t          j	        d| j        j
        z  ¦  «        z  | j        j        | j        j        dz  dœ}t          |t          ¦  «        r ||j        |d         ¦  «         d S t          |t          ¦  «        r0 ||j        |d	         ¦  «          ||j        |d
         ¦  «         d S t          |t$          ¦  «        r0 ||j        |d	         ¦  «          ||j        |d
         ¦  «         d S t          |t(          ¦  «        r ||j        |d
         ¦  «         d S t          |t,          ¦  «        r ||j        |d
         ¦  «         d S t          |t0          t2          t4          t6          f¦  «        r ||j        |d         ¦  «         d S t          |t:          ¦  «        r›|j        D ]•}|j        }|j         |         dk    rtB          |j         |                  } ||j        |¬¦  «        \  }}tE          j#        tI          ||› d�¦  «        |¦  «         tE          j#        tI          ||› d�¦  «        |¦  «         Œ”d S d S )Nr
   r	  Ústdc                 óÎ   •— t          j        | j        d|‰ |z  ‰|z  ¬¦  «         t          | t          j        ¦  «        r"| j        �t          j        | j        ¦  «         d S d S d S )NrC   )rQ   r  ÚaÚb)ÚinitÚtrunc_normal_ÚweightÚ
isinstancer   r´   rœ   Úzeros_)r	  r  Úcutoff_factors     €ra   Úinit_weightz<ModernBertPreTrainedModel._init_weights.<locals>.init_weight˜  s€   ø€ ÝÔØ”ØØØ �. 3Ñ&Ø #Ñ%ðñ ô ð õ ˜&¥"¤)Ñ,Ô,ð -Ø”;Ð*Ý”K ¤Ñ,Ô,Ð,Ð,Ð,ð-ð -Ø*Ð*rc   r9   ræ   )ÚinÚoutÚ	embeddingÚ	final_outr  r  r  r  ro   )rÃ   Ú	_inv_freqÚ_original_inv_freq)%rg   Ú_init_weightsr—   r:   r   ÚModulerŽ   r8   ÚmathÚsqrtr2   r/   r  r–   rŸ   r±   rµ   r·   r×   rÝ   ÚModernBertPredictionHeadÚdenseÚModernBertForMaskedLMÚdecoderÚ#ModernBertForSequenceClassificationÚModernBertForMultipleChoiceÚ ModernBertForTokenClassificationÚModernBertForQuestionAnsweringÚ
classifierrÀ   rE   rÅ   rn   r   r  Úcopy_Úgetattr)
ri   r	  r  ÚstdsrÃ   Úrope_init_fnÚcurr_inv_freqr  r  rk   s
           @€ra   r  z'ModernBertPreTrainedModel._init_weights‘  sö  øø€ å‰Œ×Ò˜fÑ%Ô%Ð%ØœÔ=ˆØÐ ØˆMð	-¥¤	ð 	-µð 	-ð 	-ð 	-ð 	-ð 	-ð 	-ð ”+Ô/Ø”;Ô0µ4´9¸SÀ4Ä;ÔC`Ñ=`Ñ3aÔ3aÑaØœÔ6ØœÔ0°$Ñ6ð	
ð 
ˆõ �fÕ2Ñ3Ô3ð 	^ØˆK˜Ô-¨t°KÔ/@ÑAÔAÐAÐAÐAÝ˜¥Ñ.Ô.ð 	^ØˆK˜œ	 4¨¤:Ñ.Ô.Ð.ØˆK˜œ	 4¨¤;Ñ/Ô/Ð/Ð/Ð/Ý˜Õ 3Ñ4Ô4ð 	^ØˆK˜œ T¨$¤ZÑ0Ô0Ð0ØˆK˜œ	 4¨¤;Ñ/Ô/Ð/Ð/Ð/Ý˜Õ 8Ñ9Ô9ð 	^ØˆK˜œ d¨5¤kÑ2Ô2Ð2Ð2Ð2Ý˜Õ 5Ñ6Ô6ð 	^ØˆK˜œ¨¨U¬Ñ4Ô4Ð4Ð4Ð4ÝØå3Ý+Ý0Ý.ð	ñ
ô 
ð 	^ð ˆK˜Ô)¨4°Ô+<Ñ=Ô=Ð=Ð=Ð=Ý˜Õ 9Ñ:Ô:ð 	^Ø$Ô0ð ^ð ^�
Ø%ÔE�ØÔ# JÔ/°9Ò<Ð<Ý#6°vÔ7GÈ
Ô7SÔ#T�LØ#/ <°´È*Ð#UÑ#UÔ#UÑ �˜qÝ”
�7 6¨jÐ+CÐ+CÐ+CÑDÔDÀmÑTÔTÐTÝ”
�7 6¨jÐ+LÐ+LÐ+LÑMÔMÈ}Ñ]Ô]Ð]Ð]ð	^ð 	^ð^ð ^rc   )r…   r†   r‡   r)   rŒ   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendrü   r×   Ú_can_record_outputsr­   Úno_gradr   r  r  r“   r”   s   @ra   r  r  �  s·   ø€ € € € € € àÐÐÑØÐØ&*Ð#Ø/Ð1IÐJÐØÐØ€NØÐØ"&Ðð 0Ø)ðð Ðð
 €U„]�_„_ð7^ B¤Ið 7^ð 7^ð 7^ð 7^ð 7^ñ „_ð7^ð 7^ð 7^ð 7^ð 7^rc   r  c                   óÖ   ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zeee		 	 	 	 dde
j        dz  de
j        dz  de
j        dz  d	e
j        dz  d
ee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚModernBertModelr—   c                 ó¦  •‡— t          ¦   «                              ‰¦  «         ‰| _        t          ‰¦  «        | _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _	        t          j
        ‰j        ‰j        ‰j        ¬¦  «        | _        t          ‰¬¦  «        | _        d| _        |                      ¦   «          d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS rd   )rü   )r_   rØ   r—   s     €ra   rb   z,ModernBertModel.__init__.<locals>.<listcomp>Ó  s$   ø€ ÐhÐhÐh¸9Õ# F¨IÑ6Ô6ÐhÐhÐhrc   rš   )r—   F)rg   r�   r—   r–   Ú
embeddingsr   Ú
ModuleListrf   r2   Úlayersr    r/   r;   r<   Ú
final_normrÀ   Ú
rotary_embÚgradient_checkpointingÚ	post_initr¤   s    `€ra   r�   zModernBertModel.__init__Î  sº   øø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒÝ.¨vÑ6Ô6ˆŒÝ”mØhÐhÐhÐhÍÈfÔNfÑHgÔHgÐhÑhÔhñ
ô 
ˆŒõ œ, vÔ'9¸v¼ÐU[ÔUeÐfÑfÔfˆŒÝ3¸6ÐBÑBÔBˆŒØ&+ˆÔ#Ø�ŠÑÔÐÐÐrc   c                 ó   — | j         j        S r©   ©r;  rŸ   r€   s    ra   Úget_input_embeddingsz$ModernBertModel.get_input_embeddingsÚ  s   € ØŒÔ-Ð-rc   c                 ó   — || j         _        d S r©   rC  rƒ   s     ra   Úset_input_embeddingsz$ModernBertModel.set_input_embeddingsÝ  s   € Ø).ˆŒÔ&Ð&Ð&rc   Nr¥   rã   Úposition_idsr¦   rj   r§   c                 ó–  — |d u |d uz  rt          d¦  «        ‚|�|j        d         n|j        d         }|�|j        n|j        }|€)t          j        ||¬¦  «                             d¦  «        }|                      ||¬¦  «        }t          |x}	t          ¦  «        s$| j	        ||dœ}
t          d
i |
¤Žt          d
i |
¤Ždœ}	i }t          | j	        j        ¦  «        D ]}|                      |||¦  «        ||<   Œ| j        D ]$} ||f|	|j                 ||j                 dœ|¤Ž}Œ%|                      |¦  «        }t%          |¬	¦  «        S )Nz:You must specify exactly one of input_ids or inputs_embedsrÈ   ©rÂ   r   )r¥   r¦   )r—   r¦   rã   rF   )rã   râ   )Úlast_hidden_staterd   )rÛ   rê   rÂ   r­   ÚarangerË   r;  r  r�   r—   r   r   ÚsetrE   r?  r=  r  r>  r   )ri   r¥   rã   rG  r¦   rj   Úseq_lenrÂ   rª   Úattention_mask_mappingÚmask_kwargsrâ   rÃ   Úencoder_layers                 ra   r«   zModernBertModel.forwardà  s·  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZà,9Ð,E�-Ô% aÔ(Ð(È9Ì?Ð[\ÔK]ˆØ%.Ð%:�Ô!Ð!ÀÔ@TˆàÐÝ œ<¨¸Ð?Ñ?Ô?×IÒIÈ!ÑLÔLˆLàŸš°)È=˜ÑYÔYˆå°NÐBÐ0ÅDÑIÔIð 		àœ+Ø!.Ø"0ðð ˆKõ #<Ð"JÐ"J¸kÐ"JÐ"JÝ%MÐ%\Ð%\ÐP[Ð%\Ð%\ð&ð &Ð"ð
 !ÐÝ˜dœkÔ5Ñ6Ô6ð 	gð 	gˆJØ.2¯oªo¸mÈ\Ð[eÑ.fÔ.fÐ 
Ñ+Ð+à!œ[ð 	ð 	ˆMØ)˜MØðà5°mÔ6RÔSØ$7¸Ô8TÔ$Uðð ð ð	ð ˆMˆMð Ÿš¨Ñ6Ô6ˆå°Ð?Ñ?Ô?Ð?rc   )NNNN)r…   r†   r‡   r)   r�   rD  rF  r!   r"   r   r­   r®   r¯   r   r   r   r«   r“   r”   s   @ra   r8  r8  Ì  s  ø€ € € € € ð
Ð/ð 
ð 
ð 
ð 
ð 
ð 
ð.ð .ð .ð/ð /ð /ð  ØØð .2Ø.2Ø04Ø-1ð,@ð ,@àÔ# dÑ*ð,@ð œ tÑ+ð,@ð Ô&¨Ñ-ð	,@ð
 ”| dÑ*ð,@ð Ð+Ô,ð,@ð 
ð,@ð ,@ð ,@ñ „^ñ „_ñ  Ôð,@ð ,@ð ,@ð ,@ð ,@rc   r8  c                   óH   ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Zˆ xZS )r   r—   c                 ó.  •— t          ¦   «                              ¦   «          || _        t          j        |j        |j        |j        ¦  «        | _        t          |j	                 | _
        t          j        |j        |j        |j        ¬¦  «        | _        d S )Nrš   )rg   r�   r—   r   r´   r/   rT   r!  r   rU   r¶   r    r;   r<   r¡   r¤   s     €ra   r�   z!ModernBertPredictionHead.__init__  sq   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”Y˜vÔ1°6Ô3EÀvÔG]Ñ^Ô^ˆŒ
Ý˜&Ô6Ô7ˆŒÝ”L Ô!3¸¼ÈvÔO_Ð`Ñ`Ô`ˆŒ	ˆ	ˆ	rc   rª   r§   c                 óx   — |                       |                      |                      |¦  «        ¦  «        ¦  «        S r©   )r¡   r¶   r!  )ri   rª   s     ra   r«   z ModernBertPredictionHead.forward  s,   € Ø�yŠy˜Ÿš $§*¢*¨]Ñ";Ô";Ñ<Ô<Ñ=Ô=Ð=rc   )	r…   r†   r‡   r)   r�   r­   r¯   r«   r“   r”   s   @ra   r   r     sr   ø€ € € € € ðaÐ/ð að að að að að að> U¤\ð >°e´lð >ð >ð >ð >ð >ð >ð >ð >rc   r   zd
    The ModernBert Model with a decoder head on top that is used for masked language modeling.
    )Úcustom_introc                   ó  ‡ — e Zd ZddiZdefˆ fd„Zd„ Zdej        fd„Z	e
e	 	 	 	 	 dd	ej        dz  d
ej        dz  dej        dz  dej        dz  dej        dz  dee         deej                 ez  fd„¦   «         ¦   «         Zˆ xZS )r"  zdecoder.weightz&model.embeddings.tok_embeddings.weightr—   c                 ój  •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        t          |¦  «        | _        t          j        |j	        |j
        |j        ¬¦  «        | _        | j        j        | _        | j        j        | _        |                      ¦   «          d S )Nr³   )rg   r�   r—   r8  r  r   Úheadr   r´   r/   r.   rO   r#  rW   rX   rA  r¤   s     €ra   r�   zModernBertForMaskedLM.__init__&  s“   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒÝ$ VÑ,Ô,ˆŒ
Ý,¨VÑ4Ô4ˆŒ	Ý”y Ô!3°VÔ5FÈVÔM`ÐaÑaÔaˆŒà!%¤Ô!>ˆÔØ(,¬Ô(LˆÔ%ð 	�ŠÑÔÐÐÐrc   c                 ó   — | j         S r©   ©r#  r€   s    ra   Úget_output_embeddingsz+ModernBertForMaskedLM.get_output_embeddings3  s
   € ØŒ|Ðrc   Únew_embeddingsc                 ó   — || _         d S r©   rY  )ri   r[  s     ra   Úset_output_embeddingsz+ModernBertForMaskedLM.set_output_embeddings6  s   € Ø%ˆŒˆˆrc   Nr¥   rã   rG  r¦   Úlabelsrj   r§   c                 ó²  —  | j         d||||dœ|¤Ž}|d         }| j        rS|�Q|                     d¦  «        }|                     |j        d         d¦  «        }|| j        k    }	||	         }||	         }|                      |                      |¦  «        ¦  «        }
d }|� | j        |
|fd| j        j	        i|¤Ž}t          ||
|j        |j        ¬¦  «        S )N©r¥   rã   rG  r¦   r   r¹   r.   ©ÚlossÚlogitsrª   r  rd   )r  rW   rë   rê   rX   r#  rW  Úloss_functionr—   r.   r   rª   r  )ri   r¥   rã   rG  r¦   r^  rj   ÚoutputsrJ  Úmask_tokensrc  rb  s               ra   r«   zModernBertForMaskedLM.forward9  s  € ð �$”*ð 
ØØ)Ø%Ø'ð	
ð 
ð
 ð
ð 
ˆð $ AœJÐàÔ!ð 	) fÐ&8à—[’[ ‘_”_ˆFØ 1× 6Ò 6°v´|ÀA´ÈÑ KÔ KÐð ! DÔ$AÒAˆKØ 1°+Ô >ÐØ˜KÔ(ˆFà—’˜dŸišiÐ(9Ñ:Ô:Ñ;Ô;ˆàˆØÐØ%�4Ô% f¨fÐbÐbÀÄÔAWÐbÐ[aÐbÐbˆDåØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
rc   ©NNNNN)r…   r†   r‡   Ú_tied_weights_keysr)   r�   rZ  r   r´   r]  r    r   r­   r®   r¯   r   r   rÆ   r   r«   r“   r”   s   @ra   r"  r"    s:  ø€ € € € € ð +Ð,TÐUÐðÐ/ð ð ð ð ð ð ðð ð ð&°B´Ið &ð &ð &ð &ð Øð .2Ø.2Ø,0Ø-1Ø&*ð'
ð '
àÔ# dÑ*ð'
ð œ tÑ+ð'
ð ”l TÑ)ð	'
ð
 ”| dÑ*ð'
ð ”˜tÑ#ð'
ð Ð+Ô,ð'
ð 
ˆuŒ|Ô	˜~Ñ	-ð'
ð '
ð '
ñ „^ñ Ôð'
ð '
ð '
ð '
ð '
rc   r"  z`
    The ModernBert Model with a sequence classification head on top that performs pooling.
    c                   óì   ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 ddej        dz  dej	        dz  dej	        dz  dej	        dz  dej	        dz  d	e
e         d
eej	                 ez  fd„¦   «         ¦   «         Zˆ xZS )r$  r—   c                 ó‚  •— t          ¦   «                              |¦  «         |j        | _        || _        t	          |¦  «        | _        t          |¦  «        | _        t          j	         
                    |j        ¦  «        | _        t          j        |j        |j        ¦  «        | _        |                      ¦   «          d S r©   )rg   r�   Ú
num_labelsr—   r8  r  r   rW  r­   r   r¢   rS   r£   r´   r/   r(  rA  r¤   s     €ra   r�   z,ModernBertForSequenceClassification.__init__k  s•   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒØˆŒå$ VÑ,Ô,ˆŒ
Ý,¨VÑ4Ô4ˆŒ	Ý”H×$Ò$ VÔ%>Ñ?Ô?ˆŒ	Ýœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐrc   Nr¥   rã   rG  r¦   r^  rj   r§   c                 óø  —  | j         d||||dœ|¤Ž}|d         }| j        j        dk    r|dd…df         }n‰| j        j        dk    ry|€3t          j        |j        dd…         |j        t          j        ¬¦  «        }||                     d¦  «        z   	                    d	¬
¦  «        | 	                    d	d¬¦  «        z  }|  
                    |¦  «        }	|                      |	¦  «        }	|                      |	¦  «        }
d}|��Z| j        j        €f| j        d	k    rd| j        _        nN| j        d	k    r7|j        t          j        k    s|j        t          j        k    rd| j        _        nd| j        _        | j        j        dk    rWt%          ¦   «         }| j        d	k    r1 ||
                     ¦   «         |                     ¦   «         ¦  «        }nŽ ||
|¦  «        }n�| j        j        dk    rGt)          ¦   «         } ||
                     d| j        ¦  «        |                     d¦  «        ¦  «        }n*| j        j        dk    rt-          ¦   «         } ||
|¦  «        }t/          ||
|j        |j        ¬¦  «        S )a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        r`  r   rP   NrQ   r#   )rÂ   rÊ   r¹   rÈ   rº   T©r»   ÚkeepdimÚ
regressionÚsingle_label_classificationÚmulti_label_classificationra  rd   )r  r—   rR   r­   Úonesrê   rÂ   r^   rË   ÚsumrW  r£   r(  Úproblem_typerk  rÊ   Úlongr‹   r	   Úsqueezer   rë   r   r   rª   r  )ri   r¥   rã   rG  r¦   r^  rj   re  rJ  Úpooled_outputrc  rb  Úloss_fcts                ra   r«   z+ModernBertForSequenceClassification.forwardx  sª  € ð" �$”*ð 
ØØ)Ø%Ø'ð	
ð 
ð
 ð
ð 
ˆð $ AœJÐàŒ;Ô)¨UÒ2Ð2Ø 1°!°!°!°Q°$Ô 7ÐÐØŒ[Ô+¨vÒ5Ð5ØÐ%Ý!&¤Ø%Ô+¨B¨Q¨BÔ/Ð8IÔ8PÕX]ÔXbð"ñ "ô "�ð "3°^×5MÒ5MÈbÑ5QÔ5QÑ!Q× VÒ VÐ[\Ð VÑ ]Ô ]Ð`n×`rÒ`rØ˜tð asñ aô añ !Ðð Ÿ	š	Ð"3Ñ4Ô4ˆØŸ	š	 -Ñ0Ô0ˆØ—’ Ñ/Ô/ˆàˆØÑØŒ{Ô'Ð/Ø”? aÒ'Ð'Ø/;�D”KÔ,Ð,Ø”_ qÒ(Ð(¨f¬l½e¼jÒ.HÐ.HÈFÌLÕ\aÔ\eÒLeÐLeØ/L�D”KÔ,Ð,à/K�D”KÔ,àŒ{Ô'¨<Ò7Ð7Ý"™9œ9�Ø”? aÒ'Ð'Ø#˜8 F§N¢NÑ$4Ô$4°f·n²nÑ6FÔ6FÑGÔG�D�Dà#˜8 F¨FÑ3Ô3�D�DØ”Ô)Ð-JÒJÐJÝ+Ñ-Ô-�Ø�x §¢¨B°´Ñ @Ô @À&Ç+Â+ÈbÁ/Ä/ÑRÔR��Ø”Ô)Ð-IÒIÐIÝ,Ñ.Ô.�Ø�x ¨Ñ/Ô/�å'ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
rc   rg  )r…   r†   r‡   r)   r�   r    r   r­   r®   r¯   r   r   rÆ   r   r«   r“   r”   s   @ra   r$  r$  e  s  ø€ € € € € ðÐ/ð ð ð ð ð ð ð Øð .2Ø.2Ø,0Ø-1Ø&*ðC
ð C
àÔ# dÑ*ðC
ð œ tÑ+ðC
ð ”l TÑ)ð	C
ð
 ”| dÑ*ðC
ð ”˜tÑ#ðC
ð Ð+Ô,ðC
ð 
ˆuŒ|Ô	Ð7Ñ	7ðC
ð C
ð C
ñ „^ñ ÔðC
ð C
ð C
ð C
ð C
rc   r$  zv
    The ModernBert Model with a token classification head on top, e.g. for Named Entity Recognition (NER) tasks.
    c                   óì   ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 ddej        dz  dej	        dz  dej	        dz  dej	        dz  dej	        dz  d	e
e         d
eej	                 ez  fd„¦   «         ¦   «         Zˆ xZS )r&  r—   c                 ót  •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          |¦  «        | _        t          j         	                    |j
        ¦  «        | _        t          j        |j        |j        ¦  «        | _        |                      ¦   «          d S r©   ©rg   r�   rk  r8  r  r   rW  r­   r   r¢   rS   r£   r´   r/   r(  rA  r¤   s     €ra   r�   z)ModernBertForTokenClassification.__init__Æ  sŽ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒå$ VÑ,Ô,ˆŒ
Ý,¨VÑ4Ô4ˆŒ	Ý”H×$Ò$ VÔ%>Ñ?Ô?ˆŒ	Ýœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐrc   Nr¥   rã   rG  r¦   r^  rj   r§   c                 ó‚  —  | j         d||||dœ|¤Ž}|d         }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }	d}
|�Ft	          ¦   «         } ||	                     d| j        ¦  «        |                     d¦  «        ¦  «        }
t          |
|	|j        |j	        ¬¦  «        S )zÛ
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        r`  r   Nr¹   ra  rd   )
r  rW  r£   r(  r   rë   rk  r   rª   r  )ri   r¥   rã   rG  r¦   r^  rj   re  rJ  rc  rb  rx  s               ra   r«   z(ModernBertForTokenClassification.forwardÒ  så   € ð �$”*ð 
ØØ)Ø%Ø'ð	
ð 
ð
 ð
ð 
ˆð $ AœJÐà ŸIšIÐ&7Ñ8Ô8ÐØ ŸIšIÐ&7Ñ8Ô8ÐØ—’Ð!2Ñ3Ô3ˆàˆØÐÝ'Ñ)Ô)ˆHØ�8˜FŸKšK¨¨D¬OÑ<Ô<¸f¿kºkÈ"¹o¼oÑNÔNˆDå$ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
rc   rg  )r…   r†   r‡   r)   r�   r    r   r­   r®   r¯   r   r   rÆ   r   r«   r“   r”   s   @ra   r&  r&  À  s  ø€ € € € € ð
Ð/ð 
ð 
ð 
ð 
ð 
ð 
ð Øð .2Ø.2Ø,0Ø-1Ø&*ð$
ð $
àÔ# dÑ*ð$
ð œ tÑ+ð$
ð ”l TÑ)ð	$
ð
 ”| dÑ*ð$
ð ”˜tÑ#ð$
ð Ð+Ô,ð$
ð 
ˆuŒ|Ô	Ð4Ñ	4ð$
ð $
ð $
ñ „^ñ Ôð$
ð $
ð $
ð $
ð $
rc   r&  c                   óì   ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	e	e
         d
eej                 ez  fd„¦   «         ¦   «         Zˆ xZS )r'  r—   c                 ót  •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          |¦  «        | _        t          j         	                    |j
        ¦  «        | _        t          j        |j        |j        ¦  «        | _        |                      ¦   «          d S r©   r{  r¤   s     €ra   r�   z'ModernBertForQuestionAnswering.__init__ý  sŒ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒå$ VÑ,Ô,ˆŒ
Ý,¨VÑ4Ô4ˆŒ	Ý”H×$Ò$ VÔ%>Ñ?Ô?ˆŒ	Ýœ) FÔ$6¸Ô8IÑJÔJˆŒà�ŠÑÔÐÐÐrc   Nr¥   rã   rG  Ústart_positionsÚend_positionsrj   r§   c                 óì  —  | j         |f||dœ|¤Ž}|d         }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }	|	                     dd¬¦  «        \  }
}|
                     d¦  «                             ¦   «         }
|                     d¦  «                             ¦   «         }d }|�|� | j        |
|||fi |¤Ž}t          ||
||j	        |j
        ¬¦  «        S )N)rã   rG  r   rÈ   r¹   rº   )rb  Ústart_logitsÚ
end_logitsrª   r  )r  rW  r£   r(  Úsplitrv  rò   rd  r   rª   r  )ri   r¥   rã   rG  r  r€  rj   re  rJ  rc  r‚  rƒ  rb  s                ra   r«   z&ModernBertForQuestionAnswering.forward  s.  € ð �$”*Øð
à)Ø%ð
ð 
ð ð	
ð 
ˆð $ AœJÐà ŸIšIÐ&7Ñ8Ô8ÐØ ŸIšIÐ&7Ñ8Ô8ÐØ—’Ð!2Ñ3Ô3ˆà#)§<¢<°°r <Ñ#:Ô#:Ñ ˆ�jØ#×+Ò+¨BÑ/Ô/×:Ò:Ñ<Ô<ˆØ×'Ò'¨Ñ+Ô+×6Ò6Ñ8Ô8ˆ
àˆØÐ&¨=Ð+DØ%�4Ô% l°JÀÐQ^ÐiÐiÐbhÐiÐiˆDå+ØØ%Ø!Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
rc   rg  )r…   r†   r‡   r)   r�   r    r   r­   r¯   r   r   rÆ   r   r«   r“   r”   s   @ra   r'  r'  û  s   ø€ € € € € ð	Ð/ð 	ð 	ð 	ð 	ð 	ð 	ð Øð *.Ø.2Ø,0Ø/3Ø-1ð#
ð #
à”< $Ñ&ð#
ð œ tÑ+ð#
ð ”l TÑ)ð	#
ð
 œ¨Ñ,ð#
ð ”| dÑ*ð#
ð Ð+Ô,ð#
ð 
ˆuŒ|Ô	Ð;Ñ	;ð#
ð #
ð #
ñ „^ñ Ôð#
ð #
ð #
ð #
ð #
rc   r'  z«
    The ModernBert Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a softmax) e.g. for RocStories/SWAG tasks.
    c                   óì   ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 ddej        dz  dej	        dz  dej	        dz  dej	        dz  dej	        dz  d	e
e         d
eej	                 ez  fd„¦   «         ¦   «         Zˆ xZS )r%  r—   c                 ó`  •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        t          |¦  «        | _        t          j         	                    |j
        ¦  «        | _        t          j        |j        d¦  «        | _        |                      ¦   «          d S )NrÈ   )rg   r�   r—   r8  r  r   rW  r­   r   r¢   rS   r£   r´   r/   r(  rA  r¤   s     €ra   r�   z$ModernBertForMultipleChoice.__init__6  sˆ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒå$ VÑ,Ô,ˆŒ
Ý,¨VÑ4Ô4ˆŒ	Ý”H×$Ò$ VÔ%>Ñ?Ô?ˆŒ	Ýœ) FÔ$6¸Ñ:Ô:ˆŒð 	�ŠÑÔÐÐÐrc   Nr¥   rã   rG  r¦   r^  rj   r§   c                 ó  — |�|j         d         n|j         d         }|�)|                     d|                     d¦  «        ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�=|                     d|                     d¦  «        |                     d¦  «        ¦  «        nd} | j        d
||||dœ|¤Ž}|d         }	| j        j        dk    r‰t          j        |	j         d         |	j        ¬¦  «        }
|�/| 	                    d¬	¦  «         
                    |	j        ¦  «        }n't          j        d
dt          j        |	j        ¬¦  «        }|	|
|f         }	nV| j        j        dk    rF|                     dd¬¦  «        }|	|                     d¦  «        z                       d¬	¦  «        |z  }	|                      |	¦  «        }|                      |¦  «        }|                      |¦  «        }|                     d|¦  «        }d}|�t%          j        ¦   «         } |||¦  «        }t)          |||j        |j        ¬¦  «        S )a&  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors.
        NrÈ   r¹   éþÿÿÿr`  r   rP   rI  rº   rd   )rÊ   rÂ   rQ   Trm  ra  )rê   rë   Úsizer  r—   rR   r­   rK  rÂ   ÚargmaxrÌ   Úfullru  rs  rË   rW  r£   r(  r   r   r   rª   r  )ri   r¥   rã   rG  r¦   r^  rj   Únum_choicesre  rJ  Ú	indices_0Úcls_maskÚnum_non_pad_tokensrw  rc  Úreshaped_logitsrb  rx  s                     ra   r«   z#ModernBertForMultipleChoice.forwardB  s¼  € ð  -6Ð,A�i”o aÔ(Ð(À}ÔGZÐ[\ÔG]ˆà>GÐ>S�I—N’N 2 y§~¢~°bÑ'9Ô'9Ñ:Ô:Ð:ÐY]ˆ	ØM[ÐMg˜×,Ò,¨R°×1DÒ1DÀRÑ1HÔ1HÑIÔIÐIÐmqˆØGSÐG_�|×(Ò(¨¨\×->Ò->¸rÑ-BÔ-BÑCÔCÐCÐeiˆð Ð(ð ×Ò˜r =×#5Ò#5°bÑ#9Ô#9¸=×;MÒ;MÈbÑ;QÔ;QÑRÔRÐRàð 	ð �$”*ð 
ØØ)Ø%Ø'ð	
ð 
ð
 ð
ð 
ˆð $ AœJÐð Œ;Ô)¨UÒ2Ð2ÝœÐ%6Ô%<¸QÔ%?ÐHYÔH`ÐaÑaÔaˆIàÐ)Ø)×0Ò0°RÐ0Ñ8Ô8×;Ò;Ð<MÔ<TÑUÔU��õ !œ: b¨!µ5´:ÐFWÔF^Ð_Ñ_Ô_�à 1°)¸XÐ2EÔ FÐÐð Œ[Ô+¨vÒ5Ð5Ø!/×!3Ò!3¸À4Ð!3Ñ!HÔ!HÐØ!2°^×5MÒ5MÈbÑ5QÔ5QÑ!Q× VÒ VÐ[\Ð VÑ ]Ô ]Ð`rÑ rÐàŸ	š	Ð"3Ñ4Ô4ˆØŸ	š	 -Ñ0Ô0ˆØ—’ Ñ/Ô/ˆà Ÿ+š+ b¨+Ñ6Ô6ˆàˆØÐÝÔ*Ñ,Ô,ˆHØ�8˜O¨VÑ4Ô4ˆDå(ØØ"Ø!Ô/ØÔ)ð	
ñ 
ô 
ð 	
rc   rg  )r…   r†   r‡   r)   r�   r    r   r­   r®   r¯   r   r   rÆ   r   r«   r“   r”   s   @ra   r%  r%  0  s  ø€ € € € € ð
Ð/ð 
ð 
ð 
ð 
ð 
ð 
ð Øð .2Ø.2Ø,0Ø-1Ø&*ðC
ð C
àÔ# dÑ*ðC
ð œ tÑ+ðC
ð ”l TÑ)ð	C
ð
 ”| dÑ*ðC
ð ”˜tÑ#ðC
ð Ð+Ô,ðC
ð 
ˆuŒ|Ô	Ð8Ñ	8ðC
ð C
ð C
ñ „^ñ ÔðC
ð C
ð C
ð C
ð C
rc   r%  )r)   r8  r  r"  r$  r&  r'  r%  )rÈ   )Lr  Úcollections.abcr   Útypingr   r­   Úhuggingface_hub.dataclassesr   r   Útorch.nnr   r   r	   Ú r   r  Úactivationsr   Úconfiguration_utilsr   Úintegrationsr   r   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_rope_utilsr   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr    r!   Úutils.output_capturingr"   Úalign.modeling_alignr$   Úgemma3.modeling_gemma3r%   r&   Ú
get_loggerr…   Úloggerr)   r  r–   r±   rÀ   rÕ   r×   rü   r  r8  r   r"  r$  r&  r'  r%  Ú__all__rd   rc   ra   ú<module>r§     sö  ðð  €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø Ð Ð Ð Ð Ð à €€€Ø .Ð .Ð .Ð .Ð .Ð .Ø Ð Ð Ð Ð Ð Ø AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aà &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø LÐ LÐ LÐ LÐ LÐ LÐ LÐ LØ `Ð `Ð `Ð `Ð `Ð `Ð `Ð `Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð 7Ð 6Ð 6Ð 6Ð 6Ð 6Ø FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø :Ð :Ð :Ð :Ð :Ð :Ø GÐ GÐ GÐ GÐ GÐ GÐ GÐ Gð 
ˆÔ	˜HÑ	%Ô	%€ð €Ð8Ð9Ñ9Ô9ØðG)ð G)ð G)ð G)ð G)Ð'ñ G)ô G)ñ „ñ :Ô9ðG)ðTð ð ð ð ˜2œ9ñ ô ð ð,:ð :ð :ð :ð :�B”Iñ :ô :ð :ð(Kð Kð Kð Kð KÐ 5ñ Kô Kð Kð ÐÐ-Ñ.Ô.ðBð Bð Bñ /Ô.ðBð4 ÐÐ)Ñ*Ô*ðN)ð N)ð N)ð N)ð N)˜"œ)ñ N)ô N)ñ +Ô*ðN)ðbð ð ð ð Ð7ñ ô ð ð@ ðG^ð G^ð G^ð G^ð G^ ñ G^ô G^ñ „ðG^ðT ðB@ð B@ð B@ð B@ð B@Ð/ñ B@ô B@ñ „ðB@ðJ	>ð 	>ð 	>ð 	>ð 	>˜rœyñ 	>ô 	>ð 	>ð €ððñ ô ð
?
ð ?
ð ?
ð ?
ð ?
Ð5ñ ?
ô ?
ñô ð
?
ðD €ððñ ô ð
S
ð S
ð S
ð S
ð S
Ð*Cñ S
ô S
ñô ð
S
ðl €ððñ ô ð
3
ð 3
ð 3
ð 3
ð 3
Ð'@ñ 3
ô 3
ñô ð
3
ðl ð1
ð 1
ð 1
ð 1
ð 1
Ð%>ñ 1
ô 1
ñ „ð1
ðh €ððñ ô ð
R
ð R
ð R
ð R
ð R
Ð";ñ R
ô R
ñô ð
R
ðj	ð 	ð 	€€€rc   