§
    9²¸j^D  ã                   ó  — d Z ddlZddlmZ ddlZddlmZ ddlmZ ddlm	Z
 ddlmZmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZ ddlmZmZmZmZ ddlmZmZ ddlm Z  ddl!m"Z"m#Z#m$Z$ ddl%m&Z& ddl'm(Z(m)Z)m*Z*m+Z+m,Z, ddl-m.Z.  ej/        e0¦  «        Z1 ed¬¦  «        e G d„ de&¦  «        ¦   «         ¦   «         Z2 G d„ dej3        ¦  «        Z4 G d„ de+¦  «        Z5 G d„ de*¦  «        Z6 G d„ de(¦  «        Z7 G d „ d!e)¦  «        Z8e G d"„ d#e.¦  «        ¦   «         Z9e G d$„ d%e9¦  «        ¦   «         Z: G d&„ d'ej;        ¦  «        Z<e G d(„ d)e9¦  «        ¦   «         Z= G d*„ d+e"¦  «        Z> G d,„ d-e#¦  «        Z? G d.„ d/ee9¦  «        Z@g d0¢ZAdS )1zPyTorch ESMC model.é    N)ÚCallable)Ústrict)Ú
functionalé   )Úcreate_bidirectional_maskÚpacked_sequence_mask_function)ÚGenericForTokenClassification)ÚBaseModelOutputÚMaskedLMOutput)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úis_flash_attention_requestedÚmerge_with_config_defaults)Úcapture_outputsé   )ÚEsmClassificationHeadÚEsmForSequenceClassificationÚeager_attention_forward)ÚLlamaConfig)ÚLlamaAttentionÚLlamaDecoderLayerÚLlamaMLPÚLlamaRotaryEmbeddingÚapply_rotary_pos_emb)ÚNomicBertPreTrainedModelzbiohub/ESMC-6B-hf)Ú
checkpointc                   ó>  ‡ — e Zd ZU dZdZdZeed<   dZeed<   dZ	eed<   d	Z
eed
<   dZeed<   dZedz  ed<   dZedz  ed<   dZeee         z  dz  ed<   dZedz  ed<   dZedz  ed<    e¦   «         Z e¦   «         Z e¦   «         Z e¦   «         Zˆ fd„Zˆ fd„Zˆ xZS )Ú
EsmcConfiga¤  
    mask_token_id (`int`, *optional*, defaults to 32):
        Index of the mask token in the vocabulary (``"<mask>"``), used for masked language modelling.
    classifier_dropout (`float`, *optional*, defaults to 0.1):
        Dropout ratio for the classification head.

    Examples:

    ```python
    >>> from transformers import EsmcConfig, EsmcModel

    >>> # Initializing an ESMC biohub/ESMC-6B-hf style configuration
    >>> configuration = EsmcConfig()

    >>> # Initializing a model (with random weights) from the configuration
    >>> model = EsmcModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```
    Úesmcé@   Ú
vocab_sizei 
  Úhidden_sizei   Úintermediate_sizeéP   Únum_hidden_layersé(   Únum_attention_headsé   NÚpad_token_idr   Úbos_token_idr   Úeos_token_idé    Úmask_token_idgš™™™™™¹?Úclassifier_dropoutc                 ór   •— | j         €d| _         | j        €d| _         t          ¦   «         j        di |¤Ž d S )Nr   r   © )r.   r/   ÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s     €úu/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/esmc/modular_esmc.pyr6   zEsmcConfig.__post_init__f   sJ   ø€ ð ÔÐ$Ø !ˆDÔØÔÐ$Ø !ˆDÔØ�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'ó    c                 ó¨   •— t          ¦   «                              ¦   «          | j        | j        k    r t	          d| j        › d| j        › d�¦  «        ‚d S )NzFESMC does not support grouped-query attention: `num_key_value_heads` (z$) must equal `num_attention_heads` (z).)r5   Úvalidate_architectureÚnum_key_value_headsr+   Ú
ValueError)r7   r9   s    €r:   r=   z EsmcConfig.validate_architectureo   su   ø€ Ý‰Œ×%Ò%Ñ'Ô'Ð'àÔ# tÔ'?Ò?Ð?ÝðoØÔ,ðoð oØRVÔRjðoð oð oñô ð ð @Ð?r;   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typer%   ÚintÚ__annotations__r&   r'   r)   r+   r-   r.   r/   Úlistr1   r2   ÚfloatÚAttributeErrorÚrms_norm_epsÚpretraining_tpÚ	use_cacheÚkeys_to_ignore_at_inferencer6   r=   Ú__classcell__©r9   s   @r:   r"   r"   7   s`  ø€ € € € € € ðð ð, €Jð €J�ÐÐÑØ€K�ÐÐÑØ!Ð�sÐ!Ð!Ñ!ØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø €L�#˜‘*Ð Ð Ñ Ø €L�#˜‘*Ð Ð Ñ Ø+,€L�#˜˜Sœ	‘/ DÑ(Ð,Ð,Ñ,ð !#€M�3˜‘:Ð"Ð"Ñ"Ø'*Ð˜ ™Ð*Ð*Ñ*ð "�>Ñ#Ô#€LØ#�^Ñ%Ô%€NØ�Ñ Ô €IØ"0 .Ñ"2Ô"2Ðð(ð (ð (ð (ð (ðð ð ð ð ð ð ð ð r;   r"   c                   ó@   ‡ — e Zd ZdZdej        dej        fˆ fd„Zˆ xZS )ÚEsmcLayerNorma\  LayerNorm that returns its input dtype.

    The reference fuses each LayerNorm into the projection that follows it, keeping the fp32 reduction
    inside the fused op; a bare ``nn.LayerNorm`` under autocast instead hands back fp32 and promotes
    the residual stream, leaving the rotary ``cos``/``sin`` as the only bf16 tensors in the block.
    Úhidden_statesÚreturnc                 óv   •— t          ¦   «                              |¦  «                             |j        ¦  «        S ©N)r5   ÚforwardÚtoÚdtype)r7   rR   r9   s     €r:   rV   zEsmcLayerNorm.forward�   s*   ø€ Ý‰wŒw�Š˜}Ñ-Ô-×0Ò0°Ô1DÑEÔEÐEr;   )r@   rA   rB   rC   ÚtorchÚTensorrV   rN   rO   s   @r:   rQ   rQ   y   se   ø€ € € € € ðð ðF U¤\ð F°e´lð Fð Fð Fð Fð Fð Fð Fð Fð Fð Fr;   rQ   c                   ó   — e Zd ZdS )ÚEsmcRotaryEmbeddingN©r@   rA   rB   r4   r;   r:   r\   r\   …   ó   € € € € € Ø€Dr;   r\   c                   ó   — e Zd ZdS )ÚEsmcMLPNr]   r4   r;   r:   r`   r`   ‰   r^   r;   r`   c                   óÐ   ‡ — e Zd ZdZddededz  fˆ fd„Z	 	 ddej        dej        dz  de	ej        ej        f         dz  d	e
e         d
e	ej        ej        dz  f         f
d„Zˆ xZS )ÚEsmcAttentionz5Multi-head self-attention with QK-LayerNorm and RoPE.NÚconfigÚ	layer_idxc                 óÆ   •— t          ¦   «                              ||¦  «         d| _        t          |j        d¬¦  «        | _        t          |j        d¬¦  «        | _        d S )NF©Úbias)r5   Ú__init__Ú	is_causalrQ   r&   Úq_normÚk_norm©r7   rc   rd   r9   s      €r:   rh   zEsmcAttention.__init__�   sV   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+ØˆŒå# FÔ$6¸UÐCÑCÔCˆŒÝ# FÔ$6¸UÐCÑCÔCˆŒˆˆr;   rR   Úattention_maskÚposition_embeddingsr8   rS   c                 ó”  — |j         d d…         }g |¢d‘| j        ‘R }|                      |                      |¦  «        ¦  «                             |j        ¦  «        }|                      |                      |¦  «        ¦  «                             |j        ¦  «        }|                      |¦  «        }	| 	                    |¦  «         
                    dd¦  «        }| 	                    |¦  «         
                    dd¦  «        }|	 	                    |¦  «         
                    dd¦  «        }	|\  }
}t          |||
|¦  «        \  }}t          j        | j        j        t           ¦  «        } || |||	|f| j        sdn| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        |fS )Néÿÿÿÿr,   r   g        )ÚdropoutÚscaling)ÚshapeÚhead_dimrj   Úq_projrW   rX   rk   Úk_projÚv_projÚviewÚ	transposer   r   Úget_interfacerc   Ú_attn_implementationr   ÚtrainingÚattention_dropoutrr   ÚreshapeÚ
contiguousÚo_proj)r7   rR   rm   rn   r8   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss                  r:   rV   zEsmcAttention.forward—   sÙ  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ 4§;¢;¨}Ñ#=Ô#=Ñ>Ô>×AÒAÀ-ÔBUÑVÔVˆØ—[’[ §¢¨]Ñ!;Ô!;Ñ<Ô<×?Ò?ÀÔ@SÑTÔTˆ
Ø—{’{ =Ñ1Ô1ˆà#×(Ò(¨Ñ6Ô6×@Ò@ÀÀAÑFÔFˆØ—_’_ \Ñ2Ô2×<Ò<¸QÀÑBÔBˆ
Ø#×(Ò(¨Ñ6Ô6×@Ò@ÀÀAÑFÔFˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ�{Š{˜;Ñ'Ô'¨Ð5Ð5r;   rU   ©NN©r@   rA   rB   rC   r"   rE   rh   rY   rZ   Útupler   r   rV   rN   rO   s   @r:   rb   rb   �   sé   ø€ € € € € Ø?Ð?ðDð D˜zð D°c¸D±jð Dð Dð Dð Dð Dð Dð /3ØHLð	$6ð $6à”|ð$6ð œ tÑ+ð$6ð # 5¤<°´Ð#=Ô>ÀÑEð	$6ð
 Ð+Ô,ð$6ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð$6ð $6ð $6ð $6ð $6ð $6ð $6ð $6r;   rb   c                   ó°   ‡ — e Zd ZdZddededz  fˆ fd„Z	 	 ddej        dej        dz  de	ej        ej        f         dz  d	e
e         d
ej        f
d„Zˆ xZS )Ú	EsmcLayerzRSingle transformer block: pre-norm attention + pre-norm FFN with residual scaling.Nrc   rd   c                 óò   •— t          ¦   «                              ||¦  «         t          |j        ¦  «        | _        t          |j        ¦  «        | _        t          j        |j        dz  ¦  «        | _	        d S )Né$   )
r5   rh   rQ   r&   Úinput_layernormÚpost_attention_layernormÚmathÚsqrtr)   Úscaling_factorrl   s      €r:   rh   zEsmcLayer.__init__Á   sc   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+å,¨VÔ-?Ñ@Ô@ˆÔÝ(5°fÔ6HÑ(IÔ(IˆÔ%å"œi¨Ô(@À2Ñ(EÑFÔFˆÔÐÐr;   rR   rm   rn   r8   rS   c                 óÜ   — |} | j         |                      |¦  «        |fd|i|¤Ž\  }}||| j        z  z   }|}||                      |                      |¦  «        ¦  «        | j        z  z   }|S )Nrn   )Ú	self_attnr’   r–   Úmlpr“   )r7   rR   rm   rn   r8   Úresidualr‰   Ú_s           r:   rV   zEsmcLayer.forwardÉ   s™   € ð !ˆØ'˜œØ× Ò  Ñ/Ô/Øð
ð 
ð !4ð
ð ð	
ð 
‰ˆ�Qð ! ;°Ô1DÑ#DÑDˆØ ˆØ  4§8¢8¨D×,IÒ,IÈ-Ñ,XÔ,XÑ#YÔ#YÐ\`Ô\oÑ#oÑoˆØÐr;   rU   r‹   rŒ   rO   s   @r:   r�   r�   ¾   sØ   ø€ € € € € Ø\Ð\ðGð G˜zð G°c¸D±jð Gð Gð Gð Gð Gð Gð /3ØHLð	ð à”|ðð œ tÑ+ðð # 5¤<°´Ð#=Ô>ÀÑEð	ð
 Ð+Ô,ðð 
Œðð ð ð ð ð ð ð r;   r�   c                   ó,   — e Zd ZeZdZdgZeedœZ	d„ Z
dS )ÚEsmcPreTrainedModelr#   r�   )rR   Ú
attentionsc                 ó   — t          ¦   «         ‚rU   )rI   )r7   Úmodules     r:   Ú_init_weightsz!EsmcPreTrainedModel._init_weightsè   s   € ÝÑÔÐr;   N)r@   rA   rB   r"   Úconfig_classÚbase_model_prefixÚ_no_split_modulesr�   rb   Ú_can_record_outputsr¡   r4   r;   r:   r�   r�   Þ   sI   € € € € € à€LØÐØ$˜Ðà"Ø#ðð Ðð
ð ð ð ð r;   r�   c                   óÊ   ‡ — e Zd Zdefˆ fd„Zeee	 	 	 	 ddej	        dz  dej	        dz  dej	        dz  dej	        dz  de
e         d	efd
„¦   «         ¦   «         ¦   «         Zˆ xZS )Ú	EsmcModelrc   c                 ó–  •‡— t          ¦   «                              ‰¦  «         t          j        ‰j        ‰j        ¦  «        | _        t          ‰¦  «        | _        t          j	        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        d¬¦  «        | _        d| _        |                      ¦   «          d S )Nc                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r4   )r�   )Ú.0r›   rc   s     €r:   ú
<listcomp>z&EsmcModel.__init__.<locals>.<listcomp>ò   s!   ø€ Ð$`Ð$`Ð$`¸1¥Y¨vÑ%6Ô%6Ð$`Ð$`Ð$`r;   Frf   )r5   rh   ÚnnÚ	Embeddingr%   r&   Úembed_tokensr\   Ú
rotary_embÚ
ModuleListÚranger)   ÚlayersrQ   ÚnormÚgradient_checkpointingÚ	post_init©r7   rc   r9   s    `€r:   rh   zEsmcModel.__init__î   s«   øø€ Ý‰Œ×Ò˜Ñ Ô Ð ÝœL¨Ô):¸FÔ<NÑOÔOˆÔÝ-¨fÑ5Ô5ˆŒÝ”mÐ$`Ð$`Ð$`Ð$`ÅÀfÔF^Ñ@_Ô@_Ð$`Ñ$`Ô$`ÑaÔaˆŒÝ! &Ô"4¸5ÐAÑAÔAˆŒ	Ø&+ˆÔ#Ø�ŠÑÔÐÐÐr;   NÚ	input_idsrm   Úposition_idsÚsequence_idr8   rS   c                 ó  — |                       |¦  «        }|€9t          j        |j        d         |j        ¬¦  «                             d¦  «        }|                      ||¦  «        }|�It          | j        ¦  «        rt          d¦  «        ‚t          | j        |dt          |¦  «        ¬¦  «        }nt          | j        ||¬¦  «        }| j        D ]} |||fd|i|¤Ž}Œ|                      |¦  «        }	t          |	¬	¦  «        S )
aŽ  
        position_ids (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Position of each token in the sequence, used by RoPE. Defaults to
            ``arange(sequence_length)``.

            ESMC positions are absolute over the whole input, and keep running *across* chain
            boundaries in a multi-chain input. The chain structure is carried separately by
            ``sequence_id``, which masks attention: unlike the packed-sequence format used elsewhere
            in the library, these are therefore not per-chain position ids restarting at 0, and the
            chain structure is deliberately not inferred from them.
        sequence_id (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Integer chain-ID tensor for chain-aware attention masking. Tokens with the same
            non-negative integer value can attend to each other; tokens with different values
            cannot (cross-chain masking). Padding positions should be set to ``-1`` and inputs
            must be **right-padded** (RoPE uses absolute positions starting at 0). When provided,
            ``attention_mask`` is ignored. Passing ``sequence_id`` builds a custom attention
            mask, which requires ``torch>=2.6``. Multi-chain inputs additionally require a
            non-flash ``attn_implementation`` (``'sdpa'`` / ``'eager'`` / ``'flex_attention'``);
            flash attention only supports the single-chain case.

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, EsmcModel

        >>> model = EsmcModel.from_pretrained("biohub/ESMC-300M-hf")
        >>> tokenizer = AutoTokenizer.from_pretrained("biohub/ESMC-300M-hf")
        >>> inputs = tokenizer(["MLKNVQVQLV"], return_tensors="pt")
        >>> outputs = model(**inputs)
        >>> outputs.last_hidden_state.shape
        torch.Size([1, 12, 960])
        ```
        Nr,   )Údevicer   zÔ`sequence_id` (chain-aware attention) is not supported with flash attention, which can't represent the block-diagonal chain mask. Re-load the model with attn_implementation='sdpa' (or 'eager' / 'flex_attention').)rc   Úinputs_embedsrm   Úand_mask_function)rc   r¼   rm   rn   )Úlast_hidden_state)r®   rY   Úarangers   r»   Ú	unsqueezer¯   r   rc   r?   r   r   r²   r³   r
   )
r7   r·   rm   r¸   r¹   r8   rR   rn   ÚlayerÚoutputs
             r:   rV   zEsmcModel.forward÷   sH  € ðX ×)Ò)¨)Ñ4Ô4ˆØÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\×fÒfÐghÑiÔiˆLØ"Ÿošo¨m¸\ÑJÔJÐàÐ"Ý+¨D¬KÑ8Ô8ð Ý ðRñô ð õ
 7Ø”{Ø+Ø#Ý"?ÀÑ"LÔ"Lð	ñ ô ˆNˆNõ 7Ø”{Ø+Ø-ðñ ô ˆNð ”[ð 	ð 	ˆEØ!˜EØØðð ð %8ðð ð	ð ˆMˆMð —’˜=Ñ)Ô)ˆå°Ð8Ñ8Ô8Ð8r;   )NNNN)r@   rA   rB   r"   rh   r   r   r   rY   rZ   r   r   r
   rV   rN   rO   s   @r:   r§   r§   ì   s÷   ø€ € € € € ð˜zð ð ð ð ð ð ð  ØØð *.Ø.2Ø,0Ø+/ðL9ð L9à”< $Ñ&ðL9ð œ tÑ+ðL9ð ”l TÑ)ð	L9ð
 ”\ DÑ(ðL9ð Ð+Ô,ðL9ð 
ðL9ð L9ð L9ñ „^ñ „_ñ  ÔðL9ð L9ð L9ð L9ð L9r;   r§   c            	       ó\   ‡ — e Zd Zd	dedededz  ddfˆ fd„Zdej        dej        fd„Zˆ xZS )
ÚEsmcMaskedLMHeadNr&   r%   r'   rS   c                 óî   •— t          ¦   «                              ¦   «          |�|n|}t          j        ||¦  «        | _        t          j        |¦  «        | _        t          j        ||¦  «        | _        d S rU   )r5   rh   r¬   ÚLinearÚdenseÚ	LayerNormÚ
layer_normÚdecoder)r7   r&   r%   r'   r9   s       €r:   rh   zEsmcMaskedLMHead.__init__J  si   ø€ Ý‰Œ×ÒÑÔÐØ1BÐ1NÐ-Ð-ÐT_ÐÝ”Y˜{Ð,=Ñ>Ô>ˆŒ
Ýœ,Ð'8Ñ9Ô9ˆŒÝ”yÐ!2°JÑ?Ô?ˆŒˆˆr;   rR   c                 ó¨   — |                       |¦  «        }t          j        |¦  «        }|                      |¦  «        }|                      |¦  «        S rU   )rÇ   ÚFÚgelurÉ   rÊ   )r7   rR   s     r:   rV   zEsmcMaskedLMHead.forwardQ  sE   € ØŸ
š
 =Ñ1Ô1ˆÝœ˜}Ñ-Ô-ˆØŸš¨Ñ6Ô6ˆØ�|Š|˜MÑ*Ô*Ð*r;   rU   )	r@   rA   rB   rE   rh   rY   rZ   rV   rN   rO   s   @r:   rÄ   rÄ   I  s•   ø€ € € € € ð@ð @ Cð @°Sð @ÈSÐSWÉZð @Ðcgð @ð @ð @ð @ð @ð @ð+ U¤\ð +°e´lð +ð +ð +ð +ð +ð +ð +ð +r;   rÄ   c                   ó  ‡ — e Zd Zdefˆ fd„Zdej        fd„Zdej        fd„Ze	e
	 	 	 	 	 ddej        dz  d	ej        dz  d
ej        dz  dej        dz  dej        dz  dee         deej        df         ez  fd„¦   «         ¦   «         Zˆ xZS )ÚEsmcForMaskedLMrc   c                 óØ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          |j        |j        ¦  «        | _        |                      ¦   «          d S rU   )	r5   rh   r§   r#   rÄ   r&   r%   Úlm_headrµ   r¶   s     €r:   rh   zEsmcForMaskedLM.__init__Z  sW   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý˜fÑ%Ô%ˆŒ	Ý'¨Ô(:¸FÔ<MÑNÔNˆŒØ�ŠÑÔÐÐÐr;   rS   c                 ó   — | j         j        S rU   ©rÑ   rÊ   )r7   s    r:   Úget_output_embeddingsz%EsmcForMaskedLM.get_output_embeddingsc  s   € ØŒ|Ô#Ð#r;   Únew_embeddingsc                 ó   — || j         _        d S rU   rÓ   )r7   rÕ   s     r:   Úset_output_embeddingsz%EsmcForMaskedLM.set_output_embeddingsf  s   € Ø-ˆŒÔÐÐr;   Nr·   rm   r¸   r¹   Úlabelsr8   .c           	      óÔ   —  | j         d||||ddœ|¤Ž}|                      |j        ¦  «        }d}	|� | j        ||fd| j        j        i|¤Ž}	t          |	||j        |j        ¬¦  «        S )aØ  
        sequence_id (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Integer chain-ID tensor forwarded to the encoder for chain-aware
            attention masking. See :meth:`EsmcModel.forward` for the encoding.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for masked language modelling loss.  Positions with label ``-100``
            are ignored.  Other positions must be in ``[0, config.vocab_size)``.

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, EsmcForMaskedLM
        >>> import torch

        >>> model = EsmcForMaskedLM.from_pretrained("biohub/ESMC-300M-hf")
        >>> tokenizer = AutoTokenizer.from_pretrained("biohub/ESMC-300M-hf")
        >>> inputs = tokenizer(["MLKNVQ<mask>LV"], return_tensors="pt")
        >>> outputs = model(**inputs)
        >>> outputs.logits.shape
        torch.Size([1, 11, 64])
        ```
        T)r·   rm   r¸   r¹   Úreturn_dictNr%   )ÚlossÚlogitsrR   rž   r4   )	r#   rÑ   r¾   Úloss_functionrc   r%   r   rR   rž   )
r7   r·   rm   r¸   r¹   rØ   r8   Úencoder_outputsrÜ   rÛ   s
             r:   rV   zEsmcForMaskedLM.forwardi  s¬   € ðB $˜$œ)ð 
ØØ)Ø%Ø#Øð
ð 
ð ð
ð 
ˆð —’˜oÔ?Ñ@Ô@ˆà$(ˆØÐØ%�4Ô% f¨fÐbÐbÀÄÔAWÐbÐ[aÐbÐbˆDåØØØ)Ô7Ø&Ô1ð	
ñ 
ô 
ð 	
r;   )NNNNN)r@   rA   rB   r"   rh   r¬   rÆ   rÔ   r×   r   r   rY   rZ   r   r   r�   r   rV   rN   rO   s   @r:   rÏ   rÏ   X  s=  ø€ € € € € ð˜zð ð ð ð ð ð ð$ r¤yð $ð $ð $ð $ð.°B´Ið .ð .ð .ð .ð Øð *.Ø.2Ø,0Ø+/Ø&*ð3
ð 3
à”< $Ñ&ð3
ð œ tÑ+ð3
ð ”l TÑ)ð	3
ð
 ”\ DÑ(ð3
ð ”˜tÑ#ð3
ð Ð+Ô,ð3
ð 
ˆuŒ|˜SÐ Ô	! NÑ	2ð3
ð 3
ð 3
ñ „^ñ Ôð3
ð 3
ð 3
ð 3
ð 3
r;   rÏ   c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚEsmcClassificationHeadrc   c                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        ¦  «        | _        t          j        |j        |j	        ¦  «        | _
        d S rU   )r5   rh   r¬   rÆ   r&   rÇ   ÚDropoutr2   rq   Ú
num_labelsÚout_projr¶   s     €r:   rh   zEsmcClassificationHead.__init__¢  sc   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ý”z &Ô";Ñ<Ô<ˆŒÝœ	 &Ô"4°fÔ6GÑHÔHˆŒˆˆr;   ©r@   rA   rB   r"   rh   rN   rO   s   @r:   rà   rà   ¡  sO   ø€ € € € € ðI˜zð Ið Ið Ið Ið Ið Ið Ið Ið Ið Ir;   rà   c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚEsmcForSequenceClassificationrc   c                 óÚ   •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          |¦  «        | _        |                      ¦   «          d S rU   )r5   rh   rã   r§   r#   rà   Ú
classifierrµ   r¶   s     €r:   rh   z&EsmcForSequenceClassification.__init__ª  sY   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒÝ˜fÑ%Ô%ˆŒ	Ý0°Ñ8Ô8ˆŒØ�ŠÑÔÐÐÐr;   rå   rO   s   @r:   rç   rç   ©  sD   ø€ € € € € ð˜zð ð ð ð ð ð ð ð ð ð r;   rç   c                   ó   — e Zd ZdZdS )ÚEsmcForTokenClassificationr#   N)r@   rA   rB   r£   r4   r;   r:   rë   rë   ²  s   € € € € € ØÐÐÐr;   rë   )r"   r§   rÏ   rç   rë   r�   )BrC   r”   Úcollections.abcr   rY   Útorch.nnr¬   Úhuggingface_hub.dataclassesr   r   rÌ   Úmasking_utilsr   r   Úmodeling_layersr	   Úmodeling_outputsr
   r   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   r   Úutils.output_capturingr   Úesm.modeling_esmr   r   r   Úllama.configuration_llamar   Úllama.modeling_llamar   r   r   r   r   Únomic_bert.modeling_nomic_bertr   Ú
get_loggerr@   Úloggerr"   rÈ   rQ   r\   r`   rb   r�   r�   r§   ÚModulerÄ   rÏ   rà   rç   rë   Ú__all__r4   r;   r:   ú<module>rÿ      s›  ðð Ð à €€€Ø $Ð $Ð $Ð $Ð $Ð $à €€€Ø Ð Ð Ð Ð Ð Ø .Ð .Ð .Ð .Ð .Ð .Ø $Ð $Ð $Ð $Ð $Ð $à UÐ UÐ UÐ UÐ UÐ UÐ UÐ UØ <Ð <Ð <Ð <Ð <Ð <ðð ð ð ð ð ð ð ð 6Ð 5Ð 5Ð 5Ð 5Ð 5Ø &Ð &Ð &Ð &Ð &Ð &ðð ð ð ð ð ð ð ð ð ð ð ð VÐ UÐ UÐ UÐ UÐ UÐ UÐ UØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø kÐ kÐ kÐ kÐ kÐ kÐ kÐ kÐ kÐ kØ 3Ð 3Ð 3Ð 3Ð 3Ð 3ðð ð ð ð ð ð ð ð ð ð ð ð ð ð FÐ EÐ EÐ EÐ EÐ Eð 
ˆÔ	˜HÑ	%Ô	%€ð €Ð.Ð/Ñ/Ô/Øð=ð =ð =ð =ð =�ñ =ô =ñ „ñ 0Ô/ð=ð@	Fð 	Fð 	Fð 	Fð 	F�B”Lñ 	Fô 	Fð 	Fð	ð 	ð 	ð 	ð 	Ð.ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	ˆhñ 	ô 	ð 	ð.6ð .6ð .6ð .6ð .6�Nñ .6ô .6ð .6ðbð ð ð ð Ð!ñ ô ð ð@ ð
ð 
ð 
ð 
ð 
Ð2ñ 
ô 
ñ „ð
ð ðY9ð Y9ð Y9ð Y9ð Y9Ð#ñ Y9ô Y9ñ „ðY9ðx+ð +ð +ð +ð +�r”yñ +ô +ð +ð ðE
ð E
ð E
ð E
ð E
Ð)ñ E
ô E
ñ „ðE
ðPIð Ið Ið Ið IÐ2ñ Iô Ið Iðð ð ð ð Ð$@ñ ô ð ðð ð ð ð Ð!>Ð@Sñ ô ð ðð ð €€€r;   