§
    9²¸jsf  ã                   óÊ  — d dl Z d dlmZ d dlZd dlmZ d dlmZmZmZ d dlm	Z
 ddlmZ ddlmZmZ ddlmZmZ dd	lmZmZ dd
lmZmZmZ ddlmZmZ ddlmZmZ ddl m!Z! ddl"m#Z#m$Z$m%Z% ddl&m'Z' ddl(m)Z)m*Z*m+Z+ ddl,m-Z- ddl.m/Z/  G d„ dej0        ¦  «        Z1 G d„ dej2        ¦  «        Z3 G d„ dej2        ¦  «        Z4d„ Z5 ed¦  «        d<d„¦   «         Z6	 	 d=dej2        dej7        d ej7        d!ej7        d"ej7        dz  d#e8dz  d$e8d%e!e#         fd&„Z9 ee6¦  «         G d'„ d(ej2        ¦  «        ¦   «         Z: G d)„ d*e¦  «        Z;e$ G d+„ d,e¦  «        ¦   «         Z<e$ G d-„ d.e<¦  «        ¦   «         Z= G d/„ d0ej2        ¦  «        Z>e$ G d1„ d2e<¦  «        ¦   «         Z? G d3„ d4ej2        ¦  «        Z@ e$d5¬6¦  «         G d7„ d8e<¦  «        ¦   «         ZA G d9„ d:ee<¦  «        ZBg d;¢ZCdS )>é    N)ÚCallable)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELoss)Ú
functionalé   )ÚACT2FN)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_bidirectional_maskÚpacked_sequence_mask_function)ÚGenericForTokenClassificationÚGradientCheckpointingLayer)ÚBaseModelOutputÚMaskedLMOutputÚSequenceClassifierOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úis_flash_attention_requestedÚmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )Ú
EsmcConfigc                   ó@   ‡ — e Zd ZdZdej        dej        fˆ fd„Zˆ xZS )ÚEsmcLayerNorma\  LayerNorm that returns its input dtype.

    The reference fuses each LayerNorm into the projection that follows it, keeping the fp32 reduction
    inside the fused op; a bare ``nn.LayerNorm`` under autocast instead hands back fp32 and promotes
    the residual stream, leaving the rotary ``cos``/``sin`` as the only bf16 tensors in the block.
    Úhidden_statesÚreturnc                 óv   •— t          ¦   «                              |¦  «                             |j        ¦  «        S ©N)ÚsuperÚforwardÚtoÚdtype)Úselfr$   Ú	__class__s     €úv/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/esmc/modeling_esmc.pyr)   zEsmcLayerNorm.forward4   s*   ø€ Ý‰wŒw�Š˜}Ñ-Ô-×0Ò0°Ô1DÑEÔEÐEó    )Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚtorchÚTensorr)   Ú__classcell__©r-   s   @r.   r#   r#   ,   se   ø€ € € € € ðð ðF U¤\ð F°e´lð Fð Fð Fð Fð Fð Fð Fð Fð Fð Fr/   r#   c            	       óî   ‡ — e Zd Z edd¬¦  «        d
defˆ fd„¦   «         Ze edd¬¦  «        d
dedeej	        e
f         fd„¦   «         ¦   «         Z ej        ¦   «         ed	„ ¦   «         ¦   «         Zˆ xZS )ÚEsmcRotaryEmbeddingÚdevicez5.18)ÚversionNÚconfigc                 ó¾  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        | j        j        d         | _        | j        }| j        dk    rt          | j                 } || j        |¦  «        \  }| _
        t          j        |d¬¦  «        | _        t          j        |                     ¦   «         d¬¦  «        | _        d S )NÚ	rope_typeÚdefaultF)Ú
persistent)r(   Ú__init__Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr<   Úrope_parametersr>   Úcompute_default_rope_parametersr   Úattention_scalingÚnnÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r,   r<   r:   Úrope_init_fnrJ   r-   s        €r.   rA   zEsmcRotaryEmbedding.__init__9   sÀ   ø€ å‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!àˆŒàœÔ4°[ÔAˆŒØ!%Ô!EˆØŒ>˜YÒ&Ð&Ý.¨t¬~Ô>ˆLØ+7¨<¸¼ÀVÑ+LÔ+LÑ(ˆ�$Ô(åœ	 (°uÐ=Ñ=Ô=ˆŒÝ!#¤¨8¯>ª>Ñ+;Ô+;ÈÐ!NÑ!NÔ!NˆÔÐÐr/   r%   c                 óä   — | j         d         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «        |z  z  z  }|                     |¦  «        |fS )a¾  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNg      ð?r   é   ©r+   )rE   ÚgetattrÚhidden_sizeÚnum_attention_headsr4   ÚarangeÚfloatr*   )r<   r:   ÚkwargsÚbaseÚdimÚattention_factorrJ   s          r.   rF   z3EsmcRotaryEmbedding.compute_default_rope_parametersJ   sz   € ð Ô% lÔ3ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐà˜$¥5¤<°°3¸ÅÄÐ#MÑ#MÔ#MÐPSÑ#SÑTÑUˆØ�{Š{˜6Ñ"Ô"Ð$4Ð4Ð4r/   c                 óú  — | j         d d d …d f                              |j        d         dd¦  «                             t          j        |j        ¬¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  ||z                       dd	¦  «        }t	          j        ||fd¬
¦  «        }|                     ¦   «         | j        z  }|                     ¦   «         | j        z  }	d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬¦  «        |	                     |j        ¬¦  «        fS )Nr   éÿÿÿÿr    )r+   r:   ÚmpsÚcpuF)Údevice_typeÚenabledrQ   ©rZ   rR   )rJ   ÚexpandÚshaper*   r4   rW   r:   Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosrG   Úsinr+   )
r,   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedr`   ÚfreqsÚembrj   rk   s
             r.   r)   zEsmcRotaryEmbedding.forward^   s›  € ð ŒM˜$    4˜-Ô(×/Ò/°Ô0BÀ1Ô0EÀrÈ1ÑMÔM×PÒPÕW\ÔWbÐklÔksÐPÑtÔtð 	ð !-¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆå¨¸UÐCÑCÔCð 	5ð 	5Ø&Ð)>Ñ>×IÒIÈ!ÈQÑOÔOˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCð		5ð 	5ð 	5ñ 	5ô 	5ð 	5ð 	5ð 	5ð 	5ð 	5ð 	5øøøð 	5ð 	5ð 	5ð 	5ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   ÃA*D<Ä<E ÅE r'   )r0   r1   r2   r   r!   rA   ÚstaticmethodÚtupler4   r5   rW   rF   Úno_gradr   r)   r6   r7   s   @r.   r9   r9   8   sö   ø€ € € € € Ø€_�X vÐ.Ñ.Ô.ðOð O˜zð Oð Oð Oð Oð Oñ /Ô.ðOð  Ø€_�X vÐ.Ñ.Ô.ð5ð 5°
ð 5ÐV[Ð\aÔ\hÐjoÐ\oÔVpð 5ð 5ð 5ñ /Ô.ñ „\ð5ð$ €U„]�_„_Øð<ð <ñ Ôñ „_ð<ð <ð <ð <ð <r/   r9   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚEsmcMLPc                 ó¶  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        t          j        | j        | j        |j        ¬¦  «        | _        t          j        | j        | j        |j        ¬¦  «        | _	        t          j        | j        | j        |j        ¬¦  «        | _
        t          |j                 | _        d S )N©Úbias)r(   rA   r<   rT   Úintermediate_sizerH   ÚLinearÚmlp_biasÚ	gate_projÚup_projÚ	down_projr	   Ú
hidden_actÚact_fn©r,   r<   r-   s     €r.   rA   zEsmcMLP.__init__r   s¯   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ!'Ô!9ˆÔÝœ 4Ô#3°TÔ5KÐRXÔRaÐbÑbÔbˆŒÝ”y Ô!1°4Ô3IÐPVÔP_Ð`Ñ`Ô`ˆŒÝœ 4Ô#9¸4Ô;KÐRXÔRaÐbÑbÔbˆŒÝ˜VÔ.Ô/ˆŒˆˆr/   c                 ó¨   — |                       |                      |                      |¦  «        ¦  «        |                      |¦  «        z  ¦  «        }|S r'   )r   r�   r}   r~   )r,   rl   r   s      r.   r)   zEsmcMLP.forward|   sA   € Ø—N’N 4§;¢;¨t¯~ª~¸aÑ/@Ô/@Ñ#AÔ#AÀDÇLÂLÐQRÁOÄOÑ#SÑTÔTˆ	ØÐr/   )r0   r1   r2   rA   r)   r6   r7   s   @r.   rv   rv   q   sG   ø€ € € € € ð0ð 0ð 0ð 0ð 0ðð ð ð ð ð ð r/   rv   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..Nr]   rQ   rb   )rd   r4   ri   )rl   Úx1Úx2s      r.   Úrotate_halfr‡   �   s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'r/   Úrotary_pos_embc                 ó¾   — |                      |¦  «        }|                      |¦  «        }| |z  t          | ¦  «        |z  z   }||z  t          |¦  «        |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezer‡   )ÚqÚkrj   rk   Úunsqueeze_dimÚq_embedÚk_embeds          r.   Úapply_rotary_pos_embr�   ˆ   sc   € ð& �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�GÐÐr/   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutrX   c                 ó®  — |€|                      d¦  «        dz  }t          j        ||                     dd¦  «        ¦  «        |z  }|�||z   }t          j                             |d¬¦  «        }t          j                             ||| j        ¬¦  «        }t          j        ||¦  «        }	|	                     dd¦  «         	                    ¦   «         }	|	|fS )Nr]   ç      à¿rQ   r   rb   )ÚpÚtrainingr    )
Úsizer4   Úmatmulrh   rH   r   Úsoftmaxr˜   rœ   Ú
contiguous)
r’   r“   r”   r•   r–   r—   r˜   rX   Úattn_weightsÚattn_outputs
             r.   Úeager_attention_forwardr£   ¢   sÈ   € ð €Ø—*’*˜R‘.”. DÑ(ˆõ ”<  s§}¢}°Q¸Ñ':Ô':Ñ;Ô;¸gÑE€LàÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2Ð(Ñ>Ô>€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r/   c                   óÐ   ‡ — e Zd ZdZddededz  fˆ fd„Z	 	 ddej        dej        dz  de	ej        ej        f         dz  d	e
e         d
e	ej        ej        dz  f         f
d„Zˆ xZS )ÚEsmcAttentionz5Multi-head self-attention with QK-LayerNorm and RoPE.Nr<   Ú	layer_idxc                 ó  •— t          ¦   «                              ¦   «          || _        || _        t	          |d|j        |j        z  ¦  «        | _        |j        |j        z  | _	        | j        dz  | _
        |j        | _        d| _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        | j        z  |j        |j        ¬¦  «        | _        t)          |j        d¬¦  «        | _        t)          |j        d¬¦  «        | _        d S )NrP   rš   Frx   )r(   rA   r<   r¦   rS   rT   rU   rP   Únum_key_value_headsÚnum_key_value_groupsr—   Úattention_dropoutÚ	is_causalrH   r{   Úattention_biasÚq_projÚk_projÚv_projÚo_projr#   Úq_normÚk_norm©r,   r<   r¦   r-   s      €r.   rA   zEsmcAttention.__init__Â   sr  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒÝ ¨
°FÔ4FÈ&ÔJdÑ4dÑeÔeˆŒØ$*Ô$>À&ÔB\Ñ$\ˆÔ!Ø”} dÑ*ˆŒØ!'Ô!9ˆÔØˆŒå”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ&¨¬Ñ6¸Ô8JÐQWÔQfð
ñ 
ô 
ˆŒõ $ FÔ$6¸UÐCÑCÔCˆŒÝ# FÔ$6¸UÐCÑCÔCˆŒˆˆr/   r$   r–   Úposition_embeddingsrX   r%   c                 ó”  — |j         d d…         }g |¢d‘| j        ‘R }|                      |                      |¦  «        ¦  «                             |j        ¦  «        }|                      |                      |¦  «        ¦  «                             |j        ¦  «        }|                      |¦  «        }	| 	                    |¦  «         
                    dd¦  «        }| 	                    |¦  «         
                    dd¦  «        }|	 	                    |¦  «         
                    dd¦  «        }	|\  }
}t          |||
|¦  «        \  }}t          j        | j        j        t           ¦  «        } || |||	|f| j        sdn| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        |fS )Nr]   r    rQ   r‘   )r˜   r—   )rd   rP   r±   r­   r*   r+   r²   r®   r¯   Úviewrh   r�   r   Úget_interfacer<   Ú_attn_implementationr£   rœ   rª   r—   Úreshaper    r°   )r,   r$   r–   r´   rX   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesrj   rk   Úattention_interfacer¢   r¡   s                  r.   r)   zEsmcAttention.forwardÜ   sÙ  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ 4§;¢;¨}Ñ#=Ô#=Ñ>Ô>×AÒAÀ-ÔBUÑVÔVˆØ—[’[ §¢¨]Ñ!;Ô!;Ñ<Ô<×?Ò?ÀÔ@SÑTÔTˆ
Ø—{’{ =Ñ1Ô1ˆà#×(Ò(¨Ñ6Ô6×@Ò@ÀÀAÑFÔFˆØ—_’_ \Ñ2Ô2×<Ò<¸QÀÑBÔBˆ
Ø#×(Ò(¨Ñ6Ô6×@Ò@ÀÀAÑFÔFˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ�{Š{˜;Ñ'Ô'¨Ð5Ð5r/   r'   ©NN©r0   r1   r2   r3   r!   ÚintrA   r4   r5   rs   r   r   r)   r6   r7   s   @r.   r¥   r¥   ¾   sé   ø€ € € € € à?Ð?ðDð D˜zð D°c¸D±jð Dð Dð Dð Dð Dð Dð: /3ØHLð	$6ð $6à”|ð$6ð œ tÑ+ð$6ð # 5¤<°´Ð#=Ô>ÀÑEð	$6ð
 Ð+Ô,ð$6ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð$6ð $6ð $6ð $6ð $6ð $6ð $6ð $6r/   r¥   c                   ó°   ‡ — e Zd ZdZddededz  fˆ fd„Z	 	 ddej        dej        dz  de	ej        ej        f         dz  d	e
e         d
ej        f
d„Zˆ xZS )Ú	EsmcLayerzRSingle transformer block: pre-norm attention + pre-norm FFN with residual scaling.Nr<   r¦   c                 óZ  •— t          ¦   «                              ¦   «          |j        | _        t          ||¬¦  «        | _        t          |¦  «        | _        t          |j        ¦  «        | _        t          |j        ¦  «        | _	        t          j        |j        dz  ¦  «        | _        d S )N)r<   r¦   é$   )r(   rA   rT   r¥   Ú	self_attnrv   Úmlpr#   Úinput_layernormÚpost_attention_layernormÚmathÚsqrtÚnum_hidden_layersÚscaling_factorr³   s      €r.   rA   zEsmcLayer.__init__  s‹   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔå&¨fÀ	ÐJÑJÔJˆŒå˜6‘?”?ˆŒå,¨VÔ-?Ñ@Ô@ˆÔÝ(5°fÔ6HÑ(IÔ(IˆÔ%å"œi¨Ô(@À2Ñ(EÑFÔFˆÔÐÐr/   r$   r–   r´   rX   r%   c                 óÜ   — |} | j         |                      |¦  «        |fd|i|¤Ž\  }}||| j        z  z   }|}||                      |                      |¦  «        ¦  «        | j        z  z   }|S )Nr´   )rÇ   rÉ   rÎ   rÈ   rÊ   )r,   r$   r–   r´   rX   Úresidualr¢   Ú_s           r.   r)   zEsmcLayer.forward  s™   € ð !ˆØ'˜œØ× Ò  Ñ/Ô/Øð
ð 
ð !4ð
ð ð	
ð 
‰ˆ�Qð ! ;°Ô1DÑ#DÑDˆØ ˆØ  4§8¢8¨D×,IÒ,IÈ-Ñ,XÔ,XÑ#YÔ#YÐ\`Ô\oÑ#oÑoˆØÐr/   r'   rÀ   rÁ   r7   s   @r.   rÄ   rÄ     sØ   ø€ € € € € Ø\Ð\ðGð G˜zð G°c¸D±jð Gð Gð Gð Gð Gð Gð  /3ØHLð	ð à”|ðð œ tÑ+ðð # 5¤<°´Ð#=Ô>ÀÑEð	ð
 Ð+Ô,ðð 
Œðð ð ð ð ð ð ð r/   rÄ   c                   óB   — e Zd ZeZdZdZdZdZdZ	dZ
eedœZddgZdgZdS )ÚEsmcPreTrainedModelÚesmcT)r$   Ú
attentionsrJ   rL   rÄ   N)r0   r1   r2   r!   Úconfig_classÚbase_model_prefixÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendrÄ   r¥   Ú_can_record_outputsÚ"_keys_to_ignore_on_load_unexpectedÚ_no_split_modules© r/   r.   rÓ   rÓ   (  sa   € € € € € à€LØÐØ&*Ð#ØÐØ€NØÐØ"&Ðà"Ø#ðð Ðð +5Ð6IÐ)JÐ&Ø$˜ÐÐÐr/   rÓ   c                   óÊ   ‡ — e Zd Zdefˆ fd„Zeee	 	 	 	 ddej	        dz  dej	        dz  dej	        dz  dej	        dz  de
e         d	efd
„¦   «         ¦   «         ¦   «         Zˆ xZS )Ú	EsmcModelr<   c                 ó–  •‡— t          ¦   «                              ‰¦  «         t          j        ‰j        ‰j        ¦  «        | _        t          ‰¦  «        | _        t          j	        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        d¬¦  «        | _        d| _        |                      ¦   «          d S )Nc                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS rà   )rÄ   )Ú.0rÑ   r<   s     €r.   ú
<listcomp>z&EsmcModel.__init__.<locals>.<listcomp>B  s!   ø€ Ð$`Ð$`Ð$`¸1¥Y¨vÑ%6Ô%6Ð$`Ð$`Ð$`r/   Frx   )r(   rA   rH   Ú	EmbeddingÚ
vocab_sizerT   Úembed_tokensr9   Ú
rotary_embÚ
ModuleListÚrangerÍ   Úlayersr#   ÚnormÚgradient_checkpointingÚ	post_initr‚   s    `€r.   rA   zEsmcModel.__init__>  s«   øø€ Ý‰Œ×Ò˜Ñ Ô Ð ÝœL¨Ô):¸FÔ<NÑOÔOˆÔÝ-¨fÑ5Ô5ˆŒÝ”mÐ$`Ð$`Ð$`Ð$`ÅÀfÔF^Ñ@_Ô@_Ð$`Ñ$`Ô$`ÑaÔaˆŒÝ! &Ô"4¸5ÐAÑAÔAˆŒ	Ø&+ˆÔ#Ø�ŠÑÔÐÐÐr/   NÚ	input_idsr–   rm   Úsequence_idrX   r%   c                 ó  — |                       |¦  «        }|€9t          j        |j        d         |j        ¬¦  «                             d¦  «        }|                      ||¦  «        }|�It          | j        ¦  «        rt          d¦  «        ‚t          | j        |dt          |¦  «        ¬¦  «        }nt          | j        ||¬¦  «        }| j        D ]} |||fd|i|¤Ž}Œ|                      |¦  «        }	t          |	¬	¦  «        S )
aŽ  
        position_ids (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Position of each token in the sequence, used by RoPE. Defaults to
            ``arange(sequence_length)``.

            ESMC positions are absolute over the whole input, and keep running *across* chain
            boundaries in a multi-chain input. The chain structure is carried separately by
            ``sequence_id``, which masks attention: unlike the packed-sequence format used elsewhere
            in the library, these are therefore not per-chain position ids restarting at 0, and the
            chain structure is deliberately not inferred from them.
        sequence_id (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Integer chain-ID tensor for chain-aware attention masking. Tokens with the same
            non-negative integer value can attend to each other; tokens with different values
            cannot (cross-chain masking). Padding positions should be set to ``-1`` and inputs
            must be **right-padded** (RoPE uses absolute positions starting at 0). When provided,
            ``attention_mask`` is ignored. Passing ``sequence_id`` builds a custom attention
            mask, which requires ``torch>=2.6``. Multi-chain inputs additionally require a
            non-flash ``attn_implementation`` (``'sdpa'`` / ``'eager'`` / ``'flex_attention'``);
            flash attention only supports the single-chain case.

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, EsmcModel

        >>> model = EsmcModel.from_pretrained("biohub/ESMC-300M-hf")
        >>> tokenizer = AutoTokenizer.from_pretrained("biohub/ESMC-300M-hf")
        >>> inputs = tokenizer(["MLKNVQVQLV"], return_tensors="pt")
        >>> outputs = model(**inputs)
        >>> outputs.last_hidden_state.shape
        torch.Size([1, 12, 960])
        ```
        Nr    )r:   r   zÔ`sequence_id` (chain-aware attention) is not supported with flash attention, which can't represent the block-diagonal chain mask. Re-load the model with attn_implementation='sdpa' (or 'eager' / 'flex_attention').)r<   Úinputs_embedsr–   Úand_mask_function)r<   rô   r–   r´   )Úlast_hidden_state)ré   r4   rV   rd   r:   rŠ   rê   r   r<   Ú
ValueErrorr   r   rí   rî   r   )
r,   rñ   r–   rm   rò   rX   r$   r´   ÚlayerÚoutputs
             r.   r)   zEsmcModel.forwardG  sH  € ðX ×)Ò)¨)Ñ4Ô4ˆØÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\×fÒfÐghÑiÔiˆLØ"Ÿošo¨m¸\ÑJÔJÐàÐ"Ý+¨D¬KÑ8Ô8ð Ý ðRñô ð õ
 7Ø”{Ø+Ø#Ý"?ÀÑ"LÔ"Lð	ñ ô ˆNˆNõ 7Ø”{Ø+Ø-ðñ ô ˆNð ”[ð 	ð 	ˆEØ!˜EØØðð ð %8ðð ð	ð ˆMˆMð —’˜=Ñ)Ô)ˆå°Ð8Ñ8Ô8Ð8r/   )NNNN)r0   r1   r2   r!   rA   r   r   r   r4   r5   r   r   r   r)   r6   r7   s   @r.   râ   râ   <  s÷   ø€ € € € € ð˜zð ð ð ð ð ð ð  ØØð *.Ø.2Ø,0Ø+/ðL9ð L9à”< $Ñ&ðL9ð œ tÑ+ðL9ð ”l TÑ)ð	L9ð
 ”\ DÑ(ðL9ð Ð+Ô,ðL9ð 
ðL9ð L9ð L9ñ „^ñ „_ñ  ÔðL9ð L9ð L9ð L9ð L9r/   râ   c            	       ó\   ‡ — e Zd Zd	dedededz  ddfˆ fd„Zdej        dej        fd„Zˆ xZS )
ÚEsmcMaskedLMHeadNrT   rè   rz   r%   c                 óî   •— t          ¦   «                              ¦   «          |�|n|}t          j        ||¦  «        | _        t          j        |¦  «        | _        t          j        ||¦  «        | _        d S r'   )r(   rA   rH   r{   ÚdenseÚ	LayerNormÚ
layer_normÚdecoder)r,   rT   rè   rz   r-   s       €r.   rA   zEsmcMaskedLMHead.__init__š  si   ø€ Ý‰Œ×ÒÑÔÐØ1BÐ1NÐ-Ð-ÐT_ÐÝ”Y˜{Ð,=Ñ>Ô>ˆŒ
Ýœ,Ð'8Ñ9Ô9ˆŒÝ”yÐ!2°JÑ?Ô?ˆŒˆˆr/   r$   c                 ó¨   — |                       |¦  «        }t          j        |¦  «        }|                      |¦  «        }|                      |¦  «        S r'   )rý   ÚFÚgelurÿ   r   )r,   r$   s     r.   r)   zEsmcMaskedLMHead.forward¡  sE   € ØŸ
š
 =Ñ1Ô1ˆÝœ˜}Ñ-Ô-ˆØŸš¨Ñ6Ô6ˆØ�|Š|˜MÑ*Ô*Ð*r/   r'   )	r0   r1   r2   rÂ   rA   r4   r5   r)   r6   r7   s   @r.   rû   rû   ™  s•   ø€ € € € € ð@ð @ Cð @°Sð @ÈSÐSWÉZð @Ðcgð @ð @ð @ð @ð @ð @ð+ U¤\ð +°e´lð +ð +ð +ð +ð +ð +ð +ð +r/   rû   c                   ó  ‡ — e Zd Zdefˆ fd„Zdej        fd„Zdej        fd„Ze	e
	 	 	 	 	 ddej        dz  d	ej        dz  d
ej        dz  dej        dz  dej        dz  dee         deej        df         ez  fd„¦   «         ¦   «         Zˆ xZS )ÚEsmcForMaskedLMr<   c                 óØ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          |j        |j        ¦  «        | _        |                      ¦   «          d S r'   )	r(   rA   râ   rÔ   rû   rT   rè   Úlm_headrð   r‚   s     €r.   rA   zEsmcForMaskedLM.__init__ª  sW   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý˜fÑ%Ô%ˆŒ	Ý'¨Ô(:¸FÔ<MÑNÔNˆŒØ�ŠÑÔÐÐÐr/   r%   c                 ó   — | j         j        S r'   ©r  r   )r,   s    r.   Úget_output_embeddingsz%EsmcForMaskedLM.get_output_embeddings³  s   € ØŒ|Ô#Ð#r/   Únew_embeddingsc                 ó   — || j         _        d S r'   r	  )r,   r  s     r.   Úset_output_embeddingsz%EsmcForMaskedLM.set_output_embeddings¶  s   € Ø-ˆŒÔÐÐr/   Nrñ   r–   rm   rò   ÚlabelsrX   .c           	      óÔ   —  | j         d||||ddœ|¤Ž}|                      |j        ¦  «        }d}	|� | j        ||fd| j        j        i|¤Ž}	t          |	||j        |j        ¬¦  «        S )aØ  
        sequence_id (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Integer chain-ID tensor forwarded to the encoder for chain-aware
            attention masking. See :meth:`EsmcModel.forward` for the encoding.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for masked language modelling loss.  Positions with label ``-100``
            are ignored.  Other positions must be in ``[0, config.vocab_size)``.

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, EsmcForMaskedLM
        >>> import torch

        >>> model = EsmcForMaskedLM.from_pretrained("biohub/ESMC-300M-hf")
        >>> tokenizer = AutoTokenizer.from_pretrained("biohub/ESMC-300M-hf")
        >>> inputs = tokenizer(["MLKNVQ<mask>LV"], return_tensors="pt")
        >>> outputs = model(**inputs)
        >>> outputs.logits.shape
        torch.Size([1, 11, 64])
        ```
        T)rñ   r–   rm   rò   Úreturn_dictNrè   ©ÚlossÚlogitsr$   rÕ   rà   )	rÔ   r  rö   Úloss_functionr<   rè   r   r$   rÕ   )
r,   rñ   r–   rm   rò   r  rX   Úencoder_outputsr  r  s
             r.   r)   zEsmcForMaskedLM.forward¹  s¬   € ðB $˜$œ)ð 
ØØ)Ø%Ø#Øð
ð 
ð ð
ð 
ˆð —’˜oÔ?Ñ@Ô@ˆà$(ˆØÐØ%�4Ô% f¨fÐbÐbÀÄÔAWÐbÐ[aÐbÐbˆDåØØØ)Ô7Ø&Ô1ð	
ñ 
ô 
ð 	
r/   ©NNNNN)r0   r1   r2   r!   rA   rH   r{   r
  r  r   r   r4   r5   r   r   rs   r   r)   r6   r7   s   @r.   r  r  ¨  s=  ø€ € € € € ð˜zð ð ð ð ð ð ð$ r¤yð $ð $ð $ð $ð.°B´Ið .ð .ð .ð .ð Øð *.Ø.2Ø,0Ø+/Ø&*ð3
ð 3
à”< $Ñ&ð3
ð œ tÑ+ð3
ð ”l TÑ)ð	3
ð
 ”\ DÑ(ð3
ð ”˜tÑ#ð3
ð Ð+Ô,ð3
ð 
ˆuŒ|˜SÐ Ô	! NÑ	2ð3
ð 3
ð 3
ñ „^ñ Ôð3
ð 3
ð 3
ð 3
ð 3
r/   r  c                   ó.   ‡ — e Zd ZdZdefˆ fd„Zd„ Zˆ xZS )ÚEsmcClassificationHeadz-Head for sentence-level classification tasks.r<   c                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        ¦  «        | _        t          j        |j        |j	        ¦  «        | _
        d S r'   )r(   rA   rH   r{   rT   rý   ÚDropoutÚclassifier_dropoutr˜   Ú
num_labelsÚout_projr‚   s     €r.   rA   zEsmcClassificationHead.__init__ô  sc   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ý”z &Ô";Ñ<Ô<ˆŒÝœ	 &Ô"4°fÔ6GÑHÔHˆŒˆˆr/   c                 óô   — |d d …dd d …f         }|                       |¦  «        }|                      |¦  «        }t          j        |¦  «        }|                       |¦  «        }|                      |¦  «        }|S )Nr   )r˜   rý   r4   Útanhr  )r,   ÚfeaturesrX   rl   s       r.   r)   zEsmcClassificationHead.forwardú  sj   € Ø�Q�Q�Q˜˜1˜1˜1�WÔˆØ�LŠL˜‰OŒOˆØ�JŠJ�q‰MŒMˆÝŒJ�q‰MŒMˆØ�LŠL˜‰OŒOˆØ�MŠM˜!ÑÔˆØˆr/   )r0   r1   r2   r3   r!   rA   r)   r6   r7   s   @r.   r  r  ñ  s`   ø€ € € € € Ø7Ð7ðI˜zð Ið Ið Ið Ið Ið Iðð ð ð ð ð ð r/   r  zœ
    ESMC Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled
    output) e.g. for GLUE tasks.
    )Úcustom_introc                   óÖ   ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 ddej        dz  dej	        dz  dej        dz  dej
        dz  dej        dz  d	ee         d
eez  fd„¦   «         ¦   «         Zˆ xZS )ÚEsmcForSequenceClassificationr<   c                 óè   •— t          ¦   «                              |¦  «         |j        | _        || _        t	          |¦  «        | _        t          |¦  «        | _        |                      ¦   «          d S r'   )	r(   rA   r  r<   râ   rÔ   r  Ú
classifierrð   r‚   s     €r.   rA   z&EsmcForSequenceClassification.__init__  s`   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒØˆŒÝ˜fÑ%Ô%ˆŒ	Ý0°Ñ8Ô8ˆŒà�ŠÑÔÐÐÐr/   Nrñ   r–   rm   rô   r  rX   r%   c                 óŒ  —  | j         |f|||dœ|¤Ž}|d         }|                      |¦  «        }	d}
|��t|                     |	j        ¦  «        }| j        j        €f| j        dk    rd| j        _        nN| j        dk    r7|j        t          j	        k    s|j        t          j
        k    rd| j        _        nd| j        _        | j        j        dk    rWt          ¦   «         }| j        dk    r1 ||	                     ¦   «         |                     ¦   «         ¦  «        }
nŽ ||	|¦  «        }
n�| j        j        dk    rGt          ¦   «         } ||	                     d| j        ¦  «        |                     d¦  «        ¦  «        }
n*| j        j        dk    rt          ¦   «         } ||	|¦  «        }
t!          |
|	|j        |j        ¬	¦  «        S )
a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        )r–   rm   rô   r   Nr    Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr]   r  )rÔ   r%  r*   r:   r<   Úproblem_typer  r+   r4   ÚlongrÂ   r   Úsqueezer   r¶   r   r   r$   rÕ   )r,   rñ   r–   rm   rô   r  rX   ÚoutputsÚsequence_outputr  r  Úloss_fcts               r.   r)   z%EsmcForSequenceClassification.forward  s×  € ð$ �$”)Øð
à)Ø%Ø'ð	
ð 
ð
 ð
ð 
ˆð " !œ*ˆØ—’ Ñ1Ô1ˆàˆØÑØ—Y’Y˜vœ}Ñ-Ô-ˆFàŒ{Ô'Ð/Ø”? aÒ'Ð'Ø/;�D”KÔ,Ð,Ø”_ qÒ(Ð(¨f¬l½e¼jÒ.HÐ.HÈFÌLÕ\aÔ\eÒLeÐLeØ/L�D”KÔ,Ð,à/K�D”KÔ,àŒ{Ô'¨<Ò7Ð7Ý"™9œ9�Ø”? aÒ'Ð'Ø#˜8 F§N¢NÑ$4Ô$4°f·n²nÑ6FÔ6FÑGÔG�D�Dà#˜8 F¨FÑ3Ô3�D�DØ”Ô)Ð-JÒJÐJÝ+Ñ-Ô-�Ø�x §¢¨B°´Ñ @Ô @À&Ç+Â+ÈbÁ/Ä/ÑRÔR��Ø”Ô)Ð-IÒIÐIÝ,Ñ.Ô.�Ø�x ¨Ñ/Ô/�å'ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r/   r  )r0   r1   r2   r!   rA   r   r   r4   Ú
LongTensorr5   ÚFloatTensorr   r   rs   r   r)   r6   r7   s   @r.   r#  r#    sü   ø€ € € € € ð˜zð ð ð ð ð ð ð Øð .2Ø.2Ø04Ø26Ø*.ð8
ð 8
àÔ# dÑ*ð8
ð œ tÑ+ð8
ð Ô&¨Ñ-ð	8
ð
 Ô(¨4Ñ/ð8
ð Ô  4Ñ'ð8
ð Ð+Ô,ð8
ð 
Ð)Ñ	)ð8
ð 8
ð 8
ñ „^ñ Ôð8
ð 8
ð 8
ð 8
ð 8
r/   r#  c                   ó   — e Zd ZdZdS )ÚEsmcForTokenClassificationrÔ   N)r0   r1   r2   r×   rà   r/   r.   r3  r3  Q  s   € € € € € ØÐÐÐr/   r3  )râ   r  r#  r3  rÓ   )r    )Nr‘   )DrË   Úcollections.abcr   r4   Útorch.nnrH   r   r   r   r   r  Úactivationsr	   Úintegrationsr
   r   Úmasking_utilsr   r   Úmodeling_layersr   r   Úmodeling_outputsr   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   r   Úutils.output_capturingr   Úconfiguration_esmcr!   rþ   r#   ÚModuler9   rv   r‡   r�   r5   rW   r£   r¥   rÄ   rÓ   râ   rû   r  r  r#  r3  Ú__all__rà   r/   r.   ú<module>rE     s  ðð* €€€Ø $Ð $Ð $Ð $Ð $Ð $à €€€Ø Ð Ð Ð Ð Ð Ø AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ AØ $Ð $Ð $Ð $Ð $Ð $à !Ð !Ð !Ð !Ð !Ð !Ø LÐ LÐ LÐ LÐ LÐ LÐ LÐ LØ UÐ UÐ UÐ UÐ UÐ UÐ UÐ UØ XÐ XÐ XÐ XÐ XÐ XÐ XÐ XØ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YØ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø eÐ eÐ eÐ eÐ eÐ eÐ eÐ eÐ eÐ eØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø *Ð *Ð *Ð *Ð *Ð *ð	Fð 	Fð 	Fð 	Fð 	F�B”Lñ 	Fô 	Fð 	Fð6<ð 6<ð 6<ð 6<ð 6<˜"œ)ñ 6<ô 6<ð 6<ðrð ð ð ð ˆbŒiñ ô ð ð (ð (ð (ð ÐÐ-Ñ.Ô.ðð ð ñ /Ô.ðð> !Øð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð �T‰\ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð8 ÐÐ)Ñ*Ô*ðA6ð A6ð A6ð A6ð A6�B”Iñ A6ô A6ñ +Ô*ðA6ðH"ð "ð "ð "ð "Ð*ñ "ô "ð "ðJ ð&ð &ð &ð &ð &˜/ñ &ô &ñ „ð&ð& ðY9ð Y9ð Y9ð Y9ð Y9Ð#ñ Y9ô Y9ñ „ðY9ðx+ð +ð +ð +ð +�r”yñ +ô +ð +ð ðE
ð E
ð E
ð E
ð E
Ð)ñ E
ô E
ñ „ðE
ðPð ð ð ð ˜RœYñ ô ð ð& €ððñ ô ðD
ð D
ð D
ð D
ð D
Ð$7ñ D
ô D
ñô ðD
ðNð ð ð ð Ð!>Ð@Sñ ô ð ðð ð €€€r/   