§
    9²¸jÜú  ã                   óJ  — d dl Z d dlmZ d dlmZ d dlZd dlmZ ddlmZ	 ddl
mZ ddlmZmZ dd	lmZ dd
lmZmZ ddlmZmZ ddlmZ ddlmZmZmZmZmZ ddlm Z m!Z! ddl"m#Z#m$Z$ ddl%m&Z& ddl'm(Z(m)Z)m*Z* ddl+m,Z, ddl-m.Z.m/Z/ ddl0m1Z1m2Z2 ddl3m4Z4m5Z5 d„ Z6 G d„ dej7        ¦  «        Z8 G d„ dej7        ¦  «        Z9d„ Z: ed¦  «        d`d„¦   «         Z;	 	 dad ej7        d!ej<        d"ej<        d#ej<        d$ej<        dz  d%e=dz  d&e=d'e&e(         fd(„Z> ee;¦  «         G d)„ d*ej7        ¦  «        ¦   «         Z? G d+„ d,ej7        ¦  «        Z@ G d-„ d.ej7        ¦  «        ZAd/„ ZB G d0„ d1ej7        ¦  «        ZC G d2„ d3ej7        ¦  «        ZD G d4„ d5e¦  «        ZE G d6„ d7ej7        ¦  «        ZF G d8„ d9ej7        ¦  «        ZGe) G d:„ d;e$¦  «        ¦   «         ZH G d<„ d=eH¦  «        ZI G d>„ d?ej7        ¦  «        ZJ G d@„ dAej7        ¦  «        ZK G dB„ dCej7        ¦  «        ZLe)e G dD„ dEe¦  «        ¦   «         ¦   «         ZM G dF„ dGej7        ¦  «        ZN G dH„ dIej7        ¦  «        ZO edJ¦  «         G dK„ dLej7        ¦  «        ¦   «         ZP G dM„ dNej7        ¦  «        ZQ G dO„ dPej7        ¦  «        ZRdQej<        dReSdSej<        fdT„ZT ee;¦  «         G dU„ dVej7        ¦  «        ¦   «         ZU G dW„ dXe¦  «        ZVe) G dY„ dZe$¦  «        ¦   «         ZW G d[„ d\eW¦  «        ZX G d]„ d^eWe¦  «        ZYg d_¢ZZdS )bé    N)ÚCallable)Ú	dataclass)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)Ú"BaseModelOutputWithCrossAttentionsÚBaseModelOutputWithPastÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚCausalLMOutputWithPastÚModelOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚEvollaConfigÚSaProtConfigc                 óÖ   — |                       |¦  «                             ¦   «         }t          j        |d¬¦  «                             |¦  «        |z  }|                     ¦   «         |z   S )a  
    Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
    are ignored. This is modified from fairseq's `utils.make_positions`.

    Args:
        x: torch.Tensor x:

    Returns: torch.Tensor
    r#   ©Údim)ÚneÚintÚtorchÚcumsumÚtype_asÚlong)Ú	input_idsÚpadding_idxÚmaskÚincremental_indicess       úz/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/evolla/modeling_evolla.pyÚ"create_position_ids_from_input_idsr4   4   s`   € ð �<Š<˜Ñ$Ô$×(Ò(Ñ*Ô*€DÝœ, t°Ð3Ñ3Ô3×;Ò;¸DÑAÔAÀDÑHÐØ×#Ò#Ñ%Ô%¨Ñ3Ð3ó    c                   ó8   ‡ — e Zd ZdZˆ fd„Z	 	 	 	 dd„Zd„ Zˆ xZS )ÚEvollaSaProtEmbeddingszV
    Same as BertEmbeddings with a tiny tweak for positional embeddings indexing.
    c                 ó"  •— t          ¦   «                              ¦   «          t          j        |j        |j        |j        ¬¦  «        | _        |j        r&t          j	        |j        |j
        ¬¦  «        | _        nd | _        t          j        |j        ¦  «        | _        t          |dd¦  «        | _        |j        | _        | j        dk    r+t          j        |j        |j        | j        ¬¦  «        | _        |j        | _        |j        | _        d S )N)r0   ©ÚepsÚposition_embedding_typeÚabsolute)ÚsuperÚ__init__r   Ú	EmbeddingÚ
vocab_sizeÚhidden_sizeÚpad_token_idÚword_embeddingsÚemb_layer_norm_beforeÚ	LayerNormÚlayer_norm_epsÚ
layer_normÚDropoutÚhidden_dropout_probÚdropoutÚgetattrr;   r0   Úmax_position_embeddingsÚposition_embeddingsÚtoken_dropoutÚmask_token_id©ÚselfÚconfigÚ	__class__s     €r3   r>   zEvollaSaProtEmbeddings.__init__I   s÷   ø€ Ý‰Œ×ÒÑÔÐÝ!œ|¨FÔ,=¸vÔ?QÐ_eÔ_rÐsÑsÔsˆÔàÔ'ð 	#Ý œl¨6Ô+=À6ÔCXÐYÑYÔYˆDŒOˆOà"ˆDŒOÝ”z &Ô"<Ñ=Ô=ˆŒå'.¨vÐ7PÐR\Ñ']Ô']ˆÔ$à!Ô.ˆÔØÔ'¨:Ò5Ð5Ý')¤|ØÔ.°Ô0BÐPTÔP`ð(ñ (ô (ˆDÔ$ð $Ô1ˆÔØ#Ô1ˆÔÐÐr5   Nc                 ó  — |€-|�t          || j        ¦  «        }n|                      |¦  «        }|€|                      |¦  «        }|}| j        r¾|�¼|                     || j        k                         d¦  «        d¦  «        }d}|�|                     d¦  «        n|j	        d         }|| j        k                         d¦  «         
                    ¦   «         |z  }|d|z
  z  d|z
  d d …d d f         z                       |j        ¦  «        }| j        dk    r|                      |¦  «        }	||	z   }| j        �|                      |¦  «        }|�0||                     d¦  «        z                       |j        ¦  «        }|S )Néÿÿÿÿç        g¸…ëQ¸¾?r#   r<   )r4   r0   Ú&create_position_ids_from_inputs_embedsrC   rN   Úmasked_fillrO   Ú	unsqueezeÚsumÚshapeÚfloatÚtoÚdtyper;   rM   rG   )
rQ   r/   Úattention_maskÚposition_idsÚinputs_embedsÚ
embeddingsÚmask_ratio_trainÚsrc_lengthsÚmask_ratio_observedrM   s
             r3   ÚforwardzEvollaSaProtEmbeddings.forward]   s¶  € ð ÐØÐ$åAÀ)ÈTÔM]Ñ^Ô^��à#×JÒJÈ=ÑYÔY�àÐ Ø ×0Ò0°Ñ;Ô;ˆMð #ˆ
ð Ôð 	 )Ð"7Ø#×/Ò/°¸dÔ>PÒ1P×0[Ò0[Ð\^Ñ0_Ô0_ÐadÑeÔeˆJØ)ÐØ4BÐ4N˜.×,Ò,¨RÑ0Ô0Ð0ÐT]ÔTcÐdeÔTfˆKØ#,°Ô0BÒ#B×"GÒ"GÈÑ"KÔ"K×"QÒ"QÑ"SÔ"SÐVaÑ"aÐØ$¨Ð,<Ñ(<Ñ=ÀÐEXÑAXÐZ[ÐZ[ÐZ[Ð]aÐcgÐZgÔ@hÑh×lÒlØÔ ñô ˆJð Ô'¨:Ò5Ð5Ø"&×":Ò":¸<Ñ"HÔ"HÐØ#Ð&9Ñ9ˆJàŒ?Ð&ØŸš¨Ñ4Ô4ˆJØÐ%Ø$ ~×'?Ò'?ÀÑ'CÔ'CÑC×GÒGÈ
ÔHXÑYÔYˆJð Ðr5   c                 ó  — |                      ¦   «         dd…         }|d         }t          j        | j        dz   || j        z   dz   t          j        |j        ¬¦  «        }|                     d¦  «                             |¦  «        S )z×
        We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

        Args:
            inputs_embeds: torch.Tensor

        Returns: torch.Tensor
        NrU   r#   ©r^   Údevicer   )Úsizer+   Úaranger0   r.   ri   rY   Úexpand)rQ   ra   Úinput_shapeÚsequence_lengthr`   s        r3   rW   z=EvollaSaProtEmbeddings.create_position_ids_from_inputs_embedsŽ   s‡   € ð $×(Ò(Ñ*Ô*¨3¨B¨3Ô/ˆØ% aœ.ˆå”|ØÔ˜qÑ  /°DÔ4DÑ"DÀqÑ"HÕPUÔPZÐcpÔcwð
ñ 
ô 
ˆð ×%Ò% aÑ(Ô(×/Ò/°Ñ<Ô<Ð<r5   ©NNNN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r>   rf   rW   Ú__classcell__©rS   s   @r3   r7   r7   D   st   ø€ € € € € ðð ð2ð 2ð 2ð 2ð 2ð, ØØØð/ð /ð /ð /ðb=ð =ð =ð =ð =ð =ð =r5   r7   c            	       óô   ‡ — e Zd ZdZ edd¬¦  «        ddefˆ fd„¦   «         Ze edd¬¦  «        ddedee	j
        ef         fd	„¦   «         ¦   «         Z e	j        ¦   «         edd
„¦   «         ¦   «         Zˆ xZS )ÚEvollaSaProtRotaryEmbeddingzú
    Rotary position embeddings.
    Implementation based on [ModernBERT's RotaryEmbedding](https://github.com/huggingface/transformers/blob/aad13b87ed59f2afcfaebc985f403301887a35fc/src/transformers/models/modernbert/modeling_modernbert.py#L94).
    ri   ú5.18©ÚversionNrR   c                 óò   •— t          ¦   «                              ¦   «          || _        i | _        |                      | j        ¦  «        \  }}t          j        |¦  «        | _        t          | d|¦  «         d S )NÚattention_scaling)	r=   r>   rR   Ú	rope_typeÚcompute_default_rope_parametersr   ÚBufferÚinv_freqÚsetattr)rQ   rR   ri   Úcurr_inv_freqÚcurr_attention_scalingrS   s        €r3   r>   z$EvollaSaProtRotaryEmbedding.__init__¦   so   ø€ å‰Œ×ÒÑÔÐàˆŒØˆŒà04×0TÒ0TÐUYÔU`Ñ0aÔ0aÑ-ˆÐ-Ýœ	 -Ñ0Ô0ˆŒÝ�Ð)Ð+AÑBÔBÐBÐBÐBr5   Úreturnc                 óØ   — | j         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «        |z  z  z  }|                     |¦  «        |fS )a¿  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.

        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Úhead_dimNç      ð?r   é   ©r^   )Ú
rope_thetarK   rA   Únum_attention_headsr+   rk   r\   r]   ©rR   ri   ÚkwargsÚbaser(   Úattention_factorr€   s          r3   r~   z;EvollaSaProtRotaryEmbedding.compute_default_rope_parameters±   su   € ð Ô ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐà˜$¥5¤<°°3¸ÅÄÐ#MÑ#MÔ#MÐPSÑ#SÑTÑUˆØ�{Š{˜6Ñ"Ô"Ð$4Ð4Ð4r5   c                 óp  — t          | d¦  «        }t          | d¦  «        }|d d d …d f                              ¦   «                              |j        d         dd¦  «                             |j        ¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬	¦  «        5  |                     ¦   «         |                     ¦   «         z   
                    dd
¦  «        }	t          j        |	|	fd¬¦  «        }
|
                     ¦   «         |z  }|
                     ¦   «         |z  }d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬¦  «        |                     |j        ¬¦  «        fS )Nr€   r|   r   rU   r#   ÚmpsÚcpuF©Údevice_typeÚenabledrˆ   r'   r‰   )rK   r\   rl   r[   r]   ri   Ú
isinstanceÚtypeÚstrr   Ú	transposer+   ÚcatÚcosÚsinr^   )rQ   Úxr`   Ú
layer_typer€   r|   Úinv_freq_expandedÚposition_ids_expandedr”   ÚfreqsÚembr›   rœ   s                r3   rf   z#EvollaSaProtRotaryEmbedding.forwardÆ   sÑ  € õ ˜4 Ñ,Ô,ˆÝ# DÐ*=Ñ>Ô>Ðà$ T¨1¨1¨1¨d ]Ô3×9Ò9Ñ;Ô;×BÒBÀ<ÔCUÐVWÔCXÐZ\Ð^_Ñ`Ô`×cÒcÐdeÔdlÑmÔmÐØ ,¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆÝ¨¸UÐCÑCÔCð 	0ð 	0Ø&×,Ò,Ñ.Ô.Ð1F×1LÒ1LÑ1NÔ1NÑN×YÒYÐZ[Ð]^Ñ_Ô_ˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)Ð/Ñ/ˆCØ—'’'‘)”)Ð/Ñ/ˆCð		0ð 	0ð 	0ñ 	0ô 	0ð 	0ð 	0ð 	0ð 	0ð 	0ð 	0øøøð 	0ð 	0ð 	0ð 	0ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   Ã'BE7Å7E;Å>E;©N)rp   rq   rr   rs   r   r%   r>   ÚstaticmethodÚtupler+   ÚTensorr\   r~   Úno_gradr   rf   rt   ru   s   @r3   rw   rw       s  ø€ € € € € ðð ð
 €_�X vÐ.Ñ.Ô.ðCð C˜|ð Cð Cð Cð Cð Cñ /Ô.ðCð Ø€_�X vÐ.Ñ.Ô.ð5ð 5°ð 5ÐX]Ð^cÔ^jÐlqÐ^qÔXrð 5ð 5ð 5ñ /Ô.ñ „\ð5ð& €U„]�_„_Øð<ð <ð <ñ Ôñ „_ð<ð <ð <ð <ð <r5   rw   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..NrU   rˆ   r'   )r[   r+   rš   )r�   Úx1Úx2s      r3   Úrotate_halfr«   Ù   s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'r5   Úrotary_pos_embc                 ó¨  — | j         }|                     |¦  «        }|                     |¦  «        }|                      ¦   «         |z  t          |                      ¦   «         ¦  «        |z  z   }|                     ¦   «         |z  t          |                     ¦   «         ¦  «        |z  z   }|                     |¦  «        |                     |¦  «        fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )r^   rY   r\   r«   r]   )ÚqÚkr›   rœ   Úunsqueeze_dimÚoriginal_dtypeÚq_embedÚk_embeds           r3   Úapply_rotary_pos_embr´   à   s¦   € ð& ”W€NØ
�-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�wŠw‰yŒy˜3‰¥;¨q¯wªw©y¬yÑ#9Ô#9¸CÑ#?Ñ@€GØ�wŠw‰yŒy˜3‰¥;¨q¯wªw©y¬yÑ#9Ô#9¸CÑ#?Ñ@€GØ�:Š:�nÑ%Ô% w§z¢z°.Ñ'AÔ'AÐAÐAr5   rV   ÚmoduleÚqueryÚkeyÚvaluer_   ÚscalingrJ   r�   c                 ó  — |€|                      d¦  «        dz  }t          | dd¦  «        }t          ||¦  «        }t          ||¦  «        }t          j        ||                     dd¦  «        ¦  «        |z  }	|�|	|z   }	t          j                             |	d¬¦  «        }	t          j         	                    |	|| j
        ¬¦  «        }	t          j        |	|¦  «        }
|
                     dd¦  «                             ¦   «         }
|
|	fS )	NrU   ç      à¿Únum_key_value_groupsr#   rˆ   r   r'   )ÚpÚtraining)rj   rK   Ú	repeat_kvr+   Úmatmulr™   r   Ú
functionalÚsoftmaxrJ   r¾   Ú
contiguous)rµ   r¶   r·   r¸   r_   r¹   rJ   r�   Ún_repÚattn_weightsÚattn_outputs              r3   Úeager_attention_forwardrÇ   û   sú   € ð €Ø—*’*˜R‘.”. DÑ(ˆõ
 �FÐ2°AÑ6Ô6€EÝ
�C˜Ñ
Ô
€CÝ�e˜UÑ#Ô#€Eõ ”<  s§}¢}°Q¸Ñ':Ô':Ñ;Ô;¸gÑE€LàÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2Ð(Ñ>Ô>€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r5   c                   óº   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 ddej        dej        dz  dej        dz  dej        dz  dej        dz  d	ee         d
e	ej                 fd„Z
ˆ xZS )ÚEvollaSaProtSelfAttentionNFc                 óÐ  •— t          ¦   «                              ¦   «          || _        |j        |j        z  dk    r0t          |d¦  «        s t          d|j        › d|j        › d�¦  «        ‚|j        | _        t          |j        |j        z  ¦  «        | _        | j        | j        z  | _	        t          j        |j        | j	        ¦  «        | _        t          j        |j        | j	        ¦  «        | _        t          j        |j        | j	        ¦  «        | _        |j        | _        |pt#          |dd¦  «        | _        d| _        |j        | _        || _        | j        o| | _        d S )	Nr   Úembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (ú)r;   r<   r‡   )r=   r>   rR   rA   r‹   ÚhasattrÚ
ValueErrorr*   Úattention_head_sizeÚall_head_sizer   ÚLinearr¶   r·   r¸   Úattention_probs_dropout_probrJ   rK   r;   r¹   Ú
is_decoderÚ	layer_idxÚ	is_causal)rQ   rR   r;   rÔ   Úis_cross_attentionrS   s        €r3   r>   z"EvollaSaProtSelfAttention.__init__   se  ø€ Ý‰Œ×ÒÑÔÐØˆŒàÔ Ô :Ñ:¸aÒ?Ð?ÍÐPVÐXhÑHiÔHiÐ?Ýð8 FÔ$6ð 8ð 8Ø Ô4ð8ð 8ð 8ñô ð ð
 $*Ô#=ˆÔ Ý#& vÔ'9¸FÔ<VÑ'VÑ#WÔ#WˆÔ Ø!Ô5¸Ô8PÑPˆÔå”Y˜vÔ1°4Ô3EÑFÔFˆŒ
Ý”9˜VÔ/°Ô1CÑDÔDˆŒÝ”Y˜vÔ1°4Ô3EÑFÔFˆŒ
àÔ:ˆŒà'>ð (
Å'ØÐ-¨zñC
ô C
ˆÔ$ð ˆŒØ Ô+ˆŒØ"ˆŒØœÐCÐ1CÐ-CˆŒˆˆr5   Úhidden_statesr_   Úencoder_hidden_statesÚencoder_attention_maskrM   r�   r„   c                 ó
  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }	|d u}
|
r|n|}|
r|n|}|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }|	| j        dz  z  }	| j        dk    r|\  }}t          |	|||d¬¦  «        \  }	}t          j
        | j        j        t          ¦  «        } || |	|||f| j        sdn| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }||fS )	NrU   r#   rˆ   r»   Úrotary)r°   rV   ©rJ   r¹   )r[   rÏ   r¶   Úviewr™   r·   r¸   r;   r´   r   Úget_interfacerR   Ú_attn_implementationrÇ   r¾   rJ   r¹   ÚreshaperÃ   )rQ   r×   r_   rØ   rÙ   rM   r�   rm   Úhidden_shapeÚquery_layerrÖ   Úcurrent_statesÚ	key_layerÚvalue_layerr›   rœ   Úattention_interfacerÆ   rÅ   s                      r3   rf   z!EvollaSaProtSelfAttention.forward<  sÛ  € ð $Ô)¨#¨2¨#Ô.ˆØC˜ÐC bÐC¨$Ô*BÐCÐCˆà—j’j Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆà2¸$Ð>ÐØ2DÐWÐ.Ð.È-ˆØ3EÐYÐ/Ð/È>ˆØ—H’H˜^Ñ,Ô,×1Ò1°,Ñ?Ô?×IÒIÈ!ÈQÑOÔOˆ	Ø—j’j Ñ0Ô0×5Ò5°lÑCÔC×MÒMÈaÐQRÑSÔSˆð " DÔ$<¸dÑ$BÑBˆàÔ'¨8Ò3Ð3Ø*‰HˆC�Ý%9¸+ÀyÐRUÐWZÐjkÐ%lÑ%lÔ%lÑ"ˆK˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}Ð>�C�C°$´,Ø”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ˜LÐ(Ð(r5   )NNFro   )rp   rq   rr   r>   r+   r¦   ÚFloatTensorr   r   r¥   rf   rt   ru   s   @r3   rÉ   rÉ     sÜ   ø€ € € € € ðDð Dð Dð Dð Dð Dð> 48Ø:>Ø;?Ø37ð.)ð .)à”|ð.)ð Ô)¨DÑ0ð.)ð  %Ô0°4Ñ7ð	.)ð
 !&Ô 1°DÑ 8ð.)ð #œ\¨DÑ0ð.)ð Ð+Ô,ð.)ð 
ˆuŒ|Ô	ð.)ð .)ð .)ð .)ð .)ð .)ð .)ð .)r5   rÉ   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚEvollaSaProtSelfOutputc                 óÌ   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        ¦  «        | _        d S r£   )	r=   r>   r   rÑ   rA   ÚdenserH   rI   rJ   rP   s     €r3   r>   zEvollaSaProtSelfOutput.__init__n  sJ   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ý”z &Ô"<Ñ=Ô=ˆŒˆˆr5   c                 ód   — |                       |¦  «        }|                      |¦  «        }||z   }|S r£   ©rë   rJ   ©rQ   r×   Úinput_tensors      r3   rf   zEvollaSaProtSelfOutput.forwards  ó4   € ØŸ
š
 =Ñ1Ô1ˆØŸš ]Ñ3Ô3ˆØ%¨Ñ4ˆØÐr5   ©rp   rq   rr   r>   rf   rt   ru   s   @r3   ré   ré   m  óG   ø€ € € € € ð>ð >ð >ð >ð >ð
ð ð ð ð ð ð r5   ré   c                   óV   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 ddej        dz  dee         fd„Zˆ xZ	S )	ÚEvollaSaProtAttentionNFc                 óè   •— t          ¦   «                              ¦   «          t          |||¬¦  «        | _        t	          |¦  «        | _        t          j        |j        |j	        ¬¦  «        | _        d S )N)rÔ   rÖ   r9   )
r=   r>   rÉ   rQ   ré   Úoutputr   rE   rA   rF   )rQ   rR   rÔ   rÖ   rS   s       €r3   r>   zEvollaSaProtAttention.__init__{  s_   ø€ Ý‰Œ×ÒÑÔÐÝ-¨fÀ	Ð^pÐqÑqÔqˆŒ	Ý,¨VÑ4Ô4ˆŒåœ fÔ&8¸fÔ>SÐTÑTÔTˆŒˆˆr5   rM   r�   c                 óˆ   — |                       |¦  «        } | j        |f||||dœ|¤Ž\  }}	|                      ||¦  «        }|S )N©r_   rØ   rÙ   rM   )rE   rQ   rö   )
rQ   r×   r_   rØ   rÙ   rM   r�   Úhidden_states_lnrÆ   Ú_s
             r3   rf   zEvollaSaProtAttention.forward‚  sk   € ð  Ÿ>š>¨-Ñ8Ô8ÐØ"˜œØð
à)Ø"7Ø#9Ø 3ð
ð 
ð ð
ð 
‰ˆ�Qð —k’k +¨}Ñ=Ô=ˆØÐr5   )NFro   )
rp   rq   rr   r>   r+   r¦   r   r   rf   rt   ru   s   @r3   rô   rô   z  s‹   ø€ € € € € ðUð Uð Uð Uð Uð Uð Ø"Ø#Ø37ðð ð #œ\¨DÑ0ðð Ð+Ô,ðð ð ð ð ð ð ð r5   rô   c                 óf   — | dz  dt          j        | t          j        d¦  «        z  ¦  «        z   z  S )zz
    This is the gelu implementation from the original EVOLLA_SA_PROT repo. Using F.gelu yields subtly wrong results.
    g      à?r‡   g       @)r+   ÚerfÚmathÚsqrt)r�   s    r3   Úgelurÿ   ˜  s/   € ð ˆs‰7�c�EœI a­$¬)°C©.¬.Ñ&8Ñ9Ô9Ñ9Ñ:Ð:r5   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚEvollaSaProtIntermediatec                 ó�   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        d S r£   )r=   r>   r   rÑ   rA   Úintermediate_sizerë   rP   s     €r3   r>   z!EvollaSaProtIntermediate.__init__   s6   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3KÑLÔLˆŒ
ˆ
ˆ
r5   r×   r„   c                 óN   — |                       |¦  «        }t          |¦  «        }|S r£   )rë   rÿ   )rQ   r×   s     r3   rf   z EvollaSaProtIntermediate.forward¤  s&   € ØŸ
š
 =Ñ1Ô1ˆÝ˜]Ñ+Ô+ˆØÐr5   ©rp   rq   rr   r>   r+   r¦   rf   rt   ru   s   @r3   r  r  Ÿ  sc   ø€ € € € € ðMð Mð Mð Mð Mð U¤\ð °e´lð ð ð ð ð ð ð ð r5   r  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚEvollaSaProtOutputc                 óÌ   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        ¦  «        | _	        d S r£   )
r=   r>   r   rÑ   r  rA   rë   rH   rI   rJ   rP   s     €r3   r>   zEvollaSaProtOutput.__init__«  sJ   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ7¸Ô9KÑLÔLˆŒ
Ý”z &Ô"<Ñ=Ô=ˆŒˆˆr5   c                 ód   — |                       |¦  «        }|                      |¦  «        }||z   }|S r£   rí   rî   s      r3   rf   zEvollaSaProtOutput.forward°  rð   r5   rñ   ru   s   @r3   r  r  ª  rò   r5   r  c                   óZ   ‡ — e Zd Zˆ fd„Z	 	 	 	 ddej        dz  dee         fd„Zd„ Z	ˆ xZ
S )ÚEvollaSaProtLayerc                 óÌ  •— t          ¦   «                              ¦   «          |j        | _        d| _        t	          |¦  «        | _        |j        | _        |j        | _        | j        r/| j        st          | › d�¦  «        ‚t	          |d¬¦  «        | _	        t          |¦  «        | _        t          |¦  «        | _        t          j        |j        |j        ¬¦  «        | _        d S )Nr#   z> should be used as a decoder model if cross attention is addedT)rÖ   r9   )r=   r>   Úchunk_size_feed_forwardÚseq_len_dimrô   Ú	attentionrÓ   Úadd_cross_attentionÚRuntimeErrorÚcrossattentionr  Úintermediater  rö   r   rE   rA   rF   rP   s     €r3   r>   zEvollaSaProtLayer.__init__¸  sÓ   ø€ Ý‰Œ×ÒÑÔÐØ'-Ô'EˆÔ$ØˆÔÝ.¨vÑ6Ô6ˆŒØ Ô+ˆŒØ#)Ô#=ˆÔ ØÔ#ð 	YØ”?ð lÝ" dÐ#jÐ#jÐ#jÑkÔkÐkÝ"7¸ÐSWÐ"XÑ"XÔ"XˆDÔÝ4°VÑ<Ô<ˆÔÝ(¨Ñ0Ô0ˆŒÝœ fÔ&8¸fÔ>SÐTÑTÔTˆŒˆˆr5   NrM   r�   c                 óÐ   —  | j         |f||dœ|¤Ž}| j        r8|�6t          | d¦  «        st          d| › d�¦  «        ‚ | j        |f||||dœ|¤Ž}|                      |¦  «        }|S )N©r_   rM   r  z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`rø   )r  rÓ   rÍ   ÚAttributeErrorr  Úfeed_forward_chunk)	rQ   r×   r_   rØ   rÙ   rM   r�   Úattention_outputÚlayer_outputs	            r3   rf   zEvollaSaProtLayer.forwardÇ  sÜ   € ð *˜4œ>Øð
à)Ø 3ð
ð 
ð ð	
ð 
Ðð Œ?ð 	Ð4Ð@Ý˜4Ð!1Ñ2Ô2ð Ý$ð`¸dð `ð `ð `ñô ð ð
  3˜tÔ2Ø ð à-Ø&;Ø'=Ø$7ð ð  ð ð ð  Ðð ×.Ò.Ð/?Ñ@Ô@ˆØÐr5   c                 ó†   — |                       |¦  «        }|                      |¦  «        }|                      ||¦  «        }|S r£   )rE   r  rö   )rQ   r  Úattention_output_lnÚintermediate_outputr  s        r3   r  z$EvollaSaProtLayer.feed_forward_chunkê  sE   € Ø"ŸnšnÐ-=Ñ>Ô>ÐØ"×/Ò/Ð0CÑDÔDÐØ—{’{Ð#6Ð8HÑIÔIˆØÐr5   ro   )rp   rq   rr   r>   r+   r¦   r   r   rf   r  rt   ru   s   @r3   r  r  ·  s•   ø€ € € € € ðUð Uð Uð Uð Uð$ Ø"Ø#Ø37ð!ð !ð #œ\¨DÑ0ð!ð Ð+Ô,ð!ð !ð !ð !ðFð ð ð ð ð ð r5   r  c                   ód   ‡ — e Zd Zˆ fd„Ze	 	 	 	 ddej        dz  dee         fd„¦   «         Z	ˆ xZ
S )ÚEvollaSaProtEncoderc                 ó  •‡— t          ¦   «                              ¦   «          ‰| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          j        ‰j	        ‰j
        ¬¦  «        | _        d| _        d S )Nc                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS © )r  )Ú.0rú   rR   s     €r3   ú
<listcomp>z0EvollaSaProtEncoder.__init__.<locals>.<listcomp>õ  s"   ø€ Ð#gÐ#gÐ#gÀ!Õ$5°fÑ$=Ô$=Ð#gÐ#gÐ#gr5   r9   F)r=   r>   rR   r   Ú
ModuleListÚrangeÚnum_hidden_layersÚlayerrE   rA   rF   Úemb_layer_norm_afterÚgradient_checkpointingrP   s    `€r3   r>   zEvollaSaProtEncoder.__init__ò  s}   øø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”]Ð#gÐ#gÐ#gÐ#gÅuÈVÔMeÑGfÔGfÐ#gÑ#gÔ#gÑhÔhˆŒ
Ý$&¤L°Ô1CÈÔI^Ð$_Ñ$_Ô$_ˆÔ!Ø&+ˆÔ#Ð#Ð#r5   NrM   r�   c           	      óª   — t          | j        ¦  «        D ]\  }} ||f||||dœ|¤Ž}Œ| j        r|                      |¦  «        }t          |¬¦  «        S )Nrø   )Úlast_hidden_state)Ú	enumerater'  r(  r   )	rQ   r×   r_   rØ   rÙ   rM   r�   ÚiÚlayer_modules	            r3   rf   zEvollaSaProtEncoder.forwardù  s‰   € õ  )¨¬Ñ4Ô4ð 	ð 	‰OˆAˆ|Ø(˜LØðà-Ø&;Ø'=Ø$7ðð ð ðð ˆMˆMð Ô$ð 	EØ ×5Ò5°mÑDÔDˆMå1ÀMÐRÑRÔRÐRr5   ro   )rp   rq   rr   r>   r   r+   r¦   r   r   rf   rt   ru   s   @r3   r  r  ñ  s˜   ø€ € € € € ð,ð ,ð ,ð ,ð ,ð ð Ø"Ø#Ø37ðSð Sð #œ\¨DÑ0ðSð Ð+Ô,ðSð Sð Sñ ÔðSð Sð Sð Sð Sr5   r  c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚEvollaSaProtPoolerc                 óÀ   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        ¦   «         | _        d S r£   )r=   r>   r   rÑ   rA   rë   ÚTanhÚ
activationrP   s     €r3   r>   zEvollaSaProtPooler.__init__  sC   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ýœ'™)œ)ˆŒˆˆr5   r×   r„   c                 ór   — |d d …df         }|                       |¦  «        }|                      |¦  «        }|S )Nr   )rë   r3  )rQ   r×   Úfirst_token_tensorÚpooled_outputs       r3   rf   zEvollaSaProtPooler.forward  s@   € ð +¨1¨1¨1¨a¨4Ô0ÐØŸ
š
Ð#5Ñ6Ô6ˆØŸš¨Ñ6Ô6ˆØÐr5   r  ru   s   @r3   r0  r0    s^   ø€ € € € € ð$ð $ð $ð $ð $ð
 U¤\ð °e´lð ð ð ð ð ð ð ð r5   r0  c                   óª   ‡ — e Zd ZU eed<   dgZdZdZdZdZ	e
 eedd¬¦  «        g eedd¬¦  «        gdœZ ej        ¦   «         ˆ fd	„¦   «         Zˆ xZS )
ÚEvollaSaProtPreTrainedModelrR   r  Tr#   r  )ÚindexÚ
layer_namer  )r×   Ú
attentionsÚcross_attentionsc                 óø   •— t          ¦   «                              |¦  «         t          |t          ¦  «        rB|                     |j        ¦  «        \  }}t          j        t          |d¦  «        |¦  «         d S d S )Nr€   )	r=   Ú_init_weightsr–   rw   r~   rR   ÚinitÚcopy_rK   )rQ   rµ   r‚   rú   rS   s       €r3   r>  z)EvollaSaProtPreTrainedModel._init_weights3  sw   ø€ å‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ9Ñ:Ô:ð 	CØ%×EÒEÀfÄmÑTÔTÑˆM˜1ÝŒJ•w˜v zÑ2Ô2°MÑBÔBÐBÐBÐBð	Cð 	Cr5   )rp   rq   rr   r%   Ú__annotations__Ú_no_split_modulesÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendr  r!   rÉ   Ú_can_record_outputsr+   r§   r>  rt   ru   s   @r3   r8  r8  "  sÉ   ø€ € € € € € àÐÐÑØ,Ð-ÐØÐØ€NØÐØ"&Ðð +Ø%�~Ð&?ÀqÐU`ÐaÑaÔaÐbàˆNÐ4¸AÐJZÐ[Ñ[Ô[ð
ðð Ðð €U„]�_„_ðCð Cð Cð Cñ „_ðCð Cð Cð Cð Cr5   r8  c            
       ó¤   ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zee	 d
de	j
        dz  de	j
        dz  dee	j
                 ez  fd	„¦   «         ¦   «         Zˆ xZS )ÚEvollaSaProtProteinEncoderrR   c                 óì   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          |¬¦  «        | _        t          |¦  «        | _        |                      ¦   «          d S ©N©rR   )	r=   r>   r7   rb   rw   Úrotary_embeddingsr  ÚencoderÚ	post_initrP   s     €r3   r>   z#EvollaSaProtProteinEncoder.__init__<  sb   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý0°Ñ8Ô8ˆŒÝ!<ÀFÐ!KÑ!KÔ!KˆÔÝ*¨6Ñ2Ô2ˆŒØ�ŠÑÔÐÐÐr5   c                 ó   — | j         j        S r£   ©rb   rC   ©rQ   s    r3   Úget_input_embeddingsz/EvollaSaProtProteinEncoder.get_input_embeddingsC  s   € ØŒÔ.Ð.r5   c                 ó   — || j         _        d S r£   rQ  ©rQ   r¸   s     r3   Úset_input_embeddingsz/EvollaSaProtProteinEncoder.set_input_embeddingsF  s   € Ø*/ˆŒÔ'Ð'Ð'r5   Nr/   r_   r„   c                 óÆ  — |                      ¦   «         }|\  }}|j        }|€t          j        ||f|¬¦  «        }|                      ||¬¦  «        }t          | j        ||¬¦  «        }t          j        ||¬¦  «                             d¦  «        }	|  	                    ||	¦  «        }
 | j
        |f||
dœ|¤Ž}|d         }t          ||j        |j        |j        ¬¦  «        S )N©ri   ©r/   r_   )rR   ra   r_   r   r  )r+  r×   r;  r<  )rj   ri   r+   Úonesrb   r   rR   rk   rY   rM  rN  r   r×   r;  r<  )rQ   r/   r_   r�   rm   Ú
batch_sizeÚ
seq_lengthri   ra   r`   rM   Úencoder_outputsÚsequence_outputs                r3   rf   z"EvollaSaProtProteinEncoder.forwardI  s  € ð  —n’nÑ&Ô&ˆØ!,Ñˆ
�JàÔ!ˆØÐ!Ý"œZ¨*°jÐ)AÈ6ÐRÑRÔRˆNØŸš°)ÈN˜Ñ[Ô[ˆå2Ø”;Ø'Ø)ð
ñ 
ô 
ˆõ ”| J°vÐ>Ñ>Ô>×HÒHÈÑKÔKˆØ"×4Ò4°]ÀLÑQÔQÐà&˜$œ,Øð
Ø*8ÐNað
ð 
Øekð
ð 
ˆð *¨!Ô,ˆå;Ø-Ø)Ô7Ø&Ô1Ø,Ô=ð	
ñ 
ô 
ð 	
r5   r£   )rp   rq   rr   r%   r>   rS  rV  r    r"   r+   r¦   r¥   r   rf   rt   ru   s   @r3   rI  rI  ;  sÎ   ø€ € € € € ð˜|ð ð ð ð ð ð ð/ð /ð /ð0ð 0ð 0ð  Øð /3ð!
ð !
à”< $Ñ&ð!
ð œ tÑ+ð!
ð
 
ˆuŒ|Ô	ÐKÑ	Kð!
ð !
ð !
ñ „_ñ  Ôð!
ð !
ð !
ð !
ð !
r5   rI  c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )Ú!EvollaSequenceCompressorAttentioné@   é   c                 ó†  •— t          ¦   «                              ¦   «          |dz  | _        || _        ||z  }t	          j        |¦  «        | _        t	          j        |¦  «        | _        t	          j        ||d¬¦  «        | _	        t	          j        ||dz  d¬¦  «        | _
        t	          j        ||d¬¦  «        | _        d S )Nr»   F©Úbiasrˆ   )r=   r>   ÚscaleÚheadsr   rE   Ú
norm_mediaÚnorm_latentsrÑ   Úto_qÚto_kvÚto_out)rQ   r(   Údim_headrg  Ú	inner_dimrS   s        €r3   r>   z*EvollaSequenceCompressorAttention.__init__p  s¨   ø€ Ý‰Œ×ÒÑÔÐØ˜t‘^ˆŒ
ØˆŒ
Ø˜uÑ$ˆ	åœ, sÑ+Ô+ˆŒÝœL¨Ñ-Ô-ˆÔå”I˜c 9°5Ð9Ñ9Ô9ˆŒ	Ý”Y˜s I°¡M¸Ð>Ñ>Ô>ˆŒ
Ý”i 	¨3°UÐ;Ñ;Ô;ˆŒˆˆr5   c                 ó
  — |                       |¦  «        }|                      |¦  «        }| j        }|                      |¦  «        }t	          j        ||fd¬¦  «        }|                      |¦  «                             dd¬¦  «        \  }}|                     | 	                    d¦  «        | 	                    d¦  «        |d¦  «         
                    dddd¦  «        }|                     | 	                    d¦  «        | 	                    d¦  «        |d¦  «         
                    dddd¦  «        }|                     | 	                    d¦  «        | 	                    d¦  «        |d¦  «         
                    dddd¦  «        }|| j        z  }t	          j        ||                     dd¦  «        ¦  «        }	|	|	                     dd¬	¦  «                             ¦   «         z
  }	|	j        \  }
}}}t	          j        ||¦  «                             |j        ¦  «        }|d
d
…d
d
d
d
…f         }|d
d
d
…d
d
…d
f         }||z  }|	                     d|z
                       ¦   «         d¦  «        }	|	                     d¬¦  «        }t	          j        ||¦  «        }| 
                    dddd¦  «        }|                     | 	                    d¦  «        | 	                    d¦  «        d¦  «        }|                      |¦  «        S )zÔ
        Args:
            x (torch.Tensor): image features
                shape (b, n1, D)
            latent (torch.Tensor): latent features
                shape (b, n2, D);  n2: num of latent tokens
        éþÿÿÿr'   rˆ   rU   r   r#   r   T©r(   ÚkeepdimNg     ˆÃÀ)rh  ri  rg  rj  r+   rš   rk  ÚchunkrÝ   rj   Úpermuterf  rÀ   r™   ÚamaxÚdetachr[   rZ  r]   ri   rX   ÚboolrÂ   rà   rl  )rQ   r�   Úlatentsr1   Úhr®   Úkv_inputr¯   ÚvÚsimÚbsÚnhÚskdÚokdrZ  Úmask_expÚones_expÚattnÚouts                      r3   rf   z)EvollaSequenceCompressorAttention.forward}  s¢  € ð �OŠO˜AÑÔˆØ×#Ò# GÑ,Ô,ˆàŒJˆà�IŠI�gÑÔˆÝ”9˜a ˜\¨rÐ2Ñ2Ô2ˆØ�zŠz˜(Ñ#Ô#×)Ò)Ø�2ð *ñ 
ô 
‰ˆˆ1ð �FŠF�1—6’6˜!‘9”9˜aŸfšf Q™iœi¨¨BÑ/Ô/×7Ò7¸¸1¸aÀÑCÔCˆØ�FŠF�1—6’6˜!‘9”9˜aŸfšf Q™iœi¨¨BÑ/Ô/×7Ò7¸¸1¸aÀÑCÔCˆØ�FŠF�1—6’6˜!‘9”9˜aŸfšf Q™iœi¨¨BÑ/Ô/×7Ò7¸¸1¸aÀÑCÔCˆØ�”
‰Nˆõ Œl˜1˜aŸkšk¨"¨bÑ1Ô1Ñ2Ô2ˆØ�C—H’H ¨T�HÑ2Ô2×9Ò9Ñ;Ô;Ñ;ˆØœ9ÑˆˆB��SÝŒz˜"˜cÑ"Ô"×%Ò% d¤kÑ2Ô2ˆØ˜˜˜˜4  q q qÐ(Ô)ˆØ˜˜a˜a˜a    DÐ(Ô)ˆØ˜(Ñ"ˆà�oŠo˜q 4™xŸošoÑ/Ô/°Ñ6Ô6ˆØ�{Š{˜rˆ{Ñ"Ô"ˆÝŒl˜4 Ñ#Ô#ˆØ�kŠk˜!˜Q  1Ñ%Ô%ˆð �kŠk˜#Ÿ(š( 1™+œ+ s§x¢x°¡{¤{°BÑ7Ô7ˆà�{Š{˜3ÑÔÐr5   )ra  rb  rñ   ru   s   @r3   r`  r`  o  sL   ø€ € € € € ð<ð <ð <ð <ð <ð <ð) ð ) ð ) ð ) ð ) ð ) ð ) r5   r`  c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚEvollaFeedForwardé   c                 ó>  •— t          ¦   «                              ¦   «          t          ||z  ¦  «        }t          j        |¦  «        | _        t          j        ||d¬¦  «        | _        t          j        ¦   «         | _	        t          j        ||d¬¦  «        | _
        d S ©NFrd  )r=   r>   r*   r   rE   ÚnormrÑ   Úfc1ÚGELUr3  Úfc2)rQ   r(   Úmultrn  rS   s       €r3   r>   zEvollaFeedForward.__init__ª  sz   ø€ Ý‰Œ×ÒÑÔÐÝ˜˜d™
‘O”Oˆ	å”L Ñ%Ô%ˆŒ	Ý”9˜S )°%Ð8Ñ8Ô8ˆŒÝœ'™)œ)ˆŒÝ”9˜Y¨°%Ð8Ñ8Ô8ˆŒˆˆr5   c           	      óž   — |                       |                      |                      |                      |¦  «        ¦  «        ¦  «        ¦  «        S r£   )r�  r3  r‹  rŠ  )rQ   r�   s     r3   rf   zEvollaFeedForward.forward³  s6   € Ø�xŠx˜Ÿš¨¯ª°·²¸1±´Ñ(>Ô(>Ñ?Ô?Ñ@Ô@Ð@r5   )r‡  rñ   ru   s   @r3   r†  r†  ©  sS   ø€ € € € € ð9ð 9ð 9ð 9ð 9ð 9ðAð Að Að Að Að Að Ar5   r†  c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )Ú!EvollaSequenceCompressorResamplerrR   c           
      óp  •— t          ¦   «                              ¦   «          |j        j        }|j        | _        t          j        t          j	        | j        |¦  «        d¬¦  «        | _
        t          j        g ¦  «        | _        t          |j        ¦  «        D ]^}| j                             t          j        t!          ||j        |j        ¬¦  «        t'          ||j        ¬¦  «        g¦  «        ¦  «         Œ_t          j        |j        ¦  «        | _        t          j        ||j        ¦  «        | _        d S )NT)Úrequires_grad)r(   rm  rg  )r(   rŽ  )r=   r>   Úprotein_encoder_configrA   Úresampler_num_latentsÚnum_latentsr   Ú	Parameterr+   Úrandnrx  r$  Úlayersr%  Úresampler_depthÚappendr`  Úresampler_dim_headÚresampler_headsr†  Úresampler_ff_multrE   rŠ  rÑ   Úprotein_projector)rQ   rR   Úprotein_repr_dimrú   rS   s       €r3   r>   z*EvollaSequenceCompressorResampler.__init__¸  s  ø€ Ý‰Œ×ÒÑÔÐØ!Ô8ÔDÐØ!Ô7ˆÔÝ”|¥E¤K°Ô0@ÐBRÑ$SÔ$SÐcgÐhÑhÔhˆŒÝ”m BÑ'Ô'ˆŒÝ�vÔ-Ñ.Ô.ð 
	ð 
	ˆAØŒK×ÒÝ”å9Ø 0¸6Ô;TÐ\bÔ\rðñ ô õ *Ð.>ÀVÔE]Ð^Ñ^Ô^ð	ñô ñ	ô 	ð 	ð 	õ ”L Ô!3Ñ4Ô4ˆŒ	Ý!#¤Ð+;¸VÔ=OÑ!PÔ!PˆÔÐÐr5   c                 óN  — |j         d         }|j         \  }}t          j        || j        ¦  «                             |j        ¦  «        }t          j        ||fd¬¦  «        }t          j        |¦  «                             | j        j        ¦  «        }| j        d          |                     ddd¦  «        z  }|                     |j	        ¦  «        }| j
        D ]#\  }	}
 |	|||¦  «        |z   } |
|¦  «        |z   }Œ$|                      |¦  «        }|                      |¦  «        S )Nr   r#   r'   rU   )r[   r+   rZ  r–  r]   ri   rš   rx  rÝ   r^   r™  rŸ  rŠ  )rQ   Úembedsr1   Úbr}  rú   Úlatent_maskrZ  rx  rƒ  ÚffÚtransformed_features               r3   rf   z)EvollaSequenceCompressorResampler.forwardÍ  s  € ØŒL˜ŒOˆà”
‰ˆˆAÝ”j  TÔ%5Ñ6Ô6×9Ò9¸$¼+ÑFÔFˆÝŒy˜$ Ð,°!Ð4Ñ4Ô4ˆõ Œz˜!‰}Œ}×Ò ¤Ô 3Ñ4Ô4ˆØ”,˜tÔ$ t§y¢y°°Q¸Ñ':Ô':Ñ:ˆØ—*’*˜Vœ\Ñ*Ô*ˆØœð 	,ð 	,‰HˆD�"Ø�d˜6 7¨DÑ1Ô1°GÑ;ˆGØ�b˜‘k”k GÑ+ˆGˆGà"×4Ò4°WÑ=Ô=Ðà�yŠyÐ,Ñ-Ô-Ð-r5   )rp   rq   rr   r$   r>   rf   rt   ru   s   @r3   r‘  r‘  ·  sZ   ø€ € € € € ðQ˜|ð Qð Qð Qð Qð Qð Qð*.ð .ð .ð .ð .ð .ð .r5   r‘  c                   ó¬   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
ej        df         dz  ed<   dZe
ej        df         dz  ed<   dS )ÚEvollaProteinEncoderModelOutputaN  
    sequence_compressor_output (`torch.FloatTensor` of shape `(batch_size, compressed_seq_len, hidden_size)`, *optional*):
        Compressed sequence representation produced by the sequence compressor module. The sequence length is
        reduced from the original input length to `compressed_seq_len` via learned compression.
    NÚsequence_compressor_outputr+  .r×   r;  )rp   rq   rr   rs   r©  r+   rç   rA  r+  r×   r¥   r;  r!  r5   r3   r¨  r¨  á  s•   € € € € € € ðð ð <@Ð Ô 1°DÑ 8Ð?Ð?Ñ?Ø26Ð�uÔ(¨4Ñ/Ð6Ð6Ñ6Ø:>€M�5˜Ô*¨CÐ/Ô0°4Ñ7Ð>Ð>Ñ>Ø7;€J��eÔ'¨Ð,Ô-°Ñ4Ð;Ð;Ñ;Ð;Ð;r5   r¨  c                   óX   ‡ — e Zd Zdefˆ fd„Zedej        dej        fd„¦   «         Z	ˆ xZ
S )ÚEvollaProteinEncoderrR   c                 ó¦   •— t          ¦   «                              ¦   «          t          |j        ¬¦  «        | _        t          |¬¦  «        | _        d S rK  )r=   r>   rI  r”  Úmodelr‘  Úsequence_compressor_resamplerrP   s     €r3   r>   zEvollaProteinEncoder.__init__ñ  sH   ø€ Ý‰Œ×ÒÑÔÐÝ/°vÔ7TÐUÑUÔUˆŒ
Ý-NÐV\Ð-]Ñ-]Ô-]ˆÔ*Ð*Ð*r5   r/   r_   c                 ó–   — |                       ||¬¦  «        }|j        }|                      ||¦  «        }t          ||j        ¬¦  «        S )NrY  )r©  r+  )r­  r+  r®  r¨  )rQ   r/   r_   r�   Úprotein_outputÚprotein_embedsÚsequence_reprs          r3   rf   zEvollaProteinEncoder.forwardö  sT   € àŸš¨iÈ˜ÑWÔWˆØ'Ô9ˆØ×:Ò:¸>È>ÑZÔZˆå.Ø'4Ø,Ô>ð
ñ 
ô 
ð 	
r5   )rp   rq   rr   r$   r>   r   r+   Ú
LongTensorrç   rf   rt   ru   s   @r3   r«  r«  ð  s€   ø€ € € € € ð^˜|ð ^ð ^ð ^ð ^ð ^ð ^ð
 ð
 Ô!1ð 
À5ÔCTð 
ð 
ð 
ñ Ôð
ð 
ð 
ð 
ð 
r5   r«  c                   ób   ‡ — e Zd Z	 	 	 ddedz  dedz  dedz  fˆ fd„Zd„ Z	 	 	 	 	 	 	 d	d„Zˆ xZS )
Ú#EvollaSequenceAlignerCrossAttentionNÚprotein_encoder_dimÚstructure_encoder_dimÚmsa_encoder_dimc                 óà  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        | j        dz  | _        t          | j        | j        z  ¦  «        | _        | j        | j        z  | _        |j        }|j	        }|j
        }t          j        | j        | j        ¦  «        | _        |�?t          j        || j        ¦  «        | _        t          j        || j        ¦  «        | _        nd | _        d | _        |�?t          j        || j        ¦  «        | _        t          j        || j        ¦  «        | _        nd | _        d | _        |�?t          j        || j        ¦  «        | _        t          j        || j        ¦  «        | _        nd | _        d | _        t)          | j        ¦  «        | _        t          j        |¦  «        | _        t          j        | j        | j        |¬¦  «        | _        t3          | j        |¦  «        | _        t          j        t9          j        dg¦  «        ¦  «        | _        t          j        t9          j        dg¦  «        ¦  «        | _        d S )Nr»   rd  rV   ) r=   r>   rA   r‹   rf  r*   rÏ   rÐ   Ú$aligner_attention_probs_dropout_probÚaligner_enable_biasÚaligner_ffn_multr   rÑ   r¶   Úkey_proteinÚvalue_proteinÚkey_structureÚvalue_structureÚkey_msaÚ	value_msaÚEvollaRMSNormÚattention_normrH   rJ   Úout_projr†  r¥  r—  r+   ÚtensorÚgate_attentionÚgate_ffw)	rQ   rR   r¶  r·  r¸  rÒ   Úenable_biasÚffn_multrS   s	           €r3   r>   z,EvollaSequenceAlignerCrossAttention.__init__  s
  ø€ õ 	‰Œ×ÒÑÔÐà!Ô-ˆÔØ#)Ô#=ˆÔ ØÔ-¨tÑ3ˆŒ
Ý#& tÔ'7¸$Ô:RÑ'RÑ#SÔ#SˆÔ Ø!Ô5¸Ô8PÑPˆÔà'-Ô'RÐ$ØÔ0ˆØÔ*ˆå”Y˜tÔ/°Ô1CÑDÔDˆŒ
ØÐ*Ý!œyÐ)<¸dÔ>PÑQÔQˆDÔÝ!#¤Ð+>ÀÔ@RÑ!SÔ!SˆDÔÐà#ˆDÔØ!%ˆDÔà Ð,Ý!#¤Ð+@À$ÔBTÑ!UÔ!UˆDÔÝ#%¤9Ð-BÀDÔDVÑ#WÔ#WˆDÔ Ð à!%ˆDÔØ#'ˆDÔ àÐ&Ýœ9 _°dÔ6HÑIÔIˆDŒLÝœY ¸Ô8JÑKÔKˆDŒNˆNàˆDŒLØ!ˆDŒNå+¨DÔ,<Ñ=Ô=ˆÔå”zÐ">Ñ?Ô?ˆŒåœ	 $Ô"2°DÔ4DÈ;ÐWÑWÔWˆŒå# DÔ$4°hÑ?Ô?ˆŒÝ œl­5¬<¸¸Ñ+>Ô+>Ñ?Ô?ˆÔÝœ¥U¤\°3°%Ñ%8Ô%8Ñ9Ô9ˆŒˆˆr5   c	                 óŽ  — |||g}	d„ |	D ¦   «         }	|	st          d¦  «        ‚t          j        |	d¬¦  «        }	|                      |¦  «        }
|                      |
¦  «        }
| j        �G| j        �@|                     |¦  «        }|                      |¦  «        }|                      |¦  «        }nd}d}| j        �G| j	        �@|                     |¦  «        }|                      |¦  «        }|  	                    |¦  «        }nd}d}| j
        �G| j        �@|                     |¦  «        }|  
                    |¦  «        }|                      |¦  «        }nd}d}|||g}d„ |D ¦   «         }t          j        |d¬¦  «        }|||g}d„ |D ¦   «         }t          j        |d¬¦  «        }|
                     ¦   «         dd…         | j        | j        fz   } |
j        |Ž                      d	d
dd¦  «        }
|                     ¦   «         dd…         | j        | j        fz   } |j        |Ž                      d	d
dd¦  «        }|                     ¦   «         dd…         | j        | j        fz   } |j        |Ž                      d	d
dd¦  «        }|
| j        z  }
|€St          j        |                     d	¦  «        |                     d¦  «        ¦  «                             |j        ¦  «        }|dd…ddd…df         |	dd…dddd…f         z  }t          j        |
|                     dd¦  «        ¦  «        }||                     dd¬¦  «                             ¦   «         z
  }|                     d|z
                       ¦   «         t          j        |j        ¦  «        j        ¦  «        } t;          j        d¬¦  «        |¦  «        }t          j        ||¦  «        }|                     d	d
dd¦  «                             ¦   «         }|                     ¦   «         dd…         | j         fz   } |j        |Ž }|  !                    |¦  «        }|S )zù
        query_states: text
        key_value_states: protein
        query_states: [bs, query_seq_len, dim]
        key_value_states: [bs, kv_seq_len, dim]
        query_attn_mask: [bs, query_seq_len]
        kv_attn_mask: [bs, kv_seq_len]
        c                 ó   — g | ]}|®|‘ŒS r£   r!  ©r"  rú   s     r3   r#  zGEvollaSequenceAlignerCrossAttention.cross_attention.<locals>.<listcomp>L  s   € ÐAÐAÐA˜a°1°=˜°=°=°=r5   z=At least one modality should be provided for cross attention.r#   r'   Nc                 ó   — g | ]}|®|‘ŒS r£   r!  rÍ  s     r3   r#  zGEvollaSequenceAlignerCrossAttention.cross_attention.<locals>.<listcomp>r  s   € Ð;Ð;Ð;˜1¨Q¨]�Q¨]¨]¨]r5   c                 ó   — g | ]}|®|‘ŒS r£   r!  rÍ  s     r3   r#  zGEvollaSequenceAlignerCrossAttention.cross_attention.<locals>.<listcomp>v  s   € Ð?Ð?Ð?˜Q°°�q°°°r5   rU   r   rˆ   r   rp  Trq  )"rÎ   r+   rš   rÄ  r¶   r½  r¾  r]   r¿  rÀ  rÁ  rÂ  rj   r‹   rÏ   rÝ   rt  rf  rZ  ri   rÀ   r™   ru  rv  rX   rw  Úfinfor^   Úminr   ÚSoftmaxrÃ   rÐ   rÅ  )rQ   Úquery_statesÚprotein_key_value_statesÚstructure_key_value_statesÚmsa_key_value_statesÚquery_attn_maskÚprotein_kv_attn_maskÚstructure_kv_attn_maskÚmsa_kv_attn_maskÚkv_attn_maskrâ   Úkey_layer_proteinÚvalue_layer_proteinÚkey_layer_structureÚvalue_layer_structureÚkey_layer_msaÚvalue_layer_msarä   rå   Únew_query_layer_shapeÚnew_key_layer_shapeÚnew_value_layer_shaper_   rÅ   Úattention_scoresÚattention_probsÚcontext_layerÚnew_context_layer_shapes                               r3   Úcross_attentionz3EvollaSequenceAlignerCrossAttention.cross_attention6  sª  € ð* -Ð.DÐFVÐWˆØAÐA <ÐAÑAÔAˆØð 	^ÝÐ\Ñ]Ô]Ð]Ý”y °1Ð5Ñ5Ô5ˆà×)Ò)¨,Ñ7Ô7ˆð —j’j Ñ-Ô-ˆàÔÐ'¨DÔ,>Ð,JØ'?×'BÒ'BÀ<Ñ'PÔ'PÐ$Ø $× 0Ò 0Ð1IÑ JÔ JÐØ"&×"4Ò"4Ð5MÑ"NÔ"NÐÐà $ÐØ"&ÐàÔÐ)¨dÔ.BÐ.NØ)C×)FÒ)FÀ|Ñ)TÔ)TÐ&Ø"&×"4Ò"4Ð5OÑ"PÔ"PÐØ$(×$8Ò$8Ð9SÑ$TÔ$TÐ!Ð!à"&ÐØ$(Ð!àŒ<Ð#¨¬Ð(BØ#7×#:Ò#:¸<Ñ#HÔ#HÐ Ø ŸLšLÐ)=Ñ>Ô>ˆMØ"ŸnšnÐ-AÑBÔBˆOˆOà ˆMØ"ˆOà&Ð(;¸]ÐKˆ	Ø;Ð; 	Ð;Ñ;Ô;ˆ	Ý”I˜i¨QÐ/Ñ/Ô/ˆ	à*Ð,AÀ?ÐSˆØ?Ð? +Ð?Ñ?Ô?ˆÝ”i °Ð3Ñ3Ô3ˆà +× 0Ò 0Ñ 2Ô 2°3°B°3Ô 7ØÔ$ØÔ$ð;
ñ !
Ðð '�kÔ&Ð(=Ð>×FÒFÀqÈ!ÈQÐPQÑRÔRˆà'ŸnšnÑ.Ô.¨s°¨sÔ3ØÔ$ØÔ$ð7
ñ 
Ðð #�I”NÐ$7Ð8×@Ò@ÀÀAÀqÈ!ÑLÔLˆ	à +× 0Ò 0Ñ 2Ô 2°3°B°3Ô 7ØÔ$ØÔ$ð;
ñ !
Ðð '�kÔ&Ð(=Ð>×FÒFÀqÈ!ÈQÐPQÑRÔRˆà! D¤JÑ.ˆð Ð"Ý#œj¨×):Ò):¸1Ñ)=Ô)=¸|×?PÒ?PÐQRÑ?SÔ?SÑTÔT×WÒWÐXdÔXkÑlÔlˆOØ(¨¨¨¨D°!°!°!°TÐ)9Ô:¸\È!È!È!ÈTÐSWÐYZÐYZÐYZÐJZÔ=[Ñ[ˆå”| K°×1DÒ1DÀRÈÑ1LÔ1LÑMÔMˆØ# l×&7Ò&7¸BÈÐ&7Ñ&MÔ&M×&TÒ&TÑ&VÔ&VÑVˆØ'×3Ò3Ø�Ñ×%Ò%Ñ'Ô'­¬°\Ô5GÑ)HÔ)HÔ)Lñ
ô 
Ðð -�"œ*¨Ð,Ñ,Ô,Ð-=Ñ>Ô>ˆõ œ _°kÑBÔBˆà%×-Ò-¨a°°A°qÑ9Ô9×DÒDÑFÔFˆØ"/×"4Ò"4Ñ"6Ô"6°s¸°sÔ";¸tÔ?QÐ>SÑ"SÐØ*˜Ô*Ð,CÐDˆàŸš mÑ4Ô4ˆàÐr5   c           
      ó8  — |�q|j         \  }}}|€ct          j        ||¦  «                             |	j        ¦  «        |	                     ||f¬¦  «        j        z                       |j        ¦  «        }nd }|�q|j         \  }}}|€ct          j        ||¦  «                             |	j        ¦  «        |
                     ||f¬¦  «        j        z                       |j        ¦  «        }nd }|�q|j         \  }}}|€ct          j        ||¦  «                             |	j        ¦  «        |                     ||f¬¦  «        j        z                       |j        ¦  «        }nd }|}|�|                     ¦   «         s,|�|                     ¦   «         s|�Š|                     ¦   «         rv|}|                      ||||||||¬¦  «        }t          j	        | j
        ¦  «        |z  }||z   }|}|                      |¦  «        t          j	        | j        ¦  «        z  }||z   }|S )N)rj   )rÓ  rÔ  rÕ  rÖ  r×  rØ  rÙ  rÚ  )r[   r+   rZ  r]   ri   rl   ÚTÚanyré  ÚtanhrÇ  r¥  rÈ  )rQ   rÓ  Úprotein_kv_statesÚstructure_kv_statesÚmsa_kv_statesr×  rØ  rÙ  rÚ  Úprotein_batch_maskÚstructure_batch_maskÚmsa_batch_maskÚpast_key_valuesr}  Úprotein_kv_seq_lenr(   Ústructure_kv_seq_lenÚmsa_kv_seq_lenr×   Úresiduals                       r3   rf   z+EvollaSequenceAlignerCrossAttention.forward¦  s`  € ð Ð(Ø*;Ô*AÑ'ˆBÐ" CØ#Ð+å”J˜rÐ#5Ñ6Ô6×9Ò9Ð:LÔ:SÑTÔTØ(×/Ò/Ð6HÈ"Ð5MÐ/ÑNÔNÔPñQç’"Ð&Ô-Ñ.Ô.ð %øð
 $(Ð àÐ*Ø,?Ô,EÑ)ˆBÐ$ cØ%Ð-å”J˜rÐ#7Ñ8Ô8×;Ò;Ð<NÔ<UÑVÔVØ*×1Ò1Ð8LÈbÐ7QÐ1ÑRÔRÔTñUç’"Ð(Ô/Ñ0Ô0ð 'øð
 &*Ð"àÐ$Ø&3Ô&9Ñ#ˆB� ØÐ'å”J˜r >Ñ2Ô2×5Ò5Ð6HÔ6OÑPÔPØ$×+Ò+°.À"Ð1EÐ+ÑFÔFÔHñIç’"�]Ô)Ñ*Ô*ð !øð
  $ÐØ$ˆð Ð*Ð/C×/GÒ/GÑ/IÔ/IÐ*Ø#Ð/Ð4J×4NÒ4NÑ4PÔ4PÐ/ØÐ)Ð.>×.BÒ.BÑ.DÔ.DÐ)à$ˆHØ ×0Ò0Ø*Ø):Ø+>Ø%2Ø /Ø%9Ø'=Ø!1ð 1ñ 	ô 	ˆMõ "œJ tÔ':Ñ;Ô;¸mÑKˆMà$ }Ñ4ˆMà$ˆHØ ŸGšG MÑ2Ô2µU´ZÀÄÑ5NÔ5NÑNˆMØ$ }Ñ4ˆMàÐr5   ©NNN)NNNNNNN)rp   rq   rr   r*   r>   ré  rf   rt   ru   s   @r3   rµ  rµ    s¾   ø€ € € € € ð +/Ø,0Ø&*ð1:ð 1:ð ! 4™Zð1:ð  # T™zð	1:ð
 ˜t™ð1:ð 1:ð 1:ð 1:ð 1:ð 1:ðfnð nð nðn "Ø#ØØØ!ØØðGð Gð Gð Gð Gð Gð Gð Gr5   rµ  ÚRMSNormc                   óT   ‡ — e Zd Zd	deddfˆ fd„Zdej        dej        fd„Zd„ Zˆ xZ	S )
rÃ  ç�íµ ÷Æ°>r:   r„   Nc                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )z<
        EvollaRMSNorm is equivalent to T5LayerNorm
        N)r=   r>   r   r—  r+   rZ  ÚweightÚvariance_epsilon)rQ   rA   r:   rS   s      €r3   r>   zEvollaRMSNorm.__init__ò  sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐr5   r×   c                 ó  — |j         }|                     t          j        ¦  «        }|                     d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        |                     |¦  «        z  S )Nrˆ   rU   T)rr  )	r^   r]   r+   Úfloat32ÚpowÚmeanÚrsqrtrÿ  rþ  )rQ   r×   Úinput_dtypeÚvariances       r3   rf   zEvollaRMSNorm.forwardú  s|   € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØ ×$Ò$ QÑ'Ô'×,Ò,¨R¸Ð,Ñ>Ô>ˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆØŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:r5   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)r¥   rþ  r[   rÿ  rR  s    r3   Ú
extra_reprzEvollaRMSNorm.extra_repr  s&   € Ý˜œÔ)Ñ*Ô*ÐIÐI°$Ô2GÐIÐIÐIr5   )rü  )
rp   rq   rr   r\   r>   r+   r¦   rf   r  rt   ru   s   @r3   rÃ  rÃ  ð  sŒ   ø€ € € € € ð$ð $¨ð $¸$ð $ð $ð $ð $ð $ð $ð; U¤\ð ;°e´lð ;ð ;ð ;ð ;ðJð Jð Jð Jð Jð Jð Jr5   rÃ  c            	       óî   ‡ — e Zd Z edd¬¦  «        d
defˆ fd„¦   «         Ze edd¬¦  «        d
dedeej	        e
f         fd„¦   «         ¦   «         Z ej        ¦   «         ed	„ ¦   «         ¦   «         Zˆ xZS )ÚEvollaRotaryEmbeddingri   rx   ry   NrR   c                 ó¾  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        | j        j        d         | _        | j        }| j        dk    rt          | j                 } || j        |¦  «        \  }| _
        t          j        |d¬¦  «        | _        t          j        |                     ¦   «         d¬¦  «        | _        d S )Nr}   ÚdefaultF)Ú
persistent)r=   r>   rL   Úmax_seq_len_cachedÚoriginal_max_seq_lenrR   Úrope_parametersr}   r~   r   r|   r   r   r€   ÚcloneÚoriginal_inv_freq)rQ   rR   ri   Úrope_init_fnr€   rS   s        €r3   r>   zEvollaRotaryEmbedding.__init__  sÀ   ø€ å‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!àˆŒàœÔ4°[ÔAˆŒØ!%Ô!EˆØŒ>˜YÒ&Ð&Ý.¨t¬~Ô>ˆLØ+7¨<¸¼ÀVÑ+LÔ+LÑ(ˆ�$Ô(åœ	 (°uÐ=Ñ=Ô=ˆŒÝ!#¤¨8¯>ª>Ñ+;Ô+;ÈÐ!NÑ!NÔ!NˆÔÐÐr5   r„   c                 óä   — | j         d         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «        |z  z  z  }|                     |¦  «        |fS )a¾  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        rŠ   r†   Nr‡   r   rˆ   r‰   )r  rK   rA   r‹   r+   rk   r\   r]   rŒ   s          r3   r~   z5EvollaRotaryEmbedding.compute_default_rope_parameters  sz   € ð Ô% lÔ3ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐà˜$¥5¤<°°3¸ÅÄÐ#MÑ#MÔ#MÐPSÑ#SÑTÑUˆØ�{Š{˜6Ñ"Ô"Ð$4Ð4Ð4r5   c                 óú  — | j         d d d …d f                              |j        d         dd¦  «                             t          j        |j        ¬¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  ||z                       dd	¦  «        }t	          j        ||fd¬
¦  «        }|                     ¦   «         | j        z  }|                     ¦   «         | j        z  }	d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬¦  «        |	                     |j        ¬¦  «        fS )Nr   rU   r#   rh   r‘   r’   Fr“   rˆ   r'   r‰   )r€   rl   r[   r]   r+   r\   ri   r–   r—   r˜   r   r™   rš   r›   r|   rœ   r^   )
rQ   r�   r`   rŸ   r    r”   r¡   r¢   r›   rœ   s
             r3   rf   zEvollaRotaryEmbedding.forward+  s›  € ð ŒM˜$    4˜-Ô(×/Ò/°Ô0BÀ1Ô0EÀrÈ1ÑMÔM×PÒPÕW\ÔWbÐklÔksÐPÑtÔtð 	ð !-¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆå¨¸UÐCÑCÔCð 	5ð 	5Ø&Ð)>Ñ>×IÒIÈ!ÈQÑOÔOˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCð		5ð 	5ð 	5ñ 	5ô 	5ð 	5ð 	5ð 	5ð 	5ð 	5ð 	5øøøð 	5ð 	5ð 	5ð 	5ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   ÃA*D<Ä<E ÅE r£   )rp   rq   rr   r   r$   r>   r¤   r¥   r+   r¦   r\   r~   r§   r   rf   rt   ru   s   @r3   r
  r
    sö   ø€ € € € € Ø€_�X vÐ.Ñ.Ô.ðOð O˜|ð Oð Oð Oð Oð Oñ /Ô.ðOð  Ø€_�X vÐ.Ñ.Ô.ð5ð 5°ð 5ÐX]Ð^cÔ^jÐlqÐ^qÔXrð 5ð 5ð 5ñ /Ô.ñ „\ð5ð$ €U„]�_„_Øð<ð <ñ Ôñ „_ð<ð <ð <ð <ð <r5   r
  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )Ú	EvollaMLPc                 ó¶  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        t          j        | j        | j        |j        ¬¦  «        | _        t          j        | j        | j        |j        ¬¦  «        | _	        t          j        | j        | j        |j        ¬¦  «        | _
        t          |j                 | _        d S )Nrd  )r=   r>   rR   rA   r  r   rÑ   Úmlp_biasÚ	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fnrP   s     €r3   r>   zEvollaMLP.__init__?  s¯   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ!'Ô!9ˆÔÝœ 4Ô#3°TÔ5KÐRXÔRaÐbÑbÔbˆŒÝ”y Ô!1°4Ô3IÐPVÔP_Ð`Ñ`Ô`ˆŒÝœ 4Ô#9¸4Ô;KÐRXÔRaÐbÑbÔbˆŒÝ˜VÔ.Ô/ˆŒˆˆr5   c                 ó¨   — |                       |                      |                      |¦  «        ¦  «        |                      |¦  «        z  ¦  «        }|S r£   )r  r  r  r  )rQ   r�   r  s      r3   rf   zEvollaMLP.forwardI  sA   € Ø—N’N 4§;¢;¨t¯~ª~¸aÑ/@Ô/@Ñ#AÔ#AÀDÇLÂLÐQRÁOÄOÑ#SÑTÔTˆ	ØÐr5   rñ   ru   s   @r3   r  r  >  sG   ø€ € € € € ð0ð 0ð 0ð 0ð 0ðð ð ð ð ð ð r5   r  r×   rÄ   r„   c                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r#   N)r[   rl   rà   )r×   rÄ   ÚbatchÚnum_key_value_headsÚslenr†   s         r3   r¿   r¿   N  s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTr5   c                   óÎ   ‡ — e Zd ZdZdedefˆ fd„Z	 	 	 ddej        de	ej        ej        f         dz  dej        dz  d	e
dz  d
ee         de	ej        ej        f         fd„Zˆ xZS )ÚEvollaAttentionz=Multi-headed attention from 'Attention Is All You Need' paperrR   rÔ   c                 ó®  •— t          ¦   «                              ¦   «          || _        || _        t	          |d|j        |j        z  ¦  «        | _        |j        |j        z  | _	        | j        dz  | _
        |j        | _        d| _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        | j        z  |j        |j        ¬¦  «        | _        d S )Nr†   r»   Trd  )r=   r>   rR   rÔ   rK   rA   r‹   r†   r"  r¼   r¹   Úattention_dropoutrÕ   r   rÑ   Úattention_biasÚq_projÚk_projÚv_projÚo_proj©rQ   rR   rÔ   rS   s      €r3   r>   zEvollaAttention.__init__^  sB  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒÝ ¨
°FÔ4FÈ&ÔJdÑ4dÑeÔeˆŒØ$*Ô$>À&ÔB\Ñ$\ˆÔ!Ø”} dÑ*ˆŒØ!'Ô!9ˆÔØˆŒå”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ&¨¬Ñ6¸Ô8JÐQWÔQfð
ñ 
ô 
ˆŒˆˆr5   Nr×   rM   r_   rô  r�   r„   c                 ó"  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }	|                      |¦  «                             |¦  «                             dd¦  «        }
|\  }}t          ||	||¦  «        \  }}	|�|                     |	|
| j	        ¦  «        \  }	}
t          j        | j        j        t          ¦  «        } || ||	|
|f| j        sdn| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )NrU   r#   rˆ   rV   rÜ   )r[   r†   r)  rÝ   r™   r*  r+  r´   ÚupdaterÔ   r   rÞ   rR   rß   rÇ   r¾   r'  r¹   rà   rÃ   r,  )rQ   r×   rM   r_   rô  r�   rm   rá   rÓ  Ú
key_statesÚvalue_statesr›   rœ   ræ   rÆ   rÅ   s                   r3   rf   zEvollaAttention.forwardu  sÀ  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r5   rù  )rp   rq   rr   rs   r$   r*   r>   r+   r¦   r¥   r	   r   r   rf   rt   ru   s   @r3   r%  r%  Z  så   ø€ € € € € àGÐGð
˜|ð 
¸ð 
ð 
ð 
ð 
ð 
ð 
ð4 IMØ.2Ø(,ð&)ð &)à”|ð&)ð # 5¤<°´Ð#=Ô>ÀÑEð&)ð œ tÑ+ð	&)ð
  ™ð&)ð Ð+Ô,ð&)ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð&)ð &)ð &)ð &)ð &)ð &)ð &)ð &)r5   r%  c                   ó\  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 	 	 	 	 	 	 	 ddej        deej        ej        f         dz  dej        dz  d	ej	        dz  d
e
dz  dedz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        fd„Zˆ xZS )ÚEvollaDecoderLayerrR   rÔ   c                 ó¼  •— t          ¦   «                              ¦   «          |j        | _        t          ||¬¦  «        | _        t          |¦  «        | _        t          |j        |j        ¬¦  «        | _	        t          |j        |j        ¬¦  «        | _
        |dz   t          |j        |j        z  d¦  «        z  dk    rt          ||j        ¬¦  «        | _        d S d S )N©rR   rÔ   r9   r#   r   )r¶  )r=   r>   rA   r%  Ú	self_attnr  ÚmlprÃ  Úrms_norm_epsÚinput_layernormÚpost_attention_layernormÚmaxr&  Úaligner_num_add_layersrµ  Úadapterr-  s      €r3   r>   zEvollaDecoderLayer.__init__Ÿ  sÔ   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔå(°À)ÐLÑLÔLˆŒå˜VÑ$Ô$ˆŒÝ,¨VÔ-?ÀVÔEXÐYÑYÔYˆÔÝ(5°fÔ6HÈfÔNaÐ(bÑ(bÔ(bˆÔ%Ø˜‰M�S Ô!9¸VÔ=ZÑ!ZÐ\]Ñ^Ô^Ñ^ÐbcÒcÐcÝ>ØØ$*Ô$6ðñ ô ˆDŒLˆLˆLð dÐcr5   NFr×   rM   r_   r`   rô  Ú	use_cacherî  rï  rð  rñ  rò  ró  r×  r„   c           
      ó(  — |}|                       |¦  «        } | j        d||||||dœ|¤Ž\  }}||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }t	          | d¦  «        r|                      ||||	||
||¬¦  «        }|S )N)r×   r_   r`   rô  r>  rM   r=  )rÓ  rî  rï  rð  r×  rñ  rò  ró  r!  )r9  r6  r:  r7  rÍ   r=  )rQ   r×   rM   r_   r`   rô  r>  rî  rï  rð  rñ  rò  ró  r×  r�   rø  rú   s                    r3   rf   zEvollaDecoderLayer.forward®  sç   € ð" !ˆà×,Ò,¨]Ñ;Ô;ˆð *˜4œ>ð 
Ø'Ø)Ø%Ø+ØØ 3ð
ð 
ð ð
ð 
Ñˆ�qð ! =Ñ0ˆð !ˆØ×5Ò5°mÑDÔDˆØŸš Ñ/Ô/ˆØ  =Ñ0ˆå�4˜Ñ#Ô#ð 
	Ø ŸLšLØ*Ø"3Ø$7Ø+Ø /Ø#5Ø%9Ø-ð )ñ 	ô 	ˆMð Ðr5   )NNNNFNNNNNNN)rp   rq   rr   r$   r*   r>   r+   r¦   r¥   r³  r	   rw  rf   rt   ru   s   @r3   r3  r3  ž  su  ø€ € € € € ð˜|ð ¸ð ð ð ð ð ð ð$ IMØ.2Ø04Ø(,Ø!&Ø15Ø37Ø-1Ø26Ø48Ø.2Ø/3ð3ð 3à”|ð3ð # 5¤<°´Ð#=Ô>ÀÑEð3ð œ tÑ+ð	3ð
 Ô&¨Ñ-ð3ð  ™ð3ð ˜$‘;ð3ð !œ<¨$Ñ.ð3ð #œ\¨DÑ0ð3ð ”| dÑ*ð3ð "œL¨4Ñ/ð3ð $œl¨TÑ1ð3ð œ tÑ+ð3ð œ¨Ñ,ð3ð  
Œð!3ð 3ð 3ð 3ð 3ð 3ð 3ð 3r5   r3  c                   óˆ   ‡ — e Zd ZU eed<   dZdZg d¢ZdgZdZ	dZ
dZdZdZeedœZ ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )	ÚEvollaPreTrainedModelrR   r­  T)r3  r  r‘  rµ  rô  F)r×   r;  c                 ó–  •— | j         j        }t          ¦   «                              |¦  «         t	          |t
          ¦  «        rRt          j        |j        ¦  «         t          j        |j	        ¦  «         t          j
        |j        j        ¦  «         d S t	          |t          ¦  «        rt          j        |j        d|¬¦  «         d S d S )NrV   )r  Ústd)rR   Úinitializer_ranger=   r>  r–   rµ  r?  Úzeros_rÇ  rÈ  Úones_rÄ  rþ  r‘  Únormal_rx  )rQ   rµ   rC  rS   s      €r3   r>  z#EvollaPreTrainedModel._init_weightsû  sµ   ø€ àŒkÔ+ˆÝ‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕAÑBÔBð 	<ÝŒK˜Ô-Ñ.Ô.Ð.ÝŒK˜œÑ(Ô(Ð(ÝŒJ�vÔ,Ô3Ñ4Ô4Ð4Ð4Ð4Ý˜Õ AÑBÔBð 	<ÝŒL˜œ¨c°sÐ;Ñ;Ô;Ð;Ð;Ð;ð	<ð 	<r5   )rp   rq   rr   r$   rA  Úbase_model_prefixÚsupports_gradient_checkpointingrB  Ú_skip_keys_device_placementrC  rD  rE  Ú_can_compile_fullgraphrF  r3  r%  rG  r+   r§   r>  rt   ru   s   @r3   rA  rA  ä  s³   ø€ € € € € € àÐÐÑØÐØ&*Ð#ðð ð Ðð $5Ð"5ÐØ ÐØ€NØÐà!ÐØ"'Ðà+Ø%ðð Ðð
 €U„]�_„_ð<ð <ð <ð <ñ „_ð<ð <ð <ð <ð <r5   rA  c                   óh  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zeee		 	 	 	 	 	 	 	 	 	 	 	 dde
j        dz  de
j        dz  de
j        dz  d	edz  d
e
j        dz  dedz  de
j        dz  de
j        dz  de
j        dz  de
j        dz  de
j        dz  de
j        dz  deez  fd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚEvollaModelrR   c                 ó&  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          j        | j        ‰j        | j        ¦  «        | _        t          ‰¬¦  «        | _
        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        t%          ‰dd¦  «        | _        t)          ‰¬¦  «        | _        |                      ¦   «          d S )NrL  c                 ó2   •— g | ]}t          ‰|¬ ¦  «        ‘ŒS )r5  )r3  )r"  rÔ   rR   s     €r3   r#  z(EvollaModel.__init__.<locals>.<listcomp>  s@   ø€ ð ð ð ð
 õ	 #Ø!Ø'ðñ ô ðð ð r5   r9   r)  F)r=   r>   rB   r0   r@   r   r?   rA   Úembed_tokensr«  Úprotein_encoderr$  r%  r&  r™  rÃ  r8  rŠ  rK   r)  r
  Ú
rotary_embrO  rP   s    `€r3   r>   zEvollaModel.__init__  s  øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒÝœL¨¬¸&Ô:LÈdÔN^Ñ_Ô_ˆÔÝ3¸6ÐBÑBÔBˆÔÝ”mðð ð ð õ
 "' vÔ'?Ñ!@Ô!@ðñ ô ñ
ô 
ˆŒõ " &Ô"4¸&Ô:MÐNÑNÔNˆŒ	Ý&-¨fÐ6NÐPUÑ&VÔ&VˆÔ#Ý/°vÐ>Ñ>Ô>ˆŒØ�ŠÑÔÐÐÐr5   c                 ó   — | j         S r£   ©rP  rR  s    r3   rS  z EvollaModel.get_input_embeddings  s   € ØÔ Ð r5   c                 ó   — || _         d S r£   rT  rU  s     r3   rV  z EvollaModel.set_input_embeddings   s   € Ø!ˆÔÐÐr5   Nr/   r_   r`   rô  ra   r>  Úprotein_input_idsÚprotein_attention_maskÚstructure_featsÚ	msa_featsrò  ró  r„   c                 óâ  — |du |duz  rt          d¦  «        ‚|€|                      |¦  «        }|r|€t          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   }| 	                    d¦  «        }d}d}|�Q|�O|  
                    ||¬¦  «        }|j        }t          j        |j        d         |j        t
          j        ¬¦  «        }t          | j        |||¬	¦  «        }|}|                      ||¬
¦  «        }| j        D ]} ||f||||||	|
|||||dœ|¤Ž}Œ|                      |¦  «        }t%          ||¬¦  «        }|S )a9  
        protein_input_ids (torch.LongTensor):
            The input IDs for the protein sequence in structure-aware tokens. Should be of shape `(batch_size, protein_seq_length)` and type `torch.LongTensor`.
        protein_attention_mask (torch.Tensor):
            The attention mask for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.Tensor`.
        structure_feats (torch.FloatTensor):
            The input IDs for purely structure-based features. Should be of shape `(batch_size, structure_seq_length, structure_feat_dim)` and type `torch.FloatTensor`. Dummy input for now.
        msa_feats (torch.FloatTensor):
            The input IDs for purely MSA-based features. Should be of shape `(batch_size, msa_seq_length, msa_feat_dim)` and type `torch.FloatTensor`. Dummy input for now.
        structure_batch_mask (torch.Tensor):
            The batch mask to decide which protein sequences are purely structure-based. Should be of shape `(batch_size)` and type `torch.Tensor`. Should be paired with `structure_feats`. Dummy input for now.
        msa_batch_mask (torch.Tensor):
            The batch mask to decide which protein sequences are purely MSA-based. Should be of shape `(batch_size)` and type `torch.Tensor`. Should be paired with `msa_feats`. Dummy input for now.
        Nz:You must specify exactly one of input_ids or inputs_embedsrL  r   r#   rX  rY  )ri   r^   )rR   ra   r_   rô  )r`   )r_   r`   rô  r>  rî  rï  rð  rñ  rò  ró  r×  rM   )r+  rô  )rÎ   rP  r
   rR   Úget_seq_lengthr+   rk   r[   ri   rY   rQ  r©  rZ  rw  r   rR  r™  rŠ  r   )rQ   r/   r_   r`   rô  ra   r>  rV  rW  rX  rY  rò  ró  r�   Úpast_seen_tokensÚprotein_featsrñ  Úprotein_outputsÚcausal_maskr×   rM   Údecoder_layerrö   s                          r3   rf   zEvollaModel.forward#  s  € ðB ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLàˆØ!ÐàÐ(Ð-CÐ-OØ"×2Ò2Ø+Ø5ð 3ñ ô ˆOð ,ÔFˆMÝ!&¤Ø!Ô'¨Ô*Ø(Ô/Ý”jð"ñ "ô "Ðõ )Ø”;Ø'Ø)Ø+ð	
ñ 
ô 
ˆð &ˆØ"Ÿošo¨mÈ,˜oÑWÔWÐà!œ[ð 	ð 	ˆMØ)˜MØðà*Ø)Ø /Ø#Ø"/Ø$3Ø'Ø#5Ø%9Ø-Ø .Ø$7ðð ð ðð ˆMˆMð" Ÿ	š	 -Ñ0Ô0ˆå(Ø+Ø+ð
ñ 
ô 
ˆð ˆr5   )NNNNNNNNNNNN)rp   rq   rr   r$   r>   rS  rV  r   r    r"   r+   r³  r¦   r	   rç   rw  r¥   r   rf   rt   ru   s   @r3   rM  rM    s²  ø€ € € € € ð˜|ð ð ð ð ð ð ð*!ð !ð !ð"ð "ð "ð ØØð .2Ø.2Ø04Ø(,Ø26Ø!%Ø59Ø6:Ø48Ø.2Ø48Ø.2ð]ð ]àÔ# dÑ*ð]ð œ tÑ+ð]ð Ô&¨Ñ-ð	]ð
  ™ð]ð Ô(¨4Ñ/ð]ð ˜$‘;ð]ð !Ô+¨dÑ2ð]ð !&¤¨tÑ 3ð]ð Ô*¨TÑ1ð]ð Ô$ tÑ+ð]ð $œl¨TÑ1ð]ð œ tÑ+ð]ð 
Ð(Ñ	(ð]ð ]ð ]ñ „_ñ  Ôñ „^ð]ð ]ð ]ð ]ð ]r5   rM  c                   óú   ‡ — e Zd Zˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 ddej	        dz  dej
        dz  dej        dz  d	ej	        dz  d
ej	        dz  dej
        dz  dedz  deej
        z  fd„¦   «         ¦   «         Zˆ xZS )ÚEvollaForProteinText2Textc                 óþ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        | _        t          j        |j        | j        d¬¦  «        | _        |  	                    ¦   «          d S r‰  )
r=   r>   rM  r­  r@   r   rÑ   rA   Úlm_headrO  rP   s     €r3   r>   z"EvollaForProteinText2Text.__init__‡  sg   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý  Ñ(Ô(ˆŒ
Ø Ô+ˆŒÝ”y Ô!3°T´_È5ÐQÑQÔQˆŒà�ŠÑÔÐÐÐr5   c                 ó4   — | j                              ¦   «         S r£   )r­  rS  rR  s    r3   rS  z.EvollaForProteinText2Text.get_input_embeddings�  s   € ØŒz×.Ò.Ñ0Ô0Ð0r5   c                 ó6   — | j                              |¦  «        S r£   )r­  rV  rU  s     r3   rV  z.EvollaForProteinText2Text.set_input_embeddings’  s   € ØŒz×.Ò.¨uÑ5Ô5Ð5r5   Nr   r/   r_   ra   ÚlabelsrV  rW  r>  Úlogits_to_keepc	           
      óJ  —  | j         d||||||dœ|	¤Ž}
|
j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }d}|� | j        d||| j        dœ|	¤Ž}t          |||
j	        |
j
        |
j        ¬¦  «        }|S )a,  
        protein_input_ids (torch.LongTensor):
            The input IDs for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.LongTensor`.
        protein_attention_mask (torch.Tensor):
            The attention mask for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.Tensor`.

        Example:

        ```python
        >>> from transformers import EvollaProcessor, EvollaForProteinText2Text
        >>> model = EvollaForProteinText2Text.from_pretrained("westlake/Evolla-10B-hf")
        >>> processor = EvollaProcessor.from_pretrained("westlake/Evolla-10B-hf")

        >>> protein_information = {
            "aa_seq": "your amino acid sequence",
            "foldseek": "your foldseek sequence",
        }
        >>> question = "What is the function of this protein?"
        >>> message = [
            {"role": "system", "content": "You are an AI expert that can answer any questions about protein."},
            {"role": "user", "content": question},
        ]

        >>> inputs = processor(proteins=[protein_information], messages_list=[message], return_tensors="pt", padding="longest")
        >>> outputs = model.generate(**inputs)

        >>> print(processor.batch_decode(outputs, skip_special_tokens=True))
        ```)r/   r_   ra   rV  rW  r>  N)Úlogitsrg  r@   )Úlossrj  rô  r×   r;  r!  )r­  r+  r–   r*   Úslicerd  Úloss_functionr@   r   rô  r×   r;  )rQ   r/   r_   ra   rg  rV  rW  r>  rh  r�   Úoutputsr×   Úslice_indicesrj  rk  Ú
lm_outputss                   r3   rf   z!EvollaForProteinText2Text.forward•  s÷   € ðT ,6¨4¬:ð ,
ØØ)Ø'Ø/Ø#9Øð,
ð ,
ð ð,
ð ,
ˆð  Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%Ði¨V¸FÈtÌÐiÐiÐbhÐiÐiˆDå+ØØØ#Ô3Ø!Ô/ØÔ)ð
ñ 
ô 
ˆ
ð Ðr5   )NNNNNNNr   )rp   rq   rr   r>   rS  rV  r   r   r+   r³  r¦   rç   rw  r*   rf   rt   ru   s   @r3   rb  rb  †  s8  ø€ € € € € ðð ð ð ð ð1ð 1ð 1ð6ð 6ð 6ð Øð .2Ø.2Ø26Ø*.Ø59Ø6:Ø!%Ø-.ðBð BàÔ# dÑ*ðBð œ tÑ+ðBð Ô(¨4Ñ/ð	Bð
 Ô  4Ñ'ðBð !Ô+¨dÑ2ðBð !&¤¨tÑ 3ðBð ˜$‘;ðBð ˜eœlÑ*ðBð Bð Bñ „^ñ ÔðBð Bð Bð Bð Br5   rb  )rb  rM  rA  )r#   )NrV   )[rý   Úcollections.abcr   Údataclassesr   r+   r   Ú r   r?  Úactivationsr   Úcache_utilsr	   r
   Ú
generationr   Úintegrationsr   r   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r    Úutils.output_capturingr!   r"   Úconfiguration_evollar$   r%   r4   ÚModuler7   rw   r«   r´   r¦   r\   rÇ   rÉ   ré   rô   rÿ   r  r  r  r  r0  r8  rI  r`  r†  r‘  r¨  r«  rµ  rÃ  r
  r  r*   r¿   r%  r3  rA  rM  rb  Ú__all__r!  r5   r3   ú<module>r…     s\  ðð* €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )Ø LÐ LÐ LÐ LÐ LÐ LÐ LÐ LØ JÐ JÐ JÐ JÐ JÐ JÐ JÐ JØ 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð ð ð LÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ <Ð <Ð <Ð <Ð <Ð <Ð <Ð <ð4ð 4ð 4ð Y=ð Y=ð Y=ð Y=ð Y=˜RœYñ Y=ô Y=ð Y=ðx6<ð 6<ð 6<ð 6<ð 6< "¤)ñ 6<ô 6<ð 6<ðr(ð (ð (ð ÐÐ-Ñ.Ô.ðBð Bð Bñ /Ô.ðBð@ !Øð %ð  %ØŒIð %àŒ<ð %ð 
Œð %ð Œ<ð	 %ð
 ”L 4Ñ'ð %ð �T‰\ð %ð ð %ð Ð'Ô(ð %ð  %ð  %ð  %ðF ÐÐ)Ñ*Ô*ðK)ð K)ð K)ð K)ð K) ¤	ñ K)ô K)ñ +Ô*ðK)ð\
ð 
ð 
ð 
ð 
˜RœYñ 
ô 
ð 
ðð ð ð ð ˜BœIñ ô ð ð<;ð ;ð ;ðð ð ð ð ˜rœyñ ô ð ð
ð 
ð 
ð 
ð 
˜œñ 
ô 
ð 
ð7ð 7ð 7ð 7ð 7Ð2ñ 7ô 7ð 7ðtSð Sð Sð Sð S˜"œ)ñ Sô Sð SðDð ð ð ð ˜œñ ô ð ð ðCð Cð Cð Cð C /ñ Cô Cñ „ðCð01
ð 1
ð 1
ð 1
ð 1
Ð!<ñ 1
ô 1
ð 1
ðh7 ð 7 ð 7 ð 7 ð 7 ¨¬	ñ 7 ô 7 ð 7 ðtAð Að Að Að A˜œ	ñ Aô Að Að'.ð '.ð '.ð '.ð '.¨¬	ñ '.ô '.ð '.ðT Ø
ð
<ð 
<ð 
<ð 
<ð 
< kñ 
<ô 
<ñ „ñ „ð
<ð
ð 
ð 
ð 
ð 
˜2œ9ñ 
ô 
ð 
ð$kð kð kð kð k¨"¬)ñ kô kð kð\ Ð˜YÑ'Ô'ðJð Jð Jð Jð J�B”Iñ Jô Jñ (Ô'ðJð(6<ð 6<ð 6<ð 6<ð 6<˜BœIñ 6<ô 6<ð 6<ðrð ð ð ð �”	ñ ô ð ð 	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð ÐÐ)Ñ*Ô*ð@)ð @)ð @)ð @)ð @)�b”iñ @)ô @)ñ +Ô*ð@)ðFCð Cð Cð Cð CÐ3ñ Cô Cð CðL ð<ð <ð <ð <ð <˜Oñ <ô <ñ „ð<ðD|ð |ð |ð |ð |Ð'ñ |ô |ð |ð~Sð Sð Sð Sð SÐ 5°ñ Sô Sð Sðl PÐ
OÐ
O€€€r5   