§
    9²¸jéŒ  ã                   ó  — d dl mZ d dlZd dlmZ ddlmZ ddlmZm	Z	 ddl
mZ ddlmZmZ dd	lmZmZmZmZ dd
lmZ ddlmZ ddlmZmZmZmZ ddlmZ ddlm Z m!Z! ddl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z, ddl-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4 ddl5m6Z6m7Z7  ej8        e9¦  «        Z:	 	 dSdej;        dej<        dej<        dej<        dej<        dz  de=dz  de=dee         fd„Z> G d„ de$¦  «        Z? G d „ d!e*¦  «        Z@ G d"„ d#e+¦  «        ZA G d$„ d%e,¦  «        ZB G d&„ d'e#¦  «        ZC G d(„ d)e&¦  «        ZD G d*„ d+e(¦  «        ZE G d,„ d-e'¦  «        ZF G d.„ d/e%¦  «        ZG G d0„ d1e)¦  «        ZHe G d2„ d3e¦  «        ¦   «         ZI G d4„ d5eI¦  «        ZJ G d6„ d7ej;        ¦  «        ZK G d8„ d9ej;        ¦  «        ZL G d:„ d;ej;        ¦  «        ZMee G d<„ d=e¦  «        ¦   «         ¦   «         ZN G d>„ d?ej;        ¦  «        ZO G d@„ dAej;        ¦  «        ZP G dB„ dCe2¦  «        ZQ G dD„ dEe3¦  «        ZR G dF„ dGe0¦  «        ZS G dH„ dIe.¦  «        ZT G dJ„ dKe/¦  «        ZU G dL„ dMe1¦  «        ZV G dN„ dOeV¦  «        ZW G dP„ dQeVe¦  «        ZXg dR¢ZYdS )Té    )Ú	dataclassN)Únné   )Úinitialization)ÚCacheÚDynamicCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚBaseModelOutputWithPastÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚCausalLMOutputWithPastÚModelOutput)ÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )
ÚEsmAttentionÚEsmEmbeddingsÚ
EsmEncoderÚEsmIntermediateÚEsmLayerÚ	EsmOutputÚ	EsmPoolerÚEsmRotaryEmbeddingÚEsmSelfAttentionÚEsmSelfOutput)ÚLlamaAttentionÚLlamaDecoderLayerÚLlamaMLPÚLlamaPreTrainedModelÚLlamaRMSNormÚLlamaRotaryEmbeddingÚ	repeat_kvé   )ÚEvollaConfigÚSaProtConfigç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 ó  — |€|                      d¦  «        dz  }t          | dd¦  «        }t          ||¦  «        }t          ||¦  «        }t          j        ||                     dd¦  «        ¦  «        |z  }	|�|	|z   }	t          j                             |	d¬¦  «        }	t          j         	                    |	|| j
        ¬¦  «        }	t          j        |	|¦  «        }
|
                     dd¦  «                             ¦   «         }
|
|	fS )	Néÿÿÿÿç      à¿Únum_key_value_groupsr+   r   r   ©Údim)ÚpÚtraining)ÚsizeÚgetattrr*   ÚtorchÚmatmulÚ	transposer   Ú
functionalÚsoftmaxr5   r>   Ú
contiguous)r/   r0   r1   r2   r3   r4   r5   r6   Ún_repÚattn_weightsÚattn_outputs              úy/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/evolla/modular_evolla.pyÚeager_attention_forwardrK   C   sú   € ð €Ø—*’*˜R‘.”. DÑ(ˆõ
 �FÐ2°AÑ6Ô6€EÝ
�C˜Ñ
Ô
€CÝ�e˜UÑ#Ô#€Eõ ”<  s§}¢}°Q¸Ñ':Ô':Ñ;Ô;¸gÑE€LàÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2Ð(Ñ>Ô>€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$ó    c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚEvollaSaProtEmbeddingsc                 óN   •— t          ¦   «                              |¦  «         | `d S ©N)ÚsuperÚ__init__Úposition_ids©ÚselfÚconfigÚ	__class__s     €rJ   rR   zEvollaSaProtEmbeddings.__init__g   s'   ø€ Ý‰Œ×Ò˜Ñ Ô Ð àÐÐÐrL   )Ú__name__Ú
__module__Ú__qualname__rR   Ú__classcell__©rW   s   @rJ   rN   rN   f   s8   ø€ € € € € ðð ð ð ð ð ð ð ð rL   rN   c                   óV   ‡ — e Zd Zddefˆ fd„Zddedeej        ef         fˆ fd„Z	ˆ xZ
S )ÚEvollaSaProtRotaryEmbeddingNrV   c                 óJ   •— t          ¦   «                              |¦  «         d S rP   )rQ   rR   )rU   rV   ÚdevicerW   s      €rJ   rR   z$EvollaSaProtRotaryEmbedding.__init__n   s!   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ð Ð rL   Úreturnc                 óF   •— t          ¦   «                              | ¦  «        S rP   )rQ   Úcompute_default_rope_parameters)rV   r`   r6   rW   s      €rJ   rc   z;EvollaSaProtRotaryEmbedding.compute_default_rope_parametersq   s   ø€ Ý‰wŒw×6Ò6°vÑ>Ô>Ð>rL   rP   )rX   rY   rZ   r-   rR   ÚtuplerA   ÚTensorÚfloatrc   r[   r\   s   @rJ   r^   r^   m   sŠ   ø€ € € € € ð!ð !˜|ð !ð !ð !ð !ð !ð !ð?ð ?°ð ?ÐX]Ð^cÔ^jÐlqÐ^qÔXrð ?ð ?ð ?ð ?ð ?ð ?ð ?ð ?ð ?ð ?rL   r^   c                   ó   — e Zd ZdS )ÚEvollaSaProtSelfAttentionN©rX   rY   rZ   © rL   rJ   rh   rh   u   ó   € € € € € Ø€DrL   rh   c                   ó   — e Zd ZdS )ÚEvollaSaProtSelfOutputNri   rj   rL   rJ   rm   rm   y   rk   rL   rm   c                   ó   — e Zd ZdS )ÚEvollaSaProtAttentionNri   rj   rL   rJ   ro   ro   }   rk   rL   ro   c                   ó   — e Zd ZdS )ÚEvollaSaProtIntermediateNri   rj   rL   rJ   rq   rq   �   rk   rL   rq   c                   ó   — e Zd ZdS )ÚEvollaSaProtOutputNri   rj   rL   rJ   rs   rs   …   rk   rL   rs   c                   ó   — e Zd ZdS )ÚEvollaSaProtLayerNri   rj   rL   rJ   ru   ru   ‰   rk   rL   ru   c                   ó   — e Zd ZdS )ÚEvollaSaProtEncoderNri   rj   rL   rJ   rw   rw   �   rk   rL   rw   c                   ó   — e Zd ZdS )ÚEvollaSaProtPoolerNri   rj   rL   rJ   ry   ry   ‘   rk   rL   ry   c                   óª   ‡ — e Zd ZU eed<   dgZdZdZdZdZ	e
 eedd¬¦  «        g eedd¬¦  «        gdœZ ej        ¦   «         ˆ fd	„¦   «         Zˆ xZS )
ÚEvollaSaProtPreTrainedModelrV   ru   Tr+   Ú	attention)ÚindexÚ
layer_nameÚcrossattention)Úhidden_statesÚ
attentionsÚcross_attentionsc                 óø   •— t          ¦   «                              |¦  «         t          |t          ¦  «        rB|                     |j        ¦  «        \  }}t          j        t          |d¦  «        |¦  «         d S d S )NÚinv_freq)	rQ   Ú_init_weightsÚ
isinstancer^   rc   rV   ÚinitÚcopy_r@   )rU   r/   Úcurr_inv_freqÚ_rW   s       €rJ   r…   z)EvollaSaProtPreTrainedModel._init_weights¦   sw   ø€ å‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ9Ñ:Ô:ð 	CØ%×EÒEÀfÄmÑTÔTÑˆM˜1ÝŒJ•w˜v zÑ2Ô2°MÑBÔBÐBÐBÐBð	Cð 	CrL   )rX   rY   rZ   r-   Ú__annotations__Ú_no_split_modulesÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendru   r   rh   Ú_can_record_outputsrA   Úno_gradr…   r[   r\   s   @rJ   r{   r{   •   sÉ   ø€ € € € € € àÐÐÑØ,Ð-ÐØÐØ€NØÐØ"&Ðð +Ø%�~Ð&?ÀqÐU`ÐaÑaÔaÐbàˆNÐ4¸AÐJZÐ[Ñ[Ô[ð
ðð Ðð €U„]�_„_ðCð Cð Cð Cñ „_ðCð Cð Cð Cð CrL   r{   c            
       ó¤   ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zee	 d
de	j
        dz  de	j
        dz  dee	j
                 ez  fd	„¦   «         ¦   «         Zˆ xZS )ÚEvollaSaProtProteinEncoderrV   c                 óì   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          |¬¦  «        | _        t          |¦  «        | _        |                      ¦   «          d S ©N©rV   )	rQ   rR   rN   Ú
embeddingsr^   Úrotary_embeddingsrw   ÚencoderÚ	post_initrT   s     €rJ   rR   z#EvollaSaProtProteinEncoder.__init__¯   sb   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý0°Ñ8Ô8ˆŒÝ!<ÀFÐ!KÑ!KÔ!KˆÔÝ*¨6Ñ2Ô2ˆŒØ�ŠÑÔÐÐÐrL   c                 ó   — | j         j        S rP   ©r˜   Úword_embeddings©rU   s    rJ   Úget_input_embeddingsz/EvollaSaProtProteinEncoder.get_input_embeddings¶   s   € ØŒÔ.Ð.rL   c                 ó   — || j         _        d S rP   r�   ©rU   r2   s     rJ   Úset_input_embeddingsz/EvollaSaProtProteinEncoder.set_input_embeddings¹   s   € Ø*/ˆŒÔ'Ð'Ð'rL   NÚ	input_idsr3   ra   c                 óÆ  — |                      ¦   «         }|\  }}|j        }|€t          j        ||f|¬¦  «        }|                      ||¬¦  «        }t          | j        ||¬¦  «        }t          j        ||¬¦  «                             d¦  «        }	|  	                    ||	¦  «        }
 | j
        |f||
dœ|¤Ž}|d         }t          ||j        |j        |j        ¬¦  «        S )N©r`   ©r¤   r3   )rV   Úinputs_embedsr3   r   )r3   Úposition_embeddings)Úlast_hidden_stater€   r�   r‚   )r?   r`   rA   Úonesr˜   r
   rV   ÚarangeÚ	unsqueezer™   rš   r   r€   r�   r‚   )rU   r¤   r3   r6   Úinput_shapeÚ
batch_sizeÚ
seq_lengthr`   r¨   rS   r©   Úencoder_outputsÚsequence_outputs                rJ   Úforwardz"EvollaSaProtProteinEncoder.forward¼   s  € ð  —n’nÑ&Ô&ˆØ!,Ñˆ
�JàÔ!ˆØÐ!Ý"œZ¨*°jÐ)AÈ6ÐRÑRÔRˆNØŸš°)ÈN˜Ñ[Ô[ˆå2Ø”;Ø'Ø)ð
ñ 
ô 
ˆõ ”| J°vÐ>Ñ>Ô>×HÒHÈÑKÔKˆØ"×4Ò4°]ÀLÑQÔQÐà&˜$œ,Øð
Ø*8ÐNað
ð 
Øekð
ð 
ˆð *¨!Ô,ˆå;Ø-Ø)Ô7Ø&Ô1Ø,Ô=ð	
ñ 
ô 
ð 	
rL   rP   )rX   rY   rZ   r-   rR   r    r£   r   r   rA   re   rd   r   r³   r[   r\   s   @rJ   r”   r”   ®   sÎ   ø€ € € € € ð˜|ð ð ð ð ð ð ð/ð /ð /ð0ð 0ð 0ð  Øð /3ð!
ð !
à”< $Ñ&ð!
ð œ tÑ+ð!
ð
 
ˆuŒ|Ô	ÐKÑ	Kð!
ð !
ð !
ñ „_ñ  Ôð!
ð !
ð !
ð !
ð !
rL   r”   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )Ú!EvollaSequenceCompressorAttentioné@   é   c                 ó†  •— t          ¦   «                              ¦   «          |dz  | _        || _        ||z  }t	          j        |¦  «        | _        t	          j        |¦  «        | _        t	          j        ||d¬¦  «        | _	        t	          j        ||dz  d¬¦  «        | _
        t	          j        ||d¬¦  «        | _        d S )Nr9   F©Úbiasr   )rQ   rR   ÚscaleÚheadsr   Ú	LayerNormÚ
norm_mediaÚnorm_latentsÚLinearÚto_qÚto_kvÚto_out)rU   r<   Údim_headr¼   Ú	inner_dimrW   s        €rJ   rR   z*EvollaSequenceCompressorAttention.__init__ã   s¨   ø€ Ý‰Œ×ÒÑÔÐØ˜t‘^ˆŒ
ØˆŒ
Ø˜uÑ$ˆ	åœ, sÑ+Ô+ˆŒÝœL¨Ñ-Ô-ˆÔå”I˜c 9°5Ð9Ñ9Ô9ˆŒ	Ý”Y˜s I°¡M¸Ð>Ñ>Ô>ˆŒ
Ý”i 	¨3°UÐ;Ñ;Ô;ˆŒˆˆrL   c                 ó
  — |                       |¦  «        }|                      |¦  «        }| j        }|                      |¦  «        }t	          j        ||fd¬¦  «        }|                      |¦  «                             dd¬¦  «        \  }}|                     | 	                    d¦  «        | 	                    d¦  «        |d¦  «         
                    dddd¦  «        }|                     | 	                    d¦  «        | 	                    d¦  «        |d¦  «         
                    dddd¦  «        }|                     | 	                    d¦  «        | 	                    d¦  «        |d¦  «         
                    dddd¦  «        }|| j        z  }t	          j        ||                     dd¦  «        ¦  «        }	|	|	                     dd¬	¦  «                             ¦   «         z
  }	|	j        \  }
}}}t	          j        ||¦  «                             |j        ¦  «        }|d
d
…d
d
d
d
…f         }|d
d
d
…d
d
…d
f         }||z  }|	                     d|z
                       ¦   «         d¦  «        }	|	                     d¬¦  «        }t	          j        ||¦  «        }| 
                    dddd¦  «        }|                     | 	                    d¦  «        | 	                    d¦  «        d¦  «        }|                      |¦  «        S )zÔ
        Args:
            x (torch.Tensor): image features
                shape (b, n1, D)
            latent (torch.Tensor): latent features
                shape (b, n2, D);  n2: num of latent tokens
        éþÿÿÿr;   r   r8   r   r+   r   T©r<   ÚkeepdimNg     ˆÃÀ)r¾   r¿   r¼   rÁ   rA   ÚcatrÂ   ÚchunkÚviewr?   Úpermuter»   rB   rC   ÚamaxÚdetachÚshaper«   Útor`   Úmasked_fillÚboolrE   ÚreshaperÃ   )rU   ÚxÚlatentsÚmaskÚhÚqÚkv_inputÚkÚvÚsimÚbsÚnhÚskdÚokdr«   Úmask_expÚones_expÚattnÚouts                      rJ   r³   z)EvollaSequenceCompressorAttention.forwardð   s¢  € ð �OŠO˜AÑÔˆØ×#Ò# GÑ,Ô,ˆàŒJˆà�IŠI�gÑÔˆÝ”9˜a ˜\¨rÐ2Ñ2Ô2ˆØ�zŠz˜(Ñ#Ô#×)Ò)Ø�2ð *ñ 
ô 
‰ˆˆ1ð �FŠF�1—6’6˜!‘9”9˜aŸfšf Q™iœi¨¨BÑ/Ô/×7Ò7¸¸1¸aÀÑCÔCˆØ�FŠF�1—6’6˜!‘9”9˜aŸfšf Q™iœi¨¨BÑ/Ô/×7Ò7¸¸1¸aÀÑCÔCˆØ�FŠF�1—6’6˜!‘9”9˜aŸfšf Q™iœi¨¨BÑ/Ô/×7Ò7¸¸1¸aÀÑCÔCˆØ�”
‰Nˆõ Œl˜1˜aŸkšk¨"¨bÑ1Ô1Ñ2Ô2ˆØ�C—H’H ¨T�HÑ2Ô2×9Ò9Ñ;Ô;Ñ;ˆØœ9ÑˆˆB��SÝŒz˜"˜cÑ"Ô"×%Ò% d¤kÑ2Ô2ˆØ˜˜˜˜4  q q qÐ(Ô)ˆØ˜˜a˜a˜a    DÐ(Ô)ˆØ˜(Ñ"ˆà�oŠo˜q 4™xŸošoÑ/Ô/°Ñ6Ô6ˆØ�{Š{˜rˆ{Ñ"Ô"ˆÝŒl˜4 Ñ#Ô#ˆØ�kŠk˜!˜Q  1Ñ%Ô%ˆð �kŠk˜#Ÿ(š( 1™+œ+ s§x¢x°¡{¤{°BÑ7Ô7ˆà�{Š{˜3ÑÔÐrL   )r¶   r·   ©rX   rY   rZ   rR   r³   r[   r\   s   @rJ   rµ   rµ   â   sL   ø€ € € € € ð<ð <ð <ð <ð <ð <ð) ð ) ð ) ð ) ð ) ð ) ð ) rL   rµ   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚEvollaFeedForwardé   c                 ó>  •— t          ¦   «                              ¦   «          t          ||z  ¦  «        }t          j        |¦  «        | _        t          j        ||d¬¦  «        | _        t          j        ¦   «         | _	        t          j        ||d¬¦  «        | _
        d S ©NFr¹   )rQ   rR   Úintr   r½   ÚnormrÀ   Úfc1ÚGELUÚ
activationÚfc2)rU   r<   ÚmultrÅ   rW   s       €rJ   rR   zEvollaFeedForward.__init__  sz   ø€ Ý‰Œ×ÒÑÔÐÝ˜˜d™
‘O”Oˆ	å”L Ñ%Ô%ˆŒ	Ý”9˜S )°%Ð8Ñ8Ô8ˆŒÝœ'™)œ)ˆŒÝ”9˜Y¨°%Ð8Ñ8Ô8ˆŒˆˆrL   c           	      óž   — |                       |                      |                      |                      |¦  «        ¦  «        ¦  «        ¦  «        S rP   )rñ   rð   rî   rí   )rU   rÕ   s     rJ   r³   zEvollaFeedForward.forward&  s6   € Ø�xŠx˜Ÿš¨¯ª°·²¸1±´Ñ(>Ô(>Ñ?Ô?Ñ@Ô@Ð@rL   )ré   ræ   r\   s   @rJ   rè   rè     sS   ø€ € € € € ð9ð 9ð 9ð 9ð 9ð 9ðAð Að Að Að Að Að ArL   rè   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )Ú!EvollaSequenceCompressorResamplerrV   c           
      óp  •— t          ¦   «                              ¦   «          |j        j        }|j        | _        t          j        t          j	        | j        |¦  «        d¬¦  «        | _
        t          j        g ¦  «        | _        t          |j        ¦  «        D ]^}| j                             t          j        t!          ||j        |j        ¬¦  «        t'          ||j        ¬¦  «        g¦  «        ¦  «         Œ_t          j        |j        ¦  «        | _        t          j        ||j        ¦  «        | _        d S )NT)Úrequires_grad)r<   rÄ   r¼   )r<   rò   )rQ   rR   Úprotein_encoder_configÚhidden_sizeÚresampler_num_latentsÚnum_latentsr   Ú	ParameterrA   ÚrandnrÖ   Ú
ModuleListÚlayersÚrangeÚresampler_depthÚappendrµ   Úresampler_dim_headÚresampler_headsrè   Úresampler_ff_multr½   rí   rÀ   Úprotein_projector)rU   rV   Úprotein_repr_dimrŠ   rW   s       €rJ   rR   z*EvollaSequenceCompressorResampler.__init__+  s  ø€ Ý‰Œ×ÒÑÔÐØ!Ô8ÔDÐØ!Ô7ˆÔÝ”|¥E¤K°Ô0@ÐBRÑ$SÔ$SÐcgÐhÑhÔhˆŒÝ”m BÑ'Ô'ˆŒÝ�vÔ-Ñ.Ô.ð 
	ð 
	ˆAØŒK×ÒÝ”å9Ø 0¸6Ô;TÐ\bÔ\rðñ ô õ *Ð.>ÀVÔE]Ð^Ñ^Ô^ð	ñô ñ	ô 	ð 	ð 	õ ”L Ô!3Ñ4Ô4ˆŒ	Ý!#¤Ð+;¸VÔ=OÑ!PÔ!PˆÔÐÐrL   c                 óN  — |j         d         }|j         \  }}t          j        || j        ¦  «                             |j        ¦  «        }t          j        ||fd¬¦  «        }t          j        |¦  «                             | j        j        ¦  «        }| j        d          |                     ddd¦  «        z  }|                     |j	        ¦  «        }| j
        D ]#\  }	}
 |	|||¦  «        |z   } |
|¦  «        |z   }Œ$|                      |¦  «        }|                      |¦  «        S )Nr   r+   r;   r8   )rÐ   rA   r«   rû   rÑ   r`   rÊ   rÖ   rÌ   Údtyperÿ   r  rí   )rU   Úembedsr×   ÚbrÞ   rŠ   Úlatent_maskr«   rÖ   rä   ÚffÚtransformed_features               rJ   r³   z)EvollaSequenceCompressorResampler.forward@  s  € ØŒL˜ŒOˆà”
‰ˆˆAÝ”j  TÔ%5Ñ6Ô6×9Ò9¸$¼+ÑFÔFˆÝŒy˜$ Ð,°!Ð4Ñ4Ô4ˆõ Œz˜!‰}Œ}×Ò ¤Ô 3Ñ4Ô4ˆØ”,˜tÔ$ t§y¢y°°Q¸Ñ':Ô':Ñ:ˆØ—*’*˜Vœ\Ñ*Ô*ˆØœð 	,ð 	,‰HˆD�"Ø�d˜6 7¨DÑ1Ô1°GÑ;ˆGØ�b˜‘k”k GÑ+ˆGˆGà"×4Ò4°WÑ=Ô=Ðà�yŠyÐ,Ñ-Ô-Ð-rL   )rX   rY   rZ   r,   rR   r³   r[   r\   s   @rJ   rõ   rõ   *  sZ   ø€ € € € € ðQ˜|ð Qð Qð Qð Qð Qð Qð*.ð .ð .ð .ð .ð .ð .rL   rõ   c                   ó¬   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
ej        df         dz  ed<   dZe
ej        df         dz  ed<   dS )ÚEvollaProteinEncoderModelOutputaN  
    sequence_compressor_output (`torch.FloatTensor` of shape `(batch_size, compressed_seq_len, hidden_size)`, *optional*):
        Compressed sequence representation produced by the sequence compressor module. The sequence length is
        reduced from the original input length to `compressed_seq_len` via learned compression.
    NÚsequence_compressor_outputrª   .r€   r�   )rX   rY   rZ   Ú__doc__r  rA   ÚFloatTensorr‹   rª   r€   rd   r�   rj   rL   rJ   r  r  T  s•   € € € € € € ðð ð <@Ð Ô 1°DÑ 8Ð?Ð?Ñ?Ø26Ð�uÔ(¨4Ñ/Ð6Ð6Ñ6Ø:>€M�5˜Ô*¨CÐ/Ô0°4Ñ7Ð>Ð>Ñ>Ø7;€J��eÔ'¨Ð,Ô-°Ñ4Ð;Ð;Ñ;Ð;Ð;rL   r  c                   óX   ‡ — e Zd Zdefˆ fd„Zedej        dej        fd„¦   «         Z	ˆ xZ
S )ÚEvollaProteinEncoderrV   c                 ó¦   •— t          ¦   «                              ¦   «          t          |j        ¬¦  «        | _        t          |¬¦  «        | _        d S r–   )rQ   rR   r”   rø   Úmodelrõ   Úsequence_compressor_resamplerrT   s     €rJ   rR   zEvollaProteinEncoder.__init__d  sH   ø€ Ý‰Œ×ÒÑÔÐÝ/°vÔ7TÐUÑUÔUˆŒ
Ý-NÐV\Ð-]Ñ-]Ô-]ˆÔ*Ð*Ð*rL   r¤   r3   c                 ó–   — |                       ||¬¦  «        }|j        }|                      ||¦  «        }t          ||j        ¬¦  «        S )Nr§   )r  rª   )r  rª   r  r  )rU   r¤   r3   r6   Úprotein_outputÚprotein_embedsÚsequence_reprs          rJ   r³   zEvollaProteinEncoder.forwardi  sT   € àŸš¨iÈ˜ÑWÔWˆØ'Ô9ˆØ×:Ò:¸>È>ÑZÔZˆå.Ø'4Ø,Ô>ð
ñ 
ô 
ð 	
rL   )rX   rY   rZ   r,   rR   r   rA   Ú
LongTensorr  r³   r[   r\   s   @rJ   r  r  c  s€   ø€ € € € € ð^˜|ð ^ð ^ð ^ð ^ð ^ð ^ð
 ð
 Ô!1ð 
À5ÔCTð 
ð 
ð 
ñ Ôð
ð 
ð 
ð 
ð 
rL   r  c                   ób   ‡ — e Zd Z	 	 	 ddedz  dedz  dedz  fˆ fd„Zd„ Z	 	 	 	 	 	 	 d	d„Zˆ xZS )
Ú#EvollaSequenceAlignerCrossAttentionNÚprotein_encoder_dimÚstructure_encoder_dimÚmsa_encoder_dimc                 óà  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        | j        dz  | _        t          | j        | j        z  ¦  «        | _        | j        | j        z  | _        |j        }|j	        }|j
        }t          j        | j        | j        ¦  «        | _        |�?t          j        || j        ¦  «        | _        t          j        || j        ¦  «        | _        nd | _        d | _        |�?t          j        || j        ¦  «        | _        t          j        || j        ¦  «        | _        nd | _        d | _        |�?t          j        || j        ¦  «        | _        t          j        || j        ¦  «        | _        nd | _        d | _        t)          | j        ¦  «        | _        t          j        |¦  «        | _        t          j        | j        | j        |¬¦  «        | _        t3          | j        |¦  «        | _        t          j        t9          j        dg¦  «        ¦  «        | _        t          j        t9          j        dg¦  «        ¦  «        | _        d S )Nr9   r¹   r.   ) rQ   rR   rù   Únum_attention_headsr»   rì   Úattention_head_sizeÚall_head_sizeÚ$aligner_attention_probs_dropout_probÚaligner_enable_biasÚaligner_ffn_multr   rÀ   r0   Úkey_proteinÚvalue_proteinÚkey_structureÚvalue_structureÚkey_msaÚ	value_msaÚEvollaRMSNormÚattention_normÚDropoutr5   Úout_projrè   r  rü   rA   ÚtensorÚgate_attentionÚgate_ffw)	rU   rV   r   r!  r"  Úattention_probs_dropout_probÚenable_biasÚffn_multrW   s	           €rJ   rR   z,EvollaSequenceAlignerCrossAttention.__init__v  s
  ø€ õ 	‰Œ×ÒÑÔÐà!Ô-ˆÔØ#)Ô#=ˆÔ ØÔ-¨tÑ3ˆŒ
Ý#& tÔ'7¸$Ô:RÑ'RÑ#SÔ#SˆÔ Ø!Ô5¸Ô8PÑPˆÔà'-Ô'RÐ$ØÔ0ˆØÔ*ˆå”Y˜tÔ/°Ô1CÑDÔDˆŒ
ØÐ*Ý!œyÐ)<¸dÔ>PÑQÔQˆDÔÝ!#¤Ð+>ÀÔ@RÑ!SÔ!SˆDÔÐà#ˆDÔØ!%ˆDÔà Ð,Ý!#¤Ð+@À$ÔBTÑ!UÔ!UˆDÔÝ#%¤9Ð-BÀDÔDVÑ#WÔ#WˆDÔ Ð à!%ˆDÔØ#'ˆDÔ àÐ&Ýœ9 _°dÔ6HÑIÔIˆDŒLÝœY ¸Ô8JÑKÔKˆDŒNˆNàˆDŒLØ!ˆDŒNå+¨DÔ,<Ñ=Ô=ˆÔå”zÐ">Ñ?Ô?ˆŒåœ	 $Ô"2°DÔ4DÈ;ÐWÑWÔWˆŒå# DÔ$4°hÑ?Ô?ˆŒÝ œl­5¬<¸¸Ñ+>Ô+>Ñ?Ô?ˆÔÝœ¥U¤\°3°%Ñ%8Ô%8Ñ9Ô9ˆŒˆˆrL   c	                 óŽ  — |||g}	d„ |	D ¦   «         }	|	st          d¦  «        ‚t          j        |	d¬¦  «        }	|                      |¦  «        }
|                      |
¦  «        }
| j        �G| j        �@|                     |¦  «        }|                      |¦  «        }|                      |¦  «        }nd}d}| j        �G| j	        �@|                     |¦  «        }|                      |¦  «        }|  	                    |¦  «        }nd}d}| j
        �G| j        �@|                     |¦  «        }|  
                    |¦  «        }|                      |¦  «        }nd}d}|||g}d„ |D ¦   «         }t          j        |d¬¦  «        }|||g}d„ |D ¦   «         }t          j        |d¬¦  «        }|
                     ¦   «         dd…         | j        | j        fz   } |
j        |Ž                      d	d
dd¦  «        }
|                     ¦   «         dd…         | j        | j        fz   } |j        |Ž                      d	d
dd¦  «        }|                     ¦   «         dd…         | j        | j        fz   } |j        |Ž                      d	d
dd¦  «        }|
| j        z  }
|€St          j        |                     d	¦  «        |                     d¦  «        ¦  «                             |j        ¦  «        }|dd…ddd…df         |	dd…dddd…f         z  }t          j        |
|                     dd¦  «        ¦  «        }||                     dd¬¦  «                             ¦   «         z
  }|                     d|z
                       ¦   «         t          j        |j        ¦  «        j        ¦  «        } t;          j        d¬¦  «        |¦  «        }t          j        ||¦  «        }|                     d	d
dd¦  «                             ¦   «         }|                     ¦   «         dd…         | j         fz   } |j        |Ž }|  !                    |¦  «        }|S )zù
        query_states: text
        key_value_states: protein
        query_states: [bs, query_seq_len, dim]
        key_value_states: [bs, kv_seq_len, dim]
        query_attn_mask: [bs, query_seq_len]
        kv_attn_mask: [bs, kv_seq_len]
        c                 ó   — g | ]}|®|‘ŒS rP   rj   ©Ú.0rŠ   s     rJ   ú
<listcomp>zGEvollaSequenceAlignerCrossAttention.cross_attention.<locals>.<listcomp>¿  s   € ÐAÐAÐA˜a°1°=˜°=°=°=rL   z=At least one modality should be provided for cross attention.r+   r;   Nc                 ó   — g | ]}|®|‘ŒS rP   rj   r<  s     rJ   r>  zGEvollaSequenceAlignerCrossAttention.cross_attention.<locals>.<listcomp>å  s   € Ð;Ð;Ð;˜1¨Q¨]�Q¨]¨]¨]rL   c                 ó   — g | ]}|®|‘ŒS rP   rj   r<  s     rJ   r>  zGEvollaSequenceAlignerCrossAttention.cross_attention.<locals>.<listcomp>é  s   € Ð?Ð?Ð?˜Q°°�q°°°rL   r8   r   r   r   rÇ   TrÈ   )"Ú
ValueErrorrA   rÊ   r1  r0   r*  r+  rÑ   r,  r-  r.  r/  r?   r$  r%  rÌ   rÍ   r»   r«   r`   rB   rC   rÎ   rÏ   rÒ   rÓ   Úfinfor	  Úminr   ÚSoftmaxrF   r&  r3  )rU   Úquery_statesÚprotein_key_value_statesÚstructure_key_value_statesÚmsa_key_value_statesÚquery_attn_maskÚprotein_kv_attn_maskÚstructure_kv_attn_maskÚmsa_kv_attn_maskÚkv_attn_maskÚquery_layerÚkey_layer_proteinÚvalue_layer_proteinÚkey_layer_structureÚvalue_layer_structureÚkey_layer_msaÚvalue_layer_msaÚ	key_layerÚvalue_layerÚnew_query_layer_shapeÚnew_key_layer_shapeÚnew_value_layer_shaper3   rH   Úattention_scoresÚattention_probsÚcontext_layerÚnew_context_layer_shapes                               rJ   Úcross_attentionz3EvollaSequenceAlignerCrossAttention.cross_attention©  sª  € ð* -Ð.DÐFVÐWˆØAÐA <ÐAÑAÔAˆØð 	^ÝÐ\Ñ]Ô]Ð]Ý”y °1Ð5Ñ5Ô5ˆà×)Ò)¨,Ñ7Ô7ˆð —j’j Ñ-Ô-ˆàÔÐ'¨DÔ,>Ð,JØ'?×'BÒ'BÀ<Ñ'PÔ'PÐ$Ø $× 0Ò 0Ð1IÑ JÔ JÐØ"&×"4Ò"4Ð5MÑ"NÔ"NÐÐà $ÐØ"&ÐàÔÐ)¨dÔ.BÐ.NØ)C×)FÒ)FÀ|Ñ)TÔ)TÐ&Ø"&×"4Ò"4Ð5OÑ"PÔ"PÐØ$(×$8Ò$8Ð9SÑ$TÔ$TÐ!Ð!à"&ÐØ$(Ð!àŒ<Ð#¨¬Ð(BØ#7×#:Ò#:¸<Ñ#HÔ#HÐ Ø ŸLšLÐ)=Ñ>Ô>ˆMØ"ŸnšnÐ-AÑBÔBˆOˆOà ˆMØ"ˆOà&Ð(;¸]ÐKˆ	Ø;Ð; 	Ð;Ñ;Ô;ˆ	Ý”I˜i¨QÐ/Ñ/Ô/ˆ	à*Ð,AÀ?ÐSˆØ?Ð? +Ð?Ñ?Ô?ˆÝ”i °Ð3Ñ3Ô3ˆà +× 0Ò 0Ñ 2Ô 2°3°B°3Ô 7ØÔ$ØÔ$ð;
ñ !
Ðð '�kÔ&Ð(=Ð>×FÒFÀqÈ!ÈQÐPQÑRÔRˆà'ŸnšnÑ.Ô.¨s°¨sÔ3ØÔ$ØÔ$ð7
ñ 
Ðð #�I”NÐ$7Ð8×@Ò@ÀÀAÀqÈ!ÑLÔLˆ	à +× 0Ò 0Ñ 2Ô 2°3°B°3Ô 7ØÔ$ØÔ$ð;
ñ !
Ðð '�kÔ&Ð(=Ð>×FÒFÀqÈ!ÈQÐPQÑRÔRˆà! D¤JÑ.ˆð Ð"Ý#œj¨×):Ò):¸1Ñ)=Ô)=¸|×?PÒ?PÐQRÑ?SÔ?SÑTÔT×WÒWÐXdÔXkÑlÔlˆOØ(¨¨¨¨D°!°!°!°TÐ)9Ô:¸\È!È!È!ÈTÐSWÐYZÐYZÐYZÐJZÔ=[Ñ[ˆå”| K°×1DÒ1DÀRÈÑ1LÔ1LÑMÔMˆØ# l×&7Ò&7¸BÈÐ&7Ñ&MÔ&M×&TÒ&TÑ&VÔ&VÑVˆØ'×3Ò3Ø�Ñ×%Ò%Ñ'Ô'­¬°\Ô5GÑ)HÔ)HÔ)Lñ
ô 
Ðð -�"œ*¨Ð,Ñ,Ô,Ð-=Ñ>Ô>ˆõ œ _°kÑBÔBˆà%×-Ò-¨a°°A°qÑ9Ô9×DÒDÑFÔFˆØ"/×"4Ò"4Ñ"6Ô"6°s¸°sÔ";¸tÔ?QÐ>SÑ"SÐØ*˜Ô*Ð,CÐDˆàŸš mÑ4Ô4ˆàÐrL   c           
      ó8  — |�q|j         \  }}}|€ct          j        ||¦  «                             |	j        ¦  «        |	                     ||f¬¦  «        j        z                       |j        ¦  «        }nd }|�q|j         \  }}}|€ct          j        ||¦  «                             |	j        ¦  «        |
                     ||f¬¦  «        j        z                       |j        ¦  «        }nd }|�q|j         \  }}}|€ct          j        ||¦  «                             |	j        ¦  «        |                     ||f¬¦  «        j        z                       |j        ¦  «        }nd }|}|�|                     ¦   «         s,|�|                     ¦   «         s|�Š|                     ¦   «         rv|}|                      ||||||||¬¦  «        }t          j	        | j
        ¦  «        |z  }||z   }|}|                      |¦  «        t          j	        | j        ¦  «        z  }||z   }|S )N)r?   )rE  rF  rG  rH  rI  rJ  rK  rL  )rÐ   rA   r«   rÑ   r`   ÚexpandÚTÚanyr^  Útanhr5  r  r6  )rU   rE  Úprotein_kv_statesÚstructure_kv_statesÚmsa_kv_statesrI  rJ  rK  rL  Úprotein_batch_maskÚstructure_batch_maskÚmsa_batch_maskÚpast_key_valuesrÞ   Úprotein_kv_seq_lenr<   Ústructure_kv_seq_lenÚmsa_kv_seq_lenr€   Úresiduals                       rJ   r³   z+EvollaSequenceAlignerCrossAttention.forward  s`  € ð Ð(Ø*;Ô*AÑ'ˆBÐ" CØ#Ð+å”J˜rÐ#5Ñ6Ô6×9Ò9Ð:LÔ:SÑTÔTØ(×/Ò/Ð6HÈ"Ð5MÐ/ÑNÔNÔPñQç’"Ð&Ô-Ñ.Ô.ð %øð
 $(Ð àÐ*Ø,?Ô,EÑ)ˆBÐ$ cØ%Ð-å”J˜rÐ#7Ñ8Ô8×;Ò;Ð<NÔ<UÑVÔVØ*×1Ò1Ð8LÈbÐ7QÐ1ÑRÔRÔTñUç’"Ð(Ô/Ñ0Ô0ð 'øð
 &*Ð"àÐ$Ø&3Ô&9Ñ#ˆB� ØÐ'å”J˜r >Ñ2Ô2×5Ò5Ð6HÔ6OÑPÔPØ$×+Ò+°.À"Ð1EÐ+ÑFÔFÔHñIç’"�]Ô)Ñ*Ô*ð !øð
  $ÐØ$ˆð Ð*Ð/C×/GÒ/GÑ/IÔ/IÐ*Ø#Ð/Ð4J×4NÒ4NÑ4PÔ4PÐ/ØÐ)Ð.>×.BÒ.BÑ.DÔ.DÐ)à$ˆHØ ×0Ò0Ø*Ø):Ø+>Ø%2Ø /Ø%9Ø'=Ø!1ð 1ñ 	ô 	ˆMõ "œJ tÔ':Ñ;Ô;¸mÑKˆMà$ }Ñ4ˆMà$ˆHØ ŸGšG MÑ2Ô2µU´ZÀÄÑ5NÔ5NÑNˆMØ$ }Ñ4ˆMàÐrL   )NNN)NNNNNNN)rX   rY   rZ   rì   rR   r^  r³   r[   r\   s   @rJ   r  r  u  s¾   ø€ € € € € ð +/Ø,0Ø&*ð1:ð 1:ð ! 4™Zð1:ð  # T™zð	1:ð
 ˜t™ð1:ð 1:ð 1:ð 1:ð 1:ð 1:ðfnð nð nðn "Ø#ØØØ!ØØðGð Gð Gð Gð Gð Gð Gð GrL   r  c                   ó   — e Zd ZdS )r0  Nri   rj   rL   rJ   r0  r0  c  rk   rL   r0  c                   ó   — e Zd ZdS )ÚEvollaRotaryEmbeddingNri   rj   rL   rJ   rq  rq  g  rk   rL   rq  c                   ó   — e Zd ZdS )Ú	EvollaMLPNri   rj   rL   rJ   rs  rs  k  rk   rL   rs  c                   ó   — e Zd ZdS )ÚEvollaAttentionNri   rj   rL   rJ   ru  ru  o  rk   rL   ru  c                   óN  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 	 	 	 	 	 	 	 ddej        deej        ej        f         dz  dej        dz  d	ej	        dz  d
e
dz  dedz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  fd„Zˆ xZS )ÚEvollaDecoderLayerrV   Ú	layer_idxc                 óÔ   •— t          ¦   «                              ||¦  «         |dz   t          |j        |j        z  d¦  «        z  dk    rt          ||j        ¬¦  «        | _        d S d S )Nr+   r   )r   )rQ   rR   ÚmaxÚnum_hidden_layersÚaligner_num_add_layersr  rù   Úadapter)rU   rV   rx  rW   s      €rJ   rR   zEvollaDecoderLayer.__init__t  st   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+Ø˜‰M�S Ô!9¸VÔ=ZÑ!ZÐ\]Ñ^Ô^Ñ^ÐbcÒcÐcÝ>ØØ$*Ô$6ðñ ô ˆDŒLˆLˆLð dÐcrL   NFr€   r©   r3   rS   rj  Ú	use_cacherd  re  rf  rg  rh  ri  rI  c           
      ó(  — |}|                       |¦  «        } | j        d||||||dœ|¤Ž\  }}||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }t	          | d¦  «        r|                      ||||	||
||¬¦  «        }|S )N)r€   r3   rS   rj  r~  r©   r}  )rE  rd  re  rf  rI  rg  rh  ri  rj   )Úinput_layernormÚ	self_attnÚpost_attention_layernormÚmlpÚhasattrr}  )rU   r€   r©   r3   rS   rj  r~  rd  re  rf  rg  rh  ri  rI  r6   rn  rŠ   s                    rJ   r³   zEvollaDecoderLayer.forward|  sç   € ð" !ˆà×,Ò,¨]Ñ;Ô;ˆð *˜4œ>ð 
Ø'Ø)Ø%Ø+ØØ 3ð
ð 
ð ð
ð 
Ñˆ�qð ! =Ñ0ˆð !ˆØ×5Ò5°mÑDÔDˆØŸš Ñ/Ô/ˆØ  =Ñ0ˆå�4˜Ñ#Ô#ð 
	Ø ŸLšLØ*Ø"3Ø$7Ø+Ø /Ø#5Ø%9Ø-ð )ñ 	ô 	ˆMð ÐrL   )NNNNFNNNNNNN)rX   rY   rZ   r,   rì   rR   rA   re   rd   r  r   rÓ   r³   r[   r\   s   @rJ   rw  rw  s  si  ø€ € € € € ð˜|ð ¸ð ð ð ð ð ð ð IMØ.2Ø04Ø(,Ø!&Ø15Ø37Ø-1Ø26Ø48Ø.2Ø/3ð3ð 3à”|ð3ð # 5¤<°´Ð#=Ô>ÀÑEð3ð œ tÑ+ð	3ð
 Ô&¨Ñ-ð3ð  ™ð3ð ˜$‘;ð3ð !œ<¨$Ñ.ð3ð #œ\¨DÑ0ð3ð ”| dÑ*ð3ð "œL¨4Ñ/ð3ð $œl¨TÑ1ð3ð œ tÑ+ð3ð œ¨Ñ,ð3ð 3ð 3ð 3ð 3ð 3ð 3ð 3rL   rw  c                   óR   — e Zd ZdZdZdZg d¢Z ej        ¦   «         d„ ¦   «         Z	dS )ÚEvollaPreTrainedModelF)rw  ru   rõ   r  c                 ó|  — | j         j        }t          j        | |¦  «         t	          |t
          ¦  «        rRt          j        |j        ¦  «         t          j        |j	        ¦  «         t          j
        |j        j        ¦  «         d S t	          |t          ¦  «        rt          j        |j        d|¬¦  «         d S d S )Nr.   )ÚmeanÚstd)rV   Úinitializer_ranger   r…   r†   r  r‡   Úzeros_r5  r6  Úones_r1  Úweightrõ   Únormal_rÖ   )rU   r/   r‰  s      rJ   r…   z#EvollaPreTrainedModel._init_weights½  s¯   € àŒkÔ+ˆÝÔ% d¨FÑ3Ô3Ð3Ý�fÕAÑBÔBð 	<ÝŒK˜Ô-Ñ.Ô.Ð.ÝŒK˜œÑ(Ô(Ð(ÝŒJ�vÔ,Ô3Ñ4Ô4Ð4Ð4Ð4Ý˜Õ AÑBÔBð 	<ÝŒL˜œ¨c°sÐ;Ñ;Ô;Ð;Ð;Ð;ð	<ð 	<rL   N)
rX   rY   rZ   r�   r�   r�   rŒ   rA   r’   r…   rj   rL   rJ   r†  r†  ²  s[   € € € € € Ø ÐØÐØ"'Ððð ð Ðð €U„]�_„_ð<ð <ñ „_ð<ð <ð <rL   r†  c                   óh  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zeee		 	 	 	 	 	 	 	 	 	 	 	 dde
j        dz  de
j        dz  de
j        dz  d	edz  d
e
j        dz  dedz  de
j        dz  de
j        dz  de
j        dz  de
j        dz  de
j        dz  de
j        dz  deez  fd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚEvollaModelrV   c                 ó&  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          j        | j        ‰j        | j        ¦  «        | _        t          ‰¬¦  «        | _
        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        t%          ‰dd¦  «        | _        t)          ‰¬¦  «        | _        |                      ¦   «          d S )Nr—   c                 ó2   •— g | ]}t          ‰|¬ ¦  «        ‘ŒS ))rV   rx  )rw  )r=  rx  rV   s     €rJ   r>  z(EvollaModel.__init__.<locals>.<listcomp>Ñ  s@   ø€ ð ð ð ð
 õ	 #Ø!Ø'ðñ ô ðð ð rL   )ÚepsÚgradient_checkpointingF)rQ   rR   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingrù   Úembed_tokensr  Úprotein_encoderrþ   r   r{  rÿ   r0  Úrms_norm_epsrí   r@   r”  rq  Ú
rotary_embr›   rT   s    `€rJ   rR   zEvollaModel.__init__Ê  s  øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒÝœL¨¬¸&Ô:LÈdÔN^Ñ_Ô_ˆÔÝ3¸6ÐBÑBÔBˆÔÝ”mðð ð ð õ
 "' vÔ'?Ñ!@Ô!@ðñ ô ñ
ô 
ˆŒõ " &Ô"4¸&Ô:MÐNÑNÔNˆŒ	Ý&-¨fÐ6NÐPUÑ&VÔ&VˆÔ#Ý/°vÐ>Ñ>Ô>ˆŒØ�ŠÑÔÐÐÐrL   c                 ó   — | j         S rP   ©r™  rŸ   s    rJ   r    z EvollaModel.get_input_embeddingsß  s   € ØÔ Ð rL   c                 ó   — || _         d S rP   rž  r¢   s     rJ   r£   z EvollaModel.set_input_embeddingsâ  s   € Ø!ˆÔÐÐrL   Nr¤   r3   rS   rj  r¨   r~  Úprotein_input_idsÚprotein_attention_maskÚstructure_featsÚ	msa_featsrh  ri  ra   c                 óâ  — |du |duz  rt          d¦  «        ‚|€|                      |¦  «        }|r|€t          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   }| 	                    d¦  «        }d}d}|�Q|�O|  
                    ||¬¦  «        }|j        }t          j        |j        d         |j        t
          j        ¬¦  «        }t          | j        |||¬	¦  «        }|}|                      ||¬
¦  «        }| j        D ]} ||f||||||	|
|||||dœ|¤Ž}Œ|                      |¦  «        }t%          ||¬¦  «        }|S )a9  
        protein_input_ids (torch.LongTensor):
            The input IDs for the protein sequence in structure-aware tokens. Should be of shape `(batch_size, protein_seq_length)` and type `torch.LongTensor`.
        protein_attention_mask (torch.Tensor):
            The attention mask for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.Tensor`.
        structure_feats (torch.FloatTensor):
            The input IDs for purely structure-based features. Should be of shape `(batch_size, structure_seq_length, structure_feat_dim)` and type `torch.FloatTensor`. Dummy input for now.
        msa_feats (torch.FloatTensor):
            The input IDs for purely MSA-based features. Should be of shape `(batch_size, msa_seq_length, msa_feat_dim)` and type `torch.FloatTensor`. Dummy input for now.
        structure_batch_mask (torch.Tensor):
            The batch mask to decide which protein sequences are purely structure-based. Should be of shape `(batch_size)` and type `torch.Tensor`. Should be paired with `structure_feats`. Dummy input for now.
        msa_batch_mask (torch.Tensor):
            The batch mask to decide which protein sequences are purely MSA-based. Should be of shape `(batch_size)` and type `torch.Tensor`. Should be paired with `msa_feats`. Dummy input for now.
        Nz:You must specify exactly one of input_ids or inputs_embedsr—   r   r+   r¦   r§   )r`   r	  )rV   r¨   r3   rj  )rS   )r3   rS   rj  r~  rd  re  rf  rg  rh  ri  rI  r©   )rª   rj  )rA  r™  r   rV   Úget_seq_lengthrA   r¬   rÐ   r`   r­   rš  r  r«   rÓ   r   rœ  rÿ   rí   r   )rU   r¤   r3   rS   rj  r¨   r~  r   r¡  r¢  r£  rh  ri  r6   Úpast_seen_tokensÚprotein_featsrg  Úprotein_outputsÚcausal_maskr€   r©   Údecoder_layerÚoutputs                          rJ   r³   zEvollaModel.forwardå  s  € ðB ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLàˆØ!ÐàÐ(Ð-CÐ-OØ"×2Ò2Ø+Ø5ð 3ñ ô ˆOð ,ÔFˆMÝ!&¤Ø!Ô'¨Ô*Ø(Ô/Ý”jð"ñ "ô "Ðõ )Ø”;Ø'Ø)Ø+ð	
ñ 
ô 
ˆð &ˆØ"Ÿošo¨mÈ,˜oÑWÔWÐà!œ[ð 	ð 	ˆMØ)˜MØðà*Ø)Ø /Ø#Ø"/Ø$3Ø'Ø#5Ø%9Ø-Ø .Ø$7ðð ð ðð ˆMˆMð" Ÿ	š	 -Ñ0Ô0ˆå(Ø+Ø+ð
ñ 
ô 
ˆð ˆrL   )NNNNNNNNNNNN)rX   rY   rZ   r,   rR   r    r£   r   r   r   rA   r  re   r   r  rÓ   rd   r   r³   r[   r\   s   @rJ   r�  r�  É  s²  ø€ € € € € ð˜|ð ð ð ð ð ð ð*!ð !ð !ð"ð "ð "ð ØØð .2Ø.2Ø04Ø(,Ø26Ø!%Ø59Ø6:Ø48Ø.2Ø48Ø.2ð]ð ]àÔ# dÑ*ð]ð œ tÑ+ð]ð Ô&¨Ñ-ð	]ð
  ™ð]ð Ô(¨4Ñ/ð]ð ˜$‘;ð]ð !Ô+¨dÑ2ð]ð !&¤¨tÑ 3ð]ð Ô*¨TÑ1ð]ð Ô$ tÑ+ð]ð $œl¨TÑ1ð]ð œ tÑ+ð]ð 
Ð(Ñ	(ð]ð ]ð ]ñ „_ñ  Ôñ „^ð]ð ]ð ]ð ]ð ]rL   r�  c                   óú   ‡ — e Zd Zˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 ddej	        dz  dej
        dz  dej        dz  d	ej	        dz  d
ej	        dz  dej
        dz  dedz  deej
        z  fd„¦   «         ¦   «         Zˆ xZS )ÚEvollaForProteinText2Textc                 óþ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        | _        t          j        |j        | j        d¬¦  «        | _        |  	                    ¦   «          d S rë   )
rQ   rR   r�  r  r—  r   rÀ   rù   Úlm_headr›   rT   s     €rJ   rR   z"EvollaForProteinText2Text.__init__I  sg   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý  Ñ(Ô(ˆŒ
Ø Ô+ˆŒÝ”y Ô!3°T´_È5ÐQÑQÔQˆŒà�ŠÑÔÐÐÐrL   c                 ó4   — | j                              ¦   «         S rP   )r  r    rŸ   s    rJ   r    z.EvollaForProteinText2Text.get_input_embeddingsQ  s   € ØŒz×.Ò.Ñ0Ô0Ð0rL   c                 ó6   — | j                              |¦  «        S rP   )r  r£   r¢   s     rJ   r£   z.EvollaForProteinText2Text.set_input_embeddingsT  s   € ØŒz×.Ò.¨uÑ5Ô5Ð5rL   Nr   r¤   r3   r¨   Úlabelsr   r¡  r~  Úlogits_to_keepc	           
      óJ  —  | j         d||||||dœ|	¤Ž}
|
j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }d}|� | j        d||| j        dœ|	¤Ž}t          |||
j	        |
j
        |
j        ¬¦  «        }|S )a,  
        protein_input_ids (torch.LongTensor):
            The input IDs for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.LongTensor`.
        protein_attention_mask (torch.Tensor):
            The attention mask for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.Tensor`.

        Example:

        ```python
        >>> from transformers import EvollaProcessor, EvollaForProteinText2Text
        >>> model = EvollaForProteinText2Text.from_pretrained("westlake/Evolla-10B-hf")
        >>> processor = EvollaProcessor.from_pretrained("westlake/Evolla-10B-hf")

        >>> protein_information = {
            "aa_seq": "your amino acid sequence",
            "foldseek": "your foldseek sequence",
        }
        >>> question = "What is the function of this protein?"
        >>> message = [
            {"role": "system", "content": "You are an AI expert that can answer any questions about protein."},
            {"role": "user", "content": question},
        ]

        >>> inputs = processor(proteins=[protein_information], messages_list=[message], return_tensors="pt", padding="longest")
        >>> outputs = model.generate(**inputs)

        >>> print(processor.batch_decode(outputs, skip_special_tokens=True))
        ```)r¤   r3   r¨   r   r¡  r~  N)Úlogitsr²  r—  )Úlossrµ  rj  r€   r�   rj   )r  rª   r†   rì   Úslicer¯  Úloss_functionr—  r   rj  r€   r�   )rU   r¤   r3   r¨   r²  r   r¡  r~  r³  r6   Úoutputsr€   Úslice_indicesrµ  r¶  Ú
lm_outputss                   rJ   r³   z!EvollaForProteinText2Text.forwardW  s÷   € ðT ,6¨4¬:ð ,
ØØ)Ø'Ø/Ø#9Øð,
ð ,
ð ð,
ð ,
ˆð  Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%Ði¨V¸FÈtÌÐiÐiÐbhÐiÐiˆDå+ØØØ#Ô3Ø!Ô/ØÔ)ð
ñ 
ô 
ˆ
ð ÐrL   )NNNNNNNr   )rX   rY   rZ   rR   r    r£   r   r   rA   r  re   r  rÓ   rì   r³   r[   r\   s   @rJ   r­  r­  H  s8  ø€ € € € € ðð ð ð ð ð1ð 1ð 1ð6ð 6ð 6ð Øð .2Ø.2Ø26Ø*.Ø59Ø6:Ø!%Ø-.ðBð BàÔ# dÑ*ðBð œ tÑ+ðBð Ô(¨4Ñ/ð	Bð
 Ô  4Ñ'ðBð !Ô+¨dÑ2ðBð !&¤¨tÑ 3ðBð ˜$‘;ðBð ˜eœlÑ*ðBð Bð Bñ „^ñ ÔðBð Bð Bð Bð BrL   r­  )r­  r�  r†  )Nr.   )ZÚdataclassesr   rA   r   Ú r   r‡   Úcache_utilsr   r   Ú
generationr	   Úmasking_utilsr
   r   Úmodeling_outputsr   r   r   r   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   Úutils.output_capturingr   r   Úesm.modeling_esmr   r   r   r   r   r   r    r!   r"   r#   Úllama.modeling_llamar$   r%   r&   r'   r(   r)   r*   Úconfiguration_evollar,   r-   Ú
get_loggerrX   ÚloggerÚModulere   rf   rK   rN   r^   rh   rm   ro   rq   rs   ru   rw   ry   r{   r”   rµ   rè   rõ   r  r  r  r0  rq  rs  ru  rw  r†  r�  r­  Ú__all__rj   rL   rJ   ú<module>rÎ     s”  ðð "Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )Ø JÐ JÐ JÐ JÐ JÐ JÐ JÐ Jðð ð ð ð ð ð ð ð ð ð ð ð .Ð -Ð -Ð -Ð -Ð -Ø &Ð &Ð &Ð &Ð &Ð &ðð ð ð ð ð ð ð ð ð ð ð ð 8Ð 7Ð 7Ð 7Ð 7Ð 7Ø EÐ EÐ EÐ EÐ EÐ EÐ EÐ Eðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð =Ð <Ð <Ð <Ð <Ð <Ð <Ð <ð 
ˆÔ	˜HÑ	%Ô	%€ð !Øð %ð  %ØŒIð %àŒ<ð %ð 
Œð %ð Œ<ð	 %ð
 ”L 4Ñ'ð %ð �T‰\ð %ð ð %ð Ð'Ô(ð %ð  %ð  %ð  %ðFð ð ð ð ˜]ñ ô ð ð?ð ?ð ?ð ?ð ?Ð"4ñ ?ô ?ð ?ð	ð 	ð 	ð 	ð 	Ð 0ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	˜]ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	˜Lñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	˜ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	˜ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	˜ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	˜*ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	˜ñ 	ô 	ð 	ð ðCð Cð Cð Cð C /ñ Cô Cñ „ðCð01
ð 1
ð 1
ð 1
ð 1
Ð!<ñ 1
ô 1
ð 1
ðh7 ð 7 ð 7 ð 7 ð 7 ¨¬	ñ 7 ô 7 ð 7 ðtAð Að Að Að A˜œ	ñ Aô Að Að'.ð '.ð '.ð '.ð '.¨¬	ñ '.ô '.ð '.ðT Ø
ð
<ð 
<ð 
<ð 
<ð 
< kñ 
<ô 
<ñ „ñ „ð
<ð
ð 
ð 
ð 
ð 
˜2œ9ñ 
ô 
ð 
ð$kð kð kð kð k¨"¬)ñ kô kð kð\	ð 	ð 	ð 	ð 	�Lñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð0ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	�ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	�nñ 	ô 	ð 	ð<ð <ð <ð <ð <Ð*ñ <ô <ð <ð~<ð <ð <ð <ð <Ð0ñ <ô <ð <ð.|ð |ð |ð |ð |Ð'ñ |ô |ð |ð~Sð Sð Sð Sð SÐ 5°ñ Sô Sð Sðl PÐ
OÐ
O€€€rL   