§
    9²¸j@«  ã                   óð  — d Z ddlZddlmZ ddlZddlmZ ddlmZmZm	Z	 ddl
mZ ddlmZmZ dd	lmZmZ dd
lmZ ddlmZmZmZmZmZ ddlmZ ddlmZmZ ddl m!Z! ddl"m#Z#m$Z$m%Z%m&Z& ddl'm(Z( ddl)m*Z*m+Z+ ddl,m-Z-m.Z. ddl/m0Z0  e&j1        e2¦  «        Z3d„ Z4 ed¦  «        dMd„¦   «         Z5d„ Z6d„ Z7d„ Z8 G d„ dej9        ¦  «        Z: G d„ dej9        ¦  «        Z; G d„ d ej9        ¦  «        Z<	 	 dNd"ej9        d#ej=        d$ej=        d%ej=        d&ej=        dz  d'e>dz  d(e>d)e!e#         fd*„Z? ee5¦  «         G d+„ d,ej9        ¦  «        ¦   «         Z@ G d-„ d.ej9        ¦  «        ZA G d/„ d0ej9        ¦  «        ZB G d1„ d2ej9        ¦  «        ZC G d3„ d4ej9        ¦  «        ZD G d5„ d6e¦  «        ZE G d7„ d8ej9        ¦  «        ZF G d9„ d:ej9        ¦  «        ZGe$ G d;„ d<e¦  «        ¦   «         ZHe$ G d=„ d>eH¦  «        ¦   «         ZIe$ G d?„ d@eH¦  «        ¦   «         ZJ G dA„ dBej9        ¦  «        ZK e$dC¬D¦  «         G dE„ dFeH¦  «        ¦   «         ZLe$ G dG„ dHeH¦  «        ¦   «         ZM G dI„ dJej9        ¦  «        ZNdK„ ZOg dL¢ZPdS )OzPyTorch ESM model.é    N)ÚCallable)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)Ú"BaseModelOutputWithCrossAttentionsÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚMaskedLMOutputÚSequenceClassifierOutputÚTokenClassifierOutput)Údynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )Ú	EsmConfigc                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..Néÿÿÿÿé   ©Údim)ÚshapeÚtorchÚcat)ÚxÚx1Úx2s      út/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/esm/modeling_esm.pyÚrotate_halfr/   0   s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'ó    Úrotary_pos_embc                 ó¨  — | j         }|                     |¦  «        }|                     |¦  «        }|                      ¦   «         |z  t          |                      ¦   «         ¦  «        |z  z   }|                     ¦   «         |z  t          |                     ¦   «         ¦  «        |z  z   }|                     |¦  «        |                     |¦  «        fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )ÚdtypeÚ	unsqueezeÚfloatr/   Úto)ÚqÚkÚcosÚsinÚunsqueeze_dimÚoriginal_dtypeÚq_embedÚk_embeds           r.   Úapply_rotary_pos_embr?   7   s¦   € ð& ”W€NØ
�-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�wŠw‰yŒy˜3‰¥;¨q¯wªw©y¬yÑ#9Ô#9¸CÑ#?Ñ@€GØ�wŠw‰yŒy˜3‰¥;¨q¯wªw©y¬yÑ#9Ô#9¸CÑ#?Ñ@€GØ�:Š:�nÑ%Ô% w§z¢z°.Ñ'AÔ'AÐAÐAr0   c                 óf   — | dz  dt          j        | t          j        d¦  «        z  ¦  «        z   z  S )zo
    This is the gelu implementation from the original ESM repo. Using F.gelu yields subtly wrong results.
    g      à?ç      ð?g       @)r)   ÚerfÚmathÚsqrt©r+   s    r.   ÚgelurF   R   s/   € ð ˆs‰7�c�EœI a­$¬)°C©.¬.Ñ&8Ñ9Ô9Ñ9Ñ:Ð:r0   c                 ó4   — | |                       dd¦  «        z   S )zJMake layer symmetric in final two dimensions, used for contact prediction.r$   éþÿÿÿ)Ú	transposerE   s    r.   Ú
symmetrizerJ   Y   s   € àˆq�{Š{˜2˜rÑ"Ô"Ñ"Ð"r0   c                 óÎ   — |                       dd¬¦  «        }|                       dd¬¦  «        }|                       dd¬¦  «        }||z  }|                     |¦  «         | |z
  }|S )z=Perform average product correct, used for contact prediction.r$   T)ÚkeepdimsrH   )r$   rH   )ÚsumÚdiv_)r+   Úa1Úa2Úa12ÚavgÚ
normalizeds         r.   Úaverage_product_correctrT   ^   sh   € à	
�Šˆr˜DˆÑ	!Ô	!€BØ	
�Šˆr˜DˆÑ	!Ô	!€BØ
�%Š%� 4ˆ%Ñ
(Ô
(€Cà
ˆr‰'€CØ‡H‚HˆS�M„M€MØ�S‘€JØÐr0   c            	       óô   ‡ — e Zd ZdZ edd¬¦  «        ddefˆ fd„¦   «         Ze edd¬¦  «        ddedee	j
        ef         fd	„¦   «         ¦   «         Z e	j        ¦   «         edd
„¦   «         ¦   «         Zˆ xZS )ÚEsmRotaryEmbeddingzú
    Rotary position embeddings.
    Implementation based on [ModernBERT's RotaryEmbedding](https://github.com/huggingface/transformers/blob/aad13b87ed59f2afcfaebc985f403301887a35fc/src/transformers/models/modernbert/modeling_modernbert.py#L94).
    Údevicez5.18)ÚversionNÚconfigc                 óò   •— t          ¦   «                              ¦   «          || _        i | _        |                      | j        ¦  «        \  }}t          j        |¦  «        | _        t          | d|¦  «         d S )NÚattention_scaling)	ÚsuperÚ__init__rY   Ú	rope_typeÚcompute_default_rope_parametersr   ÚBufferÚinv_freqÚsetattr)ÚselfrY   rW   Úcurr_inv_freqÚcurr_attention_scalingÚ	__class__s        €r.   r]   zEsmRotaryEmbedding.__init__p   so   ø€ å‰Œ×ÒÑÔÐàˆŒØˆŒà04×0TÒ0TÐUYÔU`Ñ0aÔ0aÑ-ˆÐ-Ýœ	 -Ñ0Ô0ˆŒÝ�Ð)Ð+AÑBÔBÐBÐBÐBr0   Úreturnc                 óØ   — | j         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «        |z  z  z  }|                     |¦  «        |fS )a¿  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.

        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Úhead_dimNrA   r   r%   ©r3   )Ú
rope_thetaÚgetattrÚhidden_sizeÚnum_attention_headsr)   Úaranger5   r6   )rY   rW   ÚkwargsÚbaser'   Úattention_factorra   s          r.   r_   z2EsmRotaryEmbedding.compute_default_rope_parameters{   su   € ð Ô ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐà˜$¥5¤<°°3¸ÅÄÐ#MÑ#MÔ#MÐPSÑ#SÑTÑUˆØ�{Š{˜6Ñ"Ô"Ð$4Ð4Ð4r0   c                 óp  — t          | d¦  «        }t          | d¦  «        }|d d d …d f                              ¦   «                              |j        d         dd¦  «                             |j        ¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬	¦  «        5  |                     ¦   «         |                     ¦   «         z   
                    dd
¦  «        }	t          j        |	|	fd¬¦  «        }
|
                     ¦   «         |z  }|
                     ¦   «         |z  }d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬¦  «        |                     |j        ¬¦  «        fS )Nra   r[   r   r$   r!   ÚmpsÚcpuF)Údevice_typeÚenabledr%   r&   rj   )rl   r5   Úexpandr(   r6   rW   Ú
isinstanceÚtypeÚstrr   rI   r)   r*   r9   r:   r3   )rc   r+   Úposition_idsÚ
layer_typera   r[   Úinv_freq_expandedÚposition_ids_expandedrv   ÚfreqsÚembr9   r:   s                r.   ÚforwardzEsmRotaryEmbedding.forward�   sÑ  € õ ˜4 Ñ,Ô,ˆÝ# DÐ*=Ñ>Ô>Ðà$ T¨1¨1¨1¨d ]Ô3×9Ò9Ñ;Ô;×BÒBÀ<ÔCUÐVWÔCXÐZ\Ð^_Ñ`Ô`×cÒcÐdeÔdlÑmÔmÐØ ,¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆÝ¨¸UÐCÑCÔCð 	0ð 	0Ø&×,Ò,Ñ.Ô.Ð1F×1LÒ1LÑ1NÔ1NÑN×YÒYÐZ[Ð]^Ñ_Ô_ˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)Ð/Ñ/ˆCØ—'’'‘)”)Ð/Ñ/ˆCð		0ð 	0ð 	0ñ 	0ô 	0ð 	0ð 	0ð 	0ð 	0ð 	0ð 	0øøøð 	0ð 	0ð 	0ð 	0ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   Ã'BE7Å7E;Å>E;©N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r"   r]   ÚstaticmethodÚtupler)   ÚTensorr5   r_   Úno_gradr   r‚   Ú__classcell__©rf   s   @r.   rV   rV   j   s  ø€ € € € € ðð ð
 €_�X vÐ.Ñ.Ô.ðCð C˜yð Cð Cð Cð Cð Cñ /Ô.ðCð Ø€_�X vÐ.Ñ.Ô.ð5ð 5°	ð 5ÐUZÐ[`Ô[gÐinÐ[nÔUoð 5ð 5ð 5ñ /Ô.ñ „\ð5ð& €U„]�_„_Øð<ð <ð <ñ Ôñ „_ð<ð <ð <ð <ð <r0   rV   c                   ó8   ‡ — e Zd ZdZ	 	 ddedefˆ fd„Zd„ Zˆ xZS )	ÚEsmContactPredictionHeadzWPerforms symmetrization, apc, and computes a logistic regression on the output featuresTr%   Úin_featuresÚeos_idxc                 óÊ   •— t          ¦   «                              ¦   «          || _        || _        t	          j        |d|¦  «        | _        t	          j        ¦   «         | _        d S )Nr!   )	r\   r]   r�   r‘   r   ÚLinearÚ
regressionÚSigmoidÚ
activation)rc   r�   Úbiasr‘   rf   s       €r.   r]   z!EsmContactPredictionHead.__init__¦   sP   ø€ õ 	‰Œ×ÒÑÔÐØ&ˆÔØˆŒÝœ) K°°DÑ9Ô9ˆŒÝœ*™,œ,ˆŒˆˆr0   c                 óª  — |                      | j        ¦  «                             |¦  «        }|                     d¦  «        |                     d¦  «        z  }||d d …d d d d …d d …f         z  }|dd d…d d…f         }|ddd …dd …f         }|                     ¦   «         \  }}}}}|                     |||z  ||¦  «        }|                     | j        j        j        ¦  «        }t          t          |¦  «        ¦  «        }|                     dddd¦  «        }|                      |                      |¦  «                             d¦  «        ¦  «        S )Nr!   r%   .r$   r   r   )Úner‘   r6   r4   ÚsizeÚviewr”   ÚweightrW   rT   rJ   Úpermuter–   Úsqueeze)	rc   ÚtokensÚ
attentionsÚeos_maskÚ
batch_sizeÚlayersÚheadsÚseqlenÚ_s	            r.   r‚   z EsmContactPredictionHead.forward²   sP  € à—9’9˜Tœ\Ñ*Ô*×-Ò-¨jÑ9Ô9ˆØ×%Ò% aÑ(Ô(¨8×+=Ò+=¸aÑ+@Ô+@Ñ@ˆØ (¨1¨1¨1¨d°D¸!¸!¸!¸Q¸Q¸QÐ+>Ô"?Ñ?ˆ
Ø  S b S¨#¨2¨# Ô.ˆ
à  Q R R¨¨¨ Ô,ˆ
Ø/9¯ªÑ/@Ô/@Ñ,ˆ
�F˜E 6¨1Ø—_’_ Z°¸%±ÀÈÑPÔPˆ
ð  —]’]ØŒOÔ"Ô)ñ
ô 
ˆ
õ -­Z¸
Ñ-CÔ-CÑDÔDˆ
Ø×'Ò'¨¨1¨a°Ñ3Ô3ˆ
Ø�Š˜tŸš¨zÑ:Ô:×BÒBÀ1ÑEÔEÑFÔFÐFr0   )Tr%   )r„   r…   r†   r‡   Úintr]   r‚   rŒ   r�   s   @r.   r�   r�   £   sx   ø€ € € € € ØaÐað
 Øð	
'ð 
'àð
'ð ð	
'ð 
'ð 
'ð 
'ð 
'ð 
'ðGð Gð Gð Gð Gð Gð Gr0   r�   c                   ó8   ‡ — e Zd ZdZˆ fd„Z	 	 	 	 dd„Zd„ Zˆ xZS )ÚEsmEmbeddingszV
    Same as BertEmbeddings with a tiny tweak for positional embeddings indexing.
    c                 ó¬  •— t          ¦   «                              ¦   «          t          j        |j        |j        |j        ¬¦  «        | _        |j        r&t          j	        |j        |j
        ¬¦  «        | _        nd | _        t          j        |j        ¦  «        | _        t          |dd¦  «        | _        t          j        t%          j        |j        ¦  «                             d¦  «        d¬¦  «        | _        |j        | _        | j        dk    r+t          j        |j        |j        | j        ¬¦  «        | _        |j        | _        |j        | _        d S )N)Úpadding_idx©ÚepsÚposition_embedding_typeÚabsolute©r!   r$   F)Ú
persistent)r\   r]   r   Ú	EmbeddingÚ
vocab_sizerm   Úpad_token_idÚword_embeddingsÚemb_layer_norm_beforeÚ	LayerNormÚlayer_norm_epsÚ
layer_normÚDropoutÚhidden_dropout_probÚdropoutrl   r®   r`   r)   ro   Úmax_position_embeddingsrx   r|   r«   Úposition_embeddingsÚtoken_dropoutÚmask_token_id©rc   rY   rf   s     €r.   r]   zEsmEmbeddings.__init__Ë   s+  ø€ Ý‰Œ×ÒÑÔÐÝ!œ|¨FÔ,=¸vÔ?QÐ_eÔ_rÐsÑsÔsˆÔàÔ'ð 	#Ý œl¨6Ô+=À6ÔCXÐYÑYÔYˆDŒOˆOà"ˆDŒOÝ”z &Ô"<Ñ=Ô=ˆŒå'.¨vÐ7PÐR\Ñ']Ô']ˆÔ$ÝœI¥e¤l°6Ô3QÑ&RÔ&R×&YÒ&YÐZaÑ&bÔ&bÐotÐuÑuÔuˆÔà!Ô.ˆÔØÔ'¨:Ò5Ð5Ý')¤|ØÔ.°Ô0BÐPTÔP`ð(ñ (ô (ˆDÔ$ð $Ô1ˆÔØ#Ô1ˆÔÐÐr0   Nc                 ó  — |€-|�t          || j        ¦  «        }n|                      |¦  «        }|€|                      |¦  «        }|}| j        r¾|�¼|                     || j        k                         d¦  «        d¦  «        }d}|�|                     d¦  «        n|j	        d         }|| j        k                         d¦  «         
                    ¦   «         |z  }|d|z
  z  d|z
  d d …d d f         z                       |j        ¦  «        }| j        dk    r|                      |¦  «        }	||	z   }| j        �|                      |¦  «        }|�0||                     d¦  «        z                       |j        ¦  «        }|S )Nr$   ç        g¸…ëQ¸¾?r!   r¯   )Ú"create_position_ids_from_input_idsr«   Ú&create_position_ids_from_inputs_embedsrµ   r¿   Úmasked_fillrÀ   r4   rM   r(   r5   r6   r3   r®   r¾   r¹   )
rc   Ú	input_idsÚattention_maskr|   Úinputs_embedsÚ
embeddingsÚmask_ratio_trainÚsrc_lengthsÚmask_ratio_observedr¾   s
             r.   r‚   zEsmEmbeddings.forwardà   s¶  € ð ÐØÐ$åAÀ)ÈTÔM]Ñ^Ô^��à#×JÒJÈ=ÑYÔY�àÐ Ø ×0Ò0°Ñ;Ô;ˆMð #ˆ
ð Ôð 	 )Ð"7Ø#×/Ò/°¸dÔ>PÒ1P×0[Ò0[Ð\^Ñ0_Ô0_ÐadÑeÔeˆJØ)ÐØ4BÐ4N˜.×,Ò,¨RÑ0Ô0Ð0ÐT]ÔTcÐdeÔTfˆKØ#,°Ô0BÒ#B×"GÒ"GÈÑ"KÔ"K×"QÒ"QÑ"SÔ"SÐVaÑ"aÐØ$¨Ð,<Ñ(<Ñ=ÀÐEXÑAXÐZ[ÐZ[ÐZ[Ð]aÐcgÐZgÔ@hÑh×lÒlØÔ ñô ˆJð Ô'¨:Ò5Ð5Ø"&×":Ò":¸<Ñ"HÔ"HÐØ#Ð&9Ñ9ˆJàŒ?Ð&ØŸš¨Ñ4Ô4ˆJØÐ%Ø$ ~×'?Ò'?ÀÑ'CÔ'CÑC×GÒGÈ
ÔHXÑYÔYˆJð Ðr0   c                 ó  — |                      ¦   «         dd…         }|d         }t          j        | j        dz   || j        z   dz   t          j        |j        ¬¦  «        }|                     d¦  «                             |¦  «        S )z×
        We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

        Args:
            inputs_embeds: torch.Tensor

        Returns: torch.Tensor
        Nr$   r!   )r3   rW   r   )rš   r)   ro   r«   ÚlongrW   r4   rx   )rc   rÉ   Úinput_shapeÚsequence_lengthr|   s        r.   rÅ   z4EsmEmbeddings.create_position_ids_from_inputs_embeds  s‡   € ð $×(Ò(Ñ*Ô*¨3¨B¨3Ô/ˆØ% aœ.ˆå”|ØÔ˜qÑ  /°DÔ4DÑ"DÀqÑ"HÕPUÔPZÐcpÔcwð
ñ 
ô 
ˆð ×%Ò% aÑ(Ô(×/Ò/°Ñ<Ô<Ð<r0   ©NNNN)r„   r…   r†   r‡   r]   r‚   rÅ   rŒ   r�   s   @r.   r©   r©   Æ   st   ø€ € € € € ðð ð2ð 2ð 2ð 2ð 2ð. ØØØð/ð /ð /ð /ðb=ð =ð =ð =ð =ð =ð =r0   r©   rÃ   ÚmoduleÚqueryÚkeyÚvaluerÈ   Úscalingr¼   rp   c                 ó®  — |€|                      d¦  «        dz  }t          j        ||                     dd¦  «        ¦  «        |z  }|�||z   }t          j                             |d¬¦  «        }t          j                             ||| j        ¬¦  «        }t          j        ||¦  «        }	|	                     dd¦  «         	                    ¦   «         }	|	|fS )Nr$   ç      à¿r%   r   r&   )ÚpÚtrainingr!   )
rš   r)   ÚmatmulrI   r   Ú
functionalÚsoftmaxr¼   rÛ   Ú
contiguous)
rÓ   rÔ   rÕ   rÖ   rÈ   r×   r¼   rp   Úattn_weightsÚattn_outputs
             r.   Úeager_attention_forwardrâ   $  sÈ   € ð €Ø—*’*˜R‘.”. DÑ(ˆõ ”<  s§}¢}°Q¸Ñ':Ô':Ñ;Ô;¸gÑE€LàÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2Ð(Ñ>Ô>€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r0   c                   óº   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 ddej        dej        dz  dej        dz  dej        dz  dej        dz  d	ee         d
e	ej                 fd„Z
ˆ xZS )ÚEsmSelfAttentionNFc                 óÐ  •— t          ¦   «                              ¦   «          || _        |j        |j        z  dk    r0t          |d¦  «        s t          d|j        › d|j        › d�¦  «        ‚|j        | _        t          |j        |j        z  ¦  «        | _        | j        | j        z  | _	        t          j        |j        | j	        ¦  «        | _        t          j        |j        | j	        ¦  «        | _        t          j        |j        | j	        ¦  «        | _        |j        | _        |pt#          |dd¦  «        | _        d| _        |j        | _        || _        | j        o| | _        d S )	Nr   Úembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (ú)r®   r¯   rA   )r\   r]   rY   rm   rn   ÚhasattrÚ
ValueErrorr§   Úattention_head_sizeÚall_head_sizer   r“   rÔ   rÕ   rÖ   Úattention_probs_dropout_probr¼   rl   r®   r×   Ú
is_decoderÚ	layer_idxÚ	is_causal)rc   rY   r®   rî   Úis_cross_attentionrf   s        €r.   r]   zEsmSelfAttention.__init__B  se  ø€ Ý‰Œ×ÒÑÔÐØˆŒàÔ Ô :Ñ:¸aÒ?Ð?ÍÐPVÐXhÑHiÔHiÐ?Ýð8 FÔ$6ð 8ð 8Ø Ô4ð8ð 8ð 8ñô ð ð
 $*Ô#=ˆÔ Ý#& vÔ'9¸FÔ<VÑ'VÑ#WÔ#WˆÔ Ø!Ô5¸Ô8PÑPˆÔå”Y˜vÔ1°4Ô3EÑFÔFˆŒ
Ý”9˜VÔ/°Ô1CÑDÔDˆŒÝ”Y˜vÔ1°4Ô3EÑFÔFˆŒ
àÔ:ˆŒà'>ð (
Å'ØÐ-¨zñC
ô C
ˆÔ$ð ˆŒØ Ô+ˆŒØ"ˆŒØœÐCÐ1CÐ-CˆŒˆˆr0   Úhidden_statesrÈ   Úencoder_hidden_statesÚencoder_attention_maskr¾   rp   rg   c                 ó
  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }	|d u}
|
r|n|}|
r|n|}|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }|	| j        dz  z  }	| j        dk    r|\  }}t          |	|||d¬¦  «        \  }	}t          j
        | j        j        t          ¦  «        } || |	|||f| j        sdn| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }||fS )	Nr$   r!   r%   rÙ   Úrotary)r;   rÃ   )r¼   r×   )r(   rê   rÔ   r›   rI   rÕ   rÖ   r®   r?   r   Úget_interfacerY   Ú_attn_implementationrâ   rÛ   r¼   r×   Úreshaperß   )rc   rñ   rÈ   rò   ró   r¾   rp   rÐ   Úhidden_shapeÚquery_layerrð   Úcurrent_statesÚ	key_layerÚvalue_layerr9   r:   Úattention_interfacerá   rà   s                      r.   r‚   zEsmSelfAttention.forward^  sÛ  € ð $Ô)¨#¨2¨#Ô.ˆØC˜ÐC bÐC¨$Ô*BÐCÐCˆà—j’j Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆà2¸$Ð>ÐØ2DÐWÐ.Ð.È-ˆØ3EÐYÐ/Ð/È>ˆØ—H’H˜^Ñ,Ô,×1Ò1°,Ñ?Ô?×IÒIÈ!ÈQÑOÔOˆ	Ø—j’j Ñ0Ô0×5Ò5°lÑCÔC×MÒMÈaÐQRÑSÔSˆð " DÔ$<¸dÑ$BÑBˆàÔ'¨8Ò3Ð3Ø*‰HˆC�Ý%9¸+ÀyÐRUÐWZÐjkÐ%lÑ%lÔ%lÑ"ˆK˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}Ð>�C�C°$´,Ø”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ˜LÐ(Ð(r0   )NNFrÒ   )r„   r…   r†   r]   r)   rŠ   ÚFloatTensorr   r   r‰   r‚   rŒ   r�   s   @r.   rä   rä   @  sÜ   ø€ € € € € ðDð Dð Dð Dð Dð Dð> 48Ø:>Ø;?Ø37ð.)ð .)à”|ð.)ð Ô)¨DÑ0ð.)ð  %Ô0°4Ñ7ð	.)ð
 !&Ô 1°DÑ 8ð.)ð #œ\¨DÑ0ð.)ð Ð+Ô,ð.)ð 
ˆuŒ|Ô	ð.)ð .)ð .)ð .)ð .)ð .)ð .)ð .)r0   rä   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚEsmSelfOutputc                 óÌ   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        ¦  «        | _        d S rƒ   )	r\   r]   r   r“   rm   Údenserº   r»   r¼   rÁ   s     €r.   r]   zEsmSelfOutput.__init__�  sJ   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ý”z &Ô"<Ñ=Ô=ˆŒˆˆr0   c                 ód   — |                       |¦  «        }|                      |¦  «        }||z   }|S rƒ   ©r  r¼   ©rc   rñ   Úinput_tensors      r.   r‚   zEsmSelfOutput.forward•  ó4   € ØŸ
š
 =Ñ1Ô1ˆØŸš ]Ñ3Ô3ˆØ%¨Ñ4ˆØÐr0   ©r„   r…   r†   r]   r‚   rŒ   r�   s   @r.   r  r  �  óG   ø€ € € € € ð>ð >ð >ð >ð >ð
ð ð ð ð ð ð r0   r  c                   óV   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 ddej        dz  dee         fd„Zˆ xZ	S )	ÚEsmAttentionNFc                 óè   •— t          ¦   «                              ¦   «          t          |||¬¦  «        | _        t	          |¦  «        | _        t          j        |j        |j	        ¬¦  «        | _        d S )N)rî   rð   r¬   )
r\   r]   rä   rc   r  Úoutputr   r·   rm   r¸   )rc   rY   rî   rð   rf   s       €r.   r]   zEsmAttention.__init__�  s_   ø€ Ý‰Œ×ÒÑÔÐÝ$ V°yÐUgÐhÑhÔhˆŒ	Ý# FÑ+Ô+ˆŒåœ fÔ&8¸fÔ>SÐTÑTÔTˆŒˆˆr0   r¾   rp   c                 óˆ   — |                       |¦  «        } | j        |f||||dœ|¤Ž\  }}	|                      ||¦  «        }|S )N©rÈ   rò   ró   r¾   )r·   rc   r  )
rc   rñ   rÈ   rò   ró   r¾   rp   Úhidden_states_lnrá   r¦   s
             r.   r‚   zEsmAttention.forward¤  sk   € ð  Ÿ>š>¨-Ñ8Ô8ÐØ"˜œØð
à)Ø"7Ø#9Ø 3ð
ð 
ð ð
ð 
‰ˆ�Qð —k’k +¨}Ñ=Ô=ˆØÐr0   )NFrÒ   )
r„   r…   r†   r]   r)   rŠ   r   r   r‚   rŒ   r�   s   @r.   r  r  œ  s‹   ø€ € € € € ðUð Uð Uð Uð Uð Uð Ø"Ø#Ø37ðð ð #œ\¨DÑ0ðð Ð+Ô,ðð ð ð ð ð ð ð r0   r  c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚEsmIntermediatec                 ó�   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        d S rƒ   )r\   r]   r   r“   rm   Úintermediate_sizer  rÁ   s     €r.   r]   zEsmIntermediate.__init__»  s6   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3KÑLÔLˆŒ
ˆ
ˆ
r0   rñ   rg   c                 óN   — |                       |¦  «        }t          |¦  «        }|S rƒ   )r  rF   )rc   rñ   s     r.   r‚   zEsmIntermediate.forward¿  s&   € ØŸ
š
 =Ñ1Ô1ˆÝ˜]Ñ+Ô+ˆØÐr0   ©r„   r…   r†   r]   r)   rŠ   r‚   rŒ   r�   s   @r.   r  r  º  sc   ø€ € € € € ðMð Mð Mð Mð Mð U¤\ð °e´lð ð ð ð ð ð ð ð r0   r  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )Ú	EsmOutputc                 óÌ   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        ¦  «        | _	        d S rƒ   )
r\   r]   r   r“   r  rm   r  rº   r»   r¼   rÁ   s     €r.   r]   zEsmOutput.__init__Æ  sJ   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ7¸Ô9KÑLÔLˆŒ
Ý”z &Ô"<Ñ=Ô=ˆŒˆˆr0   c                 ód   — |                       |¦  «        }|                      |¦  «        }||z   }|S rƒ   r  r  s      r.   r‚   zEsmOutput.forwardË  r  r0   r	  r�   s   @r.   r  r  Å  r
  r0   r  c                   óZ   ‡ — e Zd Zˆ fd„Z	 	 	 	 ddej        dz  dee         fd„Zd„ Z	ˆ xZ
S )ÚEsmLayerc                 óÌ  •— t          ¦   «                              ¦   «          |j        | _        d| _        t	          |¦  «        | _        |j        | _        |j        | _        | j        r/| j        st          | › d�¦  «        ‚t	          |d¬¦  «        | _	        t          |¦  «        | _        t          |¦  «        | _        t          j        |j        |j        ¬¦  «        | _        d S )Nr!   z> should be used as a decoder model if cross attention is addedT)rð   r¬   )r\   r]   Úchunk_size_feed_forwardÚseq_len_dimr  Ú	attentionrí   Úadd_cross_attentionÚRuntimeErrorÚcrossattentionr  Úintermediater  r  r   r·   rm   r¸   rÁ   s     €r.   r]   zEsmLayer.__init__Ó  sÒ   ø€ Ý‰Œ×ÒÑÔÐØ'-Ô'EˆÔ$ØˆÔÝ% fÑ-Ô-ˆŒØ Ô+ˆŒØ#)Ô#=ˆÔ ØÔ#ð 	PØ”?ð lÝ" dÐ#jÐ#jÐ#jÑkÔkÐkÝ".¨vÈ$Ð"OÑ"OÔ"OˆDÔÝ+¨FÑ3Ô3ˆÔÝ Ñ'Ô'ˆŒÝœ fÔ&8¸fÔ>SÐTÑTÔTˆŒˆˆr0   Nr¾   rp   c                 óÐ   —  | j         |f||dœ|¤Ž}| j        r8|�6t          | d¦  «        st          d| › d�¦  «        ‚ | j        |f||||dœ|¤Ž}|                      |¦  «        }|S )N)rÈ   r¾   r$  z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r  )r!  rí   rè   ÚAttributeErrorr$  Úfeed_forward_chunk)	rc   rñ   rÈ   rò   ró   r¾   rp   Úattention_outputÚlayer_outputs	            r.   r‚   zEsmLayer.forwardâ  sÜ   € ð *˜4œ>Øð
à)Ø 3ð
ð 
ð ð	
ð 
Ðð Œ?ð 	Ð4Ð@Ý˜4Ð!1Ñ2Ô2ð Ý$ð`¸dð `ð `ð `ñô ð ð
  3˜tÔ2Ø ð à-Ø&;Ø'=Ø$7ð ð  ð ð ð  Ðð ×.Ò.Ð/?Ñ@Ô@ˆØÐr0   c                 ó†   — |                       |¦  «        }|                      |¦  «        }|                      ||¦  «        }|S rƒ   )r·   r%  r  )rc   r)  Úattention_output_lnÚintermediate_outputr*  s        r.   r(  zEsmLayer.feed_forward_chunk  sE   € Ø"ŸnšnÐ-=Ñ>Ô>ÐØ"×/Ò/Ð0CÑDÔDÐØ—{’{Ð#6Ð8HÑIÔIˆØÐr0   rÒ   )r„   r…   r†   r]   r)   rŠ   r   r   r‚   r(  rŒ   r�   s   @r.   r  r  Ò  s•   ø€ € € € € ðUð Uð Uð Uð Uð$ Ø"Ø#Ø37ð!ð !ð #œ\¨DÑ0ð!ð Ð+Ô,ð!ð !ð !ð !ðFð ð ð ð ð ð r0   r  c                   ód   ‡ — e Zd Zˆ fd„Ze	 	 	 	 ddej        dz  dee         fd„¦   «         Z	ˆ xZ
S )Ú
EsmEncoderc                 ó  •‡— t          ¦   «                              ¦   «          ‰| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          j        ‰j	        ‰j
        ¬¦  «        | _        d| _        d S )Nc                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS © )r  )Ú.0r¦   rY   s     €r.   ú
<listcomp>z'EsmEncoder.__init__.<locals>.<listcomp>  s!   ø€ Ð#^Ð#^Ð#^¸¥H¨VÑ$4Ô$4Ð#^Ð#^Ð#^r0   r¬   F)r\   r]   rY   r   Ú
ModuleListÚrangeÚnum_hidden_layersÚlayerr·   rm   r¸   Úemb_layer_norm_afterÚgradient_checkpointingrÁ   s    `€r.   r]   zEsmEncoder.__init__  s}   øø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”]Ð#^Ð#^Ð#^Ð#^½eÀFÔD\Ñ>]Ô>]Ð#^Ñ#^Ô#^Ñ_Ô_ˆŒ
Ý$&¤L°Ô1CÈÔI^Ð$_Ñ$_Ô$_ˆÔ!Ø&+ˆÔ#Ð#Ð#r0   Nr¾   rp   c           	      óª   — t          | j        ¦  «        D ]\  }} ||f||||dœ|¤Ž}Œ| j        r|                      |¦  «        }t          |¬¦  «        S )Nr  )Úlast_hidden_state)Ú	enumerater8  r9  r   )	rc   rñ   rÈ   rò   ró   r¾   rp   ÚiÚlayer_modules	            r.   r‚   zEsmEncoder.forward  s‰   € õ  )¨¬Ñ4Ô4ð 	ð 	‰OˆAˆ|Ø(˜LØðà-Ø&;Ø'=Ø$7ðð ð ðð ˆMˆMð Ô$ð 	EØ ×5Ò5°mÑDÔDˆMå1ÀMÐRÑRÔRÐRr0   rÒ   )r„   r…   r†   r]   r   r)   rŠ   r   r   r‚   rŒ   r�   s   @r.   r/  r/    s˜   ø€ € € € € ð,ð ,ð ,ð ,ð ,ð ð Ø"Ø#Ø37ðSð Sð #œ\¨DÑ0ðSð Ð+Ô,ðSð Sð Sñ ÔðSð Sð Sð Sð Sr0   r/  c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )Ú	EsmPoolerc                 óÀ   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        ¦   «         | _        d S rƒ   )r\   r]   r   r“   rm   r  ÚTanhr–   rÁ   s     €r.   r]   zEsmPooler.__init__0  sC   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ýœ'™)œ)ˆŒˆˆr0   rñ   rg   c                 ór   — |d d …df         }|                       |¦  «        }|                      |¦  «        }|S ©Nr   )r  r–   )rc   rñ   Úfirst_token_tensorÚpooled_outputs       r.   r‚   zEsmPooler.forward5  s@   € ð +¨1¨1¨1¨a¨4Ô0ÐØŸ
š
Ð#5Ñ6Ô6ˆØŸš¨Ñ6Ô6ˆØÐr0   r  r�   s   @r.   rA  rA  /  s^   ø€ € € € € ð$ð $ð $ð $ð $ð
 U¤\ð °e´lð ð ð ð ð ð ð ð r0   rA  c                   óÄ   ‡ — e Zd ZU eed<   dZdZdZg d¢ZdgZ	dZ
dZdZdZe eedd¬	¦  «        g eedd
¬	¦  «        gdœZ ej        ¦   «         ˆ fd„¦   «         Zd„ Zˆ xZS )ÚEsmPreTrainedModelrY   ÚesmTF)r  Ú#EsmFoldTriangularSelfAttentionBlockr©   zposition_embeddings.weightr!   r!  )ÚindexÚ
layer_namer$  )rñ   r    Úcross_attentionsc                 ó$  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rt	          j        |j        ¦  «         dS t          |t          ¦  «        rQt	          j        |j	        t          j        |j	        j        d         ¦  «                             d¦  «        ¦  «         dS t          |t          ¦  «        rB|                     |j        ¦  «        \  }}t	          j        t#          |d¦  «        |¦  «         dS dS )zInitialize the weightsr$   r°   ra   N)r\   Ú_init_weightsry   Ú	EsmLMHeadÚinitÚzeros_r—   r©   Úcopy_r|   r)   ro   r(   rx   rV   r_   rY   rl   )rc   rÓ   rd   r¦   rf   s       €r.   rP  z EsmPreTrainedModel._init_weightsS  s÷   ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�f�iÑ(Ô(ð 	CÝŒK˜œÑ$Ô$Ð$Ð$Ð$Ý˜¥Ñ.Ô.ð 	CÝŒJ�vÔ*­E¬L¸Ô9LÔ9RÐSUÔ9VÑ,WÔ,W×,^Ò,^Ð_fÑ,gÔ,gÑhÔhÐhÐhÐhÝ˜Õ 2Ñ3Ô3ð 	CØ%×EÒEÀfÄmÑTÔTÑˆM˜1ÝŒJ•w˜v zÑ2Ô2°MÑBÔBÐBÐBÐBð	Cð 	Cr0   c                 ó   — d S rƒ   r2  ©rc   s    r.   Úget_output_embeddingsz(EsmPreTrainedModel.get_output_embeddings_  s	   € ð ˆtr0   )r„   r…   r†   r"   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚaccepts_loss_kwargsÚ_no_split_modulesÚ"_keys_to_ignore_on_load_unexpectedÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendr  r   rä   Ú_can_record_outputsr)   r‹   rP  rW  rŒ   r�   s   @r.   rI  rI  >  sñ   ø€ € € € € € àÐÐÑØÐØ&*Ð#ØÐØ\Ð\Ð\ÐØ*FÐ)GÐ&ØÐØ€NØÐØ"&Ðð "Ø%�~Ð&6¸aÈKÐXÑXÔXÐYàˆNÐ+°1ÐAQÐRÑRÔRð
ðð Ðð €U„]�_„_ð	Cð 	Cð 	Cð 	Cñ „_ð	Cðð ð ð ð ð ð r0   rI  c                   ó0  ‡ — e Zd ZdZdˆ fd„	Zd„ Zd„ Zd„ Zee	e
	 	 	 	 	 	 ddej        dz  d	ej        dz  d
ej        dz  dej        dz  dej        dz  dej        dz  dee         deej                 ez  fd„¦   «         ¦   «         ¦   «         Zd„ Zd„ Zˆ xZS )ÚEsmModela  

    The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
    cross-attention is added between the self-attention layers, following the architecture described in [Attention is
    all you need](https://huggingface.co/papers/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit,
    Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.

    To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
    to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
    `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.
    Tc                 óô  •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        d| _        t          |dd¦  «        | _        | j        dk    rt          |¬¦  «        | _        t          |¦  «        | _
        |rt          |¦  «        nd| _        t          |j        |j        z  d¬¦  «        | _        |                      | j        ¦  «         |                      ¦   «          dS )zv
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        Nr®   r¯   rõ   )rY   T)r�   r—   )r\   r]   rY   r©   rÊ   Úrotary_embeddingsrl   r®   rV   r/  ÚencoderrA  Úpoolerr�   r7  rn   Úcontact_headÚ"_register_load_state_dict_pre_hookÚ	load_hookÚ	post_init)rc   rY   Úadd_pooling_layerrf   s      €r.   r]   zEsmModel.__init__s  sð   ø€ õ
 	‰Œ×Ò˜Ñ Ô Ð ØˆŒå'¨Ñ/Ô/ˆŒà!%ˆÔÝ'.¨vÐ7PÐR\Ñ']Ô']ˆÔ$ØÔ'¨8Ò3Ð3Ý%7¸vÐ%FÑ%FÔ%FˆDÔ"å! &Ñ)Ô)ˆŒà+<ÐF•i Ñ'Ô'Ð'À$ˆŒå4ØÔ0°6Ô3MÑMÐTXð
ñ 
ô 
ˆÔð 	×/Ò/°´Ñ?Ô?Ð?à�ŠÑÔÐÐÐr0   c                 óÈ   ‡— ‰› d�}||vrUt          ˆfd„t          |                     ¦   «         ¦  «        D ¦   «         ¦  «        }|r||d                  ||<   |D ]}||= ŒdS dS )aÿ  Remap per-layer rotary inv_freq keys from old checkpoints to the new model-level location.

        Old checkpoints stored inv_freq per attention layer at:
            {prefix}encoder.layer.{i}.attention.self.rotary_embeddings.inv_freq
        New code stores a single shared inv_freq at:
            {prefix}rotary_embeddings.inv_freq
        The old checkpoint values must be preserved (not recomputed) because they may
        have been saved in float16, matching the precision used during training.
        zrotary_embeddings.inv_freqc              3   óp   •K  — | ]0}|                      ‰¦  «        ¯|                     d ¦  «        ¯,|V — Œ1dS )z*.attention.self.rotary_embeddings.inv_freqN)Ú
startswithÚendswith)r3  r8   Úprefixs     €r.   ú	<genexpr>z%EsmModel.load_hook.<locals>.<genexpr>š  s`   øè è € ð ð àØ—<’< Ñ'Ô'ðð -.¯JªJÐ7cÑ,dÔ,dðØðð ð ð ð ð r0   r   N)ÚsortedÚlistÚkeys)rc   Ú
state_dictrr  ÚargsÚnew_keyÚold_keysr8   s     `    r.   rk  zEsmModel.load_hookŽ  s¬   ø€ ð Ð7Ð7Ð7ˆØ˜*Ð$Ð$Ýð ð ð ð å˜jŸošoÑ/Ô/Ñ0Ô0ðñ ô ñ ô ˆHð
 ð >Ø&0°¸!´Ô&=�
˜7Ñ#Øð "ð "�Ø˜q�M�Mð %Ð$ð"ð "r0   c                 ó   — | j         j        S rƒ   ©rÊ   rµ   rV  s    r.   Úget_input_embeddingszEsmModel.get_input_embeddings¤  s   € ØŒÔ.Ð.r0   c                 ó   — || j         _        d S rƒ   r|  )rc   rÖ   s     r.   Úset_input_embeddingszEsmModel.set_input_embeddings§  s   € Ø*/ˆŒÔ'Ð'Ð'r0   NrÇ   rÈ   r|   rÉ   rò   ró   rp   rg   c                 óö  — |du |duz  rt          d¦  «        ‚|€|                      |||¬¦  «        }|                      ||||d¬¦  «        \  }}| j        dk    rT|€;|j        d         }t          j        ||j        ¬¦  «                             d¦  «        }|  	                    ||¦  «        }	nd}	 | j
        |f||||	d	œ|¤Ž}
|
d         }| j        �|                      |¦  «        nd}t          ||¬
¦  «        S )aV  
        input_ids (`torch.LongTensor` of shape `((batch_size, sequence_length))`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        position_ids (`torch.LongTensor` of shape `((batch_size, sequence_length))`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        inputs_embeds (`torch.FloatTensor` of shape `((batch_size, sequence_length), hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        Nz:You must specify exactly one of input_ids or inputs_embeds)rÇ   rÈ   r|   )rÈ   ró   Úembedding_outputrò   Úpast_key_valuesrõ   r!   )rW   r   r  )r<  Úpooler_output)ré   rÊ   Ú_create_attention_masksr®   r(   r)   ro   rW   r4   rf  rg  rh  r   )rc   rÇ   rÈ   r|   rÉ   rò   ró   rp   Úseq_lenr¾   Úencoder_outputsÚsequence_outputrG  s                r.   r‚   zEsmModel.forwardª  sk  € ð> ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ ð !ŸOšOØ#Ø-Ø)ð ,ñ ô ˆMð 26×1MÒ1MØ)Ø#9Ø*Ø"7Ø ð 2Nñ 2
ô 2
Ñ.ˆÐ.ð Ô'¨8Ò3Ð3ØÐ#Ø'Ô-¨aÔ0�Ý$œ|¨G¸MÔ<PÐQÑQÔQ×[Ò[Ð\]Ñ^Ô^�Ø"&×"8Ò"8¸ÈÑ"UÔ"UÐÐà"&Ðà&˜$œ,Øð
à)Ø"7Ø#9Ø 3ð
ð 
ð ð
ð 
ˆð *¨!Ô,ˆØ8<¼Ð8O˜Ÿš OÑ4Ô4Ð4ÐUYˆå;Ø-Ø'ð
ñ 
ô 
ð 	
r0   c                 ó¶   — | j         j        rt          | j         |||¬¦  «        }nt          | j         ||¬¦  «        }|�t          | j         |||¬¦  «        }||fS )N)rY   rÉ   rÈ   r‚  )rY   rÉ   rÈ   )rY   rÉ   rÈ   rò   )rY   rí   r   r   )rc   rÈ   ró   r�  rò   r‚  s         r.   r„  z EsmModel._create_attention_masksö  s�   € ð Œ;Ô!ð 	Ý/Ø”{Ø.Ø-Ø /ð	ñ ô ˆNˆNõ 7Ø”{Ø.Ø-ðñ ô ˆNð "Ð-Ý%>Ø”{Ø.Ø5Ø&;ð	&ñ &ô &Ð"ð Ð5Ð5Ð5r0   c                 óz  —  | ||dd¬¦  «        j         }t          j        |d¬¦  «        }||                     d¦  «                             d¦  «                             d¦  «        z  }||                     d¦  «                             d¦  «                             d¦  «        z  }|                      ||¦  «        S )NT)rÈ   Úreturn_dictÚoutput_attentionsr!   r&   r%   r   é   )r    r)   Ústackr4   ri  )rc   rŸ   rÈ   Úattnss       r.   Úpredict_contactszEsmModel.predict_contacts  s²   € Ø��V¨NÈÐ`dÐeÑeÔeÔpˆÝ”˜E qÐ)Ñ)Ô)ˆð
 	�×)Ò)¨!Ñ,Ô,×6Ò6°qÑ9Ô9×CÒCÀAÑFÔFÑFˆØ�×)Ò)¨!Ñ,Ô,×6Ò6°qÑ9Ô9×CÒCÀAÑFÔFÑFˆØ× Ò  ¨Ñ/Ô/Ð/r0   )T)NNNNNN)r„   r…   r†   r‡   r]   rk  r}  r  r   r    r   r)   rŠ   r   r   r‰   r   r‚   r„  r�  rŒ   r�   s   @r.   rd  rd  e  sz  ø€ € € € € ð
ð 
ðð ð ð ð ð ð6"ð "ð "ð,/ð /ð /ð0ð 0ð 0ð  ØØð *.Ø.2Ø,0Ø-1Ø59Ø6:ðF
ð F
à”< $Ñ&ðF
ð œ tÑ+ðF
ð ”l TÑ)ð	F
ð
 ”| dÑ*ðF
ð  %œ|¨dÑ2ðF
ð !&¤¨tÑ 3ðF
ð Ð+Ô,ðF
ð 
ˆuŒ|Ô	ÐKÑ	KðF
ð F
ð F
ñ „^ñ „_ñ  ÔðF
ðR6ð 6ð 6ð@	0ð 	0ð 	0ð 	0ð 	0ð 	0ð 	0r0   rd  c                   ó  ‡ — e Zd ZddiZˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 dde	j
        dz  de	j        dz  d	e	j
        dz  d
e	j        dz  de	j        dz  de	j        dz  de	j
        dz  dee         deez  fd„¦   «         ¦   «         Zd„ Zˆ xZS )ÚEsmForMaskedLMzlm_head.decoder.weightz%esm.embeddings.word_embeddings.weightc                 ó  •— t          ¦   «                              |¦  «         |j        rt                               d¦  «         t          |d¬¦  «        | _        t          |¦  «        | _        |  	                    ¦   «          d S )NzjIf you want to use `EsmForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention.F©rm  )
r\   r]   rí   ÚloggerÚwarningrd  rJ  rQ  Úlm_headrl  rÁ   s     €r.   r]   zEsmForMaskedLM.__init__&  s|   ø€ Ý‰Œ×Ò˜Ñ Ô Ð àÔð 	Ý�NŠNð1ñô ð õ
 ˜F°eÐ<Ñ<Ô<ˆŒÝ  Ñ(Ô(ˆŒà�ŠÑÔÐÐÐr0   c                 ó   — | j         j        S rƒ   ©r–  ÚdecoderrV  s    r.   rW  z$EsmForMaskedLM.get_output_embeddings4  s   € ØŒ|Ô#Ð#r0   c                 ó   — || j         _        d S rƒ   r˜  )rc   Únew_embeddingss     r.   Úset_output_embeddingsz$EsmForMaskedLM.set_output_embeddings7  s   € Ø-ˆŒÔÐÐr0   NrÇ   rÈ   r|   rÉ   rò   ró   Úlabelsrp   rg   c           	      óp  —  | j         |f|||||dœ|¤Ž}	|	d         }
|                      |
¦  «        }d}|�et          ¦   «         }|                     |j        ¦  «        } ||                     d| j        j        ¦  «        |                     d¦  «        ¦  «        }t          |||	j	        |	j
        ¬¦  «        S )a¢  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        )rÈ   r|   rÉ   rò   ró   r   Nr$   ©ÚlossÚlogitsrñ   r    )rJ  r–  r   r6   rW   r›   rY   r³   r   rñ   r    )rc   rÇ   rÈ   r|   rÉ   rò   ró   r�  rp   Úoutputsr‡  Úprediction_scoresÚmasked_lm_lossÚloss_fcts                 r.   r‚   zEsmForMaskedLM.forward:  så   € ð( �$”(Øð
à)Ø%Ø'Ø"7Ø#9ð
ð 
ð ð
ð 
ˆð " !œ*ˆØ ŸLšL¨Ñ9Ô9ÐàˆØÐÝ'Ñ)Ô)ˆHà—Y’YÐ0Ô7Ñ8Ô8ˆFØ%˜XÐ&7×&<Ò&<¸RÀÄÔAWÑ&XÔ&XÐZ`×ZeÒZeÐfhÑZiÔZiÑjÔjˆNåØØ$Ø!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r0   c                 ó:   — | j                              ||¬¦  «        S )N)rÈ   )rJ  r�  )rc   rŸ   rÈ   s      r.   r�  zEsmForMaskedLM.predict_contactsh  s   € ØŒx×(Ò(¨ÀÐ(ÑOÔOÐOr0   )NNNNNNN)r„   r…   r†   Ú_tied_weights_keysr]   rW  rœ  r   r   r)   Ú
LongTensorrŠ   rÿ   r   r   r‰   r   r‚   r�  rŒ   r�   s   @r.   r‘  r‘  "  sY  ø€ € € € € à2Ð4[Ð\Ððð ð ð ð ð$ð $ð $ð.ð .ð .ð Øð .2Ø.2Ø04Ø26Ø:>Ø6:Ø*.ð*
ð *
àÔ# dÑ*ð*
ð œ tÑ+ð*
ð Ô&¨Ñ-ð	*
ð
 Ô(¨4Ñ/ð*
ð  %Ô0°4Ñ7ð*
ð !&¤¨tÑ 3ð*
ð Ô  4Ñ'ð*
ð Ð+Ô,ð*
ð 
�Ñ	ð*
ð *
ð *
ñ „^ñ Ôð*
ðXPð Pð Pð Pð Pð Pð Pr0   r‘  c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )rQ  z&ESM Head for masked language modeling.c                 ó†  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t          j        |j        |j	        d¬¦  «        | _
        t          j        t          j        |j	        ¦  «        ¦  «        | _        d S )Nr¬   F)r—   )r\   r]   r   r“   rm   r  r·   r¸   r¹   r³   r™  Ú	Parameterr)   Úzerosr—   rÁ   s     €r.   r]   zEsmLMHead.__init__o  sŽ   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ýœ, vÔ'9¸vÔ?TÐUÑUÔUˆŒå”y Ô!3°VÔ5FÈUÐSÑSÔSˆŒÝ”L¥¤¨VÔ->Ñ!?Ô!?Ñ@Ô@ˆŒ	ˆ	ˆ	r0   c                 ó²   — |                       |¦  «        }t          |¦  «        }|                      |¦  «        }|                      |¦  «        | j        z   }|S rƒ   )r  rF   r¹   r™  r—   ©rc   Úfeaturesrp   r+   s       r.   r‚   zEsmLMHead.forwardw  sL   € Ø�JŠJ�xÑ Ô ˆÝ�‰GŒGˆØ�OŠO˜AÑÔˆð �LŠL˜‰OŒO˜dœiÑ'ˆØˆr0   ©r„   r…   r†   r‡   r]   r‚   rŒ   r�   s   @r.   rQ  rQ  l  sR   ø€ € € € € Ø0Ð0ðAð Að Að Að Aðð ð ð ð ð ð r0   rQ  z›
    ESM Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled
    output) e.g. for GLUE tasks.
    )Úcustom_introc                   óÐ   ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej	        dz  dej        dz  de
e         d	eez  fd
„¦   «         ¦   «         Zˆ xZS )ÚEsmForSequenceClassificationc                 óì   •— t          ¦   «                              |¦  «         |j        | _        || _        t	          |d¬¦  «        | _        t          |¦  «        | _        |                      ¦   «          d S ©NFr“  )	r\   r]   Ú
num_labelsrY   rd  rJ  ÚEsmClassificationHeadÚ
classifierrl  rÁ   s     €r.   r]   z%EsmForSequenceClassification.__init__ˆ  se   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒØˆŒå˜F°eÐ<Ñ<Ô<ˆŒÝ/°Ñ7Ô7ˆŒà�ŠÑÔÐÐÐr0   NrÇ   rÈ   r|   rÉ   r�  rp   rg   c                 óŒ  —  | j         |f|||dœ|¤Ž}|d         }|                      |¦  «        }	d}
|��t|                     |	j        ¦  «        }| j        j        €f| j        dk    rd| j        _        nN| j        dk    r7|j        t          j	        k    s|j        t          j
        k    rd| j        _        nd| j        _        | j        j        dk    rWt          ¦   «         }| j        dk    r1 ||	                     ¦   «         |                     ¦   «         ¦  «        }
nŽ ||	|¦  «        }
n�| j        j        dk    rGt          ¦   «         } ||	                     d| j        ¦  «        |                     d¦  «        ¦  «        }
n*| j        j        dk    rt          ¦   «         } ||	|¦  «        }
t!          |
|	|j        |j        ¬	¦  «        S )
a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        ©rÈ   r|   rÉ   r   Nr!   r”   Úsingle_label_classificationÚmulti_label_classificationr$   rŸ  )rJ  r¸  r6   rW   rY   Úproblem_typer¶  r3   r)   rÏ   r§   r   rž   r   r›   r   r   rñ   r    ©rc   rÇ   rÈ   r|   rÉ   r�  rp   r¢  r‡  r¡  r   r¥  s               r.   r‚   z$EsmForSequenceClassification.forward’  s×  € ð$ �$”(Øð
à)Ø%Ø'ð	
ð 
ð
 ð
ð 
ˆð " !œ*ˆØ—’ Ñ1Ô1ˆàˆØÑØ—Y’Y˜vœ}Ñ-Ô-ˆFàŒ{Ô'Ð/Ø”? aÒ'Ð'Ø/;�D”KÔ,Ð,Ø”_ qÒ(Ð(¨f¬l½e¼jÒ.HÐ.HÈFÌLÕ\aÔ\eÒLeÐLeØ/L�D”KÔ,Ð,à/K�D”KÔ,àŒ{Ô'¨<Ò7Ð7Ý"™9œ9�Ø”? aÒ'Ð'Ø#˜8 F§N¢NÑ$4Ô$4°f·n²nÑ6FÔ6FÑGÔG�D�Dà#˜8 F¨FÑ3Ô3�D�DØ”Ô)Ð-JÒJÐJÝ+Ñ-Ô-�Ø�x §¢¨B°´Ñ @Ô @À&Ç+Â+ÈbÁ/Ä/ÑRÔR��Ø”Ô)Ð-IÒIÐIÝ,Ñ.Ô.�Ø�x ¨Ñ/Ô/�å'ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r0   ©NNNNN)r„   r…   r†   r]   r   r   r)   r¨  rŠ   rÿ   r   r   r‰   r   r‚   rŒ   r�   s   @r.   r³  r³  �  sð   ø€ € € € € ðð ð ð ð ð Øð .2Ø.2Ø04Ø26Ø*.ð8
ð 8
àÔ# dÑ*ð8
ð œ tÑ+ð8
ð Ô&¨Ñ-ð	8
ð
 Ô(¨4Ñ/ð8
ð Ô  4Ñ'ð8
ð Ð+Ô,ð8
ð 
Ð)Ñ	)ð8
ð 8
ð 8
ñ „^ñ Ôð8
ð 8
ð 8
ð 8
ð 8
r0   r³  c                   óÐ   ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej	        dz  dej        dz  de
e         d	eez  fd
„¦   «         ¦   «         Zˆ xZS )ÚEsmForTokenClassificationc                 ó:  •— t          ¦   «                              |¦  «         |j        | _        t          |d¬¦  «        | _        t          j        |j        ¦  «        | _        t          j	        |j
        |j        ¦  «        | _        |                      ¦   «          d S rµ  )r\   r]   r¶  rd  rJ  r   rº   r»   r¼   r“   rm   r¸  rl  rÁ   s     €r.   r]   z"EsmForTokenClassification.__init__Ñ  s|   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒå˜F°eÐ<Ñ<Ô<ˆŒÝ”z &Ô"<Ñ=Ô=ˆŒÝœ) FÔ$6¸Ô8IÑJÔJˆŒà�ŠÑÔÐÐÐr0   NrÇ   rÈ   r|   rÉ   r�  rp   rg   c                 óŒ  —  | j         |f|||dœ|¤Ž}|d         }|                      |¦  «        }|                      |¦  «        }	d}
|�`t          ¦   «         }|                     |	j        ¦  «        } ||	                     d| j        ¦  «        |                     d¦  «        ¦  «        }
t          |
|	|j	        |j
        ¬¦  «        S )zÛ
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        rº  r   Nr$   rŸ  )rJ  r¼   r¸  r   r6   rW   r›   r¶  r   rñ   r    r¾  s               r.   r‚   z!EsmForTokenClassification.forwardÛ  sá   € ð  �$”(Øð
à)Ø%Ø'ð	
ð 
ð
 ð
ð 
ˆð " !œ*ˆàŸ,š, Ñ7Ô7ˆØ—’ Ñ1Ô1ˆàˆØÐÝ'Ñ)Ô)ˆHà—Y’Y˜vœ}Ñ-Ô-ˆFØ�8˜FŸKšK¨¨D¬OÑ<Ô<¸f¿kºkÈ"¹o¼oÑNÔNˆDå$ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r0   r¿  )r„   r…   r†   r]   r   r   r)   r¨  rŠ   rÿ   r   r   r‰   r   r‚   rŒ   r�   s   @r.   rÁ  rÁ  Ï  sð   ø€ € € € € ðð ð ð ð ð Øð .2Ø.2Ø04Ø26Ø*.ð'
ð '
àÔ# dÑ*ð'
ð œ tÑ+ð'
ð Ô&¨Ñ-ð	'
ð
 Ô(¨4Ñ/ð'
ð Ô  4Ñ'ð'
ð Ð+Ô,ð'
ð 
Ð&Ñ	&ð'
ð '
ð '
ñ „^ñ Ôð'
ð '
ð '
ð '
ð '
r0   rÁ  c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )r·  z-Head for sentence-level classification tasks.c                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        ¦  «        | _        t          j        |j        |j	        ¦  «        | _
        d S rƒ   )r\   r]   r   r“   rm   r  rº   r»   r¼   r¶  Úout_projrÁ   s     €r.   r]   zEsmClassificationHead.__init__
  sc   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ý”z &Ô"<Ñ=Ô=ˆŒÝœ	 &Ô"4°fÔ6GÑHÔHˆŒˆˆr0   c                 óô   — |d d …dd d …f         }|                       |¦  «        }|                      |¦  «        }t          j        |¦  «        }|                       |¦  «        }|                      |¦  «        }|S rE  )r¼   r  r)   ÚtanhrÆ  r®  s       r.   r‚   zEsmClassificationHead.forward  sj   € Ø�Q�Q�Q˜˜1˜1˜1�WÔˆØ�LŠL˜‰OŒOˆØ�JŠJ�q‰MŒMˆÝŒJ�q‰MŒMˆØ�LŠL˜‰OŒOˆØ�MŠM˜!ÑÔˆØˆr0   r°  r�   s   @r.   r·  r·    sR   ø€ € € € € Ø7Ð7ðIð Ið Ið Ið Iðð ð ð ð ð ð r0   r·  c                 óÖ   — |                       |¦  «                             ¦   «         }t          j        |d¬¦  «                             |¦  «        |z  }|                     ¦   «         |z   S )a  
    Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
    are ignored. This is modified from fairseq's `utils.make_positions`.

    Args:
        x: torch.Tensor x:

    Returns: torch.Tensor
    r!   r&   )r™   r§   r)   ÚcumsumÚtype_asrÏ   )rÇ   r«   ÚmaskÚincremental_indicess       r.   rÄ   rÄ     s`   € ð �<Š<˜Ñ$Ô$×(Ò(Ñ*Ô*€DÝœ, t°Ð3Ñ3Ô3×;Ò;¸DÑAÔAÀDÑHÐØ×#Ò#Ñ%Ô%¨Ñ3Ð3r0   )r‘  r³  rÁ  rd  rI  )r!   )NrÃ   )Qr‡   rC   Úcollections.abcr   r)   r   Útorch.nnr   r   r   Ú r	   rR  Úintegrationsr
   r   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   Úmodeling_rope_utilsr   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   r    Úconfiguration_esmr"   Ú
get_loggerr„   r”  r/   r?   rF   rJ   rT   ÚModulerV   r�   r©   rŠ   r5   râ   rä   r  r  r  r  r  r/  rA  rI  rd  r‘  rQ  r³  rÁ  r·  rÄ   Ú__all__r2  r0   r.   ú<module>rà     s¦  ðð Ð à €€€Ø $Ð $Ð $Ð $Ð $Ð $à €€€Ø Ð Ð Ð Ð Ð Ø AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aà &Ð &Ð &Ð &Ð &Ð &Ø LÐ LÐ LÐ LÐ LÐ LÐ LÐ LØ JÐ JÐ JÐ JÐ JÐ JÐ JÐ JØ 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð ð ð 7Ð 6Ð 6Ð 6Ð 6Ð 6Ø FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RØ 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ (Ð (Ð (Ð (Ð (Ð (ð 
ˆÔ	˜HÑ	%Ô	%€ð(ð (ð (ð ÐÐ-Ñ.Ô.ðBð Bð Bñ /Ô.ðBð4;ð ;ð ;ð#ð #ð #ð
	ð 	ð 	ð6<ð 6<ð 6<ð 6<ð 6<˜œñ 6<ô 6<ð 6<ðr Gð  Gð  Gð  Gð  G˜rœyñ  Gô  Gð  GðFZ=ð Z=ð Z=ð Z=ð Z=�B”Iñ Z=ô Z=ð Z=ðH !Øð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð �T‰\ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð8 ÐÐ)Ñ*Ô*ðK)ð K)ð K)ð K)ð K)�r”yñ K)ô K)ñ +Ô*ðK)ð\
ð 
ð 
ð 
ð 
�B”Iñ 
ô 
ð 
ðð ð ð ð �2”9ñ ô ð ð<ð ð ð ð �b”iñ ô ð ð
ð 
ð 
ð 
ð 
�”	ñ 
ô 
ð 
ð7ð 7ð 7ð 7ð 7Ð)ñ 7ô 7ð 7ðtSð Sð Sð Sð S�”ñ Sô Sð SðFð ð ð ð �”	ñ ô ð ð ð#ð #ð #ð #ð #˜ñ #ô #ñ „ð#ðL ðy0ð y0ð y0ð y0ð y0Ð!ñ y0ô y0ñ „ðy0ðx ðFPð FPð FPð FPð FPÐ'ñ FPô FPñ „ðFPðRð ð ð ð �”	ñ ô ð ð* €ððñ ô ðE
ð E
ð E
ð E
ð E
Ð#5ñ E
ô E
ñô ðE
ðP ð4
ð 4
ð 4
ð 4
ð 4
Ð 2ñ 4
ô 4
ñ „ð4
ðnð ð ð ð ˜BœIñ ô ð ð&4ð 4ð 4ð ð ð €€€r0   