§
    9²¸jöœ  ã                   óB  — d dl mZ d dlmZ d dlZd dlmc mZ d dlmZ d dl	m
Z
 ddlmZ ddlmZmZ dd	lmZ dd
lmZmZ ddlmZmZmZ ddlmZ ddlmZmZ ddlmZm Z  ddl!m"Z"m#Z# ddl$m%Z% ddl&m'Z'm(Z(m)Z) ddl*m+Z+ ddl,m-Z-m.Z. ddl/m0Z0m1Z1 ddl2m3Z3  G d„ dej4        ¦  «        Z5 ed¦  «         G d„ dej4        ¦  «        ¦   «         Z6dej7        de8dej7        fd„Z9 G d „ d!ej4        ¦  «        Z: G d"„ d#ej4        ¦  «        Z;d$„ Z<	 dGd&ej4        d'ej7        d(ej7        d)ej7        d*ej7        dz  d+e=d,e=d-e%e'         fd.„Z>dHd/„Z? G d0„ d1ej4        ¦  «        Z@ G d2„ d3e¦  «        ZA G d4„ d5ej4        ¦  «        ZB G d6„ d7ej4        ¦  «        ZC G d8„ d9ej4        ¦  «        ZDe G d:„ d;ej4        ¦  «        ¦   «         ZE G d<„ d=ej4        ¦  «        ZFe( G d>„ d?e#¦  «        ¦   «         ZGe( G d@„ dAeG¦  «        ¦   «         ZH e(dB¬C¦  «         G dD„ dEeGe¦  «        ¦   «         ZIg dF¢ZJdS )Ié    )ÚCallable)ÚAnyN)Únn)Úinité   )ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_experts_implementationÚuse_kernel_forward_from_hub)Úcreate_causal_maskÚcreate_recurrent_attention_maskÚ!create_sliding_window_causal_mask)ÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )Ú
ZayaConfigc                   óú   ‡ — e Zd Z edd¬¦  «        ddefˆ fd„¦   «         Ze edd¬¦  «        	 ddededz  dee	j
        ef         fd	„¦   «         ¦   «         Z e	j        ¦   «         ed
„ ¦   «         ¦   «         Zˆ xZS )ÚZayaRotaryEmbeddingÚdevicez5.18)ÚversionNÚconfigc           	      ó¶  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        t          t          |j        ¦  «        ¦  «        | _        i | _	        | j        D ]ã}| j        j
        |         }|€Œ|d         | j	        |<   | j        }| j	        |         dk    rt          | j	        |                  } || j        ||¬¦  «        \  }}t          | |› d�t          j        |d¬¦  «        ¦  «         t          | |› d�t          j        |                     ¦   «         d¬¦  «        ¦  «         t          | |› d�|¦  «         Œäd S )	NÚ	rope_typeÚdefault©Ú
layer_typeÚ	_inv_freqF)Ú
persistentÚ_original_inv_freqÚ_attention_scaling)ÚsuperÚ__init__Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr'   ÚsortedÚsetÚlayer_typesr)   Úrope_parametersÚcompute_default_rope_parametersr   Úsetattrr   ÚBufferÚclone)	Úselfr'   r%   r,   Úrope_paramsÚrope_init_fnÚcurr_inv_freqÚcurr_attention_scalingÚ	__class__s	           €úv/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/zaya/modeling_zaya.pyr2   zZayaRotaryEmbedding.__init__0   sl  ø€ å‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!ØˆŒÝ!¥# fÔ&8Ñ"9Ô"9Ñ:Ô:ˆÔØˆŒØÔ*ð 	Uð 	UˆJØœ+Ô5°jÔAˆKØÐ"Øà)4°[Ô)AˆDŒN˜:Ñ&Ø%)Ô%IˆLØŒ~˜jÔ)¨YÒ6Ð6Ý2°4´>À*Ô3MÔN�Ø4@°LÀÄÈfÐakÐ4lÑ4lÔ4lÑ1ˆMÐ1Ý�D˜ZÐ2Ð2Ð2µB´I¸mÐX]Ð4^Ñ4^Ô4^Ñ_Ô_Ð_Ý�D˜ZÐ;Ð;Ð;½R¼YÀ}×GZÒGZÑG\ÔG\ÐinÐ=oÑ=oÔ=oÑpÔpÐpÝ�D˜ZÐ;Ð;Ð;Ð=SÑTÔTÐTÐTð	Uð 	Uó    r,   Úreturnc                 óV  — | j         |         d         }| j         |                              dd¦  «        }t          | dd¦  «        p| j        | j        z  }t          ||z  ¦  «        }d}d|t          j        d|dt          j        ¬¦  «        |z  z  z  }	|	 	                    |¦  «        |fS )	a‘  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            layer_type (`str`, *optional*):
                The current layer type if the model has different RoPE parameters per type.
                Should not be used unless `config.layer_types is not None`
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚpartial_rotary_factorg      ð?Úhead_dimNr   é   ©Údtype)
r9   ÚgetÚgetattrÚhidden_sizeÚnum_attention_headsÚintÚtorchÚarangeÚfloatÚto)
r'   r%   r,   ÚkwargsÚbaserI   rJ   ÚdimÚattention_factorÚinv_freqs
             rD   r:   z3ZayaRotaryEmbedding.compute_default_rope_parametersF   s´   € ð" Ô% jÔ1°,Ô?ˆà &Ô 6°zÔ B× FÒ FÐG^Ð`cÑ dÔ dÐÝ˜6 :¨tÑ4Ô4Ðh¸Ô8JÈfÔNhÑ8hˆÝ�(Ð2Ñ2Ñ3Ô3ˆàÐð ˜$¥5¤<°°3¸ÅÄÐ#MÑ#MÔ#MÐPSÑ#SÑTÑUˆØ�{Š{˜6Ñ"Ô"Ð$4Ð4Ð4rE   c                 ó(  — t          | |› d�¦  «        }t          | |› d�¦  «        }|d d d …d f                              |j        d         dd¦  «                             t          j        |j        ¬¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d	¬
¦  «        5  ||z                       dd¦  «        }	t	          j        |	|	fd¬¦  «        }
|
                     ¦   «         |z  }|
                     ¦   «         |z  }d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬¦  «        |                     |j        ¬¦  «        fS )Nr-   r0   r   éÿÿÿÿr!   )rM   r%   ÚmpsÚcpuF)Údevice_typeÚenabledrK   ©rY   rL   )rO   ÚexpandÚshaperV   rS   rU   r%   Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosÚsinrM   )r>   ÚxÚposition_idsr,   r[   Úattention_scalingÚinv_freq_expandedÚposition_ids_expandedr`   ÚfreqsÚembrj   rk   s                rD   ÚforwardzZayaRotaryEmbedding.forwardc   sÈ  € õ ˜4 JÐ!9Ð!9Ð!9Ñ:Ô:ˆÝ# D¨ZÐ*KÐ*KÐ*KÑLÔLÐð �T˜1˜1˜1˜d�]Ô#×*Ò*¨<Ô+=¸aÔ+@À"ÀaÑHÔH×KÒKÕRWÔR]ÐfgÔfnÐKÑoÔoð 	ð !-¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆå¨¸UÐCÑCÔCð 	0ð 	0Ø&Ð)>Ñ>×IÒIÈ!ÈQÑOÔOˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)Ð/Ñ/ˆCØ—'’'‘)”)Ð/Ñ/ˆCð		0ð 	0ð 	0ñ 	0ô 	0ð 	0ð 	0ð 	0ð 	0ð 	0ð 	0øøøð 	0ð 	0ð 	0ð 	0ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   Ã'A EÅEÅE©N)NN)Ú__name__Ú
__module__Ú__qualname__r   r"   r2   Ústaticmethodrg   ÚtuplerS   ÚTensorrU   r:   Úno_gradr   rs   Ú__classcell__©rC   s   @rD   r$   r$   /   s  ø€ € € € € Ø€_�X vÐ.Ñ.Ô.ðUð U˜zð Uð Uð Uð Uð Uñ /Ô.ðUð* Ø€_�X vÐ.Ñ.Ô.àBFð5ð 5Øð5Ø58¸4±Zð5à	ˆuŒ|˜UÐ"Ô	#ð5ð 5ð 5ñ /Ô.ñ „\ð5ð6 €U„]�_„_Øð<ð <ñ Ôñ „_ð<ð <ð <ð <ð <rE   r$   ÚRMSNormc                   óT   ‡ — e Zd Zd	deddfˆ fd„Zdej        dej        fd„Zd„ Zˆ xZ	S )
ÚZayaRMSNormç�íµ ÷Æ°>ÚepsrF   Nc                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )z:
        ZayaRMSNorm is equivalent to T5LayerNorm
        N)r1   r2   r   Ú	ParameterrS   ÚonesÚweightÚvariance_epsilon)r>   rP   r‚   rC   s      €rD   r2   zZayaRMSNorm.__init__{   sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐrE   Úhidden_statesc                 ó  — |j         }|                     t          j        ¦  «        }|                     d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        |                     |¦  «        z  S )NrK   r]   T)Úkeepdim)	rM   rV   rS   Úfloat32ÚpowÚmeanÚrsqrtr‡   r†   )r>   rˆ   Úinput_dtypeÚvariances       rD   rs   zZayaRMSNorm.forwardƒ   s|   € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØ ×$Ò$ QÑ'Ô'×,Ò,¨R¸Ð,Ñ>Ô>ˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆØŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:rE   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)ry   r†   rd   r‡   )r>   s    rD   Ú
extra_reprzZayaRMSNorm.extra_reprŠ   s&   € Ý˜œÔ)Ñ*Ô*ÐIÐI°$Ô2GÐIÐIÐIrE   )r�   )
ru   rv   rw   rU   r2   rS   rz   rs   r’   r|   r}   s   @rD   r€   r€   y   sŒ   ø€ € € € € ð$ð $¨ð $¸$ð $ð $ð $ð $ð $ð $ð; U¤\ð ;°e´lð ;ð ;ð ;ð ;ðJð Jð Jð Jð Jð Jð JrE   r€   rˆ   Ún_reprF   c                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r!   N)rd   rc   Úreshape)rˆ   r“   ÚbatchÚnum_key_value_headsÚslenrJ   s         rD   Ú	repeat_kvr™   Ž   s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTrE   c                   ód   ‡ — e Zd ZdZdedefˆ fd„Z	 d
dej        de	dz  dej        dz  fd	„Z
ˆ xZS )ÚZayaCCAProjectionav  
    Projects hidden states into attention q/k/v states with ZAYA's Compressed Convolutional Attention (CCA) path.
    See https://huggingface.co/papers/2510.04476.

    This follows the usual q/k/v projection flow, with three ZAYA-specific changes: q/k are mixed by a causal 1D
    convolution, q/k keep residual projection paths, and v uses a delayed recurrent state.
    r'   Ú	layer_idxc                 ó¤  •— t          ¦   «                              ¦   «          || _        || _        |j        | _        |j        | _        |j        | _        | j        dz
  | j        dz
  z   | _	        |j
        | _
        |j        | _        |j        | _        | j        | j
        z  | _        | j        | j        z  }| j
        | j        z  }t          j        | j        || j        j        ¬¦  «        | _        t          j        | j        || j        j        ¬¦  «        | _        t          j        | j        |dz  | j        j        ¬¦  «        | _        t          j        | j        |dz  | j        j        ¬¦  «        | _        ||z   }t          j        ||| j        |dd¬¦  «        | _        t          j        ||| j        | j
        | j        z   dd¬¦  «        | _        d S )Nr!   ©ÚbiasrK   r   )Úin_channelsÚout_channelsÚkernel_sizeÚgroupsÚpaddingÚstride)r1   r2   r'   rœ   rP   Ú	cca_time0Údepthwise_kernel_sizeÚ	cca_time1Úgrouped_kernel_sizeÚconv_kernel_sizer—   rQ   rJ   Únum_key_value_groupsr   ÚLinearÚattention_biasÚq_projÚk_projÚv_proj_currentÚv_proj_delayedÚConv1dÚconv_qk_depthwiseÚconv_qk_grouped)r>   r'   rœ   Úquery_hidden_sizeÚkey_value_hidden_sizeÚconv_channelsrC   s         €rD   r2   zZayaCCAProjection.__init__£   sÈ  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒà!Ô-ˆÔà%+Ô%5ˆÔ"Ø#)Ô#3ˆÔ Ø!%Ô!;¸aÑ!?ÀDÔD\Ð_`ÑD`Ñ aˆÔà#)Ô#=ˆÔ Ø#)Ô#=ˆÔ ØœˆŒØ$(Ô$<ÀÔ@XÑ$XˆÔ!à Ô4°t´}ÑDÐØ $Ô 8¸4¼=Ñ HÐå”i Ô 0Ð2CÈ$Ì+ÔJdÐeÑeÔeˆŒÝ”i Ô 0Ð2GÈdÌkÔNhÐiÑiÔiˆŒÝ œi¨Ô(8Ð:OÐSTÑ:TÐ[_Ô[fÔ[uÐvÑvÔvˆÔÝ œi¨Ô(8Ð:OÐSTÑ:TÐ[_Ô[fÔ[uÐvÑvÔvˆÔà-Ð0AÑAˆÝ!#¤Ø%Ø&ØÔ2Ø ØØð"
ñ "
ô "
ˆÔõ  "œyØ%Ø&ØÔ0ØÔ,¨tÔ/GÑGØØð 
ñ  
ô  
ˆÔÐÐrE   Nrˆ   Úpast_key_valuesÚ	conv_maskc                 ó  — |�*||d d …d d …d f                               |j        ¦  «        z  }|j        d d…         }g |¢d‘| j        ‘R }|                      |¦  «        }|                      |¦  «        }t          j        ||gd¬¦  «        } |j        |Ž }	 |j        |Ž  	                    dd¦  «        }
t          |
| j        ¦  «         	                    dd¦  «        }
|	|
z   dz  }	 |	j        g |¢d‘| j        ‘| j        ‘R Ž                      d¬¦  «        }
| 	                    dd¦  «        }|d uo|                     | j        ¦  «        }|r6|j        | j                 j        d         }t          j        ||gd¬¦  «        }nt#          j        || j        df¦  «        }|�W|d| j         d …f         }t#          j        || j        |j        d         z
  df¦  «        }|                     || j        ¦  «         |                      |¦  «        }|                      |¦  «         	                    dd¦  «        }|	j        d         |	j        d         z  } |dd |…f         j        |Ž |	z   } |d|d …f         j        |Ž |
z   }|                      |¦  «        }|                      |¦  «        }|r1|j        | j                 j        d                              d¦  «        }n5|                      |                     |d         d| j        ¦  «        ¦  «        }t          j        ||d d …d d…f         gd¬¦  «        }|�(|                     |d d …dd d …f         | j        ¦  «          t          j        ||gd¬¦  «        j        |Ž }|||fS )	Nr]   rb   r!   rK   ç      à?éþÿÿÿr   .)rV   rM   rd   rJ   r®   r¯   rS   ri   Úviewrh   r™   r«   r�   Úhas_previous_staterœ   ÚlayersÚconv_statesÚFÚpadrª   Úupdate_conv_stater³   r´   r°   r±   Úrecurrent_statesÚ	unsqueezeÚ	new_zerosrP   Úupdate_recurrent_state)r>   rˆ   r¸   r¹   Úinput_shapeÚhidden_shapeÚprojected_queriesÚprojected_keysÚ	qk_statesÚquery_residualÚkey_residualÚuse_precomputed_statesÚcached_qk_statesÚnew_conv_staterµ   ÚqueryÚkeyÚvalue_currentÚdelayed_v_stateÚrecurrent_v_stateÚvalue_delayedÚvalues                         rD   rs   zZayaCCAProjection.forwardÍ   sä  € ð Ð Ø)¨I°a°a°a¸¸¸¸D°jÔ,A×,DÒ,DÀ]ÔEXÑ,YÔ,YÑYˆMà#Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà ŸKšK¨Ñ6Ô6ÐØŸš ]Ñ3Ô3ˆÝ”IÐ0°.ÐAÀrÐJÑJÔJˆ	à/Ð*Ô/°Ð>ˆØ*�~Ô*¨LÐ9×CÒCÀAÀqÑIÔIˆÝ  ¨tÔ/HÑIÔI×SÒSÐTUÐWXÑYÔYˆØ(¨<Ñ7¸3Ñ>ˆØ*�~Ô*Ðf¨KÐf¸Ðf¸TÔ=VÐfÐX\ÔXeÐfÐfÐf×kÒkÐprÐkÑsÔsˆà×'Ò'¨¨1Ñ-Ô-ˆ	Ø!0¸Ð!<Ð!sÀ×AcÒAcÐdhÔdrÑAsÔAsÐØ!ð 	EØ.Ô5°d´nÔEÔQÐRSÔTÐÝœ	Ð#3°YÐ"?ÀRÐHÑHÔHˆIˆIåœ˜i¨$Ô*?ÀÐ)CÑDÔDˆIàÐ&Ø& s¨TÔ-BÐ,BÐ,DÐ,DÐ'DÔEˆNÝœU >°DÔ4IÈNÔL`ÐacÔLdÑ4dÐfgÐ3hÑiÔiˆNØ×-Ò-¨n¸d¼nÑMÔMÐMà×*Ò*¨9Ñ5Ô5ˆ	Ø×(Ò(¨Ñ3Ô3×=Ò=¸aÀÑCÔCˆ	à*Ô0°Ô4°~Ô7KÈBÔ7OÑOÐØ7�	˜#Ð1Ð 1Ð1Ð1Ô2Ô7¸ÐFÈÑWˆØ5ˆi˜Ð.Ð/Ð/Ð/Ô0Ô5°|ÐDÀ|ÑSˆð ×+Ò+¨MÑ:Ô:ˆØ×-Ò-¨mÑ<Ô<ˆØ!ð 	rØ /Ô 6°t´~Ô FÔ WÐXYÔ Z× dÒ dÐefÑ gÔ gÐÐà $× 3Ò 3°M×4KÒ4KÈKÐXYÌNÐ\]Ð_cÔ_oÑ4pÔ4pÑ qÔ qÐÝœ	Ð#4°oÀaÀaÀaÈÈ"ÈÀfÔ6MÐ"NÐTUÐVÑVÔVˆàÐ&Ø×2Ò2°?À1À1À1ÀbÈ!È!È!À8Ô3LÈdÌnÑ]Ô]Ð]àF•”	˜=¨-Ð8¸bÐAÑAÔAÔFÈÐUˆà�c˜5Ð Ð rE   rt   )ru   rv   rw   Ú__doc__r"   rR   r2   rS   rz   r	   rs   r|   r}   s   @rD   r›   r›   š   s�   ø€ € € € € ðð ð(
˜zð (
°cð (
ð (
ð (
ð (
ð (
ð (
ð\ *.ð	9!ð 9!à”|ð9!ð  ™ð9!ð ”< $Ñ&ð	9!ð 9!ð 9!ð 9!ð 9!ð 9!ð 9!ð 9!rE   r›   c                   ót   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        deej        ej        f         fd„Z	ˆ xZ
S )Ú
ZayaQKNormzf
    L2-normalizes q/k states to sqrt(head_dim) and applies ZAYA's learned per-KV-head key scale.
    r'   c                 óÆ   •— t          ¦   «                              ¦   «          |j        dz  | _        t	          j        t          j        |j        ¦  «        ¦  «        | _	        d S )Nr»   )
r1   r2   rJ   Úhead_dim_scaler   r„   rS   Úzerosr—   Útemp©r>   r'   rC   s     €rD   r2   zZayaQKNorm.__init__  sJ   ø€ Ý‰Œ×ÒÑÔÐØ$œo¨sÑ2ˆÔÝ”L¥¤¨VÔ-GÑ!HÔ!HÑIÔIˆŒ	ˆ	ˆ	rE   Úquery_statesÚ
key_statesrF   c                 óJ  — t          j        |j        ¦  «        j        }|| j        |                     ddd¬¦  «                             |¦  «        z  z  }|| j        |                     ddd¬¦  «                             |¦  «        z  z  }|| j        d d d d …d f         z  }||fS )NrK   r]   T)ÚprY   rŠ   )rS   ÚfinforM   r‚   rÝ   ÚnormÚ	clamp_minrß   )r>   rá   râ   Únorm_epss       rD   rs   zZayaQKNorm.forward  s³   € Ý”;˜|Ô1Ñ2Ô2Ô6ˆØ#ØÔ ,×"3Ò"3°a¸RÈÐ"3Ñ"NÔ"N×"XÒ"XÐYaÑ"bÔ"bÑbñ
ˆð  ØÔ *§/¢/°A¸2Àt /Ñ"LÔ"L×"VÒ"VÐW_Ñ"`Ô"`Ñ`ñ
ˆ
ð   $¤)¨D°$¸¸¸¸4Ð,?Ô"@Ñ@ˆ
Ø˜ZÐ'Ð'rE   )ru   rv   rw   rÙ   r"   r2   rS   rz   ry   rs   r|   r}   s   @rD   rÛ   rÛ   	  s”   ø€ € € € € ðð ðJ˜zð Jð Jð Jð Jð Jð Jð
	( E¤Lð 	(¸e¼lð 	(ÈuÐUZÔUaÐchÔcoÐUoÔOpð 	(ð 	(ð 	(ð 	(ð 	(ð 	(ð 	(ð 	(rE   rÛ   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..Nr]   rK   rb   )rd   rS   ri   )rl   Úx1Úx2s      rD   Úrotate_halfrì     s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'rE   ç        ÚmodulerÒ   rÓ   rØ   Úattention_maskÚscalingÚdropoutrW   c                 ó  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        |z  }
|�|
|z   }
t
          j                             |
dt          j        ¬¦  «         	                    |j
        ¦  «        }
t
          j                             |
|| j        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «                             ¦   «         }||
fS )NrK   r   r]   )rY   rM   )rä   Útrainingr!   )r™   r«   rS   Úmatmulrh   r   Ú
functionalÚsoftmaxr‹   rV   rM   rñ   ró   Ú
contiguous)rî   rÒ   rÓ   rØ   rï   rð   rñ   rW   râ   Úvalue_statesÚattn_weightsÚattn_outputs               rD   Úeager_attention_forwardrû   &  sé   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$rE   c                 ó˜  — |                      |¦  «        }|                      |¦  «        }|j        d         }| dd|…f         | d|d…f         }}|dd|…f         |d|d…f         }	}||z  t          |¦  «        |z  z   }
||z  t          |¦  «        |z  z   }t          j        |
|gd¬¦  «        }
t          j        ||	gd¬¦  «        }|
|fS )a»  Applies Rotary Position Embedding to the query and key tensors.

    Removes the interleaving of cos and sin from GLM

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    r]   .Nrb   )rÅ   rd   rì   rS   ri   )ÚqÚkrj   rk   Úunsqueeze_dimÚ
rotary_dimÚq_rotÚq_passÚk_rotÚk_passÚq_embedÚk_embeds               rD   Úapply_rotary_pos_embr  @  sô   € ð( �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€Cð ”˜2”€JØ�c˜;˜J˜;Ð&Ô'¨¨3°
°°Ð+;Ô)<ˆ6€EØ�c˜;˜J˜;Ð&Ô'¨¨3°
°°Ð+;Ô)<ˆ6€Eð �s‰{�{¨5Ñ1Ô1°CÑ7Ñ8€GØ�s‰{�{¨5Ñ1Ô1°CÑ7Ñ8€Gõ Œi˜ &Ð)¨rÐ2Ñ2Ô2€GÝŒi˜ &Ð)¨rÐ2Ñ2Ô2€GØ�GÐÐrE   c                   óÚ   ‡ — e Zd ZdZdedefˆ fd„Z	 	 	 ddej        de	e
ef         dz  dedz  d	eej        ej        f         dz  d
ee         deej        ej        dz  f         fd„Zˆ xZS )ÚZayaAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr'   rœ   c                 óx  •— t          ¦   «                              ¦   «          || _        || _        t	          |d|j        |j        z  ¦  «        | _        |j        |j        z  | _	        |j        | _        | j        dz  | _
        |j        | _        d| _        t          j        |j        | j        z  |j        |j        ¬¦  «        | _        t#          | j        |¬¦  «        | _        |j        |         | _        | j        dk    r|j        nd | _        |j        | _        |j        | _        t-          |¦  «        | _        d S )NrJ   g      à¿Trž   ©r'   rœ   Úhybrid_sliding)r1   r2   r'   rœ   rO   rP   rQ   rJ   r—   r«   rð   Úattention_dropoutÚ	is_causalr   r¬   r­   Úo_projr›   Úqkv_projr8   r,   Úsliding_windowrÛ   Úqk_norm©r>   r'   rœ   rC   s      €rD   r2   zZayaAttention.__init__i  s+  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒÝ ¨
°FÔ4FÈ&ÔJdÑ4dÑeÔeˆŒØ$*Ô$>À&ÔB\Ñ$\ˆÔ!Ø#)Ô#=ˆÔ Ø”} dÑ*ˆŒØ!'Ô!9ˆÔØˆŒå”iØÔ&¨¬Ñ6¸Ô8JÐQWÔQfð
ñ 
ô 
ˆŒõ *Ø”;Øð
ñ 
ô 
ˆŒð !Ô,¨YÔ7ˆŒØ7;´ÐJZÒ7ZÐ7Z˜fÔ3Ð3Ð`dˆÔØ!Ô-ˆÔØ#)Ô#=ˆÔ Ý! &Ñ)Ô)ˆŒˆˆrE   Nrˆ   rï   r¸   Úposition_embeddingsrW   rF   c                 óÎ  — |j         d d…         }|pi }|                     d¦  «        }|                     d¦  «        }	|                      |||	¦  «        \  }
}}|                      |
|¦  «        \  }
}|
                     dd¦  «        }
|                     dd¦  «        }|                     dd¦  «        }|\  }}t          |
|||¦  «        \  }
}|�|                     ||| j        ¦  «        \  }}t          j	        | j
        j        t          ¦  «        } || |
|||f| j        sdn| j        | j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž }|                      |¦  «        }||fS )Nr]   ÚcausalÚconvr!   rK   rí   )rñ   rð   r  )rd   rN   r  r  rh   r  Úupdaterœ   r   Úget_interfacer'   Ú_attn_implementationrû   ró   r  rð   r  r•   r  )r>   rˆ   rï   r¸   r  rW   rÈ   Úmask_mappingÚcausal_maskr¹   rá   râ   rø   rj   rk   Úattention_interfacerú   rù   s                     rD   rs   zZayaAttention.forward�  s·  € ð $Ô)¨#¨2¨#Ô.ˆà%Ð+¨ˆØ"×&Ò& xÑ0Ô0ˆØ ×$Ò$ VÑ,Ô,ˆ	ð 26·²¸}ÈoÐ_hÑ1iÔ1iÑ.ˆ�j ,Ø#'§<¢<°¸jÑ#IÔ#IÑ ˆ�jà#×-Ò-¨a°Ñ3Ô3ˆØ×)Ò)¨!¨QÑ/Ô/ˆ
Ø#×-Ò-¨a°Ñ3Ô3ˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð  $œ}ÐH�C�C°$Ô2HØ”LØÔ.ð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;ˆØ—k’k +Ñ.Ô.ˆà˜LÐ(Ð(rE   )NNN)ru   rv   rw   rÙ   r"   rR   r2   rS   rz   Údictrg   r   r	   ry   r   r   rs   r|   r}   s   @rD   r	  r	  f  sî   ø€ € € € € ØGÐGð*˜zð *°cð *ð *ð *ð *ð *ð *ð6 15Ø(,ØHLð.)ð .)à”|ð.)ð ˜S #˜Xœ¨Ñ-ð.)ð  ™ð	.)ð
 # 5¤<°´Ð#=Ô>ÀÑEð.)ð Ð+Ô,ð.)ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð.)ð .)ð .)ð .)ð .)ð .)ð .)ð .)rE   r	  c                   óì   ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 ddej        dej        dz  dee	e
f         dz  dedz  d	eej        ej        f         dz  d
ee         deej        ej        dz  f         fd„Zˆ xZS )ÚZayaDecoderLayerr'   rœ   c                 óš  •— t          ¦   «                              ¦   «          |j        | _        t          ||¬¦  «        | _        t          ||¦  «        | _        t          |j        |j        ¬¦  «        | _	        t          |j        |j        ¬¦  «        | _
        t          |j        ¦  «        | _        t          |j        ¦  «        | _        d S )Nr  ©r‚   )r1   r2   rP   r	  Ú	self_attnÚZayaSparseMoeBlockÚmlpr€   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormÚZayaResidualScalingÚpost_attention_residual_scaleÚpost_mlp_residual_scaler  s      €rD   r2   zZayaDecoderLayer.__init__³  s«   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔå&¨fÀ	ÐJÑJÔJˆŒÝ% f¨iÑ8Ô8ˆŒÝ*¨6Ô+=À6ÔCVÐWÑWÔWˆÔÝ(3°FÔ4FÈFÔL_Ð(`Ñ(`Ô(`ˆÔ%Ý-@ÀÔASÑ-TÔ-TˆÔ*Ý':¸6Ô;MÑ'NÔ'NˆÔ$Ð$Ð$rE   Nrˆ   Úprev_router_hidden_statesrï   r¸   r  rW   rF   c                 ó¢  — |}|                       |                     | j         j        j        ¬¦  «        ¦  «        } | j        d||||dœ|¤Ž\  }}|                      ||¦  «        }|                      |                     | j        j        j        ¬¦  «        ¦  «        }|                      ||¦  «        \  }}|                      ||¦  «        }||fS )NrL   )rˆ   rï   r¸   r  © )	r'  rV   r†   rM   r#  r*  r(  r%  r+  )	r>   rˆ   r,  rï   r¸   r  rW   ÚresidualÚ_s	            rD   rs   zZayaDecoderLayer.forward¾  sð   € ð !ˆð ×,Ò,¨X¯[ª[¸tÔ?SÔ?ZÔ?`¨[Ñ-aÔ-aÑbÔbˆà)˜4œ>ð 
Ø'Ø)Ø+Ø 3ð	
ð 
ð
 ð
ð 
Ñˆ�qð ×5Ò5°mÀXÑNÔNˆØ×5Ò5°h·k²kÈÔHeÔHlÔHr°kÑ6sÔ6sÑtÔtˆà37·8²8ØØ%ñ4
ô 4
Ñ0ˆÐ0ð
 ×4Ò4°]ÀHÑMÔMˆàÐ7Ð7Ð7rE   )NNNN)ru   rv   rw   r"   rR   r2   rS   rz   r  rg   r   r	   ry   r   r   rs   r|   r}   s   @rD   r   r   ²  s  ø€ € € € € ð	O˜zð 	O°cð 	Oð 	Oð 	Oð 	Oð 	Oð 	Oð :>Ø04Ø(,ØHLð 8ð  8à”|ð 8ð $)¤<°$Ñ#6ð 8ð ˜S #˜Xœ¨Ñ-ð	 8ð
  ™ð 8ð # 5¤<°´Ð#=Ô>ÀÑEð 8ð Ð+Ô,ð 8ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð 8ð  8ð  8ð  8ð  8ð  8ð  8ð  8rE   r   c                   óH   ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Zˆ xZS )r)  rP   c                 ó   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        t          j        t	          j        |¦  «        ¦  «        | _        t          j        t	          j        |¦  «        ¦  «        | _	        t          j        t	          j        |¦  «        ¦  «        | _
        d S rt   )r1   r2   r   r„   rS   r…   Úhidden_states_scalerÞ   Úhidden_states_biasÚresidual_scaleÚresidual_bias)r>   rP   rC   s     €rD   r2   zZayaResidualScaling.__init__â  s�   ø€ Ý‰Œ×ÒÑÔÐÝ#%¤<µ´
¸;Ñ0GÔ0GÑ#HÔ#HˆÔ Ý"$¤,­u¬{¸;Ñ/GÔ/GÑ"HÔ"HˆÔÝ œl­5¬:°kÑ+BÔ+BÑCÔCˆÔÝœ\­%¬+°kÑ*BÔ*BÑCÔCˆÔÐÐrE   rˆ   r/  c                 óT   — || j         z   | j        z  }|| j        z   | j        z  }||z   S rt   )r4  r3  r6  r5  )r>   rˆ   r/  s      rD   rs   zZayaResidualScaling.forwardé  s7   € à&¨Ô)@Ñ@ÀDÔD\Ñ\ˆØ˜tÔ1Ñ1°TÔ5HÑHˆØ˜xÑ'Ð'rE   )	ru   rv   rw   rR   r2   rS   rz   rs   r|   r}   s   @rD   r)  r)  á  sq   ø€ € € € € ðD Cð Dð Dð Dð Dð Dð Dð( U¤\ð (¸U¼\ð (ð (ð (ð (ð (ð (ð (ð (rE   r)  c                   óP   ‡ — e Zd Zdededefˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚZayaRouterMLPrP   Únum_expertsr&  c                 óL  •— t          ¦   «                              ¦   «          t          ||¬¦  «        | _        t	          j        ||d¬¦  «        | _        t	          j        ||d¬¦  «        | _        t	          j        ||d¬¦  «        | _        t	          j	        ¦   «         | _
        d S )Nr"  Trž   F)r1   r2   r€   ræ   r   r¬   Úfc1Úfc2Úout_projÚGELUÚact_fn)r>   rP   r:  r&  rC   s       €rD   r2   zZayaRouterMLP.__init__ñ  s†   ø€ Ý‰Œ×ÒÑÔÐÝ °Ð>Ñ>Ô>ˆŒ	Ý”9˜[¨+¸DÐAÑAÔAˆŒÝ”9˜[¨+¸DÐAÑAÔAˆŒÝœ	 +¨{ÀÐGÑGÔGˆŒÝ”g‘i”iˆŒˆˆrE   rˆ   rF   c                 óö   — |                       |¦  «        }|                      |                      |¦  «        ¦  «        }|                      |                      |¦  «        ¦  «        }|                      |¦  «        S rt   )ræ   r@  r<  r=  r>  )r>   rˆ   s     rD   rs   zZayaRouterMLP.forwardù  s_   € ØŸ	š	 -Ñ0Ô0ˆØŸš D§H¢H¨]Ñ$;Ô$;Ñ<Ô<ˆØŸš D§H¢H¨]Ñ$;Ô$;Ñ<Ô<ˆØ�}Š}˜]Ñ+Ô+Ð+rE   )
ru   rv   rw   rR   rU   r2   rS   rz   rs   r|   r}   s   @rD   r9  r9  ð  sx   ø€ € € € € ð  Cð  °cð  Èð  ð  ð  ð  ð  ð  ð, U¤\ð ,°e´lð ,ð ,ð ,ð ,ð ,ð ,ð ,ð ,rE   r9  c                   ó–   ‡ — e Zd Zdeddfˆ fd„Z	 ddej        dej        dz  deej        ej        ej        ej        f         fd„Zˆ xZ	S )	Ú
ZayaRouterrœ   rF   Nc                 ó’  •— t          ¦   «                              ¦   «          || _        |j        | _        || _        |j        | _        |j        dz   | _        |j        | _        |j	        | _	        t          j        | j        | j	        d¬¦  «        | _        | j        dk    | _        | j        r0t          j        t          j        | j	        ¦  «        ¦  «        | _        t%          | j	        | j        |j        ¦  «        | _        t          j        t          j        | j        t          j        ¬¦  «        ¦  «        | _        d| j        d<   d S )Nr!   Trž   r   rL   ç      ð¿r]   )r1   r2   r'   rP   rœ   r:  Únum_router_classesÚnum_experts_per_tokÚtop_kÚrouter_hidden_sizer   r¬   Ú	down_projÚuse_edar„   rS   r…   Úrouter_states_scaler9  r&  Ú
router_mlpr<   rÞ   r‹   Úbalancing_biasesr  s      €rD   r2   zZayaRouter.__init__  s  ø€ õ
 	‰Œ×ÒÑÔÐàˆŒØ!Ô-ˆÔØ"ˆŒà!Ô-ˆÔà"(Ô"4°qÑ"8ˆÔØÔ/ˆŒ
Ø"(Ô";ˆÔåœ 4Ô#3°TÔ5LÐSWÐXÑXÔXˆŒà”~¨Ò*ˆŒØŒ<ð 	YÝ')¤|µE´J¸tÔ?VÑ4WÔ4WÑ'XÔ'XˆDÔ$å'¨Ô(?ÀÔAXÐZ`ÔZmÑnÔnˆŒå "¤	­%¬+°dÔ6MÕUZÔUbÐ*cÑ*cÔ*cÑ dÔ dˆÔØ$(ˆÔ˜bÑ!Ð!Ð!rE   rˆ   Úrouter_statesc                 óü  — d| j         f}|j        d         }|                      |¦  «        }| j        r|�||| j        z  z   }|d d …| d …f                              ¦   «         }|                      |¦  «        }t          j        |d¬¦  «        }| 	                    ¦   «          
                    t          j        ¦  «        | j        z   }	t          j        |	| j         d¬¦  «        \  }
}t          j        |d|¬¦  «        }|| j        j        k    }|                     |d¦  «        }|                     |d¦  «        }|                     d| j        ¦  «        |                     |¦  «        |                     |¦  «        |fS )Nr]   r!   rb   rK   )rY   Úindexr   )rH  rd   rJ  rK  rL  r=   rM  rS   rö   ÚdetachrV   r‹   rN  ÚtopkÚgatherr'   r:  Úmasked_fillr•   rF  )r>   rˆ   rO  Úfinal_shapeÚ
seq_lengthÚrouter_hidden_statesÚrouter_hidden_states_nextÚrouter_logitsÚrouter_probsÚbiased_router_probsr0  Úrouter_indicesÚskip_experts                rD   rs   zZayaRouter.forward  s}  € ð
 ˜4œ:Ð&ˆØ"Ô(¨Ô+ˆ
à#Ÿ~š~¨mÑ<Ô<ÐàŒ<ð 	c˜MÐ5Ø#7¸-È$ÔJbÑ:bÑ#bÐ à$8¸¸¸¸Z¸K¸L¸L¸Ô$I×$OÒ$OÑ$QÔ$QÐ!ØŸšÐ(<Ñ=Ô=ˆÝ”} ]¸Ð;Ñ;Ô;ˆà*×1Ò1Ñ3Ô3×6Ò6µu´}ÑEÔEÈÔH]Ñ]ÐÝ!œJÐ':¸D¼JÈBÐOÑOÔOÑˆˆ>Ý”| L°a¸~ÐNÑNÔNˆð %¨¬Ô(?Ò?ˆØ#×/Ò/°¸QÑ?Ô?ˆØ'×3Ò3°KÀÑCÔCˆð ×!Ò! " dÔ&=Ñ>Ô>Ø× Ò  Ñ-Ô-Ø×"Ò" ;Ñ/Ô/Ø%ð	
ð 	
rE   rt   ©
ru   rv   rw   rR   r2   rS   rz   ry   rs   r|   r}   s   @rD   rC  rC     sª   ø€ € € € € ð)ð ð)ð 
ð	)ð )ð )ð )ð )ð )ð> .2ð
ð 
à”|ð
ð ”| dÑ*ð
ð 
ˆuŒ|˜Uœ\¨5¬<¸¼ÐEÔ	Fð	
ð 
ð 
ð 
ð 
ð 
ð 
ð 
rE   rC  c                   ób   ‡ — e Zd ZdZˆ fd„Zdej        dej        dej        dej        fd„Zˆ xZS )ÚZayaExpertsz2Collection of expert weights stored as 3D tensors.c                 ó´  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        t          j        t          j
        | j        d| j        z  | j        ¦  «        ¦  «        | _        t          j        t          j
        | j        | j        | j        ¦  «        ¦  «        | _        t          |j                 | _        d S )NrK   )r1   r2   r:  rP   Ú
hidden_dimÚmoe_intermediate_sizeÚintermediate_dimr   r„   rS   ÚemptyÚgate_up_projrJ  r   Ú
hidden_actr@  rà   s     €rD   r2   zZayaExperts.__init__C  s£   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔØ Ô,ˆŒØ &Ô <ˆÔÝœL­¬°TÔ5EÀqÈ4ÔK`ÑG`ÐbfÔbqÑ)rÔ)rÑsÔsˆÔÝœ¥e¤k°$Ô2BÀDÄOÐUYÔUjÑ&kÔ&kÑlÔlˆŒÝ˜VÔ.Ô/ˆŒˆˆrE   rˆ   Útop_k_indexÚtop_k_weightsrF   c                 ó€  — t          j        |¦  «        }t          j        ¦   «         5  t           j        j                             || j        ¬¦  «        }|                     ddd¦  «        }t          j        | 	                    d¬¦  «        d¦  «         
                    ¦   «         }d d d ¦  «         n# 1 swxY w Y   |D ]þ}|d         }|| j        k    rŒt          j        ||         ¦  «        \  }}	||	         }
t          j                             |
| j        |         ¦  «                             dd¬¦  «        \  }}|                      |¦  «        |z  }t          j                             || j        |         ¦  «        }|||	|d f         z  }|                     d|	|                     |j        ¦  «        ¦  «         Œÿ|S )N)Únum_classesrK   r!   r   )r]   r¼   rb   r]   )rS   Ú
zeros_liker{   r   rõ   Úone_hotr:  ÚpermuteÚgreaterÚsumÚnonzeroÚwhereÚlinearrg  Úchunkr@  rJ  Ú
index_add_rV   rM   )r>   rˆ   ri  rj  Úfinal_hidden_statesÚexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stateÚgateÚupÚcurrent_hidden_statess                 rD   rs   zZayaExperts.forwardL  sø  € õ $Ô.¨}Ñ=Ô=ÐÝŒ]‰_Œ_ð 	Sð 	SÝœ(Ô-×5Ò5°kÈtÔO_Ð5Ñ`Ô`ˆKØ%×-Ò-¨a°°AÑ6Ô6ˆKÝœ {§¢¸8 Ñ'DÔ'DÀaÑHÔH×PÒPÑRÔRˆJð	Sð 	Sð 	Sñ 	Sô 	Sð 	Sð 	Sð 	Sð 	Sð 	Sð 	Søøøð 	Sð 	Sð 	Sð 	Sð
 %ð 
	nð 
	nˆJØ# AœˆJØ˜TÔ-Ò-Ð-ØÝ#(¤;¨{¸:Ô/FÑ#GÔ#GÑ ˆI�yØ)¨)Ô4ˆMÝ”}×+Ò+¨M¸4Ô;LÈZÔ;XÑYÔY×_Ò_Ð`aÐgiÐ_ÑjÔj‰HˆD�"Ø$(§K¢K°Ñ$5Ô$5¸Ñ$:Ð!Ý$&¤M×$8Ò$8Ð9NÐPTÔP^Ð_iÔPjÑ$kÔ$kÐ!Ø$9¸MÈ)ÐU^Ð`dÐJdÔ<eÑ$eÐ!Ø×*Ò*¨1¨iÐ9N×9QÒ9QÐReÔRkÑ9lÔ9lÑmÔmÐmÐmà"Ð"s   ¨A>B2Â2B6Â9B6)	ru   rv   rw   rÙ   r2   rS   rz   rs   r|   r}   s   @rD   ra  ra  ?  s€   ø€ € € € € à<Ð<ð0ð 0ð 0ð 0ð 0ð#à”|ð#ð ”\ð#ð ”|ð	#ð
 
Œð#ð #ð #ð #ð #ð #ð #ð #rE   ra  c            
       ó€   ‡ — e Zd Zdefˆ fd„Z	 ddej        dej        dz  deej        ej        dz  f         fd„Zˆ xZ	S )	r$  rœ   c                 óš   •— t          ¦   «                              ¦   «          t          ||¦  «        | _        t	          |¦  «        | _        d S rt   )r1   r2   rC  r~  ra  Úexpertsr  s      €rD   r2   zZayaSparseMoeBlock.__init__h  s=   ø€ Ý‰Œ×ÒÑÔÐÝ˜v yÑ1Ô1ˆŒ	Ý" 6Ñ*Ô*ˆŒˆˆrE   Nrˆ   r,  rF   c                 óæ   — |                       ||¬¦  «        \  }}}}|j        \  }}}|                     ||z  |¦  «        }	|                      |	||¦  «        }
|
                     |||¦  «        }
|
|fS )N)rO  )r~  rd   r½   rƒ  )r>   rˆ   r,  r0  r[  r]  Ú
batch_sizerW  Úemb_dimÚhidden_states_flatÚexpert_outputs              rD   rs   zZayaSparseMoeBlock.forwardm  s—   € ð FJÇYÂYØÐ)Bð FOñ F
ô F
ÑBˆˆ<˜Ð)Bð +8Ô*=Ñ'ˆ
�J Ø*×/Ò/°
¸ZÑ0GÈÑQÔQÐØŸšÐ%7¸ÈÑVÔVˆØ%×*Ò*¨:°zÀ7ÑKÔKˆàÐ7Ð7Ð7rE   rt   r_  r}   s   @rD   r$  r$  g  sš   ø€ € € € € ð+¨#ð +ð +ð +ð +ð +ð +ð :>ð8ð 8à”|ð8ð $)¤<°$Ñ#6ð8ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð	8ð 8ð 8ð 8ð 8ð 8ð 8ð 8rE   r$  c                   óž   ‡ — e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZdZ eed¬¦  «        eed	œZ ej        ¦   «         ˆ fd
„¦   «         Zˆ xZS )ÚZayaPreTrainedModelr'   ÚmodelTr   r¸   Fr   )rQ  )rZ  rˆ   Ú
attentionsc                 ó   •— t          ¦   «                              |¦  «         t          |t          ¦  «        rft	          j        |j        ¦  «         t	          j        |j        ¦  «         t	          j        |j	        ¦  «         t	          j        |j
        ¦  «         d S t          |t          ¦  «        r4t	          j        |j        ¦  «         t	          j        |j        ¦  «         d S t          |t          ¦  «        rt	          j        |j        ¦  «         d S t          |t           ¦  «        rE|j        rt	          j        |j        ¦  «         t	          j        |j        ¦  «         d|j        d<   d S t          |t(          ¦  «        rF| j        j        }t	          j        |j        d|¬¦  «         t	          j        |j        d|¬¦  «         d S t          |t4          ¦  «        r›|j        D ]•}|j        }|j        |         dk    rt<          |j        |                  } ||j        |¬¦  «        \  }}t?          ||› d�¦  «                              |¦  «         t?          ||› d�¦  «                              |¦  «         Œ”d S d S )	NrE  r]   rí   )r�   Ústdr*   r+   r-   r/   )!r1   Ú_init_weightsre   r)  r   Úones_r3  Úzeros_r4  r5  r6  Ú	ZayaModelÚinput_hidden_states_scaleÚinput_hidden_states_biasrÛ   rß   rC  rK  rL  rN  ra  r'   Úinitializer_rangeÚnormal_rg  rJ  r$   r8   r:   r)   r   rO   Úcopy_)r>   rî   rŽ  r,   r@   rA   r0  rC   s          €rD   r�  z!ZayaPreTrainedModel._init_weights’  sq  ø€ å‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ1Ñ2Ô2ð 	XÝŒJ�vÔ1Ñ2Ô2Ð2ÝŒK˜Ô1Ñ2Ô2Ð2ÝŒJ�vÔ,Ñ-Ô-Ð-ÝŒK˜Ô,Ñ-Ô-Ð-Ð-Ð-Ý˜¥	Ñ*Ô*ð 	XÝŒJ�vÔ7Ñ8Ô8Ð8ÝŒK˜Ô7Ñ8Ô8Ð8Ð8Ð8Ý˜¥
Ñ+Ô+ð 	XÝŒK˜œÑ$Ô$Ð$Ð$Ð$Ý˜¥
Ñ+Ô+ð 	XØŒ~ð 7Ý”
˜6Ô5Ñ6Ô6Ð6ÝŒK˜Ô/Ñ0Ô0Ð0Ø*.ˆFÔ# BÑ'Ð'Ð'Ý˜¥Ñ,Ô,ð 	XØ”+Ô/ˆCÝŒL˜Ô,°3¸CÐ@Ñ@Ô@Ð@ÝŒL˜Ô)°¸Ð=Ñ=Ô=Ð=Ð=Ð=Ý˜Õ 3Ñ4Ô4ð 	XØ$Ô0ð Xð X�
Ø%ÔE�ØÔ# JÔ/°9Ò<Ð<Ý#6°vÔ7GÈ
Ô7SÔ#T�LØ#/ <°´È*Ð#UÑ#UÔ#UÑ �˜qÝ˜ :Ð 8Ð 8Ð 8Ñ9Ô9×?Ò?ÀÑNÔNÐNÝ˜ :Ð AÐ AÐ AÑBÔB×HÒHÈÑWÔWÐWÐWð	Xð 	XðXð XrE   )ru   rv   rw   r"   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr   rC  r   r	  Ú_can_record_outputsrS   r{   r�  r|   r}   s   @rD   rŠ  rŠ    sÃ   ø€ € € € € € àÐÐÑØÐØ&*Ð#Ø+Ð,ÐØ#4Ð"5ÐØÐØ€NØÐà"ÐØ"&Ðà'˜¨
¸!Ð<Ñ<Ô<Ø)Ø#ðð Ðð €U„]�_„_ðXð Xð Xð Xñ „_ðXð Xð Xð Xð XrE   rŠ  c                   óâ   ‡ — e Zd Zdefˆ fd„Zeee	 	 	 	 	 	 ddej	        dz  dej
        dz  dej	        dz  dedz  dej        dz  d	edz  d
ee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )r’  r'   c                 óž  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          j        ‰j        ‰j        | j        ¦  «        | _        t          j	        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        t!          ‰¬¦  «        | _        d| _        t          j        t)          j        ‰j        ¦  «        ¦  «        | _        t          j        t)          j        ‰j        ¦  «        ¦  «        | _        |                      ¦   «          d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r.  )r   )Ú.0rœ   r'   s     €rD   ú
<listcomp>z&ZayaModel.__init__.<locals>.<listcomp>»  s$   ø€ ÐbÐbÐb°YÕ˜f iÑ0Ô0ÐbÐbÐbrE   r"  ©r'   F)r1   r2   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	EmbeddingrP   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersr¿   r€   r&  ræ   r$   Ú
rotary_embÚgradient_checkpointingr„   rS   r…   r“  rÞ   r”  Ú	post_initrà   s    `€rD   r2   zZayaModel.__init__´  s  øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒåœL¨Ô):¸FÔ<NÐPTÔP`ÑaÔaˆÔÝ”mØbÐbÐbÐbÅ%ÈÔH`ÑBaÔBaÐbÑbÔbñ
ô 
ˆŒõ   Ô 2¸Ô8KÐLÑLÔLˆŒ	Ý-°VÐ<Ñ<Ô<ˆŒØ&+ˆÔ#Ý)+¬µe´jÀÔASÑ6TÔ6TÑ)UÔ)UˆÔ&Ý(*¬µU´[ÀÔASÑ5TÔ5TÑ(UÔ(UˆÔ%ð 	�ŠÑÔÐÐÐrE   NÚ	input_idsrï   rm   r¸   Úinputs_embedsÚ	use_cacherW   rF   c                 ó  ‡ ‡‡‡‡— |d u |d uz  rt          d¦  «        ‚|€‰                      |¦  «        }|r|€t          ‰ j        ¬¦  «        }‰€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   Š‰ 	                    d¦  «        Št          |x}	t          ¦  «        sL‰ j        |||‰dœŠˆfd„ˆfd„d	œŠˆfd
„t          ‰ j        j        ¦  «        D ¦   «         }	t          di ‰¤Ž|	d<   |Šˆˆˆ fd„t          ‰ j        j        ¦  «        D ¦   «         }
‰‰ j        z   ‰ j        z                       t
          j        ¦  «        Šd }t'          ‰ j        ¦  «        D ]J\  }}‰ j        j        |         } |‰|f|	|         |	                     d¦  «        dœ||
|         dœ|¤Ž\  Š}ŒK‰                      ‰                     ‰ j        j        j        ¬¦  «        ¦  «        Št3          ‰|r|nd ¬¦  «        S )Nz:You must specify exactly one of input_ids or inputs_embedsr¨  r   r!   )r%   ©r'   rµ  rï   r¸   rm   c                  ó   •— t          di ‰ ¤ŽS ©Nr.  ©r   ©Úmask_kwargss   €rD   ú<lambda>z#ZayaModel.forward.<locals>.<lambda>ê  s   ø€ Õ"4Ð"CÐ"C°{Ð"CÐ"C€ rE   c                  ó   •— t          di ‰ ¤ŽS rº  ©r   r¼  s   €rD   r¾  z#ZayaModel.forward.<locals>.<lambda>ë  s   ø€ Õ*KÐ*ZÐ*ZÈkÐ*ZÐ*Z€ rE   ©Úhybridr  c                 ó2   •— i | ]}| ‰|         ¦   «         “ŒS r.  r.  ©r¦  r,   Úmask_creation_functionss     €rD   ú
<dictcomp>z%ZayaModel.forward.<locals>.<dictcomp>í  s7   ø€ ð #ð #ð #ØFP�
Ð?Ð3°JÔ?ÑAÔAð#ð #ð #rE   r  c                 ó@   •— i | ]}|‰                      ‰‰|¦  «        “ŒS r.  )r±  )r¦  r,   rˆ   rm   r>   s     €€€rD   rÆ  z%ZayaModel.forward.<locals>.<dictcomp>ô  s;   ø€ ð 
ð 
ð 
àð ˜Ÿš¨°|ÀZÑPÔPð
ð 
ð 
rE   )r  r  )rï   r¸   r  rL   )Úlast_hidden_stater¸   r.  )Ú
ValueErrorr­  r
   r'   Úget_seq_lengthrS   rT   rd   r%   rÅ   re   r  r7   r8   r   r”  r“  rV   r‹   Ú	enumerater¿   rN   ræ   r†   rM   r   )r>   r´  rï   rm   r¸   rµ  r¶  rW   Úpast_seen_tokensÚcausal_mask_mappingr  r,  ÚidxÚdecoder_layerr,   rˆ   rÅ  r½  s   `  `           @@@rD   rs   zZayaModel.forwardÆ  sÖ  øøøøø€ ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå°Ð?Ð-ÅÑFÔFð 	Yàœ+Ø!.Ø"0Ø#2Ø ,ðð ˆKð DÐCÐCÐCØ"ZÐ"ZÐ"ZÐ"Zð'ð 'Ð#ð#ð #ð #ð #ÝTWÐX\ÔXcÔXoÑTpÔTpð#ñ #ô #Ðõ +JÐ*XÐ*XÈKÐ*XÐ*XÐ Ñ'à%ˆð
ð 
ð 
ð 
ð 
ð 
å! $¤+Ô"9Ñ:Ô:ð
ñ 
ô 
Ðð (¨$Ô*GÑGÈ4ÔKiÑi×mÒmÝŒMñ
ô 
ˆð %)Ð!å"+¨D¬KÑ"8Ô"8ð 	ð 	ÑˆC�ØœÔ0°Ô5ˆJð 8E°}ØØ)ð
8ð 2°*Ô=Ø/×3Ò3°FÑ;Ô;ð ð  ð !0Ø$7¸
Ô$Cð
8ð 
8ð ð
8ð 
8Ñ4ˆMÐ4Ð4ð Ÿ	š	 -×"2Ò"2¸¼Ô9IÔ9OÐ"2Ñ"PÔ"PÑQÔQˆå%Ø+Ø/8ÐB˜O˜O¸dð
ñ 
ô 
ð 	
rE   )NNNNNN)ru   rv   rw   r"   r2   r   r    r   rS   Ú
LongTensorrz   r	   ÚFloatTensorÚboolr   r   r   rs   r|   r}   s   @rD   r’  r’  ²  s  ø€ € € € € ð˜zð ð ð ð ð ð ð$  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ðL
ð L
àÔ# dÑ*ðL
ð œ tÑ+ðL
ð Ô&¨Ñ-ð	L
ð
  ™ðL
ð Ô(¨4Ñ/ðL
ð ˜$‘;ðL
ð Ð+Ô,ðL
ð 
 ðL
ð L
ð L
ñ „^ñ „_ñ  ÔðL
ð L
ð L
ð L
ð L
rE   r’  zZyphra/ZAYA1-8B)Ú
checkpointc                   ó0  ‡ — e Zd ZddiZddiZdZˆ fd„Zee	 	 	 	 	 	 	 	 	 dd	e	j
        dz  d
e	j        dz  de	j
        dz  dedz  de	j        dz  de	j
        dz  dedz  dedz  dee	j        z  dee         defd„¦   «         ¦   «         Zedd„¦   «         Zˆ xZS )ÚZayaForCausalLMzlm_head.weightzmodel.embed_tokens.weightÚlm_headÚkeep_full_weightTc                 ó  •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        | _        t          j        |j        |j        | j        j	        ¬¦  «        | _
        |                      ¦   «          d S )Nrž   )r1   r2   r’  r‹  r«  r   r¬   rP   r'   Úlm_head_biasrÖ  r³  )r>   r'   rW   rC   s      €rD   r2   zZayaForCausalLM.__init__  sm   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý˜vÑ&Ô&ˆŒ
Ø Ô+ˆŒÝ”y Ô!3°VÔ5FÈTÌ[ÔMeÐfÑfÔfˆŒØ�ŠÑÔÐÐÐrE   Nr   r´  rï   rm   r¸   rµ  Úlabelsr¶  Úoutput_router_logitsÚlogits_to_keeprW   rF   c
                 ór  — |�|n| j         j        } | j        d|||||||dœ|
¤Ž}|j        }t	          |	t
          ¦  «        rt          |	 d¦  «        n|	}|                      |dd…|dd…f         ¦  «        }d}|� | j        ||| j	        fi |
¤Ž}t          |||j        |j        |j        |j        ¬¦  «        S )aÉ  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, ZayaForCausalLM

        >>> model = ZayaForCausalLM.from_pretrained("meta-zaya/Zaya-2-7b-hf")
        >>> tokenizer = AutoTokenizer.from_pretrained("meta-zaya/Zaya-2-7b-hf")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)r´  rï   rm   r¸   rµ  r¶  rÛ  )ÚlossÚlogitsr¸   rˆ   rŒ  rZ  r.  )r'   rÛ  r‹  rÈ  re   rR   ÚslicerÖ  Úloss_functionr«  r   r¸   rˆ   rŒ  rZ  )r>   r´  rï   rm   r¸   rµ  rÚ  r¶  rÛ  rÜ  rW   Úoutputsrˆ   Úslice_indicesrß  rÞ  s                   rD   rs   zZayaForCausalLM.forward&  s  € ðB %9Ð$DÐ Ð È$Ì+ÔJjð 	ð +5¨$¬*ð 	+
ØØ)Ø%Ø+Ø'ØØ!5ð	+
ð 	+
ð ð	+
ð 	+
ˆð  Ô1ˆÝ8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô% f¨f°d´oÐPÐPÈÐPÐPˆDå(ØØØ#Ô3Ø!Ô/ØÔ)Ø!Ô/ð
ñ 
ô 
ð 	
rE   c                 ó¶   ‡‡	— |                       ¦   «         }|||||dœŠ	ˆ	fd„ˆ	fd„dœŠˆfd„t          |j        ¦  «        D ¦   «         }t          di ‰	¤Ž|d<   |S )Nr¸  c                  ó   •— t          di ‰ ¤ŽS rº  r»  r¼  s   €rD   r¾  z;ZayaForCausalLM.create_masks_for_generate.<locals>.<lambda>s  s   ø€ Õ0Ð?Ð?°;Ð?Ð?€ rE   c                  ó   •— t          di ‰ ¤ŽS rº  rÀ  r¼  s   €rD   r¾  z;ZayaForCausalLM.create_masks_for_generate.<locals>.<lambda>t  s   ø€ Õ&GÐ&VÐ&VÈ+Ð&VÐ&V€ rE   rÁ  c                 ó2   •— i | ]}| ‰|         ¦   «         “ŒS r.  r.  rÄ  s     €rD   rÆ  z=ZayaForCausalLM.create_masks_for_generate.<locals>.<dictcomp>v  s7   ø€ ð 
ð 
ð 
ØBLˆJÐ;Ð/°
Ô;Ñ=Ô=ð
ð 
ð 
rE   r  r.  )Úget_text_configr7   r8   r   )
r'   rµ  rï   r¸   rm   r0  Útext_configr  rÅ  r½  s
           @@rD   Úcreate_masks_for_generatez)ZayaForCausalLM.create_masks_for_generatef  s®   øø€ ð ×,Ò,Ñ.Ô.ˆà!Ø*Ø,Ø.Ø(ð
ð 
ˆð @Ð?Ð?Ð?ØVÐVÐVÐVð#
ð #
Ðð
ð 
ð 
ð 
ÝPSÐT_ÔTkÑPlÔPlð
ñ 
ô 
ˆõ  ?ÐMÐMÀÐMÐMˆ�VÑØÐrE   )	NNNNNNNNr   rt   )ru   rv   rw   Ú_tied_weights_keysÚ
_fsdp_planÚ_is_statefulr2   r   r   rS   rÐ  rz   r	   rÑ  rÒ  rR   r   r   r   rs   rx   rê  r|   r}   s   @rD   rÕ  rÕ    sn  ø€ € € € € à*Ð,GÐHÐàÐ/Ð0€JØ€Lðð ð ð ð ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø,0Ø-.ð<
ð <
àÔ# dÑ*ð<
ð œ tÑ+ð<
ð Ô&¨Ñ-ð	<
ð
  ™ð<
ð Ô(¨4Ñ/ð<
ð Ô  4Ñ'ð<
ð ˜$‘;ð<
ð # T™kð<
ð ˜eœlÑ*ð<
ð Ð+Ô,ð<
ð 
#ð<
ð <
ð <
ñ „^ñ Ôð<
ð| ðð ð ñ „\ðð ð ð ð rE   rÕ  )rŠ  r’  rÕ  )rí   )r!   )KÚcollections.abcr   Útypingr   rS   Útorch.nn.functionalr   rõ   rÁ   Útorch.nnr   Úactivationsr   Úcache_utilsr	   r
   Ú
generationr   Úintegrationsr   r   Úmasking_utilsr   r   r   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   r    Úconfiguration_zayar"   ÚModuler$   r€   rz   rR   r™   r›   rÛ   rì   rU   rû   r  r	  r   r)  r9  rC  ra  r$  rŠ  r’  rÕ  Ú__all__r.  rE   rD   ú<module>r     sª  ðð, %Ð $Ð $Ð $Ð $Ð $Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )Ø SÐ SÐ SÐ SÐ SÐ SÐ SÐ SØ sÐ sÐ sÐ sÐ sÐ sÐ sÐ sÐ sÐ sØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø QÐ QÐ QÐ QÐ QÐ QÐ QÐ QØ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ *Ð *Ð *Ð *Ð *Ð *ðG<ð G<ð G<ð G<ð G<˜"œ)ñ G<ô G<ð G<ðT Ð˜YÑ'Ô'ðJð Jð Jð Jð J�"”)ñ Jô Jñ (Ô'ðJð(	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uðl!ð l!ð l!ð l!ð l!˜œ	ñ l!ô l!ð l!ð^(ð (ð (ð (ð (�”ñ (ô (ð (ð,(ð (ð (ð ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð4#ð #ð #ð #ðLI)ð I)ð I)ð I)ð I)�B”Iñ I)ô I)ð I)ðX,8ð ,8ð ,8ð ,8ð ,8Ð1ñ ,8ô ,8ð ,8ð^(ð (ð (ð (ð (˜"œ)ñ (ô (ð (ð,ð ,ð ,ð ,ð ,�B”Iñ ,ô ,ð ,ð <
ð <
ð <
ð <
ð <
�”ñ <
ô <
ð <
ð~ ð$#ð $#ð $#ð $#ð $#�"”)ñ $#ô $#ñ Ôð$#ðN8ð 8ð 8ð 8ð 8˜œñ 8ô 8ð 8ð0 ð/Xð /Xð /Xð /Xð /X˜/ñ /Xô /Xñ „ð/Xðd ðb
ð b
ð b
ð b
ð b
Ð#ñ b
ô b
ñ „ðb
ðJ €Ð,Ð-Ñ-Ô-ðað að að að aÐ)¨?ñ aô añ .Ô-ðaðH BÐ
AÐ
A€€€rE   