§
    9²¸jõ  ã            &       ó^  — d dl Z d dlmZ d dlmZ d dlZd dlmc mZ	 d dlmZ ddl
mZ ddlmZ ddlmZmZ dd	lmZ dd
lmZmZmZ ddlmZ ddlmZmZ ddlmZ ddlm Z m!Z!m"Z" ddl#m$Z$m%Z% ddl&m'Z'm(Z( ddl)m*Z* ddl+m,Z,m-Z-m.Z.m/Z/ ddl0m1Z1 ddl2m3Z3m4Z4 ddl5m6Z6 ddl7m8Z8  e/j9        e:¦  «        Z; G d„ dej        j<        ¦  «        Z= G d„ dej<        ¦  «        Z> G d„ dej<        ¦  «        Z?dej@        deAd ej@        fd!„ZB	 dwd#ej<        d$ej@        d%ej@        d&ej@        d'ej@        dz  d(eCd)eCfd*„ZDd+„ ZE ed,¦  «        dxd-„¦   «         ZF G d.„ d/ej<        ¦  «        ZGd0ej@        d1eAfd2„ZHd3„ ZId4„ ZJd5„ ZK ed6d7¦  «        	 	 dydej@        d8ej@        d9ejL        d:ejL        dz  d;eMdz  f
d<„¦   «         ZN ed=d7¦  «        	 	 dydej@        d9ejL        d:ejL        dz  d;eMdz  fd>„¦   «         ZO ed?d@¦  «        dd" eCdA¦  «        fdBdCddDdddddEfdFej@        dGej@        dHej@        dz  dIej@        dJej@        dKej@        dLeAdMej@        dz  dNePeCeCf         dOeQd;eMdPej@        dz  dQeCdRej@        dz  dSej@        dz  dTeAdz  dUeAdVeQf$dW„¦   «         ZR edXd@¦  «        	 	 	 	 dzdYej@        dej@        dZej@        dJej@        d[ej@        d\ej@        dKej@        dz  dIej@        dz  d]eQd^ej@        dz  fd_„¦   «         ZS ed`d@¦  «        ddddBd" eCdA¦  «        fdBfdej@        dZej@        dJej@        d[ej@        d\ej@        dLeAdKej@        dz  dIej@        dz  dMej@        dz  d]eQdNePeCeCf         dOeQfda„¦   «         ZT eeOeNeReSeTg¦  «         G db„ dcej<        ¦  «        ¦   «         ZU G dd„ deej<        ¦  «        ZV G df„ dgej<        ¦  «        ZW G dh„ die¦  «        ZX G dj„ dke¦  «        ZYe- G dl„ dme(¦  «        ¦   «         ZZe- G dn„ doeZ¦  «        ¦   «         Z[ G dp„ dqeZe¦  «        Z\ e-dr¬s¦  «         G dt„ dueZ¦  «        ¦   «         Z]g dv¢Z^dS ){é    N)ÚCallable)Úcycle)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hubÚ&use_kernel_func_from_hub_with_fallbackÚuse_kernelized_func)Úforce_accelerate_hooks)Úcreate_causal_maskÚcreate_recurrent_attention_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPastÚ SequenceClassifierOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )ÚZamba2Configc                   ó(   ‡ — e Zd Zdˆ fd„	Zdd„Zˆ xZS )ÚZamba2RMSNormGatedç�íµ ÷Æ°>c                 óº   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        || _        d S ©N)	ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilonÚ
group_size)ÚselfÚhidden_sizer1   ÚepsÚ	__class__s       €úz/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/zamba2/modeling_zamba2.pyr+   zZamba2RMSNormGated.__init__4   sG   ø€ Ý‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔØ$ˆŒˆˆó    Nc                 ó  — |j         }|                     t          j        ¦  «        }|�?|t          j                             |                     t          j        ¦  «        ¦  «        z  }|j        �^ }}|| j        z  } |j	        g |¢|‘| j        ‘R Ž }| 
                    d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  } |j	        g |¢|| j        z  ‘R Ž }| j        |                     |¦  «        z  S ©Né   éÿÿÿÿT)Úkeepdim)ÚdtypeÚtor-   Úfloat32r   Ú
functionalÚsiluÚshaper1   ÚviewÚpowÚmeanÚrsqrtr0   r/   )	r2   Úhidden_statesÚgateÚinput_dtypeÚprefix_dimsÚlast_dimÚgroup_countÚhidden_states_groupÚvariances	            r6   ÚforwardzZamba2RMSNormGated.forward:   s  € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØÐØ)­B¬M×,>Ò,>¸t¿wºwÅuÄ}Ñ?UÔ?UÑ,VÔ,VÑVˆMØ!.Ô!4Ñˆ�hØ $¤/Ñ1ˆØ0˜mÔ0Ð\°+Ð\¸{Ð\ÈDÌOÐ\Ð\Ð\ÐØ&×*Ò*¨1Ñ-Ô-×2Ò2°2¸tÐ2ÑDÔDˆØ1µE´KÀÈ4ÔK`Ñ@`Ñ4aÔ4aÑaÐØ0Ð+Ô0Ð]°+Ð]¸{ÈTÌ_Ñ?\Ð]Ð]Ð]ˆØŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:r7   ©r'   r)   )Ú__name__Ú
__module__Ú__qualname__r+   rO   Ú__classcell__©r5   s   @r6   r&   r&   3   sQ   ø€ € € € € ð%ð %ð %ð %ð %ð %ð;ð ;ð ;ð ;ð ;ð ;ð ;ð ;r7   r&   c                   óT   ‡ — e Zd Zd	deddfˆ fd„Zdej        dej        fd„Zd„ Zˆ xZ	S )
ÚZamba2RMSNormr'   r4   ÚreturnNc                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )z<
        Zamba2RMSNorm is equivalent to T5LayerNorm
        N)r*   r+   r   r,   r-   r.   r/   r0   )r2   r3   r4   r5   s      €r6   r+   zZamba2RMSNorm.__init__I   sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐr7   rG   c                 ó  — |j         }|                     t          j        ¦  «        }|                     d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        |                     |¦  «        z  S r9   )	r=   r>   r-   r?   rD   rE   rF   r0   r/   )r2   rG   rI   rN   s       r6   rO   zZamba2RMSNorm.forwardQ   s|   € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØ ×$Ò$ QÑ'Ô'×,Ò,¨R¸Ð,Ñ>Ô>ˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆØŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:r7   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)Útupler/   rB   r0   )r2   s    r6   Ú
extra_reprzZamba2RMSNorm.extra_reprX   s&   € Ý˜œÔ)Ñ*Ô*ÐIÐI°$Ô2GÐIÐIÐIr7   rP   )
rQ   rR   rS   Úfloatr+   r-   ÚTensorrO   r]   rT   rU   s   @r6   rW   rW   H   sŒ   ø€ € € € € ð$ð $¨ð $¸$ð $ð $ð $ð $ð $ð $ð; U¤\ð ;°e´lð ;ð ;ð ;ð ;ðJð Jð Jð Jð Jð Jð Jr7   rW   c            	       óî   ‡ — e Zd Z edd¬¦  «        d
defˆ fd„¦   «         Ze edd¬¦  «        d
dedeej	        e
f         fd„¦   «         ¦   «         Z ej        ¦   «         ed	„ ¦   «         ¦   «         Zˆ xZS )ÚZamba2RotaryEmbeddingÚdevicez5.18)ÚversionNÚconfigc                 ó¾  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        | j        j        d         | _        | j        }| j        dk    rt          | j                 } || j        |¦  «        \  }| _
        t          j        |d¬¦  «        | _        t          j        |                     ¦   «         d¬¦  «        | _        d S )NÚ	rope_typeÚdefaultF)Ú
persistent)r*   r+   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrd   Úrope_parametersrf   Úcompute_default_rope_parametersr   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r2   rd   rb   Úrope_init_fnrp   r5   s        €r6   r+   zZamba2RotaryEmbedding.__init__]   sÀ   ø€ å‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!àˆŒàœÔ4°[ÔAˆŒØ!%Ô!EˆØŒ>˜YÒ&Ð&Ý.¨t¬~Ô>ˆLØ+7¨<¸¼ÀVÑ+LÔ+LÑ(ˆ�$Ô(åœ	 (°uÐ=Ñ=Ô=ˆŒÝ!#¤¨8¯>ª>Ñ+;Ô+;ÈÐ!NÑ!NÔ!NˆÔÐÐr7   rX   c                 óä   — | j         d         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «        |z  z  z  }|                     |¦  «        |fS )a¾  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNg      ð?r   r:   ©r=   )rl   Úgetattrr3   Únum_attention_headsr-   Úaranger^   r>   )rd   rb   ÚkwargsÚbaseÚdimÚattention_factorrp   s          r6   rm   z5Zamba2RotaryEmbedding.compute_default_rope_parametersn   sz   € ð Ô% lÔ3ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐà˜$¥5¤<°°3¸ÅÄÐ#MÑ#MÔ#MÐPSÑ#SÑTÑUˆØ�{Š{˜6Ñ"Ô"Ð$4Ð4Ð4r7   c                 óú  — | j         d d d …d f                              |j        d         dd¦  «                             t          j        |j        ¬¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  ||z                       dd	¦  «        }t	          j        ||fd¬
¦  «        }|                     ¦   «         | j        z  }|                     ¦   «         | j        z  }	d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬¦  «        |	                     |j        ¬¦  «        fS )Nr   r;   r#   ©r=   rb   ÚmpsÚcpuF)Údevice_typeÚenabledr:   ©r}   rw   )rp   ÚexpandrB   r>   r-   r^   rb   Ú
isinstanceÚtypeÚstrr    Ú	transposeÚcatÚcosrn   Úsinr=   )
r2   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedrƒ   ÚfreqsÚembrŒ   r�   s
             r6   rO   zZamba2RotaryEmbedding.forward‚   s›  € ð ŒM˜$    4˜-Ô(×/Ò/°Ô0BÀ1Ô0EÀrÈ1ÑMÔM×PÒPÕW\ÔWbÐklÔksÐPÑtÔtð 	ð !-¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆå¨¸UÐCÑCÔCð 	5ð 	5Ø&Ð)>Ñ>×IÒIÈ!ÈQÑOÔOˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCð		5ð 	5ð 	5ñ 	5ô 	5ð 	5ð 	5ð 	5ð 	5ð 	5ð 	5øøøð 	5ð 	5ð 	5ð 	5ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   ÃA*D<Ä<E ÅE r)   )rQ   rR   rS   r   r$   r+   Ústaticmethodr\   r-   r_   r^   rm   Úno_gradr   rO   rT   rU   s   @r6   ra   ra   \   sö   ø€ € € € € Ø€_�X vÐ.Ñ.Ô.ðOð O˜|ð Oð Oð Oð Oð Oñ /Ô.ðOð  Ø€_�X vÐ.Ñ.Ô.ð5ð 5°ð 5ÐX]Ð^cÔ^jÐlqÐ^qÔXrð 5ð 5ð 5ñ /Ô.ñ „\ð5ð$ €U„]�_„_Øð<ð <ñ Ôñ „_ð<ð <ð <ð <ð <r7   ra   rG   Ún_reprX   c                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r#   N)rB   r†   Úreshape)rG   r–   ÚbatchÚnum_key_value_headsÚslenrv   s         r6   Ú	repeat_kvrœ   •   s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTr7   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutc                 ó  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        |z  }
|�|
|z   }
t
          j                             |
dt          j        ¬¦  «         	                    |j
        ¦  «        }
t
          j                             |
|| j        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «                             ¦   «         }||
fS )Nr:   r   r;   )r}   r=   )ÚpÚtrainingr#   )rœ   Únum_key_value_groupsr-   ÚmatmulrŠ   r   r@   Úsoftmaxr?   r>   r=   r¤   r§   Ú
contiguous)rž   rŸ   r    r¡   r¢   r£   r¤   r{   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r6   Úeager_attention_forwardr°   ¡   sé   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r7   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..Nr;   r:   r…   )rB   r-   r‹   )rŽ   Úx1Úx2s      r6   Úrotate_halfr´   º   s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'r7   Úrotary_pos_embc                 ó¾   — |                      |¦  «        }|                      |¦  «        }| |z  t          | ¦  «        |z  z   }||z  t          |¦  «        |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezer´   )ÚqÚkrŒ   r�   Úunsqueeze_dimÚq_embedÚk_embeds          r6   Úapply_rotary_pos_embr½   Á   sc   € ð& �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�GÐÐr7   c                   ó  ‡ — e Zd ZdZ	 	 	 ddededz  dedz  dedz  fˆ fd„Z	 	 	 ddej        ded	ej        dz  d
e	dz  de
ej        ej        f         dz  dee         de
ej        ej        dz  e
ej                 dz  f         fd„Zˆ xZS )ÚZamba2AttentionaZ  
    Multi-headed attention from 'Attention Is All You Need' paper.

    Adapted from transformers.models.mistral.modeling_mistral.MistralAttention:
    The input dimension here is attention_hidden_size = 2 * hidden_size, and head_dim = attention_hidden_size // num_heads.
    The extra factor of 2 comes from the input being the concatenation of original_hidden_states with the output of the previous (mamba) layer
    (see fig. 2 in https://huggingface.co/papers/2405.16712).
    Additionally, replaced
    attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) with
    attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim/2)
    Finally, this attention layer contributes to tied transformer blocks aimed to increasing compute without increasing model size. Because this
    layer is tied, un-tied adapters (formally the same as LoRA but used in the base model) modules are added to the q, k, v projectors to increase
    expressivity with a small memory overhead (see Fig. 2 of https://huggingface.co/papers/2411.15242).
    Nrd   Ú	layer_idxÚnum_fwd_mem_blocksÚblock_idc           	      ó<  •— t          ¦   «                              ¦   «          || _        || _        |j        | _        |j        | _        |j        |j        z  | _	        |j
        | _
        | j        dz  dz  | _        d| _        |j        | _        t          j        |j        |j        | j        z  d¬¦  «        | _        t          j        |j        |j        | j        z  d¬¦  «        | _        t          j        |j        |j        | j        z  d¬¦  «        | _        t          j        |j        | j        z  |j        d¬¦  «        | _        || _        |j        | _        || _        |j        �rt          j        g ¦  «        | _        t          j        g ¦  «        | _        t          j        g ¦  «        | _        t=          | j        ¦  «        D �]±}||j        z  |k    �rt          j         t          j        | j        | j        j!        d¬¦  «        t          j        | j        j!        | j        d¬¦  «        ¦  «        }t          j         t          j        | j        | j        j!        d¬¦  «        t          j        | j        j!        | j        d¬¦  «        ¦  «        }t          j         t          j        | j        | j        j!        d¬¦  «        t          j        | j        j!        | j        d¬¦  «        ¦  «        }n9t          j"        ¦   «         }t          j"        ¦   «         }t          j"        ¦   «         }| j         #                    |¦  «         | j         #                    |¦  «         | j         #                    |¦  «         �Œ³d„ tI          | j        ¦  «        D ¦   «         | _%        d S )Nr:   g      à¿TF©Úbiasc                 ó   — i | ]\  }}||“Œ	S © rÇ   ©Ú.0Úindexr¡   s      r6   ú
<dictcomp>z,Zamba2Attention.__init__.<locals>.<dictcomp>!  s   € Ð[Ð[Ð[©<¨5°%˜% Ð[Ð[Ð[r7   )&r*   r+   rd   rÀ   Úattention_hidden_sizeÚattention_head_dimrv   ry   rš   r¨   ri   r£   Ú	is_causalÚattention_dropoutr   ÚLinearÚq_projÚk_projÚv_projr3   Úo_projrÁ   Úhybrid_layer_idsÚlayer_block_maprÂ   Úuse_shared_attention_adapterÚ
ModuleListÚlinear_q_adapter_listÚlinear_k_adapter_listÚlinear_v_adapter_listÚrangeÚnum_mem_blocksÚ
SequentialÚadapter_rankÚIdentityÚappendÚ	enumerateÚ	layer_dic)
r2   rd   rÀ   rÁ   rÂ   ÚiÚlinear_q_adapterÚlinear_k_adapterÚlinear_v_adapterr5   s
            €r6   r+   zZamba2Attention.__init__ë   s*  ø€ õ 	‰Œ×ÒÑÔÐØˆŒØ"ˆŒà%+Ô%AˆÔ"ØÔ1ˆŒØ$*Ô$>À&ÔB\Ñ$\ˆÔ!Ø'-Ô'EˆÔ$Øœ¨Ñ)¨dÑ2ˆŒØˆŒØ!'Ô!9ˆÔå”i Ô <¸fÔ>XÐ[_Ô[hÑ>hÐotÐuÑuÔuˆŒÝ”i Ô <¸fÔ>XÐ[_Ô[hÑ>hÐotÐuÑuÔuˆŒÝ”i Ô <¸fÔ>XÐ[_Ô[hÑ>hÐotÐuÑuÔuˆŒÝ”i Ô :¸T¼]Ñ JÈFÔL^ÐejÐkÑkÔkˆŒØ"4ˆÔØ%Ô6ˆÔØ ˆŒàÔ.ñ 	DÝ)+¬°rÑ):Ô):ˆDÔ&Ý)+¬°rÑ):Ô):ˆDÔ&Ý)+¬°rÑ):Ô):ˆDÔ&å˜4Ô2Ñ3Ô3ð Dñ D�Ø�vÔ,Ñ,°Ò8Ñ8Ý')¤}Ýœ	 $Ô"<¸d¼kÔ>VÐ]bÐcÑcÔcÝœ	 $¤+Ô":¸DÔ<VÐ]bÐcÑcÔcñ(ô (Ð$õ (*¤}Ýœ	 $Ô"<¸d¼kÔ>VÐ]bÐcÑcÔcÝœ	 $¤+Ô":¸DÔ<VÐ]bÐcÑcÔcñ(ô (Ð$õ (*¤}Ýœ	 $Ô"<¸d¼kÔ>VÐ]bÐcÑcÔcÝœ	 $¤+Ô":¸DÔ<VÐ]bÐcÑcÔcñ(ô (Ð$Ð$õ
 (*¤{¡}¤}Ð$Ý')¤{¡}¤}Ð$Ý')¤{¡}¤}Ð$ØÔ*×1Ò1Ð2BÑCÔCÐCØÔ*×1Ò1Ð2BÑCÔCÐCØÔ*×1Ò1Ð2BÑCÔCÐCÑCà[Ð[½9ÀTÔEYÑ;ZÔ;ZÐ[Ñ[Ô[ˆŒˆˆr7   rG   r¢   Úpast_key_valuesÚposition_embeddingsr{   rX   c                 ó  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «        }	|                      |¦  «        }
|                      |¦  «        }| j        j        rX| j        |         }|	 | j        |         |¦  «        z   }	|
 | j	        |         |¦  «        z   }
| | j
        |         |¦  «        z   }|	                     |¦  «                             dd¦  «        }	|
                     |¦  «                             dd¦  «        }
|                     |¦  «                             dd¦  «        }| j        j        r|\  }}t          |	|
||¦  «        \  }	}
|�|                     |
||¦  «        \  }
}t!          j        | j        j        t&          ¦  «        } || |	|
||f| j        sdn| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Nr;   r#   r:   r�   )r¤   r£   )rB   rv   rÑ   rÒ   rÓ   rd   r×   rã   rÙ   rÚ   rÛ   rC   rŠ   Úuse_mem_roper½   Úupdater   Úget_interfaceÚ_attn_implementationr°   r§   rÏ   r£   r˜   r«   rÔ   )r2   rG   rÀ   r¢   rè   ré   r{   Úinput_shapeÚhidden_shapeÚquery_statesr¬   r­   Úadapter_layer_idxrŒ   r�   Úattention_interfacer¯   r®   s                     r6   rO   zZamba2Attention.forward#  sM  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1ˆØ—[’[ Ñ/Ô/ˆ
Ø—{’{ =Ñ1Ô1ˆØŒ;Ô3ð 	gØ $¤¨yÔ 9ÐØ'Ð*W¨$Ô*DÐEVÔ*WÐXeÑ*fÔ*fÑfˆLØ#Ð&S dÔ&@ÐARÔ&SÐTaÑ&bÔ&bÑbˆJØ'Ð*W¨$Ô*DÐEVÔ*WÐXeÑ*fÔ*fÑfˆLà#×(Ò(¨Ñ6Ô6×@Ò@ÀÀAÑFÔFˆØ—_’_ \Ñ2Ô2×<Ò<¸QÀÑBÔBˆ
Ø#×(Ò(¨Ñ6Ô6×@Ò@ÀÀAÑFÔFˆàŒ;Ô#ð 	`Ø*‰HˆC�Ý';¸LÈ*ÐVYÐ[^Ñ'_Ô'_Ñ$ˆL˜*àÐ&Ø'6×'=Ò'=¸jÈ,ÐXaÑ'bÔ'bÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r7   ©NNN)rQ   rR   rS   Ú__doc__r$   Úintr+   r-   r_   r	   r\   r   r   rO   rT   rU   s   @r6   r¿   r¿   Û   sD  ø€ € € € € ðð ð$ !%Ø)-Ø#ð6\ð 6\àð6\ð ˜‘:ð6\ð   $™Jð	6\ð
 ˜‘*ð6\ð 6\ð 6\ð 6\ð 6\ð 6\ðx /3Ø(,ØHLð1)ð 1)à”|ð1)ð ð1)ð œ tÑ+ð	1)ð
  ™ð1)ð # 5¤<°´Ð#=Ô>ÀÑEð1)ð Ð+Ô,ð1)ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	Mð1)ð 1)ð 1)ð 1)ð 1)ð 1)ð 1)ð 1)r7   r¿   Úinput_tensorÚpad_sizec                 ó¦   — t          | j        ¦  «        dk    r
ddddd|ddfnddd|ddf}t          j        j                             | |dd¬¦  «        S )z‚
    Padding x tensor with `pad_size` on the seq_len dim (dim=1)

    Assumes that we only have tensors of either size 4 or 3
    é   r   Úconstant)Úmoder¡   )ÚlenrB   r-   r   r@   Úpad)r÷   rø   Ú	pad_shapes      r6   Úpad_tensor_by_sizer   Z  sj   € õ 47°|Ô7IÑ3JÔ3JÈaÒ3OÐ3O��A�q˜!˜Q ¨!¨QÐ/Ð/ÐVWÐYZÐ\]Ð_gÐijÐlmÐUn€IåŒ8Ô×"Ò" <°ÀÐSTÐ"ÑUÔUÐUr7   c                 ó"  — t          | |¦  «        } t          | j        ¦  «        dk    r.|                      | j        d         d|| j        d         ¦  «        S |                      | j        d         d|| j        d         | j        d         ¦  «        S )zÀ
    Padding input_tensor with `pad_size` on the seq_len dim (dim=1) and
    simultaneously splitting it into chunk sequences.

    Assumes that we only have tensors of either size 4 or 3
    r   r   r;   r:   )r   rý   rB   r˜   )r÷   rø   Ú
chunk_sizes      r6   Úreshape_into_chunksr  e  s’   € õ & l°HÑ=Ô=€Lå
ˆ<ÔÑÔ !Ò#Ð#à×#Ò# LÔ$6°qÔ$9¸2¸zÈ<ÔK]Ð^_ÔK`ÑaÔaÐað ×#Ò#ØÔ˜qÔ! 2 z°<Ô3EÀaÔ3HÈ,ÔJ\Ð]^ÔJ_ñ
ô 
ð 	
r7   c                 ó  — |                       d¦  «        } | d         j        g |                       ¦   «         ¢|‘R Ž } t          j        t          j        ||| j        t          j        ¬¦  «        d¬¦  «        }|                      | d¦  «        } t          j        | d¬¦  «        }t          j        t          j        ||| j        t          j        ¬¦  «        d¬¦  «        }|                     | t          j	         ¦  «        }|S )zo
    More stable segment sum calculation. Uses cumulative sums and masking instead of direct subtractions.
    r;   ©.N©rb   r=   )Údiagonalr   éþÿÿÿr…   )
Úsizer†   r-   Útrilr.   rb   ÚboolÚmasked_fillÚcumsumÚinf)r÷   r  ÚmaskÚtensor_segsums       r6   Úsegment_sumr  y  só   € ð ×"Ò" 2Ñ&Ô&€Jð 2�< 	Ô*Ô1ÐS°<×3DÒ3DÑ3FÔ3FÐSÈ
ÐSÐSÐS€LåŒ:•e”j ¨ZÀÔ@SÕ[`Ô[eÐfÑfÔfÐqsÐtÑtÔt€DØ×+Ò+¨T¨E°1Ñ5Ô5€Lå”L °2Ð6Ñ6Ô6€Mõ Œ:•e”j ¨ZÀÔ@SÕ[`Ô[eÐfÑfÔfÐqrÐsÑsÔs€DØ!×-Ò-¨t¨eµe´i°ZÑ@Ô@€MØÐr7   c                 ób   — |�,| j         }| |dd…dd…df         z                       |¦  «        } | S )zm
    Tunes out the hidden states for padding tokens, see https://github.com/state-spaces/mamba/issues/66
    N)r=   r>   )rG   r¢   r=   s      r6   Úapply_mask_to_padding_statesr  �  sC   € ð
 Ð!ØÔ#ˆØ&¨¸¸¸¸1¸1¸1¸d¸
Ô)CÑC×GÒGÈÑNÔNˆàÐr7   Úcausal_conv1d_updateÚcausal_conv1dÚ
conv_stater/   rÅ   Ú
activationc                 óº  — | j         \  }}}|j         d         }t          j        || gd¬¦  «                             |j        ¦  «        }	|                     |	d d …d d …| d …f         ¦  «         t          j        |	|                     d¦  «        |d|¬¦  «        }
|
d d …d d …| d …f         }
|�t          |         |
¦  «        }
|
                     | j        ¦  «        S )Nr;   r…   r#   r   )ÚpaddingÚgroups)
rB   r-   r‹   r>   r=   Úcopy_ÚFÚconv1dr·   r   )rG   r  r/   rÅ   r  Ú_r3   Úseq_lenÚ	state_lenÚhidden_states_newÚouts              r6   r  r  ™  sî   € ð ,Ô1Ñ€A€{�GØÔ  Ô$€Iåœ	 :¨}Ð"=À2ÐFÑFÔF×IÒIÈ&Ì,ÑWÔWÐØ×ÒÐ& q q q¨!¨!¨!¨i¨Z¨[¨[Ð'8Ô9Ñ:Ô:Ð:Ý
Œ(Ð$ f×&6Ò&6°qÑ&9Ô&9¸4ÈÐS^Ð
_Ñ
_Ô
_€CØ
ˆaˆaˆa����W�H�I�IˆoÔ
€CØÐÝ�ZÔ  Ñ%Ô%ˆØ�6Š6�-Ô%Ñ&Ô&Ð&r7   Úcausal_conv1d_fnc                 ó@  — | j         \  }}}|j         d         dz
  }t          j        |                      |j        ¦  «        |                     d¦  «        |||¬¦  «        d d …d d …d |…f         }	|�t          |         |	¦  «        }	|	                     | j        ¦  «        S )Nr;   r#   )r/   rÅ   r  r  )rB   r  r  r>   r=   r·   r   )
rG   r/   rÅ   r  r{   r  r3   r  r  r"  s
             r6   r#  r#  ­  s²   € ð ,Ô1Ñ€A€{�GØŒl˜2Ô Ñ"€Gå
Œ(Ø×Ò˜œÑ&Ô&Ø×Ò Ñ"Ô"ØØØðñ ô ð €a€aˆˆˆˆHˆWˆH€nô€Cð ÐÝ�ZÔ  Ñ%Ô%ˆØ�6Š6�-Ô%Ñ&Ô&Ð&r7   Ú mamba_split_conv1d_scan_combinedÚ	mamba_ssmr  FrA   r'   TÚzxbcdtÚconv1d_weightÚconv1d_biasÚdt_biasÚAÚDr  Úinitial_statesÚdt_limitÚreturn_final_statesÚrmsnorm_weightÚrmsnorm_epsÚoutproj_weightÚoutproj_biasÚheaddimÚngroupsÚnorm_before_gatec                 ó   — d S r)   rÇ   )r'  r(  r)  r*  r+  r,  r  r-  r.  r/  r  r0  r1  r2  r3  r4  r5  r6  r{   s                      r6   Ú!mamba2_split_conv1d_scan_combinedr8  Ä  s	   € ð, ˆ4r7   Úselective_state_updateÚstateÚdtÚBÚCÚdt_softplusÚzc
                 óH  — |j         \  }}}|j         d         }|j         d         }|�||                     |j        ¦  «        z   }|rt          j        |¦  «        }|d         }t          j        |                     ¦   «         |                     ¦   «         z  ¦  «                             | j        ¬¦  «        }| 	                    ||d|¦  «        }| 
                    ||||z  |¦  «                             ¦   «         }| 	                    ||d|¦  «        }||z  }||d         z                       | j        ¬¦  «        }| |z  |z   }|                      |                     | j        ¦  «        ¦  «         | 	                    ||d|¦  «        }| 
                    ||||z  |¦  «                             ¦   «         }| 	                    |||¦  «        }|                     |j        |j        ¬¦  «        }|                     ||z  ||¦  «        }|                     ||z  |d¦  «        }t          j        ||¦  «        }|                     |||¦  «        }|� |||z  z                        |j        ¦  «        }|	�|t          j        |	¦  «        z  }|                     |j        ¦  «        S )Nr#   r;   r  ©rb   r  )rB   r>   r=   r  Úsoftplusr-   Úexpr^   rb   r˜   r†   r«   r  rC   ÚbmmrA   )r:  rG   r;  r+  r<  r=  r,  r*  r>  r?  r{   Ú
batch_sizeÚ	num_headsrv   Ú
num_groupsÚ
state_sizeÚdAÚdBÚdBxÚ
ssm_statesÚssm_states_reshapedÚ
C_reshapedr"  s                          r6   Úmamba2_selective_state_updaterO  Ý  sq  € ð '4Ô&9Ñ#€J�	˜8Ø”˜”€JØ”˜”€JàÐØ�'—*’*˜RœXÑ&Ô&Ñ&ˆØð ÝŒZ˜‰^Œ^ˆØ	ˆIŒ€Bõ 
Œ�2—8’8‘:”: §¢¡	¤	Ñ)Ñ	*Ô	*×	-Ò	-°U´\Ð	-Ñ	BÔ	B€Bð 	
�	Š	�*˜j¨!¨ZÑ8Ô8€AØ	�Š�˜Z¨°jÑ)@À*ÑMÔM×XÒXÑZÔZ€AØ	�	Š	�*˜i¨¨JÑ7Ô7€AØ	ˆa‰€Bð �˜iÔ(Ñ(×
,Ò
,°E´LÐ
,Ñ
AÔ
A€Cð ˜‘˜cÑ!€JØ	‡K‚K�
—’˜eœkÑ*Ô*Ñ+Ô+Ð+ð 	
�	Š	�*˜j¨!¨ZÑ8Ô8€AØ	�Š�˜Z¨°jÑ)@À*ÑMÔM×XÒXÑZÔZ€AØ	�	Š	�*˜i¨Ñ4Ô4€Að —’ a¤h°a´g�Ñ>Ô>€JØ$Ÿ/š/¨*°yÑ*@À(ÈJÑWÔWÐØ—’˜
 YÑ.°
¸AÑ>Ô>€JÝ
Œ)Ð'¨Ñ
4Ô
4€CØ
�(Š(�:˜y¨(Ñ
3Ô
3€Cð 	€}Ø�] QÑ&Ñ&×*Ò*¨3¬9Ñ5Ô5ˆà€}Ø•A”F˜1‘I”I‰oˆà�6Š6�-Ô%Ñ&Ô&Ð&r7   Úmamba_chunk_scan_combinedc                 ó	  ‡‡#— | j         \  }}}}|j         d         }|�||                     |j        ¦  «        z   }|	rt          j        |¦  «        }t          j        ||
d         |
d         ¬¦  «        }|                      ¦   «         } |                     ¦   «                              ||z  d|¬¦  «        }|                     ¦   «                              ||z  d|¬¦  «        }‰|‰z  z
  ‰z  Š#d }|�|d         t          | ‰#¦  «        z  }| |d                              ¦   «         z  } |                     | j        ¦  «        |                     ¦   «         z  }ˆˆ#fd„| |||fD ¦   «         \  } }}}| 
                    dddd¦  «        }t          j        |d	¬
¦  «        }t          j        t          |¦  «        ¦  «        }|d d …d d …d d …d d d …d d …f         |d d …d d …d d d …d d …d d …f         z                       d	¬
¦  «        }|d         | 
                    ddddd¦  «        d         z                       d	¬
¦  «        }|d         | d d …d d …d f         z                       d¬
¦  «        }t          j        |d d …d d …d d …d	d …f         |z
  ¦  «        }|| 
                    ddd	d¦  «        d         z  }|dd d d …f         | d         z                       d¬
¦  «        }|�+|d d …d f                              |j        |j        ¬¦  «        nt          j        |d d …d d…f         ¦  «        }t          j        ||gd¬
¦  «        }t          j        t          t          j        |d d …d d …d d …d	f         d¦  «        ¦  «        ¦  «                             dd¦  «        }|d         |d d …d d …d df         z                       d¬
¦  «        }|d d …d d	…f         |d d …d	f         }}t          j        |¦  «        }|dd d d …f         |d d …d d …d df         z  } |                      d	¦  «        | 
                    dddd¦  «        d         z  }!||!z   }"|"                     |d	||¦  «        }"|�|"|z   }"‰#dk    r|"d d …d |…f         }"|r|"|fS |"S )Nr:   r   r#   )ÚminÚmax)r}   Úoutput_sizer  c                 ó2   •— g | ]}t          |‰‰¦  «        ‘ŒS rÇ   )r  )rÉ   Útensorr  rø   s     €€r6   ú
<listcomp>z%mamba2_chunk_scan.<locals>.<listcomp>B  s'   ø€ ÐwÐwÐwÐTZÕ1°&¸(ÀJÑOÔOÐwÐwÐwr7   r   r;   r…   rú   r  .r€   )r#   r   ).NN)rB   r>   r=   r  rB  r-   Úclampr^   Úrepeat_interleaver   Úpermuter  rC  r  Úsumrb   Ú
zeros_liker‹   rþ   rŠ   r˜   )$rG   r;  r+  r<  r=  r  r,  r*  r-  r>  r.  r/  r{   rE  Úsequence_lengthrF  rv   rG  Ú
D_residualÚA_cumsumÚLÚGÚMÚY_diagÚdecay_statesÚB_decayÚstatesÚprevious_statesÚdecay_chunkÚ
new_statesÚfinal_stateÚstate_decay_outÚC_times_statesÚY_offÚoutputrø   s$        `                             @r6   Úmamba2_chunk_scanro    s  øø€ ð  8EÔ7JÑ4€J� ¨HØ”˜”€JàÐØ�'—*’*˜RœXÑ&Ô&Ñ&ˆØð ÝŒZ˜‰^Œ^ˆÝ	Œ�R˜X aœ[¨h°q¬kÐ	:Ñ	:Ô	:€Bà!×'Ò'Ñ)Ô)€MØ	�Š‰	Œ	×#Ò# I°Ñ$;ÀÐPYÐ#ÑZÔZ€AØ	�Š‰	Œ	×#Ò# I°Ñ$;ÀÐPYÐ#ÑZÔZ€Aà˜_¨zÑ9Ñ9¸ZÑG€HØ€JØ€}Ø�y”\Õ$6°}ÀhÑ$OÔ$OÑOˆ
ð " B y¤M×$7Ò$7Ñ$9Ô$9Ñ9€MØ	�Šˆ]Ô Ñ!Ô! B§H¢H¡J¤JÑ.€Að xÐwÐwÐwÐwÐ_lÐnoÐqrÐtuÐ^vÐwÑwÔwÑ€M�1�a˜à	�	Š	�!�Q˜˜1ÑÔ€AÝŒ|˜A 2Ð&Ñ&Ô&€Hõ 	Œ	•+˜a‘.”.Ñ!Ô!€Að 
ˆ1ˆ1ˆ1ˆaˆaˆa����D˜!˜!˜!˜Q˜Q˜QÐÔ	 ! A A A q q q¨$°°°°1°1°1°a°a°aÐ$7Ô"8Ñ	8×=Ò=À"Ð=ÑEÔE€Að 
ˆ9Œ˜Ÿ	š	 ! Q¨¨1¨aÑ0Ô0°Ô;Ñ	;×@Ò@ÀRÐ@ÑHÔH€Að �	Œl˜]¨1¨1¨1¨a¨a¨a°¨:Ô6Ñ6×;Ò;ÀÐ;ÑBÔB€Fõ ”9˜X a a a¨¨¨¨A¨A¨A¨r¨s¨s lÔ3°hÑ>Ñ?Ô?€LØ�,×&Ò& q¨"¨b°!Ñ4Ô4°YÔ?Ñ?€GØ�c˜4   �lÔ# m°IÔ&>Ñ>×CÒCÈÐCÑJÔJ€Fð Ð%ð 	�q�q�q˜$�wÔ×"Ò"¨¬¸f¼mÐ"ÑLÔLÐLåÔ˜f Q Q Q¨¨¨ UœmÑ,Ô,ð õ
 ŒY˜¨Ð0°aÐ8Ñ8Ô8€FÝ”)�K­¬¨h°q°q°q¸!¸!¸!¸Q¸Q¸QÀ°{Ô.CÀVÑ(LÔ(LÑMÔMÑNÔN×XÒXÐYZÐ\]Ñ^Ô^€KØ˜oÔ.°¸¸¸¸1¸1¸1¸dÀC¸Ô1HÑH×MÒMÐRSÐMÑTÔT€JØ$ Q Q Q¨¨¨ VÔ,¨j¸¸¸¸B¸Ô.?ˆK€Fõ ”i Ñ)Ô)€OØ�s˜D ! ! !�|”_ v¨a¨a¨a°°°°D¸#¨oÔ'>Ñ>€NØ×Ò˜rÑ"Ô" _×%<Ò%<¸QÀÀ1ÀaÑ%HÔ%HÈÔ%SÑS€Eð �e‰^€FØ�^Š^˜J¨¨I°xÑ@Ô@€FàÐØ˜*Ñ$ˆð �!‚|€|Ø˜˜˜Ð+˜OÐ+Ð+Ô,ˆàð #Ø�{Ð"Ð"à€Mr7   c            	       óÄ   ‡ — e Zd ZdZddededz  defˆ fd„Z ej	        ¦   «         d„ ¦   «         Z
 ed	¦  «        	 	 dd
ej        dedz  dej        dz  fd„¦   «         Zˆ xZS )ÚZamba2MambaMixeruƒ  
    Compute âˆ†, A, B, C, and D the state space parameters and compute the `contextualized_states`.
    A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
    âˆ†, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
    and is why Mamba is called **selective** state spaces)
    NTrd   rÀ   Úinitialize_mixer_weightsc           	      óT  •— t          ¦   «                              ¦   «          || _        | j        j        | _        |j        | _        |j        | _        |j        | _	        t          |j        | j        z  ¦  «        | _        || _        |j        | _        d| _        t!          j        ¦   «         | _        |j        | _        |j        | _        |j        | _        |j        t3          d¦  «        f| _        | j        d| j        z  | j        z  z   | _        t!          j        | j        | j        d|j        | j        |j        dz
  ¬¦  «        | _        | j        | j        z   | j        z   }t!          j        | j        ||j        ¬¦  «        | _         t!          j!        tE          j#        | j        ¦  «        ¦  «        | _$        t!          j!        tE          j#        | j        ¦  «        ¦  «        | _%        tM          | j        | j        | j        z  d¬	¦  «        | _'        t!          j!        tE          j#        | j        ¦  «        ¦  «        | _(        |r)| j$        j)        j*        d
k    r|  +                    ¦   «          t!          j        | j        | j        |j        ¬¦  «        | _,        |j-        |         | _.        d S )NrA   r  r:   Tr#   )Úin_channelsÚout_channelsrÅ   Úkernel_sizer  r  rÄ   gñhãˆµøä>)r1   r4   Úmeta)/r*   r+   rd   Ún_mamba_headsrF  r3   Úmamba_d_stateÚssm_state_sizeÚmamba_d_convÚconv_kernel_sizerö   Úmamba_expandÚintermediate_sizerÀ   Úuse_conv_biasr  r   ÚSiLUÚactÚmamba_ngroupsÚn_groupsÚmamba_headdimrv   r  Útime_step_minr^   Útime_step_limitÚconv_dimÚConv1dr  rÐ   Úadd_bias_linearÚin_projr,   r-   Úemptyr*  ÚA_logr&   Únormr,  rb   rˆ   Úinit_zamba2_weightsÚout_projÚlayer_typesÚ
layer_type)r2   rd   rÀ   rr  Úprojection_sizer5   s        €r6   r+   zZamba2MambaMixer.__init__Ž  sC  ø€ Ý‰Œ×ÒÑÔÐØˆŒØœÔ2ˆŒØ!Ô-ˆÔØ$Ô2ˆÔØ &Ô 3ˆÔÝ!$ VÔ%8¸4Ô;KÑ%KÑ!LÔ!LˆÔØ"ˆŒØ#Ô1ˆÔØ ˆŒÝ”7‘9”9ˆŒØÔ,ˆŒØÔ,ˆŒØ Ô+ˆŒà &Ô 4µe¸E±l´lÐCˆÔàÔ.°°T´]Ñ1BÀTÔEXÑ1XÑXˆŒå”iØœØœØØÔ+Ø”=ØÔ'¨!Ñ+ð
ñ 
ô 
ˆŒð Ô0°4´=Ñ@À4Ä>ÑQˆÝ”yØÔØØÔ'ð
ñ 
ô 
ˆŒõ ”|¥E¤K°´Ñ$?Ô$?Ñ@Ô@ˆŒõ ”\¥%¤+¨d¬nÑ"=Ô"=Ñ>Ô>ˆŒ
Ý&ØÔ"¨tÔ/EÈÌÑ/VÐ\`ð
ñ 
ô 
ˆŒ	õ ”�eœk¨$¬.Ñ9Ô9Ñ:Ô:ˆŒØ#ð 	'¨¬Ô(;Ô(@ÀFÒ(JÐ(JØ×$Ò$Ñ&Ô&Ð&Ýœ	 $Ô"8¸$Ô:JÐQWÔQgÐhÑhÔhˆŒà Ô,¨YÔ7ˆŒˆˆr7   c                 ó*  — t          j        d| j        dz   | j        j        t           j        ¬¦  «        }t          j        | j        t          j        |¦  «        ¦  «         t          j	        | j
        ¦  «         t          j	        | j        ¦  «         d S )Nr#   r  )r-   rz   rF  rŒ  rb   r?   Úinitr  ÚlogÚones_r,  r*  )r2   r+  s     r6   rŽ  z$Zamba2MambaMixer.init_zamba2_weightsÄ  sp   € åŒL˜˜DœN¨QÑ.°t´zÔ7HÕPUÔP]Ð^Ñ^Ô^ˆÝŒ
�4”:�uœy¨™|œ|Ñ,Ô,Ð,ÝŒ
�4”6ÑÔÐÝŒ
�4”<Ñ Ô Ð Ð Ð r7   r  rG   Úcache_paramsr¢   c                 ór
  — |j         \  }}}|j        }|d uo|                     | j        ¦  «        }	t	          ||¦  «        }|                      |¦  «        }
t          j        | j         	                    ¦   «         ¦  «         }|d| j
        iz  }| j        r�|€Žt          |
| j        j                             d¦  «        | j        j        | j        |f| j        | j        | j        | j        j        | j        j        | j        j        | j        j        | j        | j        dddœ|¤Ž}|�|S |
                     | j        | j        | j        gd¬¦  «        \  }}}|	r:|j        | j                 j        d         }|j        | j                 j         d         }| !                    dd¦  «        }|	r^|dk    rX|j        | j                 j"        sAtG          ||| j        j                             d¦  «        | j        j        | j        ¬	¦  «        }nu|�"| $                    || j        | j%        ¬
¦  «        }tM          || j        j                             d¦  «        | j        j        fd| j        i|¤Ž}|�|d d …d d …| d …f         }t	          | !                    dd¦  «        |¦  «        }t          j        || j        | j        | j'        z  | j        | j'        z  gd¬¦  «        \  }}}|	�rI|dk    �rB| (                    || j        | j        ¦  «        }| !                    dd¦  «         )                    dd| j        ¦  «        }|d d …d d f          )                    d| j        | j'        ¦  «        }| (                    || j        | j'        ¦  «        }| (                    || j        | j'        ¦  «        }| j        d d …d f          )                    d| j        ¦  «        }| j        d d …d f          )                    d| j        ¦  «        }tU          |||||||fd |ddœ|¤Ž}| (                    |dd¦  «        }nÓ|d u}tW          | (                    ||| j        | j        ¦  «        ||| (                    ||| j        | j'        ¦  «        | (                    ||| j        | j'        ¦  «        f| j        | j        d || j        d|	r|nd | j
        dœ|¤Ž}|r"|\  }}| ,                    || j        ¬¦  «         n|}| -                    ||d¦  «        }|                      ||¦  «        }|                      | .                    |¦  «        ¦  «        }|S )Nr.  r#   F)r,  r  r  r0  r1  r2  r3  r4  r5  r6  r/  r;   r…   r   r:   )r  )r|  r  T)r?  r*  r>  )r  r,  r?  r/  r*  r>  r-  r.  ©rÀ   )/rB   r=   Úhas_previous_staterÀ   r  rŠ  r-   rC  rŒ  r^   r†  r§   r8  r  r/   ÚsqueezerÅ   r*  r,  r  r  r�  r0   r�  rv   rƒ  Úsplitr~  r‡  rF  ÚlayersÚconv_statesÚrecurrent_statesrŠ   Úrecord_pastr  Úupdate_conv_stater|  r#  rz  rC   r†   rO  ro  Úupdate_recurrent_stater˜   r>   )r2   rG   r—  r¢   r{   rE  r  r  r=   Úuse_precomputed_statesÚprojected_statesr+  Úfused_kwargsÚfused_outputrH   Úhidden_states_B_Cr;  r  Úrecurrent_stater<  r=  r,  r*  Úscan_outputÚoutput_final_stateÚscan_resultrj  Úcontextualized_statess                               r6   rO   zZamba2MambaMixer.forwardË  sÍ  € ð "/Ô!4Ñˆ
�G˜QØÔ#ˆØ!-°TÐ!9Ð!m¸l×>]Ò>]Ð^bÔ^lÑ>mÔ>mÐõ 5°]ÀNÑSÔSˆØŸ<š<¨Ñ6Ô6ÐåŒY�t”z×'Ò'Ñ)Ô)Ñ*Ô*Ð*ˆØ ¨TÔ-AÐ BÑBˆØŒ=ð 	$˜\Ð1Ý<Ø Ø”Ô"×*Ò*¨1Ñ-Ô-Ø”Ô Ø”Øðð ”&Øœ?Øœ?Ø#œyÔ/Ø œIÔ6Ø#œ}Ô3Ø!œ]Ô/ØœØœØ!&Ø$)ð!ð ð" ð#ð ˆLð* Ð'Ø#Ð#à&6×&<Ò&<ØÔ# T¤]°D´NÐCÈð '=ñ '
ô '
Ñ#ˆÐ ð "ð 	VØ%Ô,¨T¬^Ô<ÔHÈÔKˆJØ*Ô1°$´.ÔAÔRÐSTÔUˆOð .×7Ò7¸¸1Ñ=Ô=ÐØ!ð 	G g°¢l l¸<Ô;NÈtÌ~Ô;^Ô;j lÝ 4Ø!ØØ”Ô"×*Ò*¨1Ñ-Ô-Ø”Ô Øœ?ð!ñ !ô !ÐÐð Ð'Ø$0×$BÒ$BØ%Ø”NØ%)Ô%:ð %Cñ %ô %Ð!õ !1Ø!Ø”Ô"×*Ò*¨1Ñ-Ô-Ø”Ô ð!ð !ð  œ?ð	!ð
 ð!ð !Ðð Ð'Ø$5°a°a°a¸¸¸¸W¸H¸I¸I°oÔ$FÐ!õ 9Ð9J×9TÒ9TÐUVÐXYÑ9ZÔ9ZÐ\jÑkÔkÐÝ#œkØØÔ# T¤]°TÔ5HÑ%HÈ$Ì-ÐZ^ÔZmÑJmÐnØð
ñ 
ô 
Ñˆ�q˜!ð "ñ 2	G g°¢l¡lØ)×.Ò.¨z¸4¼>È4Ì=ÑYÔYˆMØ—’˜a Ñ#Ô#×*Ò*¨2¨r°4´=ÑAÔAˆBØ�!�!�!�T˜4�-Ô ×'Ò'¨¨D¬M¸4Ô;NÑOÔOˆAØ—’�z 4¤=°$Ô2EÑFÔFˆAØ—’�z 4¤=°$Ô2EÑFÔFˆAØ”�q�q�q˜$�w”×&Ò& r¨4¬=Ñ9Ô9ˆAØ”l 1 1 1 d 7Ô+×2Ò2°2°t´}ÑEÔEˆGå7ØØØØØØØðð ØØ ðð ð ðð ˆKð &×*Ò*¨:°q¸"Ñ=Ô=ˆKˆKð ".°TÐ!9ÐÝ+Ø×"Ò" :¨w¸¼ÈÌÑVÔVØØØ—’�z 7¨D¬M¸4Ô;NÑOÔOØ—’�z 7¨D¬M¸4Ô;NÑOÔOðð  œ?Ø”&ØØ$6ØœØ Ø2HÐR˜˜ÈdØÔ-ðð ð ðð ˆKð" "ð *Ø+6Ñ(�˜[Ø×3Ò3°KÈ4Ì>Ð3ÑZÔZÐZÐZà)�à%×-Ò-¨j¸'À2ÑFÔFˆKà—i’i ¨TÑ2Ô2ˆð !%§¢¨k¯nªn¸UÑ.CÔ.CÑ DÔ DÐØ$Ð$r7   )NT©NN)rQ   rR   rS   rõ   r$   rö   r  r+   r-   r•   rŽ  r   r_   r	   rO   rT   rU   s   @r6   rq  rq  }  sø   ø€ € € € € ðð ð48ð 48˜|ð 48¸¸d¹
ð 48Ðeið 48ð 48ð 48ð 48ð 48ð 48ðl €U„]�_„_ð!ð !ñ „_ð!ð Ð˜HÑ%Ô%ð &*Ø.2ð	P%ð P%à”|ðP%ð ˜d‘lðP%ð œ tÑ+ð	P%ð P%ð P%ñ &Ô%ðP%ð P%ð P%ð P%ð P%r7   rq  c                   ó8   ‡ — e Zd Zddededz  fˆ fd„Zdd„Zˆ xZS )Ú	Zamba2MLPNrd   rÂ   c           	      ón  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        || _        || _        t          j        | j        d| j        z  |j	        ¬¦  «        | _
        t          j        | j        | j        |j	        ¬¦  «        | _        t          |j                 | _        t          j        g ¦  «        | _        t#          | j        ¦  «        D ]£}||j        z  |k    rft          j        t          j        | j        j        | j        j        d¬¦  «        t          j        | j        j        d| j        z  d¬¦  «        ¦  «        }nt          j        ¦   «         }| j                             |¦  «         Œ¤|j        }d„ t1          |¦  «        D ¦   «         | _        dS )aQ  
        This MLP layer contributes to tied transformer blocks aimed to increasing compute without increasing model size. Because this layer
        is tied, un-tied adapter modules (formally same as LoRA, but used in the base model) are added to the up and gate projectors to increase expressivity with a small memory overhead.
        r:   rÄ   Fc                 ó   — i | ]\  }}||“Œ	S rÇ   rÇ   rÈ   s      r6   rË   z&Zamba2MLP.__init__.<locals>.<dictcomp>|  s   € ÐVÐVÐV©<¨5°%˜% ÐVÐVÐVr7   N)r*   r+   rd   r3   r~  rÁ   rÂ   r   rÐ   r‰  Úgate_up_projÚ	down_projr   Ú
hidden_actÚact_fnrØ   Úgate_up_proj_adapter_listrÜ   rÝ   rÞ   rß   rà   rá   rÕ   râ   rã   )r2   rd   rÁ   rÂ   rä   Úgate_up_proj_adapterrÖ   r5   s          €r6   r+   zZamba2MLP.__init__`  s‰  ø€ õ
 	‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ!'Ô!9ˆÔØ"4ˆÔØ ˆŒåœI dÔ&6¸¸DÔ<RÑ8RÐY_ÔYoÐpÑpÔpˆÔÝœ 4Ô#9¸4Ô;KÐRXÔRhÐiÑiÔiˆŒÝ˜VÔ.Ô/ˆŒå)+¬°rÑ):Ô):ˆÔ&Ý�tÔ.Ñ/Ô/ð 	Hð 	HˆAØ�6Ô(Ñ(¨HÒ4Ð4Ý')¤}Ý”I˜dœkÔ5°t´{Ô7OÐV[Ð\Ñ\Ô\Ý”I˜dœkÔ6¸¸DÔ<RÑ8RÐY^Ð_Ñ_Ô_ñ(ô (Ð$Ð$õ
 (*¤{¡}¤}Ð$ØÔ*×1Ò1Ð2FÑGÔGÐGÐGà Ô1ˆØVÐV½9À_Ñ;UÔ;UÐVÑVÔVˆŒˆˆr7   c                 ó  — |                       |¦  «        }| j        |         }| | j        |         |¦  «        z   }t          j        |dd¬¦  «        }|                      |d         ¦  «        |d         z  }|                      |¦  «        }|S )Nr:   r;   r…   r   r#   )r²  rã   r¶  r-   Úchunkrµ  r³  )r2   Úhidden_staterÀ   Úgate_up_statern  s        r6   rO   zZamba2MLP.forward~  s‹   € Ø×)Ò)¨,Ñ7Ô7ˆØ”N 9Ô-ˆ	Ø%Ð(Q¨Ô(FÀyÔ(QÐR^Ñ(_Ô(_Ñ_ˆåœ M°1¸"Ð=Ñ=Ô=ˆØ—{’{ =°Ô#3Ñ4Ô4°}ÀQÔ7GÑGˆØ—’ Ñ-Ô-ˆØˆr7   r­  r)   )rQ   rR   rS   r$   rö   r+   rO   rT   rU   s   @r6   r¯  r¯  _  st   ø€ € € € € ðWð W˜|ð WÐPSÐVZÑPZð Wð Wð Wð Wð Wð Wð<ð ð ð ð ð ð ð r7   r¯  c                   óÆ   ‡ — e Zd Zddededz  dedz  fˆ fd„Z	 	 	 ddej        dej        dedej        dz  d	edz  d
ej	        dz  de
e         deej                 fd„Zˆ xZS )ÚZamba2AttentionDecoderLayerNrd   rÂ   rÀ   c                 ó\  •— t          ¦   «                              ¦   «          || _        t          |j        ¦  «        }t          |d||¬¦  «        | _        t          |||¬¦  «        | _        t          |j
        |j        ¬¦  «        | _        t          |j        |j        ¬¦  «        | _        d S )Nr;   )rÀ   rÁ   rÂ   )rÁ   rÂ   ©r4   )r*   r+   rÂ   rý   rÕ   r¿   Ú	self_attnr¯  Úfeed_forwardrW   rÌ   Úrms_norm_epsÚinput_layernormr3   Úpre_ff_layernorm)r2   rd   rÂ   rÀ   Únum_gsr5   s        €r6   r+   z$Zamba2AttentionDecoderLayer.__init__Š  sž   ø€ Ý‰Œ×ÒÑÔÐØ ˆŒÝ�VÔ,Ñ-Ô-ˆÝ(¨¸2ÐRXÐckÐlÑlÔlˆŒÝ% fÀÐRZÐ[Ñ[Ô[ˆÔÝ,¨VÔ-IÈvÔObÐcÑcÔcˆÔÝ -¨fÔ.@ÀfÔFYÐ ZÑ ZÔ ZˆÔÐÐr7   rG   Úoriginal_hidden_statesr¢   rè   ré   r{   rX   c           	      óâ   — t          j        ||gd¬¦  «        }|                      |¦  «        } | j        d|||||dœ|¤Ž\  }}|                      |¦  «        }|                      ||¦  «        }|S )a  
        Args:
            hidden_states (`torch.FloatTensor`): output of previous Mamba layer of shape `(batch, seq_len, embed_dim)`
            original_hidden_states (`torch.FloatTensor`): word embedding output of shape `(batch, seq_len, embed_dim)`.
                This is concatenated with `hidden_states` (which is the output of the previous (mamba) layer). The
                concatenated tensor is then used as input of the pre-attention RMSNorm
                (see fig. 2 in https://huggingface.co/papers/2405.16712).
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, sequence_length)` where padding elements are indicated by 0.
            past_key_values (`Cache`, *optional*): cached past key and value projection states
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
                Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
                with `head_dim` being the embedding dimension of each attention head.
        r;   r…   )rG   rÀ   r¢   rè   ré   rÇ   )r-   ÚconcatenaterÃ  rÀ  rÄ  rÁ  )	r2   rG   rÆ  rÀ   r¢   rè   ré   r{   r  s	            r6   rO   z#Zamba2AttentionDecoderLayer.forward“  sž   € õ6 Ô)¨=Ð:PÐ*QÐWYÐZÑZÔZˆØ×,Ò,¨]Ñ;Ô;ˆØ)˜4œ>ð 
Ø'ØØ)Ø+Ø 3ð
ð 
ð ð
ð 
Ñˆ�qð ×-Ò-¨mÑ<Ô<ˆØ×)Ò)¨-¸ÑCÔCˆàÐr7   r­  rô   )rQ   rR   rS   r$   rö   r+   r-   r_   r	   Ú
LongTensorr   r   r\   ÚFloatTensorrO   rT   rU   s   @r6   r½  r½  ‰  s  ø€ € € € € ð[ð [˜|ð [°s¸T±zð [ÐUXÐ[_ÑU_ð [ð [ð [ð [ð [ð [ð /3Ø(,Ø7;ð)ð )à”|ð)ð !&¤ð)ð ð	)ð
 œ tÑ+ð)ð  ™ð)ð #Ô-°Ñ4ð)ð Ð+Ô,ð)ð 
ˆuÔ Ô	!ð)ð )ð )ð )ð )ð )ð )ð )r7   r½  c                   óð   ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 	 	 	 ddej        dej        dz  dedz  dej        dz  d	ej        dz  d
edz  de	dz  dej
        dz  dej        dz  dee         dej        fd„Zˆ xZS )ÚZamba2MambaDecoderLayerrd   rÀ   c                 óÂ   •— t          ¦   «                              ¦   «          t          ||¬¦  «        | _        t	          |j        |j        ¬¦  «        | _        || _        d S )N)rd   rÀ   r¿  )	r*   r+   rq  ÚmambarW   r3   rÂ  rÃ  rÀ   )r2   rd   rÀ   r5   s      €r6   r+   z Zamba2MambaDecoderLayer.__init__À  sS   ø€ Ý‰Œ×ÒÑÔÐÝ%¨V¸yÐIÑIÔIˆŒ
Ý,¨VÔ-?ÀVÔEXÐYÑYÔYˆÔØ"ˆŒˆˆr7   NFrG   rÆ  r¢   Úcausal_maskrè   Ú	use_cacher�   Útransformer_hidden_statesr{   rX   c
                 ór   — |}|	�||	z   n|}|                       |¦  «        } | j        d|||dœ|
¤Ž}||z   }|S )aX  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, sequence_length)` where padding elements are indicated by 0.
            past_key_values (`Cache`, *optional*): cached past key and value projection states
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
        N)rG   r—  r¢   rÇ   )rÃ  rÎ  )r2   rG   rÆ  rÀ   r¢   rÏ  rè   rÐ  r�   rÑ  r{   Úresiduals               r6   rO   zZamba2MambaDecoderLayer.forwardÆ  s|   € ð0 !ˆð
 :SÐ9^ˆMÐ5Ñ5Ð5Ðdqð 	ð ×,Ò,¨]Ñ;Ô;ˆà"˜œ
ð 
Ø'Ø(Ø)ð
ð 
ð ð	
ð 
ˆð ! =Ñ0ˆàÐr7   ©NNNNNFNN)rQ   rR   rS   r$   rö   r+   r-   r_   r	   r  rÉ  r   r   rO   rT   rU   s   @r6   rÌ  rÌ  ¿  s$  ø€ € € € € ð#˜|ð #¸ð #ð #ð #ð #ð #ð #ð 7;Ø $Ø.2Ø+/Ø(,Ø!&Ø04Ø9=ð*ð *à”|ð*ð !&¤¨tÑ 3ð*ð ˜‘:ð	*ð
 œ tÑ+ð*ð ”\ DÑ(ð*ð  ™ð*ð ˜$‘;ð*ð Ô&¨Ñ-ð*ð $)¤<°$Ñ#6ð*ð Ð+Ô,ð*ð 
Œð*ð *ð *ð *ð *ð *ð *ð *r7   rÌ  c                   ó8  ‡ — e Zd Zdedej        defˆ fd„Z	 	 	 	 	 	 	 	 ddej	        dej	        dz  d	e
dz  d
ej	        dz  dej	        dz  dedz  dedz  dej        dz  dej        dz  dee         deej        eej        ej        f         dz  f         fd„Zˆ xZS )ÚZamba2HybridLayerÚshared_transformerÚlinearrÎ  c                 ór   •— t          ¦   «                              ¦   «          || _        || _        || _        d S r)   )r*   r+   rØ  Úmamba_decoderr×  )r2   r×  rØ  rÎ  r5   s       €r6   r+   zZamba2HybridLayer.__init__ô  s8   ø€ õ 	‰Œ×ÒÑÔÐØˆŒØ"ˆÔØ"4ˆÔÐÐr7   NFrG   rÆ  rÀ   r¢   rÏ  rè   rÐ  ré   r�   r{   rX   c
           
      ó‚   —  | j         |f||||||	dœ|
¤Ž}|                      |¦  «        } | j        |f|||||dœ|
¤Ž}|S )ap  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            original_hidden_states (`torch.FloatTensor`): word embedding output that will be concatenated with
            hidden activations to form the input of the shared transformer layer.
            layer_idx (`int`): layer number.
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, sequence_length)` where padding elements are indicated by 0.
            past_key_values (`Cache`, *optional*): cached past key and value projection states
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
                Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
                with `head_dim` being the embedding dimension of each attention head.
        )rÆ  rÀ   r¢   rè   ré   r�   )rÑ  r¢   rè   rÐ  ré   )r×  rØ  rÚ  )r2   rG   rÆ  rÀ   r¢   rÏ  rè   rÐ  ré   r�   r{   rÑ  s               r6   rO   zZamba2HybridLayer.forwardü  s™   € ð< %< DÔ$;Øð	%
à#9ØØ&Ø+Ø 3Ø%ð	%
ð 	%
ð ð	%
ð 	%
Ð!ð %)§K¢KÐ0IÑ$JÔ$JÐ!à*˜Ô*Øð
à&?Ø)Ø+ØØ 3ð
ð 
ð ð
ð 
ˆð Ðr7   rÔ  )rQ   rR   rS   r½  r   rÐ   rÌ  r+   r-   r_   rö   r	   r  rÉ  r   r   r\   rÊ  rO   rT   rU   s   @r6   rÖ  rÖ  ó  sR  ø€ € € € € ð5Ø"=ð5ØGIÄyð5ØYpð5ð 5ð 5ð 5ð 5ð 5ð 7;Ø $Ø.2Ø+/Ø(,Ø!&Ø7;Ø04ð4ð 4à”|ð4ð !&¤¨tÑ 3ð4ð ˜‘:ð	4ð
 œ tÑ+ð4ð ”\ DÑ(ð4ð  ™ð4ð ˜$‘;ð4ð #Ô-°Ñ4ð4ð Ô&¨Ñ-ð4ð Ð+Ô,ð4ð 
ˆuÔ  %¨Ô(9¸5Ô;LÐ(LÔ"MÐPTÑ"TÐTÔ	Uð4ð 4ð 4ð 4ð 4ð 4ð 4ð 4r7   rÖ  c                   ó„   ‡ — e Zd ZU eed<   dZdZddgZdgZdZ	dZ
dZdZeedœZ ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )	ÚZamba2PreTrainedModelrd   ÚmodelTrÖ  rÌ  rè   )rG   Ú
attentionsc                 ó  •— t          ¦   «                              |¦  «         t          |t          ¦  «        �rTt	          j        t	          j        | j        j        ¦  «        t          j
        | j        j        ¦  «        t          j
        | j        j        ¦  «        z
  z  t          j
        | j        j        ¦  «        z   ¦  «                             | j        j        ¬¦  «        }|t	          j
        t	          j        | ¦  «         ¦  «        z   }t!          j        |j        |¦  «         t	          j        d|j        dz   ¦  «        }t!          j        |j        t	          j
        |¦  «        ¦  «         t!          j        |j        ¦  «         d S d S )N)rR  r#   )r*   Ú_init_weightsr‡   rq  r-   rC  Úrandrd   rx  Úmathr•  Útime_step_maxr…  rX  Útime_step_floorÚexpm1r”  r  r*  rz   rF  rŒ  r–  r,  )r2   rž   r;  Úinv_dtr+  r5   s        €r6   rá  z#Zamba2PreTrainedModel._init_weightsC  s=  ø€ å‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ.Ñ/Ô/ñ 	!Ý”Ý”
˜4œ;Ô4Ñ5Ô5Ý”8˜DœKÔ5Ñ6Ô6½¼À$Ä+ÔB[Ñ9\Ô9\Ñ\ñ^å”(˜4œ;Ô4Ñ5Ô5ñ6ñô ÷ Še˜œÔ3ˆeÑ4Ô4ð	 ð �%œ)¥U¤[°"°Ñ%5Ô%5Ð$5Ñ6Ô6Ñ6ˆFÝŒJ�v”~ vÑ.Ô.Ð.å”˜Q Ô 0°1Ñ 4Ñ5Ô5ˆAÝŒJ�v”|¥U¤Y¨q¡\¤\Ñ2Ô2Ð2ÝŒJ�v”xÑ Ô Ð Ð Ð ð	!ð 	!r7   )rQ   rR   rS   r$   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_flex_attnÚ_supports_sdpaÚ_is_statefulrÌ  r¿   Ú_can_record_outputsr-   r•   rá  rT   rU   s   @r6   rÝ  rÝ  3  s¤   ø€ € € € € € àÐÐÑØÐØ&*Ð#Ø,Ð.GÐHÐØ#4Ð"5ÐØÐØÐØ€NØ€Là0Ø%ðð Ðð
 €U„]�_„_ð!ð !ð !ð !ñ „_ð!ð !ð !ð !ð !r7   rÝ  c                   óò   ‡ — e Zd ZdZdefˆ fd„Zeee	 	 	 	 	 	 dde	j
        dz  de	j        dz  de	j
        dz  dedz  d	e	j        dz  d
edz  dee         deez  fd„¦   «         ¦   «         ¦   «         Zd„ Zˆ xZS )ÚZamba2Modelzh
    Model consisting of *config.num_hidden_layers* layers.

    Args:
        config: Zamba2Config
    rd   c                 ó,  •— t          ¦   «                              |¦  «         || _        |j        | _        |j        | _        t          j        |j        |j        | j        ¦  «        | _	        |j
        | _
        |                      ¦   «         | _        |j        | _        t          |j        |j        ¬¦  «        | _        |j        r5|j        rt&                               d¦  «         t+          |¦  «        | _        d| _        |                      ¦   «          d S )Nr¿  ze`use_long_context` set to `True`: using rescaled `rope_theta` and extended `max_position_embeddings`.F)r*   r+   rd   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr3   Úembed_tokensÚlayers_block_typeÚ
get_layersr�  rî   rW   rÂ  Úfinal_layernormrë   Úuse_long_contextÚloggerÚwarning_oncera   Ú
rotary_embÚgradient_checkpointingÚ	post_init©r2   rd   r5   s     €r6   r+   zZamba2Model.__init__^  sû   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒØ!Ô.ˆÔØ Ô+ˆŒåœL¨Ô):¸FÔ<NÐPTÔP`ÑaÔaˆÔØ!'Ô!9ˆÔØ—o’oÑ'Ô'ˆŒà$*Ô$?ˆÔ!Ý,¨VÔ-?ÀVÔEXÐYÑYÔYˆÔØÔð 	<ØÔ&ð Ý×#Ò#Ø{ñô ð õ 4°FÑ;Ô;ˆDŒOØ&+ˆÔ#ð 	�ŠÑÔÐÐÐr7   NÚ	input_idsr¢   r�   rè   Úinputs_embedsrÐ  r{   rX   c           	      óø  — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|}t          j        |¦  «        }	|r|€t	          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}
t          j        |j        d         |j	        ¬¦  «        |
z   }| 
                    d¦  «        }t          |x}t          ¦  «        s&| j        ||||dœ}t          di |¤Žt          di |¤Ždœ}| j        j        r|                      ||¬¦  «        }nd }t#          | j        ¦  «        D ]#\  }} |||	||d	         |d
         f||||dœ|¤Ž}Œ$|                      |¦  «        }t)          ||r|nd ¬¦  «        S )NzaYou cannot specify both input_ids and inputs_embeds at the same time, and must specify either one)rd   r   r#   rA  )rd   r  r¢   rè   r�   )Úfull_attentionÚlinear_attention)r�   r  r  )rè   rÐ  ré   r�   )Úlast_hidden_staterè   rÇ   )Ú
ValueErrorrù  r-   rq   r
   rd   Úget_seq_lengthrz   rB   rb   r·   r‡   Údictr   r   rë   r   râ   r�  rü  r   )r2   r  r¢   r�   rè   r  rÐ  r{   rG   rÆ  Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsré   rÀ   Úlayers                   r6   rO   zZamba2Model.forwardu  s  € ð ˜Ð -°tÐ";Ñ<ð 	ÝØsñô ð ð Ð Ø ×-Ò-¨iÑ8Ô8ˆMà%ˆå!&¤¨]Ñ!;Ô!;Ðð ð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå°Ð?Ð-ÅÑFÔFð 	àœ+Ø!.Ø"0Ø#2Ø ,ðð ˆKõ #5Ð"CÐ"C°{Ð"CÐ"CÝ$CÐ$RÐ$RÀkÐ$RÐ$Rð#ð #Ðð Œ;Ô#ð 	'Ø"&§/¢/°-Èl /Ñ"[Ô"[ÐÐà"&Ðå )¨$¬+Ñ 6Ô 6ð 	ð 	ÑˆI�uØ!˜EØØ&ØØ#Ð$6Ô7Ø#Ð$4Ô5ðð !0Ø#Ø$7Ø)ðð ð ðð ˆMˆMð ×,Ò,¨]Ñ;Ô;ˆå&Ø+Ø/8ÐB˜O˜O¸dð
ñ 
ô 
ð 	
r7   c                 ó  — g }i | _         d| _        g }d}t          | j        ¦  «        D �]P\  }}t	          | j        |¬¦  «        }|dk    �rd|› d�}t          |t          ¦  «        rt          |¦  «        | j        j	        k    rPt          |t          ¦  «        rt          |¦  «        }t          |¦  «        }| j                              ||i¦  «         n|                     |¦  «         || j        j	        z  }	|dz  }t          | j        |	¬¦  «        }
t          j        | j        j        | j        j        d¬	¦  «        }|                     t%          |
||¦  «        ¦  «         �Œ;|                     |¦  «         �ŒRt          j        |¦  «        S )
Nr   r™  Úhybridzlayers.z.shared_transformerr#   )rÂ   FrÄ   )Ú_tied_weights_keysÚfirst_transformer_layer_idrâ   rú  rÌ  rd   r‡   Úlistrý   rÝ   r   Únextrì   rá   r½  r   rÐ   r3   rÖ  rØ   )r2   r�  Úunique_hybrid_blocksÚhybrid_layer_countÚlayer_idr‘  Úmamba_layerÚprefix_patternÚtarget_patternrÂ   Ú
attn_blockÚlinear_layers               r6   rû  zZamba2Model.get_layers¿  sž  € ØˆØ"$ˆÔØ*+ˆÔ'Ø!ÐØÐå$-¨dÔ.DÑ$EÔ$Eð 	+ñ 	+Ñ ˆH�jÝ1°$´+ÈÐRÑRÔRˆKØ˜XÒ%Ñ%Ø!H¨8Ð!HÐ!HÐ!H�õ #Ð#7½Ñ>Ô>ð
@åÐ/Ñ0Ô0°D´KÔ4NÒNÐNå!Ð"6½Ñ=Ô=ð KÝ/4Ð5IÑ/JÔ/JÐ,Ý%)Ð*>Ñ%?Ô%?�NØÔ+×2Ò2°NÀNÐ3SÑTÔTÐTÐTð )×/Ò/°Ñ?Ô?Ð?ð
 .°´Ô0JÑJ�Ø" aÑ'Ð"Ý8¸¼ÈxÐXÑXÔX�
Ý!œy¨¬Ô)@À$Ä+ÔBYÐ`eÐfÑfÔf�Ø—’Õ/°
¸LÈ+ÑVÔVÑWÔWÐWÑWà—’˜kÑ*Ô*Ð*Ñ*ÝŒ}˜VÑ$Ô$Ð$r7   )NNNNNN)rQ   rR   rS   rõ   r$   r+   r!   r"   r   r-   rÉ  r_   r	   rÊ  r  r   r   r\   r   rO   rû  rT   rU   s   @r6   ró  ró  U  s:  ø€ € € € € ðð ð˜|ð ð ð ð ð ð ð.  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ðE
ð E
àÔ# dÑ*ðE
ð œ tÑ+ðE
ð Ô&¨Ñ-ð	E
ð
  ™ðE
ð Ô(¨4Ñ/ðE
ð ˜$‘;ðE
ð Ð+Ô,ðE
ð 
Ð(Ñ	(ðE
ð E
ð E
ñ „^ñ „_ñ  ÔðE
ðN%%ð %%ð %%ð %%ð %%ð %%ð %%r7   ró  c                   ó  ‡ — e Zd ZddiZdefˆ fd„Zee	 	 	 	 	 	 	 	 ddej	        dz  dej
        dz  d	ej	        dz  d
edz  dej        dz  dej	        dz  dedz  deej
        z  dee         deez  fd„¦   «         ¦   «         Zˆ fd„Zˆ xZS )ÚZamba2ForCausalLMzlm_head.weightzmodel.embed_tokens.weightrd   c                 óþ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        | _        t          j        |j        |j        d¬¦  «        | _        |  	                    ¦   «          d S ©NFrÄ   )
r*   r+   ró  rÞ  r÷  r   rÐ   r3   Úlm_headr  r  s     €r6   r+   zZamba2ForCausalLM.__init__ë  sj   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý  Ñ(Ô(ˆŒ
Ø Ô+ˆŒÝ”y Ô!3°VÔ5FÈUÐSÑSÔSˆŒð 	�ŠÑÔÐÐÐr7   Nr   r  r¢   r�   rè   r  ÚlabelsrÐ  Úlogits_to_keepr{   rX   c	           
      óD  —  | j         d||||||dœ|	¤Ž}
|
j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }d}|� | j        ||| j        fi |	¤Ž}t          |||
j	        |
j
        |
j        ¬¦  «        S )al  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, Zamba2ForCausalLM

        >>> model = Zamba2ForCausalLM.from_pretrained("Zyphra/Zamba2-7B-v1")
        >>> tokenizer = AutoTokenizer.from_pretrained("Zyphra/Zamba2-7B-v1")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```)r  r¢   r�   rè   r  rÐ  N©ÚlossÚlogitsrè   rG   rß  rÇ   )rÞ  r	  r‡   rö   Úslicer#  Úloss_functionr÷  r   rè   rG   rß  )r2   r  r¢   r�   rè   r  r$  rÐ  r%  r{   ÚoutputsrG   Úslice_indicesr)  r(  s                  r6   rO   zZamba2ForCausalLM.forwardô  sþ   € ðH ,6¨4¬:ð ,
ØØ)Ø%Ø+Ø'Øð,
ð ,
ð ð,
ð ,
ˆð  Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%ØØØ”ðð ð ð	ð ˆDõ &ØØØ#Ô3Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r7   c                 óZ   •— | j         j        |d<    t          ¦   «         j        |fi |¤Ž}|S )Nr%  )rd   Únum_logits_to_keepr*   Úprepare_inputs_for_generation)r2   r  r{   Úmodel_inputsr5   s       €r6   r0  z/Zamba2ForCausalLM.prepare_inputs_for_generation8  s7   ø€ Ø#'¤;Ô#AˆÐÑ Ø<•u‘w”wÔ<¸YÐQÐQÈ&ÐQÐQˆØÐr7   ©NNNNNNNr   )rQ   rR   rS   r  r$   r+   r   r   r-   rÉ  r_   r	   rÊ  r  rö   r   r   r\   r   rO   r0  rT   rU   s   @r6   r   r   è  sg  ø€ € € € € Ø*Ð,GÐHÐð˜|ð ð ð ð ð ð ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ð@
ð @
àÔ# dÑ*ð@
ð œ tÑ+ð@
ð Ô&¨Ñ-ð	@
ð
  ™ð@
ð Ô(¨4Ñ/ð@
ð Ô  4Ñ'ð@
ð ˜$‘;ð@
ð ˜eœlÑ*ð@
ð Ð+Ô,ð@
ð 
Ð'Ñ	'ð@
ð @
ð @
ñ „^ñ Ôð@
ðDð ð ð ð ð ð ð ð r7   r   aÌ  
    The Zamba2 Model with a sequence classification head on top (linear layer).

    [`Zamba2ForSequenceClassification`] uses the last token in order to do the classification, as other causal models
    (e.g. GPT-2) do.

    Since it does classification on the last token, it requires to know the position of the last token. If a
    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
    each row of the batch).
    )Úcustom_introc                   ó  ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 	 	 	 ddej        dz  dej	        dz  dej        dz  de
dz  d	ej        dz  d
ej        dz  dedz  deej	        z  dee         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚZamba2ForSequenceClassificationrd   c                 óþ   •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          j        |j        | j        d¬¦  «        | _        |  	                    ¦   «          d S r"  )
r*   r+   Ú
num_labelsró  rÞ  r   rÐ   r3   Úscorer  r  s     €r6   r+   z(Zamba2ForSequenceClassification.__init__M  si   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒÝ  Ñ(Ô(ˆŒ
Ý”Y˜vÔ1°4´?ÈÐOÑOÔOˆŒ
ð 	�ŠÑÔÐÐÐr7   Nr   r  r¢   r�   rè   r  r$  rÐ  r%  r{   rX   c	           	      ó  —  | j         |f|||||dœ|	¤Ž}
|
d         }|                      |¦  «        }|�|j        d         }n|j        d         }| j        j        €|dk    rt          d¦  «        ‚| j        j        €d}n¨|�}|| j        j        k                         |j        t          j	        ¦  «        }t          j
        |j        d         |j        t          j	        ¬¦  «        }||z                       d¦  «        }n)d}t                               | j        j        › d�¦  «         |t          j
        ||j        ¬	¦  «        |f         }d}|� | j        d|||| j        d
œ|	¤Ž}t#          |||
j        |
j        |
j        ¬¦  «        S )a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        )r¢   r�   rè   r  rÐ  r   Nr#   z=Cannot handle batch sizes > 1 if no padding token is defined.r;   r  zŠ will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`rA  )r)  r$  Úpooled_logitsrd   r'  rÇ   )rÞ  r8  rB   rd   rõ  r
  r>   rb   r-   Úint32rz   Úargmaxrþ  rÿ  r5   rQ   r+  r   rè   rG   rß  )r2   r  r¢   r�   rè   r  r$  rÐ  r%  r{   Útransformer_outputsrG   r)  rE  Úlast_non_pad_tokenÚnon_pad_maskÚtoken_indicesr:  r(  s                      r6   rO   z'Zamba2ForSequenceClassification.forwardV  sç  € ð( 8B°t´zØð8
à)Ø%Ø+Ø'Øð8
ð 8
ð ð8
ð 8
Ðð ,¨AÔ.ˆØ—’˜MÑ*Ô*ˆàÐ Ø"œ¨Ô+ˆJˆJà&Ô,¨QÔ/ˆJàŒ;Ô#Ð+°
¸a²°ÝÐ\Ñ]Ô]Ð]ØŒ;Ô#Ð+Ø!#ÐÐØÐ"Ø%¨¬Ô)AÒA×EÒEÀfÄmÕUZÔU`ÑaÔaˆLÝ!œL¨¬¸Ô)<ÀVÄ]ÕZ_ÔZeÐfÑfÔfˆMØ"/°,Ñ">×!FÒ!FÀrÑ!JÔ!JÐÐà!#ÐÝ×ÒØ”>Ô*ð Zð Zð Zñô ð ð
 �uœ|¨J¸v¼}ÐMÑMÔMÐOaÐaÔbˆàˆØÐØ%�4Ô%ð Ø$¨VÀ=ÐY]ÔYdðð Øhnðð ˆDõ 0ØØ Ø/Ô?Ø-Ô;Ø*Ô5ð
ñ 
ô 
ð 	
r7   r2  )rQ   rR   rS   r$   r+   r   r   r-   rÉ  r_   r	   rÊ  r  rö   r   r   r\   r   rO   rT   rU   s   @r6   r5  r5  >  sE  ø€ € € € € ð˜|ð ð ð ð ð ð ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ð@
ð @
àÔ# dÑ*ð@
ð œ tÑ+ð@
ð Ô&¨Ñ-ð	@
ð
  ™ð@
ð Ô(¨4Ñ/ð@
ð Ô  4Ñ'ð@
ð ˜$‘;ð@
ð ˜eœlÑ*ð@
ð Ð+Ô,ð@
ð 
Ð1Ñ	1ð@
ð @
ð @
ñ „^ñ Ôð@
ð @
ð @
ð @
ð @
r7   r5  )r   r5  ró  rÝ  )r�   )r#   r­  )NNFN)_rã  Úcollections.abcr   Ú	itertoolsr   r-   Útorch.nn.functionalr   r@   r  Ú r   r”  Úactivationsr   Úcache_utilsr	   r
   Ú
generationr   Úintegrationsr   r   r   Úintegrations.accelerater   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.deprecationr   Úutils.genericr    r!   Úutils.output_capturingr"   Úconfiguration_zamba2r$   Ú
get_loggerrQ   rþ  ÚModuler&   rW   ra   r_   rö   rœ   r^   r°   r´   r½   r¿   r   r  r  r  r,   r‰   r  r#  r\   r  r8  rO  ro  rq  r¯  r½  rÌ  rÖ  rÝ  ró  r   r5  Ú__all__rÇ   r7   r6   ú<module>rX     s%
  ðð* €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )Ø tÐ tÐ tÐ tÐ tÐ tÐ tÐ tÐ tÐ tØ =Ð =Ð =Ð =Ð =Ð =Ø PÐ PÐ PÐ PÐ PÐ PÐ PÐ PØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø qÐ qÐ qÐ qÐ qÐ qÐ qÐ qÐ qÐ qØ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RØ 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø .Ð .Ð .Ð .Ð .Ð .ð 
ˆÔ	˜HÑ	%Ô	%€ð;ð ;ð ;ð ;ð ;˜œœñ ;ô ;ð ;ð*Jð Jð Jð Jð J�B”Iñ Jô Jð Jð(6<ð 6<ð 6<ð 6<ð 6<˜BœIñ 6<ô 6<ð 6<ðr	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð& ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð %ð %ð %ð2(ð (ð (ð ÐÐ-Ñ.Ô.ðð ð ñ /Ô.ðð2y)ð y)ð y)ð y)ð y)�b”iñ y)ô y)ð y)ð~V U¤\ð V¸Sð Vð Vð Vð Vð
ð 
ð 
ð(ð ð ð(	ð 	ð 	ð (Ð'Ð(>ÀÑPÔPð
 !%Ø!ð'ð 'Ø”<ð'à”ð'ð ŒLð'ð Œ,˜Ñ
ð	'ð
 �d‘
ð'ð 'ð 'ñ QÔPð'ð& (Ð'Ð(:¸OÑLÔLð !%Ø!ð	'ð 'Ø”<ð'àŒLð'ð Œ,˜Ñ
ð'ð �d‘
ð	'ð 'ð 'ñ MÔLð'ð, (Ð'Ð(JÈKÑXÔXð +/Ø%(¨%¨%°©,¬,Ð$7Ø %ØØ*.ØØ*.Ø(,ØØØ!ð%ð ØŒLðà”<ðð ” Ñ$ðð Œ\ð	ð
 „|ðð „|ðð ðð ”L 4Ñ'ðð �E˜5�LÔ!ðð ðð ðð ”L 4Ñ'ðð ðð ”L 4Ñ'ðð ”, Ñ%ðð  �4‰Zð!ð" ð#ð$ ð%ð ð ñ YÔXðð0 (Ð'Ð(@À+ÑNÔNð "Ø#'ØØ!ð:'ð :'ØŒ<ð:'à”<ð:'ð 	Œð:'ð „|ð	:'ð
 „|ð:'ð „|ð:'ð „|�dÑð:'ð Œ\˜DÑ ð:'ð ð:'ð „|�dÑð:'ð :'ð :'ñ OÔNð:'ðz (Ð'Ð(CÀ[ÑQÔQð "Ø#'Ø*.ØØ%(¨%¨%°©,¬,Ð$7Ø %ð^ð ^Ø”<ð^àŒð^ð „|ð^ð „|ð	^ð
 „|ð^ð ð^ð „|�dÑð^ð Œ\˜DÑ ð^ð ”L 4Ñ'ð^ð ð^ð �E˜5�LÔ!ð^ð ð^ð ^ð ^ñ RÔQð^ðB ÐàØØ)Ø%Øðñô ðV%ð V%ð V%ð V%ð V%�r”yñ V%ô V%ñô ðV%ðr'ð 'ð 'ð 'ð '�”	ñ 'ô 'ð 'ðT3ð 3ð 3ð 3ð 3 "¤)ñ 3ô 3ð 3ðl1ð 1ð 1ð 1ð 1Ð8ñ 1ô 1ð 1ðh=ð =ð =ð =ð =Ð2ñ =ô =ð =ð@ ð!ð !ð !ð !ð !˜Oñ !ô !ñ „ð!ðB ðN%ð N%ð N%ð N%ð N%Ð'ñ N%ô N%ñ „ðN%ðdSð Sð Sð Sð SÐ-¨ñ Sô Sð Sðl €ððñ ô ðL
ð L
ð L
ð L
ð L
Ð&;ñ L
ô L
ñô ðL
ð^ kÐ
jÐ
j€€€r7   