§
    9²¸jls  ã                   óæ  — d dl Z d dlmZ d dlmZ d dlZd dlmZ ddlmZ	 ddl
mZ ddlmZmZ dd	lmZmZ dd
lmZmZ ddlmZmZ ddlmZ ddlmZmZmZmZ ddlm Z  ddl!m"Z" ddl#m$Z$ ddl%m&Z&m'Z' ddl(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1 ddl2m3Z3  ej4        e5¦  «        Z6 G d„ dej        j7        ¦  «        Z8 G d„ de0¦  «        Z9 G d„ de&¦  «        Z: G d„ de)¦  «        Z; G d„ de$¦  «        Z< G d „ d!ej7        ¦  «        Z= G d"„ d#e*¦  «        Z> G d$„ d%e.¦  «        Z? G d&„ d'e-¦  «        Z@e G d(„ d)e¦  «        ¦   «         ZA G d*„ d+e/eA¦  «        ZB G d,„ d-e+¦  «        ZC G d.„ d/e,¦  «        ZDg d0¢ZEdS )1é    N)ÚCallable)Úcycle)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)Úcreate_causal_maskÚcreate_recurrent_attention_mask)ÚBaseModelOutputWithPastÚ SequenceClassifierOutputWithPast)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmerge_with_config_defaults)Úcapture_outputsé   )Ú
BambaMixer)ÚLlamaRotaryEmbeddingÚapply_rotary_pos_emb)	ÚZambaAttentionÚZambaAttentionDecoderLayerÚZambaForCausalLMÚZambaForSequenceClassificationÚZambaHybridLayerÚZambaMambaDecoderLayerÚ
ZambaModelÚZambaRMSNormÚeager_attention_forwardé   )ÚZamba2Configc                   ó(   ‡ — e Zd Zdˆ fd„	Zdd„Zˆ xZS )ÚZamba2RMSNormGatedç�íµ ÷Æ°>c                 óº   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        || _        d S ©N)	ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilonÚ
group_size)ÚselfÚhidden_sizer3   ÚepsÚ	__class__s       €úy/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/zamba2/modular_zamba2.pyr-   zZamba2RMSNormGated.__init__4   sG   ø€ Ý‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔØ$ˆŒˆˆó    Nc                 ó  — |j         }|                     t          j        ¦  «        }|�?|t          j                             |                     t          j        ¦  «        ¦  «        z  }|j        �^ }}|| j        z  } |j	        g |¢|‘| j        ‘R Ž }| 
                    d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  } |j	        g |¢|| j        z  ‘R Ž }| j        |                     |¦  «        z  S )Nr   éÿÿÿÿT)Úkeepdim)ÚdtypeÚtor/   Úfloat32r   Ú
functionalÚsiluÚshaper3   ÚviewÚpowÚmeanÚrsqrtr2   r1   )	r4   Úhidden_statesÚgateÚinput_dtypeÚprefix_dimsÚlast_dimÚgroup_countÚhidden_states_groupÚvariances	            r8   ÚforwardzZamba2RMSNormGated.forward:   s  € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØÐØ)­B¬M×,>Ò,>¸t¿wºwÅuÄ}Ñ?UÔ?UÑ,VÔ,VÑVˆMØ!.Ô!4Ñˆ�hØ $¤/Ñ1ˆØ0˜mÔ0Ð\°+Ð\¸{Ð\ÈDÌOÐ\Ð\Ð\ÐØ&×*Ò*¨1Ñ-Ô-×2Ò2°2¸tÐ2ÑDÔDˆØ1µE´KÀÈ4ÔK`Ñ@`Ñ4aÔ4aÑaÐØ0Ð+Ô0Ð]°+Ð]¸{ÈTÌ_Ñ?\Ð]Ð]Ð]ˆØŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:r9   )r)   r+   )Ú__name__Ú
__module__Ú__qualname__r-   rO   Ú__classcell__©r7   s   @r8   r(   r(   3   sQ   ø€ € € € € ð%ð %ð %ð %ð %ð %ð;ð ;ð ;ð ;ð ;ð ;ð ;ð ;r9   r(   c                   ó   — e Zd ZdS )ÚZamba2RMSNormN©rP   rQ   rR   © r9   r8   rV   rV   H   ó   € € € € € Ø€Dr9   rV   c                   ó   — e Zd ZdS )ÚZamba2RotaryEmbeddingNrW   rX   r9   r8   r[   r[   L   rY   r9   r[   c                   ó  ‡ — e Zd ZdZ	 	 	 ddededz  dedz  dedz  fˆ fd„Z	 	 	 ddej        ded	ej        dz  d
e	dz  de
ej        ej        f         dz  dee         de
ej        ej        dz  e
ej                 dz  f         fd„Zˆ xZS )ÚZamba2AttentionaZ  
    Multi-headed attention from 'Attention Is All You Need' paper.

    Adapted from transformers.models.mistral.modeling_mistral.MistralAttention:
    The input dimension here is attention_hidden_size = 2 * hidden_size, and head_dim = attention_hidden_size // num_heads.
    The extra factor of 2 comes from the input being the concatenation of original_hidden_states with the output of the previous (mamba) layer
    (see fig. 2 in https://huggingface.co/papers/2405.16712).
    Additionally, replaced
    attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) with
    attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim/2)
    Finally, this attention layer contributes to tied transformer blocks aimed to increasing compute without increasing model size. Because this
    layer is tied, un-tied adapters (formally the same as LoRA but used in the base model) modules are added to the q, k, v projectors to increase
    expressivity with a small memory overhead (see Fig. 2 of https://huggingface.co/papers/2411.15242).
    NÚconfigÚ	layer_idxÚnum_fwd_mem_blocksÚblock_idc           	      óú  •— t          ¦   «                              ||¦  «         || _        |j        | _        || _        |j        �rt          j        g ¦  «        | _	        t          j        g ¦  «        | _
        t          j        g ¦  «        | _        t          | j        ¦  «        D �]±}||j        z  |k    �rt          j        t          j        | j        | j        j        d¬¦  «        t          j        | j        j        | j        d¬¦  «        ¦  «        }t          j        t          j        | j        | j        j        d¬¦  «        t          j        | j        j        | j        d¬¦  «        ¦  «        }t          j        t          j        | j        | j        j        d¬¦  «        t          j        | j        j        | j        d¬¦  «        ¦  «        }n9t          j        ¦   «         }t          j        ¦   «         }t          j        ¦   «         }| j	                             |¦  «         | j
                             |¦  «         | j                             |¦  «         �Œ³d„ t+          | j        ¦  «        D ¦   «         | _        d S )NF©Úbiasc                 ó   — i | ]\  }}||“Œ	S rX   rX   ©Ú.0ÚindexÚvalues      r8   ú
<dictcomp>z,Zamba2Attention.__init__.<locals>.<dictcomp>‡   s   € Ð[Ð[Ð[©<¨5°%˜% Ð[Ð[Ð[r9   )r,   r-   r`   Úhybrid_layer_idsÚlayer_block_mapra   Úuse_shared_attention_adapterr   Ú
ModuleListÚlinear_q_adapter_listÚlinear_k_adapter_listÚlinear_v_adapter_listÚrangeÚnum_mem_blocksÚ
SequentialÚLinearÚattention_hidden_sizer^   Úadapter_rankÚIdentityÚappendÚ	enumerateÚ	layer_dic)
r4   r^   r_   r`   ra   ÚiÚlinear_q_adapterÚlinear_k_adapterÚlinear_v_adapterr7   s
            €r8   r-   zZamba2Attention.__init__`   s*  ø€ õ 	‰Œ×Ò˜ Ñ+Ô+Ð+Ø"4ˆÔØ%Ô6ˆÔØ ˆŒàÔ.ñ 	DÝ)+¬°rÑ):Ô):ˆDÔ&Ý)+¬°rÑ):Ô):ˆDÔ&Ý)+¬°rÑ):Ô):ˆDÔ&å˜4Ô2Ñ3Ô3ð Dñ D�Ø�vÔ,Ñ,°Ò8Ñ8Ý')¤}Ýœ	 $Ô"<¸d¼kÔ>VÐ]bÐcÑcÔcÝœ	 $¤+Ô":¸DÔ<VÐ]bÐcÑcÔcñ(ô (Ð$õ (*¤}Ýœ	 $Ô"<¸d¼kÔ>VÐ]bÐcÑcÔcÝœ	 $¤+Ô":¸DÔ<VÐ]bÐcÑcÔcñ(ô (Ð$õ (*¤}Ýœ	 $Ô"<¸d¼kÔ>VÐ]bÐcÑcÔcÝœ	 $¤+Ô":¸DÔ<VÐ]bÐcÑcÔcñ(ô (Ð$Ð$õ
 (*¤{¡}¤}Ð$Ý')¤{¡}¤}Ð$Ý')¤{¡}¤}Ð$ØÔ*×1Ò1Ð2BÑCÔCÐCØÔ*×1Ò1Ð2BÑCÔCÐCØÔ*×1Ò1Ð2BÑCÔCÐCÑCà[Ð[½9ÀTÔEYÑ;ZÔ;ZÐ[Ñ[Ô[ˆŒˆˆr9   rG   Úattention_maskÚpast_key_valuesÚposition_embeddingsÚkwargsÚreturnc                 ó  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «        }	|                      |¦  «        }
|                      |¦  «        }| j        j        rX| j        |         }|	 | j        |         |¦  «        z   }	|
 | j	        |         |¦  «        z   }
| | j
        |         |¦  «        z   }|	                     |¦  «                             dd¦  «        }	|
                     |¦  «                             dd¦  «        }
|                     |¦  «                             dd¦  «        }| j        j        r|\  }}t          |	|
||¦  «        \  }	}
|�|                     |
||¦  «        \  }
}t!          j        | j        j        t&          ¦  «        } || |	|
||f| j        sdn| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Nr;   r%   r   g        )ÚdropoutÚscaling)rB   Úhead_dimÚq_projÚk_projÚv_projr^   rm   r{   ro   rp   rq   rC   Ú	transposeÚuse_mem_roper   Úupdater   Úget_interfaceÚ_attn_implementationr$   ÚtrainingÚattention_dropoutr‡   ÚreshapeÚ
contiguousÚo_proj)r4   rG   r_   r€   r�   r‚   rƒ   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚadapter_layer_idxÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss                     r8   rO   zZamba2Attention.forward‰   sM  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1ˆØ—[’[ Ñ/Ô/ˆ
Ø—{’{ =Ñ1Ô1ˆØŒ;Ô3ð 	gØ $¤¨yÔ 9ÐØ'Ð*W¨$Ô*DÐEVÔ*WÐXeÑ*fÔ*fÑfˆLØ#Ð&S dÔ&@ÐARÔ&SÐTaÑ&bÔ&bÑbˆJØ'Ð*W¨$Ô*DÐEVÔ*WÐXeÑ*fÔ*fÑfˆLà#×(Ò(¨Ñ6Ô6×@Ò@ÀÀAÑFÔFˆØ—_’_ \Ñ2Ô2×<Ò<¸QÀÑBÔBˆ
Ø#×(Ò(¨Ñ6Ô6×@Ò@ÀÀAÑFÔFˆàŒ;Ô#ð 	`Ø*‰HˆC�Ý';¸LÈ*ÐVYÐ[^Ñ'_Ô'_Ñ$ˆL˜*àÐ&Ø'6×'=Ò'=¸jÈ,ÐXaÑ'bÔ'bÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r9   ©NNN)rP   rQ   rR   Ú__doc__r&   Úintr-   r/   ÚTensorr	   Útupler   r   rO   rS   rT   s   @r8   r]   r]   P   sD  ø€ € € € € ðð ð$ !%Ø)-Ø#ð'\ð '\àð'\ð ˜‘:ð'\ð   $™Jð	'\ð
 ˜‘*ð'\ð '\ð '\ð '\ð '\ð '\ðZ /3Ø(,ØHLð1)ð 1)à”|ð1)ð ð1)ð œ tÑ+ð	1)ð
  ™ð1)ð # 5¤<°´Ð#=Ô>ÀÑEð1)ð Ð+Ô,ð1)ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	Mð1)ð 1)ð 1)ð 1)ð 1)ð 1)ð 1)ð 1)r9   r]   c                   ó8   ‡ — e Zd ZdZddededz  defˆ fd„Zˆ xZS )	ÚZamba2MambaMixeruƒ  
    Compute âˆ†, A, B, C, and D the state space parameters and compute the `contextualized_states`.
    A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
    âˆ†, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
    and is why Mamba is called **selective** state spaces)
    NTr^   r_   Úinitialize_mixer_weightsc           	      óò  •— || _         t          ¦   «                              |||¦  «         |j        | _        d| _        t          j        ¦   «         | _        |j        | _	        |j
        | _        | j         j        | _        |j        | _        |j        t!          d¦  «        f| _        t          j        | j        | j        d|j        | j        |j        dz
  ¬¦  «        | _        | j        | j        z   | j        z   }t          j        | j        ||j        ¬¦  «        | _        t7          | j        | j        | j	        z  d¬¦  «        | _        t          j        | j        | j        |j        ¬¦  «        | _        | `| `d S )	NrA   ÚinfTr%   )Úin_channelsÚout_channelsrd   Úkernel_sizeÚgroupsÚpaddingrc   gñhãˆµøä>)r3   r6   ) r^   r,   r-   Úuse_conv_biasÚ
activationr   ÚSiLUÚactÚmamba_ngroupsÚn_groupsÚmamba_headdimrˆ   Ún_mamba_headsÚ	num_headsÚ
chunk_sizeÚtime_step_minÚfloatÚtime_step_limitÚConv1dÚconv_dimÚmamba_d_convÚconv1dÚintermediate_sizeru   r5   Úadd_bias_linearÚin_projr(   ÚnormÚout_projÚlayer_norm_epsilonÚuse_bias)r4   r^   r_   r¨   Úprojection_sizer7   s        €r8   r-   zZamba2MambaMixer.__init__Å   s`  ø€ ØˆŒÝ‰Œ×Ò˜ Ð,DÑEÔEÐEØ#Ô1ˆÔØ ˆŒÝ”7‘9”9ˆŒØÔ,ˆŒØÔ,ˆŒØœÔ2ˆŒØ Ô+ˆŒà &Ô 4µe¸E±l´lÐCˆÔå”iØœØœØØÔ+Ø”=ØÔ'¨!Ñ+ð
ñ 
ô 
ˆŒð Ô0°4´=Ñ@À4Ä>ÑQˆÝ”yØÔØØÔ'ð
ñ 
ô 
ˆŒõ
 'ØÔ"¨tÔ/EÈÌÑ/VÐ\`ð
ñ 
ô 
ˆŒ	õ œ	 $Ô"8¸$Ô:JÐQWÔQgÐhÑhÔhˆŒØÐ#ØˆMˆMˆMr9   )NT)	rP   rQ   rR   r¢   r&   r£   Úboolr-   rS   rT   s   @r8   r§   r§   ½   sf   ø€ € € € € ðð ð"ð "˜|ð "¸¸d¹
ð "Ðeið "ð "ð "ð "ð "ð "ð "ð "ð "ð "r9   r§   c                   ó8   ‡ — e Zd Zddededz  fˆ fd„Zdd„Zˆ xZS )Ú	Zamba2MLPNr^   ra   c           	      ón  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        || _        || _        t          j        | j        d| j        z  |j	        ¬¦  «        | _
        t          j        | j        | j        |j	        ¬¦  «        | _        t          |j                 | _        t          j        g ¦  «        | _        t#          | j        ¦  «        D ]£}||j        z  |k    rft          j        t          j        | j        j        | j        j        d¬¦  «        t          j        | j        j        d| j        z  d¬¦  «        ¦  «        }nt          j        ¦   «         }| j                             |¦  «         Œ¤|j        }d„ t1          |¦  «        D ¦   «         | _        dS )aQ  
        This MLP layer contributes to tied transformer blocks aimed to increasing compute without increasing model size. Because this layer
        is tied, un-tied adapter modules (formally same as LoRA, but used in the base model) are added to the up and gate projectors to increase expressivity with a small memory overhead.
        r   rc   Fc                 ó   — i | ]\  }}||“Œ	S rX   rX   rf   s      r8   rj   z&Zamba2MLP.__init__.<locals>.<dictcomp>  s   € ÐVÐVÐV©<¨5°%˜% ÐVÐVÐVr9   N)r,   r-   r^   r5   rÁ   r`   ra   r   ru   rÂ   Úgate_up_projÚ	down_projr   Ú
hidden_actÚact_fnrn   Úgate_up_proj_adapter_listrr   rs   rt   rw   rx   ry   rk   rz   r{   )r4   r^   r`   ra   r|   Úgate_up_proj_adapterrl   r7   s          €r8   r-   zZamba2MLP.__init__ë   s‰  ø€ õ
 	‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ!'Ô!9ˆÔØ"4ˆÔØ ˆŒåœI dÔ&6¸¸DÔ<RÑ8RÐY_ÔYoÐpÑpÔpˆÔÝœ 4Ô#9¸4Ô;KÐRXÔRhÐiÑiÔiˆŒÝ˜VÔ.Ô/ˆŒå)+¬°rÑ):Ô):ˆÔ&Ý�tÔ.Ñ/Ô/ð 	Hð 	HˆAØ�6Ô(Ñ(¨HÒ4Ð4Ý')¤}Ý”I˜dœkÔ5°t´{Ô7OÐV[Ð\Ñ\Ô\Ý”I˜dœkÔ6¸¸DÔ<RÑ8RÐY^Ð_Ñ_Ô_ñ(ô (Ð$Ð$õ
 (*¤{¡}¤}Ð$ØÔ*×1Ò1Ð2FÑGÔGÐGÐGà Ô1ˆØVÐV½9À_Ñ;UÔ;UÐVÑVÔVˆŒˆˆr9   c                 ó  — |                       |¦  «        }| j        |         }| | j        |         |¦  «        z   }t          j        |dd¬¦  «        }|                      |d         ¦  «        |d         z  }|                      |¦  «        }|S )Nr   r;   ©Údimr   r%   )rÎ   r{   rÒ   r/   ÚchunkrÑ   rÏ   )r4   Úhidden_stater_   Úgate_up_stateÚoutputs        r8   rO   zZamba2MLP.forward	  s‹   € Ø×)Ò)¨,Ñ7Ô7ˆØ”N 9Ô-ˆ	Ø%Ð(Q¨Ô(FÀyÔ(QÐR^Ñ(_Ô(_Ñ_ˆåœ M°1¸"Ð=Ñ=Ô=ˆØ—{’{ =°Ô#3Ñ4Ô4°}ÀQÔ7GÑGˆØ—’ Ñ-Ô-ˆØˆr9   ©NNr+   )rP   rQ   rR   r&   r£   r-   rO   rS   rT   s   @r8   rË   rË   ê   st   ø€ € € € € ðWð W˜|ð WÐPSÐVZÑPZð Wð Wð Wð Wð Wð Wð<ð ð ð ð ð ð ð r9   rË   c                   óÆ   ‡ — e Zd Zddededz  dedz  fˆ fd„Z	 	 	 ddej        dej        dedej        dz  d	edz  d
ej	        dz  de
e         deej                 fd„Zˆ xZS )ÚZamba2AttentionDecoderLayerNr^   ra   r_   c                 óà   •— || _         t          |j        ¦  «        }t          ¦   «                              ||¦  «         t          |d||¬¦  «        | _        t          |||¬¦  «        | _        d S )Nr;   )r_   r`   ra   )r`   ra   )	ra   Úlenrk   r,   r-   r]   Ú	self_attnrË   Úfeed_forward)r4   r^   ra   r_   Únum_gsr7   s        €r8   r-   z$Zamba2AttentionDecoderLayer.__init__  sl   ø€ Ø ˆŒÝ�VÔ,Ñ-Ô-ˆÝ‰Œ×Ò˜ Ñ+Ô+Ð+Ý(¨¸2ÐRXÐckÐlÑlÔlˆŒÝ% fÀÐRZÐ[Ñ[Ô[ˆÔÐÐr9   rG   Úoriginal_hidden_statesr€   r�   r‚   rƒ   r„   c           	      óâ   — t          j        ||gd¬¦  «        }|                      |¦  «        } | j        d|||||dœ|¤Ž\  }}|                      |¦  «        }|                      ||¦  «        }|S )a  
        Args:
            hidden_states (`torch.FloatTensor`): output of previous Mamba layer of shape `(batch, seq_len, embed_dim)`
            original_hidden_states (`torch.FloatTensor`): word embedding output of shape `(batch, seq_len, embed_dim)`.
                This is concatenated with `hidden_states` (which is the output of the previous (mamba) layer). The
                concatenated tensor is then used as input of the pre-attention RMSNorm
                (see fig. 2 in https://huggingface.co/papers/2405.16712).
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, sequence_length)` where padding elements are indicated by 0.
            past_key_values (`Cache`, *optional*): cached past key and value projection states
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
                Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
                with `head_dim` being the embedding dimension of each attention head.
        r;   rÕ   )rG   r_   r€   r�   r‚   rX   )r/   ÚconcatenateÚinput_layernormrà   Úpre_ff_layernormrá   )	r4   rG   rã   r_   r€   r�   r‚   rƒ   Ú_s	            r8   rO   z#Zamba2AttentionDecoderLayer.forward  sž   € õ6 Ô)¨=Ð:PÐ*QÐWYÐZÑZÔZˆØ×,Ò,¨]Ñ;Ô;ˆØ)˜4œ>ð 
Ø'ØØ)Ø+Ø 3ð
ð 
ð ð
ð 
Ñˆ�qð ×-Ò-¨mÑ<Ô<ˆØ×)Ò)¨-¸ÑCÔCˆàÐr9   rÛ   r¡   )rP   rQ   rR   r&   r£   r-   r/   r¤   r	   Ú
LongTensorr   r   r¥   ÚFloatTensorrO   rS   rT   s   @r8   rÝ   rÝ     s  ø€ € € € € ð\ð \˜|ð \°s¸T±zð \ÐUXÐ[_ÑU_ð \ð \ð \ð \ð \ð \ð /3Ø(,Ø7;ð)ð )à”|ð)ð !&¤ð)ð ð	)ð
 œ tÑ+ð)ð  ™ð)ð #Ô-°Ñ4ð)ð Ð+Ô,ð)ð 
ˆuÔ Ô	!ð)ð )ð )ð )ð )ð )ð )ð )r9   rÝ   c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚZamba2MambaDecoderLayerr^   r_   c                 ó¸   •— t          ¦   «                              ||¦  «         t          ||¬¦  «        | _        t	          |j        |j        ¬¦  «        | _        d S )N)r^   r_   ©r6   )r,   r-   r§   ÚmambarV   r5   Úrms_norm_epsræ   )r4   r^   r_   r7   s      €r8   r-   z Zamba2MambaDecoderLayer.__init__I  sR   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+Ý%¨V¸yÐIÑIÔIˆŒ
Ý,¨VÔ-?ÀVÔEXÐYÑYÔYˆÔÐÐr9   )rP   rQ   rR   r&   r£   r-   rS   rT   s   @r8   rì   rì   H  sW   ø€ € € € € ðZ˜|ð Z¸ð Zð Zð Zð Zð Zð Zð Zð Zð Zð Zr9   rì   c                   ó8  ‡ — e Zd Zdedej        defˆ fd„Z	 	 	 	 	 	 	 	 ddej	        dej	        dz  d	e
dz  d
ej	        dz  dej	        dz  dedz  dedz  dej        dz  dej        dz  dee         deej        eej        ej        f         dz  f         fd„Zˆ xZS )ÚZamba2HybridLayerÚshared_transformerÚlinearrï   c                 ó`   •— t          ¦   «                              |||¦  «         | `|| _        d S r+   )r,   r-   Úshared_transfró   )r4   ró   rô   rï   r7   s       €r8   r-   zZamba2HybridLayer.__init__P  s6   ø€ õ 	‰Œ×ÒÐ+¨V°UÑ;Ô;Ð;ØÐØ"4ˆÔÐÐr9   NFrG   rã   r_   r€   Úcausal_maskr�   Ú	use_cacher‚   Úposition_idsrƒ   r„   c
           
      ó‚   —  | j         |f||||||	dœ|
¤Ž}|                      |¦  «        } | j        |f|||||dœ|
¤Ž}|S )ap  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            original_hidden_states (`torch.FloatTensor`): word embedding output that will be concatenated with
            hidden activations to form the input of the shared transformer layer.
            layer_idx (`int`): layer number.
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, sequence_length)` where padding elements are indicated by 0.
            past_key_values (`Cache`, *optional*): cached past key and value projection states
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
                Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
                with `head_dim` being the embedding dimension of each attention head.
        )rã   r_   r€   r�   r‚   rù   )Útransformer_hidden_statesr€   r�   rø   r‚   )ró   rô   Úmamba_decoder)r4   rG   rã   r_   r€   r÷   r�   rø   r‚   rù   rƒ   rû   s               r8   rO   zZamba2HybridLayer.forwardW  s™   € ð< %< DÔ$;Øð	%
à#9ØØ&Ø+Ø 3Ø%ð	%
ð 	%
ð ð	%
ð 	%
Ð!ð %)§K¢KÐ0IÑ$JÔ$JÐ!à*˜Ô*Øð
à&?Ø)Ø+ØØ 3ð
ð 
ð ð
ð 
ˆð Ðr9   )NNNNNFNN)rP   rQ   rR   rÝ   r   ru   rì   r-   r/   r¤   r£   r	   rÉ   ré   r   r   r¥   rê   rO   rS   rT   s   @r8   rò   rò   O  sR  ø€ € € € € ð5Ø"=ð5ØGIÄyð5ØYpð5ð 5ð 5ð 5ð 5ð 5ð 7;Ø $Ø.2Ø+/Ø(,Ø!&Ø7;Ø04ð4ð 4à”|ð4ð !&¤¨tÑ 3ð4ð ˜‘:ð	4ð
 œ tÑ+ð4ð ”\ DÑ(ð4ð  ™ð4ð ˜$‘;ð4ð #Ô-°Ñ4ð4ð Ô&¨Ñ-ð4ð Ð+Ô,ð4ð 
ˆuÔ  %¨Ô(9¸5Ô;LÐ(LÔ"MÐPTÑ"TÐTÔ	Uð4ð 4ð 4ð 4ð 4ð 4ð 4ð 4r9   rò   c                   ó„   ‡ — e Zd ZU eed<   dZdZddgZdgZdZ	dZ
dZdZeedœZ ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )	ÚZamba2PreTrainedModelr^   ÚmodelTrò   rì   r�   )rG   Ú
attentionsc                 ó  •— t          ¦   «                              |¦  «         t          |t          ¦  «        �rTt	          j        t	          j        | j        j        ¦  «        t          j
        | j        j        ¦  «        t          j
        | j        j        ¦  «        z
  z  t          j
        | j        j        ¦  «        z   ¦  «                             | j        j        ¬¦  «        }|t	          j
        t	          j        | ¦  «         ¦  «        z   }t!          j        |j        |¦  «         t	          j        d|j        dz   ¦  «        }t!          j        |j        t	          j
        |¦  «        ¦  «         t!          j        |j        ¦  «         d S d S )N)Úminr%   )r,   Ú_init_weightsÚ
isinstancer§   r/   ÚexpÚrandr^   r·   ÚmathÚlogÚtime_step_maxrº   ÚclampÚtime_step_floorÚexpm1ÚinitÚcopy_Údt_biasÚaranger¸   ÚA_logÚones_ÚD)r4   ÚmoduleÚdtÚinv_dtÚAr7   s        €r8   r  z#Zamba2PreTrainedModel._init_weightsž  s=  ø€ å‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ.Ñ/Ô/ñ 	!Ý”Ý”
˜4œ;Ô4Ñ5Ô5Ý”8˜DœKÔ5Ñ6Ô6½¼À$Ä+ÔB[Ñ9\Ô9\Ñ\ñ^å”(˜4œ;Ô4Ñ5Ô5ñ6ñô ÷ Še˜œÔ3ˆeÑ4Ô4ð	 ð �%œ)¥U¤[°"°Ñ%5Ô%5Ð$5Ñ6Ô6Ñ6ˆFÝŒJ�v”~ vÑ.Ô.Ð.å”˜Q Ô 0°1Ñ 4Ñ5Ô5ˆAÝŒJ�v”|¥U¤Y¨q¡\¤\Ñ2Ô2Ð2ÝŒJ�v”xÑ Ô Ð Ð Ð ð	!ð 	!r9   )rP   rQ   rR   r&   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_flex_attnÚ_supports_sdpaÚ_is_statefulrì   r]   Ú_can_record_outputsr/   Úno_gradr  rS   rT   s   @r8   rþ   rþ   Ž  s¤   ø€ € € € € € àÐÐÑØÐØ&*Ð#Ø,Ð.GÐHÐØ#4Ð"5ÐØÐØÐØ€NØ€Là0Ø%ðð Ðð
 €U„]�_„_ð!ð !ð !ð !ñ „_ð!ð !ð !ð !ð !r9   rþ   c                   óè   — e Zd ZdZdefd„Zd„ Zeee		 	 	 	 	 	 dde
j        dz  de
j        dz  de
j        dz  d	edz  d
e
j        dz  dedz  dee         deez  fd„¦   «         ¦   «         ¦   «         ZdS )ÚZamba2Modelzh
    Model consisting of *config.num_hidden_layers* layers.

    Args:
        config: Zamba2Config
    r^   c                 ó  — t                                | |¦  «         || _        |j        | _        |j        | _        t          j        |j        |j        | j        ¦  «        | _	        |j
        | _
        |                      ¦   «         | _        |j        | _        t          |j        |j        ¬¦  «        | _        |j        r5|j        rt&                               d¦  «         t+          |¦  «        | _        d| _        |                      ¦   «          d S )Nrî   ze`use_long_context` set to `True`: using rescaled `rope_theta` and extended `max_position_embeddings`.F)rþ   r-   r^   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr5   Úembed_tokensÚlayers_block_typeÚ
get_layersÚlayersr�   rV   rð   Úfinal_layernormr�   Úuse_long_contextÚloggerÚwarning_oncer[   Ú
rotary_embÚgradient_checkpointingÚ	post_init)r4   r^   s     r8   r-   zZamba2Model.__init__¸  sø   € Ý×&Ò& t¨VÑ4Ô4Ð4ØˆŒØ!Ô.ˆÔØ Ô+ˆŒåœL¨Ô):¸FÔ<NÐPTÔP`ÑaÔaˆÔØ!'Ô!9ˆÔØ—o’oÑ'Ô'ˆŒà$*Ô$?ˆÔ!Ý,¨VÔ-?ÀVÔEXÐYÑYÔYˆÔØÔð 	<ØÔ&ð Ý×#Ò#Ø{ñô ð õ 4°FÑ;Ô;ˆDŒOØ&+ˆÔ#ð 	�ŠÑÔÐÐÐr9   c                 ó  — g }i | _         d| _        g }d}t          | j        ¦  «        D �]P\  }}t	          | j        |¬¦  «        }|dk    �rd|› d�}t          |t          ¦  «        rt          |¦  «        | j        j	        k    rPt          |t          ¦  «        rt          |¦  «        }t          |¦  «        }| j                              ||i¦  «         n|                     |¦  «         || j        j	        z  }	|dz  }t          | j        |	¬¦  «        }
t          j        | j        j        | j        j        d¬	¦  «        }|                     t%          |
||¦  «        ¦  «         �Œ;|                     |¦  «         �ŒRt          j        |¦  «        S )
Nr   )r_   Úhybridzlayers.z.shared_transformerr%   )ra   Frc   )Ú_tied_weights_keysÚfirst_transformer_layer_idrz   r+  rì   r^   r  Úlistrß   rs   r   ÚnextrŽ   ry   rÝ   r   ru   r5   rò   rn   )r4   r-  Úunique_hybrid_blocksÚhybrid_layer_countÚlayer_idÚ
layer_typeÚmamba_layerÚprefix_patternÚtarget_patternra   Ú
attn_blockÚlinear_layers               r8   r,  zZamba2Model.get_layersÏ  sž  € ØˆØ"$ˆÔØ*+ˆÔ'Ø!ÐØÐå$-¨dÔ.DÑ$EÔ$Eð 	+ñ 	+Ñ ˆH�jÝ1°$´+ÈÐRÑRÔRˆKØ˜XÒ%Ñ%Ø!H¨8Ð!HÐ!HÐ!H�õ #Ð#7½Ñ>Ô>ð
@åÐ/Ñ0Ô0°D´KÔ4NÒNÐNå!Ð"6½Ñ=Ô=ð KÝ/4Ð5IÑ/JÔ/JÐ,Ý%)Ð*>Ñ%?Ô%?�NØÔ+×2Ò2°NÀNÐ3SÑTÔTÐTÐTð )×/Ò/°Ñ?Ô?Ð?ð
 .°´Ô0JÑJ�Ø" aÑ'Ð"Ý8¸¼ÈxÐXÑXÔX�
Ý!œy¨¬Ô)@À$Ä+ÔBYÐ`eÐfÑfÔf�Ø—’Õ/°
¸LÈ+ÑVÔVÑWÔWÐWÑWà—’˜kÑ*Ô*Ð*Ñ*ÝŒ}˜VÑ$Ô$Ð$r9   NÚ	input_idsr€   rù   r�   Úinputs_embedsrø   rƒ   r„   c           	      óø  — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|}t          j        |¦  «        }	|r|€t	          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}
t          j        |j        d         |j	        ¬¦  «        |
z   }| 
                    d¦  «        }t          |x}t          ¦  «        s&| j        ||||dœ}t          di |¤Žt          di |¤Ždœ}| j        j        r|                      ||¬¦  «        }nd }t#          | j        ¦  «        D ]#\  }} |||	||d	         |d
         f||||dœ|¤Ž}Œ$|                      |¦  «        }t)          ||r|nd ¬¦  «        S )NzaYou cannot specify both input_ids and inputs_embeds at the same time, and must specify either one)r^   r   r%   ©Údevice)r^   rE  r€   r�   rù   )Úfull_attentionÚlinear_attention)rù   rJ  rI  )r�   rø   r‚   rù   )Úlast_hidden_stater�   rX   )Ú
ValueErrorr*  r/   Úcloner
   r^   Úget_seq_lengthr  rB   rH  Ú	unsqueezer  Údictr   r   r�   r2  rz   r-  r.  r   )r4   rD  r€   rù   r�   rE  rø   rƒ   rG   rã   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsr‚   r_   Úlayers                   r8   rO   zZamba2Model.forwardö  s  € ð ˜Ð -°tÐ";Ñ<ð 	ÝØsñô ð ð Ð Ø ×-Ò-¨iÑ8Ô8ˆMà%ˆå!&¤¨]Ñ!;Ô!;Ðð ð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå°Ð?Ð-ÅÑFÔFð 	àœ+Ø!.Ø"0Ø#2Ø ,ðð ˆKõ #5Ð"CÐ"C°{Ð"CÐ"CÝ$CÐ$RÐ$RÀkÐ$RÐ$Rð#ð #Ðð Œ;Ô#ð 	'Ø"&§/¢/°-Èl /Ñ"[Ô"[ÐÐà"&Ðå )¨$¬+Ñ 6Ô 6ð 	ð 	ÑˆI�uØ!˜EØØ&ØØ#Ð$6Ô7Ø#Ð$4Ô5ðð !0Ø#Ø$7Ø)ðð ð ðð ˆMˆMð ×,Ò,¨]Ñ;Ô;ˆå&Ø+Ø/8ÐB˜O˜O¸dð
ñ 
ô 
ð 	
r9   )NNNNNN)rP   rQ   rR   r¢   r&   r-   r,  r   r   r   r/   ré   r¤   r	   rê   rÉ   r   r   r¥   r   rO   rX   r9   r8   r$  r$  °  s'  € € € € € ðð ð˜|ð ð ð ð ð.%%ð %%ð %%ðN  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ðE
ð E
àÔ# dÑ*ðE
ð œ tÑ+ðE
ð Ô&¨Ñ-ð	E
ð
  ™ðE
ð Ô(¨4Ñ/ðE
ð ˜$‘;ðE
ð Ð+Ô,ðE
ð 
Ð(Ñ	(ðE
ð E
ð E
ñ „^ñ „_ñ  ÔðE
ð E
ð E
r9   r$  c                   ó   — e Zd ZdS )ÚZamba2ForCausalLMNrW   rX   r9   r8   rV  rV  A  rY   r9   rV  c                   ó  ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 	 	 	 ddej        dz  dej	        dz  dej        dz  de
dz  d	ej        dz  d
ej        dz  dedz  deej	        z  dee         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚZamba2ForSequenceClassificationr^   c                 óš   •— t          ¦   «                              |¦  «         t          |¦  «        | _        |                      ¦   «          d S r+   )r,   r-   r$  rÿ   r4  )r4   r^   r7   s     €r8   r-   z(Zamba2ForSequenceClassification.__init__F  s@   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý  Ñ(Ô(ˆŒ
Ø�ŠÑÔÐÐÐr9   Nr   rD  r€   rù   r�   rE  Úlabelsrø   Úlogits_to_keeprƒ   r„   c	           	      ó  —  | j         |f|||||dœ|	¤Ž}
|
d         }|                      |¦  «        }|�|j        d         }n|j        d         }| j        j        €|dk    rt          d¦  «        ‚| j        j        €d}n¨|�}|| j        j        k                         |j        t          j	        ¦  «        }t          j
        |j        d         |j        t          j	        ¬¦  «        }||z                       d¦  «        }n)d}t                               | j        j        › d�¦  «         |t          j
        ||j        ¬	¦  «        |f         }d}|� | j        d|||| j        d
œ|	¤Ž}t#          |||
j        |
j        |
j        ¬¦  «        S )a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        )r€   rù   r�   rE  rø   r   Nr%   z=Cannot handle batch sizes > 1 if no padding token is defined.r;   )rH  r=   zŠ will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`rG  )ÚlogitsrZ  Úpooled_logitsr^   )Úlossr]  r�   rG   r   rX   )rÿ   ÚscorerB   r^   r&  rL  r>   rH  r/   Úint32r  Úargmaxr0  r1  r7   rP   Úloss_functionr   r�   rG   r   )r4   rD  r€   rù   r�   rE  rZ  rø   r[  rƒ   Útransformer_outputsrG   r]  Ú
batch_sizeÚlast_non_pad_tokenÚnon_pad_maskÚtoken_indicesr^  r_  s                      r8   rO   z'Zamba2ForSequenceClassification.forwardK  sç  € ð( 8B°t´zØð8
à)Ø%Ø+Ø'Øð8
ð 8
ð ð8
ð 8
Ðð ,¨AÔ.ˆØ—’˜MÑ*Ô*ˆàÐ Ø"œ¨Ô+ˆJˆJà&Ô,¨QÔ/ˆJàŒ;Ô#Ð+°
¸a²°ÝÐ\Ñ]Ô]Ð]ØŒ;Ô#Ð+Ø!#ÐÐØÐ"Ø%¨¬Ô)AÒA×EÒEÀfÄmÕUZÔU`ÑaÔaˆLÝ!œL¨¬¸Ô)<ÀVÄ]ÕZ_ÔZeÐfÑfÔfˆMØ"/°,Ñ">×!FÒ!FÀrÑ!JÔ!JÐÐà!#ÐÝ×ÒØ”>Ô*ð Zð Zð Zñô ð ð
 �uœ|¨J¸v¼}ÐMÑMÔMÐOaÐaÔbˆàˆØÐØ%�4Ô%ð Ø$¨VÀ=ÐY]ÔYdðð Øhnðð ˆDõ 0ØØ Ø/Ô?Ø-Ô;Ø*Ô5ð
ñ 
ô 
ð 	
r9   )NNNNNNNr   )rP   rQ   rR   r&   r-   r   r   r/   ré   r¤   r	   rê   rÉ   r£   r   r   r¥   r   rO   rS   rT   s   @r8   rX  rX  E  sE  ø€ € € € € ð˜|ð ð ð ð ð ð ð
 Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ð@
ð @
àÔ# dÑ*ð@
ð œ tÑ+ð@
ð Ô&¨Ñ-ð	@
ð
  ™ð@
ð Ô(¨4Ñ/ð@
ð Ô  4Ñ'ð@
ð ˜$‘;ð@
ð ˜eœlÑ*ð@
ð Ð+Ô,ð@
ð 
Ð1Ñ	1ð@
ð @
ð @
ñ „^ñ Ôð@
ð @
ð @
ð @
ð @
r9   rX  )rV  rX  r$  rþ   )Fr  Úcollections.abcr   Ú	itertoolsr   r/   r   Ú r   r  Úactivationsr   Úcache_utilsr	   r
   Úmasking_utilsr   r   Úmodeling_outputsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   Úutils.output_capturingr   Úbamba.modeling_bambar   Úllama.modeling_llamar   r   Úzamba.modeling_zambar   r   r   r   r    r!   r"   r#   r$   Úconfiguration_zamba2r&   Ú
get_loggerrP   r0  ÚModuler(   rV   r[   r]   r§   rË   rÝ   rì   rò   rþ   r$  rV  rX  Ú__all__rX   r9   r8   ú<module>r|     sg  ðð €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø PÐ PÐ PÐ PÐ PÐ PÐ PÐ PØ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RØ 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø -Ð -Ð -Ð -Ð -Ð -Ø MÐ MÐ MÐ MÐ MÐ MÐ MÐ Mð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð /Ð .Ð .Ð .Ð .Ð .ð 
ˆÔ	˜HÑ	%Ô	%€ð;ð ;ð ;ð ;ð ;˜œœñ ;ô ;ð ;ð*	ð 	ð 	ð 	ð 	�Lñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð0ñ 	ô 	ð 	ðj)ð j)ð j)ð j)ð j)�nñ j)ô j)ð j)ðZ*ð *ð *ð *ð *�zñ *ô *ð *ðZ'ð 'ð 'ð 'ð '�”	ñ 'ô 'ð 'ðT1ð 1ð 1ð 1ð 1Ð"<ñ 1ô 1ð 1ðhZð Zð Zð Zð ZÐ4ñ Zô Zð Zð<ð <ð <ð <ð <Ð(ñ <ô <ð <ð~ ð!ð !ð !ð !ð !˜Oñ !ô !ñ „ð!ðBN
ð N
ð N
ð N
ð N
�*Ð3ñ N
ô N
ð N
ðb	ð 	ð 	ð 	ð 	Ð(ñ 	ô 	ð 	ðH
ð H
ð H
ð H
ð H
Ð&Dñ H
ô H
ð H
ðVð ð €€€r9   