§
    9²¸j}[  ã                   óÌ  — d dl mZ d dlZd dlmZ ddlmZ ddlmZ ddl	m
Z
mZ ddlmZmZ dd	lmZmZ dd
lmZmZ ddlmZmZ ddlmZ ddlmZmZmZ ddlmZ ddl m!Z!m"Z" ddl#m$Z$ ddl%m&Z&m'Z'm(Z( ddl)m*Z*m+Z+m,Z,m-Z-m.Z. ddl/m0Z0 ddl1m2Z2m3Z3 ddl4m5Z5  ej6        e7¦  «        Z8 G d„ de'¦  «        Z9 G d„ de&¦  «        Z: G d„ de$¦  «        Z; G d„ de0¦  «        Z< G d „ d!e2¦  «        Z= G d"„ d#ej>        ¦  «        Z? G d$„ d%e¦  «        Z@ G d&„ d'e¦  «        ZAe@eAd(œZB G d)„ d*e¦  «        ZCe G d+„ d,eC¦  «        ¦   «         ZD G d-„ d.e3¦  «        ZE G d/„ d0eeC¦  «        ZFg d1¢ZGdS )2é    )ÚCallableN)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)Úcreate_causal_maskÚcreate_recurrent_attention_mask)Ú GenericForSequenceClassificationÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)Úmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚFalconMambaMixer)ÚLlamaAttentionÚLlamaRMSNormÚeager_attention_forward)Úapply_mask_to_padding_statesÚcausal_conv1d_fnÚcausal_conv1d_updateÚmamba_selective_scanÚmamba_selective_state_update)Ú
MistralMLP)ÚMixtralExpertsÚMixtralForCausalLMé   )ÚJambaConfigc                   ó   — e Zd ZdS )ÚJambaRMSNormN©Ú__name__Ú
__module__Ú__qualname__© ó    úw/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/jamba/modular_jamba.pyr)   r)   4   ó   € € € € € Ø€Dr/   r)   c                   ó    ‡ — e Zd Zdedefˆ fd„Z	 	 ddej        dej        dz  dedz  de	e
         d	eej        ej        dz  f         f
d
„Zˆ xZS )ÚJambaAttentionÚconfigÚ	layer_idxc                 ó¼  •— t          ¦   «                              ||¦  «         t          j        |j        |j        | j        z  d¬¦  «        | _        t          j        |j        |j        | j        z  d¬¦  «        | _	        t          j        |j        |j        | j        z  d¬¦  «        | _
        t          j        |j        | j        z  |j        d¬¦  «        | _        d S ©NF©Úbias)ÚsuperÚ__init__r   ÚLinearÚhidden_sizeÚnum_attention_headsÚhead_dimÚq_projÚnum_key_value_headsÚk_projÚv_projÚo_proj)Úselfr4   r5   Ú	__class__s      €r0   r;   zJambaAttention.__init__9   s¿   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+Ý”i Ô 2°FÔ4NÐQUÔQ^Ñ4^ÐejÐkÑkÔkˆŒÝ”i Ô 2°FÔ4NÐQUÔQ^Ñ4^ÐejÐkÑkÔkˆŒÝ”i Ô 2°FÔ4NÐQUÔQ^Ñ4^ÐejÐkÑkÔkˆŒÝ”i Ô :¸T¼]Ñ JÈFÔL^ÐejÐkÑkÔkˆŒˆˆr/   NÚhidden_statesÚattention_maskÚpast_key_valuesÚkwargsÚreturnc                 óî  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }	|�|                     ||	| j        ¦  «        \  }}	t          j
        | j        j        t          ¦  «        }
 |
| |||	|f| j        sdn| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Néÿÿÿÿr&   r   ç        )ÚdropoutÚscaling)Úshaper?   r@   ÚviewÚ	transposerB   rC   Úupdater5   r   Úget_interfacer4   Ú_attn_implementationr   ÚtrainingÚattention_dropoutrP   ÚreshapeÚ
contiguousrD   )rE   rG   rH   rI   rJ   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚattention_interfaceÚattn_outputÚattn_weightss                r0   ÚforwardzJambaAttention.forward@   s�  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r/   ©NN)r+   r,   r-   r'   Úintr;   ÚtorchÚTensorr   r   r   Útuplerc   Ú__classcell__©rF   s   @r0   r3   r3   8   sÊ   ø€ € € € € ðl˜{ð l°sð lð lð lð lð lð lð /3Ø(,ð	")ð ")à”|ð")ð œ tÑ+ð")ð  ™ð	")ð
 Ð+Ô,ð")ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð")ð ")ð ")ð ")ð ")ð ")ð ")ð ")r/   r3   c                   ó^   — e Zd ZdZdefd„Zd„ Z	 	 d
dej        de	dz  dej        dz  fd	„Z
dS )ÚJambaMambaMixeruƒ  
    Compute âˆ†, A, B, C, and D the state space parameters and compute the `contextualized_states`.
    A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
    âˆ†, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
    and is why Mamba is called **selective** state spaces)
    r4   c           	      óŒ  — t           j                             | ¦  «         || _        || _        |j        | _        |j        | _        |j        | _	        |j
        |j        z  | _        |j        | _        |j        | _        |j        | _        t          j        | j        | j        | j        | j	        | j        | j	        dz
  ¬¦  «        | _        |j        | _        t,          |j                 | _        t          j        | j        | j        dz  | j        ¬¦  «        | _        t          j        | j        | j        | j        dz  z   d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _        t9          j        d| j        dz   ¦  «        d d d …f         }|                     | j        d¦  «                             ¦   «         }t          j         t9          j!        |¦  «        ¦  «        | _"        t          j         t9          j#        | j        ¦  «        ¦  «        | _$        t          j        | j        | j        | j        ¬¦  «        | _%        tM          | j        |j'        ¬¦  «        | _(        tM          | j        |j'        ¬¦  «        | _)        tM          | j        |j'        ¬¦  «        | _*        |j+        | _+        |j,        | _,        d S )	Nr&   )Úin_channelsÚout_channelsr9   Úkernel_sizeÚgroupsÚpaddingr   r8   FTrM   ©Úeps)-r   ÚModuler;   r4   r5   r=   Úmamba_d_stateÚssm_state_sizeÚmamba_d_convÚconv_kernel_sizeÚmamba_expandÚintermediate_sizeÚmamba_dt_rankÚtime_step_rankÚmamba_conv_biasÚuse_conv_biasÚmamba_proj_biasÚuse_biasÚConv1dÚconv1dÚ
hidden_actÚ
activationr   Úactr<   Úin_projÚx_projÚdt_projrf   ÚarangeÚexpandrZ   Ú	ParameterÚlogÚA_logÚonesÚDÚout_projr)   Úrms_norm_epsÚdt_layernormÚb_layernormÚc_layernormÚuse_mambapyÚuse_associative_scan)rE   r4   r5   ÚAs       r0   r;   zJambaMambaMixer.__init__m   s[  € Ý
Œ	×Ò˜4Ñ Ô Ð ØˆŒØ"ˆŒØ!Ô-ˆÔØ$Ô2ˆÔØ &Ô 3ˆÔØ!'Ô!4°vÔ7IÑ!IˆÔØ$Ô2ˆÔØ#Ô3ˆÔØÔ.ˆŒÝ”iØÔ.ØÔ/ØÔ#ØÔ-ØÔ)ØÔ)¨AÑ-ð
ñ 
ô 
ˆŒð !Ô+ˆŒÝ˜&Ô+Ô,ˆŒõ ”y Ô!1°4Ô3IÈAÑ3MÐTXÔTaÐbÑbÔbˆŒå”i Ô 6¸Ô8KÈdÔNaÐdeÑNeÑ8eÐlqÐrÑrÔrˆŒå”y Ô!4°dÔ6LÐSWÐXÑXÔXˆŒõ ŒL˜˜DÔ/°!Ñ3Ñ4Ô4°T¸1¸1¸1°WÔ=ˆØ�HŠH�TÔ+¨RÑ0Ô0×;Ò;Ñ=Ô=ˆå”\¥%¤)¨A¡,¤,Ñ/Ô/ˆŒ
Ý”�eœj¨Ô)?Ñ@Ô@ÑAÔAˆŒÝœ	 $Ô"8¸$Ô:JÐQUÔQ^Ð_Ñ_Ô_ˆŒå(¨Ô)<À&ÔBUÐVÑVÔVˆÔÝ'¨Ô(;ÀÔATÐUÑUÔUˆÔÝ'¨Ô(;ÀÔATÐUÑUÔUˆÔà!Ô-ˆÔØ$*Ô$?ˆÔ!Ð!Ð!r/   c                 ó    — t          d¦  «        ‚)NzNot needed for jamba)ÚNotImplementedError)rE   s    r0   Úinit_jamba_weightsz"JambaMambaMixer.init_jamba_weights›   s   € Ý!Ð"8Ñ9Ô9Ð9r/   NrG   Úcache_paramsrH   c                 ó¨  — |j         d         }|j        }|d uo|                     | j        ¦  «        }t	          ||¦  «        }|                      |¦  «                             dd¦  «        }t          j        | j	         
                    ¦   «         ¦  «         }	|                     dd¬¦  «        \  }
}|r:|j        | j                 j        d         }|j        | j                 j        d         }|r^|dk    rX|j        | j                 j        sAt!          |
|| j        j                             d¦  «        | j        j        | j        ¬¦  «        }
nu|�"|                     |
| j        | j        ¬¦  «        }
t1          |
| j        j                             d¦  «        | j        j        fd| j        i|¤Ž}
|�|
d d …d d …| d …f         }
t	          |
                     dd¦  «        |¦  «        }
t          j        |                      |
¦  «        | j        | j        | j        gd¬¦  «        \  }}}|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }tA          | j!        d	¦  «        r7|  "                    |¦  «        | j"        j        z
                       dd¦  «        }n#| j"        j        |                     dd¦  «        z  }| j"        j        �| j"        j         
                    ¦   «         nd }|rr|dk    rltG          ||
                     dd¦  «        d
         |d
         |	|d d …df         |d d …df         | j$        |d
         |d¬¦
  «
         %                    d¦  «        }n¡|d u}tM          |
                     dd¦  «        ||	|                     dd¦  «        |                     dd¦  «        | j$         
                    ¦   «         ||d|| j'        | j(        ¬¦  «        }|r!|\  }}| )                    || j        ¦  «         n|}|  *                    |                     dd¦  «         +                    |¦  «        ¦  «        }|S )Nr&   r   ©Údimr   )r…   )ry   r…   rM   Ú_is_quantized).r   T)ÚzÚdt_biasÚdt_softplus)r�   r¡   Ú
delta_biasÚdelta_softplusÚreturn_last_stater–   r—   ),rQ   ÚdtypeÚhas_previous_stater5   r   r‡   rS   rf   ÚexprŽ   ÚfloatÚchunkÚlayersÚconv_statesÚrecurrent_statesÚrecord_pastr    rƒ   ÚweightÚsqueezer9   r…   Úupdate_conv_statery   r   Úsplitrˆ   r}   rw   r“   r”   r•   Úhasattrr4   r‰   r"   r�   Ú	unsqueezer!   r–   r—   Úupdate_recurrent_stater‘   Úto)rE   rG   rœ   rH   rJ   Úseq_lenr§   Úuse_precomputed_statesÚprojected_statesr˜   Úhidden_states_B_CÚgateÚ
conv_stateÚrecurrent_stateÚ	time_stepÚBÚCÚtime_proj_biasÚscan_outputÚoutput_final_stateÚscan_resultÚfinal_stateÚcontextualized_statess                          r0   rc   zJambaMambaMixer.forwardž   s›  € ð  Ô% aÔ(ˆØÔ#ˆØ!-°TÐ!9Ð!m¸l×>]Ò>]Ð^bÔ^lÑ>mÔ>mÐõ 5°]ÀNÑSÔSˆØŸ<š<¨Ñ6Ô6×@Ò@ÀÀAÑFÔFÐõ ŒY�t”z×'Ò'Ñ)Ô)Ñ*Ô*Ð*ˆà"2×"8Ò"8¸ÀÐ"8Ñ"BÔ"BÑÐ˜4à!ð 	VØ%Ô,¨T¬^Ô<ÔHÈÔKˆJØ*Ô1°$´.ÔAÔRÐSTÔUˆOð "ð 	G g°¢l l¸<Ô;NÈtÌ~Ô;^Ô;j lÝ 4Ø!ØØ”Ô"×*Ò*¨1Ñ-Ô-Ø”Ô Øœ?ð!ñ !ô !ÐÐð Ð'Ø$0×$BÒ$BØ%Ø”NØ%)Ô%:ð %Cñ %ô %Ð!õ !1Ø!Ø”Ô"×*Ò*¨1Ñ-Ô-Ø”Ô ð!ð !ð  œ?ð	!ð
 ð!ð !Ðð Ð'Ø$5°a°a°a¸¸¸¸W¸H¸I¸I°oÔ$FÐ!õ 9Ð9J×9TÒ9TÐUVÐXYÑ9ZÔ9ZÐ\jÑkÔkÐÝœ+Ø�KŠKÐ)Ñ*Ô*ØÔ  $Ô"5°tÔ7JÐKØð
ñ 
ô 
‰ˆ	�1�að ×%Ò% iÑ0Ô0ˆ	Ø×Ò˜QÑÔˆØ×Ò˜QÑÔˆõ �4”; Ñ0Ô0ð 	HØŸš iÑ0Ô0°4´<Ô3DÑD×OÒOÐPQÐSTÑUÔUˆIˆIàœÔ+¨i×.AÒ.AÀ!ÀQÑ.GÔ.GÑGˆIØ6:´lÔ6GÐ6S˜œÔ*×0Ò0Ñ2Ô2Ð2ÐY]ˆð "ð $	* g°¢l lÝ6ØØ!×+Ò+¨A¨qÑ1Ô1°&Ô9Ø˜&Ô!ØØ�!�!�!�Q�$”Ø�!�!�!�Q�$”Ø”Ø�v”,Ø&Ø ðñ ô ÷ Ši˜‰mŒmð ˆKð ".°TÐ!9ÐÝ.Ø!×+Ò+¨A¨qÑ1Ô1ØØØ—’˜A˜qÑ!Ô!Ø—’˜A˜qÑ!Ô!Ø”&—,’,‘.”.ØØ)Ø#Ø"4Ø Ô,Ø%)Ô%>ðñ ô ˆKð "ð *Ø+6Ñ(�˜[Ø×3Ò3°KÀÄÑPÔPÐPÐPà)�ð !%§¢¨k×.CÒ.CÀAÀqÑ.IÔ.I×.LÒ.LÈUÑ.SÔ.SÑ TÔ TÐØ$Ð$r/   rd   )r+   r,   r-   Ú__doc__r'   r;   r›   rf   rg   r   rc   r.   r/   r0   rl   rl   e   s¢   € € € € € ðð ð,@˜{ð ,@ð ,@ð ,@ð ,@ð\:ð :ð :ð &*Ø.2ð	s%ð s%à”|ðs%ð ˜d‘lðs%ð œ tÑ+ð	s%ð s%ð s%ð s%ð s%ð s%r/   rl   c                   ó   — e Zd ZdS )ÚJambaMLPNr*   r.   r/   r0   rÊ   rÊ     r1   r/   rÊ   c                   ó   — e Zd ZdS )ÚJambaExpertsNr*   r.   r/   r0   rÌ   rÌ     r1   r/   rÌ   c                   óR   ‡ — e Zd ZdZdefˆ fd„Zd„ Zdej        dej        fd„Z	ˆ xZ
S )ÚJambaSparseMoeBlockaÈ  
    This implementation is
    strictly equivalent to standard MoE with full capacity (no
    dropped tokens). It's faster since it formulates MoE operations
    in terms of block-sparse operations to accommodate imbalanced
    assignments of tokens to experts, whereas standard MoE either
    (1) drop tokens at the cost of reduced performance or (2) set
    capacity factor to number of experts and thus waste computation
    and memory on padding.
    r4   c                 ó  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        |j        | _        t          j
        | j        | j        d¬¦  «        | _        t          |¦  «        | _        d S r7   )r:   r;   r=   Ú
hidden_dimr{   Úffn_dimÚnum_expertsÚnum_experts_per_tokÚtop_kr   r<   ÚrouterrÌ   Úexperts©rE   r4   rF   s     €r0   r;   zJambaSparseMoeBlock.__init__(  sr   ø€ Ý‰Œ×ÒÑÔÐØ Ô,ˆŒØÔ/ˆŒØ!Ô-ˆÔØÔ/ˆŒ
å”i ¤°Ô1AÈÐNÑNÔNˆŒÝ# FÑ+Ô+ˆŒˆˆr/   c                 óÚ   — t           j        j                             |dt           j        ¬¦  «        }t          j        || j        d¬¦  «        \  }}||                     |j        ¦  «        fS )NrM   )rŸ   r§   rž   )	rf   r   Ú
functionalÚsoftmaxrª   ÚtopkrÔ   r·   r§   )rE   rG   Úrouter_logitsÚrouting_weightsÚtop_k_weightsÚtop_k_indexs         r0   Úroute_tokens_to_expertsz+JambaSparseMoeBlock.route_tokens_to_experts2  s_   € Ýœ(Ô-×5Ò5°mÈÕSXÔS^Ð5Ñ_Ô_ˆÝ%*¤Z°ÀÄÐQSÐ%TÑ%TÔ%TÑ"ˆ�{Ø˜M×,Ò,¨]Ô-@ÑAÔAÐAÐAr/   rG   rK   c                 ó   — |j         \  }}}|                     d|¦  «        }|                      |¦  «        }|                      ||¦  «        \  }}|                      |||¦  «        }|                     |||¦  «        }|S )NrM   )rQ   rR   rÕ   rà   rÖ   rY   )rE   rG   Ú
batch_sizeÚsequence_lengthrÐ   rÜ   rß   rÞ   s           r0   rc   zJambaSparseMoeBlock.forward7  s„   € Ø2?Ô2EÑ/ˆ
�O ZØ%×*Ò*¨2¨zÑ:Ô:ˆØŸš MÑ2Ô2ˆØ%)×%AÒ%AÀ-ÐQ^Ñ%_Ô%_Ñ"ˆ�]ØŸš ]°KÀÑOÔOˆØ%×-Ò-¨j¸/È:ÑVÔVˆØÐr/   )r+   r,   r-   rÈ   r'   r;   rà   rf   rg   rc   ri   rj   s   @r0   rÎ   rÎ     s†   ø€ € € € € ð	ð 	ð,˜{ð ,ð ,ð ,ð ,ð ,ð ,ðBð Bð Bð
 U¤\ð °e´lð ð ð ð ð ð ð ð r/   rÎ   c                   ó¢   ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 ddej        dej        dz  dej        dz  d	e	dz  d
e
dz  dee         dej        fd„Zˆ xZS )ÚJambaAttentionDecoderLayerr4   r5   c                 óf  •— t          ¦   «                              ¦   «          |j        r|j        |         nd}t          ||¦  «        | _        |dk    rt
          nt          } ||¦  «        | _        t          |j	        |j
        ¬¦  «        | _        t          |j	        |j
        ¬¦  «        | _        d S )Nr&   rs   )r:   r;   Úlayers_num_expertsr3   Ú	self_attnrÎ   rÊ   Úfeed_forwardr)   r=   r’   Úinput_layernormÚpre_ff_layernorm©rE   r4   r5   rÒ   Úffn_layer_classrF   s        €r0   r;   z#JambaAttentionDecoderLayer.__init__B  s¦   ø€ Ý‰Œ×ÒÑÔÐØ>DÔ>WÐ^�fÔ/°	Ô:Ð:Ð]^ˆÝ'¨°	Ñ:Ô:ˆŒà1<¸q²°Õ-Ð-ÅhˆØ+˜O¨FÑ3Ô3ˆÔÝ+¨FÔ,>ÀFÔDWÐXÑXÔXˆÔÝ ,¨VÔ-?ÀVÔEXÐ YÑ YÔ YˆÔÐÐr/   NFrG   rH   Úposition_idsrI   Ú	use_cacherJ   rK   c           	      óÌ   — |}|                       |¦  «        } | j        d|||||dœ|¤Ž\  }}||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|S )N)rG   rH   rî   rI   rï   r.   )rê   rè   rë   ré   )	rE   rG   rH   rî   rI   rï   rJ   ÚresidualÚ_s	            r0   rc   z"JambaAttentionDecoderLayer.forwardL  sž   € ð !ˆØ×,Ò,¨]Ñ;Ô;ˆØ)˜4œ>ð 
Ø'Ø)Ø%Ø+Øð
ð 
ð ð
ð 
Ñˆ�qð ! =Ñ0ˆØ ˆØ×-Ò-¨mÑ<Ô<ˆØ×)Ò)¨-Ñ8Ô8ˆØ  =Ñ0ˆØÐr/   )NNNF)r+   r,   r-   r'   re   r;   rf   rg   Ú
LongTensorr   Úboolr   r   ÚFloatTensorrc   ri   rj   s   @r0   rå   rå   A  sà   ø€ € € € € ðZ˜{ð Z°sð Zð Zð Zð Zð Zð Zð /3Ø04Ø(,Ø!&ðð à”|ðð œ tÑ+ðð Ô&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð Ð+Ô,ðð 
Ô	ðð ð ð ð ð ð ð r/   rå   c                   ó–   ‡ — e Zd Zdedefˆ fd„Z	 	 	 ddej        dej        dz  dej        dz  de	dz  d	e
e         d
ej        fd„Zˆ xZS )ÚJambaMambaDecoderLayerr4   r5   c                 óh  •— t          ¦   «                              ¦   «          |j        r|j        |         nd}t          ||¬¦  «        | _        |dk    rt
          nt          } ||¦  «        | _        t          |j	        |j
        ¬¦  «        | _        t          |j	        |j
        ¬¦  «        | _        d S )Nr&   )r4   r5   rs   )r:   r;   rç   rl   ÚmambarÎ   rÊ   ré   r)   r=   r’   rê   rë   rì   s        €r0   r;   zJambaMambaDecoderLayer.__init__h  s©   ø€ Ý‰Œ×ÒÑÔÐØ>DÔ>WÐ^�fÔ/°	Ô:Ð:Ð]^ˆÝ$¨F¸iÐHÑHÔHˆŒ
Ø1<¸q²°Õ-Ð-ÅhˆØ+˜O¨FÑ3Ô3ˆÔÝ+¨FÔ,>ÀFÔDWÐXÑXÔXˆÔÝ ,¨VÔ-?ÀVÔEXÐ YÑ YÔ YˆÔÐÐr/   NrG   rH   rî   rI   rJ   rK   c                 óÐ   — |}|                       |¦  «        }|                      |||¬¦  «        }||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|S )N)rG   rœ   rH   )rê   rù   rë   ré   )rE   rG   rH   rî   rI   rJ   rñ   s          r0   rc   zJambaMambaDecoderLayer.forwardq  s„   € ð !ˆØ×,Ò,¨]Ñ;Ô;ˆØŸ
š
Ø'Ø(Ø)ð #ñ 
ô 
ˆð
 ! =Ñ0ˆØ ˆØ×-Ò-¨mÑ<Ô<ˆØ×)Ò)¨-Ñ8Ô8ˆØ  =Ñ0ˆØÐr/   )NNN)r+   r,   r-   r'   re   r;   rf   rg   ró   r   r   r   rõ   rc   ri   rj   s   @r0   r÷   r÷   g  sÏ   ø€ € € € € ðZ˜{ð Z°sð Zð Zð Zð Zð Zð Zð /3Ø04Ø(,ðð à”|ðð œ tÑ+ðð Ô&¨Ñ-ð	ð
  ™ðð Ð+Ô,ðð 
Ô	ðð ð ð ð ð ð ð r/   r÷   )Ú	attentionrù   c                   óª   ‡ — e Zd ZU eed<   dZdZddgZdgZdZ	dZ
dZdZeege eej        d¬¦  «        d	œZ ej        ¦   «         ˆ fd
„¦   «         Zˆ xZS )ÚJambaPreTrainedModelr4   ÚmodelTrå   r÷   rI   rÕ   )Ú
layer_name)rG   Ú
attentionsrÜ   c                 óp  •— t          ¦   «                              |¦  «         t          |t          ¦  «        r›t	          j        d|j        dz   ¦  «        d d d …f         }|                     |j        d¦  «         	                    ¦   «         }t          j        |j        t	          j        |¦  «        ¦  «         t          j        |j        ¦  «         d S t          |t           ¦  «        rNt          j        |j        d| j        j        ¬¦  «         t          j        |j        d| j        j        ¬¦  «         d S d S )Nr&   rM   rN   )ÚmeanÚstd)r:   Ú_init_weightsÚ
isinstancerl   rf   rŠ   rw   r‹   r{   rZ   ÚinitÚcopy_rŽ   r�   Úones_r�   rÌ   Únormal_Úgate_up_projr4   Úinitializer_rangeÚ	down_proj)rE   Úmoduler˜   rF   s      €r0   r  z"JambaPreTrainedModel._init_weights›  s  ø€ å‰Œ×Ò˜fÑ%Ô%Ð%Ý�f�oÑ.Ô.ð 	XÝ”˜Q Ô 5¸Ñ 9Ñ:Ô:¸4ÀÀÀ¸7ÔCˆAØ—’˜Ô1°2Ñ6Ô6×AÒAÑCÔCˆAÝŒJ�v”|¥U¤Y¨q¡\¤\Ñ2Ô2Ð2ÝŒJ�v”xÑ Ô Ð Ð Ð Ý˜¥Ñ-Ô-ð 	XÝŒL˜Ô,°3¸D¼KÔ<YÐZÑZÔZÐZÝŒL˜Ô)°¸¼Ô9VÐWÑWÔWÐWÐWÐWð	Xð 	Xr/   )r+   r,   r-   r'   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_is_statefulÚ_can_compile_fullgraphrå   r÷   r3   r   r   r<   Ú_can_record_outputsrf   Úno_gradr  ri   rj   s   @r0   rý   rý   ‹  sÇ   ø€ € € € € € ØÐÐÑØÐØ&*Ð#Ø5Ð7OÐPÐØ#4Ð"5ÐØÐØ€NØ€LØ!Ðà4Ð6LÐMØ$Ø'˜¨¬	¸hÐGÑGÔGðð Ðð €U„]�_„_ð	Xð 	Xð 	Xð 	Xñ „_ð	Xð 	Xð 	Xð 	Xð 	Xr/   rý   c                   óâ   ‡ — e Zd Zdefˆ fd„Zeee	 	 	 	 	 	 ddej	        dz  dej
        dz  dej	        dz  dedz  dej        dz  d	edz  d
ee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )Ú
JambaModelr4   c                 ó  •— t          ¦   «                              |¦  «         |j        | _        |j        | _        t          j        |j        |j        | j        ¦  «        | _        g }t          |j
        ¦  «        D ]:}t          |j        |                  }|                      |||¬¦  «        ¦  «         Œ;t          j        |¦  «        | _        t!          |j        |j        ¬¦  «        | _        d| _        |                      ¦   «          d S )N)r5   rs   F)r:   r;   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr=   Úembed_tokensÚrangeÚnum_hidden_layersÚALL_DECODER_LAYER_TYPESÚlayers_block_typeÚappendÚ
ModuleListr¬   r)   r’   Úfinal_layernormÚgradient_checkpointingÚ	post_init)rE   r4   Údecoder_layersÚiÚlayer_classrF   s        €r0   r;   zJambaModel.__init__ª  só   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒåœL¨Ô):¸FÔ<NÐPTÔP`ÑaÔaˆÔØˆÝ�vÔ/Ñ0Ô0ð 	Dð 	DˆAÝ1°&Ô2JÈ1Ô2MÔNˆKØ×!Ò! + +¨fÀÐ"BÑ"BÔ"BÑCÔCÐCÐCÝ”m NÑ3Ô3ˆŒå+¨FÔ,>ÀFÔDWÐXÑXÔXˆÔà&+ˆÔ#à�ŠÑÔÐÐÐr/   NÚ	input_idsrH   rî   rI   Úinputs_embedsrï   rJ   rK   c           	      óˆ  — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|r|€t          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   }| 	                    d¦  «        }t          |x}	t          ¦  «        s&| j        ||||dœ}
t          d
i |
¤Žt          d
i |
¤Ždœ}	|}t          | j        ¦  «        D ])\  }} ||f|	| j        j        |                  |||dœ|¤Ž}Œ*|                      |¦  «        }t%          ||¬	¦  «        S )Nz:You must specify exactly one of input_ids or inputs_embeds)r4   r   r&   )Údevice)r4   r.  rH   rI   rî   )Úfull_attentionÚlinear_attention)rH   rî   rI   rï   )Úlast_hidden_staterI   r.   )Ú
ValueErrorr   r	   r4   Úget_seq_lengthrf   rŠ   rQ   r0  rµ   r  Údictr
   r   Ú	enumerater¬   Úlayer_typesr'  r   )rE   r-  rH   rî   rI   r.  rï   rJ   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsrG   r+  Údecoder_layers                 r0   rc   zJambaModel.forward¼  sÂ  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå°Ð?Ð-ÅÑFÔFð 	ð œ+Ø!.Ø"0Ø#2Ø ,ðð ˆKõ #5Ð"CÐ"C°{Ð"CÐ"CÝ$CÐ$RÐ$RÀkÐ$RÐ$Rð#ð #Ðð &ˆÝ )¨$¬+Ñ 6Ô 6ð 	ð 	ÑˆAˆ}Ø)˜MØðà2°4´;Ô3JÈ1Ô3MÔNØ)Ø /Ø#ðð ð ðð ˆMˆMð ×,Ò,¨]Ñ;Ô;ˆå%Ø+Ø+ð
ñ 
ô 
ð 	
r/   )NNNNNN)r+   r,   r-   r'   r;   r   r   r   rf   ró   rg   r   rõ   rô   r   r   r   rc   ri   rj   s   @r0   r  r  ¨  s  ø€ € € € € ð˜{ð ð ð ð ð ð ð$  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ð6
ð 6
àÔ# dÑ*ð6
ð œ tÑ+ð6
ð Ô&¨Ñ-ð	6
ð
  ™ð6
ð Ô(¨4Ñ/ð6
ð ˜$‘;ð6
ð Ð+Ô,ð6
ð 
 ð6
ð 6
ð 6
ñ „^ñ „_ñ  Ôð6
ð 6
ð 6
ð 6
ð 6
r/   r  c                   óî   ‡ — e Zd Zdefˆ fd„Z	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dedz  d	ej	        dz  d
ej        dz  de
dz  de
dz  deej        z  dee         defˆ fd„Zˆ xZS )ÚJambaForCausalLMr4   c                 ób   •— t          ¦   «                              |¦  «         |j        | _        d S )N)r:   r;   rÒ   r×   s     €r0   r;   zJambaForCausalLM.__init__ù  s,   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô-ˆÔÐÐr/   Nr   r-  rH   rî   rI   r.  Úlabelsrï   Úoutput_router_logitsÚlogits_to_keeprJ   rK   c
           
      óF   •—  t          ¦   «         j        ||||||||	fi |
¤ŽS )aj  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, JambaForCausalLM

        >>> model = JambaForCausalLM.from_pretrained("ai21labs/Jamba-v0.1")
        >>> tokenizer = AutoTokenizer.from_pretrained("ai21labs/Jamba-v0.1")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```)r:   rc   )rE   r-  rH   rî   rI   r.  r@  rï   rA  rB  rJ   rF   s              €r0   rc   zJambaForCausalLM.forwardý  sG   ø€ ðF �u‰wŒwŒØØØØØØØØð

ð 

ð ð

ð 

ð 
	
r/   )	NNNNNNNNr   )r+   r,   r-   r'   r;   rf   ró   rg   r   rõ   rô   re   r   r   r   rc   ri   rj   s   @r0   r>  r>  ø  s0  ø€ € € € € ð.˜{ð .ð .ð .ð .ð .ð .ð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø,0Ø-.ð-
ð -
àÔ# dÑ*ð-
ð œ tÑ+ð-
ð Ô&¨Ñ-ð	-
ð
  ™ð-
ð Ô(¨4Ñ/ð-
ð Ô  4Ñ'ð-
ð ˜$‘;ð-
ð # T™kð-
ð ˜eœlÑ*ð-
ð Ð+Ô,ð-
ð 
#ð-
ð -
ð -
ð -
ð -
ð -
ð -
ð -
ð -
ð -
r/   r>  c                   ó   — e Zd ZdS )ÚJambaForSequenceClassificationNr*   r.   r/   r0   rE  rE  -  r1   r/   rE  )r>  rE  r  rý   )HÚcollections.abcr   rf   r   Ú r   r  Úactivationsr   Úcache_utilsr   r	   Úmasking_utilsr
   r   Úmodeling_layersr   r   Úmodeling_outputsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   Úutils.output_capturingr   r   Ú"falcon_mamba.modeling_falcon_mambar   Úllama.modeling_llamar   r   r   Úmamba.modeling_mambar   r   r    r!   r"   Úmistral.modeling_mistralr#   Úmixtral.modeling_mixtralr$   r%   Úconfiguration_jambar'   Ú
get_loggerr+   Úloggerr)   r3   rl   rÊ   rÌ   ru   rÎ   rå   r÷   r#  rý   r  r>  rE  Ú__all__r.   r/   r0   ú<module>r[     sD  ðð& %Ð $Ð $Ð $Ð $Ð $à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø PÐ PÐ PÐ PÐ PÐ PÐ PÐ PØ [Ð [Ð [Ð [Ð [Ð [Ð [Ð [Ø QÐ QÐ QÐ QÐ QÐ QÐ QÐ QØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ø 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ AÐ AÐ AÐ AÐ AÐ AØ XÐ XÐ XÐ XÐ XÐ XÐ XÐ XÐ XÐ Xðð ð ð ð ð ð ð ð ð ð ð ð ð ð 2Ð 1Ð 1Ð 1Ð 1Ð 1Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ ,Ð ,Ð ,Ð ,Ð ,Ð ,ð 
ˆÔ	˜HÑ	%Ô	%€ð	ð 	ð 	ð 	ð 	�<ñ 	ô 	ð 	ð*)ð *)ð *)ð *)ð *)�^ñ *)ô *)ð *)ðZl%ð l%ð l%ð l%ð l%Ð&ñ l%ô l%ð l%ð^	ð 	ð 	ð 	ð 	ˆzñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	�>ñ 	ô 	ð 	ð"ð "ð "ð "ð "˜"œ)ñ "ô "ð "ðJ#ð #ð #ð #ð #Ð!;ñ #ô #ð #ðLð ð ð ð Ð7ñ ô ð ðB )CÐMcÐdÐdÐ ðXð Xð Xð Xð X˜?ñ Xô Xð Xð: ðL
ð L
ð L
ð L
ð L
Ð%ñ L
ô L
ñ „ðL
ð^2
ð 2
ð 2
ð 2
ð 2
Ð)ñ 2
ô 2
ð 2
ðj	ð 	ð 	ð 	ð 	Ð%EÐG[ñ 	ô 	ð 	ð gÐ
fÐ
f€€€r/   