§
    9²¸j?D  ã                   óf  — d Z ddlZddlmZ ddlmZ ddlmZmZ ddl	m
Z
mZ ddlmZmZ dd	lmZ dd
lmZ ddlmZmZmZ ddlmZmZmZ ddlmZ ddlmZ ddlm Z m!Z!m"Z"m#Z#m$Z$m%Z% ddl&m'Z'm(Z( ddl)m*Z*  ej+        e,¦  «        Z- G d„ de$¦  «        Z.d*d„Z/e G d„ de ¦  «        ¦   «         Z0 G d„ de(¦  «        Z1 G d„ de'¦  «        Z2 G d„ de"¦  «        Z3 G d„ d e#¦  «        Z4 G d!„ d"e¦  «        Z5e G d#„ d$e¦  «        ¦   «         Z6e G d%„ d&e6¦  «        ¦   «         Z7 G d'„ d(e!¦  «        Z8g d)¢Z9dS )+zPyTorch Bamba model.é    N)Únné   )Úinitialization)ÚCacheÚDynamicCache)Úcreate_causal_maskÚcreate_recurrent_attention_mask)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚPreTrainedModel)ÚUnpack)Úauto_docstringÚcan_return_tupleÚlogging)ÚTransformersKwargsÚmerge_with_config_defaultsÚno_inherit_decorator)Úcapture_outputsé   )ÚJambaAttentionDecoderLayer)ÚLlamaAttentionÚLlamaForCausalLMÚLlamaMLPÚLlamaRMSNormÚLlamaRotaryEmbeddingÚrotate_half)ÚMamba2MixerÚMambaRMSNormGatedé   )ÚBambaConfigc                   ó:   — e Zd Zddedeej        ef         fd„ZdS )ÚBambaRotaryEmbeddingNÚconfigÚreturnc                 ó>  — | j         d         }t          | dd¦  «        p| j        | j        z  }| j                              dd¦  «        }t          ||z  ¦  «        }d}d|t          j        d|dt          j        ¬¦  «        |z  z  z  }| 	                    |¦  «        |fS )	a¾  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNÚpartial_rotary_factorg      ð?r   r   ©Údtype)
Úrope_parametersÚgetattrÚhidden_sizeÚnum_attention_headsÚgetÚintÚtorchÚarangeÚfloatÚto)r#   ÚdeviceÚkwargsÚbaseÚdimr(   Úattention_factorÚinv_freqs           úw/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/bamba/modular_bamba.pyÚcompute_default_rope_parametersz4BambaRotaryEmbedding.compute_default_rope_parameters5   s¨   € ð Ô% lÔ3ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆØ &Ô 6× :Ò :Ð;RÐTWÑ XÔ XÐÝ�#Ð-Ñ-Ñ.Ô.ˆàÐà˜$¥5¤<°°3¸ÅÄÐ#MÑ#MÔ#MÐPSÑ#SÑTÑUˆØ�{Š{˜6Ñ"Ô"Ð$4Ð4Ð4ó    ©N)	Ú__name__Ú
__module__Ú__qualname__r    Útupler1   ÚTensorr3   r<   © r=   r;   r"   r"   4   sK   € € € € € ð5ð 5°ð 5ÐW\Ð]bÔ]iÐkpÐ]pÔWqð 5ð 5ð 5ð 5ð 5ð 5r=   r"   c                 ó˜  — |                      |¦  «        }|                      |¦  «        }|j        d         }| dd|…f         | d|d…f         }}|dd|…f         |d|d…f         }	}||z  t          |¦  «        |z  z   }
||z  t          |¦  «        |z  z   }t          j        |
|gd¬¦  «        }
t          j        ||	gd¬¦  «        }|
|fS )a»  Applies Rotary Position Embedding to the query and key tensors.

    Removes the interleaving of cos and sin from GLM

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    éÿÿÿÿ.N©r8   )Ú	unsqueezeÚshaper   r1   Úcat)ÚqÚkÚcosÚsinÚunsqueeze_dimÚ
rotary_dimÚq_rotÚq_passÚk_rotÚk_passÚq_embedÚk_embeds               r;   Úapply_rotary_pos_embrW   K   sô   € ð( �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€Cð ”˜2”€JØ�c˜;˜J˜;Ð&Ô'¨¨3°
°°Ð+;Ô)<ˆ6€EØ�c˜;˜J˜;Ð&Ô'¨¨3°
°°Ð+;Ô)<ˆ6€Eð �s‰{�{¨5Ñ1Ô1°CÑ7Ñ8€GØ�s‰{�{¨5Ñ1Ô1°CÑ7Ñ8€Gõ Œi˜ &Ð)¨rÐ2Ñ2Ô2€GÝŒi˜ &Ð)¨rÐ2Ñ2Ô2€GØ�GÐÐr=   c                   ó   — e Zd ZdS )ÚBambaAttentionN©r?   r@   rA   rD   r=   r;   rY   rY   q   s   € € € € € à€Dr=   rY   c                   ó   — e Zd ZdS )ÚBambaRMSNormGatedNrZ   rD   r=   r;   r\   r\   v   ó   € € € € € Ø€Dr=   r\   c                   ób   ‡ — e Zd ZdZddededefˆ fd„Z ej	        ¦   «         d„ ¦   «         Z
ˆ xZS )	Ú
BambaMixeruk  
    Compute âˆ†, A, B, C, and D the state space parameters and compute the `contextualized_states`.
    A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
    âˆ†, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
    and is why Mamba is called **selective** state spaces)

    The are a few differences between this and Mamba2Mixer:
    - The variable use_precomputed_states is slightly different due to the hybrid cache structure
    - Some extra variables that our layer doesn't need have been removed
    Tr#   Ú	layer_idxÚinitialize_mixer_weightsc           	      ó  •— |j         | _        t          ¦   «                              |||¦  «         |j        | _        |j        | _        |j        | _	        t          |j        | j        z  ¦  «        | _        |j        | _        |j        | _        |j        | _        |j        | _        |j        | _        t1          | j        | j        ¬¦  «        | _        t5          j        | j        | j        |j        | j	        | j        | j	        dz
  ¬¦  «        | _        | j        | j        z   | j        z   }t5          j        | j        || j        ¬¦  «        | _        t5          j        | j        | j        | j        ¬¦  «        | _         | `!| `"| `| `#| `$d S )N©Úepsr   )Úin_channelsÚout_channelsÚbiasÚkernel_sizeÚgroupsÚpadding)rg   )%Úmamba_proj_biasÚuse_biasÚsuperÚ__init__Úmamba_n_headsÚ	num_headsÚmamba_d_stateÚssm_state_sizeÚmamba_d_convÚconv_kernel_sizer0   Úmamba_expandr-   Úintermediate_sizeÚmamba_conv_biasÚuse_conv_biasÚrms_norm_epsÚlayer_norm_epsilonÚmamba_n_groupsÚn_groupsÚmamba_d_headr'   Úmamba_chunk_sizeÚ
chunk_sizer\   Únormr   ÚConv1dÚconv_dimÚconv1dÚLinearÚin_projÚout_projÚtime_step_floorÚtime_step_rankÚtime_step_minÚtime_step_max)Úselfr#   r`   ra   Úprojection_sizeÚ	__class__s        €r;   rn   zBambaMixer.__init__†   sv  ø€ ØÔ.ˆŒÝ‰Œ×Ò˜ Ð,DÑEÔEÐEØÔ-ˆŒØ$Ô2ˆÔØ &Ô 3ˆÔÝ!$ VÔ%8¸4Ô;KÑ%KÑ!LÔ!LˆÔØ#Ô3ˆÔØ"(Ô"5ˆÔØÔ-ˆŒØÔ+ˆŒØ Ô1ˆŒÝ% dÔ&<À$ÔBYÐZÑZÔZˆŒ	Ý”iØœØœØÔ'ØÔ-Ø”=ØÔ)¨AÑ-ð
ñ 
ô 
ˆŒð Ô0°4´=Ñ@À4Ä>ÑQˆÝ”yØÔØØ”ð
ñ 
ô 
ˆŒõ
 œ	 $Ô"8¸$Ô:JÐQUÔQ^Ð_Ñ_Ô_ˆŒØÐ ØÐØˆMØÐØÐÐÐr=   c                 ó*  — t          j        d| j        dz   | j        j        t           j        ¬¦  «        }t          j        | j        t          j        |¦  «        ¦  «         t          j	        | j
        ¦  «         t          j	        | j        ¦  «         d S )Nr   )r5   r*   )r1   r2   rp   ÚA_logr5   Úfloat32ÚinitÚcopy_ÚlogÚones_ÚDÚdt_bias)r‹   ÚAs     r;   Úinit_bamba_weightszBambaMixer.init_bamba_weights¨   sp   € åŒL˜˜DœN¨QÑ.°t´zÔ7HÕPUÔP]Ð^Ñ^Ô^ˆÝŒ
�4”:�uœy¨™|œ|Ñ,Ô,Ð,ÝŒ
�4”6ÑÔÐÝŒ
�4”<Ñ Ô Ð Ð Ð r=   )T)r?   r@   rA   Ú__doc__r    r0   Úboolrn   r1   Úno_gradr˜   Ú__classcell__©r�   s   @r;   r_   r_   z   s†   ø€ € € € € ð	ð 	ð ð  ˜{ð  °sð  ÐVZð  ð  ð  ð  ð  ð  ðD €U„]�_„_ð!ð !ñ „_ð!ð !ð !ð !ð !r=   r_   c                   ó   — e Zd ZdS )ÚBambaMLPNrZ   rD   r=   r;   rŸ   rŸ   °   r]   r=   rŸ   c                   ó   — e Zd ZdS )ÚBambaRMSNormNrZ   rD   r=   r;   r¡   r¡   ´   r]   r=   r¡   c                   ó  ‡ — e Zd Zddededefˆ fd„Z	 	 	 	 	 ddej        d	ej        dz  d
ej	        dz  de
dz  dedz  deej        ej        f         dz  dee         deej        eej        ej        f         dz  f         fd„Zˆ xZS )ÚBambaDecoderLayerÚlinear_attentionr#   r`   Ú
layer_typec                 ó6  •— t          ¦   «                              ||¦  «         | `d}|dk    rt          nd } ||¦  «        | _        || _        |dk    rt          ||¬¦  «        | _        d S |dk    rt          ||¦  «        | _        d S t          d|›�¦  «        ‚)Nr   r¤   )r#   r`   Úfull_attentionzInvalid layer_type: )
rm   rn   Ú	self_attnrŸ   Úfeed_forwardÚ
block_typer_   ÚmambarY   Ú
ValueError)r‹   r#   r`   r¥   Únum_expertsÚffn_layer_classr�   s         €r;   rn   zBambaDecoderLayer.__init__¹   s¯   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+àˆNàˆØ&1°QÒ&6Ð&6�(˜(¸DˆØ+˜O¨FÑ3Ô3ˆÔà$ˆŒØÐ+Ò+Ð+Ý#¨6¸YÐGÑGÔGˆDŒJˆJˆJØÐ+Ò+Ð+Ý+¨F°IÑ>Ô>ˆDŒNˆNˆNåÐB°JÐBÐBÑCÔCÐCr=   NFÚhidden_statesÚattention_maskÚposition_idsÚpast_key_valuesÚ	use_cacheÚposition_embeddingsr6   r$   c           
      ó&  — |}|                       |¦  «        }| j        dk    r | j        d|||dœ|¤Ž}d }	n"| j        dk    r | j        d||||||dœ|¤Ž\  }}	||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }||	fS )Nr¤   )r¯   Úcache_paramsr°   r§   )r¯   r°   r±   r²   r³   r´   rD   )Úinput_layernormrª   r«   r¨   Úpre_ff_layernormr©   )
r‹   r¯   r°   r±   r²   r³   r´   r6   ÚresidualÚself_attn_weightss
             r;   ÚforwardzBambaDecoderLayer.forwardÊ   sÿ   € ð !ˆà×,Ò,¨]Ñ;Ô;ˆàŒ?Ð0Ò0Ð0Ø&˜DœJð Ø+Ø,Ø-ðð ð ð	ð ˆMð !%ÐÐØŒ_Ð 0Ò0Ð0Ø/=¨t¬~ð 0Ø+Ø-Ø)Ø /Ø#Ø$7ð0ð 0ð ð0ð 0Ñ,ˆMÐ,ð ! =Ñ0ˆà ˆØ×-Ò-¨mÑ<Ô<ˆØ×)Ò)¨-Ñ8Ô8ˆØ  =Ñ0ˆàÐ/Ð/Ð/r=   )r¤   )NNNFN)r?   r@   rA   r    r0   Ústrrn   r1   rC   Ú
LongTensorr   rš   rB   r   r   ÚFloatTensorr»   rœ   r�   s   @r;   r£   r£   ¸   s.  ø€ € € € € ðDð D˜{ð D°sð DÈð Dð Dð Dð Dð Dð Dð( /3Ø04Ø(,Ø!&ØHLð(0ð (0à”|ð(0ð œ tÑ+ð(0ð Ô&¨Ñ-ð	(0ð
  ™ð(0ð ˜$‘;ð(0ð # 5¤<°´Ð#=Ô>ÀÑEð(0ð Ð+Ô,ð(0ð 
ˆuÔ  %¨Ô(9¸5Ô;LÐ(LÔ"MÐPTÑ"TÐTÔ	Uð(0ð (0ð (0ð (0ð (0ð (0ð (0ð (0r=   r£   c                   ó‚   ‡ — e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZeedœZ ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )ÚBambaPreTrainedModelr#   ÚmodelTr£   r²   )r¯   Ú
attentionsc           
      ój  •— t          ¦   «                              |¦  «         t          |t          ¦  «        r{t	          j        |j        ¦  «         t	          j        |j        t          j
        t          j        d|j        dz   ¦  «        ¦  «        ¦  «         t	          j        |j        ¦  «         d S d S )Nr   )rm   Ú_init_weightsÚ
isinstancer_   r‘   r”   r–   r’   r�   r1   r“   r2   rp   r•   )r‹   Úmoduler�   s     €r;   rÄ   z"BambaPreTrainedModel._init_weights  s”   ø€ å‰Œ×Ò˜fÑ%Ô%Ð%Ý�f�jÑ)Ô)ð 	!ÝŒJ�v”~Ñ&Ô&Ð&ÝŒJ�v”|¥U¤Y­u¬|¸A¸vÔ?OÐRSÑ?SÑ/TÔ/TÑ%UÔ%UÑVÔVÐVÝŒJ�v”xÑ Ô Ð Ð Ð ð	!ð 	!r=   )r?   r@   rA   r    Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_is_statefulÚ_can_compile_fullgraphr£   rY   Ú_can_record_outputsr1   r›   rÄ   rœ   r�   s   @r;   rÀ   rÀ   õ   s¡   ø€ € € € € € àÐÐÑØÐØ&*Ð#Ø,Ð-ÐØ#4Ð"5ÐØÐØ€NØ€LØ!Ðà*Ø$ðð Ðð
 €U„]�_„_ð!ð !ð !ð !ñ „_ð!ð !ð !ð !ð !r=   rÀ   c                   óâ   ‡ — e Zd Zdefˆ fd„Zeee	 	 	 	 	 	 ddej	        dz  dej
        dz  dej	        dz  dedz  dej        dz  d	edz  d
ee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )Ú
BambaModelr#   c           	      óJ  •— t          ¦   «                              |¦  «         |j        | _        |j        | _        t          j        |j        |j        | j        ¦  «        | _        g }t          |j
        ¦  «        D ]2}|                     t          |||j        |         ¬¦  «        ¦  «         Œ3t          j        |¦  «        | _        |j        | _        t#          |j        |j        ¬¦  «        | _        t)          |¬¦  «        | _        d| _        |                      ¦   «          d S )N)r`   r¥   rc   ©r#   F)rm   rn   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr-   Úembed_tokensÚrangeÚnum_hidden_layersÚappendr£   Úlayers_block_typeÚ
ModuleListÚlayersÚ_attn_implementationr¡   ry   Úfinal_layernormr"   Ú
rotary_embÚgradient_checkpointingÚ	post_init)r‹   r#   Údecoder_layersÚir�   s       €r;   rn   zBambaModel.__init__  s	  ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒåœL¨Ô):¸FÔ<NÐPTÔP`ÑaÔaˆÔØˆÝ�vÔ/Ñ0Ô0ð 	rð 	rˆAØ×!Ò!Õ"3°FÀaÐTZÔTlÐmnÔToÐ"pÑ"pÔ"pÑqÔqÐqÐqÝ”m NÑ3Ô3ˆŒà$*Ô$?ˆÔ!Ý+¨FÔ,>ÀFÔDWÐXÑXÔXˆÔÝ.°fÐ=Ñ=Ô=ˆŒà&+ˆÔ#à�ŠÑÔÐÐÐr=   NÚ	input_idsr°   r±   r²   Úinputs_embedsr³   r6   r$   c           
      ó„  — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|}|r|€t          | j        ¬¦  «        }|€9t	          j        |j        d         |j        ¬¦  «                             d¦  «        }t          |x}	t          ¦  «        s&| j        ||||dœ}
t          di |
¤Žt          di |
¤Ždœ}	|                      ||¬¦  «        }t          | j        ¦  «        D ]-\  }} ||f|	| j        j        |                  ||||d	œ|¤Ž\  }}Œ.|                      |¦  «        }t%          ||¬
¦  «        S )Nz:You must specify exactly one of input_ids or inputs_embedsrÔ   r   )r5   r   )r#   rè   r°   r²   r±   )r§   r¤   )r±   )r°   r±   r²   r³   r´   )Úlast_hidden_stater²   rD   )r¬   rÙ   r   r#   r1   r2   rI   r5   rH   rÅ   Údictr   r	   râ   Ú	enumeraterß   rÝ   rá   r
   )r‹   rç   r°   r±   r²   rè   r³   r6   r¯   Úcausal_mask_mappingÚmask_kwargsr´   ræ   Údecoder_layerÚattn_weightss                  r;   r»   zBambaModel.forward#  sº  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMØ%ˆàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\×fÒfÐghÑiÔiˆLå°Ð?Ð-ÅÑFÔFð 	ð œ+Ø!.Ø"0Ø#2Ø ,ðð ˆKõ #5Ð"CÐ"C°{Ð"CÐ"CÝ$CÐ$RÐ$RÀkÐ$RÐ$Rð#ð #Ðð #Ÿošo¨mÈ,˜oÑWÔWÐå )¨$¬+Ñ 6Ô 6ð 		ð 		ÑˆAˆ}Ø*7¨-Øð+à2°4´;Ô3PÐQRÔ3SÔTØ)Ø /Ø#Ø$7ð+ð +ð ð+ð +Ñ'ˆM˜<˜<ð ×,Ò,¨]Ñ;Ô;ˆå&Ø+Ø+ð
ñ 
ô 
ð 	
r=   )NNNNNN)r?   r@   rA   r    rn   r   r   r   r1   r½   rC   r   r¾   rš   r   r   r
   r»   rœ   r�   s   @r;   rÒ   rÒ     s  ø€ € € € € ð˜{ð ð ð ð ð ð ð&  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ð7
ð 7
àÔ# dÑ*ð7
ð œ tÑ+ð7
ð Ô&¨Ñ-ð	7
ð
  ™ð7
ð Ô(¨4Ñ/ð7
ð ˜$‘;ð7
ð Ð+Ô,ð7
ð 
!ð7
ð 7
ð 7
ñ „^ñ „_ñ  Ôð7
ð 7
ð 7
ð 7
ð 7
r=   rÒ   c                   óò   ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  de	dz  dej
        dz  d	ej        dz  d
edz  deej        z  defd„¦   «         ¦   «         Zˆ fd„Zˆ xZS )ÚBambaForCausalLMc                 óŠ   •— t          ¦   «                              |¦  «         |j        | _        |                      ¦   «          d S r>   )rm   rn   Úz_loss_coefficienträ   )r‹   r#   r�   s     €r;   rn   zBambaForCausalLM.__init__a  s>   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø"(Ô";ˆÔð 	�ŠÑÔÐÐÐr=   Nr   rç   r°   r±   r²   rè   Úlabelsr³   Úlogits_to_keepr$   c	           
      ó(  —  | j         d
||||||dœ|	¤Ž}
|
j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }d}|�‡ | j        d
||| j        j        dœ|	¤Ž}| j	        dk    ra| 
                    d¬¦  «                             |j        ¬¦  «                             d¦  «                             ¦   «         }|| j	        |z  z   }t          |||
j        |
j        |
j        ¬	¦  «        S )aJ  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, BambaForCausalLM

        >>> model = BambaForCausalLM.from_pretrained("...")
        >>> tokenizer = AutoTokenizer.from_pretrained("...")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```)rç   r°   r±   r²   rè   r³   N)Úlogitsrõ   r×   r   rF   rG   r)   r   )Úlossrø   r²   r¯   rÂ   rD   )rÁ   rê   rÅ   r0   ÚsliceÚlm_headÚloss_functionr#   r×   rô   Ú	logsumexpr4   r*   ÚpowÚmeanr   r²   r¯   rÂ   )r‹   rç   r°   r±   r²   rè   rõ   r³   rö   r6   Úoutputsr¯   Úslice_indicesrø   rù   Úz_losss                   r;   r»   zBambaForCausalLM.forwardh  sX  € ðH ,6¨4¬:ð ,
ØØ)Ø%Ø+Ø'Øð,
ð ,
ð ð,
ð ,
ˆð  Ô1ˆÝ8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%Ðp¨V¸FÈtÌ{ÔOeÐpÐpÐioÐpÐpˆDØÔ&¨Ò*Ð*Ø×)Ò)¨bÐ)Ñ1Ô1×4Ò4¸4¼:Ð4ÑFÔF×JÒJÈ1ÑMÔM×RÒRÑTÔT�Ø˜dÔ5¸Ñ>Ñ>�å%ØØØ#Ô3Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r=   c                 óZ   •— | j         j        |d<    t          ¦   «         j        |fi |¤Ž}|S )Nrö   )r#   Únum_logits_to_keeprm   Úprepare_inputs_for_generation)r‹   rç   r6   Úmodel_inputsr�   s       €r;   r  z.BambaForCausalLM.prepare_inputs_for_generation©  s7   ø€ Ø#'¤;Ô#AˆÐÑ Ø<•u‘w”wÔ<¸YÐQÐQÈ&ÐQÐQˆØÐr=   )NNNNNNNr   )r?   r@   rA   rn   r   r   r1   r½   rC   r   r¾   rš   r0   r   r»   r  rœ   r�   s   @r;   rò   rò   `  s*  ø€ € € € € ðð ð ð ð ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ð=
ð =
àÔ# dÑ*ð=
ð œ tÑ+ð=
ð Ô&¨Ñ-ð	=
ð
  ™ð=
ð Ô(¨4Ñ/ð=
ð Ô  4Ñ'ð=
ð ˜$‘;ð=
ð ˜eœlÑ*ð=
ð 
 ð=
ð =
ð =
ñ „^ñ Ôð=
ð~ð ð ð ð ð ð ð ð r=   rò   )rÒ   rò   rÀ   )r   ):r™   r1   r   Ú r   r‘   Úcache_utilsr   r   Úmasking_utilsr   r	   Úmodeling_outputsr
   r   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   r   r   Úutils.output_capturingr   Újamba.modeling_jambar   Úllama.modeling_llamar   r   r   r   r   r   Úmamba2.modeling_mamba2r   r   Úconfiguration_bambar    Ú
get_loggerr?   Úloggerr"   rW   rY   r\   r_   rŸ   r¡   r£   rÀ   rÒ   rò   Ú__all__rD   r=   r;   ú<module>r     s¯  ðð& Ð à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø PÐ PÐ PÐ PÐ PÐ PÐ PÐ PØ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OØ -Ð -Ð -Ð -Ð -Ð -Ø &Ð &Ð &Ð &Ð &Ð &Ø >Ð >Ð >Ð >Ð >Ð >Ð >Ð >Ð >Ð >Ø aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø =Ð =Ð =Ð =Ð =Ð =ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ðð ð ð ð ð ð ð ð -Ð ,Ð ,Ð ,Ð ,Ð ,ð 
ˆÔ	˜HÑ	%Ô	%€ð5ð 5ð 5ð 5ð 5Ð/ñ 5ô 5ð 5ð.#ð #ð #ð #ðL ð	ð 	ð 	ð 	ð 	�^ñ 	ô 	ñ Ôð	ð	ð 	ð 	ð 	ð 	Ð)ñ 	ô 	ð 	ð3!ð 3!ð 3!ð 3!ð 3!�ñ 3!ô 3!ð 3!ðl	ð 	ð 	ð 	ð 	ˆxñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	�<ñ 	ô 	ð 	ð:0ð :0ð :0ð :0ð :0Ð2ñ :0ô :0ð :0ðz ð!ð !ð !ð !ð !˜?ñ !ô !ñ „ð!ð0 ðN
ð N
ð N
ð N
ð N
Ð%ñ N
ô N
ñ „ðN
ðbLð Lð Lð Lð LÐ'ñ Lô Lð Lð^ EÐ
DÐ
D€€€r=   