§
    9²¸jã{  ã                   óÜ  — d dl mZ d dlmZ d dlZd dlmZ ddlmZ ddl	m
Z
 ddlmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZmZ ddlmZmZ ddlmZmZ ddlm Z  ddl!m"Z"m#Z#m$Z$ ddl%m&Z& ddl'm(Z(m)Z) ddl*m+Z+ ddl,m-Z-  G d„ dej.        ¦  «        Z/d„ Z0 ed¦  «        dCd„¦   «         Z1dej2        de3dej2        fd„Z4	 dDd!ej.        d"ej2        d#ej2        d$ej2        d%ej2        dz  d&e5d'e5d(e e"         fd)„Z6 G d*„ d+ej.        ¦  «        Z7 G d,„ d-ej.        ¦  «        Z8 G d.„ d/ej.        ¦  «        Z9 G d0„ d1ej.        ¦  «        Z: G d2„ d3ej.        ¦  «        Z; G d4„ d5ej.        ¦  «        Z< G d6„ d7e¦  «        Z= G d8„ d9e¦  «        Z>e# G d:„ d;e>¦  «        ¦   «         Z?	 	 	 dEd=ej2        e@ej2                 z  dz  d>e3dz  d%ej2        dz  dej2        e3z  fd?„ZA G d@„ dAe>e¦  «        ZBg dB¢ZCdS )Fé    )ÚCallable)ÚAnyN)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hub)Úcreate_causal_mask)ÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )Ú
DbrxConfigc            	       óî   ‡ — e Zd Z edd¬¦  «        d
defˆ fd„¦   «         Ze edd¬¦  «        d
dedeej	        e
f         fd„¦   «         ¦   «         Z ej        ¦   «         ed	„ ¦   «         ¦   «         Zˆ xZS )ÚDbrxRotaryEmbeddingÚdevicez5.18)ÚversionNÚconfigc                 ó¾  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        | j        j        d         | _        | j        }| j        dk    rt          | j                 } || j        |¦  «        \  }| _
        t          j        |d¬¦  «        | _        t          j        |                     ¦   «         d¬¦  «        | _        d S )NÚ	rope_typeÚdefaultF)Ú
persistent)ÚsuperÚ__init__Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr#   Úrope_parametersr%   Úcompute_default_rope_parametersr   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)Úselfr#   r!   Úrope_init_fnr1   Ú	__class__s        €úv/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/dbrx/modeling_dbrx.pyr)   zDbrxRotaryEmbedding.__init__.   sÀ   ø€ å‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!àˆŒàœÔ4°[ÔAˆŒØ!%Ô!EˆØŒ>˜YÒ&Ð&Ý.¨t¬~Ô>ˆLØ+7¨<¸¼ÀVÑ+LÔ+LÑ(ˆ�$Ô(åœ	 (°uÐ=Ñ=Ô=ˆŒÝ!#¤¨8¯>ª>Ñ+;Ô+;ÈÐ!NÑ!NÔ!NˆÔÐÐó    Úreturnc                 óä   — | j         d         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «        |z  z  z  }|                     |¦  «        |fS )a¾  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNç      ð?r   é   ©Údtype)r-   ÚgetattrÚhidden_sizeÚnum_attention_headsÚtorchÚarangeÚfloatÚto)r#   r!   ÚkwargsÚbaseÚdimÚattention_factorr1   s          r7   r.   z3DbrxRotaryEmbedding.compute_default_rope_parameters?   sz   € ð Ô% lÔ3ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐà˜$¥5¤<°°3¸ÅÄÐ#MÑ#MÔ#MÐPSÑ#SÑTÑUˆØ�{Š{˜6Ñ"Ô"Ð$4Ð4Ð4r8   c                 óú  — | j         d d d …d f                              |j        d         dd¦  «                             t          j        |j        ¬¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  ||z                       dd	¦  «        }t	          j        ||fd¬
¦  «        }|                     ¦   «         | j        z  }|                     ¦   «         | j        z  }	d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬¦  «        |	                     |j        ¬¦  «        fS )Nr   éÿÿÿÿr   ©r@   r!   ÚmpsÚcpuF)Údevice_typeÚenabledr>   ©rJ   r?   )r1   ÚexpandÚshaperG   rD   rF   r!   Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosr/   Úsinr@   )
r4   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedrQ   ÚfreqsÚembr[   r\   s
             r7   ÚforwardzDbrxRotaryEmbedding.forwardS   s›  € ð ŒM˜$    4˜-Ô(×/Ò/°Ô0BÀ1Ô0EÀrÈ1ÑMÔM×PÒPÕW\ÔWbÐklÔksÐPÑtÔtð 	ð !-¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆå¨¸UÐCÑCÔCð 	5ð 	5Ø&Ð)>Ñ>×IÒIÈ!ÈQÑOÔOˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCð		5ð 	5ð 	5ñ 	5ô 	5ð 	5ð 	5ð 	5ð 	5ð 	5ð 	5øøøð 	5ð 	5ð 	5ð 	5ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   ÃA*D<Ä<E ÅE ©N)Ú__name__Ú
__module__Ú__qualname__r   r   r)   ÚstaticmethodÚtuplerD   ÚTensorrF   r.   Úno_gradr   rc   Ú__classcell__©r6   s   @r7   r    r    -   sö   ø€ € € € € Ø€_�X vÐ.Ñ.Ô.ðOð O˜zð Oð Oð Oð Oð Oñ /Ô.ðOð  Ø€_�X vÐ.Ñ.Ô.ð5ð 5°
ð 5ÐV[Ð\aÔ\hÐjoÐ\oÔVpð 5ð 5ð 5ñ /Ô.ñ „\ð5ð$ €U„]�_„_Øð<ð <ñ Ôñ „_ð<ð <ð <ð <ð <r8   r    c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..NrM   r>   rS   )rU   rD   rZ   )r]   Úx1Úx2s      r7   Úrotate_halfrq   f   s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'r8   Úrotary_pos_embc                 ó¾   — |                      |¦  «        }|                      |¦  «        }| |z  t          | ¦  «        |z  z   }||z  t          |¦  «        |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezerq   )ÚqÚkr[   r\   Úunsqueeze_dimÚq_embedÚk_embeds          r7   Úapply_rotary_pos_embrz   m   sc   € ð& �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�GÐÐr8   Úhidden_statesÚn_repr9   c                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r   N)rU   rT   Úreshape)r{   r|   ÚbatchÚnum_key_value_headsÚslenr<   s         r7   Ú	repeat_kvr‚   ‡   s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTr8   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutrH   c                 ó  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        |z  }
|�|
|z   }
t
          j                             |
dt          j        ¬¦  «         	                    |j
        ¦  «        }
t
          j                             |
|| j        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «                             ¦   «         }||
fS )Nr>   r   rM   ©rJ   r@   ©ÚpÚtrainingr   )r‚   Únum_key_value_groupsrD   ÚmatmulrY   r   Ú
functionalÚsoftmaxÚfloat32rG   r@   rŠ   r�   Ú
contiguous)r„   r…   r†   r‡   rˆ   r‰   rŠ   rH   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r7   Úeager_attention_forwardrš   “   sé   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r8   c                   óª   ‡ — e Zd ZdZ	 ddedz  fˆ fd„Z	 	 	 ddej        dej        dz  dej        dz  de	dz  d	e
ej        ej        f         f
d
„Zˆ xZS )ÚDbrxAttentionzYModular DBRX attention component that can be reused across different model architectures.NÚ	layer_idxc                 óT  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        | j        | j        z  | _        |j        | _	        || _
        |j        }|j        | _        |j        | _        |j        | _        | j        | j        z  | _        | j        dz  | _        |j        | _        d| _        t+          j        | j        | j        d| j        z  | j        z  z   d¬¦  «        | _        t+          j        | j        | j        d¬¦  «        | _        d S )Ng      à¿Tr>   F©Úbias)r(   r)   r#   Úd_modelrB   Ún_headsÚ	num_headsr<   Úmax_seq_lenr*   r�   Úattn_configÚ
attn_pdropÚattention_dropoutÚclip_qkvÚ
kv_n_headsr€   r�   r‰   r;   Ú	is_causalr   ÚLinearÚWqkvÚout_proj)r4   r#   r�   rH   r¥   r6   s        €r7   r)   zDbrxAttention.__init__¯   s  ø€ õ 	‰Œ×ÒÑÔÐØˆŒØ!œ>ˆÔØœˆŒØÔ(¨D¬NÑ:ˆŒØ'-Ô'9ˆÔ$Ø"ˆŒàÔ(ˆØ!,Ô!7ˆÔØ#Ô,ˆŒØ#.Ô#9ˆÔ Ø$(¤N°dÔ6NÑ$NˆÔ!Ø”} dÑ*ˆŒØ%Ô0ˆŒØˆŒå”IØÔ˜dÔ.°°TÔ5MÑ1MÐPTÔP]Ñ1]Ñ]Ðdið
ñ 
ô 
ˆŒ	õ œ	 $Ô"2°DÔ4DÈ5ÐQÑQÔQˆŒˆˆr8   r{   rˆ   Úposition_embeddingsÚpast_key_valuesr9   c                 ó®  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «        }| j        �| j         nd }	|                     |	| j        ¬¦  «        }|                     | j        | j        | j        z  | j        | j        z  gd¬¦  «        \  }
}}|
                     |¦  «         	                    dd¦  «        }
|                     |¦  «         	                    dd¦  «        }|                     |¦  «         	                    dd¦  «        }|\  }}t          |
|||¦  «        \  }
}|�|                     ||| j        ¦  «        \  }}t          j        | j        j        t"          ¦  «        } || |
|||f| j        sdn| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )NrM   )ÚminÚmaxr>   rS   r   rƒ   )rŠ   r‰   )rU   r<   r¬   r¨   ÚclampÚsplitrB   r€   ÚviewrY   rz   Úupdater�   r   Úget_interfacer#   Ú_attn_implementationrš   r�   r§   r‰   r~   r•   r­   )r4   r{   rˆ   r®   r¯   rH   Úinput_shapeÚhidden_shapeÚ
qkv_statesÚmin_valÚquery_statesr–   r—   r[   r\   Úattention_interfacer™   r˜   s                     r7   rc   zDbrxAttention.forwardË   s$  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—Y’Y˜}Ñ-Ô-ˆ
Ø$(¤MÐ$=�4”=�.�.À4ˆØ×%Ò%¨'°t´}Ð%ÑEÔEˆ
à1;×1AÒ1AàÔ ØÔ(¨4¬=Ñ8ØÔ(¨4¬=Ñ8ðð
 ð 2Bñ 2
ô 2
Ñ.ˆ�j ,ð $×(Ò(¨Ñ6Ô6×@Ò@ÀÀAÑFÔFˆØ—_’_ \Ñ2Ô2×<Ò<¸QÀÑBÔBˆ
Ø#×(Ò(¨Ñ6Ô6×@Ò@ÀÀAÑFÔFˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—m’m KÑ0Ô0ˆØ˜LÐ(Ð(r8   rd   ©NNN)re   rf   rg   Ú__doc__Úintr)   rD   rj   Ú
LongTensorr	   ri   rc   rl   rm   s   @r7   rœ   rœ   ¬   sÚ   ø€ € € € € ØcÐcð
 !%ðRð Rð ˜‘:ðRð Rð Rð Rð Rð Rð> /3Ø7;Ø(,ð3)ð 3)à”|ð3)ð œ tÑ+ð3)ð #Ô-°Ñ4ð	3)ð
  ™ð3)ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð3)ð 3)ð 3)ð 3)ð 3)ð 3)ð 3)ð 3)r8   rœ   c            
       ól   ‡ — e Zd Zˆ fd„Zdej        dej        dej        dej        dej        f
d„Zˆ xZS )ÚDbrxExpertGLUc                 ó^  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        t          j        t          j        | j        | j        z  | j        ¦  «        ¦  «        | _	        t          j        t          j        | j        | j        z  | j        ¦  «        ¦  «        | _
        t          j        t          j        | j        | j        z  | j        ¦  «        ¦  «        | _        |j                             dd¦  «        }t          |         | _        d S )NÚnameÚsilu)r(   r)   rB   Úffn_hidden_sizeÚmoe_num_expertsr   Ú	ParameterrD   ÚemptyÚw1Úv1Úw2Ú
ffn_act_fnÚgetr   Úactivation_fn)r4   r#   Úact_fn_namer6   s      €r7   r)   zDbrxExpertGLU.__init__  sé   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔØ%Ô5ˆÔØ%Ô5ˆÔå”,�uœ{¨4Ô+?À$ÔBVÑ+VÐX\ÔXhÑiÔiÑjÔjˆŒÝ”,�uœ{¨4Ô+?À$ÔBVÑ+VÐX\ÔXhÑiÔiÑjÔjˆŒÝ”,�uœ{¨4Ô+?À$ÔBVÑ+VÐX\ÔXhÑiÔiÑjÔjˆŒàÔ'×+Ò+¨F°FÑ;Ô;ˆÝ# KÔ0ˆÔÐÐr8   r]   Ú	expert_w1Ú	expert_v1Ú	expert_w2r9   c                 óÌ   — |                      |j        ¦  «        }|                      |j        ¦  «        }|                      |¦  «        }||z  }|                      |¦  «        }|S rd   )r‘   ÚTrÑ   )	r4   r]   rÓ   rÔ   rÕ   Ú	gate_projÚup_projÚintermediate_statesÚ	down_projs	            r7   rc   zDbrxExpertGLU.forward  sa   € ð —H’H˜Yœ[Ñ)Ô)ˆ	Ø—(’(˜9œ;Ñ'Ô'ˆØ×&Ò& yÑ1Ô1ˆ	Ø'¨'Ñ1ÐØ'×.Ò.¨yÑ9Ô9ˆ	ØÐr8   ©re   rf   rg   r)   rD   rj   rc   rl   rm   s   @r7   rÄ   rÄ     s   ø€ € € € € ð1ð 1ð 1ð 1ð 1ðØ”ðØ*/¬,ðØCHÄ<ðØ\aÔ\hðà	Œðð ð ð ð ð ð ð r8   rÄ   c                   ó^   ‡ — e Zd Zˆ fd„Zdej        dej        dej        dej        fd„Zˆ xZS )ÚDbrxExpertsc                 ó¸   •— t          ¦   «                              ¦   «          t          |¦  «        | _        |j        | _        |j        | _        |j        | _        d S rd   )r(   r)   rÄ   ÚmlprB   rÈ   rÉ   Únum_experts©r4   r#   r6   s     €r7   r)   zDbrxExperts.__init__  sO   ø€ Ý‰Œ×ÒÑÔÐÝ  Ñ(Ô(ˆŒØ!Ô-ˆÔØ%Ô5ˆÔØ!Ô1ˆÔÐÐr8   r{   Útop_k_indexÚtop_k_weightsr9   c                 ó‚  — |j         d         }|                     d| j        ¦  «        }t          j        ||j        |j        ¬¦  «        }t          j        ¦   «         5  t          j        j	         
                    || j        ¬¦  «        }|                     ddd¦  «        }t          j        |                     d¬¦  «        d¦  «                             ¦   «         }d d d ¦  «         n# 1 swxY w Y   d| j        | j        f}|D �]}	|	d         }	t          j        ¦   «         5  t          j        ||	         ¦  «        \  }
}d d d ¦  «         n# 1 swxY w Y   | j        j                             |¦  «        |	         }| j        j                             |¦  «        |	         }| j        j                             |¦  «        |	         }|                      ||         |||¦  «        }|                     d| j        ¦  «        |||
d f         z  }|                     d||¦  «         �Œ |                     |d| j        ¦  «        }|S )	Nr   rM   rN   )Únum_classesr>   r   )rM   éþÿÿÿrS   )rU   r~   rB   rD   Ú
zeros_liker@   r!   rk   r   r’   Úone_hotrá   ÚpermuteÚgreaterÚsumÚnonzerorÈ   Úwhererà   rÍ   rµ   rÌ   rÎ   Ú
index_add_)r4   r{   rã   rä   Ú
batch_sizeÚnext_statesÚexpert_maskÚ
expert_hitÚsplit_expert_shapeÚ
expert_idxÚidxÚ	token_idxrÍ   rÌ   rÎ   Ústatess                   r7   rc   zDbrxExperts.forward"  s©  € ð #Ô(¨Ô+ˆ
Ø%×-Ò-¨b°$Ô2BÑCÔCˆåÔ& }¸MÔ<OÐXeÔXlÐmÑmÔmˆÝŒ]‰_Œ_ð 	Sð 	SÝœ(Ô-×5Ò5°kÈtÔO_Ð5Ñ`Ô`ˆKØ%×-Ò-¨a°°AÑ6Ô6ˆKÝœ {§¢¸8 Ñ'DÔ'DÀaÑHÔH×PÒPÑRÔRˆJð	Sð 	Sð 	Sñ 	Sô 	Sð 	Sð 	Sð 	Sð 	Sð 	Sð 	Søøøð 	Sð 	Sð 	Sð 	Sð
 ! $Ô"6¸Ô8HÐIÐØ$ð 		9ñ 		9ˆJØ# AœˆJÝ”‘”ð Fð FÝ!&¤¨[¸Ô-DÑ!EÔ!E‘��YðFð Fð Fñ Fô Fð Fð Fð Fð Fð Fð Føøøð Fð Fð Fð Fà””×!Ò!Ð"4Ñ5Ô5°jÔAˆBØ””×!Ò!Ð"4Ñ5Ô5°jÔAˆBØ””×!Ò!Ð"4Ñ5Ô5°jÔAˆBØ—X’X˜m¨IÔ6¸¸BÀÑCÔCˆFØ—[’[  TÔ%5Ñ6Ô6¸ÀyÐRUÐW[ÐG[Ô9\Ñ\ˆFØ×"Ò" 1 i°Ñ8Ô8Ð8Ñ8à!×&Ò& z°2°tÔ7GÑHÔHˆØÐs%   ÁA>C'Ã'C+Ã.C+Ä!EÅE	ÅE	rÜ   rm   s   @r7   rÞ   rÞ     sz   ø€ € € € € ð2ð 2ð 2ð 2ð 2ðà”|ðð ”\ðð ”|ð	ð
 
Œðð ð ð ð ð ð ð r8   rÞ   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )Ú
DbrxRouterc                 óÄ   •— t          ¦   «                              ¦   «          |j        | _        |j        | _        t	          j        | j        |j        d¬¦  «        | _        d S ©NFrŸ   )r(   r)   rB   Úmoe_jitter_epsr   r«   rÉ   Úlayerrâ   s     €r7   r)   zDbrxRouter.__init__B  sQ   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔØ$Ô3ˆÔÝ”Y˜tÔ/°Ô1GÈeÐTÑTÔTˆŒ
ˆ
ˆ
r8   r{   r9   c                 ó  — | j         rB| j        �;|t          j        |¦  «                             d| j        z
  d| j        z   ¦  «        z  }|                     d|j        d         ¦  «        }|                      |¦  «        }|S )Nr=   rM   )r�   rý   rD   Ú
empty_likeÚuniform_rµ   rU   rþ   )r4   r{   Úrouter_logitss      r7   rc   zDbrxRouter.forwardH  s†   € ØŒ=ð 	˜TÔ0Ð<Ø�UÔ-¨mÑ<Ô<×EÒEØ�dÔ)Ñ)¨3°Ô1DÑ+Dñô ñ ˆMð &×*Ò*¨2¨}Ô/BÀ2Ô/FÑGÔGˆØŸ
š
 =Ñ1Ô1ˆØÐr8   rÜ   rm   s   @r7   rú   rú   A  sc   ø€ € € € € ðUð Uð Uð Uð Uð U¤\ð °e´lð ð ð ð ð ð ð ð r8   rú   c                   óL   ‡ — e Zd ZdZˆ fd„Zd„ Zdej        dej        fd„Zˆ xZ	S )ÚDbrxFFNz0Modular DBRX MLP/FFN component with MoE support.c                 óð   •— t          ¦   «                              ¦   «          t          |j        ¦  «        | _        t          |j        ¦  «        | _        |j        j        | _        |j        j        | _	        d S rd   )
r(   r)   rú   Ú
ffn_configÚrouterrÞ   ÚexpertsÚmoe_normalize_expert_weightsÚ	moe_top_kÚtop_k)r4   r#   rH   r6   s      €r7   r)   zDbrxFFN.__init__U  s\   ø€ Ý‰Œ×ÒÑÔÐÝ  Ô!2Ñ3Ô3ˆŒÝ" 6Ô#4Ñ5Ô5ˆŒà,2Ô,=Ô,ZˆÔ)ØÔ&Ô0ˆŒ
ˆ
ˆ
r8   c                 óî   — t           j        j                             |d|j        ¬¦  «        }t          j        || j        d¬¦  «        \  }}| j        � |t          j        || j        dd¬¦  «        z  }||fS )Nr   rŒ   rM   rS   T)rŽ   rJ   Úkeepdim)	rD   r   r’   r“   r@   Útopkr  r	  Únorm)r4   r  Úrouter_top_valueÚrouter_indicess       r7   Úroute_tokens_to_expertszDbrxFFN.route_tokens_to_experts]  s…   € ÝœÔ+×3Ò3°MÀqÐP]ÔPcÐ3ÑdÔdˆÝ+0¬:°mÀTÄZÐUWÐ+XÑ+XÔ+XÑ(Ð˜.ØÔ,Ð8Ø/µ%´*Ø  DÔ$EÈ2ÐW[ð3ñ 3ô 3ñ  Ðð   Ð/Ð/r8   r{   r9   c                 óŽ   — |                       |¦  «        }|                      |¦  «        \  }}|                      |||¦  «        }|S rd   )r  r  r  )r4   r{   r  rä   rã   Úoutputs         r7   rc   zDbrxFFN.forwardf  sE   € ØŸš MÑ2Ô2ˆØ%)×%AÒ%AÀ-Ñ%PÔ%PÑ"ˆ�{Ø—’˜m¨[¸-ÑHÔHˆØˆr8   )
re   rf   rg   rÀ   r)   r  rD   rj   rc   rl   rm   s   @r7   r  r  R  ss   ø€ € € € € Ø:Ð:ð1ð 1ð 1ð 1ð 1ð0ð 0ð 0ð U¤\ð °e´lð ð ð ð ð ð ð ð r8   r  c                   ó¤   ‡ — e Zd Zddededz  fˆ fd„Z	 	 ddej        dej        dej        dz  de	dz  d	e
d
eej        ej        f         fd„Zˆ xZS )ÚDbrxNormAttentionNormNr#   r�   c                 ó  •— t          ¦   «                              ¦   «          || _        |j        | _        t	          j        |j        d¬¦  «        | _        t          ||¬¦  «        | _	        t	          j        |j        d¬¦  «        | _
        d S )NFrŸ   ©r#   r�   )r(   r)   r�   Úresid_pdropr   Ú	LayerNormr¡   Únorm_1rœ   ÚattnÚnorm_2©r4   r#   r�   r6   s      €r7   r)   zDbrxNormAttentionNorm.__init__n  s}   ø€ Ý‰Œ×ÒÑÔÐØ"ˆŒØ!Ô-ˆÔÝ”l 6¤>¸Ð>Ñ>Ô>ˆŒÝ!ØØð
ñ 
ô 
ˆŒ	õ ”l 6¤>¸Ð>Ñ>Ô>ˆŒˆˆr8   r{   r®   rˆ   r¯   rH   r9   c                 óR  — |}|                       |¦  «                             |j        ¦  «        } | j        d||||dœ|¤Ž\  }}t          j                             || j        | j        ¬¦  «        }||z   }|}|  	                    |¦  «                             |j        ¦  «        }||fS ©N)r{   rˆ   r®   r¯   r�   © )
r  rG   r@   r  r   r’   rŠ   r  r�   r  )r4   r{   r®   rˆ   r¯   rH   Úresidual_statesÚ_s           r7   rc   zDbrxNormAttentionNorm.forwardy  sÄ   € ð (ˆØŸš MÑ2Ô2×5Ò5°mÔ6IÑJÔJˆà$˜4œ9ð 
Ø'Ø)Ø 3Ø+ð	
ð 
ð
 ð
ð 
Ñˆ�qõ œ×-Ò-¨m¸tÔ?OÐZ^ÔZgÐ-ÑhÔhˆØ%¨Ñ7ˆà'ˆØŸš MÑ2Ô2×5Ò5°mÔ6IÑJÔJˆà Ð-Ð-r8   rd   )NN)re   rf   rg   r   rÁ   r)   rD   rj   rÂ   r	   r   ri   rc   rl   rm   s   @r7   r  r  m  sÍ   ø€ € € € € ð	?ð 	?˜zð 	?°c¸D±jð 	?ð 	?ð 	?ð 	?ð 	?ð 	?ð /3Ø(,ð.ð .à”|ð.ð #Ô-ð.ð œ tÑ+ð	.ð
  ™ð.ð ð.ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð.ð .ð .ð .ð .ð .ð .ð .r8   r  c                   ó|   ‡ — e Zd Zdedefˆ fd„Z	 	 	 ddej        dej        dz  dej        dz  de	dz  d	e
f
d
„Zˆ xZS )Ú	DbrxBlockr#   r�   c                 óÜ   •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        t          ||¬¦  «        | _        t          |¬¦  «        | _	        d S )Nr  ©r#   )
r(   r)   r¡   rB   r  r�   r  Únorm_attn_normr  Úffnr  s      €r7   r)   zDbrxBlock.__init__–  sj   ø€ Ý‰Œ×ÒÑÔÐØ!œ>ˆÔØ!Ô-ˆÔØ"ˆŒÝ3ØØð
ñ 
ô 
ˆÔõ  &Ð)Ñ)Ô)ˆŒˆˆr8   Nr{   rˆ   r®   r¯   rH   c                 ó¼   —  | j         d||||dœ|¤Ž\  }}|                      |¦  «        }t          j                             || j        | j        ¬¦  «        }||z   }|S r   )r(  r)  r   r’   rŠ   r  r�   )r4   r{   rˆ   r®   r¯   rH   Úresid_statess          r7   rc   zDbrxBlock.forward¡  sƒ   € ð ': dÔ&9ð '
Ø'Ø)Ø 3Ø+ð	'
ð '
ð
 ð'
ð '
Ñ#ˆ�mð Ÿš Ñ/Ô/ˆÝœ×-Ò-¨m¸tÔ?OÐZ^ÔZgÐ-ÑhÔhˆØ$ }Ñ4ˆØÐr8   r¿   )re   rf   rg   r   rÁ   r)   rD   rj   rÂ   r	   r   rc   rl   rm   s   @r7   r%  r%  •  s´   ø€ € € € € ð	*˜zð 	*°cð 	*ð 	*ð 	*ð 	*ð 	*ð 	*ð /3Ø7;Ø(,ðð à”|ðð œ tÑ+ðð #Ô-°Ñ4ð	ð
  ™ðð ðð ð ð ð ð ð ð r8   r%  c                   ó–   ‡ — e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZdZeedœZ ej        ¦   «         dej        fˆ fd	„¦   «         Zˆ xZS )
ÚDbrxPreTrainedModelr#   ÚtransformerTr%  r¯   F)r{   Ú
attentionsr„   c                 ó8  •— t          ¦   «                              |¦  «         | j        j        }t	          |t
          ¦  «        rVt          j        |j        d|¬¦  «         t          j        |j	        d|¬¦  «         t          j        |j
        d|¬¦  «         d S d S )Nrƒ   )ÚmeanÚstd)r(   Ú_init_weightsr#   Úinitializer_rangerV   rÄ   ÚinitÚnormal_rÌ   rÍ   rÎ   )r4   r„   r2  r6   s      €r7   r3  z!DbrxPreTrainedModel._init_weightsÇ  s’   ø€ å‰Œ×Ò˜fÑ%Ô%Ð%ØŒkÔ+ˆÝ�f�mÑ,Ô,ð 	7ÝŒL˜œ¨°#Ð6Ñ6Ô6Ð6ÝŒL˜œ¨°#Ð6Ñ6Ô6Ð6ÝŒL˜œ¨°#Ð6Ñ6Ô6Ð6Ð6Ð6ð	7ð 	7r8   )re   rf   rg   r   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flex_attnÚ_supports_attention_backendÚ_supports_flash_attnÚ_supports_sdpaÚ_can_compile_fullgraphr%  rœ   Ú_can_record_outputsrD   rk   r   ÚModuler3  rl   rm   s   @r7   r-  r-  ·  sµ   ø€ € € € € € ØÐÐÑØ%ÐØ&*Ð#Ø$˜ÐØ#4Ð"5ÐØÐØ"&ÐØÐØ€NØ"Ðà"Ø#ðð Ðð
 €U„]�_„_ð7 B¤Ið 7ð 7ð 7ð 7ð 7ñ „_ð7ð 7ð 7ð 7ð 7r8   r-  c                   ó  ‡ — e Zd ZdZdefˆ fd„Zdej        fd„Zdej        fd„Z	e
ee	 	 	 	 	 	 dd	ej        dz  d
ej        dz  dej        dz  dedz  dej        dz  dedz  dee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )Ú	DbrxModela©  Transformer decoder consisting of *config.num_hidden_layers*. Each layer is a [`DbrxBlock`] layer.

    Args:
        config ([`DbrxConfig`]): Model configuration class with all parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
    r#   c                 óô  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        ‰j        | _        t          ‰¦  «        | _        t          j	        ‰j        ‰j
        | j        ¦  «        | _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          j        ‰j
        d¬¦  «        | _        d| _        |                      ¦   «          d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r!  )r%  )Ú.0r�   r#   s     €r7   ú
<listcomp>z&DbrxModel.__init__.<locals>.<listcomp>â  s#   ø€ Ð$jÐ$jÐ$jÀi¥Y¨v°yÑ%AÔ%AÐ$jÐ$jÐ$jr8   FrŸ   )r(   r)   Úpad_token_idÚpadding_idxÚ
vocab_sizeÚ	emb_pdropr    Ú
rotary_embr   Ú	Embeddingr¡   ÚwteÚ
ModuleListÚrangeÚn_layersÚblocksr  Únorm_fÚgradient_checkpointingÚ	post_initrâ   s    `€r7   r)   zDbrxModel.__init__Û  sÒ   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒØÔ)ˆŒÝ-¨fÑ5Ô5ˆŒÝ”< Ô 1°6´>À4ÔCSÑTÔTˆŒÝ”mÐ$jÐ$jÐ$jÐ$jÕSXÐY_ÔYhÑSiÔSiÐ$jÑ$jÔ$jÑkÔkˆŒÝ”l 6¤>¸Ð>Ñ>Ô>ˆŒØ&+ˆÔ#ð 	�ŠÑÔÐÐÐr8   r9   c                 ó   — | j         S rd   ©rO  ©r4   s    r7   Úget_input_embeddingszDbrxModel.get_input_embeddingsé  s	   € ØŒxˆr8   r‡   c                 ó   — || _         d S rd   rX  ©r4   r‡   s     r7   Úset_input_embeddingszDbrxModel.set_input_embeddingsì  s   € ØˆŒˆˆr8   NÚ	input_idsrˆ   r^   r¯   Úinputs_embedsÚ	use_cacherH   c           
      óF  — |d u |d uz  rt          d¦  «        ‚|r|€t          | j        ¬¦  «        }|€|                      |¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   }| 	                    d¦  «        }t          | j        ||||¬¦  «        }	|}
|                      |
|¦  «        }| j        d | j        j        …         D ]} ||
f||	|||dœ|¤Ž}
Œ|                      |
¦  «        }
t          |
|¬¦  «        S )	Nz:You must specify exactly one of input_ids or inputs_embedsr'  r   r   )r!   )r#   r_  rˆ   r¯   r^   )r®   rˆ   r^   r¯   r`  )Úlast_hidden_stater¯   )Ú
ValueErrorr
   r#   rO  Úget_seq_lengthrD   rE   rU   r!   rt   r   rM  rS  Únum_hidden_layersrT  r   )r4   r^  rˆ   r^   r¯   r_  r`  rH   Úpast_seen_tokensÚcausal_maskr{   r®   Údecoder_layers                r7   rc   zDbrxModel.forwardï  sˆ  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐ Ø ŸHšH YÑ/Ô/ˆMàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå(Ø”;Ø'Ø)Ø+Ø%ð
ñ 
ô 
ˆð &ˆð #Ÿošo¨m¸\ÑJÔJÐà!œ[Ð)H¨4¬;Ô+HÐ)HÔIð 		ð 		ˆMØ)˜MØðà$7Ø*Ø)Ø /Ø#ðð ð ðð ˆMˆMð Ÿš MÑ2Ô2ˆå%Ø+Ø+ð
ñ 
ô 
ð 	
r8   )NNNNNN)re   rf   rg   rÀ   r   r)   r   rN  rZ  r]  r   r   r   rD   rÂ   rj   r	   ÚFloatTensorÚboolr   r   r   rc   rl   rm   s   @r7   rD  rD  Ñ  sQ  ø€ € € € € ðð ð˜zð ð ð ð ð ð ð b¤lð ð ð ð ð¨"¬,ð ð ð ð ð  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ð5
ð 5
àÔ# dÑ*ð5
ð œ tÑ+ð5
ð Ô&¨Ñ-ð	5
ð
  ™ð5
ð Ô(¨4Ñ/ð5
ð ˜$‘;ð5
ð Ð+Ô,ð5
ð 
 ð5
ð 5
ð 5
ñ „^ñ „_ñ  Ôð5
ð 5
ð 5
ð 5
ð 5
r8   rD  r>   Úgate_logitsrá   c           	      óÎ  — | �t          | t          ¦  «        sdS | d         j        }t          j        |t          j        |¬¦  «        }t          j        |t          j        |¬¦  «        }d}|�4|                     d¦  «                             |t          j        ¬¦  «        }| D �]‰}	t          j        j	         
                    |	                     |¦  «        d¬¦  «        }
t          j        |
|d¬¦  «        \  }}|€z|t          j        |                     d¦  «        |¬¦  «                             ¦   «         z   }||
                     ¦   «                              d¬¦  «        z   }||
j        d         z   }ŒÒ|t          j        |t          j        |¬¦  «                             d|                     d¦  «        |                     |¦  «        ¦  «        z   }||
                     ¦   «         |                     d¦  «        z                       d¬¦  «        z   }||                     ¦   «         z   }�Œ‹||z  }||z  }t          j        ||                     d¦  «        z  ¦  «        }||z  S )	aÄ  
    Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

    See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
    function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
    experts is too unbalanced.

    Args:
        gate_logits:
            Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
            shape [batch_size X sequence_length, num_experts].
        num_experts:
            Number of experts
        top_k:
            The number of experts to route per-token, can be also interpreted as the `top-k` routing
            parameter.
        attention_mask (`torch.Tensor`, *optional*):
            The attention_mask used in forward function
            shape [batch_size X sequence_length] if not None.

    Returns:
        The auxiliary loss.
    Nr   rN   rƒ   rM   )r!   r@   rS   )Ú	minlength)rV   ri   r!   rD   Úzerosr”   r~   rG   r   r’   r“   r  ÚbincountrF   rì   rU   Úscatter_add_Úrepeat_interleavert   )rk  rá   r  rˆ   Úcompute_deviceÚtokens_per_expert_sumÚrouter_prob_sumÚ
total_rowsÚ	flat_maskÚ
layer_gateÚrouting_weightsr#  Úselected_expertsÚtokens_per_expertÚrouter_prob_per_expertÚoverall_losss                   r7   Úload_balancing_loss_funcr}  *  sV  € ð: Ð¥*¨[½%Ñ"@Ô"@ÐØˆqð ! ”^Ô*€NÝ!œK¨½5¼=ÐQ_Ð`Ñ`Ô`ÐÝ”k +µU´]È>ÐZÑZÔZ€OØ€JàÐ!à"×*Ò*¨2Ñ.Ô.×1Ò1¸ÍuÌ}Ð1Ñ]Ô]ˆ	à!ð 6ñ 6ˆ
Ýœ(Ô-×5Ò5°j·m²mÀNÑ6SÔ6SÐY[Ð5Ñ\Ô\ˆÝ#œj¨¸%ÀRÐHÑHÔHÑˆÐØÐ!ð &­¬Ð7G×7OÒ7OÐPRÑ7SÔ7SÐ_jÐ(kÑ(kÔ(k×(qÒ(qÑ(sÔ(sÑsð "ð .°×0EÒ0EÑ0GÔ0G×0KÒ0KÐPQÐ0KÑ0RÔ0RÑRˆOØ# oÔ&;¸AÔ&>Ñ>ˆJˆJð %:½E¼KØ¥5¤=¸ð=ñ =ô =çŠl˜1Ð.×6Ò6°rÑ:Ô:¸I×<WÒ<WÐX]Ñ<^Ô<^Ñ_Ô_ñ%`Ð!ð .°×1FÒ1FÑ1HÔ1HÈ9×K^ÒK^Ð_aÑKbÔKbÑ1b×0gÒ0gÐlmÐ0gÑ0nÔ0nÑnˆOØ# i§m¢m¡o¤oÑ5ˆJ‰Jà-°
Ñ:ÐØ,¨zÑ9Ðå”9Ð.Ð1G×1QÒ1QÐRSÑ1TÔ1TÑTÑUÔU€LØ˜+Ñ%Ð%r8   c                   ó¢  ‡ — e Zd ZddiZddiZddgdgfiZddiZdefˆ fd	„Zd
e	j
        fd„Zde	j
        fd„Zd
e	j        fd„Zde	j        fd„Zdefd„Zd
efd„Zee	 	 	 	 	 	 	 	 	 d!dej        dz  dej        dz  dej        dz  dedz  dej        dz  dej        dz  dedz  dedz  deej        z  dee         d
efd „¦   «         ¦   «         Zˆ xZ S )"ÚDbrxForCausalLMzlm_head.weightztransformer.wte.weightÚlm_headÚcolwise_gather_outputr{   ÚlogitsÚkeep_full_weightr#   c                 ód  •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        | _        t          j        |j        |j        d¬¦  «        | _        |j	        j
        | _        |j	        j        | _        |j	        j        | _        |                      ¦   «          d S rü   )r(   r)   rD  r.  rK  r   r«   rB   r€  r  Úmoe_loss_weightÚrouter_aux_loss_coefrÉ   rá   r
  Únum_experts_per_tokrV  râ   s     €r7   r)   zDbrxForCausalLM.__init__v  s“   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý$ VÑ,Ô,ˆÔØ Ô+ˆŒÝ”y Ô!3°VÔ5FÈUÐSÑSÔSˆŒØ$*Ô$5Ô$EˆÔ!Ø!Ô,Ô<ˆÔØ#)Ô#4Ô#>ˆÔ Ø�ŠÑÔÐÐÐr8   r9   c                 ó4   — | j                              ¦   «         S rd   )r.  rZ  rY  s    r7   rZ  z$DbrxForCausalLM.get_input_embeddings€  s   € ØÔ×4Ò4Ñ6Ô6Ð6r8   r‡   c                 ó:   — | j                              |¦  «         d S rd   )r.  r]  r\  s     r7   r]  z$DbrxForCausalLM.set_input_embeddingsƒ  s   € ØÔ×-Ò-¨eÑ4Ô4Ð4Ð4Ð4r8   c                 ó   — | j         S rd   ©r€  rY  s    r7   Úget_output_embeddingsz%DbrxForCausalLM.get_output_embeddings†  s
   € ØŒ|Ðr8   Únew_embeddingsc                 ó   — || _         d S rd   r‹  )r4   r�  s     r7   Úset_output_embeddingsz%DbrxForCausalLM.set_output_embeddings‰  s   € Ø%ˆŒˆˆr8   Údecoderc                 ó   — || _         d S rd   ©r.  )r4   r�  s     r7   Úset_decoderzDbrxForCausalLM.set_decoderŒ  s   € Ø"ˆÔÐÐr8   c                 ó   — | j         S rd   r’  rY  s    r7   Úget_decoderzDbrxForCausalLM.get_decoder�  s   € ØÔÐr8   Nr   r^  rˆ   r^   r¯   r_  Úlabelsr`  Úoutput_router_logitsÚlogits_to_keeprH   c
                 ó  — |�|n| j         j        } | j        d|||||||dœ|
¤Ž}|j        }t	          |	t
          ¦  «        rt          |	 d¦  «        n|	}|                      |dd…|dd…f         ¦  «        }d}|� | j        ||| j	        fi |
¤Ž}d}|rHt          |j        | j        | j        |¦  «        }|�%|| j        |                     |j        ¦  «        z  z  }t#          ||||j        |j        |j        |j        ¬¦  «        S )a‹  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >> from transformers import AutoTokenizer, DbrxForCausalLM

        >> model = DbrxForCausalLM.from_pretrained("transformers-community/dbrx-instruct")
        >> tokenizer = AutoTokenizer.from_pretrained("transformers-community/dbrx-instruct")

        >> prompt = "Hey, are you conscious? Can you talk to me?"
        >> inputs = tokenizer(prompt, return_tensors="pt")

        >> # Generate
        >> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```
        N)r^  rˆ   r^   r¯   r_  r`  r—  )ÚlossÚaux_lossr‚  r¯   r{   r/  r  r!  )r#   r—  r.  rb  rV   rÁ   Úslicer€  Úloss_functionrK  r}  r  rá   r‡  r†  rG   r!   r   r¯   r{   r/  )r4   r^  rˆ   r^   r¯   r_  r–  r`  r—  r˜  rH   Úoutputsr{   Úslice_indicesr‚  rš  r›  s                    r7   rc   zDbrxForCausalLM.forward’  so  € ðN %9Ð$DÐ Ð È$Ì+ÔJjð 	ð
 +;¨$Ô*:ð 	+
ØØ)Ø%Ø+Ø'ØØ!5ð	+
ð 	+
ð ð	+
ð 	+
ˆð  Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô% f¨f°d´oÐPÐPÈÐPÐPˆDàˆØð 	MÝ/ØÔ%ØÔ ØÔ(Øñ	ô ˆHð Ð!Ø˜Ô1°H·K²KÀÄÑ4LÔ4LÑLÑL�å(ØØØØ#Ô3Ø!Ô/ØÔ)Ø!Ô/ð
ñ 
ô 
ð 	
r8   )	NNNNNNNNr   )!re   rf   rg   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr   r)   r   rN  rZ  r]  r«   rŒ  r�  rD  r“  r•  r   r   rD   rÂ   rj   r	   ri  rj  rÁ   r   r   r   rc   rl   rm   s   @r7   r  r  o  s1  ø€ € € € € Ø*Ð,DÐEÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HàÐ/Ð0€Jð˜zð ð ð ð ð ð ð7 b¤lð 7ð 7ð 7ð 7ð5¨"¬,ð 5ð 5ð 5ð 5ð r¤yð ð ð ð ð&°B´Ið &ð &ð &ð &ð# 9ð #ð #ð #ð #ð ˜Yð  ð  ð  ð  ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø,0Ø-.ðP
ð P
àÔ# dÑ*ðP
ð œ tÑ+ðP
ð Ô&¨Ñ-ð	P
ð
  ™ðP
ð Ô(¨4Ñ/ðP
ð Ô  4Ñ'ðP
ð ˜$‘;ðP
ð # T™kðP
ð ˜eœlÑ*ðP
ð Ð+Ô,ðP
ð 
#ðP
ð P
ð P
ñ „^ñ ÔðP
ð P
ð P
ð P
ð P
r8   r  )r  rD  r-  )r   )rƒ   )Nr>   N)DÚcollections.abcr   Útypingr   rD   r   Ú r   r5  Úactivationsr   Úcache_utilsr	   r
   Ú
generationr   Úintegrationsr   Úmasking_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_dbrxr   rB  r    rq   rz   rj   rÁ   r‚   rF   rš   rœ   rÄ   rÞ   rú   r  r  r%  r-  rD  ri   r}  r  Ú__all__r!  r8   r7   ú<module>r·     s  ðð* %Ð $Ð $Ð $Ð $Ð $Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )Ø 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø /Ð /Ð /Ð /Ð /Ð /Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø QÐ QÐ QÐ QÐ QÐ QÐ QÐ QØ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø *Ð *Ð *Ð *Ð *Ð *ð6<ð 6<ð 6<ð 6<ð 6<˜"œ)ñ 6<ô 6<ð 6<ðr(ð (ð (ð ÐÐ-Ñ.Ô.ðð ð ñ /Ô.ðð2	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð& ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð2R)ð R)ð R)ð R)ð R)�B”Iñ R)ô R)ð R)ðjð ð ð ð �B”Iñ ô ð ð2$ð $ð $ð $ð $�"”)ñ $ô $ð $ðNð ð ð ð �”ñ ô ð ð"ð ð ð ð ˆbŒiñ ô ð ð6%.ð %.ð %.ð %.ð %.˜BœIñ %.ô %.ð %.ðPð ð ð ð Ð*ñ ô ð ðD7ð 7ð 7ð 7ð 7˜/ñ 7ô 7ð 7ð4 ðU
ð U
ð U
ð U
ð U
Ð#ñ U
ô U
ñ „ðU
ðt #Ø
Ø*.ð	B&ð B&Ø”  e¤lÔ 3Ñ3°dÑ:ðB&à�t‘ðB&ð ”L 4Ñ'ð	B&ð
 „\�CÑðB&ð B&ð B&ð B&ðJu
ð u
ð u
ð u
ð u
Ð)¨?ñ u
ô u
ð u
ðp BÐ
AÐ
A€€€r8   