§
    9²¸jæF  ã            
       ó0  — d Z ddlZddlmc mZ ddlmZ ddlmZ ddl	m
Z
 ddlmZmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZmZ ddlmZ ddlmZ ddlmZmZ ddlm Z  ddl!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z* ddl+m,Z,  ej-        e.¦  «        Z/	 	 	 d4dej0        e1ej0                 z  dz  de2dz  dej0        dz  dej0        e2z  fd„Z3e G d„ dej4        ¦  «        ¦   «         Z5 G d„ dej4        ¦  «        Z6 G d„ dej4        ¦  «        Z7 G d„ d e)¦  «        Z8 G d!„ d"e*¦  «        Z9 G d#„ d$e"¦  «        Z: G d%„ d&e¦  «        Z; G d'„ d(e(¦  «        Z< G d)„ d*e'¦  «        Z= G d+„ d,e#¦  «        Z> G d-„ d.e%¦  «        Z? G d/„ d0e&¦  «        Z@ G d1„ d2e$¦  «        ZAg d3¢ZBdS )5zPyTorch Mixtral model.é    N)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)Úuse_experts_implementation)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚlogging)ÚOutputRecorderé   )	ÚMistralAttentionÚMistralForCausalLMÚMistralForQuestionAnsweringÚ MistralForSequenceClassificationÚMistralForTokenClassificationÚMistralModelÚMistralPreTrainedModelÚMistralRMSNormÚMistralRotaryEmbeddingé   )ÚMixtralConfigÚgate_logitsÚnum_expertsÚattention_maskÚreturnc           	      óÎ  — | �t          | t          ¦  «        sdS | d         j        }t          j        |t          j        |¬¦  «        }t          j        |t          j        |¬¦  «        }d}|�4|                     d¦  «                             |t          j        ¬¦  «        }| D �]‰}	t          j        j	         
                    |	                     |¦  «        d¬¦  «        }
t          j        |
|d¬¦  «        \  }}|€z|t          j        |                     d¦  «        |¬¦  «                             ¦   «         z   }||
                     ¦   «                              d¬¦  «        z   }||
j        d         z   }ŒÒ|t          j        |t          j        |¬¦  «                             d|                     d¦  «        |                     |¦  «        ¦  «        z   }||
                     ¦   «         |                     d¦  «        z                       d¬¦  «        z   }||                     ¦   «         z   }�Œ‹||z  }||z  }t          j        ||                     d¦  «        z  ¦  «        }||z  S )	aÄ  
    Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

    See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
    function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
    experts is too unbalanced.

    Args:
        gate_logits:
            Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
            shape [batch_size X sequence_length, num_experts].
        num_experts:
            Number of experts
        top_k:
            The number of experts to route per-token, can be also interpreted as the `top-k` routing
            parameter.
        attention_mask (`torch.Tensor`, *optional*):
            The attention_mask used in forward function
            shape [batch_size X sequence_length] if not None.

    Returns:
        The auxiliary loss.
    Nr   )ÚdtypeÚdeviceç        éÿÿÿÿ)r&   r%   ©Údim)Ú	minlength)Ú
isinstanceÚtupler&   ÚtorchÚzerosÚfloat32ÚreshapeÚtor   Ú
functionalÚsoftmaxÚtopkÚbincountÚfloatÚsumÚshapeÚscatter_add_Úrepeat_interleaveÚ	unsqueeze)r    r!   Útop_kr"   Úcompute_deviceÚtokens_per_expert_sumÚrouter_prob_sumÚ
total_rowsÚ	flat_maskÚ
layer_gateÚrouting_weightsÚ_Úselected_expertsÚtokens_per_expertÚrouter_prob_per_expertÚoverall_losss                   ú{/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/mixtral/modular_mixtral.pyÚload_balancing_loss_funcrK   5   sV  € ð: Ð¥*¨[½%Ñ"@Ô"@ÐØˆqð ! ”^Ô*€NÝ!œK¨½5¼=ÐQ_Ð`Ñ`Ô`ÐÝ”k +µU´]È>ÐZÑZÔZ€OØ€JàÐ!à"×*Ò*¨2Ñ.Ô.×1Ò1¸ÍuÌ}Ð1Ñ]Ô]ˆ	à!ð 6ñ 6ˆ
Ýœ(Ô-×5Ò5°j·m²mÀNÑ6SÔ6SÐY[Ð5Ñ\Ô\ˆÝ#œj¨¸%ÀRÐHÑHÔHÑˆÐØÐ!ð &­¬Ð7G×7OÒ7OÐPRÑ7SÔ7SÐ_jÐ(kÑ(kÔ(k×(qÒ(qÑ(sÔ(sÑsð "ð .°×0EÒ0EÑ0GÔ0G×0KÒ0KÐPQÐ0KÑ0RÔ0RÑRˆOØ# oÔ&;¸AÔ&>Ñ>ˆJˆJð %:½E¼KØ¥5¤=¸ð=ñ =ô =çŠl˜1Ð.×6Ò6°rÑ:Ô:¸I×<WÒ<WÐX]Ñ<^Ô<^Ñ_Ô_ñ%`Ð!ð .°×1FÒ1FÑ1HÔ1HÈ9×K^ÒK^Ð_aÑKbÔKbÑ1b×0gÒ0gÐlmÐ0gÑ0nÔ0nÑnˆOØ# i§m¢m¡o¤oÑ5ˆJ‰Jà-°
Ñ:ÐØ,¨zÑ9Ðå”9Ð.Ð1G×1QÒ1QÐRSÑ1TÔ1TÑTÑUÔU€LØ˜+Ñ%Ð%ó    c                   óh   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        dej        dej        fd„Zˆ xZ	S )	ÚMixtralExpertsz2Collection of expert weights stored as 3D tensors.Úconfigc                 ó´  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        t          j	        t          j        | j        d| j        z  | j        ¦  «        ¦  «        | _        t          j	        t          j        | j        | j        | j        ¦  «        ¦  «        | _        t          |j                 | _        d S )Nr   )ÚsuperÚ__init__Únum_local_expertsr!   Úhidden_sizeÚ
hidden_dimÚintermediate_sizeÚintermediate_dimr   Ú	Parameterr.   ÚemptyÚgate_up_projÚ	down_projr   Ú
hidden_actÚact_fn©ÚselfrO   Ú	__class__s     €rJ   rR   zMixtralExperts.__init__~   s£   ø€ Ý‰Œ×ÒÑÔÐØ!Ô3ˆÔØ Ô,ˆŒØ &Ô 8ˆÔÝœL­¬°TÔ5EÀqÈ4ÔK`ÑG`ÐbfÔbqÑ)rÔ)rÑsÔsˆÔÝœ¥e¤k°$Ô2BÀDÄOÐUYÔUjÑ&kÔ&kÑlÔlˆŒÝ˜VÔ.Ô/ˆŒˆˆrL   Úhidden_statesÚtop_k_indexÚtop_k_weightsr#   c                 ó€  — t          j        |¦  «        }t          j        ¦   «         5  t           j        j                             || j        ¬¦  «        }|                     ddd¦  «        }t          j        | 	                    d¬¦  «        d¦  «         
                    ¦   «         }d d d ¦  «         n# 1 swxY w Y   |D ]þ}|d         }|| j        k    rŒt          j        ||         ¦  «        \  }}	||	         }
t          j                             |
| j        |         ¦  «                             dd¬¦  «        \  }}|                      |¦  «        |z  }t          j                             || j        |         ¦  «        }|||	|d f         z  }|                     d|	|                     |j        ¦  «        ¦  «         Œÿ|S )N)Únum_classesr   r   r   )r(   éþÿÿÿr)   r(   )r.   Ú
zeros_likeÚno_gradr   r3   Úone_hotr!   ÚpermuteÚgreaterr8   ÚnonzeroÚwhereÚlinearrZ   Úchunkr]   r[   Ú
index_add_r2   r%   )r_   ra   rb   rc   Úfinal_hidden_statesÚexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stateÚgateÚupÚcurrent_hidden_statess                 rJ   ÚforwardzMixtralExperts.forward‡   sø  € õ $Ô.¨}Ñ=Ô=ÐÝŒ]‰_Œ_ð 	Sð 	SÝœ(Ô-×5Ò5°kÈtÔO_Ð5Ñ`Ô`ˆKØ%×-Ò-¨a°°AÑ6Ô6ˆKÝœ {§¢¸8 Ñ'DÔ'DÀaÑHÔH×PÒPÑRÔRˆJð	Sð 	Sð 	Sñ 	Sô 	Sð 	Sð 	Sð 	Sð 	Sð 	Sð 	Søøøð 	Sð 	Sð 	Sð 	Sð
 %ð 
	nð 
	nˆJØ# AœˆJØ˜TÔ-Ò-Ð-ØÝ#(¤;¨{¸:Ô/FÑ#GÔ#GÑ ˆI�yØ)¨)Ô4ˆMÝ”}×+Ò+¨M¸4Ô;LÈZÔ;XÑYÔY×_Ò_Ð`aÐgiÐ_ÑjÔj‰HˆD�"Ø$(§K¢K°Ñ$5Ô$5¸Ñ$:Ð!Ý$&¤M×$8Ò$8Ð9NÐPTÔP^Ð_iÔPjÑ$kÔ$kÐ!Ø$9¸MÈ)ÐU^Ð`dÐJdÔ<eÑ$eÐ!Ø×*Ò*¨1¨iÐ9N×9QÒ9QÐReÔRkÑ9lÔ9lÑmÔmÐmÐmà"Ð"s   ¨A>B2Â2B6Â9B6)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   rR   r.   ÚTensorr{   Ú__classcell__©r`   s   @rJ   rN   rN   z   sŒ   ø€ € € € € à<Ð<ð0˜}ð 0ð 0ð 0ð 0ð 0ð 0ð#à”|ð#ð ”\ð#ð ”|ð	#ð
 
Œð#ð #ð #ð #ð #ð #ð #ð #rL   rN   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚMixtralTopKRouterc                 óü   •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        t          j	        t          j        | j        | j        ¦  «        ¦  «        | _        d S ©N)rQ   rR   Únum_experts_per_tokr=   rS   r!   rT   rU   r   rX   r.   rY   Úweightr^   s     €rJ   rR   zMixtralTopKRouter.__init__£   s^   ø€ Ý‰Œ×ÒÑÔÐØÔ/ˆŒ
Ø!Ô3ˆÔØ Ô,ˆŒÝ”l¥5¤;¨tÔ/?ÀÄÑ#QÔ#QÑRÔRˆŒˆˆrL   c                 ó\  — |                      d| j        ¦  «        }t          j        || j        ¦  «        }t
          j        j                             | 	                    ¦   «         d¬¦  «        }t          j
        || j        d¬¦  «        \  }}||                     dd¬¦  «        z  }|}|||fS )Nr(   r)   T)r*   Úkeepdim)r1   rU   ÚFrn   rˆ   r.   r   r3   r4   r7   r5   r=   r8   )r_   ra   Úrouter_logitsÚrouter_probsÚrouter_top_valueÚrouter_indicesÚrouter_scoress          rJ   r{   zMixtralTopKRouter.forwardª   s¥   € Ø%×-Ò-¨b°$´/ÑBÔBˆÝœ °´Ñ<Ô<ˆÝ”xÔ*×2Ò2°=×3FÒ3FÑ3HÔ3HÈbÐ2ÑQÔQˆÝ+0¬:°lÀDÄJÐTVÐ+WÑ+WÔ+WÑ(Ð˜.ØÐ,×0Ò0°RÀÐ0ÑFÔFÑFÐØ(ˆØ˜m¨^Ð;Ð;rL   )r|   r}   r~   rR   r{   r�   r‚   s   @rJ   r„   r„   ¢   sL   ø€ € € € € ðSð Sð Sð Sð Sð<ð <ð <ð <ð <ð <ð <rL   r„   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚMixtralSparseMoeBlockc                 óÈ   •— t          ¦   «                              ¦   «          |j        | _        |j        | _        t          |¦  «        | _        t          |¦  «        | _	        d S r†   )
rQ   rR   r‡   r=   Úrouter_jitter_noiseÚjitter_noiser„   rx   rN   Úexpertsr^   s     €rJ   rR   zMixtralSparseMoeBlock.__init__µ   sP   ø€ Ý‰Œ×ÒÑÔÐØÔ/ˆŒ
Ø"Ô6ˆÔÝ% fÑ-Ô-ˆŒ	Ý% fÑ-Ô-ˆŒˆˆrL   ra   r#   c                 ó†  — |j         \  }}}| j        rF| j        dk    r;|t          j        |¦  «                             d| j        z
  d| j        z   ¦  «        z  }|                     d|j         d         ¦  «        }|                      |¦  «        \  }}}|                      |||¦  «        }| 	                    |||¦  «        }|S )Nr   g      ð?r(   )
r9   Útrainingr•   r.   Ú
empty_likeÚuniform_Úviewrx   r–   r1   )r_   ra   Ú
batch_sizeÚsequence_lengthrU   rE   rc   rb   s           rJ   r{   zMixtralSparseMoeBlock.forward¼   sÇ   € Ø2?Ô2EÑ/ˆ
�O ZØŒ=ð 	x˜TÔ.°Ò2Ð2Ø�UÔ-¨mÑ<Ô<×EÒEÀcÈDÔL]ÑF]Ð_bÐeiÔevÑ_vÑwÔwÑwˆMØ%×*Ò*¨2¨}Ô/BÀ2Ô/FÑGÔGˆØ(,¯	ª	°-Ñ(@Ô(@Ñ%ˆˆ=˜+ØŸš ]°KÀÑOÔOˆØ%×-Ò-¨j¸/È:ÑVÔVˆØÐrL   )r|   r}   r~   rR   r.   r€   r{   r�   r‚   s   @rJ   r’   r’   ´   s^   ø€ € € € € ð.ð .ð .ð .ð .ð U¤\ð °e´lð ð ð ð ð ð ð ð rL   r’   c                   ó   — e Zd ZdS )ÚMixtralRMSNormN©r|   r}   r~   © rL   rJ   rŸ   rŸ   Ç   ó   € € € € € Ø€DrL   rŸ   c                   ó   — e Zd ZdS )ÚMixtralRotaryEmbeddingNr    r¡   rL   rJ   r¤   r¤   Ë   r¢   rL   r¤   c                   ó   — e Zd ZdS )ÚMixtralAttentionNr    r¡   rL   rJ   r¦   r¦   Ï   r¢   rL   r¦   c                   óÆ   ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 ddej        deej        ej        f         dz  dej        dz  dej	        dz  d	e
dz  d
ee         dej        fd„Zˆ xZS )ÚMixtralDecoderLayerrO   Ú	layer_idxc                 ó2  •— t          ¦   «                              ¦   «          |j        | _        t          ||¦  «        | _        t          |¦  «        | _        t          |j        |j        ¬¦  «        | _	        t          |j        |j        ¬¦  «        | _
        d S )N)Úeps)rQ   rR   rT   r¦   Ú	self_attnr’   ÚmlprŸ   Úrms_norm_epsÚinput_layernormÚpost_attention_layernorm)r_   rO   r©   r`   s      €rJ   rR   zMixtralDecoderLayer.__init__Ô   s€   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔå)¨&°)Ñ<Ô<ˆŒå(¨Ñ0Ô0ˆŒÝ-¨fÔ.@ÀfÔFYÐZÑZÔZˆÔÝ(6°vÔ7IÈvÔObÐ(cÑ(cÔ(cˆÔ%Ð%Ð%rL   Nra   Úposition_embeddingsr"   Úposition_idsÚpast_key_valuesÚkwargsr#   c           	      óÌ   — |}|                       |¦  «        } | j        d|||||dœ|¤Ž\  }}||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|S )N)ra   r±   r"   r²   r³   r¡   )r¯   r¬   r°   r­   )	r_   ra   r±   r"   r²   r³   r´   ÚresidualrE   s	            rJ   r{   zMixtralDecoderLayer.forwardÞ   sœ   € ð !ˆØ×,Ò,¨]Ñ;Ô;ˆØ)˜4œ>ð 
Ø'Ø 3Ø)Ø%Ø+ð
ð 
ð ð
ð 
Ñˆ�qð ! =Ñ0ˆØ ˆØ×5Ò5°mÑDÔDˆØŸš Ñ/Ô/ˆØ  =Ñ0ˆØÐrL   )NNNN)r|   r}   r~   r   ÚintrR   r.   r€   r-   Ú
LongTensorr   r   r   r{   r�   r‚   s   @rJ   r¨   r¨   Ó   sð   ø€ € € € € ðd˜}ð d¸ð dð dð dð dð dð dð IMØ.2Ø04Ø(,ðð à”|ðð # 5¤<°´Ð#=Ô>ÀÑEðð œ tÑ+ð	ð
 Ô&¨Ñ-ðð  ™ðð Ð+Ô,ðð 
Œðð ð ð ð ð ð ð rL   r¨   c                   ó`   — e Zd Z eed¬¦  «        eedœZ ej	        ¦   «         d„ ¦   «         Z
dS )ÚMixtralPreTrainedModelr   )Úindex)rŒ   ra   Ú
attentionsc                 óL  — t          j        | |¦  «         | j        j        }t	          |t
          ¦  «        r:t          j        |j        d|¬¦  «         t          j        |j	        d|¬¦  «         d S t	          |t          ¦  «        rt          j        |j        d|¬¦  «         d S d S )Nr'   )ÚmeanÚstd)r   Ú_init_weightsrO   Úinitializer_ranger,   rN   ÚinitÚnormal_rZ   r[   r„   rˆ   )r_   Úmoduler¿   s      rJ   rÀ   z$MixtralPreTrainedModel._init_weights   s§   € åÔ% d¨FÑ3Ô3Ð3ØŒkÔ+ˆÝ�f�nÑ-Ô-ð 	;ÝŒL˜Ô,°3¸CÐ@Ñ@Ô@Ð@ÝŒL˜Ô)°¸Ð=Ñ=Ô=Ð=Ð=Ð=Ý˜Õ 1Ñ2Ô2ð 	;ÝŒL˜œ¨S°cÐ:Ñ:Ô:Ð:Ð:Ð:ð	;ð 	;rL   N)r|   r}   r~   r   r„   r¨   r¦   Ú_can_record_outputsr.   rh   rÀ   r¡   rL   rJ   rº   rº   ù   s]   € € € € € à'˜Ð(9ÀÐCÑCÔCØ,Ø&ðð Ðð €U„]�_„_ð;ð ;ñ „_ð;ð ;ð ;rL   rº   c                   óœ   — e Zd Z	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dedz  dej        dz  dedz  de	e
         d	efd
„ZdS )ÚMixtralModelNÚ	input_idsr"   r²   r³   Úinputs_embedsÚ	use_cacher´   r#   c           
      ót  — |d u |d uz  rt          d¦  «        ‚|r|€t          | j        ¬¦  «        }|€|                      |¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   }| 	                    d¦  «        }| j        j
        €t          nt          }	 |	| j        ||||¬¦  «        }
|}|                      ||¬¦  «        }| j        d | j        j        …         D ]} ||f|
||||dœ|¤Ž}Œ|                      |¦  «        }t#          ||¬	¦  «        S )
Nz:You must specify exactly one of input_ids or inputs_embeds)rO   r   r   )r&   )rO   rÉ   r"   r³   r²   )r²   )r"   r²   r³   rÊ   r±   )Úlast_hidden_stater³   )Ú
ValueErrorr   rO   Úembed_tokensÚget_seq_lengthr.   Úaranger9   r&   r<   Úsliding_windowr
   r   Ú
rotary_embÚlayersÚnum_hidden_layersÚnormr   )r_   rÈ   r"   r²   r³   rÉ   rÊ   r´   Úpast_seen_tokensÚmask_functionÚcausal_maskra   r±   Údecoder_layers                 rJ   r{   zMixtralModel.forward  s¢  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLà.2¬kÔ.HÐ.PÕ*Ð*ÕVwˆØ#�mØ”;Ø'Ø)Ø+Ø%ð
ñ 
ô 
ˆð &ˆØ"Ÿošo¨mÈ,˜oÑWÔWÐà!œ[Ð)H¨4¬;Ô+HÐ)HÔIð 		ð 		ˆMØ)˜MØðà*Ø)Ø /Ø#Ø$7ðð ð ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆå%Ø+Ø+ð
ñ 
ô 
ð 	
rL   )NNNNNN)r|   r}   r~   r.   r¸   r€   r   ÚFloatTensorÚboolr   r   r   r{   r¡   rL   rJ   rÇ   rÇ     s¼   € € € € € ð .2Ø.2Ø04Ø(,Ø26Ø!%ð4
ð 4
àÔ# dÑ*ð4
ð œ tÑ+ð4
ð Ô&¨Ñ-ð	4
ð
  ™ð4
ð Ô(¨4Ñ/ð4
ð ˜$‘;ð4
ð Ð+Ô,ð4
ð 
 ð4
ð 4
ð 4
ð 4
ð 4
ð 4
rL   rÇ   c                   óô   ‡ — e Zd ZddiZddiZˆ fd„Z	 	 	 	 	 	 	 	 	 ddej        dz  d	ej        dz  d
ej        dz  de	dz  dej
        dz  dej        dz  dedz  dedz  deej        z  dee         defd„Zˆ xZS )ÚMixtralForCausalLMzlm_head.weightzmodel.embed_tokens.weightÚlm_headÚkeep_full_weightc                 óº   •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        | _        |j        | _        |j        | _        d S r†   )rQ   rR   rÇ   ÚmodelÚrouter_aux_loss_coefrS   r!   r‡   r^   s     €rJ   rR   zMixtralForCausalLM.__init__G  sQ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý! &Ñ)Ô)ˆŒ
Ø$*Ô$?ˆÔ!Ø!Ô3ˆÔØ#)Ô#=ˆÔ Ð Ð rL   Nr   rÈ   r"   r²   r³   rÉ   ÚlabelsrÊ   Úoutput_router_logitsÚlogits_to_keepr´   r#   c
                 ó  — |�|n| j         j        } | j        d|||||||dœ|
¤Ž}|j        }t	          |	t
          ¦  «        rt          |	 d¦  «        n|	}|                      |dd…|dd…f         ¦  «        }d}|� | j        ||| j	        fi |
¤Ž}d}|rHt          |j        | j        | j        |¦  «        }|�%|| j        |                     |j        ¦  «        z  z  }t#          ||||j        |j        |j        |j        ¬¦  «        S )a~  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, MixtralForCausalLM

        >>> model = MixtralForCausalLM.from_pretrained("mistralai/Mixtral-8x7B-v0.1")
        >>> tokenizer = AutoTokenizer.from_pretrained("mistralai/Mixtral-8x7B-v0.1")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)rÈ   r"   r²   r³   rÉ   rÊ   rä   )ÚlossÚaux_lossÚlogitsr³   ra   r¼   rŒ   r¡   )rO   rä   rá   rÌ   r,   r·   ÚslicerÞ   Úloss_functionÚ
vocab_sizerK   rŒ   r!   r‡   râ   r2   r&   r   r³   ra   r¼   )r_   rÈ   r"   r²   r³   rÉ   rã   rÊ   rä   rå   r´   Úoutputsra   Úslice_indicesré   rç   rè   s                    rJ   r{   zMixtralForCausalLM.forwardN  sn  € ðJ %9Ð$DÐ Ð È$Ì+ÔJjð 	ð
 +5¨$¬*ð 	+
ØØ)Ø%Ø+Ø'ØØ!5ð	+
ð 	+
ð ð	+
ð 	+
ˆð  Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô% f¨f°d´oÐPÐPÈÐPÐPˆDàˆØð 	MÝ/ØÔ%ØÔ ØÔ(Øñ	ô ˆHð Ð!Ø˜Ô1°H·K²KÀÄÑ4LÔ4LÑLÑL�å(ØØØØ#Ô3Ø!Ô/ØÔ)Ø!Ô/ð
ñ 
ô 
ð 	
rL   )	NNNNNNNNr   )r|   r}   r~   Ú_tied_weights_keysÚ
_fsdp_planrR   r.   r¸   r€   r   rÚ   rÛ   r·   r   r   r   r{   r�   r‚   s   @rJ   rÝ   rÝ   C  sE  ø€ € € € € Ø*Ð,GÐHÐØÐ/Ð0€Jð>ð >ð >ð >ð >ð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø,0Ø-.ðP
ð P
àÔ# dÑ*ðP
ð œ tÑ+ðP
ð Ô&¨Ñ-ð	P
ð
  ™ðP
ð Ô(¨4Ñ/ðP
ð Ô  4Ñ'ðP
ð ˜$‘;ðP
ð # T™kðP
ð ˜eœlÑ*ðP
ð Ð+Ô,ðP
ð 
#ðP
ð P
ð P
ð P
ð P
ð P
ð P
ð P
rL   rÝ   c                   ó   — e Zd ZdS )Ú MixtralForSequenceClassificationNr    r¡   rL   rJ   rò   rò   ¡  r¢   rL   rò   c                   ó   — e Zd ZdS )ÚMixtralForTokenClassificationNr    r¡   rL   rJ   rô   rô   ¥  r¢   rL   rô   c                   ó   — e Zd ZdS )ÚMixtralForQuestionAnsweringNr    r¡   rL   rJ   rö   rö   ©  r¢   rL   rö   )rÝ   rö   rÇ   rº   rò   rô   )Nr   N)Cr   r.   Útorch.nn.functionalr   r3   r‹   Ú r   rÂ   Úactivationsr   Úcache_utilsr   r   Úintegrationsr	   Úmasking_utilsr
   r   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   Úutils.output_capturingr   Úmistral.modeling_mistralr   r   r   r   r   r   r   r   r   Úconfiguration_mixtralr   Ú
get_loggerr|   Úloggerr€   r-   r·   rK   ÚModulerN   r„   r’   rŸ   r¤   r¦   r¨   rº   rÇ   rÝ   rò   rô   rö   Ú__all__r¡   rL   rJ   ú<module>r	     sŒ  ðð& Ð à €€€Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø RÐ RÐ RÐ RÐ RÐ RÐ RÐ RØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø QÐ QÐ QÐ QÐ QÐ QÐ QÐ QØ -Ð -Ð -Ð -Ð -Ð -Ø &Ð &Ð &Ð &Ð &Ð &Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø 4Ð 4Ð 4Ð 4Ð 4Ð 4ð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 1Ð 0Ð 0Ð 0Ð 0Ð 0ð 
ˆÔ	˜HÑ	%Ô	%€ð
 #Ø
Ø*.ð	B&ð B&Ø”  e¤lÔ 3Ñ3°dÑ:ðB&à�t‘ðB&ð ”L 4Ñ'ð	B&ð
 „\�CÑðB&ð B&ð B&ð B&ðJ ð$#ð $#ð $#ð $#ð $#�R”Yñ $#ô $#ñ Ôð$#ðN<ð <ð <ð <ð <˜œ	ñ <ô <ð <ð$ð ð ð ð ˜BœIñ ô ð ð&	ð 	ð 	ð 	ð 	�^ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð3ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð'ñ 	ô 	ð 	ð#ð #ð #ð #ð #Ð4ñ #ô #ð #ðL;ð ;ð ;ð ;ð ;Ð3ñ ;ô ;ð ;ð$5
ð 5
ð 5
ð 5
ð 5
�<ñ 5
ô 5
ð 5
ðp[
ð [
ð [
ð [
ð [
Ð+ñ [
ô [
ð [
ð|	ð 	ð 	ð 	ð 	Ð'Gñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð$Añ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð"=ñ 	ô 	ð 	ðð ð €€€rL   