§
    9²¸jî‡  ã                   óX  — d dl mZ d dlZd dlmZ d dlmZ ddlmZ	 ddl
mZ ddlmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZ ddlmZmZ ddlmZmZ ddlmZm Z  ddl!m"Z" ddl#m$Z$m%Z%m&Z&m'Z' ddl(m)Z) ddl*m+Z+m,Z, ddl-m.Z.m/Z/ ddl0m1Z1  e'j2        e3¦  «        Z4 ed¦  «         G d„ dej5        ¦  «        ¦   «         Z6 G d„ dej5        ¦  «        Z7 G d„ dej5        ¦  «        Z8 G d„ dej5        ¦  «        Z9 G d „ d!ej5        ¦  «        Z: G d"„ d#ej5        ¦  «        Z;d$„ Z< ed%¦  «        dFd&„¦   «         Z=d'ej>        d(e?d)ej>        fd*„Z@	 dGd,ej5        d-ej>        d.ej>        d/ej>        d0ej>        dz  d1eAd2eAd3e"e$         fd4„ZB G d5„ d6ej5        ¦  «        ZC G d7„ d8e¦  «        ZDe% G d9„ d:e ¦  «        ¦   «         ZEe% G d;„ d<eE¦  «        ¦   «         ZF	 	 	 dHd>ej>        eGej>                 z  dz  d?e?dz  d0ej>        dz  d)ej>        e?z  fd@„ZH G dA„ dBeEe¦  «        ZI G dC„ dDeeE¦  «        ZJg dE¢ZKdS )Ié    )ÚCallableN)Únn)Ú
functionalé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hub)Úcreate_causal_mask)Ú GenericForSequenceClassificationÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚJetMoeConfigÚRMSNormc                   óT   ‡ — e Zd Zd	deddfˆ fd„Zdej        dej        fd„Zd„ Zˆ xZ	S )
ÚJetMoeRMSNormç�íµ ÷Æ°>ÚepsÚreturnNc                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )z<
        JetMoeRMSNorm is equivalent to T5LayerNorm
        N)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizer&   Ú	__class__s      €úz/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/jetmoe/modeling_jetmoe.pyr*   zJetMoeRMSNorm.__init__2   sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐó    Úhidden_statesc                 ó  — |j         }|                     t          j        ¦  «        }|                     d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        |                     |¦  «        z  S )Né   éÿÿÿÿT)Úkeepdim)	ÚdtypeÚtor,   Úfloat32ÚpowÚmeanÚrsqrtr/   r.   )r0   r5   Úinput_dtypeÚvariances       r3   ÚforwardzJetMoeRMSNorm.forward:   s|   € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØ ×$Ò$ QÑ'Ô'×,Ò,¨R¸Ð,Ñ>Ô>ˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆØŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:r4   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)Útupler.   Úshaper/   )r0   s    r3   Ú
extra_reprzJetMoeRMSNorm.extra_reprA   s&   € Ý˜œÔ)Ñ*Ô*ÐIÐI°$Ô2GÐIÐIÐIr4   )r%   )
Ú__name__Ú
__module__Ú__qualname__Úfloatr*   r,   ÚTensorrB   rF   Ú__classcell__©r2   s   @r3   r$   r$   0   sŒ   ø€ € € € € ð$ð $¨ð $¸$ð $ð $ð $ð $ð $ð $ð; U¤\ð ;°e´lð ;ð ;ð ;ð ;ðJð Jð Jð Jð Jð Jð Jr4   r$   c            	       óî   ‡ — e Zd Z edd¬¦  «        d
defˆ fd„¦   «         Ze edd¬¦  «        d
dedeej	        e
f         fd„¦   «         ¦   «         Z ej        ¦   «         ed	„ ¦   «         ¦   «         Zˆ xZS )ÚJetMoeRotaryEmbeddingÚdevicez5.18)ÚversionNÚconfigc                 ó¾  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        | j        j        d         | _        | j        }| j        dk    rt          | j                 } || j        |¦  «        \  }| _
        t          j        |d¬¦  «        | _        t          j        |                     ¦   «         d¬¦  «        | _        d S )NÚ	rope_typeÚdefaultF)Ú
persistent)r)   r*   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrR   Úrope_parametersrT   Úcompute_default_rope_parametersr   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r0   rR   rP   Úrope_init_fnr^   r2   s        €r3   r*   zJetMoeRotaryEmbedding.__init__F   sÀ   ø€ å‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!àˆŒàœÔ4°[ÔAˆŒØ!%Ô!EˆØŒ>˜YÒ&Ð&Ý.¨t¬~Ô>ˆLØ+7¨<¸¼ÀVÑ+LÔ+LÑ(ˆ�$Ô(åœ	 (°uÐ=Ñ=Ô=ˆŒÝ!#¤¨8¯>ª>Ñ+;Ô+;ÈÐ!NÑ!NÔ!NˆÔÐÐr4   r'   c                 óä   — | j         d         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «        |z  z  z  }|                     |¦  «        |fS )a¾  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNg      ð?r   r7   ©r:   )rZ   Úgetattrr1   Únum_attention_headsr,   ÚarangerJ   r;   )rR   rP   ÚkwargsÚbaseÚdimÚattention_factorr^   s          r3   r[   z5JetMoeRotaryEmbedding.compute_default_rope_parametersW   sz   € ð Ô% lÔ3ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐà˜$¥5¤<°°3¸ÅÄÐ#MÑ#MÔ#MÐPSÑ#SÑTÑUˆØ�{Š{˜6Ñ"Ô"Ð$4Ð4Ð4r4   c                 óú  — | j         d d d …d f                              |j        d         dd¦  «                             t          j        |j        ¬¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  ||z                       dd	¦  «        }t	          j        ||fd¬
¦  «        }|                     ¦   «         | j        z  }|                     ¦   «         | j        z  }	d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬¦  «        |	                     |j        ¬¦  «        fS )Nr   r8   r    ©r:   rP   ÚmpsÚcpuF)Údevice_typeÚenabledr7   ©rk   re   )r^   ÚexpandrE   r;   r,   rJ   rP   Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosr\   Úsinr:   )
r0   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedrq   ÚfreqsÚembrz   r{   s
             r3   rB   zJetMoeRotaryEmbedding.forwardk   s›  € ð ŒM˜$    4˜-Ô(×/Ò/°Ô0BÀ1Ô0EÀrÈ1ÑMÔM×PÒPÕW\ÔWbÐklÔksÐPÑtÔtð 	ð !-¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆå¨¸UÐCÑCÔCð 	5ð 	5Ø&Ð)>Ñ>×IÒIÈ!ÈQÑOÔOˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCð		5ð 	5ð 	5ñ 	5ô 	5ð 	5ð 	5ð 	5ð 	5ð 	5ð 	5øøøð 	5ð 	5ð 	5ð 	5ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   ÃA*D<Ä<E ÅE ©N)rG   rH   rI   r   r!   r*   ÚstaticmethodrD   r,   rK   rJ   r[   Úno_gradr   rB   rL   rM   s   @r3   rO   rO   E   sö   ø€ € € € € Ø€_�X vÐ.Ñ.Ô.ðOð O˜|ð Oð Oð Oð Oð Oñ /Ô.ðOð  Ø€_�X vÐ.Ñ.Ô.ð5ð 5°ð 5ÐX]Ð^cÔ^jÐlqÐ^qÔXrð 5ð 5ð 5ñ /Ô.ñ „\ð5ð$ €U„]�_„_Øð<ð <ñ Ôñ „_ð<ð <ð <ð <ð <r4   rO   c                   ó6   ‡ — e Zd Zdedededdfˆ fd„Zd„ Zˆ xZS )ÚJetMoeParallelExpertsÚnum_expertsÚ
input_sizeÚoutput_sizer'   Nc                 óÌ   •— t          ¦   «                              ¦   «          t          j        t	          j        |||¦  «        ¦  «        | _        || _        || _        || _	        dS )aÃ  
        Initialize the JetMoeParallelExperts module.
        The experts weights are stored in [num_experts, output_size, input_size] format. Such that it's compatible with
        many MoE libraries, such as [Megablock](https://github.com/databricks/megablocks) and
        [ScatterMoE](https://github.com/shawntan/scattermoe), as well as the
        [MoE kernel](https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/fused_moe/fused_moe.py)
        used in vllm.

        Args:
            num_experts (int):
                Number of experts.
            input_size (int):
                Size of the input.
            output_size (int):
                Size of the output.
        N)
r)   r*   r   r+   r,   Úemptyr.   r‡   rˆ   r‰   )r0   r‡   rˆ   r‰   r2   s       €r3   r*   zJetMoeParallelExperts.__init__   sW   ø€ õ" 	‰Œ×ÒÑÔÐÝ”l¥5¤;¨{¸KÈÑ#TÔ#TÑUÔUˆŒØ&ˆÔØ$ˆŒØ&ˆÔÐÐr4   c                 ó  — |                      |d¬¦  «        }g }t          | j        ¦  «        D ];}|                     t	          j        ||         | j        |         ¦  «        ¦  «         Œ<t          j        |d¬¦  «        }|S )a  
        Forward pass of the JetMoeParallelExperts module.

        Args:
            inputs (Tensor):
                Input tensor.
            expert_size:
                Expert size information.

        Returns:
            Tensor: Output tensor.
        r   rs   )	ÚsplitÚranger‡   ÚappendÚFÚlinearr.   r,   ry   )r0   ÚinputsÚexpert_sizeÚ
input_listÚoutput_listÚiÚresultss          r3   rB   zJetMoeParallelExperts.forward–   s€   € ð —\’\ +°1�\Ñ5Ô5ˆ
ØˆÝ�tÔ'Ñ(Ô(ð 	Hð 	HˆAØ×Ò�qœx¨
°1¬°t´{À1´~ÑFÔFÑGÔGÐGÐGÝ”)˜K¨QÐ/Ñ/Ô/ˆØˆr4   ©rG   rH   rI   Úintr*   rB   rL   rM   s   @r3   r†   r†   ~   sh   ø€ € € € € ð' Cð '°Sð 'Àsð 'Ètð 'ð 'ð 'ð 'ð 'ð 'ð.ð ð ð ð ð ð r4   r†   c                   ó2   ‡ — e Zd Zdededefˆ fd„Zd„ Zˆ xZS )ÚJetMoeTopKGatingrˆ   r‡   Útop_kc                 óª   •— t          ¦   «                              ¦   «          || _        || _        || _        t          j        ||d¬¦  «        | _        dS )a  
        Initialize the top-k gating mechanism.

        Args:
            input_size (`int`):
                Size of the input.
            num_experts (`int`):
                Number of experts.
            top_k (`int`):
                Number of top experts to select.
        F©ÚbiasN)r)   r*   r‡   rˆ   rœ   r   ÚLinearÚlayer)r0   rˆ   r‡   rœ   r2   s       €r3   r*   zJetMoeTopKGating.__init__¬   sM   ø€ õ 	‰Œ×ÒÑÔÐà&ˆÔØ$ˆŒØˆŒ
å”Y˜z¨;¸UÐCÑCÔCˆŒ
ˆ
ˆ
r4   c                 óÐ  — |                       |¦  «                             ¦   «         }|                     | j        d¬¦  «        \  }}t	          j        |d¬¦  «                             |¦  «        }t	          j        |                     d¦  «        | j	        g|j
        |j        ¬¦  «        }|                     d|d¦  «        }|                     ¦   «                              d¦  «        }|                     ¦   «         }|                     ¦   «         }	|	                     d¦  «        \  }
}|                     | j        d¬¦  «        }|                     ¦   «         }||         }|||||fS )Nr    rs   r   rn   Útrunc)Úrounding_mode)r¡   rJ   Útopkrœ   r,   ÚsoftmaxÚtype_asÚzerosÚsizer‡   r:   rP   ÚscatterÚlongÚsumÚtolistÚflattenÚsortÚdiv)r0   r5   ÚlogitsÚtop_k_logitsÚtop_k_indicesÚtop_k_gatesr¨   Úgatesr“   Útop_k_expertsÚ_Úindex_sorted_expertsÚbatch_indexÚbatch_gatess                 r3   rB   zJetMoeTopKGating.forwardÀ   sS  € à—’˜MÑ*Ô*×0Ò0Ñ2Ô2ˆØ&,§k¢k°$´*À! kÑ&DÔ&DÑ#ˆ�mÝ”m L°aÐ8Ñ8Ô8×@Ò@ÀÑOÔOˆõ ”Ø×Ò˜aÑ Ô  $Ô"2Ð3¸;Ô;LÐU`ÔUgð
ñ 
ô 
ˆð —’˜a °Ñ2Ô2ˆØ—j’j‘l”l×&Ò& qÑ)Ô)ˆð "×(Ò(Ñ*Ô*ˆð &×-Ò-Ñ/Ô/ˆØ"/×"4Ò"4°QÑ"7Ô"7ÑˆÐØ*×.Ò.¨t¬zÈÐ.ÑQÔQˆð "×)Ò)Ñ+Ô+ˆØ!Ð"6Ô7ˆà# [°+¸{ÈFÐRÐRr4   r˜   rM   s   @r3   r›   r›   «   sq   ø€ € € € € ðD 3ð D°Sð DÀð Dð Dð Dð Dð Dð Dð(Sð Sð Sð Sð Sð Sð Sr4   r›   c                   ó.   ‡ — e Zd ZdZdefˆ fd„Zd„ Zˆ xZS )Ú	JetMoeMoEzº
    A Sparsely gated mixture of experts layer with 1-layer Feed-Forward networks as experts.

    Args:
        config:
            Configuration object with model hyperparameters.
    rR   c                 ó  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        t
          |j                 | _        t          j	         
                    t          j        | j        ¦  «        ¦  «        | _        t          |j        | j        | j        dz  ¦  «        | _        t          |j        | j        | j        ¦  «        | _        t#          | j        |j        |j        ¬¦  «        | _        d S )Nr7   ©rˆ   r‡   rœ   )r)   r*   r1   rˆ   Úintermediate_sizer   Úactivation_functionÚ
activationr,   r   r+   r‹   rŸ   r†   Únum_local_expertsÚinput_linearÚoutput_linearr›   Únum_experts_per_tokÚrouter©r0   rR   r2   s     €r3   r*   zJetMoeMoE.__init__å   sÌ   ø€ Ý‰Œ×ÒÑÔÐà Ô,ˆŒØ!Ô3ˆÔÝ  Ô!;Ô<ˆŒÝ”H×&Ò&¥u¤{°4´?Ñ'CÔ'CÑDÔDˆŒ	Ý1°&Ô2JÈDÌOÐ]aÔ]mÐpqÑ]qÑrÔrˆÔÝ2°6Ô3KÈTÔM]Ð_cÔ_nÑoÔoˆÔå&Ø”ØÔ0ØÔ,ð
ñ 
ô 
ˆŒˆˆr4   c                 ód  — |                      ¦   «         \  }}}|                     d|¦  «        }|                      |¦  «        \  }}}}}	||         }
|                      |
|¦  «        }|                     dd¬¦  «        }|                      |d         ¦  «        |d         z  }|                      ||¦  «        }||dd…df         z  }t          j        ||z  | j	        f|j
        |j        ¬¦  «        }|                     d||¦  «        }|                     ||| j	        ¦  «        }|| j        z   }|S )a  
        Forward pass of the mixture of experts layer.

        Args:
            layer_input (Tensor):
                Input tensor.

        Returns:
            Tensor:
                Output tensor.
            Tensor:
                Router logits.
        r8   r7   rs   r   r    Nrn   )r©   ÚreshaperÆ   rÃ   ÚchunkrÁ   rÄ   r,   r¨   rˆ   r:   rP   Ú	index_addÚviewrŸ   )r0   Úlayer_inputÚbszÚlengthÚemb_sizer·   r¹   rº   r“   Úrouter_logitsÚexpert_inputsr5   Úchunked_hidden_statesÚexpert_outputsr¨   Úlayer_outputs                   r3   rB   zJetMoeMoE.forwardõ   s<  € ð !,× 0Ò 0Ñ 2Ô 2ÑˆˆV�XØ!×)Ò)¨"¨hÑ7Ô7ˆØBFÇ+Â+ÈkÑBZÔBZÑ?ˆˆ;˜ [°-à# KÔ0ˆØ×)Ò)¨-¸ÑEÔEˆØ -× 3Ò 3°A¸2Ð 3Ñ >Ô >ÐØŸšÐ(=¸aÔ(@ÑAÔAÐDYÐZ[ÔD\Ñ\ˆØ×+Ò+¨M¸;ÑGÔGˆà'¨+°a°a°a¸°gÔ*>Ñ>ˆå”˜S 6™\¨4¬?Ð;À>ÔCWÐ`nÔ`uÐvÑvÔvˆØ—’ q¨+°~ÑFÔFˆØ#×(Ò(¨¨f°d´oÑFÔFˆØ# d¤iÑ/ˆØÐr4   )rG   rH   rI   Ú__doc__r!   r*   rB   rL   rM   s   @r3   r¼   r¼   Ü   s]   ø€ € € € € ðð ð
˜|ð 
ð 
ð 
ð 
ð 
ð 
ð ð ð ð ð ð ð r4   r¼   c                   ó:   ‡ — e Zd ZdZdefˆ fd„Zd„ Zd„ Zd„ Zˆ xZ	S )Ú	JetMoeMoAzÅ
    A Sparsely gated mixture of attention layer with pairs of query- and output-projections as experts.

    Args:
        config:
            Configuration object with model hyperparameters.
    rR   c                 ó  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        |j        z  | _        |j        | _	        t          j                             t          j        | j        ¦  «        ¦  «        | _        t          | j        | j        | j        ¦  «        | _        t          | j        | j        | j        ¦  «        | _        t%          | j        | j        | j	        ¬¦  «        | _        d S )Nr¾   )r)   r*   rÂ   r‡   r1   rˆ   Úkv_channelsÚnum_key_value_headsrÅ   rœ   r,   r   r+   r‹   rŸ   r†   rÃ   rÄ   r›   rÆ   rÇ   s     €r3   r*   zJetMoeMoA.__init__  sÓ   ø€ Ý‰Œ×ÒÑÔÐà!Ô3ˆÔØ Ô,ˆŒØ!Ô-°Ô0JÑJˆÔØÔ/ˆŒ
Ý”H×&Ò&¥u¤{°4´?Ñ'CÔ'CÑDÔDˆŒ	å1°$Ô2BÀDÄOÐUYÔUeÑfÔfˆÔÝ2°4Ô3CÀTÔEUÐW[ÔWfÑgÔgˆÔå&Ø”ØÔ(Ø”*ð
ñ 
ô 
ˆŒˆˆr4   c                 ó´  — |                      ¦   «         \  }}}|                     d|¦  «        }|                      |¦  «        \  }}}}}	||||f}
||         }|                      ||¦  «        }t	          j        ||z  | j        z  | j        f|j        |j	        ¬¦  «        }| 
                    d||¦  «        }|                     ||| j        d¦  «        }||	|
fS )z�
        Map inputs to attention experts according to routing decision and compute query projection inside each experts.
        r8   rn   r   )r©   rÉ   rÆ   rÃ   r,   r¨   rœ   r1   r:   rP   rË   rÌ   )r0   rÍ   rÎ   rÏ   rÐ   r¸   r¹   rº   r“   rÑ   Ú	topo_inforÒ   rÔ   r¨   rÕ   s                  r3   ÚmapzJetMoeMoA.map1  sù   € ð !,× 0Ò 0Ñ 2Ô 2ÑˆˆV�XØ!×)Ò)¨"¨hÑ7Ô7ˆØUY×U`ÒU`ÐalÑUmÔUmÑRÐ˜k¨;¸À]Ø)¨;¸À[ÐQˆ	ð $ KÔ0ˆØ×*Ò*¨=¸+ÑFÔFˆõ ”Ø�6‰\˜DœJÑ&¨Ô(8Ð9ÀÔAUÐ^lÔ^sð
ñ 
ô 
ˆð —’ qÐ*>ÀÑOÔOˆØ#×(Ò(¨¨f°d´jÀ"ÑEÔEˆØ˜]¨IÐ5Ð5r4   c                 óœ  — |                      ¦   «         \  }}}}|                     d|¦  «        }|\  }}}	}
||         }|                      ||
¦  «        }||	dd…df         z  }t          j        ||z  | j        f|j        |j        ¬¦  «        }|                     d||¦  «        }| 	                    ||| j        ¦  «        }|| j
        z   }|S )zu
        Compute output projection inside each attention experts and merge the outputs of different experts.
        r8   Nrn   r   )r©   rÉ   rÄ   r,   r¨   rˆ   r:   rP   rË   rÌ   rŸ   )r0   rÍ   rÝ   rÎ   rÏ   Úkr1   r¸   r¹   rº   r“   rÒ   rÔ   r¨   rÕ   s                  r3   ÚreducezJetMoeMoA.reduceH  sç   € ð '2×&6Ò&6Ñ&8Ô&8Ñ#ˆˆV�Q˜Ø!×)Ò)¨"¨kÑ:Ô:ˆØFOÑCÐ˜k¨;¸ð $Ð$8Ô9ˆØ×+Ò+¨M¸;ÑGÔGˆð (¨+°a°a°a¸°gÔ*>Ñ>ˆõ ”˜S 6™\¨4¬?Ð;À>ÔCWÐ`nÔ`uÐvÑvÔvˆØ—’ q¨+°~ÑFÔFˆØ#×(Ò(¨¨f°d´oÑFÔFˆØ# d¤iÑ/ˆØÐr4   c                 ó    — t          d¦  «        ‚)Nz-This module doesn't support call and forward.)ÚNotImplementedError)r0   rÍ   s     r3   rB   zJetMoeMoA.forward^  s   € Ý!Ð"QÑRÔRÐRr4   )
rG   rH   rI   rÖ   r!   r*   rÞ   rá   rB   rL   rM   s   @r3   rØ   rØ     s‚   ø€ € € € € ðð ð
˜|ð 
ð 
ð 
ð 
ð 
ð 
ð$6ð 6ð 6ð.ð ð ð,Sð Sð Sð Sð Sð Sð Sr4   rØ   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..Nr8   r7   rs   )rE   r,   ry   )r|   Úx1Úx2s      r3   Úrotate_halfrç   b  s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'r4   Úrotary_pos_embc                 ó¾   — |                      |¦  «        }|                      |¦  «        }| |z  t          | ¦  «        |z  z   }||z  t          |¦  «        |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezerç   )Úqrà   rz   r{   Úunsqueeze_dimÚq_embedÚk_embeds          r3   Úapply_rotary_pos_embrï   i  sc   € ð& �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�GÐÐr4   r5   Ún_repr'   c                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r    N)rE   rt   rÉ   )r5   rð   ÚbatchrÛ   Úslenrd   s         r3   Ú	repeat_kvrô   ƒ  s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTr4   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutri   c                 ó  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        |z  }
|�|
|z   }
t
          j                             |
dt          j        ¬¦  «         	                    |j
        ¦  «        }
t
          j                             |
|| j        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «                             ¦   «         }||
fS )Nr7   r   r8   )rk   r:   )ÚpÚtrainingr    )rô   Únum_key_value_groupsr,   Úmatmulrx   r   r   r¦   r<   r;   r:   rü   rÿ   Ú
contiguous)rö   r÷   rø   rù   rú   rû   rü   ri   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r3   Úeager_attention_forwardr  �  sé   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r4   c                   óÐ   ‡ — e Zd ZdZddededz  fˆ fd„Z	 	 	 ddej        dej        dz  dej	        dz  d	e
dz  d
eej        ej        dz  eej                 dz  f         f
d„Zˆ xZS )ÚJetMoeAttentionzH
    Multi-headed attention from 'Attention Is All You Need' paper.
    NrR   Ú	layer_idxc                 ó"  •— t          ¦   «                              ¦   «          || _        || _        d| _        |€(t
                               d| j        j        › d�¦  «         d| _	        |j
        | _        |j        | _        |j        |j        z  | _        |j        | _        |j        | _        |j        | _        | j        dz  | _        t)          |¦  «        | _        t,          j                             |j        | j        dz  d¬	¦  «        | _        dS )
zä
        Initialize the JetMoeAttention module.

        Args:
            config:
                Configuration object with model hyperparameters.
            layer_idx:
                Index of the layer in the model.
        TNzInstantiating z¹ without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.r    g      à¿r7   Frž   )r)   r*   rR   r
  Ú	is_causalÚloggerÚwarning_oncer2   rG   r   rÅ   rœ   Úattention_dropoutrÚ   rÛ   Úkv_projection_sizerg   Ú	num_headsrd   rû   rØ   Úexpertsr,   r   r    r1   Úkv_proj©r0   rR   r
  r2   s      €r3   r*   zJetMoeAttention.__init__­  s  ø€ õ 	‰Œ×ÒÑÔÐØˆŒØ"ˆŒØˆŒØÐÝ×Òð, ¤Ô!8ð ,ð ,ð ,ñô ð ð %&ˆÔ!ØÔ/ˆŒ
Ø!'Ô!9ˆÔØ"(Ô"4°vÔ7QÑ"QˆÔØ#)Ô#=ˆÔ ØÔ3ˆŒØÔ*ˆŒØ”} dÑ*ˆŒÝ  Ñ(Ô(ˆŒå”x—’ vÔ'9¸4Ô;RÐUVÑ;VÐ]b�ÑcÔcˆŒˆˆr4   r5   rú   Úposition_embeddingsÚpast_key_valuesr'   c                 óÐ  — |j         d d…         }g |¢d‘| j        ‘R }| j                             |¦  «        \  }}	}
|                      |¦  «                             dd¬¦  «        \  }}|                     |¦  «                             dd¦  «        }|                     |¦  «                             dd¦  «        }|                     |¦  «                             dd¦  «        }|\  }}t          ||||¦  «        \  }}|�| 	                    ||| j
        ¦  «        \  }}t          j        | j        j        t          ¦  «        }|                     d| j        dd¦  «        }|                     d| j        dd¦  «        } || ||||f| j        sdn| j        | j        dœ|¤Ž\  }} |j        g |¢| j        ‘d‘R Ž }| j                             ||
¦  «        } |j        g |¢d‘R Ž }|||	fS )Nr8   r7   rs   r    rõ   )rü   rû   )rE   rd   r  rÞ   r  rÊ   rÌ   rx   rï   Úupdater
  r   Úget_interfacerR   Ú_attn_implementationr  Úrepeatrœ   rÿ   r  rû   rá   )r0   r5   rú   r  r  ri   Úinput_shapeÚhidden_shapeÚquery_statesrÑ   rÝ   r  r  rz   r{   Úattention_interfacer  r  s                     r3   rB   zJetMoeAttention.forwardÎ  s2  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà15´×1AÒ1AÀ-Ñ1PÔ1PÑ.ˆ�m YØ#'§<¢<°Ñ#>Ô#>×#DÒ#DÀQÈBÐ#DÑ#OÔ#OÑ ˆ
�Là#×(Ò(¨Ñ6Ô6×@Ò@ÀÀAÑFÔFˆØ—_’_ \Ñ2Ô2×<Ò<¸QÀÑBÔBˆ
Ø#×(Ò(¨Ñ6Ô6×@Ò@ÀÀAÑFÔFˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð  ×&Ò& q¨$¬*°a¸Ñ;Ô;ˆ
Ø#×*Ò*¨1¨d¬j¸!¸QÑ?Ô?ˆà$7Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð '�kÔ&ÐD¨ÐD°T´ZÐDÀÐDÐDÐDˆØ”l×)Ò)¨+°yÑAÔAˆØ&�kÔ&Ð8¨Ð8°RÐ8Ð8Ð8ˆØ˜L¨-Ð7Ð7r4   r‚   )NNN)rG   rH   rI   rÖ   r!   r™   r*   r,   rK   Ú
LongTensorr	   rD   rB   rL   rM   s   @r3   r	  r	  ¨  sò   ø€ € € € € ðð ðdð d˜|ð d¸¸d¹
ð dð dð dð dð dð dðH /3Ø7;Ø(,ð/8ð /8à”|ð/8ð œ tÑ+ð/8ð #Ô-°Ñ4ð	/8ð
  ™ð/8ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	Mð/8ð /8ð /8ð /8ð /8ð /8ð /8ð /8r4   r	  c                   óÚ   ‡ — e Zd Zddededz  fˆ fd„Z	 	 	 	 	 ddej        dej        dz  dej        dz  d	e	dz  d
e
dz  deej        ej        f         dz  dee         dej        fd„Zˆ xZS )ÚJetMoeDecoderLayerNrR   r
  c                 ó  •— t          ¦   «                              ¦   «          |j        | _        t          |¦  «        | _        t          |j        ¦  «        | _        t          |j        ¦  «        | _        t          ||¦  «        | _	        d S r‚   )
r)   r*   r1   r¼   Úmlpr$   Úinput_layernormÚpost_attention_layernormr	  Úself_attentionr  s      €r3   r*   zJetMoeDecoderLayer.__init__  sq   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔÝ˜VÑ$Ô$ˆŒÝ,¨VÔ-?Ñ@Ô@ˆÔÝ(5°fÔ6HÑ(IÔ(IˆÔ%Ý-¨f°iÑ@Ô@ˆÔÐÐr4   Fr5   rú   r}   r  Ú	use_cacher  ri   r'   c           
      óÐ   — |}|                       |¦  «        } | j        d||||||dœ|¤Ž\  }}	}	||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|S )N)r5   rú   r}   r  r(  r  © )r%  r'  r&  r$  )
r0   r5   rú   r}   r  r(  r  ri   Úresidualr·   s
             r3   rB   zJetMoeDecoderLayer.forward	  s¤   € ð !ˆØ×,Ò,¨]Ñ;Ô;ˆà1˜dÔ1ð 
Ø'Ø)Ø%Ø+ØØ 3ð
ð 
ð ð
ð 
Ñˆ�q˜!ð ! =Ñ0ˆð !ˆØ×5Ò5°mÑDÔDˆØŸš Ñ/Ô/ˆØ  =Ñ0ˆØÐr4   r‚   )NNNFN)rG   rH   rI   r!   r™   r*   r,   rK   r   r	   ÚboolrD   r   r   rB   rL   rM   s   @r3   r"  r"     s	  ø€ € € € € ðAð A˜|ð A¸¸d¹
ð Að Að Að Að Að Að /3Ø04Ø(,Ø!&ØHLðð à”|ðð œ tÑ+ðð Ô&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð # 5¤<°´Ð#=Ô>ÀÑEðð Ð+Ô,ðð 
Œðð ð ð ð ð ð ð r4   r"  c                   óÎ   ‡ — e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZdZ eed¬¦  «         eed	¬¦  «        ge eed
¬¦  «        dœZ ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )ÚJetMoePreTrainedModelrR   ÚmodelFr"  r  Tr7   )Úindexé   r    )rÑ   r5   Ú
attentionsc                 ó4  •— t          ¦   «                              |¦  «         t          |t          ¦  «        r(t	          j        |j        d| j        j        ¬¦  «         dS t          |t          t          z  ¦  «        rt	          j        |j        ¦  «         dS dS )zInitialize the weights.rõ   )r>   ÚstdN)r)   Ú_init_weightsru   r†   ÚinitÚnormal_r.   rR   Úinitializer_rangerØ   r¼   Úzeros_rŸ   )r0   rö   r2   s     €r3   r5  z#JetMoePreTrainedModel._init_weights;  s�   ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ3Ñ4Ô4ð 	%ÝŒL˜œ¨S°d´kÔ6SÐTÑTÔTÐTÐTÐTÝ˜¥	­IÑ 5Ñ6Ô6ð 	%ÝŒK˜œÑ$Ô$Ð$Ð$Ð$ð	%ð 	%r4   )rG   rH   rI   r!   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr   r	  r›   r"  Ú_can_record_outputsr,   r„   r5  rL   rM   s   @r3   r.  r.  )  sß   ø€ € € € € € àÐÐÑØÐØ&+Ð#Ø-Ð.ÐØ#4Ð"5ÐØÐØ€NØÐØ"ÐØ"&Ðà(˜.¨ÀÐBÑBÔBÀNÀNÐScÐklÐDmÑDmÔDmÐnØ+Ø$�n _¸AÐ>Ñ>Ô>ðð Ðð €U„]�_„_ð%ð %ð %ð %ñ „_ð%ð %ð %ð %ð %r4   r.  c                   óâ   ‡ — e Zd Zdefˆ fd„Zeee	 	 	 	 	 	 ddej	        dz  dej
        dz  dej	        dz  dedz  dej        dz  d	edz  d
ee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚJetMoeModelrR   c                 óö  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          j        ‰j        ‰j        | j        ¦  «        | _        t          j	        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        t!          ‰¬¦  «        | _        d| _        ‰j        | _        |                      ¦   «          d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r*  )r"  )Ú.0r
  rR   s     €r3   ú
<listcomp>z(JetMoeModel.__init__.<locals>.<listcomp>N  s$   ø€ ÐdÐdÐd°yÕ ¨	Ñ2Ô2ÐdÐdÐdr4   )r&   ©rR   F)r)   r*   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr1   Úembed_tokensÚ
ModuleListrŽ   Únum_hidden_layersÚlayersr$   Úrms_norm_epsÚnormrO   Ú
rotary_embÚgradient_checkpointingr  Ú	post_initrÇ   s    `€r3   r*   zJetMoeModel.__init__G  sß   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒåœL¨Ô):¸FÔ<NÐPTÔP`ÑaÔaˆÔÝ”mØdÐdÐdÐdÅEÈ&ÔJbÑDcÔDcÐdÑdÔdñ
ô 
ˆŒõ " &Ô"4¸&Ô:MÐNÑNÔNˆŒ	Ý/°vÐ>Ñ>Ô>ˆŒØ&+ˆÔ#Ø$*Ô$?ˆÔ!ð 	�ŠÑÔÐÐÐr4   NÚ	input_idsrú   r}   r  Úinputs_embedsr(  ri   r'   c           
      óF  — |d u |d uz  rt          d¦  «        ‚|r|€t          | j        ¬¦  «        }|€|                      |¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   }| 	                    d¦  «        }t          | j        ||||¬¦  «        }	|}
|                      |
|¦  «        }| j        d | j        j        …         D ]} ||
f||	|||dœ|¤Ž}
Œ|                      |
¦  «        }
t          |
|¬¦  «        S )	Nz:You must specify exactly one of input_ids or inputs_embedsrK  r   r    )rP   )rR   rZ  rú   r  r}   )r  rú   r  r(  r}   )Úlast_hidden_stater  )Ú
ValueErrorr
   rR   rP  Úget_seq_lengthr,   rh   rE   rP   rê   r   rV  rS  rR  rU  r   )r0   rY  rú   r}   r  rZ  r(  ri   Úpast_seen_tokensÚcausal_maskr5   r  Údecoder_layers                r3   rB   zJetMoeModel.forwardX  sŠ  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå(Ø”;Ø'Ø)Ø+Ø%ð
ñ 
ô 
ˆð &ˆð #Ÿošo¨m¸\ÑJÔJÐà!œ[Ð)H¨4¬;Ô+HÐ)HÔIð 		ð 		ˆMØ)˜MØðà$7Ø*Ø /Ø#Ø)ðð ð ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆå%Ø+Ø+ð
ñ 
ô 
ð 	
r4   )NNNNNN)rG   rH   rI   r!   r*   r   r   r   r,   r   rK   r	   ÚFloatTensorr,  r   r   r   rB   rL   rM   s   @r3   rF  rF  E  s  ø€ € € € € ð˜|ð ð ð ð ð ð ð"  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ð5
ð 5
àÔ# dÑ*ð5
ð œ tÑ+ð5
ð Ô&¨Ñ-ð	5
ð
  ™ð5
ð Ô(¨4Ñ/ð5
ð ˜$‘;ð5
ð Ð+Ô,ð5
ð 
 ð5
ð 5
ð 5
ñ „^ñ „_ñ  Ôð5
ð 5
ð 5
ð 5
ð 5
r4   rF  r7   Úgate_logitsr‡   c           	      óÎ  — | �t          | t          ¦  «        sdS | d         j        }t          j        |t          j        |¬¦  «        }t          j        |t          j        |¬¦  «        }d}|�4|                     d¦  «                             |t          j        ¬¦  «        }| D �]‰}	t          j        j	         
                    |	                     |¦  «        d¬¦  «        }
t          j        |
|d¬¦  «        \  }}|€z|t          j        |                     d¦  «        |¬¦  «                             ¦   «         z   }||
                     ¦   «                              d¬¦  «        z   }||
j        d         z   }ŒÒ|t          j        |t          j        |¬¦  «                             d|                     d¦  «        |                     |¦  «        ¦  «        z   }||
                     ¦   «         |                     d¦  «        z                       d¬¦  «        z   }||                     ¦   «         z   }�Œ‹||z  }||z  }t          j        ||                     d¦  «        z  ¦  «        }||z  S )	aÄ  
    Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

    See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
    function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
    experts is too unbalanced.

    Args:
        gate_logits:
            Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
            shape [batch_size X sequence_length, num_experts].
        num_experts:
            Number of experts
        top_k:
            The number of experts to route per-token, can be also interpreted as the `top-k` routing
            parameter.
        attention_mask (`torch.Tensor`, *optional*):
            The attention_mask used in forward function
            shape [batch_size X sequence_length] if not None.

    Returns:
        The auxiliary loss.
    Nr   rn   rõ   r8   )rP   r:   rs   )Ú	minlength)ru   rD   rP   r,   r¨   r<   rÉ   r;   r   r   r¦   r¥   ÚbincountrJ   r¬   rE   Úscatter_add_Úrepeat_interleaverê   )rc  r‡   rœ   rú   Úcompute_deviceÚtokens_per_expert_sumÚrouter_prob_sumÚ
total_rowsÚ	flat_maskÚ
layer_gateÚrouting_weightsr·   Úselected_expertsÚtokens_per_expertÚrouter_prob_per_expertÚoverall_losss                   r3   Úload_balancing_loss_funcrt  “  sV  € ð: Ð¥*¨[½%Ñ"@Ô"@ÐØˆqð ! ”^Ô*€NÝ!œK¨½5¼=ÐQ_Ð`Ñ`Ô`ÐÝ”k +µU´]È>ÐZÑZÔZ€OØ€JàÐ!à"×*Ò*¨2Ñ.Ô.×1Ò1¸ÍuÌ}Ð1Ñ]Ô]ˆ	à!ð 6ñ 6ˆ
Ýœ(Ô-×5Ò5°j·m²mÀNÑ6SÔ6SÐY[Ð5Ñ\Ô\ˆÝ#œj¨¸%ÀRÐHÑHÔHÑˆÐØÐ!ð &­¬Ð7G×7OÒ7OÐPRÑ7SÔ7SÐ_jÐ(kÑ(kÔ(k×(qÒ(qÑ(sÔ(sÑsð "ð .°×0EÒ0EÑ0GÔ0G×0KÒ0KÐPQÐ0KÑ0RÔ0RÑRˆOØ# oÔ&;¸AÔ&>Ñ>ˆJˆJð %:½E¼KØ¥5¤=¸ð=ñ =ô =çŠl˜1Ð.×6Ò6°rÑ:Ô:¸I×<WÒ<WÐX]Ñ<^Ô<^Ñ_Ô_ñ%`Ð!ð .°×1FÒ1FÑ1HÔ1HÈ9×K^ÒK^Ð_aÑKbÔKbÑ1b×0gÒ0gÐlmÐ0gÑ0nÔ0nÑnˆOØ# i§m¢m¡o¤oÑ5ˆJ‰Jà-°
Ñ:ÐØ,¨zÑ9Ðå”9Ð.Ð1G×1QÒ1QÐRSÑ1TÔ1TÑTÑUÔU€LØ˜+Ñ%Ð%r4   c                   óü   ‡ — e Zd ZddiZˆ fd„Zee	 	 	 	 	 	 	 	 	 ddej        dz  dej	        dz  d	ej        dz  d
e
dz  dej        dz  dej        dz  dedz  deej	        z  dedz  defd„¦   «         ¦   «         Zˆ xZS )ÚJetMoeForCausalLMzlm_head.weightzmodel.embed_tokens.weightc                 ó^  •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        | _        |j        | _        t          j        |j        |j        d¬¦  «        | _	        |j
        | _
        |j        | _        |j        | _        |                      ¦   «          d S )NFrž   )r)   r*   rF  r/  rN  Úaux_loss_coefr   r    r1   Úlm_headÚtie_word_embeddingsrÂ   r‡   rÅ   rX  rÇ   s     €r3   r*   zJetMoeForCausalLM.__init__Û  s–   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý  Ñ(Ô(ˆŒ
Ø Ô+ˆŒØ#Ô1ˆÔÝ”y Ô!3°VÔ5FÈUÐSÑSÔSˆŒØ#)Ô#=ˆÔ Ø!Ô3ˆÔØ#)Ô#=ˆÔ ð 	�ŠÑÔÐÐÐr4   Nr   FrY  rú   r}   r  rZ  Úlabelsr(  Úlogits_to_keepÚoutput_router_logitsr'   c
                 óø  —  | j         d|||||||	dœ|
¤Ž}|j        }t          |t          ¦  «        rt	          | d ¦  «        n|}|                      |d d …|d d …f         ¦  «        }d }|� | j        ||fd| j        j        i|
¤Ž}d }|	rHt          |j
        | j        | j        |¦  «        }|�%|| j        |                     |j        ¦  «        z  z  }t!          ||||j        |j        |j        |j
        ¬¦  «        S )N)rY  rú   r}   r  rZ  r(  r}  rN  )ÚlossÚaux_lossr±   r  r5   r2  rÑ   r*  )r/  r\  ru   r™   Úslicery  Úloss_functionrR   rN  rt  rÑ   r‡   rÅ   rx  r;   rP   r   r  r5   r2  )r0   rY  rú   r}   r  rZ  r{  r(  r|  r}  ri   Úoutputsr5   Úslice_indicesr±   r  r€  s                    r3   rB   zJetMoeForCausalLM.forwardè  sm  € ð +5¨$¬*ð 	+
ØØ)Ø%Ø+Ø'ØØ!5ð	+
ð 	+
ð ð	+
ð 	+
ˆð  Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%ØØðð ð  œ;Ô1ðð ð	ð ˆDð ˆØð 	FÝ/ØÔ%ØÔ ØÔ(Øñ	ô ˆHð Ð!Ø˜Ô*¨X¯[ª[¸¼Ñ-EÔ-EÑEÑE�å(ØØØØ#Ô3Ø!Ô/ØÔ)Ø!Ô/ð
ñ 
ô 
ð 	
r4   )	NNNNNNNr   F)rG   rH   rI   Ú_tied_weights_keysr*   r   r   r,   r   rK   r	   rb  r,  r™   r   rB   rL   rM   s   @r3   rv  rv  Ø  s-  ø€ € € € € Ø*Ð,GÐHÐðð ð ð ð ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.Ø,1ð9
ð 9
àÔ# dÑ*ð9
ð œ tÑ+ð9
ð Ô&¨Ñ-ð	9
ð
  ™ð9
ð Ô(¨4Ñ/ð9
ð Ô  4Ñ'ð9
ð ˜$‘;ð9
ð ˜eœlÑ*ð9
ð # T™kð9
ð 
#ð9
ð 9
ð 9
ñ „^ñ Ôð9
ð 9
ð 9
ð 9
ð 9
r4   rv  c                   ó   — e Zd ZdS )ÚJetMoeForSequenceClassificationN)rG   rH   rI   r*  r4   r3   r‡  r‡  &  s   € € € € € € € r4   r‡  )rv  rF  r.  r‡  )r    )rõ   )Nr7   N)LÚcollections.abcr   r,   r   Útorch.nnr   r�   Ú r   r6  Úactivationsr   Úcache_utilsr	   r
   Ú
generationr   Úintegrationsr   Úmasking_utilsr   Úmodeling_layersr   r   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   r   Úconfiguration_jetmoer!   Ú
get_loggerrG   r  ÚModuler$   rO   r†   r›   r¼   rØ   rç   rï   rK   r™   rô   rJ   r  r	  r"  r.  rF  rD   rt  rv  r‡  Ú__all__r*  r4   r3   ú<module>r�     s¡  ðð* %Ð $Ð $Ð $Ð $Ð $à €€€Ø Ð Ð Ð Ð Ð Ø $Ð $Ð $Ð $Ð $Ð $à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )Ø 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø /Ð /Ð /Ð /Ð /Ð /Ø [Ð [Ð [Ð [Ð [Ð [Ð [Ð [Ø QÐ QÐ QÐ QÐ QÐ QÐ QÐ QØ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RØ 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ .Ð .Ð .Ð .Ð .Ð .ð 
ˆÔ	˜HÑ	%Ô	%€ð Ð˜YÑ'Ô'ðJð Jð Jð Jð J�B”Iñ Jô Jñ (Ô'ðJð(6<ð 6<ð 6<ð 6<ð 6<˜BœIñ 6<ô 6<ð 6<ðr*ð *ð *ð *ð *˜BœIñ *ô *ð *ðZ.Sð .Sð .Sð .Sð .S�r”yñ .Sô .Sð .Sðb7ð 7ð 7ð 7ð 7�”	ñ 7ô 7ð 7ðtISð ISð ISð ISð IS�”	ñ ISô ISð ISðX(ð (ð (ð ÐÐ-Ñ.Ô.ðð ð ñ /Ô.ðð2	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð& ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð2U8ð U8ð U8ð U8ð U8�b”iñ U8ô U8ð U8ðp&ð &ð &ð &ð &Ð3ñ &ô &ð &ðR ð%ð %ð %ð %ð %˜Oñ %ô %ñ „ð%ð6 ðJ
ð J
ð J
ð J
ð J
Ð'ñ J
ô J
ñ „ðJ
ð^ #Ø
Ø*.ð	B&ð B&Ø”  e¤lÔ 3Ñ3°dÑ:ðB&à�t‘ðB&ð ”L 4Ñ'ð	B&ð
 „\�CÑðB&ð B&ð B&ð B&ðJK
ð K
ð K
ð K
ð K
Ð-¨ñ K
ô K
ð K
ð\ dÐ cÐ cÐ cÐ cÐ&FÐH]Ñ cÔ cÐ cð kÐ
jÐ
j€€€r4   