§
    9²¸j§X  ã                   óH  — d Z ddlZddlmc mZ ddlmZ ddlmZ ddlm	Z
 ddlmZ ddlmZmZ dd	lmZ dd
lmZmZmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl m!Z!m"Z"m#Z# ddl$m%Z% ddl&m'Z'm(Z( ddl)m*Z* ddl+m,Z, ddl-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8  e#j9        e:¦  «        Z; e"d¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z< G d„ de6¦  «        Z= G d„ de¦  «        Z> G d„ d ej?        ¦  «        Z@ G d!„ d"e*¦  «        ZA G d#„ d$e.¦  «        ZB G d%„ d&e8¦  «        ZC G d'„ d(e7¦  «        ZD G d)„ d*e/e¦  «        ZE G d+„ d,e5¦  «        ZF G d-„ d.e4¦  «        ZG G d/„ d0e0¦  «        ZH G d1„ d2e2¦  «        ZI G d3„ d4e3¦  «        ZJ G d5„ d6e1¦  «        ZKg d7¢ZLdS )8zPyTorch MiniMax model.é    N)Ústrict)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚPreTrainedConfig)Úcreate_causal_maskÚcreate_recurrent_attention_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚMoeModelOutputWithPast)ÚRopeParameters)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)Úmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚGemma2RotaryEmbedding)Úapply_mask_to_padding_states)ÚMixtralAttentionÚMixtralDecoderLayerÚMixtralForCausalLMÚMixtralForQuestionAnsweringÚ MixtralForSequenceClassificationÚMixtralForTokenClassificationÚMixtralModelÚMixtralPreTrainedModelÚMixtralRMSNormÚMixtralSparseMoeBlockÚMixtralTopKRouterzMiniMaxAI/MiniMax-Text-01-hf)Ú
checkpointc                   óº  ‡ — e Zd ZU dZdZdgZdZdddddddd	œZd
gdgfddgdgfdgdgfdœZdddddœZ	ddiZ
dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZedz  ed <   d!Zeed"<   d#Zeed$<   d%Zeed&<   d'Zeed(<   d)Zeed*<   dZedz  ed+<   d,Zedz  ed-<   d.Zeee         z  dz  ed/<   d0Z eed1<   dZ!edz  ed2<   d3Z"eez  ed4<   d.Z#eed5<   dZ$eed<   d0Z%eed6<   d7Z&eed8<   d3Z'eed9<   dZ(e)e*z  dz  ed:<   dZ+ee         dz  ed;<   d<Z,eed=<   d,Z-eez  ed><   d,Z.eez  ed?<   d,Z/eez  ed@<   d,Z0eez  edA<   d,Z1eez  edB<   d,Z2eez  edC<   ˆ fdD„Z3ˆ xZ4S )EÚMiniMaxConfiga¡  
    block_size (`int`, *optional*, defaults to 256):
        The length of each attention block, determining how queries, keys, and values
        are grouped and processed for intra- and inter-block attention.
    full_attn_alpha_factor (`float`, *optional*, defaults to 1):
        Weight for residual value in residual connection after normal attention.
    full_attn_beta_factor (`float`, *optional*, defaults to 1):
        Weight for hidden state value in residual connection after normal attention.
    linear_attn_alpha_factor (`float`, *optional*, defaults to 1):
        Weight for residual value in residual connection after lightning attention.
    linear_attn_beta_factor (`float`, *optional*, defaults to 1):
        Weight for hidden state value in residual connection after lightning attention.
    mlp_alpha_factor (`float`, *optional*, defaults to 1):
        Weight for residual value in residual connection after MLP.
    mlp_beta_factor (`float`, *optional*, defaults to 1):
        Weight for hidden state value in residual connection after MLP.

    ```python
    >>> from transformers import MiniMaxModel, MiniMaxConfig

    >>> # Initializing a MiniMax style configuration
    >>> configuration = MiniMaxConfig()

    >>> # Initializing a model from the MiniMax style configuration
    >>> model = MiniMaxModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```ÚminimaxÚpast_key_valuesg    €„.AÚcolwiseÚrowwiseÚpacked_colwiseÚmoe_tp_experts)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projú!layers.*.mlp.experts.gate_up_projúlayers.*.mlp.experts.down_projúlayers.*.mlp.expertsÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormÚ	ep_routerÚgrouped_gemm)zlayers.*.mlp.gater0   r1   r2   Únum_expertsÚnum_local_expertsi }  Ú
vocab_sizei   Úhidden_sizei 8  Úintermediate_sizeé    Únum_hidden_layersÚnum_attention_headsé   Únum_key_value_headsNÚhead_dimÚsiluÚ
hidden_acti   Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangeçñhãˆµøä>Úrms_norm_epsTÚ	use_cacheÚpad_token_idé   Úbos_token_idr   Úeos_token_idFÚtie_word_embeddingsÚsliding_windowg        Úattention_dropoutÚnum_experts_per_tokÚoutput_router_logitsgü©ñÒMbP?Úrouter_aux_loss_coefÚrouter_jitter_noiseÚrope_parametersÚlayer_typesé   Ú
block_sizeÚfull_attn_alpha_factorÚfull_attn_beta_factorÚlinear_attn_alpha_factorÚlinear_attn_beta_factorÚmlp_alpha_factorÚmlp_beta_factorc                 ó´   •— | j         €| j        | _         | j        €#d„ t          | j        ¦  «        D ¦   «         | _         t          ¦   «         j        di |¤Ž d S )Nc                 ó@   — g | ]}t          |d z   dz  ¦  «        rdnd‘ŒS )rO   r   Úfull_attentionÚlinear_attention)Úbool)Ú.0Úis     ú{/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/minimax/modular_minimax.pyú
<listcomp>z/MiniMaxConfig.__post_init__.<locals>.<listcomp>˜   sA   € ð  ð  ð  ØRS¥D¨!¨a©%°1©Ñ$5Ô$5ÐMÐ Ð Ð;Mð ð  ð  ó    © )rE   rC   rZ   ÚrangerB   ÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s     €rj   rp   zMiniMaxConfig.__post_init__“   sr   ø€ ØÔ#Ð+Ø'+Ô'?ˆDÔ$àÔÐ#ð ð  ÝW\Ð]aÔ]sÑWtÔWtð ñ  ô  ˆDÔð 	�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'rl   )5Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚdefault_thetaÚbase_model_tp_planÚbase_model_pp_planÚbase_model_ep_planÚattribute_mapr>   ÚintÚ__annotations__r?   r@   rB   rC   rE   rF   rH   ÚstrrI   rJ   ÚfloatrL   rM   rg   rN   rP   rQ   ÚlistrR   rS   rT   rU   r=   rV   rW   rX   rY   r   ÚdictrZ   r\   r]   r^   r_   r`   ra   rb   rp   Ú__classcell__©rs   s   @rj   r)   r)   7   s<  ø€ € € € € € ðð ð< €JØ#4Ð"5ÐØ€Mà%.Ø%.Ø%.Ø%.Ø-=Ø*3Ø 0ðð Ðð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ðð Ðð )Ø-;Ø*8Ø 0ð	ð Ðð #Ð$7Ð8€Mà€J�ÐÐÑØ€K�ÐÐÑØ"Ð�sÐ"Ð"Ñ"ØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø Ð˜Ð Ð Ñ Ø€Hˆc�D‰jÐÐÑØ€J�ÐÐÑØ#,Ð˜SÐ,Ð,Ñ,Ø#Ð�uÐ#Ð#Ñ#Ø€L�%ÐÐÑØ€IˆtÐÐÑØ#€L�#˜‘*Ð#Ð#Ñ#Ø €L�#˜‘*Ð Ð Ñ Ø+,€L�#˜˜Sœ	‘/ DÑ(Ð,Ð,Ñ,Ø %Ð˜Ð%Ð%Ñ%Ø!%€N�C˜$‘JÐ%Ð%Ñ%Ø%(Ð�u˜s‘{Ð(Ð(Ñ(Ø Ð˜Ð Ð Ñ ØÐ�sÐÐÑØ!&Ð˜$Ð&Ð&Ñ&Ø"'Ð˜%Ð'Ð'Ñ'Ø!$Ð˜Ð$Ð$Ñ$Ø48€O�^ dÑ*¨TÑ1Ð8Ð8Ñ8Ø$(€K��c”˜TÑ!Ð(Ð(Ñ(Ø€J�ÐÐÑØ*+Ð˜C %™KÐ+Ð+Ñ+Ø)*Ð˜3 ™;Ð*Ð*Ñ*Ø,-Ð˜c E™kÐ-Ð-Ñ-Ø+,Ð˜S 5™[Ð,Ð,Ñ,Ø$%Ð�c˜E‘kÐ%Ð%Ñ%Ø#$€O�S˜5‘[Ð$Ð$Ñ$ð	(ð 	(ð 	(ð 	(ð 	(ð 	(ð 	(ð 	(ð 	(rl   r)   c                   ó   — e Zd ZdS )ÚMiniMaxRMSNormN©rt   ru   rv   rm   rl   rj   rˆ   rˆ   Ÿ   ó   € € € € € Ø€Drl   rˆ   c                   óp   ‡ — e Zd ZdZˆ fd„Zd„ Zdefd„Zˆ fd„Zdefd„Z	d	e
j        fd
„Zdeddfd„Zˆ xZS )ÚMiniMaxCacheFc                 óV   •— t          ¦   «                              ¦   «          g | _        d S ©N)ro   Ú__init__Úlinear_cache©rq   rs   s    €rj   r�   zMiniMaxCache.__init__¨   s'   ø€ Ý‰Œ×ÒÑÔÐØ02ˆÔÐÐrl   c                 óž   — t          t          | j        ¦  «        |dz   ¦  «        D ]}| j                             g ¦  «         Œ|| j        |<   d S )NrO   )rn   Úlenr�   Úappend)rq   Ú	layer_idxr�   Ú_s       rj   Úset_linear_cachezMiniMaxCache.set_linear_cache¬   sW   € å•s˜4Ô,Ñ-Ô-¨y¸1©}Ñ=Ô=ð 	)ð 	)ˆAØÔ×$Ò$ RÑ(Ô(Ð(Ð(Ø'3ˆÔ˜)Ñ$Ð$Ð$rl   r•   c                 óF   — |t          | ¦  «        k     r| j        |         S d S rŽ   )r“   r�   )rq   r•   s     rj   Úget_linear_cachezMiniMaxCache.get_linear_cache²   s&   € Ø•s˜4‘y”yÒ Ð ØÔ$ YÔ/Ð/Øˆtrl   c                 ó„   •— t          t          ¦   «                              ¦   «         t          | j        ¦  «        ¦  «        S rŽ   )Úmaxro   Ú__len__r“   r�   r‘   s    €rj   rœ   zMiniMaxCache.__len__·   s,   ø€ Ý•5‘7”7—?’?Ñ$Ô$¥c¨$Ô*;Ñ&<Ô&<Ñ=Ô=Ð=rl   Úrepeatsc                 óü   — t          t          | ¦  «        ¦  «        D ]^}| j        |         g k    r+| j        |                              |d¬¦  «        | j        |<   Œ>| j        |                              |¦  «         Œ_d S )Nr   ©Údim)rn   r“   r�   Úrepeat_interleaver8   Úbatch_repeat_interleave)rq   r�   r•   s      rj   r¢   z$MiniMaxCache.batch_repeat_interleaveº   s‹   € Ý�s 4™yœyÑ)Ô)ð 	Hð 	HˆIØÔ  Ô+¨rÒ1Ð1Ø/3Ô/@ÀÔ/K×/]Ò/]Ð^eÐklÐ/]Ñ/mÔ/m�Ô! )Ñ,Ð,à”˜IÔ&×>Ò>¸wÑGÔGÐGÐGð		Hð 	Hrl   Úindicesc                 óâ   — t          t          | ¦  «        ¦  «        D ]Q}| j        |         g k    r| j        |         |df         | j        |<   Œ1| j        |                              |¦  «         ŒRd S )N.)rn   r“   r�   r8   Úbatch_select_indices)rq   r£   r•   s      rj   r¥   z!MiniMaxCache.batch_select_indicesÁ   s€   € Ý�s 4™yœyÑ)Ô)ð 	Eð 	EˆIØÔ  Ô+¨rÒ1Ð1Ø/3Ô/@ÀÔ/KÈGÐUXÈLÔ/Y�Ô! )Ñ,Ð,à”˜IÔ&×;Ò;¸GÑDÔDÐDÐDð		Eð 	Erl   Útokens_to_removeÚreturnNc                 ó    — t          d¦  «        ‚)Nz+MiniMaxCache does not support `crop` method)ÚRuntimeError)rq   r¦   s     rj   ÚcropzMiniMaxCache.cropÈ   s   € ÝÐHÑIÔIÐIrl   )rt   ru   rv   Úis_croppabler�   r—   r   r™   rœ   r¢   ÚtorchÚTensorr¥   rª   r…   r†   s   @rj   rŒ   rŒ   £   sð   ø€ € € € € ð €Lð3ð 3ð 3ð 3ð 3ð4ð 4ð 4ð¨#ð ð ð ð ð
>ð >ð >ð >ð >ðH¨sð Hð Hð Hð HðE¨E¬Lð Eð Eð Eð EðJ Sð J¨Tð Jð Jð Jð Jð Jð Jð Jð Jrl   rŒ   c                   óð   ‡ — e Zd Zdedefˆ fd„Zd„ Zd„ Z	 ddej	        de
ej	        ej	        f         d	ej	        dz  d
edz  dee         de
ej	        ej	        dz  e
ej	                 dz  f         fd„Zˆ xZS )ÚMiniMaxLightningAttentionÚconfigr•   c                 ó¸  •— t          ¦   «                              ¦   «          || _        t          |dd ¦  «        p|j        |j        z  | _        |j        | _        |j        | _        |j        | _        t          |j
                 | _        t          | j        | j        z  ¦  «        | _        t          j        |j        | j        | j        z  dz  d¬¦  «        | _        t          j        | j        | j        z  |j        d¬¦  «        | _        t          j        |j        | j        | j        z  d¬¦  «        | _        |                      ¦   «         }|                      |¦  «        \  }}}t          j        |¦  «        | _        t          j        |¦  «        | _        t          j        |¦  «        | _        t          j        |¦  «        | _        |j        |         | _        d S )NrF   r   F)Úbias)ro   r�   r•   Úgetattrr?   rC   rF   rB   r\   r   rH   Úact_fnrˆ   r9   r   ÚLinearÚqkv_projÚout_projÚoutput_gateÚget_slope_rateÚdecay_factorsÚBufferÚ
slope_rateÚquery_decayÚ	key_decayÚdiagonal_decayrZ   Ú
layer_type)rq   r°   r•   r¼   r½   r¾   r¿   rs   s          €rj   r�   z"MiniMaxLightningAttention.__init__Í   s�  ø€ Ý‰Œ×ÒÑÔÐØ"ˆŒÝ ¨
°DÑ9Ô9Ðm¸VÔ=OÐSYÔSmÑ=mˆŒØ#)Ô#=ˆÔ Ø!'Ô!9ˆÔØ Ô+ˆŒå˜VÔ.Ô/ˆŒÝ" 4¤=°4Ô3KÑ#KÑLÔLˆŒ	Ýœ	 &Ô"4°dÔ6NÐQUÔQ^Ñ6^ÐabÑ6bÐinÐoÑoÔoˆŒÝœ	 $Ô":¸T¼]Ñ"JÈFÔL^ÐejÐkÑkÔkˆŒÝœ9 VÔ%7¸Ô9QÐTXÔTaÑ9aÐhmÐnÑnÔnˆÔà×(Ò(Ñ*Ô*ˆ
Ø15×1CÒ1CÀJÑ1OÔ1OÑ.ˆ�Y åœ) JÑ/Ô/ˆŒÝœ9 [Ñ1Ô1ˆÔÝœ 9Ñ-Ô-ˆŒÝ œi¨Ñ7Ô7ˆÔà Ô,¨YÔ7ˆŒˆˆrl   c                 óÂ   — ddd| j         z  z  z  }t          j        | j         ¦  «        dz   }d| j        | j        dz
  dz   z  z
  dz   }||z  }||z  }|d d …d d f         }|S )NrO   r   rD   rK   )rC   r¬   Úaranger•   rB   )rq   ÚbaseÚexponentÚfactorÚrates        rj   r¹   z(MiniMaxLightningAttention.get_slope_rateå   s~   € Ø�A˜!˜dÔ6Ñ6Ñ7Ñ8ˆÝ”< Ô 8Ñ9Ô9¸AÑ=ˆØ�T”^ tÔ'=ÀÑ'AÀDÑ'HÑIÑIÈDÑPˆà�X‰~ˆØ�f‰}ˆØ�A�A�A�t˜T�MÔ"ˆàˆrl   c                 ó°  — t          j        | j        ¦  «        dz   }t          j        | |d d …d f         z  ¦  «        }t          j        | | j        |d d …d f         z
  z  ¦  «        }|d d …d f         |d d d …f         z
  }|d d d d …d d …f         }||z  }t          j        |dk    | t          d¦  «        ¦  «        }t          j        |¦  «        }|||fS )NrO   r   z-inf)r¬   rÂ   r\   ÚexpÚwherer‚   )rq   r¼   Úblock_size_ranger½   r¾   r¿   s         rj   rº   z'MiniMaxLightningAttention.decay_factorsð   sô   € Ý œ<¨¬Ñ8Ô8¸1Ñ<Ðå”i  Ð.>¸q¸q¸qÀ$¸wÔ.GÑ GÑHÔHˆÝ”I˜z˜k¨T¬_Ð?OÐPQÐPQÐPQÐSWÐPWÔ?XÑ-XÑYÑZÔZˆ	à)¨!¨!¨!¨T¨'Ô2Ð5EÀdÈAÈAÈAÀgÔ5NÑNˆØ'¨¨d°A°A°A°q°q°qÐ(8Ô9ˆØ# nÑ4ˆÝœ ^°qÒ%8¸>¸/Í5ÐQWÉ=Ì=ÑYÔYˆÝœ >Ñ2Ô2ˆà˜I ~Ð5Ð5rl   Nr5   Úposition_embeddingsr6   r+   rr   r§   c                 ór  — |j         \  }}}|| j        z   dz
  | j        z  }	|                      |                      |¦  «        ¦  «        }
t	          |
|¦  «        }
|
                     ||| j        d| j        z  ¦  «        }
t          j	        |
| j        d¬¦  «        \  }}}| 
                    dd¦  «        }| 
                    dd¦  «        }| 
                    dd¦  «        }d }|�|                     | j        ¦  «        }|�€®t          j        || j        | j        | j        ¦  «                             |¦  «        }g }t          |	¦  «        D �]a}|| j        z  }t!          || j        z   |¦  «        }||z
  }|d d …d d …||…f         }|d d …d d …||…f         }|d d …d d …||…f         }| j        d d …d |…f         }| j        d d …| d …f         }| j        d d …d d …d |…d |…f         }t          j        | j         |z  ¦  «        }t          j        || 
                    dd¦  «        ¦  «        }t          j        ||z  |¦  «        }t          j        ||z  |¦  «        }||z   }|                     |¦  «         t          j        ||z   
                    dd¦  «        |¦  «        }||z  |z   }�ŒcnÅt          j        | j         ¦  «        } g }t          |¦  «        D ]™}|d d …d d …||dz   …f         }|d d …d d …||dz   …f         }|d d …d d …||dz   …f         }t          j        | 
                    dd¦  «        |¦  «        }!| |z  |!z   }t          j        ||¦  «        }|                     |¦  «         Œšt          j        |d¬¦  «        }| 
                    dd¦  «        }|                     ||| j        | j        z  ¦  «        }|                      |¦  «        }t5          j        |                      |¦  «        ¦  «        |z  }|                      |¦  «        }|�|                     | j        |¦  «         ||fS )NrO   r   rŸ   r   éÿÿÿÿéþÿÿÿ)Úshaper\   r´   r¶   r   ÚreshaperC   rF   r¬   ÚsplitÚ	transposer™   r•   ÚzerosÚtorn   Úminr½   r¾   r¿   rÈ   r¼   Úmatmulr”   Úcatr9   ÚFÚsigmoidr¸   r·   r—   )"rq   r5   rË   r6   r+   rr   Ú
batch_sizeÚseq_lenr?   Ú
num_blocksÚ
qkv_statesÚquery_statesÚ
key_statesÚvalue_statesÚattn_weights_interÚattn_outputri   Ú	start_idxÚend_idxÚcurrent_block_sizeÚcurrent_query_statesÚcurrent_key_statesÚcurrent_value_statesÚcurrent_query_decayÚcurrent_key_decayÚcurrent_diagonal_decayÚblock_decayÚattn_weights_intraÚattn_output_intraÚattn_output_interÚcurrent_attn_outputÚnext_attn_weights_interÚratioÚcurrent_attn_weights_inters"                                     rj   Úforwardz!MiniMaxLightningAttention.forwardþ   sï  € ð ,9Ô+>Ñ(ˆ
�G˜[Ø ¤Ñ/°!Ñ3¸¼ÑGˆ
à—[’[ §¢¨}Ñ!=Ô!=Ñ>Ô>ˆ
Ý1°*¸nÑMÔMˆ
Ø×'Ò'¨
°G¸TÔ=UÐWXÐ[_Ô[hÑWhÑiÔiˆ
å16´¸ZÈÌÐ\]Ð1^Ñ1^Ô1^Ñ.ˆ�j ,à#×-Ò-¨a°Ñ3Ô3ˆØ×)Ò)¨!¨QÑ/Ô/ˆ
Ø#×-Ò-¨a°Ñ3Ô3ˆð "ÐØÐ&Ø!0×!AÒ!AÀ$Ä.Ñ!QÔ!QÐàÑ%Ý!&¤¨Z¸Ô9QÐSWÔS`ÐbfÔboÑ!pÔ!p×!sÒ!sØñ"ô "Ðð ˆKÝ˜:Ñ&Ô&ð `ñ `�Ø ¤Ñ/�	Ý˜i¨$¬/Ñ9¸7ÑCÔC�Ø%,¨yÑ%8Ð"à'3°A°A°A°q°q°q¸)ÀGÐ:KÐ4KÔ'LÐ$Ø%/°°°°1°1°1°iÀÐ6GÐ0GÔ%HÐ"Ø'3°A°A°A°q°q°q¸)ÀGÐ:KÐ4KÔ'LÐ$à&*Ô&6°q°q°qÐ:MÐ;MÐ:MÐ7MÔ&NÐ#Ø$(¤N°1°1°1Ð7IÐ6IÐ6JÐ6JÐ3JÔ$KÐ!Ø)-Ô)<¸Q¸Q¸QÀÀÀÐCVÐDVÐCVÐXkÐYkÐXkÐ=kÔ)lÐ&Ý#œi¨¬Ð(8Ð;MÑ(MÑNÔN�õ &+¤\Ð2FÐHZ×HdÒHdÐegÐikÑHlÔHlÑ%mÔ%mÐ"Ý$)¤LÐ1CÐF\Ñ1\Ð^rÑ$sÔ$sÐ!õ %*¤LÐ1EÐH[Ñ1[Ð]oÑ$pÔ$pÐ!ð '8Ð:KÑ&KÐ#Ø×"Ò"Ð#6Ñ7Ô7Ð7õ +0¬,Ø'Ð*;Ñ;×FÒFÀrÈ2ÑNÔNÐPdñ+ô +Ð'ð &8¸+Ñ%EÐH_Ñ%_Ð"Ñ"ð;`õ@ ”I˜tœÐ.Ñ/Ô/ˆEØˆKÝ˜7‘^”^ð 	8ð 	8�Ø'3°A°A°A°q°q°q¸!¸aÀ!¹e¸)°OÔ'DÐ$Ø%/°°°°1°1°1°a¸!¸a¹%°i°Ô%@Ð"Ø'3°A°A°A°q°q°q¸!¸aÀ!¹e¸)°OÔ'DÐ$å-2¬\Ð:L×:VÒ:VÐWYÐ[]Ñ:^Ô:^Ð`tÑ-uÔ-uÐ*Ø%*Ð-?Ñ%?ÐB\Ñ%\Ð"Ý&+¤lÐ3GÐI[Ñ&\Ô&\Ð#à×"Ò"Ð#6Ñ7Ô7Ð7Ð7õ ”i °Ð4Ñ4Ô4ˆð "×+Ò+¨A¨qÑ1Ô1ˆØ!×)Ò)¨*°g¸tÔ?WÐZ^ÔZgÑ?gÑhÔhˆØ—i’i Ñ,Ô,ˆÝ”i × 0Ò 0°Ñ ?Ô ?Ñ@Ô@À;ÑNˆØ—m’m KÑ0Ô0ˆð Ð&Ø×,Ò,¨T¬^Ð=OÑPÔPÐPàÐ.Ð.Ð.rl   rŽ   )rt   ru   rv   r)   r   r�   r¹   rº   r¬   r­   Útupler   r   r   rô   r…   r†   s   @rj   r¯   r¯   Ì   s  ø€ € € € € ð8˜}ð 8¸ð 8ð 8ð 8ð 8ð 8ð 8ð0	ð 	ð 	ð6ð 6ð 6ð& )-ð[/ð [/à”|ð[/ð # 5¤<°´Ð#=Ô>ð[/ð œ tÑ+ð	[/ð
  ™ð[/ð Ð-Ô.ð[/ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	Mð[/ð [/ð [/ð [/ð [/ð [/ð [/ð [/rl   r¯   c                   ó   — e Zd ZdS )ÚMiniMaxRotaryEmbeddingNr‰   rm   rl   rj   r÷   r÷   \  rŠ   rl   r÷   c                   ó   — e Zd ZdS )ÚMiniMaxAttentionNr‰   rm   rl   rj   rù   rù   `  rŠ   rl   rù   c                   ó   — e Zd ZdS )ÚMiniMaxTopKRouterNr‰   rm   rl   rj   rû   rû   d  rŠ   rl   rû   c                   ó   — e Zd ZdS )ÚMiniMaxSparseMoeBlockNr‰   rm   rl   rj   rý   rý   h  rŠ   rl   rý   c                   ó  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 ddej        deej        ej        f         dz  dej        dz  d	ej	        dz  d
e
dz  dedz  dee         deej        eej        ej        f         dz  f         fd„Zˆ xZS )ÚMiniMaxDecoderLayerr°   r•   c                 óÌ  •— t          ¦   «                              ||¦  «         || _        t          |d¦  «        r|j        |         nd | _        |j        | _        |j        | _        | `t          |¦  «        | _        | j        dk    r/t          ||¦  «        | _        |j        | _        |j        | _        d S t!          ||¦  «        | _        |j        | _        |j        | _        d S )NrZ   rf   )ro   r�   r•   ÚhasattrrZ   Ú
block_typera   rb   Úmlprý   r¯   Ú	self_attnr_   Úattn_alpha_factorr`   Úattn_beta_factorrù   r]   r^   )rq   r°   r•   rs   s      €rj   r�   zMiniMaxDecoderLayer.__init__m  sÚ   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+à"ˆŒÝ;BÀ6È=Ñ;YÔ;YÐc˜&Ô,¨YÔ7Ð7Ð_cˆŒØ &Ô 7ˆÔØ%Ô5ˆÔØˆHÝ(¨Ñ0Ô0ˆŒØŒ?Ð0Ò0Ð0Ý6°v¸yÑIÔIˆDŒNØ%+Ô%DˆDÔ"Ø$*Ô$BˆDÔ!Ð!Ð!å-¨f°iÑ@Ô@ˆDŒNØ%+Ô%BˆDÔ"Ø$*Ô$@ˆDÔ!Ð!Ð!rl   NFr5   rË   r6   Úposition_idsr+   rM   rr   r§   c           
      ó  — |                       |¦  «        }|} | j        d||||||dœ|¤Ž\  }}	|| j        z  || j        z  z   }|                      |¦  «        }|}|                      |¦  «        }|| j        z  || j        z  z   }|S )N)r5   rË   r6   r  r+   rM   rm   )Úinput_layernormr  r  r  Úpost_attention_layernormr  ra   rb   )
rq   r5   rË   r6   r  r+   rM   rr   Úresidualr–   s
             rj   rô   zMiniMaxDecoderLayer.forward  s¿   € ð ×,Ò,¨]Ñ;Ô;ˆØ ˆØ)˜4œ>ð 
Ø'Ø 3Ø)Ø%Ø+Øð
ð 
ð ð
ð 
Ñˆ�qð ! 4Ô#9Ñ9¸MÈDÔLaÑ<aÑaˆØ×5Ò5°mÑDÔDˆØ ˆØŸš Ñ/Ô/ˆØ  4Ô#8Ñ8¸=È4ÔK_Ñ;_Ñ_ˆàÐrl   )NNNNF)rt   ru   rv   r)   r   r�   r¬   r­   rõ   Ú
LongTensorr   rg   r   r   ÚFloatTensorrô   r…   r†   s   @rj   rÿ   rÿ   l  s"  ø€ € € € € ðA˜}ð A¸ð Að Að Að Að Að Að* IMØ.2Ø04Ø(,Ø!&ðð à”|ðð # 5¤<°´Ð#=Ô>ÀÑEðð œ tÑ+ð	ð
 Ô&¨Ñ-ðð  ™ðð ˜$‘;ðð Ð-Ô.ðð 
ˆuÔ  %¨Ô(9¸5Ô;LÐ(LÔ"MÐPTÑ"TÐTÔ	Uðð ð ð ð ð ð ð rl   rÿ   c                   óJ   ‡ — e Zd ZdZ eedd¬¦  «        eeegdœZ	ˆ fd„Z
ˆ xZS )ÚMiniMaxPreTrainedModelFzmlp.gater   )Ú
layer_nameÚindex)Úrouter_logitsr5   Ú
attentionsc                 ó¢  •— t          ¦   «                              |¦  «         t          |t          ¦  «        r—|                     ¦   «         }|                     |¦  «        \  }}}t          j        |j        |¦  «         t          j        |j	        |¦  «         t          j        |j
        |¦  «         t          j        |j        |¦  «         d S d S rŽ   )ro   Ú_init_weightsÚ
isinstancer¯   r¹   rº   ÚinitÚcopy_r¼   r½   r¾   r¿   )rq   Úmoduler¼   r½   r¾   r¿   rs   s         €rj   r  z$MiniMaxPreTrainedModel._init_weights¥  s½   ø€ Ý‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ7Ñ8Ô8ð 	>Ø×.Ò.Ñ0Ô0ˆJØ5;×5IÒ5IÈ*Ñ5UÔ5UÑ2ˆK˜ NÝŒJ�vÔ(¨*Ñ5Ô5Ð5ÝŒJ�vÔ)¨;Ñ7Ô7Ð7ÝŒJ�vÔ'¨Ñ3Ô3Ð3ÝŒJ�vÔ,¨nÑ=Ô=Ð=Ð=Ð=ð	>ð 	>rl   )rt   ru   rv   Ú_can_compile_fullgraphr   rû   rÿ   rù   r¯   Ú_can_record_outputsr  r…   r†   s   @rj   r  r  �  sm   ø€ € € € € Ø"Ðà'˜Ð(9ÀjÐXYÐZÑZÔZØ,Ø'Ð)BÐCðð Ðð>ð >ð >ð >ð >ð >ð >ð >ð >rl   r  c                   óÂ   — e Zd Zee	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dedz  dej	        dz  de
dz  dee         d	eez  fd
„¦   «         ¦   «         ZdS )ÚMiniMaxModelNr3   r6   r  r+   r4   rM   rr   r§   c           
      óD  — |d u |d uz  rt          d¦  «        ‚|r|€t          ¦   «         }n7|r5t          |t          ¦  «        s t          dt          |¦  «        › d�¦  «        ‚|€|                      |¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j	        ¬¦  «        |z   }| 
                    d¦  «        }t          |x}	t          ¦  «        s<| j        j        €t          nt          }
| j        ||||dœ} |
di |¤Žt!          di |¤Ždœ}	|}|                      ||¦  «        }t%          | j        ¦  «        D ]*\  }} ||f|	| j        j        |                  ||||d	œ|¤Ž}Œ+|                      |¦  «        }t-          ||¬
¦  «        S )Nz:You must specify exactly one of input_ids or inputs_embedszSMiniMax uses cache of its own and is not compatible with `past_key_values` of type ú.r   rO   )Údevice)r°   r4   r6   r+   r  )re   rf   )r6   rË   r  r+   rM   )Úlast_hidden_stater+   rm   )Ú
ValueErrorrŒ   r  Útyper7   Úget_seq_lengthr¬   rÂ   rÏ   r   Ú	unsqueezer„   r°   rS   r   r   r   Ú
rotary_embÚ	enumerater8   rZ   r9   r   )rq   r3   r6   r  r+   r4   rM   rr   Úpast_seen_tokensÚcausal_mask_mappingÚmask_functionÚmask_kwargsr5   rË   ri   Údecoder_layers                   rj   rô   zMiniMaxModel.forward±  s0  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàð 	˜Ð0Ý*™nœnˆOˆOØð 	�z¨/½<ÑHÔHð 	ÝØ~ÕfjÐkzÑf{Ôf{Ð~Ð~Ð~ñô ð ð Ð Ø ×-Ò-¨iÑ8Ô8ˆMàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå°Ð?Ð-ÅÑFÔFð 	à&*¤kÔ&@Ð&HÕ"Ð"ÕNoð ð œ+Ø!.Ø"0Ø#2Ø ,ðð ˆKð #0 -Ð">Ð">°+Ð">Ð">Ý$CÐ$RÐ$RÀkÐ$RÐ$Rð#ð #Ðð
 &ˆØ"Ÿošo¨m¸\ÑJÔJÐå )¨$¬+Ñ 6Ô 6ð 		ð 		ÑˆAˆ}Ø)˜MØðà2°4´;Ô3JÈ1Ô3MÔNØ$7Ø)Ø /Ø#ðð ð ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆå%Ø+Ø+ð
ñ 
ô 
ð 	
rl   )NNNNNN)rt   ru   rv   r   r   r¬   r  r­   rŒ   r  rg   r   r   rõ   r   rô   rm   rl   rj   r  r  °  sØ   € € € € € ØØð .2Ø.2Ø04Ø/3Ø26Ø!%ð?
ð ?
àÔ# dÑ*ð?
ð œ tÑ+ð?
ð Ô&¨Ñ-ð	?
ð
 &¨Ñ,ð?
ð Ô(¨4Ñ/ð?
ð ˜$‘;ð?
ð Ð+Ô,ð?
ð 
Ð'Ñ	'ð?
ð ?
ð ?
ñ „_ñ  Ôð?
ð ?
ð ?
rl   r  c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚMiniMaxForCausalLMc                 ó6   •—  t          ¦   «         j        di |¤ŽS )a€  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, MiniMaxForCausalLM

        >>> model = MiniMaxForCausalLM.from_pretrained("MiniMaxAI/MiniMax-Text-01-hf")
        >>> tokenizer = AutoTokenizer.from_pretrained("MiniMaxAI/MiniMax-Text-01-hf")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```rm   )ro   rô   )rq   Úsuper_kwargsrs   s     €rj   rô   zMiniMaxForCausalLM.forwardö  s!   ø€ ð. �u‰wŒwŒÐ.Ð. Ð.Ð.Ð.rl   )rt   ru   rv   rô   r…   r†   s   @rj   r.  r.  õ  s8   ø€ € € € € ð/ð /ð /ð /ð /ð /ð /ð /ð /rl   r.  c                   ó   — e Zd ZdS )Ú MiniMaxForSequenceClassificationNr‰   rm   rl   rj   r2  r2    rŠ   rl   r2  c                   ó   — e Zd ZdS )ÚMiniMaxForTokenClassificationNr‰   rm   rl   rj   r4  r4    rŠ   rl   r4  c                   ó   — e Zd ZdS )ÚMiniMaxForQuestionAnsweringNr‰   rm   rl   rj   r6  r6    rŠ   rl   r6  )r)   r  r  r.  r2  r4  r6  )Mrw   r¬   Útorch.nn.functionalr   Ú
functionalrØ   Úhuggingface_hub.dataclassesr   Ú r   r  Úactivationsr   Úcache_utilsr   r	   Úconfiguration_utilsr
   Úmasking_utilsr   r   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   Úmodeling_rope_utilsr   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   Úutils.output_capturingr   r   Úgemma2.modeling_gemma2r   Úmamba2.modeling_mamba2r   Úmixtral.modeling_mixtralr   r   r   r   r    r!   r"   r#   r$   r%   r&   Ú
get_loggerrt   Úloggerr)   rˆ   rŒ   ÚModuler¯   r÷   rù   rû   rý   rÿ   r  r  r.  r2  r4  r6  Ú__all__rm   rl   rj   ú<module>rN     s  ðð Ð à €€€Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø .Ð .Ð .Ð .Ð .Ð .Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø sÐ sÐ sÐ sÐ sÐ sÐ sÐ sÐ sÐ sØ BÐ BÐ BÐ BÐ BÐ BØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø &Ð &Ð &Ð &Ð &Ð &Ø @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ø 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ :Ð :Ð :Ð :Ð :Ð :Ø AÐ AÐ AÐ AÐ AÐ Aðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð 
ˆÔ	˜HÑ	%Ô	%€ð €Ð9Ð:Ñ:Ô:Øðc(ð c(ð c(ð c(ð c(Ð$ñ c(ô c(ñ „ñ ;Ô:ðc(ðL	ð 	ð 	ð 	ð 	�^ñ 	ô 	ð 	ð&Jð &Jð &Jð &Jð &J�<ñ &Jô &Jð &JðRM/ð M/ð M/ð M/ð M/ ¤	ñ M/ô M/ð M/ð`	ð 	ð 	ð 	ð 	Ð2ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð'ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð)ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð1ñ 	ô 	ð 	ð.ð .ð .ð .ð .Ð-Ð/Iñ .ô .ð .ðb>ð >ð >ð >ð >Ð3ñ >ô >ð >ð&B
ð B
ð B
ð B
ð B
�<ñ B
ô B
ð B
ðJ/ð /ð /ð /ð /Ð+ñ /ô /ð /ð6	ð 	ð 	ð 	ð 	Ð'Gñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð$Añ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð"=ñ 	ô 	ð 	ðð ð €€€rl   