§
    9²¸jÓ5  ã                   óŽ  — d dl Z d dlmZ d dlZd dlmZ ddlmZ ddl	m
Z
 ddlmZmZ ddlmZmZ dd	lmZ dd
lmZmZmZmZ ddlmZ ddlmZ ddlmZ ddlmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(  ej)        e*¦  «        Z+ ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z, G d„ de¦  «        Z- G d„ de%¦  «        Z. G d„ de&¦  «        Z/ G d„ de"¦  «        Z0 G d„ de¦  «        Z1 G d„ de¦  «        Z2 G d „ d!e$¦  «        Z3e G d"„ d#e#¦  «        ¦   «         Z4e G d$„ d%e ¦  «        ¦   «         Z5 G d&„ d'e!¦  «        Z6g d(¢Z7dS ))é    N)ÚCallable)Ústricté   )Úinitialization)ÚCache)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚloggingé   )ÚDeepseekV2Attention)ÚGemma3TextScaledWordEmbedding)ÚLlamaConfig)
ÚLlamaDecoderLayerÚLlamaForCausalLMÚLlamaForSequenceClassificationÚLlamaMLPÚ
LlamaModelÚLlamaPreTrainedModelÚLlamaRMSNormÚLlamaRotaryEmbeddingÚapply_rotary_pos_embÚeager_attention_forwardzopenbmb/MiniCPM3-4B)Ú
checkpointc            	       ó€  ‡ — e Zd ZU dZdddddddddœZdZdZeed<   d	Z	eed
<   dZ
eed<   dZeed<   dZeed<   dZedz  ed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZedz  ed<   dZeed <   d!Zeed"<   dZedz  ed#<   d$Zeez  ed%<   d&Zeez  dz  ed'<   dZedz  ed(<   ˆ fd)„Zed*efd+„¦   «         Zˆ xZS ),ÚMiniCPM3Configa‰  
    kv_lora_rank (`int`, *optional*, defaults to 256):
        Rank of the low-rank KV projection in multi-head latent attention.
    q_lora_rank (`int`, *optional*, defaults to 768):
        Rank of the low-rank query projection in multi-head latent attention. If `None`, the query projection
        is a single dense projection rather than a low-rank one.
    qk_nope_head_dim (`int`, *optional*, defaults to 64):
        Dimension of the non-RoPE part of each query/key head.
    qk_rope_head_dim (`int`, *optional*, defaults to 32):
        Dimension of the RoPE part of each query/key head.
    v_head_dim (`int`, *optional*):
        Dimension of each value head. If `None`, defaults to `hidden_size // num_attention_heads`.
    scale_emb (`int` or `float`, *optional*, defaults to 12):
        Multiplier applied to input embeddings.
    scale_depth (`int` or `float`, *optional*, defaults to 1.4):
        Multiplier for residual connections; the effective scaling is `scale_depth / sqrt(num_hidden_layers)`.
        If `None`, defaults to `sqrt(num_hidden_layers)` (no-op scaling).
    dim_model_base (`int`, *optional*, defaults to 256):
        Base model dimension used to scale logits before the language model head. If `None`,
        defaults to `hidden_size` (no-op scaling).

    Example:

    ```python
    >>> from transformers import MiniCPM3Model, MiniCPM3Config
    >>> # Initializing a MiniCPM3 style configuration
    >>> configuration = MiniCPM3Config()
    >>> # Initializing a model from the configuration
    >>> model = MiniCPM3Model(configuration)
    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```
    ÚcolwiseÚmla_kv_a_projÚrowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.q_b_projz%layers.*.self_attn.kv_a_proj_with_mqazlayers.*.self_attn.kv_b_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚminicpm3iè Ú
vocab_sizei 
  Úhidden_sizei   Úintermediate_sizeé>   Únum_hidden_layersé(   Únum_attention_headsNÚnum_key_value_headsi €  Úmax_position_embeddingsgš™™™™™¹?Úinitializer_rangegñhãˆµøä>Úrms_norm_epsTÚtie_word_embeddingsé   Úkv_lora_ranki   Úq_lora_ranké@   Úqk_nope_head_dimé    Úqk_rope_head_dimÚ
v_head_dimé   Ú	scale_embgffffffö?Úscale_depthÚdim_model_basec                 ó   •— | j         | _        | j        | j         z   | _        | j        €| j        | j        z  | _        | j        €t          j	        | j
        ¦  «        | _        | j        €| j        | _         t          ¦   «         j        di |¤Ž d S )N© )r8   Úhead_dimr6   Úqk_head_dimr9   r'   r,   r<   ÚmathÚsqrtr*   r=   ÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s     €ú}/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/minicpm3/modular_minicpm3.pyrE   zMiniCPM3Config.__post_init__v   s‘   ø€ àÔ-ˆŒàÔ0°4Ô3HÑHˆÔð Œ?Ð"Ø"Ô.°$Ô2JÑJˆDŒOØÔÐ#Ý#œy¨Ô)?Ñ@Ô@ˆDÔØÔÐ&Ø"&Ô"2ˆDÔØ�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'ó    Úreturnc                 ó    — | j         | j        z  S ©N)r'   r=   )rF   s    rI   Úlogits_scalingzMiniCPM3Config.logits_scaling…   s   € ð Ô $Ô"5Ñ5Ð5rJ   ) Ú__name__Ú
__module__Ú__qualname__Ú__doc__Úbase_model_tp_planÚ
model_typer&   ÚintÚ__annotations__r'   r(   r*   r,   r-   r.   r/   Úfloatr0   r1   Úboolr3   r4   r6   r8   r9   r;   r<   r=   rE   ÚpropertyrN   Ú__classcell__©rH   s   @rI   r!   r!   /   sâ  ø€ € € € € € ð ð  ðF &/Ø'0Ø1@Ø(1Ø%.Ø"+Ø )Ø"+ð	ð 	Ðð €Jð €J�ÐÐÑØ€K�ÐÐÑØ!Ð�sÐ!Ð!Ñ!ØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø&(Ð˜˜t™Ð(Ð(Ñ(Ø#(Ð˜SÐ(Ð(Ñ(Ø"Ð�uÐ"Ð"Ñ"Ø€L�%ÐÐÑØ $Ð˜Ð$Ð$Ñ$Ø€L�#ÐÐÑØ!€K��t‘Ð!Ð!Ñ!ØÐ�cÐÐÑØÐ�cÐÐÑØ!€J��d‘
Ð!Ð!Ñ!Ø€Iˆs�U‰{ÐÐÑØ&)€K��u‘˜tÑ#Ð)Ð)Ñ)Ø!$€N�C˜$‘JÐ$Ð$Ñ$ð(ð (ð (ð (ð (ð ð6 ð 6ð 6ð 6ñ „Xð6ð 6ð 6ð 6ð 6rJ   r!   c                   ó   — e Zd ZdS )ÚMiniCPM3ScaledWordEmbeddingN©rO   rP   rQ   r?   rJ   rI   r]   r]   ‹   ó   € € € € € Ø€DrJ   r]   c                   ó   — e Zd ZdS )ÚMiniCPM3RMSNormNr^   r?   rJ   rI   ra   ra   �   r_   rJ   ra   c                   ó   — e Zd ZdS )ÚMiniCPM3RotaryEmbeddingNr^   r?   rJ   rI   rc   rc   “   r_   rJ   rc   c                   ó   — e Zd ZdS )ÚMiniCPM3MLPNr^   r?   rJ   rI   re   re   —   r_   rJ   re   c                   ó¢   — e Zd ZdZ	 	 d	dej        deej        ej        f         dej        dz  dedz  deej        ej        dz  f         f
d„ZdS )
ÚMiniCPM3AttentionaE  
    Multi-head Latent Attention (MLA), structurally identical to `DeepseekV2Attention`.
    The only difference is the rotary convention: MiniCPM3 keeps the original cos/sin RoPE
    (`apply_rotary_pos_emb`) instead of DeepSeek-V2's complex rotary, so we inherit the
    module construction and override only `forward`.
    NÚhidden_statesÚposition_embeddingsÚattention_maskÚpast_key_valuesrK   c                 óŠ  — |j         d d…         \  }}||d| j        f}| j        €|                      |¦  «        }	n;|                      |                      |                      |¦  «        ¦  «        ¦  «        }	|	                     |¦  «                             dd¦  «        }	t          j
        |	| j        | j        gd¬¦  «        \  }
}|                      |¦  «        }t          j
        || j        | j        gd¬¦  «        \  }}|                      |¦  «                             |d|| j        ¦  «        }|                     |d|| j        ¦  «        }|\  }}t!          ||||¦  «        \  }}|�|                     ||| j        ¦  «        \  }}t          j        |
|fd¬¦  «        }|                      ||¦  «        \  }}t+          j        | j        j        t2          ¦  «        } || ||||f| j        sdn| j        | j        dœ|¤Ž\  }}|                     ||d¦  «                             ¦   «         }|                      |¦  «        }||fS )Néÿÿÿÿé   r   )Údimg        )ÚdropoutÚscaling) ÚshaperA   r4   Úq_projÚq_b_projÚq_a_layernormÚq_a_projÚviewÚ	transposeÚtorchÚsplitr6   r8   Úkv_a_proj_with_mqar3   Úkv_a_layernormr   ÚupdateÚ	layer_idxÚcatÚ	expand_kvr
   Úget_interfaceÚconfigÚ_attn_implementationr   ÚtrainingÚattention_dropoutrq   ÚreshapeÚ
contiguousÚo_proj)rF   rh   ri   rj   rk   rG   Ú
batch_sizeÚ
seq_lengthÚquery_shapeÚq_statesÚq_passÚq_rotÚcompressed_kvÚkv_passÚk_rotÚk_passÚcosÚsinÚquery_statesÚ
key_statesÚvalue_statesÚattention_interfaceÚattn_outputÚattn_weightss                           rI   ÚforwardzMiniCPM3Attention.forward£   s`  € ð "/Ô!4°S°b°SÔ!9Ñˆ
�JØ! :¨r°4Ô3CÐDˆàÔÐ#Ø—{’{ =Ñ1Ô1ˆHˆHà—}’} T×%7Ò%7¸¿ºÀmÑ8TÔ8TÑ%UÔ%UÑVÔVˆHØ—=’= Ñ-Ô-×7Ò7¸¸1Ñ=Ô=ˆÝœ H¨tÔ/DÀdÔF[Ð.\ÐbdÐeÑeÔe‰ˆ�à×/Ò/°Ñ>Ô>ˆÝœ ]°TÔ5FÈÔH]Ð4^ÐdfÐgÑgÔg‰ˆ�à×$Ò$ WÑ-Ô-×2Ò2°:¸qÀ*ÈdÔN_Ñ`Ô`ˆØ—
’
˜: q¨*°dÔ6KÑLÔLˆà&‰ˆˆSõ ,¨E°5¸#¸sÑCÔC‰ˆˆuð Ð&Ø+×2Ò2°6¸5À$Ä.ÑQÔQ‰MˆF�Eå”y &¨% °bÐ9Ñ9Ô9ˆà#'§>¢>°&¸%Ñ#@Ô#@Ñ ˆ
�Lå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð "×)Ò)¨*°jÀ"ÑEÔE×PÒPÑRÔRˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(rJ   )NN)	rO   rP   rQ   rR   ry   ÚTensorÚtupler   r›   r?   rJ   rI   rg   rg   ›   s�   € € € € € ðð ð /3Ø(,ð6)ð 6)à”|ð6)ð # 5¤<°´Ð#=Ô>ð6)ð œ tÑ+ð	6)ð
  ™ð6)ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð6)ð 6)ð 6)ð 6)ð 6)ð 6)rJ   rg   c                   óÒ   ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 ddej        dej        dz  dej        dz  d	e	dz  d
e
dz  deej        ej        f         dz  dee         dej        fd„Zˆ xZS )ÚMiniCPM3DecoderLayerr‚   r~   c                 ó˜   •— t          ¦   «                              ||¦  «         |j        t          j        |j        ¦  «        z  | _        d S rM   )rD   Ú__init__r<   rB   rC   r*   Úresidual_scale)rF   r‚   r~   rH   s      €rI   r¡   zMiniCPM3DecoderLayer.__init__Ý   sB   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+ð %Ô0µ4´9¸VÔ=UÑ3VÔ3VÑVˆÔÐÐrJ   NFrh   rj   Úposition_idsrk   Ú	use_cacheri   rG   rK   c           
      óî   — |}|                       |¦  «        } | j        d||||||dœ|¤Ž\  }}	||| j        z  z   }|}|                      |¦  «        }|                      |¦  «        }||| j        z  z   }|S )N)rh   rj   r£   rk   r¤   ri   r?   )Úinput_layernormÚ	self_attnr¢   Úpost_attention_layernormÚmlp)
rF   rh   rj   r£   rk   r¤   ri   rG   ÚresidualÚ_s
             rI   r›   zMiniCPM3DecoderLayer.forwardã   s¯   € ð !ˆØ×,Ò,¨]Ñ;Ô;ˆØ)˜4œ>ð 
Ø'Ø)Ø%Ø+ØØ 3ð
ð 
ð ð
ð 
Ñˆ�qð ! =°4Ô3FÑ#FÑFˆà ˆØ×5Ò5°mÑDÔDˆØŸš Ñ/Ô/ˆØ  =°4Ô3FÑ#FÑFˆØÐrJ   )NNNFN)rO   rP   rQ   r!   rU   r¡   ry   rœ   Ú
LongTensorr   rX   r�   r   r   r›   rZ   r[   s   @rI   rŸ   rŸ   Ü   sÿ   ø€ € € € € ðW˜~ð W¸#ð Wð Wð Wð Wð Wð Wð /3Ø04Ø(,Ø!&ØHLðð à”|ðð œ tÑ+ðð Ô&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð # 5¤<°´Ð#=Ô>ÀÑEðð Ð+Ô,ðð 
Œðð ð ð ð ð ð ð rJ   rŸ   c                   ó>   — e Zd Z ej        ¦   «         d„ ¦   «         ZdS )ÚMiniCPM3PreTrainedModelc                 óœ   — t          j        | |¦  «         t          |t          ¦  «        r!t	          j        |j        |j        ¦  «         d S d S rM   )r   Ú_init_weightsÚ
isinstancer]   ÚinitÚ	constant_Úembed_scaleÚscalar_embed_scale)rF   Úmodules     rI   r°   z%MiniCPM3PreTrainedModel._init_weights  sS   € åÔ% d¨FÑ3Ô3Ð3Ý�fÕ9Ñ:Ô:ð 	JÝŒN˜6Ô-¨vÔ/HÑIÔIÐIÐIÐIð	Jð 	JrJ   N)rO   rP   rQ   ry   Úno_gradr°   r?   rJ   rI   r®   r®     s:   € € € € € Ø€U„]�_„_ðJð Jñ „_ðJð Jð JrJ   r®   c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚMiniCPM3Modelr‚   c                 ó¢   •— t          ¦   «                              |¦  «         t          |j        |j        | j        |j        ¬¦  «        | _        d S )N)r´   )rD   r¡   r]   r&   r'   Úpadding_idxr;   Úembed_tokens)rF   r‚   rH   s     €rI   r¡   zMiniCPM3Model.__init__  sN   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å7ØÔ˜vÔ1°4Ô3CÐQWÔQað
ñ 
ô 
ˆÔÐÐrJ   )rO   rP   rQ   r!   r¡   rZ   r[   s   @rI   r¹   r¹   	  sD   ø€ € € € € ð
˜~ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
rJ   r¹   c                   óè   — e Zd Zee	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dedz  dej	        dz  dej        dz  d	e
dz  d
eej        z  dee         defd„¦   «         ¦   «         ZdS )ÚMiniCPM3ForCausalLMNr   Ú	input_idsrj   r£   rk   Úinputs_embedsÚlabelsr¤   Úlogits_to_keeprG   rK   c	           
      ón  —  | j         d||||||dœ|	¤Ž}
|
j        }|| j        j        z  }t	          |t
          ¦  «        rt          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }d}|� | j        d||| j        j	        dœ|	¤Ž}t          |||
j        |
j        |
j        ¬¦  «        S )a´  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, MiniCPM3ForCausalLM

        >>> model = MiniCPM3ForCausalLM.from_pretrained("openbmb/MiniCPM3-4B")
        >>> tokenizer = AutoTokenizer.from_pretrained("openbmb/MiniCPM3-4B")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```)r¿   rj   r£   rk   rÀ   r¤   N)ÚlogitsrÁ   r&   )ÚlossrÄ   rk   rh   Ú
attentionsr?   )ÚmodelÚlast_hidden_stater‚   rN   r±   rU   ÚsliceÚlm_headÚloss_functionr&   r	   rk   rh   rÆ   )rF   r¿   rj   r£   rk   rÀ   rÁ   r¤   rÂ   rG   Úoutputsrh   Úslice_indicesrÄ   rÅ   s                  rI   r›   zMiniCPM3ForCausalLM.forward  s  € ð< ,6¨4¬:ð ,
ØØ)Ø%Ø+Ø'Øð,
ð ,
ð ð,
ð ,
ˆð  Ô1ˆà%¨¬Ô(BÑBˆÝ8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%Ðp¨V¸FÈtÌ{ÔOeÐpÐpÐioÐpÐpˆDå%ØØØ#Ô3Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
rJ   )NNNNNNNr   )rO   rP   rQ   r   r   ry   r¬   rœ   r   ÚFloatTensorrX   rU   r   r   r	   r›   r?   rJ   rI   r¾   r¾     sú   € € € € € àØð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ð6
ð 6
àÔ# dÑ*ð6
ð œ tÑ+ð6
ð Ô&¨Ñ-ð	6
ð
  ™ð6
ð Ô(¨4Ñ/ð6
ð Ô  4Ñ'ð6
ð ˜$‘;ð6
ð ˜eœlÑ*ð6
ð Ð+Ô,ð6
ð 
 ð6
ð 6
ð 6
ñ „^ñ Ôð6
ð 6
ð 6
rJ   r¾   c                   ó   — e Zd ZdS )Ú!MiniCPM3ForSequenceClassificationNr^   r?   rJ   rI   rÐ   rÐ   P  r_   rJ   rÐ   )r!   r®   r¹   r¾   rÐ   )8rB   Úcollections.abcr   ry   Úhuggingface_hub.dataclassesr   Ú r   r²   Úcache_utilsr   Úmodeling_outputsr   r	   Úmodeling_utilsr
   r   Úprocessing_utilsr   Úutilsr   r   r   r   Ú deepseek_v2.modeling_deepseek_v2r   Úgemma3.modeling_gemma3r   Úllama.configuration_llamar   Úllama.modeling_llamar   r   r   r   r   r   r   r   r   r   Ú
get_loggerrO   Úloggerr!   r]   ra   rc   re   rg   rŸ   r®   r¹   r¾   rÐ   Ú__all__r?   rJ   rI   ú<module>rà      sÄ  ðð €€€Ø $Ð $Ð $Ð $Ð $Ð $à €€€Ø .Ð .Ð .Ð .Ð .Ð .à &Ð &Ð &Ð &Ð &Ð &Ø  Ð  Ð  Ð  Ð  Ð  Ø OÐ OÐ OÐ OÐ OÐ OÐ OÐ OØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RØ BÐ BÐ BÐ BÐ BÐ BØ BÐ BÐ BÐ BÐ BÐ BØ 3Ð 3Ð 3Ð 3Ð 3Ð 3ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð 
ˆÔ	˜HÑ	%Ô	%€ð €Ð0Ð1Ñ1Ô1ØðW6ð W6ð W6ð W6ð W6�[ñ W6ô W6ñ „ñ 2Ô1ðW6ðt	ð 	ð 	ð 	ð 	Ð"?ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	�lñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð2ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	�(ñ 	ô 	ð 	ð>)ð >)ð >)ð >)ð >)Ð+ñ >)ô >)ð >)ðB"ð "ð "ð "ð "Ð,ñ "ô "ð "ðJJð Jð Jð Jð JÐ2ñ Jô Jð Jð ð
ð 
ð 
ð 
ð 
�Jñ 
ô 
ñ „ð
ð ð9
ð 9
ð 9
ð 9
ð 9
Ð*ñ 9
ô 9
ñ „ð9
ðx	ð 	ð 	ð 	ð 	Ð(Fñ 	ô 	ð 	ðð ð €€€rJ   