§
    9²¸jô  ã                   ó~  — d Z ddlZddlmZ ddlmZ ddlmZm	Z	 ddl
mZ ddlmZmZ d	d
lmZ d	dlmZmZmZmZmZmZmZmZmZmZmZ  ej        e¦  «        Z ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z  G d„ de¦  «        Z! G d„ de¦  «        Z" G d„ de¦  «        Z# G d„ de¦  «        Z$ G d„ de¦  «        Z% G d„ de¦  «        Z& G d„ de¦  «        Z' G d„ de¦  «        Z( G d „ d!e¦  «        Z) G d"„ d#e¦  «        Z* G d$„ d%e¦  «        Z+ G d&„ d'ee)¦  «        Z, G d(„ d)e	e)¦  «        Z-g d*¢Z.dS )+u²  PyTorch A.X-K1 model (modular).

A.X-K1 is SK Telecom's Mixture-of-Experts LLM. Architecturally it is DeepSeek-V3 (Multi-head Latent
Attention + grouped sigmoid top-k MoE with a shared expert) with a single SK Telecom modification: an
extra `post_mlp_layernorm` applied to the MoE block output. Everything else â€” MLA, RoPE, the router,
the experts, and the model / causal-LM scaffolding â€” is inherited unchanged from DeepSeek-V3.
é    N)Ústricté   )Úinitialization)Ú GenericForSequenceClassificationÚGenericForTokenClassification)ÚPreTrainedModel)Úauto_docstringÚloggingé   )ÚDeepseekV3Config)ÚDeepseekV3AttentionÚDeepseekV3DecoderLayerÚDeepseekV3ExpertsÚDeepseekV3ForCausalLMÚDeepseekV3MLPÚDeepseekV3ModelÚDeepseekV3MoEÚDeepseekV3PreTrainedModelÚDeepseekV3RMSNormÚDeepseekV3RotaryEmbeddingÚDeepseekV3TopkRouterz
skt/A.X-K1)Ú
checkpointc                   ó”  — e Zd ZU dZdZddiZdZeed<   dZ	eed<   d	Z
eed
<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed <   d!Zeed"<   dZeed#<   dZeed$<   d%Zeed&<   d'Zeed(<   d)Zed*z  ed+<   d)Zee e         z  d*z  ed,<    e!¦   «         Z"d*S )-Ú
AXK1Configac  
    n_group (`int`, *optional*, defaults to 8):
        Number of groups for routed experts.
    topk_group (`int`, *optional*, defaults to 4):
        Number of selected groups per token (each token's experts are drawn from these groups).
    first_k_dense_replace (`int`, *optional*, defaults to 1):
        Number of leading layers that use a dense MLP; the rest use the MoE block.
    rope_interleave (`bool`, *optional*, defaults to `True`):
        Whether to use the interleaved rotary position embedding layout.

    ```python
    >>> from transformers import AXK1Config, AXK1Model

    >>> # Initializing an A.X-K1 style configuration
    >>> configuration = AXK1Config()

    >>> # Initializing a model from the configuration
    >>> model = AXK1Model(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```Úaxk1Únum_local_expertsÚn_routed_expertsi € Ú
vocab_sizei   Úhidden_sizei H  Úintermediate_sizei   Úmoe_intermediate_sizeé=   Únum_hidden_layersé@   Únum_attention_headsÚnum_key_value_headsé   Ún_shared_expertséÀ   g      @Úrouted_scaling_factori   Úkv_lora_ranki   Úq_lora_rankÚqk_rope_head_dimé€   Ú
v_head_dimÚqk_nope_head_dimé   Ún_groupé   Ú
topk_groupÚnum_experts_per_tokÚfirst_k_dense_replacei €  Úmax_position_embeddingsg�íµ ÷Æ°>Úrms_norm_epsik NÚbos_token_idÚeos_token_id)#Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚattribute_mapr   ÚintÚ__annotations__r   r    r!   r#   r%   r&   r(   r   r*   Úfloatr+   r,   r-   r/   r0   r2   r4   r5   r6   r7   r8   r9   r:   ÚlistÚAttributeErrorÚnum_mtp_layers© ó    úu/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/axk1/modular_axk1.pyr   r   0   sË  € € € € € € ðð ð. €JàÐ/ð€Mð €J�ÐÐÑØ€K�ÐÐÑØ"Ð�sÐ"Ð"Ñ"Ø!%Ð˜3Ð%Ð%Ñ%ØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø!Ð˜Ð!Ð!Ñ!ØÐ�cÐÐÑØÐ�cÐÐÑØ#&Ð˜5Ð&Ð&Ñ&Ø€L�#ÐÐÑØ€K�ÐÐÑØÐ�cÐÐÑØ€J�ÐÐÑØÐ�cÐÐÑØ€GˆSÐÐÑØ€J�ÐÐÑØ Ð˜Ð Ð Ñ Ø!"Ð˜3Ð"Ð"Ñ"Ø#(Ð˜SÐ(Ð(Ñ(Ø€L�%ÐÐÑØ%€L�#˜‘*Ð%Ð%Ñ%Ø+1€L�#˜˜Sœ	‘/ DÑ(Ð1Ð1Ñ1à#�^Ñ%Ô%€N€N€NrH   r   c                   ó   — e Zd ZdS )ÚAXK1RMSNormN©r;   r<   r=   rG   rH   rI   rK   rK   j   ó   € € € € € Ø€DrH   rK   c                   ó   — e Zd ZdS )ÚAXK1RotaryEmbeddingNrL   rG   rH   rI   rO   rO   n   rM   rH   rO   c                   ó   — e Zd ZdS )ÚAXK1MLPNrL   rG   rH   rI   rQ   rQ   r   rM   rH   rQ   c                   ó   — e Zd ZdS )ÚAXK1TopkRouterNrL   rG   rH   rI   rS   rS   v   rM   rH   rS   c                   ó   — e Zd ZdS )ÚAXK1ExpertsNrL   rG   rH   rI   rU   rU   z   rM   rH   rU   c                   óL   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚAXK1MoEa%  DeepSeek-V3 MoE with an extra `post_mlp_layernorm` on the block output (A.X-K1's single delta).

    The released checkpoints store the norm at the decoder-layer level (`post_mlp_layernorm.*`); the
    checkpoint conversion mapping renames it into `mlp.post_mlp_layernorm.*` at load time.
    Úconfigc                 óŠ   •— t          ¦   «                              |¦  «         t          |j        |j        ¬¦  «        | _        d S )N)Úeps)ÚsuperÚ__init__rK   r   r8   Úpost_mlp_layernorm)ÚselfrX   Ú	__class__s     €rI   r\   zAXK1MoE.__init__…   s<   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý"-¨fÔ.@ÀfÔFYÐ"ZÑ"ZÔ"ZˆÔÐÐrH   Úhidden_statesÚreturnc                 ó   — |}|j         }|                      |¦  «        \  }}}|                     d|j         d         ¦  «        } |                      |||¦  «        j        |Ž }||                      |¦  «        z   }|                      |¦  «        S )Néÿÿÿÿ)ÚshapeÚgateÚviewÚexpertsÚshared_expertsr]   )r^   r`   Ú	residualsÚ
orig_shapeÚ_Útopk_weightsÚtopk_indicess          rI   ÚforwardzAXK1MoE.forward‰   s�   € Ø!ˆ	Ø"Ô(ˆ
Ø(,¯	ª	°-Ñ(@Ô(@Ñ%ˆˆ<˜Ø%×*Ò*¨2¨}Ô/BÀ2Ô/FÑGÔGˆØT˜Ÿš ]°LÀ,ÑOÔOÔTÐV`ÐaˆØ%¨×(;Ò(;¸IÑ(FÔ(FÑFˆØ×&Ò& }Ñ5Ô5Ð5rH   )
r;   r<   r=   r>   r   r\   ÚtorchÚTensorrn   Ú__classcell__)r_   s   @rI   rW   rW   ~   s{   ø€ € € € € ðð ð[˜zð [ð [ð [ð [ð [ð [ð6 U¤\ð 6°e´lð 6ð 6ð 6ð 6ð 6ð 6ð 6ð 6rH   rW   c                   ó   — e Zd ZdS )ÚAXK1AttentionNrL   rG   rH   rI   rs   rs   “   rM   rH   rs   c                   ó   — e Zd ZdS )ÚAXK1DecoderLayerNrL   rG   rH   rI   ru   ru   —   rM   rH   ru   c                   óD   — e Zd ZdgZ ej        ¦   «         d„ ¦   «         ZdS )ÚAXK1PreTrainedModelÚinv_freqc                 ó¢  — t          j        | |¦  «         t          |t          ¦  «        rAt	          j        |j        d| j        j        ¬¦  «         t	          j	        |j
        ¦  «         d S t          |t          ¦  «        rNt	          j        |j        d| j        j        ¬¦  «         t	          j        |j        d| j        j        ¬¦  «         d S d S )Ng        )ÚmeanÚstd)r   Ú_init_weightsÚ
isinstancerS   ÚinitÚnormal_ÚweightrX   Úinitializer_rangeÚzeros_Úe_score_correction_biasrU   Úgate_up_projÚ	down_proj)r^   Úmodules     rI   r|   z!AXK1PreTrainedModel._init_weightsž   sÂ   € åÔ% d¨FÑ3Ô3Ð3Ý�f�nÑ-Ô-ð 	XÝŒL˜œ¨S°d´kÔ6SÐTÑTÔTÐTÝŒK˜Ô6Ñ7Ô7Ð7Ð7Ð7Ý˜¥Ñ,Ô,ð 	XÝŒL˜Ô,°3¸D¼KÔ<YÐZÑZÔZÐZÝŒL˜Ô)°¸¼Ô9VÐWÑWÔWÐWÐWÐWð	Xð 	XrH   N)r;   r<   r=   Ú"_keys_to_ignore_on_load_unexpectedro   Úno_gradr|   rG   rH   rI   rw   rw   ›   sB   € € € € € Ø*4¨Ð&à€U„]�_„_ðXð Xñ „_ðXð Xð XrH   rw   c                   ó   — e Zd ZdS )Ú	AXK1ModelNrL   rG   rH   rI   rŠ   rŠ   ©   rM   rH   rŠ   c                   ó   — e Zd ZdS )ÚAXK1ForCausalLMNrL   rG   rH   rI   rŒ   rŒ   ­   rM   rH   rŒ   c                   ó   — e Zd ZdS )ÚAXK1ForSequenceClassificationNrL   rG   rH   rI   rŽ   rŽ   ±   rM   rH   rŽ   c                   ó   — e Zd ZdS )ÚAXK1ForTokenClassificationNrL   rG   rH   rI   r�   r�   µ   rM   rH   r�   )r   rw   rŠ   rŒ   rŽ   r�   )/r>   ro   Úhuggingface_hub.dataclassesr   Ú r   r~   Úmodeling_layersr   r   Úmodeling_utilsr   Úutilsr	   r
   Ú%deepseek_v3.configuration_deepseek_v3r   Ú deepseek_v3.modeling_deepseek_v3r   r   r   r   r   r   r   r   r   r   r   Ú
get_loggerr;   Úloggerr   rK   rO   rQ   rS   rU   rW   rs   ru   rw   rŠ   rŒ   rŽ   r�   Ú__all__rG   rH   rI   ú<module>r›      sÇ  ððð ð €€€Ø .Ð .Ð .Ð .Ð .Ð .à &Ð &Ð &Ð &Ð &Ð &Ø ^Ð ^Ð ^Ð ^Ð ^Ð ^Ð ^Ð ^Ø -Ð -Ð -Ð -Ð -Ð -Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ø DÐ DÐ DÐ DÐ DÐ Dðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð 
ˆÔ	˜HÑ	%Ô	%€ð €˜<Ð(Ñ(Ô(Øð5&ð 5&ð 5&ð 5&ð 5&Ð!ñ 5&ô 5&ñ „ñ )Ô(ð5&ðp	ð 	ð 	ð 	ð 	Ð#ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð3ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	ˆmñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð)ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð#ñ 	ô 	ð 	ð6ð 6ð 6ð 6ð 6ˆmñ 6ô 6ð 6ð*	ð 	ð 	ð 	ð 	Ð'ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð-ñ 	ô 	ð 	ðXð Xð Xð Xð XÐ3ñ Xô Xð Xð	ð 	ð 	ð 	ð 	�ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð+ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð$DÐFYñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð!>Ð@Sñ 	ô 	ð 	ðð ð €€€rH   