§
    9²¸j$F  ã                   ó>  — d Z ddlmZ ddlZddlmc mZ ddlm	Z	 ddlmZ ddl
mZ ddlmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZmZ ddlmZ ddlmZmZ ddlmZ ddl m!Z!m"Z"m#Z# ddl$m%Z% ddl&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1  ej2        e3¦  «        Z4 ed¬¦  «        e	 G d„ de%¦  «        ¦   «         ¦   «         Z5 G d„ de!¦  «        Z6 G d„ de¦  «        Z7 G d„ dej8        ¦  «        Z9 G d„ de/¦  «        Z: G d „ d!e+¦  «        Z; G d"„ d#e0¦  «        Z< G d$„ d%e)¦  «        Z= G d&„ d'e-¦  «        Z> G d(„ d)e'¦  «        Z? G d*„ d+e(¦  «        Z@ G d,„ d-e.¦  «        ZA G d.„ d/e,¦  «        ZB G d0„ d1e*¦  «        ZC G d2„ d3eeA¦  «        ZD G d4„ d5eeA¦  «        ZEg d6¢ZFdS )7zPyTorch A.X-K2 model.é    )ÚCallableN)Ústrict)Únné   )Úinitialization)ÚCache)ÚPreTrainedConfig)ÚFlashAttentionKwargs)Ú GenericForSequenceClassificationÚGenericForTokenClassification)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Úauto_docstringÚloggingé   )ÚCLIPMLP)ÚDeepseekV3RMSNormÚapply_rotary_pos_emb_interleaveÚeager_attention_forward)ÚDeepseekV32Config)ÚDeepseekV32AttentionÚDeepseekV32DecoderLayerÚDeepseekV32ExpertsÚDeepseekV32ForCausalLMÚDeepseekV32IndexerÚDeepseekV32ModelÚDeepseekV32MoEÚDeepseekV32PreTrainedModelÚDeepseekV32RotaryEmbeddingÚDeepseekV32TopkRouterÚyarn_apply_mscalez
skt/A.X-K2)Ú
checkpointc                   ó  — e Zd ZU dZdZdZeed<   dZeed<   dZ	eed<   d	Z
eed
<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed <   d!Zeed"<   d#Zed$z  ed%<   d#Zeee         z  d$z  ed&<   dZeed'<   dZeed(<   d)Zeed*<   d)Z eed+<   d$Z!ed$z  ed,<   d$Z"ed$z  ed-<    e#¦   «         Z$ e#¦   «         Z%d.d/d.d0d1d0d2d.d.d0d.d.d0d3œZ&d4„ Z'd5„ Z(d$S )6Ú
AXK2Configaå  
    n_group (`int`, *optional*):
        Number of expert groups for grouped routing, used by the larger A.X-K2 releases. `None` (the
        A.X-K2-Light default) routes over all experts without group restriction.
    topk_group (`int`, *optional*):
        Number of expert groups the top-k selection is restricted to when `n_group` is set.
    mlp_layer_types (`list`, *optional*):
        MLP type pattern for each layer (`"dense"` or `"sparse"`). Derived from the (legacy) kwargs
        `first_k_dense_replace` / `moe_layer_freq` when not provided.
    index_topk (`int`, *optional*, defaults to 2048):
        Number of top tokens selected by the indexer for sparse attention.
    index_head_dim (`int`, *optional*, defaults to 128):
        Head dimension for the indexer projections (DSA).
    index_n_heads (`int`, *optional*, defaults to 16):
        Number of heads for the indexer projections (DSA).
    gated_norm_rank (`int`, *optional*, defaults to 16):
        Bottleneck rank for the low-rank input-dependent gate used by `AXK2GatedRMSNorm`. The gate wraps
        `input_layernorm` on every layer and `post_attention_layernorm` on MoE layers.

    ```python
    >>> from transformers import AXK2Config, AXK2Model

    >>> # Initializing an A.X-K2 style configuration
    >>> configuration = AXK2Config()

    >>> # Initializing a model from the configuration
    >>> model = AXK2Model(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```Úaxk2i € Ú
vocab_sizei   Úhidden_sizei   Úintermediate_sizei   Úmoe_intermediate_sizeé0   Únum_hidden_layersé    Únum_attention_headsÚnum_key_value_headsé   Ún_shared_expertsé€   Ún_routed_expertsg      @Úrouted_scaling_factorÚkv_lora_ranki€  Úq_lora_rankÚqk_rope_head_dimé@   Ú
v_head_dimÚqk_nope_head_dimé   Únum_experts_per_toki   Úmax_position_embeddingsg�íµ ÷Æ°>Úrms_norm_epsik NÚbos_token_idÚeos_token_idÚ
index_topkÚindex_head_dimé   Úindex_n_headsÚgated_norm_rankÚn_groupÚ
topk_groupÚcolwiseÚmla_kv_a_projÚrowwiseÚpacked_colwiseÚmoe_tp_experts)zlayers.*.self_attn.q_gate_projz%layers.*.self_attn.kv_a_proj_with_mqazlayers.*.self_attn.kv_b_projzlayers.*.self_attn.o_projz!layers.*.mlp.experts.gate_up_projzlayers.*.mlp.experts.down_projzlayers.*.mlp.expertsz%layers.*.mlp.shared_experts.gate_projz#layers.*.mlp.shared_experts.up_projz%layers.*.mlp.shared_experts.down_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projc                 óN  ‡‡— | j         | j        z   | _        | j        | _        | j        €R|                     dd¦  «        Š|                     dd¦  «        Šˆˆfd„t          | j        ¦  «        D ¦   «         | _        | j        €dg| j        z  | _        t          j
        | fi |¤Ž d S )NÚfirst_k_dense_replacer0   Úmoe_layer_freqc                 ó6   •— g | ]}|‰k    r|‰z  d k    rdnd‘ŒS )r   ÚsparseÚdense© )Ú.0ÚirN   rO   s     €€úu/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/axk2/modular_axk2.pyú
<listcomp>z,AXK2Config.__post_init__.<locals>.<listcomp>—   sI   ø€ ð $ð $ð $àð Ð!6Ò6Ð6¸1¸~Ñ;MÐQRÒ;RÐ;R��ÐX_ð$ð $ð $ó    Údeepseek_sparse_attention)r:   r7   Úqk_head_dimÚhead_dimÚmlp_layer_typesÚpopÚranger,   Úlayer_typesr	   Ú__post_init__)ÚselfÚkwargsrN   rO   s     @@rV   r`   zAXK2Config.__post_init__�   sÏ   øø€ ØÔ0°4Ô3HÑHˆÔð Ô-ˆŒð ÔÐ'Ø$*§J¢JÐ/FÈÑ$JÔ$JÐ!Ø#ŸZšZÐ(8¸!Ñ<Ô<ˆNð$ð $ð $ð $ð $å˜tÔ5Ñ6Ô6ð$ñ $ô $ˆDÔ ð ÔÐ#Ø ;Ð<¸tÔ?UÑUˆDÔåÔ& tÐ6Ð6¨vÐ6Ð6Ð6Ð6Ð6rX   c                 óÂ  — t          j        | ¦  «         | j        �| j        dk    rt          d| j        › d�¦  «        ‚| j        d u | j        d u k    r t          d| j        › d| j        › d�¦  «        ‚| j        �c| j        | j        z  dk    r t          d| j        › d| j        › d�¦  «        ‚| j        | j        k    r"t          d	| j        › d
| j        › d�¦  «        ‚d S d S )Nr   zkA.X-K2 requires a positive `q_lora_rank` (the indexer and output gate read the query LoRA bottleneck), got ú.zc`n_group` and `topk_group` must be set together (both `None` for non-grouped routing), got n_group=z, topk_group=z`n_routed_experts` (z") must be divisible by `n_group` (z).z`topk_group` (z) cannot exceed `n_group` ()r	   Úvalidate_architecturer6   Ú
ValueErrorrF   rG   r3   )ra   s    rV   re   z AXK2Config.validate_architecture¢   sG  € ÝÔ.¨tÑ4Ô4Ð4ØÔÐ# tÔ'7¸1Ò'<Ð'<Ýð8Ø$(Ô$4ð8ð 8ð 8ñô ð ð ŒL˜DÐ  d¤o¸Ð&=Ò>Ð>ÝðIØœ<ðIð IØ6:´oðIð Ið Iñô ð ð Œ<Ð#ØÔ$ t¤|Ñ3°qÒ8Ð8Ý Øt¨4Ô+@ÐtÐtÐdhÔdpÐtÐtÐtñô ð ð Œ ¤Ò-Ð-Ý Ð!n°$´/Ð!nÐ!nÐ^bÔ^jÐ!nÐ!nÐ!nÑoÔoÐoð $Ð#ð
 .Ð-rX   ))Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typer'   ÚintÚ__annotations__r(   r)   r*   r,   r.   r/   r1   r3   r4   Úfloatr5   r6   r7   r9   r:   r<   r=   r>   r?   r@   ÚlistrA   rB   rD   rE   rF   rG   ÚAttributeErrorrN   Úmlp_biasÚbase_model_tp_planr`   re   rS   rX   rV   r%   r%   8   sc  € € € € € € ðð ð@ €Jà€J�ÐÐÑØ€K�ÐÐÑØ!Ð�sÐ!Ð!Ñ!Ø!$Ð˜3Ð$Ð$Ñ$ØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø!Ð˜Ð!Ð!Ñ!ØÐ�cÐÐÑØÐ�cÐÐÑØ#&Ð˜5Ð&Ð&Ñ&Ø€L�#ÐÐÑØ€K�ÐÐÑØÐ�cÐÐÑØ€J�ÐÐÑØÐ�cÐÐÑØ Ð˜Ð Ð Ñ Ø#)Ð˜SÐ)Ð)Ñ)Ø€L�%ÐÐÑØ%€L�#˜‘*Ð%Ð%Ñ%Ø+1€L�#˜˜Sœ	‘/ DÑ(Ð1Ð1Ñ1Ø€J�ÐÐÑØ€N�CÐÐÑØ€M�3ÐÐÑØ€O�SÐÐÑð €GˆS�4‰ZÐÐÑØ!€J��d‘
Ð!Ð!Ñ!à*˜NÑ,Ô,ÐØˆ~ÑÔ€Hð +4Ø1@Ø(1Ø%.Ø-=Ø*3Ø 0Ø1:Ø/8Ø1:Ø"+Ø )Ø"+ðð Ðð 7ð 7ð 7ð*pð pð pð pð prX   r%   c                   ó   — e Zd ZdS )ÚAXK2RMSNormN©rg   rh   ri   rS   rX   rV   rt   rt   ·   ó   € € € € € Ø€DrX   rt   c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚAXK2GateMLPÚconfigc                 ó  •— t          ¦   «                              |¦  «         t          j        ¦   «         | _        t          j        |j        |j        d¬¦  «        | _        t          j        |j        |j        d¬¦  «        | _	        d S )NF©Úbias)
ÚsuperÚ__init__r   ÚSiLUÚactivation_fnÚLinearr(   rE   Úfc1Úfc2©ra   ry   Ú	__class__s     €rV   r~   zAXK2GateMLP.__init__¼   si   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ÝœW™YœYˆÔÝ”9˜VÔ/°Ô1GÈeÐTÑTÔTˆŒÝ”9˜VÔ3°VÔ5GÈeÐTÑTÔTˆŒˆˆrX   )rg   rh   ri   r%   r~   Ú__classcell__©r…   s   @rV   rx   rx   »   sO   ø€ € € € € ðU˜zð Uð Uð Uð Uð Uð Uð Uð Uð Uð UrX   rx   c                   óL   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚAXK2GatedRMSNormz—RMSNorm followed by a low-rank input-dependent sigmoid gate (Megatron `GatedNormWrapper`):

    y = RMSNorm(x)
    return y * sigmoid(gate_mlp(y))
    ry   c                 ó°   •— t          ¦   «                              ¦   «          t          |j        |j        ¬¦  «        | _        t          |¦  «        | _        d S )N©Úeps)r}   r~   rt   r(   r>   Únormrx   Úmlpr„   s     €rV   r~   zAXK2GatedRMSNorm.__init__Ê   sF   ø€ Ý‰Œ×ÒÑÔÐÝ Ô 2¸Ô8KÐLÑLÔLˆŒ	Ý˜vÑ&Ô&ˆŒˆˆrX   ÚxÚreturnc                 óÔ   — |                       |¦  «        }|t          j        |                      |¦  «                             ¦   «         ¦  «        z                       |j        ¦  «        S )N)r�   ÚtorchÚsigmoidrŽ   rn   ÚtoÚdtype)ra   r�   Úys      rV   ÚforwardzAXK2GatedRMSNorm.forwardÏ   sL   € Ø�IŠI�a‰LŒLˆØ•E”M $§(¢(¨1¡+¤+×"3Ò"3Ñ"5Ô"5Ñ6Ô6Ñ6×:Ò:¸1¼7ÑCÔCÐCrX   )
rg   rh   ri   rj   r%   r~   r’   ÚTensorr—   r†   r‡   s   @rV   r‰   r‰   Ã   s~   ø€ € € € € ðð ð'˜zð 'ð 'ð 'ð 'ð 'ð 'ð
D˜œð D¨%¬,ð Dð Dð Dð Dð Dð Dð Dð DrX   r‰   c                   ó   — e Zd ZdS )ÚAXK2RotaryEmbeddingNru   rS   rX   rV   rš   rš   Ô   rv   rX   rš   c                   ó   — e Zd ZdS )ÚAXK2IndexerNru   rS   rX   rV   rœ   rœ   Ø   rv   rX   rœ   c                   ó   — e Zd Zd„ Zd„ ZdS )ÚAXK2TopkRouterc                 óN  — |                      d| j        | j        | j        z  ¦  «                             dd¬¦  «        d                              d¬¦  «        }t          j        || j        dd¬¦  «        d         }t          j        |¦  «        }|                     d|d¦  «         | 	                    d¦  «         
                    d| j        | j        | j        z  ¦  «                             d| j        ¦  «        }|                     |                     ¦   «          d¦  «        }|S )	z$Apply DeepSeek style grouped scoringéÿÿÿÿr   ©Údimr   F©Úkr¢   Úsortedr0   ç        )ÚviewÚ	num_groupÚnum_expertsÚtopkÚsumr’   rG   Ú
zeros_likeÚscatter_Ú	unsqueezeÚexpandÚreshapeÚmasked_fillÚbool)ra   Úscores_for_choiceÚgroup_scoresÚ	group_idxÚ
group_maskÚ
score_masks         rV   Úapply_group_scoringz"AXK2TopkRouter.apply_group_scoringÝ   s  € ð ×"Ò" 2 t¤~°tÔ7GÈ4Ì>Ñ7YÑZÔZßŠT�!˜ˆT‰_Œ_˜Qô çŠS�RˆS‰[Œ[ð 	õ
 ”J˜|¨t¬ÀBÈuÐUÑUÔUÐVWÔXˆ	ÝÔ% lÑ3Ô3ˆ
Ø×Ò˜A˜y¨!Ñ,Ô,Ð,à× Ò  Ñ$Ô$ßŠV�B˜œ¨Ô(8¸D¼NÑ(JÑKÔKßŠW�R˜Ô)Ñ*Ô*ð 	ð .×9Ò9¸:¿?º?Ñ;LÔ;LÐ:LÈcÑRÔRÐØ Ð rX   c                 ó0  — |                      d| j        ¦  «        }t          j        |                     t
          j        ¦  «        | j                             t
          j        ¦  «        ¦  «        }|                     ¦   «         }|| j	        z   }| j
        �|                      |¦  «        }t          j        || j        dd¬¦  «        d         }|                     d|¦  «        }| j        r|                     dd¬¦  «        dz   }||z  }|| j        z  }|||fS )Nr    Fr£   r0   T)r¢   Úkeepdimg#B’¡œÇ;)r§   Ú
hidden_dimÚFÚlinearÚtyper’   Úfloat32Úweightr“   Úe_score_correction_biasr¨   r¸   rª   Útop_kÚgatherÚnorm_topk_probr«   r4   )ra   Úhidden_statesÚrouter_logitsÚscoresr³   Útopk_indicesÚtopk_weightsÚdenominators           rV   r—   zAXK2TopkRouter.forwardð   s  € Ø%×*Ò*¨2¨t¬Ñ?Ô?ˆÝœ ×!3Ò!3µE´MÑ!BÔ!BÀDÄK×DTÒDTÕUZÔUbÑDcÔDcÑdÔdˆØ×&Ò&Ñ(Ô(ˆØ" TÔ%AÑAÐð Œ>Ð%Ø $× 8Ò 8Ð9JÑ KÔ KÐå”zÐ"3°t´zÀrÐRWÐXÑXÔXÐYZÔ[ˆØ—}’} Q¨Ñ5Ô5ˆØÔð 	(Ø&×*Ò*¨r¸4Ð*Ñ@Ô@À5ÑHˆKØ˜KÑ'ˆLØ# dÔ&@Ñ@ˆØ˜l¨LÐ8Ð8rX   N)rg   rh   ri   r¸   r—   rS   rX   rV   rž   rž   Ü   s2   € € € € € ð!ð !ð !ð&9ð 9ð 9ð 9ð 9rX   rž   c                   ó   — e Zd ZdS )ÚAXK2ExpertsNru   rS   rX   rV   rÌ   rÌ     rv   rX   rÌ   c                   ó   — e Zd ZdS )ÚAXK2MoENru   rS   rX   rV   rÎ   rÎ     rv   rX   rÎ   c                   óÌ   — e Zd Zdedefd„Z	 	 ddej        deej        ej        f         dej        de	dz  d	ej        dz  d
e
e         deej        ej        dz  f         fd„ZdS )ÚAXK2Attentionry   Ú	layer_idxc                 óP  — t           j                             | ¦  «         || _        || _        |j        | _        |j        | _        |j        | _        |j	        | _	        |j
        | _
        |j        | _        |j        | _        |j        | _        |j        |j
        z   | _        |j        |j        z  | _        d| _        t          j        | j        | j	        |j        ¬¦  «        | _        t+          | j	        ¦  «        | _        t          j        d| j	        z  | j        | j        | j        z   z  d¬¦  «        | _        t          j        | j        | j        | j
        z   |j        ¬¦  «        | _        t+          | j        ¦  «        | _        t          j        | j        | j        | j        | j        z   z  d¬¦  «        | _        t          j        | j        | j        z  | j        |j        ¬¦  «        | _        t9          |j        | j        dz  ¦  «        | _        t?          ||¦  «        | _         d S )NTr{   r   Fg      à¿)!r   ÚModuler~   ry   rÑ   Úattention_dropoutr(   r.   Ú	num_headsr6   r7   r5   r9   r:   rZ   r/   Únum_key_value_groupsÚ	is_causalr�   Úattention_biasÚq_a_projrt   Úq_a_layernormÚq_gate_projÚkv_a_proj_with_mqaÚkv_a_layernormÚ	kv_b_projÚo_projr"   Úrope_parametersÚscalingrœ   Úindexer)ra   ry   rÑ   s      rV   r~   zAXK2Attention.__init__  sü  € Ý
Œ	×Ò˜4Ñ Ô Ð ØˆŒØ"ˆŒØ!'Ô!9ˆÔØ!Ô-ˆÔØÔ3ˆŒà!Ô-ˆÔØ &Ô 7ˆÔØ"Ô/ˆÔØ Ô+ˆŒØ &Ô 7ˆÔØ!Ô2°VÔ5LÑLˆÔØ$*Ô$>À&ÔB\Ñ$\ˆÔ!àˆŒåœ	 $Ô"2°DÔ4DÈ6ÔK`ÐaÑaÔaˆŒÝ(¨Ô)9Ñ:Ô:ˆÔåœ9Ø�Ô Ñ  $¤.°DÔ4DÀtÄÑ4VÑ"WÐ^cð
ñ 
ô 
ˆÔõ #%¤)ØÔØÔ Ô 5Ñ5ØÔ&ð#
ñ #
ô #
ˆÔõ
 *¨$Ô*;Ñ<Ô<ˆÔÝœØÔØŒN˜dÔ3°d´oÑEÑFØð
ñ 
ô 
ˆŒõ ”iØŒN˜Tœ_Ñ,ØÔØÔ&ð
ñ 
ô 
ˆŒõ )¨Ô)?ÀÔAQÐVZÑA[Ñ\Ô\ˆŒÝ" 6¨9Ñ5Ô5ˆŒˆˆrX   NrÅ   Úposition_embeddingsÚattention_maskÚpast_key_valuesÚposition_idsrb   r�   c                 ó|  — |j         d d…         \  }}||d| j        | j        z   f}	|                      |¦  «        }
|                      |
¦  «        }|                      t          j        ||
gd¬¦  «        ¦  «                             |	¦  «        }t          j	        || j        | j        gd¬¦  «        \  }}| 
                    dd¦  «        }|                     ||d¦  «        }t          j	        || j        | j        gd¬¦  «        \  }}|                      |¦  «        }t          j	        || j        | j        gd¬¦  «        \  }}|                      |¦  «                             |d|| j        ¦  «        }|                     |d|| j        ¦  «        }|\  }}t#          ||||¦  «        \  }}t          j        ||fd¬¦  «        }|                      ||¦  «        \  }}|�|                     ||| j        ¦  «        \  }}|                      ||||d d …dd d …d d …f         ||¬¦  «        }d }| j        j        dv r¸|                     |||j         d         ft          j        ¬¦  «                             d|                     ¦   «         d	¦  «                             d¦  «        }|j        t          j        k    r|| z  }n5|                     |t          j        |j        ¦  «        j         ¦  «        }n|}tC          j"        | j        j        tF          ¦  «        } || ||||f| j$        sd
n| j%        | j&        |dœ|¤Ž\  }}|                     ||d¦  «         '                    ¦   «         }|t          j(        | )                    ¦   «         ¦  «        z   *                    |j        ¦  «        }|  +                    |¦  «        }||fS )Nr    r¡   r0   r   r   )rå   )ÚeagerÚsdpa)r•   Fr¦   )Údropoutrá   Úindices),ÚshaperZ   r9   rÙ   rÚ   rÛ   r’   Úcatr§   ÚsplitÚ	transposer°   r:   r7   rÜ   r5   rÝ   r   Ú	expand_kvÚupdaterÑ   râ   ry   Ú_attn_implementationÚnew_onesr²   ÚscatterÚlongr®   r•   r±   ÚfinfoÚminr   Úget_interfacer   ÚtrainingrÔ   rá   Ú
contiguousr“   rn   r”   rß   )ra   rÅ   rã   rä   rå   ræ   rb   Ú
batch_sizeÚ
seq_lengthÚquery_gate_shapeÚq_compressedÚq_residÚquery_gate_statesÚquery_statesÚgate_statesÚq_passÚq_rotÚcompressed_kvÚkv_passÚk_rotÚk_passÚcosÚsinÚ
key_statesÚvalue_statesrÈ   Úsparse_indicesÚ
index_maskÚattention_interfaceÚattn_outputÚattn_weightss                                  rV   r—   zAXK2Attention.forward:  sé  € ð "/Ô!4°S°b°SÔ!9Ñˆ
�JØ&¨
°B¸Ô8HÈ4Ì?Ñ8ZÐ[Ðð —}’} ]Ñ3Ô3ˆØ×$Ò$ \Ñ2Ô2ˆà ×,Ò,­U¬Y¸ÀÐ7NÐTVÐ-WÑ-WÔ-WÑXÔX×]Ò]Ð^nÑoÔoÐÝ$)¤KÐ0AÀDÔDTÐVZÔVeÐCfÐlnÐ$oÑ$oÔ$oÑ!ˆ�kØ#×-Ò-¨a°Ñ3Ô3ˆØ!×)Ò)¨*°jÀ"ÑEÔEˆåœ L°4Ô3HÈ$ÔJ_Ð2`ÐfhÐiÑiÔi‰ˆ�à×/Ò/°Ñ>Ô>ˆÝœ ]°TÔ5FÈÔH]Ð4^ÐdfÐgÑgÔg‰ˆ�à×$Ò$ WÑ-Ô-×2Ò2°:¸qÀ*ÈdÔN_Ñ`Ô`ˆØ—
’
˜: q¨*°dÔ6KÑLÔLˆà&‰ˆˆSÝ6°u¸eÀSÈ#ÑNÔN‰ˆˆuå”y &¨% °bÐ9Ñ9Ô9ˆà#'§>¢>°&¸%Ñ#@Ô#@Ñ ˆ
�Lð Ð&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜ð —|’|ØØØØ˜1˜1˜1˜a    A A A˜:Ô&ØØ+ð $ñ 
ô 
ˆð ˆØŒ;Ô+Ð/@Ð@Ð@à×%Ò% z°:¸zÔ?OÐPQÔ?RÐ&SÕ[`Ô[eÐ%ÑfÔfß’˜˜\×.Ò.Ñ0Ô0°%Ñ8Ô8ß’˜1‘”ð ð Ô#¥u¤zÒ1Ð1Ø!/°:°+Ñ!=��à!/×!;Ò!;¸JÍÌÐTaÔTgÑHhÔHhÔHlÑ!mÔ!m��à)ˆNå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð  $œ}ÐH�C�C°$Ô2HØ”LØ"ð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð "×)Ò)¨*°jÀ"ÑEÔE×PÒPÑRÔRˆà"¥U¤]°;×3DÒ3DÑ3FÔ3FÑ%GÔ%GÑG×KÒKÈKÔL]Ñ^Ô^ˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(rX   )NN)rg   rh   ri   r%   rl   r~   r’   r˜   Útupler   r   r
   r—   rS   rX   rV   rÐ   rÐ     sà   € € € € € ð,6˜zð ,6°cð ,6ð ,6ð ,6ð ,6ðf )-Ø,0ðT)ð T)à”|ðT)ð # 5¤<°´Ð#=Ô>ðT)ð œð	T)ð
  ™ðT)ð ”l TÑ)ðT)ð Ð-Ô.ðT)ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ðT)ð T)ð T)ð T)ð T)ð T)rX   rÐ   c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚAXK2DecoderLayerry   rÑ   c                 óô   •— t          ¦   «                              ||¦  «         t          |¦  «        | _        |j        |         dk    rt          |¦  «        nt          |j        |j        ¬¦  «        | _        d S )NrQ   r‹   )	r}   r~   r‰   Úinput_layernormr\   rt   r(   r>   Úpost_attention_layernorm)ra   ry   rÑ   r…   s      €rV   r~   zAXK2DecoderLayer.__init__’  sv   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+Ý/°Ñ7Ô7ˆÔð Ô% iÔ0°HÒ<Ð<õ ˜VÑ$Ô$Ð$å˜VÔ/°VÔ5HÐIÑIÔIð 	Ô%Ð%Ð%rX   )rg   rh   ri   r%   rl   r~   r†   r‡   s   @rV   r  r  ‘  sK   ø€ € € € € ð
˜zð 
°cð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
rX   r  c                   óD   — e Zd ZdgZ ej        ¦   «         d„ ¦   «         ZdS )ÚAXK2PreTrainedModelÚinv_freqc                 ó¢  — t          j        | |¦  «         t          |t          ¦  «        rAt	          j        |j        d| j        j        ¬¦  «         t	          j	        |j
        ¦  «         d S t          |t          ¦  «        rNt	          j        |j        d| j        j        ¬¦  «         t	          j        |j        d| j        j        ¬¦  «         d S d S )Nr¦   )ÚmeanÚstd)r   Ú_init_weightsÚ
isinstancerž   ÚinitÚnormal_rÀ   ry   Úinitializer_rangeÚzeros_rÁ   rÌ   Úgate_up_projÚ	down_proj)ra   Úmodules     rV   r  z!AXK2PreTrainedModel._init_weightsŸ  sÂ   € åÔ% d¨FÑ3Ô3Ð3Ý�f�nÑ-Ô-ð 	XÝŒL˜œ¨S°d´kÔ6SÐTÑTÔTÐTÝŒK˜Ô6Ñ7Ô7Ð7Ð7Ð7Ý˜¥Ñ,Ô,ð 	XÝŒL˜Ô,°3¸D¼KÔ<YÐZÑZÔZÐZÝŒL˜Ô)°¸¼Ô9VÐWÑWÔWÐWÐWÐWð	Xð 	XrX   N)rg   rh   ri   Ú"_keys_to_ignore_on_load_unexpectedr’   Úno_gradr  rS   rX   rV   r  r  œ  sB   € € € € € Ø*4¨Ð&à€U„]�_„_ðXð Xñ „_ðXð Xð XrX   r  c                   ó   — e Zd ZdS )Ú	AXK2ModelNru   rS   rX   rV   r*  r*  ª  rv   rX   r*  c                   ó   — e Zd ZdS )ÚAXK2ForCausalLMNru   rS   rX   rV   r,  r,  ®  rv   rX   r,  c                   ó   — e Zd ZdS )ÚAXK2ForSequenceClassificationNru   rS   rX   rV   r.  r.  ²  rv   rX   r.  c                   ó   — e Zd ZdS )ÚAXK2ForTokenClassificationNru   rS   rX   rV   r0  r0  ¶  rv   rX   r0  )r%   r  r*  r,  r.  r0  )Grj   Úcollections.abcr   r’   Útorch.nn.functionalr   Ú
functionalr¼   Úhuggingface_hub.dataclassesr   Ú r   r   Úcache_utilsr   Úconfiguration_utilsr	   Úmodeling_flash_attention_utilsr
   Úmodeling_layersr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   Úclip.modeling_clipr   Ú deepseek_v3.modeling_deepseek_v3r   r   r   Ú'deepseek_v32.configuration_deepseek_v32r   Ú"deepseek_v32.modeling_deepseek_v32r   r   r   r   r   r   r   r   r    r!   r"   Ú
get_loggerrg   Úloggerr%   rt   rx   rÓ   r‰   rš   rœ   rž   rÌ   rÎ   rÐ   r  r  r*  r,  r.  r0  Ú__all__rS   rX   rV   ú<module>rD     s  ðð Ð à $Ð $Ð $Ð $Ð $Ð $à €€€Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø .Ð .Ð .Ð .Ð .Ð .Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø  Ð  Ð  Ð  Ð  Ð  Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø BÐ BÐ BÐ BÐ BÐ BØ ^Ð ^Ð ^Ð ^Ð ^Ð ^Ð ^Ð ^Ø FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ø (Ð (Ð (Ð (Ð (Ð (ðð ð ð ð ð ð ð ð ð ð
 HÐ GÐ GÐ GÐ GÐ Gðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð 
ˆÔ	˜HÑ	%Ô	%€ð €˜<Ð(Ñ(Ô(Øðzpð zpð zpð zpð zpÐ"ñ zpô zpñ „ñ )Ô(ðzpðz	ð 	ð 	ð 	ð 	Ð#ñ 	ô 	ð 	ðUð Uð Uð Uð U�'ñ Uô Uð UðDð Dð Dð Dð D�r”yñ Dô Dð Dð"	ð 	ð 	ð 	ð 	Ð4ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð$ñ 	ô 	ð 	ð$9ð $9ð $9ð $9ð $9Ð*ñ $9ô $9ð $9ðN	ð 	ð 	ð 	ð 	Ð$ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	ˆnñ 	ô 	ð 	ðC)ð C)ð C)ð C)ð C)Ð(ñ C)ô C)ð C)ðL
ð 
ð 
ð 
ð 
Ð.ñ 
ô 
ð 
ðXð Xð Xð Xð XÐ4ñ Xô Xð Xð	ð 	ð 	ð 	ð 	Ð ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð,ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð$DÐFYñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð!>Ð@Sñ 	ô 	ð 	ðð ð €€€rX   