§
    9²¸jk  ã                   ó€   — d dl mZ ddlmZ ddlmZ  ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         ZdgZd	S )
é    )Ústricté   )ÚPreTrainedConfig)Úauto_docstringz
skt/A.X-K2)Ú
checkpointc                   ó  ‡ — e Zd ZU dZdZdgZdddddddddddddd	œZd
gdgfddgdgfdgdgfdœZdddddœZddiZ	dZ
eed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed <   d!Zeed"<   d#Zeed<   d$Zeed%<   d#Zeed&<   d'Zeed(<   dZeed)<   d*Zeed+<   d*Zeed,<   d-Zed-z  ed.<   d-Zed-z  ed/<   d0Zeed1<   d2Ze ed3<   d4Z!e"ed5<   d6Z#eed7<   d8Z$eed9<   d:Z%eed;<   d2Z&e ed<<   d-Z'ed-z  ed=<   d>Z(ed-z  ed?<   d>Z)ee*e         z  d-z  ed@<   dAZ+e edB<   d-Z,e-d-z  edC<   d-Z.e*e"         d-z  edD<   dAZ/e edE<   dFZ0eez  edG<   dZ1eedH<   d#Z2eedI<   dJZ3eedK<   d*Z4eedL<   d-Z5e*e"         d-z  edM<   dJZ6eedN<   ˆ fdO„Z7ˆ fdP„Z8ˆ xZ9S )QÚ
AXK2Configaå  
    n_group (`int`, *optional*):
        Number of expert groups for grouped routing, used by the larger A.X-K2 releases. `None` (the
        A.X-K2-Light default) routes over all experts without group restriction.
    topk_group (`int`, *optional*):
        Number of expert groups the top-k selection is restricted to when `n_group` is set.
    mlp_layer_types (`list`, *optional*):
        MLP type pattern for each layer (`"dense"` or `"sparse"`). Derived from the (legacy) kwargs
        `first_k_dense_replace` / `moe_layer_freq` when not provided.
    index_topk (`int`, *optional*, defaults to 2048):
        Number of top tokens selected by the indexer for sparse attention.
    index_head_dim (`int`, *optional*, defaults to 128):
        Head dimension for the indexer projections (DSA).
    index_n_heads (`int`, *optional*, defaults to 16):
        Number of heads for the indexer projections (DSA).
    gated_norm_rank (`int`, *optional*, defaults to 16):
        Bottleneck rank for the low-rank input-dependent gate used by `AXK2GatedRMSNorm`. The gate wraps
        `input_layernorm` on every layer and `post_attention_layernorm` on MoE layers.

    ```python
    >>> from transformers import AXK2Config, AXK2Model

    >>> # Initializing an A.X-K2 style configuration
    >>> configuration = AXK2Config()

    >>> # Initializing a model from the configuration
    >>> model = AXK2Model(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```Úaxk2Úpast_key_valuesÚcolwiseÚmla_kv_a_projÚrowwiseÚpacked_colwiseÚmoe_tp_experts)zlayers.*.self_attn.q_gate_projz%layers.*.self_attn.kv_a_proj_with_mqazlayers.*.self_attn.kv_b_projzlayers.*.self_attn.o_projú!layers.*.mlp.experts.gate_up_projúlayers.*.mlp.experts.down_projúlayers.*.mlp.expertsz%layers.*.mlp.shared_experts.gate_projz#layers.*.mlp.shared_experts.up_projz%layers.*.mlp.shared_experts.down_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormÚ	ep_routerÚgrouped_gemm)zlayers.*.mlp.gater   r   r   Únum_local_expertsÚn_routed_expertsi € Ú
vocab_sizei   Úhidden_sizei   Úintermediate_sizei   Úmoe_intermediate_sizeé0   Únum_hidden_layersé    Únum_attention_headsÚnum_key_value_headsé   Ún_shared_expertsé€   g      @Úrouted_scaling_factorÚkv_lora_ranki€  Úq_lora_rankÚqk_rope_head_dimé@   Ú
v_head_dimÚqk_nope_head_dimNÚn_groupÚ
topk_groupé   Únum_experts_per_tokTÚnorm_topk_probÚsiluÚ
hidden_acti   Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangeg�íµ ÷Æ°>Úrms_norm_epsÚ	use_cacheÚpad_token_idik Úbos_token_idÚeos_token_idFÚtie_word_embeddingsÚrope_parametersÚmlp_layer_typesÚattention_biasg        Úattention_dropoutÚ
index_topkÚindex_head_dimé   Úindex_n_headsÚhead_dimÚlayer_typesÚgated_norm_rankc                 ó^  •‡‡— | j         | j        z   | _        | j        | _        | j        €R|                     dd¦  «        Š|                     dd¦  «        Šˆˆfd„t          | j        ¦  «        D ¦   «         | _        | j        €dg| j        z  | _         t          ¦   «         j
        di |¤Ž d S )NÚfirst_k_dense_replacer(   Úmoe_layer_freqc                 ó6   •— g | ]}|‰k    r|‰z  d k    rdnd‘ŒS )r   ÚsparseÚdense© )Ú.0ÚirM   rN   s     €€ú{/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/axk2/configuration_axk2.pyú
<listcomp>z,AXK2Config.__post_init__.<locals>.<listcomp>�   sI   ø€ ð $ð $ð $àð Ð!6Ò6Ð6¸1¸~Ñ;MÐQRÒ;RÐ;R��ÐX_ð$ð $ð $ó    Údeepseek_sparse_attentionrR   )r1   r.   Úqk_head_dimrI   rB   ÚpopÚranger$   rJ   ÚsuperÚ__post_init__)ÚselfÚkwargsrM   rN   Ú	__class__s     @@€rU   r]   zAXK2Config.__post_init__†   sÔ   øøø€ ØÔ0°4Ô3HÑHˆÔð Ô-ˆŒð ÔÐ'Ø$*§J¢JÐ/FÈÑ$JÔ$JÐ!Ø#ŸZšZÐ(8¸!Ñ<Ô<ˆNð$ð $ð $ð $ð $å˜tÔ5Ñ6Ô6ð$ñ $ô $ˆDÔ ð ÔÐ#Ø ;Ð<¸tÔ?UÑUˆDÔà�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'rW   c                 óÜ  •— t          ¦   «                              ¦   «          | j        �| j        dk    rt          d| j        › d�¦  «        ‚| j        d u | j        d u k    r t          d| j        › d| j        › d�¦  «        ‚| j        �c| j        | j        z  dk    r t          d| j        › d| j        › d�¦  «        ‚| j        | j        k    r"t          d	| j        › d
| j        › d�¦  «        ‚d S d S )Nr   zkA.X-K2 requires a positive `q_lora_rank` (the indexer and output gate read the query LoRA bottleneck), got ú.zc`n_group` and `topk_group` must be set together (both `None` for non-grouped routing), got n_group=z, topk_group=z`n_routed_experts` (z") must be divisible by `n_group` (z).z`topk_group` (z) cannot exceed `n_group` ()r\   Úvalidate_architecturer-   Ú
ValueErrorr2   r3   r   )r^   r`   s    €rU   rc   z AXK2Config.validate_architecture›   sM  ø€ Ý‰Œ×%Ò%Ñ'Ô'Ð'ØÔÐ# tÔ'7¸1Ò'<Ð'<Ýð8Ø$(Ô$4ð8ð 8ð 8ñô ð ð ŒL˜DÐ  d¤o¸Ð&=Ò>Ð>ÝðIØœ<ðIð IØ6:´oðIð Ið Iñô ð ð Œ<Ð#ØÔ$ t¤|Ñ3°qÒ8Ð8Ý Øt¨4Ô+@ÐtÐtÐdhÔdpÐtÐtÐtñô ð ð Œ ¤Ò-Ð-Ý Ð!n°$´/Ð!nÐ!nÐ^bÔ^jÐ!nÐ!nÐ!nÑoÔoÐoð $Ð#ð
 .Ð-rW   ):Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚbase_model_tp_planÚbase_model_pp_planÚbase_model_ep_planÚattribute_mapr   ÚintÚ__annotations__r    r!   r"   r$   r&   r'   r)   r   r+   Úfloatr,   r-   r.   r0   r1   r2   r3   r5   r6   Úboolr8   Ústrr9   r:   r;   r<   r=   r>   r?   Úlistr@   rA   ÚdictrB   rC   rD   rE   rF   rH   rI   rJ   rK   r]   rc   Ú__classcell__)r`   s   @rU   r	   r	      s¸  ø€ € € € € € ðð ð@ €JØ#4Ð"5Ðð +4Ø1@Ø(1Ø%.Ø-=Ø*3Ø 0Ø1:Ø/8Ø1:Ø"+Ø )Ø"+ðð Ðð  &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ðð Ðð )Ø-;Ø*8Ø 0ð	ð Ðð )Ð*<Ð=€Mà€J�ÐÐÑØ€K�ÐÐÑØ!Ð�sÐ!Ð!Ñ!Ø!$Ð˜3Ð$Ð$Ñ$ØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø!Ð˜Ð!Ð!Ñ!ØÐ�cÐÐÑØÐ�cÐÐÑØ#&Ð˜5Ð&Ð&Ñ&Ø€L�#ÐÐÑØ€K�ÐÐÑØÐ�cÐÐÑØ€J�ÐÐÑØÐ�cÐÐÑð €GˆS�4‰ZÐÐÑØ!€J��d‘
Ð!Ð!Ñ!Ø Ð˜Ð Ð Ñ Ø€N�DÐÐÑØ€J�ÐÐÑØ#)Ð˜SÐ)Ð)Ñ)Ø#Ð�uÐ#Ð#Ñ#Ø€L�%ÐÐÑØ€IˆtÐÐÑØ#€L�#˜‘*Ð#Ð#Ñ#Ø%€L�#˜‘*Ð%Ð%Ñ%Ø+1€L�#˜˜Sœ	‘/ DÑ(Ð1Ð1Ñ1Ø %Ð˜Ð%Ð%Ñ%Ø#'€O�T˜D‘[Ð'Ð'Ñ'Ø(,€O�T˜#”Y Ñ%Ð,Ð,Ñ,Ø €N�DÐ Ð Ñ Ø%(Ð�u˜s‘{Ð(Ð(Ñ(Ø€J�ÐÐÑØ€N�CÐÐÑØ€M�3ÐÐÑØ€HˆcÐÐÑØ$(€K��c”˜TÑ!Ð(Ð(Ñ(Ø€O�SÐÐÑð(ð (ð (ð (ð (ð*pð pð pð pð pð pð pð pð prW   r	   N)Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úutilsr   r	   Ú__all__rR   rW   rU   ú<module>r{      sª   ðð( /Ð .Ð .Ð .Ð .Ð .à 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø #Ð #Ð #Ð #Ð #Ð #ð €˜<Ð(Ñ(Ô(ØðQpð Qpð Qpð Qpð QpÐ!ñ Qpô Qpñ „ñ )Ô(ðQpðh ˆ.€€€rW   