§
    9²¸jl  ã                   óŒ   — d dl mZ ddlmZ ddlmZ ddlmZ  ed¬¦  «        e G d„ d	e¦  «        ¦   «         ¦   «         Zd	gZ	d
S )é    )Ústricté   )ÚPreTrainedConfig)ÚRopeParameters)Úauto_docstringz
skt/A.X-K1)Ú
checkpointc            
       ó¬  ‡ — e Zd ZU dZdZdgZddddddddddœ	Zd	gd
gfddgdgfdgdgfdœZdddddœZddiZ	dZ
eed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   d Zeed!<   d"Zeed<   d#Zeed$<   d%Zeed&<   d'Zeed(<   dZeed)<   d*Zeed+<   d*Zeed,<   d-Zeed.<   d/Zeed0<   d-Zeed1<   d Zeed2<   d3Z e!d4z  ed5<   d6Z"e#ed7<   d8Z$eed9<   d:Z%eed;<   d<Z&eed=<   d3Z'e!ed><   d4Z(ed4z  ed?<   d@Z)ed4z  edA<   d@Z*ee+e         z  d4z  edB<   d Z,ed4z  edC<   dDZ-e!edE<   d4Z.e/e0z  d4z  edF<   d3Z1e!d4z  edG<   dDZ2e!edH<   dIZ3eez  d4z  edJ<   ˆ fdK„Z4ˆ xZ5S )LÚ
AXK1Configac  
    n_group (`int`, *optional*, defaults to 8):
        Number of groups for routed experts.
    topk_group (`int`, *optional*, defaults to 4):
        Number of selected groups per token (each token's experts are drawn from these groups).
    first_k_dense_replace (`int`, *optional*, defaults to 1):
        Number of leading layers that use a dense MLP; the rest use the MoE block.
    rope_interleave (`bool`, *optional*, defaults to `True`):
        Whether to use the interleaved rotary position embedding layout.

    ```python
    >>> from transformers import AXK1Config, AXK1Model

    >>> # Initializing an A.X-K1 style configuration
    >>> configuration = AXK1Config()

    >>> # Initializing a model from the configuration
    >>> model = AXK1Model(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```Úaxk1Úpast_key_valuesÚpacked_colwiseÚrowwiseÚmoe_tp_expertsÚcolwise)	ú!layers.*.mlp.experts.gate_up_projúlayers.*.mlp.experts.down_projúlayers.*.mlp.expertsz%layers.*.mlp.shared_experts.gate_projz#layers.*.mlp.shared_experts.up_projz%layers.*.mlp.shared_experts.down_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormÚ	ep_routerÚgrouped_gemm)zlayers.*.mlp.gater   r   r   Únum_local_expertsÚn_routed_expertsi € Ú
vocab_sizei   Úhidden_sizei H  Úintermediate_sizei   Úmoe_intermediate_sizeé=   Únum_hidden_layersé@   Únum_attention_headsÚnum_key_value_headsé   Ún_shared_expertséÀ   g      @Úrouted_scaling_factori   Úkv_lora_ranki   Úq_lora_rankÚqk_rope_head_dimé€   Ú
v_head_dimÚqk_nope_head_dimé   Ún_groupé   Ú
topk_groupÚnum_experts_per_tokÚfirst_k_dense_replaceTNÚnorm_topk_probÚsiluÚ
hidden_acti €  Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangeg�íµ ÷Æ°>Úrms_norm_epsÚ	use_cacheÚpad_token_idik Úbos_token_idÚeos_token_idÚpretraining_tpFÚtie_word_embeddingsÚrope_parametersÚrope_interleaveÚattention_biasg        Úattention_dropoutc                 ó    •— | j         €| j        | _         | j        | j        z   | _        | j        | _         t          ¦   «         j        di |¤Ž d S )N© )r'   r&   r1   r.   Úqk_head_dimÚhead_dimÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s     €ú{/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/axk1/configuration_axk1.pyrM   zAXK1Config.__post_init__t   sW   ø€ ØÔ#Ð+Ø'+Ô'?ˆDÔ$àÔ0°4Ô3HÑHˆÔØÔ-ˆŒØ�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'ó    )6Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚbase_model_tp_planÚbase_model_pp_planÚbase_model_ep_planÚattribute_mapr   ÚintÚ__annotations__r    r!   r"   r$   r&   r'   r)   r   r+   Úfloatr,   r-   r.   r0   r1   r3   r5   r6   r7   r8   Úboolr:   Ústrr;   r<   r=   r>   r?   r@   rA   ÚlistrB   rC   rD   r   ÚdictrE   rF   rG   rM   Ú__classcell__)rP   s   @rQ   r
   r
      sG  ø€ € € € € € ðð ð. €JØ#4Ð"5Ðà-=Ø*3Ø 0Ø1:Ø/8Ø1:Ø"+Ø )Ø"+ð
ð 
Ðð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ðð Ðð )Ø-;Ø*8Ø 0ð	ð Ðð 	Ð/ð€Mð €J�ÐÐÑØ€K�ÐÐÑØ"Ð�sÐ"Ð"Ñ"Ø!%Ð˜3Ð%Ð%Ñ%ØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø!Ð˜Ð!Ð!Ñ!ØÐ�cÐÐÑØÐ�cÐÐÑØ#&Ð˜5Ð&Ð&Ñ&Ø€L�#ÐÐÑØ€K�ÐÐÑØÐ�cÐÐÑØ€J�ÐÐÑØÐ�cÐÐÑØ€GˆSÐÐÑØ€J�ÐÐÑØ Ð˜Ð Ð Ñ Ø!"Ð˜3Ð"Ð"Ñ"Ø"&€N�D˜4‘KÐ&Ð&Ñ&Ø€J�ÐÐÑØ#(Ð˜SÐ(Ð(Ñ(Ø#Ð�uÐ#Ð#Ñ#Ø€L�%ÐÐÑØ€IˆtÐÐÑØ#€L�#˜‘*Ð#Ð#Ñ#Ø%€L�#˜‘*Ð%Ð%Ñ%Ø+1€L�#˜˜Sœ	‘/ DÑ(Ð1Ð1Ñ1Ø!"€N�C˜$‘JÐ"Ð"Ñ"Ø %Ð˜Ð%Ð%Ñ%Ø48€O�^ dÑ*¨TÑ1Ð8Ð8Ñ8Ø#'€O�T˜D‘[Ð'Ð'Ñ'Ø €N�DÐ Ð Ñ Ø,/Ð�u˜s‘{ TÑ)Ð/Ð/Ñ/ð(ð (ð (ð (ð (ð (ð (ð (ð (rR   r
   N)
Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úmodeling_rope_utilsr   Úutilsr   r
   Ú__all__rI   rR   rQ   ú<module>rj      s´   ðð( /Ð .Ð .Ð .Ð .Ð .à 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø #Ð #Ð #Ð #Ð #Ð #ð €˜<Ð(Ñ(Ô(Øð](ð ](ð ](ð ](ð ](Ð!ñ ](ô ](ñ „ñ )Ô(ð](ð@ ˆ.€€€rR   