§
    9²¸j®6  ã                   ót  — d dl mZ ddlmZ ddlmZ  ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z ed¬¦  «        e G d	„ d
e¦  «        ¦   «         ¦   «         Z ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z edd¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z	g d¢Z
dS )é    )Ústricté   )ÚPreTrainedConfig)Úauto_docstringzthinkingmachines/Inkling)Ú
checkpointc                   óT  ‡ — e Zd ZU dZdZdZdddddddddddd	œZd
gdgfddgdgfdgdgfdœZdddddœZddddddœZ	dZ
eed<   dZedz  ed<   dZeed<   dZeed<   d Zeed!<   d"Zeed#<   d$Zeed%<   d Zeed&<   d'Zeed(<   d$Zeed)<   d*Zeed<   d'Zeed+<   d,Zeed-<   dZedz  ed.<   d/Zeed0<   dZee         dz  ed1<   dZee         dz  ed2<   d3Z eed<   d4Z!eed5<   d6Z"eed<   dZ#ee         dz  ed7<   d8Z$eed9<   d:Z%eed;<   d<Z&eed=<   d>Z'eed<   d?Z(eed@<   dAZ)eedB<   dCZ*e+edD<   dEZ,eedF<   dGZ-eed<   d4Z.eedH<   dIZ/eedJ<   dKZ0eedL<   dZ1edz  edM<   dNZ2edz  edO<   dAZ3edz  edP<   dZ4edz  edQ<   dRZ5e+edS<   dCZ6e+edT<   dZ7ee         dz  edU<   ˆ fdV„Z8e9dW„ ¦   «         Z:e9dX„ ¦   «         Z;ˆ xZ<S )YÚInklingTextConfigaÐ  
    unpadded_vocab_size (`int`, *optional*, defaults to `None`):
        Number of rows the checkpoint's unembedding matrix actually holds when the head is not padded to
        `vocab_size`. Logits beyond it are dropped. If `None`, the head is not padded.
    swa_num_attention_heads (`int`, *optional*, defaults to 64):
        Number of attention heads in the sliding-window layers.
    swa_num_key_value_heads (`int`, *optional*, defaults to 16):
        Number of key/value heads in the sliding-window layers.
    swa_head_dim (`int`, *optional*, defaults to 128):
        Dimension of query and key heads in the sliding-window layers.
    sliding_window_size (`int`, *optional*, defaults to 512):
        Size of the sliding attention window used by layers whose `layer_types` entry is `"hybrid_sliding"`.
    d_rel (`int`, *optional*, defaults to 16):
        Per-head dimension of the relative states that are mixed into the relative position bias.
    rel_extent (`int`, *optional*, defaults to 1024):
        Backward distance, in tokens, over which the relative position bias is applied. The bias is zero beyond it.
    log_scaling_n_floor (`int`, *optional*, defaults to `None`):
        Position from which logits start being scaled up logarithmically in the full-attention layers. If `None`,
        the scaling is disabled.
    log_scaling_alpha (`float`, *optional*, defaults to 0.1):
        Strength of the logarithmic logit scaling controlled by `log_scaling_n_floor`.
    local_layer_ids (`list[int]`, *optional*, defaults to `None`):
        Indices of the layers using sliding window attention. Used to derive `layer_types` when it is not provided.
        If `None`, every layer whose index is not a multiple of 6 uses sliding window attention.
    mlp_layer_types (`list[str]`, *optional*, defaults to `None`):
        MLP type pattern for each layer (`"dense"` or `"sparse"`). If `None`, every layer is sparse.
    shared_expert_sink (`bool`, *optional*, defaults to `True`):
        Whether the router scores the shared experts alongside the routed ones, so that they act as a sink in the
        softmax over expert weights.
    logits_mup_width_multiplier (`float`, *optional*, defaults to 24.0):
        muP width multiplier the final hidden states are divided by before the language modeling head.
    rms_norm_eps_moe_gate (`float`, *optional*, defaults to 1e-6):
        Epsilon of the RMS normalization applied inside the mixture-of-experts router.
    num_mtp_layers (`int`, *optional*, defaults to `None`):
        Number of multi-token-prediction layers. If `None`, multi-token prediction is disabled.
    chain_hidden_post_norm (`bool`, *optional*, defaults to `False`):
        Whether the hidden states chained between multi-token-prediction layers are normalized after each layer.
    mtp_hidden_states_first (`bool`, *optional*, defaults to `True`):
        Whether the hidden states come before the token embeddings when the two are concatenated as the input of a
        multi-token-prediction layer.
    mtp_local_layer_ids (`list[int]`, *optional*, defaults to `None`):
        Indices of the multi-token-prediction layers using sliding window attention. If `None`, every
        multi-token-prediction layer uses full attention.
    Úinkling_textÚtext_configÚembedding_rowwiseÚcolwiseÚrowwiseÚmoe_tp_expertsÚ
all_reduce)Úembed_tokensú!layers.*.mlp.experts.gate_up_projúlayers.*.mlp.experts.down_projúlayers.*.mlp.expertsz%layers.*.mlp.shared_experts.gate_projz#layers.*.mlp.shared_experts.up_projz%layers.*.mlp.shared_experts.down_projzlayers.*.mlp.shared_expertszlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)r   ÚlayersÚnormÚ	ep_routerÚgrouped_gemm)zlayers.*.mlp.gater   r   r   Úlogits_mup_width_multiplierÚsliding_window_sizeÚn_routed_expertsÚconv_kernel_sizeÚmax_position_embeddings)Úembedding_multiplierÚsliding_windowÚnum_local_expertsÚsconv_kernel_sizeÚmodel_max_lengthi@ Ú
vocab_sizeNÚunpadded_vocab_sizeé   Úhidden_sizeéB   Únum_hidden_layersé@   Únum_attention_headsé   Únum_key_value_headsé€   Úhead_dimÚswa_num_attention_headsé   Úswa_num_key_value_headsÚswa_head_dimi   Úd_relé   Ú
rel_extentÚlog_scaling_n_floorgš™™™™™¹?Úlog_scaling_alphaÚlocal_layer_idsÚlayer_typesi   ç�íµ ÷Æ°>Úrms_norm_epsé   Úmlp_layer_typesi `  Úintermediate_sizeÚsiluÚ
hidden_acti   Úmoe_intermediate_sizeé   é   Únum_experts_per_toké   Ún_shared_expertsTÚshared_expert_sinkg       @Úroute_scaleg      8@Úrms_norm_eps_moe_gateg        Úattention_dropoutç{®Gáz”?Úinitializer_rangeÚpad_token_idé   Úbos_token_idÚeos_token_idÚnum_mtp_layersFÚchain_hidden_post_normÚmtp_hidden_states_firstÚmtp_local_layer_idsc                 óú  •‡‡— | j         €_| j        �t          | j        ¦  «        Šnd„ t          | j        ¦  «        D ¦   «         Šˆfd„t          | j        ¦  «        D ¦   «         | _         | j        €;|                     dd¦  «        Šˆfd„t          | j        ¦  «        D ¦   «         | _        |                     d¦  «        �|                     d¦  «        | _        d| _	         t          ¦   «         j        di |¤Ž d S )	Nc                 ó"   — h | ]}|d z   dz  ¯
|’ŒS )rR   rG   © )Ú.0Úis     ú�/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/inkling/configuration_inkling.pyú	<setcomp>z2InklingTextConfig.__post_init__.<locals>.<setcomp>¤   s)   € Ð"[Ð"[Ð"[¨ÐPQÐTUÑPUÐYZÉ{Ð"[ 1Ð"[Ð"[Ð"[ó    c                 ó    •— g | ]
}|‰v rd nd‘ŒS ©Úhybrid_slidingÚhybridr[   )r\   r]   r<   s     €r^   ú
<listcomp>z3InklingTextConfig.__post_init__.<locals>.<listcomp>¥   s6   ø€ ð  ð  ð  ØKL A¨Ð$8Ð$8Ð Ð ¸hð ð  ð  r`   Údense_mlp_idxr   c                 ó$   •— g | ]}|‰k     rd nd‘ŒS )ÚdenseÚsparser[   )r\   r]   rf   s     €r^   re   z3InklingTextConfig.__post_init__.<locals>.<listcomp>ª   s(   ø€ Ð#vÐ#vÐ#vÐST¨q°=Ò/@Ð/@ G GÀhÐ#vÐ#vÐ#vr`   Údense_intermediate_sizer@   r[   )r=   r<   ÚsetÚranger,   rA   ÚpopÚgetrB   Únumber_of_conv_statesÚsuperÚ__post_init__)ÚselfÚkwargsrf   r<   Ú	__class__s     @@€r^   rq   zInklingTextConfig.__post_init__Ÿ   s  øøø€ ØÔÐ#ØÔ#Ð/Ý"% dÔ&:Ñ";Ô";��à"[Ð"[­e°DÔ4JÑ.KÔ.KÐ"[Ñ"[Ô"[�ð ð  ð  ð  ÝPUÐVZÔVlÑPmÔPmð ñ  ô  ˆDÔð ÔÐ'Ø"ŸJšJ ¸Ñ:Ô:ˆMØ#vÐ#vÐ#vÐ#vÕX]Ð^bÔ^tÑXuÔXuÐ#vÑ#vÔ#vˆDÔ à�:Š:Ð/Ñ0Ô0Ð<Ø%+§Z¢ZÐ0IÑ%JÔ%JˆDÔ"ð &'ˆÔ"à�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'r`   c                 óz   ‡ — ‰ j         �2‰ j        €dg‰ j         z  S ˆ fd„t          ‰ j         ¦  «        D ¦   «         S d S )Nrd   c                 ó*   •— g | ]}|‰j         v rd nd‘ŒS rb   )rX   )r\   r]   rr   s     €r^   re   z5InklingTextConfig.mtp_layer_types.<locals>.<listcomp>º   s9   ø€ ð ð ð ØXY¨¨TÔ-EÐ(EÐ(EÐ$Ð$È8ðð ð r`   )rU   rX   rl   ©rr   s   `r^   Úmtp_layer_typesz!InklingTextConfig.mtp_layer_types´   sc   ø€ àÔÐ*ØÔ'Ð/Ø �z DÔ$7Ñ7Ð7ðð ð ð Ý]bÐcgÔcvÑ]wÔ]wðñ ô ð ð ˆtr`   c                 ó*   — | j         �dg| j         z  S d S )Nrh   )rU   rw   s    r^   Úmtp_mlp_layer_typesz%InklingTextConfig.mtp_mlp_layer_typesÀ   s    € àÔÐ*Ø�9˜tÔ2Ñ2Ð2Øˆtr`   )=Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚbase_config_keyÚbase_model_tp_planÚbase_model_pp_planÚbase_model_ep_planÚattribute_mapr'   ÚintÚ__annotations__r(   r*   r,   r.   r0   r2   r3   r5   r6   r   r7   r9   r:   r;   Úfloatr<   Úlistr=   Ústrr!   r?   r    rA   rB   rD   rE   r   rH   rJ   rK   ÚboolrL   r   rM   rN   rP   rQ   rS   rT   rU   rV   rW   rX   rq   Úpropertyrx   rz   Ú__classcell__©rt   s   @r^   r	   r	      sý  ø€ € € € € € ð+ð +ðZ  €JØ#€Oà+Ø-6Ø*3Ø 0Ø1:Ø/8Ø1:Ø'3Ø"+Ø )Ø"+ðð Ðð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ðð Ðð )Ø-;Ø*8Ø 0ð	ð Ðð !>Ø/Ø/ð 0à5ð	ð 	€Mð €J�ÐÐÑà&*Ð˜˜t™Ð*Ð*Ñ*Ø€K�ÐÐÑØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø Ð˜Ð Ð Ñ Ø€HˆcÐÐÑØ#%Ð˜SÐ%Ð%Ñ%Ø#%Ð˜SÐ%Ð%Ñ%Ø€L�#ÐÐÑØ"Ð˜Ð"Ð"Ñ"Ø€Eˆ3€O€O�OØ€J�ÐÐÑØ&*Ð˜˜t™Ð*Ð*Ñ*Ø"Ð�uÐ"Ð"Ñ"Ø(,€O�T˜#”Y Ñ%Ð,Ð,Ñ,Ø$(€K��c”˜TÑ!Ð(Ð(Ñ(Ø#)Ð˜SÐ)Ð)Ñ)Ø€L�%ÐÐÑØÐ�cÐÐÑØ(,€O�T˜#”Y Ñ%Ð,Ð,Ñ,Ø"Ð�sÐ"Ð"Ñ"Ø€J�ÐÐÑà!%Ð˜3Ð%Ð%Ñ%ØÐ�cÐÐÑØ Ð˜Ð Ð Ñ ØÐ�cÐÐÑØ#Ð˜Ð#Ð#Ñ#Ø€K�ÐÐÑà)-Ð Ð-Ð-Ñ-Ø#'Ð˜5Ð'Ð'Ñ'Ø"Ð�uÐ"Ð"Ñ"Ø#Ð�uÐ#Ð#Ñ#Ø#€L�#˜‘*Ð#Ð#Ñ#Ø €L�#˜‘*Ð Ð Ñ Ø €L�#˜‘*Ð Ð Ñ à!%€N�C˜$‘JÐ%Ð%Ñ%Ø#(Ð˜DÐ(Ð(Ñ(Ø$(Ð˜TÐ(Ð(Ñ(Ø,0Ð˜˜cœ TÑ)Ð0Ð0Ñ0ð(ð (ð (ð (ð (ð* ðð ñ „Xðð ðð ñ „Xðð ð ð ð r`   r	   c                   ón   — e Zd ZU dZdZdZddddœZdZee	d<   d	Z
ee	d<   d
Zee	d<   dZee	d<   dZee	d<   dS )ÚInklingAudioConfiga–  
    n_mel_bins (`int`, *optional*, defaults to 80):
        Number of mel-frequency bins per audio frame.
    mel_vocab_size (`int`, *optional*, defaults to 256):
        Number of discrete bins each mel value is quantized into before being embedded.
    text_hidden_size (`int`, *optional*, defaults to 6144):
        Dimensionality the audio embeddings are projected to, matching the text backbone.
    Úinkling_audioÚaudio_configÚ
n_mel_binsÚmel_vocab_sizeÚtext_hidden_size)Únum_codebooksÚcodebook_sizer*   éP   rF   r)   r>   r?   rO   rP   N)r{   r|   r}   r~   r   r€   r„   r’   r…   r†   r“   r”   r?   r‡   rP   r[   r`   r^   r�   r�   Ç   s‘   € € € € € € ðð ð !€JØ$€Oà%Ø)Ø)ðð €Mð €J�ÐÐÑØ€N�CÐÐÑØ Ð�cÐ Ð Ñ Ø€L�%ÐÐÑØ#Ð�uÐ#Ð#Ñ#Ð#Ð#r`   r�   c                   ó¢   — e Zd ZU dZdZdZddiZdZee	d<   dZ
ee	d	<   d
Zee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dS )ÚInklingVisionConfigz´
    text_hidden_size (`int`, *optional*, defaults to 6144):
        Dimensionality the vision features are projected to by the last encoder layer, matching the text backbone.
    Úinkling_visionÚvision_configr,   Ún_layersr)   r”   é(   Ú
patch_sizerI   Útemporal_patch_sizer   Únum_channelsr8   r*   é   r4   r.   r>   r?   rO   rP   N)r{   r|   r}   r~   r   r€   r„   r”   r…   r†   rž   rŸ   r    r*   r,   r.   r?   r‡   rP   r[   r`   r^   r™   r™   â   sÇ   € € € € € € ðð ð
 "€JØ%€OØ(¨*Ð5€Mà Ð�cÐ Ð Ñ Ø€J�ÐÐÑØ Ð˜Ð Ð Ñ Ø€L�#ÐÐÑØ€K�ÐÐÑØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø€L�%ÐÐÑØ#Ð�uÐ#Ð#Ñ#Ð#Ð#r`   r™   zETop-level multimodal config (`InklingMMConfig` in the SGLang source).)r   Úcustom_introc                   óº   ‡ — e Zd ZU dZdZeeedœZdZ	ee
z  dz  ed<   dZee
z  dz  ed<   dZee
z  dz  ed<   dZeed	<   d
Zeed<   dZeed<   dZeed<   ˆ fd„Zˆ xZS )ÚInklingConfiga)  
    image_bos_token_id (`int`, *optional*, defaults to 200005):
        The beginning-of-image token index used to mark the start of image spans.
    audio_bos_token_id (`int`, *optional*, defaults to 200020):
        The beginning-of-audio token index used to mark the start of audio spans.
    Úinkling_mm_model)r   r‘   r›   Nr   r‘   r›   iv Úimage_token_idiu Úaudio_token_idiE Úimage_bos_token_idiT Úaudio_bos_token_idc                 ó   •— |                      d¦  «        pi }t          | j        t          ¦  «        r‹| j                             d|                      d¦  «        ¦  «         | j                             d|                      dd¦  «        ¦  «         | j                             d|                      d¦  «        ¦  «         t          | j        t          ¦  «        r | j        d         di | j        ¤Ž| _        n!| j        € | j        d         ¦   «         | _        t          | j        t          ¦  «        r | j        d	         di | j        ¤Ž| _        n!| j        € | j        d	         ¦   «         | _        t          | j        t          ¦  «        r | j        d
         di | j        ¤Ž| _        n!| j        € | j        d
         ¦   «         | _        | j        j        | j        _	        | j        j        | j        _	         t          ¦   «         j        di |¤Ž d S )NÚ
mtp_configrU   Únum_nextn_predict_layersrV   FrX   r<   r‘   r›   r   r[   )rn   Ú
isinstancer   ÚdictÚ
setdefaultr‘   Úsub_configsr›   r*   r”   rp   rq   )rr   rs   r«   rt   s      €r^   rq   zInklingConfig.__post_init__  s  ø€ à—Z’Z Ñ-Ô-Ð3°ˆ
Ý�dÔ&­Ñ-Ô-ð 	bØÔ×'Ò'Ð(8¸*¿.º.ÐIcÑ:dÔ:dÑeÔeÐeØÔ×'Ò'Ð(@À*Ç.Â.ÐQiÐkpÑBqÔBqÑrÔrÐrØÔ×'Ò'Ð(=¸z¿~º~ÐN_Ñ?`Ô?`ÑaÔaÐaå�dÔ'­Ñ.Ô.ð 	CØ @ Ô 0°Ô @Ð UÐ UÀ4ÔCTÐ UÐ UˆDÔÐØÔÐ&Ø @ Ô 0°Ô @Ñ BÔ BˆDÔå�dÔ(­$Ñ/Ô/ð 	EØ!B Ô!1°/Ô!BÐ!XÐ!XÀTÔEWÐ!XÐ!XˆDÔÐØÔÐ'Ø!B Ô!1°/Ô!BÑ!DÔ!DˆDÔå�dÔ&­Ñ-Ô-ð 	AØ>˜tÔ/°Ô>ÐRÐRÀÔAQÐRÐRˆDÔÐØÔÐ%Ø>˜tÔ/°Ô>Ñ@Ô@ˆDÔà.2Ô.>Ô.JˆÔÔ+Ø-1Ô-=Ô-IˆÔÔ*Ø�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'r`   )r{   r|   r}   r~   r   r	   r�   r™   r°   r   r®   r†   r‘   r›   r¦   r…   r§   r¨   r©   rq   rŒ   r�   s   @r^   r¤   r¤   ù   sõ   ø€ € € € € € ðð ð $€Jà(Ø*Ø,ðð €Kð 48€KÐ" TÑ)¨DÑ0Ð7Ð7Ñ7Ø59€LÐ$ tÑ+¨dÑ2Ð9Ð9Ñ9Ø7;€MÐ&¨Ñ-°Ñ4Ð;Ð;Ñ;Ø €N�CÐ Ð Ñ Ø €N�CÐ Ð Ñ Ø$Ð˜Ð$Ð$Ñ$Ø$Ð˜Ð$Ð$Ñ$ð(ð (ð (ð (ð (ð (ð (ð (ð (r`   r¤   )r¤   r	   r�   r™   N)Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úutilsr   r	   r�   r™   r¤   Ú__all__r[   r`   r^   ú<module>rµ      s¦  ðð, /Ð .Ð .Ð .Ð .Ð .à 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø #Ð #Ð #Ð #Ð #Ð #ð €Ð5Ð6Ñ6Ô6Øðfð fð fð fð fÐ(ñ fô fñ „ñ 7Ô6ðfðR €Ð5Ð6Ñ6Ô6Øð$ð $ð $ð $ð $Ð)ñ $ô $ñ „ñ 7Ô6ð$ð2 €Ð5Ð6Ñ6Ô6Øð$ð $ð $ð $ð $Ð*ñ $ô $ñ „ñ 7Ô6ð$ð* €Ø)ØXðñ ô ð ð0(ð 0(ð 0(ð 0(ð 0(Ð$ñ 0(ô 0(ñ „ñ	ô ð
0(ðf ^Ð
]Ð
]€€€r`   