§
    9²¸j7û  ã                   óà  — d dl Z d dlmZ d dlZd dlmZ d dlmc mZ d dl	m
Z
 ddlmZ ddlmZ ddlmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZmZ ddlm Z  ddl!m"Z"m#Z# ddl$m%Z%m&Z& ddl'm(Z( ddl)m*Z*m+Z+m,Z,m-Z-m.Z. ddl/m0Z0 ddl1m2Z2 ddl3m4Z4m5Z5m6Z6m7Z7 ddl8m9Z9 ddl:m;Z;m<Z< ddl=m>Z> ddl?m@Z@mAZAmBZB  e-jC        eD¦  «        ZE e+d¬¦  «        e
 G d„ de¦  «        ¦   «         ¦   «         ZF e+d¬¦  «        e
 G d„ de¦  «        ¦   «         ¦   «         ZG e+d¬¦  «        e
 G d „ d!e¦  «        ¦   «         ¦   «         ZH e+dd"¬#¦  «        e
 G d$„ d%e¦  «        ¦   «         ¦   «         ZI G d&„ d'e7¦  «        ZJ G d(„ d)e4¦  «        ZK G d*„ d+e;¦  «        ZL G d,„ d-ejM        ¦  «        ZN	 	 dfd/ejM        d0ejO        d1ejO        d2ejO        d3ejO        dz  d4ePd5ePd6ejO        dz  fd7„ZQ G d8„ d9ejM        ¦  «        ZR G d:„ d;e6¦  «        ZS G d<„ d=e>¦  «        ZT G d>„ d?ejM        ¦  «        ZU G d@„ dAejM        ¦  «        ZV G dB„ dCejM        ¦  «        ZW eeBeAg¦  «         G dD„ dEejM        ¦  «        ¦   «         ZX G dF„ dGe ¦  «        ZYe+ G dH„ dIe&¦  «        ¦   «         ZZe+ G dJ„ dKeZ¦  «        ¦   «         Z[ G dL„ dMe5¦  «        Z\ G dN„ dOe9¦  «        Z]e+ G dP„ dQeZ¦  «        ¦   «         Z^ G dR„ dSejM        ¦  «        Z_dTe`dUeae`         fdV„Zb	 dgdXe`dYe`dZe`d[e`dUejc        f
d\„Zde+ G d]„ d^eZ¦  «        ¦   «         Ze e+d_¬`¦  «         G da„ dbeZ¦  «        ¦   «         Zf e+d_¬`¦  «         G dc„ ddeZe¦  «        ¦   «         Zgg de¢ZhdS )hé    N)ÚCallable)Ústricté   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚPreTrainedConfig)ÚGenerationMixin)Úuse_kernelized_func)Úforce_accelerate_hooks)Úcreate_causal_maskÚcreate_recurrent_attention_maskÚ!create_sliding_window_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚBaseModelOutputWithPooling)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚloggingÚtorch_compilable_check)Úmerge_with_config_defaults)Úcapture_outputsé   )ÚGemma3CausalLMOutputWithPastÚGemma3ForCausalLMÚ	Gemma3MLPÚGemma3ModelOutputWithPast)ÚHiggsAudioV2Embeddings)ÚLlamaRMSNormÚ	repeat_kv)ÚMixtralExperts)Úapply_mask_to_padding_statesÚcausal_conv1d_fnÚcausal_conv1d_updatezthinkingmachines/Inkling)Ú
checkpointc                   óT  ‡ — e Zd ZU dZdZdZdddddddddddd	œZd
gdgfddgdgfdgdgfdœZdddddœZddddddœZ	dZ
eed<   dZedz  ed<   dZeed<   dZeed<   d Zeed!<   d"Zeed#<   d$Zeed%<   d Zeed&<   d'Zeed(<   d$Zeed)<   d*Zeed<   d'Zeed+<   d,Zeed-<   dZedz  ed.<   d/Zeed0<   dZee         dz  ed1<   dZee         dz  ed2<   d3Z eed<   d4Z!eed5<   d6Z"eed<   dZ#ee         dz  ed7<   d8Z$eed9<   d:Z%eed;<   d<Z&eed=<   d>Z'eed<   d?Z(eed@<   dAZ)eedB<   dCZ*e+edD<   dEZ,eedF<   dGZ-eed<   d4Z.eedH<   dIZ/eedJ<   dKZ0eedL<   dZ1edz  edM<   dNZ2edz  edO<   dAZ3edz  edP<   dZ4edz  edQ<   dRZ5e+edS<   dCZ6e+edT<   dZ7ee         dz  edU<   ˆ fdV„Z8e9dW„ ¦   «         Z:e9dX„ ¦   «         Z;ˆ xZ<S )YÚInklingTextConfigaÐ  
    unpadded_vocab_size (`int`, *optional*, defaults to `None`):
        Number of rows the checkpoint's unembedding matrix actually holds when the head is not padded to
        `vocab_size`. Logits beyond it are dropped. If `None`, the head is not padded.
    swa_num_attention_heads (`int`, *optional*, defaults to 64):
        Number of attention heads in the sliding-window layers.
    swa_num_key_value_heads (`int`, *optional*, defaults to 16):
        Number of key/value heads in the sliding-window layers.
    swa_head_dim (`int`, *optional*, defaults to 128):
        Dimension of query and key heads in the sliding-window layers.
    sliding_window_size (`int`, *optional*, defaults to 512):
        Size of the sliding attention window used by layers whose `layer_types` entry is `"hybrid_sliding"`.
    d_rel (`int`, *optional*, defaults to 16):
        Per-head dimension of the relative states that are mixed into the relative position bias.
    rel_extent (`int`, *optional*, defaults to 1024):
        Backward distance, in tokens, over which the relative position bias is applied. The bias is zero beyond it.
    log_scaling_n_floor (`int`, *optional*, defaults to `None`):
        Position from which logits start being scaled up logarithmically in the full-attention layers. If `None`,
        the scaling is disabled.
    log_scaling_alpha (`float`, *optional*, defaults to 0.1):
        Strength of the logarithmic logit scaling controlled by `log_scaling_n_floor`.
    local_layer_ids (`list[int]`, *optional*, defaults to `None`):
        Indices of the layers using sliding window attention. Used to derive `layer_types` when it is not provided.
        If `None`, every layer whose index is not a multiple of 6 uses sliding window attention.
    mlp_layer_types (`list[str]`, *optional*, defaults to `None`):
        MLP type pattern for each layer (`"dense"` or `"sparse"`). If `None`, every layer is sparse.
    shared_expert_sink (`bool`, *optional*, defaults to `True`):
        Whether the router scores the shared experts alongside the routed ones, so that they act as a sink in the
        softmax over expert weights.
    logits_mup_width_multiplier (`float`, *optional*, defaults to 24.0):
        muP width multiplier the final hidden states are divided by before the language modeling head.
    rms_norm_eps_moe_gate (`float`, *optional*, defaults to 1e-6):
        Epsilon of the RMS normalization applied inside the mixture-of-experts router.
    num_mtp_layers (`int`, *optional*, defaults to `None`):
        Number of multi-token-prediction layers. If `None`, multi-token prediction is disabled.
    chain_hidden_post_norm (`bool`, *optional*, defaults to `False`):
        Whether the hidden states chained between multi-token-prediction layers are normalized after each layer.
    mtp_hidden_states_first (`bool`, *optional*, defaults to `True`):
        Whether the hidden states come before the token embeddings when the two are concatenated as the input of a
        multi-token-prediction layer.
    mtp_local_layer_ids (`list[int]`, *optional*, defaults to `None`):
        Indices of the multi-token-prediction layers using sliding window attention. If `None`, every
        multi-token-prediction layer uses full attention.
    Úinkling_textÚtext_configÚembedding_rowwiseÚcolwiseÚrowwiseÚmoe_tp_expertsÚ
all_reduce)Úembed_tokensú!layers.*.mlp.experts.gate_up_projúlayers.*.mlp.experts.down_projúlayers.*.mlp.expertsz%layers.*.mlp.shared_experts.gate_projz#layers.*.mlp.shared_experts.up_projz%layers.*.mlp.shared_experts.down_projzlayers.*.mlp.shared_expertszlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)r4   ÚlayersÚnormÚ	ep_routerÚgrouped_gemm)zlayers.*.mlp.gater5   r6   r7   Úlogits_mup_width_multiplierÚsliding_window_sizeÚn_routed_expertsÚconv_kernel_sizeÚmax_position_embeddings)Úembedding_multiplierÚsliding_windowÚnum_local_expertsÚsconv_kernel_sizeÚmodel_max_lengthi@ Ú
vocab_sizeNÚunpadded_vocab_sizeé   Úhidden_sizeéB   Únum_hidden_layersé@   Únum_attention_headsé   Únum_key_value_headsé€   Úhead_dimÚswa_num_attention_headsé   Úswa_num_key_value_headsÚswa_head_dimi   Úd_relé   Ú
rel_extentÚlog_scaling_n_floorgš™™™™™¹?Úlog_scaling_alphaÚlocal_layer_idsÚlayer_typesi   ç�íµ ÷Æ°>Úrms_norm_epsé   Úmlp_layer_typesi `  Úintermediate_sizeÚsiluÚ
hidden_acti   Úmoe_intermediate_sizeé   é   Únum_experts_per_tokr   Ún_shared_expertsTÚshared_expert_sinkg       @Úroute_scaleg      8@Úrms_norm_eps_moe_gateç        Úattention_dropoutç{®Gáz”?Úinitializer_rangeÚpad_token_idé   Úbos_token_idÚeos_token_idÚnum_mtp_layersFÚchain_hidden_post_normÚmtp_hidden_states_firstÚmtp_local_layer_idsc                 óú  •‡‡— | j         €_| j        �t          | j        ¦  «        Šnd„ t          | j        ¦  «        D ¦   «         Šˆfd„t          | j        ¦  «        D ¦   «         | _         | j        €;|                     dd¦  «        Šˆfd„t          | j        ¦  «        D ¦   «         | _        |                     d¦  «        �|                     d¦  «        | _        d| _	         t          ¦   «         j        di |¤Ž d S )	Nc                 ó"   — h | ]}|d z   dz  ¯
|’ŒS )ru   rj   © )Ú.0Úis     ú{/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/inkling/modular_inkling.pyú	<setcomp>z2InklingTextConfig.__post_init__.<locals>.<setcomp>Ä   s)   € Ð"[Ð"[Ð"[¨ÐPQÐTUÑPUÐYZÉ{Ð"[ 1Ð"[Ð"[Ð"[ó    c                 ó    •— g | ]
}|‰v rd nd‘ŒS ©Úhybrid_slidingÚhybridr~   )r   r€   r_   s     €r�   ú
<listcomp>z3InklingTextConfig.__post_init__.<locals>.<listcomp>Å   s6   ø€ ð  ð  ð  ØKL A¨Ð$8Ð$8Ð Ð ¸hð ð  ð  rƒ   Údense_mlp_idxr   c                 ó$   •— g | ]}|‰k     rd nd‘ŒS )ÚdenseÚsparser~   )r   r€   r‰   s     €r�   rˆ   z3InklingTextConfig.__post_init__.<locals>.<listcomp>Ê   s(   ø€ Ð#vÐ#vÐ#vÐST¨q°=Ò/@Ð/@ G GÀhÐ#vÐ#vÐ#vrƒ   Údense_intermediate_sizerc   r~   )r`   r_   ÚsetÚrangerO   rd   ÚpopÚgetre   Únumber_of_conv_statesÚsuperÚ__post_init__)ÚselfÚkwargsr‰   r_   Ú	__class__s     @@€r�   r”   zInklingTextConfig.__post_init__¿   s  øøø€ ØÔÐ#ØÔ#Ð/Ý"% dÔ&:Ñ";Ô";��à"[Ð"[­e°DÔ4JÑ.KÔ.KÐ"[Ñ"[Ô"[�ð ð  ð  ð  ÝPUÐVZÔVlÑPmÔPmð ñ  ô  ˆDÔð ÔÐ'Ø"ŸJšJ ¸Ñ:Ô:ˆMØ#vÐ#vÐ#vÐ#vÕX]Ð^bÔ^tÑXuÔXuÐ#vÑ#vÔ#vˆDÔ à�:Š:Ð/Ñ0Ô0Ð<Ø%+§Z¢ZÐ0IÑ%JÔ%JˆDÔ"ð &'ˆÔ"à�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'rƒ   c                 óz   ‡ — ‰ j         �2‰ j        €dg‰ j         z  S ˆ fd„t          ‰ j         ¦  «        D ¦   «         S d S )Nr‡   c                 ó*   •— g | ]}|‰j         v rd nd‘ŒS r…   )r{   )r   r€   r•   s     €r�   rˆ   z5InklingTextConfig.mtp_layer_types.<locals>.<listcomp>Ú   s9   ø€ ð ð ð ØXY¨¨TÔ-EÐ(EÐ(EÐ$Ð$È8ðð ð rƒ   )rx   r{   r�   ©r•   s   `r�   Úmtp_layer_typesz!InklingTextConfig.mtp_layer_typesÔ   sc   ø€ àÔÐ*ØÔ'Ð/Ø �z DÔ$7Ñ7Ð7ðð ð ð Ý]bÐcgÔcvÑ]wÔ]wðñ ô ð ð ˆtrƒ   c                 ó*   — | j         �dg| j         z  S d S )Nr‹   )rx   rš   s    r�   Úmtp_mlp_layer_typesz%InklingTextConfig.mtp_mlp_layer_typesà   s    € àÔÐ*Ø�9˜tÔ2Ñ2Ð2Øˆtrƒ   )=Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚbase_config_keyÚbase_model_tp_planÚbase_model_pp_planÚbase_model_ep_planÚattribute_maprJ   ÚintÚ__annotations__rK   rM   rO   rQ   rS   rU   rV   rX   rY   rA   rZ   r\   r]   r^   Úfloatr_   Úlistr`   ÚstrrD   rb   rC   rd   re   rg   rh   rB   rk   rl   rm   Úboolrn   r@   ro   rq   rs   rt   rv   rw   rx   ry   rz   r{   r”   Úpropertyr›   r�   Ú__classcell__©r—   s   @r�   r,   r,   <   sý  ø€ € € € € € ð+ð +ðZ  €JØ#€Oà+Ø-6Ø*3Ø 0Ø1:Ø/8Ø1:Ø'3Ø"+Ø )Ø"+ðð Ðð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ðð Ðð )Ø-;Ø*8Ø 0ð	ð Ðð !>Ø/Ø/ð 0à5ð	ð 	€Mð €J�ÐÐÑà&*Ð˜˜t™Ð*Ð*Ñ*Ø€K�ÐÐÑØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø Ð˜Ð Ð Ñ Ø€HˆcÐÐÑØ#%Ð˜SÐ%Ð%Ñ%Ø#%Ð˜SÐ%Ð%Ñ%Ø€L�#ÐÐÑØ"Ð˜Ð"Ð"Ñ"Ø€Eˆ3€O€O�OØ€J�ÐÐÑØ&*Ð˜˜t™Ð*Ð*Ñ*Ø"Ð�uÐ"Ð"Ñ"Ø(,€O�T˜#”Y Ñ%Ð,Ð,Ñ,Ø$(€K��c”˜TÑ!Ð(Ð(Ñ(Ø#)Ð˜SÐ)Ð)Ñ)Ø€L�%ÐÐÑØÐ�cÐÐÑØ(,€O�T˜#”Y Ñ%Ð,Ð,Ñ,Ø"Ð�sÐ"Ð"Ñ"Ø€J�ÐÐÑà!%Ð˜3Ð%Ð%Ñ%ØÐ�cÐÐÑØ Ð˜Ð Ð Ñ ØÐ�cÐÐÑØ#Ð˜Ð#Ð#Ñ#Ø€K�ÐÐÑà)-Ð Ð-Ð-Ñ-Ø#'Ð˜5Ð'Ð'Ñ'Ø"Ð�uÐ"Ð"Ñ"Ø#Ð�uÐ#Ð#Ñ#Ø#€L�#˜‘*Ð#Ð#Ñ#Ø €L�#˜‘*Ð Ð Ñ Ø €L�#˜‘*Ð Ð Ñ à!%€N�C˜$‘JÐ%Ð%Ñ%Ø#(Ð˜DÐ(Ð(Ñ(Ø$(Ð˜TÐ(Ð(Ñ(Ø,0Ð˜˜cœ TÑ)Ð0Ð0Ñ0ð(ð (ð (ð (ð (ð* ðð ñ „Xðð ðð ñ „Xðð ð ð ð rƒ   r,   c                   ón   — e Zd ZU dZdZdZddddœZdZee	d<   d	Z
ee	d<   d
Zee	d<   dZee	d<   dZee	d<   dS )ÚInklingAudioConfiga–  
    n_mel_bins (`int`, *optional*, defaults to 80):
        Number of mel-frequency bins per audio frame.
    mel_vocab_size (`int`, *optional*, defaults to 256):
        Number of discrete bins each mel value is quantized into before being embedded.
    text_hidden_size (`int`, *optional*, defaults to 6144):
        Dimensionality the audio embeddings are projected to, matching the text backbone.
    Úinkling_audioÚaudio_configÚ
n_mel_binsÚmel_vocab_sizeÚtext_hidden_size)Únum_codebooksÚcodebook_sizerM   éP   ri   rL   ra   rb   rr   rs   N)rž   rŸ   r    r¡   r¢   r£   r§   rµ   r¨   r©   r¶   r·   rb   rª   rs   r~   rƒ   r�   r²   r²   ç   s‘   € € € € € € ðð ð !€JØ$€Oà%Ø)Ø)ðð €Mð €J�ÐÐÑØ€N�CÐÐÑØ Ð�cÐ Ð Ñ Ø€L�%ÐÐÑØ#Ð�uÐ#Ð#Ñ#Ð#Ð#rƒ   r²   c                   ó¢   — e Zd ZU dZdZdZddiZdZee	d<   dZ
ee	d	<   d
Zee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dS )ÚInklingVisionConfigz´
    text_hidden_size (`int`, *optional*, defaults to 6144):
        Dimensionality the vision features are projected to by the last encoder layer, matching the text backbone.
    Úinkling_visionÚvision_configrO   Ún_layersrL   r·   é(   Ú
patch_sizer   Útemporal_patch_sizer   Únum_channelsr[   rM   é   rW   rQ   ra   rb   rr   rs   N)rž   rŸ   r    r¡   r¢   r£   r§   r·   r¨   r©   rÁ   rÂ   rÃ   rM   rO   rQ   rb   rª   rs   r~   rƒ   r�   r¼   r¼     sÇ   € € € € € € ðð ð
 "€JØ%€OØ(¨*Ð5€Mà Ð�cÐ Ð Ñ Ø€J�ÐÐÑØ Ð˜Ð Ð Ñ Ø€L�#ÐÐÑØ€K�ÐÐÑØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø€L�%ÐÐÑØ#Ð�uÐ#Ð#Ñ#Ð#Ð#rƒ   r¼   zETop-level multimodal config (`InklingMMConfig` in the SGLang source).)r*   Úcustom_introc                   óº   ‡ — e Zd ZU dZdZeeedœZdZ	ee
z  dz  ed<   dZee
z  dz  ed<   dZee
z  dz  ed<   dZeed	<   d
Zeed<   dZeed<   dZeed<   ˆ fd„Zˆ xZS )ÚInklingConfiga)  
    image_bos_token_id (`int`, *optional*, defaults to 200005):
        The beginning-of-image token index used to mark the start of image spans.
    audio_bos_token_id (`int`, *optional*, defaults to 200020):
        The beginning-of-audio token index used to mark the start of audio spans.
    Úinkling_mm_model)r.   r´   r¾   Nr.   r´   r¾   iv Úimage_token_idiu Úaudio_token_idiE Úimage_bos_token_idiT Úaudio_bos_token_idc                 ó   •— |                      d¦  «        pi }t          | j        t          ¦  «        r‹| j                             d|                      d¦  «        ¦  «         | j                             d|                      dd¦  «        ¦  «         | j                             d|                      d¦  «        ¦  «         t          | j        t          ¦  «        r | j        d         di | j        ¤Ž| _        n!| j        € | j        d         ¦   «         | _        t          | j        t          ¦  «        r | j        d	         di | j        ¤Ž| _        n!| j        € | j        d	         ¦   «         | _        t          | j        t          ¦  «        r | j        d
         di | j        ¤Ž| _        n!| j        € | j        d
         ¦   «         | _        | j        j        | j        _	        | j        j        | j        _	         t          ¦   «         j        di |¤Ž d S )NÚ
mtp_configrx   Únum_nextn_predict_layersry   Fr{   r_   r´   r¾   r.   r~   )r‘   Ú
isinstancer.   ÚdictÚ
setdefaultr´   Úsub_configsr¾   rM   r·   r“   r”   )r•   r–   rÎ   r—   s      €r�   r”   zInklingConfig.__post_init__5  s  ø€ à—Z’Z Ñ-Ô-Ð3°ˆ
Ý�dÔ&­Ñ-Ô-ð 	bØÔ×'Ò'Ð(8¸*¿.º.ÐIcÑ:dÔ:dÑeÔeÐeØÔ×'Ò'Ð(@À*Ç.Â.ÐQiÐkpÑBqÔBqÑrÔrÐrØÔ×'Ò'Ð(=¸z¿~º~ÐN_Ñ?`Ô?`ÑaÔaÐaå�dÔ'­Ñ.Ô.ð 	CØ @ Ô 0°Ô @Ð UÐ UÀ4ÔCTÐ UÐ UˆDÔÐØÔÐ&Ø @ Ô 0°Ô @Ñ BÔ BˆDÔå�dÔ(­$Ñ/Ô/ð 	EØ!B Ô!1°/Ô!BÐ!XÐ!XÀTÔEWÐ!XÐ!XˆDÔÐØÔÐ'Ø!B Ô!1°/Ô!BÑ!DÔ!DˆDÔå�dÔ&­Ñ-Ô-ð 	AØ>˜tÔ/°Ô>ÐRÐRÀÔAQÐRÐRˆDÔÐØÔÐ%Ø>˜tÔ/°Ô>Ñ@Ô@ˆDÔà.2Ô.>Ô.JˆÔÔ+Ø-1Ô-=Ô-IˆÔÔ*Ø�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'rƒ   )rž   rŸ   r    r¡   r¢   r,   r²   r¼   rÓ   r.   rÑ   r©   r´   r¾   rÉ   r¨   rÊ   rË   rÌ   r”   r¯   r°   s   @r�   rÇ   rÇ     sõ   ø€ € € € € € ðð ð $€Jà(Ø*Ø,ðð €Kð 48€KÐ" TÑ)¨DÑ0Ð7Ð7Ñ7Ø59€LÐ$ tÑ+¨dÑ2Ð9Ð9Ñ9Ø7;€MÐ&¨Ñ-°Ñ4Ð;Ð;Ñ;Ø €N�CÐ Ð Ñ Ø €N�CÐ Ð Ñ Ø$Ð˜Ð$Ð$Ñ$Ø$Ð˜Ð$Ð$Ñ$ð(ð (ð (ð (ð (ð (ð (ð (ð (rƒ   rÇ   c                   ó   — e Zd ZdS )ÚInklingModelOutputWithPastN©rž   rŸ   r    r~   rƒ   r�   rÕ   rÕ   Q  ó   € € € € € Ø€Drƒ   rÕ   c                   ó   — e Zd ZdS )ÚInklingCausalLMOutputWithPastNrÖ   r~   rƒ   r�   rÙ   rÙ   U  r×   rƒ   rÙ   c                   ó   — e Zd ZdS )ÚInklingRMSNormNrÖ   r~   rƒ   r�   rÛ   rÛ   Y  r×   rƒ   rÛ   c                   ól   ‡ — e Zd ZdZdedefˆ fd„Zdej        dej        dej        dej        fd	„Zˆ xZ	S )
ÚInklingRelativeLogitsa|  hidden states conditioned relative position bias. `proj` is a trained bank of bias-vs-distance profiles; each token's
    `relative_states` mixes them into one bias value per backward distance
    (`sglang RelLogitsProj` + the FA4 `score_mod`, materialized densely). The bias is zero
    outside `0 <= distance < rel_extent`; causality and padding stay in the attention mask.
    rZ   r\   c                 ó®   •— t          ¦   «                              ¦   «          || _        t          j        t          j        ||¦  «        ¦  «        | _        d S ©N)r“   Ú__init__r\   ÚnnÚ	ParameterÚtorchÚemptyÚproj)r•   rZ   r\   r—   s      €r�   rà   zInklingRelativeLogits.__init__d  sA   ø€ Ý‰Œ×ÒÑÔÐØ$ˆŒÝ”L¥¤¨U°JÑ!?Ô!?Ñ@Ô@ˆŒ	ˆ	ˆ	rƒ   Úrelative_statesÚquery_positionsÚkey_positionsÚreturnc                 óx  — || j         z                       dd¦  «        }|d d …d f         |d d d …f         z
  d d d d …d d …f         } |                     d| j        dz
  ¦  «        j        g |j        d d…         ¢d‘d‘R Ž }|                     d|¦  «        }|                     |dk     || j        k    z  d¦  «        S )Nru   r   r   éÿÿÿÿrp   )rå   Ú	transposeÚclampr\   ÚexpandÚshapeÚgatherÚmasked_fill)r•   ræ   rç   rè   Ú
rel_logitsÚdistanceÚgather_indexÚposition_biass           r�   ÚforwardzInklingRelativeLogits.forwardi  sä   € ð &¨¬	Ñ1×<Ò<¸QÀÑBÔBˆ
Ø# A A A t GÔ,¨}¸TÀ1À1À1¸WÔ/EÑEÀtÈTÐSTÐSTÐSTÐVWÐVWÐVWÐGWÔXˆØD�x—~’~ a¨¬¸1Ñ)<Ñ=Ô=ÔDÐcÀjÔFVÐWYÐXYÐWYÔFZÐcÐ\^ÐcÐ`bÐcÐcÐcˆØ"×)Ò)¨"¨lÑ;Ô;ˆØ×(Ò(¨(°Qª,¸8ÀtÄÒ;VÑ)WÐY\Ñ]Ô]Ð]rƒ   )
rž   rŸ   r    r¡   r¨   rà   rã   ÚTensorrö   r¯   r°   s   @r�   rÝ   rÝ   ]  s«   ø€ € € € € ðð ðA˜cð A¨sð Að Að Að Að Að Að
^àœð^ð œð^ð ”|ð	^ð
 
Œð^ð ^ð ^ð ^ð ^ð ^ð ^ð ^rƒ   rÝ   rp   ÚmoduleÚqueryÚkeyÚvaluer;   ÚscalingÚdropoutrõ   c                 ó"  — t          || j        ¦  «        }	t          || j        ¦  «        }
t          j        ||	                     dd¦  «        ¦  «        |z  }|�||z   }|�||z   }t
          j                             |dt          j        ¬¦  «         	                    |j
        ¦  «        }t
          j                             ||| j        ¬¦  «        }t          j        ||
¦  «        }|                     dd¦  «                             ¦   «         }||fS )Nr   r   rë   )ÚdimÚdtype)ÚpÚtrainingru   )r%   Únum_key_value_groupsrã   Úmatmulrì   rá   Ú
functionalÚsoftmaxÚfloat32Útor   rý   r  Ú
contiguous)rø   rù   rú   rû   r;   rü   rý   rõ   r–   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs                r�   Úeager_attention_forwardr  w  sù   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ Ø# mÑ3ˆØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$rƒ   c                   ó´   ‡ — e Zd Zdedefˆ fd„Z	 	 ddej        dej        dz  dej        dz  dedz  d	e	e
         d
eej        ej        dz  f         fd„Zˆ xZS )ÚInklingAttentionÚconfigÚ	layer_idxc                 ó2  •— t          ¦   «                              ¦   «          || _        || _        |j        | j                 dk    | _        | j        r|j        n|j        | _        | j        r|j        n|j	        | _
        | j        r|j        n|j        | _        | j
        | j        z  | _        | j        r|j        nd | _        | j        r|j        n|j        | _        d| j        z  | _        |j        | _        d| _        t)          j        |j        | j
        | j        z  d¬¦  «        | _        t)          j        |j        | j        | j        z  d¬¦  «        | _        t)          j        |j        | j        | j        z  d¬¦  «        | _        t)          j        |j        | j
        |j        z  d¬¦  «        | _        t)          j        | j
        | j        z  |j        d¬¦  «        | _        t;          | j        | j        z  |j        |d¬¦  «        | _        t;          | j        | j        z  |j        |d¬¦  «        | _         tC          | j        |j"        ¬	¦  «        | _#        tC          | j        |j"        ¬	¦  «        | _$        tK          |j        | j        ¦  «        | _&        d S )
Nr†   ç      ð?TF©Úbiasr   )Úconv_idxru   ©Úeps)'r“   rà   r  r  r`   Ú
is_slidingrY   rU   rV   rQ   Ú	num_headsrX   rS   r  rA   rF   r\   rü   rq   Ú	is_causalrá   ÚLinearrM   Úq_projÚk_projÚv_projrZ   Úr_projÚo_projÚInklingShortConvolutionrH   Úk_sconvÚv_sconvrÛ   rb   Úq_normÚk_normrÝ   Úrel_logits_proj©r•   r  r  r—   s      €r�   rà   zInklingAttention.__init__”  sW  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒØ Ô,¨T¬^Ô<Ð@PÒPˆŒØ/3¬ÐS˜Ô+Ð+ÀFÄOˆŒØ;?¼?Ðj˜Ô7Ð7ÐPVÔPjˆŒØEIÄ_Ð#t 6Ô#AÐ#AÐZ`ÔZtˆÔ Ø$(¤N°dÔ6NÑ$NˆÔ!Ø<@¼OÐU˜fÔ8Ð8ÐQUˆÔØ8<¼Ð^˜&Ô4Ð4ÈVÔM^ˆŒà˜Tœ]Ñ*ˆŒØ!'Ô!9ˆÔØˆŒå”i Ô 2°D´NÀTÄ]Ñ4RÐY^Ð_Ñ_Ô_ˆŒÝ”i Ô 2°DÔ4LÈtÌ}Ñ4\ÐchÐiÑiÔiˆŒÝ”i Ô 2°DÔ4LÈtÌ}Ñ4\ÐchÐiÑiÔiˆŒÝ”i Ô 2°D´NÀVÄ\Ñ4QÐX]Ð^Ñ^Ô^ˆŒÝ”i ¤°´Ñ >ÀÔ@RÐY^Ð_Ñ_Ô_ˆŒÝ.ØÔ$ t¤}Ñ4°fÔ6NÐPYÐdeð
ñ 
ô 
ˆŒõ /ØÔ$ t¤}Ñ4°fÔ6NÐPYÐdeð
ñ 
ô 
ˆŒõ % T¤]¸Ô8KÐLÑLÔLˆŒÝ$ T¤]¸Ô8KÐLÑLÔLˆŒÝ4°V´\À4Ä?ÑSÔSˆÔÐÐrƒ   Nr:   r;   Ú	conv_maskÚpast_key_valuesr–   ré   c                 ó"  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «        }|                      |                      |¦  «        ||¬¦  «        }	|                      |                      |¦  «        ||¬¦  «        }
|                      |¦  «        }|                      | 	                    |¦  «        ¦  «         
                    dd¦  «        }|                      |	 	                    |¦  «        ¦  «         
                    dd¦  «        }	|
 	                    |¦  «         
                    dd¦  «        }
|j         d         }|�X|                     || j        ¦  «        \  }}|                     | j        ¦  «        }|                     |	|
| j        ¦  «        \  }	}
n|	j         d         }d\  }}t!          j        ||j        ¬¦  «        |z   }t!          j        ||j        ¬¦  «        |z   } |j	        g |¢| j        ‘d‘R Ž }|                      |||¦  «        }| j        sÞ| j        j        �Ò|dz                        ¦   «         }d| j        j        t!          j        || j        j        z                       d¬¦  «        ¦  «        z  z   }| 	                    dddd¦  «        }|                     ¦   «         |z                       |j        ¦  «        }|                     ¦   «         |z                       |j        ¦  «        }t=          j        | j        j         tB          ¦  «        } || ||	|
|f| j"        sd	n| j#        | j$        | j%        |d
œ|¤Ž\  }} |j&        g |¢d‘R Ž  '                    ¦   «         }|  (                    |¦  «        }||fS )Nrë   ©r+  r*  ru   r   )r   r   ©Údevicer  )Úminrp   )rý   rü   rF   rõ   ))rï   rU   r  r$  r  r%  r   r!  r&  Úviewrì   r'  Úget_mask_sizesr  Úget_query_offsetÚupdaterã   Úaranger/  r  r(  r  r  r]   rª   r^   Úlogrí   r  r   r   Úget_interfaceÚ_attn_implementationr  r  rq   rü   rF   Úreshaper	  r"  )r•   r:   r;   r*  r+  r–   Úinput_shapeÚhidden_shapeÚquery_statesr
  r  ræ   Úq_lengthÚ	kv_lengthÚ	kv_offsetÚq_offsetÚkv_positionsÚq_positionsrõ   Úeffective_nÚtauÚattention_interfacer  r  s                           r�   rö   zInklingAttention.forward³  s   € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1ˆØ—\’\ $§+¢+¨mÑ"<Ô"<ÈoÐir�\ÑsÔsˆ
Ø—|’| D§K¢K°Ñ$>Ô$>ÐP_Ðkt�|ÑuÔuˆØŸ+š+ mÑ4Ô4ˆà—{’{ <×#4Ò#4°\Ñ#BÔ#BÑCÔC×MÒMÈaÐQRÑSÔSˆØ—[’[ §¢°Ñ!>Ô!>Ñ?Ô?×IÒIÈ!ÈQÑOÔOˆ
Ø#×(Ò(¨Ñ6Ô6×@Ò@ÀÀAÑFÔFˆàÔ% aÔ(ˆØÐ&à#2×#AÒ#AÀ(ÈDÌNÑ#[Ô#[Ñ ˆI�yØ&×7Ò7¸¼ÑGÔGˆHà'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜˜à"Ô(¨Ô+ˆIØ"&ÑˆH�iå”| I°mÔ6JÐKÑKÔKÈiÑWˆÝ”l 8°MÔ4HÐIÑIÔIÈHÑTˆØ.˜/Ô.ÐP°ÐP¸T¼^ÐPÈRÐPÐPÐPˆØ×,Ò,¨_¸kÈ<ÑXÔXˆð Œð 	R 4¤;Ô#BÐ#NØ&¨™?×1Ò1Ñ3Ô3ˆKØ˜œÔ5½¼	Ø˜tœ{Ô>Ñ>×EÒEÈ#ÐEÑNÔNñ9ô 9ñ ñ ˆCð —(’(˜1˜a  QÑ'Ô'ˆCØ(×.Ò.Ñ0Ô0°3Ñ6×:Ò:¸<Ô;MÑNÔNˆLØ*×0Ò0Ñ2Ô2°SÑ8×<Ò<¸]Ô=PÑQÔQˆMå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð%
ð  $œ}ÐH�C�C°$Ô2HØ”LØÔ.Ø'ð%
ð %
ð ð%
ð %
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(rƒ   ©NN)rž   rŸ   r    r,   r¨   rà   rã   r÷   r   r   r   Útuplerö   r¯   r°   s   @r�   r  r  “  sì   ø€ € € € € ðTÐ0ð T¸Sð Tð Tð Tð Tð Tð TðF *.Ø(,ðA)ð A)à”|ðA)ð œ tÑ+ðA)ð ”< $Ñ&ð	A)ð
  ™ðA)ð Ð+Ô,ðA)ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ðA)ð A)ð A)ð A)ð A)ð A)ð A)ð A)rƒ   r  c                   óH   ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Zˆ xZS )Ú
InklingMLPr  c                 óÎ   •— t          ¦   «                              |¦  «         t          |j                 | _        t          j        t          j        d¦  «        ¦  «        | _	        d S ©Nru   )
r“   rà   r   rg   Úact_fnrá   râ   rã   ÚonesÚglobal_scale©r•   r  r—   s     €r�   rà   zInklingMLP.__init__ø  sJ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý˜VÔ.Ô/ˆŒÝœL­¬°A©¬Ñ7Ô7ˆÔÐÐrƒ   r:   ré   c                 ó¸   — |                       |                      |                      |¦  «        ¦  «        |                      |¦  «        z  ¦  «        }|| j        z  S rß   )Ú	down_projrL  Ú	gate_projÚup_projrN  ©r•   r:   s     r�   rö   zInklingMLP.forwardý  sN   € ØŸš t§{¢{°4·>²>À-Ñ3PÔ3PÑ'QÔ'QÐTX×T`ÒT`ÐanÑToÔToÑ'oÑpÔpˆØ˜tÔ0Ñ0Ð0rƒ   )	rž   rŸ   r    r,   rà   rã   r÷   rö   r¯   r°   s   @r�   rI  rI  ÷  sk   ø€ € € € € ð8Ð0ð 8ð 8ð 8ð 8ð 8ð 8ð
1 U¤\ð 1°e´lð 1ð 1ð 1ð 1ð 1ð 1ð 1ð 1rƒ   rI  c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚInklingExpertsr  c                 óz   •— t          ¦   «                              |¦  «         |j        | _        |j        | _        d S rß   )r“   rà   rB   Únum_expertsrh   Úintermediate_dimrO  s     €r�   rà   zInklingExperts.__init__  s7   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô2ˆÔØ &Ô <ˆÔÐÐrƒ   )rž   rŸ   r    r,   rà   r¯   r°   s   @r�   rV  rV    sE   ø€ € € € € ð=Ð0ð =ð =ð =ð =ð =ð =ð =ð =ð =ð =rƒ   rV  c                   óZ   ‡ — e Zd Zˆ fd„Zdeej        ej        ej        f         fd„Zˆ xZS )ÚInklingTopkRouterc                 ó
  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        | j        | j        z   | _        |j        | _        |j        | _        |j	        | _
        t          j        t          j        | j        |j        ¦  «        ¦  «        | _        t          j        t          j        d¦  «        ¦  «        | _        t          j        t          j        | j        ¦  «        ¦  «        | _        d S rK  )r“   rà   rB   rX  rl   Ún_total_expertsrM   Ú
hidden_dimrn   rk   Útop_krá   râ   rã   rä   ÚweightrM  rN  ÚBufferÚzerosÚe_score_correction_biasrO  s     €r�   rà   zInklingTopkRouter.__init__
  sÃ   ø€ Ý‰Œ×ÒÑÔÐØ!Ô2ˆÔØ &Ô 7ˆÔØ#Ô/°$Ô2GÑGˆÔØ Ô,ˆŒØ!Ô-ˆÔØÔ/ˆŒ
å”l¥5¤;¨tÔ/CÀVÔEWÑ#XÔ#XÑYÔYˆŒÝœL­¬°A©¬Ñ7Ô7ˆÔÝ')¤yµ´¸TÔ=MÑ1NÔ1NÑ'OÔ'OˆÔ$Ð$Ð$rƒ   ré   c                 óð  — |                      d| j        ¦  «        }t          j        || j        ¦  «        }|                     ¦   «         }|dd | j         …f         }|| j        z   }t          j	        || j
        dd¬¦  «        d         }|dd | j         …f         }|d| j         d …f         }	t          j        |                     d|¦  «        |	gd¬¦  «        }
t          j        |
¦  «        }t          j        |t          j        |dd¬¦  «        z
  ¦  «        }|| j        z  | j        z  }|d| j         d …f                              ¦   «         }|dd | j
        …f                              ¦   «         }||||fS )	Nrë   .F)rÿ   Úsortedru   ©rÿ   T)rÿ   Úkeepdim)r9  r^  ÚFÚlinearr`  Úsigmoidrl   rc  rã   Útopkr_  Úcatrð   Ú
logsigmoidÚexpÚ	logsumexprn   rN  r	  )r•   r:   ÚflatÚrouter_logitsÚscoresÚrouted_scoresÚscores_for_choiceÚtopk_indicesÚrouted_logitsÚshared_logitsÚtopk_logitsÚtopk_log_probsÚtopk_weightsÚshared_gammass                 r�   rö   zInklingTopkRouter.forward  sŒ  € Ø×$Ò$ R¨¬Ñ9Ô9ˆÝœ  t¤{Ñ3Ô3ˆð ×&Ò&Ñ(Ô(ˆØ˜sÐ$< tÔ'<Ð&<Ð$<Ð<Ô=ˆØ)¨DÔ,HÑHÐÝ”zÐ"3°T´ZÀRÐPUÐVÑVÔVÐWXÔYˆà% cÐ+C¨dÔ.CÐ-CÐ+CÐ&CÔDˆØ% c¨DÔ,AÐ+AÐ+CÐ+CÐ&CÔDˆÝ”i ×!5Ò!5°b¸,Ñ!GÔ!GÈÐ WÐ]_Ð`Ñ`Ô`ˆÝœ kÑ2Ô2ˆÝ”y µ%´/À.ÐVXÐbfÐ2gÑ2gÔ2gÑ!gÑhÔhˆà# dÔ&6Ñ6¸Ô9JÑJˆà$ S¨4Ô+@Ð*@Ð*BÐ*BÐ%BÔC×NÒNÑPÔPˆØ# C¨¨4¬:¨Ð$5Ô6×AÒAÑCÔCˆà˜l¨L¸-ÐGÐGrƒ   )	rž   rŸ   r    rà   rG  rã   r÷   rö   r¯   r°   s   @r�   r[  r[  	  ss   ø€ € € € € ðPð Pð Pð Pð PðH¨¨e¬l¸E¼LÈ%Ì,Ð.VÔ(Wð Hð Hð Hð Hð Hð Hð Hð Hrƒ   r[  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚInklingSharedExpertsc                 óæ  •— t          ¦   «                              ¦   «          |j        | _        |j        }t	          j        t          j        |j        ||j        ¦  «        ¦  «        | _	        t	          j        t          j        |j        ||j        ¦  «        ¦  «        | _
        t	          j        t          j        |j        |j        |¦  «        ¦  «        | _        t          |j                 | _        d S rß   )r“   rà   rl   rh   rá   râ   rã   rä   rM   rR  rS  rQ  r   rg   rL  )r•   r  rY  r—   s      €r�   rà   zInklingSharedExperts.__init__0  s¶   ø€ Ý‰Œ×ÒÑÔÐØ &Ô 7ˆÔØ!Ô7Ðõ œ¥e¤k°&Ô2IÐK[Ð]cÔ]oÑ&pÔ&pÑqÔqˆŒÝ”|¥E¤K°Ô0GÐIYÐ[aÔ[mÑ$nÔ$nÑoÔoˆŒÝœ¥e¤k°&Ô2IÈ6ÔK]Ð_oÑ&pÔ&pÑqÔqˆŒÝ˜VÔ.Ô/ˆŒˆˆrƒ   c                 óÒ  — |j         }|                     dd|d         ¦  «                             | j        dd¦  «        }|                     d| j        d¦  «                             dd¦  «        }t          j        || j                             dd¦  «        ¦  «        }t          j        || j                             dd¦  «        ¦  «        }|  	                    |¦  «        |z  |z  }t          j        || j
                             dd¦  «        ¦  «        }|                     ¦   «                              d¬¦  «                             |j        ¦  «        }|                     |¦  «        S )Nru   rë   r   r   rf  )rï   r9  rî   rl   rì   rã   ÚbmmrR  rS  rL  rQ  rª   Úsumr  r   r1  )	r•   r:   Úgammasr:  ÚgateÚupÚ	activatedÚdownÚouts	            r�   rö   zInklingSharedExperts.forward<  s)  € Ø#Ô)ˆØ%×-Ò-¨a°°[À´_ÑEÔE×LÒLÈTÔMbÐdfÐhjÑkÔkˆØ—’  DÔ$9¸1Ñ=Ô=×GÒGÈÈ1ÑMÔMˆåŒy˜¨¬×(@Ò(@ÀÀAÑ(FÔ(FÑGÔGˆÝŒY�} d¤l×&<Ò&<¸QÀÑ&BÔ&BÑCÔCˆØ—K’K Ñ%Ô%¨Ñ*¨VÑ3ˆ	ÝŒy˜ D¤N×$<Ò$<¸QÀÑ$BÔ$BÑCÔCˆà�jŠj‰lŒl×Ò 1ÐÑ%Ô%×(Ò(¨Ô)<Ñ=Ô=ˆØ�xŠx˜Ñ$Ô$Ð$rƒ   )rž   rŸ   r    rà   rö   r¯   r°   s   @r�   r}  r}  /  sG   ø€ € € € € ð
0ð 
0ð 
0ð 
0ð 
0ð%ð %ð %ð %ð %ð %ð %rƒ   r}  c                   ó8   ‡ — e Zd ZdZˆ fd„Zdej        fd„Zˆ xZS )Ú
InklingMoEz7Gate -> routed experts (+ shared experts), TML flavour.c                 óÎ   •— t          ¦   «                              ¦   «          || _        t          |¦  «        | _        t          |¦  «        | _        t          |¦  «        | _        d S rß   )	r“   rà   r  r[  rƒ  rV  Úexpertsr}  Úshared_expertsrO  s     €r�   rà   zInklingMoE.__init__M  sT   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ% fÑ-Ô-ˆŒ	Ý% fÑ-Ô-ˆŒÝ2°6Ñ:Ô:ˆÔÐÐrƒ   ré   c                 ó   — |}|j         }|                      |¦  «        \  }}}}|                     d|j         d         ¦  «        } |                      |||¦  «        j        |Ž }||                      ||¬¦  «        z   }|S )Nrë   )r‚  )rï   rƒ  r1  r‹  rŒ  )r•   r:   Ú	residualsr:  Ú_rz  ru  r{  s           r�   rö   zInklingMoE.forwardT  s‰   € Ø!ˆ	Ø#Ô)ˆØ7;·y²yÀÑ7OÔ7OÑ4ˆˆ<˜ }Ø%×*Ò*¨2¨}Ô/BÀ2Ô/FÑGÔGˆØT˜Ÿš ]°LÀ,ÑOÔOÔTÐVaÐbˆØ%¨×(;Ò(;¸IÈmÐ(;Ñ(\Ô(\Ñ\ˆØÐrƒ   )	rž   rŸ   r    r¡   rà   rã   r÷   rö   r¯   r°   s   @r�   r‰  r‰  J  s[   ø€ € € € € ØAÐAð;ð ;ð ;ð ;ð ;ð¨¬ð ð ð ð ð ð ð ð rƒ   r‰  c                   óœ   ‡ — e Zd Zdedededefˆ fd„Z ed¦  «        	 	 ddej        d	edz  d
ej        dz  de	e
         fd„¦   «         Zˆ xZS )r#  rM   rC   r  r  c                 ó¶   •— t          ¦   «                              ¦   «          || _        || _        || _        t          j        |||||dz
  d¬¦  «        | _        d S )Nru   F)Úin_channelsÚout_channelsÚkernel_sizeÚgroupsÚpaddingr  )r“   rà   r  r  rC   rá   ÚConv1dÚconv1d)r•   rM   rC   r  r  r—   s        €r�   rà   z InklingShortConvolution.__init__`  sb   ø€ Ý‰Œ×ÒÑÔÐØ"ˆŒØ ˆŒØ 0ˆÔå”iØ#Ø$Ø(ØØ$ qÑ(Øð
ñ 
ô 
ˆŒˆˆrƒ   r˜  Nr:   r+  r*  r–   c                 óX  — |j         }|                     ¦   «         }|}t          ||¦  «        }|j        d         }|                     dd¦  «        }|d uo|                     | j        | j        ¦  «        }|ry|dk    rs|j        | j                 j	        s\|j        | j                 j
        | j                 }	t          ||	| j        j                             d¦  «        | j        j        ¦  «        }n‹|�(|                     || j        | j        | j        ¬¦  «        }t%          || j        j                             d¦  «        | j        j        |                     d¦  «        ¬¦  «        }|�|d d …d d …| d …f         }|                     dd¦  «        }||z                        |¬¦  «        }|S )Nru   r   )Ú	state_idxrC   Úseq_idx)r›  )r   )r   rª   r'   rï   rì   Úhas_previous_stater  r  r<   Úrecord_pastÚconv_statesr)   r˜  r`  Úsqueezer  Úupdate_conv_staterC   r(   r‘   r  )
r•   r:   r+  r*  r–   Úinput_dtypeÚresidualÚseq_lenÚuse_precomputed_statesÚ
conv_states
             r�   rö   zInklingShortConvolution.forwardo  sÇ  € ð $Ô)ˆØ%×+Ò+Ñ-Ô-ˆà ˆÝ4°]ÀIÑNÔNˆØÔ% aÔ(ˆØ%×/Ò/°°1Ñ5Ô5ˆà!0¸Ð!<ð "
À×AcÒAcØŒN˜DœMñB
ô B
Ðð "ð 	? g°¢l l¸?Ô;QÐRVÔR`Ô;aÔ;m lØ(Ô/°´Ô?ÔKÈDÌMÔZˆJå0Ø˜z¨4¬;Ô+=×+EÒ+EÀaÑ+HÔ+HÈ$Ì+ÔJZñô ˆMˆMð Ð*Ø /× AÒ AØ! 4¤>¸T¼]Ð]aÔ]rð !Bñ !ô !�õ -Ø˜tœ{Ô1×9Ò9¸!Ñ<Ô<¸d¼kÔ>NÐX^×XbÒXbÐclÑXmÔXmðñ ô ˆMð
 Ð*Ø -¨a¨a¨a°°°°W°H°I°I¨oÔ >�à%×/Ò/°°1Ñ5Ô5ˆØ&¨Ñ1×5Ò5¸KÐ5ÑHÔHˆØÐrƒ   rF  )rž   rŸ   r    r¨   rà   r   rã   r÷   r   r   r   rö   r¯   r°   s   @r�   r#  r#  ^  sÌ   ø€ € € € € ð
 Cð 
¸3ð 
È3ð 
ÐZ]ð 
ð 
ð 
ð 
ð 
ð 
ð Ð˜HÑ%Ô%ð )-Ø)-ð	*ð *à”|ð*ð  ™ð*ð ”< $Ñ&ð	*ð
 Ð+Ô,ð*ð *ð *ñ &Ô%ð*ð *ð *ð *ð *rƒ   r#  c                   ó–   ‡ — e Zd Zdedefˆ fd„Z	 	 	 ddej        dej        dz  dej        dz  dedz  d	e	e
         d
ej        fd„Zˆ xZS )ÚInklingDecoderLayerr  r  c                 ó&  •— t          ¦   «                              ¦   «          |j        | _        t          ||¦  «        | _        |j        |         dk    rt          |¦  «        | _        nt          |¦  «        | _        t          |j        |j
        ¦  «        | _        t          |j        |j
        ¦  «        | _        |j        |         | _        t          |j        |j        |d¬¦  «        | _        t          |j        |j        |d¬¦  «        | _        d S )NrŒ   r   )r  r  r   )r“   rà   rM   r  Ú	self_attnrd   r‰  ÚmlprI  rÛ   rb   Úinput_layernormÚpost_attention_layernormr`   Ú
layer_typer#  rC   Ú
attn_sconvÚ	mlp_sconvr)  s      €r�   rà   zInklingDecoderLayer.__init__ž  sù   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔÝ)¨&°)Ñ<Ô<ˆŒàÔ! )Ô,°Ò8Ð8Ý! &Ñ)Ô)ˆDŒHˆHå! &Ñ)Ô)ˆDŒHå-¨fÔ.@À&ÔBUÑVÔVˆÔÝ(6°vÔ7IÈ6ÔK^Ñ(_Ô(_ˆÔ%Ø Ô,¨YÔ7ˆŒÝ1ØÔ Ô 7À9ÐWXð
ñ 
ô 
ˆŒõ 1ØÔ Ô 7À9ÐWXð
ñ 
ô 
ˆŒˆˆrƒ   Nr:   r;   r*  r+  r–   ré   c                 ó*  — |}|                       |¦  «        } | j        d||||dœ|¤Ž\  }}|                      |||¬¦  «        }||z   }|}|                      |¦  «        }|                      |¦  «        }|                      |||¬¦  «        }||z   }|S )N)r:   r;   r*  r+  r-  r~   )r«  r©  r®  r¬  rª  r¯  )r•   r:   r;   r*  r+  r–   r¢  r�  s           r�   rö   zInklingDecoderLayer.forward²  sÉ   € ð !ˆØ×,Ò,¨]Ñ;Ô;ˆØ)˜4œ>ð 
Ø'Ø)ØØ+ð	
ð 
ð
 ð
ð 
Ñˆ�qð Ÿš¨ÀÐbk˜ÑlÔlˆØ  =Ñ0ˆà ˆØ×5Ò5°mÑDÔDˆØŸš Ñ/Ô/ˆØŸš }ÀoÐaj˜ÑkÔkˆØ  =Ñ0ˆØÐrƒ   )NNN)rž   rŸ   r    r,   r¨   rà   rã   r÷   r   r   r   rö   r¯   r°   s   @r�   r§  r§  �  sÆ   ø€ € € € € ð
Ð0ð 
¸Sð 
ð 
ð 
ð 
ð 
ð 
ð. /3Ø)-Ø(,ðð à”|ðð œ tÑ+ðð ”< $Ñ&ð	ð
  ™ðð Ð+Ô,ðð 
Œðð ð ð ð ð ð ð rƒ   r§  c                   óŒ   ‡ — e Zd ZeZdZdZdgZdgZdZ	dZ
dZdZdZdgZg d¢ZeedœZ ej        ¦   «         ˆ fd	„¦   «         Zˆ xZS )
ÚInklingPreTrainedModelÚmodelTr§  r+  Fzmodel\.mtp\..*)r®  r¯  r$  r%  )r:   Ú
attentionsc                 ó`  •— t          ¦   «                              |¦  «         | j                             ¦   «         j        }t          |t          ¦  «        rt          j        |j	        d|¬¦  «         d S t          |t          ¦  «        rt          j        |j        ¦  «         d S t          |t          ¦  «        r:t          j        |j        d|¬¦  «         t          j        |j        d|¬¦  «         d S t          |t           ¦  «        rPt          j        |j        d|¬¦  «         t          j        |j        ¦  «         t          j        |j        ¦  «         d S t          |t(          ¦  «        rVt          j        |j        d|¬¦  «         t          j        |j        d|¬¦  «         t          j        |j        d|¬¦  «         d S t          |t.          ¦  «        rVt1          | j        d| j        ¦  «        }t          j        |j        t7          j        |j        ¦  «        |j        z  ¦  «         d S d S )Nrp   )ÚmeanÚstdr´   )r“   Ú_init_weightsr  Úget_text_configrs   rÐ   rÝ   ÚinitÚnormal_rå   rI  Úones_rN  rV  Úgate_up_projrQ  r[  r`  Úzeros_rc  r}  rR  rS  ÚInklingAudioModelEmbeddingsÚgetattrÚcopy_Úaudio_tokens_offsetsrã   r5  rµ   r¶   )r•   rø   r·  r´   r—   s       €r�   r¸  z$InklingPreTrainedModel._init_weightsã  s  ø€ å‰Œ×Ò˜fÑ%Ô%Ð%ØŒk×)Ò)Ñ+Ô+Ô=ˆÝ�fÕ3Ñ4Ô4ð 	ÝŒL˜œ¨3°CÐ8Ñ8Ô8Ð8Ð8Ð8Ý˜¥
Ñ+Ô+ð 	ÝŒJ�vÔ*Ñ+Ô+Ð+Ð+Ð+Ý˜¥Ñ/Ô/ð 	ÝŒL˜Ô,°3¸CÐ@Ñ@Ô@Ð@ÝŒL˜Ô)°¸Ð=Ñ=Ô=Ð=Ð=Ð=Ý˜Õ 1Ñ2Ô2ð 	ÝŒL˜œ¨S°cÐ:Ñ:Ô:Ð:ÝŒJ�vÔ*Ñ+Ô+Ð+ÝŒK˜Ô6Ñ7Ô7Ð7Ð7Ð7Ý˜Õ 4Ñ5Ô5ð 	ÝŒL˜Ô)°¸Ð=Ñ=Ô=Ð=ÝŒL˜œ¨c°sÐ;Ñ;Ô;Ð;ÝŒL˜Ô)°¸Ð=Ñ=Ô=Ð=Ð=Ð=Ý˜Õ ;Ñ<Ô<ð 	õ # 4¤;°ÀÄÑLÔLˆLÝŒJØÔ+Ý”˜\Ô4Ñ5Ô5¸Ô8SÑSñô ð ð ð ð		ð 	rƒ   )rž   rŸ   r    rÇ   Úconfig_classÚbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendÚ"_keys_to_ignore_on_load_unexpectedÚ_keep_in_fp32_modules_strictr§  r  Ú_can_record_outputsrã   Úno_gradr¸  r¯   r°   s   @r�   r²  r²  Î  s¶   ø€ € € € € à €LØÐØ&*Ð#Ø.Ð/ÐØ#4Ð"5Ðð !ÐØ€NØÐØ"ÐØ"'ÐØ*;Ð)<Ð&Ø#TÐ#TÐ#TÐ à,Ø&ðð Ðð
 €U„]�_„_ðð ð ð ñ „_ðð ð ð ð rƒ   r²  c                   óî   ‡ — e Zd ZU eed<   defˆ fd„Zeee	 	 	 	 	 	 dde	j
        dz  de	j        dz  de	j
        dz  dedz  de	j        dz  d	edz  d
ee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚInklingTextModelr  c                 óô  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          j        ‰j        ‰j        | j        ¦  «        | _        t          j	        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        d| _        |                      ¦   «          d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r~   )r§  )r   r  r  s     €r�   rˆ   z-InklingTextModel.__init__.<locals>.<listcomp>  s$   ø€ ÐeÐeÐe¸	Õ  ¨Ñ3Ô3ÐeÐeÐerƒ   r  F)r“   rà   rt   Úpadding_idxrJ   rá   Ú	EmbeddingrM   r4   Ú
ModuleListr�   rO   r<   rÛ   rb   r=   Ú
embed_normÚgradient_checkpointingÚ	post_initrO  s    `€r�   rà   zInklingTextModel.__init__  sÜ   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒåœL¨Ô):¸FÔ<NÐPTÔP`ÑaÔaˆÔÝ”mØeÐeÐeÐeÅUÈ6ÔKcÑEdÔEdÐeÑeÔeñ
ô 
ˆŒõ # 6Ô#5¸6Ô;NÐOÑOÔOˆŒ	Ý(¨Ô);ÀÔATÐUÑUÔUˆŒØ&+ˆÔ#ð 	�ŠÑÔÐÐÐrƒ   Nr8   r;   Úposition_idsr+  r9   Ú	use_cacher–   ré   c                 óæ  — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|                      |¦  «        }|r|€t          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j	        ¬¦  «        |z   }| 
                    d¦  «        }t          |x}	t          ¦  «        s1| j        ||||dœ}
t          di |
¤Žt          di |
¤Žt          di |
¤Ždœ}	|}t!          | j        ¦  «        D ]8\  }}| j        j        |         dk    rd	nd
} ||f|	|         |	d         |dœ|¤Ž}Œ9|                      |¦  «        }t)          ||¬¦  «        S )Nú:You must specify exactly one of input_ids or inputs_embeds)r  r   ru   r.  ©r  r9   r;   r+  rÛ  ©Úfull_attentionÚsliding_attentionÚlinear_attentionr‡   rá  râ  rã  )r;   r*  r+  )Úlast_hidden_stater+  r~   )Ú
ValueErrorr4   rØ  r	   r  Úget_seq_lengthrã   r5  rï   r/  Ú	unsqueezerÐ   rÑ   r   r   r   Ú	enumerater<   r`   r=   r   )r•   r8   r;   rÛ  r+  r9   rÜ  r–   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsr:   r€   Údecoder_layerÚattention_types                  r�   rö   zInklingTextModel.forward  sû  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàŸš¨Ñ6Ô6ˆàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLõ °Ð?Ð-ÅÑFÔFð 	àœ+Ø!.Ø"0Ø#2Ø ,ðð ˆKõ #5Ð"CÐ"C°{Ð"CÐ"CÝ%FÐ%UÐ%UÈÐ%UÐ%UÝ$CÐ$RÐ$RÀkÐ$RÐ$Rð#ð #Ðð &ˆÝ )¨$¬+Ñ 6Ô 6ð 	ð 	ÑˆAˆ}Ø15´Ô1HÈÔ1KÈxÒ1WÐ1WÐ-Ð-Ð]pˆNØ)˜MØðà2°>ÔBØ-Ð.@ÔAØ /ð	ð ð
 ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆÝ&Ø+Ø+ð
ñ 
ô 
ð 	
rƒ   )NNNNNN)rž   rŸ   r    r,   r©   rà   r   r   r   rã   Ú
LongTensorr÷   r   ÚFloatTensorr­   r   r   r   rö   r¯   r°   s   @r�   rÒ  rÒ     s  ø€ € € € € € àÐÐÑðÐ0ð ð ð ð ð ð ð   ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ð8
ð 8
àÔ# dÑ*ð8
ð œ tÑ+ð8
ð Ô&¨Ñ-ð	8
ð
  ™ð8
ð Ô(¨4Ñ/ð8
ð ˜$‘;ð8
ð Ð+Ô,ð8
ð 
!ð8
ð 8
ð 8
ñ „^ñ „_ñ  Ôð8
ð 8
ð 8
ð 8
ð 8
rƒ   rÒ  c                   óü   — e Zd Zi ZddiZddiZee	 	 	 	 	 	 	 	 ddej	        dz  dej
        dz  dej	        dz  d	edz  d
ej        dz  dej	        dz  dedz  deej
        z  dee         defd„¦   «         ¦   «         ZdS )ÚInklingForCausalLMÚlm_headÚrowwise_split_inputÚkeep_full_weightNr   r8   r;   rÛ  r+  r9   ÚlabelsrÜ  Úlogits_to_keepr–   ré   c	           
      óÂ  —  | j         d||||||dœ|	¤Ž}
|
j        | j        j        z  }t	          |t
          ¦  «        rt          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }| j        j        }|�||j	        d         k     r|dd|…f         }d}|� | j
        d|||j	        d         dœ|	¤Ž}t          |||
j        |
j        |
j        ¬¦  «        S )aš  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, InklingForCausalLM

        >>> model = InklingForCausalLM.from_pretrained("thinkingmachines/Inkling-NVFP4")
        >>> tokenizer = AutoTokenizer.from_pretrained("thinkingmachines/Inkling-NVFP4")

        >>> prompt = "What is your favorite condiment?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(**inputs, max_new_tokens=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "What is your favorite condiment?"
        ```)r8   r;   rÛ  r+  r9   rÜ  Nrë   .©Úlogitsrõ  rJ   )Úlossrù  r+  r:   r´  r~   )r³  rä  r  r@   rÐ   r¨   Úslicerò  rK   rï   Úloss_functionrÙ   r+  r:   r´  )r•   r8   r;   rÛ  r+  r9   rõ  rÜ  rö  r–   Úoutputsr:   Úslice_indicesrù  rK   rú  s                   r�   rö   zInklingForCausalLM.forwardX  s;  € ð> �$”*ð 
ØØ)Ø%Ø+Ø'Øð
ð 
ð ð
ð 
ˆð  Ô1°D´KÔ4[Ñ[ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆØ"œkÔ=ÐØÐ*Ð/BÀVÄ\ÐRTÔEUÒ/UÐ/UØ˜CÐ!5Ð"5Ð!5Ð5Ô6ˆFàˆØÐØ%�4Ô%Ðj¨V¸FÈvÌ|Ð\^ÔO_ÐjÐjÐciÐjÐjˆDå,ØØØ#Ô3Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
rƒ   )NNNNNNNr   )rž   rŸ   r    Ú_tied_weights_keysÚ_tp_planÚ
_fsdp_planr   r   rã   rî  r÷   r   rï  r­   r¨   r   r   rÙ   rö   r~   rƒ   r�   rñ  rñ  R  s  € € € € € àÐØÐ0Ð1€HØÐ/Ð0€JàØð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ð9
ð 9
àÔ# dÑ*ð9
ð œ tÑ+ð9
ð Ô&¨Ñ-ð	9
ð
  ™ð9
ð Ô(¨4Ñ/ð9
ð Ô  4Ñ'ð9
ð ˜$‘;ð9
ð ˜eœlÑ*ð9
ð Ð+Ô,ð9
ð 
'ð9
ð 9
ð 9
ñ „^ñ Ôð9
ð 9
ð 9
rƒ   rñ  c                   ó   — e Zd ZdS )r¿  NrÖ   r~   rƒ   r�   r¿  r¿  –  s   € € € € € € € rƒ   r¿  c                   óN   ‡ — e Zd Zdefˆ fd„Zedej        defd„¦   «         Z	ˆ xZ
S )ÚInklingAudioModelr  c                 ó¨   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          |j        d¬¦  «        | _        d S )Nra   r  )r“   rà   r¿  Úembed_audio_tokensrÛ   r·   r=   rO  s     €r�   rà   zInklingAudioModel.__init__›  sF   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý"=¸fÑ"EÔ"EˆÔÝ" 6Ô#:ÀÐEÑEÔEˆŒ	ˆ	ˆ	rƒ   Úaudio_input_idsré   c                 óx   — |                       |¦  «        }|                      |¦  «        }t          ||¬¦  «        S )zŸ
        audio_input_ids (`torch.Tensor` of shape `(num_audios, max_num_frames, n_mel_bins)`):
            Mel-spectrogram frames of the input audios.
        ©rä  Úpooler_output)r  r=   r   )r•   r  r–   r:   s       r�   rö   zInklingAudioModel.forward   sE   € ð ×/Ò/°Ñ@Ô@ˆØŸ	š	 -Ñ0Ô0ˆÝ)Ø+Ø'ð
ñ 
ô 
ð 	
rƒ   )rž   rŸ   r    r²   rà   r   rã   r÷   r   rö   r¯   r°   s   @r�   r  r  ™  s}   ø€ € € € € ðFÐ1ð Fð Fð Fð Fð Fð Fð
 ð

 u¤|ð 

ÐB\ð 

ð 

ð 

ñ „^ð

ð 

ð 

ð 

ð 

rƒ   r  c            
       ó|   ‡ — e Zd Zdededededef
ˆ fd„Zdej        dej        fd	„Zdej        dej        fd
„Z	ˆ xZ
S )ÚInklingVisionEncoderLayerÚ	input_dimÚ
output_dimÚt_foldÚhw_foldÚadd_normc                 óÖ   •— t          ¦   «                              ¦   «          t          j        ||d¬¦  «        | _        |rt          |¦  «        | _        || _        || _        || _	        d S ©NFr  )
r“   rà   rá   r  Ú
projectionrÛ   Ú
layer_normr  r  r  )r•   r  r  r  r  r  r—   s         €r�   rà   z"InklingVisionEncoderLayer.__init__¯  sa   ø€ Ý‰Œ×ÒÑÔÐÝœ) I¨zÀÐFÑFÔFˆŒØð 	9Ý,¨ZÑ8Ô8ˆDŒOØˆŒØˆŒØ ˆŒˆˆrƒ   r:   ré   c           
      óL  — |j         \  }}}}}|| j        z  }|| j        z  }|| j        z  }	|                     ||| j        || j        |	| j        |¦  «        }|                     dddddddd¦  «        }|                     ||||	| j        | j        z  | j        z  |z  ¦  «        }|S )	z€
        Convert a tensor of shape (B, T, H, W, C) to a tensor of shape (B, T // t, H // hw, W //  hw, C * (t * hw**2))
        r   ru   r   é   r   rc   rj   é   )rï   r  r  r9  Úpermute)
r•   r:   ÚBÚTÚHÚWÚCÚt_newÚh_newÚw_news
             r�   Úfold_timespace_to_depthz1InklingVisionEncoderLayer.fold_timespace_to_depth¸  sÁ   € ð &Ô+‰ˆˆ1ˆa��Aà�T”[Ñ ˆØ�T”\Ñ!ˆØ�T”\Ñ!ˆà%×-Ò-¨a°¸¼ÀUÈDÌLÐZ_ÐaeÔamÐopÑqÔqˆà%×-Ò-¨a°°A°q¸!¸QÀÀ1ÑEÔEˆØ%×-Ò-¨a°¸¸uÀdÄkÐTXÔT`ÑF`ÐcgÔcoÑFoÐrsÑFsÑtÔtˆØÐrƒ   c                 óæ   — | j         dk    s| j        dk    r|                      |¦  «        }|                      |¦  «        }| j        r)|                      |¦  «        }t          j        |¦  «        }|S rK  )r  r  r"  r  r  r  rh  ÚgelurT  s     r�   rö   z!InklingVisionEncoderLayer.forwardÈ  sm   € ØŒ<˜!ÒÐ˜tœ{¨Qš˜Ø ×8Ò8¸ÑGÔGˆMàŸš¨Ñ6Ô6ˆØŒ=ð 	2Ø ŸOšO¨MÑ:Ô:ˆMÝœF =Ñ1Ô1ˆMØÐrƒ   )rž   rŸ   r    r¨   r­   rà   rã   r÷   r"  rö   r¯   r°   s   @r�   r  r  ®  s­   ø€ € € € € ð! #ð !°3ð !Àð !Ècð !Ð]að !ð !ð !ð !ð !ð !ð°U´\ð ÀeÄlð ð ð ð ð  U¤\ð °e´lð ð ð ð ð ð ð ð rƒ   r  Únumberré   c                 óB  — g }| dz  dk    r#|                      d¦  «         | dz  } | dz  dk    °#t          dt          j        | ¦  «        dz   d¦  «        D ].}| |z  dk    r#|                      |¦  «         | |z  } | |z  dk    °#Œ/| dk    r|                      | ¦  «         |S )Nr   r   r   ru   )Úappendr�   ÚmathÚisqrt)r%  Úfactorsr  s      r�   Úprime_factorsr+  Ó  sÄ   € Ø€Gà
�1‰*˜Š/ˆ/Ø�Š�qÑÔÐØ�1‰ˆð �1‰*˜Š/ˆ/õ �1•d”j Ñ(Ô(¨1Ñ,¨aÑ0Ô0ð ð ˆØ�q‰j˜AŠoˆoØ�NŠN˜1ÑÔÐØ�q‰LˆFð �q‰j˜AŠoˆoøð �‚z€zØ�Š�vÑÔÐØ€Nrƒ   ÚcpurÂ   rÁ   r¿   Ú
n_channelsc           	      óø  — t          j        t          j        t          |¦  «        ddd…         |¬¦  «        d¬¦  «        }t          j        t          j        t          | ¦  «        ddd…         |¬¦  «        d¬¦  «        }t          j        |dz  |z  dz  ¦  «                             ¦   «         dz  }t          j        |d         dz  |z  |z  ¦  «                             ¦   «         dz  }t          j        ddd|gg|¬¦  «        }	t          j        t          j        |¦  «        |||gd¬¦  «        }
t          j        |t          j        ||d         ¦  «        t          j        ||d         ¦  «        |gd¬¦  «        }t          j	        |	|
|gd¬¦  «        }t          j
        |dd…dd…f         d¬¦  «                             ¦   «         }||z  | z  |z  }t          j        dt          j        t          j        ||¬¦  «        ¦  «        |dz   |¬¦  «        }t          j        |                     d¦  «        t          j        |¦  «                             d¦  «        z
  ¦  «        }||j        d         k    rt          j        |d¬¦  «        }nNdd	lm}  ||                     ¦   «                              ¦   «         ¦  «        \  }}t          j        ||¬¦  «        }d|d<   |j        d         dz
  |d<   ||         S )
a  
    Plan out the dimensions for each layer in the HMLP encoder.

    This function determines the progression of dimensions (temporal, height, width, channels)
    for a multi-layer perceptual model that processes image/video patches. It follows these
    principles:
    1. Start with small dimensions and increase to full size
    2. Expand spatial dimensions (height/width) first, then temporal
    3. Increase channel count to avoid information bottlenecks
    4. Round channel dimensions to multiples of 64 for hardware efficiency

    The function computes optimal assignments of scale configurations to layers using either:
    - For n_layers >= len(scales): Individual best matching scales for each layer (allowing duplicates)
    - For n_layers < len(scales): Global optimal assignment via linear_sum_assignment

    The first and last scales are always fixed to ensure the proper input and output dimensions.

    Args:
        temporal_patch_size: Temporal dimension of input patches
        patch_size: Spatial dimension (height/width) of input patches
        n_layers: Number of layers in the encoder
        n_channels: Number of input channels (default: 3 for RGB)

    Returns:
        torch.LongTensor of shape `(n_layers + 1, 4)` where the last dim holds values for (t, h, w, c) grids.
    Nrë   r.  r   rf  r   rP   ru   )Úlinear_sum_assignment)rã   ÚcumprodÚtensorr+  Úceilr¨   ÚstackÚ	ones_likeÚ	full_likerl  Úprodrª   Úlinspacer6  Úabsrç  rï   ÚargminÚscipy.optimizer/  r,  Únumpy)rÂ   rÁ   r¿   r-  r/  ÚhÚtÚh_chÚt_chÚbaseÚspatialÚtemporalÚscalesÚsize_reductionÚtotal_elementsÚlog_ideal_scalesÚcost_matrixÚidxsr/  r�  Úidxs_nps                        r�   Úplan_out_scalesrJ  ä  sÖ  € õ: 	Œ•e”l¥=°Ñ#<Ô#<¸T¸T¸r¸TÔ#BÈ6ÐRÑRÔRÐXYÐZÑZÔZ€AÝŒ•e”l¥=Ð1DÑ#EÔ#EÀdÀdÈÀdÔ#KÐTZÐ[Ñ[Ô[ÐabÐcÑcÔc€AåŒ:�a˜‘d˜ZÑ'¨"Ñ,Ñ-Ô-×1Ò1Ñ3Ô3°bÑ8€DÝŒ:�a˜”e˜q‘j :Ñ-°Ñ1Ñ2Ô2×6Ò6Ñ8Ô8¸2Ñ=€DåŒ<˜!˜Q  :Ð.Ð/¸Ð?Ñ?Ô?€DÝŒk�5œ?¨1Ñ-Ô-¨q°!°TÐ:ÀÐBÑBÔB€GÝŒ{˜A�uœ¨q°!°B´%Ñ8Ô8½%¼/È!ÈQÈrÌUÑ:SÔ:SÐUYÐZÐ`aÐbÑbÔb€HÝŒY˜˜g xÐ0°aÐ8Ñ8Ô8€Få”Z  q q q¨#¨2¨# v¤°AÐ6Ñ6Ô6×<Ò<Ñ>Ô>€Nà *Ñ,Ð/BÑBÀZÑO€NÝ”~Ø	�5Œ9•U”\ .¸Ð@Ñ@Ô@ÑAÔAÀ8ÈaÁ<ÐX^ðñ ô Ðõ ”)Ð,×6Ò6°qÑ9Ô9½E¼IÀnÑ<UÔ<U×<_Ò<_Ð`aÑ<bÔ<bÑbÑcÔc€Kà�6”< ”?Ò"Ð"ÝŒ|˜K¨QÐ/Ñ/Ô/ˆˆà8Ð8Ð8Ð8Ð8Ð8à*Ð*¨;¯?ª?Ñ+<Ô+<×+BÒ+BÑ+DÔ+DÑEÔE‰
ˆˆ7ÝŒ|˜G¨FÐ3Ñ3Ô3ˆð €Dˆ�GØŒ|˜AŒ Ñ"€Dˆ�HØ�$Œ<Ðrƒ   c                   ó^   ‡ — e Zd Zdefˆ fd„Zedej        dee	         de
fd„¦   «         Zˆ xZS )ÚInklingVisionModelr  c                 óH  •— t          ¦   «                              |¦  «         t          |j        |j        |j        |j        ¦  «        | _        t          j	        ¦   «         | _
        t          t          | j        d d…         | j        dd …         ¦  «        ¦  «        D ]×\  }\  }}|d         |d         z  |d         |d         z  z  |d         |d         z  z  }||j        dz
  k    r|j        n|d         }t          |d         |d         z  ¦  «        }t          |d         |d         z  ¦  «        }| j
                             t!          |d         |z  |||||j        dz
  k    ¬¦  «        ¦  «         ŒØt#          |j        ¦  «        | _        |                      ¦   «          d S )Nrë   ru   r   r   r   )r  r  r  r  r  )r“   rà   rJ  rÂ   rÁ   rO   rÃ   rC  rá   r×  Úencoder_layersrè  Úzipr·   r¨   r'  r  rÛ   Ú
final_normrÚ  )
r•   r  r€   Ústart_scaleÚ	end_scaleÚshuffle_multr  r  r  r—   s
            €r�   rà   zInklingVisionModel.__init__$  sµ  ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý%ØÔ&ØÔØÔ$ØÔñ	
ô 
ˆŒõ !œm™oœoˆÔÝ+4µS¸¼ÀSÀbÀSÔ9IÈ4Ì;ÐWXÐWYÐWYÌ?Ñ5[Ô5[Ñ+\Ô+\ð 	ð 	Ñ'ˆAÑ'�˜Yà˜1” ¨Q¤Ñ/°I¸a´LÀKÐPQÄNÑ4RÑSÐW`ÐabÔWcÐgrÐstÔguÑWuÑvð ð 56¸Ô9QÐTUÑ9UÒ4UÐ4U˜Ô0Ð0Ð[dÐefÔ[gˆJÝ˜) Aœ,¨+°a¬.Ñ8Ñ9Ô9ˆGÝ˜ 1œ¨°Q¬Ñ7Ñ8Ô8ˆFØÔ×&Ò&Ý)Ø)¨!œn¨|Ñ;Ø)Ø#Ø!Ø &Ô":¸QÑ">Ò>ðñ ô ñô ð ð õ )¨Ô)@ÑAÔAˆŒØ�ŠÑÔÐÐÐrƒ   Úpixel_valuesr–   ré   c                 óÆ   — |j         d         }|}| j        D ]} ||¬¦  «        }Œ|                      |¦  «        }|                     |dd¦  «        }t	          ||¬¦  «        S )Nr   )r:   ru   rë   r	  )rï   rN  rP  r9  r   )r•   rT  r–   Únum_patchesr:   Úlayers         r�   rö   zInklingVisionModel.forwardC  s   € à"Ô(¨Ô+ˆØ$ˆØÔ(ð 	?ð 	?ˆEØ!˜E°Ð>Ñ>Ô>ˆMˆMàŸš¨Ñ6Ô6ˆØ%×-Ò-¨k¸1¸bÑAÔAˆÝ)Ø+Ø'ð
ñ 
ô 
ð 	
rƒ   )rž   rŸ   r    r¼   rà   r   rã   r÷   r   r   r   rö   r¯   r°   s   @r�   rL  rL  "  sƒ   ø€ € € € € ðÐ2ð ð ð ð ð ð ð> ð
 E¤Lð 
¸FÐCUÔ<Vð 
Ð[uð 
ð 
ð 
ñ „^ð
ð 
ð 
ð 
ð 
rƒ   rL  zz
    The Base Inkling model which consists of a vision backbone and a language model without language modeling head.,
    ©rÅ   c                   óP  ‡ — e Zd ZdZdefˆ fd„Ze ed¬¦  «        dej	        de
e         deez  fd	„¦   «         ¦   «         Ze ed
¬¦  «        	 ddej        dej        dz  deez  fd„¦   «         ¦   «         Zdej        dej	        dej	        defd„Zee	 	 	 	 	 	 	 	 	 	 	 ddej        dz  dej	        dz  dej        dz  dej        dz  dej        dz  dej        dz  dedz  dej        dz  dej	        dz  dej        dz  dedz  de
e         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚInklingModelFr  c                 ó*  •— t          ¦   «                              |¦  «         |j        j        | _        t	          |j        ¦  «        | _        t          |j        ¦  «        | _        t          |j
        ¦  «        | _        |                      ¦   «          d S rß   )r“   rà   r.   rJ   rÒ  Úlanguage_modelr  r´   Úaudio_towerrL  r¾   Úvision_towerrÚ  rO  s     €r�   rà   zInklingModel.__init__[  sw   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô,Ô7ˆŒÝ.¨vÔ/AÑBÔBˆÔÝ,¨VÔ-@ÑAÔAˆÔÝ.¨vÔ/CÑDÔDˆÔØ�ŠÑÔÐÐÐrƒ   zOProjects the last hidden state from the vision model into language model space.rX  rT  r–   ré   c                 ó    —  | j         dd|i|¤ŽS )NrT  r~   )r^  ©r•   rT  r–   s      r�   Úget_image_featureszInklingModel.get_image_featuresc  s"   € ð
 !ˆtÔ ÐEÐE¨lÐE¸fÐEÐEÐErƒ   zCProjects discretized dMel bin tokens into the language model space.Nr  Úaudio_input_ids_maskc                 ó¨   — |�||                      ¦   «                  }n!|                     d|j        d         ¦  «        }|                      |¦  «        S )aì  
        audio_input_ids (`torch.LongTensor` of shape `(num_audios, max_num_frames, n_mel_bins)`):
            Batch of (padded) dMel bin tokens produced by [`InklingProcessor`].
        audio_input_ids_mask (`torch.Tensor` of shape `(num_audios, max_num_frames)`, *optional*):
            Mask marking valid (non-padding) frames. When provided, only valid frames are encoded so that the
            number of returned audio embeddings matches the number of audio placeholder tokens.
        Nrë   )r­   r9  rï   r]  )r•   r  rb  s      r�   Úget_audio_featureszInklingModel.get_audio_featuresj  sU   € ð  Ð+Ø-Ð.B×.GÒ.GÑ.IÔ.IÔJˆOˆOà-×5Ò5°b¸/Ô:OÐPRÔ:SÑTÔTˆOØ×Ò Ñ0Ô0Ð0rƒ   r8   r9   ÚfeaturesÚtoken_idc           
      óB  — |€s|| j                               |                      ¦   «         t          j        |t          j        |j        ¬¦  «        ¦  «        ¦  «        k    }|                     d¦  «        }n||k    }|                     ¦   «         }| 	                    d¦  «         
                    |¦  «                             |j        ¦  «        }t          ||                              ¦   «         |                     ¦   «         k    d|› d|j        d         › �¦  «         |S )zö
        Obtains a multimodal placeholder mask from `input_ids` or `inputs_embeds` for the given `token_id`, and checks
        that the placeholder token count matches the length of `features`. If the lengths differ, an error is raised.
        N)r   r/  rë   zAMultimodal features and placeholder tokens do not match, tokens: z, features: r   )r\  rØ  Úget_input_embeddingsrã   r1  Úlongr/  Úallr�  rç  Ú	expand_asr  r   Únumelrï   )r•   r8   r9   re  rf  Úspecial_maskÚn_tokenss          r�   Úget_placeholder_maskz!InklingModel.get_placeholder_mask~  s  € ð ÐØ(¨DÔ,?×,JÒ,JØ+�×)Ò)Ñ+Ô+­E¬L¸ÍÌÐ\iÔ\pÐ,qÑ,qÔ,qÑrÔrñ-ô -ò ˆLð (×+Ò+¨BÑ/Ô/ˆLˆLà$¨Ò0ˆLà×#Ò#Ñ%Ô%ˆØ#×-Ò-¨bÑ1Ô1×;Ò;¸MÑJÔJ×MÒMÈmÔNbÑcÔcˆÝØ˜,Ô'×-Ò-Ñ/Ô/°8·>²>Ñ3CÔ3CÒCØyÐPXÐyÐyÐfnÔftÐuvÔfwÐyÐyñ	
ô 	
ð 	
ð Ðrƒ   r;   rÛ  r+  Útoken_type_idsrõ  rÜ  Ú	lm_kwargsc           	      ó¬  — |du |	duz  rt          d¦  «        ‚|	€ |                      ¦   «         |¦  «        }	| j                             |	¦  «        }	|�r|                      |¦  «        j        }|                     |	j        |	j        ¦  «        }|  	                    ||	|| j
        j        ¦  «        }|	                     ||¦  «        }	d}|�s|                      ||¦  «        j        }|                     |	j        |	j        ¦  «        }|  	                    ||	|| j
        j        ¦  «        }|	                     ||¦  «        }	t!          |x}t"          ¦  «        sC| j
                             ¦   «         |	|||dœ}t'          di |¤Žt)          di |¤Žt+          di |¤Ždœ} | j        d||||	|dœ|¤Ž}t-          |j        |j        |j        |j        |�|nd¬¦  «        S )a7  
        audio_input_ids (`torch.LongTensor` of shape `(num_audios, max_num_frames, n_mel_bins)`, *optional*):
            Batch of (padded) discretized dMel bin tokens produced by [`InklingProcessor`].
        audio_input_ids_mask (`torch.Tensor` of shape `(num_audios, max_num_frames)`, *optional*):
            Mask marking valid (non-padding) audio frames in `audio_input_ids`.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`.

        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, InklingForConditionalGeneration

        >>> model = InklingForConditionalGeneration.from_pretrained("thinkingmachines/Inkling-NVFP4")
        >>> processor = AutoProcessor.from_pretrained("thinkingmachines/Inkling-NVFP4")

        >>> prompt = "Where is the cat standing?"
        >>> url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> inputs = processor(images=image, text=prompt,  return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(**inputs,)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Where is the cat standing?\nsnow"
        ```NrÞ  rß  rà  )r;   rÛ  r+  r9   rÜ  )rä  r+  r:   r´  Úimage_hidden_statesr~   )rå  rh  r\  rØ  ra  r
  r  r/  r   ro  r  rÉ   Úmasked_scatterrd  rä  rÊ   rÐ   rÑ   r¹  r   r   r   rÕ   r+  r:   r´  )r•   r8   rT  r  rb  r;   rÛ  r+  rp  r9   rõ  rÜ  rq  Úimage_featuresÚspecial_image_maskÚaudio_featuresÚspecial_audio_maskrê  rë  rý  s                       r�   rö   zInklingModel.forward™  sQ  € ðd ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø7˜D×5Ò5Ñ7Ô7¸	ÑBÔBˆMàÔ+×6Ò6°}ÑEÔEˆð Ð#Ø!×4Ò4°\ÑBÔBÔPˆNØ+×.Ò.¨}Ô/CÀ]ÔEXÑYÔYˆNØ!%×!:Ò!:Ø˜=¨.¸$¼+Ô:Tñ"ô "Ðð *×8Ò8Ð9KÈ^Ñ\Ô\ˆMð ˆØÐ&Ø!×4Ò4°_ÐFZÑ[Ô[ÔmˆNØ+×.Ò.¨}Ô/CÀ]ÔEXÑYÔYˆNØ!%×!:Ò!:Ø˜=¨.¸$¼+Ô:Tñ"ô "Ðð *×8Ò8Ð9KÈ^Ñ\Ô\ˆMõ °Ð?Ð-ÅÑFÔFð 	àœ+×5Ò5Ñ7Ô7Ø!.Ø"0Ø#2Ø ,ðð ˆKõ #5Ð"CÐ"C°{Ð"CÐ"CÝ%FÐ%UÐ%UÈÐ%UÐ%UÝ$CÐ$RÐ$RÀkÐ$RÐ$Rð#ð #Ðð &�$Ô%ð 
Ø.Ø%Ø+Ø'Øð
ð 
ð ð
ð 
ˆõ *Ø%Ô7Ø#Ô3Ø!Ô/ØÔ)Ø2>Ð2J  ÐPTð
ñ 
ô 
ð 	
rƒ   rß   )NNNNNNNNNNN)rž   rŸ   r    Úaccepts_loss_kwargsrÇ   rà   r   r   rã   rï  r   r   rG  r   ra  rî  r÷   rd  r¨   ro  r   r­   rÕ   rö   r¯   r°   s   @r�   rZ  rZ  R  s¬  ø€ € € € € ð  Ðð˜}ð ð ð ð ð ð ð Ø€^Ð!rÐsÑsÔsðFØ!Ô-ðFØ9?Ð@RÔ9SðFà	Ð+Ñ	+ðFð Fð Fñ tÔsñ ÔðFð
 Ø€^Ð!fÐgÑgÔgð 59ð1ð 1àÔ)ð1ð $œl¨TÑ1ð1ð 
Ð+Ñ	+ð	1ð 1ð 1ñ hÔgñ Ôð1ð$àÔ#ðð Ô(ðð Ô#ð	ð
 ðð ð ð ð6 Øð .2Ø15Ø37Ø48Ø.2Ø04Ø(,Ø26Ø26Ø*.Ø!%ðj
ð j
àÔ# dÑ*ðj
ð Ô'¨$Ñ.ðj
ð Ô)¨DÑ0ð	j
ð
 $œl¨TÑ1ðj
ð œ tÑ+ðj
ð Ô&¨Ñ-ðj
ð  ™ðj
ð Ô(¨4Ñ/ðj
ð Ô(¨4Ñ/ðj
ð Ô  4Ñ'ðj
ð ˜$‘;ðj
ð Ð.Ô/ðj
ð 
Ð+Ñ	+ðj
ð j
ð j
ñ „^ñ Ôðj
ð j
ð j
ð j
ð j
rƒ   rZ  c                   ó®  ‡ — e Zd Zi ZddiZdZdefˆ fd„Zede	j
        dee         fd„¦   «         Zee	 	 	 	 	 	 	 	 	 	 	 dde	j        d	z  de	j
        d	z  de	j        d	z  de	j        d	z  ded	z  de	j        d	z  de	j        d	z  de	j
        d	z  de	j        d	z  ded	z  dee	j        z  dee         deez  fd„¦   «         ¦   «         Z	 	 	 	 	 	 	 	 	 	 	 dˆ fd„	Zˆ xZS )ÚInklingForConditionalGenerationrò  ró  Fr  c                 óú   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        j        |j        j        d¬¦  «        | _	        |  
                    ¦   «          d S r  )r“   rà   rZ  r³  rá   r  r.   rM   rJ   rò  rÚ  rO  s     €r�   rà   z(InklingForConditionalGeneration.__init__  sg   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý! &Ñ)Ô)ˆŒ
õ ”y Ô!3Ô!?ÀÔASÔA^ÐejÐkÑkÔkˆŒØ�ŠÑÔÐÐÐrƒ   rT  r–   c                 ó(   —  | j         j        |fi |¤ŽS rß   )r³  ra  r`  s      r�   ra  z2InklingForConditionalGeneration.get_image_features  s   € à,ˆtŒzÔ,¨\ÐDÐD¸VÐDÐDÐDrƒ   Nr   r8   r;   rÛ  r+  r  rb  r9   rõ  rÜ  rö  ré   c                 óì  —  | j         d|||||||||
|	dœ
|¤Ž}|d         | j        j        j        z  }t	          |t
          ¦  «        rt          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }| j        j        j        }|�||j	        d         k     r|dd|…f         }d}|	� | j
        d||	|j	        d         dœ|¤Ž}t          |||j        |j        |j        |j        ¬¦  «        S )	a”	  
        audio_input_ids (`torch.LongTensor` of shape `(num_audios, max_num_frames, n_mel_bins)`, *optional*):
            Batch of (padded) discretized dMel bin tokens produced by [`InklingProcessor`].
        audio_input_ids_mask (`torch.Tensor` of shape `(num_audios, max_num_frames)`, *optional*):
            Mask marking valid (non-padding) audio frames in `audio_input_ids`.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`.

        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, InklingForConditionalGeneration

        >>> model = InklingForConditionalGeneration.from_pretrained("thinkingmachines/Inkling-NVFP4")
        >>> processor = AutoProcessor.from_pretrained("thinkingmachines/Inkling-NVFP4")

        >>> messages = [
        ...     {
        ...         "role": "system",
        ...         "content": [
        ...             {"type": "text", "text": "You are a helpful assistant."}
        ...         ]
        ...     },
        ...     {
        ...         "role": "user", "content": [
        ...             {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"},
        ...             {"type": "text", "text": "Where is the cat standing?"},
        ...         ]
        ...     },
        ... ]

        >>> inputs = processor.apply_chat_template(
        ...     messages,
        ...     tokenize=True,
        ...     return_dict=True,
        ...     return_tensors="pt",
        ...     add_generation_prompt=True
        ... )
        >>> # Generate
        >>> generate_ids = model.generate(**inputs)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "user\nYou are a helpful assistant.\n\n\n\n\n\nWhere is the cat standing?\nmodel\nBased on the image, the cat is standing in a snowy area, likely outdoors. It appears to"
        ```
        )
r8   rT  r  rb  r;   rÛ  r+  r9   rÜ  rõ  r   Nrë   .rø  )rú  rù  r+  r:   r´  rs  r~   )r³  r  r.   r@   rÐ   r¨   rû  rò  rK   rï   rü  rÙ   r+  r:   r´  rs  )r•   r8   rT  r;   rÛ  r+  r  rb  r9   rõ  rÜ  rö  r–   rý  r:   rþ  rù  rK   rú  s                      r�   rö   z'InklingForConditionalGeneration.forward!  sU  € ðD �$”*ð 
ØØ%Ø+Ø!5Ø)Ø%Ø+Ø'ØØð
ð 
ð ð
ð 
ˆð   œ
 T¤[Ô%<Ô%XÑXˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆØ"œkÔ5ÔIÐØÐ*Ð/BÀVÄ\ÐRTÔEUÒ/UÐ/UØ˜CÐ!5Ð"5Ð!5Ð5Ô6ˆFàˆØÐØ%�4Ô%Ðj¨V¸FÈvÌ|Ð\^ÔO_ÐjÐjÐciÐjÐjˆDå,ØØØ#Ô3Ø!Ô/ØÔ)Ø 'Ô ;ð
ñ 
ô 
ð 	
rƒ   Tc                 ór   •—  t          ¦   «         j        |f|||||	|
|dœ|¤Ž}|s|	s||d<   ||d<   ||d<   |S )N)r+  r9   r;   rÛ  rÜ  rö  Úis_first_iterationrT  r  rb  )r“   Úprepare_inputs_for_generation)r•   r8   r+  r9   rÛ  rT  r;   r  rb  rÜ  rö  rõ  r€  r–   Úmodel_inputsr—   s                  €r�   r�  z=InklingForConditionalGeneration.prepare_inputs_for_generation†  s‚   ø€ ð" =•u‘w”wÔ<Øð

à+Ø'Ø)Ø%ØØ)Ø1ð

ð 

ð ð

ð 

ˆð ð 	H Yð 	HØ+7ˆL˜Ñ(Ø.=ˆLÐ*Ñ+Ø3GˆLÐ/Ñ0àÐrƒ   )NNNNNNNNNNr   )NNNNNNNTr   NF)rž   rŸ   r    rÿ  r   ry  rÇ   rà   r   rã   rï  r   r   ra  r   rî  r÷   r   r­   r¨   rG  rÙ   rö   r�  r¯   r°   s   @r�   r{  r{    s  ø€ € € € € ð ÐØÐ0Ð1€Hð  Ðð˜}ð ð ð ð ð ð ð ðE¨uÔ/@ð EÈFÐSeÔLfð Eð Eð Eñ „^ðEð Øð .2Ø15Ø.2Ø04Ø(,Ø37Ø48Ø26Ø*.Ø!%Ø-.ða
ð a
àÔ# dÑ*ða
ð Ô'¨$Ñ.ða
ð œ tÑ+ð	a
ð
 Ô&¨Ñ-ða
ð  ™ða
ð Ô)¨DÑ0ða
ð $œl¨TÑ1ða
ð Ô(¨4Ñ/ða
ð Ô  4Ñ'ða
ð ˜$‘;ða
ð ˜eœlÑ*ða
ð Ð+Ô,ða
ð 
Ð.Ñ	.ða
ð a
ð a
ñ „^ñ Ôða
ðL ØØØØØØ!ØØØØ ð"ð "ð "ð "ð "ð "ð "ð "ð "ð "rƒ   r{  )rÇ   r,   r²   r¼   r²  rÒ  rñ  r  rL  rZ  r{  )rp   N)r,  )ir(  Úcollections.abcr   rã   Útorch.nnrá   Útorch.nn.functionalr  rh  Úhuggingface_hub.dataclassesr   Ú r   rº  Úactivationsr   Úcache_utilsr   r	   Úconfiguration_utilsr
   Ú
generationr   Úintegrationsr   Úintegrations.accelerater   Úmasking_utilsr   r   r   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   Úutils.genericr   Úutils.output_capturingr   Úgemma3.modeling_gemma3r   r    r!   r"   Ú&higgs_audio_v2.modeling_higgs_audio_v2r#   Úllama.modeling_llamar$   r%   Úmixtral.modeling_mixtralr&   Úqwen3_next.modeling_qwen3_nextr'   r(   r)   Ú
get_loggerrž   Úloggerr,   r²   r¼   rÇ   rÕ   rÙ   rÛ   ÚModulerÝ   r÷   rª   r  r  rI  rV  r[  r}  r‰  r#  r§  r²  rÒ  rñ  r¿  r  r  r¨   r«   r+  rî  rJ  rL  rZ  r{  Ú__all__r~   rƒ   r�   ú<module>rŸ     s5	  ðð  €€€Ø $Ð $Ð $Ð $Ð $Ð $à €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø .Ð .Ð .Ð .Ð .Ð .à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø )Ð )Ð )Ð )Ð )Ð )Ø /Ð /Ð /Ð /Ð /Ð /Ø =Ð =Ð =Ð =Ð =Ð =Ø sÐ sÐ sÐ sÐ sÐ sÐ sÐ sÐ sÐ sØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø SÐ SÐ SÐ SÐ SÐ SÐ SÐ SØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &ðð ð ð ð ð ð ð ð ð ð ð ð ð ð 8Ð 7Ð 7Ð 7Ð 7Ð 7Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5ðð ð ð ð ð ð ð ð ð ð ð ð LÐ KÐ KÐ KÐ KÐ KØ :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø qÐ qÐ qÐ qÐ qÐ qÐ qÐ qÐ qÐ qð 
ˆÔ	˜HÑ	%Ô	%€ð €Ð5Ð6Ñ6Ô6Øðfð fð fð fð fÐ(ñ fô fñ „ñ 7Ô6ðfðR €Ð5Ð6Ñ6Ô6Øð$ð $ð $ð $ð $Ð)ñ $ô $ñ „ñ 7Ô6ð$ð2 €Ð5Ð6Ñ6Ô6Øð$ð $ð $ð $ð $Ð*ñ $ô $ñ „ñ 7Ô6ð$ð* €Ø)ØXðñ ô ð ð0(ð 0(ð 0(ð 0(ð 0(Ð$ñ 0(ô 0(ñ „ñ	ô ð
0(ðf	ð 	ð 	ð 	ð 	Ð!:ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð$@ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	�\ñ 	ô 	ð 	ð^ð ^ð ^ð ^ð ^˜BœIñ ^ô ^ð ^ðB Ø)-ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð ”< $Ñ&ð%ð %ð %ð %ð8a)ð a)ð a)ð a)ð a)�r”yñ a)ô a)ð a)ðH1ð 1ð 1ð 1ð 1�ñ 1ô 1ð 1ð=ð =ð =ð =ð =�^ñ =ô =ð =ð#Hð #Hð #Hð #Hð #H˜œ	ñ #Hô #Hð #HðL%ð %ð %ð %ð %˜2œ9ñ %ô %ð %ð6ð ð ð ð �”ñ ô ð ð( ÐÐ*Ð,<Ð=Ñ>Ô>ð;ð ;ð ;ð ;ð ;˜bœiñ ;ô ;ñ ?Ô>ð;ð|.ð .ð .ð .ð .Ð4ñ .ô .ð .ðb ð.ð .ð .ð .ð .˜_ñ .ô .ñ „ð.ðb ðN
ð N
ð N
ð N
ð N
Ð-ñ N
ô N
ñ „ðN
ðbA
ð A
ð A
ð A
ð A
Ð*ñ A
ô A
ð A
ðH ?Ð >Ð >Ð >Ð >Ð"8Ñ >Ô >Ð >ð ð
ð 
ð 
ð 
ð 
Ð.ñ 
ô 
ñ „ð
ð("ð "ð "ð "ð " ¤	ñ "ô "ð "ðJ˜#ð  $ s¤)ð ð ð ð ð$ W\ð;ð ;Øð;Ø*-ð;Ø9<ð;ØJMð;à
Ôð;ð ;ð ;ð ;ð| ð,
ð ,
ð ,
ð ,
ð ,
Ð/ñ ,
ô ,
ñ „ð,
ð^ €ððñ ô ð
n
ð n
ð n
ð n
ð n
Ð)ñ n
ô n
ñô ð
n
ðb €ððñ ô ð
[ð [ð [ð [ð [Ð&<¸oñ [ô [ñô ð
[ð|ð ð €€€rƒ   