§
    9²¸jKO  ã                   ó   — d Z ddlmZ ddlmZmZ ddlZddlmc m	Z
 ddlmZ ddlmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl m!Z! ddl"m#Z#m$Z$ ddl%m&Z&m'Z' ddl(m)Z) ddl*m+Z+ ddl,m-Z- ddl.m/Z/m0Z0 ddl1m2Z2 ddl3m4Z4m5Z5m6Z6m7Z7 ddl8m9Z9m:Z: ddl;m<Z<m=Z=  e$j>        e?¦  «        Z@ e#d¬¦  «        e G d„ de2¦  «        ¦   «         ¦   «         ZA G d „ d!e7¦  «        ZB G d"„ d#e+¦  «        ZC G d$„ d%e5¦  «        ZD G d&„ d'e9¦  «        ZE G d(„ d)e<¦  «        ZF G d*„ d+e=¦  «        ZGe' G d,„ d-e)¦  «        ¦   «         ZH G d.„ d/e-¦  «        ZI G d0„ d1e6¦  «        ZJ G d2„ d3e/¦  «        ZK G d4„ d5e4¦  «        ZLg d6¢ZMdS )7zPyTorch Laguna model.é    )ÚCallable)ÚAnyÚLiteralN)Ústrict)Únné   )Úinitialization)ÚCacheÚDynamicCache)ÚPreTrainedConfig)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONS)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)Úauto_docstringÚlogging)ÚTransformersKwargsÚno_inherit_decoratoré   )ÚAfmoeAttention)ÚGemma3RotaryEmbedding)ÚGlm4MoeLiteDecoderLayer)Ú
LlamaModelÚeager_attention_forward)ÚQwen2MoeConfig)ÚQwen2MoeForCausalLMÚQwen2MoeMLPÚQwen2MoePreTrainedModelÚQwen2MoeRMSNorm)ÚQwen3_5MoeTopKRouterÚapply_rotary_pos_emb)ÚQwen3MoeExpertsÚQwen3MoeSparseMoeBlockzpoolside/laguna-XS.2)Ú
checkpointc                   ó:  — e Zd ZU dZdZi dd“dd“dd“dd“dd	“d
d“dd“dd“dd“dd	“dd“dd	“dd“dd“dd“dd	“ZdZeed<   dZ	eed<   dZ
eed<   dZeed<   d Zeed!<   d"Zeed#<   d$Zeed%<   d Zeed&<   d'Zeed(<   d'Zeed)<   d'Zeed*<   d+Zeed,<   d-Zeed.<   d/Zeez  ed0<   d1Zee         d1z  ed2<   d1Zee         d1z  ed3<   d4Zeed5<   d-Zeed6<   d7Zeed8<    e¦   «         Z  e¦   «         Z! e¦   «         Z" e¦   «         Z# e¦   «         Z$ e¦   «         Z%d9„ Z&d:„ Z'd;„ Z(d1S )<ÚLagunaConfigu¤  
    gating (`bool` or `str`, *optional*, defaults to `True`):
        Softplus output-gate granularity. ``True`` or ``"per-head"`` applies one gate per head,
        broadcast across ``head_dim``; ``"per-element"`` applies one gate per ``(head, head_dim)``
        channel.
    num_attention_heads_per_layer (`list[int]`, *optional*):
        Per-layer override for ``num_attention_heads``. Length must equal ``num_hidden_layers``.
    mlp_layer_types (`list[str]`, *optional*):
        Per-layer MLP type â€” ``"dense"`` or ``"sparse"``. Length must equal
        ``num_hidden_layers``. Defaults to first layer dense, rest sparse.
    moe_routed_scaling_factor (`float`, *optional*, defaults to 1.0):
        Scalar applied to routed-expert output before combining with the shared-expert output.
    moe_apply_router_weight_on_input (`bool`, *optional*, defaults to `False`):
        Whether to apply router weights to the MoE input rather than the output. Not supported
        in transformers yet; ``True`` will raise a ``NotImplementedError`` for now.
    moe_router_logit_softcapping (`float`, *optional*, defaults to 0.0):
        Scaling factor when applying tanh softcapping on the logits of the MoE router logits.

    Example:

    ```python
    >>> from transformers import LagunaModel, LagunaConfig

    >>> configuration = LagunaConfig()
    >>> model = LagunaModel(configuration)
    >>> configuration = model.config
    ```
    Úlagunazlayers.*.self_attn.q_projÚcolwisezlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.g_projzlayers.*.self_attn.o_projÚrowwisezlayers.*.self_attn.q_normÚreplicated_with_grad_allreducezlayers.*.self_attn.k_normzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projz!layers.*.mlp.experts.gate_up_projÚpacked_colwisezlayers.*.mlp.experts.down_projzlayers.*.mlp.expertsÚmoe_tp_expertsz%layers.*.mlp.shared_experts.gate_projz#layers.*.mlp.shared_experts.up_projz%layers.*.mlp.shared_experts.down_proji ˆ Ú
vocab_sizei    Úintermediate_sizeé(   Únum_hidden_layersé0   Únum_attention_headsé   Únum_key_value_headsi   Úmax_position_embeddingsé   Únum_expertsÚnum_experts_per_toki   Úmoe_intermediate_sizeÚshared_expert_intermediate_sizeÚsliding_windowé€   Úhead_dimFÚattention_biasTÚgatingNÚnum_attention_heads_per_layerÚmlp_layer_typesç      ð?Úmoe_routed_scaling_factorÚ moe_apply_router_weight_on_inputç        Úmoe_router_logit_softcappingc                 ó  — | j         €dg| j        z  | _         | j        €dgdg| j        dz
  z  z   | _        | j        €| j        g| j        z  | _        ddddœdd	d
dœdœ}| j        €|| _        t          j        | fi |¤dddhi¤Ž d S )NÚfull_attentionÚdenseÚsparseé   Údefaultg    €„Ag      à?)Ú	rope_typeÚ
rope_thetaÚpartial_rotary_factorg     ˆÃ@rE   ©rK   Úsliding_attentionÚignore_keys_at_rope_validationrT   )Úlayer_typesr3   rD   rC   r5   Úrope_parametersr   Ú__post_init__)ÚselfÚkwargsÚdefault_rope_paramss      úy/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/laguna/modular_laguna.pyrX   zLagunaConfig.__post_init__„   sî   € ØÔÐ#Ø 0Ð1°DÔ4JÑJˆDÔØÔÐ'Ø$+ 9°¨z¸TÔ=SÐVWÑ=WÑ/XÑ#XˆDÔ ØÔ-Ð5Ø26Ô2JÐ1KÈdÔNdÑ1dˆDÔ.ð -6ÀXÐhkÐlÐlØ/8ÈÐjmÐ!nÐ!nðe
ð e
Ðð ÔÐ'Ø#6ˆDÔ õ 	Ô&Øð	
ð 	
Øð	
ð 	
Ø<OÐQaÐ;bð	
ð 	
ð 	
ð 	
ð 	
ð 	
ó    c                 ó   — |S ©N© )rY   rZ   s     r\   Úconvert_rope_params_to_dictz(LagunaConfig.convert_rope_params_to_dict˜   s   € Øˆr]   c                 óü  — | j         rt          d¦  «        ‚| j        �Jt          | j        ¦  «        | j        k    r-t          dt          | j        ¦  «        › d| j        › d�¦  «        ‚t          | j        ¦  «        | j        k    r-t          dt          | j        ¦  «        › d| j        › d�¦  «        ‚t          | j        ¦  «        | j        k    r-t          dt          | j        ¦  «        › d| j        › d�¦  «        ‚dS )z'Part of ``@strict``-powered validation.zhmoe_apply_router_weight_on_input=True is not yet supported in the transformers implementation of Laguna.Nz&num_attention_heads_per_layer length (z ) must equal num_hidden_layers (z).zlayer_types length (zmlp_layer_types length ()rG   ÚNotImplementedErrorrC   Úlenr3   Ú
ValueErrorrV   rD   )rY   s    r\   Úvalidate_architecturez"LagunaConfig.validate_architecture›   sa  € àÔ0ð 	Ý%ð9ñô ð ð
 Ô.Ð:Ý�DÔ6Ñ7Ô7¸4Ô;QÒQÐQåðL½¸TÔ=_Ñ9`Ô9`ð Lð LØ15Ô1GðLð Lð Lñô ð õ ˆtÔÑ Ô  DÔ$:Ò:Ð:ÝðL¥s¨4Ô+;Ñ'<Ô'<ð Lð LØ15Ô1GðLð Lð Lñô ð õ ˆtÔ#Ñ$Ô$¨Ô(>Ò>Ð>ÝðL­3¨tÔ/CÑ+DÔ+Dð Lð LØ15Ô1GðLð Lð Lñô ð ð ?Ð>r]   ))Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚbase_model_tp_planr0   ÚintÚ__annotations__r1   r3   r5   r7   r8   r:   r;   r<   r=   r>   r@   rA   ÚboolrB   ÚstrrC   ÚlistrD   rF   ÚfloatrG   rI   ÚAttributeErrorÚdecoder_sparse_stepÚmlp_only_layersÚqkv_biasÚnorm_topk_probÚuse_sliding_windowÚmax_window_layersrX   ra   rf   r`   r]   r\   r)   r)   0   sÅ  € € € € € € ðð ð: €JðØ# Yðà# Yðð 	$ Yðð 	$ Yð	ð
 	$ Yðð 	$Ð%Eðð 	$Ð%Eðð 	! )ðð 	 	ðð 	! )ðð 	,Ð-=ðð 	)¨)ðð 	Ð 0ðð 	0°ðð 	.¨yðð  	0°ð!Ðð& €J�ÐÐÑØ!Ð�sÐ!Ð!Ñ!ØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø Ð˜Ð Ð Ñ Ø#)Ð˜SÐ)Ð)Ñ)Ø€K�ÐÐÑØ Ð˜Ð Ð Ñ Ø!$Ð˜3Ð$Ð$Ñ$Ø+.Ð# SÐ.Ð.Ñ.Ø€N�CÐÐÑð €HˆcÐÐÑØ €N�DÐ Ð Ñ Ø€FˆD�3‰JÐÐÑØ6:Ð! 4¨¤9¨tÑ#3Ð:Ð:Ñ:à(,€O�T˜#”Y Ñ%Ð,Ð,Ñ,Ø'*Ð˜uÐ*Ð*Ñ*Ø-2Ð$ dÐ2Ð2Ñ2Ø*-Ð  %Ð-Ð-Ñ-ð )˜.Ñ*Ô*ÐØ$�nÑ&Ô&€OØˆ~ÑÔ€HØ#�^Ñ%Ô%€NØ'˜Ñ)Ô)ÐØ&˜Ñ(Ô(Ðð
ð 
ð 
ð(ð ð ðð ð ð ð r]   r)   c                   ó   — e Zd ZdS )ÚLagunaRMSNormN©rg   rh   ri   r`   r]   r\   r{   r{   ¶   ó   € € € € € Ø€Dr]   r{   c            	       ó^   ‡ — e Zd Zddefˆ fd„Z	 ddededz  deej        e	f         fd„Z
ˆ xZS )	ÚLagunaRotaryEmbeddingNÚconfigc                 óJ   •— t          ¦   «                              |¦  «         d S r_   )ÚsuperÚ__init__)rY   r€   ÚdeviceÚ	__class__s      €r\   rƒ   zLagunaRotaryEmbedding.__init__»   s!   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ð Ð r]   Ú
layer_typeÚreturnc                 óV  — | j         |         d         }| j         |                              dd¦  «        }t          | dd¦  «        p| j        | j        z  }t          ||z  ¦  «        }d}d|t          j        d|dt          j        ¬¦  «        |z  z  z  }	|	 	                    |¦  «        |fS )	a‘  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            layer_type (`str`, *optional*):
                The current layer type if the model has different RoPE parameters per type.
                Should not be used unless `config.layer_types is not None`
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        rQ   rR   rE   r@   Nr   r   )Údtype)
rW   ÚgetÚgetattrÚhidden_sizer5   rm   ÚtorchÚarangerr   Úto)
r€   r„   r†   rZ   ÚbaserR   r@   ÚdimÚattention_factorÚinv_freqs
             r\   Úcompute_default_rope_parametersz5LagunaRotaryEmbedding.compute_default_rope_parameters¾   s´   € ð Ô% jÔ1°,Ô?ˆà &Ô 6°zÔ B× FÒ FÐG^Ð`cÑ dÔ dÐÝ˜6 :¨tÑ4Ô4Ðh¸Ô8JÈfÔNhÑ8hˆÝ�(Ð2Ñ2Ñ3Ô3ˆàÐð ˜$¥5¤<°°3¸ÅÄÐ#MÑ#MÔ#MÐPSÑ#SÑTÑUˆØ�{Š{˜6Ñ"Ô"Ð$4Ð4Ð4r]   r_   )NN)rg   rh   ri   r)   rƒ   rp   Útupler�   ÚTensorrr   r”   Ú__classcell__©r…   s   @r\   r   r   º   s‘   ø€ € € € € ð!ð !˜|ð !ð !ð !ð !ð !ð !ð EIð5ð 5Øð5Ø7:¸T±zð5à	ˆuŒ|˜UÐ"Ô	#ð5ð 5ð 5ð 5ð 5ð 5ð 5ð 5r]   r   c                   ó   — e Zd ZdS )Ú	LagunaMLPNr|   r`   r]   r\   rš   rš   Ú   r}   r]   rš   c                   óh   ‡ — e Zd Zˆ fd„Zdej        deej        ej        ej        f         fd„Zˆ xZS )ÚLagunaTopKRouterc                 óÄ   •— t          ¦   «                              ¦   «          t          j        t	          j        |j        ¦  «        d¬¦  «        | _        |j        | _	        d S )NF)Úrequires_grad)
r‚   rƒ   r   Ú	Parameterr�   Úzerosr:   Úe_score_correction_biasrI   Úrouter_logit_softcapping©rY   r€   r…   s     €r\   rƒ   zLagunaTopKRouter.__init__ß   sO   ø€ Ý‰Œ×ÒÑÔÐÝ')¤|µE´KÀÔ@RÑ4SÔ4SÐchÐ'iÑ'iÔ'iˆÔ$Ø(.Ô(KˆÔ%Ð%Ð%r]   Úhidden_statesr‡   c                 ó6  — |                      d| j        ¦  «        }t          j        || j        ¦  «                             ¦   «         }| j        dk    r$t          j        || j        z  ¦  «        | j        z  }t          j	        |¦  «        }|| j
                             |j        ¦  «        z   }t          j        || j        d¬¦  «        \  }}|                     d|¦  «        }||                     dd¬¦  «        z  }|                     |j        ¦  «        }|||fS )NéÿÿÿÿrH   )r‘   T)r‘   Úkeepdim)ÚreshapeÚ
hidden_dimÚFÚlinearÚweightrr   r¢   r�   ÚtanhÚsigmoidr¡   r�   r‰   ÚtopkÚtop_kÚgatherÚsum)rY   r¤   Úrouter_logitsÚrouting_scoresÚscores_for_selectionÚ_Úselected_expertsÚrouting_weightss           r\   ÚforwardzLagunaTopKRouter.forwardä   s  € ð &×-Ò-¨b°$´/ÑBÔBˆÝœ °´Ñ<Ô<×BÒBÑDÔDˆàÔ(¨3Ò.Ð.Ý!œJ }°tÔ7TÑ'TÑUÔUÐX\ÔXuÑuˆMåœ }Ñ5Ô5ˆà-°Ô0L×0OÒ0OÐP^ÔPdÑ0eÔ0eÑeÐÝ#œjÐ)=¸t¼zÈrÐRÑRÔRÑˆÐØ(×/Ò/°Ð4DÑEÔEˆØ)¨O×,?Ò,?ÀBÐPTÐ,?Ñ,UÔ,UÑUˆØ)×,Ò,¨]Ô-@ÑAÔAˆà˜oÐ/?Ð?Ð?r]   )	rg   rh   ri   rƒ   r�   r–   r•   r¹   r—   r˜   s   @r\   rœ   rœ   Þ   s�   ø€ € € € € ðLð Lð Lð Lð Lð
@à”|ð@ð 
ˆuŒ|˜Uœ\¨5¬<Ð7Ô	8ð@ð @ð @ð @ð @ð @ð @ð @r]   rœ   c                   ó   — e Zd ZdS )ÚLagunaExpertsNr|   r`   r]   r\   r»   r»   ù   r}   r]   r»   c                   óH   ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Zˆ xZS )ÚLagunaSparseMoeBlockr€   c                 ó˜   •— t          ¦   «                              |¦  «         t          ||j        ¬¦  «        | _        |j        | _        d S )N©r1   )r‚   rƒ   rš   r=   Úshared_expertsrF   Úrouted_scaling_factorr£   s     €r\   rƒ   zLagunaSparseMoeBlock.__init__þ   sD   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý'¨À&ÔBhÐiÑiÔiˆÔØ%+Ô%EˆÔ"Ð"Ð"r]   r¤   r‡   c                 ó  — |j         \  }}}|                     d|¦  «        }|                      |¦  «        }|                      |¦  «        \  }}}|                      |||¦  «        }|| j        z  }||z   }|                     |||¦  «        }|S )Nr¦   )ÚshapeÚviewrÀ   ÚgateÚexpertsrÁ   r¨   )	rY   r¤   Ú
batch_sizeÚsequence_lengthr©   Úshared_outputr¶   r¸   r·   s	            r\   r¹   zLagunaSparseMoeBlock.forward  sœ   € Ø2?Ô2EÑ/ˆ
�O ZØ%×*Ò*¨2¨zÑ:Ô:ˆØ×+Ò+¨MÑ:Ô:ˆà/3¯yªy¸Ñ/GÔ/GÑ,ˆˆ?Ð,ØŸš ]Ð4DÀoÑVÔVˆà%¨Ô(BÑBˆØ%¨Ñ5ˆà%×-Ò-¨j¸/È:ÑVÔVˆØÐr]   )	rg   rh   ri   r)   rƒ   r�   r–   r¹   r—   r˜   s   @r\   r½   r½   ý   sq   ø€ € € € € ðF˜|ð Fð Fð Fð Fð Fð Fð
 U¤\ð °e´lð ð ð ð ð ð ð ð r]   r½   c                   óÎ   ‡ — e Zd ZdZdededefˆ fd„Z	 ddej        de	ej        ej        f         d	ej        dz  d
e
dz  dee         de	ej        ej        dz  f         fd„Zˆ xZS )ÚLagunaAttentionzSAfmoe-style SWA/GQA attention with Laguna-specific gating and per-layer head count.r€   Ú	layer_idxÚ	num_headsc                 ó   •— || _         t          ¦   «                              ||¦  «         | j         |j        z  | _        t          j        |j        | j         | j        z  |j	        ¬¦  «        | _
        t          j        | j         | j        z  |j        |j	        ¬¦  «        | _        | `|j        du p
|j        dk    | _        | j        r| j         n| j         | j        z  }t          j        |j        |d¬¦  «        | _        d S )N)ÚbiasTzper-headF)rÍ   r‚   rƒ   r7   Únum_key_value_groupsr   ÚLinearrŒ   r@   rA   Úq_projÚo_projÚ	gate_projrB   Úgate_per_headÚg_proj)rY   r€   rÌ   rÍ   Ú
g_proj_dimr…   s        €r\   rƒ   zLagunaAttention.__init__  sã   ø€ à"ˆŒå‰Œ×Ò˜ Ñ+Ô+Ð+Ø$(¤N°fÔ6PÑ$PˆÔ!å”i Ô 2°D´NÀTÄ]Ñ4RÐY_ÔYnÐoÑoÔoˆŒÝ”i ¤°´Ñ >ÀÔ@RÐY_ÔYnÐoÑoÔoˆŒàˆNØ#œ]¨dÐ2ÐQ°f´mÀzÒ6QˆÔØ'+Ô'9Ð]�T”^�^¸t¼~ÐPTÔP]Ñ?]ˆ
Ý”i Ô 2°JÀUÐKÑKÔKˆŒˆˆr]   Nr¤   Úposition_embeddingsÚattention_maskÚpast_key_valuesrZ   r‡   c                 ó´  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «        }|                      |¦  «                             |¦  «        }	|                      |¦  «                             |¦  «        }
|                      |¦  «                             dd¦  «        }|                      |	¦  «                             dd¦  «        }	|
                     dd¦  «        }
|\  }}t          ||	||¦  «        \  }}	|�| 
                    |	|
| j        ¦  «        \  }	}
t          j        | j        j        t           ¦  «        } || ||	|
|f| j        sdn| j        | j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }t/          j        |                      |¦  «                             ¦   «         ¦  «                             |j        ¦  «        }| j        r:  |j        g |¢d‘| j        ‘R Ž |                     d¦  «        z  j        g |¢d‘R Ž }n||z  }|                      |¦  «        }||fS )Nr¦   rN   r   rH   )ÚdropoutÚscalingr>   ) rÃ   r@   rÒ   rÄ   Úk_projÚv_projÚq_normÚ	transposeÚk_normr$   ÚupdaterÌ   r   Úget_interfacer€   Ú_attn_implementationr   ÚtrainingÚattention_dropoutrÝ   r>   r¨   Ú
contiguousrª   ÚsoftplusrÖ   rr   r�   r‰   rÕ   Ú	unsqueezerÓ   )rY   r¤   rØ   rÙ   rÚ   rZ   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightsrÅ   s                    r\   r¹   zLagunaAttention.forward%  s–  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔDˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔBˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔDˆà—{’{ <Ñ0Ô0×:Ò:¸1¸aÑ@Ô@ˆØ—[’[ Ñ,Ô,×6Ò6°q¸!Ñ<Ô<ˆ
Ø#×-Ò-¨a°Ñ3Ô3ˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð  $œ}ÐH�C�C°$Ô2HØ”LØÔ.ð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆåŒz˜$Ÿ+š+ mÑ4Ô4×:Ò:Ñ<Ô<Ñ=Ô=×@Ò@ÀÔARÑSÔSˆØÔð 	-ØgÐ+˜;Ô+ÐL¨[ÐL¸"ÐL¸d¼mÐLÐLÐLÈtÏ~Ê~Ð^`ÑOaÔOaÑaÔgð ØðØ ðð ð ˆKˆKð &¨Ñ,ˆKà—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r]   r_   )rg   rh   ri   rj   r)   rm   rƒ   r�   r–   r•   r
   r   r   r¹   r—   r˜   s   @r\   rË   rË     sí   ø€ € € € € à]Ð]ðL˜|ð L¸ð LÈð Lð Lð Lð Lð Lð Lð( )-ð3)ð 3)à”|ð3)ð # 5¤<°´Ð#=Ô>ð3)ð œ tÑ+ð	3)ð
  ™ð3)ð Ð-Ô.ð3)ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð3)ð 3)ð 3)ð 3)ð 3)ð 3)ð 3)ð 3)r]   rË   c                   ó   — e Zd Zdedefd„ZdS )ÚLagunaDecoderLayerr€   rÌ   c                 ó   — t           j                             | ¦  «         |j        | _        t	          |||j        |         ¦  «        | _        |j        |         dk    rt          |¦  «        | _	        nt          ||j        ¬¦  «        | _	        t          |j        |j        ¬¦  «        | _        t          |j        |j        ¬¦  «        | _        d S )NrM   r¿   )Úeps)r   ÚModulerƒ   rŒ   rË   rC   Ú	self_attnrD   r½   Úmlprš   r1   r{   Úrms_norm_epsÚinput_layernormÚpost_attention_layernorm)rY   r€   rÌ   s      r\   rƒ   zLagunaDecoderLayer.__init__\  s¶   € Ý
Œ	×Ò˜4Ñ Ô Ð Ø!Ô-ˆÔÝ(¨°¸FÔ<`ÐajÔ<kÑlÔlˆŒØÔ! )Ô,°Ò8Ð8Ý+¨FÑ3Ô3ˆDŒHˆHå  ¸6Ô;SÐTÑTÔTˆDŒHÝ,¨VÔ-?ÀVÔEXÐYÑYÔYˆÔÝ(5°fÔ6HÈfÔNaÐ(bÑ(bÔ(bˆÔ%Ð%Ð%r]   N)rg   rh   ri   r)   rm   rƒ   r`   r]   r\   rö   rö   [  s>   € € € € € ð	c˜|ð 	c¸ð 	cð 	cð 	cð 	cð 	cð 	cr]   rö   c                   óH   ‡ — e Zd Z ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )ÚLagunaPreTrainedModelc                 ó.  •— t          ¦   «                              |¦  «         t          |t          ¦  «        r+t          j        j                             |j        ¦  «         d S t          |t          ¦  «        r›|j
        D ]•}|j        }|j        |         dk    rt          |j        |                  } ||j        |¬¦  «        \  }}t          j        t!          ||› d�¦  «        |¦  «         t          j        t!          ||› d�¦  «        |¦  «         Œ”d S d S )NrO   )r†   Ú	_inv_freqÚ_original_inv_freq)r‚   Ú_init_weightsÚ
isinstancerœ   r�   r   ÚinitÚzeros_r¡   r   rV   r”   rP   r   r€   Úcopy_r‹   )rY   Úmoduler†   Úrope_init_fnÚcurr_inv_freqr¶   r…   s         €r\   r  z#LagunaPreTrainedModel._init_weightsi  s%  ø€ å‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ.Ñ/Ô/ð 		^ÝŒHŒM× Ò  Ô!?Ñ@Ô@Ð@Ð@Ð@Ý˜Õ 5Ñ6Ô6ð 	^Ø$Ô0ð ^ð ^�
Ø%ÔE�ØÔ# JÔ/°9Ò<Ð<Ý#6°vÔ7GÈ
Ô7SÔ#T�LØ#/ <°´È*Ð#UÑ#UÔ#UÑ �˜qÝ”
�7 6¨jÐ+CÐ+CÐ+CÑDÔDÀmÑTÔTÐTÝ”
�7 6¨jÐ+LÐ+LÐ+LÑMÔMÈ}Ñ]Ô]Ð]Ð]ð	^ð 	^ð^ð ^r]   )rg   rh   ri   r�   Úno_gradr  r—   r˜   s   @r\   r   r   h  sS   ø€ € € € € Ø€U„]�_„_ð^ð ^ð ^ð ^ñ „_ð^ð ^ð ^ð ^ð ^r]   r   c                   óœ   — e Zd Z	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dedz  dej        dz  dedz  de	e
         d	efd
„ZdS )ÚLagunaModelNÚ	input_idsrÙ   Úposition_idsrÚ   Úinputs_embedsÚ	use_cacherZ   r‡   c           	      ó˜  ‡— |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|r|€t          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   }| 	                    d¦  «        }t          |x}	t          ¦  «        sI| j        ||||dœŠˆfd„ˆfd„d	œ}
i }	t          | j        j        ¦  «        D ]} |
|         ¦   «         |	|<   Œ|}i }t          | j        j        ¦  «        D ]}|                      |||¦  «        ||<   Œt          | j        d | j        j        …         ¦  «        D ]?\  }} ||f|	| j        j        |                  || j        j        |                  ||d
œ|¤Ž}Œ@|                      |¦  «        }t'          ||r|nd ¬¦  «        S )Nz:You must specify exactly one of input_ids or inputs_embeds)r€   r   rN   )r„   )r€   r  rÙ   rÚ   r  c                  ó   •— t          di ‰ ¤ŽS ©Nr`   )r   ©Úmask_kwargss   €r\   ú<lambda>z%LagunaModel.forward.<locals>.<lambda>š  s   ø€ Õ*<Ð*KÐ*K¸{Ð*KÐ*K€ r]   c                  ó   •— t          di ‰ ¤ŽS r  )r   r  s   €r\   r  z%LagunaModel.forward.<locals>.<lambda>›  s   ø€ Õ-NÐ-]Ð-]ÐQ\Ð-]Ð-]€ r]   rS   )rÙ   rØ   r  rÚ   )Úlast_hidden_staterÚ   )re   Úembed_tokensr   r€   Úget_seq_lengthr�   rŽ   rÃ   r„   rê   r  ÚdictÚsetrV   Ú
rotary_embÚ	enumerateÚlayersr3   Únormr   )rY   r  rÙ   r  rÚ   r  r  rZ   Úpast_seen_tokensÚcausal_mask_mappingÚmask_creation_functionsr†   r¤   rØ   ÚiÚdecoder_layerr  s                   @r\   r¹   zLagunaModel.forwardy  sc  ø€ ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå°Ð?Ð-ÅÑFÔFð 	Xàœ+Ø!.Ø"0Ø#2Ø ,ðð ˆKð #LÐ"KÐ"KÐ"KØ%]Ð%]Ð%]Ð%]ð'ð 'Ð#ð #%ÐÝ! $¤+Ô"9Ñ:Ô:ð Xð X�
Ø2UÐ2IÈ*Ô2UÑ2WÔ2WÐ# JÑ/Ð/à%ˆØ ÐÝ˜dœkÔ5Ñ6Ô6ð 	gð 	gˆJØ.2¯oªo¸mÈ\Ð[eÑ.fÔ.fÐ 
Ñ+Ð+å )¨$¬+Ð6U¸¼Ô8UÐ6UÔ*VÑ WÔ Wð 	ð 	ÑˆAˆ}Ø)˜MØðà2°4´;Ô3JÈ1Ô3MÔNØ$7¸¼Ô8OÐPQÔ8RÔ$SØ)Ø /ðð ð ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆå%Ø+Ø/8ÐB˜O˜O¸dð
ñ 
ô 
ð 	
r]   )NNNNNN)rg   rh   ri   r�   Ú
LongTensorr–   r
   ÚFloatTensorro   r   r   r   r¹   r`   r]   r\   r  r  x  s¼   € € € € € ð .2Ø.2Ø04Ø(,Ø26Ø!%ð<
ð <
àÔ# dÑ*ð<
ð œ tÑ+ð<
ð Ô&¨Ñ-ð	<
ð
  ™ð<
ð Ô(¨4Ñ/ð<
ð ˜$‘;ð<
ð Ð+Ô,ð<
ð 
 ð<
ð <
ð <
ð <
ð <
ð <
r]   r  c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚLagunaForCausalLMc                 ó6   •—  t          ¦   «         j        di |¤ŽS )a­  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        r`   )r‚   r¹   )rY   Úsuper_kwargsr…   s     €r\   r¹   zLagunaForCausalLM.forward¹  s!   ø€ ð �u‰wŒwŒÐ.Ð. Ð.Ð.Ð.r]   )rg   rh   ri   r¹   r—   r˜   s   @r\   r+  r+  ¸  s8   ø€ € € € € ð/ð /ð /ð /ð /ð /ð /ð /ð /r]   r+  )r)   r+  r  r   )Nrj   Úcollections.abcr   Útypingr   r   r�   Útorch.nn.functionalr   Ú
functionalrª   Úhuggingface_hub.dataclassesr   Ú r	   r  Úcache_utilsr
   r   Úconfiguration_utilsr   Úmasking_utilsr   r   Úmodeling_flash_attention_utilsr   Úmodeling_outputsr   Úmodeling_rope_utilsr   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   Úutils.genericr   r   Úafmoe.modeling_afmoer   Úgemma3.modeling_gemma3r   Ú$glm4_moe_lite.modeling_glm4_moe_liter   Úllama.modeling_llamar   r   Ú!qwen2_moe.configuration_qwen2_moer   Úqwen2_moe.modeling_qwen2_moer   r    r!   r"   Ú qwen3_5_moe.modeling_qwen3_5_moer#   r$   Úqwen3_moe.modeling_qwen3_moer%   r&   Ú
get_loggerrg   Úloggerr)   r{   r   rš   rœ   r»   r½   rË   rö   r   r  r+  Ú__all__r`   r]   r\   ú<module>rI     s–  ðð Ð à $Ð $Ð $Ð $Ð $Ð $Ø Ð Ð Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø .Ð .Ð .Ð .Ð .Ð .Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø RÐ RÐ RÐ RÐ RÐ RÐ RÐ RØ BÐ BÐ BÐ BÐ BÐ BØ 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø &Ð &Ð &Ð &Ð &Ð &Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ø EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø :Ð :Ð :Ð :Ð :Ð :Ø JÐ JÐ JÐ JÐ JÐ JØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ >Ð >Ð >Ð >Ð >Ð >Ø uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uØ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YØ RÐ RÐ RÐ RÐ RÐ RÐ RÐ Rð 
ˆÔ	˜HÑ	%Ô	%€ð €Ð1Ð2Ñ2Ô2ØðAð Að Að Að A�>ñ Aô Añ „ñ 3Ô2ðAðH	ð 	ð 	ð 	ð 	�Oñ 	ô 	ð 	ð5ð 5ð 5ð 5ð 5Ð1ñ 5ô 5ð 5ð@	ð 	ð 	ð 	ð 	�ñ 	ô 	ð 	ð@ð @ð @ð @ð @Ð+ñ @ô @ð @ð6	ð 	ð 	ð 	ð 	�Oñ 	ô 	ð 	ðð ð ð ð Ð1ñ ô ð ð* ðE)ð E)ð E)ð E)ð E)�nñ E)ô E)ñ ÔðE)ðP
cð 
cð 
cð 
cð 
cÐ0ñ 
cô 
cð 
cð^ð ^ð ^ð ^ð ^Ð3ñ ^ô ^ð ^ð =
ð =
ð =
ð =
ð =
�*ñ =
ô =
ð =
ð@/ð /ð /ð /ð /Ð+ñ /ô /ð /ðð ð €€€r]   