§
    9²¸jî¡  ã                   ó¦  — d dl Z d dlmZ d dlZd dlmc mZ d dlmZ ddlm	Z
 ddlmZ ddlmZmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZ ddlmZmZmZ ddlmZmZ ddl m!Z!m"Z" ddl#m$Z$m%Z% ddl&m'Z' ddl(m)Z)m*Z* ddl+m,Z, ddl-m.Z.m/Z/m0Z0 ddl1m2Z2 ddl3m4Z4  ed¦  «         G d„ dej5        ¦  «        ¦   «         Z6 G d„ dej5        ¦  «        Z7 G d„ dej5        ¦  «        Z8 G d„ dej5        ¦  «        Z9d „ Z: ed!¦  «        dMd"„¦   «         Z; G d#„ d$ej5        ¦  «        Z< G d%„ d&ej5        ¦  «        Z=e G d'„ d(ej5        ¦  «        ¦   «         Z> G d)„ d*ej5        ¦  «        Z? G d+„ d,ej5        ¦  «        Z@d-ejA        d.eBd/ejA        fd0„ZC	 dNd2ej5        d3ejA        d4ejA        d5ejA        d6ejA        dz  d7eDd8eDd9e'e)         fd:„ZEdOd;„ZFd<„ ZGdPd=„ZH G d>„ d?ej5        ¦  «        ZI G d@„ dAe¦  «        ZJe* G dB„ dCe%¦  «        ¦   «         ZKe* G dD„ dEeK¦  «        ¦   «         ZLe* G dF„ dGeKe¦  «        ¦   «         ZM G dH„ dIeeK¦  «        ZN G dJ„ dKeeK¦  «        ZOg dL¢ZPdS )Qé    N)ÚCallable)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_experts_implementationÚuse_kernel_forward_from_hub)Úcreate_causal_mask)ÚFlashAttentionKwargs)Ú GenericForSequenceClassificationÚGenericForTokenClassificationÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstring)Údeprecate_kwarg)Úcan_return_tupleÚmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )Ú
AXK2ConfigÚRMSNormc                   óT   ‡ — e Zd Zd	deddfˆ fd„Zdej        dej        fd„Zd„ Zˆ xZ	S )
ÚAXK2RMSNormç�íµ ÷Æ°>ÚepsÚreturnNc                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )z:
        AXK2RMSNorm is equivalent to T5LayerNorm
        N)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizer&   Ú	__class__s      €úv/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/axk2/modeling_axk2.pyr*   zAXK2RMSNorm.__init__5   sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐó    Úhidden_statesc                 ó  — |j         }|                     t          j        ¦  «        }|                     d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        |                     |¦  «        z  S )Né   éÿÿÿÿT)Úkeepdim)	ÚdtypeÚtor,   Úfloat32ÚpowÚmeanÚrsqrtr/   r.   )r0   r5   Úinput_dtypeÚvariances       r3   ÚforwardzAXK2RMSNorm.forward=   s|   € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØ ×$Ò$ QÑ'Ô'×,Ò,¨R¸Ð,Ñ>Ô>ˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆØŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:r4   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)Útupler.   Úshaper/   )r0   s    r3   Ú
extra_reprzAXK2RMSNorm.extra_reprD   s&   € Ý˜œÔ)Ñ*Ô*ÐIÐI°$Ô2GÐIÐIÐIr4   )r%   )
Ú__name__Ú
__module__Ú__qualname__Úfloatr*   r,   ÚTensorrB   rF   Ú__classcell__©r2   s   @r3   r$   r$   3   sŒ   ø€ € € € € ð$ð $¨ð $¸$ð $ð $ð $ð $ð $ð $ð; U¤\ð ;°e´lð ;ð ;ð ;ð ;ðJð Jð Jð Jð Jð Jð Jr4   r$   c                   óH   ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Zˆ xZS )ÚAXK2GateMLPÚconfigc                 ó  •— t          ¦   «                              ¦   «          || _        t          j        ¦   «         | _        t          j        |j        |j        d¬¦  «        | _	        t          j        |j        |j        d¬¦  «        | _
        d S ©NF©Úbias)r)   r*   rP   r   ÚSiLUÚactivation_fnÚLinearr1   Úgated_norm_rankÚfc1Úfc2©r0   rP   r2   s     €r3   r*   zAXK2GateMLP.__init__I   sn   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝœW™YœYˆÔÝ”9˜VÔ/°Ô1GÈeÐTÑTÔTˆŒÝ”9˜VÔ3°VÔ5GÈeÐTÑTÔTˆŒˆˆr4   r5   r'   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S ©N)rY   rV   rZ   )r0   r5   s     r3   rB   zAXK2GateMLP.forwardP   s=   € ØŸš Ñ/Ô/ˆØ×*Ò*¨=Ñ9Ô9ˆØŸš Ñ/Ô/ˆØÐr4   )	rG   rH   rI   r!   r*   r,   rK   rB   rL   rM   s   @r3   rO   rO   H   sq   ø€ € € € € ðU˜zð Uð Uð Uð Uð Uð Uð U¤\ð °e´lð ð ð ð ð ð ð ð r4   rO   c                   óL   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚAXK2GatedRMSNormz—RMSNorm followed by a low-rank input-dependent sigmoid gate (Megatron `GatedNormWrapper`):

    y = RMSNorm(x)
    return y * sigmoid(gate_mlp(y))
    rP   c                 ó°   •— t          ¦   «                              ¦   «          t          |j        |j        ¬¦  «        | _        t          |¦  «        | _        d S )N©r&   )r)   r*   r$   r1   Úrms_norm_epsÚnormrO   Úmlpr[   s     €r3   r*   zAXK2GatedRMSNorm.__init__^   sF   ø€ Ý‰Œ×ÒÑÔÐÝ Ô 2¸Ô8KÐLÑLÔLˆŒ	Ý˜vÑ&Ô&ˆŒˆˆr4   Úxr'   c                 óÔ   — |                       |¦  «        }|t          j        |                      |¦  «                             ¦   «         ¦  «        z                       |j        ¦  «        S r]   )rc   r,   Úsigmoidrd   rJ   r;   r:   )r0   re   Úys      r3   rB   zAXK2GatedRMSNorm.forwardc   sL   € Ø�IŠI�a‰LŒLˆØ•E”M $§(¢(¨1¡+¤+×"3Ò"3Ñ"5Ô"5Ñ6Ô6Ñ6×:Ò:¸1¼7ÑCÔCÐCr4   ©
rG   rH   rI   Ú__doc__r!   r*   r,   rK   rB   rL   rM   s   @r3   r_   r_   W   s~   ø€ € € € € ðð ð'˜zð 'ð 'ð 'ð 'ð 'ð 'ð
D˜œð D¨%¬,ð Dð Dð Dð Dð Dð Dð Dð Dr4   r_   c            	       óî   ‡ — e Zd Z edd¬¦  «        d
defˆ fd„¦   «         Ze edd¬¦  «        d
dedeej	        e
f         fd„¦   «         ¦   «         Z ej        ¦   «         ed	„ ¦   «         ¦   «         Zˆ xZS )ÚAXK2RotaryEmbeddingÚdevicez5.18)ÚversionNrP   c                 ó¾  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        | j        j        d         | _        | j        }| j        dk    rt          | j                 } || j        |¦  «        \  }| _
        t          j        |d¬¦  «        | _        t          j        |                     ¦   «         d¬¦  «        | _        d S )NÚ	rope_typeÚdefaultF)Ú
persistent)r)   r*   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrP   Úrope_parametersrp   Úcompute_default_rope_parametersr   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r0   rP   rm   Úrope_init_fnrz   r2   s        €r3   r*   zAXK2RotaryEmbedding.__init__i   sÀ   ø€ å‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!àˆŒàœÔ4°[ÔAˆŒØ!%Ô!EˆØŒ>˜YÒ&Ð&Ý.¨t¬~Ô>ˆLØ+7¨<¸¼ÀVÑ+LÔ+LÑ(ˆ�$Ô(åœ	 (°uÐ=Ñ=Ô=ˆŒÝ!#¤¨8¯>ª>Ñ+;Ô+;ÈÐ!NÑ!NÔ!NˆÔÐÐr4   r'   c                 óä   — | j         d         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «        |z  z  z  }|                     |¦  «        |fS )a¾  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNç      ð?r   r7   ©r:   )rv   Úgetattrr1   Únum_attention_headsr,   ÚarangerJ   r;   )rP   rm   ÚkwargsÚbaseÚdimÚattention_factorrz   s          r3   rw   z3AXK2RotaryEmbedding.compute_default_rope_parametersz   sz   € ð Ô% lÔ3ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐà˜$¥5¤<°°3¸ÅÄÐ#MÑ#MÔ#MÐPSÑ#SÑTÑUˆØ�{Š{˜6Ñ"Ô"Ð$4Ð4Ð4r4   c                 óú  — | j         d d d …d f                              |j        d         dd¦  «                             t          j        |j        ¬¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  ||z                       dd	¦  «        }t	          j        ||fd¬
¦  «        }|                     ¦   «         | j        z  }|                     ¦   «         | j        z  }	d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬¦  «        |	                     |j        ¬¦  «        fS )Nr   r8   r    )r:   rm   ÚmpsÚcpuF)Údevice_typeÚenabledr7   ©rˆ   r‚   )rz   ÚexpandrE   r;   r,   rJ   rm   Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosrx   Úsinr:   )
r0   re   Úposition_idsÚinv_freq_expandedÚposition_ids_expandedr�   ÚfreqsÚembr–   r—   s
             r3   rB   zAXK2RotaryEmbedding.forwardŽ   s›  € ð ŒM˜$    4˜-Ô(×/Ò/°Ô0BÀ1Ô0EÀrÈ1ÑMÔM×PÒPÕW\ÔWbÐklÔksÐPÑtÔtð 	ð !-¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆå¨¸UÐCÑCÔCð 	5ð 	5Ø&Ð)>Ñ>×IÒIÈ!ÈQÑOÔOˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCð		5ð 	5ð 	5ñ 	5ô 	5ð 	5ð 	5ð 	5ð 	5ð 	5ð 	5øøøð 	5ð 	5ð 	5ð 	5ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   ÃA*D<Ä<E ÅE r]   )rG   rH   rI   r   r!   r*   ÚstaticmethodrD   r,   rK   rJ   rw   Úno_gradr   rB   rL   rM   s   @r3   rl   rl   h   sö   ø€ € € € € Ø€_�X vÐ.Ñ.Ô.ðOð O˜zð Oð Oð Oð Oð Oñ /Ô.ðOð  Ø€_�X vÐ.Ñ.Ô.ð5ð 5°
ð 5ÐV[Ð\aÔ\hÐjoÐ\oÔVpð 5ð 5ð 5ñ /Ô.ñ „\ð5ð$ €U„]�_„_Øð<ð <ñ Ôñ „_ð<ð <ð <ð <ð <r4   rl   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..Nr8   r7   r�   )rE   r,   r•   )re   Úx1Úx2s      r3   Úrotate_halfr¢   ¡   s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'r4   Úrotary_pos_embc                 ó¾   — |                      |¦  «        }|                      |¦  «        }| |z  t          | ¦  «        |z  z   }||z  t          |¦  «        |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezer¢   )ÚqÚkr–   r—   Úunsqueeze_dimÚq_embedÚk_embeds          r3   Úapply_rotary_pos_embr«   ¨   sc   € ð& �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�GÐÐr4   c                   óÚ   ‡ — e Zd ZdZdddefˆ fd„Z ej        ¦   «         	 ddej        dej        d	e	ej        ej        f         d
ej        dej        de
dz  dej        fd„¦   «         Zˆ xZS )ÚAXK2IndexeraU  
    DeepSeek Sparse Attention (DSA) indexer for selecting top-k tokens.

    The Indexer has its own lightweight projections (wq_b, wk) separate from the main MLA attention,
    and returns the additive top-k sparse mask directly (`0` at the selected tokens, `-inf` elsewhere);
    the raw top-k indices are only ever scattered into that mask, so they are not surfaced.

    **Cache strategy**: the indexer key cache lives on the per-layer `DynamicIndexedLayer` (or the
    `StaticIndexedLayer` for static caches) inside the shared cache, accessed via
    `past_key_values.update_indexer()`.
    rP   r!   Ú	layer_idxc                 óF  •— t          ¦   «                              ¦   «          || _        || _        |j        | _        |j        | _        |j        | _        |j	        | _	        |j
        | _
        |j        | _        t          j        | j        | j        | j        z  d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _        | j        dz  | _        d S )NFrS   r%   ra   ç      à¿)r)   r*   rP   r®   r1   Úindex_n_headsÚn_headsÚindex_head_dimr€   Úqk_rope_head_dimÚ
index_topkÚq_lora_rankr   rW   Úwq_bÚwkÚ	LayerNormÚk_normÚweights_projÚsoftmax_scale©r0   rP   r®   r2   s      €r3   r*   zAXK2Indexer.__init__Ï   só   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒà &Ô 2ˆÔØ"Ô0ˆŒØ#Ô2ˆŒØ%+Ô%<ˆÔØ%Ô0ˆŒØ &Ô 2ˆÔå”I˜dÔ.°´¸t¼}Ñ0LÐSXÐYÑYÔYˆŒ	Ý”)˜DÔ,¨d¬mÀ%ÐHÑHÔHˆŒÝ”l 4¤=°dÐ;Ñ;Ô;ˆŒÝœI dÔ&6¸¼È5ÐQÑQÔQˆÔØ!œ]¨DÑ0ˆÔÐÐr4   Nr5   Úq_residÚposition_embeddingsÚattention_maskr˜   Úpast_key_valuesr'   c                 óò  — |j         \  }}}	|\  }
}|                      |¦  «        }|                     ||| j        | j        ¦  «        }t          j        || j        | j        | j        z
  gd¬¦  «        \  }}|                      |  	                    |¦  «        ¦  «         
                    d¦  «        }t          j        || j        | j        | j        z
  gd¬¦  «        \  }}t          |||
|d¬¦  «        \  }}t          j        ||gd¬¦  «        }t          j        ||gd¬¦  «                             d¦  «        }|�|                     || j        ¦  «        }t          j        |                     ¦   «         |                     dd¦  «                             ¦   «          
                    d¦  «        ¦  «        | j        z  }t)          j        |¦  «        }|                      |                     | j        j        j        ¦  «        ¦  «                             ¦   «         | j        dz  z  }t          j        | 
                    d¦  «        |¦  «                             d¦  «        }|j        t
          j        k    r%|                     | t#          d	¦  «        ¦  «        }n||z   }t9          | j        |j         d         ¦  «        }|                     |d¬¦  «        j                             t
          j         ¦  «        S )
uß  
        Selects the top-k tokens per query for DeepSeek Sparse Attention (DSA).

        This is the bf16 equivalent of the reference Indexer which uses `rotate_activation` (Hadamard transform)
        and `fp8_index` (FP8 quantized scoring kernel). Since the Hadamard transform is orthogonal (dot products
        are preserved: HqÂ·Hk = qÂ·k), and FP8 quantization is a precision optimization, we skip both and compute
        scores directly in bf16/fp32.

        The scoring logic computes:
            index_score[b,s,t] = Î£_h (weight[b,s,h] Â· softmax_scale Â· q[b,s,h,:] Â· k[b,t,:])

        Args:
            hidden_states: Input hidden states `[B, S, hidden_size]`.
            q_resid: Query residual from `q_a_layernorm(q_a_proj(x))`, shape `[B, S, q_lora_rank]`.
            position_embeddings: `(cos, sin)` from RotaryEmbedding.
            attention_mask: Causal mask, broadcastable to `[B, S, T]`.
            past_key_values: Cache object containing the indexer key cache for this layer.

        Returns:
            `torch.Tensor`: the `int32` top-k token indices of shape `[B, S, topk]`. The eager / SDPA paths
                turn these into an additive sparse mask; the `flash-mla` kernel consumes them directly.
        r8   r�   r7   )r¨   Néþÿÿÿr    r°   z-inf)!rE   r·   Úviewr²   r€   r,   Úsplitr´   rº   r¸   r¥   r«   r•   ÚsqueezeÚupdate_indexerr®   ÚmatmulrJ   r”   r¼   ÚFÚrelur»   r;   r.   r:   ÚboolÚmasked_fillÚminrµ   ÚtopkÚindicesÚint32)r0   r5   r¾   r¿   rÀ   r˜   rÁ   Ú
batch_sizeÚseq_lenÚ_r–   r—   r¦   Úq_rotÚq_passr§   Úk_rotÚk_passÚscoresÚweightsÚindex_scoresrÎ   s                         r3   rB   zAXK2Indexer.forwardá   s“  € ð@ "/Ô!4Ñˆ
�G˜QØ&‰ˆˆSØ�IŠI�gÑÔˆØ�FŠF�:˜w¨¬°d´mÑDÔDˆÝœ A¨Ô(=¸t¼}ÈtÔOdÑ?dÐ'eÐkmÐnÑnÔn‰ˆˆvà�KŠK˜Ÿš Ñ.Ô.Ñ/Ô/×9Ò9¸!Ñ<Ô<ˆÝœ A¨Ô(=¸t¼}ÈtÔOdÑ?dÐ'eÐkmÐnÑnÔn‰ˆˆvõ ,¨E°5¸#¸sÐRSÐTÑTÔT‰ˆˆuÝŒI�u˜f�o¨2Ð.Ñ.Ô.ˆÝŒI�u˜f�o¨2Ð.Ñ.Ô.×6Ò6°qÑ9Ô9ˆàÐ&Ø×.Ò.¨q°$´.ÑAÔAˆAå”˜aŸgšg™iœi¨¯ª°R¸Ñ)<Ô)<×)BÒ)BÑ)DÔ)D×)NÒ)NÈqÑ)QÔ)QÑRÔRÐUYÔUgÑgˆÝ”˜‘”ˆð ×#Ò# M×$4Ò$4°TÔ5FÔ5MÔ5SÑ$TÔ$TÑUÔU×[Ò[Ñ]Ô]ÐaeÔamÐosÑasÑtˆÝ”| G×$5Ò$5°bÑ$9Ô$9¸6ÑBÔB×JÒJÈ2ÑNÔNˆð Ô¥5¤:Ò-Ð-Ø'×3Ò3°^°OÅUÈ6Á]Ä]ÑSÔSˆLˆLà'¨.Ñ8ˆLå�4”? LÔ$6°rÔ$:Ñ;Ô;ˆØ× Ò  ¨2Ð Ñ.Ô.Ô6×9Ò9½%¼+ÑFÔFÐFr4   r]   )rG   rH   rI   rj   Úintr*   r,   rž   rK   rD   r   rB   rL   rM   s   @r3   r­   r­   Â   s÷   ø€ € € € € ð
ð 
ð1˜|ð 1¸ð 1ð 1ð 1ð 1ð 1ð 1ð$ €U„]�_„_ð )-ð>Gð >Gà”|ð>Gð ”ð>Gð # 5¤<°´Ð#=Ô>ð	>Gð
 œð>Gð ”lð>Gð  ™ð>Gð 
Œð>Gð >Gð >Gñ „_ð>Gð >Gð >Gð >Gð >Gr4   r­   c                   ó0   ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zˆ xZS )ÚAXK2TopkRouterrP   c                 óÔ  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        t          j	        t          j        | j        | j        ¦  «        ¦  «        | _        |j        | _        |j        | _        |j        | _        |j        | _        t          j        t          j        | j        t          j        ¬¦  «        ¦  «        | _        d S )Nr‚   )r)   r*   Únum_experts_per_tokÚtop_kÚnum_local_expertsÚnum_expertsr1   Ú
hidden_dimr   r+   r,   Úzerosr.   Úrouted_scaling_factorÚn_groupÚ	num_groupÚ
topk_groupÚnorm_topk_probry   r<   Úe_score_correction_biasr[   s     €r3   r*   zAXK2TopkRouter.__init__$  s±   ø€ Ý‰Œ×ÒÑÔÐØÔ/ˆŒ
Ø!Ô3ˆÔØ Ô,ˆŒÝ”l¥5¤;¨tÔ/?ÀÄÑ#QÔ#QÑRÔRˆŒØ%+Ô%AˆÔ"ØœˆŒØ Ô+ˆŒØ$Ô3ˆÔÝ')¤yµ´¸dÔ>NÕW\ÔWdÐ1eÑ1eÔ1eÑ'fÔ'fˆÔ$Ð$Ð$r4   c                 ó0  — |                      d| j        ¦  «        }t          j        |                     t
          j        ¦  «        | j                             t
          j        ¦  «        ¦  «        }|                     ¦   «         }|| j	        z   }| j
        �|                      |¦  «        }t          j        || j        dd¬¦  «        d         }|                     d|¦  «        }| j        r|                     dd¬¦  «        dz   }||z  }|| j        z  }|||fS )Nr8   F©r§   rˆ   Úsortedr    T)rˆ   r9   g#B’¡œÇ;)rÄ   rã   rÉ   Úlinearr’   r,   r<   r.   rg   rê   rç   Úapply_group_scoringrÎ   rà   Úgatherré   Úsumrå   )r0   r5   Úrouter_logitsrØ   Úscores_for_choiceÚtopk_indicesÚtopk_weightsÚdenominators           r3   rB   zAXK2TopkRouter.forward0  s  € Ø%×*Ò*¨2¨t¬Ñ?Ô?ˆÝœ ×!3Ò!3µE´MÑ!BÔ!BÀDÄK×DTÒDTÕUZÔUbÑDcÔDcÑdÔdˆØ×&Ò&Ñ(Ô(ˆØ" TÔ%AÑAÐð Œ>Ð%Ø $× 8Ò 8Ð9JÑ KÔ KÐå”zÐ"3°t´zÀrÐRWÐXÑXÔXÐYZÔ[ˆØ—}’} Q¨Ñ5Ô5ˆØÔð 	(Ø&×*Ò*¨r¸4Ð*Ñ@Ô@À5ÑHˆKØ˜KÑ'ˆLØ# dÔ&@Ñ@ˆØ˜l¨LÐ8Ð8r4   c                 óN  — |                      d| j        | j        | j        z  ¦  «                             dd¬¦  «        d                              d¬¦  «        }t          j        || j        dd¬¦  «        d         }t          j        |¦  «        }|                     d|d¦  «         | 	                    d¦  «         
                    d| j        | j        | j        z  ¦  «                             d| j        ¦  «        }|                     |                     ¦   «          d¦  «        }|S )	z$Apply DeepSeek style grouped scoringr8   r7   r�   r   Frì   r    ç        )rÄ   rç   râ   rÎ   rñ   r,   rè   Ú
zeros_likeÚscatter_r¥   r�   ÚreshaperÌ   rË   )r0   ró   Úgroup_scoresÚ	group_idxÚ
group_maskÚ
score_masks         r3   rï   z"AXK2TopkRouter.apply_group_scoringB  s  € ð ×"Ò" 2 t¤~°tÔ7GÈ4Ì>Ñ7YÑZÔZßŠT�!˜ˆT‰_Œ_˜Qô çŠS�RˆS‰[Œ[ð 	õ
 ”J˜|¨t¬ÀBÈuÐUÑUÔUÐVWÔXˆ	ÝÔ% lÑ3Ô3ˆ
Ø×Ò˜A˜y¨!Ñ,Ô,Ð,à× Ò  Ñ$Ô$ßŠV�B˜œ¨Ô(8¸D¼NÑ(JÑKÔKßŠW�R˜Ô)Ñ*Ô*ð 	ð .×9Ò9¸:¿?º?Ñ;LÔ;LÐ:LÈcÑRÔRÐØ Ð r4   )rG   rH   rI   r!   r*   rB   rï   rL   rM   s   @r3   rÝ   rÝ   #  si   ø€ € € € € ð
g˜zð 
gð 
gð 
gð 
gð 
gð 
gð9ð 9ð 9ð$!ð !ð !ð !ð !ð !ð !r4   rÝ   c                   ób   ‡ — e Zd ZdZˆ fd„Zdej        dej        dej        dej        fd„Zˆ xZS )ÚAXK2Expertsz2Collection of expert weights stored as 3D tensors.c                 ó´  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        t          j	        t          j        | j        d| j        z  | j        ¦  «        ¦  «        | _        t          j	        t          j        | j        | j        | j        ¦  «        ¦  «        | _        t          |j                 | _        d S )Nr7   )r)   r*   rá   râ   r1   rã   Úmoe_intermediate_sizeÚintermediate_dimr   r+   r,   ÚemptyÚgate_up_projÚ	down_projr   Ú
hidden_actÚact_fnr[   s     €r3   r*   zAXK2Experts.__init__Z  s£   ø€ Ý‰Œ×ÒÑÔÐØ!Ô3ˆÔØ Ô,ˆŒØ &Ô <ˆÔÝœL­¬°TÔ5EÀqÈ4ÔK`ÑG`ÐbfÔbqÑ)rÔ)rÑsÔsˆÔÝœ¥e¤k°$Ô2BÀDÄOÐUYÔUjÑ&kÔ&kÑlÔlˆŒÝ˜VÔ.Ô/ˆŒˆˆr4   r5   Útop_k_indexÚtop_k_weightsr'   c                 ó€  — t          j        |¦  «        }t          j        ¦   «         5  t           j        j                             || j        ¬¦  «        }|                     ddd¦  «        }t          j        | 	                    d¬¦  «        d¦  «         
                    ¦   «         }d d d ¦  «         n# 1 swxY w Y   |D ]þ}|d         }|| j        k    rŒt          j        ||         ¦  «        \  }}	||	         }
t          j                             |
| j        |         ¦  «                             dd¬¦  «        \  }}|                      |¦  «        |z  }t          j                             || j        |         ¦  «        }|||	|d f         z  }|                     d|	|                     |j        ¦  «        ¦  «         Œÿ|S )N)Únum_classesr7   r    r   )r8   rÃ   r�   r8   )r,   rù   rž   r   Ú
functionalÚone_hotrâ   ÚpermuteÚgreaterrñ   ÚnonzeroÚwhererî   r  Úchunkr	  r  Ú
index_add_r;   r:   )r0   r5   r
  r  Úfinal_hidden_statesÚexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stateÚgateÚupÚcurrent_hidden_statess                 r3   rB   zAXK2Experts.forwardc  sø  € õ $Ô.¨}Ñ=Ô=ÐÝŒ]‰_Œ_ð 	Sð 	SÝœ(Ô-×5Ò5°kÈtÔO_Ð5Ñ`Ô`ˆKØ%×-Ò-¨a°°AÑ6Ô6ˆKÝœ {§¢¸8 Ñ'DÔ'DÀaÑHÔH×PÒPÑRÔRˆJð	Sð 	Sð 	Sñ 	Sô 	Sð 	Sð 	Sð 	Sð 	Sð 	Sð 	Søøøð 	Sð 	Sð 	Sð 	Sð
 %ð 
	nð 
	nˆJØ# AœˆJØ˜TÔ-Ò-Ð-ØÝ#(¤;¨{¸:Ô/FÑ#GÔ#GÑ ˆI�yØ)¨)Ô4ˆMÝ”}×+Ò+¨M¸4Ô;LÈZÔ;XÑYÔY×_Ò_Ð`aÐgiÐ_ÑjÔj‰HˆD�"Ø$(§K¢K°Ñ$5Ô$5¸Ñ$:Ð!Ý$&¤M×$8Ò$8Ð9NÐPTÔP^Ð_iÔPjÑ$kÔ$kÐ!Ø$9¸MÈ)ÐU^Ð`dÐJdÔ<eÑ$eÐ!Ø×*Ò*¨1¨iÐ9N×9QÒ9QÐReÔRkÑ9lÔ9lÑmÔmÐmÐmà"Ð"s   ¨A>B2Â2B6Â9B6)	rG   rH   rI   rj   r*   r,   rK   rB   rL   rM   s   @r3   r  r  V  s€   ø€ € € € € à<Ð<ð0ð 0ð 0ð 0ð 0ð#à”|ð#ð ”\ð#ð ”|ð	#ð
 
Œð#ð #ð #ð #ð #ð #ð #ð #r4   r  c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚAXK2MLPNc                 ó   •— t          ¦   «                              ¦   «          || _        |j        | _        |€|j        n|| _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _	        t          |j                 | _        d S rR   )r)   r*   rP   r1   Úintermediate_sizer   rW   Ú	gate_projÚup_projr  r   r  r	  )r0   rP   r#  r2   s      €r3   r*   zAXK2MLP.__init__  s±   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ=NÐ=V Ô!9Ð!9Ð\mˆÔÝœ 4Ô#3°TÔ5KÐRWÐXÑXÔXˆŒÝ”y Ô!1°4Ô3IÐPUÐVÑVÔVˆŒÝœ 4Ô#9¸4Ô;KÐRWÐXÑXÔXˆŒÝ˜VÔ.Ô/ˆŒˆˆr4   c                 ó¨   — |                       |                      |                      |¦  «        ¦  «        |                      |¦  «        z  ¦  «        }|S r]   )r  r	  r$  r%  )r0   re   r  s      r3   rB   zAXK2MLP.forward‰  sA   € Ø—N’N 4§;¢;¨t¯~ª~¸aÑ/@Ô/@Ñ#AÔ#AÀDÇLÂLÐQRÁOÄOÑ#SÑTÔTˆ	ØÐr4   r]   )rG   rH   rI   r*   rB   rL   rM   s   @r3   r!  r!  ~  sL   ø€ € € € € ð0ð 0ð 0ð 0ð 0ð 0ðð ð ð ð ð ð r4   r!  c                   óL   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚAXK2MoEz:
    A mixed expert module containing shared experts.
    rP   c                 óì   •— t          ¦   «                              ¦   «          || _        t          |¦  «        | _        t          |¦  «        | _        t          ||j        |j	        z  ¬¦  «        | _
        d S )N)rP   r#  )r)   r*   rP   r  ÚexpertsrÝ   r  r!  r  Ún_shared_expertsÚshared_expertsr[   s     €r3   r*   zAXK2MoE.__init__“  sk   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ" 6Ñ*Ô*ˆŒÝ" 6Ñ*Ô*ˆŒ	Ý%Ø¨VÔ-IÈFÔLcÑ-cð
ñ 
ô 
ˆÔÐÐr4   r5   r'   c                 óú   — |}|j         }|                      |¦  «        \  }}}|                     d|j         d         ¦  «        } |                      |||¦  «        j        |Ž }||                      |¦  «        z   }|S )Nr8   )rE   r  rÄ   r*  r,  )r0   r5   Ú	residualsÚ
orig_shaperÓ   rõ   rô   s          r3   rB   zAXK2MoE.forwardœ  s‚   € Ø!ˆ	Ø"Ô(ˆ
Ø(,¯	ª	°-Ñ(@Ô(@Ñ%ˆˆ<˜Ø%×*Ò*¨2¨}Ô/BÀ2Ô/FÑGÔGˆØT˜Ÿš ]°LÀ,ÑOÔOÔTÐV`ÐaˆØ%¨×(;Ò(;¸IÑ(FÔ(FÑFˆØÐr4   ri   rM   s   @r3   r(  r(  Ž  st   ø€ € € € € ðð ð
˜zð 
ð 
ð 
ð 
ð 
ð 
ð U¤\ð °e´lð ð ð ð ð ð ð ð r4   r(  r5   Ún_repr'   c                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r    N)rE   r�   rû   )r5   r0  ÚbatchÚnum_key_value_headsÚslenr€   s         r3   Ú	repeat_kvr5  ¦  s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTr4   rø   ÚmoduleÚqueryÚkeyÚvaluerÀ   ÚscalingÚdropoutr†   c                 ó  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        |z  }
|�|
|z   }
t
          j                             |
dt          j        ¬¦  «         	                    |j
        ¦  «        }
t
          j                             |
|| j        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «                             ¦   «         }||
fS )Nr7   r   r8   )rˆ   r:   )ÚpÚtrainingr    )r5  Únum_key_value_groupsr,   rÈ   r”   r   r  Úsoftmaxr<   r;   r:   r;  r>  Ú
contiguous)r6  r7  r8  r9  rÀ   r:  r;  r†   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r3   Úeager_attention_forwardrF  ²  sé   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r4   c                 óL   — | dk    rdS d|z  t          j        | ¦  «        z  dz   S )Nr    r�   gš™™™™™¹?)ÚmathÚlog)ÚscaleÚmscales     r3   Úyarn_get_mscalerL  Ë  s,   € Ø�‚z€zØˆsØ�‰<�$œ( 5™/œ/Ñ)¨CÑ/Ð/r4   c                 óª   — |                       dd¦  «        dk    r8|                       dd¦  «        }| d         }|rt          ||¦  «        }||z  |z  }|S )Nrp   rq   Úmscale_all_dimr   Úfactor)ÚgetrL  )rv   r:  rN  Úscaling_factorrK  s        r3   Úyarn_apply_mscalerR  Ñ  sh   € Ø×Ò˜;¨	Ñ2Ô2°iÒ?Ð?Ø(×,Ò,Ð-=¸qÑAÔAˆØ(¨Ô2ˆØð 	0Ý$ ^°^ÑDÔDˆFØ Ñ&¨Ñ/ˆGØ€Nr4   c                 óÎ  — |dd|j         d         dz  …f                              |¦  «        }|dd|j         d         dz  …f                              |¦  «        }| dddd…f         | dddd…f         }}|dddd…f         |dddd…f         }	}t          j        ||z  ||z  z
  ||z  ||z  z   gd¬¦  «        }
t          j        ||z  |	|z  z
  |	|z  ||z  z   gd¬¦  «        }|
|fS )aû  
    Applies interleaved Rotary Position Embedding to the query and key tensors.

    DeepSeek lays the rotary dimensions out in interleaved pairs `(x0, x1), (x2, x3), ...`, each rotated by a
    single frequency. We compute that rotation directly on the even/odd slices instead of de-interleaving with a
    `view`/`transpose`/`reshape`; the output is bit-identical to the de-interleaved `rotate_half` formulation while
    avoiding the extra contiguous copy.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`):
            The position indices of the tokens corresponding to the query and key tensors. For example, this can be
            used to pass offsetted position ids when working with a KV-cache.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    .Nr8   r7   r   r    r�   )rE   r¥   r,   r•   )r¦   r§   r–   r—   r˜   r¨   Úq1Úq2Úk1Úk2r©   rª   s               r3   Úapply_rotary_pos_emb_interleaverX  Û  s  € ð8 ˆcÐ'�S”Y˜r”] aÑ'Ð'Ð'Ô
(×
2Ò
2°=Ñ
AÔ
A€CØ
ˆcÐ'�S”Y˜r”] aÑ'Ð'Ð'Ô
(×
2Ò
2°=Ñ
AÔ
A€Càˆs�A�D�q�DˆyŒ\˜1˜S ! $ Q $˜Yœ<ˆ€BØˆs�A�D�q�DˆyŒ\˜1˜S ! $ Q $˜Yœ<ˆ€BåŒi˜˜c™ B¨¡HÑ,¨b°3©h¸¸c¹Ñ.AÐBÈÐKÑKÔK€GÝŒi˜˜c™ B¨¡HÑ,¨b°3©h¸¸c¹Ñ.AÐBÈÐKÑKÔK€GØ�GÐÐr4   c                   ó&  ‡ — e Zd ZdZdedefˆ fd„Zdej        dej        de	ej        ej        f         fd„Z
	 	 dd
ej        de	ej        ej        f         dej        ded	z  dej        d	z  dee         de	ej        ej        d	z  f         fd„Zˆ xZS )ÚAXK2Attentionz¶
    DeepSeek-V3 MLA, with a DSA indexer whose top-k sparse mask is folded into the attention mask.
    Qlora rank formulation is dropped as it is never used in released models.
    rP   r®   c                 óT  •— t          ¦   «                              ¦   «          || _        || _        |j        | _        |j        | _        |j        | _        |j        | _        |j	        | _	        |j
        | _
        |j        | _        |j        | _        |j        |j	        z   | _        |j        |j        z  | _        d| _        t#          j        | j        | j        |j        ¬¦  «        | _        t+          | j        ¦  «        | _        t#          j        d| j        z  | j        | j        | j        z   z  d¬¦  «        | _        t#          j        | j        | j
        | j	        z   |j        ¬¦  «        | _        t+          | j
        ¦  «        | _        t#          j        | j
        | j        | j        | j        z   z  d¬¦  «        | _        t#          j        | j        | j        z  | j        |j        ¬¦  «        | _        t9          |j        | j        dz  ¦  «        | _        t?          ||¦  «        | _         d S )NTrS   r7   Fr°   )!r)   r*   rP   r®   Úattention_dropoutr1   r„   Ú	num_headsr¶   r´   Úkv_lora_rankÚ
v_head_dimÚqk_nope_head_dimÚqk_head_dimr3  r?  Ú	is_causalr   rW   Úattention_biasÚq_a_projr$   Úq_a_layernormÚq_gate_projÚkv_a_proj_with_mqaÚkv_a_layernormÚ	kv_b_projÚo_projrR  rv   r:  r­   Úindexerr½   s      €r3   r*   zAXK2Attention.__init__  sý  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒØ!'Ô!9ˆÔØ!Ô-ˆÔØÔ3ˆŒà!Ô-ˆÔØ &Ô 7ˆÔØ"Ô/ˆÔØ Ô+ˆŒØ &Ô 7ˆÔØ!Ô2°VÔ5LÑLˆÔØ$*Ô$>À&ÔB\Ñ$\ˆÔ!àˆŒåœ	 $Ô"2°DÔ4DÈ6ÔK`ÐaÑaÔaˆŒÝ(¨Ô)9Ñ:Ô:ˆÔåœ9Ø�Ô Ñ  $¤.°DÔ4DÀtÄÑ4VÑ"WÐ^cð
ñ 
ô 
ˆÔõ #%¤)ØÔØÔ Ô 5Ñ5ØÔ&ð#
ñ #
ô #
ˆÔõ
 *¨$Ô*;Ñ<Ô<ˆÔÝœØÔØŒN˜dÔ3°d´oÑEÑFØð
ñ 
ô 
ˆŒõ ”iØŒN˜Tœ_Ñ,ØÔØÔ&ð
ñ 
ô 
ˆŒõ )¨Ô)?ÀÔAQÐVZÑA[Ñ\Ô\ˆŒÝ" 6¨9Ñ5Ô5ˆŒˆˆr4   Úkv_noperÖ   r'   c                 ó4  — |j         \  }}}}||d| j        | j        z   f}|                      |¦  «                             |¦  «                             dd¦  «        }t          j        || j        | j        gd¬¦  «        \  }}|                     d|j         d         dd¦  «        } |j	        g |j         dd…         ¢| j        | j
        z   ‘R Ž }	|	dd| j        …f                              |¦  «         |	d| j        d…f                              |¦  «         |	|fS )a“  Expands the compressed latents into key and value states. Args:
            - kv_nope: key + value without positional encoding, shape [batch_size, 1, seqlen, self.kv_lora_rank]
            - k_rot: shared key with positional encoding, shape [batch_size, 1, seqlen, self.qk_rope_head_dim]
        Returns the key and value states, two tensors of shape [batch, num_heads, seq, (k or v)_head_dim].
        r8   r    r7   r�   N.)rE   r`  r_  ri  rÄ   r”   r,   rÅ   r�   Ú	new_emptyr´   Úcopy_)
r0   rl  rÖ   rÑ   rÓ   Ú
seq_lengthÚ	key_shapeÚk_noperC  rB  s
             r3   Ú	expand_kvzAXK2Attention.expand_kv6  s-  € ð (/¤}Ñ$ˆ
�A�z 1Ø ¨R°Ô1FÈÌÑ1XÐYˆ	à—.’. Ñ)Ô)×.Ò.¨yÑ9Ô9×CÒCÀAÀqÑIÔIˆÝ$œ{¨7°TÔ5JÈDÌOÐ4\ÐbdÐeÑeÔeÑˆ�Ø—’˜R ¤¨a¤°"°bÑ9Ô9ˆð '�WÔ&Ðj¨¬°c°r°cÔ(:Ðj¸DÔ<QÐTXÔTiÑ<iÐjÐjÐjˆ
Ø�3Ð/˜$Ô/Ð/Ð/Ô0×6Ò6°vÑ>Ô>Ð>Ø�3˜Ô-Ð/Ð/Ð/Ô0×6Ò6°uÑ=Ô=Ð=Ø˜<Ð'Ð'r4   Nr5   r¿   rÀ   rÁ   r˜   r†   c                 ó|  — |j         d d…         \  }}||d| j        | j        z   f}	|                      |¦  «        }
|                      |
¦  «        }|                      t          j        ||
gd¬¦  «        ¦  «                             |	¦  «        }t          j	        || j        | j        gd¬¦  «        \  }}| 
                    dd¦  «        }|                     ||d¦  «        }t          j	        || j        | j        gd¬¦  «        \  }}|                      |¦  «        }t          j	        || j        | j        gd¬¦  «        \  }}|                      |¦  «                             |d|| j        ¦  «        }|                     |d|| j        ¦  «        }|\  }}t#          ||||¦  «        \  }}t          j        ||fd¬¦  «        }|                      ||¦  «        \  }}|�|                     ||| j        ¦  «        \  }}|                      ||||d d …dd d …d d …f         ||¬¦  «        }d }| j        j        dv r¸|                     |||j         d         ft          j        ¬¦  «                             d|                     ¦   «         d	¦  «                             d¦  «        }|j        t          j        k    r|| z  }n5|                     |t          j        |j        ¦  «        j         ¦  «        }n|}tC          j"        | j        j        tF          ¦  «        } || ||||f| j$        sd
n| j%        | j&        |dœ|¤Ž\  }}|                     ||d¦  «         '                    ¦   «         }|t          j(        | )                    ¦   «         ¦  «        z   *                    |j        ¦  «        }|  +                    |¦  «        }||fS )Nr8   r�   r    r7   r   )rÁ   )ÚeagerÚsdpar‚   Frø   )r;  r:  rÏ   ),rE   ra  r_  rd  re  rf  r,   r•   rÄ   rÅ   r”   rû   r`  r´   rg  r^  rh  rX  rs  Úupdater®   rk  rP   Ú_attn_implementationÚnew_onesrË   ÚscatterÚlongr¥   r:   rÌ   ÚfinforÍ   r   Úget_interfacerF  r>  r\  r:  rA  rg   rJ   r;   rj  )r0   r5   r¿   rÀ   rÁ   r˜   r†   rÑ   rp  Úquery_gate_shapeÚq_compressedr¾   Úquery_gate_statesÚquery_statesÚgate_statesrÕ   rÔ   Úcompressed_kvÚkv_passrÖ   r×   r–   r—   rB  rC  rô   Úsparse_indicesÚ
index_maskÚattention_interfacerE  rD  s                                  r3   rB   zAXK2Attention.forwardI  sé  € ð "/Ô!4°S°b°SÔ!9Ñˆ
�JØ&¨
°B¸Ô8HÈ4Ì?Ñ8ZÐ[Ðð —}’} ]Ñ3Ô3ˆØ×$Ò$ \Ñ2Ô2ˆà ×,Ò,­U¬Y¸ÀÐ7NÐTVÐ-WÑ-WÔ-WÑXÔX×]Ò]Ð^nÑoÔoÐÝ$)¤KÐ0AÀDÔDTÐVZÔVeÐCfÐlnÐ$oÑ$oÔ$oÑ!ˆ�kØ#×-Ò-¨a°Ñ3Ô3ˆØ!×)Ò)¨*°jÀ"ÑEÔEˆåœ L°4Ô3HÈ$ÔJ_Ð2`ÐfhÐiÑiÔi‰ˆ�à×/Ò/°Ñ>Ô>ˆÝœ ]°TÔ5FÈÔH]Ð4^ÐdfÐgÑgÔg‰ˆ�à×$Ò$ WÑ-Ô-×2Ò2°:¸qÀ*ÈdÔN_Ñ`Ô`ˆØ—
’
˜: q¨*°dÔ6KÑLÔLˆà&‰ˆˆSÝ6°u¸eÀSÈ#ÑNÔN‰ˆˆuå”y &¨% °bÐ9Ñ9Ô9ˆà#'§>¢>°&¸%Ñ#@Ô#@Ñ ˆ
�Lð Ð&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜ð —|’|ØØØØ˜1˜1˜1˜a    A A A˜:Ô&ØØ+ð $ñ 
ô 
ˆð ˆØŒ;Ô+Ð/@Ð@Ð@à×%Ò% z°:¸zÔ?OÐPQÔ?RÐ&SÕ[`Ô[eÐ%ÑfÔfß’˜˜\×.Ò.Ñ0Ô0°%Ñ8Ô8ß’˜1‘”ð ð Ô#¥u¤zÒ1Ð1Ø!/°:°+Ñ!=��à!/×!;Ò!;¸JÍÌÐTaÔTgÑHhÔHhÔHlÑ!mÔ!m��à)ˆNå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð  $œ}ÐH�C�C°$Ô2HØ”LØ"ð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð "×)Ò)¨*°jÀ"ÑEÔE×PÒPÑRÔRˆà"¥U¤]°;×3DÒ3DÑ3FÔ3FÑ%GÔ%GÑG×KÒKÈKÔL]Ñ^Ô^ˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r4   )NN)rG   rH   rI   rj   r!   rÛ   r*   r,   rK   rD   rs  r   r   r   rB   rL   rM   s   @r3   rZ  rZ    s>  ø€ € € € € ðð ð
,6˜zð ,6°cð ,6ð ,6ð ,6ð ,6ð ,6ð ,6ð\( ¤ð (°e´lð (ÀuÈUÌ\Ð[`Ô[gÐMgÔGhð (ð (ð (ð (ð0 )-Ø,0ðT)ð T)à”|ðT)ð # 5¤<°´Ð#=Ô>ðT)ð œð	T)ð
  ™ðT)ð ”l TÑ)ðT)ð Ð-Ô.ðT)ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ðT)ð T)ð T)ð T)ð T)ð T)ð T)ð T)r4   rZ  c                   óÒ   ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 ddej        dej        dz  dej        dz  d	e	dz  d
e
dz  deej        ej        f         dz  dee         dej        fd„Zˆ xZS )ÚAXK2DecoderLayerrP   r®   c                 óš  •— t          ¦   «                              ¦   «          |j        | _        t          ||¦  «        | _        |j        |         dk    rt          |¦  «        nt          |¦  «        | _        t          |¦  «        | _
        |j        |         dk    rt          |¦  «        nt          |j        |j        ¬¦  «        | _        d S )NÚsparsera   )r)   r*   r1   rZ  Ú	self_attnÚmlp_layer_typesr(  r!  rd   r_   Úinput_layernormr$   rb   Úpost_attention_layernormr½   s      €r3   r*   zAXK2DecoderLayer.__init__¡  s»   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔÝ& v¨yÑ9Ô9ˆŒà&,Ô&<¸YÔ&GÈ8Ò&SÐ&S•7˜6‘?”?�?ÕY`ÐagÑYhÔYhˆŒÝ/°Ñ7Ô7ˆÔð Ô% iÔ0°HÒ<Ð<õ ˜VÑ$Ô$Ð$å˜VÔ/°VÔ5HÐIÑIÔIð 	Ô%Ð%Ð%r4   NFr5   rÀ   r˜   rÁ   Ú	use_cacher¿   r†   r'   c           
      óÎ   — |}|                       |¦  «        } | j        d||||||dœ|¤Ž\  }}	||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|S )N)r5   rÀ   r˜   rÁ   r�  r¿   © )rŽ  rŒ  r�  rd   )
r0   r5   rÀ   r˜   rÁ   r�  r¿   r†   ÚresidualrÓ   s
             r3   rB   zAXK2DecoderLayer.forward®  s¡   € ð !ˆØ×,Ò,¨]Ñ;Ô;ˆà)˜4œ>ð 
Ø'Ø)Ø%Ø+ØØ 3ð
ð 
ð ð
ð 
Ñˆ�qð ! =Ñ0ˆð !ˆØ×5Ò5°mÑDÔDˆØŸš Ñ/Ô/ˆØ  =Ñ0ˆØÐr4   )NNNFN)rG   rH   rI   r!   rÛ   r*   r,   rK   Ú
LongTensorr   rË   rD   r   r   rB   rL   rM   s   @r3   r‰  r‰     s÷   ø€ € € € € ð
˜zð 
°cð 
ð 
ð 
ð 
ð 
ð 
ð  /3Ø04Ø(,Ø!&ØHLðð à”|ðð œ tÑ+ðð Ô&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð # 5¤<°´Ð#=Ô>ÀÑEðð Ð+Ô,ðð 
Œðð ð ð ð ð ð ð r4   r‰  c                   ó˜   ‡ — e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZdZeedœZdgZd	gZd
gZ ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )ÚAXK2PreTrainedModelrP   ÚmodelTr‰  rÁ   F)r5   Ú
attentionsrê   rz   zindexer.weights_projc                 ó¼  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rAt	          j        |j        d| j        j        ¬¦  «         t	          j	        |j
        ¦  «         d S t          |t          ¦  «        rNt	          j        |j        d| j        j        ¬¦  «         t	          j        |j        d| j        j        ¬¦  «         d S d S )Nrø   )r>   Ústd)r)   Ú_init_weightsr‘   rÝ   ÚinitÚnormal_r.   rP   Úinitializer_rangeÚzeros_rê   r  r  r  )r0   r6  r2   s     €r3   r›  z!AXK2PreTrainedModel._init_weightsã  sÈ   ø€ å‰Œ×Ò˜fÑ%Ô%Ð%Ý�f�nÑ-Ô-ð 	XÝŒL˜œ¨S°d´kÔ6SÐTÑTÔTÐTÝŒK˜Ô6Ñ7Ô7Ð7Ð7Ð7Ý˜¥Ñ,Ô,ð 	XÝŒL˜Ô,°3¸D¼KÔ<YÐZÑZÔZÐZÝŒL˜Ô)°¸¼Ô9VÐWÑWÔWÐWÐWÐWð	Xð 	Xr4   )rG   rH   rI   r!   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr‰  rZ  Ú_can_record_outputsÚ_keep_in_fp32_modules_strictÚ"_keys_to_ignore_on_load_unexpectedÚ_keep_in_fp32_modulesr,   rž   r›  rL   rM   s   @r3   r–  r–  Î  sË   ø€ € € € € € àÐÐÑØÐØ&*Ð#Ø+Ð,ÐØ#4Ð"5ÐØ ÐØ€NØÐà!ÐØ"&Ðà)Ø#ðð Ðð %>Ð#>Ð Ø*4¨Ð&Ø3Ð4Ðà€U„]�_„_ðXð Xð Xð Xñ „_ðXð Xð Xð Xð Xr4   r–  c                   óâ   ‡ — e Zd Zdefˆ fd„Zeee	 	 	 	 	 	 ddej	        dz  dej
        dz  dej	        dz  dedz  dej        dz  d	edz  d
ee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )Ú	AXK2ModelrP   c                 óÞ  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          j        ‰j        ‰j        | j        ¦  «        | _        t          j	        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        t!          ‰¬¦  «        | _        d| _        |                      ¦   «          d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r’  )r‰  )Ú.0r®   rP   s     €r3   ú
<listcomp>z&AXK2Model.__init__.<locals>.<listcomp>÷  s$   ø€ ÐbÐbÐb°YÕ˜f iÑ0Ô0ÐbÐbÐbr4   ra   ©rP   F)r)   r*   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr1   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersÚlayersr$   rb   rc   rl   Ú
rotary_embÚgradient_checkpointingÚ	post_initr[   s    `€r3   r*   zAXK2Model.__init__ð  sÔ   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒåœL¨Ô):¸FÔ<NÐPTÔP`ÑaÔaˆÔÝ”mØbÐbÐbÐbÅ%ÈÔH`ÑBaÔBaÐbÑbÔbñ
ô 
ˆŒõ   Ô 2¸Ô8KÐLÑLÔLˆŒ	Ý-°VÐ<Ñ<Ô<ˆŒØ&+ˆÔ#ð 	�ŠÑÔÐÐÐr4   NÚ	input_idsrÀ   r˜   rÁ   Úinputs_embedsr�  r†   r'   c           
      óÈ  — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|r|€t          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   }| 	                    d¦  «        }t          |x}	t          ¦  «        s| j        ||||ddœ}
dt          di |
¤Ži}	|}|                      ||¬	¦  «        }t          | j        d | j        j        …         ¦  «        D ]*\  }} ||f|	| j        j        |                  ||||d
œ|¤Ž}Œ+|                      |¦  «        }t'          ||¬¦  «        S )Nz:You must specify exactly one of input_ids or inputs_embedsr´  r   r    )rm   F)rP   rÂ  rÀ   rÁ   r˜   Úallow_is_causal_skipÚdeepseek_sparse_attention)r˜   )rÀ   r¿   r˜   rÁ   r�  )Úlast_hidden_staterÁ   r’  )Ú
ValueErrorr¹  r	   rP   Úget_seq_lengthr,   r…   rE   rm   r¥   r‘   Údictr   r¾  Ú	enumerater½  r¼  Úlayer_typesrc   r   )r0   rÁ  rÀ   r˜   rÁ   rÂ  r�  r†   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsr5   r¿   ÚiÚdecoder_layers                  r3   rB   zAXK2Model.forward   s×  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø*.×*;Ò*;¸IÑ*FÔ*FˆMàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLõ °Ð?Ð-ÅÑFÔFð 		càœ+Ø!.Ø"0Ø#2Ø ,Ø(-ðð ˆKð $?Õ@RÐ@aÐ@aÐU`Ð@aÐ@aÐ"bÐà%ˆØ"Ÿošo¨mÈ,˜oÑWÔWÐå )¨$¬+Ð6U¸¼Ô8UÐ6UÔ*VÑ WÔ Wð 		ð 		ÑˆAˆ}Ø)˜MØðà2°4´;Ô3JÈ1Ô3MÔNØ$7Ø)Ø /Ø#ðð ð ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆÝ&Ø+Ø+ð
ñ 
ô 
ð 	
r4   )NNNNNN)rG   rH   rI   r!   r*   r   r   r   r,   r”  rK   r   ÚFloatTensorrË   r   r   r   rB   rL   rM   s   @r3   r¯  r¯  î  s  ø€ € € € € ð˜zð ð ð ð ð ð ð   ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ð6
ð 6
àÔ# dÑ*ð6
ð œ tÑ+ð6
ð Ô&¨Ñ-ð	6
ð
  ™ð6
ð Ô(¨4Ñ/ð6
ð ˜$‘;ð6
ð Ð+Ô,ð6
ð 
!ð6
ð 6
ð 6
ñ „^ñ „_ñ  Ôð6
ð 6
ð 6
ð 6
ð 6
r4   r¯  c                   ó   ‡ — e Zd ZddiZddiZddgdgfiZddiZˆ fd„Zee		 	 	 	 	 	 	 	 dde
j        d	z  de
j        d	z  de
j        d	z  ded	z  de
j        d	z  de
j        d	z  ded	z  dee
j        z  dee         defd„¦   «         ¦   «         Zˆ xZS )ÚAXK2ForCausalLMzlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr5   ÚlogitsÚkeep_full_weightc                 óþ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        | _        t          j        |j        |j        d¬¦  «        | _        |  	                    ¦   «          d S rR   )
r)   r*   r¯  r—  r·  r   rW   r1   rÔ  rÀ  r[   s     €r3   r*   zAXK2ForCausalLM.__init__C  sj   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý˜vÑ&Ô&ˆŒ
Ø Ô+ˆŒÝ”y Ô!3°VÔ5FÈUÐSÑSÔSˆŒð 	�ŠÑÔÐÐÐr4   Nr   rÁ  rÀ   r˜   rÁ   rÂ  Úlabelsr�  Úlogits_to_keepr†   r'   c	           
      óP  —  | j         d||||||dœ|	¤Ž}
|
j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }d}|� | j        d||| j        j        dœ|	¤Ž}t          |||
j
        |
j        |
j        ¬¦  «        S )aÉ  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, AXK2ForCausalLM

        >>> model = AXK2ForCausalLM.from_pretrained("meta-axk2/AXK2-2-7b-hf")
        >>> tokenizer = AutoTokenizer.from_pretrained("meta-axk2/AXK2-2-7b-hf")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```)rÁ  rÀ   r˜   rÁ   rÂ  r�  N)rÖ  rÙ  r·  )ÚlossrÖ  rÁ   r5   r˜  r’  )r—  rÆ  r‘   rÛ   ÚslicerÔ  Úloss_functionrP   r·  r   rÁ   r5   r˜  )r0   rÁ  rÀ   r˜   rÁ   rÂ  rÙ  r�  rÚ  r†   Úoutputsr5   Úslice_indicesrÖ  rÜ  s                  r3   rB   zAXK2ForCausalLM.forwardL  sô   € ð> ,6¨4¬:ð ,
ØØ)Ø%Ø+Ø'Øð,
ð ,
ð ð,
ð ,
ˆð  Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%Ðp¨V¸FÈtÌ{ÔOeÐpÐpÐioÐpÐpˆDå%ØØØ#Ô3Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r4   )NNNNNNNr   )rG   rH   rI   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr*   r   r   r,   r”  rK   r   rÑ  rË   rÛ   r   r   r   rB   rL   rM   s   @r3   rÓ  rÓ  <  sV  ø€ € € € € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€Jðð ð ð ð ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ð6
ð 6
àÔ# dÑ*ð6
ð œ tÑ+ð6
ð Ô&¨Ñ-ð	6
ð
  ™ð6
ð Ô(¨4Ñ/ð6
ð Ô  4Ñ'ð6
ð ˜$‘;ð6
ð ˜eœlÑ*ð6
ð Ð+Ô,ð6
ð 
 ð6
ð 6
ð 6
ñ „^ñ Ôð6
ð 6
ð 6
ð 6
ð 6
r4   rÓ  c                   ó   — e Zd ZdS )ÚAXK2ForSequenceClassificationN©rG   rH   rI   r’  r4   r3   ræ  ræ  ‡  ó   € € € € € Ø€Dr4   ræ  c                   ó   — e Zd ZdS )ÚAXK2ForTokenClassificationNrç  r’  r4   r3   rê  rê  ‹  rè  r4   rê  )r–  r¯  rÓ  ræ  rê  )r    )rø   )r    r    )Nr    )QrH  Úcollections.abcr   r,   Útorch.nn.functionalr   r  rÉ   Ú r   rœ  Úactivationsr   Úcache_utilsr   r	   Ú
generationr
   Úintegrationsr   r   Úmasking_utilsr   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   r   r   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   Úutils.deprecationr   Úutils.genericr   r   r   Úutils.output_capturingr   Úconfiguration_axk2r!   ÚModuler$   rO   r_   rl   r¢   r«   r­   rÝ   r  r!  r(  rK   rÛ   r5  rJ   rF  rL  rR  rX  rZ  r‰  r–  r¯  rÓ  ræ  rê  Ú__all__r’  r4   r3   ú<module>r      sk  ðð* €€€Ø $Ð $Ð $Ð $Ð $Ð $à €€€Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )Ø SÐ SÐ SÐ SÐ SÐ SÐ SÐ SØ /Ð /Ð /Ð /Ð /Ð /Ø BÐ BÐ BÐ BÐ BÐ Bðð ð ð ð ð ð ð ð ð ð
 PÐ OÐ OÐ OÐ OÐ OÐ OÐ OØ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø 7Ð 7Ð 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø *Ð *Ð *Ð *Ð *Ð *ð Ð˜YÑ'Ô'ðJð Jð Jð Jð J�"”)ñ Jô Jñ (Ô'ðJð(ð ð ð ð �"”)ñ ô ð ðDð Dð Dð Dð D�r”yñ Dô Dð Dð"6<ð 6<ð 6<ð 6<ð 6<˜"œ)ñ 6<ô 6<ð 6<ðr(ð (ð (ð ÐÐ-Ñ.Ô.ðð ð ñ /Ô.ðð2^Gð ^Gð ^Gð ^Gð ^G�"”)ñ ^Gô ^Gð ^GðB0!ð 0!ð 0!ð 0!ð 0!�R”Yñ 0!ô 0!ð 0!ðf ð$#ð $#ð $#ð $#ð $#�"”)ñ $#ô $#ñ Ôð$#ðNð ð ð ð ˆbŒiñ ô ð ð ð ð ð ð ˆbŒiñ ô ð ð0	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð& ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð20ð 0ð 0ð 0ðð ð ð$ð $ð $ð $ðN[)ð [)ð [)ð [)ð [)�B”Iñ [)ô [)ð [)ð|+ð +ð +ð +ð +Ð1ñ +ô +ð +ð\ ðXð Xð Xð Xð X˜/ñ Xô Xñ „ðXð> ðJ
ð J
ð J
ð J
ð J
Ð#ñ J
ô J
ñ „ðJ
ðZ ðG
ð G
ð G
ð G
ð G
Ð)¨?ñ G
ô G
ñ „ðG
ðT	ð 	ð 	ð 	ð 	Ð$DÐFYñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð!>Ð@Sñ 	ô 	ð 	ðð ð €€€r4   