§
    9²¸j‡�  ã                   ó‚  — d dl mZ d dlmZ d dlZd dlmc mZ d dlmZ ddl	m
Z ddlmZ ddlmZmZ dd	lmZ dd
lmZmZmZ ddlmZmZ ddlmZmZ ddlmZ ddlm Z m!Z!m"Z" ddl#m$Z$ ddl%m&Z&m'Z' ddl(m)Z) ddl*m+Z+  G d„ dej,        ¦  «        Z- G d„ dej,        ¦  «        Z. G d„ dej,        ¦  «        Z/ G d„ dej,        ¦  «        Z0dej1        de2dej1        fd„Z3 G d „ d!ej,        ¦  «        Z4 G d"„ d#ej,        ¦  «        Z5d$„ Z6dJd%„Z7	 dKd'ej,        d(ej1        d)ej1        d*ej1        d+ej1        dz  d,e8d-e8d.ee          fd/„Z9 G d0„ d1ej,        ¦  «        Z: G d2„ d3ej,        ¦  «        Z; G d4„ d5e¦  «        Z<e! G d6„ d7e¦  «        ¦   «         Z= e!d8¬9¦  «         G d:„ d;e=¦  «        ¦   «         Z> e!d<¬9¦  «         G d=„ d>e=¦  «        ¦   «         Z? e!d?¬9¦  «        e G d@„ dAe¦  «        ¦   «         ¦   «         Z@ G dB„ dCej,        ¦  «        ZA G dD„ dEej,        ¦  «        ZB e!dF¬9¦  «         G dG„ dHe=¦  «        ¦   «         ZCg dI¢ZDdS )Lé    )ÚCallable)Ú	dataclassN)Únné   )Úinitialization)ÚACT2FN)Úcreate_bidirectional_maskÚ(create_bidirectional_sliding_window_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚBaseModelOutputWithPoolingÚMaskedLMOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚtorch_compilable_check)Údeprecate_kwarg)Úcan_return_tupleÚmaybe_autocast)Úcapture_outputsé   )ÚNeoMMEConfigc                   óh   ‡ — e Zd Zddededefˆ fd„Zdej        fd„Z	dej        d	ej        fd
„Z
ˆ xZS )ÚNeoMMERMSNormç�íµ ÷Æ°>TÚdimÚepsÚ
with_scalec                 óÐ   •— t          ¦   «                              ¦   «          || _        || _        | j        r/t	          j        t          j        |¦  «        d¬¦  «        | _        d S d S )NT)Úrequires_grad)	ÚsuperÚ__init__r!   r"   r   Ú	ParameterÚtorchÚonesÚweight)Úselfr    r!   r"   Ú	__class__s       €úz/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/neomme/modeling_neomme.pyr&   zNeoMMERMSNorm.__init__,   s`   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ$ˆŒàŒ?ð 	LÝœ,¥u¤z°#¡¤ÀdÐKÑKÔKˆDŒKˆKˆKð	Lð 	Ló    Úhidden_statesc                 ó–   — |                      d¦  «                             dd¬¦  «        | j        z   }|t          j         |d¦  «        z  S )Né   éÿÿÿÿT©Úkeepdimç      à¿)ÚpowÚmeanr!   r(   )r+   r/   Úmean_squareds      r-   Ú_normzNeoMMERMSNorm._norm4   sF   € Ø$×(Ò(¨Ñ+Ô+×0Ò0°¸TÐ0ÑBÔBÀTÄXÑMˆà�uœy¨°tÑ<Ô<Ñ<Ð<r.   Úreturnc                 óÀ   — |                       |                     ¦   «         ¦  «        }| j        r|| j                             ¦   «         z  }|                     |¦  «        S ©N)r9   Úfloatr"   r*   Útype_as)r+   r/   Únormed_outputs      r-   ÚforwardzNeoMMERMSNorm.forward9   sV   € ØŸ
š
 =×#6Ò#6Ñ#8Ô#8Ñ9Ô9ˆØŒ?ð 	@Ø)¨D¬K×,=Ò,=Ñ,?Ô,?Ñ?ˆMØ×$Ò$ ]Ñ3Ô3Ð3r.   )r   T)Ú__name__Ú
__module__Ú__qualname__Úintr=   Úboolr&   r(   ÚTensorr9   r@   Ú__classcell__©r,   s   @r-   r   r   +   s¤   ø€ € € € € ðLð L˜Cð L eð LÀð Lð Lð Lð Lð Lð Lð= 5¤<ð =ð =ð =ð =ð
4 U¤\ð 4°e´lð 4ð 4ð 4ð 4ð 4ð 4ð 4ð 4r.   r   c                   óL   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        fd„Z	ˆ xZ
S )ÚNeoMMEEmbeddingszZFactorized (ALBERT-style) token embeddings: `vocab_size -> embedding_rank -> hidden_size`.Úconfigc                 óÜ   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        |j        d¬¦  «        | _	        d S ©NF©Úbias)
r%   r&   r   Ú	EmbeddingÚ
vocab_sizeÚembedding_rankÚword_embeddingsÚLinearÚhidden_sizeÚembedding_projection©r+   rK   r,   s     €r-   r&   zNeoMMEEmbeddings.__init__C   sY   ø€ Ý‰Œ×ÒÑÔÐÝ!œ|¨FÔ,=¸vÔ?TÑUÔUˆÔÝ$&¤I¨fÔ.CÀVÔEWÐ^cÐ$dÑ$dÔ$dˆÔ!Ð!Ð!r.   Ú	input_idsr:   c                 óR   — |                       |                      |¦  «        ¦  «        S r<   )rV   rS   )r+   rX   s     r-   r@   zNeoMMEEmbeddings.forwardH   s$   € Ø×(Ò(¨×)=Ò)=¸iÑ)HÔ)HÑIÔIÐIr.   )rA   rB   rC   Ú__doc__r   r&   r(   Ú
LongTensorrF   r@   rG   rH   s   @r-   rJ   rJ   @   s‚   ø€ € € € € ØdÐdðe˜|ð eð eð eð eð eð eð
J Ô!1ð J°e´lð Jð Jð Jð Jð Jð Jð Jð Jr.   rJ   c                   óL   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚNeoMMEPatchEmbeddingsz<Patch stem that maps flattened image patches to hidden size.rK   c                 óX  •— t          ¦   «                              ¦   «          t          j        |j        ¦  «        | _        t          j        |j        |j        dz  d¬¦  «        | _        t          j	        ¦   «         | _
        t          j        |j        dz  |j        d¬¦  «        | _        d S )Nr1   FrN   T)r%   r&   r   Ú	LayerNormÚ	patch_dimÚnormrT   rU   Úup_projÚGELUÚact_fnÚ	down_projrW   s     €r-   r&   zNeoMMEPatchEmbeddings.__init__O   s†   ø€ Ý‰Œ×ÒÑÔÐÝ”L Ô!1Ñ2Ô2ˆŒ	Ý”y Ô!1°6Ô3EÈÑ3IÐPUÐVÑVÔVˆŒÝ”g‘i”iˆŒÝœ 6Ô#5¸Ñ#9¸6Ô;MÐTXÐYÑYÔYˆŒˆˆr.   Úpixel_valuesr:   c                 ó¢   — |                       |                      |¦  «        ¦  «        }|                      |                      |¦  «        ¦  «        S r<   )rb   ra   re   rd   )r+   rf   r/   s      r-   r@   zNeoMMEPatchEmbeddings.forwardV   s=   € ØŸš T§Y¢Y¨|Ñ%<Ô%<Ñ=Ô=ˆØ�~Š~˜dŸkšk¨-Ñ8Ô8Ñ9Ô9Ð9r.   )
rA   rB   rC   rZ   r   r&   r(   rF   r@   rG   rH   s   @r-   r]   r]   L   sw   ø€ € € € € ØFÐFðZ˜|ð Zð Zð Zð Zð Zð Zð: E¤Lð :°U´\ð :ð :ð :ð :ð :ð :ð :ð :r.   r]   c                   óX  ‡ — e Zd ZdZ edd¬¦  «        ddefˆ fd„¦   «         Ze edd¬¦  «        	 ddededz  d	e	e
j        ef         fd
„¦   «         ¦   «         Z e
j        ¦   «         e	 dde
j        de
j        dedz  d	e	e
j        e
j        f         fd„¦   «         ¦   «         Zd„ Zˆ xZS )ÚNeoMMERotaryEmbeddingzBTwo-axis interleaved M-RoPE with per-layer-type frequency spectra.Údevicez5.18)ÚversionNrK   c           	      ó¶  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        t          t          |j        ¦  «        ¦  «        | _        i | _	        | j        D ]ã}| j        j
        |         }|€Œ|d         | j	        |<   | j        }| j	        |         dk    rt          | j	        |                  } || j        ||¬¦  «        \  }}t          | |› d�t          j        |d¬¦  «        ¦  «         t          | |› d�t          j        |                     ¦   «         d¬¦  «        ¦  «         t          | |› d�|¦  «         Œäd S )	NÚ	rope_typeÚdefault©Ú
layer_typeÚ	_inv_freqF)Ú
persistentÚ_original_inv_freqÚ_attention_scaling)r%   r&   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrK   ÚsortedÚsetÚlayer_typesrm   Úrope_parametersÚcompute_default_rope_parametersr   Úsetattrr   ÚBufferÚclone)	r+   rK   rj   rp   Úrope_paramsÚrope_init_fnÚcurr_inv_freqÚcurr_attention_scalingr,   s	           €r-   r&   zNeoMMERotaryEmbedding.__init__^   sl  ø€ å‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!ØˆŒÝ!¥# fÔ&8Ñ"9Ô"9Ñ:Ô:ˆÔØˆŒØÔ*ð 	Uð 	UˆJØœ+Ô5°jÔAˆKØÐ"Øà)4°[Ô)AˆDŒN˜:Ñ&Ø%)Ô%IˆLØŒ~˜jÔ)¨YÒ6Ð6Ý2°4´>À*Ô3MÔN�Ø4@°LÀÄÈfÐakÐ4lÑ4lÔ4lÑ1ˆMÐ1Ý�D˜ZÐ2Ð2Ð2µB´I¸mÐX]Ð4^Ñ4^Ô4^Ñ_Ô_Ð_Ý�D˜ZÐ;Ð;Ð;½R¼YÀ}×GZÒGZÑG\ÔG\ÐinÐ=oÑ=oÔ=oÑpÔpÐpÝ�D˜ZÐ;Ð;Ð;Ð=SÑTÔTÐTÐTð	Uð 	Ur.   rp   r:   c                 óV  — | j         |         d         }| j         |                              dd¦  «        }t          | dd¦  «        p| j        | j        z  }t          ||z  ¦  «        }d}d|t          j        d|dt          j        ¬¦  «        |z  z  z  }	|	 	                    |¦  «        |fS )	a‘  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            layer_type (`str`, *optional*):
                The current layer type if the model has different RoPE parameters per type.
                Should not be used unless `config.layer_types is not None`
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚpartial_rotary_factorç      ð?Úhead_dimNr   r1   )Údtype)
r{   ÚgetÚgetattrrU   Únum_attention_headsrD   r(   Úaranger=   Úto)
rK   rj   rp   ÚkwargsÚbaser†   rˆ   r    Úattention_factorÚinv_freqs
             r-   r|   z5NeoMMERotaryEmbedding.compute_default_rope_parameterst   s´   € ð" Ô% jÔ1°,Ô?ˆà &Ô 6°zÔ B× FÒ FÐG^Ð`cÑ dÔ dÐÝ˜6 :¨tÑ4Ô4Ðh¸Ô8JÈfÔNhÑ8hˆÝ�(Ð2Ñ2Ñ3Ô3ˆàÐð ˜$¥5¤<°°3¸ÅÄÐ#MÑ#MÔ#MÐPSÑ#SÑTÑUˆØ�{Š{˜6Ñ"Ô"Ð$4Ð4Ð4r.   ÚxÚposition_idsc                 ó.  — t          | |› d�¦  «        }t          | |› d�¦  «        }|d d d d …d f                              ¦   «                              d|j        d         dd¦  «        }|d d …d d …d d d …f                              ¦   «         }t	          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬	¦  «        5  ||z   	                    dd
¦  «        }	|	 
                    ¦   «         |z  }
|	                     ¦   «         |z  }d d d ¦  «         n# 1 swxY w Y   |                      |
¦  «        }
|                      |¦  «        }|
                     |j        ¦  «        |                     |j        ¦  «        fS )Nrq   rt   r1   r   r2   ÚmpsÚcpuF)Údevice_typeÚenabledr   )r‹   r=   ÚexpandÚshapeÚ
isinstancerj   ÚtypeÚstrr   Ú	transposeÚcosÚsinÚrecomposition_frequenciesrŽ   r‰   )r+   r“   r”   rp   r’   Úattention_scalingÚinv_freq_expandedÚposition_ids_expandedr˜   Úfreqsr    r¡   s               r-   r@   zNeoMMERotaryEmbedding.forward‘   sÇ  € õ
 ˜4 JÐ!9Ð!9Ð!9Ñ:Ô:ˆÝ# D¨ZÐ*KÐ*KÐ*KÑLÔLÐà$ T¨4°°°°DÐ%8Ô9×?Ò?ÑAÔA×HÒHÈÈLÔL^Ð_`ÔLaÐceÐghÑiÔiÐØ ,¨Q¨Q¨Q°°°°4¸¸¸¨]Ô ;× AÒ AÑ CÔ CÐå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆÝ¨¸UÐCÑCÔCð 	2ð 	2à&Ð)>Ñ>×IÒIÈ!ÈQÑOÔOˆEØ—)’)‘+”+Ð 1Ñ1ˆCØ—)’)‘+”+Ð 1Ñ1ˆCð		2ð 	2ð 	2ñ 	2ô 	2ð 	2ð 	2ð 	2ð 	2ð 	2ð 	2øøøð 	2ð 	2ð 	2ð 	2ð ×,Ò,¨SÑ1Ô1ˆØ×,Ò,¨SÑ1Ô1ˆØ�vŠv�a”g‰Œ §¢ q¤w¡¤Ð/Ð/s   ÃAD.Ä.D2Ä5D2c                 óÔ   — |d         dddd…f         |d         dddd…f         }}t          j        ||gd¬¦  «                             d¦  «        }t          j        ||gd¬¦  «        S )	z]
        Recompose the frequencies into the final spatial layout used per each grid.
        r   .Nr1   r   r2   ©r    éþÿÿÿ)r(   ÚstackÚflattenÚcat)r+   ÚfreqÚfreq_rowÚfreq_colÚangless        r-   r¢   z/NeoMMERotaryEmbedding.recomposition_frequencies§   st   € ð " !œW S¨!¨$¨Q¨$ YÔ/°°a´¸¸a¸dÀ¸d¸Ô1C�(ˆÝ”˜h¨Ð1°rÐ:Ñ:Ô:×BÒBÀ2ÑFÔFˆÝŒy˜& &Ð)¨rÐ2Ñ2Ô2Ð2r.   r<   ©NN)rA   rB   rC   rZ   r   r   r&   Ústaticmethodrž   Útupler(   rF   r=   r|   Úno_gradr   r[   r@   r¢   rG   rH   s   @r-   ri   ri   [   s]  ø€ € € € € ØLÐLà€_�X vÐ.Ñ.Ô.ðUð U˜|ð Uð Uð Uð Uð Uñ /Ô.ðUð* Ø€_�X vÐ.Ñ.Ô.àDHð5ð 5Øð5Ø7:¸T±zð5à	ˆuŒ|˜UÐ"Ô	#ð5ð 5ð 5ñ /Ô.ñ „\ð5ð6 €U„]�_„_ØàX\ð0ð 0Ø”ð0Ø-2Ô-=ð0ØKNÐQUÉ:ð0à	ˆuŒ|˜Uœ\Ð)Ô	*ð0ð 0ð 0ñ Ôñ „_ð0ð(3ð 3ð 3ð 3ð 3ð 3ð 3r.   ri   r/   Ún_repr:   c                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r   N)r›   rš   Úreshape)r/   rµ   ÚbatchÚnum_key_value_headsÚslenrˆ   s         r-   Ú	repeat_kvr»   ²   s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTr.   c                   óV   ‡ — e Zd Zdefˆ fd„Zdej        dej        dej        fd„Zˆ xZS )ÚNeoMMEExclusiveSelfAttentionrK   c                 óÐ   •— t          ¦   «                              ¦   «          |j        |j        z  | _        t          j        t          j        |j        ¦  «        ¦  «        | _	        d S r<   )
r%   r&   rŒ   r¹   Únum_key_value_groupsr   r'   r(   ÚzerosÚalpharW   s     €r-   r&   z%NeoMMEExclusiveSelfAttention.__init__¿   sN   ø€ Ý‰Œ×ÒÑÔÐØ$*Ô$>À&ÔB\Ñ$\ˆÔ!Ý”\¥%¤+¨fÔ.HÑ"IÔ"IÑJÔJˆŒ
ˆ
ˆ
r.   Úattn_outputÚvalue_statesr:   c                 ó¬  — t          || j        ¦  «                             dd¦  «        }t          j        |                     ¦   «         d¬¦  «                             |j        ¦  «        }||z                       dd¬¦  «        }t          j
        | j        ¦  «                             |j        ¦  «                             dddd¦  «        }|||z  |z  z
  S )Nr   r1   r2   r¨   Tr3   )r»   r¿   rŸ   ÚFÚ	normalizer=   rŽ   r‰   Úsumr(   ÚtanhrÁ   Úview)r+   rÂ   rÃ   Ú
value_unitÚ
projectionÚscales         r-   r@   z$NeoMMEExclusiveSelfAttention.forwardÄ   s¿   € Ý  ¨tÔ/HÑIÔI×SÒSÐTUÐWXÑYÔYˆÝ”[ ×!3Ò!3Ñ!5Ô!5¸2Ð>Ñ>Ô>×AÒAÀ+ÔBSÑTÔTˆ
Ø! JÑ.×3Ò3°BÀÐ3ÑEÔEˆ
Ý”
˜4œ:Ñ&Ô&×)Ò)¨+Ô*;Ñ<Ô<×AÒAÀ!ÀQÈÈAÑNÔNˆØ˜e jÑ0°JÑ>Ñ>Ð>r.   ©	rA   rB   rC   r   r&   r(   rF   r@   rG   rH   s   @r-   r½   r½   ¾   s|   ø€ € € € € ðK˜|ð Kð Kð Kð Kð Kð Kð
? 5¤<ð ?¸u¼|ð ?ÐPUÔP\ð ?ð ?ð ?ð ?ð ?ð ?ð ?ð ?r.   r½   c                   óV   ‡ — e Zd Zdefˆ fd„Zdej        dej        dej        fd„Zˆ xZS )ÚNeoMMESigmoidGatedProjectionrK   c                 óô   •— t          ¦   «                              ¦   «          |j        |j        z  }t	          j        |j        |d¬¦  «        | _        t	          j        ||j        |j        ¬¦  «        | _	        d S rM   )
r%   r&   rŒ   rˆ   r   rT   rU   Ú	gate_projÚattention_biasÚo_proj)r+   rK   Úprojection_sizer,   s      €r-   r&   z%NeoMMESigmoidGatedProjection.__init__Í   se   ø€ Ý‰Œ×ÒÑÔÐØ Ô4°v´ÑFˆÝœ 6Ô#5°ÈUÐSÑSÔSˆŒÝ”i °Ô1CÈ&ÔJ_Ð`Ñ`Ô`ˆŒˆˆr.   rÂ   r/   r:   c                 ó|   — |                       |t          j        |                      |¦  «        ¦  «        z  ¦  «        S r<   )rÓ   r(   ÚsigmoidrÑ   )r+   rÂ   r/   s      r-   r@   z$NeoMMESigmoidGatedProjection.forwardÓ   s/   € Ø�{Š{˜;­¬°t·~²~ÀmÑ7TÔ7TÑ)UÔ)UÑUÑVÔVÐVr.   rÍ   rH   s   @r-   rÏ   rÏ   Ì   s‡   ø€ € € € € ða˜|ð að að að að að aðW 5¤<ð WÀÄð WÐQVÔQ]ð Wð Wð Wð Wð Wð Wð Wð Wr.   rÏ   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..Nr2   r1   r¨   )r›   r(   r¬   )r“   Úx1Úx2s      r-   Úrotate_halfrÚ   ×   s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'r.   c                 ó˜  — |                      |¦  «        }|                      |¦  «        }|j        d         }| dd|…f         | d|d…f         }}|dd|…f         |d|d…f         }	}||z  t          |¦  «        |z  z   }
||z  t          |¦  «        |z  z   }t          j        |
|gd¬¦  «        }
t          j        ||	gd¬¦  «        }|
|fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    r2   .Nr¨   )Ú	unsqueezer›   rÚ   r(   r¬   )ÚqÚkr    r¡   Úunsqueeze_dimÚ
rotary_dimÚq_rotÚq_passÚk_rotÚk_passÚq_embedÚk_embeds               r-   Úapply_rotary_pos_embrç   Þ   sô   € ð$ �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€Cð ”˜2”€JØ�c˜;˜J˜;Ð&Ô'¨¨3°
°°Ð+;Ô)<ˆ6€EØ�c˜;˜J˜;Ð&Ô'¨¨3°
°°Ð+;Ô)<ˆ6€Eð �s‰{�{¨5Ñ1Ô1°CÑ7Ñ8€GØ�s‰{�{¨5Ñ1Ô1°CÑ7Ñ8€Gõ Œi˜ &Ð)¨rÐ2Ñ2Ô2€GÝŒi˜ &Ð)¨rÐ2Ñ2Ô2€GØ�GÐÐr.   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutr�   c                 ó  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        |z  }
|�|
|z   }
t
          j                             |
dt          j        ¬¦  «         	                    |j
        ¦  «        }
t
          j                             |
|| j        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «                             ¦   «         }||
fS )Nr1   r   r2   )r    r‰   )ÚpÚtrainingr   )r»   r¿   r(   ÚmatmulrŸ   r   Ú
functionalÚsoftmaxÚfloat32rŽ   r‰   rï   rò   Ú
contiguous)ré   rê   rë   rì   rí   rî   rï   r�   Ú
key_statesrÃ   Úattn_weightsrÂ   s               r-   Úeager_attention_forwardrú     sé   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r.   c                   óÖ   ‡ — e Zd ZdZdedefˆ fd„Z	 	 ddej        de	ej        ej        f         dej        dz  d	ej        dz  d
e
e         de	ej        ej        dz  f         fd„Zˆ xZS )ÚNeoMMEAttentionzõBidirectional grouped-query attention with QK-norm, M-RoPE, and a sigmoid output gate.

    QK-norm runs before rotary embedding, value embeddings are added after rotation, and
    exclusive self-attention is applied before the output gate.
    rK   Ú	layer_idxc                 óÊ  •— t          ¦   «                              ¦   «          || _        || _        t	          |d|j        |j        z  ¦  «        | _        |j        |j        z  | _	        | j        dz  | _
        |j        | _        d| _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        |j        |         dk    | _        | j        r|j        |         j        dz   nd | _        |j        |         | _        |j        | _        t1          |j        |j        d¬¦  «        | _        t1          |j        |j        d¬¦  «        | _        t9          |¦  «        | _        t=          |¦  «        | _        d S )Nrˆ   r5   FrN   Úsliding_attentionr   ©r"   ) r%   r&   rK   rý   r‹   rU   rŒ   rˆ   r¹   r¿   rî   Úattention_dropoutÚ	is_causalr   rT   rÒ   Úq_projÚk_projÚv_projrz   Úis_local_attentionÚper_layer_configÚsliding_windowÚattention_typer   Únorm_epsÚq_normÚk_normr½   Úexclusive_self_attentionrÏ   Úoutput_projection©r+   rK   rý   r,   s      €r-   r&   zNeoMMEAttention.__init__"  sÊ  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒÝ ¨
°FÔ4FÈ&ÔJdÑ4dÑeÔeˆŒØ$*Ô$>À&ÔB\Ñ$\ˆÔ!Ø”} dÑ*ˆŒØ!'Ô!9ˆÔØˆŒå”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒð
 #)Ô"4°YÔ"?ÐCVÒ"VˆÔð
 FJÔE\ÐfˆFÔ# IÔ.Ô=ÀÑAÐAÐbfð 	Ôð %Ô0°Ô;ˆÔØ#)Ô#=ˆÔ Ý# F¤O°V´_ÐQVÐWÑWÔWˆŒÝ# F¤O°V´_ÐQVÐWÑWÔWˆŒÝ(DÀVÑ(LÔ(LˆÔ%Ý!=¸fÑ!EÔ!EˆÔÐÐr.   Nr/   Úposition_embeddingsrí   Úvalue_embedsr�   r:   c                 ó¦  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }	|                      |¦  «                             |¦  «                             dd¦  «        }
|                      |¦  «        }|                      |	¦  «        }	|\  }}t          ||	||¦  «        \  }}	|�,|
|                     |¦  «                             dd¦  «        z   }
t          j        | j        j        t          ¦  «        } || ||	|
|f| j        r| j        nd| j        | j        dœ|¤Ž\  }}|                      ||
¦  «        } |j        g |¢d‘R Ž }|                      ||¦  «        }||fS )Nr2   r   r1   rè   )rï   rî   r  )r›   rˆ   r  rÉ   rŸ   r  r  r  r  rç   r   Úget_interfacerK   Ú_attn_implementationrú   rò   r  rî   r  r  r·   r  )r+   r/   r  rí   r  r�   Úinput_shapeÚhidden_shapeÚquery_statesrø   rÃ   r    r¡   Úattention_interfacerÂ   rù   s                   r-   r@   zNeoMMEAttention.forwardF  s   € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆà—{’{ <Ñ0Ô0ˆØ—[’[ Ñ,Ô,ˆ
à&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jØÐ#Ø'¨,×*;Ò*;¸LÑ*IÔ*I×*SÒ*SÐTUÐWXÑ*YÔ*YÑYˆLå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð /3¬mÐD�DÔ*Ð*ÀØ”LØÔ.ð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð ×3Ò3°KÀÑNÔNˆØ)�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;ˆØ×,Ò,¨[¸-ÑHÔHˆØ˜LÐ(Ð(r.   r±   ©rA   rB   rC   rZ   r   rD   r&   r(   rF   r³   r   r   r@   rG   rH   s   @r-   rü   rü     sð   ø€ € € € € ðð ð"F˜|ð "F¸ð "Fð "Fð "Fð "Fð "Fð "FðP /3Ø,0ð*)ð *)à”|ð*)ð # 5¤<°´Ð#=Ô>ð*)ð œ tÑ+ð	*)ð
 ”l TÑ)ð*)ð Ð+Ô,ð*)ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð*)ð *)ð *)ð *)ð *)ð *)ð *)ð *)r.   rü   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )Ú	NeoMMEMLPc                 ó`  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        t          j        | j        | j        |j        ¬¦  «        | _        t          j        | j        | j        |j        ¬¦  «        | _	        t          |j                 | _        d S )NrN   )r%   r&   rK   rU   Úintermediate_sizer   rT   Úmlp_biasrb   re   r   Ú
hidden_actrd   rW   s     €r-   r&   zNeoMMEMLP.__init__t  s�   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ!'Ô!9ˆÔÝ”y Ô!1°4Ô3IÐPVÔP_Ð`Ñ`Ô`ˆŒÝœ 4Ô#9¸4Ô;KÐRXÔRaÐbÑbÔbˆŒÝ˜VÔ.Ô/ˆŒˆˆr.   c                 óx   — |                       |                      |                      |¦  «        ¦  «        ¦  «        S r<   )re   rd   rb   )r+   r“   s     r-   r@   zNeoMMEMLP.forward}  s*   € Ø�~Š~˜dŸkšk¨$¯,ª,°q©/¬/Ñ:Ô:Ñ;Ô;Ð;r.   )rA   rB   rC   r&   r@   rG   rH   s   @r-   r  r  s  sG   ø€ € € € € ð0ð 0ð 0ð 0ð 0ð<ð <ð <ð <ð <ð <ð <r.   r  c                   óÌ   ‡ — e Zd ZdZdedefˆ fd„Z	 	 	 ddej        dej        dej        dz  d	e	ej        ej        f         dz  d
ej        dz  de
e         dej        fd„Zˆ xZS )ÚNeoMMEEncoderLayerzGPre-norm encoder layer with initial-state mixing and muP depth scaling.rK   rý   c                 ó´  •— t          ¦   «                              ¦   «          t          ||¦  «        | _        t	          |¦  «        | _        t          j        t          j	        ddg¦  «        ¦  «        | _
        t          |j        |j        d¬¦  «        | _        t          |j        |j        d¬¦  «        | _        |j        | _        |j        |         | _        d S )Nr‡   rè   Fr   )r%   r&   rü   Ú	self_attnr  Úmlpr   r'   r(   ÚtensorÚlambdasr   rU   r
  Úinput_layernormÚpost_attention_layernormÚresidual_multiplierrz   r	  r  s      €r-   r&   zNeoMMEEncoderLayer.__init__„  s³   ø€ Ý‰Œ×ÒÑÔÐÝ(¨°Ñ;Ô;ˆŒÝ˜VÑ$Ô$ˆŒÝ”|¥E¤L°#°s°Ñ$<Ô$<Ñ=Ô=ˆŒÝ,¨VÔ-?ÀÄÐ]bÐcÑcÔcˆÔÝ(5°fÔ6HÈ&Ì/ÐfkÐ(lÑ(lÔ(lˆÔ%Ø#)Ô#=ˆÔ Ø$Ô0°Ô;ˆÔÐÐr.   Nr/   Úinitial_hidden_statesr  r  rí   r�   r:   c                 ó   — | j         d         |z  | j         d         |z  z   }|                      |¦  «        } | j        |f|||dœ|¤Ž\  }	}
|| j        |	z  z   }|                      |¦  «        }|                      |¦  «        }|| j        |z  z   S )Nr   r   )r  rí   r  )r'  r(  r$  r*  r)  r%  )r+   r/   r+  r  r  rí   r�   Úmixed_statesÚnormed_statesrÂ   Ú_Ú
mlp_outputs               r-   r@   zNeoMMEEncoderLayer.forwardŽ  sº   € ð ”| A”¨Ñ6¸¼Àa¼ÐK`Ñ9`Ñ`ˆØ×,Ò,¨\Ñ:Ô:ˆØ'˜œØð
à 3Ø)Ø%ð	
ð 
ð
 ð
ð 
‰ˆ�Qð &¨Ô(@À;Ñ(NÑNˆà×5Ò5°mÑDÔDˆØ—X’X˜mÑ,Ô,ˆ
Ø˜tÔ7¸*ÑDÑDÐDr.   ©NNNr  rH   s   @r-   r"  r"  �  sö   ø€ € € € € ØQÐQð<˜|ð <¸ð <ð <ð <ð <ð <ð <ð -1ØHLØ.2ðEð Eà”|ðEð  %œ|ðEð ”l TÑ)ð	Eð
 # 5¤<°´Ð#=Ô>ÀÑEðEð œ tÑ+ðEð Ð+Ô,ðEð 
ŒðEð Eð Eð Eð Eð Eð Eð Er.   r"  c            	       óú   ‡ — e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZdZeedœZdZdej        fd	„Zd
ej        ddfd„Z ej        ¦   «         dej        fˆ fd„¦   «         Z	 ddededz  dedej        fˆ fd„Zˆ xZS )ÚNeoMMEPreTrainedModelrK   ÚmodelTr"  Úpast_key_values)r/   Ú
attentions)ÚimageÚtextr:   c                 óF   — t          | | j        | ¦  «        }|j        j        S r<   ©r‹   Úbase_model_prefixÚembed_tokensrS   )r+   Úbackbones     r-   Úget_input_embeddingsz*NeoMMEPreTrainedModel.get_input_embeddingsº  s"   € Ý˜4 Ô!7¸Ñ>Ô>ˆØÔ$Ô4Ð4r.   rì   Nc                 óJ   — t          | | j        | ¦  «        }||j        _        d S r<   r:  )r+   rì   r=  s      r-   Úset_input_embeddingsz*NeoMMEPreTrainedModel.set_input_embeddings¾  s'   € Ý˜4 Ô!7¸Ñ>Ô>ˆØ05ˆÔÔ-Ð-Ð-r.   ré   c                 óê  •— t          ¦   «                              |¦  «         t          |t          ¦  «        r0t	          j        |j        j        d| j        j	        dz  ¬¦  «         d S t          |t          ¦  «        rt	          j        |j        ¦  «         d S t          |t          ¦  «        r t	          j        |j        j        ¦  «         d S t          |t          ¦  «        r t	          j        |j        j        ¦  «         d S t          |t"          ¦  «        r0t	          j        |j        t)          j        ddg¦  «        ¦  «         d S t          |t,          ¦  «        r t	          j        |j        j        ¦  «         d S t          |t0          ¦  «        r0t	          j        |j        j        d| j        j	        dz  ¬¦  «         d S t          |t4          ¦  «        r›|j        D ]•}|j        }|j        |         dk    rt<          |j        |                  } ||j        |¬¦  «        \  }}t	          j        t?          ||› d�¦  «        |¦  «         t	          j        t?          ||› d�¦  «        |¦  «         Œ”d S d S )	Nrè   r5   )r7   Ústdr‡   rn   ro   rq   rs   ) r%   Ú_init_weightsrœ   rJ   ÚinitÚnormal_rS   r*   rK   rR   r½   Úzeros_rÁ   rÏ   rÓ   r  re   r"  Úcopy_r'  r(   r&  ÚNeoMMEModelÚvalue_embeddingsÚNeoMMEForMaskedLMÚlm_headri   rz   r|   rm   r   r‹   )r+   ré   rp   r�   r’   r/  r,   s         €r-   rC  z#NeoMMEPreTrainedModel._init_weightsÂ  si  ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%å�fÕ.Ñ/Ô/ð 	YÝŒL˜Ô/Ô6¸SÀdÄkÔF`ÐbfÑFfÐgÑgÔgÐgÐgÐgÝ˜Õ <Ñ=Ô=ð 	YÝŒK˜œÑ%Ô%Ð%Ð%Ð%Ý˜Õ <Ñ=Ô=ð 	YåŒK˜œÔ,Ñ-Ô-Ð-Ð-Ð-Ý˜¥	Ñ*Ô*ð 	YÝŒK˜Ô(Ô/Ñ0Ô0Ð0Ð0Ð0Ý˜Õ 2Ñ3Ô3ð 	YÝŒJ�v”~¥u¤|°S¸#°JÑ'?Ô'?Ñ@Ô@Ð@Ð@Ð@Ý˜¥Ñ,Ô,ð 	YÝŒK˜Ô/Ô6Ñ7Ô7Ð7Ð7Ð7Ý˜Õ 1Ñ2Ô2ð 		YÝŒL˜œÔ.°S¸d¼kÔ>XÐZ^Ñ>^Ð_Ñ_Ô_Ð_Ð_Ð_Ý˜Õ 5Ñ6Ô6ð 	YØ$Ô0ð Yð Y�
Ø%ÔE�ØÔ# JÔ/°9Ò<Ð<Ý#6°vÔ7GÈ
Ô7SÔ#T�LØ*˜l¨6¬=ÀZÐPÑPÔP‘�˜!Ý”
�7 6¨jÐ+CÐ+CÐ+CÑDÔDÀhÑOÔOÐOÝ”
�7 6¨jÐ+LÐ+LÐ+LÑMÔMÈxÑXÔXÐXÐXð	Yð 	YðYð Yr.   Únew_num_tokensÚpad_to_multiple_ofÚmean_resizingc                 ó>  •— t          ¦   «                              |||¦  «        }t          | | j        | ¦  «        }|                      |j        |j        j        d         |¬¦  «        }t          j	        |j
        |j        ¦  «        |_        |j        |j        _        |S )z0Resize word and value embedding tables together.r   )rN  )r%   Ú_resize_token_embeddingsr‹   r;  Ú_get_resized_embeddingsrI  r*   r›   r   rP   Únum_embeddingsÚembedding_dim)r+   rL  rM  rN  rS   r=  Úresizedr,   s          €r-   rP  z.NeoMMEPreTrainedModel._resize_token_embeddingsß  s•   ø€ õ  ™'œ'×:Ò:¸>ÐK]Ð_lÑmÔmˆÝ˜4 Ô!7¸Ñ>Ô>ˆØ×.Ò.ØÔ% Ô'=Ô'CÀAÔ'FÐVcð /ñ 
ô 
ˆõ %'¤L°Ô1GÈÔI^Ñ$_Ô$_ˆÔ!Ø+2¬>ˆÔ!Ô(ØÐr.   )NT)rA   rB   rC   r   Ú__annotations__r;  Úsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr"  rü   Ú_can_record_outputsÚinput_modalitiesr   rP   r>  r@  r(   r´   ÚModulerC  rD   rE   rP  rG   rH   s   @r-   r3  r3  §  sV  ø€ € € € € € àÐÐÑØÐØ&*Ð#Ø-Ð.ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&Ðà+Ø%ðð Ðð )Ðð5 b¤lð 5ð 5ð 5ð 5ð6¨"¬,ð 6¸4ð 6ð 6ð 6ð 6ð €U„]�_„_ðY B¤Ið Yð Yð Yð Yð Yñ „_ðYð: aeðð Ø!ðØ7:¸T±zðØY]ðà	Œðð ð ð ð ð ð ð ð ð r.   r3  zu
    The bare NeoMME model. It encodes text tokens and image patches with one bidirectional Transformer encoder.
    ©Úcustom_introc                   óB  ‡ — e Zd Zdefˆ fd„Ze ed¬¦  «        dej        de	e
         defd„¦   «         ¦   «         Zd	ej        d
ej        dej        fd„Zee	 	 	 dd	ej        dej        dz  dej        dz  dej        dz  de	e
         defd„¦   «         ¦   «         Zˆ xZS )rH  rK   c                 óª  •‡— t          ¦   «                              ‰¦  «         t          ‰¦  «        | _        t	          ‰¦  «        | _        t          ‰¦  «        | _        t          ‰j	        ‰j
        d¬¦  «        | _        t          ‰j	        ‰j
        d¬¦  «        | _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        d„ t%          ‰j        ¦  «        D ¦   «         }t          j        ‰j        ‰j        ‰j        z  ¦  «        | _        |d         |d         h| _        d| _        |                      ¦   «          d S )NFr   c                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS © )r"  )Ú.0rý   rK   s     €r-   ú
<listcomp>z(NeoMMEModel.__init__.<locals>.<listcomp>û  s$   ø€ ÐdÐdÐd°yÕ ¨	Ñ2Ô2ÐdÐdÐdr.   c                 ó$   — g | ]\  }}|d k    ¯|‘ŒS )Úfull_attentionrf  )rg  Úirp   s      r-   rh  z(NeoMMEModel.__init__.<locals>.<listcomp>þ  s(   € ÐpÐpÐp™}˜q *ÐQ[Ð_oÒQoÐQo˜ÐQoÐQoÐQor.   r   r2   )r%   r&   rJ   r<  r]   Úpatch_embeddingsri   Ú
rotary_embr   rU   r
  Úembedding_normÚ
final_normr   Ú
ModuleListÚrangeÚnum_hidden_layersÚlayersÚ	enumeraterz   rP   rQ   r¹   rˆ   rI  Úvalue_embedding_layersÚgradient_checkpointingÚ	post_init)r+   rK   Úglobal_layersr,   s    ` €r-   r&   zNeoMMEModel.__init__ó  s2  øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý,¨VÑ4Ô4ˆÔÝ 5°fÑ =Ô =ˆÔÝ/°Ñ7Ô7ˆŒÝ+¨FÔ,>ÀÄÐ\aÐbÑbÔbˆÔÝ'¨Ô(:¸F¼OÐX]Ð^Ñ^Ô^ˆŒÝ”mØdÐdÐdÐdÅEÈ&ÔJbÑDcÔDcÐdÑdÔdñ
ô 
ˆŒð qÐpµ	¸&Ô:LÑ0MÔ0MÐpÑpÔpˆÝ "¤¨VÔ->ÀÔ@ZÐ]cÔ]lÑ@lÑ mÔ mˆÔØ'4°QÔ'7¸ÀrÔ9JÐ&KˆÔ#Ø&+ˆÔ#Ø�ŠÑÔÐÐÐr.   z=Projects flattened image patches into the model hidden space.ra  rf   r�   r:   c           	      óB  — |j         d         | j        j        k    r8t          d|j         d         › d| j        j        › d| j        j        › d�¦  «        ‚|                      |                     | j        j        j        j	        ¦  «        ¦  «        }t          ||¬¦  «        S )Nr2   zpixel_values has patch width z but the model expects z( (= 3 * patch_size ** 2 with patch_size=ú))Úlast_hidden_stateÚpooler_output)r›   rK   r`   Ú
ValueErrorÚ
patch_sizerl  rŽ   ra   r*   r‰   r   )r+   rf   r�   Úimage_featuress       r-   Úget_image_featureszNeoMMEModel.get_image_features  sº   € ð
 Ô˜bÔ! T¤[Ô%:Ò:Ð:Ýðl°Ô0BÀ2Ô0Fð lð lØ”;Ô(ðlð lØRVÔR]ÔRhðlð lð lñô ð ð ×.Ò.¨|¯ª¸tÔ?TÔ?YÔ?`Ô?fÑ/gÔ/gÑhÔhˆÝ)¸NÐZhÐiÑiÔiÐir.   rX   r  c                 ó~  ‡‡— t          j        |dd…dd…f         d| j        j        pd¬¦  «        }|| j        j        k    || j        j        k    z  }|                     ¦   «         Št          ‰‰j        d         k    ˆˆfd„¦  «         | 	                    d¦  «         
                    dd‰j        d         ¦  «        S )zPFind patch placeholders and validate that every image feature has a destination.Nr2   )r   r   r   )rì   c                  óF   •— d‰ j         d         › dt          ‰¦  «        › d�S )NzGot r   z image patches for z image placeholder tokens)r›   rD   )r  Únum_image_tokenss   €€r-   ú<lambda>z2NeoMMEModel.get_placeholder_mask.<locals>.<lambda>  s,   ø€ Ðw˜>Ô/°Ô2ÐwÐwÅsÐK[ÑG\ÔG\ÐwÐwÐw€ r.   )rÅ   ÚpadrK   Úpad_token_idÚimage_token_idÚdocument_token_idrÇ   r   r›   rÜ   rš   )r+   rX   r  Úprevious_idsÚ
image_maskrƒ  s     `  @r-   Úget_placeholder_maskz NeoMMEModel.get_placeholder_mask  sÇ   øø€ å”u˜Y q q q¨#¨2¨# vÔ.°¸d¼kÔ>VÐ>[ÐZ[Ð\Ñ\Ô\ˆØ 4¤;Ô#=Ò=À,ÐRVÔR]ÔRoÒBoÑpˆ
à%Ÿ>š>Ñ+Ô+ÐÝØ Ô 4°QÔ 7Ò7ØwÐwÐwÐwÐwñ	
ô 	
ð 	
ð ×#Ò# BÑ'Ô'×.Ò.¨r°2°~Ô7KÈBÔ7OÑPÔPÐPr.   Nrí   r”   c                 ó$  ‡ ‡‡— ‰                       |¦  «        Š|�M‰                      |d¬¦  «        }‰                      ||j        ¦  «        }‰                     ||j        ¦  «        Š‰j        dd…         \  }}	‰€/t          j        |	‰j        ¬¦  «         	                    |d¦  «        Š‰j
        dk    r*‰                     d¦  «         	                    ddd¦  «        Š‰                      ‰¦  «        xŠ}
t          |x}t          ¦  «        s_i }‰|dœ}t          ‰ j        j        ¦  «        D ]>}‰ j        j        |         }|j        �t)          dd	|i|¤Ž||<   Œ-t+          dd	|i|¤Ž||<   Œ?ˆˆˆ fd
„t-          ‰ j        j        ¦  «        D ¦   «         }‰                      |¦  «        }t3          ‰ j        ¦  «        D ]/\  }} |‰|
|‰ j        v r|ndf||j                 ||         dœ|¤ŽŠŒ0‰                      ‰¦  «        Št=          ‰¬¦  «        S )a6  
        position_ids (`torch.LongTensor` of shape `(2, batch_size, sequence_length)` or `(batch_size, sequence_length)`, *optional*):
            Positions for the input tokens. [`NeoMMEProcessor`] returns two-axis positions for document images. A
            one-axis position tensor is used for text inputs.
        pixel_values (`torch.Tensor` of shape `(num_patches, 3 * patch_size ** 2)`, *optional*):
            Flattened image patches returned by [`NeoMMEProcessor`]. The model places these patches at image
            placeholders in `input_ids`.
        NT)Úreturn_dictr1   )rj   r2   r   )Úinputs_embedsrí   rK   c                 ó@   •— i | ]}|‰                      ‰‰|¦  «        “ŒS rf  )rm  )rg  rp   r/   r”   r+   s     €€€r-   ú
<dictcomp>z'NeoMMEModel.forward.<locals>.<dictcomp>O  s;   ø€ ð 
ð 
ð 
àð ˜Ÿš¨°|ÀZÑPÔPð
ð 
ð 
r.   )r  rí   )r{  rf  )r<  r€  r‹  r|  Úmasked_scatterr›   r(   r�   rj   rš   ÚndimrÜ   rn  rœ   Údictrq  rK   rr  r  r  r
   r	   ry   rz   rI  rt  rs  ru  r	  ro  r   )r+   rX   rí   r”   rf   r�   Úimage_outputsrŠ  Ú
batch_sizeÚseq_lenr+  Úattention_mask_mappingÚmask_kwargsÚlayer_idr  r  r  rý   Úencoder_layerr/   s   `  `               @r-   r@   zNeoMMEModel.forward  s¥  øøø€ ð$ ×)Ò)¨)Ñ4Ô4ˆØÐ#Ø ×3Ò3°LÈdÐ3ÑSÔSˆMØ×2Ò2°9¸mÔ>YÑZÔZˆJØ)×8Ò8¸À]ÔE`ÑaÔaˆMà+Ô1°"°1°"Ô5Ñˆ
�GàÐÝ œ<¨¸Ô8LÐMÑMÔM×TÒTÐU_ÐacÑdÔdˆLØÔ Ò!Ð!Ø'×1Ò1°!Ñ4Ô4×;Ò;¸A¸rÀ2ÑFÔFˆLð 15×0CÒ0CÀMÑ0RÔ0RÐRˆÐ-å°NÐBÐ0ÅDÑIÔIð 	ØEGÐ"Ø,9È^Ð\Ð\ˆKÝ! $¤+Ô"?Ñ@Ô@ð ð �Ø#'¤;Ô#?ÀÔ#IÐ Ø#Ô2Ð>Ý7_ð 8ð 8Ø/ð8à%ð8ð 8Ð*¨8Ñ4Ð4õ
 8Qð 8ð 8Ø/ð8à%ð8ð 8Ð*¨8Ñ4Ð4ð

ð 
ð 
ð 
ð 
ð 
å! $¤+Ô"9Ñ:Ô:ð
ñ 
ô 
Ðð
 ×,Ò,¨YÑ7Ô7ˆå(1°$´+Ñ(>Ô(>ð 		ð 		Ñ$ˆI�}à)˜MØØ%Ø )¨TÔ-HÐ HÐ H��Èdðð %8¸Ô8TÔ$UØ5°iÔ@ðð ð ðð ˆMˆMð Ÿš¨Ñ6Ô6ˆÝ°Ð?Ñ?Ô?Ð?r.   r1  )rA   rB   rC   r   r&   r   r   r(   rF   r   r   r   r€  r[   Ú
BoolTensorr‹  r   r   r@   rG   rH   s   @r-   rH  rH  í  s’  ø€ € € € € ð˜|ð ð ð ð ð ð ð" Ø€^Ð!`ÐaÑaÔað	jØ!œLð	jØ4:Ð;MÔ4Nð	jà	#ð	jð 	jð 	jñ bÔañ Ôð	jð
Q¨eÔ.>ð 
QÐPUÔP\ð 
QÐafÔaqð 
Qð 
Qð 
Qð 
Qð Øð /3Ø04Ø,0ðC@ð C@àÔ#ðC@ð œ tÑ+ðC@ð Ô&¨Ñ-ð	C@ð
 ”l TÑ)ðC@ð Ð+Ô,ðC@ð 
ðC@ð C@ð C@ñ „^ñ „_ðC@ð C@ð C@ð C@ð C@r.   rH  zB
    The NeoMME model with a factorized masked token decoder.
    c                   óÚ   ‡ — e Zd ZdddœZdefˆ fd„Zee	 	 	 	 	 ddej	        dz  dej
        dz  d	ej	        dz  d
ej
        dz  dej	        dz  dee         defd„¦   «         ¦   «         Zˆ xZS )rJ  z)model.embed_tokens.word_embeddings.weightz.model.embed_tokens.embedding_projection.weight)zlm_head.weightzunembedding_projection.weightrK   c                 ó2  •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        |j        d¬¦  «        | _        t	          j        |j        |j	        d¬¦  «        | _
        |                      ¦   «          d S rM   )r%   r&   rH  r4  r   rT   rR   rU   Úunembedding_projectionrQ   rK  rw  rW   s     €r-   r&   zNeoMMEForMaskedLM.__init__p  s   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý  Ñ(Ô(ˆŒ
Ý&(¤i°Ô0EÀvÔGYÐ`eÐ&fÑ&fÔ&fˆÔ#Ý”y Ô!6¸Ô8IÐPUÐVÑVÔVˆŒØ�ŠÑÔÐÐÐr.   NrX   rí   r”   rf   Úlabelsr�   r:   c                 óô   —  | j         d||||dœ|¤Ž}|j        }|| j        j        z  }|                      |¦  «        }	d}
|� | j        |	|fd| j        j        i|¤Ž}
t          |
|	|j	        |j
        ¬¦  «        S )a&  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for the masked-language-modeling loss. Indices should be in `[0, ..., config.vocab_size - 1]`
            or `-100`; only tokens with a label different from `-100` contribute.
        ©rX   rí   r”   rf   NrQ   )ÚlossÚlogitsr/   r6  rf  )r4  r{  rž  r*   rK  Úloss_functionrK   rQ   r   r/   r6  )r+   rX   rí   r”   rf   rŸ  r�   Úoutputsr/   r£  r¢  s              r-   r@   zNeoMMEForMaskedLM.forwardw  s½   € ð  $. 4¤:ð $
ØØ)Ø%Ø%ð	$
ð $
ð
 ð$
ð $
ˆð  Ô1ˆØ%¨Ô(CÔ(JÑJˆØ—’˜mÑ,Ô,ˆàˆØÐØ%�4Ô% f¨fÐbÐbÀÄÔAWÐbÐ[aÐbÐbˆDåØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r.   )NNNNN)rA   rB   rC   Ú_tied_weights_keysr   r&   r   r   r(   r[   rF   r   r   r   r@   rG   rH   s   @r-   rJ  rJ  e  s  ø€ € € € € ð FØ)Yðð Ðð
˜|ð ð ð ð ð ð ð Øð .2Ø.2Ø04Ø,0Ø*.ð"
ð "
àÔ# dÑ*ð"
ð œ tÑ+ð"
ð Ô&¨Ñ-ð	"
ð
 ”l TÑ)ð"
ð Ô  4Ñ'ð"
ð Ð+Ô,ð"
ð 
ð"
ð "
ð "
ñ „^ñ Ôð"
ð "
ð "
ð "
ð "
r.   rJ  z1
    Output type for [`NeoMMEForRetrieval`].
    c                   ón   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	ej        dz  ed<   dS )ÚNeoMMEForRetrievalOutputau  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*):
        Retrieval loss. This value is always `None`.
    embeddings (`torch.FloatTensor` of shape `(batch_size, sequence_length, embedding_dim)`, *optional*):
        Normalized token embeddings for late-interaction retrieval. Padding rows are zeroed. Score them with MeanMaxSim.
    dense_embeddings (`torch.FloatTensor` of shape `(batch_size, hidden_size)` or `(batch_size, dense_dim)`, *optional*):
        A normalized mean-pooled embedding for each input. When `dense_dim` is set, the last dimension is `dense_dim`.
        Score them with cosine similarity.
    Nr¢  Ú
embeddingsÚdense_embeddings)
rA   rB   rC   rZ   r¢  r(   ÚFloatTensorrU  r©  rª  rf  r.   r-   r¨  r¨  ž  sg   € € € € € € ðð ð &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø+/€J�Ô! DÑ(Ð/Ð/Ñ/Ø15Ð�eÔ'¨$Ñ.Ð5Ð5Ñ5Ð5Ð5r.   r¨  c                   óV   ‡ — e Zd Zdefˆ fd„Zdej        dej        dej        fd„Zˆ xZS )ÚNeoMMEMultiVectorHeadrK   c                 ó”   •— t          ¦   «                              ¦   «          t          j        |j        |j        d¬¦  «        | _        d S rM   )r%   r&   r   rT   rU   rS  ÚprojrW   s     €r-   r&   zNeoMMEMultiVectorHead.__init__µ  s;   ø€ Ý‰Œ×ÒÑÔÐÝ”I˜fÔ0°&Ô2FÈUÐSÑSÔSˆŒ	ˆ	ˆ	r.   r/   rí   r:   c                 ó  — |                       |                     | j         j        j        ¦  «        ¦  «        }t	          j        |d¬¦  «        }|                     |                     ¦   «                              d¦  «         d¦  «        S )Nr2   r¨   rè   )	r¯  rŽ   r*   r‰   rÅ   rÆ   Úmasked_fillrE   rÜ   )r+   r/   rí   r©  s       r-   r@   zNeoMMEMultiVectorHead.forward¹  sp   € Ø—Y’Y˜}×/Ò/°´	Ô0@Ô0FÑGÔGÑHÔHˆ
Ý”[ °Ð4Ñ4Ô4ˆ
à×%Ò% ~×':Ò':Ñ'<Ô'<×'FÒ'FÀrÑ'JÔ'JÐ&JÈCÑPÔPÐPr.   rÍ   rH   s   @r-   r­  r­  ´  s‡   ø€ € € € € ðT˜|ð Tð Tð Tð Tð Tð TðQ U¤\ð QÀ5Ä<ð QÐTYÔT`ð Qð Qð Qð Qð Qð Qð Qð Qr.   r­  c            	       óN   — e Zd Z	 ddej        dej        dedz  dej        fd„ZdS )ÚNeoMMEDenseHeadNr/   rí   Ú	dense_dimr:   c                 óò  — |                      d¦  «                             |j        ¦  «                             |j        ¦  «        }||z                       d¦  «        |                     d¦  «                             d¦  «        z  }|€t          j        |d¬¦  «        S d|cxk     r|j        d         k    s"n t          d|j        d         › d|› �¦  «        ‚t          j        |dd |…f         d¬¦  «        S )	Nr2   r   g•Ö&è.>r¨   r   zdense_dim must be in 1..z (the pooled width), got .)
rÜ   rš   r›   rŽ   r‰   rÇ   Ú	clamp_minrÅ   rÆ   r}  )r+   r/   rí   r´  Úexpanded_maskÚpooleds         r-   r@   zNeoMMEDenseHead.forwardÁ  sþ   € ð '×0Ò0°Ñ4Ô4×;Ò;¸MÔ<OÑPÔP×SÒSÐTaÔTgÑhÔhˆØ -Ñ/×4Ò4°QÑ7Ô7¸-×:KÒ:KÈAÑ:NÔ:N×:XÒ:XÐY]Ñ:^Ô:^Ñ^ˆØÐÝ”;˜v¨2Ð.Ñ.Ô.Ð.à�9Ð0Ð0Ò0Ð0 ¤¨RÔ 0Ò0Ð0Ð0Ð0ÝÐn¸¼ÀRÔ8HÐnÐnÐclÐnÐnÑoÔoÐoÝŒ{˜6 # z¨	 z /Ô2¸Ð;Ñ;Ô;Ð;r.   r<   )rA   rB   rC   r(   rF   rD   r@   rf  r.   r-   r³  r³  À  s\   € € € € € àaeð
<ð 
<Ø"œ\ð
<Ø;@¼<ð
<ØTWÐZ^ÑT^ð
<à	Œð
<ð 
<ð 
<ð 
<ð 
<ð 
<r.   r³  z½
    The NeoMME model with multi-vector and dense retrieval heads. One forward pass can return token embeddings for
    MaxSim scoring and mean-pooled embeddings for cosine similarity.
    c                   óÒ   ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 	 	 ddej        dz  dej	        dz  dej        dz  dej	        dz  d	e
d
e
dedz  dee         defd„¦   «         ¦   «         Zˆ xZS )ÚNeoMMEForRetrievalrK   c                 óè   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          |¦  «        | _        t          ¦   «         | _        |                      ¦   «          d S r<   )	r%   r&   rH  r4  r­  Úmulti_vector_headr³  Ú
dense_headrw  rW   s     €r-   r&   zNeoMMEForRetrieval.__init__Õ  s]   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý  Ñ(Ô(ˆŒ
Ý!6°vÑ!>Ô!>ˆÔÝ)Ñ+Ô+ˆŒØ�ŠÑÔÐÐÐr.   NTrX   rí   r”   rf   Úoutput_multivectorÚoutput_denser´  r�   r:   c                 ój  — |s|st          d¦  «        ‚ | j        d||||dœ|¤Ž}	|	j        }
|€3t          j        |
j        dd…         t          j        |
j        ¬¦  «        }|r|                      |
|¦  «        nd}|r|  	                    |
||¦  «        nd}t          |||
|	j        |	j        ¬¦  «        S )aà  
        output_multivector (`bool`, *optional*, defaults to `True`):
            Whether to return token embeddings for late-interaction retrieval.
        output_dense (`bool`, *optional*, defaults to `True`):
            Whether to return one mean-pooled dense embedding per input.
        dense_dim (`int`, *optional*):
            Width of the Matryoshka prefix to return for dense embeddings. The model truncates the pooled vector
            before normalizing it.
        zCAt least one of `output_multivector` or `output_dense` must be Truer¡  Nr1   )r‰   rj   )r©  rª  r{  r/   r6  rf  )r}  r4  r{  r(   r)   r›   rE   rj   r¼  r½  r¨  r/   r6  )r+   rX   rí   r”   rf   r¾  r¿  r´  r�   r¥  r/   r©  rª  s                r-   r@   zNeoMMEForRetrieval.forwardÜ  sü   € ð, #ð 	d lð 	dÝÐbÑcÔcÐcà#- 4¤:ð $
ØØ)Ø%Ø%ð	$
ð $
ð
 ð$
ð $
ˆð  Ô1ˆØÐ!Ý"œZ¨Ô(;¸B¸Q¸BÔ(?ÅuÄzÐZgÔZnÐoÑoÔoˆNàN`Ðj�T×+Ò+¨M¸>ÑJÔJÐJÐfjˆ
ØXdÐn˜4Ÿ?š?¨=¸.È)ÑTÔTÐTÐjnÐÝ'Ø!Ø-Ø+Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r.   )NNNNTTN)rA   rB   rC   r   r&   r   r   r(   r[   rF   rE   rD   r   r   r¨  r@   rG   rH   s   @r-   rº  rº  Î  s  ø€ € € € € ð˜|ð ð ð ð ð ð ð Øð .2Ø.2Ø04Ø,0Ø#'Ø!Ø $ð*
ð *
àÔ# dÑ*ð*
ð œ tÑ+ð*
ð Ô&¨Ñ-ð	*
ð
 ”l TÑ)ð*
ð !ð*
ð ð*
ð ˜‘:ð*
ð Ð+Ô,ð*
ð 
"ð*
ð *
ð *
ñ „^ñ Ôð*
ð *
ð *
ð *
ð *
r.   rº  )rJ  rº  rH  r3  )r   )rè   )EÚcollections.abcr   Údataclassesr   r(   Útorch.nn.functionalr   rô   rÅ   Ú r   rD  Úactivationsr   Úmasking_utilsr	   r
   Úmodeling_layersr   Úmodeling_outputsr   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_neommer   r`  r   rJ   r]   ri   rF   rD   r»   r½   rÏ   rÚ   rç   r=   rú   rü   r  r"  r3  rH  rJ  r¨  r­  r³  rº  Ú__all__rf  r.   r-   ú<module>rÒ     s  ðð* %Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø `Ð `Ð `Ð `Ð `Ð `Ð `Ð `Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø [Ð [Ð [Ð [Ð [Ð [Ð [Ð [Ð [Ð [Ø KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OØ 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø =Ð =Ð =Ð =Ð =Ð =Ð =Ð =Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø .Ð .Ð .Ð .Ð .Ð .ð4ð 4ð 4ð 4ð 4�B”Iñ 4ô 4ð 4ð*	Jð 	Jð 	Jð 	Jð 	J�r”yñ 	Jô 	Jð 	Jð:ð :ð :ð :ð :˜BœIñ :ô :ð :ðT3ð T3ð T3ð T3ð T3˜BœIñ T3ô T3ð T3ðn	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð?ð ?ð ?ð ?ð ? 2¤9ñ ?ô ?ð ?ðWð Wð Wð Wð W 2¤9ñ Wô Wð Wð(ð (ð (ð!ð !ð !ð !ðV ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð2U)ð U)ð U)ð U)ð U)�b”iñ U)ô U)ð U)ðp<ð <ð <ð <ð <�”	ñ <ô <ð <ð#Eð #Eð #Eð #Eð #EÐ3ñ #Eô #Eð #EðL ðBð Bð Bð Bð B˜Oñ Bô Bñ „ðBðJ €ððñ ô ð
p@ð p@ð p@ð p@ð p@Ð'ñ p@ô p@ñô ð
p@ðf €ððñ ô ð
1
ð 1
ð 1
ð 1
ð 1
Ð-ñ 1
ô 1
ñô ð
1
ðh €ððñ ô ð
 ð6ð 6ð 6ð 6ð 6˜ñ 6ô 6ñ „ñô ð6ð 	Qð 	Qð 	Qð 	Qð 	Q˜BœIñ 	Qô 	Qð 	Qð<ð <ð <ð <ð <�b”iñ <ô <ð <ð €ððñ ô ð4
ð 4
ð 4
ð 4
ð 4
Ð.ñ 4
ô 4
ñô ð4
ðn ^Ð
]Ð
]€€€r.   