§
    9²¸j&  ã                   óÌ   — d Z ddlZddlmZ ddlmZ ddlmZ ddlm	Z	m
Z
 ddlmZ  e
j        e¦  «        Zd	Z e	d
¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         ZdgZdS )zNeoMME model configuration.é    N)ÚLiteral)Ústricté   )ÚPreTrainedConfig)Úauto_docstringÚlogging)Úpositive_int_field©Úfull_attentionÚsliding_attentionzHcompany/NeoMME-260M)Ú
checkpointc                   ó*  ‡ — e Zd ZU dZdZddddddddœZdddœZd	d
dœZdZ e	d¬¦  «        Z
eed<    e	d¬¦  «        Zeed<    e	d¬¦  «        Zeed<    e	d¬¦  «        Zeed<   dZed         ed<   dZeed<    e	d¬¦  «        Zeed<    e	d¬¦  «        Zeed<    e	d¬¦  «        Zeed<    e	d¬¦  «        Zeed<    e	d ¬¦  «        Zeed!<   d"Zeed#<   d$Zeed%<   d&Zeez  ed'<   dZeed(<   d)Zee          d)z  ed*<   d)Z!e"ed         e"f         d)z  ed+<   dZ#ed)z  ed,<   d)Z$ed)z  ed-<    e	d.¬¦  «        Z%eed/<    e	d0¬¦  «        Z&eed1<   d2Z'ed)z  ed3<   d4Z(ed)z  ed5<   d6Z)ed)z  ed7<   d8Z*eed9<   ˆ fd:„Z+d;„ Z,d<„ Z-dAd>„Z.dAd?„Z/e0d=efd@„¦   «         Z1ˆ xZ2S )BÚNeoMMEConfigaü  
    embedding_rank (`int`, *optional*, defaults to 256):
        Width of the factorized token embedding table before projection to `hidden_size`.
    layer_types (`list[str]`, *optional*):
        By default, every sixth layer and the final layer use full attention.
    rope_parameters (`dict`, *optional*):
        Rotary-position settings for `"full_attention"` and `"sliding_attention"` layers. The rotated dimensions,
        `head_dim * partial_rotary_factor`, must be a positive multiple of four.
    residual_multiplier (`float`, *optional*):
        Scale applied to attention and MLP residual branches. Defaults to `1 / sqrt(2 * num_hidden_layers)`.
    embedding_dim (`int`, *optional*, defaults to 128):
        Width of the token-level embeddings returned by [`NeoMMEForRetrieval`]. This setting is unrelated to
        `embedding_rank`.
    document_token_id (`int`, *optional*, defaults to 5):
        Token ID for the `<doc>` marker.
    tie_word_embeddings (`bool`, *optional*, defaults to `True`):
        Whether the masked token decoder reuses both factorized token embedding weights.

    ```python
    >>> from transformers import NeoMMEModel, NeoMMEConfig

    >>> configuration = NeoMMEConfig()
    >>> model = NeoMMEModel(configuration)
    ```
    ÚneommeÚcolwiseÚrowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projz.layers.*.self_attn.output_projection.gate_projz+layers.*.self_attn.output_projection.o_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projg    €„.Ag     ˆÃ@r
   g      Ð?ç      ð?i   i   )ÚdefaultÚ
vocab_sizeé   Úembedding_rankÚhidden_sizei   Úintermediate_sizeÚrelu2Ú
hidden_actFÚmlp_biasé   Únum_hidden_layersé   Únum_attention_headsé   Únum_key_value_headsé@   Úhead_dimi @  Úmax_position_embeddingsg�íµ ÷Æ°>Únorm_epsg{®Gáz”?Úinitializer_rangeç        Úattention_dropoutÚattention_biasNÚlayer_typesÚrope_parametersÚsliding_windowÚresidual_multiplieré    Ú
patch_sizeé€   Úembedding_dimr   Úpad_token_idé   Údocument_token_idé   Úimage_token_idTÚtie_word_embeddingsc                 ó¤  •‡ — ‰ j         €%ˆ fd„t          ‰ j        ¦  «        D ¦   «         ‰ _         ‰                      ¦   «          d|vrWd}i |d<   t	          ‰ j         ¦  «        D ];\  }}|dk    rdd i|d         |<   Œ|dk    r|dz  rd‰ j        i|d         |<   |dz  }Œ<‰ j        €d‰ j        z  d	z  ‰ _         t          ¦   «         j        d
i |¤Ž d S )Nc                 óL   •— g | ] }|d z   dz  dk    s|‰j         d z
  k    rdnd‘Œ!S )é   r6   r   r   r   )r   )Ú.0ÚiÚselfs     €ú/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/neomme/configuration_neomme.pyú
<listcomp>z.NeoMMEConfig.__post_init__.<locals>.<listcomp>h   sU   ø€ ð  ð  ð  àð &'¨¡U¨a¡K°1Ò$4Ð$4¸¸TÔ=SÐVWÑ=WÒ8WÐ8WÐ Ð Ð]pð ð  ð  ó    Úper_layer_configr   r   r-   r   é   r;   g      à¿© )	r+   Úranger   Úvalidate_layer_typeÚ	enumerateÚdefault_long_sliding_windowr.   ÚsuperÚ__post_init__)r>   ÚkwargsÚsliding_idxÚ	layer_idxÚ
layer_typeÚ	__class__s   `    €r?   rJ   zNeoMMEConfig.__post_init__f   s3  øø€ ØÔÐ#ð ð  ð  ð  å˜tÔ5Ñ6Ô6ð ñ  ô  ˆDÔð 	× Ò Ñ"Ô"Ð"à VÐ+Ð+ØˆKØ)+ˆFÐ%Ñ&Ý)2°4Ô3CÑ)DÔ)Dð %ð %Ñ%�	˜:ØÐ!1Ò1Ð1Ø=MÈtÐ<T�FÐ-Ô.¨yÑ9Ð9ØÐ#6Ò6Ð6à" Q‘ð uØAQÐSWÔSsÐ@t˜Ð1Ô2°9Ñ=Ø 1Ñ$�KøØÔ#Ð+Ø()¨DÔ,BÑ(BÀtÑ'KˆDÔ$à�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'rA   c                 óP  — | j         | j        z  rt          d¦  «        ‚t          | j        ¦  «        D ]?\  }}|j        }|�1t          |t          ¦  «        r|dk    rt          d|› d|›d�¦  «        ‚Œ@t          j	        | j
        ¦  «        r| j
        dk    rt          d¦  «        ‚dS )zOPart of `@strict`-powered validation. Validates the architecture of the config.z3num_key_value_heads must divide num_attention_headsNr   zsliding_window for layer=z) must be a positive integer or None, got ú.z/residual_multiplier must be finite and positive)r    r"   Ú
ValueErrorrG   rB   r-   Ú
isinstanceÚintÚmathÚisfiniter.   )r>   rM   Úlayer_configr-   s       r?   Úvalidate_architecturez"NeoMMEConfig.validate_architecture~   sÕ   € àÔ# dÔ&>Ñ>ð 	TÝÐRÑSÔSÐSå'0°Ô1FÑ'GÔ'Gð 	ð 	Ñ#ˆI�|Ø)Ô8ˆNØÐ)µ:¸nÍcÑ3RÔ3RÐ)ÐVdÐhiÒViÐViÝ Øw°	ÐwÐwÐdrÐwÐwÐwñô ð øõ Œ}˜TÔ5Ñ6Ô6ð 	P¸$Ô:RÐVWÒ:WÐ:WÝÐNÑOÔOÐOð ;XÐ:WrA   c                 óü  — |                      dd ¦  «        }| j        �| j        ni | _        t          | j        ¦  «        D ]“}| j                             |i ¦  «        }|                     dd¦  «         |                     d|�|n| j                             |¦  «        ¦  «         |                     d| j                             |¦  «        ¦  «         Œ”|                      ¦   «          |  	                    ¦   «          |S )NÚ
rope_thetaÚ	rope_typer   Úpartial_rotary_factor)
Úpopr,   Úsetr+   Ú
setdefaultÚdefault_thetaÚgetÚdefault_partial_rotary_factorÚstandardize_rope_paramsÚ_validate_rotary_dims)r>   rK   rZ   rN   Úlayer_paramss        r?   Úconvert_rope_params_to_dictz(NeoMMEConfig.convert_rope_params_to_dict�   s  € Ø—Z’Z ¨dÑ3Ô3ˆ
Ø7;Ô7KÐ7W˜tÔ3Ð3Ð]_ˆÔå˜dÔ.Ñ/Ô/ð 	qð 	qˆJØÔ/×:Ò:¸:ÀrÑJÔJˆLØ×#Ò# K°Ñ;Ô;Ð;Ø×#Ò#Ø¨JÐ,B˜j˜jÈÔHZ×H^ÒH^Ð_iÑHjÔHjñô ð ð ×#Ò#Ð$;¸TÔ=_×=cÒ=cÐdnÑ=oÔ=oÑpÔpÐpÐpà×$Ò$Ñ&Ô&Ð&Ø×"Ò"Ñ$Ô$Ð$ØˆrA   Úreturnc                 óv  — t          | j        ¦  «        | j        k    r-t          d| j        › dt          | j        ¦  «        › d�¦  «        ‚t	          t          | j        ¦  «                             t          ¦  «        ¦  «        }|rt          dt          › d|› d�¦  «        ‚d| j        vrt          d¦  «        ‚d S )	Nz`num_hidden_layers` (z0) must be equal to the number of `layer_types` (ú)z%`layer_types` entries must be one of z for NeoMME, got rQ   r   z;layer_types must contain at least one full_attention layer.)Úlenr+   r   rR   Úsortedr^   Ú
differenceÚNEOMME_LAYER_TYPES)r>   Úinvalid_layer_typess     r?   rF   z NeoMMEConfig.validate_layer_type�   sá   € ÝˆtÔÑ Ô  DÔ$:Ò:Ð:Ýð;¨Ô(>ð ;ð ;Ý"% dÔ&6Ñ"7Ô"7ð;ð ;ð ;ñô ð õ %¥S¨Ô)9Ñ%:Ô%:×%EÒ%EÕFXÑ%YÔ%YÑZÔZÐØð 	ÝØsÕ8JÐsÐsÐ]pÐsÐsÐsñô ð ð  4Ô#3Ð3Ð3ÝÐZÑ[Ô[Ð[ð 4Ð3rA   c                 óŽ  — t          t          | j        ¦  «        ¦  «        D �]!}| j        |         d         }t	          |t
          t          f¦  «        r/t	          |t          ¦  «        st          j	        |¦  «        r|dk    rt          d|›d�¦  «        ‚| j        |                              dd¦  «        }t          | j        |z  ¦  «        }d|cxk     rdk    sn t          d|›d	|› �¦  «        ‚|d
k     s|d
z  rFt          d|›d	|› d|› d| j        › d||d
z  z
  | j        z  › d|d
z   |d
z  z
  | j        z  › d�¦  «        ‚�Œ#dS )z?Ensure each layer type rotates a multiple of 4 head dimensions.rZ   r   zrope_parameters[z+]['rope_theta'] must be finite and positiver\   r   r(   zF`partial_rotary_factor` must be in (0.0, 1.0] but got rope_parameters[z]['partial_rotary_factor']=r!   z	 rotates z of head_dim=z{ dims, which is not a multiple of 4: the two M-RoPE axes consume frequencies in alternating pairs. Nearest usable factors: z or rQ   N)rk   r^   r+   r,   rS   rT   ÚfloatÚboolrU   rV   rR   ra   r$   )r>   rN   rZ   r\   Ú
rotary_dims        r?   rd   z"NeoMMEConfig._validate_rotary_dims«   sè  € å ¥ TÔ%5Ñ!6Ô!6Ñ7Ô7ð 	ñ 	ˆJØÔ-¨jÔ9¸,ÔGˆJå˜z­Cµ¨<Ñ8Ô8ðoå˜j­$Ñ/Ô/ðoõ ”} ZÑ0Ô0ðoð  ’?�?å Ð!m°JÐ!mÐ!mÐ!mÑnÔnÐnà$(Ô$8¸Ô$D×$HÒ$HÐI`ÐbeÑ$fÔ$fÐ!Ý˜Tœ]Ð-BÑBÑCÔCˆJØÐ.Ð5Ð5Ò5Ð5°#Ò5Ð5Ð5Ð5Ý ðhØ'1ðhð hØPeðhð hñô ð ð ˜AŠ~ˆ~ ¨a¡ˆ~Ý ðL zð Lð LÐPeð Lð LØ!ðLð LØ04´ðLð Lð # Z°!¡^Ñ3°t´}ÑDðLð Lð # Q™¨°a©Ñ7¸4¼=ÑHð	Lð Lð Lñô ð ñ ð#	ð 	rA   c                 ó   — d| j         dz  z  S )z:Width of one flattened image patch, `3 * patch_size ** 2`.r   rC   )r0   )r>   s    r?   Ú	patch_dimzNeoMMEConfig.patch_dimÇ   s   € ð �4”? AÑ%Ñ%Ð%rA   )rg   N)3Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚbase_model_tp_planr`   rb   rH   r	   r   rT   Ú__annotations__r   r   r   r   r   r   rq   r   r    r"   r$   r%   r&   rp   r'   r)   r*   r+   ÚlistÚstrr,   Údictr-   r.   r0   r2   r3   r5   r7   r8   rJ   rX   rf   rF   rd   Úpropertyrt   Ú__classcell__)rO   s   @r?   r   r      su  ø€ € € € € € ðð ð4 €Jà%.Ø%.Ø%.Ø:CØ7@Ø )Ø"+ðð Ðð (3ÈÐRÐR€MØ7;ÐRUÐ$VÐ$VÐ!Ø"&Ðà(Ð(°Ð8Ñ8Ô8€J�Ð8Ð8Ñ8Ø,Ð,°SÐ9Ñ9Ô9€N�CÐ9Ð9Ñ9Ø)Ð)°$Ð7Ñ7Ô7€K�Ð7Ð7Ñ7Ø/Ð/¸Ð=Ñ=Ô=Ð�sÐ=Ð=Ñ=Ø#*€J�˜Ô Ð*Ð*Ñ*Ø€HˆdÐÐÑØ/Ð/¸Ð;Ñ;Ô;Ð�sÐ;Ð;Ñ;Ø1Ð1¸"Ð=Ñ=Ô=Ð˜Ð=Ð=Ñ=Ø1Ð1¸!Ð<Ñ<Ô<Ð˜Ð<Ð<Ñ<Ø&Ð&¨rÐ2Ñ2Ô2€HˆcÐ2Ð2Ñ2Ø#5Ð#5¸eÐ#DÑ#DÔ#DÐ˜SÐDÐDÑDØ€HˆeÐÐÑØ#Ð�uÐ#Ð#Ñ#Ø%(Ð�u˜s‘{Ð(Ð(Ñ(Ø €N�DÐ Ð Ñ Ø$(€K��c”˜TÑ!Ð(Ð(Ñ(ØY]€O�T˜'Ð"GÔHÈ$ÐNÔOÐRVÑVÐ]Ð]Ñ]Ø!$€N�C˜$‘JÐ$Ð$Ñ$Ø(,Ð˜ ™Ð,Ð,Ñ,Ø(Ð(°Ð4Ñ4Ô4€J�Ð4Ð4Ñ4Ø+Ð+°CÐ8Ñ8Ô8€M�3Ð8Ð8Ñ8Ø €L�#˜‘*Ð Ð Ñ Ø$%Ð�s˜T‘zÐ%Ð%Ñ%Ø!"€N�C˜$‘JÐ"Ð"Ñ"Ø $Ð˜Ð$Ð$Ñ$ð(ð (ð (ð (ð (ð0Pð Pð Pðð ð ð \ð \ð \ð \ðð ð ð ð8 ð&˜3ð &ð &ð &ñ „Xð&ð &ð &ð &ð &rA   r   )rx   rU   Útypingr   Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úutilsr   r   Úutils.type_validatorsr	   Ú
get_loggerru   Úloggerrm   r   Ú__all__rD   rA   r?   ú<module>r‰      s÷   ðð "Ð !à €€€Ø Ð Ð Ð Ð Ð à .Ð .Ð .Ð .Ð .Ð .à 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ø 7Ð 7Ð 7Ð 7Ð 7Ð 7ð 
ˆÔ	˜HÑ	%Ô	%€à<Ð ð €Ð1Ð2Ñ2Ô2Øði&ð i&ð i&ð i&ð i&Ð#ñ i&ô i&ñ „ñ 3Ô2ði&ðX Ð
€€€rA   