§
    9²¸jÈÂ  ã                   ó  — d dl mZ d dlZd dlZd dlmZ d dlmc mZ	 d dl
mZ d dlmZ ddlmZ ddlmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZ ddlmZ ddlm Z  ddl!m"Z" ddl#m$Z$m%Z%m&Z&m'Z'm(Z( ddl)m*Z*m+Z+ ddl,m-Z- ddl.m/Z/m0Z0 ddl1m2Z2m3Z3m4Z4 ddl5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@ ddlAmBZB ddlCmDZDmEZE  e'jF        eG¦  «        ZH e%d¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         ZI e%d¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         ZJ e%d¬¦  «        e G d„ d e¦  «        ¦   «         ¦   «         ZK G d!„ d"e3¦  «        ZL G d#„ d$e9¦  «        ZM G d%„ d&e6¦  «        ZN G d'„ d(e@¦  «        ZO G d)„ d*ejP        ¦  «        ZQ G d+„ d,ejP        ¦  «        ZR G d-„ d.e>¦  «        ZS G d/„ d0e?¦  «        ZT G d1„ d2e<¦  «        ZUd3„ ZVdOd5„ZW G d6„ d7ejP        ¦  «        ZX G d8„ d9e2¦  «        ZY G d:„ d;e¦  «        ZZ G d<„ d=e:¦  «        Z[ G d>„ d?e;¦  «        Z\ G d@„ dAe\¦  «        Z] G dB„ dCe=¦  «        Z^ G dD„ dEeB¦  «        Z_ G dF„ dGe7¦  «        Z` G dH„ dIe8¦  «        Za G dJ„ dKeE¦  «        Zb G dL„ dMeD¦  «        Zcg dN¢ZddS )Pé    )ÚCallableN)Ústrict)Ú	LayerNormé   )ÚACT2FN)ÚCacheÚDynamicCache)ÚPreTrainedConfig)Úcreate_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚBaseModelOutputWithPooling)ÚRopeParameters)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚloggingÚtorch_compilable_check)Úaccepts_precomputed_kwargsÚmerge_with_config_defaults)Úcapture_outputs)Úget_vision_attention_seqlensÚget_vision_position_idsé   )ÚGlm4MLPÚGlm4RMSNormÚeager_attention_forward)ÚQwen2_5_VisionPatchEmbedÚ Qwen2_5_VLCausalLMOutputWithPastÚ"Qwen2_5_VLForConditionalGenerationÚQwen2_5_VLMLPÚQwen2_5_VLModelOutputWithPastÚQwen2_5_VLPreTrainedModelÚQwen2_5_VLRotaryEmbeddingÚQwen2_5_VLTextModelÚQwen2_5_VLVisionAttentionÚQwen2_5_VLVisionBlockÚQwen2_5_VLVisionRotaryEmbedding)ÚQwen2VLModel)ÚQwen2VLProcessorÚQwen2VLProcessorKwargszzai-org/GLM-4.1V-9B-Thinking)Ú
checkpointc                   óŒ  — e Zd ZU dZdZdZdZddiZdZe	e
d<   d	Ze	e
d
<   dZee
d<   dZee
d<   dZee	z  e
d<   dZe	e
d<   dZe	e
d<   dZe	ee	         z  ee	e	f         z  e
d<   dZe	ee	         z  ee	e	f         z  e
d<   dZee
d<   dZe	e
d<   dZe	ee	         z  ee	e	f         z  e
d<   dZe	e
d<   dZe	e
d <   d!Zee
d"<   d#Zed#z  e
d$<   d#S )%ÚGlm4vVisionConfiga,  
    out_hidden_size (`int`, *optional*, defaults to 4096):
        The output hidden size of the vision model.

    Example:

    ```python
    >>> from transformers import Glm4vVisionConfig, Glm4vVisionModel

    >>> # Initializing a Glm4vVisionConfig GLM-4.1V-9B style configuration
    >>> configuration = Glm4vVisionConfig()

    >>> # Initializing a model (with random weights) from the GLM-4.1V-9B configuration
    >>> model = Glm4vVisionModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```Úglm4v_visionÚvision_configÚaxialÚnum_attention_headsÚ	num_headsé   Údepthi   Úhidden_sizeÚsiluÚ
hidden_actFÚattention_biasç        Úattention_dropouté   r   Úin_channelsiP  Ú
image_sizeé   Ú
patch_sizeçñhãˆµøä>Úrms_norm_epsr   Úspatial_merge_sizeÚtemporal_patch_sizeé   Úout_hidden_sizeé€5  Úintermediate_sizeç{®Gáz”?Úinitializer_rangeNÚrope_parameters) Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚbase_config_keyÚdefault_rope_typeÚattribute_mapr8   ÚintÚ__annotations__r9   r;   Ústrr<   Úboolr>   Úfloatr6   r@   rA   ÚlistÚtuplerC   rE   rF   rG   rI   rK   rM   rN   Údict© ó    úw/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/glm4v/modular_glm4v.pyr1   r1   F   s�  € € € € € € ðð ð&  €JØ%€OØÐØ*¨KÐ8€Mà€Eˆ3€O€O�OØ€K�ÐÐÑØ€J�ÐÐÑØ €N�DÐ Ð Ñ Ø%(Ð�u˜s‘{Ð(Ð(Ñ(Ø€IˆsÐÐÑØ€K�ÐÐÑØ47€J��d˜3”i‘ %¨¨S¨¤/Ñ1Ð7Ð7Ñ7Ø46€J��d˜3”i‘ %¨¨S¨¤/Ñ1Ð6Ð6Ñ6Ø€L�%ÐÐÑØÐ˜ÐÐÑØ=>Ð˜˜t Cœy™¨5°°c°¬?Ñ:Ð>Ð>Ñ>Ø€O�SÐÐÑØ"Ð�sÐ"Ð"Ñ"Ø#Ð�uÐ#Ð#Ñ#Ø#'€O�T˜D‘[Ð'Ð'Ñ'Ð'Ð'r`   r1   c                   óR  ‡ — e Zd ZU dZdZdZdgZddddddd	œZd
gdgfddgdgfdgdgfdœZdhZ	dZ
eed<   dZeed<   dZeed<   dZeed<   dZeed<   dZedz  ed<   dZeed<   dZeed <   d!Zeed"<   d#Zeed$<   d%Zeed&<   d'Zeez  ed(<   dZeez  dz  ed)<   dZedz  ed*<   ˆ fd+„Zˆ xZ S ),ÚGlm4vTextConfiga„  
    Example:

    ```python
    >>> from transformers import Glm4vTextModel, Glm4vConfig

    >>> # Initializing a GLM-4.1V style configuration
    >>> configuration = Glm4vConfig()

    >>> # Initializing a model from the GLM-4.1V style configuration
    >>> model = Glm4vTextModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```Ú
glm4v_textÚtext_configÚpast_key_valuesÚcolwiseÚrowwiseÚcolwise_gather_outputÚrowwise_split_input)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_up_projzlayers.*.mlp.down_projÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormÚmrope_sectioni P Ú
vocab_sizerH   r9   rJ   rK   é(   Únum_hidden_layersé    r5   r   NÚnum_key_value_headsr:   r;   i €  Úmax_position_embeddingsrL   rM   rD   rE   TÚ	use_cacher=   r>   rN   Úpad_token_idc                 ó`   •— | j         €| j        | _          t          ¦   «         j        di |¤Ž d S )Nr_   )rw   r5   ÚsuperÚ__post_init__©ÚselfÚkwargsÚ	__class__s     €ra   r}   zGlm4vTextConfig.__post_init__¨   s:   ø€ ØÔ#Ð+Ø'+Ô'?ˆDÔ$à�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'r`   )!rO   rP   rQ   rR   rS   rT   Úkeys_to_ignore_at_inferenceÚbase_model_tp_planÚbase_model_pp_planÚignore_keys_at_rope_validationrs   rW   rX   r9   rK   ru   r5   rw   r;   rY   rx   rM   r[   rE   ry   rZ   r>   rN   r   r^   rz   r}   Ú__classcell__©r�   s   @ra   rc   rc   s   s½  ø€ € € € € € ðð ð  €JØ#€OØ#4Ð"5Ðð &/Ø%.Ø%.Ø%.Ø%<Ø"7ðð Ðð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ðð Ðð
 '6Ð%6Ð"à€J�ÐÐÑØ€K�ÐÐÑØ"Ð�sÐ"Ð"Ñ"ØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø&'Ð˜˜t™Ð'Ð'Ñ'Ø€J�ÐÐÑØ#(Ð˜SÐ(Ð(Ñ(Ø#Ð�uÐ#Ð#Ñ#Ø€L�%ÐÐÑØ€IˆtÐÐÑØ%(Ð�u˜s‘{Ð(Ð(Ñ(Ø48€O�^ dÑ*¨TÑ1Ð8Ð8Ñ8Ø#€L�#˜‘*Ð#Ð#Ñ#ð(ð (ð (ð (ð (ð (ð (ð (ð (r`   rc   c                   óÎ   ‡ — e Zd ZU dZdZeedœZdgZdZ	e
ez  dz  ed<   dZe
ez  dz  ed<   dZeed	<   d
Zeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   ˆ fd„Zˆ xZS )ÚGlm4vConfiga™  
    image_start_token_id (`int`, *optional*, defaults to 151339):
        The image start token index to encode the start of image.
    image_end_token_id (`int`, *optional*, defaults to 151340):
        The image end token index to encode the end of image.
    video_start_token_id (`int`, *optional*, defaults to 151341):
        The video start token index to encode the start of video.
    video_end_token_id (`int`, *optional*, defaults to 151342):
        The video end token index to encode the end of video.

    ```python
    >>> from transformers import Glm4vForConditionalGeneration, Glm4vConfig

    >>> # Initializing a GLM-4.1V style configuration
    >>> configuration = Glm4vConfig()

    >>> # Initializing a model from the GLM-4.1V style configuration
    >>> model = Glm4vForConditionalGeneration(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```Úglm4v)r3   re   rf   Nre   r3   i/O Úimage_token_idi0O Úvideo_token_idi+O Úimage_start_token_idi,O Úimage_end_token_idi-O Úvideo_start_token_idi.O Úvideo_end_token_idFÚtie_word_embeddingsc                 óÞ  •— t          | j        t          ¦  «        r | j        d         di | j        ¤Ž| _        n| j        € | j        d         di |¤Ž| _        t          | j        t          ¦  «        r | j        d         di | j        ¤Ž| _        n| j        € | j        d         di |¤Ž| _        | j        st          | j        dd¦  «        rd| _         t          ¦   «         j        di |¤Ž d S )Nr3   re   r‘   FTr_   )	Ú
isinstancer3   r^   Úsub_configsre   r‘   Úgetattrr|   r}   r~   s     €ra   r}   zGlm4vConfig.__post_init__×   s  ø€ Ý�dÔ(­$Ñ/Ô/ð 	MØ!B Ô!1°/Ô!BÐ!XÐ!XÀTÔEWÐ!XÐ!XˆDÔÐØÔÐ'Ø!B Ô!1°/Ô!BÐ!LÐ!LÀVÐ!LÐ!LˆDÔå�dÔ&­Ñ-Ô-ð 	IØ>˜tÔ/°Ô>ÐRÐRÀÔAQÐRÐRˆDÔÐØÔÐ%Ø>˜tÔ/°Ô>ÐHÐHÀÐHÐHˆDÔð
 Ô'ð 	,­G°DÔ4DÐF[Ð]bÑ,cÔ,cð 	,Ø'+ˆDÔ$à�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'r`   )rO   rP   rQ   rR   rS   r1   rc   r”   r‚   re   r^   r
   rX   r3   r‹   rW   rŒ   r�   rŽ   r�   r�   r‘   rZ   r}   r†   r‡   s   @ra   r‰   r‰   ¯   s  ø€ € € € € € ðð ð. €JØ$5ÀoÐVÐV€KØ#4Ð"5Ðà26€K�Ð(Ñ(¨4Ñ/Ð6Ð6Ñ6Ø48€M�4Ð*Ñ*¨TÑ1Ð8Ð8Ñ8Ø €N�CÐ Ð Ñ Ø €N�CÐ Ð Ñ Ø &Ð˜#Ð&Ð&Ñ&Ø$Ð˜Ð$Ð$Ñ$Ø &Ð˜#Ð&Ð&Ñ&Ø$Ð˜Ð$Ð$Ñ$Ø %Ð˜Ð%Ð%Ñ%ð(ð (ð (ð (ð (ð (ð (ð (ð (r`   r‰   c                   ó   — e Zd ZdS )ÚGlm4vRMSNormN©rO   rP   rQ   r_   r`   ra   r—   r—   ì   ó   € € € € € Ø€Dr`   r—   c                   ó&   ‡ — e Zd Zddefˆ fd„Zˆ xZS )ÚGlm4VisionMlpFÚbiasc                 ód   •— t          ¦   «                              ||¦  «         |j        | _        d S ©N)r|   Ú__init__rI   rK   )r   Úconfigrœ   r�   s      €ra   rŸ   zGlm4VisionMlp.__init__ñ   s.   ø€ Ý‰Œ×Ò˜ Ñ&Ô&Ð&Ø!'Ô!7ˆÔÐÐr`   ©F)rO   rP   rQ   rZ   rŸ   r†   r‡   s   @ra   r›   r›   ð   sI   ø€ € € € € ð8ð 8 Tð 8ð 8ð 8ð 8ð 8ð 8ð 8ð 8ð 8ð 8r`   r›   c                   ó   — e Zd Zdeddfd„ZdS )ÚGlm4vVisionPatchEmbedr    ÚreturnNc                 ó  — t           j                             | ¦  «         |j        | _        |j        | _        |j        | _        |j        | _        | j        | j        | j        g}t          j        | j        | j        ||¬¦  «        | _	        d S )N)Úkernel_sizeÚstride)
ÚnnÚModulerŸ   rC   rG   r@   r9   Ú	embed_dimÚConv3dÚproj)r   r    r¦   s      ra   rŸ   zGlm4vVisionPatchEmbed.__init__÷   sy   € Ý
Œ	×Ò˜4Ñ Ô Ð Ø Ô+ˆŒØ#)Ô#=ˆÔ Ø!Ô-ˆÔØÔ+ˆŒàÔ/°´À$Ä/ÐRˆÝ”I˜dÔ.°´ÈKÐ`kÐlÑlÔlˆŒ	ˆ	ˆ	r`   )rO   rP   rQ   r1   rŸ   r_   r`   ra   r£   r£   ö   s?   € € € € € ðmÐ0ð m°Tð mð mð mð mð mð mr`   r£   c                   ó   — e Zd ZdS )ÚGlm4vVisionRotaryEmbeddingNr˜   r_   r`   ra   r®   r®     r™   r`   r®   c                   óZ   ‡ — e Zd Zddededededdf
ˆ fd„Zd	ej        dej        fd
„Z	ˆ xZ
S )ÚGlm4vVisionPatchMergerFÚdimÚcontext_dimr;   rœ   r¤   Nc                 ó¤  •— t          ¦   «                              ¦   «          t          j        |||¬¦  «        | _        t          |¦  «        | _        t          j        |||¬¦  «        | _        t          j        |||¬¦  «        | _        t          j        |||¬¦  «        | _	        t          j
        ¦   «         | _        t          |         | _        d S )N©rœ   )r|   rŸ   r¨   ÚLinearr¬   r   Úpost_projection_normÚ	gate_projÚup_projÚ	down_projÚGELUÚact1r   Úact_fn)r   r±   r²   r;   rœ   r�   s        €ra   rŸ   zGlm4vVisionPatchMerger.__init__  s¤   ø€ Ý‰Œ×ÒÑÔÐÝ”I˜c 3¨TÐ2Ñ2Ô2ˆŒ	Ý$-¨c¡N¤NˆÔ!Ýœ 3¨¸$Ð?Ñ?Ô?ˆŒÝ”y  k¸Ð=Ñ=Ô=ˆŒÝœ ;°¸$Ð?Ñ?Ô?ˆŒÝ”G‘I”IˆŒ	Ý˜ZÔ(ˆŒˆˆr`   Úhidden_statec                 ó  — |                       |¦  «        }|                      |                      |¦  «        ¦  «        }|                      |                      |                      |¦  «        ¦  «        |                      |¦  «        z  ¦  «        S rž   )r¬   r»   r¶   r¹   r¼   r·   r¸   )r   r½   s     ra   ÚforwardzGlm4vVisionPatchMerger.forward  sn   € Ø—y’y Ñ.Ô.ˆØ—y’y ×!:Ò!:¸<Ñ!HÔ!HÑIÔIˆØ�~Š~˜dŸkšk¨$¯.ª.¸Ñ*FÔ*FÑGÔGÈ$Ï,Ê,ÐWcÑJdÔJdÑdÑeÔeÐer`   r¡   )rO   rP   rQ   rW   rY   rZ   rŸ   ÚtorchÚTensorr¿   r†   r‡   s   @ra   r°   r°     s–   ø€ € € € € ð)ð )˜Cð )¨cð )¸sð )È$ð )Ð[_ð )ð )ð )ð )ð )ð )ðf E¤Lð f°U´\ð fð fð fð fð fð fð fð fr`   r°   c                   ó:   ‡ — e Zd Zdefˆ fd„Zdej        fd„Zˆ xZS )ÚGlm4vVisionEmbeddingsr    c                 ó:  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        |j        | _        | j        | j        z  dz  | _        | j        | _        t          j
        | j        | j        ¦  «        | _        d| _        d S )Nr   Úbicubic)r|   rŸ   r    r9   rª   rA   rC   Únum_patchesÚnum_positionsr¨   Ú	EmbeddingÚposition_embeddingÚinterpolated_method©r   r    r�   s     €ra   rŸ   zGlm4vVisionEmbeddings.__init__  s†   ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ+ˆŒØ Ô+ˆŒØ Ô+ˆŒà œO¨t¬Ñ>À1ÑDˆÔØ!Ô-ˆÔÝ"$¤,¨tÔ/AÀ4Ä>Ñ"RÔ"RˆÔØ#,ˆÔ Ð Ð r`   r¤   c                 ó  — | j         j        }|j        d         }|j        }t	          |t
          ¦  «        r!t          j        ||t          j        ¬¦  «        }|j        d         }	t          |	dz  ¦  «        }
| 
                    |
|
|¦  «                             ddd¦  «                             d¦  «                             |t          j        ¬¦  «        }|j        d         }t          j        ||j        ¬¦  «        }|                     d¦  «        |                     d¦  «                             d¦  «        k                         d¦  «        }||df                              t          j        ¬¦  «        }||df                              t          j        ¬¦  «        }|dz   |z  dz  dz
  }|dz   |z  dz  dz
  }t          j        ||fd¬	¦  «                             d¦  «                             d¦  «        }t'          j        ||| j        d
d¬¦  «        }|                     d¦  «                             d¦  «                             dd¦  «        }|                     |j        ¦  «                             |j        ¦  «        }||z   }|S )a¡  
        Forward pass with integrated position encoding adaptation using 2D interpolation.

        Args:
            embeddings: Input embeddings tensor
            lengths (torch.Tensor): Sequence lengths for each image in the batch.
            image_shapes (torch.Tensor): Tensor of shape [batch_size, 3] representing the image shapes (t, h, w).
            h_coords (torch.Tensor): Tensor of shape [total_seq] representing the h coordinate for each patch.
            w_coords (torch.Tensor): Tensor of shape [total_seq] representing the w coordinate for each patch.

        Returns:
            torch.Tensor: Embeddings with adapted position encoding added.
        é   )ÚdeviceÚdtyper   g      à?r   ©rÎ   ©rÏ   éÿÿÿÿ©r±   FÚborder)ÚmodeÚalign_cornersÚpadding_mode)rÉ   ÚweightÚshaperÎ   r“   r\   rÀ   ÚtensorÚlongrW   ÚviewÚpermuteÚ	unsqueezeÚtoÚfloat32ÚarangeÚcumsumÚsumÚstackÚFÚgrid_samplerÊ   ÚsqueezerÏ   )r   Ú
embeddingsÚlengthsÚimage_shapesÚh_coordsÚw_coordsÚpos_embed_weightr9   rÎ   Úorig_size_sqÚ	orig_sizeÚpos_embed_2dÚ
num_tokensÚtoken_positionsÚseq_idsÚtarget_hÚtarget_wÚnorm_wÚnorm_hÚgridÚinterpolated_embed_fp32Úadapted_pos_embed_fp32Úadapted_pos_embeds                          ra   r¿   zGlm4vVisionEmbeddings.forward$  si  € ð  Ô2Ô9ÐØ&Ô,¨QÔ/ˆØ!Ô(ˆõ �g�tÑ$Ô$ð 	MÝ”l 7°6ÅÄÐLÑLÔLˆGð (Ô-¨aÔ0ˆÝ˜ cÑ)Ñ*Ô*ˆ	à×!Ò! )¨Y¸ÑDÔDßŠW�Q˜˜1ÑÔßŠY�q‰\Œ\ßŠR�v¥U¤]ˆRÑ3Ô3ð	 	ð  Ô% aÔ(ˆ
Ýœ, z¸*Ô:KÐLÑLÔLˆØ"×,Ò,¨QÑ/Ô/°7·>²>À!Ñ3DÔ3D×3NÒ3NÈqÑ3QÔ3QÒQ×VÒVÐWXÑYÔYˆØ ¨ 
Ô+×.Ò.µU´]Ð.ÑCÔCˆØ ¨ 
Ô+×.Ò.µU´]Ð.ÑCÔCˆð ˜c‘> XÑ-°Ñ2°QÑ6ˆØ˜c‘> XÑ-°Ñ2°QÑ6ˆõ Œ{˜F FÐ+°Ð4Ñ4Ô4×>Ò>¸qÑAÔA×KÒKÈAÑNÔNˆõ #$¤-Ø˜$ TÔ%=ÈUÐaið#
ñ #
ô #
Ðð
 "9×!@Ò!@ÀÑ!CÔ!C×!KÒ!KÈBÑ!OÔ!O×!WÒ!WÐXYÐ[\Ñ!]Ô!]ÐØ2×5Ò5Ð6FÔ6LÑMÔM×PÒPÐQ[ÔQbÑcÔcÐð  Ð"3Ñ3ˆ
ØÐr`   )	rO   rP   rQ   r1   rŸ   rÀ   rÁ   r¿   r†   r‡   s   @ra   rÃ   rÃ     sd   ø€ € € € € ð
-Ð0ð 
-ð 
-ð 
-ð 
-ð 
-ð 
-ð:ÐPUÔP\ð :ð :ð :ð :ð :ð :ð :ð :r`   rÃ   c                   ó(   ‡ — e Zd Zdeddfˆ fd„Zˆ xZS )ÚGlm4vVisionAttentionr    r¤   Nc                 ó
  •— t          ¦   «                              |¦  «         |j        | _        t          j        |j        |j        dz  |j        ¬¦  «        | _        t          j        |j        |j        d¬¦  «        | _        d S )Nr   r´   F)	r|   rŸ   r>   r¨   rµ   r9   r<   Úqkvr¬   rË   s     €ra   rŸ   zGlm4vVisionAttention.__init__b  sn   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!'Ô!9ˆÔÝ”9˜VÔ/°Ô1CÀaÑ1GÈfÔNcÐdÑdÔdˆŒÝ”I˜fÔ0°&Ô2DÈ5ÐQÑQÔQˆŒ	ˆ	ˆ	r`   )rO   rP   rQ   r1   rŸ   r†   r‡   s   @ra   rý   rý   a  sX   ø€ € € € € ðRÐ0ð R°Tð Rð Rð Rð Rð Rð Rð Rð Rð Rð Rr`   rý   c                   ó    ‡ — e Zd Zdˆ fd„Zˆ xZS )ÚGlm4vVisionBlockr¤   Nc                 ó  •— t          ¦   «                              |¦  «         t          |j        |j        ¬¦  «        | _        t          |j        |j        ¬¦  «        | _        t          |¦  «        | _        t          |d¬¦  «        | _
        d S )N©ÚepsFr´   )r|   rŸ   r—   r9   rE   Únorm1Únorm2rý   Úattnr›   ÚmlprË   s     €ra   rŸ   zGlm4vVisionBlock.__init__j  sv   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý! &Ô"4¸&Ô:MÐNÑNÔNˆŒ
Ý! &Ô"4¸&Ô:MÐNÑNÔNˆŒ
Ý(¨Ñ0Ô0ˆŒ	Ý  ¨eÐ4Ñ4Ô4ˆŒˆˆr`   ©r¤   N)rO   rP   rQ   rŸ   r†   r‡   s   @ra   r  r  i  s=   ø€ € € € € ð5ð 5ð 5ð 5ð 5ð 5ð 5ð 5ð 5ð 5r`   r  c                   óX   ‡ — e Zd Zddefˆ fd„Zddedeej        e	f         fd„Z
d„ Zˆ xZS )ÚGlm4vTextRotaryEmbeddingNr    c                 óŒ   •— t          ¦   «                              ¦   «          |j                             dg d¢¦  «        | _        d S )Nrr   )é   r?   r?   )r|   rŸ   rN   Úgetrr   )r   r    rÎ   r�   s      €ra   rŸ   z!Glm4vTextRotaryEmbedding.__init__s  s>   ø€ Ý‰Œ×ÒÑÔÐØ#Ô3×7Ò7¸ÈÈÈÑUÔUˆÔÐÐr`   r¤   c                 ó>  — | j         d         }| j                              dd¦  «        }t          | dd¦  «        p| j        | j        z  }t          ||z  ¦  «        }d}d|t          j        d|dt          j        ¬¦  «        |z  z  z  }| 	                    |¦  «        |fS )	a¾  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚpartial_rotary_factorg      ð?Úhead_dimNr   r   rÑ   )
rN   r  r•   r9   r5   rW   rÀ   rá   r[   rß   )	r    rÎ   r€   Úbaser  r  r±   Úattention_factorÚinv_freqs	            ra   Úcompute_default_rope_parametersz8Glm4vTextRotaryEmbedding.compute_default_rope_parametersw  s¨   € ð Ô% lÔ3ˆØ &Ô 6× :Ò :Ð;RÐTWÑ XÔ XÐÝ˜6 :¨tÑ4Ô4Ðh¸Ô8JÈfÔNhÑ8hˆÝ�(Ð2Ñ2Ñ3Ô3ˆàÐà˜$¥5¤<°°3¸ÅÄÐ#MÑ#MÔ#MÐPSÑ#SÑTÑUˆØ�{Š{˜6Ñ"Ô"Ð$4Ð4Ð4r`   c           	      ó¾   — t          j        d„ t          |                     | j        d¬¦  «        ¦  «        D ¦   «         d¬¦  «        }|                     dd¬¦  «        S )z]
        Recompose the frequencies into the final spatial layout used per each grid.
        c                 ó*   — g | ]\  }}||d z           ‘ŒS )r   r_   )Ú.0ÚiÚms      ra   ú
<listcomp>zFGlm4vTextRotaryEmbedding.recomposition_frequencies.<locals>.<listcomp>�  s$   € ÐaÐaÐa¡t q¨!˜!˜A ™Eœ(ÐaÐaÐar`   rÒ   rÓ   r   )rÀ   ÚcatÚ	enumerateÚsplitrr   Úrepeat_interleave)r   Úfreqs     ra   Úrecomposition_frequenciesz2Glm4vTextRotaryEmbedding.recomposition_frequencies‹  s]   € õ ŒyÐaÐa­y¸¿ºÀDÔDVÐ\^¸Ñ9_Ô9_Ñ/`Ô/`ÐaÑaÔaÐgiÐjÑjÔjˆØ×%Ò% a¨RÐ%Ñ0Ô0Ð0r`   rž   )rO   rP   rQ   rc   rŸ   r‰   r]   rÀ   rÁ   r[   r  r"  r†   r‡   s   @ra   r  r  r  s—   ø€ € € € € ðVð V˜ð Vð Vð Vð Vð Vð Vð5ð 5°ð 5ÐW\Ð]bÔ]iÐkpÐ]pÔWqð 5ð 5ð 5ð 5ð(1ð 1ð 1ð 1ð 1ð 1ð 1r`   r  c                 óŽ   — | dddd…f         }| dddd…f         }t          j        | |fd¬¦  «                             d¦  «        S )	z*Rotates half the hidden dims of the input..r   Nr   rÍ   rÒ   rÓ   éþÿÿÿ)rÀ   rä   Úflatten)ÚxÚx1Úx2s      ra   Úrotate_half_llmr)  “  sQ   € à	
ˆ3���1�ˆ9Œ€BØ	
ˆ3���1�ˆ9Œ€BÝŒ;˜˜˜R�y bÐ)Ñ)Ô)×1Ò1°"Ñ5Ô5Ð5r`   rÍ   c                 ó˜  — |                      |¦  «        }|                      |¦  «        }|j        d         }| dd|…f         | d|d…f         }}|dd|…f         |d|d…f         }	}||z  t          |¦  «        |z  z   }
||z  t          |¦  «        |z  z   }t          j        |
|gd¬¦  «        }
t          j        ||	gd¬¦  «        }|
|fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    rÒ   .NrÓ   )rÞ   rÙ   r)  rÀ   r  )ÚqÚkÚcosÚsinÚunsqueeze_dimÚ
rotary_dimÚq_rotÚq_passÚk_rotÚk_passÚq_embedÚk_embeds               ra   Úapply_rotary_pos_embr7  š  sô   € ð$ �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€Cð ”˜2”€JØ�c˜;˜J˜;Ð&Ô'¨¨3°
°°Ð+;Ô)<ˆ6€EØ�c˜;˜J˜;Ð&Ô'¨¨3°
°°Ð+;Ô)<ˆ6€Eð �s‰{�¨uÑ5Ô5¸Ñ;Ñ<€GØ�s‰{�¨uÑ5Ô5¸Ñ;Ñ<€Gõ Œi˜ &Ð)¨rÐ2Ñ2Ô2€GÝŒi˜ &Ð)¨rÐ2Ñ2Ô2€GØ�GÐÐr`   c                   óú   ‡ — e Zd ZdZddededz  fˆ fd„Z	 	 	 ddej        de	ej        ej        f         dz  dej        dz  d	e
dz  d
ee         de	ej        ej        dz  e	ej                 dz  f         fd„Zˆ xZS )ÚGlm4vTextAttentionz†
    Multi-headed attention from 'Attention Is All You Need' paper.
    and "Generating Long Sequences with Sparse Transformers".
    Nr    Ú	layer_idxc                 óÈ  •— t          ¦   «                              ¦   «          || _        || _        |j        | _        |j        | _        | j        | j        z  | _        |j        | _        | j        | j        z  | _	        d| _
        |j        | _        |j        | _        | j        dz  | _        t          j        | j        | j        | j        z  d¬¦  «        | _        t          j        | j        | j        | j        z  d¬¦  «        | _        t          j        | j        | j        | j        z  d¬¦  «        | _        t          j        | j        | j        z  | j        d¬¦  «        | _        d S )NTg      à¿r´   F)r|   rŸ   r    r:  r9   r5   r6   r  rw   Únum_key_value_groupsÚ	is_causalr>   rN   Úscalingr¨   rµ   Úq_projÚk_projÚv_projÚo_proj©r   r    r:  r�   s      €ra   rŸ   zGlm4vTextAttention.__init__Ä  s/  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒà!Ô-ˆÔØÔ3ˆŒØÔ(¨D¬NÑ:ˆŒØ#)Ô#=ˆÔ Ø$(¤N°dÔ6NÑ$NˆÔ!ØˆŒØ!'Ô!9ˆÔØ%Ô5ˆÔØ”} dÑ*ˆŒå”i Ô 0°$´.À4Ä=Ñ2PÐW[Ð\Ñ\Ô\ˆŒÝ”i Ô 0°$Ô2JÈTÌ]Ñ2ZÐaeÐfÑfÔfˆŒÝ”i Ô 0°$Ô2JÈTÌ]Ñ2ZÐaeÐfÑfÔfˆŒÝ”i ¤°´Ñ >ÀÔ@PÐW\Ð]Ñ]Ô]ˆŒˆˆr`   rm   Úposition_embeddingsrn   rf   r€   r¤   c                 ód  — |                      ¦   «         \  }}}|                      |¦  «        }	|                      |¦  «        }
|                      |¦  «        }|	                     ||d| j        ¦  «                             dd¦  «        }	|
                     ||d| j        ¦  «                             dd¦  «        }
|                     ||d| j        ¦  «                             dd¦  «        }|\  }}t          |	|
||¦  «        \  }	}
|�|                     |
|| j	        ¦  «        \  }
}t          j        | j        j        t          ¦  «        } || |	|
||f| j        sdn| j        | j        dœ|¤Ž\  }}|                     ||d¦  «                             ¦   «         }|                      |¦  «        }||fS )NrÒ   rÍ   r   r=   )Údropoutr>  )Úsizer?  r@  rA  rÜ   r  Ú	transposer7  Úupdater:  r   Úget_interfacer    Ú_attn_implementationr    Útrainingr>   r>  ÚreshapeÚ
contiguousrB  )r   rm   rD  rn   rf   r€   ÚbszÚq_lenÚ_Úquery_statesÚ
key_statesÚvalue_statesr-  r.  Úattention_interfaceÚattn_outputÚattn_weightss                    ra   r¿   zGlm4vTextAttention.forwardØ  sÎ  € ð &×*Ò*Ñ,Ô,‰ˆˆU�Aà—{’{ =Ñ1Ô1ˆØ—[’[ Ñ/Ô/ˆ
Ø—{’{ =Ñ1Ô1ˆà#×(Ò(¨¨e°R¸¼ÑGÔG×QÒQÐRSÐUVÑWÔWˆØ—_’_ S¨%°°T´]ÑCÔC×MÒMÈaÐQRÑSÔSˆ
Ø#×(Ò(¨¨e°R¸¼ÑGÔG×QÒQÐRSÐUVÑWÔWˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð "×)Ò)¨#¨u°bÑ9Ô9×DÒDÑFÔFˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r`   rž   )NNN)rO   rP   rQ   rR   rc   rW   rŸ   rÀ   rÁ   r]   r   r   r   r¿   r†   r‡   s   @ra   r9  r9  ¾  s  ø€ € € € € ðð ð
^ð ^˜ð ^¸3À¹:ð ^ð ^ð ^ð ^ð ^ð ^ð. IMØ.2Ø(,ð))ð ))à”|ð))ð # 5¤<°´Ð#=Ô>ÀÑEð))ð œ tÑ+ð	))ð
  ™ð))ð Ð-Ô.ð))ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	Mð))ð ))ð ))ð ))ð ))ð ))ð ))ð ))r`   r9  c                   ó   — e Zd ZdS )ÚGlm4vTextMLPNr˜   r_   r`   ra   rY  rY    r™   r`   rY  c                   ó  ‡ — e Zd Zdedefˆ fd„Ze	 	 	 	 	 ddej        de	ej        ej        f         dz  dej        dz  d	ej
        dz  d
edz  dedz  de	ej        e	ej        ej        f         dz  f         fd„¦   «         Zˆ xZS )ÚGlm4vTextDecoderLayerr    r:  c                 ó²  •— t          ¦   «                              ¦   «          |j        | _        t          ||¦  «        | _        t          |¦  «        | _        t          |j        |j        ¬¦  «        | _	        t          |j        |j        ¬¦  «        | _
        t          |j        |j        ¬¦  «        | _        t          |j        |j        ¬¦  «        | _        d S )Nr  )r|   rŸ   r9   r9  Ú	self_attnrY  r  r—   rE   Úinput_layernormÚpost_attention_layernormÚpost_self_attn_layernormÚpost_mlp_layernormrC  s      €ra   rŸ   zGlm4vTextDecoderLayer.__init__	  s¶   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔÝ+¨F°IÑ>Ô>ˆŒÝ Ñ'Ô'ˆŒÝ+¨FÔ,>ÀFÔDWÐXÑXÔXˆÔÝ(4°VÔ5GÈVÔM`Ð(aÑ(aÔ(aˆÔ%Ý(4°VÔ5GÈVÔM`Ð(aÑ(aÔ(aˆÔ%Ý".¨vÔ/AÀvÔGZÐ"[Ñ"[Ô"[ˆÔÐÐr`   NFrm   rD  rn   Úposition_idsrf   ry   r¤   c           
      ó"  — |}|                       |¦  «        } | j        d||||||dœ|¤Ž\  }}	|                      |¦  «        }||z   }|}|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }||z   }|S )N)rm   rD  rn   rb  rf   ry   r_   )r^  r]  r`  r_  r  ra  )
r   rm   rD  rn   rb  rf   ry   r€   ÚresidualrQ  s
             ra   r¿   zGlm4vTextDecoderLayer.forward  sÉ   € ð !ˆà×,Ò,¨]Ñ;Ô;ˆð *˜4œ>ð 
Ø'Ø 3Ø)Ø%Ø+Øð
ð 
ð ð
ð 
Ñˆ�qð ×5Ò5°mÑDÔDˆØ  =Ñ0ˆð !ˆØ×5Ò5°mÑDÔDˆØŸš Ñ/Ô/ˆØ×/Ò/°Ñ>Ô>ˆØ  =Ñ0ˆàÐr`   )NNNNF)rO   rP   rQ   rc   rW   rŸ   r   rÀ   rÁ   r]   Ú
LongTensorr   rZ   ÚFloatTensorr¿   r†   r‡   s   @ra   r[  r[    s  ø€ € € € € ð\˜ð \¸3ð \ð \ð \ð \ð \ð \ð ð IMØ.2Ø04Ø(,Ø!&ð#ð #à”|ð#ð # 5¤<°´Ð#=Ô>ÀÑEð#ð œ tÑ+ð	#ð
 Ô&¨Ñ-ð#ð  ™ð#ð ˜$‘;ð#ð 
ˆuÔ  %¨Ô(9¸5Ô;LÐ(LÔ"MÐPTÑ"TÐTÔ	Uð#ð #ð #ñ „^ð#ð #ð #ð #ð #r`   r[  c                   ó   — e Zd ZdS )ÚGlm4vModelOutputWithPastNr˜   r_   r`   ra   rh  rh  :  r™   r`   rh  c                   ó   — e Zd ZddgZdS )ÚGlm4vPreTrainedModelr[  r  N)rO   rP   rQ   Ú_no_split_modulesr_   r`   ra   rj  rj  >  s   € € € € € Ø0Ð2DÐEÐÐÐr`   rj  c                   ó®   ‡ — e Zd ZU eed<   dZdgZeedœZ	dˆ fd„Z
eeedej        d	ej        d
ee         deez  fd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚGlm4vVisionModelr    )ÚimageÚvideor  ©rm   Ú
attentionsr¤   Nc                 óÆ  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t	          ‰¦  «        | _        t          ‰¦  «        | _        t          ‰¦  «        | _	        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ‰j        ¬¦  «        | _        t)          ‰j        ‰j        ¬¦  «        | _        t          j        ‰j        ‰j        ‰j        ‰j        ¬¦  «        | _        t)          ‰j        ‰j        ¬¦  «        | _        d| _        |                      ¦   «          d S )Nc                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r_   )r  )r  rQ  r    s     €ra   r  z-Glm4vVisionModel.__init__.<locals>.<listcomp>U  s"   ø€ Ð$[Ð$[Ð$[À!Õ%5°fÑ%=Ô%=Ð$[Ð$[Ð$[r`   )r±   r²   r;   r  )r@   Úout_channelsr¦   r§   F)r|   rŸ   rF   rC   rÃ   rè   r£   Úpatch_embedr®   Úrotary_pos_embr¨   Ú
ModuleListÚranger8   Úblocksr°   rI   rK   r;   Úmergerr—   r9   rE   Úpost_conv_layernormÚConv2dÚ
downsampleÚpost_layernormÚgradient_checkpointingÚ	post_initrË   s    `€ra   rŸ   zGlm4vVisionModel.__init__K  s?  øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø"(Ô";ˆÔØ Ô+ˆŒå/°Ñ7Ô7ˆŒÝ0°Ñ8Ô8ˆÔå8¸Ñ@Ô@ˆÔå”mÐ$[Ð$[Ð$[Ð$[ÅuÈVÌ\ÑGZÔGZÐ$[Ñ$[Ô$[Ñ\Ô\ˆŒÝ,ØÔ&°FÔ4LÐY_ÔYjð
ñ 
ô 
ˆŒõ $0°Ô0BÈÔH[Ð#\Ñ#\Ô#\ˆÔ Ýœ)ØÔ*ØÔ/ØÔ1ØÔ,ð	
ñ 
ô 
ˆŒõ +¨6Ô+=À6ÔCVÐWÑWÔWˆÔà&+ˆÔ#Ø�ŠÑÔÐÐÐr`   rm   Úgrid_thwr€   c           	      ó\  — t          || j        |¬¦  «        }t          || j        |¬¦  «        \  }}|                      |¦  «        }|                      |¦  «        }|                      ||¦  «        }|dd…         |dd…         z
  }|                      ||||dd…df                              |j	        ¦  «        |dd…df                              |j	        ¦  «        ¦  «        }| j
        D ]}	 |	|f|||dœ|¤Ž}Œ|                      |¦  «        }|                     d| j        | j        |j        d         ¦  «        }|                     dddd¦  «        }|                      |¦  «                             d| j        j        ¦  «        }|                      |¦  «        }
t%          ||
¬	¦  «        S )
a\  
        hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`):
            The final hidden states of the model.
        grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`):
            The temporal, height and width of feature shape of each image in LLM.

        Returns:
            `torch.Tensor`: hidden_states.
        )r€   rÍ   NrÒ   r   )Ú
cu_seqlensÚ
max_seqlenrD  r   r   )Úlast_hidden_stateÚpooler_output)r   rF   r   r    ru  r{  rv  rè   rß   rÎ   ry  r~  rÜ   rÙ   rÝ   r}  rI   rz  r   )r   rm   r�  r€   rb  rƒ  r„  rD  ÚseqlensÚblkÚmerged_hidden_statess              ra   r¿   zGlm4vVisionModel.forwardf  sä  € õ /¨x¸Ô9PÐY_Ð`Ñ`Ô`ˆÝ!=¸hÈÌÐ\bÐ!cÑ!cÔ!cÑˆ
�Jà×(Ò(¨Ñ7Ô7ˆØ×0Ò0°Ñ?Ô?ˆØ"×1Ò1°-ÀÑNÔNÐà˜Q˜R˜R”. :¨c¨r¨c¤?Ñ2ˆØŸšØØØØ˜˜˜˜A˜Ô×!Ò! -Ô"6Ñ7Ô7Ø˜˜˜˜A˜Ô×!Ò! -Ô"6Ñ7Ô7ñ
ô 
ˆð ”;ð 	ð 	ˆCØ˜CØðà%Ø%Ø$7ð	ð ð
 ðð ˆMˆMð ×+Ò+¨MÑ:Ô:ˆà%×*Ò*Ø�Ô'¨Ô)@À-ÔBUÐVXÔBYñ
ô 
ˆð &×-Ò-¨a°°A°qÑ9Ô9ˆØŸš¨Ñ6Ô6×;Ò;¸BÀÄÔ@[Ñ\Ô\ˆà#Ÿ{š{¨=Ñ9Ô9Ðå)Ø+Ø.ð
ñ 
ô 
ð 	
r`   r	  )rO   rP   rQ   r1   rX   Úinput_modalitiesrk  r  rý   Ú_can_record_outputsrŸ   r   r   r   rÀ   rÁ   r   r   r]   r   r¿   r†   r‡   s   @ra   rm  rm  B  sÒ   ø€ € € € € € ØÐÐÑØ)ÐØ+Ð,Ðà)Ø*ðð Ðð
ð ð ð ð ð ð6  ØØð2
Ø"œ\ð2
Ø5:´\ð2
ØMSÐTfÔMgð2
à	Ð+Ñ	+ð2
ð 2
ð 2
ñ „^ñ „_ñ  Ôð2
ð 2
ð 2
ð 2
ð 2
r`   rm  c                   óò   ‡ — e Zd ZeedœZdefˆ fd„Zee	e
	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dedz  d	ej        dz  d
edz  dee         deez  fd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚGlm4vTextModelrp  r    c                 ó,  •‡— t          ¦   «                              ‰¦  «         t          j        ˆfd„t	          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j	        ¬¦  «        | _
        t          ‰¬¦  «        | _        | `| `d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r_   )r[  )r  r:  r    s     €ra   r  z+Glm4vTextModel.__init__.<locals>.<listcomp>§  s$   ø€ ÐgÐgÐg¸)Õ" 6¨9Ñ5Ô5ÐgÐgÐgr`   r  ©r    )r|   rŸ   r¨   rw  rx  ru   rp   r—   r9   rE   rq   r  Ú
rotary_embrK  Úhas_sliding_layersrË   s    `€ra   rŸ   zGlm4vTextModel.__init__¤  s‘   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý”mØgÐgÐgÐgÅuÈVÔMeÑGfÔGfÐgÑgÔgñ
ô 
ˆŒõ ! Ô!3¸Ô9LÐMÑMÔMˆŒ	Ý2¸&ÐAÑAÔAˆŒØÐ%ØÐ#Ð#Ð#r`   Nrk   rn   rb  rf   rl   ry   r€   r¤   c           	      óp  — |d u |d uz  rt          d¦  «        ‚|r5|€3t          j                             ¦   «         st	          | j        ¬¦  «        }|€|                      |¦  «        }|€y|�|                     ¦   «         nd}t          j        |j	        d         |j
        ¬¦  «        |z   }|                     ddd¦  «                             d|j	        d         d¦  «        }n3|j        dk    r(|d	                              d|j	        d         d¦  «        }|j        dk    r$|j	        d         d
k    r|d         }	|dd …         }nd }	| j        ||||	dœ}
t          di |
¤Ž}|}|                      ||¬¦  «        }| j        D ]} ||f||	||dœ|¤Ž}|}Œ|                      |¦  «        }t%          ||¬¦  «        S )Nú:You must specify exactly one of input_ids or inputs_embedsr�  r   rÍ   rÐ   rÒ   r   r   )N.é   )r    rl   rn   rf   rb  )rb  )rn   rb  rf   rD  )r…  rf   r_   )Ú
ValueErrorrÀ   ÚjitÚ
is_tracingr	   r    ro   Úget_seq_lengthrá   rÙ   rÎ   rÜ   ÚexpandÚndimr   r‘  rp   rq   r   )r   rk   rn   rb  rf   rl   ry   r€   Úpast_seen_tokensÚtext_position_idsÚmask_kwargsÚcausal_maskrm   rD  Údecoder_layerÚlayer_outputss                   ra   r¿   zGlm4vTextModel.forward®  s:  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZð ð 	?˜Ð0½¼×9MÒ9MÑ9OÔ9OÐ0Ý*°$´+Ð>Ñ>Ô>ˆOàÐ Ø ×-Ò-¨iÑ8Ô8ˆMð ÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×,Ò,¨Q°°2Ñ6Ô6×=Ò=¸aÀÔATÐUVÔAWÐY[Ñ\Ô\ˆLˆLØÔ !Ò#Ð#Ø'¨	Ô2×9Ò9¸!¸\Ô=OÐPQÔ=RÐTVÑWÔWˆLð Ô Ò!Ð! lÔ&8¸Ô&;¸qÒ&@Ð&@Ø ,¨Q¤ÐØ'¨¨¨Ô+ˆLˆLð !%Ðð ”kØ*Ø,Ø.Ø-ð
ð 
ˆõ )Ð7Ð7¨;Ð7Ð7ˆà%ˆØ"Ÿošo¨mÈ,˜oÑWÔWÐà!œ[ð 		*ð 		*ˆMØ)˜MØðà*Ø.Ø /Ø$7ðð ð ðð ˆMð *ˆMˆMàŸ	š	 -Ñ0Ô0ˆå&Ø+Ø+ð
ñ 
ô 
ð 	
r`   )NNNNNN)rO   rP   rQ   r[  r9  r‹  rc   rŸ   r   r   r   rÀ   re  rÁ   r   rf  rZ   r   r   r]   r   r¿   r†   r‡   s   @ra   r�  r�  ž  s7  ø€ € € € € à.Ø(ðð Ðð
$˜ð $ð $ð $ð $ð $ð $ð ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ðJ
ð J
àÔ# dÑ*ðJ
ð œ tÑ+ðJ
ð Ô&¨Ñ-ð	J
ð
  ™ðJ
ð Ô(¨4Ñ/ðJ
ð ˜$‘;ðJ
ð Ð-Ô.ðJ
ð 
Ð(Ñ	(ðJ
ð J
ð J
ñ „_ñ  Ôñ „^ðJ
ð J
ð J
ð J
ð J
r`   r�  c                   óf  ‡ — e Zd ZddgZˆ fd„Z ed¬¦  «        ee	 ddej	        dej
        dz  d	ee         d
eez  fd„¦   «         ¦   «         ¦   «         Z	 	 ddej
        dej	        dej	        dz  dej	        dz  fd„Z	 ddej
        dz  d
eej        ej        f         fˆ fd„Zee	 	 	 	 	 	 	 	 	 	 ddej
        dz  dej        dz  dej
        dz  dedz  dej	        dz  dej        dz  dej	        dz  dej
        dz  dej
        dz  dej        dz  d	ee         d
eez  fd„¦   «         ¦   «         Zˆ xZS )Ú
Glm4vModelr[  r  c                 ó’   •— t          ¦   «                              |¦  «         t                               |j        ¦  «        | _        d S rž   )r|   rŸ   rm  Ú_from_configr3   ÚvisualrË   s     €ra   rŸ   zGlm4vModel.__init__  s7   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý&×3Ò3°FÔ4HÑIÔIˆŒˆˆr`   ro  )ÚmodalityNÚpixel_values_videosÚvideo_grid_thwr€   r¤   c                 óê  — |                      | j        j        ¦  «        }|d d …df         }|d d …dd …f         }t          j        ||d¬¦  «        }|                     |j        d         d¦  «        }t          j        ||gd¬¦  «        } | j        |f|ddœ|¤Ž}	|                     d¦  «        | j        j	        dz  z   
                    ¦   «         }
t          j        |	j        |
¦  «        }||	_        |	S )Nr   rÍ   rÓ   T)r�  Úreturn_dictrÒ   r   )Útyper¦  rÏ   rÀ   r   Únew_onesrÙ   r  ÚprodrF   Útolistr  r†  )r   r¨  r©  r€   ÚtÚhwÚflattened_hwÚprefix_onesÚflattened_video_grid_thwÚvision_outputsÚsplit_sizesÚvideo_embedss               ra   Úget_video_featureszGlm4vModel.get_video_features  s  € ð 2×6Ò6°t´{Ô7HÑIÔIÐà˜1˜1˜1˜a˜4Ô ˆØ˜A˜A˜A˜q˜r˜r˜EÔ"ˆåÔ.¨r°1¸!Ð<Ñ<Ô<ˆØ$×-Ò-¨lÔ.@ÀÔ.CÀQÑGÔGˆÝ#(¤9¨k¸<Ð-HÈaÐ#PÑ#PÔ#PÐ Ø$˜œØð
Ø*BÐPTð
ð 
ØX^ð
ð 
ˆð &×*Ò*¨2Ñ.Ô.°$´+Ô2PÐRSÑ2SÑS×[Ò[Ñ]Ô]ˆÝ”{ >Ô#?ÀÑMÔMˆØ'3ˆÔ$àÐr`   rk   rl   Úimage_featuresÚvideo_featuresc           	      ó  — |€Ë| |                       ¦   «         t          j        d| j        j        t          j        |j        ¬¦  «        ¦  «        k    }|                     d¦  «        }| |                       ¦   «         t          j        d| j        j        t          j        |j        ¬¦  «        ¦  «        k    }|                     d¦  «        }n || j        j        k    }|| j        j        k    }| 	                    ¦   «         }| 
                    d¦  «                             |j        ¦  «        }|�Et          ||j        d         z  |                     ¦   «         k    d|› d|j        d         › �¦  «         | 	                    ¦   «         }| 
                    d¦  «                             |j        ¦  «        }|�Et          ||j        d         z  |                     ¦   «         k    d|› d|j        d         › �¦  «         ||fS )	zï
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        Nr_   ©rÏ   rÎ   rÒ   z6Image features and image tokens do not match, tokens: z, features: r   z6Video features and video tokens do not match, tokens: )Úget_input_embeddingsrÀ   Úfullr    r‹   rÛ   rÎ   ÚallrŒ   rã   rÞ   rß   r   rÙ   Únumel)	r   rk   rl   r¹  rº  Úspecial_image_maskÚspecial_video_maskÚn_image_tokensÚn_video_tokenss	            ra   Úget_placeholder_maskzGlm4vModel.get_placeholder_mask  s  € ð ÐØ!.Ð2M°$×2KÒ2KÑ2MÔ2MÝ”
˜2˜tœ{Ô9ÅÄÐTaÔThÐiÑiÔiñ3ô 3ò "Ðð "4×!7Ò!7¸Ñ!;Ô!;ÐØ!.Ð2M°$×2KÒ2KÑ2MÔ2MÝ”
˜2˜tœ{Ô9ÅÄÐTaÔThÐiÑiÔiñ3ô 3ò "Ðð "4×!7Ò!7¸Ñ!;Ô!;ÐÐð "+¨d¬kÔ.HÒ!HÐØ!*¨d¬kÔ.HÒ!HÐà+×/Ò/Ñ1Ô1ˆØ/×9Ò9¸"Ñ=Ô=×@Ò@ÀÔAUÑVÔVÐØÐ%Ý"Ø Ô!4°RÔ!8Ñ8¸N×<PÒ<PÑ<RÔ<RÒRØ~ÈÐ~Ð~ÐesÔeyÐz{Ôe|Ð~Ð~ñô ð ð
 ,×/Ò/Ñ1Ô1ˆØ/×9Ò9¸"Ñ=Ô=×@Ò@ÀÔAUÑVÔVÐØÐ%Ý"Ø Ô!4°RÔ!8Ñ8¸N×<PÒ<PÑ<RÔ<RÒRØ~ÈÐ~Ð~ÐesÔeyÐz{Ôe|Ð~Ð~ñô ð ð "Ð#5Ð5Ð5r`   c                 ó’   •— |�*t          j        ||dd…df         d¬¦  «        }d|dd…df<    t          ¦   «         j        dd|i|¤ŽS )aÙ  
        Difference from Qwen2VL/Qwen2.5VL's get_rope_index:
        - GLM4V uses timestamps to separate each video frame, so the video_grid_thw should also be split too.

        Args:
            input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
                Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
                it.
            mm_token_type_ids (`torch.IntTensor` of shape `(batch_size, sequence_length)`):
                Token type ids matching each modality to a different value in the input sequence, i.e. text (0), image (1), video (2).
            image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*):
                The temporal, height and width of feature shape of each image in LLM.
            video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*):
                The temporal, height and width of feature shape of each video in LLM.
            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

        Returns:
            position_ids (`torch.LongTensor` of shape `(3, batch_size, sequence_length)`)
            mrope_position_deltas (`torch.Tensor` of shape `(batch_size)`)
        Nr   rÓ   rÍ   r©  r_   )rÀ   r   r|   Úget_rope_index)r   r©  Úsuper_kwargsr�   s      €ra   rÇ  zGlm4vModel.get_rope_indexI  si   ø€ ð> Ð%Ý"Ô4°^À^ÐTUÐTUÐTUÐWXÐTXÔEYÐ_`ÐaÑaÔaˆNØ#$ˆN˜1˜1˜1˜a˜4Ñ à%�u‰wŒwÔ%ÐTÐT°^ÐTÀ|ÐTÐTÐTr`   rn   rb  rf   Úpixel_valuesÚimage_grid_thwÚmm_token_type_idsc           	      óð  — |d u |d uz  rt          d¦  «        ‚|€ |                      ¦   «         |¦  «        }|�{ | j        ||fddi|¤Žj        }t	          j        |d¬¦  «                             |j        |j        ¦  «        }|  	                    |||¬¦  «        \  }}| 
                    ||¦  «        }|�{ | j        ||	fddi|¤Žj        }t	          j        |d¬¦  «                             |j        |j        ¦  «        }|  	                    |||¬¦  «        \  }}| 
                    ||¦  «        }|€|                      |||	||||
¬¦  «        } | j        dd ||||d	œ|¤Ž}t          di |¤d
| j        i¤ŽS )Nr”  r«  Tr   rÓ   )r¹  )rº  )rk   rÊ  r©  rl   rn   rf   rË  )rk   rb  rn   rf   rl   Úrope_deltasr_   )r–  r½  Úget_image_featuresr†  rÀ   r  rß   rÎ   rÏ   rÅ  Úmasked_scatterr¸  Úcompute_3d_position_idsÚlanguage_modelrh  rÍ  )r   rk   rn   rb  rf   rl   rÉ  r¨  rÊ  r©  rË  r€   Úimage_embedsÚ
image_maskrQ  r·  Ú
video_maskÚoutputss                     ra   r¿   zGlm4vModel.forwardn  s&  € ð  ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø7˜D×5Ò5Ñ7Ô7¸	ÑBÔBˆMàÐ#Ø2˜4Ô2Ø˜nðð Ø:>ðØBHðð äð õ !œ9 \°qÐ9Ñ9Ô9×<Ò<¸]Ô=QÐS`ÔSfÑgÔgˆLØ ×5Ò5°iÀÐ_kÐ5ÑlÔl‰MˆJ˜Ø)×8Ò8¸À\ÑRÔRˆMàÐ*Ø2˜4Ô2Ø# ^ðð ØAEðØIOðð äð õ !œ9 \°qÐ9Ñ9Ô9×<Ò<¸]Ô=QÐS`ÔSfÑgÔgˆLØ ×5Ò5°iÀÐ_kÐ5ÑlÔl‰MˆAˆzØ)×8Ò8¸À\ÑRÔRˆMàÐØ×7Ò7Ø#Ø-Ø-Ø+Ø-Ø /Ø"3ð 8ñ ô ˆLð &�$Ô%ð 
ØØ%Ø)Ø+Ø'ð
ð 
ð ð
ð 
ˆõ (ð 
ð 
Øð
ð 
àÔ(ð
ð 
ð 
ð 	
r`   rž   )NN)
NNNNNNNNNN)rO   rP   rQ   rk  rŸ   r   r   r   rÀ   rf  re  r   r   r]   r   r¸  rÅ  rÁ   rÇ  r   Ú	IntTensorrh  r¿   r†   r‡   s   @ra   r£  r£  þ  s§  ø€ € € € € Ø0Ð2DÐEÐðJð Jð Jð Jð Jð  Ð¨Ð1Ñ1Ô1ØØð 37ðð à"Ô.ðð Ô(¨4Ñ/ðð Ð+Ô,ð	ð
 
Ð+Ñ	+ðð ð ñ „^ñ Ôñ 2Ô1ðð6 48Ø37ð(6ð (6àÔ#ð(6ð Ô(ð(6ð Ô)¨DÑ0ð	(6ð
 Ô)¨DÑ0ð(6ð (6ð (6ð (6ðX 37ð#Uð #UàÔ(¨4Ñ/ð#Uð 
ˆuŒ|˜Uœ\Ð)Ô	*ð	#Uð #Uð #Uð #Uð #Uð #UðJ Øð .2Ø.2Ø04Ø(,Ø26Ø,0Ø8<Ø26Ø26Ø48ð;
ð ;
àÔ# dÑ*ð;
ð œ tÑ+ð;
ð Ô&¨Ñ-ð	;
ð
  ™ð;
ð Ô(¨4Ñ/ð;
ð ”l TÑ)ð;
ð #Ô.°Ñ5ð;
ð Ô(¨4Ñ/ð;
ð Ô(¨4Ñ/ð;
ð !œ?¨TÑ1ð;
ð Ð+Ô,ð;
ð 
Ð)Ñ	)ð;
ð ;
ð ;
ñ Ôñ „^ð;
ð ;
ð ;
ð ;
ð ;
r`   r£  c                   ó   — e Zd ZdS )ÚGlm4vCausalLMOutputWithPastNr˜   r_   r`   ra   rØ  rØ  ®  r™   r`   rØ  c                   óŒ  — e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dedz  dej        dz  dej        dz  d	ej        dz  d
ej        dz  dej        dz  dej        dz  dej        dz  de	ej        z  de
e         deez  fd„Z	 ddej        dz  dej        dz  deej        ej        f         fd„ZdS )ÚGlm4vForConditionalGenerationNr   rk   rn   rb  rf   rl   ÚlabelsrÉ  r¨  rÊ  r©  rË  Úlogits_to_keepr€   r¤   c                 óp  —  | j         d||||	|
|||||dœ
|¤Ž}|d         }t          |t          ¦  «        rt          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }d}|�  | j        d||| j        j        j        dœ|¤Ž}t          |||j
        |j        |j        |j        ¬¦  «        S )aÇ  
        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, Glm4vForConditionalGeneration

        >>> model = Glm4vForConditionalGeneration.from_pretrained("zai-org/GLM-4.1V-9B-Thinking")
        >>> processor = AutoProcessor.from_pretrained("zai-org/GLM-4.1V-9B-Thinking")

        >>> messages = [
            {
                "role": "user",
                "content": [
                    {"type": "image", "url": "https://www.ilankelman.org/stopsigns/australia.jpg"},
                    {"type": "text", "text": "What is shown in this image?"},
                ],
            },
        ]
        >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
        >>> inputs = processor(text=[text], images=[image], vision_infos=[vision_infos])

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "The image shows a street scene with a red stop sign in the foreground. In the background, there is a large red gate with Chinese characters ..."
        ```)
rk   rÉ  r¨  rÊ  r©  rË  rb  rn   rf   rl   r   N)ÚlogitsrÛ  rs   )ÚlossrÞ  rf   rm   rq  rÍ  r_   )Úmodelr“   rW   ÚsliceÚlm_headÚloss_functionr    re   rs   rØ  rf   rm   rq  rÍ  )r   rk   rn   rb  rf   rl   rÛ  rÉ  r¨  rÊ  r©  rË  rÜ  r€   rÕ  rm   Úslice_indicesrÞ  rß  s                      ra   r¿   z%Glm4vForConditionalGeneration.forward³  s  € ðb �$”*ð 
ØØ%Ø 3Ø)Ø)Ø/Ø%Ø)Ø+Ø'ð
ð 
ð ð
ð 
ˆð   œ
ˆõ 9CÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%ð Ø f¸¼Ô9PÔ9[ðð Ø_eðð ˆDõ +ØØØ#Ô3Ø!Ô/ØÔ)ØÔ+ð
ñ 
ô 
ð 	
r`   c           	      ó`  — |��| |                       ¦   «         t          j        d| j        j        t          j        |j        ¬¦  «        ¦  «        k    d         }| |                       ¦   «         t          j        d| j        j        t          j        |j        ¬¦  «        ¦  «        k    d         }| |                       ¦   «         t          j        d| j        j        t          j        |j        ¬¦  «        ¦  «        k    d         }n0|| j        j        k    }|| j        j        k    }|| j        j        k    }t          j	        | 
                    ¦   «         | 
                    ¦   «         z
  d¬¦  «        }|dk    }|| z  }|                     d¬¦  «        }	|                     d¬¦  «        }
|	|
fS )aa  
        Get the number of images and videos for each sample to calculate the separation length of the sample tensor.
        These parameters are not passed through the processor to avoid unpredictable impacts from interface modifications.

        Args:
            input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
                Indices of input sequence tokens in the vocabulary.

        Returns:
            image_nums (`torch.LongTensor` of shape `(batch_size, num_images_sample)`)
            video_nums (`torch.LongTensor` of shape `(batch_size, num_videos_sample)`)
        Nr_   r¼  ).r   rÍ   rÓ   r   )r½  rÀ   r¾  r    r�   rÛ   rÎ   r�   r�   râ   rW   rã   )r   rk   rl   Úis_imageÚis_video_startÚis_video_endÚvideo_levelÚinside_videoÚstandalone_imagesÚimage_countsÚvideo_countss              ra   Ú_get_image_nums_and_video_numsz<Glm4vForConditionalGeneration._get_image_nums_and_video_nums  s´  € ð$ Ñ$àØ.�4×,Ò,Ñ.Ô.Ý”J˜r 4¤;Ô#CÍ5Ì:Ð^kÔ^rÐsÑsÔsñô òð ôˆHð Ø.�4×,Ò,Ñ.Ô.Ý”J˜r 4¤;Ô#CÍ5Ì:Ð^kÔ^rÐsÑsÔsñô òð ôˆNð Ø.�4×,Ò,Ñ.Ô.Ý”J˜r 4¤;Ô#AÍÌÐ\iÔ\pÐqÑqÔqñô òð ôˆLˆLð ! D¤KÔ$DÒDˆHØ&¨$¬+Ô*JÒJˆNØ$¨¬Ô(FÒFˆLõ ”l >×#5Ò#5Ñ#7Ô#7¸,×:JÒ:JÑ:LÔ:LÑ#LÐRSÐTÑTÔTˆØ" Q’ˆð %¨¨Ñ6Ðð )×,Ò,°Ð,Ñ3Ô3ˆØ%×)Ò)¨aÐ)Ñ0Ô0ˆà˜\Ð)Ð)r`   )NNNNNNNNNNNr   rž   )rO   rP   rQ   rÀ   re  rÁ   r   rf  rÖ  rW   r   r   r]   rØ  r¿   rî  r_   r`   ra   rÚ  rÚ  ²  s¬  € € € € € ð .2Ø.2Ø04Ø(,Ø26Ø*.Ø,0Ø8<Ø26Ø26Ø48Ø-.ðR
ð R
àÔ# dÑ*ðR
ð œ tÑ+ðR
ð Ô&¨Ñ-ð	R
ð
  ™ðR
ð Ô(¨4Ñ/ðR
ð Ô  4Ñ'ðR
ð ”l TÑ)ðR
ð #Ô.°Ñ5ðR
ð Ô(¨4Ñ/ðR
ð Ô(¨4Ñ/ðR
ð !œ?¨TÑ1ðR
ð ˜eœlÑ*ðR
ð Ð+Ô,ðR
ð 
Ð,Ñ	,ðR
ð R
ð R
ð R
ðn .2ð6*ð 6*àÔ# dÑ*ð6*ð ”| dÑ*ð6*ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð	6*ð 6*ð 6*ð 6*ð 6*ð 6*r`   rÚ  c                   ó$   — e Zd ZddddœddidœZdS )ÚGlm4vProcessorKwargsFT)ÚpaddingÚreturn_token_type_idsÚreturn_mm_token_type_idsÚreturn_metadata)Útext_kwargsÚvideos_kwargsN)rO   rP   rQ   Ú	_defaultsr_   r`   ra   rð  rð  @  s9   € € € € € ð Ø%*Ø(,ð
ð 
ð
 ,¨TÐ2ðð €I€I€Ir`   rð  c                   ój   ‡ — e Zd Zdˆ fd„	Zdededefd„Zdedeee                  fd„Z	dd
efd„Z
ˆ xZS )ÚGlm4vProcessorNc                 ó2  •— t          ¦   «                              ||||¬¦  «         t          |d¦  «        sdn|j        | _        t          |d¦  «        sdn|j        | _        |                     d¦  «        | _        |                     d¦  «        | _        d S )N)Úchat_templateÚimage_tokenz	<|image|>Úvideo_tokenz	<|video|>z<|begin_of_video|>z<|end_of_video|>)r|   rŸ   Úhasattrrü  rý  Úconvert_tokens_to_idsÚvideo_start_idÚvideo_end_id)r   Úimage_processorÚ	tokenizerÚvideo_processorrû  r€   r�   s         €ra   rŸ   zGlm4vProcessor.__init__L  s—   ø€ Ý‰Œ×Ò˜¨)°_ÐTaÐÑbÔbÐbÝ.5°iÀÑ.OÔ.OÐj˜;˜;ÐU^ÔUjˆÔÝ.5°iÀÑ.OÔ.OÐj˜;˜;ÐU^ÔUjˆÔØ'×=Ò=Ð>RÑSÔSˆÔØ%×;Ò;Ð<NÑOÔOˆÔÐÐr`   Úvideo_inputsÚ	video_idxr¤   c                 óÈ  — | j         j        dz  }|d         |         d         }|d         |                              ¦   «         |z  |z  }|d         |         }d}|j        €t                               d¦  «         |j        €dn|j        |_        |j        d d d…         }	g }
t          dt          |	¦  «        ¦  «        D ]}|
 	                    |	|         ¦  «         Œ|
d |…         }t          |¦  «        |k     r2| 	                    |r|d         nd¦  «         t          |¦  «        |k     °2t          |¦  «        D ]&}||         }|  
                    ||¬	¦  «        }||z  }Œ'|S )
Nr   r©  r   Úvideo_metadataÚ a  GLM4V requires frame timestamps to construct prompts, but the `fps` of the input video could not be inferred. Probably `video_metadata` was missing from inputs and you passed pre-sampled frames. Defaulting to `fps=24`. Please provide `video_metadata` for more accurate results.r7   rÒ   )Únum_image_tokens)r  Ú
merge_sizer®  ÚfpsÚloggerÚwarning_onceÚ
timestampsrx  ÚlenÚappendÚreplace_frame_token_id)r   r  r  r€   Úmerge_lengthÚ
num_framesr
  ÚmetadataÚvideo_structurer  Úunique_timestampsÚidxÚselected_timestampsÚ	frame_idxÚtimestamp_secÚframe_structures                   ra   Úreplace_video_tokenz"Glm4vProcessor.replace_video_tokenS  s   € ØÔ+Ô6¸Ñ9ˆØ!Ð"2Ô3°IÔ>¸qÔAˆ
Ø'Ð(8Ô9¸)ÔD×IÒIÑKÔKÈ|Ñ[Ð_iÑiÐØÐ 0Ô1°)Ô<ˆØˆàŒ<ÐÝ×Òðeñô ð ð
 &œ\Ð1�r�r°x´|ˆŒØÔ(¨¨¨1¨Ô-ˆ
àÐÝ˜�C 
™OœOÑ,Ô,ð 	6ð 	6ˆCØ×$Ò$ Z°¤_Ñ5Ô5Ð5Ð5à/°°°Ô<ÐÝÐ%Ñ&Ô&¨Ò3Ð3Ø×&Ò&ÐBUÐ'\Ð':¸2Ô'>Ð'>Ð[\Ñ]Ô]Ð]õ Ð%Ñ&Ô&¨Ò3Ð3õ ˜zÑ*Ô*ð 	/ð 	/ˆIØ/°	Ô:ˆMØ"×9Ò9¸-ÐZjÐ9ÑkÔkˆOØ˜Ñ.ˆOˆOàÐr`   rk   c                 ó€  — g }|D ]¸}t          j        |¦  «        }t          j        |¦  «        }t          j        || j        k    d¬¦  «        }t          j        || j        k    d¬¦  «        }||k    }d||| j        k    |z  <   d||| j        k    | z  <   |                     |                     ¦   «         ¦  «         Œ¹|S )Nr   )Úaxisr   rÍ   )	ÚnpÚarrayÚ
zeros_likerâ   r   r  r‹   r  r¯  )	r   rk   rË  ÚinputÚ	array_idsÚmm_token_typesÚstartsÚendsÚis_video_modalitys	            ra   Úcreate_mm_token_type_idsz'Glm4vProcessor.create_mm_token_type_idsr  sÕ   € ð ÐØð 	>ð 	>ˆEÝœ ™œˆIÝœ]¨5Ñ1Ô1ˆNõ
 ”Y˜y¨DÔ,?Ò?ÀaÐHÑHÔHˆFÝ”9˜Y¨$Ô*;Ò;À!ÐDÑDÔDˆDØ &¨¢ÐàUVˆN˜I¨Ô)<Ò<Ð@QÑQÑRØXYˆN˜I¨Ô)<Ò<ÐBSÐASÑTÑUØ×$Ò$ ^×%:Ò%:Ñ%<Ô%<Ñ=Ô=Ð=Ð=Ø Ð r`   rÍ   r
  c                 ó<   — d| j         |z  › dt          |¦  «        › �S )Nz<|begin_of_image|>z<|end_of_image|>)rü  rW   )r   r  r
  s      ra   r  z%Glm4vProcessor.replace_frame_token_id‡  s+   € Øm DÔ$4Ð7GÑ$GÐmÐmÕY\Ð]jÑYkÔYkÐmÐmÐmr`   )NNNN©rÍ   )rO   rP   rQ   rŸ   r^   rW   rY   r  r\   r)  r  r†   r‡   s   @ra   rù  rù  K  sÁ   ø€ € € € € ðPð Pð Pð Pð Pð Pð°ð Àð ÐSVð ð ð ð ð>!°$ð !¸4ÀÀSÄ	¼?ð !ð !ð !ð !ð*nð nÀcð nð nð nð nð nð nð nð nr`   rù  )	r‰   rc   r1   rÚ  r£  rj  rù  r�  rm  r+  )eÚcollections.abcr   Únumpyr   rÀ   Útorch.nnr¨   Útorch.nn.functionalÚ
functionalrå   Úhuggingface_hub.dataclassesr   r   Úactivationsr   Úcache_utilsr   r	   Úconfiguration_utilsr
   Úmasking_utilsr   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   r   r   r   Úutils.genericr   r   Úutils.output_capturingr   Úvision_utilsr   r   Úglm4.modeling_glm4r   r   r    Úqwen2_5_vl.modeling_qwen2_5_vlr!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   Úqwen2_vl.modeling_qwen2_vlr,   Úqwen2_vl.processing_qwen2_vlr-   r.   Ú
get_loggerrO   r  r1   rc   r‰   r—   r›   r£   r®   r©   r°   rÃ   rý   r  r  r)  r7  r9  rY  r[  rh  rj  rm  r�  r£  rØ  rÚ  rð  rù  Ú__all__r_   r`   ra   ú<module>rF     sÜ  ðð %Ð $Ð $Ð $Ð $Ð $à Ð Ð Ð Ø €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø .Ð .Ð .Ð .Ð .Ð .Ø Ð Ð Ð Ð Ð à !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø /Ð /Ð /Ð /Ð /Ð /Ø BÐ BÐ BÐ BÐ BÐ BØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø SÐ SÐ SÐ SÐ SÐ SÐ SÐ SØ 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø &Ð &Ð &Ð &Ð &Ð &ðð ð ð ð ð ð ð ð ð ð ð ð ð ðð ð ð ð ð ð ð ð 6Ð 5Ð 5Ð 5Ð 5Ð 5Ø QÐ QÐ QÐ QÐ QÐ QÐ QÐ QØ NÐ NÐ NÐ NÐ NÐ NÐ NÐ NÐ NÐ Nðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð 6Ð 5Ð 5Ð 5Ð 5Ð 5ðð ð ð ð ð ð ð ð 
ˆÔ	˜HÑ	%Ô	%€ð €Ð9Ð:Ñ:Ô:Øð((ð ((ð ((ð ((ð ((Ð(ñ ((ô ((ñ „ñ ;Ô:ð((ðV €Ð9Ð:Ñ:Ô:Øð7(ð 7(ð 7(ð 7(ð 7(Ð&ñ 7(ô 7(ñ „ñ ;Ô:ð7(ðt €Ð9Ð:Ñ:Ô:Øð7(ð 7(ð 7(ð 7(ð 7(Ð"ñ 7(ô 7(ñ „ñ ;Ô:ð7(ðv	ð 	ð 	ð 	ð 	�;ñ 	ô 	ð 	ð8ð 8ð 8ð 8ð 8�Mñ 8ô 8ð 8ð	mð 	mð 	mð 	mð 	mÐ4ñ 	mô 	mð 	mð	ð 	ð 	ð 	ð 	Ð!@ñ 	ô 	ð 	ðfð fð fð fð f˜RœYñ fô fð fð"Gð Gð Gð Gð G˜BœIñ Gô Gð GðTRð Rð Rð Rð RÐ4ñ Rô Rð Rð5ð 5ð 5ð 5ð 5Ð,ñ 5ô 5ð 5ð1ð 1ð 1ð 1ð 1Ð8ñ 1ô 1ð 1ðB6ð 6ð 6ð!ð !ð !ð !ðHC)ð C)ð C)ð C)ð C)˜œñ C)ô C)ð C)ðL	ð 	ð 	ð 	ð 	�7ñ 	ô 	ð 	ð/ð /ð /ð /ð /Ð6ñ /ô /ð /ðd	ð 	ð 	ð 	ð 	Ð<ñ 	ô 	ð 	ðFð Fð Fð Fð FÐ4ñ Fô Fð FðY
ð Y
ð Y
ð Y
ð Y
Ð+ñ Y
ô Y
ð Y
ðx]
ð ]
ð ]
ð ]
ð ]
Ð(ñ ]
ô ]
ð ]
ð@m
ð m
ð m
ð m
ð m
�ñ m
ô m
ð m
ð`	ð 	ð 	ð 	ð 	Ð"Bñ 	ô 	ð 	ðK*ð K*ð K*ð K*ð K*Ð$Fñ K*ô K*ð K*ð\ð ð ð ð Ð1ñ ô ð ð=nð =nð =nð =nð =nÐ%ñ =nô =nð =nð@
ð 
ð 
€€€r`   