§
    9²¸j0 ã                   ó”  — d dl Z d dlmZ d dlmZ d dlmZ d dlZd dlm	Z	 d dl
m	c mZ d dlmZ ddlmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZmZm Z  ddl!m"Z"m#Z# ddl$m%Z%m&Z& ddl'm(Z( ddl)m*Z*m+Z+m,Z,m-Z- ddl.m/Z/ ddl0m1Z1m2Z2m3Z3m4Z4m5Z5 ddl6m7Z7 ddl8m9Z9m:Z: ddl;m<Z<m=Z=m>Z>  ed¦  «         G d„ de	j?        ¦  «        ¦   «         Z@ G d„ de	j?        ¦  «        ZA G d„ de	j?        ¦  «        ZB G d „ d!e	j?        ¦  «        ZC G d"„ d#e	j?        ¦  «        ZD G d$„ d%e	j?        ¦  «        ZEd&„ ZFd'ejG        d(ejG        d)ejG        d*ejG        d+eHejG        ejG        f         f
d,„ZId-ejG        d.eJd+ejG        fd/„ZK	 dWd1e	j?        d2ejG        d3ejG        d4ejG        d5ejG        dz  d6eLd7eLd8e(e*         fd9„ZM G d:„ d;e	j?        ¦  «        ZN G d<„ d=e¦  «        ZO G d>„ d?e	j?        ¦  «        ZPd@„ ZQdXdA„ZR G dB„ dCe	j?        ¦  «        ZS G dD„ dEe	j?        ¦  «        ZT G dF„ dGe¦  «        ZUe+e G dH„ dIe¦  «        ¦   «         ¦   «         ZVe+ G dJ„ dKe&¦  «        ¦   «         ZW G dL„ dMeW¦  «        ZXe+ G dN„ dOeW¦  «        ¦   «         ZYe+ G dP„ dQeW¦  «        ¦   «         ZZe+e G dR„ dSe ¦  «        ¦   «         ¦   «         Z[ G dT„ dUeWe¦  «        Z\g dV¢Z]dS )Yé    N)ÚCallable)Ú	dataclass)ÚAny)Ú	LayerNormé   )ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hub)Úcreate_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚBaseModelOutputWithPoolingÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚtorch_compilable_check)Údeprecate_kwarg)Úaccepts_precomputed_kwargsÚget_max_seqlenÚis_flash_attention_requestedÚmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputs)Úget_vision_attention_seqlensÚget_vision_position_idsé   )ÚGlm4vConfigÚGlm4vTextConfigÚGlm4vVisionConfigÚRMSNormc                   óT   ‡ — e Zd Zd	deddfˆ fd„Zdej        dej        fd„Zd„ Zˆ xZ	S )
ÚGlm4vRMSNormç�íµ ÷Æ°>ÚepsÚreturnNc                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )z;
        Glm4vRMSNorm is equivalent to T5LayerNorm
        N)ÚsuperÚ__init__ÚnnÚ	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizer-   Ú	__class__s      €úx/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/glm4v/modeling_glm4v.pyr1   zGlm4vRMSNorm.__init__9   sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐó    Úhidden_statesc                 ó  — |j         }|                     t          j        ¦  «        }|                     d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        |                     |¦  «        z  S )Né   éÿÿÿÿT)Úkeepdim)	ÚdtypeÚtor4   Úfloat32ÚpowÚmeanÚrsqrtr7   r6   )r8   r=   Úinput_dtypeÚvariances       r;   ÚforwardzGlm4vRMSNorm.forwardA   s|   € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØ ×$Ò$ QÑ'Ô'×,Ò,¨R¸Ð,Ñ>Ô>ˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆØŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:r<   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)Útupler6   Úshaper7   )r8   s    r;   Ú
extra_reprzGlm4vRMSNorm.extra_reprH   s&   € Ý˜œÔ)Ñ*Ô*ÐIÐI°$Ô2GÐIÐIÐIr<   )r,   )
Ú__name__Ú
__module__Ú__qualname__Úfloatr1   r4   ÚTensorrJ   rN   Ú__classcell__©r:   s   @r;   r+   r+   7   sŒ   ø€ € € € € ð$ð $¨ð $¸$ð $ð $ð $ð $ð $ð $ð; U¤\ð ;°e´lð ;ð ;ð ;ð ;ðJð Jð Jð Jð Jð Jð Jr<   r+   c                   ó,   ‡ — e Zd Zddefˆ fd„Zd„ Zˆ xZS )ÚGlm4VisionMlpFÚbiasc                 óŠ  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        t          j        | j        | j        |¬¦  «        | _        t          j        | j        | j        |¬¦  «        | _        t          j        | j        | j        |¬¦  «        | _	        t          |j                 | _        d S ©N©rX   )r0   r1   r9   Úout_hidden_sizeÚintermediate_sizer2   ÚLinearÚ	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fn)r8   ÚconfigrX   r:   s      €r;   r1   zGlm4VisionMlp.__init__M   sŸ   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔØ!'Ô!7ˆÔÝœ 4Ô#3°TÔ5KÐRVÐWÑWÔWˆŒÝ”y Ô!1°4Ô3IÐPTÐUÑUÔUˆŒÝœ 4Ô#9¸4Ô;KÐRVÐWÑWÔWˆŒÝ˜VÔ.Ô/ˆŒˆˆr<   c                 ó¤   — |                       |                      |                      |¦  «        ¦  «        |                      |¦  «        z  ¦  «        S ©N)ra   rc   r_   r`   ©r8   Úhidden_states     r;   rJ   zGlm4VisionMlp.forwardV   s>   € Ø�~Š~˜dŸkšk¨$¯.ª.¸Ñ*FÔ*FÑGÔGÈ$Ï,Ê,ÐWcÑJdÔJdÑdÑeÔeÐer<   ©F)rO   rP   rQ   Úboolr1   rJ   rT   rU   s   @r;   rW   rW   L   s_   ø€ € € € € ð0ð 0 Tð 0ð 0ð 0ð 0ð 0ð 0ðfð fð fð fð fð fð fr<   rW   c                   óL   ‡ — e Zd Zdeddfˆ fd„Zdej        dej        fd„Zˆ xZS )ÚGlm4vVisionPatchEmbedrd   r.   Nc                 ó  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        |j        | _        | j        | j        | j        g}t          j        | j        | j        ||¬¦  «        | _	        d S )N)Úkernel_sizeÚstride)
r0   r1   Ú
patch_sizeÚtemporal_patch_sizeÚin_channelsr9   Ú	embed_dimr2   ÚConv3dÚproj)r8   rd   rn   r:   s      €r;   r1   zGlm4vVisionPatchEmbed.__init__[   sz   ø€ Ý‰Œ×ÒÑÔÐØ Ô+ˆŒØ#)Ô#=ˆÔ Ø!Ô-ˆÔØÔ+ˆŒàÔ/°´À$Ä/ÐRˆÝ”I˜dÔ.°´ÈKÐ`kÐlÑlÔlˆŒ	ˆ	ˆ	r<   r=   c                 ó  — | j         j        j        }|                     d| j        | j        | j        | j        ¦  «        }|                       |                     |¬¦  «        ¦  «                             d| j        ¦  «        }|S )Nr@   ©rB   )	ru   r6   rB   Úviewrr   rq   rp   rC   rs   )r8   r=   Útarget_dtypes      r;   rJ   zGlm4vVisionPatchEmbed.forwarde   sw   € Ø”yÔ'Ô-ˆØ%×*Ò*Ø�Ô  $Ô":¸D¼OÈTÌ_ñ
ô 
ˆð Ÿ	š	 -×"2Ò"2¸Ð"2Ñ"FÔ"FÑGÔG×LÒLÈRÐQUÔQ_Ñ`Ô`ˆØÐr<   ©	rO   rP   rQ   r(   r1   r4   rS   rJ   rT   rU   s   @r;   rl   rl   Z   sz   ø€ € € € € ðmÐ0ð m°Tð mð mð mð mð mð mð U¤\ð °e´lð ð ð ð ð ð ð ð r<   rl   c            	       óè   ‡ — e Zd ZdZ edd¬¦  «        ddefˆ fd„¦   «         Ze edd¬¦  «        ddedee	j
        ef         fd	„¦   «         ¦   «         Z e	j        ¦   «         d
„ ¦   «         Zd„ Zˆ xZS )ÚGlm4vVisionRotaryEmbeddingzý
    Simple axial 2D rope with same freqs used for H and W grids. The freqs are
    pre-computed using `head-dim//4` which is later used to concat H and W positions.
    The final angles rotate over the whole head dim, no partial rotation involved.
    Údeviceú5.18©ÚversionNrd   c                 ó°  •— t          ¦   «                              ¦   «          || _        | j        j        d         | _        | j        }| j        dk    r#t          | j        j        › d| j        › �¦  «        ‚ || j        |¦  «        \  }| _	        t          j        |d¬¦  «        | _        t          j        |                     ¦   «         d¬¦  «        | _        d S )NÚ	rope_typeÚaxialz) supports only axial rope, but requested F©Ú
persistent)r0   r1   rd   Úrope_parametersr‚   Úcompute_axial_rope_parametersÚ
ValueErrorr:   rO   Úattention_scalingr2   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq©r8   rd   r}   Úrope_init_fnr‹   r:   s        €r;   r1   z#Glm4vVisionRotaryEmbedding.__init__u   sÃ   ø€ å‰Œ×ÒÑÔÐØˆŒàœÔ4°[ÔAˆŒØ!%Ô!CˆØŒ>˜WÒ$Ð$Ý ¤Ô 7ÐrÐrÐbfÔbpÐrÐrÑsÔsÐsØ+7¨<¸¼ÀVÑ+LÔ+LÑ(ˆ�$Ô(åœ	 (°uÐ=Ñ=Ô=ˆŒÝ!#¤¨8¯>ª>Ñ+;Ô+;ÈÐ!NÑ!NÔ!NˆÔÐÐr<   r.   c                 óî   — | j         d         }t          | dd¦  «        p| j        | j        z  }|dz  }d}d|t	          j        d|dt          j        ¬¦  «        |z  z  z  }|                     |¦  «        |fS )á¾  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNr?   ç      ð?r   rw   )r†   Úgetattrr9   Únum_attention_headsr4   ÚarangerR   rC   )rd   r}   ÚkwargsÚbaseÚdimÚspatial_dimÚattention_factorr‹   s           r;   r‡   z8Glm4vVisionRotaryEmbedding.compute_axial_rope_parametersƒ   sƒ   € ð Ô% lÔ3ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆØ˜Q‘hˆàÐØ˜$¥5¤<°°;ÀÍÌÐ#UÑ#UÔ#UÐXcÑ#cÑdÑeˆØ�{Š{˜6Ñ"Ô"Ð$4Ð4Ð4r<   c                 ó  — |d                               ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  || j                              ¦   «         z  }|                     ¦   «         | j        z  }| 	                    ¦   «         | j        z  }d d d ¦  «         n# 1 swxY w Y   |  
                    |¦  «        }|  
                    |¦  «        }||fS )N).NÚmpsÚcpuF©Údevice_typeÚenabled)rR   Ú
isinstancer}   ÚtypeÚstrr    r‹   Úcosr‰   ÚsinÚrecomposition_frequencies)r8   ÚxÚposition_idsÚposition_ids_expandedr¡   Úfreqsr¦   r§   s           r;   rJ   z"Glm4vVisionRotaryEmbedding.forward—   s&  € ð !-¨YÔ 7× =Ò =Ñ ?Ô ?ÐÝ'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆÝ¨¸UÐCÑCÔCð 	7ð 	7Ø)¨D¬M×,?Ò,?Ñ,AÔ,AÑAˆEØ—)’)‘+”+ Ô 6Ñ6ˆCØ—)’)‘+”+ Ô 6Ñ6ˆCð	7ð 	7ð 	7ñ 	7ô 	7ð 	7ð 	7ð 	7ð 	7ð 	7ð 	7øøøð 	7ð 	7ð 	7ð 	7ð
 ×,Ò,¨SÑ1Ô1ˆØ×,Ò,¨SÑ1Ô1ˆØ�Cˆxˆs   Á)AC
Ã
CÃCc                 ó’   — |dd…df         |dd…df         }}t          j        ||gd¬¦  «        }t          j        ||gd¬¦  «        S )ú]
        Recompose the frequencies into the final spatial layout used per each grid.
        Nr   r%   r@   ©rš   )r4   Úcat)r8   ÚfreqÚfreq_hÚfreq_wÚfreq_hws        r;   r¨   z4Glm4vVisionRotaryEmbedding.recomposition_frequencies¥   sX   € ð ˜a˜a˜a ˜dœ T¨!¨!¨!¨Q¨$¤Z�ˆÝ”)˜V VÐ,°"Ð5Ñ5Ô5ˆÝŒy˜' 7Ð+°Ð4Ñ4Ô4Ð4r<   rf   )rO   rP   rQ   Ú__doc__r   r(   r1   ÚstaticmethodrL   r4   rS   rR   r‡   Úno_gradrJ   r¨   rT   rU   s   @r;   r|   r|   n   s  ø€ € € € € ðð ð €_�X vÐ.Ñ.Ô.ðOð OÐ0ð Oð Oð Oð Oð Oñ /Ô.ðOð Ø€_�X vÐ.Ñ.Ô.ð5ð 5Ð.?ð 5Ð[`ÐafÔamÐotÐatÔ[uð 5ð 5ð 5ñ /Ô.ñ „\ð5ð$ €U„]�_„_ðð ñ „_ðð5ð 5ð 5ð 5ð 5ð 5ð 5r<   r|   c                   óZ   ‡ — e Zd Zddededededdf
ˆ fd„Zd	ej        dej        fd
„Z	ˆ xZ
S )ÚGlm4vVisionPatchMergerFrš   Úcontext_dimrb   rX   r.   Nc                 ó¤  •— t          ¦   «                              ¦   «          t          j        |||¬¦  «        | _        t          |¦  «        | _        t          j        |||¬¦  «        | _        t          j        |||¬¦  «        | _        t          j        |||¬¦  «        | _	        t          j
        ¦   «         | _        t          |         | _        d S rZ   )r0   r1   r2   r^   ru   r   Úpost_projection_normr_   r`   ra   ÚGELUÚact1r   rc   )r8   rš   rº   rb   rX   r:   s        €r;   r1   zGlm4vVisionPatchMerger.__init__¯   s¤   ø€ Ý‰Œ×ÒÑÔÐÝ”I˜c 3¨TÐ2Ñ2Ô2ˆŒ	Ý$-¨c¡N¤NˆÔ!Ýœ 3¨¸$Ð?Ñ?Ô?ˆŒÝ”y  k¸Ð=Ñ=Ô=ˆŒÝœ ;°¸$Ð?Ñ?Ô?ˆŒÝ”G‘I”IˆŒ	Ý˜ZÔ(ˆŒˆˆr<   rh   c                 ó  — |                       |¦  «        }|                      |                      |¦  «        ¦  «        }|                      |                      |                      |¦  «        ¦  «        |                      |¦  «        z  ¦  «        S rf   )ru   r¾   r¼   ra   rc   r_   r`   rg   s     r;   rJ   zGlm4vVisionPatchMerger.forward¹   sn   € Ø—y’y Ñ.Ô.ˆØ—y’y ×!:Ò!:¸<Ñ!HÔ!HÑIÔIˆØ�~Š~˜dŸkšk¨$¯.ª.¸Ñ*FÔ*FÑGÔGÈ$Ï,Ê,ÐWcÑJdÔJdÑdÑeÔeÐer<   ri   )rO   rP   rQ   Úintr¥   rj   r1   r4   rS   rJ   rT   rU   s   @r;   r¹   r¹   ®   s–   ø€ € € € € ð)ð )˜Cð )¨cð )¸sð )È$ð )Ð[_ð )ð )ð )ð )ð )ð )ðf E¤Lð f°U´\ð fð fð fð fð fð fð fð fr<   r¹   c                   ó:   ‡ — e Zd Zdefˆ fd„Zdej        fd„Zˆ xZS )ÚGlm4vVisionEmbeddingsrd   c                 ó:  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        |j        | _        | j        | j        z  dz  | _        | j        | _        t          j
        | j        | j        ¦  «        | _        d| _        d S )Nr?   Úbicubic)r0   r1   rd   r9   rs   Ú
image_sizerp   Únum_patchesÚnum_positionsr2   Ú	EmbeddingÚposition_embeddingÚinterpolated_method©r8   rd   r:   s     €r;   r1   zGlm4vVisionEmbeddings.__init__À   s†   ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ+ˆŒØ Ô+ˆŒØ Ô+ˆŒà œO¨t¬Ñ>À1ÑDˆÔØ!Ô-ˆÔÝ"$¤,¨tÔ/AÀ4Ä>Ñ"RÔ"RˆÔØ#,ˆÔ Ð Ð r<   r.   c                 ó  — | j         j        }|j        d         }|j        }t	          |t
          ¦  «        r!t          j        ||t          j        ¬¦  «        }|j        d         }	t          |	dz  ¦  «        }
| 
                    |
|
|¦  «                             ddd¦  «                             d¦  «                             |t          j        ¬¦  «        }|j        d         }t          j        ||j        ¬¦  «        }|                     d¦  «        |                     d¦  «                             d¦  «        k                         d¦  «        }||df                              t          j        ¬¦  «        }||df                              t          j        ¬¦  «        }|dz   |z  dz  dz
  }|dz   |z  dz  dz
  }t          j        ||fd¬	¦  «                             d¦  «                             d¦  «        }t'          j        ||| j        d
d¬¦  «        }|                     d¦  «                             d¦  «                             dd¦  «        }|                     |j        ¦  «                             |j        ¦  «        }||z   }|S )a¡  
        Forward pass with integrated position encoding adaptation using 2D interpolation.

        Args:
            embeddings: Input embeddings tensor
            lengths (torch.Tensor): Sequence lengths for each image in the batch.
            image_shapes (torch.Tensor): Tensor of shape [batch_size, 3] representing the image shapes (t, h, w).
            h_coords (torch.Tensor): Tensor of shape [total_seq] representing the h coordinate for each patch.
            w_coords (torch.Tensor): Tensor of shape [total_seq] representing the w coordinate for each patch.

        Returns:
            torch.Tensor: Embeddings with adapted position encoding added.
        r%   )r}   rB   r   g      à?r?   ©r}   rw   r@   r¯   FÚborder)ÚmodeÚalign_cornersÚpadding_mode)rÉ   r6   rM   r}   r£   Úlistr4   ÚtensorÚlongrÀ   rx   ÚpermuteÚ	unsqueezerC   rD   r—   ÚcumsumÚsumÚstackÚFÚgrid_samplerÊ   ÚsqueezerB   )r8   Ú
embeddingsÚlengthsÚimage_shapesÚh_coordsÚw_coordsÚpos_embed_weightr9   r}   Úorig_size_sqÚ	orig_sizeÚpos_embed_2dÚ
num_tokensÚtoken_positionsÚseq_idsÚtarget_hÚtarget_wÚnorm_wÚnorm_hÚgridÚinterpolated_embed_fp32Úadapted_pos_embed_fp32Úadapted_pos_embeds                          r;   rJ   zGlm4vVisionEmbeddings.forwardÌ   si  € ð  Ô2Ô9ÐØ&Ô,¨QÔ/ˆØ!Ô(ˆõ �g�tÑ$Ô$ð 	MÝ”l 7°6ÅÄÐLÑLÔLˆGð (Ô-¨aÔ0ˆÝ˜ cÑ)Ñ*Ô*ˆ	à×!Ò! )¨Y¸ÑDÔDßŠW�Q˜˜1ÑÔßŠY�q‰\Œ\ßŠR�v¥U¤]ˆRÑ3Ô3ð	 	ð  Ô% aÔ(ˆ
Ýœ, z¸*Ô:KÐLÑLÔLˆØ"×,Ò,¨QÑ/Ô/°7·>²>À!Ñ3DÔ3D×3NÒ3NÈqÑ3QÔ3QÒQ×VÒVÐWXÑYÔYˆØ ¨ 
Ô+×.Ò.µU´]Ð.ÑCÔCˆØ ¨ 
Ô+×.Ò.µU´]Ð.ÑCÔCˆð ˜c‘> XÑ-°Ñ2°QÑ6ˆØ˜c‘> XÑ-°Ñ2°QÑ6ˆõ Œ{˜F FÐ+°Ð4Ñ4Ô4×>Ò>¸qÑAÔA×KÒKÈAÑNÔNˆõ #$¤-Ø˜$ TÔ%=ÈUÐaið#
ñ #
ô #
Ðð
 "9×!@Ò!@ÀÑ!CÔ!C×!KÒ!KÈBÑ!OÔ!O×!WÒ!WÐXYÐ[\Ñ!]Ô!]ÐØ2×5Ò5Ð6FÔ6LÑMÔM×PÒPÐQ[ÔQbÑcÔcÐð  Ð"3Ñ3ˆ
ØÐr<   rz   rU   s   @r;   rÂ   rÂ   ¿   sd   ø€ € € € € ð
-Ð0ð 
-ð 
-ð 
-ð 
-ð 
-ð 
-ð:ÐPUÔP\ð :ð :ð :ð :ð :ð :ð :ð :r<   rÂ   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )ú*Rotates half the hidden dims of the input..Nr@   r?   r¯   )rM   r4   r°   ©r©   Úx1Úx2s      r;   Úrotate_halfrö   	  s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'r<   ÚqÚkr¦   r§   r.   c                 óÆ  — | j         }|j         }|                      ¦   «         |                     ¦   «         }} |                     d¦  «                             ¦   «         |                     d¦  «                             ¦   «         }}| |z  t          | ¦  «        |z  z   }||z  t          |¦  «        |z  z   }|                     |¦  «        }|                     |¦  «        }||fS )Néþÿÿÿ)rB   rR   rÖ   rö   rC   )r÷   rø   r¦   r§   Úorig_q_dtypeÚorig_k_dtypeÚq_embedÚk_embeds           r;   Úapply_rotary_pos_emb_visionrÿ     sÃ   € ð ”7€LØ”7€LØ�7Š7‰9Œ9�a—g’g‘i”i€q€AØ�}Š}˜RÑ Ô ×&Ò&Ñ(Ô(¨#¯-ª-¸Ñ*;Ô*;×*AÒ*AÑ*CÔ*Cˆ€CØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�jŠj˜Ñ&Ô&€GØ�jŠj˜Ñ&Ô&€GØ�GÐÐr<   r=   Ún_repc                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r%   N)rM   ÚexpandÚreshape)r=   r   ÚbatchÚnum_key_value_headsÚslenr“   s         r;   Ú	repeat_kvr    s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTr<   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutr˜   c                 ó  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        |z  }
|�|
|z   }
t
          j                             |
dt          j        ¬¦  «         	                    |j
        ¦  «        }
t
          j                             |
|| j        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «                             ¦   «         }||
fS )Nr?   r   r@   )rš   rB   )ÚpÚtrainingr%   )r  Únum_key_value_groupsr4   ÚmatmulÚ	transposer2   Ú
functionalÚsoftmaxrD   rC   rB   r  r  Ú
contiguous)r	  r
  r  r  r  r  r  r˜   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r;   Úeager_attention_forwardr  *  sé   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r<   c                   ó˜   ‡ — e Zd Zdeddfˆ fd„Z	 	 d
dej        dej        deej        ej        f         dz  dedz  dej        f
d	„Z	ˆ xZ
S )ÚGlm4vVisionAttentionrd   r.   Nc                 ó¨  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        | j        | j        z  | _        d| _        t          j        |j        |j        dz  |j	        ¬¦  «        | _
        t          j        |j        |j        d¬¦  «        | _        | j        dz  | _        || _        |j        | _        d| _        d S )Nr%   r   r[   Fç      à¿)r0   r1   r9   rš   Ú	num_headsr“   r  r2   r^   Úattention_biasÚqkvru   r  rd   Úattention_dropoutÚ	is_causalrË   s     €r;   r1   zGlm4vVisionAttention.__init__D  s´   ø€ Ý‰Œ×ÒÑÔÐØÔ%ˆŒØÔ)ˆŒØœ D¤NÑ2ˆŒØ$%ˆÔ!Ý”9˜VÔ/°Ô1CÀaÑ1GÈfÔNcÐdÑdÔdˆŒÝ”I˜fÔ0°&Ô2DÈ5ÐQÑQÔQˆŒ	Ø”} dÑ*ˆŒØˆŒØ!'Ô!9ˆÔØˆŒˆˆr<   r=   Ú
cu_seqlensÚposition_embeddingsÚ
max_seqlenc                 ó&  ‡ ‡‡‡— |j         d         }‰                      |¦  «                             |d‰ j        d¦  «                             dddd¦  «                             d¦  «        \  }}}	|\  }
}t          |||
|¦  «        \  }}|                     dd¦  «                             d¦  «        }|                     dd¦  «                             d¦  «        }|	                     dd¦  «                             d¦  «        }	t          j
        ‰ j        j        t          ¦  «        Št          ‰ j        ¦  «        rEt          |‰ j        d|i¬¦  «        } ‰‰ |||	fd ‰ j        ‰ j        sdn‰ j        ||||d	d
œ‰¤Ž\  }}nS|dd …         |d d…         z
  Šˆfd„|||	fD ¦   «         }ˆˆˆ fd„t'          |Ž D ¦   «         }t)          j        |d¬¦  «        }|                     |d¦  «                             ¦   «         }‰                      |¦  «        }|S )Nr   r   r@   r%   r?   r)  ©r˜   r  F)r  r  r  Úcu_seq_lens_qÚcu_seq_lens_kÚmax_length_qÚmax_length_kr&  c                 ób   •— g | ]+}t          j        |‰                     ¦   «         d ¬¦  «        ‘Œ,S )r?   r¯   )r4   ÚsplitÚtolist)Ú.0rÓ   rÞ   s     €r;   ú
<listcomp>z0Glm4vVisionAttention.forward.<locals>.<listcomp>}  sA   ø€ ð ð ð ØAG•”˜F G§N¢NÑ$4Ô$4¸!Ð<Ñ<Ô<ðð ð r<   c           
      ól   •— g | ]0\  }}} ‰‰|||fd ‰j         ‰j        sdn‰j        ddœ‰¤Žd         ‘Œ1S )Nr  F)r  r  r  r&  r   )r  r  r%  )r3  r÷   rø   ÚvÚattention_interfacer˜   r8   s       €€€r;   r4  z0Glm4vVisionAttention.forward.<locals>.<listcomp>�  s†   ø€ ð ð ð ñ �A�q˜!ð $Ð#ØØØØð	
ð
 $(Ø œLØ'+¤}ÐP˜C˜C¸$Ô:PØ#ð
ð 
ð ð
ð 
ð ô
ðð ð r<   r¯   )rM   r$  r  r"  rÕ   Úunbindrÿ   r  rÖ   r   Úget_interfacerd   Ú_attn_implementationr  r   r   r  r  r%  Úzipr4   r°   r  ru   )r8   r=   r'  r(  r)  r˜   Ú
seq_lengthÚquery_statesr  r  r¦   r§   r  Ú_ÚsplitsÚattn_outputsr7  rÞ   s   `    `          @@r;   rJ   zGlm4vVisionAttention.forwardQ  s}  øøøø€ ð #Ô(¨Ô+ˆ
à�HŠH�]Ñ#Ô#×+Ò+¨J¸¸4¼>È2ÑNÔN×VÒVÐWXÐZ[Ð]^Ð`aÑbÔb×iÒiÐjkÑlÔlñ 	/ˆ�j ,ð '‰ˆˆSÝ#>¸|ÈZÐY\Ð^aÑ#bÔ#bÑ ˆ�jà#×-Ò-¨a°Ñ3Ô3×=Ò=¸aÑ@Ô@ˆØ×)Ò)¨!¨QÑ/Ô/×9Ò9¸!Ñ<Ô<ˆ
Ø#×-Ò-¨a°Ñ3Ô3×=Ò=¸aÑ@Ô@ˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðõ (¨¬Ñ4Ô4ð '	9å'¨
°D´KÈÐWaÐHbÐcÑcÔcˆJØ0Ð0ØØØØð	ð
  $ØœØ#'¤=ÐL˜˜°dÔ6LØ(Ø(Ø'Ø'Øðð ð ðð ‰NˆK˜˜ð" !   ”n z°#°2°#¤Ñ6ˆGðð ð ð ØLXÐZdÐfrÐKsðñ ô ˆFðð ð ð ð ð õ  # F˜|ðñ ô ˆLõ  œ) L°aÐ8Ñ8Ô8ˆKà!×)Ò)¨*°bÑ9Ô9×DÒDÑFÔFˆØ—i’i Ñ,Ô,ˆØÐr<   ©NN)rO   rP   rQ   r(   r1   r4   rS   rL   rÀ   rJ   rT   rU   s   @r;   r  r  C  sÊ   ø€ € € € € ðÐ0ð °Tð ð ð ð ð ð ð" IMØ!%ðBð Bà”|ðBð ”LðBð # 5¤<°´Ð#=Ô>ÀÑEð	Bð
 ˜$‘JðBð 
ŒðBð Bð Bð Bð Bð Bð Bð Br<   r  c            
       ó”   ‡ — e Zd Zdˆ fd„Ze	 d	dej        dej        deej        ej        f         dz  dej        fd„¦   «         Zˆ xZ	S )
ÚGlm4vVisionBlockr.   Nc                 ó  •— t          ¦   «                              ¦   «          t          |j        |j        ¬¦  «        | _        t          |j        |j        ¬¦  «        | _        t          |¦  «        | _        t          |d¬¦  «        | _
        d S )N©r-   Fr[   )r0   r1   r+   r9   Úrms_norm_epsÚnorm1Únorm2r  ÚattnrW   ÚmlprË   s     €r;   r1   zGlm4vVisionBlock.__init__—  st   ø€ Ý‰Œ×ÒÑÔÐÝ! &Ô"4¸&Ô:MÐNÑNÔNˆŒ
Ý! &Ô"4¸&Ô:MÐNÑNÔNˆŒ
Ý(¨Ñ0Ô0ˆŒ	Ý  ¨eÐ4Ñ4Ô4ˆŒˆˆr<   r=   r'  r(  c                 óª   — | | j         |                      |¦  «        f||dœ|¤Žz   }||                      |                      |¦  «        ¦  «        z   }|S )zœ
        cu_seqlens (`torch.Tensor`):
            Cumulative sequence lengths used for packed variable-length attention in Flash Attention kernels.
        )r'  r(  )rI  rG  rJ  rH  )r8   r=   r'  r(  r˜   s        r;   rJ   zGlm4vVisionBlock.forwardž  sq   € ð &¨	¨¬	Ø�JŠJ�}Ñ%Ô%ð)
à!Ø 3ð)
ð )
ð ð	)
ð )
ñ 
ˆð &¨¯ª°·²¸MÑ1JÔ1JÑ(KÔ(KÑKˆØÐr<   ©r.   Nrf   )
rO   rP   rQ   r1   r   r4   rS   rL   rJ   rT   rU   s   @r;   rC  rC  –  s¨   ø€ € € € € ð5ð 5ð 5ð 5ð 5ð 5ð ð
 IMð	ð à”|ðð ”Lðð # 5¤<°´Ð#=Ô>ÀÑEð	ð 
Œðð ð ñ „^ðð ð ð ð r<   rC  c            	       óô   ‡ — e Zd Z edd¬¦  «        ddefˆ fd„¦   «         Ze edd¬¦  «        ddedee	j
        ef         fd„¦   «         ¦   «         Z e	j        ¦   «         ed	„ ¦   «         ¦   «         Zd
„ Zˆ xZS )ÚGlm4vTextRotaryEmbeddingr}   r~   r   Nrd   c                 ó  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        | j        j        d         | _        | j        }| j        dk    rt          | j                 } || j        |¦  «        \  }| _
        t          j        |d¬¦  «        | _        t          j        |                     ¦   «         d¬¦  «        | _        |j                             dg d¢¦  «        | _        d S )Nr‚   ÚdefaultFr„   Úmrope_section)é   é   rS  )r0   r1   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrd   r†   r‚   Úcompute_default_rope_parametersr   r‰   r2   rŠ   r‹   rŒ   r�   ÚgetrQ  rŽ   s        €r;   r1   z!Glm4vTextRotaryEmbedding.__init__µ  sß   ø€ å‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!àˆŒàœÔ4°[ÔAˆŒØ!%Ô!EˆØŒ>˜YÒ&Ð&Ý.¨t¬~Ô>ˆLØ+7¨<¸¼ÀVÑ+LÔ+LÑ(ˆ�$Ô(åœ	 (°uÐ=Ñ=Ô=ˆŒÝ!#¤¨8¯>ª>Ñ+;Ô+;ÈÐ!NÑ!NÔ!NˆÔØ#Ô3×7Ò7¸ÈÈÈÑUÔUˆÔÐÐr<   r.   c                 ó>  — | j         d         }| j                              dd¦  «        }t          | dd¦  «        p| j        | j        z  }t          ||z  ¦  «        }d}d|t          j        d|dt          j        ¬¦  «        |z  z  z  }| 	                    |¦  «        |fS )	r‘   r’   Úpartial_rotary_factorr”   r“   Nr   r?   rw   )
r†   rX  r•   r9   r–   rÀ   r4   r—   rR   rC   )	rd   r}   r˜   r™   rZ  r“   rš   rœ   r‹   s	            r;   rW  z8Glm4vTextRotaryEmbedding.compute_default_rope_parametersÇ  s¨   € ð Ô% lÔ3ˆØ &Ô 6× :Ò :Ð;RÐTWÑ XÔ XÐÝ˜6 :¨tÑ4Ô4Ðh¸Ô8JÈfÔNhÑ8hˆÝ�(Ð2Ñ2Ñ3Ô3ˆàÐà˜$¥5¤<°°3¸ÅÄÐ#MÑ#MÔ#MÐPSÑ#SÑTÑUˆØ�{Š{˜6Ñ"Ô"Ð$4Ð4Ð4r<   c                 óL  — | j         d d d d …d f                              ¦   «                              d|j        d         dd¦  «        }|d d …d d …d d d …f                              ¦   «         }t	          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  |                     ¦   «         |                     ¦   «         z   	                    dd¦  «        }| 
                    ¦   «         | j        z  }|                     ¦   «         | j        z  }d d d ¦  «         n# 1 swxY w Y   |                      |¦  «        }|                      |¦  «        }|                     |j        ¬	¦  «        |                     |j        ¬	¦  «        fS )
Nr   r%   r@   rž   rŸ   Fr    r?   rw   )r‹   rR   r  rM   r£   r}   r¤   r¥   r    r  r¦   r‰   r§   r¨   rC   rB   )	r8   r©   rª   Úinv_freq_expandedr«   r¡   r¬   r¦   r§   s	            r;   rJ   z Glm4vTextRotaryEmbedding.forwardÝ  s¿  € ð
 !œM¨$°°a°a°a¸Ð*=Ô>×DÒDÑFÔF×MÒMÈaÐQ]ÔQcÐdeÔQfÐhjÐlmÑnÔnÐØ ,¨Q¨Q¨Q°°°°4¸¸¸¨]Ô ;× AÒ AÑ CÔ CÐå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆÝ¨¸UÐCÑCÔCð 	7ð 	7Ø&×,Ò,Ñ.Ô.Ð1F×1LÒ1LÑ1NÔ1NÑN×YÒYÐZ[Ð]^Ñ_Ô_ˆEØ—)’)‘+”+ Ô 6Ñ6ˆCØ—)’)‘+”+ Ô 6Ñ6ˆCð	7ð 	7ð 	7ñ 	7ô 	7ð 	7ð 	7ð 	7ð 	7ð 	7ð 	7øøøð 	7ð 	7ð 	7ð 	7ð
 ×,Ò,¨SÑ1Ô1ˆØ×,Ò,¨SÑ1Ô1ˆØ�vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   Â9A6D;Ä;D?ÅD?c           	      ó¾   — t          j        d„ t          |                     | j        d¬¦  «        ¦  «        D ¦   «         d¬¦  «        }|                     dd¬¦  «        S )r®   c                 ó*   — g | ]\  }}||d z           ‘ŒS )r   © )r3  ÚiÚms      r;   r4  zFGlm4vTextRotaryEmbedding.recomposition_frequencies.<locals>.<listcomp>ó  s$   € ÐaÐaÐa¡t q¨!˜!˜A ™Eœ(ÐaÐaÐar<   r@   r¯   r?   )r4   r°   Ú	enumerater1  rQ  Úrepeat_interleave)r8   r±   s     r;   r¨   z2Glm4vTextRotaryEmbedding.recomposition_frequenciesï  s]   € õ ŒyÐaÐa­y¸¿ºÀDÔDVÐ\^¸Ñ9_Ô9_Ñ/`Ô/`ÐaÑaÔaÐgiÐjÑjÔjˆØ×%Ò% a¨RÐ%Ñ0Ô0Ð0r<   rf   )rO   rP   rQ   r   r'   r1   r¶   r&   rL   r4   rS   rR   rW  r·   r   rJ   r¨   rT   rU   s   @r;   rN  rN  ´  s  ø€ € € € € Ø€_�X vÐ.Ñ.Ô.ðVð V˜ð Vð Vð Vð Vð Vñ /Ô.ðVð" Ø€_�X vÐ.Ñ.Ô.ð5ð 5°ð 5ÐW\Ð]bÔ]iÐkpÐ]pÔWqð 5ð 5ð 5ñ /Ô.ñ „\ð5ð( €U„]�_„_Øð<ð <ñ Ôñ „_ð<ð 1ð 1ð 1ð 1ð 1ð 1ð 1r<   rN  c                 óŽ   — | dddd…f         }| dddd…f         }t          j        | |fd¬¦  «                             d¦  «        S )	rò   .r   Nr?   r%   r@   r¯   rú   )r4   rÙ   Úflattenró   s      r;   Úrotate_half_llmrf  ÷  sQ   € à	
ˆ3���1�ˆ9Œ€BØ	
ˆ3���1�ˆ9Œ€BÝŒ;˜˜˜R�y bÐ)Ñ)Ô)×1Ò1°"Ñ5Ô5Ð5r<   c                 ó˜  — |                      |¦  «        }|                      |¦  «        }|j        d         }| dd|…f         | d|d…f         }}|dd|…f         |d|d…f         }	}||z  t          |¦  «        |z  z   }
||z  t          |¦  «        |z  z   }t          j        |
|gd¬¦  «        }
t          j        ||	gd¬¦  «        }|
|fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    r@   .Nr¯   )rÖ   rM   rf  r4   r°   )r÷   rø   r¦   r§   Úunsqueeze_dimÚ
rotary_dimÚq_rotÚq_passÚk_rotÚk_passrý   rþ   s               r;   Úapply_rotary_pos_embrn  þ  sô   € ð$ �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€Cð ”˜2”€JØ�c˜;˜J˜;Ð&Ô'¨¨3°
°°Ð+;Ô)<ˆ6€EØ�c˜;˜J˜;Ð&Ô'¨¨3°
°°Ð+;Ô)<ˆ6€Eð �s‰{�¨uÑ5Ô5¸Ñ;Ñ<€GØ�s‰{�¨uÑ5Ô5¸Ñ;Ñ<€Gõ Œi˜ &Ð)¨rÐ2Ñ2Ô2€GÝŒi˜ &Ð)¨rÐ2Ñ2Ô2€GØ�GÐÐr<   c                   óú   ‡ — e Zd ZdZddededz  fˆ fd„Z	 	 	 ddej        de	ej        ej        f         dz  dej        dz  d	e
dz  d
ee         de	ej        ej        dz  e	ej                 dz  f         fd„Zˆ xZS )ÚGlm4vTextAttentionz†
    Multi-headed attention from 'Attention Is All You Need' paper.
    and "Generating Long Sequences with Sparse Transformers".
    Nrd   Ú	layer_idxc                 óÈ  •— t          ¦   «                              ¦   «          || _        || _        |j        | _        |j        | _        | j        | j        z  | _        |j        | _        | j        | j        z  | _	        d| _
        |j        | _        |j        | _        | j        dz  | _        t          j        | j        | j        | j        z  d¬¦  «        | _        t          j        | j        | j        | j        z  d¬¦  «        | _        t          j        | j        | j        | j        z  d¬¦  «        | _        t          j        | j        | j        z  | j        d¬¦  «        | _        d S )NTr!  r[   F)r0   r1   rd   rq  r9   r–   r"  r“   r  r  r&  r%  r†   r  r2   r^   Úq_projÚk_projÚv_projÚo_proj©r8   rd   rq  r:   s      €r;   r1   zGlm4vTextAttention.__init__(  s/  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒà!Ô-ˆÔØÔ3ˆŒØÔ(¨D¬NÑ:ˆŒØ#)Ô#=ˆÔ Ø$(¤N°dÔ6NÑ$NˆÔ!ØˆŒØ!'Ô!9ˆÔØ%Ô5ˆÔØ”} dÑ*ˆŒå”i Ô 0°$´.À4Ä=Ñ2PÐW[Ð\Ñ\Ô\ˆŒÝ”i Ô 0°$Ô2JÈTÌ]Ñ2ZÐaeÐfÑfÔfˆŒÝ”i Ô 0°$Ô2JÈTÌ]Ñ2ZÐaeÐfÑfÔfˆŒÝ”i ¤°´Ñ >ÀÔ@PÐW\Ð]Ñ]Ô]ˆŒˆˆr<   r=   r(  r  Úpast_key_valuesr˜   r.   c                 ód  — |                      ¦   «         \  }}}|                      |¦  «        }	|                      |¦  «        }
|                      |¦  «        }|	                     ||d| j        ¦  «                             dd¦  «        }	|
                     ||d| j        ¦  «                             dd¦  «        }
|                     ||d| j        ¦  «                             dd¦  «        }|\  }}t          |	|
||¦  «        \  }	}
|�|                     |
|| j	        ¦  «        \  }
}t          j        | j        j        t          ¦  «        } || |	|
||f| j        sdn| j        | j        dœ|¤Ž\  }}|                     ||d¦  «                             ¦   «         }|                      |¦  «        }||fS )Nr@   r%   r?   r  )r  r  )Úsizers  rt  ru  rx   r“   r  rn  Úupdaterq  r   r9  rd   r:  r  r  r%  r  r  r  rv  )r8   r=   r(  r  rx  r˜   ÚbszÚq_lenr>  r=  r  r  r¦   r§   r7  r  r  s                    r;   rJ   zGlm4vTextAttention.forward<  sÎ  € ð &×*Ò*Ñ,Ô,‰ˆˆU�Aà—{’{ =Ñ1Ô1ˆØ—[’[ Ñ/Ô/ˆ
Ø—{’{ =Ñ1Ô1ˆà#×(Ò(¨¨e°R¸¼ÑGÔG×QÒQÐRSÐUVÑWÔWˆØ—_’_ S¨%°°T´]ÑCÔC×MÒMÈaÐQRÑSÔSˆ
Ø#×(Ò(¨¨e°R¸¼ÑGÔG×QÒQÐRSÐUVÑWÔWˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð "×)Ò)¨#¨u°bÑ9Ô9×DÒDÑFÔFˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r<   rf   ©NNN)rO   rP   rQ   rµ   r'   rÀ   r1   r4   rS   rL   r	   r   r   rJ   rT   rU   s   @r;   rp  rp  "  s  ø€ € € € € ðð ð
^ð ^˜ð ^¸3À¹:ð ^ð ^ð ^ð ^ð ^ð ^ð. IMØ.2Ø(,ð))ð ))à”|ð))ð # 5¤<°´Ð#=Ô>ÀÑEð))ð œ tÑ+ð	))ð
  ™ð))ð Ð-Ô.ð))ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	Mð))ð ))ð ))ð ))ð ))ð ))ð ))ð ))r<   rp  c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚGlm4vTextMLPc                 ó"  •— t          ¦   «                              ¦   «          || _        t          j        |j        d|j        z  d¬¦  «        | _        t          j        |j        |j        d¬¦  «        | _        t          |j
                 | _        d S )Nr?   Fr[   )r0   r1   rd   r2   r^   r9   r]   Úgate_up_projra   r   rb   Úactivation_fnrË   s     €r;   r1   zGlm4vTextMLP.__init__i  sz   ø€ Ý‰Œ×ÒÑÔÐàˆŒÝœI fÔ&8¸!¸fÔ>VÑ:VÐ]bÐcÑcÔcˆÔÝœ 6Ô#;¸VÔ=OÐV[Ð\Ñ\Ô\ˆŒÝ# FÔ$5Ô6ˆÔÐÐr<   r=   r.   c                 óº   — |                       |¦  «        }|                     dd¬¦  «        \  }}||                      |¦  «        z  }|                      |¦  «        S )Nr?   r@   r¯   )r‚  Úchunkrƒ  ra   )r8   r=   Ú	up_statesÚgates       r;   rJ   zGlm4vTextMLP.forwardq  sX   € Ø×%Ò% mÑ4Ô4ˆ	à#Ÿ/š/¨!°˜/Ñ4Ô4‰ˆˆiØ × 2Ò 2°4Ñ 8Ô 8Ñ8ˆ	à�~Š~˜iÑ(Ô(Ð(r<   )rO   rP   rQ   r1   r4   ÚFloatTensorrJ   rT   rU   s   @r;   r€  r€  h  s`   ø€ € € € € ð7ð 7ð 7ð 7ð 7ð) UÔ%6ð )¸5Ô;Lð )ð )ð )ð )ð )ð )ð )ð )r<   r€  c                   ó  ‡ — e Zd Zdedefˆ fd„Ze	 	 	 	 	 ddej        de	ej        ej        f         dz  dej        dz  d	ej
        dz  d
edz  dedz  de	ej        e	ej        ej        f         dz  f         fd„¦   «         Zˆ xZS )ÚGlm4vTextDecoderLayerrd   rq  c                 ó²  •— t          ¦   «                              ¦   «          |j        | _        t          ||¦  «        | _        t          |¦  «        | _        t          |j        |j        ¬¦  «        | _	        t          |j        |j        ¬¦  «        | _
        t          |j        |j        ¬¦  «        | _        t          |j        |j        ¬¦  «        | _        d S )NrE  )r0   r1   r9   rp  Ú	self_attnr€  rJ  r+   rF  Úinput_layernormÚpost_attention_layernormÚpost_self_attn_layernormÚpost_mlp_layernormrw  s      €r;   r1   zGlm4vTextDecoderLayer.__init__{  s¶   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔÝ+¨F°IÑ>Ô>ˆŒÝ Ñ'Ô'ˆŒÝ+¨FÔ,>ÀFÔDWÐXÑXÔXˆÔÝ(4°VÔ5GÈVÔM`Ð(aÑ(aÔ(aˆÔ%Ý(4°VÔ5GÈVÔM`Ð(aÑ(aÔ(aˆÔ%Ý".¨vÔ/AÀvÔGZÐ"[Ñ"[Ô"[ˆÔÐÐr<   NFr=   r(  r  rª   rx  Ú	use_cacher.   c           
      ó"  — |}|                       |¦  «        } | j        d||||||dœ|¤Ž\  }}	|                      |¦  «        }||z   }|}|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }||z   }|S )N)r=   r(  r  rª   rx  r‘  r_  )r�  rŒ  r�  rŽ  rJ  r�  )
r8   r=   r(  r  rª   rx  r‘  r˜   Úresidualr>  s
             r;   rJ   zGlm4vTextDecoderLayer.forward…  sÉ   € ð !ˆà×,Ò,¨]Ñ;Ô;ˆð *˜4œ>ð 
Ø'Ø 3Ø)Ø%Ø+Øð
ð 
ð ð
ð 
Ñˆ�qð ×5Ò5°mÑDÔDˆØ  =Ñ0ˆð !ˆØ×5Ò5°mÑDÔDˆØŸš Ñ/Ô/ˆØ×/Ò/°Ñ>Ô>ˆØ  =Ñ0ˆàÐr<   )NNNNF)rO   rP   rQ   r'   rÀ   r1   r   r4   rS   rL   Ú
LongTensorr	   rj   rˆ  rJ   rT   rU   s   @r;   rŠ  rŠ  z  s  ø€ € € € € ð\˜ð \¸3ð \ð \ð \ð \ð \ð \ð ð IMØ.2Ø04Ø(,Ø!&ð#ð #à”|ð#ð # 5¤<°´Ð#=Ô>ÀÑEð#ð œ tÑ+ð	#ð
 Ô&¨Ñ-ð#ð  ™ð#ð ˜$‘;ð#ð 
ˆuÔ  %¨Ô(9¸5Ô;LÐ(LÔ"MÐPTÑ"TÐTÔ	Uð#ð #ð #ñ „^ð#ð #ð #ð #ð #r<   rŠ  c                   ó2   — e Zd ZU dZdZej        dz  ed<   dS )ÚGlm4vModelOutputWithPastá  
    rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*):
        The rope index difference between sequence length and multimodal rope.
        The attribute is deprecated and will be removed in v5.20, use `model.base_model.rope_deltas` instead.
    NÚrope_deltas©rO   rP   rQ   rµ   r˜  r4   r”  Ú__annotations__r_  r<   r;   r–  r–  ¬  ó6   € € € € € € ðð ð ,0€K�Ô! DÑ(Ð/Ð/Ñ/Ð/Ð/r<   r–  c                   óD   — e Zd ZU eed<   dZdZdZddgZdgZ	dZ
dZdZdZdS )	ÚGlm4vPreTrainedModelrd   Úmodel)ÚimageÚvideoÚtextTrŠ  rC  rx  N)rO   rP   rQ   r&   rš  Úbase_model_prefixÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_can_compile_fullgraphÚ_supports_attention_backendr_  r<   r;   r�  r�  ¸  s\   € € € € € € àÐÐÑØÐØ1ÐØ&*Ð#Ø0Ð2DÐEÐØ#4Ð"5ÐØÐØ€Nà!ÐØ"&ÐÐÐr<   r�  c                   ó®   ‡ — e Zd ZU eed<   dZdgZeedœZ	dˆ fd„Z
eeedej        d	ej        d
ee         deez  fd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚGlm4vVisionModelrd   )rŸ  r   rC  ©r=   Ú
attentionsr.   Nc                 óÆ  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t	          ‰¦  «        | _        t          ‰¦  «        | _        t          ‰¦  «        | _	        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ‰j        ¬¦  «        | _        t)          ‰j        ‰j        ¬¦  «        | _        t          j        ‰j        ‰j        ‰j        ‰j        ¬¦  «        | _        t)          ‰j        ‰j        ¬¦  «        | _        d| _        |                      ¦   «          d S )Nc                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r_  )rC  )r3  r>  rd   s     €r;   r4  z-Glm4vVisionModel.__init__.<locals>.<listcomp>Ú  s"   ø€ Ð$[Ð$[Ð$[À!Õ%5°fÑ%=Ô%=Ð$[Ð$[Ð$[r<   )rš   rº   rb   rE  )rr   Úout_channelsrn   ro   F)r0   r1   Úspatial_merge_sizerp   rÂ   rÝ   rl   Úpatch_embedr|   Úrotary_pos_embr2   Ú
ModuleListÚrangeÚdepthÚblocksr¹   r\   r]   rb   Úmergerr+   r9   rF  Úpost_conv_layernormÚConv2dÚ
downsampleÚpost_layernormÚgradient_checkpointingÚ	post_initrË   s    `€r;   r1   zGlm4vVisionModel.__init__Ð  s?  øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø"(Ô";ˆÔØ Ô+ˆŒå/°Ñ7Ô7ˆŒÝ0°Ñ8Ô8ˆÔå8¸Ñ@Ô@ˆÔå”mÐ$[Ð$[Ð$[Ð$[ÅuÈVÌ\ÑGZÔGZÐ$[Ñ$[Ô$[Ñ\Ô\ˆŒÝ,ØÔ&°FÔ4LÐY_ÔYjð
ñ 
ô 
ˆŒõ $0°Ô0BÈÔH[Ð#\Ñ#\Ô#\ˆÔ Ýœ)ØÔ*ØÔ/ØÔ1ØÔ,ð	
ñ 
ô 
ˆŒõ +¨6Ô+=À6ÔCVÐWÑWÔWˆÔà&+ˆÔ#Ø�ŠÑÔÐÐÐr<   r=   Úgrid_thwr˜   c           	      ó\  — t          || j        |¬¦  «        }t          || j        |¬¦  «        \  }}|                      |¦  «        }|                      |¦  «        }|                      ||¦  «        }|dd…         |dd…         z
  }|                      ||||dd…df                              |j	        ¦  «        |dd…df                              |j	        ¦  «        ¦  «        }| j
        D ]}	 |	|f|||dœ|¤Ž}Œ|                      |¦  «        }|                     d| j        | j        |j        d         ¦  «        }|                     dddd¦  «        }|                      |¦  «                             d| j        j        ¦  «        }|                      |¦  «        }
t%          ||
¬	¦  «        S )
a\  
        hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`):
            The final hidden states of the model.
        grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`):
            The temporal, height and width of feature shape of each image in LLM.

        Returns:
            `torch.Tensor`: hidden_states.
        r+  r%   Nr@   r   )r'  r)  r(  r   r?   )Úlast_hidden_stateÚpooler_output)r$   r²  r#   rd   r³  rº  r´  rÝ   rC   r}   r¸  r½  rx   rM   rÕ   r¼  r\   r¹  r   )r8   r=   rÀ  r˜   rª   r'  r)  r(  ÚseqlensÚblkÚmerged_hidden_statess              r;   rJ   zGlm4vVisionModel.forwardë  sä  € õ /¨x¸Ô9PÐY_Ð`Ñ`Ô`ˆÝ!=¸hÈÌÐ\bÐ!cÑ!cÔ!cÑˆ
�Jà×(Ò(¨Ñ7Ô7ˆØ×0Ò0°Ñ?Ô?ˆØ"×1Ò1°-ÀÑNÔNÐà˜Q˜R˜R”. :¨c¨r¨c¤?Ñ2ˆØŸšØØØØ˜˜˜˜A˜Ô×!Ò! -Ô"6Ñ7Ô7Ø˜˜˜˜A˜Ô×!Ò! -Ô"6Ñ7Ô7ñ
ô 
ˆð ”;ð 	ð 	ˆCØ˜CØðà%Ø%Ø$7ð	ð ð
 ðð ˆMˆMð ×+Ò+¨MÑ:Ô:ˆà%×*Ò*Ø�Ô'¨Ô)@À-ÔBUÐVXÔBYñ
ô 
ˆð &×-Ò-¨a°°A°qÑ9Ô9ˆØŸš¨Ñ6Ô6×;Ò;¸BÀÄÔ@[Ñ\Ô\ˆà#Ÿ{š{¨=Ñ9Ô9Ðå)Ø+Ø.ð
ñ 
ô 
ð 	
r<   rL  )rO   rP   rQ   r(   rš  r£  r¥  rC  r  Ú_can_record_outputsr1   r!   r"   r   r4   rS   r   r   rL   r   rJ   rT   rU   s   @r;   r¬  r¬  Ç  sÒ   ø€ € € € € € ØÐÐÑØ)ÐØ+Ð,Ðà)Ø*ðð Ðð
ð ð ð ð ð ð6  ØØð2
Ø"œ\ð2
Ø5:´\ð2
ØMSÐTfÔMgð2
à	Ð+Ñ	+ð2
ð 2
ð 2
ñ „^ñ „_ñ  Ôð2
ð 2
ð 2
ð 2
ð 2
r<   r¬  c                   ó  ‡ — e Zd ZU eed<   dZeedœZdefˆ fd„Z	e
ee	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  d	edz  d
ej        dz  dedz  dee         deez  fd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚGlm4vTextModelrd   )r¡  r­  c                 óÞ  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          j        ‰j        ‰j        | j        ¦  «        | _        t          j	        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        t!          ‰¬¦  «        | _        d| _        |                      ¦   «          d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r_  )rŠ  )r3  rq  rd   s     €r;   r4  z+Glm4vTextModel.__init__.<locals>.<listcomp>3  s$   ø€ ÐgÐgÐg¸)Õ" 6¨9Ñ5Ô5ÐgÐgÐgr<   rE  ©rd   F)r0   r1   Úpad_token_idÚpadding_idxÚ
vocab_sizer2   rÈ   r9   Úembed_tokensrµ  r¶  Únum_hidden_layersÚlayersr+   rF  ÚnormrN  Ú
rotary_embr¾  r¿  rË   s    `€r;   r1   zGlm4vTextModel.__init__,  sÒ   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒåœL¨Ô):¸FÔ<NÐPTÔP`ÑaÔaˆÔÝ”mØgÐgÐgÐgÅuÈVÔMeÑGfÔGfÐgÑgÔgñ
ô 
ˆŒõ ! Ô!3¸Ô9LÐMÑMÔMˆŒ	Ý2¸&ÐAÑAÔAˆŒà&+ˆÔ#à�ŠÑÔÐÐÐr<   NÚ	input_idsr  rª   rx  Úinputs_embedsr‘  r˜   r.   c           	      óp  — |d u |d uz  rt          d¦  «        ‚|r5|€3t          j                             ¦   «         st	          | j        ¬¦  «        }|€|                      |¦  «        }|€y|�|                     ¦   «         nd}t          j        |j	        d         |j
        ¬¦  «        |z   }|                     ddd¦  «                             d|j	        d         d¦  «        }n3|j        dk    r(|d	                              d|j	        d         d¦  «        }|j        dk    r$|j	        d         d
k    r|d         }	|dd …         }nd }	| j        ||||	dœ}
t          di |
¤Ž}|}|                      ||¬¦  «        }| j        D ]} ||f||	||dœ|¤Ž}|}Œ|                      |¦  «        }t%          ||¬¦  «        S )Nú:You must specify exactly one of input_ids or inputs_embedsrÌ  r   r%   rÍ   r@   r   r?   ©N.é   )rd   rÖ  r  rx  rª   )rª   )r  rª   rx  r(  )rÂ  rx  r_  )rˆ   r4   ÚjitÚ
is_tracingr
   rd   rÐ  Úget_seq_lengthr—   rM   r}   rx   r  Úndimr   rÔ  rÒ  rÓ  r   )r8   rÕ  r  rª   rx  rÖ  r‘  r˜   Úpast_seen_tokensÚtext_position_idsÚmask_kwargsÚcausal_maskr=   r(  Údecoder_layerÚlayer_outputss                   r;   rJ   zGlm4vTextModel.forward<  s:  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZð ð 	?˜Ð0½¼×9MÒ9MÑ9OÔ9OÐ0Ý*°$´+Ð>Ñ>Ô>ˆOàÐ Ø ×-Ò-¨iÑ8Ô8ˆMð ÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×,Ò,¨Q°°2Ñ6Ô6×=Ò=¸aÀÔATÐUVÔAWÐY[Ñ\Ô\ˆLˆLØÔ !Ò#Ð#Ø'¨	Ô2×9Ò9¸!¸\Ô=OÐPQÔ=RÐTVÑWÔWˆLð Ô Ò!Ð! lÔ&8¸Ô&;¸qÒ&@Ð&@Ø ,¨Q¤ÐØ'¨¨¨Ô+ˆLˆLð !%Ðð ”kØ*Ø,Ø.Ø-ð
ð 
ˆõ )Ð7Ð7¨;Ð7Ð7ˆà%ˆØ"Ÿošo¨mÈ,˜oÑWÔWÐà!œ[ð 		*ð 		*ˆMØ)˜MØðà*Ø.Ø /Ø$7ðð ð ðð ˆMð *ˆMˆMàŸ	š	 -Ñ0Ô0ˆå&Ø+Ø+ð
ñ 
ô 
ð 	
r<   )NNNNNN)rO   rP   rQ   r'   rš  r£  rŠ  rp  rÇ  r1   r   r!   r"   r4   r”  rS   r	   rˆ  rj   r   r   rL   r   rJ   rT   rU   s   @r;   rÉ  rÉ  #  sK  ø€ € € € € € àÐÐÑØ Ðà.Ø(ðð Ðð
˜ð ð ð ð ð ð ð  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ðJ
ð J
àÔ# dÑ*ðJ
ð œ tÑ+ðJ
ð Ô&¨Ñ-ð	J
ð
  ™ðJ
ð Ô(¨4Ñ/ðJ
ð ˜$‘;ðJ
ð Ð-Ô.ðJ
ð 
Ð(Ñ	(ðJ
ð J
ð J
ñ „_ñ  Ôñ „^ðJ
ð J
ð J
ð J
ð J
r<   rÉ  c                   óT  ‡ — e Zd ZdZdZddgZˆ fd„Z	 	 	 	 d&ded	eeeef         e	j
        z  d
edededee	j        z  dz  fd„Z	 	 	 d'de	j        de	j        de	j        dz  de	j        dz  de	j
        dz  dee	j
        e	j
        f         fd„Z ed¬¦  «        ee	 d(de	j        de	j        dz  dee         deez  fd„¦   «         ¦   «         ¦   «         Z ed¬¦  «        ee	 d(de	j        de	j        dz  dee         deez  fd„¦   «         ¦   «         ¦   «         Z	 	 d)de	j        de	j        de	j        dz  d e	j        dz  fd!„Z	 	 	 	 	 d*de	j
        dz  de	j
        dz  de	j
        dz  de	j
        dz  de	j
        dz  d"e	j
        dz  de	j        dz  de	j
        dz  fd#„Zee	 	 	 	 	 	 	 	 	 	 d+de	j        dz  de	j
        dz  d$e	j        dz  d"edz  de	j        dz  de	j
        dz  de	j        dz  de	j        dz  de	j        dz  de	j        dz  dee         deez  fd%„¦   «         ¦   «         Zˆ xZ S ),Ú
Glm4vModelrž  FrŠ  rC  c                 ó  •— t          ¦   «                              |¦  «         t                               |j        ¦  «        | _        t                               |j        ¦  «        | _        d | _	        |  
                    ¦   «          d S rf   )r0   r1   r¬  Ú_from_configÚvision_configÚvisualrÉ  Útext_configÚlanguage_modelr˜  r¿  rË   s     €r;   r1   zGlm4vModel.__init__“  sl   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý&×3Ò3°FÔ4HÑIÔIˆŒÝ,×9Ò9¸&Ô:LÑMÔMˆÔØˆÔð 	�ŠÑÔÐÐÐr<   r%   NÚstart_positionrÀ  Útemp_merge_sizer²  Útime_intervalr}   c                 óü  — |d                               ¦   «         |z  |d                               ¦   «         |z  |d                               ¦   «         |z  }	}}t          j        ||¬¦  «        |z  }
t          j        ||¬¦  «        |z   }t          j        |	|¬¦  «        |z   }t          j        |
||d¬¦  «        \  }}}t          j        |||gd¬¦  «                             dd	¦  «        }|dxx         |z  cc<   |S )
a¢  
        Compute 3D positional indices for vision tokens derived from a single image or video input.

        The positions are generated from the input grid defined by temporal (T), height (H), and
        width (W) dimensions. Temporal and spatial dimensions can be downscaled according to the
        merge sizes used in the vision backbone. The resulting positions are offset by `start_position`.

        Args:
            start_position (`int`):
                Offset added to all computed positional indices.
            grid_thw (`Sequence[int]` or `torch.Tensor` of shape `(3,)`):
                The (T, H, W) grid representing the feature layout of the current image or video after patch embedding.
            temp_merge_size (`int`, *optional*):
                Factor by which the temporal dimension is reduced in the backbone. The temporal grid size is divided
                by this value. Defaults to 1.
            spatial_merge_size (`int`, *optional*):
                Factor by which the spatial dimensions (H and W) are reduced in the backbone. Both H and W are divided
                by this value. Defaults to 1.
            time_interval (`int`, *optional*):
                Spacing factor applied between consecutive temporal position indices.Defaults to 1.
            device (`str` or `torch.device`, *optional*):
                Device on which the resulting tensor is allocated. If `None`, uses the current default device.

        Returns:
            torch.LongTensor of shape (3, sequence_length):
                Positional indices for temporal, height, and width dimensions,
                flattened into sequence form and offset by `start_position`.
        r   r%   r?   rÍ   Úij)Úindexingr¯   r   r@   )Úitemr4   r—   ÚmeshgridrÙ   r  )r8   rí  rÀ  rî  r²  rï  r}   Ú
llm_grid_tÚ
llm_grid_hÚ
llm_grid_wÚposition_temporalÚposition_heightÚposition_widthÚT_gridÚH_gridÚW_gridÚvision_position_idss                    r;   r$   z"Glm4vModel.get_vision_position_idsœ  s  € ðL �QŒK×ÒÑÔ /Ñ1Ø�QŒK×ÒÑÔÐ"4Ñ4Ø�QŒK×ÒÑÔÐ"4Ñ4ð !+�Jˆ
õ "œL¨¸FÐCÑCÔCÀmÑSÐÝœ, z¸&ÐAÑAÔAÀNÑRˆÝœ j¸Ð@Ñ@Ô@À>ÑQˆå!&¤Ð0AÀ?ÐTbÐmqÐ!rÑ!rÔ!rÑˆ�˜Ý#œk¨6°6¸6Ð*BÈÐJÑJÔJ×RÒRÐSTÐVXÑYÔYÐØ˜AÐÐÔ .Ñ0ÐÐÑØ"Ð"r<   rÕ  Úmm_token_type_idsÚimage_grid_thwÚvideo_grid_thwr  r.   c           	      ó´  — |�*t          j        ||dd…df         d¬¦  «        }d|dd…df<   | j        j        j        }g }t          j        d|j        d         |j        d         |j        |j        ¬¦  «        }	|�t          |¦  «        nd|�t          |¦  «        nddœ}
t          |¦  «        D �]w\  }}||         }|�@|||                              ¦   «                  }|||                              ¦   «                  }g }t          j        t          |                     ¦   «         ¦  «        d„ ¦  «        D ]K\  }}t          |¦  «        }|d         d         }|d	         d         dz   }|                     |||f¦  «         ŒLd}g }|D ]Û\  }}}|dk    rd||z
  }|                     t          j        ||j        ¬
¦  «                             dd	¦  «                             dd	¦  «        |z   ¦  «         ||z  }Œpt)          |
|         ¦  «        }|                      ||d||j        ¬
¦  «        }|                     |¦  «         |t-          |d         |d         ¦  «        |z  z  }ŒÜt          j        |d¬¦  «                             dd	¦  «        }|�;|                     |	j        ¦  «        |	dd…|||                              ¦   «         f<   n!|                     |	j        ¦  «        |	dd…|f<   |                     |                     ¦   «         dz   t5          |¦  «        z
  ¦  «         �Œyt          j        ||j        ¬
¦  «                             d¦  «        }|	|fS )aÙ  
        Difference from Qwen2VL/Qwen2.5VL's get_rope_index:
        - GLM4V uses timestamps to separate each video frame, so the video_grid_thw should also be split too.

        Args:
            input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
                Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
                it.
            mm_token_type_ids (`torch.IntTensor` of shape `(batch_size, sequence_length)`):
                Token type ids matching each modality to a different value in the input sequence, i.e. text (0), image (1), video (2).
            image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*):
                The temporal, height and width of feature shape of each image in LLM.
            video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*):
                The temporal, height and width of feature shape of each video in LLM.
            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

        Returns:
            position_ids (`torch.LongTensor` of shape `(3, batch_size, sequence_length)`)
            mrope_position_deltas (`torch.Tensor` of shape `(batch_size)`)
        Nr   r¯   r%   r   ©rB   r}   )r%   r?   c                 ó   — | d         S )Nr%   r_  )r©   s    r;   ú<lambda>z+Glm4vModel.get_rope_index.<locals>.<lambda>  s   € Ð`aÐbcÔ`d€ r<   r@   rÍ   r?   )r4   rc  rd   ré  r²  ÚzerosrM   rB   r}   Úiterrb  rj   Ú	itertoolsÚgroupbyr2  rÒ   Úappendr—   rx   r  Únextr$   Úmaxr°   r  rC   ÚlenrÓ   rÖ   )r8   rÕ  rÿ  r   r  r  r˜   r²  Úmrope_position_deltasrª   Ú
grid_itersÚ	batch_idxÚcurrent_input_idsÚinput_token_typeÚinput_type_groupr  ÚgroupÚstart_indexÚ	end_indexÚcurrent_posÚllm_pos_ids_listÚmodality_typeÚ	start_idxÚend_idxÚtext_lenrÀ  rþ  Úllm_positionss                               r;   Úget_rope_indexzGlm4vModel.get_rope_indexÐ  sÇ  € ðF Ð%Ý"Ô4°^À^ÐTUÐTUÐTUÐWXÐTXÔEYÐ_`ÐaÑaÔaˆNØ#$ˆN˜1˜1˜1˜a˜4Ñ Ø!œ[Ô6ÔIÐà "ÐÝ”{ØØŒO˜AÔØŒO˜AÔØ”/ØÔ#ð
ñ 
ô 
ˆð (6Ð'A�t�NÑ#Ô#Ð#ÀtØ'5Ð'A�t�NÑ#Ô#Ð#Àtð
ð 
ˆ
õ
 -6°iÑ,@Ô,@ð $	[ñ $	[Ñ(ˆIÐ(Ø0°Ô;ÐØÐ)Ø$5°nÀYÔ6O×6TÒ6TÑ6VÔ6VÔ$WÐ!Ø#3°NÀ9Ô4M×4RÒ4RÑ4TÔ4TÔ#UÐ à!ÐÝ'Ô/µ	Ð:J×:QÒ:QÑ:SÔ:SÑ0TÔ0TÐVdÐVdÑeÔeð Gð G‘
��UÝ˜U™œ�Ø# Aœh qœk�Ø! "œI aœL¨1Ñ,�	Ø ×'Ò'¨¨k¸9Ð(EÑFÔFÐFÐFàˆKØ!ÐØ5Eð Wð WÑ1�˜y¨'à  AÒ%Ð%Ø&¨Ñ2�HØ$×+Ò+Ýœ X°iÔ6FÐGÑGÔG×LÒLÈQÐPRÑSÔS×ZÒZÐ[\Ð^`ÑaÔaÐdoÑoñô ð ð   8Ñ+�K�Kõ  $ J¨}Ô$=Ñ>Ô>�HØ*.×*FÒ*FØ# X¨qÐ2DÈYÔM]ð +Gñ +ô +Ð'ð %×+Ò+Ð,?Ñ@Ô@Ð@Ø¥3 x°¤{°H¸Q´KÑ#@Ô#@ÐDVÑ#VÑV�K�KÝ!œIÐ&6¸AÐ>Ñ>Ô>×FÒFÀqÈ"ÑMÔMˆMØÐ)ØO\×O_ÒO_Ð`lÔ`sÑOtÔOt�˜Q˜Q˜Q 	¨>¸)Ô+D×+IÒ+IÑ+KÔ+KÐKÑLÐLà-:×-=Ò-=¸lÔ>QÑ-RÔ-R�˜Q˜Q˜Q 	˜\Ñ*Ø!×(Ò(¨×):Ò):Ñ)<Ô)<¸qÑ)@Å3ÐGXÑCYÔCYÑ)YÑZÔZÐZÑZÝ %¤Ð-BÈ9ÔK[Ð \Ñ \Ô \× fÒ fÐghÑ iÔ iÐØÐ2Ð2Ð2r<   r   )ÚmodalityÚpixel_values_videosr˜   c                 óê  — |                      | j        j        ¦  «        }|d d …df         }|d d …dd …f         }t          j        ||d¬¦  «        }|                     |j        d         d¦  «        }t          j        ||gd¬¦  «        } | j        |f|ddœ|¤Ž}	|                     d¦  «        | j        j	        dz  z   
                    ¦   «         }
t          j        |	j        |
¦  «        }||	_        |	S )Nr   r%   r¯   T)rÀ  Úreturn_dictr@   r?   )r¤   rê  rB   r4   rc  Únew_onesrM   r°   Úprodr²  r2  r1  rÃ  )r8   r   r  r˜   ÚtÚhwÚflattened_hwÚprefix_onesÚflattened_video_grid_thwÚvision_outputsÚsplit_sizesÚvideo_embedss               r;   Úget_video_featureszGlm4vModel.get_video_features-  s  € ð 2×6Ò6°t´{Ô7HÑIÔIÐà˜1˜1˜1˜a˜4Ô ˆØ˜A˜A˜A˜q˜r˜r˜EÔ"ˆåÔ.¨r°1¸!Ð<Ñ<Ô<ˆØ$×-Ò-¨lÔ.@ÀÔ.CÀQÑGÔGˆÝ#(¤9¨k¸<Ð-HÈaÐ#PÑ#PÔ#PÐ Ø$˜œØð
Ø*BÐPTð
ð 
ØX^ð
ð 
ˆð &×*Ò*¨2Ñ.Ô.°$´+Ô2PÐRSÑ2SÑS×[Ò[Ñ]Ô]ˆÝ”{ >Ô#?ÀÑMÔMˆØ'3ˆÔ$àÐr<   rŸ  Úpixel_valuesc                 ó  — |                      | j        j        ¦  «        } | j        |fd|i|¤Ž}|                     d¦  «        | j        j        dz  z                       ¦   «         }t          j        |j        |¦  «        }||_        |S )NrÀ  r@   r?   )	r¤   rê  rB   r$  r²  r2  r4   r1  rÃ  )r8   r.  r   r˜   r*  r+  Úimage_embedss          r;   Úget_image_featureszGlm4vModel.get_image_featuresG  s‹   € ð $×(Ò(¨¬Ô):Ñ;Ô;ˆØ$˜œ \ÐUÐU¸NÐUÈfÐUÐUˆØ%×*Ò*¨2Ñ.Ô.°$´+Ô2PÐRSÑ2SÑS×[Ò[Ñ]Ô]ˆÝ”{ >Ô#?ÀÑMÔMˆØ'3ˆÔ$àÐr<   rÖ  Úimage_featuresÚvideo_featuresc           	      ó  — |€Ë| |                       ¦   «         t          j        d| j        j        t          j        |j        ¬¦  «        ¦  «        k    }|                     d¦  «        }| |                       ¦   «         t          j        d| j        j        t          j        |j        ¬¦  «        ¦  «        k    }|                     d¦  «        }n || j        j        k    }|| j        j        k    }| 	                    ¦   «         }| 
                    d¦  «                             |j        ¦  «        }|�Et          ||j        d         z  |                     ¦   «         k    d|› d|j        d         › �¦  «         | 	                    ¦   «         }| 
                    d¦  «                             |j        ¦  «        }|�Et          ||j        d         z  |                     ¦   «         k    d|› d|j        d         › �¦  «         ||fS )	zï
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        Nr_  r  r@   z6Image features and image tokens do not match, tokens: z, features: r   z6Video features and video tokens do not match, tokens: )Úget_input_embeddingsr4   Úfullrd   Úimage_token_idrÔ   r}   ÚallÚvideo_token_idrØ   rÖ   rC   r   rM   Únumel)	r8   rÕ  rÖ  r2  r3  Úspecial_image_maskÚspecial_video_maskÚn_image_tokensÚn_video_tokenss	            r;   Úget_placeholder_maskzGlm4vModel.get_placeholder_maskX  s  € ð ÐØ!.Ð2M°$×2KÒ2KÑ2MÔ2MÝ”
˜2˜tœ{Ô9ÅÄÐTaÔThÐiÑiÔiñ3ô 3ò "Ðð "4×!7Ò!7¸Ñ!;Ô!;ÐØ!.Ð2M°$×2KÒ2KÑ2MÔ2MÝ”
˜2˜tœ{Ô9ÅÄÐTaÔThÐiÑiÔiñ3ô 3ò "Ðð "4×!7Ò!7¸Ñ!;Ô!;ÐÐð "+¨d¬kÔ.HÒ!HÐØ!*¨d¬kÔ.HÒ!HÐà+×/Ò/Ñ1Ô1ˆØ/×9Ò9¸"Ñ=Ô=×@Ò@ÀÔAUÑVÔVÐØÐ%Ý"Ø Ô!4°RÔ!8Ñ8¸N×<PÒ<PÑ<RÔ<RÒRØ~ÈÐ~Ð~ÐesÔeyÐz{Ôe|Ð~Ð~ñô ð ð
 ,×/Ò/Ñ1Ô1ˆØ/×9Ò9¸"Ñ=Ô=×@Ò@ÀÔAUÑVÔVÐØÐ%Ý"Ø Ô!4°RÔ!8Ñ8¸N×<PÒ<PÑ<RÔ<RÒRØ~ÈÐ~Ð~ÐesÔeyÐz{Ôe|Ð~Ð~ñô ð ð "Ð#5Ð5Ð5r<   rx  c                 ó”  — |€dn|                      ¦   «         }|d up|d u}	|	r|€|�t          d¦  «        ‚|d uo|d uo|	}
|
r3| j        �|dk    r&|                      |||||¬¦  «        \  }}|| _        �nS| j        ��I|dk    s|�€@|j        \  }}}|�‰|                     ¦   «                              d¦  «        dz
  }|                     |dk    d¦  «        }|                     d|d¦  «         	                    ddd¦  «         
                    |j        ¦  «        }n\t          j        |||z   ¦  «        }|                     ddd¦  «                             d|d¦  «         
                    |j        ¦  «        }| j                             || j        j        d         z  d¬¦  «        }|| 
                    |j        ¬¦  «        z   }nd }|S )	Nr   a  Multimodal data was passed (via `image_grid_thw` or `video_grid_thw`) but `mm_token_type_ids` is missing. Please pass `mm_token_type_ids` to the model so that multimodal RoPE (M-RoPE) can be computed correctly. `mm_token_type_ids` is returned by the processor alongside `input_ids`.)r   r  r  rÿ  r@   r%   r   r¯   rÍ   )rÝ  rˆ   r˜  r  rM   rÔ   r×   Úmasked_fillrx   ÚrepeatrC   r}   r4   r—   r  rc  )r8   rÕ  rÖ  r   r  r  rx  rÿ  Úpast_key_values_lengthÚhas_multimodalÚcan_compute_mroperª   r˜  Ú
batch_sizer<  r>  Údeltas                    r;   Úcompute_3d_position_idsz"Glm4vModel.compute_3d_position_ids‚  s+  € ð '6Ð&=  À?×CaÒCaÑCcÔCcÐØ'¨tÐ3ÐQ°~ÈTÐ7QˆØð 	Ð/Ð7¸IÐ<QÝðnñô ð ð
 &¨TÐ1ÐfÐ6GÈtÐ6SÐfÐXfÐàð 	  $Ô"2Ð":Ð>TÐXYÒ>YÐ>YØ(,×(;Ò(;ØØ-Ø-Ø-Ø"3ð )<ñ )ô )Ñ%ˆL˜+ð  +ˆDÔÑð
 ÔÑ)Ð/EÈÒ/IÐ/IÈYÑM^Ø(5Ô(;Ñ%ˆJ˜
 AØÐ)Ø-×2Ò2Ñ4Ô4×;Ò;¸BÑ?Ô?À!ÑC�Ø+×7Ò7¸È!Ò8KÈQÑOÔO�Ø+×0Ò0°°JÀÑCÔC×JÒJÈ1ÈaÐQRÑSÔS×VÒVÐWdÔWkÑlÔl��å$œ|Ð,BÐDZÐ]gÑDgÑhÔh�Ø+×0Ò0°°A°rÑ:Ô:×AÒAÀ!ÀZÐQSÑTÔT×WÒWÐXeÔXlÑmÔm�ØÔ$×6Ò6°zÀTÔEUÔE[Ð\]ÔE^Ñ7^ÐdeÐ6ÑfÔfˆEØ'¨%¯(ª(¸-Ô:N¨(Ñ*OÔ*OÑOˆLˆLð  ˆLØÐr<   rª   c           	      óð  — |d u |d uz  rt          d¦  «        ‚|€ |                      ¦   «         |¦  «        }|�{ | j        ||fddi|¤Žj        }t	          j        |d¬¦  «                             |j        |j        ¦  «        }|  	                    |||¬¦  «        \  }}| 
                    ||¦  «        }|�{ | j        ||	fddi|¤Žj        }t	          j        |d¬¦  «                             |j        |j        ¦  «        }|  	                    |||¬¦  «        \  }}| 
                    ||¦  «        }|€|                      |||	||||
¬¦  «        } | j        dd ||||d	œ|¤Ž}t          di |¤d
| j        i¤ŽS )NrØ  r"  Tr   r¯   )r2  )r3  )rÕ  r   r  rÖ  r  rx  rÿ  )rÕ  rª   r  rx  rÖ  r˜  r_  )rˆ   r5  r1  rÃ  r4   r°   rC   r}   rB   r?  Úmasked_scatterr-  rH  rì  r–  r˜  )r8   rÕ  r  rª   rx  rÖ  r.  r   r   r  rÿ  r˜   r0  Ú
image_maskr>  r,  Ú
video_maskÚoutputss                     r;   rJ   zGlm4vModel.forward³  s&  € ð  ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø7˜D×5Ò5Ñ7Ô7¸	ÑBÔBˆMàÐ#Ø2˜4Ô2Ø˜nðð Ø:>ðØBHðð äð õ !œ9 \°qÐ9Ñ9Ô9×<Ò<¸]Ô=QÐS`ÔSfÑgÔgˆLØ ×5Ò5°iÀÐ_kÐ5ÑlÔl‰MˆJ˜Ø)×8Ò8¸À\ÑRÔRˆMàÐ*Ø2˜4Ô2Ø# ^ðð ØAEðØIOðð äð õ !œ9 \°qÐ9Ñ9Ô9×<Ò<¸]Ô=QÐS`ÔSfÑgÔgˆLØ ×5Ò5°iÀÐ_kÐ5ÑlÔl‰MˆAˆzØ)×8Ò8¸À\ÑRÔRˆMàÐØ×7Ò7Ø#Ø-Ø-Ø+Ø-Ø /Ø"3ð 8ñ ô ˆLð &�$Ô%ð 
ØØ%Ø)Ø+Ø'ð
ð 
ð ð
ð 
ˆõ (ð 
ð 
Øð
ð 
àÔ(ð
ð 
ð 
ð 	
r<   )r%   r%   r%   Nr~  rf   rA  )NNNNN)
NNNNNNNNNN)!rO   rP   rQ   r¢  Úaccepts_loss_kwargsr¥  r1   rÀ   rÒ   r4   rS   r¥   r}   r$   r”  Ú	IntTensorrL   r  r   r   r   rˆ  r   r   r   r-  r1  r?  rH  r	   r–  rJ   rT   rU   s   @r;   ræ  ræ  Œ  s¤  ø€ € € € € àÐàÐØ0Ð2DÐEÐðð ð ð ð ð  !Ø"#ØØ,0ð2#ð 2#àð2#ð �s˜C �}Ô%¨¬Ñ4ð2#ð ð	2#ð
  ð2#ð ð2#ð �e”lÑ" TÑ)ð2#ð 2#ð 2#ð 2#ðp 37Ø26Ø.2ð[3ð [3àÔ#ð[3ð !œ?ð[3ð Ô(¨4Ñ/ð	[3ð
 Ô(¨4Ñ/ð[3ð œ tÑ+ð[3ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð[3ð [3ð [3ð [3ðz  Ð¨Ð1Ñ1Ô1ØØð 37ðð à"Ô.ðð Ô(¨4Ñ/ðð Ð+Ô,ð	ð
 
Ð+Ñ	+ðð ð ñ „^ñ Ôñ 2Ô1ðð.  Ð¨Ð1Ñ1Ô1ØØð 37ðð àÔ'ðð Ô(¨4Ñ/ðð Ð+Ô,ð	ð
 
Ð+Ñ	+ðð ð ñ „^ñ Ôñ 2Ô1ðð$ 48Ø37ð(6ð (6àÔ#ð(6ð Ô(ð(6ð Ô)¨DÑ0ð	(6ð
 Ô)¨DÑ0ð(6ð (6ð (6ð (6ð\ /3Ø.2Ø.2Ø/3Ø48ð/ð /à”< $Ñ&ð/ð ”| dÑ*ð/ð œ tÑ+ð	/ð
 œ tÑ+ð/ð œ tÑ+ð/ð œ¨Ñ,ð/ð !œ?¨TÑ1ð/ð 
Œ˜Ñ	ð/ð /ð /ð /ðb Øð .2Ø.2Ø04Ø(,Ø26Ø,0Ø8<Ø26Ø26Ø48ð;
ð ;
àÔ# dÑ*ð;
ð œ tÑ+ð;
ð Ô&¨Ñ-ð	;
ð
  ™ð;
ð Ô(¨4Ñ/ð;
ð ”l TÑ)ð;
ð #Ô.°Ñ5ð;
ð Ô(¨4Ñ/ð;
ð Ô(¨4Ñ/ð;
ð !œ?¨TÑ1ð;
ð Ð+Ô,ð;
ð 
Ð)Ñ	)ð;
ð ;
ð ;
ñ Ôñ „^ð;
ð ;
ð ;
ð ;
ð ;
r<   ræ  c                   ó2   — e Zd ZU dZdZej        dz  ed<   dS )ÚGlm4vCausalLMOutputWithPastr—  Nr˜  r™  r_  r<   r;   rQ  rQ  ó  r›  r<   rQ  c                    óÜ  ‡ — e Zd ZddiZdZˆ fd„Ze	 ddej        dej	        dz  de
e         d	eez  fd
„¦   «         Ze	 ddej        dej	        dz  de
e         d	eez  fd„¦   «         Zee	 	 	 	 	 	 	 	 	 	 	 	 ddej	        dz  dej        dz  dej	        dz  dedz  dej        dz  dej	        dz  dej        dz  dej        dz  dej	        dz  dej	        dz  dej        dz  deej        z  de
e         d	eez  fd„¦   «         ¦   «         Zˆ fd„Z	 ddej	        dz  dej        dz  d	eej        ej        f         fd„Z	 	 	 d dededej	        dz  d	eej	        eeef         f         fd„Zˆ xZS )!ÚGlm4vForConditionalGenerationzlm_head.weightz(model.language_model.embed_tokens.weightFc                 óú   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        j        |j        j        d¬¦  «        | _	        |  
                    ¦   «          d S )NFr[   )r0   r1   ræ  rž  r2   r^   rë  r9   rÏ  Úlm_headr¿  rË   s     €r;   r1   z&Glm4vForConditionalGeneration.__init__  se   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý Ñ'Ô'ˆŒ
Ý”y Ô!3Ô!?ÀÔASÔA^ÐejÐkÑkÔkˆŒà�ŠÑÔÐÐÐr<   Nr   r  r˜   r.   c                 ó*   —  | j         j        ||fi |¤ŽS )zµ
        pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
            The tensors corresponding to the input videos.
        )rž  r-  )r8   r   r  r˜   s       r;   r-  z0Glm4vForConditionalGeneration.get_video_features  s%   € ð -ˆtŒzÔ,Ð-@À.Ð[Ð[ÐTZÐ[Ð[Ð[r<   r.  r   c                 ó*   —  | j         j        ||fi |¤ŽS )z®
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
            The tensors corresponding to the input images.
        )rž  r1  )r8   r.  r   r˜   s       r;   r1  z0Glm4vForConditionalGeneration.get_image_features  s#   € ð -ˆtŒzÔ,¨\¸>ÐTÐTÈVÐTÐTÐTr<   r   rÕ  r  rª   rx  rÖ  Úlabelsrÿ  Úlogits_to_keepc                 óp  —  | j         d||||	|
|||||dœ
|¤Ž}|d         }t          |t          ¦  «        rt          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }d}|�  | j        d||| j        j        j        dœ|¤Ž}t          |||j
        |j        |j        |j        ¬¦  «        S )aÇ  
        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, Glm4vForConditionalGeneration

        >>> model = Glm4vForConditionalGeneration.from_pretrained("zai-org/GLM-4.1V-9B-Thinking")
        >>> processor = AutoProcessor.from_pretrained("zai-org/GLM-4.1V-9B-Thinking")

        >>> messages = [
            {
                "role": "user",
                "content": [
                    {"type": "image", "url": "https://www.ilankelman.org/stopsigns/australia.jpg"},
                    {"type": "text", "text": "What is shown in this image?"},
                ],
            },
        ]
        >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
        >>> inputs = processor(text=[text], images=[image], vision_infos=[vision_infos])

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "The image shows a street scene with a red stop sign in the foreground. In the background, there is a large red gate with Chinese characters ..."
        ```)
rÕ  r.  r   r   r  rÿ  rª   r  rx  rÖ  r   N)ÚlogitsrX  rÏ  )Úlossr[  rx  r=   r®  r˜  r_  )rž  r£   rÀ   ÚslicerU  Úloss_functionrd   rë  rÏ  rQ  rx  r=   r®  r˜  )r8   rÕ  r  rª   rx  rÖ  rX  r.  r   r   r  rÿ  rY  r˜   rM  r=   Úslice_indicesr[  r\  s                      r;   rJ   z%Glm4vForConditionalGeneration.forward%  s  € ðf �$”*ð 
ØØ%Ø 3Ø)Ø)Ø/Ø%Ø)Ø+Ø'ð
ð 
ð ð
ð 
ˆð   œ
ˆõ 9CÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%ð Ø f¸¼Ô9PÔ9[ðð Ø_eðð ˆDõ +ØØØ#Ô3Ø!Ô/ØÔ)ØÔ+ð
ñ 
ô 
ð 	
r<   c                 ó¼  •— t          ¦   «                              ||¦  «        }d}|                     d¦  «        x}�|                     ¦   «         }|dk    r#| j        j        �|d         | j        j        z   }|S d|v r|d         j        d         dk    r|d         }t          |j        ¦  «        dk    o|j        t          j
        t          j        fv }|r€|                     d¦  «        �k|                     d¦  «        €|                     d	¦  «        �Ad
„ |                     ¦   «         D ¦   «         } | j        j        |fi |¤Ž\  }}	|	| j        _        nf|                     d¦  «                             ddd¦  «        }t          j        |j        d         dt          j        |j        ¬¦  «        | j        _        |d         }t          j        ||gd¬¦  «        }|S )Nr   rx  rÙ  rÕ  r%   r?   rÿ  r   r  c                 ó&   — i | ]\  }}|d k    ¯||“ŒS )rÕ  r_  )r3  rø   r6  s      r;   ú
<dictcomp>zVGlm4vForConditionalGeneration._prepare_position_ids_for_generation.<locals>.<dictcomp>’  s(   € ÐVÐVÐV¡T Q¨ÀQÈ+ÒEUÐEU˜A˜qÐEUÐEUÐEUr<   r   r@   r  r¯   )r0   Ú$_prepare_position_ids_for_generationrX  rÝ  rž  r˜  rM   r  rB   r4   rÀ   rÔ   Úitemsr  rÖ   r  r  r}   r°   )r8   Úinputs_tensorÚmodel_kwargsÚtext_positionsÚpast_lengthÚcacherª   Úis_input_idsÚvision_positionsr˜  r:   s             €r;   rc  zBGlm4vForConditionalGeneration._prepare_position_ids_for_generation{  sö  ø€ õ ™œ×EÒEÀmÐUaÑbÔbˆð ˆØ!×%Ò%Ð&7Ñ8Ô8Ð8ˆEÐEØ×.Ò.Ñ0Ô0ˆKØ˜!ÒÐ ¤
Ô 6Ð BØ)¨)Ô4°t´zÔ7MÑMˆLØÐð ˜,Ð&Ð&¨<¸Ô+DÔ+JÈ1Ô+MÐPQÒ+QÐ+QØ(¨Ô5ˆMå˜=Ô.Ñ/Ô/°1Ò4Ðg¸Ô9LÕQVÔQZÕ\aÔ\fÐPgÐ9gˆàð	à× Ò Ð!4Ñ5Ô5ÐAØ×!Ò!Ð"2Ñ3Ô3Ð?À<×CSÒCSÐTdÑCeÔCeÐCqàVÐV¨\×-?Ò-?Ñ-AÔ-AÐVÑVÔVˆLØ,E¨D¬JÔ,EÀmÐ,dÐ,dÐWcÐ,dÐ,dÑ)Ð˜kØ%0ˆDŒJÔ"Ð"à-×7Ò7¸Ñ:Ô:×AÒAÀ!ÀRÈÑLÔLÐÝ%*¤[ØÔ# AÔ&¨µ´ÀMÔDXð&ñ &ô &ˆDŒJÔ"ð
 (¨	Ô2ˆÝ”y .Ð2BÐ!CÈÐKÑKÔKˆàÐr<   c           	      ó`  — |��| |                       ¦   «         t          j        d| j        j        t          j        |j        ¬¦  «        ¦  «        k    d         }| |                       ¦   «         t          j        d| j        j        t          j        |j        ¬¦  «        ¦  «        k    d         }| |                       ¦   «         t          j        d| j        j        t          j        |j        ¬¦  «        ¦  «        k    d         }n0|| j        j        k    }|| j        j        k    }|| j        j        k    }t          j	        | 
                    ¦   «         | 
                    ¦   «         z
  d¬¦  «        }|dk    }|| z  }|                     d¬¦  «        }	|                     d¬¦  «        }
|	|
fS )aa  
        Get the number of images and videos for each sample to calculate the separation length of the sample tensor.
        These parameters are not passed through the processor to avoid unpredictable impacts from interface modifications.

        Args:
            input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
                Indices of input sequence tokens in the vocabulary.

        Returns:
            image_nums (`torch.LongTensor` of shape `(batch_size, num_images_sample)`)
            video_nums (`torch.LongTensor` of shape `(batch_size, num_videos_sample)`)
        Nr_  r  ).r   r%   r¯   r   )r5  r4   r6  rd   Úimage_start_token_idrÔ   r}   Úvideo_start_token_idÚvideo_end_token_idr×   rÀ   rØ   )r8   rÕ  rÖ  Úis_imageÚis_video_startÚis_video_endÚvideo_levelÚinside_videoÚstandalone_imagesÚimage_countsÚvideo_countss              r;   Ú_get_image_nums_and_video_numsz<Glm4vForConditionalGeneration._get_image_nums_and_video_nums¡  s´  € ð$ Ñ$àØ.�4×,Ò,Ñ.Ô.Ý”J˜r 4¤;Ô#CÍ5Ì:Ð^kÔ^rÐsÑsÔsñô òð ôˆHð Ø.�4×,Ò,Ñ.Ô.Ý”J˜r 4¤;Ô#CÍ5Ì:Ð^kÔ^rÐsÑsÔsñô òð ôˆNð Ø.�4×,Ò,Ñ.Ô.Ý”J˜r 4¤;Ô#AÍÌÐ\iÔ\pÐqÑqÔqñô òð ôˆLˆLð ! D¤KÔ$DÒDˆHØ&¨$¬+Ô*JÒJˆNØ$¨¬Ô(FÒFˆLõ ”l >×#5Ò#5Ñ#7Ô#7¸,×:JÒ:JÑ:LÔ:LÑ#LÐRSÐTÑTÔTˆØ" Q’ˆð %¨¨Ñ6Ðð )×,Ò,°Ð,Ñ3Ô3ˆØ%×)Ò)¨aÐ)Ñ0Ô0ˆà˜\Ð)Ð)r<   r%   Úexpand_sizeÚis_encoder_decoderc                 ó  ‡ ‡‡‡‡— ‰dk    r‰‰fS g d¢Šˆˆˆˆ fd„}ˆˆfd„} |‰¦  «        Š‰�‰                      ‰d¬¦  «        Š |‰¦  «        Š|r8‰                     d¦  «        €t          d¦  «        ‚ |‰d         ¦  «        ‰d<   ‰‰fS )	Nr%   )r.  r   r   r  Úsecond_per_grid_tsc                 ó  •— ‰                      dd ¦  «        }‰                      dd ¦  «        }‰                     ‰
‰                      dd ¦  «        ¬¦  «        \  }}d„ }| D �]}|dk    rFt          j        |t	          |¦  «        ¦  «        }d„ |D ¦   «         } || |         |‰	¬¦  «        | |<   ŒO|dk    r't	          |¦  «        } || |         |‰	¬¦  «        | |<   Œ||d	k    rFt          j        |t	          |¦  «        ¦  «        }d
„ |D ¦   «         } || |         |‰	¬¦  «        | |<   ŒÈ|dk    r't	          |¦  «        } || |         |‰	¬¦  «        | |<   Œõ|dk    r$ || |         t	          |¦  «        ‰	¬¦  «        | |<   �Œ!| S )Nr   r  rÖ  )rÖ  c                 ó´   ‡— t          j        | |¦  «        }|gdg|                      ¦   «         dz
  z  z   Št          j        ˆfd„|D ¦   «         d¬¦  «        }|S )Nr%   c                 ó$   •— g | ]} |j         ‰Ž ‘ŒS r_  )rB  )r3  ÚsampleÚrepeat_argss     €r;   r4  zŸGlm4vForConditionalGeneration._expand_inputs_for_generation.<locals>._expand_dict_for_generation_visual.<locals>._repeat_interleave_samples.<locals>.<listcomp>ô  s"   ø€ Ð#VÐ#VÐ#VÀF M F¤M°;Ð$?Ð#VÐ#VÐ#Vr<   r   r¯   )r4   r1  rš   r°   )r©   rÞ   Úrepeat_timesÚsamplesÚresultr�  s        @r;   Ú_repeat_interleave_samplesz‹Glm4vForConditionalGeneration._expand_inputs_for_generation.<locals>._expand_dict_for_generation_visual.<locals>._repeat_interleave_samplesñ  sb   ø€ Ýœ+ a¨Ñ1Ô1�Ø+˜n°¨s°a·e²e±g´gÀ±kÑ/BÑB�ÝœÐ#VÐ#VÐ#VÐ#VÈgÐ#VÑ#VÔ#VÐ\]Ð^Ñ^Ô^�Ø�r<   r.  c                 ó^   — g | ]*}t          j        |d ¬¦  «                             ¦   «         ‘Œ+S ©r%   r¯   ©r4   r$  rØ   ©r3  r€  s     r;   r4  z{Glm4vForConditionalGeneration._expand_inputs_for_generation.<locals>._expand_dict_for_generation_visual.<locals>.<listcomp>ü  ó3   € ÐUÐUÐUÀ6�uœz¨&°aÐ8Ñ8Ô8×<Ò<Ñ>Ô>ÐUÐUÐUr<   )rÞ   r‚  r   c                 ó^   — g | ]*}t          j        |d ¬¦  «                             ¦   «         ‘Œ+S r‡  rˆ  r‰  s     r;   r4  z{Glm4vForConditionalGeneration._expand_inputs_for_generation.<locals>._expand_dict_for_generation_visual.<locals>.<listcomp>  rŠ  r<   r|  )rX  rx  r4   r1  rÒ   )Údict_to_expandr   r  Ú
image_numsÚ
video_numsr…  r  rƒ  rÞ   ry  rÕ  rf  r8   s            €€€€r;   Ú"_expand_dict_for_generation_visualzgGlm4vForConditionalGeneration._expand_inputs_for_generation.<locals>._expand_dict_for_generation_visualê  s  ø€ Ø)×-Ò-Ð.>ÀÑEÔEˆNØ)×-Ò-Ð.>ÀÑEÔEˆNØ%)×%HÒ%HØ¨×)9Ò)9¸/È4Ñ)PÔ)Pð &Iñ &ô &Ñ"ˆJ˜
ðð ð ð &ð ñ �Ø˜.Ò(Ð(å#œk¨.½$¸zÑ:JÔ:JÑKÔK�GàUÐUÈWÐUÑUÔU�GØ*DÐ*DØ& sÔ+°WÈ;ð+ñ +ô +�N 3Ñ'Ð'ð Ð,Ò,Ð,å" :Ñ.Ô.�GØ*DÐ*DØ& sÔ+°WÈ;ð+ñ +ô +�N 3Ñ'Ð'ð Ð1Ò1Ð1Ý#œk¨.½$¸zÑ:JÔ:JÑKÔK�GØUÐUÈWÐUÑUÔU�GØ*DÐ*DØ& sÔ+°WÈ;ð+ñ +ô +�N 3Ñ'Ð'ð Ð,Ò,Ð,Ý" :Ñ.Ô.�GØ*DÐ*DØ& sÔ+°WÈ;ð+ñ +ô +�N 3Ñ'Ð'ð Ð0Ò0Ð0Ø*DÐ*DØ& sÔ+µT¸*Ñ5EÔ5EÐT_ð+ñ +ô +�N 3Ñ'ùð "Ð!r<   c                 ó  •— | D ]†}|dk    r2| |         j         dk    r!| |                              ‰d¬¦  «        | |<   Œ:| |         �Dt          | |         t          j        ¦  «        r$|‰vr | |                              ‰d¬¦  «        | |<   Œ‡| S )Nrª   r   r%   r¯   r   )rÞ  rc  r£   r4   rS   )rŒ  r  ry  Úvisual_keyss     €€r;   Ú_expand_dict_for_generationz`Glm4vForConditionalGeneration._expand_inputs_for_generation.<locals>._expand_dict_for_generation  s²   ø€ Ø%ð dð d�Ø˜.Ò(Ð(¨^¸CÔ-@Ô-EÈÒ-JÐ-JØ*8¸Ô*=×*OÒ*OÐP[ÐabÐ*OÑ*cÔ*c�N 3Ñ'Ð'à" 3Ô'Ð3Ý" >°#Ô#6½¼ÑEÔEð 4à ;Ð.Ð.à*8¸Ô*=×*OÒ*OÐP[ÐabÐ*OÑ*cÔ*c�N 3Ñ'øØ!Ð!r<   r   r¯   Úencoder_outputszMIf `is_encoder_decoder` is True, make sure that `encoder_outputs` is defined.)rc  rX  rˆ   )r8   ry  rz  rÕ  rf  r�  r’  r‘  s   `` ``  @r;   Ú_expand_inputs_for_generationz;Glm4vForConditionalGeneration._expand_inputs_for_generationÙ  s  øøøøø€ ð ˜!ÒÐØ˜lÐ*Ð*àwÐwÐwˆð+	"ð +	"ð +	"ð +	"ð +	"ð +	"ð +	"ð +	"ðZ
	"ð 
	"ð 
	"ð 
	"ð 
	"ð 
	"ð :Ð9¸,ÑGÔGˆàÐ Ø!×3Ò3°KÀQÐ3ÑGÔGˆIà2Ð2°<Ñ@Ô@ˆàð 	kØ×ÒÐ 1Ñ2Ô2Ð:Ý Ð!pÑqÔqÐqØ.IÐ.IÈ,ÐWhÔJiÑ.jÔ.jˆLÐ*Ñ+à˜,Ð&Ð&r<   rf   )NNNNNNNNNNNr   )r%   FN)rO   rP   rQ   Ú_tied_weights_keysrN  r1   r   r4   rˆ  r”  r   r   rL   r   r-  r1  r   rS   r	   rO  rÀ   rQ  rJ   rc  rx  rj   Údictr¥   r   r”  rT   rU   s   @r;   rS  rS  ÿ  sh  ø€ € € € € Ø*Ð,VÐWÐàÐðð ð ð ð ð ð 37ð
\ð 
\à"Ô.ð
\ð Ô(¨4Ñ/ð
\ð Ð+Ô,ð	
\ð
 
Ð+Ñ	+ð
\ð 
\ð 
\ñ „^ð
\ð ð 37ð
Uð 
UàÔ'ð
Uð Ô(¨4Ñ/ð
Uð Ð+Ô,ð	
Uð
 
Ð+Ñ	+ð
Uð 
Uð 
Uñ „^ð
Uð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø,0Ø8<Ø26Ø26Ø48Ø-.ðR
ð R
àÔ# dÑ*ðR
ð œ tÑ+ðR
ð Ô&¨Ñ-ð	R
ð
  ™ðR
ð Ô(¨4Ñ/ðR
ð Ô  4Ñ'ðR
ð ”l TÑ)ðR
ð #Ô.°Ñ5ðR
ð Ô(¨4Ñ/ðR
ð Ô(¨4Ñ/ðR
ð !œ?¨TÑ1ðR
ð ˜eœlÑ*ðR
ð Ð+Ô,ðR
ð 
Ð,Ñ	,ðR
ð R
ð R
ñ „^ñ ÔðR
ðh$ð $ð $ð $ð $ðR .2ð6*ð 6*àÔ# dÑ*ð6*ð ”| dÑ*ð6*ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð	6*ð 6*ð 6*ð 6*ðt Ø#(Ø-1ð	V'ð V'àðV'ð !ðV'ð Ô# dÑ*ð	V'ð 
ˆuÔ  c¨3 h¤Ð/Ô	0ðV'ð V'ð V'ð V'ð V'ð V'ð V'ð V'r<   rS  )rS  ræ  r�  rÉ  r¬  )r  )r%   )^r  Úcollections.abcr   Údataclassesr   Útypingr   r4   Útorch.nnr2   Útorch.nn.functionalr  rÚ   r   Úactivationsr   Úcache_utilsr	   r
   Ú
generationr   Úintegrationsr   Úmasking_utilsr   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.deprecationr   Úutils.genericr   r   r   r    r!   Úutils.output_capturingr"   Úvision_utilsr#   r$   Úconfiguration_glm4vr&   r'   r(   ÚModuler+   rW   rl   r|   r¹   rÂ   rö   rS   rL   rÿ   rÀ   r  rR   r  r  rC  rN  rf  rn  rp  r€  rŠ  r–  r�  r¬  rÉ  ræ  rQ  rS  Ú__all__r_  r<   r;   ú<module>r¯     s‹  ðð( Ð Ð Ð Ø $Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )Ø 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø /Ð /Ð /Ð /Ð /Ð /Ø BÐ BÐ BÐ BÐ BÐ BØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø kÐ kÐ kÐ kÐ kÐ kÐ kÐ kÐ kÐ kØ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aØ 0Ð 0Ð 0Ð 0Ð 0Ð 0ðð ð ð ð ð ð ð ð ð ð ð ð ð ð 6Ð 5Ð 5Ð 5Ð 5Ð 5Ø QÐ QÐ QÐ QÐ QÐ QÐ QÐ QØ PÐ PÐ PÐ PÐ PÐ PÐ PÐ PÐ PÐ Pð Ð˜YÑ'Ô'ðJð Jð Jð Jð J�2”9ñ Jô Jñ (Ô'ðJð(fð fð fð fð f�B”Iñ fô fð fðð ð ð ð ˜BœIñ ô ð ð(=5ð =5ð =5ð =5ð =5 ¤ñ =5ô =5ð =5ð@fð fð fð fð f˜RœYñ fô fð fð"Gð Gð Gð Gð G˜BœIñ Gô Gð GðT(ð (ð (ðØ„|ðØœðØ+0¬<ðØ>C¼lðà
ˆ5Œ<˜œÐ%Ô&ðð ð ð ð	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð& ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð2Pð Pð Pð Pð P˜2œ9ñ Pô Pð Pðfð ð ð ð Ð1ñ ô ð ð<@1ð @1ð @1ð @1ð @1˜rœyñ @1ô @1ð @1ðF6ð 6ð 6ð!ð !ð !ð !ðHC)ð C)ð C)ð C)ð C)˜œñ C)ô C)ð C)ðL)ð )ð )ð )ð )�2”9ñ )ô )ð )ð$/ð /ð /ð /ð /Ð6ñ /ô /ð /ðd Ø
ð0ð 0ð 0ð 0ð 0Ð6ñ 0ô 0ñ „ñ „ð0ð ð'ð 'ð 'ð 'ð '˜?ñ 'ô 'ñ „ð'ðY
ð Y
ð Y
ð Y
ð Y
Ð+ñ Y
ô Y
ð Y
ðx ðe
ð e
ð e
ð e
ð e
Ð)ñ e
ô e
ñ „ðe
ðP ðc
ð c
ð c
ð c
ð c
Ð%ñ c
ô c
ñ „ðc
ðL Ø
ð0ð 0ð 0ð 0ð 0Ð"8ñ 0ô 0ñ „ñ „ð0ðp'ð p'ð p'ð p'ð p'Ð$8¸/ñ p'ô p'ð p'ðf	 xÐ
wÐ
w€€€r<   