§
    9²¸jI  ã                   óØ  — d Z ddlmZ ddlZddlmZ ddlmZ ddlmZm	Z	 ddl
mZ dd	lmZ dd
lmZmZ ddlmZ ddlmZmZmZ ddlmZ ddlmZmZmZ ddlmZ ddlm Z   ej!        e"¦  «        Z# G d„ dej$        ¦  «        Z%d„ Z& ed¦  «        d/d„¦   «         Z'	 d0dej$        dej(        dej(        dej(        dej(        dz  de)de)fd„Z* e	e'¦  «         G d „ d!ej$        ¦  «        ¦   «         Z+ G d"„ d#ej$        ¦  «        Z, G d$„ d%ej$        ¦  «        Z- G d&„ d'e¦  «        Z. G d(„ d)ej$        ¦  «        Z/e G d*„ d+e¦  «        ¦   «         Z0d,„ Z1e G d-„ d.e0¦  «        ¦   «         Z2d.d+gZ3dS )1zPyTorch Pixtral model.é    )ÚCallableN)Únné   )ÚACT2FN)Úuse_kernel_forward_from_hubÚuse_kernelized_func)ÚGradientCheckpointingLayer)ÚBaseModelOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)Údeprecate_kwarg)Úis_flash_attention_requestedÚmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )ÚPixtralVisionConfigc            	       óê   ‡ — e Zd ZdZ edd¬¦  «        ddefˆ fd„¦   «         Ze edd¬¦  «        	 ddedee	j
        ef         fd	„¦   «         ¦   «         Z e	j        ¦   «         d
„ ¦   «         Zd„ Zˆ xZS )ÚPixtralVisionRotaryEmbeddingzý
    Simple axial 2D rope with same freqs used for H and W grids. The freqs are
    pre-computed using `head-dim//4` which is later used to concat H and W positions.
    The final angles rotate over the whole head dim, no partial rotation involved.
    Údevicez5.18)ÚversionNÚconfigc                 ó°  •— t          ¦   «                              ¦   «          || _        | j        j        d         | _        | j        }| j        dk    r#t          | j        j        › d| j        › �¦  «        ‚ || j        |¦  «        \  }| _	        t          j        |d¬¦  «        | _        t          j        |                     ¦   «         d¬¦  «        | _        d S )NÚ	rope_typeÚaxialz) supports only axial rope, but requested F)Ú
persistent)ÚsuperÚ__init__r   Úrope_parametersr   Úcompute_axial_rope_parametersÚ
ValueErrorÚ	__class__Ú__name__Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)Úselfr   r   Úrope_init_fnr*   r&   s        €ú|/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/pixtral/modeling_pixtral.pyr"   z%PixtralVisionRotaryEmbedding.__init__-   sÃ   ø€ å‰Œ×ÒÑÔÐØˆŒàœÔ4°[ÔAˆŒØ!%Ô!CˆØŒ>˜WÒ$Ð$Ý ¤Ô 7ÐrÐrÐbfÔbpÐrÐrÑsÔsÐsØ+7¨<¸¼ÀVÑ+LÔ+LÑ(ˆ�$Ô(åœ	 (°uÐ=Ñ=Ô=ˆŒÝ!#¤¨8¯>ª>Ñ+;Ô+;ÈÐ!NÑ!NÔ!NˆÔÐÐó    Úreturnc                 ó4  — | j         d         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «        |z  z  z  }t	          j        |ddd…         |ddd…         g¦  «        }|                     |¦  «        |fS )	a»  
        Computes the inverse frequencies according to the axial RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNg      ð?r   é   ©Údtyper   )	r#   ÚgetattrÚhidden_sizeÚnum_attention_headsÚtorchÚarangeÚfloatÚcatÚto)r   r   ÚkwargsÚbaseÚdimÚattention_factorr*   Úinv_freq_2ds           r/   r$   z:PixtralVisionRotaryEmbedding.compute_axial_rope_parameters;   s¢   € ð Ô% lÔ3ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐà˜$¥5¤<°°3¸ÅÄÐ#MÑ#MÔ#MÐPSÑ#SÑTÑUˆÝ”i ¨!¨$¨Q¨$¤°¸!¸$¸Q¸$´Ð @ÑAÔAˆà�~Š~˜fÑ%Ô%Ð'7Ð7Ð7r0   c                 óv  — | j         d                              ¦   «         }|d                              ¦   «         }t          |j        j        t
          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  ||z  }|                     ¦   «         | j        z  }| 	                    ¦   «         | j        z  }d d d ¦  «         n# 1 swxY w Y   |  
                    |¦  «        }|  
                    |¦  «        }|                     |j        ¦  «        |                     |j        ¦  «        fS )N)N.).NÚmpsÚcpuF)Údevice_typeÚenabled)r*   r=   Ú
isinstancer   ÚtypeÚstrr   Úcosr(   ÚsinÚrecomposition_frequenciesr?   r7   )	r-   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedrH   ÚfreqsrM   rN   s	            r/   Úforwardz$PixtralVisionRotaryEmbedding.forwardS   sJ  € à œM¨)Ô4×:Ò:Ñ<Ô<ÐØ ,¨YÔ 7× =Ò =Ñ ?Ô ?Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆÝ¨¸UÐCÑCÔCð 	7ð 	7Ø)Ð,=Ñ=ˆEØ—)’)‘+”+ Ô 6Ñ6ˆCØ—)’)‘+”+ Ô 6Ñ6ˆCð	7ð 	7ð 	7ñ 	7ô 	7ð 	7ð 	7ð 	7ð 	7ð 	7ð 	7øøøð 	7ð 	7ð 	7ð 	7ð
 ×,Ò,¨SÑ1Ô1ˆØ×,Ò,¨SÑ1Ô1ˆØ�vŠv�a”g‰Œ §¢ q¤w¡¤Ð/Ð/s   Â>CÃCÃCc                 óÄ   — d„ t          |                     dd¬¦  «        ¦  «        D ¦   «         \  }}t          j        ||gd¬¦  «        }t          j        ||gd¬¦  «        S )z]
        Recompose the frequencies into the final spatial layout used per each grid.
        c              3   ó:   K  — | ]\  }}|d d …|dz  f         V — Œd S )Nr5   © )Ú.0ÚiÚms      r/   ú	<genexpr>zIPixtralVisionRotaryEmbedding.recomposition_frequencies.<locals>.<genexpr>g   s6   è è € ÐSÐS©$¨!¨Q˜!˜A˜A˜A˜q 1™u˜Hœ+ÐSÐSÐSÐSÐSÐSr0   r5   éÿÿÿÿ©rB   )Ú	enumerateÚchunkr;   r>   )r-   ÚfreqÚfreq_hÚfreq_wÚfreq_hws        r/   rO   z6PixtralVisionRotaryEmbedding.recomposition_frequenciesb   sg   € ð
 TÐSµ)¸D¿JºJÀqÈb¸JÑ<QÔ<QÑ2RÔ2RÐSÑSÔS‰ˆ�Ý”)˜V VÐ,°"Ð5Ñ5Ô5ˆÝŒy˜' 7Ð+°Ð4Ñ4Ô4Ð4r0   ©N)r'   Ú
__module__Ú__qualname__Ú__doc__r   r   r"   ÚstaticmethodÚtupler;   ÚTensorr=   r$   Úno_gradrU   rO   Ú__classcell__©r&   s   @r/   r   r   &   s  ø€ € € € € ðð ð €_�X vÐ.Ñ.Ô.ðOð OÐ2ð Oð Oð Oð Oð Oñ /Ô.ðOð Ø€_�X vÐ.Ñ.Ô.à,0ð8ð 8Ø#ð8à	ˆuŒ|˜UÐ"Ô	#ð8ð 8ð 8ñ /Ô.ñ „\ð8ð, €U„]�_„_ð0ð 0ñ „_ð0ð5ð 5ð 5ð 5ð 5ð 5ð 5r0   r   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..Nr]   r5   r^   )Úshaper;   r>   )rP   Úx1Úx2s      r/   Úrotate_halfrs   m   s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'r0   Úrotary_pos_embc                 ó¾   — |                      |¦  «        }|                      |¦  «        }| |z  t          | ¦  «        |z  z   }||z  t          |¦  «        |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezers   )ÚqÚkrM   rN   Úunsqueeze_dimÚq_embedÚk_embeds          r/   Úapply_rotary_pos_embr|   t   sc   € ð( �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�GÐÐr0   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutc                 óÀ  — t          j        ||                     dd¦  «        ¦  «        |z  }|�||z   }t          j                             |dt           j        ¬¦  «                             |j        ¦  «        }t          j         	                    ||| j
        ¬¦  «        }t          j        ||¦  «        }	|	                     dd¦  «                             ¦   «         }	|	|fS )Nr]   éþÿÿÿ)rB   r7   )ÚpÚtrainingr   r5   )r;   ÚmatmulÚ	transposer   Ú
functionalÚsoftmaxÚfloat32r?   r7   r„   rˆ   Ú
contiguous)
r~   r   r€   r�   r‚   rƒ   r„   r@   Úattn_weightsÚattn_outputs
             r/   Úeager_attention_forwardr‘   �   sÃ   € õ ”<  s§}¢}°R¸Ñ'<Ô'<Ñ=Ô=ÀÑG€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r0   c                   ó¾   ‡ — e Zd ZdZˆ fd„Z	 	 d
dej        dej        dz  deej        ej        f         dz  dee	         deej        ej        dz  f         f
d	„Z
ˆ xZS )ÚPixtralAttentionzI
    Multi-headed attention compatible with ALL_ATTENTION_FUNCTIONS.
    c                 ó0  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        | j        | j        z  | _        d| _        | j        dz  | _	        d| _        |j
        | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _        d S )NFg      à¿©Úbias)r!   r"   r   r9   Ú	embed_dimr:   Ú	num_headsr4   Ú	is_causalrƒ   Úattention_dropoutr„   r   ÚLinearÚk_projÚv_projÚq_projÚo_proj©r-   r   r&   s     €r/   r"   zPixtralAttention.__init__­   sÞ   ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ+ˆŒØÔ3ˆŒØœ¨$¬.Ñ8ˆŒØˆŒà”} dÑ*ˆŒØˆŒàÔ/ˆŒå”i ¤°´ÀUÐKÑKÔKˆŒÝ”i ¤°´ÀUÐKÑKÔKˆŒÝ”i ¤°´ÀUÐKÑKÔKˆŒÝ”i ¤°´ÀUÐKÑKÔKˆŒˆˆr0   NÚhidden_statesr‚   Úposition_embeddingsr@   r1   c                 óD  — |                      ¦   «         \  }}}|                      |¦  «        }|                      |¦  «        }	|                      |¦  «        }
|                     ||| j        | j        ¦  «                             dd¦  «        }|	                     ||| j        | j        ¦  «                             dd¦  «        }	|
                     ||| j        | j        ¦  «                             dd¦  «        }
|\  }}t          ||	||d¬¦  «        \  }}	t          j
        | j        j        t          ¦  «        } || ||	|
|f| j        sdn| j        | j        dœ|¤Ž\  }}|                     ||d¦  «                             ¦   «         }|                      |¦  «        }||fS )z#Input shape: Batch x Time x Channelr   r5   r   )ry   r}   )r„   rƒ   r]   )Úsizerž   rœ   r�   Úviewr˜   r4   rŠ   r|   r   Úget_interfacer   Ú_attn_implementationr‘   rˆ   r„   rƒ   ÚreshaperŽ   rŸ   )r-   r¡   r‚   r¢   r@   Ú
batch_sizeÚpatchesÚ_Úquery_statesÚ
key_statesÚvalue_statesrM   rN   Úattention_interfacer�   r�   s                   r/   rU   zPixtralAttention.forward¿   sµ  € ð "/×!3Ò!3Ñ!5Ô!5Ñˆ
�G˜Qà—{’{ =Ñ1Ô1ˆØ—[’[ Ñ/Ô/ˆ
Ø—{’{ =Ñ1Ô1ˆà#×(Ò(¨°W¸d¼nÈdÌmÑ\Ô\×fÒfÐghÐjkÑlÔlˆØ—_’_ Z°¸$¼.È$Ì-ÑXÔX×bÒbÐcdÐfgÑhÔhˆ
Ø#×(Ò(¨°W¸d¼nÈdÌmÑ\Ô\×fÒfÐghÐjkÑlÔlˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÐjkÐ#lÑ#lÔ#lÑ ˆ�jå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}Ð>�C�C°$´,Ø”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð "×)Ò)¨*°g¸rÑBÔB×MÒMÑOÔOˆØ—k’k +Ñ.Ô.ˆà˜LÐ(Ð(r0   ©NN)r'   rf   rg   rh   r"   r;   rk   rj   r   r   rU   rm   rn   s   @r/   r“   r“   §   sÍ   ø€ € € € € ðð ðLð Lð Lð Lð Lð* /3ØHLð	()ð ()à”|ð()ð œ tÑ+ð()ð # 5¤<°´Ð#=Ô>ÀÑEð	()ð
 Ð+Ô,ð()ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð()ð ()ð ()ð ()ð ()ð ()ð ()ð ()r0   r“   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )Ú
PixtralMLPc                 ó˜  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _	        t          |j                 | _        d S )NFr•   )r!   r"   r   r9   Úintermediate_sizer   r›   Ú	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fnr    s     €r/   r"   zPixtralMLP.__init__ì   s¦   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ!'Ô!9ˆÔÝœ 4Ô#3°TÔ5KÐRWÐXÑXÔXˆŒÝ”y Ô!1°4Ô3IÐPUÐVÑVÔVˆŒÝœ 4Ô#9¸4Ô;KÐRWÐXÑXÔXˆŒÝ˜VÔ.Ô/ˆŒˆˆr0   c                 ó¨   — |                       |                      |                      |¦  «        ¦  «        |                      |¦  «        z  ¦  «        }|S re   )r·   r¹   rµ   r¶   )r-   rP   r·   s      r/   rU   zPixtralMLP.forwardö   sA   € Ø—N’N 4§;¢;¨t¯~ª~¸aÑ/@Ô/@Ñ#AÔ#AÀDÇLÂLÐQRÁOÄOÑ#SÑTÔTˆ	ØÐr0   )r'   rf   rg   r"   rU   rm   rn   s   @r/   r²   r²   ë   sG   ø€ € € € € ð0ð 0ð 0ð 0ð 0ðð ð ð ð ð ð r0   r²   c                   óT   ‡ — e Zd Zd	deddfˆ fd„Zdej        dej        fd„Zd„ Zˆ xZ	S )
ÚPixtralRMSNormç�íµ ÷Æ°>Úepsr1   Nc                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )z=
        PixtralRMSNorm is equivalent to T5LayerNorm
        N)r!   r"   r   Ú	Parameterr;   ÚonesÚweightÚvariance_epsilon)r-   r9   r¾   r&   s      €r/   r"   zPixtralRMSNorm.__init__ý   sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐr0   r¡   c                 ó  — |j         }|                     t          j        ¦  «        }|                     d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        |                     |¦  «        z  S )Nr5   r]   T)Úkeepdim)	r7   r?   r;   r�   ÚpowÚmeanÚrsqrtrÃ   rÂ   )r-   r¡   Úinput_dtypeÚvariances       r/   rU   zPixtralRMSNorm.forward  s|   € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØ ×$Ò$ QÑ'Ô'×,Ò,¨R¸Ð,Ñ>Ô>ˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆØŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:r0   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)rj   rÂ   rp   rÃ   ©r-   s    r/   Ú
extra_reprzPixtralRMSNorm.extra_repr  s&   € Ý˜œÔ)Ñ*Ô*ÐIÐI°$Ô2GÐIÐIÐIr0   )r½   )
r'   rf   rg   r=   r"   r;   rk   rU   rÍ   rm   rn   s   @r/   r¼   r¼   ü   sŒ   ø€ € € € € ð$ð $¨ð $¸$ð $ð $ð $ð $ð $ð $ð; U¤\ð ;°e´lð ;ð ;ð ;ð ;ðJð Jð Jð Jð Jð Jð Jr0   r¼   c                   ó’   ‡ — e Zd Zˆ fd„Z	 d	dej        dej        deej        ej        f         dz  dee         dej        f
d„Z	ˆ xZ
S )
ÚPixtralAttentionLayerc                 ó  •— t          ¦   «                              ¦   «          t          |j        d¬¦  «        | _        t          |¦  «        | _        t          |¦  «        | _        t          |j        d¬¦  «        | _	        d S )Nçñhãˆµøä>©r¾   )
r!   r"   r¼   r9   Úattention_normr²   Úfeed_forwardr“   Ú	attentionÚffn_normr    s     €r/   r"   zPixtralAttentionLayer.__init__  sk   ø€ Ý‰Œ×ÒÑÔÐÝ,¨VÔ-?ÀTÐJÑJÔJˆÔÝ& vÑ.Ô.ˆÔÝ)¨&Ñ1Ô1ˆŒÝ& vÔ'9¸tÐDÑDÔDˆŒˆˆr0   Nr¡   r‚   r¢   r@   r1   c                 óÈ   — |}|                       |¦  «        } | j        d|||dœ|¤Ž\  }}||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|S )aR  
        Args:
            hidden_states (`torch.FloatTensor`):
                Input to the layer of shape `(batch, seq_len, embed_dim)`.
            attention_mask (`torch.FloatTensor`):
                Attention mask of shape `(batch, 1, q_len, k_v_seq_len)` where padding elements are indicated by very large negative values.
        )r¡   r‚   r¢   rX   )rÓ   rÕ   rÖ   rÔ   )r-   r¡   r‚   r¢   r@   Úresidualr«   s          r/   rU   zPixtralAttentionLayer.forward  s–   € ð !ˆà×+Ò+¨MÑ:Ô:ˆØ)˜4œ>ð 
Ø'Ø)Ø 3ð
ð 
ð ð	
ð 
Ñˆ�qð ! =Ñ0ˆà ˆØŸš mÑ4Ô4ˆØ×)Ò)¨-Ñ8Ô8ˆØ  =Ñ0ˆàÐr0   re   )r'   rf   rg   r"   r;   rk   rj   r   r   rU   rm   rn   s   @r/   rÏ   rÏ     s¬   ø€ € € € € ðEð Eð Eð Eð Eð IMð	ð à”|ðð œðð # 5¤<°´Ð#=Ô>ÀÑEð	ð
 Ð+Ô,ðð 
Œðð ð ð ð ð ð ð r0   rÏ   c            
       óˆ   ‡ — e Zd Zˆ fd„Z	 	 ddej        dz  deej        ej        f         dz  dee         dee	z  fd„Z
ˆ xZS )	ÚPixtralTransformerc                 ó&  •— t          ¦   «                              ¦   «          || _        t          j                             ¦   «         | _        t          |j        ¦  «        D ])}| j         	                    t          |¦  «        ¦  «         Œ*d| _        d S )NF)r!   r"   r   r;   r   Ú
ModuleListÚlayersÚrangeÚnum_hidden_layersÚappendrÏ   Úgradient_checkpointing)r-   r   r«   r&   s      €r/   r"   zPixtralTransformer.__init__:  s   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”h×)Ò)Ñ+Ô+ˆŒÝ�vÔ/Ñ0Ô0ð 	>ð 	>ˆAØŒK×ÒÕ4°VÑ<Ô<Ñ=Ô=Ð=Ð=Ø&+ˆÔ#Ð#Ð#r0   Nr‚   r¢   r@   r1   c                 óR   — |}| j         D ]} |||fd|i|¤Ž}Œt          |¬¦  «        S )aI  
        Args:
            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
                Embeddings which serve as input to the Transformer.
            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
        r¢   )Úlast_hidden_state)rÝ   r
   )r-   Úinputs_embedsr‚   r¢   r@   r¡   Úencoder_layers          r/   rU   zPixtralTransformer.forwardB  s_   € ð& &ˆØ!œ[ð 	ð 	ˆMØ)˜MØØðð ð %8ðð ð	ð ˆMˆMõ °Ð?Ñ?Ô?Ð?r0   r°   )r'   rf   rg   r"   r;   rk   rj   r   r   r
   rU   rm   rn   s   @r/   rÚ   rÚ   9  s³   ø€ € € € € ð,ð ,ð ,ð ,ð ,ð /3ØHLð	@ð @ð œ tÑ+ð@ð # 5¤<°´Ð#=Ô>ÀÑEð	@ð
 Ð+Ô,ð@ð 
�Ñ	 ð@ð @ð @ð @ð @ð @ð @ð @r0   rÚ   c                   óJ   — e Zd ZU eed<   dZdZdZdZdZ	dZ
dZdZdgZeedœZdS )	ÚPixtralPreTrainedModelr   ÚmodelÚpixel_values)ÚimageTrÏ   )r¡   Ú
attentionsN)r'   rf   rg   r   Ú__annotations__Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_supports_attention_backendÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_no_split_modulesrÏ   r“   Ú_can_record_outputsrX   r0   r/   rç   rç   a  sh   € € € € € € àÐÐÑØÐØ$€OØ!ÐØ&*Ð#Ø"&ÐØÐØ€NØÐØ0Ð1Ðà.Ø&ðð ÐÐÐr0   rç   c                 óú  — |j         }|j        }|j        d         }t          j        |¦  «        j        }t          j        ||f|||¬¦  «        }t          j        | ¦  «                             d¦  «        }t          j        dg| d d…         z   ¦  «                             d¦  «        }t          ||¦  «        D ]\  }	}
d||	|
…|	|
…f<   Œ|d d d d …d d …f          
                    |j        d         ddd¦  «        }|S )Nr   )Ú
fill_valuer7   r   r]   r   )r7   r   rp   r;   ÚfinfoÚminÚfullÚtensorÚcumsumÚzipÚexpand)Úpatch_embeds_listrü   r7   r   Úseq_lenÚd_minÚcausal_maskÚblock_end_idxÚblock_start_idxÚstartÚends              r/   Úgenerate_block_attention_maskr  s  s	  € ØŒL€EØŒ]€FØŒl˜1Œo€GÝŒK˜ÑÔÔ"€EÝ”*˜g wÐ/¸EÈÐW]Ð^Ñ^Ô^€Kå”LÐ!2Ñ3Ô3×:Ò:¸2Ñ>Ô>€MÝ”l A 3Ð):¸3¸B¸3Ô)?Ñ#?Ñ@Ô@×GÒGÈÑKÔK€OÝ˜/¨=Ñ9Ô9ð .ð .‰
ˆˆsØ,-ˆ�E˜#�I˜u S˜yÐ(Ñ)Ð)à˜d D¨!¨!¨!¨Q¨Q¨QÐ.Ô/×6Ò6°v´|ÀA´ÈÈ2ÈrÑRÔR€KØÐr0   c                   ó    ‡ — e Zd ZdZˆ fd„Zd„ Zeee	 d
de	j
        de	j
        dz  dee         deez  fd	„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚPixtralVisionModelÚvision_encoderc                 ó‚  •— t          ¦   «                              |¦  «         || _        t          j        |j        |j        |j        |j        d¬¦  «        | _        |j        | _        t          |j        d¬¦  «        | _
        t          |¦  «        | _        t          |¦  «        | _        |                      ¦   «          d S )NF)Úin_channelsÚout_channelsÚkernel_sizeÚstrider–   rÑ   rÒ   )r!   r"   r   r   ÚConv2dÚnum_channelsr9   Ú
patch_sizeÚ
patch_convr¼   Úln_prerÚ   Útransformerr   Úpatch_positional_embeddingÚ	post_initr    s     €r/   r"   zPixtralVisionModel.__init__‡  s®   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒÝœ)ØÔ+ØÔ+ØÔ)ØÔ$Øð
ñ 
ô 
ˆŒð !Ô+ˆŒÝ$ VÔ%7¸TÐBÑBÔBˆŒÝ-¨fÑ5Ô5ˆÔÝ*FÀvÑ*NÔ*NˆÔ'à�ŠÑÔÐÐÐr0   c                 ó   — | j         S re   )r  rÌ   s    r/   Úget_input_embeddingsz'PixtralVisionModel.get_input_embeddings˜  s
   € ØŒÐr0   Nré   Úimage_sizesr@   r1   c                 óº  ‡ — |€|j         \  }}}}||fg|z  }‰ j        j        j        }‰                      |                     |¬¦  «        ¦  «        }	ˆ fd„t          |	|¦  «        D ¦   «         }
t          j        d„ |
D ¦   «         d¬¦  «                             d¦  «        }	‰  	                    |	¦  «        }	g }|
D ]³}t          j
        t          j        |j         d         |j        ¬¦  «        t          j        |j         d         |j        ¬¦  «        d	¬
¦  «        \  }}|                     t          j        |                     ¦   «         |                     ¦   «         gd¬¦  «        ¦  «         Œ´t          j        |d¬¦  «        }‰                      |	|¦  «        }t#          ‰ j        ¦  «        rd }nt'          d„ |
D ¦   «         |	¦  «        } ‰ j        |	f|||dœ|¤ŽS )Nr6   c                 ól   •— g | ]0\  }}|d d|d         ‰j         z  …d|d         ‰j         z  …f         ‘Œ1S ).Nr   r   )r  )rY   Úembedr¤   r-   s      €r/   ú
<listcomp>z.PixtralVisionModel.forward.<locals>.<listcomp>«  s[   ø€ ð 
ð 
ð 
á��tð �#Ð5˜$˜qœ' T¤_Ñ4Ð5Ð7U¸$¸q¼'ÀTÄ_Ñ:TÐ7UÐUÔVð
ð 
ð 
r0   c                 óB   — g | ]}|                      d ¦  «        j        ‘ŒS ©r   )ÚflattenÚT©rY   r‡   s     r/   r  z.PixtralVisionModel.forward.<locals>.<listcomp>±  s$   € Ð!LÐ!LÐ!L°Q !§)¢)¨A¡,¤,¤.Ð!LÐ!LÐ!Lr0   r   r^   r†   )r   r]   Úij)Úindexingc                 óD   — g | ]}|j         d          |j         d         z  ‘ŒS )r†   r]   )rp   r$  s     r/   r  z.PixtralVisionModel.forward.<locals>.<listcomp>Å  s)   € ÐFÐFÐF¨q�”˜”˜qœw rœ{Ñ*ÐFÐFÐFr0   )r‚   r¢   rQ   )rp   r  rÂ   r7   r?   rþ   r;   r>   rv   r  Úmeshgridr<   r   rà   Ústackr"  r  r   r   r  r  )r-   ré   r  r@   r©   r«   ÚheightÚwidthÚtarget_dtypeÚpatch_embedsr   rQ   ÚpatchÚhpos_idsÚwpos_idsr¢   r‚   s   `                r/   rU   zPixtralVisionModel.forward›  s$  ø€ ð ÐØ+7Ô+=Ñ(ˆJ˜˜6 5Ø" E˜?Ð+¨jÑ8ˆKð ”Ô-Ô3ˆØ—’ |§¢¸\ Ñ'JÔ'JÑKÔKˆð
ð 
ð 
ð 
å" <°Ñ=Ô=ð
ñ 
ô 
Ðõ ”yÐ!LÐ!LÐ:KÐ!LÑ!LÔ!LÐRSÐTÑTÔT×^Ò^Ð_`ÑaÔaˆØ—{’{ <Ñ0Ô0ˆð ˆØ&ð 	_ð 	_ˆEÝ!&¤Ý”˜Uœ[¨œ_°U´\ÐBÑBÔBÝ”˜Uœ[¨œ_°U´\ÐBÑBÔBØð"ñ "ô "ÑˆH�hð
 ×Ò¥¤¨X×-=Ò-=Ñ-?Ô-?À×AQÒAQÑASÔASÐ,TÐZ\Ð ]Ñ ]Ô ]Ñ^Ô^Ð^Ð^Ý”y °1Ð5Ñ5Ô5ˆØ"×=Ò=¸lÈLÑYÔYÐå'¨¬Ñ4Ô4ð 	à!ˆNˆNå:ØFÐFÐ4EÐFÑFÔFÈñô ˆNð  ˆtÔØð
à)Ø 3Ø%ð	
ð 
ð
 ð
ð 
ð 	
r0   re   )r'   rf   rg   rí   r"   r  r   r   r   r;   rk   r   r   rj   r
   rU   rm   rn   s   @r/   r
  r
  ƒ  sÆ   ø€ € € € € à(Ððð ð ð ð ð"ð ð ð  ØØð ,0ð0
ð 0
à”lð0
ð ”\ DÑ(ð0
ð Ð+Ô,ð	0
ð
 
�Ñ	 ð0
ð 0
ð 0
ñ „^ñ „_ñ  Ôð0
ð 0
ð 0
ð 0
ð 0
r0   r
  r!  )r}   )4rh   Úcollections.abcr   r;   r   Úactivationsr   Úintegrationsr   r   Úmodeling_layersr	   Úmodeling_outputsr
   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   r   Úutils.output_capturingr   Úconfiguration_pixtralr   Ú
get_loggerr'   ÚloggerÚModuler   rs   r|   rk   r=   r‘   r“   r²   r¼   rÏ   rÚ   rç   r  r
  Ú__all__rX   r0   r/   ú<module>rA     s¶  ðð Ð à $Ð $Ð $Ð $Ð $Ð $à €€€Ø Ð Ð Ð Ð Ð à !Ð !Ð !Ð !Ð !Ð !Ø LÐ LÐ LÐ LÐ LÐ LÐ LÐ LØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø /Ð /Ð /Ð /Ð /Ð /Ø FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø eÐ eÐ eÐ eÐ eÐ eÐ eÐ eÐ eÐ eØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6ð 
ˆÔ	˜HÑ	%Ô	%€ðC5ð C5ð C5ð C5ð C5 2¤9ñ C5ô C5ð C5ðN(ð (ð (ð ÐÐ-Ñ.Ô.ðð ð ñ /Ô.ððB ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð %ð %ð %ð. ÐÐ)Ñ*Ô*ð?)ð ?)ð ?)ð ?)ð ?)�r”yñ ?)ô ?)ñ +Ô*ð?)ðFð ð ð ð �”ñ ô ð ð"Jð Jð Jð Jð J�R”Yñ Jô Jð Jð(&ð &ð &ð &ð &Ð6ñ &ô &ð &ðR%@ð %@ð %@ð %@ð %@˜œñ %@ô %@ð %@ðP ðð ð ð ð ˜_ñ ô ñ „ðð"ð ð ð  ðJ
ð J
ð J
ð J
ð J
Ð/ñ J
ô J
ñ „ðJ
ðZ  Ð!9Ð
:€€€r0   