§
    9²¸jåÞ  ã                   óê  — d dl Z d dlmZ d dlmZ d dlZd dlmZ d dlmc m	Z
 d dlmZ ddlmZ ddlmZ ddlmZmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZ ddlmZm Z m!Z!m"Z"m#Z# ddl$m%Z%m&Z& ddl'm(Z(m)Z) ddl*m+Z+ ddl,m-Z-m.Z.m/Z/m0Z0m1Z1 ddl2m3Z3 ddl4m5Z5m6Z6 ddl7m8Z8 ddl9m:Z:m;Z;  e0j<        e=¦  «        Z> G d„ dej?        ¦  «        Z@ G d„ dej?        ¦  «        ZA G d„ dej        j?        ¦  «        ZB G d„ dej?        ¦  «        ZC G d„ d ejD        ¦  «        ZE G d!„ d"ej?        ¦  «        ZF G d#„ d$ej?        ¦  «        ZGd%ejH        d&ejH        d'ejH        d(eIejH        ejH        f         fd)„ZJd*ejH        d+eKd(ejH        fd,„ZL	 d`d.ej?        d/ejH        d0ejH        d1ejH        d2ejH        dz  d3eMd4eMfd5„ZN	 d`d.ej?        d/ejH        d0ejH        d1ejH        d2ejH        dz  d3eMd4eMfd6„ZO G d7„ d8ej?        ¦  «        ZP G d9„ d:e¦  «        ZQe. G d;„ d<e)¦  «        ¦   «         ZRe. G d=„ d>eR¦  «        ¦   «         ZS G d?„ d@eRe¦  «        ZT e.dA¬B¦  «        e G dC„ dDe#¦  «        ¦   «         ¦   «         ZU G dE„ dFej        j?        ¦  «        ZV G dG„ dHej?        ¦  «        ZWdI„ ZX G dJ„ dKej?        ¦  «        ZYdLejH        d/ejH        fdM„ZZd/ejH        d0ejH        dLejH        d(eIejH        ejH        f         fdN„Z[ G dO„ dPej?        ¦  «        Z\ G dQ„ dRej?        ¦  «        Z] G dS„ dTe¦  «        Z^ G dU„ dVej?        ¦  «        Z_ G dW„ dXej?        ¦  «        Z` G dY„ dZej?        ¦  «        Za G d[„ d\eR¦  «        Zb G d]„ d^eRe¦  «        Zcg d_¢ZddS )aé    N)ÚCallable)Ú	dataclass)ÚLlama4VisionConfigé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úcreate_causal_maskÚcreate_chunked_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚBaseModelOutputWithPastÚBaseModelOutputWithPoolingÚCausalLMOutputWithPastÚModelOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚloggingÚtorch_compilable_check)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )ÚLlama4ConfigÚLlama4TextConfigc                   óH   ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Zˆ xZS )ÚLlama4TextExpertsÚconfigc                 óÎ  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        | j        | _        t          j        t          j
        | j        | j        d| j        z  ¦  «        ¦  «        | _        t          j        t          j        | j        | j        | j        f¦  «        ¦  «        | _        t          |j                 | _        d S ©Né   )ÚsuperÚ__init__Únum_local_expertsÚnum_expertsÚintermediate_sizeÚhidden_sizeÚ
expert_dimÚnnÚ	ParameterÚtorchÚzerosÚgate_up_projÚemptyÚ	down_projr   Ú
hidden_actÚact_fn©Úselfr(   Ú	__class__s     €úz/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/llama4/modeling_llama4.pyr-   zLlama4TextExperts.__init__8   s²   ø€ Ý‰Œ×ÒÑÔÐØ!Ô3ˆÔØ!'Ô!9ˆÔØ!Ô-ˆÔØÔ0ˆŒÝœL­¬°TÔ5EÀtÔGWÐYZÐ]aÔ]lÑYlÑ)mÔ)mÑnÔnˆÔÝœ¥e¤k°4Ô3CÀTÄ_ÐVZÔVfÐ2gÑ&hÔ&hÑiÔiˆŒÝ˜VÔ.Ô/ˆŒˆˆó    Úhidden_statesÚreturnc                 ó\  — |                      | j        j        d         d| j        ¦  «        }t	          j        || j        ¦  «        }|                     dd¬¦  «        \  }}t	          j        ||                      |¦  «        z  | j        ¦  «        }|                      d| j        ¦  «        }|S )a2  
        This should really not be run on a single machine, as we are reaching compute bound:
        - the inputs are expected to be "sorted" per expert already.
        - the weights are viewed with another dim, to match num_expert, 1, shape * num_tokens, shape

        Args:
            hidden_states (torch.Tensor): (batch_size * token_num, hidden_size)
            selected_experts (torch.Tensor): (batch_size * token_num, top_k)
            routing_weights (torch.Tensor): (batch_size * token_num, top_k)
        Returns:
            torch.Tensor
        r   éÿÿÿÿr+   ©Údim)	Úviewr7   Úshaper1   r5   ÚbmmÚchunkr;   r9   )r=   rA   Úgate_upÚgateÚupÚnext_statess         r?   ÚforwardzLlama4TextExperts.forwardB   s˜   € ð &×*Ò*¨4Ô+<Ô+BÀ1Ô+EÀrÈ4ÔK[Ñ\Ô\ˆÝ”)˜M¨4Ô+<Ñ=Ô=ˆØ—=’= ¨�=Ñ+Ô+‰ˆˆbÝ”i  d§k¢k°$Ñ&7Ô&7Ñ!7¸$¼.ÑIÔIˆØ!×&Ò& r¨4Ô+;Ñ<Ô<ˆØÐr@   )	Ú__name__Ú
__module__Ú__qualname__r%   r-   r5   ÚTensorrO   Ú__classcell__©r>   s   @r?   r'   r'   7   sk   ø€ € € € € ð0Ð/ð 0ð 0ð 0ð 0ð 0ð 0ð U¤\ð °e´lð ð ð ð ð ð ð ð r@   r'   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚLlama4TextMLPNc                 ó\  •— t          ¦   «                              ¦   «          |€|j        }|| _        t	          j        |j        |d¬¦  «        | _        t	          j        |j        |d¬¦  «        | _        t	          j        ||j        d¬¦  «        | _	        t          |j                 | _        d S ©NF©Úbias)r,   r-   r0   r(   r3   ÚLinearr1   Ú	gate_projÚup_projr9   r   r:   Úactivation_fn)r=   r(   r0   r>   s      €r?   r-   zLlama4TextMLP.__init__Y   s™   ø€ Ý‰Œ×ÒÑÔÐàÐ$Ø &Ô 8ÐàˆŒÝœ 6Ô#5Ð7HÈuÐUÑUÔUˆŒÝ”y Ô!3Ð5FÈUÐSÑSÔSˆŒÝœÐ#4°fÔ6HÈuÐUÑUÔUˆŒÝ# FÔ$5Ô6ˆÔÐÐr@   c                 ó¨   — |                       |                      |¦  «        ¦  «        |                      |¦  «        z  }|                      |¦  «        S ©N)r_   r]   r^   r9   )r=   Úxr9   s      r?   rO   zLlama4TextMLP.forwarde   sB   € Ø×&Ò& t§~¢~°aÑ'8Ô'8Ñ9Ô9¸D¿LºLÈ¹O¼OÑKˆ	Ø�~Š~˜iÑ(Ô(Ð(r@   ra   ©rP   rQ   rR   r-   rO   rT   rU   s   @r?   rW   rW   X   sL   ø€ € € € € ð
7ð 
7ð 
7ð 
7ð 
7ð 
7ð)ð )ð )ð )ð )ð )ð )r@   rW   c                   ó8   ‡ — e Zd Zddefˆ fd„Zd„ Zd„ Zd„ Zˆ xZS )ÚLlama4TextL2Normç�íµ ÷Æ°>Úepsc                 óV   •— t          ¦   «                              ¦   «          || _        d S ra   )r,   r-   rg   )r=   rg   r>   s     €r?   r-   zLlama4TextL2Norm.__init__k   s$   ø€ Ý‰Œ×ÒÑÔÐØˆŒˆˆr@   c                 ó�   — |t          j        |                     d¦  «                             dd¬¦  «        | j        z   ¦  «        z  S ©Nr+   rD   T)Úkeepdim©r5   ÚrsqrtÚpowÚmeanrg   ©r=   rb   s     r?   Ú_normzLlama4TextL2Norm._normo   ó8   € Ø•5”;˜qŸušu Q™xœxŸ}š}¨R¸˜}Ñ>Ô>ÀÄÑIÑJÔJÑJÐJr@   c                 óv   — |                       |                     ¦   «         ¦  «                             |¦  «        S ra   )rq   ÚfloatÚtype_asrp   s     r?   rO   zLlama4TextL2Norm.forwardr   s*   € Ø�zŠz˜!Ÿ'š'™)œ)Ñ$Ô$×,Ò,¨QÑ/Ô/Ð/r@   c                 ó   — d| j         › �S )Nzeps=©rg   ©r=   s    r?   Ú
extra_reprzLlama4TextL2Norm.extra_repru   s   € Ø �d”hÐ Ð Ð r@   )rf   )	rP   rQ   rR   rt   r-   rq   rO   ry   rT   rU   s   @r?   re   re   j   sy   ø€ € € € € ðð ˜Eð ð ð ð ð ð ðKð Kð Kð0ð 0ð 0ð!ð !ð !ð !ð !ð !ð !r@   re   c                   ó2   ‡ — e Zd Zdˆ fd„	Zd„ Zd„ Zd„ Zˆ xZS )ÚLlama4TextRMSNormçñhãˆµøä>c                 ó¬   •— t          ¦   «                              ¦   «          || _        t          j        t          j        |¦  «        ¦  «        | _        dS )z<
        Llama4RMSNorm is equivalent to T5LayerNorm
        N)r,   r-   rg   r3   r4   r5   ÚonesÚweight)r=   r1   rg   r>   s      €r?   r-   zLlama4TextRMSNorm.__init__z   sA   ø€ õ 	‰Œ×ÒÑÔÐØˆŒÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒˆˆr@   c                 ó�   — |t          j        |                     d¦  «                             dd¬¦  «        | j        z   ¦  «        z  S rj   rl   rp   s     r?   rq   zLlama4TextRMSNorm._norm‚   rr   r@   c                 óŠ   — |                       |                     ¦   «         ¦  «                             |¦  «        }|| j        z  S ra   )rq   rt   ru   r   )r=   rb   Úoutputs      r?   rO   zLlama4TextRMSNorm.forward…   s6   € Ø—’˜AŸGšG™IœIÑ&Ô&×.Ò.¨qÑ1Ô1ˆØ˜œÑ#Ð#r@   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)Útupler   rH   rg   rx   s    r?   ry   zLlama4TextRMSNorm.extra_repr‰   s%   € Ý˜œÔ)Ñ*Ô*Ð<Ð<°$´(Ð<Ð<Ð<r@   )r|   )rP   rQ   rR   r-   rq   rO   ry   rT   rU   s   @r?   r{   r{   y   sm   ø€ € € € € ð<ð <ð <ð <ð <ð <ðKð Kð Kð$ð $ð $ð=ð =ð =ð =ð =ð =ð =r@   r{   c                   ó(   ‡ — e Zd Zˆ fd„Zˆ fd„Zˆ xZS )ÚLlama4Routerc                 ó”   •— t          ¦   «                              |j        |j        d¬¦  «         |j        | _        |j        | _        d S rY   )r,   r-   r1   r.   r/   Únum_experts_per_tokÚtop_kr<   s     €r?   r-   zLlama4Router.__init__Ž   sA   ø€ Ý‰Œ×Ò˜Ô+¨VÔ-EÈEÐÑRÔRÐRØ!Ô3ˆÔØÔ/ˆŒ
ˆ
ˆ
r@   c                 ó–  •— t          ¦   «                              |¦  «        }t          j        || j        d¬¦  «        \  }}t          j        |t          d¦  «        ¦  «                             d||¦  «        }t          j        j	         
                    |                     ¦   «         ¦  «                             |j        ¦  «        }||fS )Nr#   rE   z-inf)r,   rO   r5   Útopkr‰   Ú	full_likert   Úscatter_r3   Ú
functionalÚsigmoidÚtoÚdtype)r=   rA   Úrouter_logitsÚrouter_top_valueÚrouter_indicesÚrouter_scoresr>   s         €r?   rO   zLlama4Router.forward“   s¢   ø€ Ý™œŸš¨Ñ6Ô6ˆÝ+0¬:°mÀTÄZÐUVÐ+WÑ+WÔ+WÑ(Ð˜.Ýœ¨µu¸V±}´}ÑEÔE×NÒNÈqÐR`ÐbrÑsÔsˆÝœÔ+×3Ò3°M×4GÒ4GÑ4IÔ4IÑJÔJ×MÒMÈmÔNaÑbÔbˆØ˜mÐ+Ð+r@   rc   rU   s   @r?   r†   r†   �   sQ   ø€ € € € € ð0ð 0ð 0ð 0ð 0ð
,ð ,ð ,ð ,ð ,ð ,ð ,ð ,ð ,r@   r†   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚLlama4TextMoec                 ó  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        t          |¦  «        | _	        t          |¦  «        | _        t          |¦  «        | _        d S ra   )r,   r-   rˆ   r‰   r1   Ú
hidden_dimr.   r/   r'   Úexpertsr†   ÚrouterrW   Úshared_expertr<   s     €r?   r-   zLlama4TextMoe.__init__�   sl   ø€ Ý‰Œ×ÒÑÔÐØÔ/ˆŒ
Ø Ô,ˆŒØ!Ô3ˆÔÝ(¨Ñ0Ô0ˆŒÝ" 6Ñ*Ô*ˆŒÝ*¨6Ñ2Ô2ˆÔÐÐr@   c                 ó  — |                      d| j        ¦  «        }|                      |¦  «        \  }}|                     |j        d         d¦  «        }||                     dd¦  «                              dd¦  «        z  }|                      |¦  «        }|                      |¦  «        }|                     |                      |j        d         d|j        d         ¦  «         	                    d¬¦  «        ¦  «         ||fS )NrD   r#   r   rE   )
Úreshaper™   r›   ÚrepeatrH   Ú	transposerš   rœ   Úadd_Úsum)r=   rA   r•   r’   Ú	routed_inÚ
routed_outÚouts          r?   rO   zLlama4TextMoe.forward¦   sì   € Ø%×-Ò-¨b°$´/ÑBÔBˆØ'+§{¢{°=Ñ'AÔ'AÑ$ˆ�}Ø!×(Ò(¨Ô)<¸QÔ)?ÀÑCÔCˆ	Ø × 7Ò 7¸¸1Ñ =Ô =× EÒ EÀbÈ!Ñ LÔ LÑLˆ	Ø—\’\ )Ñ,Ô,ˆ
Ø× Ò  Ñ/Ô/ˆØ�Š�×#Ò# MÔ$7¸Ô$:¸BÀ
Ô@PÐQSÔ@TÑUÔU×YÒYÐ^_ÐYÑ`Ô`ÑaÔaÐaØ�MÐ!Ð!r@   rc   rU   s   @r?   r—   r—   œ   sG   ø€ € € € € ð3ð 3ð 3ð 3ð 3ð"ð "ð "ð "ð "ð "ð "r@   r—   c            	       óî   ‡ — e Zd Z edd¬¦  «        d
defˆ fd„¦   «         Ze edd¬¦  «        d
dedeej	        e
f         fd„¦   «         ¦   «         Z ej        ¦   «         ed	„ ¦   «         ¦   «         Zˆ xZS )ÚLlama4TextRotaryEmbeddingÚdevicez5.18)ÚversionNr(   c                 ó¾  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        | j        j        d         | _        | j        }| j        dk    rt          | j                 } || j        |¦  «        \  }| _
        t          j        |d¬¦  «        | _        t          j        |                     ¦   «         d¬¦  «        | _        d S )NÚ	rope_typeÚdefaultF©Ú
persistent)r,   r-   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr(   Úrope_parametersr«   Úcompute_default_rope_parametersr   Úattention_scalingr3   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r=   r(   r¨   Úrope_init_fnr¶   r>   s        €r?   r-   z"Llama4TextRotaryEmbedding.__init__´   sÀ   ø€ å‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!àˆŒàœÔ4°[ÔAˆŒØ!%Ô!EˆØŒ>˜YÒ&Ð&Ý.¨t¬~Ô>ˆLØ+7¨<¸¼ÀVÑ+LÔ+LÑ(ˆ�$Ô(åœ	 (°uÐ=Ñ=Ô=ˆŒÝ!#¤¨8¯>ª>Ñ+;Ô+;ÈÐ!NÑ!NÔ!NˆÔÐÐr@   rB   c                 óä   — | j         d         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «        |z  z  z  }|                     |¦  «        |fS )a¾  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNç      ð?r   r+   ©r‘   )r²   Úgetattrr1   Únum_attention_headsr5   Úarangert   r�   )r(   r¨   ÚkwargsÚbaserF   Úattention_factorr¶   s          r?   r³   z9Llama4TextRotaryEmbedding.compute_default_rope_parametersÅ   sz   € ð Ô% lÔ3ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐà˜$¥5¤<°°3¸ÅÄÐ#MÑ#MÔ#MÐPSÑ#SÑTÑUˆØ�{Š{˜6Ñ"Ô"Ð$4Ð4Ð4r@   c                 ó`  — | j         d d d …d f                              ¦   «                              |j        d         dd¦  «        }|d d …d d d …f                              ¦   «         }t	          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  | 	                    |j        ¦  «        |z   
                    dd¦  «        }t          j        t          j        |¦  «        |¦  «        }|| j        z  }d d d ¦  «         n# 1 swxY w Y   |S )	Nr   rD   r#   ÚmpsÚcpuF)Údevice_typeÚenabledr+   )r¶   rt   ÚexpandrH   Ú
isinstancer¨   ÚtypeÚstrr    r�   r    r5   ÚpolarÚ	ones_liker´   )r=   rb   Úposition_idsÚinv_freq_expandedÚposition_ids_expandedrÈ   ÚfreqsÚ	freqs_ciss           r?   rO   z!Llama4TextRotaryEmbedding.forwardÚ   s^  € ð !œM¨$°°°°4¨-Ô8×>Ò>Ñ@Ô@×GÒGÈÔHZÐ[\ÔH]Ð_aÐcdÑeÔeÐØ ,¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆÝ¨¸UÐCÑCÔCð 	;ð 	;Ø&×)Ò)¨!¬(Ñ3Ô3Ð6KÑK×VÒVÐWXÐZ[Ñ\Ô\ˆEÝœ¥E¤O°EÑ$:Ô$:¸EÑBÔBˆIØ! DÔ$:Ñ:ˆIð	;ð 	;ð 	;ñ 	;ô 	;ð 	;ð 	;ð 	;ð 	;ð 	;ð 	;øøøð 	;ð 	;ð 	;ð 	;ð
 Ðs   Â4A#D#Ä#D'Ä*D'ra   )rP   rQ   rR   r   r%   r-   Ústaticmethodr„   r5   rS   rt   r³   Úno_gradr   rO   rT   rU   s   @r?   r§   r§   ²   sø   ø€ € € € € à€_�X vÐ.Ñ.Ô.ðOð OÐ/ð Oð Oð Oð Oð Oñ /Ô.ðOð  Ø€_�X vÐ.Ñ.Ô.ð5ð 5Ð0@ð 5Ð\aÐbgÔbnÐpuÐbuÔ\vð 5ð 5ð 5ñ /Ô.ñ „\ð5ð& €U„]�_„_Øð
ð 
ñ Ôñ „_ð
ð 
ð 
ð 
ð 
r@   r§   ÚxqÚxkrÔ   rB   c           	      óF  — t          j         |                      ¦   «         j        g | j        d d…         ¢d‘d‘R Ž ¦  «        }t          j         |                     ¦   «         j        g |j        d d…         ¢d‘d‘R Ž ¦  «        }t          j        ||d d …d d …d d d …f         z  ¦  «                             d¦  «        }t          j        ||d d …d d …d d d …f         z  ¦  «                             d¦  «        }|                     | ¦  «        |                     |¦  «        fS )NrD   r+   r   )r5   Úview_as_complexrt   rž   rH   Úview_as_realÚflattenru   )r×   rØ   rÔ   Úxq_Úxk_Úxq_outÚxk_outs          r?   Úapply_rotary_embrá   é   s  € õ
 Ô
Ð 2 §¢¡
¤
Ô 2Ð I°B´H¸S¸b¸S´MÐ IÀ2Ð IÀqÐ IÐ IÐ IÑ
JÔ
J€CÝ
Ô
Ð 2 §¢¡
¤
Ô 2Ð I°B´H¸S¸b¸S´MÐ IÀ2Ð IÀqÐ IÐ IÐ IÑ
JÔ
J€CÝÔ  i°°°°1°1°1°d¸A¸A¸A°Ô&>Ñ >Ñ?Ô?×GÒGÈÑJÔJ€FÝÔ  i°°°°1°1°1°d¸A¸A¸A°Ô&>Ñ >Ñ?Ô?×GÒGÈÑJÔJ€FØ�>Š>˜"ÑÔ˜vŸ~š~¨bÑ1Ô1Ð1Ð1r@   rA   Ún_repc                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r#   N)rH   rÊ   rž   )rA   râ   ÚbatchÚnum_key_value_headsÚslenr¼   s         r?   Ú	repeat_kvrç   õ   s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTr@   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutc                 óÎ  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        |z  }
|�|
|z   }
t
          j                             |
d¬¦  «        }
t
          j                             |
|| j	        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «         
                    ¦   «         }||
fS )Nr+   r   rD   rE   ©ÚpÚtrainingr#   )rç   Únum_key_value_groupsr5   Úmatmulr    r3   rŽ   Úsoftmaxrï   ró   Ú
contiguous©ré   rê   rë   rì   rí   rî   rï   rÂ   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r?   Úeager_attention_forwardrý     sÓ   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2Ð(Ñ>Ô>€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r@   c                 óÞ  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        | j        dz  z  }
|�|
|z   }
t          j                             |
d¬¦  «        }
t          j         	                    |
|| j
        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «                             ¦   «         }||
fS )Nr+   r   ç      à¿rD   rE   rñ   r#   )rç   rô   r5   rõ   r    r¼   r3   rŽ   rö   rï   ró   r÷   rø   s               r?   Úvision_eager_attention_forwardr     sÛ   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀVÄ_ÐVZÑEZÑZ€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2Ð(Ñ>Ô>€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r@   c                   óä   ‡ — e Zd ZdZdefˆ fd„Z	 ddej        deej        ej        f         dej        dz  de	dz  d	e
e         d
eej        ej        dz  eej                 dz  f         fd„Zˆ xZS )ÚLlama4TextAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr(   c                 óª  •— t          ¦   «                              ¦   «          || _        || _        t	          |d|j        |j        z  ¦  «        | _        |j        | _        |j        |j        z  | _	        |j        | _        | j        dz  | _
        |j        | _        |j        | _        |j        | _        |j        | _        d| _        |j        |         | _        t%          j        |j        |j        | j        z  |j        ¬¦  «        | _        t%          j        |j        |j        | j        z  |j        ¬¦  «        | _        t%          j        |j        |j        | j        z  |j        ¬¦  «        | _        t%          j        |j        | j        z  |j        |j        ¬¦  «        | _        | j        j        r"| j        rt5          |j        ¦  «        | _        d S d S d S )Nr¼   rÿ   TrZ   )r,   r-   r(   Ú	layer_idxr¿   r1   rÀ   r¼   rå   rô   rî   Ú
attn_scaleÚfloor_scaleÚattn_temperature_tuningÚattention_dropoutÚ	is_causalÚno_rope_layersÚuse_roper3   r\   Úattention_biasÚq_projÚk_projÚv_projÚo_projÚuse_qk_normre   Úrms_norm_epsÚqk_norm©r=   r(   r  r>   s      €r?   r-   zLlama4TextAttention.__init__8  sË  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒÝ ¨
°FÔ4FÈ&ÔJdÑ4dÑeÔeˆŒØ#)Ô#=ˆÔ Ø$*Ô$>À&ÔB\Ñ$\ˆÔ!Ø#)Ô#=ˆÔ Ø”} dÑ*ˆŒØ Ô+ˆŒØ!Ô-ˆÔØ'-Ô'EˆÔ$Ø!'Ô!9ˆÔØˆŒØÔ-¨iÔ8ˆŒÝ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ&¨¬Ñ6¸Ô8JÐQWÔQfð
ñ 
ô 
ˆŒð Œ;Ô"ð 	A t¤}ð 	AÝ+¨FÔ,?Ñ@Ô@ˆDŒLˆLˆLð	Að 	Að 	Að 	Ar@   NrA   Úposition_embeddingsrí   Úpast_key_valuesrÂ   rB   c                 óÄ  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «        } |                      |¦  «        j        g |¢d‘| j        ‘R Ž }	|                      |¦  «                             |¦  «                             dd¦  «        }
| j        r,t          ||	| 	                    |j
        ¦  «        ¦  «        \  }}	t          | d¦  «        r*|                      |¦  «        }|                      |	¦  «        }	| j        rò| j        së|�|                     | j        ¦  «        nd}t!          j        |j         d         |j
        ¬¦  «        |z   }t!          j        t!          j        |                     ¦   «         dz   | j        z  ¦  «        ¦  «        | j        z  dz   }|                     d|d         ddf¦  «                             g |¢d‘d‘R ¦  «        }||z   	                    |j        ¦  «        }|                     dd¦  «        }|	                     dd¦  «        }	|�|                     |	|
| j        ¦  «        \  }	}
t5          j        | j        j        t<          ¦  «        } || ||	|
|f| j        sdn| j         | j!        d	œ|¤Ž\  }} |j"        g |¢d‘R Ž  #                    ¦   «         }|  $                    |¦  «        }||fS )
NrD   r#   r+   r  r   ©r¨   r½   rè   )rï   rî   )%rH   r¼   r  rG   r  r  r    r  rá   r�   r¨   Úhasattrr  r  Úget_seq_lengthr  r5   rÁ   Úlog1pÚfloorrt   r  r  rÊ   r‘   Úupdater   Úget_interfacer(   Ú_attn_implementationrý   ró   r  rî   rž   r÷   r  )r=   rA   r  rí   r  rÂ   Úinput_shapeÚhidden_shapeÚquery_statesrù   rú   Úpast_seen_tokensÚ	positionsÚattn_scalesÚattention_interfacerü   rû   s                    r?   rO   zLlama4TextAttention.forwardV  s$  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔDˆØ4�T—[’[ Ñ/Ô/Ô4ÐU°kÐUÀ2ÐUÀtÄ}ÐUÐUÐUˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆàŒ=ð 	Ý'7Ø˜jÐ*=×*@Ò*@ÀÔATÑ*UÔ*Uñ(ô (Ñ$ˆL˜*õ �4˜Ñ#Ô#ð 	2ØŸ<š<¨Ñ5Ô5ˆLØŸš jÑ1Ô1ˆJð Ô'ð 	O°´ð 	OØQ`ÐQl˜×=Ò=¸d¼nÑMÔMÐMÐrsÐÝœ ]Ô%8¸Ô%;ÀMÔDXÐYÑYÔYÐ\lÑlˆIå”�EœK¨¯ªÑ):Ô):¸SÑ)@ÀDÔDTÑ(TÑUÔUÑVÔVÐY]ÔYhÑhÐknÑnð ð &×*Ò*¨A¨{¸2¬ÀÀ1Ð+EÑFÔF×MÒMÐNbÐP[ÐNbÐ]^ÐNbÐ`aÐNbÐNbÑcÔcˆKØ(¨;Ñ6×:Ò:¸<Ô;MÑNÔNˆLà#×-Ò-¨a°Ñ3Ô3ˆØ×)Ò)¨!¨QÑ/Ô/ˆ
àÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r@   ra   )rP   rQ   rR   Ú__doc__r%   r-   r5   rS   r„   r	   r   r   rO   rT   rU   s   @r?   r  r  5  sí   ø€ € € € € ØGÐGðAÐ/ð Að Að Að Að Að AðF )-ð8)ð 8)à”|ð8)ð # 5¤<°´Ð#=Ô>ð8)ð œ tÑ+ð	8)ð
  ™ð8)ð Ð-Ô.ð8)ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	Mð8)ð 8)ð 8)ð 8)ð 8)ð 8)ð 8)ð 8)r@   r  c                   ó  ‡ — e Zd Zˆ fd„Z	 	 	 	 	 ddej        dej        dz  dej        dz  dedz  dedz  d	e	ej        ej        f         dz  d
e
e         de	ej        e	ej        ej        f         dz  f         fd„Zˆ xZS )ÚLlama4TextDecoderLayerc                 ó¢  •— t          ¦   «                              ¦   «          |j        | _        || _        t	          ||¦  «        | _        ||j        v | _        | j        rt          |¦  «        | _	        nt          ||j        ¬¦  «        | _	        t          |j        |j        ¬¦  «        | _        t          |j        |j        ¬¦  «        | _        d S )N)r0   rw   )r,   r-   r1   r  r  Ú	self_attnÚ
moe_layersÚis_moe_layerr—   Úfeed_forwardrW   Úintermediate_size_mlpr{   r  Úinput_layernormÚpost_attention_layernormr  s      €r?   r-   zLlama4TextDecoderLayer.__init__’  sÀ   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔØ"ˆŒÝ,¨V°YÑ?Ô?ˆŒØ%¨Ô):Ð:ˆÔØÔð 	fÝ -¨fÑ 5Ô 5ˆDÔÐå -¨fÈÔHdÐ eÑ eÔ eˆDÔå0°Ô1CÈÔI\Ð]Ñ]Ô]ˆÔÝ(9¸&Ô:LÐRXÔReÐ(fÑ(fÔ(fˆÔ%Ð%Ð%r@   NFrA   rí   rÐ   r  Ú	use_cacher  rÂ   rB   c           	      ó  — |}|                       |¦  «        } | j        d|||||dœ|¤Ž\  }	}
||	z   }|}|                      |¦  «        }|                      |¦  «        }| j        r|\  }}
||                     |j        ¦  «        z   }|S )N)rA   r  rí   r  r2  © )r0  r+  r1  r.  r-  rG   rH   )r=   rA   rí   rÐ   r  r2  r  rÂ   ÚresidualÚattention_statesÚ_s              r?   rO   zLlama4TextDecoderLayer.forward   sÉ   € ð !ˆà×,Ò,¨]Ñ;Ô;ˆð -˜dœnð 
Ø'Ø 3Ø)Ø+Øð
ð 
ð ð
ð 
ÑÐ˜!ð !Ð#3Ñ3ˆð !ˆØ×5Ò5°mÑDÔDˆØ×)Ò)¨-Ñ8Ô8ˆØÔð 	-Ø,ÑˆM˜1Ø  =×#5Ò#5°h´nÑ#EÔ#EÑEˆØÐr@   )NNNFN)rP   rQ   rR   r-   r5   rS   Ú
LongTensorr	   Úboolr„   r   r   ÚFloatTensorrO   rT   rU   s   @r?   r)  r)  ‘  s
  ø€ € € € € ðgð gð gð gð gð" /3Ø04Ø(,Ø!&ØHLð ð  à”|ð ð œ tÑ+ð ð Ô&¨Ñ-ð	 ð
  ™ð ð ˜$‘;ð ð # 5¤<°´Ð#=Ô>ÀÑEð ð Ð-Ô.ð ð 
ˆuÔ  %¨Ô(9¸5Ô;LÐ(LÔ"MÐPTÑ"TÐTÔ	Uð ð  ð  ð  ð  ð  ð  ð  r@   r)  c                   óv   ‡ — e Zd ZU eed<   dZdZdgZdZdZ	dZ
dZdZ ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )ÚLlama4PreTrainedModelr(   )ÚimageÚtextTr  Fc                 óŒ  •— t          ¦   «                              |¦  «         t          | j        d¦  «        r| j        j        n| j        j        j        }t          |t          ¦  «        r:t          j	        |j
        d|¬¦  «         t          j	        |j        d|¬¦  «         d S t          |t          ¦  «        r4t          j        |j        |                     |j        ¦  «        ¦  «         d S t          |t           ¦  «        rBt          j	        |j        |j        ¬¦  «         t          j	        |j        |j        ¬¦  «         d S d S )NÚinitializer_rangerè   )ro   Ústd)rA  )r,   Ú_init_weightsr  r(   r@  Útext_configrË   r'   ÚinitÚnormal_r7   r9   ÚLlama4VisionRotaryEmbeddingÚcopy_Úfreqs_ciÚ_compute_freqs_ciÚLlama4VisionModelÚclass_embeddingÚscaleÚpositional_embedding_vlm)r=   ré   rA  r>   s      €r?   rB  z#Llama4PreTrainedModel._init_weightsÐ  s4  ø€ å‰Œ×Ò˜fÑ%Ô%Ð%õ �t”{Ð$7Ñ8Ô8ð;ˆDŒKÔ)Ð)à”Ô(Ô:ð 	õ
 �fÕ/Ñ0Ô0ð 	LÝŒL˜Ô,°3¸CÐ@Ñ@Ô@Ð@ÝŒL˜Ô)°¸Ð=Ñ=Ô=Ð=Ð=Ð=Ý˜Õ ;Ñ<Ô<ð 	LÝŒJ�v”¨×(@Ò(@ÀÄÑ(OÔ(OÑPÔPÐPÐPÐPÝ˜Õ 1Ñ2Ô2ð 	LÝŒL˜Ô/°V´\ÐBÑBÔBÐBÝŒL˜Ô8¸f¼lÐKÑKÔKÐKÐKÐKð	Lð 	Lr@   )rP   rQ   rR   r$   Ú__annotations__Úinput_modalitiesÚsupports_gradient_checkpointingÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr5   rÖ   rB  rT   rU   s   @r?   r<  r<  Ã  s“   ø€ € € € € € àÐÐÑØ(ÐØ&*Ð#Ø#4Ð"5ÐØ ÐØ€NØÐà!ÐØ"&Ðà€U„]�_„_ðLð Lð Lð Lñ „_ðLð Lð Lð Lð Lr@   r<  c                   ó  ‡ — e Zd ZU dgZdZdZeed<   ee	e
dœZdefˆ fd„Zeee	 	 	 	 	 	 ddej        dz  d	ej        dz  d
ej        dz  dedz  dej        dz  dedz  dee         deez  fd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚLlama4TextModelr)  Úmodel)r>  r(   )Ú
attentionsrA   r’   c                 óÞ  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          j        ‰j        ‰j        | j        ¦  «        | _        t          j	        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        t!          ‰¬¦  «        | _        d| _        |                      ¦   «          d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r4  )r)  )Ú.0r  r(   s     €r?   ú
<listcomp>z,Llama4TextModel.__init__.<locals>.<listcomp>õ  s$   ø€ ÐhÐhÐh¸9Õ# F¨IÑ6Ô6ÐhÐhÐhr@   rw   ©r(   F)r,   r-   Úpad_token_idÚpadding_idxÚ
vocab_sizer3   Ú	Embeddingr1   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersÚlayersr{   r  Únormr§   Ú
rotary_embÚgradient_checkpointingÚ	post_initr<   s    `€r?   r-   zLlama4TextModel.__init__î  sÔ   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒåœL¨Ô):¸FÔ<NÐPTÔP`ÑaÔaˆÔÝ”mØhÐhÐhÐhÍÈfÔNfÑHgÔHgÐhÑhÔhñ
ô 
ˆŒõ & fÔ&8¸fÔ>QÐRÑRÔRˆŒ	Ý3¸6ÐBÑBÔBˆŒØ&+ˆÔ#ð 	�ŠÑÔÐÐÐr@   NÚ	input_idsrí   rÐ   r  Úinputs_embedsr2  rÂ   rB   c           
      ó&  — |d u |d uz  rt          d¦  «        ‚|€7|                      |                     | j        j        j        ¦  «        ¦  «        }|r|€t          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}t          j	        |j
        d         |j        ¬¦  «        |z   }|                     d¦  «        }t          |x}	t          ¦  «        s&| j        ||||dœ}
t          d
i |
¤Žt          d
i |
¤Ždœ}	|}|                      ||¦  «        }t#          | j        d | j        j        …         ¦  «        D ]*\  }} ||f|	| j        j        |                  ||||dœ|¤Ž}Œ+|                      |¦  «        }t-          ||r|nd ¬	¦  «        S )Nú:You must specify exactly one of input_ids or inputs_embedsr_  r   r#   r  )r(   rn  rí   r  rÐ   )Úfull_attentionÚchunked_attention)rí   rÐ   r  r2  r  )Úlast_hidden_stater  r4  )Ú
ValueErrorrd  r�   r   r¨   r
   r(   r  r5   rÁ   rH   Ú	unsqueezerË   Údictr   r   rj  Ú	enumeraterh  rg  Úlayer_typesri  r   )r=   rm  rí   rÐ   r  rn  r2  rÂ   r#  Úcausal_mask_mappingÚmask_kwargsrA   Úfreq_cisÚiÚdecoder_layers                  r?   rO   zLlama4TextModel.forwardþ  s  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨i¯lªl¸4Ô;LÔ;SÔ;ZÑ.[Ô.[Ñ\Ô\ˆMàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLõ °Ð?Ð-ÅÑFÔFð 	ð œ+Ø!.Ø"0Ø#2Ø ,ðð ˆKõ #5Ð"CÐ"C°{Ð"CÐ"CÝ%?Ð%NÐ%NÀ+Ð%NÐ%Nð#ð #Ðð
 &ˆð —?’? =°,Ñ?Ô?ˆå )¨$¬+Ð6U¸¼Ô8UÐ6UÔ*VÑ WÔ Wð 		ð 		ÑˆAˆ}Ø)˜MØðà2°4´;Ô3JÈ1Ô3MÔNØ)Ø /Ø#Ø$,ðð ð ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆå&Ø+Ø/8ÐB˜O˜O¸dð
ñ 
ô 
ð 	
r@   )NNNNNN)rP   rQ   rR   Ú_no_split_modulesÚbase_model_prefixrO  r%   rN  r  r)  r—   Ú_can_record_outputsr-   r!   r"   r   r5   r8  rS   r	   r:  r9  r   r   r„   r   rO   rT   rU   s   @r?   rX  rX  â  sM  ø€ € € € € € à1Ð2ÐØÐØ ÐØÐÐÑà)Ø/Ø&ðð ÐðÐ/ð ð ð ð ð ð ð   ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ð<
ð <
àÔ# dÑ*ð<
ð œ tÑ+ð<
ð Ô&¨Ñ-ð	<
ð
  ™ð<
ð Ô(¨4Ñ/ð<
ð ˜$‘;ð<
ð Ð+Ô,ð<
ð 
Ð(Ñ	(ð<
ð <
ð <
ñ „^ñ „_ñ  Ôð<
ð <
ð <
ð <
ð <
r@   rX  c                   ó2  ‡ — e Zd ZU dgZdZddiZddiZddiZee	d<   defˆ fd	„Z
ee	 	 	 	 	 	 	 	 ddej        d
z  dej        d
z  dej        d
z  ded
z  dej        d
z  dej        d
z  ded
z  deej        z  dee         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚLlama4ForCausalLMr)  Úlanguage_modelzlm_head.weightzmodel.embed_tokens.weightÚlm_headÚkeep_full_weightÚcolwise_gather_outputr(   c                 óþ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        | _        t          j        |j        |j        d¬¦  «        | _        |  	                    ¦   «          d S rY   )
r,   r-   rX  rY  rb  r3   r\   r1   r„  rl  r<   s     €r?   r-   zLlama4ForCausalLM.__init__H  sj   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý$ VÑ,Ô,ˆŒ
Ø Ô+ˆŒÝ”y Ô!3°VÔ5FÈUÐSÑSÔSˆŒð 	�ŠÑÔÐÐÐr@   Nr   rm  rí   rÐ   r  rn  Úlabelsr2  Úlogits_to_keeprÂ   rB   c	           
      óR  —  | j         d||||||dœ|	¤Ž}
|
d         }t          |t          ¦  «        rt          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }d}|� | j        d||| j        j        dœ|	¤Ž}t          |||
j	        |
j
        |
j        ¬¦  «        S )az  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, Llama4ForCausalLM

        >>> model = Llama4ForCausalLM.from_pretrained("meta-llama4/Llama4-2-7b-hf")
        >>> tokenizer = AutoTokenizer.from_pretrained("meta-llama4/Llama4-2-7b-hf")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```)rm  rí   rÐ   r  rn  r2  r   N©Úlogitsrˆ  rb  )ÚlossrŒ  r  rA   rZ  r4  )rY  rË   ÚintÚslicer„  Úloss_functionr(   rb  r   r  rA   rZ  )r=   rm  rí   rÐ   r  rn  rˆ  r2  r‰  rÂ   ÚoutputsrA   Úslice_indicesrŒ  r�  s                  r?   rO   zLlama4ForCausalLM.forwardQ  sö   € ðH �$”*ð 
ØØ)Ø%Ø+Ø'Øð
ð 
ð ð
ð 
ˆð   œ
ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆØˆØÐØ%�4Ô%Ðp¨V¸FÈtÌ{ÔOeÐpÐpÐioÐpÐpˆDå%ØØØ#Ô3Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r@   )NNNNNNNr   )rP   rQ   rR   r~  r  Ú_tied_weights_keysÚ
_fsdp_planÚ_tp_planr%   rN  r-   r   r   r5   r8  rS   r	   r:  r9  rŽ  r   r   r„   r   rO   rT   rU   s   @r?   r‚  r‚  @  sr  ø€ € € € € € Ø1Ð2ÐØ(ÐØ*Ð,GÐHÐØÐ/Ð0€JØÐ2Ð3€HØÐÐÑðÐ/ð ð ð ð ð ð ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ð:
ð :
àÔ# dÑ*ð:
ð œ tÑ+ð:
ð Ô&¨Ñ-ð	:
ð
  ™ð:
ð Ô(¨4Ñ/ð:
ð Ô  4Ñ'ð:
ð ˜$‘;ð:
ð ˜eœlÑ*ð:
ð Ð+Ô,ð:
ð 
Ð'Ñ	'ð:
ð :
ð :
ñ „^ñ Ôð:
ð :
ð :
ð :
ð :
r@   r‚  zQ
    Base class for Llava causal language model (or autoregressive) outputs.
    ©Úcustom_introc                   óÖ   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dZej        dz  ed<   dS )	ÚLlama4CausalLMOutputWithPasta3  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size (batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
    Nr�  rŒ  r  rA   rZ  Úimage_hidden_states)rP   rQ   rR   r'  r�  r5   r:  rN  rŒ  r  r	   rA   r„   rZ  rš  r4  r@   r?   r™  r™  �  sµ   € € € € € € ðð ð &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø$(€O�U˜T‘\Ð(Ð(Ñ(Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ø48Ð˜Ô*¨TÑ1Ð8Ð8Ñ8Ð8Ð8r@   r™  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚLlama4VisionMLP2c                 óX  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        t	          j        | j        |j        d¬¦  «        | _        t	          j        |j        |j        d¬¦  «        | _	        t	          j
        ¦   «         | _        |j        | _        d S rY   )r,   r-   r1   r0   r3   r\   Úprojector_input_dimÚfc1Úprojector_output_dimÚfc2ÚGELUr_   Úprojector_dropoutrï   r<   s     €r?   r-   zLlama4VisionMLP2.__init__¯  s‰   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔØ!'Ô!9ˆÔÝ”9˜TÔ3°VÔ5OÐV[Ð\Ñ\Ô\ˆŒÝ”9˜VÔ8¸&Ô:UÐ\aÐbÑbÔbˆŒÝœW™YœYˆÔØÔ/ˆŒˆˆr@   c                 óè   — |                       |¦  «        }|                      |¦  «        }t          j        || j        | j        ¬¦  «        }|                      |                      |¦  «        ¦  «        S )Nrñ   )rŸ  r_   ÚFrï   ró   r¡  ©r=   rA   s     r?   rO   zLlama4VisionMLP2.forward¸  s`   € ØŸš Ñ/Ô/ˆØ×*Ò*¨=Ñ9Ô9ˆÝœ	 -°4´<È$Ì-ÐXÑXÔXˆØ×!Ò! $§(¢(¨=Ñ"9Ô"9Ñ:Ô:Ð:r@   rc   rU   s   @r?   rœ  rœ  ®  sG   ø€ € € € € ð0ð 0ð 0ð 0ð 0ð;ð ;ð ;ð ;ð ;ð ;ð ;r@   rœ  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚLlama4MultiModalProjectorc                 ó¨   •— t          ¦   «                              ¦   «          t          j        |j        j        |j        j        d¬¦  «        | _        d S rY   )	r,   r-   r3   r\   Úvision_configÚvision_output_dimrC  r1   Úlinear_1r<   s     €r?   r-   z"Llama4MultiModalProjector.__init__À  sJ   ø€ Ý‰Œ×ÒÑÔÐÝœ	ØÔ Ô2ØÔÔ*Øð
ñ 
ô 
ˆŒˆˆr@   c                 ó0   — |                       |¦  «        }|S ra   )r¬  )r=   Úimage_featuresrA   s      r?   rO   z!Llama4MultiModalProjector.forwardÈ  s   € ØŸš nÑ5Ô5ˆØÐr@   rc   rU   s   @r?   r¨  r¨  ¿  sG   ø€ € € € € ð
ð 
ð 
ð 
ð 
ðð ð ð ð ð ð r@   r¨  c           
      ó²  — | j         \  }}}t          t          j        |¦  «        ¦  «        }|                      |||d¦  «        } |                      ¦   «         \  }}}}|                      ||t          ||z  ¦  «        t          ||z  ¦  «        ¦  «        }|                     dddd¦  «                             ¦   «         }|                     |t          ||z  ¦  «        t          ||z  ¦  «        t          ||dz  z  ¦  «        ¦  «        }|                     dddd¦  «                             ¦   «         }|                     |d|j         d         ¦  «        }	|	S )NrD   r   r+   r#   r   )rH   rŽ  ÚmathÚsqrtrG   ÚsizeÚpermuter÷   )
Úinput_tensorÚshuffle_ratioÚ
batch_sizeÚnum_patchesÚchannelsÚ
patch_sizeÚheightÚwidthÚreshaped_tensorÚoutput_tensors
             r?   Úpixel_shuffler¾  Í  sO  € à(4Ô(:Ñ%€J�˜XÝ•T”Y˜{Ñ+Ô+Ñ,Ô,€Jà×$Ò$ Z°¸ZÈÑLÔL€LØ*6×*;Ò*;Ñ*=Ô*=Ñ'€J�˜˜xà"×'Ò'¨
°F½CÀÈÑ@UÑ<VÔ<VÕX[Ð\dÐgtÑ\tÑXuÔXuÑvÔv€OØ%×-Ò-¨a°°A°qÑ9Ô9×DÒDÑFÔF€Oà%×*Ò*Ø•C˜ Ñ.Ñ/Ô/µ°U¸]Ñ5JÑ1KÔ1KÍSÐQYÐ]jÐlmÑ]mÑQnÑMoÔMoñô €Oð &×-Ò-¨a°°A°qÑ9Ô9×DÒDÑFÔF€Oà#×(Ò(¨°R¸Ô9NÈrÔ9RÑSÔS€MØÐr@   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚLlama4VisionPixelShuffleMLPc                 óè   •— t          ¦   «                              ¦   «          |j        | _        t          |j        | j        dz  z  ¦  «        | _        |j        | _        t          |¦  «        | _	        d S r*   )
r,   r-   Úpixel_shuffle_ratiorŽ  rž  Ú	inner_dimr   Ú
output_dimrœ  Úmlpr<   s     €r?   r-   z$Llama4VisionPixelShuffleMLP.__init__â  sa   ø€ Ý‰Œ×ÒÑÔÐØ#)Ô#=ˆÔ Ý˜VÔ7¸DÔ<TÐVWÑ<WÑXÑYÔYˆŒØ Ô5ˆŒÝ# FÑ+Ô+ˆŒˆˆr@   Úencoded_patchesrB   c                 óV   — t          || j        ¦  «        }|                      |¦  «        S ra   )r¾  rÂ  rÅ  )r=   rÆ  s     r?   rO   z#Llama4VisionPixelShuffleMLP.forwardé  s&   € Ý'¨¸Ô9QÑRÔRˆØ�xŠx˜Ñ(Ô(Ð(r@   ©rP   rQ   rR   r-   r5   rS   rO   rT   rU   s   @r?   rÀ  rÀ  á  s^   ø€ € € € € ð,ð ,ð ,ð ,ð ,ð) u¤|ð )¸¼ð )ð )ð )ð )ð )ð )ð )ð )r@   rÀ  rH  c                 óf   ‡— |j         Šˆfd„t          |j        ¦  «        D ¦   «         } | j        |Ž S )Nc                 ó<   •— g | ]\  }}|d k    s	|‰d z
  k    r|nd ‘ŒS )r#   r4  )r]  r|  ÚdÚndims      €r?   r^  z)reshape_for_broadcast.<locals>.<listcomp>ñ  s5   ø€ ÐTÐTÐT±T°Q¸�!�q’&�&˜A ¨¡šM˜MˆQˆQ¨qÐTÐTÐTr@   )rÌ  rw  rH   rG   )rH  rê   rH   rÌ  s      @r?   Úreshape_for_broadcastrÍ  ï  s<   ø€ ØŒ:€DØTÐTÐTÐT½YÀuÄ{Ñ=SÔ=SÐTÑTÔT€EØˆ8Œ=˜%Ð Ð r@   c                 ó\  — t          j         |                      ¦   «         j        g | j        d d…         ¢d‘d‘R Ž ¦  «        }t          j         |                     ¦   «         j        g |j        d d…         ¢d‘d‘R Ž ¦  «        }t          ||¬¦  «        }|                     |j        ¦  «        }t          j        ||z  ¦  «         	                    d¦  «        }t          j        ||z  ¦  «         	                    d¦  «        }| 
                    | ¦  «        | 
                    |¦  «        fS )NrD   r+   )rH  rê   r   )r5   rÚ   rt   rž   rH   rÍ  r�   r¨   rÛ   rÜ   ru   )rê   rë   rH  Úquery_Úkey_Ú	query_outÚkey_outs          r?   Úvision_apply_rotary_embrÓ  õ  s  € õ
 Ô"Ð#8 5§;¢;¡=¤=Ô#8Ð#R¸%¼+ÀcÀrÀcÔ:JÐ#RÈBÐ#RÐPQÐ#RÐ#RÐ#RÑSÔS€FÝÔ Ð!4 §¢¡¤Ô!4Ð!L°c´iÀÀÀ´nÐ!LÀbÐ!LÈ!Ð!LÐ!LÐ!LÑMÔM€DÝ$¨h¸fÐEÑEÔE€HØ�{Š{˜6œ=Ñ)Ô)€HÝÔ" 6¨HÑ#4Ñ5Ô5×=Ò=¸aÑ@Ô@€IÝÔ  ¨¡Ñ1Ô1×9Ò9¸!Ñ<Ô<€GØ×Ò˜UÑ#Ô# W§_¢_°SÑ%9Ô%9Ð9Ð9r@   c                   óÈ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej        dej        dej        dz  dedz  dee	         d	e
ej        ej        dz  e
ej                 dz  f         fd
„Zˆ xZS )ÚLlama4VisionAttentionr(   c                 ób  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        |j        |j        z  | _        d| _        |j	        | _	        | j        dz  | _
        t          j        | j        | j        | j        z  d¬¦  «        | _        t          j        | j        | j        | j        z  d¬¦  «        | _        t          j        | j        | j        | j        z  d¬¦  «        | _        t          j        | j        | j        z  | j        d¬¦  «        | _        d S )Nr#   rÿ   TrZ   )r,   r-   r(   r1   Ú	embed_dimrÀ   Ú	num_headsr¼   rô   r  rî   r3   r\   r  r  r  r  r<   s     €r?   r-   zLlama4VisionAttention.__init__  sû   ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ+ˆŒØÔ3ˆŒØÔ*¨fÔ.HÑHˆŒØ$%ˆÔ!Ø!'Ô!9ˆÔØ”} dÑ*ˆŒå”i ¤°´ÀÄÑ0NÐUYÐZÑZÔZˆŒÝ”i ¤°´ÀÄÑ0NÐUYÐZÑZÔZˆŒÝ”i ¤°´ÀÄÑ0NÐUYÐZÑZÔZˆŒÝ”i ¤°´Ñ >ÀÄÐUYÐZÑZÔZˆŒˆˆr@   NrA   rH  rí   r  rÂ   rB   c                 óÚ  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «        }|                      |¦  «                             |¦  «        }	|                      |¦  «                             |¦  «        }
t          ||	|¬¦  «        \  }}	|                     dd¦  «        }|	                     dd¦  «        }	|
                     dd¦  «        }
t          j	        | j
        j        t          ¦  «        } || ||	|
d f| j        sdn| j        d ddœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )NrD   )rH  r#   r+   rè   F)rï   rî   r	  )rH   r¼   r  rG   r  r  rÓ  r    r   r  r(   r  r   ró   r  rž   r÷   r  )r=   rA   rH  rí   r  rÂ   r   r!  r"  rù   rú   r&  rü   rû   s                 r?   rO   zLlama4VisionAttention.forward  sž  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔDˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔBˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔDˆå#:¸<ÈÐ^fÐ#gÑ#gÔ#gÑ ˆ�jà#×-Ò-¨a°Ñ3Ô3ˆØ×)Ò)¨!¨QÑ/Ô/ˆ
Ø#×-Ò-¨a°Ñ3Ô3ˆå(?Ô(MØŒKÔ,Õ.Lñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð  $œ}ÐH�C�C°$Ô2HØØð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r@   ©NN)rP   rQ   rR   r   r-   r5   rS   r	   r   r   r„   rO   rT   rU   s   @r?   rÕ  rÕ    sÝ   ø€ € € € € ð[Ð1ð [ð [ð [ð [ð [ð [ð& /3Ø(,ð')ð ')à”|ð')ð ”,ð')ð œ tÑ+ð	')ð
  ™ð')ð Ð-Ô.ð')ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	Mð')ð ')ð ')ð ')ð ')ð ')ð ')ð ')r@   rÕ  c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚLlama4VisionMLPc                 ó  •— t          ¦   «                              ¦   «          || _        t          j        ¦   «         | _        t          j        |j        |j        d¬¦  «        | _	        t          j        |j        |j        d¬¦  «        | _
        d S )NTrZ   )r,   r-   r(   r3   r¢  r_   r\   r1   r0   rŸ  r¡  r<   s     €r?   r-   zLlama4VisionMLP.__init__>  sp   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝœW™YœYˆÔÝ”9˜VÔ/°Ô1IÐPTÐUÑUÔUˆŒÝ”9˜VÔ5°vÔ7IÐPTÐUÑUÔUˆŒˆˆr@   rA   rB   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S ra   )rŸ  r_   r¡  r¦  s     r?   rO   zLlama4VisionMLP.forwardE  s=   € ØŸš Ñ/Ô/ˆØ×*Ò*¨=Ñ9Ô9ˆØŸš Ñ/Ô/ˆØÐr@   rÈ  rU   s   @r?   rÜ  rÜ  =  sc   ø€ € € € € ðVð Vð Vð Vð Vð U¤\ð °e´lð ð ð ð ð ð ð ð r@   rÜ  c            
       ól   ‡ — e Zd Zdefˆ fd„Z	 	 d	dej        dej        dej        dz  dedz  fd„Zˆ xZ	S )
ÚLlama4VisionEncoderLayerr(   c                 ó(  •— t          ¦   «                              ¦   «          |j        | _        t          |¦  «        | _        t          |¦  «        | _        t          j        |j        ¦  «        | _	        t          j        |j        ¦  «        | _
        d S ra   )r,   r-   r1   rÕ  r+  rÜ  rÅ  r3   Ú	LayerNormr0  r1  r<   s     €r?   r-   z!Llama4VisionEncoderLayer.__init__M  sr   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔå.¨vÑ6Ô6ˆŒÝ" 6Ñ*Ô*ˆŒå!œ|¨FÔ,>Ñ?Ô?ˆÔÝ(*¬°VÔ5GÑ(HÔ(HˆÔ%Ð%Ð%r@   NÚhidden_staterH  rí   Úoutput_attentionsc                 óì   — |}|                       |¦  «        }|                      |||¬¦  «        \  }}||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|f}|r||fz  }|S )N)rH  rí   )r0  r+  r1  rÅ  )r=   rã  rH  rí   rä  r5  rû   r‘  s           r?   rO   z Llama4VisionEncoderLayer.forwardW  s£   € ð  ˆà×+Ò+¨LÑ9Ô9ˆà%)§^¢^ØØØ)ð &4ñ &
ô &
Ñ"ˆ�lð
   ,Ñ.ˆð  ˆØ×4Ò4°\ÑBÔBˆØ—x’x Ñ-Ô-ˆØ ,Ñ.ˆà�/ˆàð 	'Ø˜�Ñ&ˆGàˆr@   rÚ  )
rP   rQ   rR   r   r-   r5   rS   r9  rO   rT   rU   s   @r?   rà  rà  L  s¢   ø€ € € € € ðIÐ1ð Ið Ið Ið Ið Ið Ið /3Ø)-ðð à”lðð ”,ðð œ tÑ+ð	ð
   $™;ðð ð ð ð ð ð ð r@   rà  c                   ó’   ‡ — e Zd ZdZdefˆ fd„Z	 	 	 	 ddej        dej        dej        dz  dedz  d	edz  d
edz  de	e
z  fd„Zˆ xZS )ÚLlama4VisionEncoderz½
    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
    [`Llama4VisionEncoderLayer`].

    Args:
        config: Llama4VisionConfig
    r(   c                 óâ   •‡— t          ¦   «                              ¦   «          ‰| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        d| _        ‰| _        d S )Nc                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r4  )rà  )r]  r7  r(   s     €r?   r^  z0Llama4VisionEncoder.__init__.<locals>.<listcomp>„  s"   ø€ Ð$oÐ$oÐ$oÈ!Õ%=¸fÑ%EÔ%EÐ$oÐ$oÐ$or@   F)	r,   r-   r(   r3   re  rf  rg  rh  rk  r<   s    `€r?   r-   zLlama4VisionEncoder.__init__�  sf   øø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”mÐ$oÐ$oÐ$oÐ$oÍuÐU[ÔUmÑOnÔOnÐ$oÑ$oÔ$oÑpÔpˆŒØ&+ˆÔ#ØˆŒˆˆr@   NrA   rH  rí   rä  Úoutput_hidden_statesÚreturn_dictrB   c                 óX  — |�|n| j         j        }|�|n| j         j        }|�|n| j         j        }|rdnd}|rdnd}| j        D ]/}	|r||fz   } |	||||¬¦  «        }
|r||
d         fz   }|
d         }Œ0|r||fz   }|st          d„ |||fD ¦   «         ¦  «        S t          |||¬¦  «        S )ad  
        Args:
            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
                Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
                This is useful if you want more control over how to convert `input_ids` indices into associated vectors
                than the model's internal embedding lookup matrix.
            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            output_hidden_states (`bool`, *optional*):
                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
                for more detail.
            return_dict (`bool`, *optional*):
                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
        Nr4  )rã  rí   rä  rH  r#   r   c              3   ó   K  — | ]}|®|V — Œ	d S ra   r4  ©r]  Úvs     r?   ú	<genexpr>z.Llama4VisionEncoder.forward.<locals>.<genexpr>Ä  s(   è è € ÐeÐe˜qÐWXÐWd˜ÐWdÐWdÐWdÐWdÐeÐer@   ©rs  rA   rZ  )r(   rä  rê  rë  rh  r„   r   )r=   rA   rH  rí   rä  rê  rë  Úencoder_statesÚall_attentionsÚencoder_layerÚlayer_outputss              r?   rO   zLlama4VisionEncoder.forwardˆ  s9  € ð> 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆà3Ð=˜˜¸ˆØ0Ð:˜˜°dˆà!œ[ð 	-ð 	-ˆMØ#ð CØ!/°=Ð2BÑ!B�à)˜MØ*Ø-Ø"3Ø!ð	ñ ô ˆMð !ð FØ!/°=ÀÔ3CÐ2EÑ!E�à)¨!Ô,ˆMˆMàð 	?Ø+¨}Ð.>Ñ>ˆNàð 	fÝÐeÐe ]°NÀNÐ$SÐeÑeÔeÑeÔeÐeÝØ+¸>ÐVdð
ñ 
ô 
ð 	
r@   ©NNNN)rP   rQ   rR   r'  r   r-   r5   rS   r9  r„   r   rO   rT   rU   s   @r?   rç  rç  x  sÖ   ø€ € € € € ðð ðÐ1ð ð ð ð ð ð ð /3Ø)-Ø,0Ø#'ð?
ð ?
à”|ð?
ð ”,ð?
ð œ tÑ+ð	?
ð
   $™;ð?
ð # T™kð?
ð ˜D‘[ð?
ð 
�Ñ	 ð?
ð ?
ð ?
ð ?
ð ?
ð ?
ð ?
ð ?
r@   rç  c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚLlama4UnfoldConvolutionc                 óN  •— t          ¦   «                              ¦   «          |j        }t          |t          ¦  «        r||f}t
          j                             ||j        ¬¦  «        | _        t          j	        |j
        |d         z  |d         z  |j        d¬¦  «        | _        d S )N)Úkernel_sizeÚstrider   r#   FrZ   )r,   r-   r¹  rË   rŽ  r5   r3   ÚUnfoldÚunfoldr\   Únum_channelsr1   Úlinear)r=   r(   rú  r>   s      €r?   r-   z Llama4UnfoldConvolution.__init__Ë  s–   ø€ Ý‰Œ×ÒÑÔÐØÔ'ˆÝ�k¥3Ñ'Ô'ð 	5Ø&¨Ð4ˆKÝ”h—o’o°+ÀfÔFW�oÑXÔXˆŒÝ”iØÔ +¨a¤.Ñ0°;¸q´>ÑAØÔØð
ñ 
ô 
ˆŒˆˆr@   rA   rB   c                 óˆ   — |                       |¦  «        }|                     ddd¦  «        }|                      |¦  «        }|S )Nr   r+   r#   )rý  r³  rÿ  r¦  s     r?   rO   zLlama4UnfoldConvolution.forward×  sA   € ØŸš MÑ2Ô2ˆØ%×-Ò-¨a°°AÑ6Ô6ˆØŸš MÑ2Ô2ˆØÐr@   rÈ  rU   s   @r?   rø  rø  Ê  s^   ø€ € € € € ð

ð 

ð 

ð 

ð 

ð U¤\ð °e´lð ð ð ð ð ð ð ð r@   rø  c                   ó@   ‡ — e Zd Zdefˆ fd„Zed„ ¦   «         Zd„ Zˆ xZS )rF  r(   c                 ó²   •— t          ¦   «                              ¦   «          || _        t          j        |                      |¦  «        d¬¦  «        | _        d S )NFr­   )r,   r-   r(   r3   rµ   rI  rH  r<   s     €r?   r-   z$Llama4VisionRotaryEmbedding.__init__à  sH   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝœ	 $×"8Ò"8¸Ñ"@Ô"@ÈUÐSÑSÔSˆŒˆˆr@   c                 óè  — | j         | j        z  }t          j        |dz  t          j        ¬¦  «                             |dz  d¦  «        }t          j        ||d d…         gd¬¦  «        }d|d<   ||z  }||z  }| j        | j        z  dz  }d| j	        d	         t          j        d|d¦  «        d |dz  …          
                    ¦   «         |z  z  z  }|dz   d
         |d d d d …f         z                       dd¬¦  «        }|dz   d
         |d d d d …f         z                       dd¬¦  «        }t          j        ||gd¬¦  «         
                    ¦   «                              ¦   «         dd d d…f         }	|	                     |                     ddd¦  «        dk     d¦  «        }	t          j        t          j        t          j        |	¦  «        t          j        |	¦  «        gd¬¦  «        ¦  «        }
|
S )Nr+   r¾   r#   r   rE   éþÿÿÿ)rD   rD   r½   r»   ).NrD   .)Ú
image_sizer¹  r5   rÁ   Úint32rž   Úcatr1   rÀ   r²   rt   Úrepeat_interleaver÷   Úmasked_fillrÚ   ÚstackÚcosÚsin)r(   ÚidxÚimg_idxÚfrequencies_xÚfrequencies_yÚfreq_dimÚ	rope_freqÚfreqs_xÚfreqs_yrÓ   r{  s              r?   rI  z-Llama4VisionRotaryEmbedding._compute_freqs_ciå  sü  € àÔ 6Ô#4Ñ4ˆÝ”,˜s A™v­U¬[Ð9Ñ9Ô9×AÒAÀ#ÀqÁ&È!ÑLÔLˆÝ”)˜W g¨b¨q¨b¤kÐ2¸Ð:Ñ:Ô:ˆØˆ�‰Ø #™ˆØ 3™ˆØÔ%¨Ô)CÑCÀqÑHˆØØÔ" <Ô0Ý”˜Q ¨!Ñ,Ô,Ð->°¸A±Ð->Ô?×EÒEÑGÔGÈ(ÑRñTñ
ˆ	ð " AÑ% yÔ1°I¸dÀDÈ!È!È!¸mÔ4LÑL×_Ò_Ð`aÐgiÐ_ÑjÔjˆØ! AÑ% yÔ1°I¸dÀDÈ!È!È!¸mÔ4LÑL×_Ò_Ð`aÐgiÐ_ÑjÔjˆÝ”	˜7 GÐ,°"Ð5Ñ5Ô5×;Ò;Ñ=Ô=×HÒHÑJÔJÈ3ÐPSÐPSÐRSÐPSÈ8ÔTˆØ×!Ò! '§/¢/°"°a¸Ñ";Ô";¸aÒ"?ÀÑCÔCˆÝÔ(­¬µe´iÀÑ6FÔ6FÍÌ	ÐRWÑHXÔHXÐ5YÐ_aÐ)bÑ)bÔ)bÑcÔcˆØˆr@   c                 ó@   — | j                              |j        ¦  «        S ra   )rH  r�   r¨   r¦  s     r?   rO   z#Llama4VisionRotaryEmbedding.forwardù  s   € ØŒ}×Ò Ô 4Ñ5Ô5Ð5r@   )	rP   rQ   rR   r   r-   rÕ   rI  rO   rT   rU   s   @r?   rF  rF  ß  sv   ø€ € € € € ðTÐ1ð Tð Tð Tð Tð Tð Tð
 ðð ñ „\ðð&6ð 6ð 6ð 6ð 6ð 6ð 6r@   rF  c                   óº   ‡ — e Zd ZU dZdZdgZeed<   defˆ fd„Zd„ Z		 	 	 	 dde
j        d	e
j        dz  d
edz  dedz  dedz  deee
j        df         z  fd„Zˆ xZS )rJ  Úvision_model)r=  rà  r(   c                 ó(  •— t          ¦   «                              |¦  «         |j        | _        |j        | _        |j        | _        |j        | _        | j        | j        z  dz  dz   | _        |j        dz  | _        t          |¦  «        | _	        t          j        | j        t          j        | j        ¦  «        z  ¦  «        | _        t          j        | j        t          j        | j        | j        ¦  «        z  ¦  «        | _        t!          |¦  «        | _        t          j        | j        ¦  «        | _        t          j        | j        ¦  «        | _        t+          |¦  «        | _        t/          |¦  «        | _        |                      ¦   «          d S )Nr+   r#   rÿ   )r,   r-   r  r¹  r1   rþ  r·  rL  rø  Úpatch_embeddingr3   r4   r5   ÚrandnrK  rM  rF  Úrotary_embeddingrâ  Úlayernorm_preÚlayernorm_postrç  rY  rÀ  Úvision_adapterrl  r<   s     €r?   r-   zLlama4VisionModel.__init__  sD  ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒØ Ô+ˆŒØ!Ô-ˆÔØ"Ô/ˆÔà œO¨t¬Ñ>À1ÑDÀqÑHˆÔØÔ'¨Ñ-ˆŒ
å6°vÑ>Ô>ˆÔå!œ|¨D¬J½¼ÀTÔEUÑ9VÔ9VÑ,VÑWÔWˆÔÝ(*¬°T´ZÅ%Ä+ÈdÔN^Ð`dÔ`pÑBqÔBqÑ5qÑ(rÔ(rˆÔ%Ý ;¸FÑ CÔ CˆÔõ  œ\¨$Ô*:Ñ;Ô;ˆÔÝ œl¨4Ô+;Ñ<Ô<ˆÔõ )¨Ñ0Ô0ˆŒ
Ý9¸&ÑAÔAˆÔØ�ŠÑÔÐÐÐr@   c                 ó   — | j         S )zg
        This function is used to fetch the first embedding layer to activate grads on inputs.
        )r  rx   s    r?   Úget_input_embeddingsz&Llama4VisionModel.get_input_embeddings  s   € ð Ô#Ð#r@   NÚpixel_valuesrí   rä  rê  rë  rB   .c                 óð  — |�|n| j         j        }|�|n| j         j        }|�|n| j         j        }|j        \  }}}	}
d}d}|                      |¦  «        }|j        \  }}}|                     ||z  |z  ||¦  «        }| j                             |j        d         d|j        d         ¦  «        }t          j
        ||gd¬¦  «        }|dz  }|                     ||z  |||¦  «        }| j                             |j        |j        ¬¦  «        }||z   }|                      |¦  «        }|                     |d|¦  «        }|                      |¦  «        }|                      |d|||¬¦  «        }|j        }|                      |¦  «        }|dd…dd…dd…f         }|                      |¦  «        }|r|j        nd}|r	|d         }nd}|st/          d	„ |||fD ¦   «         ¦  «        S t1          |||¬
¦  «        S )a  

        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, MllamaVisionModel

        >>> checkpoint = "meta-llama/Llama-3.2-11B-Vision"
        >>> model = MllamaVisionModel.from_pretrained(checkpoint)
        >>> processor = AutoProcessor.from_pretrained(checkpoint)

        >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))
        >>> inputs = processor(images=image, return_tensors="pt")

        >>> output = model(**inputs)

        >>> print(output.last_hidden_state.shape)
        torch.Size([1, 1, 4, 1025, 7680])
        ```
        Nr#   r   rD   rE   ©r‘   r¨   )rí   rê  rä  rH  r+   c              3   ó   K  — | ]}|®|V — Œ	d S ra   r4  rî  s     r?   rð  z,Llama4VisionModel.forward.<locals>.<genexpr>~  s(   è è € Ð_Ð_˜qÐQRÐQ^˜ÐQ^ÐQ^ÐQ^ÐQ^Ð_Ð_r@   rñ  )r(   rä  rê  rë  rH   r  rž   rK  rÊ   r5   r  rM  r�   r‘   r¨   r  rG   r  rY  rs  r  r  rA   r„   r   )r=   r!  rí   rä  rê  rë  rÂ   Úbatch_size_times_num_tilesrþ  rº  r»  Únum_concurrent_mediaÚ
num_chunksrã  r7  r·  r™   rK  Úpositional_embeddingrH  r‚   rA   rZ  s                          r?   rO   zLlama4VisionModel.forward"  sŠ  € ðD 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆð COÔBTÑ?Ð" L°&¸%Ø ÐØˆ
Ø×+Ò+¨LÑ9Ô9ˆØ%1Ô%7Ñ"ˆˆ;˜
ð $×+Ò+Ø&Ð)=Ñ=À
ÑJÈKÐYcñ
ô 
ˆð Ô.×5Ò5°lÔ6HÈÔ6KÈQÐP\ÔPbÐceÔPfÑgÔgˆÝ”y ,°Ð!@ÀaÐHÑHÔHˆØ�qÑˆð $×+Ò+Ø&Ð)=Ñ=¸zÈ;ÐXbñ
ô 
ˆð  $Ô<×?Ò?ÀlÔFXÐamÔatÐ?ÑuÔuÐØ#Ð&:Ñ:ˆà×)Ò)¨,Ñ7Ô7ˆà#×(Ò(Ð)CÀRÈÑTÔTˆØ×(Ò(¨Ñ6Ô6ˆà—’ØØØ!5Ø/Øð ñ 
ô 
ˆð Ô/ˆà×*Ò*¨<Ñ8Ô8ˆà# A A A s¨ s¨A¨A¨A IÔ.ˆð ×*Ò*¨<Ñ8Ô8ˆà0DÐN˜Ô,Ð,È$ˆàð 	Ø œˆJˆJàˆJàð 	`ÝÐ_Ð_ \°=À*Ð$MÐ_Ñ_Ô_Ñ_Ô_Ð_å)Ø*Ø'Ø!ð
ñ 
ô 
ð 	
r@   rö  )rP   rQ   rR   r  rO  r~  r   rN  r-   r   r5   rS   r9  r   r„   rO   rT   rU   s   @r?   rJ  rJ  ý  s  ø€ € € € € € Ø&ÐØ!ÐØ3Ð4ÐØÐÐÑðÐ1ð ð ð ð ð ð ð2$ð $ð $ð /3Ø)-Ø,0Ø#'ðb
ð b
à”lðb
ð œ tÑ+ðb
ð   $™;ð	b
ð
 # T™kðb
ð ˜D‘[ðb
ð 
$ e¨E¬L¸#Ð,=Ô&>Ñ	>ðb
ð b
ð b
ð b
ð b
ð b
ð b
ð b
r@   rJ  c            #       ó`  ‡ — e Zd ZU ddgZi ZdZeed<   defˆ fd„Zd„ Z	d„ Z
d„ Zd	„ Ze ed
¬¦  «         ed¬¦  «        dej        dedee         deez  fd„¦   «         ¦   «         ¦   «         Zdej        dej        dej        fd„Ze ed
¬¦  «        e	 	 	 	 	 	 	 	 	 	 	 	 	 d#dej        dz  dej        dz  dej        dz  dej        dz  dedz  dej        dz  dedz  dej        dz  dedz  dedz  dedz  d edz  d!eej        z  dee         deez  fd"„¦   «         ¦   «         ¦   «         Zˆ xZ S )$ÚLlama4ForConditionalGenerationr)  rà  rY  r(   c                 óž  •— t          ¦   «                              |¦  «         t          |j        ¦  «        | _        t          |¦  «        | _        t          |j        ¦  «        | _	        |j        j
        | _
        t          | j        d¦  «        r| j        j        | _        n| j        j        j        pd| _        |                      ¦   «          d S )Nr`  rD   )r,   r-   rJ  rª  r  r¨  Úmulti_modal_projectorr‚  rC  rƒ  rb  r  r(   r`  rl  r<   s     €r?   r-   z'Llama4ForConditionalGeneration.__init__�  s®   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý-¨fÔ.BÑCÔCˆÔå%>¸vÑ%FÔ%FˆÔ"Ý/°Ô0BÑCÔCˆÔØ Ô,Ô7ˆŒÝ�4”; Ñ/Ô/ð 	KØ $¤Ô 8ˆDÔÐà $¤Ô 7Ô DÐ JÈˆDÔà�ŠÑÔÐÐÐr@   c                 ó4   — | j                              ¦   «         S ra   )rƒ  Úget_output_embeddingsrx   s    r?   r.  z4Llama4ForConditionalGeneration.get_output_embeddings›  s   € ØÔ"×8Ò8Ñ:Ô:Ð:r@   c                 ó:   — | j                              |¦  «         d S ra   )rƒ  Úset_output_embeddings)r=   Únew_embeddingss     r?   r0  z4Llama4ForConditionalGeneration.set_output_embeddingsž  s   € ØÔ×1Ò1°.ÑAÔAÐAÐAÐAr@   c                 ó:   — | j                              |¦  «         d S ra   )rƒ  Úset_decoder)r=   Údecoders     r?   r3  z*Llama4ForConditionalGeneration.set_decoder¡  s   € ØÔ×'Ò'¨Ñ0Ô0Ð0Ð0Ð0r@   c                 ó4   — | j                              ¦   «         S ra   )rƒ  Úget_decoderrx   s    r?   r6  z*Llama4ForConditionalGeneration.get_decoder¤  s   € ØÔ"×.Ò.Ñ0Ô0Ð0r@   F)Útie_last_hidden_stateszOObtains image last hidden states from the vision tower and apply al projection.r–  r!  Úvision_feature_select_strategyrÂ   rB   c                 óZ   — d„ |                      ¦   «         D ¦   «         } | j        |fi |¤ŽS )ai  
        pixel_values (`torch.FloatTensor` of shape `(batch_size, channels, height, width)`)
            The tensors corresponding to the input images.
        vision_feature_select_strategy (`str`):
            The feature selection strategy used to select the vision feature from the vision backbone.
            Can be one of `"default"` or `"full"`
        c                 ó   — i | ]
\  }}|®||“ŒS ra   r4  )r]  Úkrï  s      r?   ú
<dictcomp>zELlama4ForConditionalGeneration.get_image_features.<locals>.<dictcomp>·  s   € ÐCÐCÐC™4˜1˜a°Q°]�!�Q°]°]°]r@   )Úitemsr  )r=   r!  r8  rÂ   s       r?   Úget_image_featuresz1Llama4ForConditionalGeneration.get_image_features§  s;   € ð  DÐC 6§<¢<¡>¤>ÐCÑCÔCˆØ ˆtÔ  Ð8Ð8°Ð8Ð8Ð8r@   rm  rn  r®  c           	      ó  — |€f| |                       ¦   «         t          j        d| j        j        t          j        |j        ¬¦  «        ¦  «        k    }|                     d¦  «        }n|| j        j        k    }|                     ¦   «         }| 	                    d¦  «         
                    |j        ¦  «        }t          ||j        d         z  |                     ¦   «         k    d|› d|j        d         › �¦  «         |S )zï
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        Nr4  r#  rD   z6Image features and image tokens do not match, tokens: z, features: r   )r   r5   Úfullr(   Úimage_token_idÚlongr¨   Úallr¢   ru  r�   r   rH   Únumel)r=   rm  rn  r®  Úspecial_image_maskÚn_image_tokenss         r?   Úget_placeholder_maskz3Llama4ForConditionalGeneration.get_placeholder_maskº  s  € ð ÐØ!.Ð2M°$×2KÒ2KÑ2MÔ2MÝ”
˜2˜tœ{Ô9ÅÄÐTaÔThÐiÑiÔiñ3ô 3ò "Ðð "4×!7Ò!7¸Ñ!;Ô!;ÐÐà!*¨d¬kÔ.HÒ!HÐà+×/Ò/Ñ1Ô1ˆØ/×9Ò9¸"Ñ=Ô=×@Ò@ÀÔAUÑVÔVÐÝØ˜]Ô0°Ô4Ñ4¸×8LÒ8LÑ8NÔ8NÒNØzÀ^ÐzÐzÐaoÔauÐvwÔaxÐzÐzñ	
ô 	
ð 	
ð "Ð!r@   Nr   rí   rÐ   r  rˆ  r2  rä  rê  rë  r‰  c                 óR  — |
�|
n| j         j        }
|�|n| j         j        }|�|n| j         j        }|du |duz  rt	          d¦  «        ‚|�|�t	          d¦  «        ‚|€ |                      ¦   «         |¦  «        }|�§|                      ||d¬¦  «        j        }|                     d| 	                    d¦  «        ¦  «        }|  
                    |¦  «                             |j        |j        ¦  «        }|                      |||¬¦  «        }|                     ||¦  «        } | j        d|||||	|
|||dœ	|¤Ž}|d	         }d}|�  | j        d||| j         j        j        d
œ|¤Ž}|s|f|dd…         z   }|�|f|z   n|S t)          |||j        |j        |j        |�|nd¬¦  «        S )añ  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, LlavaForConditionalGeneration

        >>> model = LlavaForConditionalGeneration.from_pretrained("llava-hf/llava-1.5-7b-hf")
        >>> processor = AutoProcessor.from_pretrained("llava-hf/llava-1.5-7b-hf")

        >>> prompt = "USER: <image>\nWhat's the content of the image? ASSISTANT:"
        >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> inputs = processor(images=image, text=prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(**inputs, max_new_tokens=15)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "USER:  \nWhat's the content of the image? ASSISTANT: The image features a busy city street with a stop sign prominently displayed"
        ```Nrp  zdYou cannot specify both pixel_values and inputs_embeds at the same time, and must specify either oneT)r!  r8  rë  rD   )rn  r®  )	rí   rÐ   r  rn  r2  rä  rê  rë  r‰  r   r‹  r#   )r�  rŒ  r  rA   rZ  rš  r4  )r(   rä  rê  rë  rt  r   r>  rs  rG   r²  r,  r�   r¨   r‘   rG  Úmasked_scatterrƒ  r�  rC  rb  r™  r  rA   rZ  )r=   rm  r!  rí   rÐ   r  rn  r8  rˆ  r2  rä  rê  rë  r‰  rÂ   r®  Úvision_flatÚprojected_vision_flatrE  r‘  rŒ  r�  r‚   s                          r?   rO   z&Llama4ForConditionalGeneration.forwardÑ  sx  € ðd 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆà˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ#¨Ð(AÝØvñô ð ð Ð Ø7˜D×5Ò5Ñ7Ô7¸	ÑBÔBˆMàÐ#Ø!×4Ò4Ø)Ø/MØ ð 5ñ ô ô  ð	 ð )×-Ò-¨b°.×2EÒ2EÀbÑ2IÔ2IÑJÔJˆKØ$(×$>Ò$>¸{Ñ$KÔ$K×$NÒ$NØÔ$ mÔ&9ñ%ô %Ð!ð "&×!:Ò!:Ø¨ÐG\ð ";ñ "ô "Ðð *×8Ò8Ð9KÐMbÑcÔcˆMà%�$Ô%ð 
Ø)Ø%Ø+Ø'ØØ/Ø!5Ø#Ø)ð
ð 
ð ð
ð 
ˆð ˜”ˆàˆØÐØ%�4Ô%ð Ø f¸¼Ô9PÔ9[ðð Ø_eðð ˆDð ð 	DØ�Y ¨¨¨¤Ñ,ˆFØ'+Ð'7�D�7˜VÑ#Ð#¸VÐCå+ØØØ#Ô3Ø!Ô/ØÔ)Ø2>Ð2J  ÐPTð
ñ 
ô 
ð 	
r@   )NNNNNNNNNNNNr   )!rP   rQ   rR   r~  r•  r  r$   rN  r-   r.  r0  r3  r6  r!   r"   r   r5   r:  rÍ   r   r   r„   r   r>  r8  rG  rS   r	   r9  rŽ  r™  rO   rT   rU   s   @r?   r*  r*  ‡  sÜ  ø€ € € € € € Ø1Ð3MÐNÐØ€HØÐØÐÐÑð˜|ð ð ð ð ð ð ð;ð ;ð ;ðBð Bð Bð1ð 1ð 1ð1ð 1ð 1ð  Ø€_¨EÐ2Ñ2Ô2Ø€^Ð!rÐsÑsÔsð9àÔ'ð9ð ),ð9ð Ð+Ô,ð	9ð
 
Ð+Ñ	+ð9ð 9ð 9ñ tÔsñ 3Ô2ñ  Ôð9ð "ØÔ)ð"Ø:?Ô:Kð"Ø]bÔ]nð"ð "ð "ð "ð.  Ø€_¨EÐ2Ñ2Ô2Øð .2Ø15Ø.2Ø04Ø(,Ø26Ø59Ø*.Ø!%Ø)-Ø,0Ø#'Ø-.ðp
ð p
àÔ# dÑ*ðp
ð Ô'¨$Ñ.ðp
ð œ tÑ+ð	p
ð
 Ô&¨Ñ-ðp
ð  ™ðp
ð Ô(¨4Ñ/ðp
ð ),¨d©
ðp
ð Ô  4Ñ'ðp
ð ˜$‘;ðp
ð   $™;ðp
ð # T™kðp
ð ˜D‘[ðp
ð ˜eœlÑ*ðp
ð Ð+Ô,ðp
ð  
Ð-Ñ	-ð!p
ð p
ð p
ñ „^ñ 3Ô2ñ  Ôðp
ð p
ð p
ð p
ð p
r@   r*  )r<  rX  rJ  r‚  r*  )rè   )er°  Úcollections.abcr   Údataclassesr   r5   Útorch.nnr3   Útorch.nn.functionalrŽ   r¥  Ú/transformers.models.llama4.configuration_llama4r   Ú r   rD  Úactivationsr   Úcache_utilsr	   r
   Ú
generationr   Úmasking_utilsr   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   Úutils.deprecationr   Úutils.genericr    r!   Úutils.output_capturingr"   Úconfiguration_llama4r$   r%   Ú
get_loggerrP   ÚloggerÚModuler'   rW   re   r{   r\   r†   r—   r§   rS   r„   rá   rŽ  rç   rt   rý   r   r  r)  r<  rX  r‚  r™  rœ  r¨  r¾  rÀ  rÍ  rÓ  rÕ  rÜ  rà  rç  rø  rF  rJ  r*  Ú__all__r4  r@   r?   ú<module>re     sÒ  ðð €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð à NÐ NÐ NÐ NÐ NÐ Nà &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )Ø KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ BÐ BÐ BÐ BÐ BÐ BØ 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð ð ðð ð ð ð ð ð ð ð GÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jØ 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø @Ð @Ð @Ð @Ð @Ð @Ð @Ð @ð 
ˆÔ	˜HÑ	%Ô	%€ðð ð ð ð ˜œ	ñ ô ð ðB)ð )ð )ð )ð )�B”Iñ )ô )ð )ð$!ð !ð !ð !ð !�u”x”ñ !ô !ð !ð=ð =ð =ð =ð =˜œ	ñ =ô =ð =ð(,ð ,ð ,ð ,ð ,�2”9ñ ,ô ,ð ,ð"ð "ð "ð "ð "�B”Iñ "ô "ð "ð,4ð 4ð 4ð 4ð 4 ¤	ñ 4ô 4ð 4ðn	2ØŒð	2àŒð	2ð Œ|ð	2ð ˆ5Œ<˜œÐ%Ô&ð		2ð 	2ð 	2ð 	2ð	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð( ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð %ð %ð %ðB ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð %ð %ð %ð2Y)ð Y)ð Y)ð Y)ð Y)˜"œ)ñ Y)ô Y)ð Y)ðx/ð /ð /ð /ð /Ð7ñ /ô /ð /ðd ðLð Lð Lð Lð L˜Oñ Lô Lñ „ðLð< ðZ
ð Z
ð Z
ð Z
ð Z
Ð+ñ Z
ô Z
ñ „ðZ
ðzM
ð M
ð M
ð M
ð M
Ð-¨ñ M
ô M
ð M
ð` €ððñ ô ð
 ð9ð 9ð 9ð 9ð 9 ;ñ 9ô 9ñ „ñô ð9ð0;ð ;ð ;ð ;ð ;�u”x”ñ ;ô ;ð ;ð"ð ð ð ð  ¤	ñ ô ð ðð ð ð(
)ð 
)ð 
)ð 
)ð 
) "¤)ñ 
)ô 
)ð 
)ð! E¤Lð !¸¼ð !ð !ð !ð !ð:ØŒ<ð:à	Œð:ð Œlð:ð ˆ5Œ<˜œÐ%Ô&ð	:ð :ð :ð :ð7)ð 7)ð 7)ð 7)ð 7)˜BœIñ 7)ô 7)ð 7)ðtð ð ð ð �b”iñ ô ð ð)ð )ð )ð )ð )Ð9ñ )ô )ð )ðXO
ð O
ð O
ð O
ð O
˜"œ)ñ O
ô O
ð O
ðdð ð ð ð ˜bœiñ ô ð ð*6ð 6ð 6ð 6ð 6 "¤)ñ 6ô 6ð 6ð<G
ð G
ð G
ð G
ð G
Ð-ñ G
ô G
ð G
ðT}
ð }
ð }
ð }
ð }
Ð%:¸Oñ }
ô }
ð }
ð@ð ð €€€r@   