§
    9²¸jl¼  ã                   óÀ  — d dl mZ d dlmZ d dlZd dlmZ ddlmZ ddl	m
Z
 ddlmZmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZ ddlmZ ddlmZmZmZmZ ddlmZm Z  ddl!m"Z"m#Z# ddl$m%Z% ddl&m'Z'm(Z(m)Z)m*Z* ddl+m,Z, ddl-m.Z.m/Z/ ddl0m1Z1 ddl2m3Z3 ddl4m5Z5m6Z6  ed¦  «         G d„ dej7        ¦  «        ¦   «         Z8 G d„ dej7        ¦  «        Z9 G d„ d ej7        ¦  «        Z: G d!„ d"ej7        ¦  «        Z; G d#„ d$ej7        ¦  «        Z<d%„ Z= G d&„ d'ej7        ¦  «        Z> G d(„ d)ej7        ¦  «        Z? G d*„ d+ej7        ¦  «        Z@d,„ ZA ed-¦  «        dYd.„¦   «         ZBd/ejC        d0eDd1ejC        fd2„ZE	 dZd4ej7        d5ejC        d6ejC        d7ejC        d8ejC        dz  d9eFd:eFd;e%e'         fd<„ZG eeB¦  «         G d=„ d>ej7        ¦  «        ¦   «         ZH G d?„ d@e¦  «        ZIe( G dA„ dBe#¦  «        ¦   «         ZJe( G dC„ dDe#¦  «        ¦   «         ZK G dE„ dFej7        ¦  «        ZLe( G dG„ dHeJ¦  «        ¦   «         ZMe( G dI„ dJeJe¦  «        ¦   «         ZN e(dK¬L¦  «        e G dM„ dNe¦  «        ¦   «         ¦   «         ZO e(dO¬L¦  «        e G dP„ dQe¦  «        ¦   «         ¦   «         ZP e(dR¬L¦  «         G dS„ dTeK¦  «        ¦   «         ZQ e(dU¬L¦  «         G dV„ dWeKe¦  «        ¦   «         ZRg dX¢ZSdS )[é    )ÚCallable)Ú	dataclassN)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚBaseModelOutputWithPoolingÚCausalLMOutputWithPastÚModelOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚtorch_compilable_check)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )Ú	AutoModelé   )Ú
AriaConfigÚAriaTextConfigÚRMSNormc                   óT   ‡ — e Zd Zd	deddfˆ fd„Zdej        dej        fd„Zd„ Zˆ xZ	S )
ÚAriaTextRMSNormç�íµ ÷Æ°>ÚepsÚreturnNc                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )z>
        AriaTextRMSNorm is equivalent to T5LayerNorm
        N)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizer+   Ú	__class__s      €úv/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/aria/modeling_aria.pyr/   zAriaTextRMSNorm.__init__6   sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐó    Úhidden_statesc                 ó  — |j         }|                     t          j        ¦  «        }|                     d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        |                     |¦  «        z  S )Nr"   éÿÿÿÿT)Úkeepdim)	ÚdtypeÚtor1   Úfloat32ÚpowÚmeanÚrsqrtr4   r3   )r5   r:   Úinput_dtypeÚvariances       r8   ÚforwardzAriaTextRMSNorm.forward>   s|   € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØ ×$Ò$ QÑ'Ô'×,Ò,¨R¸Ð,Ñ>Ô>ˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆØŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:r9   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)Útupler3   Úshaper4   ©r5   s    r8   Ú
extra_reprzAriaTextRMSNorm.extra_reprE   s&   € Ý˜œÔ)Ñ*Ô*ÐIÐI°$Ô2GÐIÐIÐIr9   )r*   )
Ú__name__Ú
__module__Ú__qualname__Úfloatr/   r1   ÚTensorrF   rK   Ú__classcell__©r7   s   @r8   r)   r)   4   sŒ   ø€ € € € € ð$ð $¨ð $¸$ð $ð $ð $ð $ð $ð $ð; U¤\ð ;°e´lð ;ð ;ð ;ð ;ðJð Jð Jð Jð Jð Jð Jr9   r)   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚAriaProjectorMLPa!  
    Feed-Forward Network module for the Aria Projector.

    Args:
        in_features (`int`):
            Input embedding dimension.
        hidden_features (`int`):
            Hidden dimension of the feed-forward network.
        output_dim (`int`):
            Output dimension.
    c                 óÜ   •— t          ¦   «                              ¦   «          t          j        ||d¬¦  «        | _        t          j        ||d¬¦  «        | _        t          d         | _        d S )NF©ÚbiasÚgelu_new)r.   r/   r   ÚLinearÚ	linear_inÚ
linear_outr   Úact)r5   Úin_featuresÚhidden_featuresÚ
output_dimr7   s       €r8   r/   zAriaProjectorMLP.__init__V   sY   ø€ Ý‰Œ×ÒÑÔÐÝœ ;°ÀeÐLÑLÔLˆŒÝœ) O°ZÀeÐLÑLÔLˆŒÝ˜*Ô%ˆŒˆˆr9   c                 ó€   — |                       |                      |¦  «        ¦  «        }|                      |¦  «        }|S ©N)r\   rZ   r[   )r5   r:   s     r8   rF   zAriaProjectorMLP.forward\   s6   € ØŸš §¢°Ñ!>Ô!>Ñ?Ô?ˆØŸš¨Ñ6Ô6ˆØÐr9   ©rL   rM   rN   Ú__doc__r/   rF   rQ   rR   s   @r8   rT   rT   I   sQ   ø€ € € € € ð
ð 
ð&ð &ð &ð &ð &ðð ð ð ð ð ð r9   rT   c                   ó6   ‡ — e Zd ZdZddedefˆ fd„Zd	d„Zˆ xZS )
ÚAriaCrossAttentionzv
    Aria Cross-Attention module.

    Args:
        config (`AriaConfig`):
            The configuration to use.
    r   ÚconfigÚdropout_ratec                 ó0  •— t          ¦   «                              ¦   «          |j        j        }|j        j        }|| _        t          j        ||d¬¦  «        | _        t          j        ||d¬¦  «        | _	        t          j        ||d¬¦  «        | _
        t          j        ||d¬¦  «        | _        t          j        ||¦  «        | _        t          j        |¦  «        | _        t          j        |¦  «        | _        t          j        |¦  «        | _        d S )NFrV   T)Úbatch_first)r.   r/   Úvision_configr6   Únum_attention_headsÚ	num_headsr   rY   Úq_projÚk_projÚv_projÚMultiheadAttentionÚmultihead_attnÚlinearÚDropoutÚdropoutÚ	LayerNormÚ
layer_normÚlayer_norm_kv)r5   rf   rg   r6   rl   r7   s        €r8   r/   zAriaCrossAttention.__init__k   sè   ø€ Ý‰Œ×ÒÑÔÐØÔ*Ô6ˆØÔ(Ô<ˆ	Ø"ˆŒÝ”i ¨[¸uÐEÑEÔEˆŒÝ”i ¨[¸uÐEÑEÔEˆŒÝ”i ¨[¸uÐEÑEÔEˆŒõ !Ô3°KÀÐX\Ð]Ñ]Ô]ˆÔÝ”i ¨[Ñ9Ô9ˆŒÝ”z ,Ñ/Ô/ˆŒåœ, {Ñ3Ô3ˆŒÝœ\¨+Ñ6Ô6ˆÔÐÐr9   Nc                 ó\  — |                       |                      |¦  «        ¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      ||||¬¦  «        \  }}|                      |                      |¦  «        ¦  «        }|S )aÉ  
        Forward pass of the AriaCrossAttention module.

        Args:
            key_value_states (`torch.Tensor`):
                Input tensor for key and value.
            hidden_states (`torch.Tensor`):
                Input tensor for query.
            attn_mask (`torch.Tensor`, *optional*, defaults to None):
                Attention mask.

        Returns:
            torch.Tensor:
                Output tensor after cross-attention.
        ©Ú	attn_mask)rm   rv   rw   rn   ro   rq   rt   rr   )	r5   Úkey_value_statesr:   rz   ÚqueryÚkeyÚvalueÚattn_outputÚ_s	            r8   rF   zAriaCrossAttention.forward|   s�   € ð  —’˜DŸOšO¨MÑ:Ô:Ñ;Ô;ˆà×-Ò-Ð.>Ñ?Ô?ÐØ�kŠkÐ*Ñ+Ô+ˆØ—’Ð,Ñ-Ô-ˆà×,Ò,¨U°C¸È)Ð,ÑTÔT‰ˆ�Qà—l’l 4§;¢;¨{Ñ#;Ô#;Ñ<Ô<ˆàÐr9   )r   ra   )	rL   rM   rN   rc   r%   rO   r/   rF   rQ   rR   s   @r8   re   re   b   sn   ø€ € € € € ðð ð7ð 7˜zð 7¸ð 7ð 7ð 7ð 7ð 7ð 7ð"ð ð ð ð ð ð ð r9   re   c                   óT   ‡ — e Zd ZdZdefˆ fd„Zddej        dej        dz  fd„Zˆ xZ	S )	ÚAriaProjectora  
    Aria Projector module.

    This module projects vision features into the language model's embedding space, enabling interaction between vision and language components.

    Args:
        config (`AriaConfig`):
            Configuration object for the model.
    rf   c                 ó$  •— t          ¦   «                              ¦   «          |j        | _        |j        j        | _        |j        j        | _        |j        j        | _	        |j
        j        | _        |j
        j        | _        t          j        t          j        |j        | j        ¦  «        ¦  «        | _        t'          |¦  «        | _        t          j        | j        ¦  «        | _        t/          | j        | j        | j        ¦  «        | _        d S ra   )r.   r/   Úprojector_patch_to_query_dictÚpatch_to_query_dictrj   r6   r]   rk   rl   Úkv_dimÚtext_configr^   r_   r   r0   r1   ÚzerosÚ'max_value_projector_patch_to_query_dictr|   re   Ú
cross_attnru   rv   rT   Úfeed_forward©r5   rf   r7   s     €r8   r/   zAriaProjector.__init__¤   sÔ   ø€ õ 	‰Œ×ÒÑÔÐà#)Ô#GˆÔ Ø!Ô/Ô;ˆÔØÔ-ÔAˆŒØÔ*Ô6ˆŒØ%Ô1Ô=ˆÔØ Ô,Ô8ˆŒå”\¥%¤+¨fÔ.\Ð^bÔ^nÑ"oÔ"oÑpÔpˆŒ
å,¨VÑ4Ô4ˆŒåœ, tÔ'7Ñ8Ô8ˆŒÝ,¨TÔ-=¸tÔ?SÐUYÔUdÑeÔeˆÔÐÐr9   Nr{   rz   c                 ób  — |j         d         |j         d         }}|| j        vr-t          d|› d| j                             ¦   «         › d�¦  «        ‚| j        |         }| j        d|…                              d¦  «                             |dd¦  «        }|�X|                     | j        d¦  «        }|                     d¦  «         	                    d| 
                    d¦  «        d¦  «        }|                      |||¬¦  «        }|                      |                      |¦  «        ¦  «        }|S )	a�  
        Forward pass of the Projector module.

        Args:
            key_value_states (`torch.Tensor`):
                Input tensor of shape (batch_size, num_patches, kv_dim).
            attn_mask (`torch.Tensor`, *optional*, default is None):
                Attention mask.

        Returns:
            `torch.Tensor`: Output tensor of shape (batch_size, query_number, output_dim).
        r   r$   zNumber of patches z: not found in patch_to_query_dict amongst possible values ú.Nr<   ry   )rI   r…   ÚKeyErrorÚkeysr|   Ú	unsqueezeÚrepeatÚrepeat_interleaverl   ÚexpandÚsizerŠ   r‹   rv   )	r5   r{   rz   Ú
batch_sizeÚnum_patchesÚ	query_numÚqueriesÚattention_outÚouts	            r8   rF   zAriaProjector.forward¸   sR  € ð #3Ô"8¸Ô";Ð=MÔ=SÐTUÔ=V�Kˆ
à˜dÔ6Ð6Ð6Ýð O [ð  Oð  OÐlpô  mE÷  mJò  mJñ  mLô  mLð  Oð  Oð  Oñô ð ð Ô,¨[Ô9ˆ	à”*˜Z˜i˜ZÔ(×2Ò2°1Ñ5Ô5×<Ò<¸ZÈÈAÑNÔNˆàÐ Ø!×3Ò3°D´NÀAÑFÔFˆIØ!×+Ò+¨AÑ.Ô.×5Ò5°b¸'¿,º,Àq¹/¼/È2ÑNÔNˆIàŸšÐ(8¸'ÈY˜ÑWÔWˆà×Ò §¢°Ñ >Ô >Ñ?Ô?ˆàˆ
r9   ra   )
rL   rM   rN   rc   r%   r/   r1   rP   rF   rQ   rR   s   @r8   r‚   r‚   ™   s‡   ø€ € € € € ðð ðfàðfð fð fð fð fð fð(ð ¨¬ð ÀÄÐPTÑATð ð ð ð ð ð ð ð r9   r‚   c                   ó.   ‡ — e Zd ZdZdefˆ fd„Zd„ Zˆ xZS )ÚAriaSharedExpertsMLPa/  
    Shared Expert MLP for shared experts.

    Unlike routed experts, shared experts process all tokens without routing.
    This class reconfigures the intermediate size in comparison to the LlamaMLP.

    Args:
        config (`AriaTextConfig`): Configuration object for the Aria language model.
    rf   c                 óÆ  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        |j        z  | _        t          j        | j        | j        |j        ¬¦  «        | _	        t          j        | j        | j        |j        ¬¦  «        | _
        t          j        | j        | j        |j        ¬¦  «        | _        t          |j                 | _        d S )NrV   )r.   r/   rf   r6   Úintermediate_sizeÚmoe_num_shared_expertsr   rY   Úmlp_biasÚ	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fnrŒ   s     €r8   r/   zAriaSharedExpertsMLP.__init__å   s·   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ!'Ô!9¸FÔ<YÑ!YˆÔÝœ 4Ô#3°TÔ5KÐRXÔRaÐbÑbÔbˆŒÝ”y Ô!1°4Ô3IÐPVÔP_Ð`Ñ`Ô`ˆŒÝœ 4Ô#9¸4Ô;KÐRXÔRaÐbÑbÔbˆŒÝ˜VÔ.Ô/ˆŒˆˆr9   c                 ó¨   — |                       |                      |                      |¦  «        ¦  «        |                      |¦  «        z  ¦  «        }|S ra   )r¤   r¦   r¢   r£   )r5   Úxr¤   s      r8   rF   zAriaSharedExpertsMLP.forwardï   sA   € Ø—N’N 4§;¢;¨t¯~ª~¸aÑ/@Ô/@Ñ#AÔ#AÀDÇLÂLÐQRÁOÄOÑ#SÑTÔTˆ	ØÐr9   )rL   rM   rN   rc   r&   r/   rF   rQ   rR   s   @r8   r�   r�   Ú   s]   ø€ € € € € ðð ð0˜~ð 0ð 0ð 0ð 0ð 0ð 0ðð ð ð ð ð ð r9   r�   c                 óÚ  — | j         d         }|j         d         }t          j        ||| j        | j        ¬¦  «        }t          j        |d¬¦  «        }t          j        dt          j        |j        ¬¦  «        }t          j        ||f¦  «        }t          |j         d         ¦  «        D ]A}||         }	||dz            }
| |	|
…         }t          j	        |||         ¦  «        }|||	|
…<   ŒB|S )a*  
    Compute the matrix multiplication (GEMM) for each expert sequentially. This approach is computationally inefficient, especially when dealing with a large number of experts.

    Args:
        token_states (torch.Tensor): Input tensor of shape (num_tokens, in_features).
        expert_weights (torch.Tensor): Weight tensor of shape (num_experts, in_features, out_features).
        tokens_per_expert (torch.Tensor): Number of tokens assigned to each expert.

    Returns:
        torch.Tensor: Output tensor of shape (num_tokens, out_features).
    r   r<   ©r>   Údevice©Údimr$   )
rI   r1   rˆ   r>   r«   ÚcumsumÚlongÚcatÚrangeÚmatmul)Útoken_statesÚexpert_weightsÚtokens_per_expertÚ
num_tokensÚout_featuresÚoutputÚcumsum_num_tokensÚzero_tensorÚ
expert_numÚstartÚendÚtokensr›   s                r8   Úsequential_experts_gemmr¿   ô   sö   € ð Ô# AÔ&€JØ!Ô'¨Ô+€LÝŒ[˜ \¸Ô9KÐT`ÔTgÐhÑhÔh€FåœÐ%6¸AÐ>Ñ>Ô>Ðå”+˜a¥u¤zÐ:KÔ:RÐSÑSÔS€KÝœ	 ;Ð0AÐ"BÑCÔCÐå˜NÔ0°Ô3Ñ4Ô4ð  ð  ˆ
Ø! *Ô-ˆØ 
¨Q¡Ô/ˆØ˜e C˜iÔ(ˆåŒl˜6 >°*Ô#=Ñ>Ô>ˆØˆˆu�SˆyÑÐØ€Mr9   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚAriaGroupedExpertsGemmaP  
    Grouped GEMM (General Matrix Multiplication) module for efficient expert computation.
    This module utilizes the grouped_gemm library (https://github.com/fanshiqing/grouped_gemm)
    for optimized performance. If the grouped_gemm library is not installed, it gracefully
    falls back to a sequential GEMM implementation, which may be slower but ensures
    functionality.

    Args:
        in_features (`int`):
            Number of input features.
        out_features (`int`):
            Number of output features.
        groups (`int`):
            Number of expert groups.
    c                 óÌ   •— t          ¦   «                              ¦   «          || _        || _        || _        t          j        t          j        |||¦  «        ¦  «        | _	        d S ra   )
r.   r/   r]   r·   Úgroupsr   r0   r1   Úemptyr3   )r5   r]   r·   rÃ   r7   s       €r8   r/   zAriaGroupedExpertsGemm.__init__$  sS   ø€ Ý‰Œ×ÒÑÔÐØ&ˆÔØ(ˆÔØˆŒÝ”l¥5¤;¨v°{ÀLÑ#QÔ#QÑRÔRˆŒˆˆr9   c                 óR   — t          || j        |                     ¦   «         ¦  «        S )au  
        Perform grouped matrix multiplication.

        Args:
            input (`torch.Tensor`):
                Input tensor of shape (num_tokens, in_features).
            tokens_per_expert (`torch.Tensor`):
                Number of tokens assigned to each expert.

        Returns:
            torch.Tensor: Output tensor of shape (num_tokens, out_features).
        )r¿   r3   Úcpu)r5   Úinputrµ   s      r8   rF   zAriaGroupedExpertsGemm.forward+  s-   € õ 'ØØŒKØ×!Ò!Ñ#Ô#ñ
ô 
ð 	
r9   rb   rR   s   @r8   rÁ   rÁ     sV   ø€ € € € € ðð ð Sð Sð Sð Sð Sð
ð 
ð 
ð 
ð 
ð 
ð 
r9   rÁ   c                   óD   ‡ — e Zd Zdeddfˆ fd„Zd„ Zdej        fd„Zˆ xZ	S )ÚAriaExpertsrf   r,   Nc                 óð   •— t          ¦   «                              ¦   «          || _        t          |j        |j        dz  |j        ¦  «        | _        t          |j        |j        |j        ¦  «        | _        d S )Nr"   )	r.   r/   rf   rÁ   r6   rŸ   Úmoe_num_expertsÚfc1Úfc2rŒ   s     €r8   r/   zAriaExperts.__init__@  sd   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ)¨&Ô*<¸fÔ>VÐYZÑ>ZÐ\bÔ\rÑsÔsˆŒÝ)¨&Ô*BÀFÔDVÐX^ÔXnÑoÔoˆŒˆˆr9   c                 ó”   — t          j        || j        j        d¬¦  «        \  }}t          j                             |d¬¦  «        }||fS )Nr$   )Úkr­   r<   r¬   )r1   Útopkrf   Úmoe_topkr   Ú
functionalÚsoftmax)r5   Úrouter_logitsÚ
top_logitsÚtop_indicesÚscoress        r8   Úroute_tokens_to_expertsz#AriaExperts.route_tokens_to_expertsF  sH   € Ý"'¤*¨]¸d¼kÔ>RÐXYÐ"ZÑ"ZÔ"ZÑˆ
�KÝ”×&Ò& z°rÐ&Ñ:Ô:ˆØ˜FÐ"Ð"r9   c                 ó  — |                       |¦  «        \  }}|j        }t          j        |                     ¦   «                              t          j        ¦  «        | j        j        d| j        j        dz
  ¬¦  «                             |¦  «        }|}| 	                    d¦  «        }t          j
        |¦  «        }	|                     d|	| j        j        z  ¦  «        }
|                      |
|¦  «        }t          j        |dd¬¦  «        \  }}t          j                             |¦  «        |z  }|                      ||¦  «        }t          j        |j        d         | j        j        z  |                     d¦  «        f|j        |j        ¬¦  «        }|                     d|	|¦  «         | 	                    d| j        j        |                     d¦  «        ¦  «        }||                     d¦  «        z                       d¬¦  «        }|S )Nr   r$   )ÚbinsÚminÚmaxr<   r"   r¬   rª   )rØ   r>   r1   ÚhistcÚflattenr?   r@   rf   rË   ÚviewÚargsortÚindex_selectrÑ   rÌ   Úchunkr   rÒ   ÚsilurÍ   rˆ   rI   r•   r«   Úindex_copy_r‘   Úsum)r5   r:   rÔ   Útop_k_indexÚtop_k_weightsÚoriginal_dtyperµ   ÚindicesÚflatten_indicesÚsorted_indicesÚpermuted_tokensÚ
fc1_outputÚ
projectionÚgateÚexpert_outputÚunpermuted_tokensr¸   s                    r8   rF   zAriaExperts.forwardK  sã  € Ø%)×%AÒ%AÀ-Ñ%PÔ%PÑ"ˆ�]Ø$Ô*ˆÝ!œKØ×ÒÑ!Ô!×$Ò$¥U¤]Ñ3Ô3Ø”Ô,ØØ”Ô+¨aÑ/ð	
ñ 
ô 
÷
 Š"ˆ^Ñ
Ô
ð 	ð ˆà!Ÿ,š, rÑ*Ô*ˆÝœ Ñ7Ô7ˆØ'×4Ò4°Q¸È$Ì+ÔJ^Ñ8^Ñ_Ô_ˆà—X’X˜oÐ/@ÑAÔAˆ
Ý œ; z°1¸"Ð=Ñ=Ô=Ñˆ
�DÝ”]×'Ò'¨
Ñ3Ô3°dÑ:ˆ
ØŸš Ð->Ñ?Ô?ˆå!œKØÔ  Ô# d¤kÔ&:Ñ:¸M×<NÒ<NÈqÑ<QÔ<QÐRØÔ%Ø Ô'ð
ñ 
ô 
Ðð
 	×%Ò% a¨¸ÑGÔGÐGØ-×2Ò2°2°t´{Ô7KÈ]×M_ÒM_Ð`aÑMbÔMbÑcÔcÐà# m×&=Ò&=¸bÑ&AÔ&AÑA×FÒFÈ1ÐFÑMÔMˆØˆr9   )
rL   rM   rN   r&   r/   rØ   r1   rP   rF   rQ   rR   s   @r8   rÉ   rÉ   ?  s   ø€ € € € € ðp˜~ð p°$ð pð pð pð pð pð pð#ð #ð #ð
°u´|ð ð ð ð ð ð ð ð r9   rÉ   c                   óH   ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Zˆ xZS )ÚAriaTextMoELayerrf   c                 óò   •— t          ¦   «                              ¦   «          t          j        |j        |j        d¬¦  «        | _        t          |¦  «        | _        t          |¦  «        | _
        || _        d S ©NFrV   )r.   r/   r   rY   r6   rË   ÚrouterrÉ   Úexpertsr�   Úshared_expertsrf   rŒ   s     €r8   r/   zAriaTextMoELayer.__init__l  sb   ø€ Ý‰Œ×ÒÑÔÐÝ”i Ô 2°FÔ4JÐQVÐWÑWÔWˆŒÝ" 6Ñ*Ô*ˆŒÝ2°6Ñ:Ô:ˆÔØˆŒˆˆr9   r:   r,   c                 ó8  — |j         }|                     d|                     d¦  «        ¦  «        }|                      |¦  «        }|                      ||¦  «                             |¦  «        }|                      |                     |¦  «        ¦  «        }||z   S ©Nr<   )rI   rß   r•   rö   r÷   rø   )r5   r:   Úoriginal_shaperÔ   rð   Úshared_expert_outputs         r8   rF   zAriaTextMoELayer.forwards  s�   € Ø&Ô,ˆØ%×*Ò*¨2¨}×/AÒ/AÀ"Ñ/EÔ/EÑFÔFˆØŸš MÑ2Ô2ˆØŸš ]°MÑBÔB×GÒGÈÑWÔWˆØ#×2Ò2°=×3EÒ3EÀnÑ3UÔ3UÑVÔVÐØÐ3Ñ3Ð3r9   )	rL   rM   rN   r&   r/   r1   rP   rF   rQ   rR   s   @r8   ró   ró   k  sj   ø€ € € € € ð˜~ð ð ð ð ð ð ð4 U¤\ð 4°e´lð 4ð 4ð 4ð 4ð 4ð 4ð 4ð 4r9   ró   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..Nr<   r"   r¬   )rI   r1   r°   )r¨   Úx1Úx2s      r8   Úrotate_halfr   |  s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'r9   Úrotary_pos_embc                 ó¾   — |                      |¦  «        }|                      |¦  «        }| |z  t          | ¦  «        |z  z   }||z  t          |¦  «        |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )r‘   r   )ÚqrÏ   ÚcosÚsinÚunsqueeze_dimÚq_embedÚk_embeds          r8   Úapply_rotary_pos_embr	  ƒ  sc   € ð& �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�GÐÐr9   r:   Ún_repr,   c                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r$   N)rI   r”   Úreshape)r:   r
  ÚbatchÚnum_key_value_headsÚslenÚhead_dims         r8   Ú	repeat_kvr  �  s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTr9   ç        Úmoduler|   r}   r~   Úattention_maskÚscalingrt   Úkwargsc                 ó  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        |z  }
|�|
|z   }
t
          j                             |
dt          j        ¬¦  «         	                    |j
        ¦  «        }
t
          j                             |
|| j        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «                             ¦   «         }||
fS )Nr"   r   r<   )r­   r>   )ÚpÚtrainingr$   )r  Únum_key_value_groupsr1   r²   Ú	transposer   rÒ   rÓ   r@   r?   r>   rt   r  Ú
contiguous)r  r|   r}   r~   r  r  rt   r  Ú
key_statesÚvalue_statesÚattn_weightsr   s               r8   Úeager_attention_forwardr   ©  sé   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r9   c                   óÎ   ‡ — e Zd ZdZdedefˆ fd„Z	 	 	 ddej        de	ej        ej        f         dz  dej        dz  d	e
dz  d
ee         de	ej        ej        f         fd„Zˆ xZS )ÚAriaTextAttentionz=Multi-headed attention from 'Attention Is All You Need' paperrf   Ú	layer_idxc                 ó®  •— t          ¦   «                              ¦   «          || _        || _        t	          |d|j        |j        z  ¦  «        | _        |j        |j        z  | _	        | j        dz  | _
        |j        | _        d| _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        | j        z  |j        |j        ¬¦  «        | _        d S )Nr  g      à¿TrV   )r.   r/   rf   r#  Úgetattrr6   rk   r  r  r  r  Úattention_dropoutÚ	is_causalr   rY   Úattention_biasrm   rn   ro   Úo_proj©r5   rf   r#  r7   s      €r8   r/   zAriaTextAttention.__init__Æ  sB  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒÝ ¨
°FÔ4FÈ&ÔJdÑ4dÑeÔeˆŒØ$*Ô$>À&ÔB\Ñ$\ˆÔ!Ø”} dÑ*ˆŒØ!'Ô!9ˆÔØˆŒå”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ&¨¬Ñ6¸Ô8JÐQWÔQfð
ñ 
ô 
ˆŒˆˆr9   Nr:   Úposition_embeddingsr  Úpast_key_valuesr  r,   c                 ó"  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }	|                      |¦  «                             |¦  «                             dd¦  «        }
|\  }}t          ||	||¦  «        \  }}	|�|                     |	|
| j	        ¦  «        \  }	}
t          j        | j        j        t          ¦  «        } || ||	|
|f| j        sdn| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Nr<   r$   r"   r  )rt   r  )rI   r  rm   rß   r  rn   ro   r	  Úupdater#  r   Úget_interfacerf   Ú_attn_implementationr   r  r&  r  r  r  r)  )r5   r:   r+  r  r,  r  Úinput_shapeÚhidden_shapeÚquery_statesr  r  r  r  Úattention_interfacer   r  s                   r8   rF   zAriaTextAttention.forwardÝ  sÀ  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r9   )NNN)rL   rM   rN   rc   r&   Úintr/   r1   rP   rH   r	   r   r   rF   rQ   rR   s   @r8   r"  r"  Â  så   ø€ € € € € àGÐGð
˜~ð 
¸#ð 
ð 
ð 
ð 
ð 
ð 
ð4 IMØ.2Ø(,ð&)ð &)à”|ð&)ð # 5¤<°´Ð#=Ô>ÀÑEð&)ð œ tÑ+ð	&)ð
  ™ð&)ð Ð+Ô,ð&)ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð&)ð &)ð &)ð &)ð &)ð &)ð &)ð &)r9   r"  c                   óÖ   ‡ — e Zd ZdZdedefˆ fd„Z	 	 	 	 	 ddej        dej        dz  d	ej	        dz  d
e
dz  dedz  deej        ej        f         dz  dee         dej        fd„Zˆ xZS )ÚAriaTextDecoderLayerag  
    Aria Text Decoder Layer.

    This class defines a single decoder layer in the language model, incorporating self-attention and Mixture of Experts (MoE) feed-forward network.

    Args:
        config (`AriaTextConfig`):
            Configuration object for the text component of the model.
        layer_idx (`int`):
            Index of the layer.
    rf   r#  c                 ó4  •— t          ¦   «                              ¦   «          |j        | _        t          ||¬¦  «        | _        t          |¦  «        | _        t          |j        |j        ¬¦  «        | _	        t          |j        |j        ¬¦  «        | _
        d S )N)rf   r#  ©r+   )r.   r/   r6   r"  Ú	self_attnró   Úmlpr)   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormr*  s      €r8   r/   zAriaTextDecoderLayer.__init__  s„   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔå*°&ÀIÐNÑNÔNˆŒÝ# FÑ+Ô+ˆŒÝ.¨vÔ/AÀvÔGZÐ[Ñ[Ô[ˆÔÝ(7¸Ô8JÐPVÔPcÐ(dÑ(dÔ(dˆÔ%Ð%Ð%r9   NFr:   r  Úposition_idsr,  Ú	use_cacher+  r  r,   c           
      óÎ   — |}|                       |¦  «        } | j        d||||||dœ|¤Ž\  }}	||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|S )N)r:   r  r?  r,  r@  r+  © )r=  r:  r>  r;  )
r5   r:   r  r?  r,  r@  r+  r  Úresidualr€   s
             r8   rF   zAriaTextDecoderLayer.forward  s¡   € ð !ˆØ×,Ò,¨]Ñ;Ô;ˆà)˜4œ>ð 
Ø'Ø)Ø%Ø+ØØ 3ð
ð 
ð ð
ð 
Ñˆ�qð ! =Ñ0ˆð !ˆØ×5Ò5°mÑDÔDˆØŸš Ñ/Ô/ˆØ  =Ñ0ˆØÐr9   )NNNFN)rL   rM   rN   rc   r&   r5  r/   r1   rP   Ú
LongTensorr	   ÚboolrH   r   r   rF   rQ   rR   s   @r8   r7  r7    s	  ø€ € € € € ð
ð 
ðe˜~ð e¸#ð eð eð eð eð eð eð /3Ø04Ø(,Ø!&ØHLðð à”|ðð œ tÑ+ðð Ô&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð # 5¤<°´Ð#=Ô>ÀÑEðð Ð+Ô,ðð 
Œðð ð ð ð ð ð ð r9   r7  c                   ó„   ‡ — e Zd ZU eed<   dZdZddgZdZdgZ	dZ
dZdZeedœZ ej        ¦   «         ˆ fd	„¦   «         Zˆ xZS )
ÚAriaTextPreTrainedModelrf   Úmodel)ÚimageÚtextr7  rÁ   Tr,  ©r:   Ú
attentionsc                 óÄ   •— t          ¦   «                              |¦  «         t          |t          ¦  «        r(t	          j        |j        d| j        j        ¬¦  «         d S d S )Nr  )rB   Ústd)	r.   Ú_init_weightsÚ
isinstancerÁ   ÚinitÚnormal_r3   rf   Úinitializer_range©r5   r  r7   s     €r8   rO  z%AriaTextPreTrainedModel._init_weightsM  s_   ø€ å‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ4Ñ5Ô5ð 	UÝŒL˜œ¨S°d´kÔ6SÐTÑTÔTÐTÐTÐTð	Uð 	Ur9   )rL   rM   rN   r&   Ú__annotations__Úbase_model_prefixÚinput_modalitiesÚ_no_split_modulesÚsupports_gradient_checkpointingÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_attention_backendr7  r"  Ú_can_record_outputsr1   Úno_gradrO  rQ   rR   s   @r8   rG  rG  <  s®   ø€ € € € € € àÐÐÑØÐØ(ÐØ/Ð1IÐJÐØ&*Ð#Ø#4Ð"5ÐØÐØ€Nà"&Ðà-Ø'ðð Ðð
 €U„]�_„_ðUð Uð Uð Uñ „_ðUð Uð Uð Uð Ur9   rG  c                   ó†   ‡ — e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZdZeedœZ ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )	ÚAriaPreTrainedModelrf   rH  TÚAriaDecoderLayerr,  FrK  c                 óÂ   •— t          ¦   «                              |¦  «         t          |t          ¦  «        r't	          j        |j        | j        j        ¬¦  «         d S d S )N)rN  )	r.   rO  rP  r‚   rQ  Útrunc_normal_r|   rf   rS  rT  s     €r8   rO  z!AriaPreTrainedModel._init_weightse  s]   ø€ å‰Œ×Ò˜fÑ%Ô%Ð%Ý�f�mÑ,Ô,ð 	PÝÔ˜vœ|°´Ô1NÐOÑOÔOÐOÐOÐOð	Pð 	Pr9   )rL   rM   rN   r%   rU  rV  rY  rX  rZ  r[  r\  Ú_supports_flex_attnÚ_can_compile_fullgraphr]  r7  r"  r^  r1   r_  rO  rQ   rR   s   @r8   ra  ra  T  s±   ø€ € € € € € àÐÐÑØÐØ&*Ð#Ø+Ð,ÐØ#4Ð"5ÐØÐØ€NØÐØ"ÐØ"&Ðà-Ø'ðð Ðð
 €U„]�_„_ðPð Pð Pð Pñ „_ðPð Pð Pð Pð Pr9   ra  c            	       óî   ‡ — e Zd Z edd¬¦  «        d
defˆ fd„¦   «         Ze edd¬¦  «        d
dedeej	        e
f         fd„¦   «         ¦   «         Z ej        ¦   «         ed	„ ¦   «         ¦   «         Zˆ xZS )ÚAriaTextRotaryEmbeddingr«   z5.18)ÚversionNrf   c                 ó¾  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        | j        j        d         | _        | j        }| j        dk    rt          | j                 } || j        |¦  «        \  }| _
        t          j        |d¬¦  «        | _        t          j        |                     ¦   «         d¬¦  «        | _        d S )NÚ	rope_typeÚdefaultF)Ú
persistent)r.   r/   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrf   Úrope_parametersrk  Úcompute_default_rope_parametersr   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r5   rf   r«   Úrope_init_fnru  r7   s        €r8   r/   z AriaTextRotaryEmbedding.__init__m  sÀ   ø€ å‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!àˆŒàœÔ4°[ÔAˆŒØ!%Ô!EˆØŒ>˜YÒ&Ð&Ý.¨t¬~Ô>ˆLØ+7¨<¸¼ÀVÑ+LÔ+LÑ(ˆ�$Ô(åœ	 (°uÐ=Ñ=Ô=ˆŒÝ!#¤¨8¯>ª>Ñ+;Ô+;ÈÐ!NÑ!NÔ!NˆÔÐÐr9   r,   c                 óä   — | j         d         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «        |z  z  z  }|                     |¦  «        |fS )a¾  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetar  Ng      ð?r   r"   ©r>   )rq  r%  r6   rk   r1   ÚarangerO   r?   )rf   r«   r  Úbaser­   Úattention_factorru  s          r8   rr  z7AriaTextRotaryEmbedding.compute_default_rope_parameters~  sz   € ð Ô% lÔ3ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐà˜$¥5¤<°°3¸ÅÄÐ#MÑ#MÔ#MÐPSÑ#SÑTÑUˆØ�{Š{˜6Ñ"Ô"Ð$4Ð4Ð4r9   c                 óú  — | j         d d d …d f                              |j        d         dd¦  «                             t          j        |j        ¬¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  ||z                       dd	¦  «        }t	          j        ||fd¬
¦  «        }|                     ¦   «         | j        z  }|                     ¦   «         | j        z  }	d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬¦  «        |	                     |j        ¬¦  «        fS )Nr   r<   r$   rª   ÚmpsrÆ   F)Údevice_typeÚenabledr"   r¬   r{  )ru  r”   rI   r?   r1   rO   r«   rP  ÚtypeÚstrr   r  r°   r  rs  r  r>   )
r5   r¨   r?  Úinv_freq_expandedÚposition_ids_expandedr�  ÚfreqsÚembr  r  s
             r8   rF   zAriaTextRotaryEmbedding.forward’  s›  € ð ŒM˜$    4˜-Ô(×/Ò/°Ô0BÀ1Ô0EÀrÈ1ÑMÔM×PÒPÕW\ÔWbÐklÔksÐPÑtÔtð 	ð !-¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆå¨¸UÐCÑCÔCð 	5ð 	5Ø&Ð)>Ñ>×IÒIÈ!ÈQÑOÔOˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCð		5ð 	5ð 	5ñ 	5ô 	5ð 	5ð 	5ð 	5ð 	5ð 	5ð 	5øøøð 	5ð 	5ð 	5ð 	5ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   ÃA*D<Ä<E ÅE ra   )rL   rM   rN   r   r&   r/   ÚstaticmethodrH   r1   rP   rO   rr  r_  r   rF   rQ   rR   s   @r8   rh  rh  l  sö   ø€ € € € € Ø€_�X vÐ.Ñ.Ô.ðOð O˜~ð Oð Oð Oð Oð Oñ /Ô.ðOð  Ø€_�X vÐ.Ñ.Ô.ð5ð 5°ð 5ÐZ_Ð`eÔ`lÐnsÐ`sÔZtð 5ð 5ð 5ñ /Ô.ñ „\ð5ð$ €U„]�_„_Øð<ð <ñ Ôñ „_ð<ð <ð <ð <ð <r9   rh  c                   óâ   ‡ — e Zd Zdefˆ fd„Zeee	 	 	 	 	 	 ddej	        dz  dej
        dz  dej	        dz  dedz  dej        dz  d	edz  d
ee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚAriaTextModelrf   c                 óÞ  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          j        ‰j        ‰j        | j        ¦  «        | _        t          j	        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        t!          ‰¬¦  «        | _        d| _        |                      ¦   «          d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS rB  )r7  )Ú.0r#  rf   s     €r8   ú
<listcomp>z*AriaTextModel.__init__.<locals>.<listcomp>®  s$   ø€ ÐfÐfÐf¸Õ! &¨)Ñ4Ô4ÐfÐfÐfr9   r9  ©rf   F)r.   r/   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr6   Úembed_tokensÚ
ModuleListr±   Únum_hidden_layersÚlayersr)   r<  Únormrh  Ú
rotary_embÚgradient_checkpointingÚ	post_initrŒ   s    `€r8   r/   zAriaTextModel.__init__§  sÔ   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒåœL¨Ô):¸FÔ<NÐPTÔP`ÑaÔaˆÔÝ”mØfÐfÐfÐfÅeÈFÔLdÑFeÔFeÐfÑfÔfñ
ô 
ˆŒõ $ FÔ$6¸FÔ<OÐPÑPÔPˆŒ	Ý1¸Ð@Ñ@Ô@ˆŒØ&+ˆÔ#ð 	�ŠÑÔÐÐÐr9   NÚ	input_idsr  r?  r,  Úinputs_embedsr@  r  r,   c           
      óH  — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|r|€t          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   }| 	                    d¦  «        }t          | j        ||||¬¦  «        }	|}
|                      |
|¬¦  «        }| j        d | j        j        …         D ]} ||
f|	||||dœ|¤Ž}
Œ|                      |
¦  «        }
t          |
|¬	¦  «        S )
Nz:You must specify exactly one of input_ids or inputs_embedsr�  r   r$   )r«   )rf   rž  r  r,  r?  )r?  )r  r+  r?  r,  r@  )Úlast_hidden_stater,  )Ú
ValueErrorr•  r
   rf   Úget_seq_lengthr1   r|  rI   r«   r‘   r   rš  r˜  r—  r™  r   )r5   r�  r  r?  r,  rž  r@  r  Úpast_seen_tokensÚcausal_maskr:   r+  Údecoder_layers                r8   rF   zAriaTextModel.forward·  sŠ  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø*.×*;Ò*;¸IÑ*FÔ*FˆMàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå(Ø”;Ø'Ø)Ø+Ø%ð
ñ 
ô 
ˆð &ˆØ"Ÿošo¨mÈ,˜oÑWÔWÐà!œ[Ð)H¨4¬;Ô+HÐ)HÔIð 		ð 		ˆMØ)˜MØðà*Ø$7Ø)Ø /Ø#ðð ð ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆÝ&Ø+Ø+ð
ñ 
ô 
ð 	
r9   )NNNNNN)rL   rM   rN   r&   r/   r    r!   r   r1   rD  rP   r	   ÚFloatTensorrE  r   r   r   rF   rQ   rR   s   @r8   r‹  r‹  ¥  s  ø€ € € € € ð˜~ð ð ð ð ð ð ð   ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ð2
ð 2
àÔ# dÑ*ð2
ð œ tÑ+ð2
ð Ô&¨Ñ-ð	2
ð
  ™ð2
ð Ô(¨4Ñ/ð2
ð ˜$‘;ð2
ð Ð+Ô,ð2
ð 
!ð2
ð 2
ð 2
ñ „^ñ „_ñ  Ôð2
ð 2
ð 2
ð 2
ð 2
r9   r‹  c                   ó  ‡ — e Zd ZddiZddiZddgdgfiZddiZdefˆ fd	„Ze		 	 	 	 	 	 	 	 dde
j        d
z  de
j        d
z  de
j        d
z  ded
z  de
j        d
z  de
j        d
z  ded
z  dee
j        z  dee         defd„¦   «         Zˆ xZS )ÚAriaTextForCausalLMúlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr:   ÚlogitsÚkeep_full_weightrf   c                 óþ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        | _        t          j        |j        |j        d¬¦  «        | _        |  	                    ¦   «          d S rõ   )
r.   r/   r‹  rH  r“  r   rY   r6   rª  rœ  rŒ   s     €r8   r/   zAriaTextForCausalLM.__init__ö  sj   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý" 6Ñ*Ô*ˆŒ
Ø Ô+ˆŒÝ”y Ô!3°VÔ5FÈUÐSÑSÔSˆŒð 	�ŠÑÔÐÐÐr9   Nr   r�  r  r?  r,  rž  Úlabelsr@  Úlogits_to_keepr  r,   c	           
      óP  —  | j         d||||||dœ|	¤Ž}
|
j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }d}|� | j        d||| j        j        dœ|	¤Ž}t          |||
j
        |
j        |
j        ¬¦  «        S )aã  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, AriaTextForCausalLM

        >>> model = AriaTextForCausalLM.from_pretrained("meta-aria_text/AriaText-2-7b-hf")
        >>> tokenizer = AutoTokenizer.from_pretrained("meta-aria_text/AriaText-2-7b-hf")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```)r�  r  r?  r,  rž  r@  N©r¬  r¯  r“  ©Úlossr¬  r,  r:   rL  rB  )rH  r   rP  r5  Úslicerª  Úloss_functionrf   r“  r   r,  r:   rL  )r5   r�  r  r?  r,  rž  r¯  r@  r°  r  Úoutputsr:   Úslice_indicesr¬  r´  s                  r8   rF   zAriaTextForCausalLM.forwardÿ  sô   € ð< ,6¨4¬:ð ,
ØØ)Ø%Ø+Ø'Øð,
ð ,
ð ð,
ð ,
ˆð  Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%Ðp¨V¸FÈtÌ{ÔOeÐpÐpÐioÐpÐpˆDå%ØØØ#Ô3Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r9   )NNNNNNNr   )rL   rM   rN   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr&   r/   r   r1   rD  rP   r	   r¦  rE  r5  r   r   r   rF   rQ   rR   s   @r8   r¨  r¨  ï  sW  ø€ € € € € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€Jð˜~ð ð ð ð ð ð ð ð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ð6
ð 6
àÔ# dÑ*ð6
ð œ tÑ+ð6
ð Ô&¨Ñ-ð	6
ð
  ™ð6
ð Ô(¨4Ñ/ð6
ð Ô  4Ñ'ð6
ð ˜$‘;ð6
ð ˜eœlÑ*ð6
ð Ð+Ô,ð6
ð 
 ð6
ð 6
ð 6
ñ „^ð6
ð 6
ð 6
ð 6
ð 6
r9   r¨  zP
    Base class for Aria causal language model (or autoregressive) outputs.
    ©Úcustom_introc                   óÖ   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dZej        dz  ed<   dS )	ÚAriaCausalLMOutputWithPasta4  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
    Nr´  r¬  r,  r:   rL  Úimage_hidden_states)rL   rM   rN   rc   r´  r1   r¦  rU  r¬  r,  r	   r:   rH   rL  rÁ  rB  r9   r8   rÀ  rÀ  9  sµ   € € € € € € ðð ð &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø$(€O�U˜T‘\Ð(Ð(Ñ(Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ø48Ð˜Ô*¨TÑ1Ð8Ð8Ñ8Ð8Ð8r9   rÀ  zI
    Base class for Aria outputs, with hidden states and attentions.
    c                   ó2   — e Zd ZU dZdZej        dz  ed<   dS )ÚAriaModelOutputWithPastaÏ  
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
    NrÁ  )rL   rM   rN   rc   rÁ  r1   r¦  rU  rB  r9   r8   rÃ  rÃ  W  s7   € € € € € € ð	ð 	ð 59Ð˜Ô*¨TÑ1Ð8Ð8Ñ8Ð8Ð8r9   rÃ  zt
    The Aria model which consists of a vision backbone and a language model, without a language modeling head.
    c                   óì  ‡ — e Zd Zdefˆ fd„Zee ed¬¦  «        	 	 	 ddej	        dej	        dz  d	e
ee
         z  d
edz  dee         deez  fd„¦   «         ¦   «         ¦   «         Zdej        dej	        dej	        fd„Zee	 	 	 	 	 	 	 	 ddej        dz  dej	        dz  dej        dz  dej        dz  dej        dz  dedz  dej	        dz  dedz  dee         deez  fd„¦   «         ¦   «         Zd„ Zˆ xZS )Ú	AriaModelrf   c                 ó  •— t          ¦   «                              |¦  «         t          j        |j        ¦  «        | _        t          |¦  «        | _        t          j        |j        ¦  «        | _	        |  
                    ¦   «          d S ra   )r.   r/   r#   Úfrom_configrj   Úvision_towerr‚   Úmulti_modal_projectorr‡   Úlanguage_modelrœ  rŒ   s     €r8   r/   zAriaModel.__init__r  sm   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý%Ô1°&Ô2FÑGÔGˆÔÝ%2°6Ñ%:Ô%:ˆÔ"Ý'Ô3°FÔ4FÑGÔGˆÔØ�ŠÑÔÐÐÐr9   zWObtains image last hidden states from the vision tower and apply multimodal projection.r½  Nr<   Úpixel_valuesÚ
pixel_maskÚvision_feature_layerÚoutput_hidden_statesr  r,   c                 ó   — |                       |¦  «        } | j        |f|dddœ|¤Ž}d }|�)|                     d¦  «        }	t          j        |	¦  «        }|j        |         }
|                      |
|¬¦  «        |_        |S )NT)Úpatch_attention_maskrÎ  Úreturn_dictr$   ry   )Ú_create_patch_attention_maskrÈ  rÞ   r1   Úlogical_notr:   rÉ  Úpooler_output)r5   rË  rÌ  rÍ  rÎ  r  rÐ  Úimage_outputsÚimage_attn_maskÚflattened_maskÚselected_image_features              r8   Úget_image_featureszAriaModel.get_image_featuresy  s®   € ð  $×@Ò@ÀÑLÔLÐØ)˜Ô)Øð
à!5Ø!%Øð	
ð 
ð
 ð
ð 
ˆð ˆØÐ+Ø1×9Ò9¸!Ñ<Ô<ˆNÝ#Ô/°Ñ?Ô?ˆOà!.Ô!<Ð=QÔ!RÐØ&*×&@Ò&@ÐAWÐcrÐ&@Ñ&sÔ&sˆÔ#àÐr9   r�  rž  Úimage_featuresc           	      ó"  — |€f| |                       ¦   «         t          j        d| j        j        t          j        |j        ¬¦  «        ¦  «        k    }|                     d¦  «        }n|| j        j        k    }|                     ¦   «         }|j	        d         |j	        d         z  }| 
                    d¦  «                             |j        ¦  «        }t          ||j	        d         z  |                     ¦   «         k    d|› d|› �¦  «         |S )	zï
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        NrB  rª   r<   r   r$   z6Image features and image tokens do not match, tokens: z, features: )Úget_input_embeddingsr1   Úfullrf   Úimage_token_idr¯   r«   Úallrå   rI   r‘   r?   r   Únumel)r5   r�  rž  rÚ  Úspecial_image_maskÚn_image_tokensÚn_image_featuress          r8   Úget_placeholder_maskzAriaModel.get_placeholder_mask˜  s  € ð ÐØ!.Ð2M°$×2KÒ2KÑ2MÔ2MÝ”
˜2˜tœ{Ô9ÅÄÐTaÔThÐiÑiÔiñ3ô 3ò "Ðð "4×!7Ò!7¸Ñ!;Ô!;ÐÐà!*¨d¬kÔ.HÒ!HÐà+×/Ò/Ñ1Ô1ˆØ)Ô/°Ô2°^Ô5IÈ!Ô5LÑLÐØ/×9Ò9¸"Ñ=Ô=×@Ò@ÀÔAUÑVÔVÐÝØ˜]Ô0°Ô4Ñ4¸×8LÒ8LÑ8NÔ8NÒNØsÀ^ÐsÐsÐaqÐsÐsñ	
ô 	
ð 	
ð "Ð!r9   r  r?  r,  r@  c	           	      óØ  — |€ |                       ¦   «         |¦  «        }|�‡|j        d         dk    rv|                      ||| j        j        d¬¦  «        j        }
|
                     |j        |j        ¦  «        }
|  	                    |||
¬¦  «        }| 
                    ||
¦  «        } | j        d|||||dœ|	¤Ž}t          |j        |r|j        nd |j        |j        |�|
nd ¬¦  «        S )Nr$   T)rË  rÌ  rÍ  rÑ  )rž  rÚ  )r  r?  r,  rž  r@  )r   r,  r:   rL  rÁ  rB  )rÜ  rI   rÙ  rf   rÍ  rÔ  r?   r«   r>   rä  Úmasked_scatterrÊ  rÃ  r   r,  r:   rL  )r5   r�  rË  rÌ  r  r?  r,  rž  r@  r  rÚ  rá  r·  s                r8   rF   zAriaModel.forward°  sH  € ð Ð Ø7˜D×5Ò5Ñ7Ô7¸	ÑBÔBˆMð Ð#¨Ô(;¸AÔ(>À!Ò(CÐ(CØ!×4Ò4Ø)Ø%Ø%)¤[Ô%EØ ð	 5ñ ô ô
 ð ð ,×.Ò.¨}Ô/CÀ]ÔEXÑYÔYˆNØ!%×!:Ò!:Ø¨À~ð ";ñ "ô "Ðð *×8Ò8Ð9KÈ^Ñ\Ô\ˆMà%�$Ô%ð 
Ø)Ø%Ø+Ø'Øð
ð 
ð ð
ð 
ˆõ 'Ø%Ô7Ø7@ÐJ˜GÔ3Ð3ÀdØ!Ô/ØÔ)Ø2>Ð2J  ÐPTð
ñ 
ô 
ð 	
r9   c                 ó:  — |€d S |                      d| j        j        j        | j        j        j        ¬¦  «        }|                      d| j        j        j        | j        j        j        ¬¦  «        }|                     d¬¦  «        dk                         ¦   «         S )Nr$   )Ú	dimensionr•   Ústepr"   )r<   éþÿÿÿr¬   r   )ÚunfoldrÈ  rf   Ú
patch_sizerå   rE  )r5   rÌ  Úpatches_subgrids      r8   rÒ  z&AriaModel._create_patch_attention_maskà  s£   € ØÐØ�4à$×+Ò+ØØÔ"Ô)Ô4ØÔ"Ô)Ô4ð ,ñ 
ô 
ˆð
 *×0Ò0ØØÔ"Ô)Ô4ØÔ"Ô)Ô4ð 1ñ 
ô 
ˆð
  ×#Ò#¨Ð#Ñ1Ô1°AÒ5×;Ò;Ñ=Ô=Ð=r9   )Nr<   N)NNNNNNNN)rL   rM   rN   r%   r/   r    r   r   r1   r¦  r5  ÚlistrE  r   r   rH   r   rÙ  rD  rä  rP   r	   r   rÃ  rF   rÒ  rQ   rR   s   @r8   rÅ  rÅ  l  s2  ø€ € € € € ð˜zð ð ð ð ð ð ð  ØØ€^Ønðñ ô ð 04Ø02Ø,0ðð àÔ'ðð Ô%¨Ñ,ðð " D¨¤I™oð	ð
 # T™kðð Ð+Ô,ðð 
Ð+Ñ	+ðð ð ñô ñ Ôñ  Ôð
ð4"ØÔ)ð"Ø:?Ô:Kð"Ø]bÔ]nð"ð "ð "ð "ð0 Øð .2Ø15Ø.2Ø.2Ø04Ø(,Ø26Ø!%ð,
ð ,
àÔ# dÑ*ð,
ð Ô'¨$Ñ.ð,
ð Ô$ tÑ+ð	,
ð
 œ tÑ+ð,
ð Ô&¨Ñ-ð,
ð  ™ð,
ð Ô(¨4Ñ/ð,
ð ˜$‘;ð,
ð Ð-Ô.ð,
ð 
Ð(Ñ	(ð,
ð ,
ð ,
ñ „^ñ Ôð,
ð\>ð >ð >ð >ð >ð >ð >r9   rÅ  zË
    Aria model for conditional generation tasks.

    This model combines a vision tower, a multi-modal projector, and a language model
    to perform tasks that involve both image and text inputs.
    c                   ó¾  ‡ — e Zd ZddiZdefˆ fd„Zdej        fd„Ze		 	 dd	e
j        d
e
j        dz  deee         z  dee         deez  f
d„¦   «         Zee		 	 	 	 	 	 	 	 	 	 dde
j        dz  d	e
j        dz  d
e
j        dz  de
j        dz  de
j        dz  dedz  de
j        dz  de
j        dz  dedz  dee
j        z  dee         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚAriaForConditionalGenerationr©  z(model.language_model.embed_tokens.weightrf   c                 óú   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        j        |j        j        d¬¦  «        | _	        |  
                    ¦   «          d S rõ   )r.   r/   rÅ  rH  r   rY   r‡   r6   r“  rª  rœ  rŒ   s     €r8   r/   z%AriaForConditionalGeneration.__init__ü  se   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý˜vÑ&Ô&ˆŒ
Ý”y Ô!3Ô!?ÀÔASÔA^ÐejÐkÑkÔkˆŒØ�ŠÑÔÐÐÐr9   r,   c                 ó   — | j         S ra   )rª  rJ   s    r8   Úget_output_embeddingsz2AriaForConditionalGeneration.get_output_embeddings  s
   € ØŒ|Ðr9   Nr<   rË  rÌ  rÍ  r  c                 ó.   —  | j         j        d|||dœ|¤ŽS )N)rË  rÌ  rÍ  rB  )rH  rÙ  )r5   rË  rÌ  rÍ  r  s        r8   rÙ  z/AriaForConditionalGeneration.get_image_features  s:   € ð -ˆtŒzÔ,ð 
Ø%Ø!Ø!5ð
ð 
ð ð	
ð 
ð 	
r9   r   r�  r  r?  r,  rž  r¯  r@  r°  c                 ó`  —  | j         d||||||||	dœ|¤Ž}|d         }t          |
t          ¦  «        rt          |
 d¦  «        n|
}|                      |dd…|dd…f         ¦  «        }d}|�  | j        d||| j        j        j        dœ|¤Ž}t          |||j
        |j        |j        ¬¦  «        S )a{  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or `model.image_token_id` (where `model` is your instance of `AriaForConditionalGeneration`).
            Tokens with indices set to `model.image_token_id` are ignored (masked), the loss is only
            computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> import httpx
        >>> from io import BytesIO
        >>> import torch
        >>> from PIL import Image
        >>> from io import BytesIO

        >>> from transformers import AutoProcessor, AutoModel
        >>> from transformers.image_utils import load_image

        >>> # Note that passing the image urls (instead of the actual pil images) to the processor is also possible
        >>> image1 = load_image("https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg")
        >>> image2 = load_image("https://cdn.britannica.com/59/94459-050-DBA42467/Skyline-Chicago.jpg")
        >>> image3 = load_image("https://cdn.britannica.com/68/170868-050-8DDE8263/Golden-Gate-Bridge-San-Francisco.jpg")

        >>> processor = AutoProcessor.from_pretrained("Rhymes-AI/Aria")
        >>> model = AutoModel.from_pretrained("Rhymes-AI/Aria", dtype=torch.bfloat16, device_map="auto")

        >>> # Create inputs
        >>> messages = [
        ...     {
        ...         "role": "user",
        ...         "content": [
        ...             {"type": "image"},
        ...             {"type": "text", "text": "In this image, we can see the city of New York, and more specifically the Statue of Liberty."},
        ...             {"type": "image"},
        ...             {"type": "text", "text": "What can we see in this image?"},
        ...         ]
        ...     },
        ...     {
        ...         "role": "user",
        ...         "content": [
        ...             {"type": "image"},
        ...             {"type": "text", "text": "In which city is that bridge located?"},
        ...         ]
        ...     }
        ... ]

        >>> prompts = [processor.apply_chat_template([message], add_generation_prompt=True) for message in messages]
        >>> images = [[image1, image2], [image3]]
        >>> inputs = processor(text=prompts, images=images, padding=True, return_tensors="pt").to(model.device)

        >>> # Generate
        >>> generated_ids = model.generate(**inputs, max_new_tokens=256)
        >>> generated_texts = processor.batch_decode(generated_ids, skip_special_tokens=True)

        >>> print(generated_texts[0])
        Assistant: There are buildings, trees, lights, and water visible in this image.

        >>> print(generated_texts[1])
        Assistant: The bridge is in San Francisco.
        ```)r�  rË  rÌ  r  r?  r,  rž  r@  r   Nr²  r³  rB  )rH  rP  r5  rµ  rª  r¶  rf   r‡   r“  rÀ  r,  r:   rL  )r5   r�  rË  rÌ  r  r?  r,  rž  r¯  r@  r°  r  r·  r:   r¸  r¬  r´  s                    r8   rF   z$AriaForConditionalGeneration.forward  s  € ðZ �$”*ð 

ØØ%Ø!Ø)Ø%Ø+Ø'Øð

ð 

ð ð

ð 

ˆð   œ
ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%ð Ø f¸¼Ô9PÔ9[ðð Ø_eðð ˆDõ *ØØØ#Ô3Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r9   rú   )
NNNNNNNNNr   )rL   rM   rN   r¹  r%   r/   r   ÚModuleró  r   r1   r¦  r5  rî  r   r   rH   r   rÙ  r   rD  rP   r	   rE  rÀ  rF   rQ   rR   s   @r8   rð  rð  ñ  s  ø€ € € € € ð +Ð,VÐWÐð˜zð ð ð ð ð ð ð r¤yð ð ð ð ð ð 04Ø02ð	
ð 
àÔ'ð
ð Ô%¨Ñ,ð
ð " D¨¤I™oð	
ð
 Ð+Ô,ð
ð 
Ð+Ñ	+ð
ð 
ð 
ñ „^ð
ð Øð .2Ø15Ø.2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ðh
ð h
àÔ# dÑ*ðh
ð Ô'¨$Ñ.ðh
ð Ô$ tÑ+ð	h
ð
 œ tÑ+ðh
ð Ô&¨Ñ-ðh
ð  ™ðh
ð Ô(¨4Ñ/ðh
ð Ô  4Ñ'ðh
ð ˜$‘;ðh
ð ˜eœlÑ*ðh
ð Ð+Ô,ðh
ð 
Ð+Ñ	+ðh
ð h
ð h
ñ „^ñ Ôðh
ð h
ð h
ð h
ð h
r9   rð  )rð  ra  rG  r‹  rÅ  r¨  )r$   )r  )TÚcollections.abcr   Údataclassesr   r1   r   Ú r   rQ  Úactivationsr   Úcache_utilsr	   r
   Ú
generationr   Úintegrationsr   r   Úmasking_utilsr   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.deprecationr   Úutils.genericr   r    Úutils.output_capturingr!   Úautor#   Úconfiguration_ariar%   r&   rö  r)   rT   re   r‚   r�   r¿   rÁ   rÉ   ró   r   r	  rP   r5  r  rO   r   r"  r7  rG  ra  rh  r‹  r¨  rÀ  rÃ  rÅ  rð  Ú__all__rB  r9   r8   ú<module>r     s”  ðð* %Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )Ø LÐ LÐ LÐ LÐ LÐ LÐ LÐ LØ /Ð /Ð /Ð /Ð /Ð /Ø BÐ BÐ BÐ BÐ BÐ BØ 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð LÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aØ 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø Ð Ð Ð Ð Ð Ø :Ð :Ð :Ð :Ð :Ð :Ð :Ð :ð Ð˜YÑ'Ô'ðJð Jð Jð Jð J�b”iñ Jô Jñ (Ô'ðJð(ð ð ð ð �r”yñ ô ð ð24ð 4ð 4ð 4ð 4˜œñ 4ô 4ð 4ðn>ð >ð >ð >ð >�B”Iñ >ô >ð >ðBð ð ð ð ˜2œ9ñ ô ð ð4ð ð ð>)
ð )
ð )
ð )
ð )
˜RœYñ )
ô )
ð )
ðX)ð )ð )ð )ð )�"”)ñ )ô )ð )ðX4ð 4ð 4ð 4ð 4�r”yñ 4ô 4ð 4ð"(ð (ð (ð ÐÐ-Ñ.Ô.ðð ð ñ /Ô.ðð2	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð& ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð2 ÐÐ)Ñ*Ô*ð@)ð @)ð @)ð @)ð @)˜œ	ñ @)ô @)ñ +Ô*ð@)ðF3ð 3ð 3ð 3ð 3Ð5ñ 3ô 3ð 3ðl ðUð Uð Uð Uð U˜oñ Uô Uñ „ðUð. ðPð Pð Pð Pð P˜/ñ Pô Pñ „ðPð.6<ð 6<ð 6<ð 6<ð 6<˜bœiñ 6<ô 6<ð 6<ðr ðF
ð F
ð F
ð F
ð F
Ð+ñ F
ô F
ñ „ðF
ðR ðF
ð F
ð F
ð F
ð F
Ð1°?ñ F
ô F
ñ „ðF
ðR €ððñ ô ð
 ð9ð 9ð 9ð 9ð 9 ñ 9ô 9ñ „ñô ð9ð0 €ððñ ô ð
 ð9ð 9ð 9ð 9ð 9Ð5ñ 9ô 9ñ „ñô ð9ð €ððñ ô ð
}>ð }>ð }>ð }>ð }>Ð#ñ }>ô }>ñô ð
}>ð@ €ððñ ô ðE
ð E
ð E
ð E
ð E
Ð#6¸ñ E
ô E
ñô ðE
ðPð ð €€€r9   