§
    9²¸j¡Ú  ã                   óV  — d Z ddlmZ ddlmZ ddlmZ ddlZddlm	c m
Z ddlm	Z	 ddlmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZm Z  ddl!m"Z"m#Z#m$Z$ ddl%m&Z& ddl'm(Z(m)Z)m*Z*m+Z+ ddl,m-Z- ddl.m/Z/m0Z0 ddl1m2Z2m3Z3 ddl4m5Z5 ddl6m7Z7 ddl8m9Z9m:Z:  e+j;        e<¦  «        Z= e)d¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z> e)d¬¦  «        e G d „ d!e¦  «        ¦   «         ¦   «         Z?	 	 	 	 dId#„Z@dJd%„ZA G d&„ d'e	jB        ¦  «        ZC G d(„ d)e	jD        ¦  «        ZE G d*„ d+e	jF        ¦  «        ZG G d,„ d-e	jF        ¦  «        ZHd.„ ZI ed/¦  «        dKd0„¦   «         ZJ G d1„ d2e	jF        ¦  «        ZK	 dLd4e	jF        d5ejL        d6ejL        d7ejL        d8ejL        dz  d9eMd:eMfd;„ZN G d<„ d=e	jF        ¦  «        ZO G d>„ d?e¦  «        ZP G d@„ dAe¦  «        ZQe) G dB„ dCe$¦  «        ¦   «         ZRe) G dD„ dEeR¦  «        ¦   «         ZS G dF„ dGeRe¦  «        ZTg dH¢ZUdS )MzPyTorch Idefics model.é    )ÚCallable)Ú	dataclass)ÚAnyN)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hub)Úcreate_causal_mask)ÚGradientCheckpointingLayer)ÚModelOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedConfigÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚIdeficsConfig)ÚIdeficsPerceiverResampler)ÚIdeficsVisionEmbeddingsÚIdeficsVisionTransformerz{
    Base class for Idefics model's outputs that may also contain a past key/values (to speed up sequential decoding).
    )Úcustom_introc                   óÄ   — e Zd ZU dZdZej        dz  ed<   dZe	dz  ed<   dZ
eej                 dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dS )ÚIdeficsBaseModelOutputWithPasta…  
    last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
        Sequence of hidden-states at the output of the last layer of the model.

        If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,
        hidden_size)` is output.
    image_hidden_states (`tuple(torch.FloatTensor)`, *optional*):
        Tuple of `torch.FloatTensor` (one for the output of the image embeddings, `(batch_size, num_images,
        sequence_length, hidden_size)`.

        image_hidden_states of the model produced by the vision encoder, and optionally by the perceiver
    NÚlast_hidden_stateÚpast_key_valuesÚhidden_statesÚ
attentionsÚimage_hidden_states)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r(   ÚtorchÚFloatTensorÚ__annotations__r)   r
   r*   Útupler+   r,   © ó    ú|/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/idefics/modeling_idefics.pyr'   r'   4   s£   € € € € € € ðð ð 37Ð�uÔ(¨4Ñ/Ð6Ð6Ñ6Ø$(€O�U˜T‘\Ð(Ð(Ñ(Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ø;?Ð˜˜uÔ0Ô1°DÑ8Ð?Ð?Ñ?Ð?Ð?r6   r'   zS
    Base class for Idefics causal language model (or autoregressive) outputs.
    c                   óâ   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dS )	ÚIdeficsCausalLMOutputWithPastae  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`tuple(torch.FloatTensor)`, *optional*):
        Tuple of `torch.FloatTensor` (one for the output of the image embeddings, `(batch_size, num_images,
        sequence_length, hidden_size)`.

        image_hidden_states of the model produced by the vision encoder, and optionally by the perceiver
    NÚlossÚlogitsr)   r*   r+   r,   )r-   r.   r/   r0   r:   r1   r2   r3   r;   r)   r
   r*   r4   r+   r,   r5   r6   r7   r9   r9   O   sº   € € € € € € ðð ð" &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø$(€O�U˜T‘\Ð(Ð(Ñ(Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ø;?Ð˜˜uÔ0Ô1°DÑ8Ð?Ð?Ñ?Ð?Ð?r6   r9   Fc                 ó–  — t          j        | j        d         ¦  «                             dd¦  «                             d|¦  «                             d¦  «                             | j        ¦  «        }|                      d|¦  «        } |                     d¦  «        |d<   |                     d¦  «        |d<   |                     d¦  «        |d<   |                     d¦  «        |d<   d|v r!|d         }|                     d|¦  «        |d<   |�|                     d|¦  «        |d	<   |d         �|d                              d|¦  «        |d<   |d         � |d                              d|¦  «        |d<   nO|d         � |d                              d|¦  «        |d<   n'|d         �|d                              d|¦  «        |d<   | |fS )
Nr   éÿÿÿÿr    Úpixel_valuesÚimage_encoder_embeddingsÚperceiver_embeddingsÚimage_attention_maskÚtoken_type_idsÚattention_mask)	r1   ÚarangeÚshapeÚviewÚrepeatÚtoÚdeviceÚindex_selectÚget)Ú	input_idsÚexpand_sizeÚis_encoder_decoderrC   Úencoder_outputsÚmodel_kwargsÚexpanded_return_idxrB   s           r7   Úexpand_inputs_for_generationrR   o   s  € õ 	Œ�Y”_ QÔ'Ñ(Ô(×-Ò-¨b°!Ñ4Ô4×;Ò;¸A¸{ÑKÔK×PÒPÐQSÑTÔT×WÒWÐXaÔXhÑiÔið ð ×&Ò& qÐ*=Ñ>Ô>€IØ#/×#3Ò#3°NÑ#CÔ#C€L�Ñ Ø/;×/?Ò/?Ð@ZÑ/[Ô/[€LÐ+Ñ,Ø+7×+;Ò+;Ð<RÑ+SÔ+S€LÐ'Ñ(Ø+7×+;Ò+;Ð<RÑ+SÔ+S€LÐ'Ñ(à˜<Ð'Ð'Ø%Ð&6Ô7ˆØ)7×)DÒ)DÀQÐH[Ñ)\Ô)\ˆÐ%Ñ&àÐ!Ø)7×)DÒ)DÀQÐH[Ñ)\Ô)\ˆÐ%Ñ&àÐ*Ô+Ð7Ø/;Ð<RÔ/S×/`Ò/`ØÐ"ñ0
ô 0
ˆÐ+Ñ,ð �NÔ#Ð/Ø'3°NÔ'C×'PÒ'PÐQRÐTgÑ'hÔ'hˆ�^Ñ$Ð$à	Ð0Ô	1Ð	=Ø3?Ð@ZÔ3[×3hÒ3hØÐ"ñ4
ô 4
ˆÐ/Ñ0Ð0ð 
Ð,Ô	-Ð	9Ø/;Ð<RÔ/S×/`Ò/`ØÐ"ñ0
ô 0
ˆÐ+Ñ,ð �lÐ"Ð"r6   r5   c                 ó,  ‡‡— t           j        t           j        t           j        dœŠˆfd„|D ¦   «         }|                      ¦   «         D ]JŠ|r1t          ˆfd„|D ¦   «         ¦  «        r‰                     d¦  «         Œ5‰                     d¦  «         ŒK| S )N)Ú	LayerNormÚLinearÚ	Embeddingc                 ó    •— g | ]
}‰|         ‘ŒS r5   r5   )Ú.0ÚmÚmappings     €r7   ú
<listcomp>z freeze_model.<locals>.<listcomp>¢   s   ø€ ÐFÐFÐF¨q ¨¤
ÐFÐFÐFr6   c              3   ó8   •K  — | ]}t          ‰|¦  «        V — Œd S ©N)Ú
isinstance)rX   ÚtÚmodules     €r7   ú	<genexpr>zfreeze_model.<locals>.<genexpr>¤   s-   øè è € Ð$]Ð$]¸q¥Z°¸Ñ%:Ô%:Ð$]Ð$]Ð$]Ð$]Ð$]Ð$]r6   TF)r   rT   rU   rV   ÚmodulesÚanyÚrequires_grad_)ÚmodelÚmodule_exceptionsÚmodule_exceptions_mappedrZ   r`   s      @@r7   Úfreeze_modelrh   œ   s´   øø€ å”\Ý”)Ý”\ðð €Gð
  GÐFÐFÐFÐ4EÐFÑFÔFÐØ—-’-‘/”/ð )ð )ˆØð 	)¥Ð$]Ð$]Ð$]Ð$]ÐD\Ð$]Ñ$]Ô$]Ñ!]Ô!]ð 	)Ø×!Ò! $Ñ'Ô'Ð'Ð'à×!Ò! %Ñ(Ô(Ð(Ð(Ø€Lr6   c                   óN   ‡ — e Zd ZdZ	 	 	 	 d	dedz  ddfˆ fd„Zd„ Zdefd„Zˆ xZ	S )
ÚIdeficsDecoupledEmbeddinga¿  
    Implements a decoupling of parameters to allow freezing (or not) a subset of the embeddings. In practise, the
    regular `weight` can be trained or frozen (i.e. `partially_freeze=True`), and if `num_additional_embeddings` > 0,
    then it will create `num_additional_embeddings` additional parameters that are always trained. If
    `num_additional_embeddings=0`, then the module defaults back to the regular behavior of `nn.Embedding`.
    FNÚpartially_freezeÚreturnc           	      óN  •— |�||k    rt          d|› d|› �¦  «        ‚ t          ¦   «         j        d|||||dœ|¤Ž || _        || _        || _        || _        |r| j                             d¦  «         | j        dk    r$t          j
        | j        |||¬¦  «        | _        dS dS )	a)  
        Args:
            num_embeddings (`int`):
                Size of the dictionary of embeddings
            num_additional_embeddings (`int`):
                Number of additional embeddings. Only useful when you `partially_freeze=True`.
            embedding_dim (`int`):
                The size of each embedding vector
            partially_freeze: (`bool`, *optional*, defaults to `False`):
                If `True`, the regular `weight` will be frozen. `additional_weight` is never frozen.
            padding_idx (`int`, *optional*):
                The padding index (needs to be less than num_embeddings)

        Note: there are a lot of other parameters to initialize a standard `nn.Embedding` such as `padding_idx`,
        `max_norm` or `norm_type`. We are not supporting these.
        Nz/padding_idx must be within num_embeddings. Got z and )Únum_embeddingsÚembedding_dimrI   ÚdtypeÚpadding_idxFr   )rn   ro   rI   rp   r5   )Ú
ValueErrorÚsuperÚ__init__rn   rq   Únum_additional_embeddingsrk   Úweightrd   r   rV   Úadditional_embedding)
Úselfrn   ru   ro   rk   rI   rp   rq   ÚkwargsÚ	__class__s
            €r7   rt   z"IdeficsDecoupledEmbedding.__init__´   s÷   ø€ ð6 Ð" {°^Ò'CÐ'CÝÐqÈ{ÐqÐqÐaoÐqÐqÑrÔrÐrØ�‰ŒÔð 	
Ø)Ø'ØØØ#ð	
ð 	
ð ð	
ð 	
ð 	
ð -ˆÔØ&ˆÔØ)BˆÔ&Ø 0ˆÔàð 	.ØŒK×&Ò& uÑ-Ô-Ð-àÔ)¨AÒ-Ð-Ý(*¬Ø#Ô=Ø+ØØð	)ñ )ô )ˆDÔ%Ð%Ð%ð .Ð-r6   c                 óD  — | j         dk    rt          j        || j        ¦  «        S |                     ¦   «         }t          j        || j        k    ¦  «        }||         }|                      || j        z
  ¦  «        }d||<   t          j        || j        ¦  «        }|||<   |S )aû  
        we have 2 embeddings, with different indices - one pretrained self.weight and another
        self.additional_embedding.weight that is being trained.

        in order to make a lookup of the input ids, we:
        1. find out the indices of the entries belonging to the 2nd embedding
        2. extract those values while subtracting the size of the first embedding (num_embeddings), since the 2nd
           embedding starts from 0 and not num_embeddings
        3. perform the 2nd embedding lookup
        4. now we handle the 1st embedding, we overwrite indices belonging to the 2nd embedding with a padding index
        5. perform the 1st embedding lookup
        6. now we overwrite the values in the 1st embedding lookup with the values of the 2nd embedding lookup

        note: for the 1st embedding lookup we could have looked up only the low indices and not do the padding, but
        then we have to create a new tensor and populate it with 2 tensors that are spread out across various indices -
        i.e. not a simple concat - I haven't benchmarked the complex case if it's any faster, given that seqlens are
        usually relatively short it's probably not faster or if faster not by much - but might be a good idea to
        measure.

        r   )	ru   ÚFÚ	embeddingrv   Úcloner1   Úwherern   rw   )rx   rL   Úadditional_vocab_indicesÚinput_ids_additional_vocabÚadditional_embeddingsÚfull_vectors         r7   Úforwardz!IdeficsDecoupledEmbedding.forwardé   s­   € ð* Ô)¨QÒ.Ð.Ý”;˜y¨$¬+Ñ6Ô6Ð6ð —O’OÑ%Ô%ˆ	Ý#(¤;¨y¸DÔ<OÒ/OÑ#PÔ#PÐ Ø%.Ð/GÔ%HÐ"Ø $× 9Ò 9Ð:TÐW[ÔWjÑ:jÑ kÔ kÐð /0ˆ	Ð*Ñ+Ý”k )¨T¬[Ñ9Ô9ˆð 1FˆÐ,Ñ-àÐr6   c                 óF   — d| j         › d| j        › d| j        › d| j        › �S )Nznum_embeddings=z, num_additional_embeddings=z, embedding_dim=ú, partially_freeze=)rn   ru   ro   rk   ©rx   s    r7   Ú
extra_reprz$IdeficsDecoupledEmbedding.extra_repr  se   € ð A Ô!4ð  Að  AÐRVÔRpð  Að  Að  CGô  CUð  Að  Að  jnô  jð  Að  Að  	Ar6   )FNNN)
r-   r.   r/   r0   Úboolrt   r„   Ústrrˆ   Ú__classcell__©rz   s   @r7   rj   rj   «   s§   ø€ € € € € ðð ð ).ØØØð3ð 3ð
  ™+ð3ð 
ð3ð 3ð 3ð 3ð 3ð 3ðj%ð %ð %ðNA˜Cð Að Að Að Að Að Að Að Ar6   rj   c                   óx   ‡ — e Zd ZdZ	 	 	 	 	 ddedededed	ed
dfˆ fd„Zdej        d
ej        fd„Z	d
e
fd„Zˆ xZS )ÚIdeficsDecoupledLinearaÄ  
    Implements a decoupling of parameters to allow freezing (or not) a subset of the parameters. In practise, the
    regular `weight` can be trained or frozen (i.e. `partially_freeze=True`), and if `out_additional_features` > 0,
    then it will create `out_additional_features * in_features` additional parameters that are always trained. If
    `out_additional_features=0`, then the module defaults back to the regular behavior of `nn.Linear`.
    r   TNÚin_featuresÚout_featuresÚout_additional_featuresÚbiasrk   rl   c                 óF  •— t          ¦   «                              |||||¦  «         || _        || _        || _        || _        |r6| j                             d¦  «         |r| j                             d¦  «         |dk    r t          j
        |||||¬¦  «        | _        dS dS )aG  
        out_additional_features: int. Number of additional trainable dimensions. Only makes sense when
        `partially_freeze=True`. partially_freeze: bool. If True, the regular `weight` will be frozen and extra
        parameters (if any) will be trainable. If False, default to the regular behavior of nn.Linear.
        Fr   )r�   r�   r’   rI   rp   N)rs   rt   r‘   rk   r�   r�   rv   rd   r’   r   rU   Úadditional_fc)	rx   r�   r�   r‘   r’   rk   rI   rp   rz   s	           €r7   rt   zIdeficsDecoupledLinear.__init__  sÂ   ø€ õ 	‰Œ×Ò˜ l°D¸&À%ÑHÔHÐHØ'>ˆÔ$Ø 0ˆÔà&ˆÔØ(ˆÔàð 	0ØŒK×&Ò& uÑ-Ô-Ð-Øð 0Ø”	×(Ò(¨Ñ/Ô/Ð/à" QÒ&Ð&Ý!#¤Ø'Ø4ØØØð"ñ "ô "ˆDÔÐÐð 'Ð&r6   Úinputc                 ó´   — t          j        || j        | j        ¦  «        }| j        dk    r,|                      |¦  «        }t          j        ||fd¦  «        }|S )Nr   r=   )r|   Úlinearrv   r’   r‘   r”   r1   Úcat)rx   r•   ÚoutputÚadditional_featuress       r7   r„   zIdeficsDecoupledLinear.forwardA  sW   € Ý”˜% ¤¨d¬iÑ8Ô8ˆàÔ'¨!Ò+Ð+Ø"&×"4Ò"4°UÑ";Ô";ÐÝ”Y Ð(;Ð<¸bÑAÔAˆFàˆr6   c           
      óZ   — d| j         › d| j        › d| j        › d| j        du› d| j        › �
S )z=Overwriting `nn.Linear.extra_repr` to include new parameters.zin_features=z, out_features=z, out_additional_features=z, bias=Nr†   ©r�   r�   r‘   r’   rk   r‡   s    r7   rˆ   z!IdeficsDecoupledLinear.extra_reprJ  sˆ   € ð S˜dÔ.ð  Sð  S¸tÔ?Pð  Sð  SÐlpô  mIð  Sð  Sð  RVô  R[ð  cgð  Rgð  Sð  Sð  |@ô  |Qð  Sð  Sð  	Sr6   )r   TTNN)r-   r.   r/   r0   Úintr‰   rt   r1   ÚTensorr„   rŠ   rˆ   r‹   rŒ   s   @r7   rŽ   rŽ     sâ   ø€ € € € € ðð ð ()ØØ!%ØØð"ð "àð"ð ð"ð "%ð	"ð
 ð"ð ð"ð 
ð"ð "ð "ð "ð "ð "ðH˜Uœ\ð ¨e¬lð ð ð ð ðS˜Cð Sð Sð Sð Sð Sð Sð Sð Sr6   rŽ   c                   ó,   ‡ — e Zd Zdˆ fd„	Zd„ Zd„ Zˆ xZS )ÚIdeficsRMSNormç�íµ ÷Æ°>c                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )z=
        IdeficsRMSNorm is equivalent to T5LayerNorm
        N)rs   rt   r   Ú	Parameterr1   Úonesrv   Úvariance_epsilon)rx   Úhidden_sizeÚepsrz   s      €r7   rt   zIdeficsRMSNorm.__init__Q  sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐr6   c                 óh  — |                      t          j        ¦  «                             d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        j        t          j	        t          j
        fv r|                      | j        j        ¦  «        }| j        |z  S )Né   r=   T)Úkeepdim)rH   r1   Úfloat32ÚpowÚmeanÚrsqrtr¥   rv   rp   Úfloat16Úbfloat16)rx   r*   Úvariances      r7   r„   zIdeficsRMSNorm.forwardY  s”   € Ø ×#Ò#¥E¤MÑ2Ô2×6Ò6°qÑ9Ô9×>Ò>¸rÈ4Ð>ÑPÔPˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆð Œ;Ô¥¤µ´Ð ?Ð?Ð?Ø)×,Ò,¨T¬[Ô->Ñ?Ô?ˆMàŒ{˜]Ñ*Ð*r6   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)r4   rv   rE   r¥   r‡   s    r7   rˆ   zIdeficsRMSNorm.extra_reprc  s&   € Ý˜œÔ)Ñ*Ô*ÐIÐI°$Ô2GÐIÐIÐIr6   )r¡   )r-   r.   r/   rt   r„   rˆ   r‹   rŒ   s   @r7   r    r    P  sb   ø€ € € € € ð$ð $ð $ð $ð $ð $ð+ð +ð +ðJð Jð Jð Jð Jð Jð Jr6   r    c            	       óî   ‡ — e Zd Z edd¬¦  «        d
defˆ fd„¦   «         Ze edd¬¦  «        d
dedeej	        e
f         fd„¦   «         ¦   «         Z ej        ¦   «         ed	„ ¦   «         ¦   «         Zˆ xZS )ÚIdeficsRotaryEmbeddingrI   z5.18)ÚversionNÚconfigc                 ó¾  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        | j        j        d         | _        | j        }| j        dk    rt          | j                 } || j        |¦  «        \  }| _
        t          j        |d¬¦  «        | _        t          j        |                     ¦   «         d¬¦  «        | _        d S )NÚ	rope_typeÚdefaultF)Ú
persistent)rs   rt   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr¶   Úrope_parametersr¸   Úcompute_default_rope_parametersr   Úattention_scalingr   ÚBufferÚinv_freqr~   Úoriginal_inv_freq)rx   r¶   rI   Úrope_init_fnrÂ   rz   s        €r7   rt   zIdeficsRotaryEmbedding.__init__i  sÀ   ø€ å‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!àˆŒàœÔ4°[ÔAˆŒØ!%Ô!EˆØŒ>˜YÒ&Ð&Ý.¨t¬~Ô>ˆLØ+7¨<¸¼ÀVÑ+LÔ+LÑ(ˆ�$Ô(åœ	 (°uÐ=Ñ=Ô=ˆŒÝ!#¤¨8¯>ª>Ñ+;Ô+;ÈÐ!NÑ!NÔ!NˆÔÐÐr6   rl   c                 óä   — | j         d         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «        |z  z  z  }|                     |¦  «        |fS )a¾  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNg      ð?r   r©   ©rp   )r¾   Úgetattrr¦   Únum_attention_headsr1   rD   ÚfloatrH   )r¶   rI   ry   ÚbaseÚdimÚattention_factorrÂ   s          r7   r¿   z6IdeficsRotaryEmbedding.compute_default_rope_parametersz  sz   € ð Ô% lÔ3ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐà˜$¥5¤<°°3¸ÅÄÐ#MÑ#MÔ#MÐPSÑ#SÑTÑUˆØ�{Š{˜6Ñ"Ô"Ð$4Ð4Ð4r6   c                 óú  — | j         d d d …d f                              |j        d         dd¦  «                             t          j        |j        ¬¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  ||z                       dd	¦  «        }t	          j        ||fd¬
¦  «        }|                     ¦   «         | j        z  }|                     ¦   «         | j        z  }	d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬¦  «        |	                     |j        ¬¦  «        fS )Nr   r=   r    ©rp   rI   ÚmpsÚcpuF)Údevice_typeÚenabledr©   ©rÍ   rÈ   )rÂ   ÚexpandrE   rH   r1   rË   rI   r^   ÚtyperŠ   r   Ú	transposer˜   ÚcosrÀ   Úsinrp   )
rx   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedrÓ   ÚfreqsÚembrÙ   rÚ   s
             r7   r„   zIdeficsRotaryEmbedding.forwardŽ  s›  € ð ŒM˜$    4˜-Ô(×/Ò/°Ô0BÀ1Ô0EÀrÈ1ÑMÔM×PÒPÕW\ÔWbÐklÔksÐPÑtÔtð 	ð !-¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆå¨¸UÐCÑCÔCð 	5ð 	5Ø&Ð)>Ñ>×IÒIÈ!ÈQÑOÔOˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCð		5ð 	5ð 	5ñ 	5ô 	5ð 	5ð 	5ð 	5ð 	5ð 	5ð 	5øøøð 	5ð 	5ð 	5ð 	5ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   ÃA*D<Ä<E ÅE r]   )r-   r.   r/   r   r!   rt   Ústaticmethodr4   r1   rž   rË   r¿   Úno_gradr   r„   r‹   rŒ   s   @r7   r´   r´   h  sö   ø€ € € € € Ø€_�X vÐ.Ñ.Ô.ðOð O˜}ð Oð Oð Oð Oð Oñ /Ô.ðOð  Ø€_�X vÐ.Ñ.Ô.ð5ð 5°ð 5ÐY^Ð_dÔ_kÐmrÐ_rÔYsð 5ð 5ð 5ñ /Ô.ñ „\ð5ð$ €U„]�_„_Øð<ð <ñ Ôñ „_ð<ð <ð <ð <ð <r6   r´   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..Nr=   r©   rÕ   )rE   r1   r˜   )rÛ   Úx1Úx2s      r7   Úrotate_halfræ   ¢  s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'r6   Úrotary_pos_embc                 ó¾   — |                      |¦  «        }|                      |¦  «        }| |z  t          | ¦  «        |z  z   }||z  t          |¦  «        |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezeræ   )ÚqÚkrÙ   rÚ   Úunsqueeze_dimÚq_embedÚk_embeds          r7   Úapply_rotary_pos_embrï   ©  sc   € ð( �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�GÐÐr6   c                   ó2   ‡ — e Zd Zdededefˆ fd„Zd„ Zˆ xZS )Ú
IdeficsMLPr¦   Úintermediate_sizeÚ
hidden_actc                 ó  •— t          ¦   «                              ¦   «          t          j        ||d¬¦  «        | _        t          j        ||d¬¦  «        | _        t          j        ||d¬¦  «        | _        t          |         | _        d S )NF©r’   )	rs   rt   r   rU   Ú	gate_projÚ	down_projÚup_projr	   Úact_fn)rx   r¦   rò   ró   rz   s       €r7   rt   zIdeficsMLP.__init__Æ  sv   ø€ õ 	‰Œ×ÒÑÔÐÝœ ;Ð0AÈÐNÑNÔNˆŒÝœÐ#4°kÈÐNÑNÔNˆŒÝ”y Ð.?ÀeÐLÑLÔLˆŒÝ˜ZÔ(ˆŒˆˆr6   c                 ó¤   — |                       |                      |                      |¦  «        ¦  «        |                      |¦  «        z  ¦  «        S r]   )r÷   rù   rö   rø   )rx   rÛ   s     r7   r„   zIdeficsMLP.forwardÒ  s;   € Ø�~Š~˜dŸkšk¨$¯.ª.¸Ñ*;Ô*;Ñ<Ô<¸t¿|º|ÈA¹¼ÑNÑOÔOÐOr6   )r-   r.   r/   r�   rŠ   rt   r„   r‹   rŒ   s   @r7   rñ   rñ   Å  so   ø€ € € € € ð
)àð
)ð ð
)ð ð	
)ð 
)ð 
)ð 
)ð 
)ð 
)ðPð Pð Pð Pð Pð Pð Pr6   rñ   ç        r`   ÚqueryÚkeyÚvaluerC   ÚscalingÚdropoutc                 óÀ  — t          j        ||                     dd¦  «        ¦  «        |z  }|�||z   }t          j                             |dt           j        ¬¦  «                             |j        ¦  «        }t          j         	                    ||| j
        ¬¦  «        }t          j        ||¦  «        }	|	                     dd¦  «                             ¦   «         }	|	|fS )Nr=   éþÿÿÿ)rÍ   rp   ©ÚpÚtrainingr    r©   )r1   ÚmatmulrØ   r   Ú
functionalÚsoftmaxr«   rH   rp   r   r  Ú
contiguous)
r`   rü   rý   rþ   rC   rÿ   r   ry   Úattn_weightsÚattn_outputs
             r7   Úeager_attention_forwardr  ×  sÃ   € õ ”<  s§}¢}°R¸Ñ'<Ô'<Ñ=Ô=ÀÑG€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r6   c                   ó  ‡ — e Zd ZdZ	 	 	 	 	 ddedededed	edz  d
ededz  fˆ fd„Zde	j
        dedefd„Z	 	 	 	 dde	j
        de	j
        dz  de	j
        dz  de	j        dz  dedz  dee         dee	j
        e	j
        f         fd„Zˆ xZS )ÚIdeficsAttentionz=Multi-headed attention from 'Attention Is All You Need' paperrû   FNr¦   Ú	num_headsr   Úis_cross_attentionr¶   Úqk_layer_normsÚ	layer_idxc                 ó&  •— t          ¦   «                              ¦   «          || _        || _        || _        ||z  | _        || _        d| _        | j        dz  | _        || _	        |€(t                               d| j        j        › d�¦  «         | j        |z  | j        k    rt          d| j        › d|› d�¦  «        ‚|| _        t!          t"          j        d¦  «        st          d	¦  «        ‚| j        ršt!          |j        d
¦  «        s| j        n|j        j        }t#          j        | j        || j        z  d¬¦  «        | _        t#          j        ||| j        z  d¬¦  «        | _        t#          j        ||| j        z  d¬¦  «        | _        n{t#          j        | j        || j        z  d¬¦  «        | _        t#          j        | j        || j        z  d¬¦  «        | _        t#          j        | j        || j        z  d¬¦  «        | _        t#          j        || j        z  |d¬¦  «        | _        t5          |¦  «        | _        || _        | j        rBt;          | j        |j        ¬¦  «        | _        t;          | j        |j        ¬¦  «        | _         d S d S )NTg      à¿zInstantiating z¹ without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.z?hidden_size must be divisible by num_heads (got `hidden_size`: z and `num_heads`: z).Úscaled_dot_product_attentionz)this model requires pytorch 2.0 or higherÚ	embed_dimFrõ   ©r§   )!rs   rt   r¶   r¦   r  rÇ   r   Ú	is_causalrÿ   r  ÚloggerÚwarning_oncerz   r-   rr   r  Úhasattrr   r  Úvision_configr  rU   Úq_projÚk_projÚv_projÚo_projr´   Ú
rotary_embr  r    Úrms_norm_epsÚq_layer_normÚk_layer_norm)
rx   r¦   r  r   r  r¶   r  r  Úkv_input_dimrz   s
            €r7   rt   zIdeficsAttention.__init__ò  sá  ø€ õ 	‰Œ×ÒÑÔÐØˆŒØ&ˆÔØ"ˆŒØ# yÑ0ˆŒØˆŒØˆŒØ”} dÑ*ˆŒà"ˆŒØÐÝ×Òð, ¤Ô!8ð ,ð ,ð ,ñô ð ð ŒM˜IÑ%¨$Ô*:Ò:Ð:Ýð3ÐRVÔRbð 3ð 3Ø%.ð3ð 3ð 3ñô ð ð
 #5ˆÔå•r”}Ð&DÑEÔEð 	JÝÐHÑIÔIÐIàÔ"ð 	å(/°Ô0DÀkÑ(RÔ(RÐv�Ô Ð ÐX^ÔXlÔXvð õ œ)ØÔ Ø˜DœMÑ)Øðñ ô ˆDŒKõ
 œ) L°)¸d¼mÑ2KÐRWÐXÑXÔXˆDŒKÝœ)ØØ˜DœMÑ)Øðñ ô ˆDŒKˆKõ œ)ØÔ Ø˜DœMÑ)Øðñ ô ˆDŒKõ
 œ)ØÔ Ø˜DœMÑ)Øðñ ô ˆDŒKõ
 œ)ØÔ Ø˜DœMÑ)Øðñ ô ˆDŒKõ
 ”iØ˜œÑ%ØØð
ñ 
ô 
ˆŒõ
 1°Ñ8Ô8ˆŒà,ˆÔØÔð 	WÝ .¨t¬}À&ÔBUÐ VÑ VÔ VˆDÔÝ .¨t¬}À&ÔBUÐ VÑ VÔ VˆDÔÐÐð	Wð 	Wr6   ÚtensorÚseq_lenÚbszc                 ó’   — |                      ||| j        | j        ¦  «                             dd¦  «                             ¦   «         S )Nr    r©   )rF   r  rÇ   rØ   r	  )rx   r%  r&  r'  s       r7   Ú_shapezIdeficsAttention._shapeC  s<   € Ø�{Š{˜3 ¨¬¸¼ÑGÔG×QÒQÐRSÐUVÑWÔW×bÒbÑdÔdÐdr6   r*   Úkey_value_statesrC   rÜ   r)   ry   rl   c                 óÀ  — | j         p|d u}|                     ¦   «         \  }}	}
|                      |¦  «                             ||	| j        | j        ¦  «                             dd¦  «        }|s“|                      |¦  «                             ||	| j        | j        ¦  «                             dd¦  «        }|                      |¦  «                             ||	| j        | j        ¦  «                             dd¦  «        }nª|                     ¦   «         \  }
}}
|                      |¦  «                             ||| j        | j        ¦  «                             dd¦  «        }|                      |¦  «                             ||| j        | j        ¦  «                             dd¦  «        }|j	        d         }|�|| 
                    ¦   «         z  }|s.|                      ||¦  «        \  }}t          ||||¦  «        \  }}|�|                     ||| j        ¦  «        \  }}| j        r*|                      |¦  «        }|                      |¦  «        }t%          j        | j        j        t,          ¦  «        } || ||||f| j        sdn| j        | j        dœ|¤Ž\  }}|                     ||	d¦  «                             ¦   «         }|                      |¦  «        }||fS )Nr    r©   r  rû   )r   rÿ   r=   )r  Úsizer  rF   r  rÇ   rØ   r  r  rE   Úget_seq_lengthr   rï   Úupdater  r  r"  r#  r   Úget_interfacer¶   Ú_attn_implementationr  r  r   rÿ   Úreshaper	  r  )rx   r*   r*  rC   rÜ   r)   ry   r  r'  Úq_lenÚ_Úquery_statesÚ
key_statesÚvalue_statesÚkv_lenÚ
kv_seq_lenrÙ   rÚ   Úattention_interfacer  r
  s                        r7   r„   zIdeficsAttention.forwardF  s÷  € ð "Ô4ÐTÐ8HÐPTÐ8TÐà%×*Ò*Ñ,Ô,‰ˆˆU�Aà—{’{ =Ñ1Ô1×6Ò6°s¸EÀ4Ä>ÐSWÔS`ÑaÔa×kÒkÐlmÐopÑqÔqˆØ!ð 	ØŸš ]Ñ3Ô3×8Ò8¸¸eÀTÄ^ÐUYÔUbÑcÔc×mÒmÐnoÐqrÑsÔsˆJØŸ;š; }Ñ5Ô5×:Ò:¸3ÀÀtÄ~ÐW[ÔWdÑeÔe×oÒoÐpqÐstÑuÔuˆLˆLà+×0Ò0Ñ2Ô2‰LˆAˆv�qØŸšÐ%5Ñ6Ô6×;Ò;¸CÀÈÌÐY]ÔYfÑgÔg×qÒqÐrsÐuvÑwÔwˆJà—’Ð,Ñ-Ô-×2Ò2°3¸ÀÄÐPTÔP]Ñ^Ô^×hÒhÐijÐlmÑnÔnð ð  Ô% bÔ)ˆ
ØÐ&Ø˜/×8Ò8Ñ:Ô:Ñ:ˆJà!ð 	`Ø—’ |°\ÑBÔB‰HˆC�Ý';¸LÈ*ÐVYÐ[^Ñ'_Ô'_Ñ$ˆL˜*ð Ð&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜àÔð 	7Ø×,Ò,¨\Ñ:Ô:ˆLØ×*Ò*¨:Ñ6Ô6ˆJå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}Ð>�C�C°$´,Ø”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð "×)Ò)¨#¨u°bÑ9Ô9×DÒDÑFÔFˆØ—k’k +Ñ.Ô.ˆà˜LÐ(Ð(r6   )rû   FNFN)NNNN)r-   r.   r/   r0   r�   rË   r‰   r   rt   r1   rž   r)  Ú
LongTensorr
   r   r   r4   r„   r‹   rŒ   s   @r7   r  r  ï  s�  ø€ € € € € ØGÐGð Ø#(Ø*.Ø$Ø $ðOWð OWàðOWð ðOWð ð	OWð
 !ðOWð ! 4Ñ'ðOWð ðOWð ˜‘:ðOWð OWð OWð OWð OWð OWðbe˜Uœ\ð e°Cð e¸cð eð eð eð eð 15Ø.2Ø04Ø(,ð;)ð ;)à”|ð;)ð  œ,¨Ñ-ð;)ð œ tÑ+ð	;)ð
 Ô&¨Ñ-ð;)ð  ™ð;)ð Ð+Ô,ð;)ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð;)ð ;)ð ;)ð ;)ð ;)ð ;)ð ;)ð ;)r6   r  c                   ó®   ‡ — e Zd Zddededz  fˆ fd„Ze	 	 	 ddej        dej        dz  dej	        dz  de
dz  d	ee         d
ej        fd„¦   «         Zˆ xZS )ÚIdeficsDecoderLayerNr¶   r  c                 ó”  •— t          ¦   «                              ¦   «          |j        | _        t          | j        |j        |j        ||¬¦  «        | _        t          | j        |j        |j	        ¬¦  «        | _
        t          |j        |j        ¬¦  «        | _        t          |j        |j        ¬¦  «        | _        |j        | _        d S )N)r¦   r  r   r¶   r  ©r¦   rò   ró   r  )rs   rt   r¦   r  rÊ   r   Ú	self_attnrñ   rò   ró   Úmlpr    r!  Úinput_layernormÚpost_attention_layernorm©rx   r¶   r  rz   s      €r7   rt   zIdeficsDecoderLayer.__init__†  sÀ   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔÝ)ØÔ(ØÔ0Ø”NØØð
ñ 
ô 
ˆŒõ ØÔ(Ø$Ô6ØÔ(ð
ñ 
ô 
ˆŒõ
  .¨fÔ.@ÀfÔFYÐZÑZÔZˆÔÝ(6°vÔ7IÈvÔObÐ(cÑ(cÔ(cˆÔ%Ø”~ˆŒˆˆr6   r*   rC   rÜ   r)   ry   rl   c                 óz  — |}|                       |¦  «        } | j        d||||dœ|¤Ž\  }}t          j                             || j        | j        ¬¦  «        }||z   }|}|                      |¦  «        }|                      |¦  «        }t          j                             || j        | j        ¬¦  «        }||z   }|S )N)r*   rC   rÜ   r)   r  r5   )rA  r?  r   r  r   r  rB  r@  )rx   r*   rC   rÜ   r)   ry   Úresidualr3  s           r7   r„   zIdeficsDecoderLayer.forward™  sá   € ð !ˆà×,Ò,¨]Ñ;Ô;ˆð *˜4œ>ð 
Ø'Ø)Ø%Ø+ð	
ð 
ð
 ð
ð 
Ñˆ�qõ œ×-Ò-¨m¸t¼|ÐVZÔVcÐ-ÑdÔdˆØ  =Ñ0ˆð !ˆØ×5Ò5°mÑDÔDˆØŸš Ñ/Ô/ˆÝœ×-Ò-¨m¸t¼|ÐVZÔVcÐ-ÑdÔdˆØ  =Ñ0ˆàÐr6   r]   )NNN)r-   r.   r/   r!   r�   rt   r   r1   rž   r:  r
   r   r   r2   r„   r‹   rŒ   s   @r7   r<  r<  …  sÜ   ø€ € € € € ð&ð &˜}ð &¸¸t¹ð &ð &ð &ð &ð &ð &ð& ð /3Ø04Ø(,ðð à”|ðð œ tÑ+ðð Ô&¨Ñ-ð	ð
  ™ðð Ð+Ô,ðð 
Ô	ðð ð ñ „^ðð ð ð ð r6   r<  c                   óÚ   ‡ — e Zd Zddededz  fˆ fd„Ze	 	 	 	 	 ddej        dej        dz  dej        dz  dej        dz  d	ej        dz  d
e	dz  de
e         dej        fd„¦   «         Zˆ xZS )ÚIdeficsGatedCrossAttentionLayerNr¶   r  c           	      ó˜  •— t          ¦   «                              ¦   «          |j        | _        t          | j        |j        d|j        ||j        |¬¦  «        | _        t          | j        |j	        |j
        ¬¦  «        | _        t          |j        |j        ¬¦  «        | _        t          |j        |j        ¬¦  «        | _        |j        | _        t#          j        ¦   «         | _        t#          j        ¦   «         | _        |j        dk    rì|j        dk    rft#          j        t1          j        dd| j        ¦  «        ¦  «        | _        t#          j        t1          j        dd| j        ¦  «        ¦  «        | _        �n |j        dk    rXt#          j        t1          j        d¦  «        ¦  «        | _        t#          j        t1          j        d¦  «        ¦  «        | _        �n=t9          d	|j        › d
�¦  «        ‚|j        dk    rì|j        dk    rft#          j        t1          j        dd| j        ¦  «        ¦  «        | _        t#          j        t1          j        dd| j        ¦  «        ¦  «        | _        �n©|j        dk    rXt#          j        t1          j        d¦  «        ¦  «        | _        t#          j        t1          j        d¦  «        ¦  «        | _        �nFt9          d	|j        › d
�¦  «        ‚|j        dv �r|j        dk    rwt#          j        t1          j        d|j        dd| j        f¬¦  «        ¦  «        | _        t#          j        t1          j        d|j        dd| j        f¬¦  «        ¦  «        | _        n¢|j        dk    rgt#          j        t1          j        d|j        d¬¦  «        ¦  «        | _        t#          j        t1          j        d|j        d¬¦  «        ¦  «        | _        n0t9          d	|j        › d
�¦  «        ‚tA          d|j        › d�¦  «        ‚tC          | d¦  «        rtC          | d¦  «        st9          d¦  «        ‚d S )NT)r¦   r  r  r   r¶   r  r  r>  r  ÚzerosÚvectorr    rË   z Unknown value for `alpha_type` (ú)r¤   >   ÚnormalÚrandomÚgaussianrû   )r­   Ústdr,  zAlpha initialization scheme z not yet implemented!Úalpha_cross_attnÚalpha_densez+Alpha parameters not initialized correctly!)"rs   rt   r¦   r  rÊ   r   r  Ú
cross_attnrñ   rò   ró   r@  r    r!  rA  rB  r¶   r   ÚTanhÚact_cross_attnÚ	act_denseÚalpha_initializerÚ
alpha_typer£   r1   rI  rP  rQ  rr   r¤   rL  Úalphas_initializer_rangeÚNotImplementedErrorr  rC  s      €r7   rt   z(IdeficsGatedCrossAttentionLayer.__init__¼  så  ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔÝ*ØÔ(ØÔ0Ø#Ø”NØØ!Ô0Øð
ñ 
ô 
ˆŒõ ØÔ(Ø$Ô6ØÔ(ð
ñ 
ô 
ˆŒõ
  .¨fÔ.@ÀfÔFYÐZÑZÔZˆÔÝ(6°vÔ7IÈvÔObÐ(cÑ(cÔ(cˆÔ%Ø”nˆŒå œg™iœiˆÔÝœ™œˆŒàÔ# wÒ.Ð.ØÔ  HÒ,Ð,Ý(*¬µU´[ÀÀAÀtÔGWÑ5XÔ5XÑ(YÔ(Y�Ô%Ý#%¤<µ´¸A¸qÀ$ÔBRÑ0SÔ0SÑ#TÔ#T�Ô Ñ ØÔ" gÒ-Ð-Ý(*¬µU´[À±^´^Ñ(DÔ(D�Ô%Ý#%¤<µ´¸A±´Ñ#?Ô#?�Ô Ñ å Ð!XÀFÔDUÐ!XÐ!XÐ!XÑYÔYÐYàÔ%¨Ò/Ð/ØÔ  HÒ,Ð,Ý(*¬µU´ZÀÀ1ÀdÔFVÑ5WÔ5WÑ(XÔ(X�Ô%Ý#%¤<µ´
¸1¸aÀÔAQÑ0RÔ0RÑ#SÔ#S�Ô Ñ ØÔ" gÒ-Ð-Ý(*¬µU´ZÀ±]´]Ñ(CÔ(C�Ô%Ý#%¤<µ´
¸1±´Ñ#>Ô#>�Ô Ñ å Ð!XÀFÔDUÐ!XÐ!XÐ!XÑYÔYÐYàÔ%Ð)IÐIÑIØÔ  HÒ,Ð,Ý(*¬Ý”L c¨vÔ/NÐVWÐYZÐ\`Ô\lÐUmÐnÑnÔnñ)ô )�Ô%õ $&¤<Ý”L c¨vÔ/NÐVWÐYZÐ\`Ô\lÐUmÐnÑnÔnñ$ô $�Ô Ð ð Ô" gÒ-Ð-Ý(*¬Ý”L c¨vÔ/NÐVWÐYÑYÔYñ)ô )�Ô%õ $&¤<µ´À#È6ÔKjÐrsÐ0uÑ0uÔ0uÑ#vÔ#v�Ô Ð å Ð!XÀFÔDUÐ!XÐ!XÐ!XÑYÔYÐYõ &Ð&tÀVÔE]Ð&tÐ&tÐ&tÑuÔuÐuå˜Ð0Ñ1Ô1ð 	Lµg¸dÀMÑ6RÔ6Rð 	LÝÐJÑKÔKÐKð	Lð 	Lr6   r*   rC   r,   rA   Úcross_attention_gater)   ry   rl   c                 ó˜  — |€t          d¦  «        ‚|€t          d¦  «        ‚|�t          d¦  «        ‚|}|                      |¦  «        } | j        d	|||dœ|¤Ž\  }}	t          j                             || j        | j        ¬¦  «        }| 	                    |dk    dd…dd…df         d¦  «        }||  
                    | j        ¦  «        |z  z   }|}|                      |¦  «        }|                      |¦  «        }t          j                             || j        | j        ¬¦  «        }||                      | j        ¦  «        |z  z   }|S )
a  
        image_hidden_states (`torch.FloatTensor`):
            Input to the layer of shape `(batch, seq_len, embed_dim)`
        image_attention_mask (`torch.FloatTensor`, *optional*):
            image attention mask of size
            `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
        cross_attention_gate (`torch.FloatTensor`, *optional*):
            gate of size `(batch, seq_len)` used to zero-out cross-attention output for tokens attending no images.
        Nzt`image_hidden_states` is required for Idefics cross attention module which are visual features to be conditioned on.z‹`cross_attention_gate` is required for Idefics cross attention module to zero-out the cross-attention hidden_states attending to no images.zMPast key value states are not implemented for Idefics cross attention module.)r*   r*  rC   r  r   rû   r5   )rr   rY  rA  rR  r   r  r   r¶   r  Úmasked_fillrT  rP  rB  r@  rU  rQ  )
rx   r*   rC   r,   rA   rZ  r)   ry   rE  r3  s
             r7   r„   z'IdeficsGatedCrossAttentionLayer.forwardþ  s�  € ð( Ð&Ýð#ñô ð ð
  Ð'Ýð ^ñô ð ð Ð&Ý%Ð&uÑvÔvÐvà ˆà×,Ò,¨]Ñ;Ô;ˆð +˜4œ?ð 
Ø'Ø0Ø/ð
ð 
ð ð	
ð 
Ñˆ�qõ œ×-Ò-¨m¸t¼{ÐUYÔUbÐ-ÑcÔcˆà%×1Ò1Ð3GÈ1Ò3LÈaÈaÈaÐQRÐQRÐQRÐTXÈjÔ2YÐ[^Ñ_Ô_ˆØ  4×#6Ò#6°tÔ7LÑ#MÔ#MÐP]Ñ#]Ñ]ˆð !ˆØ×5Ò5°mÑDÔDˆØŸš Ñ/Ô/ˆÝœ×-Ò-¨m¸t¼{ÐUYÔUbÐ-ÑcÔcˆØ  4§>¢>°$Ô2BÑ#CÔ#CÀmÑ#SÑSˆàÐr6   r]   )NNNNN)r-   r.   r/   r!   r�   rt   r   r1   rž   r
   r   r   r2   r„   r‹   rŒ   s   @r7   rG  rG  »  s  ø€ € € € € ð@Lð @L˜}ð @L¸¸t¹ð @Lð @Lð @Lð @Lð @Lð @LðD ð /3Ø37Ø48Ø48Ø(,ð8ð 8à”|ð8ð œ tÑ+ð8ð #œ\¨DÑ0ð	8ð
 $œl¨TÑ1ð8ð $œl¨TÑ1ð8ð  ™ð8ð Ð+Ô,ð8ð 
Ô	ð8ð 8ð 8ñ „^ð8ð 8ð 8ð 8ð 8r6   rG  c                   óš   ‡ — e Zd ZU eed<   dZdZdZg d¢ZdZ	dZ
dZdZe eedd¬	¦  «        d
œZ ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )ÚIdeficsPreTrainedModelr¶   re   )ÚimageÚtextT)r<  rG  ÚIdeficsVisionEncoderLayerFr    r?  )ÚindexÚ
layer_name)r*   r+   c                 óž  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rjt	          j        |j        ¦  «         t	          j        |j        t          j
        |j        j        d         ¦  «                             d¦  «        ¦  «         d S t          |t          ¦  «        ræ| j        j        dk    r4t	          j        |j        ¦  «         t	          j        |j        ¦  «         d S | j        j        dk    r4t	          j        |j        ¦  «         t	          j        |j        ¦  «         d S | j        j        dv rNt	          j        |j        d| j        j        ¬¦  «         t	          j        |j        d| j        j        ¬¦  «         d S d S t          |t*          ¦  «        rt	          j        |j        ¦  «         d S d S )Nr=   )r    r=   rI  r¤   >   rL  rM  rN  rû   )r­   rO  )rs   Ú_init_weightsr^   r#   ÚinitÚnormal_Úclass_embeddingÚcopy_rÜ   r1   rD   rE   rÖ   rG  r¶   rV  Úzeros_rP  rQ  Úones_rX  r"   Úlatents)rx   r`   rz   s     €r7   re  z$IdeficsPreTrainedModel._init_weightsL  s£  ø€ õ
 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ5Ñ6Ô6ð 	)ÝŒL˜Ô/Ñ0Ô0Ð0ÝŒJ�vÔ*­E¬L¸Ô9LÔ9RÐSUÔ9VÑ,WÔ,W×,^Ò,^Ð_fÑ,gÔ,gÑhÔhÐhÐhÐhÝ˜Õ ?Ñ@Ô@ð 	)ØŒ{Ô,°Ò7Ð7Ý”˜FÔ3Ñ4Ô4Ð4Ý”˜FÔ.Ñ/Ô/Ð/Ð/Ð/Ø”Ô.°&Ò8Ð8Ý”
˜6Ô2Ñ3Ô3Ð3Ý”
˜6Ô-Ñ.Ô.Ð.Ð.Ð.Ø”Ô.Ð2RÐRÐRÝ”˜VÔ4¸3ÀDÄKÔDhÐiÑiÔiÐiÝ”˜VÔ/°c¸t¼{Ô?cÐdÑdÔdÐdÐdÐdð SÐRõ ˜Õ 9Ñ:Ô:ð 	)ÝŒL˜œÑ(Ô(Ð(Ð(Ð(ð	)ð 	)r6   )r-   r.   r/   r!   r3   Úbase_model_prefixÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_sdpaÚ_supports_flash_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr<  r   r  Ú_can_record_outputsr1   râ   re  r‹   rŒ   s   @r7   r^  r^  :  s¶   ø€ € € € € € àÐÐÑØÐØ(ÐØ&*Ð#ØoÐoÐoÐØ€Nà ÐØ"ÐØ"&Ðð -Ø$�nÐ%5¸QÈ;ÐWÑWÔWðð Ðð
 €U„]�_„_ð)ð )ð )ð )ñ „_ð)ð )ð )ð )ð )r6   r^  c                   óh  ‡ — e Zd ZdZdefˆ fd„Zdd„Zdd„Zdd„Ze	e
e	 	 	 	 	 	 	 	 	 	 	 dd
ej        dz  dej        dz  dej        dz  dedz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dedz  dedz  dee         deez  fd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚIdeficsModelzŸ
    Transformer decoder consisting of `config.num_hidden_layers` layers. Each layer is a [`IdeficsDecoderLayer`]

    Args:
        config: IdeficsConfig
    r¶   c                 ó®  •‡— t          ¦   «                              ‰¦  «         ‰| _        ‰j        | _        ‰j        | _        t          ‰j        ‰j        ‰j        ‰j	        | j        ¬¦  «        | _
        ‰j        j        | _        ‰j        | _        ‰j        | j        _        t          ‰j        ¦  «        | _        ‰j        r>‰j        }t%          ‰‰j        j        |j        |j        |j        |j        ¦  «        | _        t3          j        ˆfd„t7          ‰j        ¦  «        D ¦   «         ¦  «        | _        ‰j        | _        ‰j        | j        z  }t3          j        ˆfd„t7          |¦  «        D ¦   «         ¦  «        | _        d| _         tC          ‰j        ‰j"        ¬¦  «        | _#        |  $                    ¦   «          |  %                    ‰¦  «         d S )N)rn   ru   ro   rk   rq   c                 ó2   •— g | ]}t          ‰|¬ ¦  «        ‘ŒS ©)r  )r<  ©rX   Úir¶   s     €r7   r[   z)IdeficsModel.__init__.<locals>.<listcomp>�  s'   ø€ Ð_Ð_Ð_¸!Õ  °1Ð5Ñ5Ô5Ð_Ð_Ð_r6   c                 ó2   •— g | ]}t          ‰|¬ ¦  «        ‘ŒS rz  )rG  r{  s     €r7   r[   z)IdeficsModel.__init__.<locals>.<listcomp>“  s'   ø€ ÐcÐcÐcÀaÕ,¨V¸qÐAÑAÔAÐcÐcÐcr6   Fr  )&rs   rt   r¶   Úpad_token_idrq   Ú
vocab_sizerj   Úadditional_vocab_sizer¦   Úfreeze_text_layersÚembed_tokensr  Ú
image_sizer0  r$   Úvision_modelÚuse_resamplerÚperceiver_configr"   r  Úresampler_depthÚresampler_n_headsÚresampler_head_dimÚresampler_n_latentsÚperceiver_resamplerr   Ú
ModuleListÚrangeÚnum_hidden_layersÚlayersÚcross_layer_intervalÚgated_cross_attn_layersÚgradient_checkpointingr    r!  ÚnormÚ	post_initÚfreeze_relevant_params)rx   r¶   r†  Únum_cross_layersrz   s    `  €r7   rt   zIdeficsModel.__init__l  sÍ  øø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒØ!Ô.ˆÔØ Ô+ˆŒå5Ø!Ô,Ø&,Ô&BØ Ô,Ø#Ô6ØÔ(ð
ñ 
ô 
ˆÔð !Ô.Ô9ˆŒØ#Ô1ˆÔà28Ô2MˆÔÔ/Ý4°VÔ5IÑJÔJˆÔð Ôð 		Ø%Ô6ÐÝ'@ØØÔ$Ô.Ø Ô0Ø Ô2Ø Ô3Ø Ô4ñ(ô (ˆDÔ$õ ”mØ_Ð_Ð_Ð_½uÀVÔE]Ñ?^Ô?^Ð_Ñ_Ô_ñ
ô 
ˆŒð %+Ô$?ˆÔ!Ø!Ô3°tÔ7PÑPÐÝ')¤}ØcÐcÐcÐcÍ5ÐQaÑKbÔKbÐcÑcÔcñ(
ô (
ˆÔ$ð ',ˆÔ#å" 6Ô#5¸6Ô;NÐOÑOÔOˆŒ	ð 	�ŠÑÔÐà×#Ò# FÑ+Ô+Ð+Ð+Ð+r6   Nc                 ó¢   — |€| j         }|j        r|                      |j        ¦  «         |j        rt	          | j        |j        ¬¦  «         d S d S ©N)rf   )r¶   r�  Úfreeze_text_module_exceptionsÚfreeze_vision_layersrh   r„  Úfreeze_vision_module_exceptions)rx   r¶   s     r7   r•  z#IdeficsModel.freeze_relevant_paramsž  sh   € Øˆ>Ø”[ˆFàÔ$ð 	JØ×#Ò# FÔ$HÑIÔIÐIàÔ&ð 	fÝ˜Ô*¸fÔ>dÐeÑeÔeÐeÐeÐeð	fð 	fr6   r5   c                 óJ   — | j         | j        fD ]}t          ||¬¦  «         Œd S r˜  )r�  r“  rh   )rx   rf   r`   s      r7   r�  zIdeficsModel.freeze_text_layers¨  s?   € Ø”{ D¤IÐ.ð 	Fð 	FˆFÝ˜Ð3DÐEÑEÔEÐEÐEð	Fð 	Fr6   c                 ó2   — t          | j        |¬¦  «         d S r˜  )rh   r„  )rx   rf   s     r7   rš  z!IdeficsModel.freeze_vision_layers¬  s   € Ý�TÔ&Ð:KÐLÑLÔLÐLÐLÐLr6   FrL   rC   rÜ   r)   Úinputs_embedsr>   r?   r@   rA   Ú	use_cacheÚinterpolate_pos_encodingry   rl   c           	      ó2	  — |�|j         n|j         }|du |duz  rt          d¦  «        ‚|€|                      |¦  «        }|
r|€t          | j        ¬¦  «        }|j        d         }|�|                     ¦   «         nd}||z   }|�V|€T|                     ¦   «                              d¦  «        dz
  }| 	                    |dk    d¦  «         |dd…| d…f         }n5|€3t          j        ||j         ¬¦  «        |z   }|                     d¦  «        }t          d„ |||fD ¦   «         ¦  «        d	k    rt          d
¦  «        ‚|�{|                     | j        |¬¦  «        }|j        dd	…         \  }} |                     ¦   «         j        ||z  g|j        d	d…         ¢R Ž }|                      ||¬¦  «        j        }nQ|�O|                     ¦   «         \  }}}}|                     | j        |¬¦  «        }|                     ||z  ||¦  «        }| j        j        r^|€@|                      |¦  «        }|                     d¦  «        |                     d	¦  «        }}n|                     ¦   «         \  }}}}|}n<|€+|                     d¦  «        |                     d	¦  «        }}nt          d¦  «        ‚|                     |||z  |¦  «        } |	d                              ddd|¦  «        j        g |	j        dd	…         ¢d‘R Ž }	t          j        |	dd…ddd…dd…f                              ¦   «         t          j        dd||j        ¬¦  «        t          j        |j        ¦  «        j        ¦  «        }	|	dk                         d¬¦  «                             | j        |¬¦  «                             d¬¦  «        }|€(t          j         ||ft          j        |j         ¬¦  «        }tC          | j        ||||¬¦  «        }|}tE          | j#        ¦  «        D ]D\  }}|| j$        z  dk    r$| j%        || j$        z           } ||||f|	|ddœ|¤Ž} ||f|||dœ|¤Ž}ŒE|  &                    |¦  «        }|                     ||||¦  «        }tO          |||¬¦  «        S )ab  
        image_encoder_embeddings (`torch.FloatTensor`, *optional*):
            The output of the image encoder.
        perceiver_embeddings (`torch.FloatTensor`, *optional*):
            The output of the perceiver resampler.
        image_attention_mask (`torch.LongTensor`, *optional*):
            The attention mask for the image encoder.
        Nz:You must specify exactly one of input_ids or inputs_embeds)r¶   r    r   r=   )rI   c              3   ó   K  — | ]}|d u V — Œ	d S r]   r5   )rX   rÛ   s     r7   ra   z'IdeficsModel.forward.<locals>.<genexpr>á  s&   è è € ÐaÐa˜Qˆq�DˆyÐaÐaÐaÐaÐaÐar6   r©   z_Exactly 1 of pixel_values, image_encoder_embeddings or perceiver_embeddings has to be not-None.rÐ   )r>   r   zBIf `perceiver_embeddings` are passed, use_resampler should be True).Nr5   rû   )rI   rp   rÕ   )r¶   rž  rC   r)   rÜ   )rA   rZ  r)   )rC   rÜ   r)   )r(   r,   r)   )(rI   rr   r‚  r   r¶   rE   r-  ÚlongÚcumsumÚmasked_fill_r1   rD   ré   ÚsumrH   rp   r	  rF   r„  r(   r,  r…  r‹  rÖ   r1  r   r‰   ÚfullÚfinfoÚminrc   Úsqueezer¤   r   Ú	enumerater�  r�  r‘  r“  r'   )rx   rL   rC   rÜ   r)   rž  r>   r?   r@   rA   rŸ  r   ry   rI   Ú
seq_lengthÚpast_key_values_lengthÚseq_length_with_pastÚ
batch_sizeÚ
num_imagesr,   Úimage_seq_lenÚimage_hidden_sizerZ  Úcausal_maskr*   ÚidxÚdecoder_layerÚcross_attn_blocks                               r7   r„   zIdeficsModel.forward¯  s¾  € ð4 &/Ð%:�Ô!Ð!ÀÔ@Tˆà˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOà"Ô(¨Ô+ˆ
ØETÐE` ×!?Ò!?Ñ!AÔ!AÐ!AÐfgÐØ)Ð,BÑBÐàÐ%¨,Ð*>à)×.Ò.Ñ0Ô0×7Ò7¸Ñ;Ô;¸aÑ?ˆLØ×%Ò% n¸Ò&9¸1Ñ=Ô=Ð=Ø'¨¨¨¨J¨;¨<¨<¨Ô8ˆLˆLØÐ!Ý œ<¨
¸=Ô;OÐPÑPÔPÐSiÑiˆLØ'×1Ò1°!Ñ4Ô4ˆLåÐaÐa <Ð1IÐK_Ð"`ÐaÑaÔaÑaÔaÐefÒfÐfÝØqñô ð ð Ð%Ø'Ÿ?š?°´ÀF˜?ÑKÔKˆLØ%1Ô%7¸¸¸Ô%;Ñ"ˆJ˜
Ø9˜<×2Ò2Ñ4Ô4Ô9¸*ÀzÑ:QÐkÐT`ÔTfÐghÐgiÐgiÔTjÐkÐkÐkˆLð #'×"3Ò"3Ø)ÐD\ð #4ñ #ô #äð  Ðð &Ð1ØG_×GdÒGdÑGfÔGfÑDˆJ˜
 MÐ3DØ":×"=Ò"=ÀDÄJÐW]Ð"=Ñ"^Ô"^ÐØ"5×":Ò":¸:È
Ñ;RÐTaÐctÑ"uÔ"uÐàŒ;Ô$ð 
	cØ#Ð+Ø'+×'?Ò'?Ð@SÑ'TÔ'TÐ$Ø3G×3LÒ3LÈQÑ3OÔ3OÐQe×QjÒQjÐklÑQmÔQmÐ0��àK_×KdÒKdÑKfÔKfÑH�
˜J¨Ð7HØ"6ÐÐØ!Ð)Ø/B×/GÒ/GÈÑ/JÔ/JÐL_×LdÒLdÐefÑLgÔLgÐ,ˆMˆMåÐaÑbÔbÐbà1×6Ò6°zÀ:ÐP]ÑC]Ð_pÑqÔqÐðÐ  Ô+ßŠV�B˜˜B Ñ.Ô.Üð:à*Ô0°°!°Ô4ð:à68ð:ð :ð :ð 	õ
  %œ{Ø     D¨!¨!¨!¨Q¨Q¨Q Ô/×4Ò4Ñ6Ô6ÝŒJ�r˜3 vÐ5HÔ5NÐOÑOÔOÝŒKÐ+Ô1Ñ2Ô2Ô6ñ 
ô  
Ðð " SÒ(×-Ò-°"Ð-Ñ5Ô5×8Ò8¸t¼zÐRXÐ8ÑYÔY×aÒaÐfgÐaÑhÔhð 	ð
 Ð!Ý"œZØÐ1Ð2½%¼*È]ÔMaðñ ô ˆNõ )Ø”;Ø'Ø)Ø+Ø%ð
ñ 
ô 
ˆð &ˆå"+¨D¬KÑ"8Ô"8ð 	ð 	ÑˆC�à�TÔ.Ñ.°!Ò3Ð3Ø#'Ô#?ÀÀtÔG`Ñ@`Ô#aÐ Ø 0Ð 0Ø!ØØ'ð!ð *>Ø)=Ø$(ð!ð !ð ð!ð !�ð *˜MØðà*Ø)Ø /ð	ð ð
 ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆØ1×6Ò6°zÀ:È}Ð^oÑpÔpÐå-Ø+Ø 3Ø+ð
ñ 
ô 
ð 	
r6   r]   ©r5   )NNNNNNNNNNF)r-   r.   r/   r0   r!   rt   r•  r�  rš  r   r   r   r1   r:  rž   r
   r2   r‰   r   r   r4   r'   r„   r‹   rŒ   s   @r7   rw  rw  c  sà  ø€ € € € € ðð ð0,˜}ð 0,ð 0,ð 0,ð 0,ð 0,ð 0,ðdfð fð fð fðFð Fð Fð FðMð Mð Mð Mð  ØØð .2Ø.2Ø04Ø(,Ø26Ø15Ø=AØ9=Ø48Ø!%Ø05ðS
ð S
àÔ# dÑ*ðS
ð œ tÑ+ðS
ð Ô&¨Ñ-ð	S
ð
  ™ðS
ð Ô(¨4Ñ/ðS
ð Ô'¨$Ñ.ðS
ð #(Ô"3°dÑ":ðS
ð $Ô/°$Ñ6ðS
ð $œl¨TÑ1ðS
ð ˜$‘;ðS
ð #'¨¡+ðS
ð Ð+Ô,ðS
ð 
Ð/Ñ	/ðS
ð S
ð S
ñ „^ñ „_ñ  ÔðS
ð S
ð S
ð S
ð S
r6   rw  c            "       óÈ  ‡ — e Zd ZddiZdˆ fd„	Zee	 	 	 	 	 	 	 	 	 	 	 	 	 ddej        dz  dej	        dz  d	ej        dz  d
e
dz  dej        dz  dej        dz  dej        dz  dej        dz  dej	        dz  dej        dz  dedz  dedz  deej	        z  dee         deez  fd„¦   «         ¦   «         Z	 	 	 	 	 	 	 	 dˆ fd„	Z	 ddedeeef         dedeeef         fˆ fd„Zˆ xZS ) ÚIdeficsForVisionText2Textúlm_head.weightúmodel.embed_tokens.weightNc                 ó  •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          |j        |j        |j        d|j        ¬¦  «        | _	        |j        dk    r
dddœ| _
        |                      ¦   «          d S )NFrœ   r   r»  z.model.embed_tokens.additional_embedding.weight)rº  zlm_head.additional_fc.weight)rs   rt   rw  re   rŽ   r¦   r  r€  Úfreeze_lm_headÚlm_headÚ_tied_weights_keysr”  )rx   r¶   r„  rz   s      €r7   rt   z"IdeficsForVisionText2Text.__init__K  s˜   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý! &Ñ)Ô)ˆŒ
å-ØÔ*ØÔ*Ø$*Ô$@ØØ#Ô2ð
ñ 
ô 
ˆŒð Ô'¨!Ò+Ð+à"=Ø0`ð'ð 'ˆDÔ#ð 	�ŠÑÔÐÐÐr6   Fr   rL   rC   rÜ   r)   rž  r>   r?   r@   rA   ÚlabelsrŸ  r   Úlogits_to_keepry   rl   c                 óh  —  | j         d|||||||||	||ddœ|¤Ž}|j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }d}|
� | j        d||
| j        j        dœ|¤Ž}t          |||j
        |j        |j        |j        ¬¦  «        S )aC  
        image_encoder_embeddings (`torch.FloatTensor`, *optional*):
            The output of the image encoder.
        perceiver_embeddings (`torch.FloatTensor`, *optional*):
            The output of the perceiver resampler.
        image_attention_mask (`torch.LongTensor`, *optional*):
            The attention mask for the image encoder.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoProcessor, IdeficsForVisionText2Text

        >>> model = IdeficsForVisionText2Text.from_pretrained("HuggingFaceM4/idefics-9b")
        >>> processor = AutoProcessor.from_pretrained("HuggingFaceM4/idefics-9b")

        >>> dogs_image_url_1 = "https://huggingface.co/datasets/hf-internal-testing/fixtures_nlvr2/raw/main/image1.jpeg"
        >>> dogs_image_url_2 = "https://huggingface.co/datasets/hf-internal-testing/fixtures_nlvr2/raw/main/image2.jpeg"

        >>> prompts = [
        ...     [
        ...         "User:",
        ...         dogs_image_url_1,
        ...         "Describe this image.\nAssistant: An image of two dogs.\n",
        ...         "User:",
        ...         dogs_image_url_2,
        ...         "Describe this image.\nAssistant:",
        ...     ]
        ... ]
        >>> inputs = processor(prompts, return_tensors="pt")
        >>> generate_ids = model.generate(**inputs, max_new_tokens=6)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True)
        ```T)rL   rC   rÜ   r)   rž  r>   r?   r@   rA   rŸ  r   Úreturn_dictN)r;   rÀ  r  )r:   r;   r)   r*   r+   r,   r5   )re   r(   r^   r�   Úslicer¾  Úloss_functionr¶   r  r9   r)   r*   r+   r,   )rx   rL   rC   rÜ   r)   rž  r>   r?   r@   rA   rÀ  rŸ  r   rÁ  ry   Úoutputsr*   Úslice_indicesr;   r:   s                       r7   r„   z!IdeficsForVisionText2Text.forward_  s  € ðp 3=°$´*ð 3
ØØ)Ø%Ø+Ø'Ø%Ø%=Ø!5Ø!5ØØ%=Øð3
ð 3
ð ð3
ð 3
ˆð   Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%Ðp¨V¸FÈtÌ{ÔOeÐpÐpÐioÐpÐpˆDå,ØØØ#Ô3Ø!Ô/ØÔ)Ø 'Ô ;ð
ñ 
ô 
ð 	
r6   c
           
      ó  •—  t          ¦   «         j        |f|||||	|dœ|
¤Ž}|�| j        j        r||d<   n||d<   n||d<   |
                     dd¦  «        |d<   |�'|€%|d         j        d         }|d d …| d …f         |d	<   |S )
N)r)   rC   rž  rÜ   rŸ  rA   r@   r?   r>   r   FrL   r    rA   )rs   Úprepare_inputs_for_generationr¶   r…  ÚpoprE   )rx   rL   rC   rÜ   rž  r)   r>   r,   rA   rŸ  ry   Úmodel_inputsr¬  rz   s                €r7   rÉ  z7IdeficsForVisionText2Text.prepare_inputs_for_generation¹  sÞ   ø€ ð =•u‘w”wÔ<Øð	
à+Ø)Ø'Ø%ØØ!5ð	
ð 	
ð ð	
ð 	
ˆð Ð*ØŒ{Ô(ð OØ7J�Ð3Ñ4Ð4à;N�Ð7Ñ8Ð8à+7ˆL˜Ñ(Ø39·:²:Ð>XÐZ_Ñ3`Ô3`ˆÐ/Ñ0àÐ+°Ð0EØ% kÔ2Ô8¸Ô;ˆJØ3GÈÈÈÈJÈ;È<È<ÈÔ3XˆLÐ/Ñ0àÐr6   rÆ  rP   rN   c                 ó  •—  t          ¦   «         j        |||fi |¤Ž}d|v ra|d         }|d d …dd d …f                              d¦  «        }|                     dd¦  «        r||d<   nt	          j        ||gd¬¦  «        |d<   |j        |d<   |S )NrA   r=   r    rŸ  TrÕ   r,   )rs   Ú#_update_model_kwargs_for_generationré   rK   r1   r˜   r,   )rx   rÆ  rP   rN   ry   rA   Ú	last_maskrz   s          €r7   rÍ  z=IdeficsForVisionText2Text._update_model_kwargs_for_generationá  sÔ   ø€ ð C•u‘w”wÔBØØØð
ð 
ð ð	
ð 
ˆð " \Ð1Ð1Ø#/Ð0FÔ#GÐ Ø,¨Q¨Q¨Q°°A°A°A¨XÔ6×@Ò@ÀÑCÔCˆIØ×Ò ¨TÑ2Ô2ð kØ7@�Ð3Ñ4Ð4å7<´yÐBVÐXaÐAbÐhiÐ7jÑ7jÔ7j�Ð3Ñ4ð /6Ô.IˆÐ*Ñ+ØÐr6   r]   )NNNNNNNNNNNFr   )NNNNNNNN)F)r-   r.   r/   r¿  rt   r   r   r1   r:  rž   r
   r2   r‰   r�   r   r   r4   r9   r„   rÉ  r   ÚdictrŠ   r   rÍ  r‹   rŒ   s   @r7   r¹  r¹  H  sG  ø€ € € € € Ø*Ð,GÐHÐðð ð ð ð ð ð( Øð .2Ø.2Ø04Ø(,Ø26Ø15Ø=AØ9=Ø48Ø*.Ø!%Ø05Ø-.ðV
ð V
àÔ# dÑ*ðV
ð œ tÑ+ðV
ð Ô&¨Ñ-ð	V
ð
  ™ðV
ð Ô(¨4Ñ/ðV
ð Ô'¨$Ñ.ðV
ð #(Ô"3°dÑ":ðV
ð $Ô/°$Ñ6ðV
ð $œl¨TÑ1ðV
ð Ô  4Ñ'ðV
ð ˜$‘;ðV
ð #'¨¡+ðV
ð ˜eœlÑ*ðV
ð Ð+Ô,ðV
ð  
Ð.Ñ	.ð!V
ð V
ð V
ñ „^ñ ÔðV
ðv ØØØØØ Ø!Øð&ð &ð &ð &ð &ð &ðX $)ð	ð àðð ˜3 ˜8”nðð !ð	ð 
ˆc�3ˆhŒðð ð ð ð ð ð ð ð ð r6   r¹  )r¹  rw  r^  )r    FNNr·  )r    )rû   )Vr0   Úcollections.abcr   Údataclassesr   Útypingr   r1   Útorch.nn.functionalr   r  r|   Ú r   rf  Úactivationsr	   Úcache_utilsr
   r   Ú
generationr   Úintegrationsr   Úmasking_utilsr   Úmodeling_layersr   Úmodeling_outputsr   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   r   Úconfiguration_ideficsr!   Ú	perceiverr"   Úvisionr#   r$   Ú
get_loggerr-   r  r'   r9   rR   rh   rV   rj   rU   rŽ   ÚModuler    r´   ræ   rï   rñ   rž   rË   r  r  r<  rG  r^  rw  r¹  Ú__all__r5   r6   r7   ú<module>ré     s  ðð& Ð à $Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )Ø 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø /Ð /Ð /Ð /Ð /Ð /Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø +Ð +Ð +Ð +Ð +Ð +Ø KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ XÐ XÐ XÐ XÐ XÐ XÐ XÐ XÐ XÐ XØ &Ð &Ð &Ð &Ð &Ð &Ø RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RØ 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø EÐ EÐ EÐ EÐ EÐ EÐ EÐ Eð 
ˆÔ	˜HÑ	%Ô	%€ð €ððñ ô ð
 ð@ð @ð @ð @ð @ [ñ @ô @ñ „ñô ð@ð* €ððñ ô ð
 ð@ð @ð @ð @ð @ Kñ @ô @ñ „ñô ð@ð8 ØØØð*#ð *#ð *#ð *#ðZð ð ð ðfAð fAð fAð fAð fA ¤ñ fAô fAð fAðR8Sð 8Sð 8Sð 8Sð 8S˜RœYñ 8Sô 8Sð 8SðxJð Jð Jð Jð J�R”Yñ Jô Jð Jð06<ð 6<ð 6<ð 6<ð 6<˜RœYñ 6<ô 6<ð 6<ðt(ð (ð (ð ÐÐ-Ñ.Ô.ðð ð ñ /Ô.ðð4Pð Pð Pð Pð P�”ñ Pô Pð Pð2 ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð %ð %ð %ð0R)ð R)ð R)ð R)ð R)�r”yñ R)ô R)ð R)ðl3ð 3ð 3ð 3ð 3Ð4ñ 3ô 3ð 3ðl|ð |ð |ð |ð |Ð&@ñ |ô |ð |ð~ ð%)ð %)ð %)ð %)ð %)˜_ñ %)ô %)ñ „ð%)ðP ða
ð a
ð a
ð a
ð a
Ð)ñ a
ô a
ñ „ða
ðHqð qð qð qð qÐ 6¸ñ qô qð qðh RÐ
QÐ
Q€€€r6   