§
    9²¸j¦`  ã                   ó  — d dl mZ d dlZd dlmZ ddlmZ ddlm	Z	 ddl
mZ ddlmZmZ ddlmZmZ dd	lmZ dd
lmZmZmZ ddlmZ ddlmZmZ ddlmZ ddlm Z   G d„ dej!        ¦  «        Z" G d„ dej!        ¦  «        Z# G d„ dej!        ¦  «        Z$	 d6dej!        dej%        dej%        dej%        dej%        dz  de&de&dee         fd„Z'd „ Z(d!ej%        d"e)d#ej%        fd$„Z*d%ej%        d&ej%        d'ej%        d(ej%        d#e+ej%        ej%        f         f
d)„Z, G d*„ d+ej!        ¦  «        Z- G d,„ d-e¦  «        Z. G d.„ d/ej!        ¦  «        Z/e G d0„ d1e¦  «        ¦   «         Z0 ed2¬3¦  «         G d4„ d5e0¦  «        ¦   «         Z1d1d5gZ2dS )7é    )ÚCallableNé   )Úinitialization)ÚACT2FN)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚBaseModelOutputWithPooling)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚ	torch_int)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )ÚMLCDVisionConfigc                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚMLCDMLPc                 ó  •— t          ¦   «                              ¦   «          || _        t          |j                 | _        t          j        |j        |j	        ¦  «        | _
        t          j        |j	        |j        ¦  «        | _        d S ©N)ÚsuperÚ__init__Úconfigr   Ú
hidden_actÚactivation_fnÚnnÚLinearÚhidden_sizeÚintermediate_sizeÚfc1Úfc2©Úselfr   Ú	__class__s     €úv/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/mlcd/modeling_mlcd.pyr   zMLCDMLP.__init__'   sf   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ# FÔ$5Ô6ˆÔÝ”9˜VÔ/°Ô1IÑJÔJˆŒÝ”9˜VÔ5°vÔ7IÑJÔJˆŒˆˆó    Úhidden_statesÚreturnc                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r   )r#   r   r$   )r&   r*   s     r(   ÚforwardzMLCDMLP.forward.   s=   € ØŸš Ñ/Ô/ˆØ×*Ò*¨=Ñ9Ô9ˆØŸš Ñ/Ô/ˆØÐr)   )Ú__name__Ú
__module__Ú__qualname__r   ÚtorchÚTensorr-   Ú__classcell__©r'   s   @r(   r   r   &   sc   ø€ € € € € ðKð Kð Kð Kð Kð U¤\ð °e´lð ð ð ð ð ð ð ð r)   r   c            	       óè   ‡ — e Zd ZdZ edd¬¦  «        ddefˆ fd„¦   «         Ze edd¬¦  «        ddedee	j
        ef         fd	„¦   «         ¦   «         Z e	j        ¦   «         d
„ ¦   «         Zd„ Zˆ xZS )ÚMLCDRotaryEmbeddingzý
    Simple axial 2D rope with same freqs used for H and W grids. The freqs are
    pre-computed using `head-dim//4` which is later used to concat H and W positions.
    The final angles rotate over the whole head dim, no partial rotation involved.
    Údevicez5.18)ÚversionNr   c                 ó°  •— t          ¦   «                              ¦   «          || _        | j        j        d         | _        | j        }| j        dk    r#t          | j        j        › d| j        › �¦  «        ‚ || j        |¦  «        \  }| _	        t          j        |d¬¦  «        | _        t          j        |                     ¦   «         d¬¦  «        | _        d S )NÚ	rope_typeÚaxialz) supports only axial rope, but requested F©Ú
persistent)r   r   r   Úrope_parametersr:   Úcompute_axial_rope_parametersÚ
ValueErrorr'   r.   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r&   r   r7   Úrope_init_fnrC   r'   s        €r(   r   zMLCDRotaryEmbedding.__init__<   sÃ   ø€ å‰Œ×ÒÑÔÐØˆŒàœÔ4°[ÔAˆŒØ!%Ô!CˆØŒ>˜WÒ$Ð$Ý ¤Ô 7ÐrÐrÐbfÔbpÐrÐrÑsÔsÐsØ+7¨<¸¼ÀVÑ+LÔ+LÑ(ˆ�$Ô(åœ	 (°uÐ=Ñ=Ô=ˆŒÝ!#¤¨8¯>ª>Ñ+;Ô+;ÈÐ!NÑ!NÔ!NˆÔÐÐr)   r+   c                 óî   — | j         d         }t          | dd¦  «        p| j        | j        z  }|dz  }d}d|t	          j        d|dt          j        ¬¦  «        |z  z  z  }|                     |¦  «        |fS )a¾  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNé   g      ð?r   ©Údtype)r>   Úgetattrr!   Únum_attention_headsr1   ÚarangeÚfloatÚto)r   r7   ÚkwargsÚbaseÚdimÚspatial_dimÚattention_factorrC   s           r(   r?   z1MLCDRotaryEmbedding.compute_axial_rope_parametersJ   sƒ   € ð Ô% lÔ3ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆØ˜Q‘hˆàÐØ˜$¥5¤<°°;ÀÍÌÐ#UÑ#UÔ#UÐXcÑ#cÑdÑeˆØ�{Š{˜6Ñ"Ô"Ð$4Ð4Ð4r)   c                 ó  — |d                               ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  || j                              ¦   «         z  }|                     ¦   «         | j        z  }| 	                    ¦   «         | j        z  }d d d ¦  «         n# 1 swxY w Y   |  
                    |¦  «        }|  
                    |¦  «        }||fS )N).NÚmpsÚcpuF)Údevice_typeÚenabled)rP   Ú
isinstancer7   ÚtypeÚstrr   rC   ÚcosrA   ÚsinÚrecomposition_frequencies)r&   ÚxÚposition_idsÚposition_ids_expandedrZ   Úfreqsr_   r`   s           r(   r-   zMLCDRotaryEmbedding.forward^   s&  € ð !-¨YÔ 7× =Ò =Ñ ?Ô ?ÐÝ'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆÝ¨¸UÐCÑCÔCð 	7ð 	7Ø)¨D¬M×,?Ò,?Ñ,AÔ,AÑAˆEØ—)’)‘+”+ Ô 6Ñ6ˆCØ—)’)‘+”+ Ô 6Ñ6ˆCð	7ð 	7ð 	7ñ 	7ô 	7ð 	7ð 	7ð 	7ð 	7ð 	7ð 	7øøøð 	7ð 	7ð 	7ð 	7ð
 ×,Ò,¨SÑ1Ô1ˆØ×,Ò,¨SÑ1Ô1ˆØ�Cˆxˆs   Á)AC
Ã
CÃCc                 ó’   — |dd…df         |dd…df         }}t          j        ||gd¬¦  «        }t          j        ||gd¬¦  «        S )z]
        Recompose the frequencies into the final spatial layout used per each grid.
        Nr   r   éÿÿÿÿ©rT   )r1   Úcat)r&   ÚfreqÚfreq_hÚfreq_wÚfreq_hws        r(   ra   z-MLCDRotaryEmbedding.recomposition_frequenciesl   sX   € ð ˜a˜a˜a ˜dœ T¨!¨!¨!¨Q¨$¤Z�ˆÝ”)˜V VÐ,°"Ð5Ñ5Ô5ˆÝŒy˜' 7Ð+°Ð4Ñ4Ô4Ð4r)   r   )r.   r/   r0   Ú__doc__r   r   r   ÚstaticmethodÚtupler1   r2   rP   r?   Úno_gradr-   ra   r3   r4   s   @r(   r6   r6   5   s  ø€ € € € € ðð ð €_�X vÐ.Ñ.Ô.ðOð OÐ/ð Oð Oð Oð Oð Oñ /Ô.ðOð Ø€_�X vÐ.Ñ.Ô.ð5ð 5Ð.>ð 5ÐZ_Ð`eÔ`lÐnsÐ`sÔZtð 5ð 5ð 5ñ /Ô.ñ „\ð5ð$ €U„]�_„_ðð ñ „_ðð5ð 5ð 5ð 5ð 5ð 5ð 5r)   r6   c                   ót   ‡ — e Zd Zdefˆ fd„Zdej        dededej        fd„Zdej	        dej        fd	„Z
ˆ xZS )
ÚMLCDVisionEmbeddingsr   c                 ó8  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        |j        | _        t          j        t          j
        | j        ¦  «        ¦  «        | _        t          j        |j        | j        | j        | j        d¬¦  «        | _        | j        | j        z  dz  | _        | j        dz   | _        t          j        t          j        | j        ¦  «                             d¦  «        d¬¦  «        | _        d S )NF)Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚbiasrJ   r   ©r   rg   r<   )r   r   r   r!   Ú	embed_dimÚ
image_sizeÚ
patch_sizer   Ú	Parameterr1   ÚrandnÚclass_embeddingÚConv2dÚnum_channelsÚpatch_embeddingÚnum_patchesÚnum_positionsrB   rO   Úexpandrc   r%   s     €r(   r   zMLCDVisionEmbeddings.__init__v   sî   ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ+ˆŒØ Ô+ˆŒØ Ô+ˆŒå!œ|­E¬K¸¼Ñ,GÔ,GÑHÔHˆÔå!œyØÔ+ØœØœØ”?Øð 
ñ  
ô  
ˆÔð !œO¨t¬Ñ>À1ÑDˆÔØ!Ô-°Ñ1ˆÔÝœI¥e¤l°4Ô3EÑ&FÔ&F×&MÒ&MÈgÑ&VÔ&VÐchÐiÑiÔiˆÔÐÐr)   Ú
embeddingsÚheightÚwidthr+   c                 óÚ  — |j         d         dz
  }| j        j                             d¦  «        }|j         d         dz
  }t          j                             ¦   «         s&||k    r ||k    r|                      | j        ¦  «        S |dd…dd…f         }|dd…dd…f         }|j         d         }	|| j        z  }
|| j        z  }t          |dz  ¦  «        }| 
                    d|||	¦  «        }|                     dddd¦  «        }t          j                             ||
|fdd	¬
¦  «        }|                     dddd¦  «                             dd|	¦  «        }t	          j        ||fd¬¦  «        S )a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   r   Nrg   g      à?r   rJ   ÚbicubicF)ÚsizeÚmodeÚalign_cornersrh   )ÚshapeÚposition_embeddingÚweightÚ	unsqueezer1   ÚjitÚ
is_tracingrc   r}   r   ÚreshapeÚpermuter   Ú
functionalÚinterpolateÚviewri   )r&   r‡   rˆ   r‰   r„   r�   r…   Úclass_pos_embedÚpatch_pos_embedrT   Ú
new_heightÚ	new_widthÚsqrt_num_positionss                r(   Úinterpolate_pos_encodingz-MLCDVisionEmbeddings.interpolate_pos_encoding‹   s‘  € ð !Ô& qÔ)¨AÑ-ˆØ!Ô4Ô;×EÒEÀaÑHÔHÐØ*Ô0°Ô3°aÑ7ˆõ Œy×#Ò#Ñ%Ô%ð 	>¨+¸Ò*FÐ*FÈ6ÐUZÊ?È?Ø×*Ò*¨4Ô+<Ñ=Ô=Ð=à,¨Q¨Q¨Q°°°¨UÔ3ˆØ,¨Q¨Q¨Q°°°¨UÔ3ˆàÔ˜rÔ"ˆà˜tœÑ.ˆ
Ø˜Tœ_Ñ,ˆ	å& }°cÑ'9Ñ:Ô:ÐØ)×1Ò1°!Ð5GÐI[Ð]`ÑaÔaˆØ)×1Ò1°!°Q¸¸1Ñ=Ô=ˆåœ-×3Ò3ØØ˜iÐ(ØØð	 4ñ 
ô 
ˆð *×1Ò1°!°Q¸¸1Ñ=Ô=×BÒBÀ1ÀbÈ#ÑNÔNˆåŒy˜/¨?Ð;ÀÐCÑCÔCÐCr)   Úpixel_valuesc                 óN  — |j         d         }| j        j        j        }|                      |                     |¬¦  «        ¦  «        }|                     d¦  «                             dd¦  «        }| j                             |dd¦  «        }t          j
        ||gd¬¦  «        }|S )Nr   rK   rJ   r   rg   rh   )r�   rƒ   r‘   rL   rQ   ÚflattenÚ	transposer€   r†   r1   ri   )r&   r    Ú
batch_sizeÚtarget_dtypeÚpatch_embedsÚclass_embedsr‡   s          r(   r-   zMLCDVisionEmbeddings.forward´   sš   € Ø!Ô'¨Ô*ˆ
ØÔ+Ô2Ô8ˆà×+Ò+¨L¯OªOÀ,¨OÑ,OÔ,OÑPÔPˆØ#×+Ò+¨AÑ.Ô.×8Ò8¸¸AÑ>Ô>ˆàÔ+×2Ò2°:¸qÀ"ÑEÔEˆÝ”Y ¨lÐ;ÀÐCÑCÔCˆ
àÐr)   )r.   r/   r0   r   r   r1   r2   ÚintrŸ   ÚFloatTensorr-   r3   r4   s   @r(   rs   rs   u   s³   ø€ € € € € ðjÐ/ð jð jð jð jð jð jð*'D°5´<ð 'DÈð 'DÐUXð 'DÐ]bÔ]ið 'Dð 'Dð 'Dð 'DðR
 EÔ$5ð 
¸%¼,ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r)   rs   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutrR   c                 ó  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        |z  }
|�|
|z   }
t
          j                             |
dt          j        ¬¦  «         	                    |j
        ¦  «        }
t
          j                             |
|| j        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «                             ¦   «         }||
fS )NrJ   r   rg   )rT   rL   )ÚpÚtrainingr   )Ú	repeat_kvÚnum_key_value_groupsr1   Úmatmulr£   r   r—   ÚsoftmaxÚfloat32rQ   rL   r±   r´   Ú
contiguous)r«   r¬   r­   r®   r¯   r°   r±   rR   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r(   Úeager_attention_forwardr¿   Á   sé   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r)   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..Nrg   rJ   rh   )r�   r1   ri   )rb   Úx1Úx2s      r(   Úrotate_halfrÃ   Ú   s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'r)   r*   Ún_repr+   c                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r   N)r�   r†   r•   )r*   rÄ   ÚbatchÚnum_key_value_headsÚslenrI   s         r(   rµ   rµ   á   s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTr)   ÚqÚkr_   r`   c                 óÆ  — | j         }|j         }|                      ¦   «         |                     ¦   «         }} |                     d¦  «                             ¦   «         |                     d¦  «                             ¦   «         }}| |z  t          | ¦  «        |z  z   }||z  t          |¦  «        |z  z   }|                     |¦  «        }|                     |¦  «        }||fS )Néþÿÿÿ)rL   rP   r’   rÃ   rQ   )rÉ   rÊ   r_   r`   Úorig_q_dtypeÚorig_k_dtypeÚq_embedÚk_embeds           r(   Úapply_rotary_pos_emb_visionrÑ   í   sÃ   € ð ”7€LØ”7€LØ�7Š7‰9Œ9�a—g’g‘i”i€q€AØ�}Š}˜RÑ Ô ×&Ò&Ñ(Ô(¨#¯-ª-¸Ñ*;Ô*;×*AÒ*AÑ*CÔ*Cˆ€CØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�jŠj˜Ñ&Ô&€GØ�jŠj˜Ñ&Ô&€GØ�GÐÐr)   c                   ó¼   ‡ — e Zd ZdZdefˆ fd„Z	 ddej        deej        ej        f         dej        dz  de	e
         d	eej        ej        dz  f         f
d
„Zˆ xZS )ÚMLCDAttentionzÿMulti-headed attention with RoPE. Refer to papers:
    - Attention is all you need:
        https://huggingface.co/papers/1706.03762
    - RoFormer: Enhanced Transformer with Rotary Position Embedding:
        https://huggingface.co/papers/2104.09864
    r   c                 ó*  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        | j        | j        z  | _        | j        dz  | _        |j	        | _
        d| _        t          j        | j        | j        ¦  «        | _        t          j        | j        | j        ¦  «        | _        t          j        | j        | j        ¦  «        | _        t          j        | j        | j        ¦  «        | _        |j        | _        d S )Nç      à¿F)r   r   r   r!   r{   rN   Ú	num_headsrI   ÚscaleÚattention_dropoutr±   Ú	is_causalr   r    Úk_projÚv_projÚq_projÚout_projr¶   r%   s     €r(   r   zMLCDAttention.__init__  sÐ   ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ+ˆŒØÔ3ˆŒØœ¨$¬.Ñ8ˆŒØ”] DÑ(ˆŒ
ØÔ/ˆŒØˆŒå”i ¤°´Ñ?Ô?ˆŒÝ”i ¤°´Ñ?Ô?ˆŒÝ”i ¤°´Ñ?Ô?ˆŒÝœ	 $¤.°$´.ÑAÔAˆŒØ$*Ô$?ˆÔ!Ð!Ð!r)   Nr*   Úposition_embeddingsr¯   rR   r+   c                 óê  — |j         dd…         \  }}|                      |¦  «                             ||| j        | j        f¦  «        }|                      |¦  «                             ||| j        | j        f¦  «        }|                      |¦  «                             ||| j        | j        f¦  «        }	|d                              d¦  «                             ¦   «         }
|d                              d¦  «                             ¦   «         }t          |||
|¦  «        \  }}| 
                    dddd¦  «                             ¦   «         }| 
                    dddd¦  «                             ¦   «         }|	 
                    dddd¦  «                             ¦   «         }	t          j        | j        j        t           ¦  «        } || |||	|f| j        sdn| j        | j        | j        dœ|¤Ž\  }}| 
                    dddd¦  «                             ¦   «         }|                     ||d¦  «        }|                      |¦  «        }| 
                    ddd¦  «                             ¦   «         }||fS )	z#Input shape: Batch x Time x ChannelNrg   r   r   rJ   r   rª   )r±   r°   rÙ   )r�   rÜ   r•   rÖ   rI   rÚ   rÛ   r’   rP   rÑ   r–   rº   r
   Úget_interfacer   Ú_attn_implementationr¿   r´   r±   r×   rÙ   r™   rÝ   )r&   r*   rÞ   r¯   rR   r¤   Ú
seq_lengthÚquery_statesr»   r¼   r_   r`   Úattention_interfacer¾   r½   s                  r(   r-   zMLCDAttention.forward  st  € ð "/Ô!4°S°b°SÔ!9Ñˆ
�Jð —{’{ =Ñ1Ô1×9Ò9¸:ÀzÐSWÔSaÐcgÔcpÐ:qÑrÔrˆØ—[’[ Ñ/Ô/×7Ò7¸ÀZÐQUÔQ_ÐaeÔanÐ8oÑpÔpˆ
Ø—{’{ =Ñ1Ô1×9Ò9¸:ÀzÐSWÔSaÐcgÔcpÐ:qÑrÔrˆð " !Ô$×.Ò.¨qÑ1Ô1×7Ò7Ñ9Ô9ˆØ! !Ô$×.Ò.¨qÑ1Ô1×7Ò7Ñ9Ô9ˆÝ#>¸|ÈZÐY\Ð^aÑ#bÔ#bÑ ˆ�jð $×+Ò+¨A¨q°!°QÑ7Ô7×BÒBÑDÔDˆØ×'Ò'¨¨1¨a°Ñ3Ô3×>Ò>Ñ@Ô@ˆ
Ø#×+Ò+¨A¨q°!°QÑ7Ô7×BÒBÑDÔDˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð  $œ}Ð>�C�C°$´,Ø”JØ”nð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð "×)Ò)¨!¨Q°°1Ñ5Ô5×@Ò@ÑBÔBˆØ!×&Ò& z°:¸rÑBÔBˆØ—m’m KÑ0Ô0ˆØ!×)Ò)¨!¨Q°Ñ2Ô2×=Ò=Ñ?Ô?ˆØ˜LÐ(Ð(r)   r   )r.   r/   r0   rn   r   r   r1   r2   rp   r   r   r-   r3   r4   s   @r(   rÓ   rÓ   û   sÔ   ø€ € € € € ðð ð@Ð/ð @ð @ð @ð @ð @ð @ð( /3ð	-)ð -)à”|ð-)ð # 5¤<°´Ð#=Ô>ð-)ð œ tÑ+ð	-)ð
 Ð+Ô,ð-)ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð-)ð -)ð -)ð -)ð -)ð -)ð -)ð -)r)   rÓ   c                   ó¤   ‡ — e Zd Zdefˆ fd„Z	 d
dej        deej        ej        f         dej        dz  dee	         deej
                 f
d	„Zˆ xZS )ÚMLCDEncoderLayerr   c                 óD  •— t          ¦   «                              ¦   «          |j        | _        t	          |¦  «        | _        t          j        | j        |j        ¬¦  «        | _	        t          |¦  «        | _        t          j        | j        |j        ¬¦  «        | _        d S )N©Úeps)r   r   r!   r{   rÓ   Ú	self_attnr   Ú	LayerNormÚlayer_norm_epsÚlayer_norm1r   ÚmlpÚlayer_norm2r%   s     €r(   r   zMLCDEncoderLayer.__init__D  s}   ø€ Ý‰Œ×ÒÑÔÐØÔ+ˆŒÝ& vÑ.Ô.ˆŒÝœ<¨¬¸FÔ<QÐRÑRÔRˆÔÝ˜6‘?”?ˆŒÝœ<¨¬¸FÔ<QÐRÑRÔRˆÔÐÐr)   Nr*   rÞ   r¯   rR   r+   c                 óÈ   — |}|                       |¦  «        } | j        d|||dœ|¤Ž\  }}||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|S )a¸  
        Args:
            hidden_states (`torch.FloatTensor`):
                Input to the layer of shape `(batch, seq_len, embed_dim)`.
                Represents the hidden states from the previous layer or the input embeddings.
            position_embeddings (`tuple[torch.Tensor, torch.Tensor]`):
                A tuple of two tensors, each of shape `(batch, seq_len, embed_dim)`.
                Represents absolute positional embeddings for the query and key in the attention mechanism.
            attention_mask (`torch.FloatTensor`):
                Attention mask of shape `(batch, 1, q_len, k_v_seq_len)` where padding elements are indicated by very large negative values.
        )r*   rÞ   r¯   © )rí   rê   rï   rî   )r&   r*   rÞ   r¯   rR   ÚresidualÚ_s          r(   r-   zMLCDEncoderLayer.forwardL  s–   € ð$ !ˆà×(Ò(¨Ñ7Ô7ˆØ)˜4œ>ð 
Ø'Ø 3Ø)ð
ð 
ð ð	
ð 
Ñˆ�qð ! =Ñ0ˆà ˆØ×(Ò(¨Ñ7Ô7ˆØŸš Ñ/Ô/ˆØ  =Ñ0ˆàÐr)   r   )r.   r/   r0   r   r   r1   r2   rp   r   r   r©   r-   r3   r4   s   @r(   ræ   ræ   C  s¿   ø€ € € € € ðSÐ/ð Sð Sð Sð Sð Sð Sð /3ð	"ð "à”|ð"ð # 5¤<°´Ð#=Ô>ð"ð œ tÑ+ð	"ð
 Ð+Ô,ð"ð 
ˆuÔ Ô	!ð"ð "ð "ð "ð "ð "ð "ð "r)   ræ   c                   ó˜   ‡ — e Zd ZdZdefˆ fd„Z	 ddej        deej	        ej	        f         dej	        dz  de
e         d	eez  f
d
„Zˆ xZS )ÚMLCDEncoderz³
    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
    [`MLCDEncoderLayer`].

    Args:
        config: MLCDVisionConfig
    r   c                 óÔ   •‡— t          ¦   «                              ¦   «          ‰| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        d| _        dS )z3Overwrite dummy `MLCDConfig` to `MLCDVisionConfig`.c                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS rñ   )ræ   )Ú.0ró   r   s     €r(   ú
<listcomp>z(MLCDEncoder.__init__.<locals>.<listcomp>~  s"   ø€ Ð$gÐ$gÐ$gÀ!Õ%5°fÑ%=Ô%=Ð$gÐ$gÐ$gr)   FN)	r   r   r   r   Ú
ModuleListÚrangeÚnum_hidden_layersÚlayersÚgradient_checkpointingr%   s    `€r(   r   zMLCDEncoder.__init__z  s`   øø€ å‰Œ×ÒÑÔÐØˆŒÝ”mÐ$gÐ$gÐ$gÐ$gÅuÈVÔMeÑGfÔGfÐ$gÑ$gÔ$gÑhÔhˆŒØ&+ˆÔ#Ð#Ð#r)   NÚinputs_embedsrÞ   r¯   rR   r+   c                 óP   — |}| j         D ]} ||||fi |¤Ž}Œt          |¬¦  «        S )a=  
        Args:
            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
                Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
                This is useful if you want more control over how to convert `input_ids` indices into associated vectors
                than the model's internal embedding lookup matrix.
            position_embeddings (`tuple[torch.Tensor, torch.Tensor]`):
                A tuple of two tensors, each of shape `(batch, seq_len, embed_dim)`.
                Represents absolute positional embeddings for the query and key in the attention mechanism.
            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.
                [What are attention masks?](../glossary#attention-mask)
        )Úlast_hidden_state)rý   r   )r&   rÿ   rÞ   r¯   rR   r*   Úencoder_layers          r(   r-   zMLCDEncoder.forward�  sa   € ð, &ˆØ!œ[ð 	ð 	ˆMØ)˜MØØ#Øðð ð ð	ð ˆMˆMõ Ø+ð
ñ 
ô 
ð 	
r)   r   )r.   r/   r0   rn   r   r   r1   r©   rp   r2   r   r   r   r-   r3   r4   s   @r(   rõ   rõ   q  sÀ   ø€ € € € € ðð ð,Ð/ð ,ð ,ð ,ð ,ð ,ð ,ð /3ð	!
ð !
àÔ(ð!
ð # 5¤<°´Ð#=Ô>ð!
ð œ tÑ+ð	!
ð
 Ð+Ô,ð!
ð 
�Ñ	 ð!
ð !
ð !
ð !
ð !
ð !
ð !
ð !
r)   rõ   c                   ó€   ‡ — e Zd ZU eed<   dZdgZdZdZdZ	dZ
dZdZeedœZ ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )ÚMLCDPreTrainedModelr   Úvision_modelræ   TF)r*   Ú
attentionsc                 óŠ  •— t          ¦   «                              |¦  «         | j        j        }t	          |t
          ¦  «        r±| j        j        }t          j        |j        d|j	        dz  |z  ¬¦  «         t          j        |j
        j        |j        j        |z  ¬¦  «         t          j        |j        t          j        |j        j        d         ¦  «                             d¦  «        ¦  «         dS t	          |t&          ¦  «        r»| j        j        }|j	        dz  d|j        j        z  dz  z  |z  }|j	        dz  |z  }t          j        |j        j        |¬¦  «         t          j        |j        j        |¬¦  «         t          j        |j        j        |¬¦  «         t          j        |j        j        |¬¦  «         dS t	          |t2          ¦  «        rˆ| j        j        }|j        j        dz  d|j        j        z  dz  z  |z  }d|j        j        z  dz  |z  }t          j        |j        j        |¬¦  «         t          j        |j        j        |¬¦  «         dS t	          |t:          ¦  «        rL| j        j        }|j        j        |j        j        z  dz  dz  |z  }t          j        |j        d|¬¦  «         dS dS )	zInitialize the weightsrª   rÕ   )ÚmeanÚstd)r	  rg   rz   rJ   N) r   Ú_init_weightsr   Úinitializer_factorr\   rs   ÚinitÚnormal_r€   r{   rƒ   r‘   Úinitializer_rangeÚcopy_rc   r1   rO   r�   r†   rÓ   rü   rÜ   rÚ   rÛ   rÝ   r   r!   r#   r$   ÚMLCDVisionModelrN   Úclass_pos_emb)r&   r«   ÚfactorÚin_proj_stdÚout_proj_stdÚfc_stdÚpos_emb_stdr'   s          €r(   r
  z!MLCDPreTrainedModel._init_weightsµ  s¢  ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ø”Ô/ˆÝ�fÕ2Ñ3Ô3ð 	JØ”[Ô3ˆFÝŒL˜Ô/°c¸vÔ?OÐQUÑ?UÐX^Ñ?^Ð_Ñ_Ô_Ð_ÝŒL˜Ô/Ô6¸F¼MÔ<[Ð^dÑ<dÐeÑeÔeÐeÝŒJ�vÔ*­E¬L¸Ô9LÔ9RÐSUÔ9VÑ,WÔ,W×,^Ò,^Ð_fÑ,gÔ,gÑhÔhÐhÐhÐhÝ˜¥Ñ.Ô.ð 	JØ”[Ô3ˆFØ!Ô+¨TÑ1°q¸6¼=Ô;ZÑ7ZÐ_cÑ6cÑdÐgmÑmˆKØ"Ô,¨dÑ2°fÑ<ˆLÝŒL˜œÔ-°;Ð?Ñ?Ô?Ð?ÝŒL˜œÔ-°;Ð?Ñ?Ô?Ð?ÝŒL˜œÔ-°;Ð?Ñ?Ô?Ð?ÝŒL˜œÔ/°\ÐBÑBÔBÐBÐBÐBÝ˜¥Ñ(Ô(ð 		JØ”[Ô3ˆFØ!œ=Ô4°dÑ:ÀÀFÄMÔDcÑ@cÐhlÑ?lÑmÐpvÑvˆKØ˜&œ-Ô3Ñ3¸Ñ<¸vÑEˆFÝŒL˜œÔ*°Ð7Ñ7Ô7Ð7ÝŒL˜œÔ*°Ð<Ñ<Ô<Ð<Ð<Ð<Ý˜¥Ñ0Ô0ð 	JØ”[Ô3ˆFØ!œ=Ô4¸¼Ô8YÑYÐ]^Ñ^ÐcgÑgÐjpÑpˆKÝŒL˜Ô-°C¸[ÐIÑIÔIÐIÐIÐIð	Jð 	Jr)   )r.   r/   r0   r   Ú__annotations__Úbase_model_prefixÚ_no_split_modulesÚsupports_gradient_checkpointingÚaccepts_loss_kwargsÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendræ   rÓ   Ú_can_record_outputsr1   rq   r
  r3   r4   s   @r(   r  r  ¥  s¨   ø€ € € € € € àÐÐÑØ&ÐØ+Ð,ÐØ&*Ð#ØÐØÐØ€NØÐØ"&Ðà)Ø#ðð Ðð
 €U„]�_„_ðJð Jð Jð Jñ „_ðJð Jð Jð Jð Jr)   r  zN
    The vision model from M_L_C_D without any head or projection on top.
    )Úcustom_introc                   óº   ‡ — e Zd ZU eed<   dZdZdZdefˆ fd„Ze	 e
d¬¦  «        e	 ddej        dz  d	ee         d
eez  fd„¦   «         ¦   «         ¦   «         Zˆ xZS )r  r   r    )Úimagerƒ   c                 óð  •— t          ¦   «                              |¦  «         |j        }t          |¦  «        | _        t          j        ||j        ¬¦  «        | _        t          |¦  «        | _
        t          j        ||j        ¬¦  «        | _        t          |¦  «        | _        t          j        t          j        d|j        |j        z  dz  ¦  «        ¦  «        | _        |                      ¦   «          d S )Nrè   r   rJ   )r   r   r!   rs   r‡   r   rë   rì   Úpre_layrnormrõ   ÚencoderÚpost_layernormr6   Úvision_rotary_embeddingr~   r1   r   rN   r  Ú	post_init)r&   r   r{   r'   s      €r(   r   zMLCDVisionModel.__init__Þ  sÊ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØÔ&ˆ	å.¨vÑ6Ô6ˆŒÝœL¨¸Ô8MÐNÑNÔNˆÔÝ" 6Ñ*Ô*ˆŒÝ œl¨9¸&Ô:OÐPÑPÔPˆÔÝ':¸6Ñ'BÔ'BˆÔ$Ýœ\­%¬+°a¸Ô9KÈvÔOiÑ9iÐmnÑ9nÑ*oÔ*oÑpÔpˆÔØ�ŠÑÔÐÐÐr)   F)Útie_last_hidden_statesNrR   r+   c                 ó*  — |€t          d¦  «        ‚|                      |¦  «        }|                      |¦  «        }|j        d         | j        j        z  }|j        d         | j        j        z  }t          j        ||j        ¬¦  «         	                    d¦  «         
                    d|¦  «        }t          j        ||j        ¬¦  «         	                    d¦  «         
                    |d¦  «        }t          j        |                     ¦   «         |                     ¦   «         gd¬¦  «        }|                      ||¦  «        }	t          j        | j        | j        gd¬¦  «        }
t          j        |
                     ¦   «         |	d         gd¬¦  «        t          j        |
                     ¦   «         |	d         gd¬¦  «        g}	 | j        d||	d	œ|¤Ž}|d         }|dd…ddd…f         }|                      |¦  «        }t)          ||¬
¦  «        S )aû  
        Example:

        ```python
        >>> import httpx
        >>> from io import BytesIO
        >>> from PIL import Image
        >>> from transformers import AutoProcessor, MLCDVisionModel
        >>> model = MLCDVisionModel.from_pretrained("DeepGlint-AI/mlcd-vit-bigG-patch14-448")
        >>> processor = AutoProcessor.from_pretrained("DeepGlint-AI/mlcd-vit-bigG-patch14-448")

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))
        >>> inputs = processor(images=image, return_tensors="pt")

        >>> with torch.no_grad():
        ...     outputs = model(**inputs, output_attentions=True)

        >>> features = outputs.last_hidden_state
        >>> print(f"Extracted features shape: {features.shape}")
        >>> print(f"Number of attention layers: {len(outputs.attentions)}")
        >>> print(f"Attention shape: {outputs.attentions[0].shape}")
        ```Nz You have to specify pixel_valuesrÌ   rg   )r7   r   r   rh   )rÿ   rÞ   )r  Úpooler_outputrñ   )r@   r‡   r%  r�   r   r}   r1   rO   r7   r’   r†   Ústackr¢   r(  ri   r  r_   r`   r&  r'  r	   )r&   r    rR   r*   Únum_patches_heightÚnum_patches_widthÚhpos_idsÚwpos_idsÚpos_idsrÞ   r  Úencoder_outputsr  Úpooled_outputs                 r(   r-   zMLCDVisionModel.forwardê  s/  € ð@ ÐÝÐ?Ñ@Ô@Ð@àŸš¨Ñ5Ô5ˆØ×)Ò)¨-Ñ8Ô8ˆà)Ô/°Ô3°t´{Ô7MÑMÐØ(Ô.¨rÔ2°d´kÔ6LÑLÐåŒLÐ+°LÔ4GÐHÑHÔH×RÒRÐSTÑUÔU×\Ò\Ð]_ÐarÑsÔsð 	õ ŒLÐ*°<Ô3FÐGÑGÔG×QÒQÐRSÑTÔT×[Ò[Ð\nÐprÑsÔsð 	õ ”+˜x×/Ò/Ñ1Ô1°8×3CÒ3CÑ3EÔ3EÐFÈBÐOÑOÔOˆØ"×:Ò:¸=È'ÑRÔRÐÝœ	 4Ô#5°tÔ7IÐ"JÐPRÐSÑSÔSˆåŒI�}×(Ò(Ñ*Ô*Ð,?ÀÔ,BÐCÈÐKÑKÔKÝŒI�}×(Ò(Ñ*Ô*Ð,?ÀÔ,BÐCÈÐKÑKÔKð
Ðð
 '˜$œ,ð 
Ø'Ø 3ð
ð 
ð ð
ð 
ˆð ,¨AÔ.ÐØ)¨!¨!¨!¨Q°°°¨'Ô2ˆØ×+Ò+¨MÑ:Ô:ˆå)Ø/Ø'ð
ñ 
ô 
ð 	
r)   r   )r.   r/   r0   r   r  Úmain_input_nameÚinput_modalitiesÚ_input_embed_layerr   r   r   r   r1   r©   r   r   rp   r	   r-   r3   r4   s   @r(   r  r  Ó  sï   ø€ € € € € € ð ÐÐÑØ$€OØ!ÐØ*Ðð
Ð/ð 
ð 
ð 
ð 
ð 
ð 
ð  Ø€_¨EÐ2Ñ2Ô2Øð 26ð@
ð @
àÔ'¨$Ñ.ð@
ð Ð+Ô,ð@
ð 
Ð+Ñ	+ð	@
ð @
ð @
ñ „^ñ 3Ô2ñ  Ôð@
ð @
ð @
ð @
ð @
r)   r  )rª   )3Úcollections.abcr   r1   Útorch.nnr   Ú r   r  Úactivationsr   Úmodeling_layersr   Úmodeling_outputsr   r	   Úmodeling_utilsr
   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_mlcdr   ÚModuler   r6   rs   r2   rP   r¿   rÃ   r¨   rµ   rp   rÑ   rÓ   ræ   rõ   r  r  Ú__all__rñ   r)   r(   ú<module>rG     sñ  ðð( %Ð $Ð $Ð $Ð $Ð $à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø BÐ BÐ BÐ BÐ BÐ BÐ BÐ BÐ BÐ BØ 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0ðð ð ð ð ˆbŒiñ ô ð ð=5ð =5ð =5ð =5ð =5˜"œ)ñ =5ô =5ð =5ð@Ið Ið Ið Ið I˜2œ9ñ Iô Ið Iðf ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð2(ð (ð (ð	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	UðØ„|ðØœðØ+0¬<ðØ>C¼lðà
ˆ5Œ<˜œÐ%Ô&ðð ð ð ðE)ð E)ð E)ð E)ð E)�B”Iñ E)ô E)ð E)ðP+ð +ð +ð +ð +Ð1ñ +ô +ð +ð\1
ð 1
ð 1
ð 1
ð 1
�"”)ñ 1
ô 1
ð 1
ðh ð*Jð *Jð *Jð *Jð *J˜/ñ *Jô *Jñ „ð*JðZ €ððñ ô ð
U
ð U
ð U
ð U
ð U
Ð)ñ U
ô U
ñô ð
U
ðp !Ð"3Ð
4€€€r)   