§
    9²¸j•á  ã                   óæ  — d dl mZ d dlmZ d dlZd dlmZ d dlmc mZ	 ddl
mZ ddlmZ ddlmZmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZmZmZmZ ddlmZm Z  ddl!m"Z"m#Z# ddl$m%Z% ddl&m'Z'm(Z(m)Z)m*Z*m+Z+ ddl,m-Z- ddl.m/Z/m0Z0 ddl1m2Z2 ddl3m4Z4 ddl5m6Z6m7Z7m8Z8  G d„ dej9        ¦  «        Z: G d„ dej9        ¦  «        Z;dej<        de=dej<        fd„Z>	 dZdej9        d ej<        d!ej<        d"ej<        d#ej<        dz  d$e?d%e?d&e%e'         fd'„Z@d(„ ZAd)ej<        d*ej<        d+ej<        d,ej<        deBej<        ej<        f         f
d-„ZC G d.„ d/ej9        ¦  «        ZD G d0„ d1e¦  «        ZE G d2„ d3ej9        ¦  «        ZFe( G d4„ d5e#¦  «        ¦   «         ZGe( G d6„ d7eG¦  «        ¦   «         ZH G d8„ d9ej9        ¦  «        ZI G d:„ d;ej9        ¦  «        ZJ G d<„ d=ej9        ¦  «        ZK G d>„ d?ej9        ¦  «        ZL G d@„ dAej9        ¦  «        ZM G dB„ dCej9        ¦  «        ZNd[dD„ZO G dE„ dFej9        ¦  «        ZP G dG„ dHe¦  «        ZQdIe=dee=e=e=e=geRf         fdJ„ZSe( G dK„ dLeG¦  «        ¦   «         ZT e(dM¬N¦  «        e G dO„ dPe¦  «        ¦   «         ¦   «         ZU e(dQ¬N¦  «         G dR„ dSeG¦  «        ¦   «         ZV e(dT¬N¦  «        e G dU„ dVe¦  «        ¦   «         ¦   «         ZWe( G dW„ dXeGe¦  «        ¦   «         ZXg dY¢ZYdS )\é    )ÚCallable)Ú	dataclassNé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚBaseModelOutputWithPastÚBaseModelOutputWithPoolingÚModelOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚtorch_compilable_checkÚ	torch_int)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputs)Úget_vision_position_idsé   )ÚStep3p7ConfigÚStep3p7TextConfigÚStep3p7VisionConfigc            	       óê   ‡ — e Zd ZdZ edd¬¦  «        ddefˆ fd„¦   «         Ze edd¬¦  «        	 ddedee	j
        ef         fd	„¦   «         ¦   «         Z e	j        ¦   «         d
„ ¦   «         Zd„ Zˆ xZS )ÚStep3p7VisionRotaryEmbeddingzý
    Simple axial 2D rope with same freqs used for H and W grids. The freqs are
    pre-computed using `head-dim//4` which is later used to concat H and W positions.
    The final angles rotate over the whole head dim, no partial rotation involved.
    Údeviceú5.18©ÚversionNÚconfigc                 ó°  •— t          ¦   «                              ¦   «          || _        | j        j        d         | _        | j        }| j        dk    r#t          | j        j        › d| j        › �¦  «        ‚ || j        |¦  «        \  }| _	        t          j        |d¬¦  «        | _        t          j        |                     ¦   «         d¬¦  «        | _        d S )NÚ	rope_typeÚaxialz) supports only axial rope, but requested F©Ú
persistent)ÚsuperÚ__init__r+   Úrope_parametersr-   Úcompute_axial_rope_parametersÚ
ValueErrorÚ	__class__Ú__name__Úattention_scalingÚnnÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)Úselfr+   r'   Úrope_init_fnr;   r6   s        €ú|/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/step3p7/modeling_step3p7.pyr2   z%Step3p7VisionRotaryEmbedding.__init__5   sÃ   ø€ å‰Œ×ÒÑÔÐØˆŒàœÔ4°[ÔAˆŒØ!%Ô!CˆØŒ>˜WÒ$Ð$Ý ¤Ô 7ÐrÐrÐbfÔbpÐrÐrÑsÔsÐsØ+7¨<¸¼ÀVÑ+LÔ+LÑ(ˆ�$Ô(åœ	 (°uÐ=Ñ=Ô=ˆŒÝ!#¤¨8¯>ª>Ñ+;Ô+;ÈÐ!NÑ!NÔ!NˆÔÐÐó    Úreturnc                 óî   — | j         d         }t          | dd¦  «        p| j        | j        z  }|dz  }d}d|t	          j        d|dt          j        ¬¦  «        |z  z  z  }|                     |¦  «        |fS )a¾  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNé   ç      ð?r   ©Údtype)r3   ÚgetattrÚhidden_sizeÚnum_attention_headsÚtorchÚarangeÚfloatÚto)r+   r'   ÚkwargsÚbaseÚdimÚspatial_dimÚattention_factorr;   s           r@   r4   z:Step3p7VisionRotaryEmbedding.compute_axial_rope_parametersC   sƒ   € ð Ô% lÔ3ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆØ˜Q‘hˆàÐØ˜$¥5¤<°°;ÀÍÌÐ#UÑ#UÔ#UÐXcÑ#cÑdÑeˆØ�{Š{˜6Ñ"Ô"Ð$4Ð4Ð4rA   c                 ó  — |d                               ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  || j                              ¦   «         z  }|                     ¦   «         | j        z  }| 	                    ¦   «         | j        z  }d d d ¦  «         n# 1 swxY w Y   |  
                    |¦  «        }|  
                    |¦  «        }||fS )N).NÚmpsÚcpuF©Údevice_typeÚenabled)rO   Ú
isinstancer'   ÚtypeÚstrr   r;   Úcosr8   ÚsinÚrecomposition_frequencies)r>   ÚxÚposition_idsÚposition_ids_expandedrZ   Úfreqsr_   r`   s           r@   Úforwardz$Step3p7VisionRotaryEmbedding.forwardY   s&  € ð !-¨YÔ 7× =Ò =Ñ ?Ô ?ÐÝ'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆÝ¨¸UÐCÑCÔCð 	7ð 	7Ø)¨D¬M×,?Ò,?Ñ,AÔ,AÑAˆEØ—)’)‘+”+ Ô 6Ñ6ˆCØ—)’)‘+”+ Ô 6Ñ6ˆCð	7ð 	7ð 	7ñ 	7ô 	7ð 	7ð 	7ð 	7ð 	7ð 	7ð 	7øøøð 	7ð 	7ð 	7ð 	7ð
 ×,Ò,¨SÑ1Ô1ˆØ×,Ò,¨SÑ1Ô1ˆØ�Cˆxˆs   Á)AC
Ã
CÃCc                 ó\   — |                      d¦  «        }t          j        ||fd¬¦  «        S )z]
        Recompose the frequencies into the final spatial layout used per each grid.
        r!   éÿÿÿÿ©rS   )ÚflattenrM   Úcat)r>   ÚfreqÚfreq_hws      r@   ra   z6Step3p7VisionRotaryEmbedding.recomposition_frequenciesg   s-   € ð —,’,˜q‘/”/ˆÝŒy˜' 7Ð+°Ð4Ñ4Ô4Ð4rA   ©N)r7   Ú
__module__Ú__qualname__Ú__doc__r   r$   r2   ÚstaticmethodÚtuplerM   ÚTensorrO   r4   Úno_gradrf   ra   Ú__classcell__©r6   s   @r@   r&   r&   .   s  ø€ € € € € ðð ð €_�X vÐ.Ñ.Ô.ðOð OÐ2ð Oð Oð Oð Oð Oñ /Ô.ðOð Ø€_�X vÐ.Ñ.Ô.à,0ð5ð 5Ø#ð5à	ˆuŒ|˜UÐ"Ô	#ð5ð 5ð 5ñ /Ô.ñ „\ð5ð( €U„]�_„_ðð ñ „_ðð5ð 5ð 5ð 5ð 5ð 5ð 5rA   r&   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚStep3p7VisionMLPc                 ó  •— t          ¦   «                              ¦   «          || _        t          |j                 | _        t          j        |j        |j	        ¦  «        | _
        t          j        |j	        |j        ¦  «        | _        d S rn   )r1   r2   r+   r   Ú
hidden_actÚactivation_fnr9   ÚLinearrK   Úintermediate_sizeÚfc1Úfc2©r>   r+   r6   s     €r@   r2   zStep3p7VisionMLP.__init__p   sf   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ# FÔ$5Ô6ˆÔÝ”9˜VÔ/°Ô1IÑJÔJˆŒÝ”9˜VÔ5°vÔ7IÑJÔJˆŒˆˆrA   Úhidden_statesrB   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S rn   )r   r|   r€   )r>   r‚   s     r@   rf   zStep3p7VisionMLP.forwardw   s=   € ØŸš Ñ/Ô/ˆØ×*Ò*¨=Ñ9Ô9ˆØŸš Ñ/Ô/ˆØÐrA   ©r7   ro   rp   r2   rM   rt   rf   rv   rw   s   @r@   ry   ry   o   sc   ø€ € € € € ðKð Kð Kð Kð Kð U¤\ð °e´lð ð ð ð ð ð ð ð rA   ry   r‚   Ún_reprB   c                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r!   N)ÚshapeÚexpandÚreshape)r‚   r…   ÚbatchÚnum_key_value_headsÚslenrE   s         r@   Ú	repeat_kvr�   ~   s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTrA   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutrQ   c                 ó  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        |z  }
|�|
|z   }
t
          j                             |
dt          j        ¬¦  «         	                    |j
        ¦  «        }
t
          j                             |
|| j        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «                             ¦   «         }||
fS )NrF   r   rh   )rS   rI   )ÚpÚtrainingr!   )r�   Únum_key_value_groupsrM   ÚmatmulÚ	transposer9   Ú
functionalÚsoftmaxÚfloat32rP   rI   r•   r˜   Ú
contiguous)r�   r�   r‘   r’   r“   r”   r•   rQ   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r@   Úeager_attention_forwardr¤   Š   sé   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$rA   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..Nrh   rF   ri   )r‡   rM   rk   )rb   Úx1Úx2s      r@   Úrotate_halfr¨   £   s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'rA   ÚqÚkr_   r`   c                 óÆ  — | j         }|j         }|                      ¦   «         |                     ¦   «         }} |                     d¦  «                             ¦   «         |                     d¦  «                             ¦   «         }}| |z  t          | ¦  «        |z  z   }||z  t          |¦  «        |z  z   }|                     |¦  «        }|                     |¦  «        }||fS )Néþÿÿÿ)rI   rO   Ú	unsqueezer¨   rP   )r©   rª   r_   r`   Úorig_q_dtypeÚorig_k_dtypeÚq_embedÚk_embeds           r@   Úapply_rotary_pos_emb_visionr²   ª   sÃ   € ð ”7€LØ”7€LØ�7Š7‰9Œ9�a—g’g‘i”i€q€AØ�}Š}˜RÑ Ô ×&Ò&Ñ(Ô(¨#¯-ª-¸Ñ*;Ô*;×*AÒ*AÑ*CÔ*Cˆ€CØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�jŠj˜Ñ&Ô&€GØ�jŠj˜Ñ&Ô&€GØ�GÐÐrA   c                   ó¼   ‡ — e Zd ZdZdefˆ fd„Z	 ddej        deej        ej        f         dej        dz  de	e
         d	eej        ej        dz  f         f
d
„Zˆ xZS )ÚStep3p7VisionAttentionzËCLIP-style vision attention; the only difference from [`CLIPAttention`] is
    that queries and keys are rotated by the tower's 3D RoPE before the
    (interface-dispatched) scaled dot-product attention.r+   c                 ó   •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        | j        | j        z  | _        | j        dz  | _        |j	        | _
        d| _        t          j        | j        | j        ¦  «        | _        t          j        | j        | j        ¦  «        | _        t          j        | j        | j        ¦  «        | _        t          j        | j        | j        ¦  «        | _        d| _        d S )Nç      à¿Fr!   )r1   r2   r+   rK   Ú	embed_dimrL   Ú	num_headsrE   ÚscaleÚattention_dropoutr•   Ú	is_causalr9   r}   Úk_projÚv_projÚq_projÚout_projr™   r�   s     €r@   r2   zStep3p7VisionAttention.__init__½   sÏ   ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ+ˆŒØÔ3ˆŒØœ¨$¬.Ñ8ˆŒØ”] DÑ(ˆŒ
ØÔ/ˆŒØˆŒå”i ¤°´Ñ?Ô?ˆŒÝ”i ¤°´Ñ?Ô?ˆŒÝ”i ¤°´Ñ?Ô?ˆŒÝœ	 $¤.°$´.ÑAÔAˆŒð %&ˆÔ!Ð!Ð!rA   Nr‚   Úposition_embeddingsr“   rQ   rB   c                 óä  — |j         dd…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «        }|                      |¦  «                             |¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }	|\  }
}t          |||
|¦  «        \  }}|                     dd¦  «        |                     dd¦  «        }}t          j	        | j
        j        t          ¦  «        } || |||	|f| j        | j        sdn| j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        |fS )z#Input shape: Batch x Time x ChannelNrh   r!   rF   rŽ   )r”   r•   )r‡   rE   r¾   Úviewr¼   r½   r›   r²   r   Úget_interfacer+   Ú_attn_implementationr¤   r¹   r˜   r•   r‰   rŸ   r¿   )r>   r‚   rÀ   r“   rQ   Úinput_shapeÚhidden_shapeÚqueriesÚkeysÚvaluesr_   r`   Úattention_interfacer£   r¢   s                  r@   rf   zStep3p7VisionAttention.forwardÏ   s•  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—+’+˜mÑ,Ô,×1Ò1°,Ñ?Ô?ˆØ�{Š{˜=Ñ)Ô)×.Ò.¨|Ñ<Ô<ˆØ—’˜]Ñ+Ô+×0Ò0°Ñ>Ô>×HÒHÈÈAÑNÔNˆà&‰ˆˆSÝ3°G¸TÀ3ÈÑLÔL‰ˆ�Ø×)Ò)¨!¨QÑ/Ô/°·²ÀÀ1Ñ1EÔ1E�ˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð ”JØ#œ}Ð>�C�C°$´,ð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ�}Š}˜[Ñ)Ô)¨<Ð7Ð7rA   rn   )r7   ro   rp   rq   r$   r2   rM   rt   rs   r   r   rf   rv   rw   s   @r@   r´   r´   ¸   sÍ   ø€ € € € € ð<ð <ð&Ð2ð &ð &ð &ð &ð &ð &ð, /3ð	!8ð !8à”|ð!8ð # 5¤<°´Ð#=Ô>ð!8ð œ tÑ+ð	!8ð
 Ð+Ô,ð!8ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð!8ð !8ð !8ð !8ð !8ð !8ð !8ð !8rA   r´   c            	       ó�   ‡ — e Zd Zdefˆ fd„Z	 	 d	dej        dej        dz  deej        ej        f         dz  dej        fd„Zˆ xZ	S )
ÚStep3p7VisionEncoderLayerr+   c                 ó  •— t          ¦   «                              ¦   «          || _        t          |¦  «        | _        t          |¦  «        | _        t          j        |j	        |j
        ¬¦  «        | _        t          j        |j	        |j
        ¬¦  «        | _        t          j        |j        t          j        |j	        ¦  «        z  ¦  «        | _        t          j        |j        t          j        |j	        ¦  «        z  ¦  «        | _        d S )N©Úeps)r1   r2   r+   r´   Ú	self_attnry   Úmlpr9   Ú	LayerNormrK   Úlayer_norm_epsÚlayernorm_beforeÚlayernorm_afterÚ	ParameterÚlayer_scale_init_valuerM   ÚonesÚlambda_1Úlambda_2r�   s     €r@   r2   z"Step3p7VisionEncoderLayer.__init__ô   sÊ   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ/°Ñ7Ô7ˆŒÝ# FÑ+Ô+ˆŒÝ "¤¨VÔ-?ÀVÔEZÐ [Ñ [Ô [ˆÔÝ!œ|¨FÔ,>ÀFÔDYÐZÑZÔZˆÔÝœ VÔ%BÅUÄZÐPVÔPbÑEcÔEcÑ%cÑdÔdˆŒÝœ VÔ%BÅUÄZÐPVÔPbÑEcÔEcÑ%cÑdÔdˆŒˆˆrA   Nr‚   r“   rÀ   rB   c                 óð   — |}|                       |¦  «        } | j        |f||dœ|¤Ž\  }}| j        |z  }||z   }|}|                      |¦  «        }|                      |¦  «        }| j        |z  }||z   }|S )N)r“   rÀ   )rÔ   rÐ   rÙ   rÕ   rÑ   rÚ   )r>   r‚   r“   rÀ   rQ   ÚresidualÚ_s          r@   rf   z!Step3p7VisionEncoderLayer.forwardþ   s®   € ð !ˆØ×-Ò-¨mÑ<Ô<ˆØ)˜4œ>Øð
à)Ø 3ð
ð 
ð ð	
ð 
Ñˆ�qð œ¨Ñ5ˆØ%¨Ñ0ˆà ˆØ×,Ò,¨]Ñ;Ô;ˆØŸš Ñ/Ô/ˆØœ¨Ñ5ˆØ%¨Ñ0ˆØÐrA   ©NN)
r7   ro   rp   r$   r2   rM   rt   rs   rf   rv   rw   s   @r@   rÌ   rÌ   ó   s±   ø€ € € € € ðeÐ2ð eð eð eð eð eð eð /3ØHLð	ð à”|ðð œ tÑ+ðð # 5¤<°´Ð#=Ô>ÀÑEð	ð 
Œðð ð ð ð ð ð ð rA   rÌ   c                   óv   ‡ — e Zd Zdefˆ fd„Zdej        dededej        fd„Zdd	ej	        dej        fd
„Z
ˆ xZS )ÚStep3p7VisionEmbeddingsr+   c                 ó  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        |j        | _        t          j        |j	        |j        |j        |j        d¬¦  «        | _
        | j        | j        z  dz  | _        | j        | _        t          j        | j        | j        ¦  «        | _        t          j        t!          j        | j        ¦  «                             d¦  «        d¬¦  «        | _        d S )NF)Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚbiasrF   ©r!   rh   r/   )r1   r2   r+   rK   r·   Ú
image_sizeÚ
patch_sizer9   ÚConv2dÚnum_channelsÚpatch_embeddingÚnum_patchesÚnum_positionsÚ	EmbeddingÚposition_embeddingr:   rM   rN   rˆ   rc   r�   s     €r@   r2   z Step3p7VisionEmbeddings.__init__  sç   ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ+ˆŒØ Ô+ˆŒØ Ô+ˆŒÝ!œyØÔ+ØÔ+ØÔ)ØÔ$Øð 
ñ  
ô  
ˆÔð !œO¨t¬Ñ>À1ÑDˆÔØ!Ô-ˆÔÝ"$¤,¨tÔ/AÀ4Ä>Ñ"RÔ"RˆÔÝœI¥e¤l°4Ô3EÑ&FÔ&F×&MÒ&MÈgÑ&VÔ&VÐchÐiÑiÔiˆÔÐÐrA   Ú
embeddingsÚheightÚwidthrB   c                 ón  — | j         j        j        d         }|| j        z  }|| j        z  }t	          |dz  ¦  «        }t
          j                             ¦   «         s+||k    r%||k    r| j         j                             d¦  «        S | j         j                             d¦  «        }|j        d         }	| 	                    d|||	¦  «        }| 
                    dddd¦  «        }t          j                             |||fdd¬	¦  «        }| 
                    dddd¦  «                             dd|	¦  «        }|S )
a  
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing and no class embeddings.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   g      à?rh   r!   r   rF   ÚbilinearF)ÚsizeÚmodeÚalign_corners)rð   Úweightr‡   ré   r   rM   ÚjitÚ
is_tracingr­   r‰   Úpermuter9   rœ   ÚinterpolaterÂ   )
r>   rñ   rò   ró   rî   Ú
new_heightÚ	new_widthÚsqrt_num_positionsÚpatch_pos_embedrS   s
             r@   Úinterpolate_pos_encodingz0Step3p7VisionEmbeddings.interpolate_pos_encoding,  sC  € ð Ô/Ô6Ô<¸QÔ?ˆØ˜tœÑ.ˆ
Ø˜Tœ_Ñ,ˆ	Ý& }°cÑ'9Ñ:Ô:ÐÝŒy×#Ò#Ñ%Ô%ð 	?¨*Ð8JÒ*JÐ*JÈyÐ\nÒOnÐOnØÔ*Ô1×;Ò;¸AÑ>Ô>Ð>ØÔ1Ô8×BÒBÀ1ÑEÔEˆØÔ˜rÔ"ˆØ)×1Ò1°!Ð5GÐI[Ð]`ÑaÔaˆØ)×1Ò1°!°Q¸¸1Ñ=Ô=ˆÝœ-×3Ò3ØØ˜iÐ(ØØð	 4ñ 
ô 
ˆð *×1Ò1°!°Q¸¸1Ñ=Ô=×BÒBÀ1ÀbÈ#ÑNÔNˆØÐrA   FÚpixel_valuesc                 óX  — |j         \  }}}}| j        j        j        }|                      |                     |¬¦  «        ¦  «        }|                     d¦  «                             dd¦  «        }|r||                      |||¦  «        z   }n||                      | j	        ¦  «        z   }|S )NrH   rF   r!   )
r‡   rì   rù   rI   rP   rj   r›   r  rð   rc   )	r>   r  r  rÝ   rò   ró   Útarget_dtypeÚpatch_embedsrñ   s	            r@   rf   zStep3p7VisionEmbeddings.forwardH  s¯   € Ø*Ô0Ñˆˆ1ˆf�eØÔ+Ô2Ô8ˆØ×+Ò+¨L¯OªOÀ,¨OÑ,OÔ,OÑPÔPˆØ!×)Ò)¨!Ñ,Ô,×6Ò6°q¸!Ñ<Ô<ˆ
à#ð 	QØ# d×&CÒ&CÀJÐPVÐX]Ñ&^Ô&^Ñ^ˆJˆJà# d×&=Ò&=¸dÔ>OÑ&PÔ&PÑPˆJØÐrA   ©F)r7   ro   rp   r$   r2   rM   rt   Úintr  ÚFloatTensorrf   rv   rw   s   @r@   rà   rà     s±   ø€ € € € € ðjÐ2ð jð jð jð jð jð jð&°5´<ð Èð ÐUXð Ð]bÔ]ið ð ð ð ð8
ð 
 EÔ$5ð 
ÐZ_ÔZfð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
rA   rà   c                   ó~   ‡ — e Zd ZU eed<   dZdZddgZdgZdZ	dZ
dZdZdZ ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )	ÚStep3p7PreTrainedModelr+   ÚmodelTrÌ   ÚStep3p7DecoderLayerÚpast_key_valuesFc                 ó<  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rFt	          j        |j        t          j        |j	        ¦  «         
                    d¦  «        ¦  «         d S t          |t          ¦  «        r`t          j                             |j        |j        j        ¦  «         t          j                             |j        |j        j        ¦  «         d S t          |t$          ¦  «        r�|j        D ]“}|j        }|j        |         dk    rt,          |j        |                  } ||j        |¬¦  «        \  }}t	          j        t/          ||› d�¦  «        |¦  «         t	          j        t/          ||› d�¦  «        |¦  «         Œ”d S t          |t0          ¦  «        rPt/          | j        dd¦  «        }t	          j        |j        d|¬	¦  «         t	          j        |j        d|¬	¦  «         d S t          |t8          ¦  «        rMt/          | j        dd¦  «        }t	          j        |j        d|¬	¦  «         t	          j        |j        ¦  «         d S t          |t@          ¦  «        rt	          j        |j        ¦  «         d S d S )
Nrç   Údefault©Ú
layer_typeÚ	_inv_freqÚ_original_inv_freqÚinitializer_rangeg{®Gáz”?rŽ   )ÚmeanÚstd)!r1   Ú_init_weightsr\   rà   ÚinitÚcopy_rc   rM   rN   rî   rˆ   rÌ   r9   Ú	constant_rÙ   r+   r×   rÚ   ÚStep3p7RotaryEmbeddingÚlayer_typesÚcompute_default_rope_parametersr-   r   rJ   ÚStep3p7ExpertsÚnormal_Úgate_up_projÚ	down_projÚStep3p7TopKRouterrù   Úzeros_Úe_score_correction_biasÚStep3p7RMSNorm)r>   r�   r  r?   Úcurr_inv_freqrÝ   r  r6   s          €r@   r  z$Step3p7PreTrainedModel._init_weightsb  su  ø€ å‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ5Ñ6Ô6ð 	'ÝŒJ�vÔ*­E¬L¸Ô9MÑ,NÔ,N×,UÒ,UÐV]Ñ,^Ô,^Ñ_Ô_Ð_Ð_Ð_Ý˜Õ 9Ñ:Ô:ð 	'ÝŒG×Ò˜fœo¨v¬}Ô/SÑTÔTÐTÝŒG×Ò˜fœo¨v¬}Ô/SÑTÔTÐTÐTÐTÝ˜Õ 6Ñ7Ô7ð 	'Ø$Ô0ð ^ð ^�
Ø%ÔE�ØÔ# JÔ/°9Ò<Ð<Ý#6°vÔ7GÈ
Ô7SÔ#T�LØ#/ <°´È*Ð#UÑ#UÔ#UÑ �˜qÝ”
�7 6¨jÐ+CÐ+CÐ+CÑDÔDÀmÑTÔTÐTÝ”
�7 6¨jÐ+LÐ+LÐ+LÑMÔMÈ}Ñ]Ô]Ð]Ð]ð^ð ^õ ˜¥Ñ/Ô/ð 	'õ ˜$œ+Ð':¸DÑAÔAˆCÝŒL˜Ô,°3¸CÐ@Ñ@Ô@Ð@ÝŒL˜Ô)°¸Ð=Ñ=Ô=Ð=Ð=Ð=Ý˜Õ 1Ñ2Ô2ð 	'Ý˜$œ+Ð':¸DÑAÔAˆCÝŒL˜œ¨S°cÐ:Ñ:Ô:Ð:ÝŒK˜Ô6Ñ7Ô7Ð7Ð7Ð7Ý˜¥Ñ/Ô/ð 	'ÝŒK˜œÑ&Ô&Ð&Ð&Ð&ð	'ð 	'rA   )r7   ro   rp   r"   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_static_cacheÚ_supports_attention_backendrM   ru   r  rv   rw   s   @r@   r  r  U  s–   ø€ € € € € € àÐÐÑØÐØ&*Ð#Ø4Ð6KÐLÐØ#4Ð"5ÐØ ÐØ€NØÐØ!ÐØ"&Ðà€U„]�_„_ð'ð 'ð 'ð 'ñ „_ð'ð 'ð 'ð 'ð 'rA   r  c                   óœ   ‡ — e Zd ZU dZeed<   eedœZdefˆ fd„Z	 e
d¬¦  «        edej        dee         d	efd
„¦   «         ¦   «         Zˆ xZS )ÚStep3p7VisionModelu1  Vision encoder: patch embeddings â†’ 2-D RoPE transformer layers â†’ conv downsampler.

    The rotary embedding (``self.rotary_emb``) and layer stack (``self.layers``) are
    held directly on this module, following the Gemma4 convention of not wrapping
    them in a separate ``Encoder`` submodule.
    r+   ©r‚   Ú
attentionsc                 ó.  •‡— t          ¦   «                              ‰¦  «         t          ‰¦  «        | _        t	          j        ‰j        ‰j        ¬¦  «        | _        t          ‰¦  «        | _
        t	          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t	          j        ‰j        ‰j        dz  ddd¬¦  «        | _        t	          j        ‰j        dz  ‰j        dz  ddd¬¦  «        | _        |                      ¦   «          d S )NrÎ   c                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS © )rÌ   )Ú.0rÝ   r+   s     €r@   ú
<listcomp>z/Step3p7VisionModel.__init__.<locals>.<listcomp>•  s"   ø€ Ð$pÐ$pÐ$pÈ1Õ%>¸vÑ%FÔ%FÐ$pÐ$pÐ$prA   rF   r   r!   )rä   rå   Úpaddingé   )r1   r2   rà   rñ   r9   rÒ   rK   rÓ   Úpre_layernormr&   Ú
rotary_embÚ
ModuleListÚrangeÚnum_hidden_layersÚlayersrê   Údownsampler1Údownsampler2Ú	post_initr�   s    `€r@   r2   zStep3p7VisionModel.__init__�  s  øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý1°&Ñ9Ô9ˆŒÝœ\¨&Ô*<À&ÔBWÐXÑXÔXˆÔÝ6°vÑ>Ô>ˆŒÝ”mÐ$pÐ$pÐ$pÐ$pÕPUÐV\ÔVnÑPoÔPoÐ$pÑ$pÔ$pÑqÔqˆŒÝœI fÔ&8¸&Ô:LÈqÑ:PÐ^_ÐhiÐstÐuÑuÔuˆÔÝœIØÔ Ñ" FÔ$6¸Ñ$:ÈÐRSÐ]^ð
ñ 
ô 
ˆÔð 	�ŠÑÔÐÐÐrA   F)Útie_last_hidden_statesr  rQ   rB   c                 óœ  — |j         \  }}}}|| j        j        z  }|| j        j        z  }|                      |d¬¦  «        }	|                      |	¦  «        }	t	          j        d||gg|	j        ¬¦  «        }
t          |
d¬¦  «        }|                      |	|¦  «        }| j	        D ]} ||	fd|i|¤Ž}	Œ|	j         d         }|	 
                    dd	d¦  «                             ||||¦  «        }	|                      |	¦  «        }	|                      |	¦  «        }	t          |	                     d	¦  «         
                    dd	d¦  «        ¬
¦  «        S )NT)r  r!   ©r'   )Úspatial_merge_sizerÀ   rh   r   rF   )Úlast_hidden_state)r‡   rñ   ré   r=  rM   Útensorr'   r    r>  rB  rü   rÂ   rC  rD  r   rj   )r>   r  rQ   Ú
batch_sizerÝ   rò   ró   Úgrid_hÚgrid_wÚhidden_stateÚgrid_thwrc   rÀ   ÚlayerÚchannelss                  r@   rf   zStep3p7VisionModel.forwardœ  sk  € ð (4Ô'9Ñ$ˆ
�A�v˜uØ˜4œ?Ô5Ñ5ˆØ˜$œ/Ô4Ñ4ˆØ—’ |Èd�ÑSÔSˆØ×)Ò)¨,Ñ7Ô7ˆõ ”< ! V¨VÐ!4Ð 5¸lÔ>QÐRÑRÔRˆÝ.¨xÈAÐNÑNÔNˆØ"Ÿošo¨l¸LÑIÔIÐØ”[ð 	bð 	bˆEØ ˜5 ÐaÐaÐCVÐaÐZ`ÐaÐaˆLˆLàÔ% bÔ)ˆØ#×+Ò+¨A¨q°!Ñ4Ô4×9Ò9¸*ÀhÐPVÐX^Ñ_Ô_ˆØ×(Ò(¨Ñ6Ô6ˆØ×(Ò(¨Ñ6Ô6ˆÝ°×1EÒ1EÀaÑ1HÔ1H×1PÒ1PÐQRÐTUÐWXÑ1YÔ1YÐZÑZÔZÐZrA   )r7   ro   rp   rq   r$   r(  rÌ   r´   Ú_can_record_outputsr2   r   r   rM   rt   r   r   r   rf   rv   rw   s   @r@   r3  r3  �  sÓ   ø€ € € € € € ðð ð  ÐÐÑà2Ø,ðð Ðð

Ð2ð 
ð 
ð 
ð 
ð 
ð 
ð €_¨EÐ2Ñ2Ô2Øð[ E¤Lð [¸FÐCUÔ<Vð [Ð[jð [ð [ð [ñ „^ñ 3Ô2ð[ð [ð [ð [ð [rA   r3  c                   óú   ‡ — e Zd Z edd¬¦  «        ddefˆ fd„¦   «         Ze edd¬¦  «        	 ddededz  dee	j
        ef         fd	„¦   «         ¦   «         Z e	j        ¦   «         ed
„ ¦   «         ¦   «         Zˆ xZS )r  r'   r(   r)   Nr+   c           	      ó¶  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        t          t          |j        ¦  «        ¦  «        | _        i | _	        | j        D ]ã}| j        j
        |         }|€Œ|d         | j	        |<   | j        }| j	        |         dk    rt          | j	        |                  } || j        ||¬¦  «        \  }}t          | |› d�t          j        |d¬¦  «        ¦  «         t          | |› d�t          j        |                     ¦   «         d¬¦  «        ¦  «         t          | |› d�|¦  «         Œäd S )	Nr-   r  r  r  Fr/   r  Ú_attention_scaling)r1   r2   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr+   ÚsortedÚsetr  r-   r3   r  r   Úsetattrr9   r:   r<   )	r>   r+   r'   r  Úrope_paramsr?   r'  Úcurr_attention_scalingr6   s	           €r@   r2   zStep3p7RotaryEmbedding.__init__µ  sl  ø€ å‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!ØˆŒÝ!¥# fÔ&8Ñ"9Ô"9Ñ:Ô:ˆÔØˆŒØÔ*ð 	Uð 	UˆJØœ+Ô5°jÔAˆKØÐ"Øà)4°[Ô)AˆDŒN˜:Ñ&Ø%)Ô%IˆLØŒ~˜jÔ)¨YÒ6Ð6Ý2°4´>À*Ô3MÔN�Ø4@°LÀÄÈfÐakÐ4lÑ4lÔ4lÑ1ˆMÐ1Ý�D˜ZÐ2Ð2Ð2µB´I¸mÐX]Ð4^Ñ4^Ô4^Ñ_Ô_Ð_Ý�D˜ZÐ;Ð;Ð;½R¼YÀ}×GZÒGZÑG\ÔG\ÐinÐ=oÑ=oÔ=oÑpÔpÐpÝ�D˜ZÐ;Ð;Ð;Ð=SÑTÔTÐTÐTð	Uð 	UrA   r  rB   c                 óV  — | j         |         d         }| j         |                              dd¦  «        }t          | dd¦  «        p| j        | j        z  }t          ||z  ¦  «        }d}d|t          j        d|dt          j        ¬¦  «        |z  z  z  }	|	 	                    |¦  «        |fS )	a‘  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            layer_type (`str`, *optional*):
                The current layer type if the model has different RoPE parameters per type.
                Should not be used unless `config.layer_types is not None`
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        rD   Úpartial_rotary_factorrG   rE   Nr   rF   rH   )
r3   ÚgetrJ   rK   rL   r  rM   rN   rO   rP   )
r+   r'   r  rQ   rR   r`  rE   rS   rU   r;   s
             r@   r  z6Step3p7RotaryEmbedding.compute_default_rope_parametersË  s´   € ð" Ô% jÔ1°,Ô?ˆà &Ô 6°zÔ B× FÒ FÐG^Ð`cÑ dÔ dÐÝ˜6 :¨tÑ4Ô4Ðh¸Ô8JÈfÔNhÑ8hˆÝ�(Ð2Ñ2Ñ3Ô3ˆàÐð ˜$¥5¤<°°3¸ÅÄÐ#MÑ#MÔ#MÐPSÑ#SÑTÑUˆØ�{Š{˜6Ñ"Ô"Ð$4Ð4Ð4rA   c                 ó(  — t          | |› d�¦  «        }t          | |› d�¦  «        }|d d d …d f                              |j        d         dd¦  «                             t          j        |j        ¬¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d	¬
¦  «        5  ||z                       dd¦  «        }	t	          j        |	|	fd¬¦  «        }
|
                     ¦   «         |z  }|
                     ¦   «         |z  }d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬¦  «        |                     |j        ¬¦  «        fS )Nr  rV  r   rh   r!   ©rI   r'   rW   rX   FrY   rF   ri   rH   )rJ   rˆ   r‡   rP   rM   rO   r'   r\   r]   r^   r   r›   rk   r_   r`   rI   )r>   rb   rc   r  r;   r8   Úinv_freq_expandedrd   rZ   re   Úembr_   r`   s                r@   rf   zStep3p7RotaryEmbedding.forwardè  sÈ  € õ ˜4 JÐ!9Ð!9Ð!9Ñ:Ô:ˆÝ# D¨ZÐ*KÐ*KÐ*KÑLÔLÐð �T˜1˜1˜1˜d�]Ô#×*Ò*¨<Ô+=¸aÔ+@À"ÀaÑHÔH×KÒKÕRWÔR]ÐfgÔfnÐKÑoÔoð 	ð !-¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆå¨¸UÐCÑCÔCð 	0ð 	0Ø&Ð)>Ñ>×IÒIÈ!ÈQÑOÔOˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)Ð/Ñ/ˆCØ—'’'‘)”)Ð/Ñ/ˆCð		0ð 	0ð 	0ñ 	0ô 	0ð 	0ð 	0ð 	0ð 	0ð 	0ð 	0øøøð 	0ð 	0ð 	0ð 	0ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   Ã'A EÅEÅErn   rÞ   )r7   ro   rp   r   r"   r2   rr   r^   rs   rM   rt   rO   r  ru   r   rf   rv   rw   s   @r@   r  r  ´  s  ø€ € € € € Ø€_�X vÐ.Ñ.Ô.ðUð U˜}ð Uð Uð Uð Uð Uñ /Ô.ðUð* Ø€_�X vÐ.Ñ.Ô.àEIð5ð 5Øð5Ø8;¸d¹
ð5à	ˆuŒ|˜UÐ"Ô	#ð5ð 5ð 5ñ /Ô.ñ „\ð5ð6 €U„]�_„_Øð<ð <ñ Ôñ „_ð<ð <ð <ð <ð <rA   r  c                   ó@   ‡ — e Zd ZdZd	dedefˆ fd„Zd„ Zd„ Zd„ Z	ˆ xZ
S )
r&  zCGemma-style RMSNorm: normalizes in fp32 and scales by `weight + 1`.ç�íµ ÷Æ°>rS   rÏ   c                 ó¬   •— t          ¦   «                              ¦   «          || _        t          j        t          j        |¦  «        ¦  «        | _        d S rn   )r1   r2   rÏ   r9   rÖ   rM   Úzerosrù   )r>   rS   rÏ   r6   s      €r@   r2   zStep3p7RMSNorm.__init__  s?   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”l¥5¤;¨sÑ#3Ô#3Ñ4Ô4ˆŒˆˆrA   c                 ó�   — |t          j        |                     d¦  «                             dd¬¦  «        | j        z   ¦  «        z  S )NrF   rh   T)Úkeepdim)rM   ÚrsqrtÚpowr  rÏ   )r>   rb   s     r@   Ú_normzStep3p7RMSNorm._norm  s8   € Ø•5”;˜qŸušu Q™xœxŸ}š}¨R¸˜}Ñ>Ô>ÀÄÑIÑJÔJÑJÐJrA   c                 ó¸   — |                       |                     ¦   «         ¦  «        }|d| j                             ¦   «         z   z  }|                     |¦  «        S )NrG   )rn  rO   rù   Útype_as)r>   rb   Úoutputs      r@   rf   zStep3p7RMSNorm.forward	  sL   € Ø—’˜AŸGšG™IœIÑ&Ô&ˆð ˜3 ¤×!2Ò!2Ñ!4Ô!4Ñ4Ñ5ˆØ�~Š~˜aÑ Ô Ð rA   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)rs   rù   r‡   rÏ   ©r>   s    r@   Ú
extra_reprzStep3p7RMSNorm.extra_repr  s%   € Ý˜œÔ)Ñ*Ô*Ð<Ð<°$´(Ð<Ð<Ð<rA   )rg  )r7   ro   rp   rq   r  rO   r2   rn  rf   rt  rv   rw   s   @r@   r&  r&  þ  s†   ø€ € € € € ØMÐMð5ð 5˜Cð 5 eð 5ð 5ð 5ð 5ð 5ð 5ð
Kð Kð Kð!ð !ð !ð=ð =ð =ð =ð =ð =ð =rA   r&  c                   ó6   ‡ — e Zd Zdˆ fd„	Zdej        fd„Zˆ xZS )Ú
Step3p7MLPFc                 ó  •— t          ¦   «                              ¦   «          || _        |j        | _        |r|j        n|j        | _        t          j        | j        | j        |j        ¬¦  «        | _	        t          j        | j        | j        |j        ¬¦  «        | _
        t          j        | j        | j        |j        ¬¦  «        | _        t          |j                 | _        |j        r|j        |         ndpt!          d¦  «        | _        d S )N©ræ   r   Úinf)r1   r2   r+   rK   Úshare_expert_dimr~   r9   r}   Úmlp_biasÚ	gate_projÚup_projr"  r   r{   Úact_fnÚswiglu_limits_sharedrO   Úlimit)r>   r+   Ú	layer_idxÚis_shared_expertr6   s       €r@   r2   zStep3p7MLP.__init__  sí   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ<LÐ!j Ô!8Ð!8ÐRXÔRjˆÔÝœ 4Ô#3°TÔ5KÐRXÔRaÐbÑbÔbˆŒÝ”y Ô!1°4Ô3IÐPVÔP_Ð`Ñ`Ô`ˆŒÝœ 4Ô#9¸4Ô;KÐRXÔRaÐbÑbÔbˆŒÝ˜VÔ.Ô/ˆŒð AGÔ@[Ðb�fÔ1°)Ô<Ð<ÐabÐsÕglÐmrÑgsÔgsˆŒ
ˆ
ˆ
rA   rB   c                 ó  — |                       |                      |¦  «        ¦  «                             | j        ¬¦  «        }|                      |¦  «                             | j         | j        ¬¦  «        }|                      ||z  ¦  «        S )N©Úmax©Úminr…  )r~  r|  Úclampr€  r}  r"  )r>   rb   ÚgateÚups       r@   rf   zStep3p7MLP.forward"  sp   € Ø�{Š{˜4Ÿ>š>¨!Ñ,Ô,Ñ-Ô-×3Ò3¸¼
Ð3ÑCÔCˆØ�\Š\˜!‰_Œ_×"Ò"¨¬
 {¸¼
Ð"ÑCÔCˆØ�~Š~˜d R™iÑ(Ô(Ð(rA   r  r„   rw   s   @r@   rv  rv    s`   ø€ € € € € ðtð tð tð tð tð tð)˜EœLð )ð )ð )ð )ð )ð )ð )ð )rA   rv  c                   óŒ   ‡ — e Zd ZdZdZdˆ fd„	Zdej        dej        dej        dej        fd	„Zd
ej        dej        fd„Z	ˆ xZ
S )r  z2Collection of expert weights stored as 3D tensors.TNc                 óä  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        t          j	        t          j        | j        d| j        z  | j        ¦  «        ¦  «        | _        t          j	        t          j        | j        | j        | j        ¦  «        ¦  «        | _        t          |j                 | _        |€t#          d¦  «        n|| _        d S )NrF   ry  )r1   r2   Únum_local_expertsÚnum_expertsrK   Ú
hidden_dimÚmoe_intermediate_sizeÚintermediate_dimr9   rÖ   rM   Úemptyr!  r"  r   r{   r~  rO   r€  )r>   r+   Úswiglu_limitr6   s      €r@   r2   zStep3p7Experts.__init__-  s¹   ø€ Ý‰Œ×ÒÑÔÐØ!Ô3ˆÔØ Ô,ˆŒØ &Ô <ˆÔÝœL­¬°TÔ5EÀqÈ4ÔK`ÑG`ÐbfÔbqÑ)rÔ)rÑsÔsˆÔÝœ¥e¤k°$Ô2BÀDÄOÐUYÔUjÑ&kÔ&kÑlÔlˆŒÝ˜VÔ.Ô/ˆŒØ%1Ð%9•U˜5‘\”\�\¸|ˆŒ
ˆ
ˆ
rA   r‚   Útop_k_indexÚtop_k_weightsrB   c                 óî  — t          j        |¦  «        }t          j        ¦   «         5  t          j        || j        ¬¦  «                             ddd¦  «        }t          j        |                     d¬¦  «        d¦  «         	                    ¦   «         }d d d ¦  «         n# 1 swxY w Y   |D ]Ç}|d         }|| j        k    rŒt          j
        ||         ¦  «        \  }}	|                      t          j        ||	         | j        |         ¦  «        ¦  «        }
t          j        |
| j        |         ¦  «        ||	|d f         z  }
|                     d|	|
                     |j        ¦  «        ¦  «         ŒÈ|S )N)Únum_classesrF   r!   r   )rh   r¬   ri   )rM   Ú
zeros_likeru   ÚFÚone_hotrŽ  rü   ÚgreaterÚsumÚnonzeroÚwhereÚ_apply_gateÚlinearr!  r"  Ú
index_add_rP   rI   )r>   r‚   r”  r•  ÚfinalÚmaskÚhitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrents              r@   rf   zStep3p7Experts.forward7  sª  € õ Ô  Ñ/Ô/ˆÝŒ]‰_Œ_ð 	Eð 	EÝ”9˜[°dÔ6FÐGÑGÔG×OÒOÐPQÐSTÐVWÑXÔXˆDÝ”- §¢¨X Ñ 6Ô 6¸Ñ:Ô:×BÒBÑDÔDˆCð	Eð 	Eð 	Eñ 	Eô 	Eð 	Eð 	Eð 	Eð 	Eð 	Eð 	Eøøøð 	Eð 	Eð 	Eð 	Eð ð 	Dð 	DˆJØ# AœˆJØ˜TÔ-Ò-Ð-ØÝ#(¤;¨t°JÔ/?Ñ#@Ô#@Ñ ˆI�yØ×&Ò&¥q¤x°¸iÔ0HÈ$ÔJ[Ð\fÔJgÑ'hÔ'hÑiÔiˆGÝ”h˜w¨¬°zÔ(BÑCÔCÀmÐT]Ð_hÐjnÐTnÔFoÑoˆGØ×Ò˜Q 	¨7¯:ª:°e´kÑ+BÔ+BÑCÔCÐCÐCØˆs   ¨A,B Â B$Â'B$Úgate_upc                 óà   — |                      dd¬¦  «        \  }}|                      |¦  «                             | j        ¬¦  «        }|                     | j         | j        ¬¦  «        }||z  S )NrF   rh   ri   r„  r†  )Úchunkr~  rˆ  r€  )r>   r©  r‰  rŠ  s       r@   rŸ  zStep3p7Experts._apply_gateH  sd   € Ø—=’= ¨�=Ñ+Ô+‰ˆˆbØ�{Š{˜4Ñ Ô ×&Ò&¨4¬:Ð&Ñ6Ô6ˆØ�XŠX˜4œ:˜+¨4¬:ˆXÑ6Ô6ˆØ�b‰yÐrA   rn   )r7   ro   rp   rq   Ú#_fp8_experts_clamp_after_activationr2   rM   rt   rf   rŸ  rv   rw   s   @r@   r  r  (  s²   ø€ € € € € Ø<Ð<à*.Ð'ðLð Lð Lð Lð Lð LðØ"œ\ðØ8=¼ðØUZÔUaðà	Œðð ð ð ð" 5¤<ð °E´Lð ð ð ð ð ð ð ð rA   r  c                   ón   ‡ — e Zd Zdefˆ fd„Zdej        deej        ej        ej        f         fd„Zˆ xZ	S )r#  r+   c                 ó\  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        t          j	        t          j        | j        | j        ¦  «        ¦  «        | _        t          j        t          j        |j        ¦  «        ¦  «        | _        d S rn   )r1   r2   Únum_experts_per_tokÚtop_kr�  rŽ  rK   r�  r9   rÖ   rM   r’  rù   r:   ri  r%  r�   s     €r@   r2   zStep3p7TopKRouter.__init__P  s   ø€ Ý‰Œ×ÒÑÔÐØÔ/ˆŒ
Ø!Ô3ˆÔØ Ô,ˆŒÝ”l¥5¤;¨tÔ/?ÀÄÑ#QÔ#QÑRÔRˆŒÝ')¤yµ´¸VÔ=UÑ1VÔ1VÑ'WÔ'WˆÔ$Ð$Ð$rA   r‚   rB   c                 ó°  — |                      d| j        ¦  «        }t          j        |                     | j        j        ¦  «        | j        ¦  «        }t          j        |                     ¦   «         ¦  «        }|| j	        z   }t          j        || j        dd¬¦  «        \  }}|                     d|¦  «        }||                     dd¬¦  «        z  }|||fS )Nrh   F)rS   rZ  r!   T)rS   rk  )r‰   r�  r™  r   rP   rù   rI   ÚsigmoidrO   r%  rM   Útopkr°  Úgatherrœ  )r>   r‚   Úrouter_logitsÚrouting_weightsÚscores_for_choicerÝ   r”  r•  s           r@   rf   zStep3p7TopKRouter.forwardX  sÇ   € Ø%×-Ò-¨b°$´/ÑBÔBˆÝœ ×!1Ò!1°$´+Ô2CÑ!DÔ!DÀdÄkÑRÔRˆåœ) M×$7Ò$7Ñ$9Ô$9Ñ:Ô:ˆØ+¨dÔ.JÑJÐÝœÐ$5°t´zÀrÐRWÐXÑXÔX‰ˆˆ;Ø'×.Ò.¨q°+Ñ>Ô>ˆØ˜×*Ò*¨r¸4Ð*Ñ@Ô@Ñ@ˆØ˜m¨[Ð8Ð8rA   )
r7   ro   rp   r#   r2   rM   rt   rs   rf   rv   rw   s   @r@   r#  r#  O  s„   ø€ € € € € ðXÐ0ð Xð Xð Xð Xð Xð Xð	9 U¤\ð 	9°e¸E¼LÈ%Ì,ÐX]ÔXdÐ<dÔ6eð 	9ð 	9ð 	9ð 	9ð 	9ð 	9ð 	9ð 	9rA   r#  c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚStep3p7SparseMoeBlockc                 ó  •— t          ¦   «                              ¦   «          |j        r|j        |         pd nd }t          |¦  «        | _        t          ||¬¦  «        | _        t          ||d¬¦  «        | _        |j	        | _
        d S )N)r“  T)r‚  )r1   r2   Úswiglu_limitsr#  r‰  r  Úexpertsrv  Úshared_expertsÚmoe_router_scaling_factorÚrouted_scaling_factor)r>   r+   r�  r“  r6   s       €r@   r2   zStep3p7SparseMoeBlock.__init__e  s†   ø€ Ý‰Œ×ÒÑÔÐð EKÔDXÐb˜Ô,¨YÔ7Ð?¸4øÐ^bˆÝ% fÑ-Ô-ˆŒ	Ý% f¸<ÐHÑHÔHˆŒÝ(¨°ÈTÐRÑRÔRˆÔØ%+Ô%EˆÔ"Ð"Ð"rA   r‚   rB   c                 ó  — |j         \  }}}|                     d|¦  «        }|                      |¦  «        }|                      |¦  «        \  }}}|                      |||¦  «        }|| j        z  }||z   }|                     |||¦  «        }|S )Nrh   )r‡   rÂ   r½  r‰  r¼  r¿  r‰   )	r>   r‚   rL  Úsequence_lengthr�  Úshared_outputrÝ   r¶  Úselected_expertss	            r@   rf   zStep3p7SparseMoeBlock.forwardo  sœ   € Ø2?Ô2EÑ/ˆ
�O ZØ%×*Ò*¨2¨zÑ:Ô:ˆØ×+Ò+¨MÑ:Ô:ˆà/3¯yªy¸Ñ/GÔ/GÑ,ˆˆ?Ð,ØŸš ]Ð4DÀoÑVÔVˆà%¨Ô(BÑBˆØ%¨Ñ5ˆà%×-Ò-¨j¸/È:ÑVÔVˆØÐrA   r„   rw   s   @r@   r¹  r¹  d  sc   ø€ € € € € ðFð Fð Fð Fð Fð U¤\ð °e´lð ð ð ð ð ð ð ð rA   r¹  c                 ó˜  — |                      |¦  «        }|                      |¦  «        }|j        d         }| dd|…f         | d|d…f         }}|dd|…f         |d|d…f         }	}||z  t          |¦  «        |z  z   }
||z  t          |¦  «        |z  z   }t          j        |
|gd¬¦  «        }
t          j        ||	gd¬¦  «        }|
|fS )a»  Applies Rotary Position Embedding to the query and key tensors.

    Removes the interleaving of cos and sin from GLM

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    rh   .Nri   )r­   r‡   r¨   rM   rk   )r©   rª   r_   r`   Úunsqueeze_dimÚ
rotary_dimÚq_rotÚq_passÚk_rotÚk_passr°   r±   s               r@   Úapply_rotary_pos_embrË    sô   € ð( �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€Cð ”˜2”€JØ�c˜;˜J˜;Ð&Ô'¨¨3°
°°Ð+;Ô)<ˆ6€EØ�c˜;˜J˜;Ð&Ô'¨¨3°
°°Ð+;Ô)<ˆ6€Eð �s‰{�{¨5Ñ1Ô1°CÑ7Ñ8€GØ�s‰{�{¨5Ñ1Ô1°CÑ7Ñ8€Gõ Œi˜ &Ð)¨rÐ2Ñ2Ô2€GÝŒi˜ &Ð)¨rÐ2Ñ2Ô2€GØ�GÐÐrA   c                   óÊ   ‡ — e Zd ZdZdedefˆ fd„Z	 ddej        de	ej        ej        f         dej        dz  d	e
dz  d
ee         de	ej        ej        dz  f         fd„Zˆ xZS )ÚStep3p7AttentionzTAfmoe-style SWA/GQA attention with Step3p7-specific gating and per-layer head count.r+   r�  c                 óN  •— t          ¦   «                              ¦   «          |j        }|| _        || _        || _        t          |d|j        |j        z  ¦  «        | _        | j        |j	        z  | _
        |j        dz  | _        |j        | _        d| _        t          j        |j        | j        | j        z  |j        ¬¦  «        | _        t          j        |j        |j	        | j        z  |j        ¬¦  «        | _        t          j        |j        |j	        | j        z  |j        ¬¦  «        | _        t          j        | j        | j        z  |j        |j        ¬¦  «        | _        |j        |         dk    | _        | j        r|j        nd | _        t3          | j        |j        ¬¦  «        | _        t3          | j        |j        ¬¦  «        | _        |j        du p
|j        dk    | _        | j        r| j        n| j        | j        z  }t          j        |j        |d¬¦  «        | _        d S )	NrE   r¶   Trx  Úsliding_attentionrÎ   zper-headF) r1   r2   rL   r¸   r+   r�  rJ   rK   rE   r‹   r™   Úquery_pre_attn_scalarr”   rº   r»   r9   r}   Úattention_biasr¾   r¼   r½   Úo_projr  Úis_local_attentionÚsliding_windowr&  Úrms_norm_epsÚq_normÚk_normÚgatingÚgate_per_headÚg_proj)r>   r+   r�  r¸   Ú
g_proj_dimr6   s        €r@   r2   zStep3p7Attention.__init__¨  sö  ø€ Ý‰Œ×ÒÑÔÐØÔ.ˆ	à"ˆŒØˆŒØ"ˆŒÝ ¨
°FÔ4FÈ&ÔJdÑ4dÑeÔeˆŒØ$(¤N°fÔ6PÑ$PˆÔ!ØÔ3°TÑ9ˆŒØ!'Ô!9ˆÔØˆŒå”i Ô 2°D´NÀTÄ]Ñ4RÐY_ÔYnÐoÑoÔoˆŒÝ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”i ¤°´Ñ >ÀÔ@RÐY_ÔYnÐoÑoÔoˆŒð #)Ô"4°YÔ"?ÐCVÒ"VˆÔØ7;Ô7NÐX˜fÔ3Ð3ÐTXˆÔå$ T¤]¸Ô8KÐLÑLÔLˆŒÝ$ T¤]¸Ô8KÐLÑLÔLˆŒØ#œ]¨dÐ2ÐQ°f´mÀzÒ6QˆÔØ'+Ô'9Ð]�T”^�^¸t¼~ÐPTÔP]Ñ?]ˆ
Ý”i Ô 2°JÀUÐKÑKÔKˆŒˆˆrA   Nr‚   rÀ   r“   r  rQ   rB   c                 ó<  — |j         d d…         }g |¢d‘| j        ‘R }|                      |                      |¦  «                             |¦  «        ¦  «                             dd¦  «        }|                      |                      |¦  «                             |¦  «        ¦  «                             dd¦  «        }	|                      |¦  «                             |¦  «                             dd¦  «        }
|  	                    |¦  «        }|\  }}t          ||	||¦  «        \  }}	|�|                     |	|
| j        ¦  «        \  }	}
t          j        | j        j        t"          ¦  «        } || ||	|
|f| j        sdn| j        | j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž }|j         } |j        g |d d…         ¢| j        ‘| j        ‘R Ž }||                     d¦  «                             ¦   «         z  } |j        |Ž }|                      |¦  «        }||fS )Nrh   r!   rF   rŽ   )r•   r”   rÔ  )r‡   rE   rÖ  r¾   rÂ   r›   r×  r¼   r½   rÚ  rË  Úupdater�  r   rÃ   r+   rÄ   r¤   r˜   rº   r”   rÔ  r‰   r¸   r­   r²  rÒ  )r>   r‚   rÀ   r“   r  rQ   rÅ   rÆ   Úquery_statesr    r¡   Úgate_statesr_   r`   rÊ   r£   r¢   Úoutput_shapes                     r@   rf   zStep3p7Attention.forwardÈ  sO  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ 4§;¢;¨}Ñ#=Ô#=×#BÒ#BÀ<Ñ#PÔ#PÑQÔQ×[Ò[Ð\]Ð_`ÑaÔaˆØ—[’[ §¢¨]Ñ!;Ô!;×!@Ò!@ÀÑ!NÔ!NÑOÔO×YÒYÐZ[Ð]^Ñ_Ô_ˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆà—k’k -Ñ0Ô0ˆØ&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð  $œ}ÐH�C�C°$Ô2HØ”LØÔ.ð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;ˆØ"Ô(ˆØ&�kÔ&ÐY¨°S°b°SÔ(9ÐY¸4¼>ÐYÈ4Ì=ÐYÐYÐYˆØ! K×$9Ò$9¸"Ñ$=Ô$=×$EÒ$EÑ$GÔ$GÑGˆØ&�kÔ&¨Ð5ˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(rA   rn   )r7   ro   rp   rq   r#   r  r2   rM   rt   rs   r   r   r   rf   rv   rw   s   @r@   rÍ  rÍ  ¥  sç   ø€ € € € € Ø^Ð^ðLÐ0ð L¸Sð Lð Lð Lð Lð Lð LðJ )-ð*)ð *)à”|ð*)ð # 5¤<°´Ð#=Ô>ð*)ð œ tÑ+ð	*)ð
  ™ð*)ð Ð+Ô,ð*)ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð*)ð *)ð *)ð *)ð *)ð *)ð *)ð *)rA   rÍ  c                   óÈ   ‡ — e Zd Zˆ fd„Z	 	 	 	 	 ddej        dej        dz  dej        dz  dedz  dedz  d	e	ej        ej        f         dz  d
e
e         dej        fd„Zˆ xZS )r  c                 óÀ  •— t          ¦   «                              ¦   «          |j        | _        |j        |         }t	          ||¦  «        | _        || j        _        |j        |         | _        |j	        |         dk    rt          nt          } |||¦  «        | _        t          |j        |j        ¬¦  «        | _        t          |j        |j        ¬¦  «        | _        d S )NÚsparserÎ   )r1   r2   rK   Úper_layer_configrÍ  rÐ   r+   r  Úattention_typeÚmlp_layer_typesr¹  rv  rÑ   r&  rÕ  Úinput_layernormÚpost_attention_layernorm)r>   r+   r�  Úlayer_configÚ	mlp_classr6   s        €r@   r2   zStep3p7DecoderLayer.__init__ö  sÎ   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔð Ô.¨yÔ9ˆÝ)¨,¸	ÑBÔBˆŒØ &ˆŒÔØ$Ô0°Ô;ˆÔð .4Ô-CÀIÔ-NÐRZÒ-ZÐ-ZÕ)Ð)Õ`jˆ	Ø�9˜V YÑ/Ô/ˆŒå-¨fÔ.@ÀfÔFYÐZÑZÔZˆÔÝ(6°vÔ7IÈvÔObÐ(cÑ(cÔ(cˆÔ%Ð%Ð%rA   NFr‚   r“   rc   r  Ú	use_cacherÀ   rQ   rB   c           
      óÎ   — |}|                       |¦  «        } | j        d||||||dœ|¤Ž\  }}	||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|S )N)r‚   r“   rc   r  rë  rÀ   r8  )rç  rÐ   rè  rÑ   )
r>   r‚   r“   rc   r  rë  rÀ   rQ   rÜ   rÝ   s
             r@   rf   zStep3p7DecoderLayer.forward	  s¡   € ð !ˆØ×,Ò,¨]Ñ;Ô;ˆà)˜4œ>ð 
Ø'Ø)Ø%Ø+ØØ 3ð
ð 
ð ð
ð 
Ñˆ�qð ! =Ñ0ˆð !ˆØ×5Ò5°mÑDÔDˆØŸš Ñ/Ô/ˆØ  =Ñ0ˆØÐrA   )NNNFN)r7   ro   rp   r2   rM   rt   Ú
LongTensorr   Úboolrs   r   r   rf   rv   rw   s   @r@   r  r  õ  sé   ø€ € € € € ðdð dð dð dð dð, /3Ø04Ø(,Ø!&ØHLðð à”|ðð œ tÑ+ðð Ô&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð # 5¤<°´Ð#=Ô>ÀÑEðð Ð+Ô,ðð 
Œðð ð ð ð ð ð ð rA   r  rÔ  c           
      óZ   ‡ — dt           dt           dt           dt           dt          f
ˆ fd„}|S )zA
    Enables a bidirectional mask within the sliding window.
    Ú	batch_idxÚhead_idxÚq_idxÚkv_idxrB   c                 ó0   •— t          ||z
  ¦  «        ‰k     S )z“A token can attend to any other token if their absolute distance is within
        the (exclusive) sliding window size (distance < sliding_window).)Úabs)rð  rñ  rò  ró  rÔ  s       €r@   Ú
inner_maskz1_bidirectional_window_overlay.<locals>.inner_mask.  s   ø€ õ �5˜6‘>Ñ"Ô" ^Ò3Ð3rA   )r  rî  )rÔ  rö  s   ` r@   Ú_bidirectional_window_overlayr÷  )  sL   ø€ ð
4�cð 4­Sð 4½ð 4Åcð 4Ídð 4ð 4ð 4ð 4ð 4ð 4ð
 ÐrA   c                   óü   ‡ — e Zd ZU eed<   dZeedœZdefˆ fd„Z	e
ee	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  d	edz  d
ej        dz  dedz  dee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚStep3p7TextModelr+   )Útextr4  c                 ó„  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          j        ‰j        ‰j        | j        ¦  «        | _        t          j	        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        t!          ‰¬¦  «        | _        d| _        ‰j        rLt          ‰j        ‰j        ‰j        z   ¦  «        }d„ |D ¦   «         }t)          | j        pg ¦  «        |z  | _        |                      ¦   «          d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r8  )r  )r9  Úir+   s     €r@   r:  z-Step3p7TextModel.__init__.<locals>.<listcomp>D  s$   ø€ Ð$mÐ$mÐ$mÈÕ%8¸ÀÑ%CÔ%CÐ$mÐ$mÐ$mrA   rÎ   ©r+   Fc                 ó   — h | ]}d |› d�’Œ	S )z(^|\.)layers\.z\.r8  )r9  rý  s     r@   ú	<setcomp>z,Step3p7TextModel.__init__.<locals>.<setcomp>P  s$   € ÐDÐDÐD°AÐ/¨!Ð/Ð/Ð/ÐDÐDÐDrA   )r1   r2   Úpad_token_idÚpadding_idxÚ
vocab_sizer9   rï   rK   Úembed_tokensr?  r@  rA  rB  r&  rÕ  Únormr  r>  Úgradient_checkpointingÚnum_nextn_predict_layersr[  Ú"_keys_to_ignore_on_load_unexpectedrE  )r>   r+   Ú
mtp_layersÚpatternsr6   s    `  €r@   r2   zStep3p7TextModel.__init__?  s/  øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒÝœL¨Ô):¸FÔ<NÐPTÔP`ÑaÔaˆÔÝ”mÐ$mÐ$mÐ$mÐ$mÍUÐSYÔSkÑMlÔMlÐ$mÑ$mÔ$mÑnÔnˆŒÝ" 6Ô#5¸6Ô;NÐOÑOÔOˆŒ	Ý0¸Ð?Ñ?Ô?ˆŒØ&+ˆÔ#ð Ô*ð 	tõ ˜vÔ7¸Ô9QÐTZÔTsÑ9sÑtÔtˆJØDÐD¸ÐDÑDÔDˆHÝ69¸$Ô:aÐ:gÐegÑ6hÔ6hÐksÑ6sˆDÔ3ð 	�ŠÑÔÐÐÐrA   NÚ	input_idsr“   rc   r  Úinputs_embedsrë  rQ   rB   c           	      óÌ  — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|r|€t          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   }| 	                    d¦  «        }t          |x}	t          ¦  «        sh| j        ||||dœ}
|
                     ¦   «         }| j        j        r"d„ |
d<   t          | j        j        ¦  «        |d<   t!          di |
¤Žt#          di |¤Žd	œ}	|}i }t%          | j        j        ¦  «        D ]}|                      |||¦  «        ||<   Œt+          | j        d | j        j        …         ¦  «        D ]?\  }} ||f|	| j        j        |                  || j        j        |                  ||d
œ|¤Ž}Œ@|                      |¦  «        }t3          ||¬¦  «        S )Nz:You must specify exactly one of input_ids or inputs_embedsrþ  r   r!   rH  )r+   r  r“   r  rc   c                  óB   — t          j        dt           j        ¬¦  «        S )NTrH   )rM   rK  rî  )Úargss    r@   ú<lambda>z*Step3p7TextModel.forward.<locals>.<lambda>~  s   € ÅÄÈTÕY^ÔYcÐ@dÑ@dÔ@d€ rA   Úor_mask_function)Úfull_attentionrÏ  )r“   rÀ   rc   r  )rJ  r  r8  )r5   r  r	   r+   Úget_seq_lengthrM   rN   r‡   r'   r­   r\   ÚdictÚcopyÚuse_bidirectional_attentionr÷  rÔ  r   r   r[  r  r>  Ú	enumeraterB  rA  r  r   )r>   r  r“   rc   r  r  rë  rQ   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsÚsliding_mask_kwargsr‚   rÀ   r  rý  Údecoder_layers                    r@   rf   zStep3p7TextModel.forwardV  sq  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLõ °Ð?Ð-ÅÑFÔFð 	ð œ+Ø!.Ø"0Ø#2Ø ,ðð ˆKð #.×"2Ò"2Ñ"4Ô"4ÐàŒ{Ô6ð tØ2dÐ2d�Ð.Ñ/Ý:WÐX\ÔXcÔXrÑ:sÔ:sÐ#Ð$6Ñ7õ #5Ð"CÐ"C°{Ð"CÐ"CÝ%FÐ%]Ð%]ÐI\Ð%]Ð%]ð#ð #Ðð &ˆØ ÐÝ˜dœkÔ5Ñ6Ô6ð 	gð 	gˆJØ.2¯oªo¸mÈ\Ð[eÑ.fÔ.fÐ 
Ñ+Ð+å )¨$¬+Ð6U¸¼Ô8UÐ6UÔ*VÑ WÔ Wð 	ð 	ÑˆAˆ}Ø)˜MØðà2°4´;Ô3JÈ1Ô3MÔNØ$7¸¼Ô8OÐPQÔ8RÔ$SØ)Ø /ðð ð ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆå&Ø+Ø+ð
ñ 
ô 
ð 	
rA   )NNNNNN)r7   ro   rp   r#   r(  Úinput_modalitiesr  rÍ  rS  r2   r   r   r   rM   rí  rt   r   r	  rî  r   r   r   rf   rv   rw   s   @r@   rù  rù  6  sF  ø€ € € € € € àÐÐÑØ Ðà,Ø&ðð Ðð
Ð0ð ð ð ð ð ð ð.  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ðC
ð C
àÔ# dÑ*ðC
ð œ tÑ+ðC
ð Ô&¨Ñ-ð	C
ð
  ™ðC
ð Ô(¨4Ñ/ðC
ð ˜$‘;ðC
ð Ð+Ô,ðC
ð 
!ðC
ð C
ð C
ñ „^ñ „_ñ  ÔðC
ð C
ð C
ð C
ð C
rA   rù  zJ
    Base class for Llava outputs, with hidden states and attentions.
    )Úcustom_introc                   ó2   — e Zd ZU dZdZej        dz  ed<   dS )ÚStep3p7ModelOutputWithPastaÏ  
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
    NÚimage_hidden_states)r7   ro   rp   rq   r!  rM   r	  r(  r8  rA   r@   r   r   Ÿ  s7   € € € € € € ð	ð 	ð 59Ð˜Ô*¨TÑ1Ð8Ð8Ñ8Ð8Ð8rA   r   zw
    The Llava-Next model which consists of a vision backbone and a language model without language modeling head.
    c                   óÞ  ‡ — e Zd ZU dZeed<   defˆ fd„Zee	 	 dde	j
        de	j
        dz  dee         dz  deez  fd	„¦   «         ¦   «         Zd
e	j        de	j        de	j        fd„Zee	 	 	 	 	 	 	 	 	 dd
e	j        dz  de	j        dz  de	j        dz  dee         e	j
        z  dz  de	j
        dz  de	j        dz  dedz  de	j        dz  dedz  dee         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚStep3p7Modelr  r+   c                 ó€  •— t          ¦   «                              |¦  «         t          |j        ¦  «        | _        t          |j        ¦  «        | _        |j        j        | _        t          j
        |j        j        dz  |j        j        |j        ¬¦  «        | _        |j        | _        |                      ¦   «          d S )Nr<  rx  )r1   r2   r3  Úvision_configÚvision_modelrù  Útext_configÚlanguage_modelr  r9   r}   rK   Úprojector_biasÚmulti_modal_projectorÚimage_token_idÚimage_placeholder_token_idrE  r�   s     €r@   r2   zStep3p7Model.__init__½  s¨   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý.¨vÔ/CÑDÔDˆÔÝ.¨vÔ/AÑBÔBˆÔØ Ô,Ô7ˆŒõ &(¤YØÔ Ô,¨qÑ0°&Ô2DÔ2PÐW]ÔWlð&
ñ &
ô &
ˆÔ"ð +1Ô*?ˆÔ'Ø�ŠÑÔÐÐÐrA   Nr  Úpixel_values_localÚnum_local_patchesrB   c           	      óâ  —  | j         |fi |¤Ž}|                      |j        ¦  «        }|�,|                       | j         |fi i |¥ddi¥¤Žj        ¦  «        nd}|€dg|j        d         z  }g }d}	t	          |¦  «        D ]Ï\  }
}g }|dk    rA||	|	|z   …         }|                     |                     d|j        d         ¦  «        ¦  «         |                     ||
                              d|j        d         ¦  «        ¦  «         |	|z  }	|                     t          |¦  «        dk    rt          j	        |¦  «        n|d         ¦  «         ŒÐt          |j        ||j        |j        ¬¦  «        S )aS  
        pixel_values_local (`torch.FloatTensor` of shape `(total_patches, 3, height, width)`, *optional*):
            All local patches flattened across the batch, or `None` if no local views.
        num_local_patches (`list[int]` or `torch.Tensor`, *optional*):
            Number of local patches per image, e.g. `[6, 0, 4]`.
        NÚoutput_hidden_statesFr   rh   r!   )rJ  Úpooler_outputr‚   r5  )r&  r*  rJ  r‡   r  ÚappendrÂ   ÚlenrM   rk   r   r‚   r5  )r>   r  r-  r.  rQ   Úvision_outputÚimage_featuresÚpatch_image_featuresÚmergedÚcur_patch_idxrý  Ú	num_patchÚcur_featureÚpatch_slices                 r@   Úget_image_featureszStep3p7Model.get_image_featuresÊ  sÄ  € ð *˜Ô)¨,ÐAÐA¸&ÐAÐAˆØ×3Ò3°MÔ4SÑTÔTˆð
 "Ð-ð ×&Ò&Ø!�Ô!Ð"4ÐbÐbÐ8a¸6Ð8aÐCYÐ[`Ð8aÐ8aÐbÐbÔtñô ð ð ð 	ð Ð$Ø!"  nÔ&:¸1Ô&=Ñ =ÐàˆØˆÝ%Ð&7Ñ8Ô8ð 	^ð 	^‰LˆAˆyØˆKØ˜1Š}ˆ}Ø2°=À=ÐS\ÑC\Ð3\Ô]�Ø×"Ò" ;×#3Ò#3°B¸Ô8IÈ"Ô8MÑ#NÔ#NÑOÔOÐOØ×Ò˜~¨aÔ0×5Ò5°b¸.Ô:NÈrÔ:RÑSÔSÑTÔTÐTØ˜YÑ&ˆMØ�MŠMµC¸Ñ4DÔ4DÀqÒ4HÐ4H�%œ) KÑ0Ô0Ð0ÈkÐZ[ÌnÑ]Ô]Ð]Ð]Ý)Ø+Ô=Ø Ø'Ô5Ø$Ô/ð	
ñ 
ô 
ð 	
rA   r  r  r5  c           	      ó  — |€f| |                       ¦   «         t          j        d| j        j        t          j        |j        ¬¦  «        ¦  «        k    }|                     d¦  «        }n|| j        j        k    }|                     ¦   «         }| 	                    d¦  «         
                    |j        ¦  «        }t          ||j        d         z  |                     ¦   «         k    d|› d|j        d         › �¦  «         |S )zï
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        Nr8  rc  rh   z6Image features and image tokens do not match, tokens: z, features: r   )Úget_input_embeddingsrM   Úfullr+   r+  Úlongr'   Úallrœ  r­   rP   r   r‡   Únumel)r>   r  r  r5  Úspecial_image_maskÚn_image_tokenss         r@   Úget_placeholder_maskz!Step3p7Model.get_placeholder_mask÷  s  € ð ÐØ!.Ð2M°$×2KÒ2KÑ2MÔ2MÝ”
˜2˜tœ{Ô9ÅÄÐTaÔThÐiÑiÔiñ3ô 3ò "Ðð "4×!7Ò!7¸Ñ!;Ô!;ÐÐà!*¨d¬kÔ.HÒ!HÐà+×/Ò/Ñ1Ô1ˆØ/×9Ò9¸"Ñ=Ô=×@Ò@ÀÔAUÑVÔVÐÝØ˜]Ô0°Ô4Ñ4¸×8LÒ8LÑ8NÔ8NÒNØzÀ^ÐzÐzÐaoÔauÐvwÔaxÐzÐzñ	
ô 	
ð 	
ð "Ð!rA   r“   rc   r  rë  rQ   c
           
      ó¾  — |€ |                       ¦   «         |¦  «        }d}|�|                      |||d¬¦  «        j        }t          j        |d¬¦  «                             |j        |j        ¦  «        }|                      |||¦  «        }| 	                    ||¦  «        } | j
        dd|||||	dœ|
¤Ž}t          |j        |j        |j        |j        |¬¦  «        S )	á  
        pixel_values_local (`torch.FloatTensor`, *optional*):
            Local patch pixel values of shape `(total_patches, 3, H, W)`.
        num_local_patches (`list[int]` or `torch.Tensor`, *optional*):
            Number of local patches per image in the batch.
        NT)Úreturn_dictr   ri   )r  r  r“   rc   r  rë  )rJ  r  r‚   r5  r!  r8  )r>  r<  r1  rM   rk   rP   r'   rI   rE  Úmasked_scatterr(  r   rJ  r  r‚   r5  )r>   r  r  r-  r.  r“   rc   r  r  rë  rQ   r5  rC  Úoutputss                 r@   rf   zStep3p7Model.forward  s&  € ð* Ð Ø7˜D×5Ò5Ñ7Ô7¸	ÑBÔBˆMàˆØÐ#Ø!×4Ò4ØÐ0Ð2CÐQUð 5ñ ô äð õ #œY ~¸1Ð=Ñ=Ô=×@Ò@ÀÔAUÐWdÔWjÑkÔkˆNà!%×!:Ò!:¸9ÀmÐUcÑ!dÔ!dÐØ)×8Ò8Ð9KÈ^Ñ\Ô\ˆMà%�$Ô%ð 
ØØ'Ø)Ø%Ø+Øð
ð 
ð ð
ð 
ˆõ *Ø%Ô7Ø#Ô3Ø!Ô/ØÔ)Ø .ð
ñ 
ô 
ð 	
rA   rÞ   )	NNNNNNNNN)r7   ro   rp   r)  r"   r(  r2   r   r   rM   rt   Úlistr  rs   r   r<  rí  r	  rE  r   rî  r   r   r   rf   rv   rw   s   @r@   r#  r#  ´  s  ø€ € € € € € ð  ÐØÐÐÑð˜}ð ð ð ð ð ð ð Øð 37Ø.2ð	)
ð )
à”lð)
ð "œL¨4Ñ/ð)
ð   œ9 tÑ+ð	)
ð 
Ð+Ñ	+ð)
ð )
ð )
ñ „^ñ Ôð)
ðV"ØÔ)ð"Ø:?Ô:Kð"Ø]bÔ]nð"ð "ð "ð "ð. Øð .2Ø15Ø7;Ø=AØ.2Ø04Ø(,Ø26Ø!%ð0
ð 0
àÔ# dÑ*ð0
ð Ô'¨$Ñ.ð0
ð "Ô-°Ñ4ð	0
ð
   œ9 u¤|Ñ3°dÑ:ð0
ð œ tÑ+ð0
ð Ô&¨Ñ-ð0
ð  ™ð0
ð Ô(¨4Ñ/ð0
ð ˜$‘;ð0
ð Ð+Ô,ð0
ð 
Ð+Ñ	+ð0
ð 0
ð 0
ñ „^ñ Ôð0
ð 0
ð 0
ð 0
ð 0
rA   r#  zS
    Base class for Step3p7 causal language model (or autoregressive) outputs.
    c                   óÖ   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dZej        dz  ed<   dS )	ÚStep3p7CausalLMOutputWithPastaA  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size (batch_size * num_patches, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
    NÚlossÚlogitsr  r‚   r5  r!  )r7   ro   rp   rq   rN  rM   r	  r(  rO  r  r   r‚   rs   r5  r!  r8  rA   r@   rM  rM  C  sµ   € € € € € € ðð ð &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø$(€O�U˜T‘\Ð(Ð(Ñ(Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ø48Ð˜Ô*¨TÑ1Ð8Ð8Ñ8Ð8Ð8rA   rM  c                   ó  ‡ — e Zd ZU ddiZeed<   defˆ fd„Zdej        fd„Z	d„ Z
ee	 	 dd	ej        d
ej        dz  dee         ej        z  dz  dee         deez  f
d„¦   «         ¦   «         Zee	 	 	 	 	 	 	 	 	 	 	 ddej        dz  d	ej        dz  d
ej        dz  dee         ej        z  dz  dej        dz  dej        dz  dedz  dej        dz  dej        dz  dedz  deej        z  dee         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚStep3p7ForConditionalGenerationzlm_head.weightz(model.language_model.embed_tokens.weightr+   c                 óú   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        j        |j        j        d¬¦  «        | _	        |  
                    ¦   «          d S )NFrx  )r1   r2   r#  r  r9   r}   r'  rK   r  Úlm_headrE  r�   s     €r@   r2   z(Step3p7ForConditionalGeneration.__init__f  se   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý! &Ñ)Ô)ˆŒ
Ý”y Ô!3Ô!?ÀÔASÔA^ÐejÐkÑkÔkˆŒØ�ŠÑÔÐÐÐrA   rB   c                 ó   — | j         S rn   )rS  rs  s    r@   Úget_output_embeddingsz5Step3p7ForConditionalGeneration.get_output_embeddingsl  s
   € ØŒ|ÐrA   c                 ó    — t          d¦  «        ‚)Nz(Step3p7 does not use pack_image_features)ÚNotImplementedErrorrs  s    r@   Úpack_image_featuresz3Step3p7ForConditionalGeneration.pack_image_featureso  s   € Ý!Ð"LÑMÔMÐMrA   Nr  r-  r.  rQ   c                 ó.   —  | j         j        d|||dœ|¤ŽS )að  
        pixel_values (`torch.FloatTensor` of shape `(batch_size, 3, height, width)`):
            The tensors corresponding to the global view input images.
        pixel_values_local (`torch.FloatTensor` of shape `(total_patches, 3, height, width)`, *optional*):
            All local patches flattened across the batch, or `None` if no local views.
        num_local_patches (`list[int]` or `torch.Tensor`, *optional*):
            Number of local patches per image, e.g. `[6, 0, 4]`.
        )r  r-  r.  r8  )r  r<  )r>   r  r-  r.  rQ   s        r@   r<  z2Step3p7ForConditionalGeneration.get_image_featuresr  s:   € ð" -ˆtŒzÔ,ð 
Ø%Ø1Ø/ð
ð 
ð ð	
ð 
ð 	
rA   r   r  r“   rc   r  r  Úlabelsrë  Úlogits_to_keepc                 ór  —  | j         d|||||||||
dœ	|¤Ž}|d         }t          |t          ¦  «        rt          | d¦  «        n|}|dd…|dd…f         }|                      |¦  «        }d}|	�  | j        d||	| j        j        j        dœ|¤Ž}t          |||j
        |j        |j        |j        ¬¦  «        S )rG  )	r  r  r-  r.  r“   rc   r  r  rë  r   N)rO  rZ  r  )rN  rO  r  r‚   r5  r!  r8  )r  r\   r  ÚslicerS  Úloss_functionr+   r'  r  rM  r  r‚   r5  r!  )r>   r  r  r-  r.  r“   rc   r  r  rZ  rë  r[  rQ   rJ  r‚   Úslice_indicesrO  rN  s                     r@   rf   z'Step3p7ForConditionalGeneration.forwardŠ  s  € ð. �$”*ð 
ØØ%Ø1Ø/Ø)Ø%Ø+Ø'Øð
ð 
ð ð
ð 
ˆð   œ
ˆÝ8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ% a a a¨¸¸¸Ð&9Ô:ˆØ—’˜mÑ,Ô,ˆàˆØÐØ%�4Ô%ð ØØØœ;Ô2Ô=ðð ð ð	ð ˆDõ -ØØØ#Ô3Ø!Ô/ØÔ)Ø 'Ô ;ð
ñ 
ô 
ð 	
rA   rÞ   )NNNNNNNNNNr   )r7   ro   rp   Ú_tied_weights_keysr"   r(  r2   r9   ÚModulerU  rX  r   r   rM   r	  rK  r  rt   r   r   rs   r   r<  rí  r   rî  rM  rf   rv   rw   s   @r@   rQ  rQ  a  sQ  ø€ € € € € € à*Ð,VÐWÐØÐÐÑð˜}ð ð ð ð ð ð ð r¤yð ð ð ð ðNð Nð Nð Øð 8<Ø=Að	
ð 
àÔ'ð
ð "Ô-°Ñ4ð
ð   œ9 u¤|Ñ3°dÑ:ð	
ð
 Ð+Ô,ð
ð 
Ð+Ñ	+ð
ð 
ð 
ñ „^ñ Ôð
ð, Øð .2Ø15Ø7;Ø=AØ.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ð7
ð 7
àÔ# dÑ*ð7
ð Ô'¨$Ñ.ð7
ð "Ô-°Ñ4ð	7
ð
   œ9 u¤|Ñ3°dÑ:ð7
ð œ tÑ+ð7
ð Ô&¨Ñ-ð7
ð  ™ð7
ð Ô(¨4Ñ/ð7
ð Ô  4Ñ'ð7
ð ˜$‘;ð7
ð ˜eœlÑ*ð7
ð Ð+Ô,ð7
ð 
Ð.Ñ	.ð7
ð 7
ð 7
ñ „^ñ Ôð7
ð 7
ð 7
ð 7
ð 7
rA   rQ  )rQ  r#  r  rù  r3  )rŽ   )r!   )ZÚcollections.abcr   Údataclassesr   rM   Útorch.nnr9   Útorch.nn.functionalrœ   r™  Ú r   r  Úactivationsr   Úcache_utilsr   r	   Ú
generationr
   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úvision_utilsr    Úconfiguration_step3p7r"   r#   r$   ra  r&   ry   rt   r  r�   rO   r¤   r¨   rs   r²   r´   rÌ   rà   r  r3  r  r&  rv  r  r#  r¹  rË  rÍ  r  rî  r÷  rù  r   r#  rM  rQ  Ú__all__r8  rA   r@   ú<module>rw     s¥  ðð* %Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )Ø RÐ RÐ RÐ RÐ RÐ RÐ RÐ RØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø qÐ qÐ qÐ qÐ qÐ qÐ qÐ qÐ qÐ qÐ qÐ qØ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lØ 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø XÐ XÐ XÐ XÐ XÐ XÐ XÐ XÐ XÐ Xð>5ð >5ð >5ð >5ð >5 2¤9ñ >5ô >5ð >5ðBð ð ð ð �r”yñ ô ð ð	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð& ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð2(ð (ð (ðØ„|ðØœðØ+0¬<ðØ>C¼lðà
ˆ5Œ<˜œÐ%Ô&ðð ð ð ð88ð 88ð 88ð 88ð 88˜RœYñ 88ô 88ð 88ðv"ð "ð "ð "ð "Ð :ñ "ô "ð "ðJ:ð :ð :ð :ð :˜bœiñ :ô :ð :ðz ð('ð ('ð ('ð ('ð ('˜_ñ ('ô ('ñ „ð('ðV ð/[ð /[ð /[ð /[ð /[Ð/ñ /[ô /[ñ „ð/[ðdG<ð G<ð G<ð G<ð G<˜RœYñ G<ô G<ð G<ðT=ð =ð =ð =ð =�R”Yñ =ô =ð =ð,)ð )ð )ð )ð )�”ñ )ô )ð )ð($ð $ð $ð $ð $�R”Yñ $ô $ð $ðN9ð 9ð 9ð 9ð 9˜œ	ñ 9ô 9ð 9ð*ð ð ð ð ˜BœIñ ô ð ð6#ð #ð #ð #ðLM)ð M)ð M)ð M)ð M)�r”yñ M)ô M)ð M)ð`1ð 1ð 1ð 1ð 1Ð4ñ 1ô 1ð 1ðh
°#ð 
¸(ÀCÈÈcÐSVÐCWÐY]ÐC]Ô:^ð 
ð 
ð 
ð 
ð ðe
ð e
ð e
ð e
ð e
Ð-ñ e
ô e
ñ „ðe
ðP €ððñ ô ð
 ð9ð 9ð 9ð 9ð 9Ð!8ñ 9ô 9ñ „ñô ð9ð €ððñ ô ð
G
ð G
ð G
ð G
ð G
Ð)ñ G
ô G
ñô ð
G
ðT €ððñ ô ð
 ð9ð 9ð 9ð 9ð 9 Kñ 9ô 9ñ „ñô ð9ð0 ða
ð a
ð a
ð a
ð a
Ð&<¸oñ a
ô a
ñ „ða
ðHð ð €€€rA   