§
    9²¸jþ  ã                   óÆ  — d dl Z d dlmZ d dlmZ d dlmZ d dlZd dlm	Z	 d dl
m	c mZ ddlmZ ddlmZ ddlmZmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZ ddlmZm Z m!Z! ddl"m#Z#m$Z$ ddl%m&Z&m'Z' ddl(m)Z) ddl*m+Z+m,Z,m-Z-m.Z. ddl/m0Z0 ddl1m2Z2m3Z3 ddl4m5Z5 ddl6m7Z7m8Z8m9Z9 e,e G d„ de ¦  «        ¦   «         ¦   «         Z:d„ Z; ed¦  «        dfd„¦   «         Z<dej=        de>dej=        fd„Z?	 dgd!e	j@        d"ej=        d#ej=        d$ej=        d%ej=        dz  d&eAd'eAd(e)e+         fd)„ZB ee<¦  «         G d*„ d+e	j@        ¦  «        ¦   «         ZC ed,¦  «         G d-„ d.e	j@        ¦  «        ¦   «         ZD G d/„ d0e	j@        ¦  «        ZE G d1„ d2e¦  «        ZF G d3„ d4e	j@        ¦  «        ZG G d5„ d6e	j@        ¦  «        ZH G d7„ d8e	j@        ¦  «        ZI G d9„ d:e	j@        ¦  «        ZJ G d;„ d<e	j@        ¦  «        ZK G d=„ d>e	j@        ¦  «        ZL G d?„ d@e	j@        ¦  «        ZM G dA„ dBe	j@        ¦  «        ZN G dC„ dDe	j@        ¦  «        ZO G dE„ dFe	j@        ¦  «        ZP G dG„ dHe	jQ        ¦  «        ZR G dI„ dJe	j@        ¦  «        ZS G dK„ dLe	j@        ¦  «        ZT G dM„ dNe	j@        ¦  «        ZU G dO„ dPe	j@        ¦  «        ZV G dQ„ dRe	j@        ¦  «        ZW e,dS¬T¦  «         G dU„ dVe'¦  «        ¦   «         ZX G dW„ dX¦  «        ZYe, G dY„ dZe'¦  «        ¦   «         ZZ G d[„ d\e	j@        ¦  «        Z[e, G d]„ d^eZ¦  «        ¦   «         Z\e, G d_„ d`eZe¦  «        ¦   «         Z] G da„ dbeZ¦  «        Z^ G dc„ ddeZe¦  «        Z_g de¢Z`dS )hé    N)ÚCallable)Ú	dataclass)Úcached_propertyé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚBaseModelOutputWithPoolingÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚtorch_compilable_check)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )Ú
Emu3ConfigÚEmu3TextConfigÚEmu3VQVAEConfigc                   ó2   — e Zd ZU dZdZej        dz  ed<   dS )ÚEmu3VQVAEModelOutputz™
    image_tokens (`torch.LongTensor` of shape `(batch_size, config.vocab_size`):
        Indices of the image tokens predicted by the VQ-VAE model.
    NÚimage_tokens)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r&   ÚtorchÚ
LongTensorÚ__annotations__© ó    úv/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/emu3/modeling_emu3.pyr%   r%   1   s6   € € € € € € ðð ð
 -1€L�%Ô" TÑ)Ð0Ð0Ñ0Ð0Ð0r/   r%   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..Néÿÿÿÿé   ©Údim)Úshaper+   Úcat)ÚxÚx1Úx2s      r0   Úrotate_halfr;   <   s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'r/   Úrotary_pos_embc                 ó¾   — |                      |¦  «        }|                      |¦  «        }| |z  t          | ¦  «        |z  z   }||z  t          |¦  «        |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezer;   )ÚqÚkÚcosÚsinÚunsqueeze_dimÚq_embedÚk_embeds          r0   Úapply_rotary_pos_embrF   C   sc   € ð& �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�GÐÐr/   Úhidden_statesÚn_repÚreturnc                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r    N)r6   ÚexpandÚreshape)rG   rH   ÚbatchÚnum_key_value_headsÚslenÚhead_dims         r0   Ú	repeat_kvrQ   ]   s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTr/   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 ó  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        |z  }
|�|
|z   }
t
          j                             |
dt          j        ¬¦  «         	                    |j
        ¦  «        }
t
          j                             |
|| j        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «                             ¦   «         }||
fS )Nr3   r   r2   )r5   Údtype)ÚpÚtrainingr    )rQ   Únum_key_value_groupsr+   ÚmatmulÚ	transposeÚnnÚ
functionalÚsoftmaxÚfloat32Útor\   rY   r^   Ú
contiguous)rS   rT   rU   rV   rW   rX   rY   rZ   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r0   Úeager_attention_forwardrl   i   sé   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r/   c                   óÎ   ‡ — e Zd ZdZdedefˆ fd„Z	 	 	 ddej        de	ej        ej        f         dz  dej        dz  d	e
dz  d
ee         de	ej        ej        f         fd„Zˆ xZS )ÚEmu3Attentionú=Multi-headed attention from 'Attention Is All You Need' paperÚconfigÚ	layer_idxc                 ó®  •— t          ¦   «                              ¦   «          || _        || _        t	          |d|j        |j        z  ¦  «        | _        |j        |j        z  | _	        | j        dz  | _
        |j        | _        d| _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        | j        z  |j        |j        ¬¦  «        | _        d S )NrP   ç      à¿T©Úbias)ÚsuperÚ__init__rp   rq   ÚgetattrÚhidden_sizeÚnum_attention_headsrP   rN   r_   rX   Úattention_dropoutÚ	is_causalrb   ÚLinearÚattention_biasÚq_projÚk_projÚv_projÚo_proj©Úselfrp   rq   Ú	__class__s      €r0   rw   zEmu3Attention.__init__†   sB  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒÝ ¨
°FÔ4FÈ&ÔJdÑ4dÑeÔeˆŒØ$*Ô$>À&ÔB\Ñ$\ˆÔ!Ø”} dÑ*ˆŒØ!'Ô!9ˆÔØˆŒå”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ&¨¬Ñ6¸Ô8JÐQWÔQfð
ñ 
ô 
ˆŒˆˆr/   NrG   Úposition_embeddingsrW   Úpast_key_valuesrZ   rI   c                 ó"  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }	|                      |¦  «                             |¦  «                             dd¦  «        }
|\  }}t          ||	||¦  «        \  }}	|�|                     |	|
| j	        ¦  «        \  }	}
t          j        | j        j        t          ¦  «        } || ||	|
|f| j        sdn| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Nr2   r    r3   rR   )rY   rX   )r6   rP   r   Úviewra   r€   r�   rF   Úupdaterq   r   Úget_interfacerp   Ú_attn_implementationrl   r^   r{   rX   rL   rg   r‚   )r„   rG   r†   rW   r‡   rZ   Úinput_shapeÚhidden_shapeÚquery_statesrh   ri   rA   rB   Úattention_interfacerk   rj   s                   r0   ÚforwardzEmu3Attention.forward�   sÀ  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r/   )NNN)r'   r(   r)   r*   r!   Úintrw   r+   ÚTensorÚtupler	   r   r   r‘   Ú__classcell__©r…   s   @r0   rn   rn   ‚   så   ø€ € € € € àGÐGð
˜zð 
°cð 
ð 
ð 
ð 
ð 
ð 
ð4 IMØ.2Ø(,ð&)ð &)à”|ð&)ð # 5¤<°´Ð#=Ô>ÀÑEð&)ð œ tÑ+ð	&)ð
  ™ð&)ð Ð+Ô,ð&)ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð&)ð &)ð &)ð &)ð &)ð &)ð &)ð &)r/   rn   ÚRMSNormc                   óT   ‡ — e Zd Zd	deddfˆ fd„Zdej        dej        fd„Zd„ Zˆ xZ	S )
ÚEmu3RMSNormç�íµ ÷Æ°>ÚepsrI   Nc                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )z:
        Emu3RMSNorm is equivalent to T5LayerNorm
        N)rv   rw   rb   Ú	Parameterr+   ÚonesÚweightÚvariance_epsilon)r„   ry   r›   r…   s      €r0   rw   zEmu3RMSNorm.__init__È   sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐr/   rG   c                 ó  — |j         }|                     t          j        ¦  «        }|                     d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        |                     |¦  «        z  S )Nr3   r2   T)Úkeepdim)	r\   rf   r+   re   ÚpowÚmeanÚrsqrtr    rŸ   )r„   rG   Úinput_dtypeÚvariances       r0   r‘   zEmu3RMSNorm.forwardÐ   s|   € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØ ×$Ò$ QÑ'Ô'×,Ò,¨R¸Ð,Ñ>Ô>ˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆØŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:r/   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)r”   rŸ   r6   r    ©r„   s    r0   Ú
extra_reprzEmu3RMSNorm.extra_repr×   s&   € Ý˜œÔ)Ñ*Ô*ÐIÐI°$Ô2GÐIÐIÐIr/   )rš   )
r'   r(   r)   Úfloatrw   r+   r“   r‘   rª   r•   r–   s   @r0   r™   r™   Æ   sŒ   ø€ € € € € ð$ð $¨ð $¸$ð $ð $ð $ð $ð $ð $ð; U¤\ð ;°e´lð ;ð ;ð ;ð ;ðJð Jð Jð Jð Jð Jð Jr/   r™   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚEmu3MLPc                 ó¶  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        t          j        | j        | j        |j        ¬¦  «        | _        t          j        | j        | j        |j        ¬¦  «        | _	        t          j        | j        | j        |j        ¬¦  «        | _
        t          |j                 | _        d S )Nrt   )rv   rw   rp   ry   Úintermediate_sizerb   r}   Úmlp_biasÚ	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fn©r„   rp   r…   s     €r0   rw   zEmu3MLP.__init__Ü   s¯   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ!'Ô!9ˆÔÝœ 4Ô#3°TÔ5KÐRXÔRaÐbÑbÔbˆŒÝ”y Ô!1°4Ô3IÐPVÔP_Ð`Ñ`Ô`ˆŒÝœ 4Ô#9¸4Ô;KÐRXÔRaÐbÑbÔbˆŒÝ˜VÔ.Ô/ˆŒˆˆr/   c                 ó¨   — |                       |                      |                      |¦  «        ¦  «        |                      |¦  «        z  ¦  «        }|S ©N)r³   rµ   r±   r²   )r„   r8   r³   s      r0   r‘   zEmu3MLP.forwardæ   sA   € Ø—N’N 4§;¢;¨t¯~ª~¸aÑ/@Ô/@Ñ#AÔ#AÀDÇLÂLÐQRÁOÄOÑ#SÑTÔTˆ	ØÐr/   ©r'   r(   r)   rw   r‘   r•   r–   s   @r0   r­   r­   Û   sG   ø€ € € € € ð0ð 0ð 0ð 0ð 0ðð ð ð ð ð ð r/   r­   c                   óÒ   ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 ddej        dej        dz  dej        dz  d	e	dz  d
e
dz  deej        ej        f         dz  dee         dej        fd„Zˆ xZS )ÚEmu3DecoderLayerrp   rq   c                 óp  •— t          ¦   «                              ¦   «          |j        | _        t          ||¬¦  «        | _        t          |¦  «        | _        t          |j        |j        ¬¦  «        | _	        t          |j        |j        ¬¦  «        | _
        t          j        |j        ¦  «        | _        d S )N)rp   rq   ©r›   )rv   rw   ry   rn   Ú	self_attnr­   Úmlpr™   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormrb   ÚDropoutr{   rY   rƒ   s      €r0   rw   zEmu3DecoderLayer.__init__ì   s“   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔå&¨fÀ	ÐJÑJÔJˆŒå˜6‘?”?ˆŒÝ*¨6Ô+=À6ÔCVÐWÑWÔWˆÔÝ(3°FÔ4FÈFÔL_Ð(`Ñ(`Ô(`ˆÔ%Ý”z &Ô":Ñ;Ô;ˆŒˆˆr/   NFrG   rW   Úposition_idsr‡   Ú	use_cacher†   rZ   rI   c           
      ó  — |}|                       |¦  «        } | j        d||||||dœ|¤Ž\  }}	||                      |¦  «        z   }|}|                      |¦  «        }|                      |¦  «        }||                      |¦  «        z   }|S )N)rG   rW   rÄ   r‡   rÅ   r†   r.   )rÁ   r¾   rY   rÂ   r¿   )
r„   rG   rW   rÄ   r‡   rÅ   r†   rZ   ÚresidualÚ_s
             r0   r‘   zEmu3DecoderLayer.forward÷   s·   € ð !ˆØ×,Ò,¨]Ñ;Ô;ˆà)˜4œ>ð 
Ø'Ø)Ø%Ø+ØØ 3ð
ð 
ð ð
ð 
Ñˆ�qð ! 4§<¢<°Ñ#>Ô#>Ñ>ˆà ˆØ×5Ò5°mÑDÔDˆØŸš Ñ/Ô/ˆØ  4§<¢<°Ñ#>Ô#>Ñ>ˆØÐr/   )NNNFN)r'   r(   r)   r!   r’   rw   r+   r“   r,   r	   Úboolr”   r   r   r‘   r•   r–   s   @r0   r»   r»   ë   s÷   ø€ € € € € ð	<˜zð 	<°cð 	<ð 	<ð 	<ð 	<ð 	<ð 	<ð /3Ø04Ø(,Ø!&ØHLðð à”|ðð œ tÑ+ðð Ô&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð # 5¤<°´Ð#=Ô>ÀÑEðð Ð+Ô,ðð 
Œðð ð ð ð ð ð ð r/   r»   c                   ó>   ‡ — e Zd ZdZdefˆ fd„Zdej        fd„Zˆ xZ	S )ÚEmu3VQVAEVectorQuantizeraã  
    A module for vector quantization using learned embedding vectors.

    This module implements the quantization process similar to the one described in
    the VQ-VAE (Vector Quantized Variational AutoEncoder) paper. It quantizes continuous
    input vectors into discrete codebook vectors, which are learned during training.
    Current implementation improves over previous ones by avoiding costly matrix multiplications
    and allowing for post-hoc remapping of indices.
    rp   c                 óú   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        | j        j        j         	                    d|j        z  d|j        z  ¦  «         d S )Ng      ð¿ç      ð?)
rv   rw   rb   Ú	EmbeddingÚcodebook_sizeÚ	embed_dimÚ	embeddingrŸ   ÚdataÚuniform_r¶   s     €r0   rw   z!Emu3VQVAEVectorQuantizer.__init__!  sf   ø€ Ý‰Œ×ÒÑÔÐÝœ fÔ&:¸FÔ<LÑMÔMˆŒØŒÔÔ"×+Ò+¨D°6Ô3GÑ,GÈÈvÔOcÑIcÑdÔdÐdÐdÐdr/   Úhidden_statec                 óô  — |j         \  }}}}}|                     ddddd¦  «                             ¦   «         }|                     d|¦  «        }t	          j        |dz  dd¬¦  «        }t	          j        | j        j        dz  d¬	¦  «        }	dt	          j        || j        j         	                    dd¦  «        ¦  «        z  }
||	z   |
z
  }
t	          j
        |
d¬	¦  «        }|                     ||||¦  «        }|S )
Nr   r    r   é   r3   r2   T)r5   r¢   r4   )r6   Úpermuterg   r‰   r+   ÚsumrÑ   rŸ   r`   ra   Úargmin)r„   rÔ   Ú
batch_sizeÚtemporalÚchannelsÚheightÚwidthÚhidden_state_flattenedÚhidden_state_sumÚembedding_sumÚ	distancesÚmin_encoding_indicess               r0   r‘   z Emu3VQVAEVectorQuantizer.forward&  s  € Ø8DÔ8JÑ5ˆ
�H˜h¨°Ø#×+Ò+¨A¨q°!°Q¸Ñ:Ô:×EÒEÑGÔGˆØ!-×!2Ò!2°2°xÑ!@Ô!@Ðõ !œ9Ð%;¸QÑ%>ÀAÈtÐTÑTÔTÐÝœ	 $¤.Ô"7¸Ñ":ÀÐBÑBÔBˆð �œÐ%;¸T¼^Ô=R×=\Ò=\Ð]^Ð`aÑ=bÔ=bÑcÔcÑcˆ	Ø$ }Ñ4°yÑ@ˆ	å$œ|¨I¸1Ð=Ñ=Ô=ÐØ3×8Ò8¸ÀXÈvÐW\Ñ]Ô]ÐØ#Ð#r/   )
r'   r(   r)   r*   r#   rw   r+   r“   r‘   r•   r–   s   @r0   rË   rË     sr   ø€ € € € € ðð ðe˜ð eð eð eð eð eð eð
$ E¤Lð $ð $ð $ð $ð $ð $ð $ð $r/   rË   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚEmu3VQVAEEncoderConvDownsamplec                 ó„   •— t          ¦   «                              ¦   «          t          j        ||ddd¬¦  «        | _        d S )Nr   r3   r   ©Úkernel_sizeÚstrideÚpadding©rv   rw   rb   ÚConv2dÚconv©r„   Úin_channelsr…   s     €r0   rw   z'Emu3VQVAEEncoderConvDownsample.__init__9  ó:   ø€ Ý‰Œ×ÒÑÔÐÝ”I˜k¨;ÀAÈaÐYZÐ[Ñ[Ô[ˆŒ	ˆ	ˆ	r/   c                 ó`   — t          j        |ddd¬¦  «        }|                      |¦  «        }|S )N)r   r    r   r    Úconstantr   )ÚpadÚmoderV   )ÚFró   rí   ©r„   rG   s     r0   r‘   z&Emu3VQVAEEncoderConvDownsample.forward=  s2   € åœ˜m°ÀJÐVWÐXÑXÔXˆØŸ	š	 -Ñ0Ô0ˆØÐr/   r¹   r–   s   @r0   rå   rå   8  sL   ø€ € € € € ð\ð \ð \ð \ð \ðð ð ð ð ð ð r/   rå   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚEmu3VQVAEEncoderConvUpsamplec                 ó„   •— t          ¦   «                              ¦   «          t          j        ||ddd¬¦  «        | _        d S )Nr   r    rç   rë   rî   s     €r0   rw   z%Emu3VQVAEEncoderConvUpsample.__init__E  rð   r/   c                 ó^   — t          j        |dd¬¦  «        }|                      |¦  «        }|S )Nç       @Únearest©Úscale_factorrô   )rõ   Úinterpolaterí   rö   s     r0   r‘   z$Emu3VQVAEEncoderConvUpsample.forwardI  s/   € Ýœ mÀ#ÈIÐVÑVÔVˆØŸ	š	 -Ñ0Ô0ˆØÐr/   r¹   r–   s   @r0   rø   rø   D  sL   ø€ € € € € ð\ð \ð \ð \ð \ðð ð ð ð ð ð r/   rø   c            	       ó^   ‡ — e Zd Zdededee         dee         fˆ fd„Zdej        fd„Zˆ xZ	S )ÚEmu3VQVAEConv3dÚ
in_channelÚout_channelrè   ré   c                 ó\  •— t          ¦   «                              ¦   «          d„ t          |dd …         |dd …         ¦  «        D ¦   «         }d| _        |d d d…         D ] }| xj        |dz  |dz  z   |dz  fz  c_        Œ!| xj        dz  c_        t	          j        ||||¬¦  «        | _        d S )Nc                 ó   — g | ]
\  }}||z
  ‘ŒS r.   r.   )Ú.0Ú
one_kernelÚ
one_strides      r0   ú
<listcomp>z,Emu3VQVAEConv3d.__init__.<locals>.<listcomp>Y  s"   € ÐpÐpÐpÑ5K°ZÀ˜ jÑ0ÐpÐpÐpr/   r    r.   r2   r3   )r3   r   )ré   )rv   rw   Úziprê   rb   ÚConv3drí   )r„   r  r  rè   ré   Úpadding_sizesÚpad_sizer…   s          €r0   rw   zEmu3VQVAEConv3d.__init__P  sÕ   ø€ õ 	‰Œ×ÒÑÔÐàpÐpÍsÐS^Ð_`Ð_aÐ_aÔSbÐdjÐklÐkmÐkmÔdnÑOoÔOoÐpÑpÔpˆØˆŒØ% d d¨ dÔ+ð 	Jð 	JˆHØˆLŒL˜X¨™]¨X¸©\Ñ9¸8Àq¹=ÐIÑIˆLŒLˆLØˆŒ˜ÑˆŒå”IØØØØð	
ñ 
ô 
ˆŒ	ˆ	ˆ	r/   rG   c                 ód   — t          j        || j        ¦  «        }|                      |¦  «        }|S r¸   )rõ   ró   rê   rí   rö   s     r0   r‘   zEmu3VQVAEConv3d.forwardf  s,   € Ýœ˜m¨T¬\Ñ:Ô:ˆØŸ	š	 -Ñ0Ô0ˆØÐr/   )
r'   r(   r)   r’   r”   rw   r+   r“   r‘   r•   r–   s   @r0   r  r  O  sˆ   ø€ € € € € ð
àð
ð ð
ð ˜3”Zð	
ð
 �c”
ð
ð 
ð 
ð 
ð 
ð 
ð, U¤\ð ð ð ð ð ð ð ð r/   r  c                   óL   ‡ — e Zd Zdedefˆ fd„Zdej        dej        fd„Zˆ xZS )ÚEmu3VQVAESpatialNormrï   Úout_channelsc                 óú   •— t          ¦   «                              ¦   «          t          j        |ddd¬¦  «        | _        t          j        ||ddd¬¦  «        | _        t          j        ||ddd¬¦  «        | _        d S )Né    rš   T©Únum_channelsÚ
num_groupsr›   Úaffiner    r   rç   )rv   rw   rb   Ú	GroupNormÚ
norm_layerrì   Úconv_yÚconv_b©r„   rï   r  r…   s      €r0   rw   zEmu3VQVAESpatialNorm.__init__m  s•   ø€ õ
 	‰Œ×ÒÑÔÐÝœ,Ø%ØØØð	
ñ 
ô 
ˆŒõ ”iØØØØØð
ñ 
ô 
ˆŒõ ”iØØØØØð
ñ 
ô 
ˆŒˆˆr/   rG   Úquant_statesc                 óÔ   — t          j        ||j        dd …         d¬¦  «        }|                      |¦  «        }||                      |¦  «        z  |                      |¦  «        z   }|S )Néþÿÿÿrü   )Úsizerô   )rõ   rÿ   r6   r  r  r  )r„   rG   r  s      r0   r‘   zEmu3VQVAESpatialNorm.forward‰  sd   € Ý”} \¸Ô8KÈBÈCÈCÔ8PÐW`ÐaÑaÔaˆØŸš¨Ñ6Ô6ˆØ%¨¯ª°LÑ(AÔ(AÑAÀDÇKÂKÐP\ÑD]ÔD]Ñ]ˆØÐr/   ©	r'   r(   r)   r’   rw   r+   r“   r‘   r•   r–   s   @r0   r  r  l  su   ø€ € € € € ð
àð
ð ð
ð 
ð 
ð 
ð 
ð 
ð8 U¤\ð ÀÄð ð ð ð ð ð ð ð r/   r  c                   ó>   ‡ — e Zd Zdedefˆ fd„Zdej        fd„Zˆ xZS )ÚEmu3VQVAETemporalUpsampler  r  c                 óx   •— t          ¦   «                              ¦   «          t          ||dd¬¦  «        | _        d S )N©r   r   r   ©r    r    r    ©rè   ré   ©rv   rw   r  rí   ©r„   r  r  r…   s      €r0   rw   z"Emu3VQVAETemporalUpsample.__init__‘  óA   ø€ õ
 	‰Œ×ÒÑÔÐÝ#ØØØ!Øð	
ñ 
ô 
ˆŒ	ˆ	ˆ	r/   rG   c                 ó|  — |j         \  }}}}}|                     ddddd¦  «                             ¦   «                              |d|¦  «        }t	          j        |dd¬	¦  «        }|                     ||||d¦  «                             ddddd¦  «                             ¦   «         }|                      |¦  «        }|S )
Nr   r    r   rÖ   r3   r2   rû   rü   rý   )r6   r×   rg   r‰   rõ   rÿ   rí   )r„   rG   rÚ   rÜ   rÛ   rÝ   rÞ   s          r0   r‘   z!Emu3VQVAETemporalUpsample.forwardž  sÀ   € Ø8EÔ8KÑ5ˆ
�H˜h¨°Ø%×-Ò-¨a°°A°q¸!Ñ<Ô<×GÒGÑIÔI×NÒNÈzÐ[]Ð_gÑhÔhˆÝœ mÀ#ÈIÐVÑVÔVˆØ%×*Ò*¨:°xÀÈÐPRÑSÔS×[Ò[Ð\]Ð_`ÐbcÐefÐhiÑjÔj×uÒuÑwÔwˆØŸ	š	 -Ñ0Ô0ˆØÐr/   r!  r–   s   @r0   r#  r#  �  sl   ø€ € € € € ð
àð
ð ð
ð 
ð 
ð 
ð 
ð 
ð U¤\ð ð ð ð ð ð ð ð r/   r#  c                   ó>   ‡ — e Zd Zdedefˆ fd„Zdej        fd„Zˆ xZS )ÚEmu3VQVAETemporalDownsampler  r  c                 óx   •— t          ¦   «                              ¦   «          t          ||dd¬¦  «        | _        d S )N)rÖ   r   r   )r3   r    r    r'  r(  r)  s      €r0   rw   z$Emu3VQVAETemporalDownsample.__init__¨  r*  r/   rG   c                 ó0   — |                       |¦  «        }|S r¸   )rí   rö   s     r0   r‘   z#Emu3VQVAETemporalDownsample.forwardµ  s   € ØŸ	š	 -Ñ0Ô0ˆØÐr/   r!  r–   s   @r0   r-  r-  §  sl   ø€ € € € € ð
àð
ð ð
ð 
ð 
ð 
ð 
ð 
ð U¤\ð ð ð ð ð ð ð ð r/   r-  c                   ó(   ‡ — e Zd Z	 dˆ fd„	Zd„ Zˆ xZS )ÚEmu3VQVAETemporalResnetBlockNc                 ó�  •— t          ¦   «                              ¦   «          || _        |€|n|| _        t	          j        |¦  «        | _        t          ||dd¬¦  «        | _        t	          j        |¦  «        | _	        t          ||dd¬¦  «        | _
        | j        | j        k    r t	          j        ||ddd¬¦  «        | _        d S d S )Nr%  r&  r'  r    r   rç   )rv   rw   rï   r  rb   ÚBatchNorm3dÚnorm1r  Úconv1Únorm2Úconv2r  Únin_shortcutr  s      €r0   rw   z%Emu3VQVAETemporalResnetBlock.__init__»  så   ø€ õ
 	‰Œ×ÒÑÔÐØ&ˆÔØ+7Ð+?˜K˜KÀ\ˆÔå”^ KÑ0Ô0ˆŒ
Ý$ØØØ!Øð	
ñ 
ô 
ˆŒ
õ ”^ LÑ1Ô1ˆŒ
Ý$ØØØ!Øð	
ñ 
ô 
ˆŒ
ð Ô˜tÔ0Ò0Ð0Ý "¤	ØØØØØð!ñ !ô !ˆDÔÐÐð 1Ð0r/   c                 ó^  — |}|                       |¦  «        }|t          j        |¦  «        z  }|                      |¦  «        }|                      |¦  «        }|t          j        |¦  «        z  }|                      |¦  «        }| j        | j        k    r|                      |¦  «        }||z   S r¸   )	r4  r+   Úsigmoidr5  r6  r7  rï   r  r8  )r„   rG   rÇ   s      r0   r‘   z$Emu3VQVAETemporalResnetBlock.forwardÛ  s¢   € Ø ˆØŸ
š
 =Ñ1Ô1ˆØ�œ }Ñ5Ô5Ñ5ˆØŸ
š
 =Ñ1Ô1ˆàŸ
š
 =Ñ1Ô1ˆØ�œ }Ñ5Ô5Ñ5ˆØŸ
š
 =Ñ1Ô1ˆàÔ˜tÔ0Ò0Ð0Ø×(Ò(¨Ñ2Ô2ˆHà˜-Ñ'Ð'r/   r¸   r¹   r–   s   @r0   r1  r1  º  sR   ø€ € € € € ð ðð ð ð ð ð ð@(ð (ð (ð (ð (ð (ð (r/   r1  c                   ój   ‡ — e Zd Z	 	 ddededz  dedz  fˆ fd„Zd	dej        dej        dz  fd„Zˆ xZS )
ÚEmu3VQVAEResnetBlockNrï   r  Úquant_channelsc                 ó$  •— t          ¦   «                              ¦   «          || _        |€|n|}|| _        || _        |€;t          j        |ddd¬¦  «        | _        t          j        |ddd¬¦  «        | _        n*t          ||¦  «        | _        t          ||¦  «        | _        t          j
        ||ddd¬¦  «        | _        t          j
        ||ddd¬¦  «        | _        | j        | j        k    r t          j
        ||ddd¬¦  «        | _        d S d S )	Nr  rš   Tr  r   r    rç   r   )rv   rw   rï   r  r=  rb   r  r4  r6  r  rì   r5  r7  r8  )r„   rï   r  r=  r…   s       €r0   rw   zEmu3VQVAEResnetBlock.__init__ì  s<  ø€ õ 	‰Œ×ÒÑÔÐØ&ˆÔØ&2Ð&:�{�{ÀˆØ(ˆÔØ,ˆÔàÐ!Ýœ°;È2ÐSWÐ`dÐeÑeÔeˆDŒJÝœ°<ÈBÐTXÐaeÐfÑfÔfˆDŒJˆJå-¨n¸kÑJÔJˆDŒJÝ-¨n¸lÑKÔKˆDŒJå”YØØØØØð
ñ 
ô 
ˆŒ
õ ”YØØØØØð
ñ 
ô 
ˆŒ
ð Ô˜tÔ0Ò0Ð0Ý "¤	ØØØØØð!ñ !ô !ˆDÔÐÐð 1Ð0r/   rG   c                 óZ  — | j         €dn|f}|} | j        |g|¢R Ž }|t          j        |¦  «        z  }|                      |¦  «        } | j        |g|¢R Ž }|t          j        |¦  «        z  }|                      |¦  «        }| j        | j        k    r|  	                    |¦  «        }||z   S ©Nr.   )
r=  r4  r+   r:  r5  r6  r7  rï   r  r8  )r„   rG   r=  Ú	norm_argsrÇ   s        r0   r‘   zEmu3VQVAEResnetBlock.forward  sÆ   € ØÔ-Ð5�B�B¸NÐ;Lˆ	à ˆØ"˜œ
 =Ð=°9Ð=Ð=Ð=ˆØ�œ }Ñ5Ô5Ñ5ˆØŸ
š
 =Ñ1Ô1ˆà"˜œ
 =Ð=°9Ð=Ð=Ð=ˆØ�œ }Ñ5Ô5Ñ5ˆØŸ
š
 =Ñ1Ô1ˆàÔ˜tÔ0Ò0Ð0Ø×(Ò(¨Ñ2Ô2ˆHà˜-Ñ'Ð'r/   )NNr¸   r!  r–   s   @r0   r<  r<  ë  s    ø€ € € € € ð $(Ø%)ð	*ð *àð*ð ˜D‘jð*ð ˜d™
ð	*ð *ð *ð *ð *ð *ðX(ð ( U¤\ð (À5Ä<ÐRVÑCVð (ð (ð (ð (ð (ð (ð (ð (r/   r<  c            
       ó„   ‡ — e Zd ZdZdefˆ fd„Z	 d	dej        dej        dz  deej        ej        dz  f         fd„Z	ˆ xZ
S )
ÚEmu3VQVAEAttentionBlockro   rp   c                 ó�  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        | j        | j        z  | _        | j        | j        z  | j        k    r t          d| j        › d| j        › d�¦  «        ‚| j        dz  | _	        |j
        | _        d| _        t          j        | j        | j        ¦  «        | _        t          j        | j        | j        ¦  «        | _        t          j        | j        | j        ¦  «        | _        t          j        | j        | j        ¦  «        | _        d| _        d S )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).rs   Fr    )rv   rw   rp   ry   rÐ   rz   Ú	num_headsrP   Ú
ValueErrorÚscaler{   rY   r|   rb   r}   r€   r�   r   Úout_projr_   r¶   s     €r0   rw   z Emu3VQVAEAttentionBlock.__init__-  s   ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ+ˆŒØÔ3ˆŒØœ¨$¬.Ñ8ˆŒØŒ=˜4œ>Ñ)¨T¬^Ò;Ð;Ýð'ÈdÌnð 'ð 'Ø”Nð'ð 'ð 'ñô ð ð ”] DÑ(ˆŒ
ØÔ/ˆŒØˆŒå”i ¤°´Ñ?Ô?ˆŒÝ”i ¤°´Ñ?Ô?ˆŒÝ”i ¤°´Ñ?Ô?ˆŒÝœ	 $¤.°$´.ÑAÔAˆŒð %&ˆÔ!Ð!Ð!r/   NrG   rW   rI   c           
      ó¼  — |j         dd…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }t          j        | j	        j
        t          ¦  «        }	 |	| ||||| j        | j        | j        sdn| j        ¬¦  «        \  }
} |
j        g |¢d‘R Ž                      ¦   «         }
|                      |
¦  «        }
|
|fS )z#Input shape: Batch x Time x ChannelNr2   r    r3   rR   )r|   rX   rY   )r6   rP   r   r‰   ra   r€   r�   r   r‹   rp   rŒ   rl   r|   rG  r^   rY   rL   rg   rH  )r„   rG   rW   rZ   r�   rŽ   ÚqueriesÚkeysÚvaluesr�   rk   rj   s               r0   r‘   zEmu3VQVAEAttentionBlock.forwardD  sg  € ð $Ô)¨#¨2¨#Ô.ˆà8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆØ—+’+˜mÑ,Ô,×1Ò1°,Ñ?Ô?×IÒIÈ!ÈQÑOÔOˆØ�{Š{˜=Ñ)Ô)×.Ò.¨|Ñ<Ô<×FÒFÀqÈ!ÑLÔLˆØ—’˜]Ñ+Ô+×0Ò0°Ñ>Ô>×HÒHÈÈAÑNÔNˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØØ”nØ”JØ#œ}Ð>�C�C°$´,ð	%
ñ 	%
ô 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—m’m KÑ0Ô0ˆà˜LÐ(Ð(r/   r¸   )r'   r(   r)   r*   r#   rw   r+   r“   r”   r‘   r•   r–   s   @r0   rC  rC  *  s    ø€ € € € € ØGÐGð&˜ð &ð &ð &ð &ð &ð &ð4 /3ð!)ð !)à”|ð!)ð œ tÑ+ð!)ð
 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð!)ð !)ð !)ð !)ð !)ð !)ð !)ð !)r/   rC  c                   ó*   ‡ — e Zd ZdZˆ fd„Zdd„Zˆ xZS )ÚEmu3VQVAEGroupNormzá
    Same as the torch GroupNorm with the only difference that this ones accepts
    an optional kwarg `quant_states` which is not used. This class makes it easier to
    use SpatialNorm or GroupNorm without conditionals
    c                 ó:   •—  t          ¦   «         j        di |¤Ž d S r@  )rv   rw   )r„   rZ   r…   s     €r0   rw   zEmu3VQVAEGroupNorm.__init__o  s&   ø€ Ø�‰ŒÔÐ"Ð"˜6Ð"Ð"Ð"Ð"Ð"r/   Nc                 óZ   — t          j        || j        | j        | j        | j        ¦  «        S r¸   )rõ   Ú
group_normr  rŸ   ru   r›   )r„   Úinputr  s      r0   r‘   zEmu3VQVAEGroupNorm.forwardr  s"   € ÝŒ|˜E 4¤?°D´KÀÄÈDÌHÑUÔUÐUr/   r¸   )r'   r(   r)   r*   rw   r‘   r•   r–   s   @r0   rN  rN  h  s^   ø€ € € € € ðð ð#ð #ð #ð #ð #ðVð Vð Vð Vð Vð Vð Vð Vr/   rN  c                   óL   ‡ — e Zd Zdˆ fd„	Zddej        dej        dz  fd„Zˆ xZS )ÚEmu3VQVAEMiddleBlockNc                 ó,  •— t          ¦   «                              ¦   «          t          |||¬¦  «        | _        t	          |¦  «        | _        |€t          |ddd¬¦  «        | _        nt          ||¦  «        | _        t          |||¬¦  «        | _	        d S )N©rï   r  r=  r  rš   Tr  )
rv   rw   r<  Úblock_1rC  Úattn_1rN  Ú	attn_normr  Úblock_2)r„   rp   rï   r=  r…   s       €r0   rw   zEmu3VQVAEMiddleBlock.__init__w  s    ø€ Ý‰Œ×ÒÑÔÐå+Ø#Ø$Ø)ð
ñ 
ô 
ˆŒõ
 .¨fÑ5Ô5ˆŒØÐ!Ý/¸[ÐUWÐ]aÐjnÐoÑoÔoˆDŒNˆNå1°.À+ÑNÔNˆDŒNå+Ø#Ø$Ø)ð
ñ 
ô 
ˆŒˆˆr/   rG   r  c                 óž  — |                       ||¦  «        }|}|                      ||¦  «        }|j        \  }}}}|                     ||||z  ¦  «                             dd¦  «        }|                      |¦  «        d         }|                     ||||¦  «                             dddd¦  «        }||z   }|                      ||¦  «        }|S )Nr    r3   r   r   )	rW  rY  r6   r‰   ra   rX  rL   r×   rZ  )r„   rG   r  rÇ   rÚ   rÜ   rÝ   rÞ   s           r0   r‘   zEmu3VQVAEMiddleBlock.forward‹  sÕ   € ØŸš ]°LÑAÔAˆØ ˆØŸš }°lÑCÔCˆØ.;Ô.AÑ+ˆ
�H˜f eØ%×*Ò*¨:°xÀÈ%ÁÑPÔP×ZÒZÐ[\Ð^_Ñ`Ô`ˆØŸš MÑ2Ô2°1Ô5ˆØ%×-Ò-¨j¸&À%ÈÑRÔR×ZÒZÐ[\Ð^_ÐabÐdeÑfÔfˆØ  =Ñ0ˆØŸš ]°LÑAÔAˆØÐr/   r¸   ©r'   r(   r)   rw   r+   ÚFloatTensorr‘   r•   r–   s   @r0   rT  rT  v  sp   ø€ € € € € ð
ð 
ð 
ð 
ð 
ð 
ð(
ð 
 UÔ%6ð 
ÀeÔFWÐZ^ÑF^ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r/   rT  c                   ó4   ‡ — e Zd Zˆ fd„Zdej        fd„Zˆ xZS )ÚEmu3VQVAEDownBlockc           
      óÜ  •— t          ¦   «                              ¦   «          t          |j        ¦  «        | _        |j        | _        |j        }|j        }dt          |¦  «        z   }|| _        t          j
        ¦   «         | _        t          | j        ¦  «        D �]P}t          j
        ¦   «         }t          j
        ¦   «         }t          j
        ¦   «         }|||         z  }	|||         z  }
t          | j        ¦  «        D ]…}|                     t          |	|
¬¦  «        ¦  «         |
}	|j        �V||j        v rM|                     t!          |¦  «        ¦  «         |                     t          j        |	ddd¬¦  «        ¦  «         Œ†t          j        ¦   «         }||_        ||_        ||_        || j        dz
  k    rt-          |	¦  «        |_        | j                             |¦  «         �ŒRd S )N©r    ©rï   r  r  rš   Tr  r    )rv   rw   ÚlenÚchannel_multiplierÚnum_resolutionsÚnum_res_blocksÚbase_channelsr”   Úin_channel_multiplierrb   Ú
ModuleListÚdownÚrangeÚappendr<  Úattn_resolutionsrC  r  ÚModuleÚblockÚattnÚ
attn_normsrå   Ú
downsample)r„   rp   rg  rd  rh  Úi_levelro  rp  rq  Úblock_inÚ	block_outÚi_blockrj  r…   s                €r0   rw   zEmu3VQVAEDownBlock.__init__™  sÞ  ø€ Ý‰Œ×ÒÑÔÐå" 6Ô#<Ñ=Ô=ˆÔØ$Ô3ˆÔØÔ,ˆØ#Ô6Ðà $¥uÐ-?Ñ'@Ô'@Ñ @ÐØ%:ˆÔ"Ý”M‘O”OˆŒ	Ý˜TÔ1Ñ2Ô2ð 	#ñ 	#ˆGÝ”M‘O”OˆEÝ”=‘?”?ˆDÝœ™œˆJØ$Ð'<¸WÔ'EÑEˆHØ%Ð(:¸7Ô(CÑCˆIÝ  Ô!4Ñ5Ô5ð 
qð 
q�Ø—’Ý(Ø$,Ø%.ðñ ô ñô ð ð %�ØÔ*Ð6¸7ÀfÔF]Ð;]Ð;]Ø—K’KÕ 7¸Ñ ?Ô ?Ñ@Ô@Ð@Ø×%Ò%¥b¤lÀÐUWÐ]aÐjnÐ&oÑ&oÔ&oÑpÔpÐpøå”9‘;”;ˆDØˆDŒJØˆDŒIØ(ˆDŒOØ˜$Ô.°Ñ2Ò2Ð2Ý"@ÀÑ"JÔ"J�”ØŒI×Ò˜TÑ"Ô"Ð"Ñ"ð1	#ð 	#r/   rG   c                 óP  — t          | j        ¦  «        D �]\  }}t          | j        ¦  «        D ]Ñ} |j        |         |¦  «        }t          |j        ¦  «        dk    r¡|} |j        |         |¦  «        }|j        \  }}}}	| 	                    ||||	z  ¦  «         
                    dd¦  «        } |j        |         |¦  «        d         }|                     |||	|¦  «                             dddd¦  «        }||z   }ŒÒ|| j        dz
  k    r|                     |¦  «        }�Œ|S )Nr   r    r3   r   )Ú	enumeraterj  rk  rf  ro  rc  rp  rq  r6   r‰   ra   rL   r×   re  rr  )
r„   rG   rs  Úblocksrv  rÇ   rÚ   rÜ   rÝ   rÞ   s
             r0   r‘   zEmu3VQVAEDownBlock.forward¾  sF  € Ý(¨¬Ñ3Ô3ð 	Añ 	A‰OˆG�VÝ  Ô!4Ñ5Ô5ð =ð =�Ø 5 ¤¨WÔ 5°mÑ DÔ D�Ý�v”{Ñ#Ô# aÒ'Ð'Ø,�HØ$> FÔ$5°gÔ$>¸}Ñ$MÔ$M�Mà:GÔ:MÑ7�J ¨&°%Ø$1×$6Ò$6°zÀ8ÈVÐV[É^Ñ$\Ô$\×$fÒ$fÐghÐjkÑ$lÔ$l�MØ$8 F¤K°Ô$8¸Ñ$GÔ$GÈÔ$J�Mà$1×$9Ò$9¸*ÀfÈeÐU]Ñ$^Ô$^×$fÒ$fÐghÐjkÐmnÐpqÑ$rÔ$r�MØ$,¨}Ñ$<�Møà˜$Ô.°Ñ2Ò2Ð2Ø &× 1Ò 1°-Ñ @Ô @�ùàÐr/   r\  r–   s   @r0   r_  r_  ˜  sW   ø€ € € € € ð##ð ##ð ##ð ##ð ##ðJ UÔ%6ð ð ð ð ð ð ð ð r/   r_  c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚEmu3VQVAEUpBlockc           	      óº  •— t          ¦   «                              ¦   «          t          |j        ¦  «        | _        |j        | _        |j        }|j        |j        d         z  }t          j	        ¦   «         | _
        t          t          | j        ¦  «        ¦  «        D �]=}t          j	        ¦   «         }t          j	        ¦   «         }t          j	        ¦   «         }|j        |j        |         z  }t          | j        dz   ¦  «        D ]w}	|                     t          |||¬¦  «        ¦  «         |}||j        v rE|                     t!          |¦  «        ¦  «         |                     t#          ||¦  «        ¦  «         Œxt          j        ¦   «         }
||
_        ||
_        ||
_        |dk    rt-          |¦  «        |
_        | j
                             d|
¦  «         �Œ?d S )Nr2   r    rV  r   )rv   rw   rc  rd  re  rf  rÐ   rg  rb   ri  ÚupÚreversedrk  rl  r<  rm  rC  r  rn  ro  rp  rq  rø   ÚupsampleÚinsert)r„   rp   r=  rt  rs  ro  rp  rq  ru  rv  r}  r…   s              €r0   rw   zEmu3VQVAEUpBlock.__init__Ô  s¶  ø€ Ý‰Œ×ÒÑÔÐå" 6Ô#<Ñ=Ô=ˆÔØ$Ô3ˆÔàÔ)ˆØÔ'¨&Ô*CÀBÔ*GÑGˆå”-‘/”/ˆŒÝ¥ dÔ&:Ñ ;Ô ;Ñ<Ô<ð 	"ñ 	"ˆGÝ”M‘O”OˆEÝ”=‘?”?ˆDÝœ™œˆJØÔ,¨vÔ/HÈÔ/QÑQˆIÝ  Ô!4°qÑ!8Ñ9Ô9ð Vð V�Ø—’Ý(Ø$,Ø%.Ø'5ðñ ô ñô ð ð %�Ø˜fÔ5Ð5Ð5Ø—K’KÕ 7¸Ñ ?Ô ?Ñ@Ô@Ð@Ø×%Ò%Õ&:¸>È8Ñ&TÔ&TÑUÔUÐUøå”‘”ˆBØˆBŒHØˆBŒGØ&ˆBŒMØ˜!Š|ˆ|Ý:¸8ÑDÔD�”àŒG�NŠN˜1˜bÑ!Ô!Ð!Ñ!ð3	"ð 	"r/   rG   r  c                 ó†  — t          | j        d d d…         ¦  «        D �]!\  }}t          | j        dz   ¦  «        D ]Ó} |j        |         ||¦  «        }t          |j        ¦  «        dk    r¢|} |j        |         ||¦  «        }|j        \  }}}	}
| 	                    |||	|
z  ¦  «         
                    dd¦  «        } |j        |         |¦  «        d         }|                     ||	|
|¦  «                             dddd¦  «        }||z   }ŒÔ|t          | j        ¦  «        dz
  k    r|                     |¦  «        }�Œ#|S )Nr2   r    r   r3   r   )rx  r}  rk  rf  ro  rc  rp  rq  r6   r‰   ra   rL   r×   r  )r„   rG   r  rs  ry  rv  rÇ   rÚ   rÜ   rÝ   rÞ   s              r0   r‘   zEmu3VQVAEUpBlock.forwardù  sZ  € Ý(¨¬°°°2°¬Ñ7Ô7ð 	?ñ 	?‰OˆG�VÝ  Ô!4°qÑ!8Ñ9Ô9ð =ð =�Ø 5 ¤¨WÔ 5°mÀ\Ñ RÔ R�Ý�v”{Ñ#Ô# aÒ'Ð'Ø,�HØ$> FÔ$5°gÔ$>¸}ÈlÑ$[Ô$[�Mà:GÔ:MÑ7�J ¨&°%Ø$1×$6Ò$6°zÀ8ÈVÐV[É^Ñ$\Ô$\×$fÒ$fÐghÐjkÑ$lÔ$l�MØ$8 F¤K°Ô$8¸Ñ$GÔ$GÈÔ$J�Mà$1×$9Ò$9¸*ÀfÈeÐU]Ñ$^Ô$^×$fÒ$fÐghÐjkÐmnÐpqÑ$rÔ$r�MØ$,¨}Ñ$<�MøØ�#˜dœg™,œ,¨Ñ*Ò*Ð*Ø &§¢°Ñ >Ô >�ùàÐr/   r\  r–   s   @r0   r{  r{  Ó  sa   ø€ € € € € ð#"ð #"ð #"ð #"ð #"ðJ UÔ%6ð ÀeÔFWð ð ð ð ð ð ð ð r/   r{  c                   ó4   ‡ — e Zd Zˆ fd„Zdej        fd„Zˆ xZS )ÚEmu3VQVAEEncoderc                 ó¤  •— t          ¦   «                              ¦   «          |j        }|j        }|j        }|j        }|j        }|rd|z  n|}||d         z  }t          j         	                    ||ddd¬¦  «        | _
        t          |¦  «        | _        t          ||¦  «        | _        t          j                             d|dd¬	¦  «        | _        t          j         	                    ||ddd¬¦  «        | _        t%          t'          j        |j        ¦  «        ¦  «        }	t          j        ¦   «         | _        t          j        ¦   «         | _        t3          |	¦  «        D ],}
t5          ||¦  «        }| j                             |¦  «         Œ-t3          |j        ¦  «        D ]-}t;          ||¬
¦  «        }| j                             |¦  «         Œ.d S )Nr3   r2   r   r    rç   r  rš   T)r  r  r›   r  rb  )rv   rw   rg  rï   Údouble_latentÚlatent_channelsrd  r+   rb   rì   Úconv_inr_  Ú
down_blockrT  Úmiddle_blockr  Únorm_outÚconv_outr’   ÚmathÚlog2Útemporal_downsample_factorri  Ú	time_convÚtime_res_stackrk  r-  rl  rf  r1  )r„   rp   rg  rï   r…  r†  rd  r  rt  Útemporal_down_blocksÚirí   rÈ   Útime_res_convr…   s                 €r0   rw   zEmu3VQVAEEncoder.__init__  sÇ  ø€ Ý‰Œ×ÒÑÔÐàÔ,ˆØÔ(ˆØÔ,ˆØ Ô0ˆØ#Ô6ÐØ.;ÐP�q˜?Ñ*Ð*ÀˆØ Ð#5°bÔ#9Ñ9ˆå”x—’ {°MÈqÐYZÐde�ÑfÔfˆŒÝ,¨VÑ4Ô4ˆŒÝ0°¸ÑBÔBˆÔåœ×*Ò*°bÀxÐUYÐbfÐ*ÑgÔgˆŒÝœŸšØØØØØð (ñ 
ô 
ˆŒõ  #¥4¤9¨VÔ-NÑ#OÔ#OÑPÔPÐÝœ™œˆŒÝ œm™oœoˆÔåÐ+Ñ,Ô,ð 	(ð 	(ˆAÝ.¨|¸\ÑJÔJˆDØŒN×!Ò! $Ñ'Ô'Ð'Ð'å�vÔ,Ñ-Ô-ð 	6ð 	6ˆAÝ8Ø(Ø)ðñ ô ˆMð Ô×&Ò& }Ñ5Ô5Ð5Ð5ð	6ð 	6r/   Úpixel_valuesc                 ót  — |j         d         } |j        dg|j         dd …         ¢R Ž }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|t          j        |¦  «        z  }|                      |¦  «        } |j        d|g|j         dd …         ¢R Ž }| 	                    ddddd¦  «        }| j
        D ]$} ||¦  «        }|t          j        |¦  «        z  }Œ%| j        D ]} ||¦  «        }Œ| 	                    ddddd¦  «        }|S )Nr    r2   r3   r   r   rÖ   )r6   rL   r‡  rˆ  r‰  rŠ  r+   r:  r‹  r×   r�  r�  )r„   r”  Útemporal_dimrG   rí   Úlayers         r0   r‘   zEmu3VQVAEEncoder.forward5  sf  € Ø#Ô)¨!Ô,ˆØ+�|Ô+¨BÐH°Ô1CÀAÀBÀBÔ1GÐHÐHÐHˆð Ÿš \Ñ2Ô2ˆØŸš¨Ñ6Ô6ˆØ×)Ò)¨-Ñ8Ô8ˆð Ÿš mÑ4Ô4ˆØ�œ }Ñ5Ô5Ñ5ˆØŸš mÑ4Ô4ˆà-˜Ô-¨b°,ÐYÀÔATÐUVÐUWÐUWÔAXÐYÐYÐYˆØ%×-Ò-¨a°°A°q¸!Ñ<Ô<ˆð ”Nð 	:ð 	:ˆDØ ˜D Ñ/Ô/ˆMØ�Uœ]¨=Ñ9Ô9Ñ9ˆMˆMàÔ(ð 	1ð 	1ˆEØ!˜E -Ñ0Ô0ˆMˆMà%×-Ò-¨a°°A°q¸!Ñ<Ô<ˆàÐr/   )r'   r(   r)   rw   r+   r,   r‘   r•   r–   s   @r0   rƒ  rƒ    sW   ø€ € € € € ð%6ð %6ð %6ð %6ð %6ðN EÔ$4ð ð ð ð ð ð ð ð r/   rƒ  c                   óH   ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Zˆ xZS )ÚEmu3VQVAEDecoderrp   c                 ó†  •— t          ¦   «                              ¦   «          |j        }|j        |j        d         z  }t          j        ¦   «         | _        t          |j	        ¦  «        D ]7}t          |j        |j        ¬¦  «        }| j                             |¦  «         Œ8t          t          j        |j        ¦  «        ¦  «        }t          j        ¦   «         | _        t          |¦  «        D ]6}t%          |j        |j        ¦  «        }| j                             |¦  «         Œ7t          j        |j        |ddd¬¦  «        | _        t+          |||¬¦  «        | _        t/          |¦  «        | _        |j        |j        d         z  }t3          ||¦  «        | _        t          j        ||j        ddd¬¦  «        | _        d S )Nr2   rb  r   r    rç   )r=  r   )rv   rw   rÐ   rg  rd  rb   ri  r�  rk  rf  r1  r†  rl  r’   rŒ  r�  rŽ  r�  r#  rì   r‡  rT  r‰  r{  Úup_blockr  rŠ  r  r‹  )
r„   rp   r=  rt  rÈ   r“  Útemp_upsample_block_numr’  rí   r…   s
            €r0   rw   zEmu3VQVAEDecoder.__init__T  s®  ø€ Ý‰Œ×ÒÑÔÐàÔ)ˆØÔ'¨&Ô*CÀBÔ*GÑGˆÝ œm™oœoˆÔÝ�vÔ,Ñ-Ô-ð 	6ð 	6ˆAÝ8Ø"Ô2ÀÔAWðñ ô ˆMð Ô×&Ò& }Ñ5Ô5Ð5Ð5å"%¥d¤i°Ô0QÑ&RÔ&RÑ"SÔ"SÐÝœ™œˆŒÝÐ.Ñ/Ô/ð 	(ð 	(ˆAÝ,¨VÔ-CÀVÔE[Ñ\Ô\ˆDØŒN×!Ò! $Ñ'Ô'Ð'Ð'å”yØÔ"ØØØØð
ñ 
ô 
ˆŒõ 1°¸ÐR`ÐaÑaÔaˆÔÝ(¨Ñ0Ô0ˆŒàÔ'¨&Ô*CÀAÔ*FÑFˆÝ,¨^¸XÑFÔFˆŒÝœ	ØØÔØØØð
ñ 
ô 
ˆŒˆˆr/   rG   r  c                 óÂ  — t          j        ||fd¬¦  «        }|                     ddddd¦  «        }| j        D ]} ||¦  «        }Œ| j        D ]$} ||¦  «        }|t          j        |¦  «        z  }Œ%|                     ddddd¦  «        }t          j        |dd¬¦  «        \  }} |j        dg|j        dd …         ¢R Ž } |j        dg|j        dd …         ¢R Ž }|  	                    |¦  «        }|  
                    ||¦  «        }|                      ||¦  «        }|                      ||¦  «        }|t          j        |¦  «        z  }|                      |¦  «        }|S )Nr   r4   r3   r    r   rÖ   r2   )r+   r7   r×   r�  r�  r:  ÚchunkrL   r6   r‡  r‰  r›  rŠ  r‹  )r„   rG   r  Úhidden_quant_statesr—  s        r0   r‘   zEmu3VQVAEDecoder.forward{  sš  € Ý#œi¨¸Ð(EÈ1ÐMÑMÔMÐØ1×9Ò9¸!¸QÀÀ1ÀaÑHÔHÐð Ô(ð 	=ð 	=ˆEØ"' %Ð(;Ñ"<Ô"<ÐÐà”^ð 	Fð 	FˆEØ"' %Ð(;Ñ"<Ô"<ÐØ¥5¤=Ð1DÑ#EÔ#EÑEÐÐà1×9Ò9¸!¸QÀÀ1ÀaÑHÔHÐÝ&+¤kÐ2EÀqÈaÐ&PÑ&PÔ&PÑ#ˆ�|Ø-˜Ô-¨bÐK°=Ô3FÀqÀrÀrÔ3JÐKÐKÐKˆØ+�|Ô+¨BÐH°Ô1CÀAÀBÀBÔ1GÐHÐHÐHˆàŸš ]Ñ3Ô3ˆð ×)Ò)¨-¸ÑFÔFˆØŸš m°\ÑBÔBˆàŸš m°\ÑBÔBˆØ�œ }Ñ5Ô5Ñ5ˆØŸš mÑ4Ô4ˆàÐr/   )	r'   r(   r)   r#   rw   r+   r“   r‘   r•   r–   s   @r0   r™  r™  S  sk   ø€ € € € € ð%
˜ð %
ð %
ð %
ð %
ð %
ð %
ðN U¤\ð ÀÄð ð ð ð ð ð ð ð r/   r™  aR  
    The VQ-VAE model used in Emu3 for encoding/decoding images into discrete tokens.
    This model follows the "Make-a-scene: Scene-based text-to-image generation with human priors" paper from
    [ Oran Gafni, Adam Polyak, Oron Ashual, Shelly Sheynin, Devi Parikh, and Yaniv
    Taigman](https://huggingface.co/papers/2203.13131).
    ©Úcustom_introc            
       ó  ‡ — e Zd ZU eed<   dZdZdZdZdZ	dZ
dZg d¢ZeegedœZ ej        ¦   «         ˆ fd„¦   «         Zdefˆ fd	„Zeedej        d
ej        dee         defd„¦   «         ¦   «         Zdej        fd„Zˆ xZS )Ú	Emu3VQVAErp   Ú
emuvideovqr”  )ÚimageT)r1  rC  r<  rË   ©rG   Ú
attentionsc                 ó  •— t          ¦   «                              |¦  «         t          |t          j        t          j        f¦  «        r†t          j        |j        dd¬¦  «         |j	        �at          j        j                             |j        ¦  «        \  }}dt          j        |¦  «        z  }t          j        |j	        | |¦  «         d S d S t          |t          j        ¦  «        rŸt          j        |j        t          j        d¦  «        ¬¦  «         |j	        �it          j        j                             |j        ¦  «        \  }}|dk    rdt          j        |¦  «        z  nd}t          j        |j	        | |¦  «         d S d S t          |t          j        ¦  «        r\t          j        |j        ¦  «         |j        �>t)          |j        dd	¦  «        s*t          j        |j        |j                 ¦  «         d S d S d S d S )
NÚfan_outÚrelu)rô   Únonlinearityr    é   )Úar   Ú_is_hf_initializedF)rv   Ú_init_weightsÚ
isinstancerb   rì   r  ÚinitÚkaiming_normal_rŸ   ru   r+   Ú_calculate_fan_in_and_fan_outrŒ  ÚsqrtrÓ   r}   Úkaiming_uniform_rÎ   Únormal_Úpadding_idxrx   Úzeros_)r„   rS   Úfan_inrÈ   Úboundr…   s        €r0   r¯  zEmu3VQVAE._init_weightsµ  sÏ  ø€ å‰Œ×Ò˜fÑ%Ô%Ð%Ý�f�rœy­"¬)Ð4Ñ5Ô5ð 	?ÝÔ  ¤°YÈVÐTÑTÔTÐTØŒ{Ð&Ý!œHœM×GÒGÈÌÑVÔV‘	�˜Ø�DœI fÑ-Ô-Ñ-�Ý”˜fœk¨E¨6°5Ñ9Ô9Ð9Ð9Ð9ð 'Ð&õ ˜¥¤	Ñ*Ô*ð 
	?ÝÔ! &¤-µ4´9¸Q±<´<Ð@Ñ@Ô@Ð@ØŒ{Ð&Ý!œHœM×GÒGÈÌÑVÔV‘	�˜Ø17¸!²°˜�DœI fÑ-Ô-Ñ-Ð-À�Ý”˜fœk¨E¨6°5Ñ9Ô9Ð9Ð9Ð9ð 'Ð&õ ˜¥¤Ñ-Ô-ð 	?ÝŒL˜œÑ'Ô'Ð'àÔ!Ð-µg¸f¼mÐMaÐchÑ6iÔ6iÐ-Ý”˜FœM¨&Ô*<Ô=Ñ>Ô>Ð>Ð>Ð>ð		?ð 	?ð .Ð-Ð-Ð-r/   c                 ó$  •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        t          |¦  «        | _        t          |¦  «        | _        dt          |j
        ¦  «        dz
  z  | _        t          |j        |j        dd¬¦  «        | _        t          |j        |j        dd¬¦  «        | _        dt          |j
        ¦  «        dz
  z  | _        |                      ¦   «          |                      ¦   «          d S )Nr3   r    )r   r    r    r&  r'  )rv   rw   rp   rƒ  Úencoderr™  ÚdecoderrË   Úquantizerc  rd  Úvision_spatial_factorr  r†  rÐ   Ú
quant_convÚpost_quant_convÚspatial_scale_factorÚevalÚ	post_initr¶   s     €r0   rw   zEmu3VQVAE.__init__Ê  sý   ø€ Ý‰Œ×Ò˜Ñ Ô Ð àˆŒå'¨Ñ/Ô/ˆŒÝ'¨Ñ/Ô/ˆŒÝ0°Ñ8Ô8ˆŒØ%&­3¨vÔ/HÑ+IÔ+IÈAÑ+MÑ%NˆÔ"å)ØÔ" FÔ$4À)ÐT]ð
ñ 
ô 
ˆŒõ  /ØÔ˜fÔ4À)ÐT]ð 
ñ  
ô  
ˆÔð %&­#¨fÔ.GÑ*HÔ*HÈ1Ñ*LÑ$MˆÔ!Ø�	Š	‰Œˆà�ŠÑÔÐÐÐr/   Úimage_sizesrZ   rI   c                 ód  ‡ — |j         dk    }|rE‰ j        j        }|j        \  }}}}	|                     d¦  «                             d|ddd¦  «        }n|j        \  }}}}}	|                     ‰ j        ¦  «        }‰                      |¦  «        }
|
 	                    ddddd¦  «        }‰  
                    |¦  «        }| 	                    ddddd¦  «        }‰                      |¦  «        }|r|                     d¦  «        n|}ˆ fd„t          ||¦  «        D ¦   «         }t          |
|¬¦  «        S )NrÖ   r    r   r3   r   c           	      óž   •— g | ]I\  }}|d t          |d         ‰j        z  ¦  «        …d t          |d         ‰j        z  ¦  «        …f         ‘ŒJS )Nr   r    )r’   r¿  )r  Úsingle_imager   r„   s      €r0   r	  z$Emu3VQVAE.encode.<locals>.<listcomp>ù  sm   ø€ ð 
ð 
ð 
á"�˜dð ÐD�3˜t Aœw¨Ô)CÑCÑDÔDÐDÐFqÍÈDÐQRÌGÐVZÔVpÑLpÑHqÔHqÐFqÐqÔrð
ð 
ð 
r/   )Úlast_hidden_stater&   )Úndimrp   rŽ  r6   r>   Úrepeatrf   r\   r¼  r×   rÀ  r¾  Úsqueezer
  r%   )r„   r”  rÅ  rZ   Úis_imagerÛ   rÚ   rÜ   rÝ   rÞ   rG   Úconv_hidden_statesÚcodesr&   s   `             r0   ÚencodezEmu3VQVAE.encodeß  se  ø€ ð
  Ô$¨Ò)ˆØð 	OØ”{Ô=ˆHØ2>Ô2DÑ/ˆJ˜ &¨%Ø'×1Ò1°!Ñ4Ô4×;Ò;¸A¸xÈÈAÈqÑQÔQˆLˆLà<HÔ<NÑ9ˆJ˜ (¨F°Eà#—’ t¤zÑ2Ô2ˆØŸš \Ñ2Ô2ˆð +×2Ò2°1°a¸¸A¸qÑAÔAÐØ!Ÿ_š_Ð-?Ñ@Ô@Ðð 0×7Ò7¸¸1¸aÀÀAÑFÔFÐØ—’Ð0Ñ1Ô1ˆà+3Ð>�u—}’} QÑ'Ô'Ð'¸ˆð
ð 
ð 
ð 
å&)¨,¸Ñ&DÔ&Dð
ñ 
ô 
ˆõ
 $Ø+Ø%ð
ñ 
ô 
ð 	
r/   rG   c                 ó®  — |j         dk    }|r|                     d¦  «        }|j        \  }}}}| j                             |                     ¦   «         ¦  «        }|j        d         }|                     |||||¦  «                             ddddd¦  «                             ¦   «         }|  	                    |¦  «        }	|                     ddddd¦  «        }|	                     ddddd¦  «        }	|  
                    |	|¦  «        }
|
                     ||| j        j        z  | j        j        || j        z  || j        z  ¦  «        }
|r|
d d …df         n|
S )Nr   r    r2   r   rÖ   r3   )rÊ  r>   r6   r¾  rÑ   Úflattenr‰   r×   rg   rÁ  r½  rL   rp   rŽ  r  rÂ  )r„   rG   rÍ  rÚ   rÛ   rÝ   rÞ   ÚquantrÜ   Ú
post_quantÚvideos              r0   ÚdecodezEmu3VQVAE.decode  s_  € Ø Ô%¨Ò*ˆØð 	7Ø)×3Ò3°AÑ6Ô6ˆMà.;Ô.AÑ+ˆ
�H˜f eØ”×'Ò'¨×(=Ò(=Ñ(?Ô(?Ñ@Ô@ˆà”;˜r”?ˆØ—
’
˜: x°¸ÀÑIÔI×QÒQÐRSÐUVÐXYÐ[\Ð^_Ñ`Ô`×kÒkÑmÔmˆØ×)Ò)¨%Ñ0Ô0ˆ
à—’˜a  A q¨!Ñ,Ô,ˆØ×'Ò'¨¨1¨a°°AÑ6Ô6ˆ
à—’˜Z¨Ñ/Ô/ˆØ—’ØØ�t”{Ô=Ñ=ØŒKÔ$Ø�TÔ.Ñ.Ø�DÔ-Ñ-ñ
ô 
ˆð 'Ð1ˆu�Q�Q�Q˜�TŒ{ˆ{¨EÐ1r/   )r'   r(   r)   r#   r-   Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚ_supports_sdpaÚ_supports_flash_attnÚ_supports_flex_attnÚ_supports_attention_backendÚ_no_split_modulesr<  r1  rC  Ú_can_record_outputsr+   Úno_gradr¯  rw   r   r   r“   r   r   r%   rÐ  rÖ  r•   r–   s   @r0   r£  r£  ™  sI  ø€ € € € € € ð ÐÐÑØ$ÐØ$€OØ!ÐØ€NØÐØÐØ"&Ððð ð Ðð /Ð0LÐMØ-ðð Ðð
 €U„]�_„_ð?ð ?ð ?ð ?ñ „_ð?ð(˜ð ð ð ð ð ð ð*  Øð 
Ø!œLð 
Ø7<´|ð 
ØOUÐVhÔOið 
à	ð 
ð  
ð  
ñ „_ñ  Ôð 
ðD2 E¤Lð 2ð 2ð 2ð 2ð 2ð 2ð 2ð 2r/   r£  c                   óð   — e Zd ZdZd„ Zed„ ¦   «         Zed„ ¦   «         Zed„ ¦   «         Zed„ ¦   «         Z	ed„ ¦   «         Z
ed„ ¦   «         Zd	eej                 d
ej        fd„Zd	ej        d
ej        fd„ZdS )ÚEmu3ImageVocabularyMappingzM
    A class for mapping discrete image tokens from VQGAN to BPE tokens.
    c                 ó|   — || _         |                     d¦  «        | _        |                     d¦  «        | _        d S )Nz<|extra_200|>z<image>)Ú	vocab_mapÚgetÚeol_token_idÚimage_token_id)r„   rä  s     r0   rw   z#Emu3ImageVocabularyMapping.__init__"  s7   € Ø"ˆŒØ%ŸMšM¨/Ñ:Ô:ˆÔØ'Ÿmšm¨IÑ6Ô6ˆÔÐÐr/   c                 ób   — t          d„ | j                             ¦   «         D ¦   «         ¦  «        S )Nc                 óB   — g | ]\  }}|                      d ¦  «        ¯|‘ŒS ©z<|visual token©Ú
startswith©r  ÚnameÚvals      r0   r	  z;Emu3ImageVocabularyMapping.image_tokens.<locals>.<listcomp>)  s.   € ÐhÐhÐh™y˜t SÀdÇoÂoÐVfÑFgÔFgÐh�sÐhÐhÐhr/   ©Úsortedrä  Úitemsr©   s    r0   r&   z'Emu3ImageVocabularyMapping.image_tokens'  s-   € åÐhÐh¨D¬N×,@Ò,@Ñ,BÔ,BÐhÑhÔhÑiÔiÐir/   c                 ób   — t          d„ | j                             ¦   «         D ¦   «         ¦  «        S )Nc                 óB   — g | ]\  }}|                      d ¦  «        ¯|‘ŒS rê  rë  rí  s      r0   r	  z?Emu3ImageVocabularyMapping.image_tokens_str.<locals>.<listcomp>-  s.   € ÐiÐiÐi¡	  cÀtÇÂÐWgÑGhÔGhÐi�tÐiÐiÐir/   rð  r©   s    r0   Úimage_tokens_strz+Emu3ImageVocabularyMapping.image_tokens_str+  s-   € åÐiÐi¨T¬^×-AÒ-AÑ-CÔ-CÐiÑiÔiÑjÔjÐjr/   c                 ó*   ‡ — ˆ fd„‰ j         D ¦   «         S )Nc                 óV   •— i | ]%}t          |d d…         ¦  «        ‰j        |         “Œ&S )iøÿÿÿr  )r’   rä  )r  Útokenr„   s     €r0   ú
<dictcomp>z6Emu3ImageVocabularyMapping.img2bpe.<locals>.<dictcomp>1  s2   ø€ Ð\Ð\Ð\¸U•�E˜"˜R˜%”LÑ!Ô! 4¤>°%Ô#8Ð\Ð\Ð\r/   )rõ  r©   s   `r0   Úimg2bpez"Emu3ImageVocabularyMapping.img2bpe/  s    ø€ à\Ð\Ð\Ð\ÀdÔF[Ð\Ñ\Ô\Ð\r/   c                 óH   — d„ | j                              ¦   «         D ¦   «         S )Nc                 ó   — i | ]\  }}||“Œ	S r.   r.   )r  r@   Úvs      r0   rù  z6Emu3ImageVocabularyMapping.bpe2img.<locals>.<dictcomp>5  s   € Ð6Ð6Ð6™˜˜A��1Ð6Ð6Ð6r/   )rú  rò  r©   s    r0   Úbpe2imgz"Emu3ImageVocabularyMapping.bpe2img3  s$   € à6Ð6 ¤×!3Ò!3Ñ!5Ô!5Ð6Ñ6Ô6Ð6r/   c                 óÜ   — t          j        t          | j                             ¦   «         ¦  «        dz   t           j        ¬¦  «        }| j                             ¦   «         D ]
\  }}|||<   Œ|S ©Nr    ©r\   )r+   ÚzerosÚmaxrþ  rK  r’   rò  ©r„   Úmappingr@   rý  s       r0   Úbpe2img_mapping_tensorz1Emu3ImageVocabularyMapping.bpe2img_mapping_tensor7  ód   € å”+�c $¤,×"3Ò"3Ñ"5Ô"5Ñ6Ô6¸Ñ:Å%Ä)ÐLÑLÔLˆØ”L×&Ò&Ñ(Ô(ð 	ð 	‰DˆAˆqØˆG�A‰JˆJØˆr/   c                 óÜ   — t          j        t          | j                             ¦   «         ¦  «        dz   t           j        ¬¦  «        }| j                             ¦   «         D ]
\  }}|||<   Œ|S r   )r+   r  r  rú  rK  r’   rò  r  s       r0   Úimg2bpe_mapping_tensorz1Emu3ImageVocabularyMapping.img2bpe_mapping_tensor>  r  r/   Ú	img_batchrI   c                 ó  — |j         }t          j        |j        d         dft          j        ¬¦  «        | j        z  }| j        |                     d¦  «                 }t          j        ||gd¬¦  «        }|                     |¦  «        S )Nr   r    r  Úcpur2   r4   )	Údevicer+   rž   r6   r’   ræ  r	  rf   r7   )r„   r
  r  Úeol_rowÚ
img_tokenss        r0   Úconvert_img2bpez*Emu3ImageVocabularyMapping.convert_img2bpeE  sx   € ØÔ!ˆÝ”*˜iœo¨aÔ0°!Ð4½E¼IÐFÑFÔFÈÔIZÑZˆØÔ0°·²¸eÑ1DÔ1DÔEˆ
Ý”Y 
¨GÐ4¸"Ð=Ñ=Ô=ˆ
Ø�}Š}˜VÑ$Ô$Ð$r/   c                 ó’   — |j         }|dd d…f         }| j        |                     d¦  «                 }|                     |¦  «        S )N.r2   r  )r  r  rf   )r„   r
  r  r  s       r0   Úconvert_bpe2imgz*Emu3ImageVocabularyMapping.convert_bpe2imgL  sG   € ØÔ!ˆØ˜c 3 B 3˜hÔ'ˆ	ØÔ0°·²¸eÑ1DÔ1DÔEˆ
Ø�}Š}˜VÑ$Ô$Ð$r/   N)r'   r(   r)   r*   rw   r   r&   rõ  rú  rþ  r  r	  Úlistr+   r“   r  r  r.   r/   r0   râ  râ    s)  € € € € € ðð ð7ð 7ð 7ð
 ðjð jñ „_ðjð ðkð kñ „_ðkð ð]ð ]ñ „_ð]ð ð7ð 7ñ „_ð7ð ðð ñ „_ðð ðð ñ „_ðð%¨¨e¬lÔ);ð %ÀÄð %ð %ð %ð %ð%¨¬ð %¸%¼,ð %ð %ð %ð %ð %ð %r/   râ  c                   óR   — e Zd ZU eed<   dZdZdZdgZddgZ	dZ
dZdZdZdZeedœZd	S )
ÚEmu3PreTrainedModelrp   Úmodel©r¥  ÚtextTr»   r‡   Úcausal_maskr¦  N)r'   r(   r)   r!   r-   r×  rÙ  Úsupports_gradient_checkpointingrÞ  Ú_skip_keys_device_placementrÛ  rÚ  Ú_can_compile_fullgraphrÜ  rÝ  r»   rn   rß  r.   r/   r0   r  r  S  sx   € € € € € € àÐÐÑØÐØ(ÐØ&*Ð#àðÐð $5°mÐ"DÐØÐØ€Nà!ÐØÐØ"&Ðà)Ø#ðð ÐÐÐr/   r  c            	       óî   ‡ — e Zd Z edd¬¦  «        d
defˆ fd„¦   «         Ze edd¬¦  «        d
dedeej	        e
f         fd„¦   «         ¦   «         Z ej        ¦   «         ed	„ ¦   «         ¦   «         Zˆ xZS )ÚEmu3RotaryEmbeddingr  z5.18)ÚversionNrp   c                 ó¾  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        | j        j        d         | _        | j        }| j        dk    rt          | j                 } || j        |¦  «        \  }| _
        t          j        |d¬¦  «        | _        t          j        |                     ¦   «         d¬¦  «        | _        d S )NÚ	rope_typeÚdefaultF)Ú
persistent)rv   rw   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrp   Úrope_parametersr!  Úcompute_default_rope_parametersr   Úattention_scalingrb   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r„   rp   r  Úrope_init_fnr+  r…   s        €r0   rw   zEmu3RotaryEmbedding.__init__j  sÀ   ø€ å‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!àˆŒàœÔ4°[ÔAˆŒØ!%Ô!EˆØŒ>˜YÒ&Ð&Ý.¨t¬~Ô>ˆLØ+7¨<¸¼ÀVÑ+LÔ+LÑ(ˆ�$Ô(åœ	 (°uÐ=Ñ=Ô=ˆŒÝ!#¤¨8¯>ª>Ñ+;Ô+;ÈÐ!NÑ!NÔ!NˆÔÐÐr/   rI   c                 óä   — | j         d         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «        |z  z  z  }|                     |¦  «        |fS )a¾  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetarP   NrÍ   r   r3   r  )r'  rx   ry   rz   r+   Úaranger«   rf   )rp   r  rZ   Úbaser5   Úattention_factorr+  s          r0   r(  z3Emu3RotaryEmbedding.compute_default_rope_parameters{  sz   € ð Ô% lÔ3ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐà˜$¥5¤<°°3¸ÅÄÐ#MÑ#MÔ#MÐPSÑ#SÑTÑUˆØ�{Š{˜6Ñ"Ô"Ð$4Ð4Ð4r/   c                 óú  — | j         d d d …d f                              |j        d         dd¦  «                             t          j        |j        ¬¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  ||z                       dd	¦  «        }t	          j        ||fd¬
¦  «        }|                     ¦   «         | j        z  }|                     ¦   «         | j        z  }	d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬¦  «        |	                     |j        ¬¦  «        fS )Nr   r2   r    ©r\   r  Úmpsr  F)Údevice_typeÚenabledr3   r4   r  )r+  rK   r6   rf   r+   r«   r  r°  ÚtypeÚstrr   ra   r7   rA   r)  rB   r\   )
r„   r8   rÄ   Úinv_freq_expandedÚposition_ids_expandedr7  ÚfreqsÚembrA   rB   s
             r0   r‘   zEmu3RotaryEmbedding.forward�  s›  € ð ŒM˜$    4˜-Ô(×/Ò/°Ô0BÀ1Ô0EÀrÈ1ÑMÔM×PÒPÕW\ÔWbÐklÔksÐPÑtÔtð 	ð !-¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆå¨¸UÐCÑCÔCð 	5ð 	5Ø&Ð)>Ñ>×IÒIÈ!ÈQÑOÔOˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCð		5ð 	5ð 	5ñ 	5ô 	5ð 	5ð 	5ð 	5ð 	5ð 	5ð 	5øøøð 	5ð 	5ð 	5ð 	5ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   ÃA*D<Ä<E ÅE r¸   )r'   r(   r)   r   r!   rw   Ústaticmethodr”   r+   r“   r«   r(  rà  r   r‘   r•   r–   s   @r0   r  r  i  sö   ø€ € € € € Ø€_�X vÐ.Ñ.Ô.ðOð O˜zð Oð Oð Oð Oð Oñ /Ô.ðOð  Ø€_�X vÐ.Ñ.Ô.ð5ð 5°
ð 5ÐV[Ð\aÔ\hÐjoÐ\oÔVpð 5ð 5ð 5ñ /Ô.ñ „\ð5ð$ €U„]�_„_Øð<ð <ñ Ôñ „_ð<ð <ð <ð <ð <r/   r  c                   óî   ‡ — e Zd ZU eed<   defˆ fd„Zeee	 	 	 	 	 	 dde	j
        dz  de	j        dz  de	j
        dz  dedz  de	j        dz  d	edz  d
ee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚEmu3TextModelrp   c                 óÞ  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          j        ‰j        ‰j        | j        ¦  «        | _        t          j	        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        t!          ‰¬¦  «        | _        d| _        |                      ¦   «          d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r.   )r»   )r  rq   rp   s     €r0   r	  z*Emu3TextModel.__init__.<locals>.<listcomp>­  s$   ø€ ÐbÐbÐb°YÕ˜f iÑ0Ô0ÐbÐbÐbr/   r½   ©rp   F)rv   rw   Úpad_token_idr·  Ú
vocab_sizerb   rÎ   ry   Úembed_tokensri  rk  Únum_hidden_layersÚlayersr™   rÀ   Únormr  Ú
rotary_embÚgradient_checkpointingrÄ  r¶   s    `€r0   rw   zEmu3TextModel.__init__¦  sÔ   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒåœL¨Ô):¸FÔ<NÐPTÔP`ÑaÔaˆÔÝ”mØbÐbÐbÐbÅ%ÈÔH`ÑBaÔBaÐbÑbÔbñ
ô 
ˆŒõ   Ô 2¸Ô8KÐLÑLÔLˆŒ	Ý-°VÐ<Ñ<Ô<ˆŒØ&+ˆÔ#ð 	�ŠÑÔÐÐÐr/   NÚ	input_idsrW   rÄ   r‡   Úinputs_embedsrÅ   rZ   rI   c           
      óH  — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|r|€t          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   }| 	                    d¦  «        }t          | j        ||||¬¦  «        }	|}
|                      |
|¬¦  «        }| j        d | j        j        …         D ]} ||
f|	||||dœ|¤Ž}
Œ|                      |
¦  «        }
t          |
|¬	¦  «        S )
Nz:You must specify exactly one of input_ids or inputs_embedsrD  r   r    )r  )rp   rN  rW   r‡   rÄ   )rÄ   )rW   r†   rÄ   r‡   rÅ   )rÉ  r‡   )rF  rG  r
   rp   Úget_seq_lengthr+   r1  r6   r  r>   r   rK  rI  rH  rJ  r   )r„   rM  rW   rÄ   r‡   rN  rÅ   rZ   Úpast_seen_tokensr  rG   r†   Údecoder_layers                r0   r‘   zEmu3TextModel.forward¶  sŠ  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø*.×*;Ò*;¸IÑ*FÔ*FˆMàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå(Ø”;Ø'Ø)Ø+Ø%ð
ñ 
ô 
ˆð &ˆØ"Ÿošo¨mÈ,˜oÑWÔWÐà!œ[Ð)H¨4¬;Ô+HÐ)HÔIð 		ð 		ˆMØ)˜MØðà*Ø$7Ø)Ø /Ø#ðð ð ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆÝ&Ø+Ø+ð
ñ 
ô 
ð 	
r/   )NNNNNN)r'   r(   r)   r"   r-   rw   r   r   r   r+   r,   r“   r	   r]  rÉ   r   r   r   r‘   r•   r–   s   @r0   rA  rA  ¢  s  ø€ € € € € € àÐÐÑð˜~ð ð ð ð ð ð ð   ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ð2
ð 2
àÔ# dÑ*ð2
ð œ tÑ+ð2
ð Ô&¨Ñ-ð	2
ð
  ™ð2
ð Ô(¨4Ñ/ð2
ð ˜$‘;ð2
ð Ð+Ô,ð2
ð 
!ð2
ð 2
ð 2
ñ „^ñ „_ñ  Ôð2
ð 2
ð 2
ð 2
ð 2
r/   rA  c                   ó,  ‡ — e Zd ZU ddiZddiZddgdgfiZddiZeed<   ˆ fd	„Z	e
e	 	 	 	 	 	 	 	 ddej        d
z  dej        d
z  dej        d
z  ded
z  dej        d
z  dej        d
z  ded
z  deej        z  dee         defd„¦   «         ¦   «         Zˆ xZS )ÚEmu3ForCausalLMúlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputrG   ÚlogitsÚkeep_full_weightrp   c                 óþ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        | _        t          j        |j        |j        d¬¦  «        | _        |  	                    ¦   «          d S ©NFrt   )
rv   rw   rA  r  rF  rb   r}   ry   rV  rÄ  r¶   s     €r0   rw   zEmu3ForCausalLM.__init__ö  sj   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý" 6Ñ*Ô*ˆŒ
Ø Ô+ˆŒÝ”y Ô!3°VÔ5FÈUÐSÑSÔSˆŒð 	�ŠÑÔÐÐÐr/   Nr   rM  rW   rÄ   r‡   rN  ÚlabelsrÅ   Úlogits_to_keeprZ   rI   c	           
      óP  —  | j         d||||||dœ|	¤Ž}
|
j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }d}|� | j        d||| j        j        dœ|	¤Ž}t          |||
j
        |
j        |
j        ¬¦  «        S )aÃ  
        Example:

        ```python
        >>> from transformers import Emu3Processor, Emu3ForConditionalGeneration
        >>> import torch
        >>> import httpx
        >>> from io import BytesIO
        >>> from PIL import Image

        >>> model = Emu3ForCausalLM.from_pretrained("BAAI/Emu3-Chat-hf", dtype=torch.bfloat16)
        >>> processor = Emu3Processor.from_pretrained("BAAI/Emu3-Chat-hf")

        >>> inputs = processor(text=["Can you write me a poem about winter."], return_tensors="pt").to(model.device)

        >>> generated_ids = model.generate(**inputs, max_new_tokens=100, do_sample=False)
        >>> processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        ```)rM  rW   rÄ   r‡   rN  rÅ   N©rX  r\  rF  ©ÚlossrX  r‡   rG   r§  r.   )r  rÉ  r°  r’   ÚslicerV  Úloss_functionrp   rF  r   r‡   rG   r§  )r„   rM  rW   rÄ   r‡   rN  r\  rÅ   r]  rZ   ÚoutputsrG   Úslice_indicesrX  ra  s                  r0   r‘   zEmu3ForCausalLM.forwardÿ  sõ   € ð@ ,6¨4¬:ð ,
ØØ)Ø%Ø+Ø'Øð,
ð ,
ð ð,
ð ,
ˆð  Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%Ðp¨V¸FÈtÌ{ÔOeÐpÐpÐioÐpÐpˆDå%ØØØ#Ô3Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r/   )NNNNNNNr   )r'   r(   r)   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr"   r-   rw   r   r   r+   r,   r“   r	   r]  rÉ   r’   r   r   r   r‘   r•   r–   s   @r0   rT  rT  î  sd  ø€ € € € € € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€JØÐÐÑðð ð ð ð ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ð7
ð 7
àÔ# dÑ*ð7
ð œ tÑ+ð7
ð Ô&¨Ñ-ð	7
ð
  ™ð7
ð Ô(¨4Ñ/ð7
ð Ô  4Ñ'ð7
ð ˜$‘;ð7
ð ˜eœlÑ*ð7
ð Ð+Ô,ð7
ð 
 ð7
ð 7
ð 7
ñ „^ñ Ôð7
ð 7
ð 7
ð 7
ð 7
r/   rT  c                   ó(  ‡ — e Zd Zˆ fd„Zd„ Zd„ Zdej        dej        dej        fd„Z	e
 ed¬	¦  «        dej        dej        d
ee         deez  fd„¦   «         ¦   «         Z ej        ¦   «         dej        dedefd„¦   «         Zdej        dej        dej        fd„Ze
e	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dedz  dej        dz  dedz  d
ee         deez  fd„¦   «         ¦   «         Zˆ xZS )Ú	Emu3Modelc                 ó  •— t          ¦   «                              |¦  «         t                               |j        ¦  «        | _        t          |j        ¦  «        | _        t          |j
        ¦  «        | _        |                      ¦   «          d S r¸   )rv   rw   rA  Ú_from_configÚtext_configÚ
text_modelr£  Ú	vq_configÚvqmodelrâ  Úvocabulary_mapÚvocabulary_mappingrÄ  r¶   s     €r0   rw   zEmu3Model.__init__<  sp   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý'×4Ò4°VÔ5GÑHÔHˆŒÝ  Ô!1Ñ2Ô2ˆŒÝ"<¸VÔ=RÑ"SÔ"SˆÔð 	�ŠÑÔÐÐÐr/   c                 ó4   — | j                              ¦   «         S r¸   )ro  Úget_input_embeddingsr©   s    r0   ru  zEmu3Model.get_input_embeddingsE  s   € ØŒ×3Ò3Ñ5Ô5Ð5r/   c                 ó:   — | j                              |¦  «         d S r¸   )ro  Úset_input_embeddings©r„   rV   s     r0   rw  zEmu3Model.set_input_embeddingsH  s   € ØŒ×,Ò,¨UÑ3Ô3Ð3Ð3Ð3r/   r”  rÅ  rI   c                 ó�   ‡ — ‰ j                              ||d¬¦  «        }ˆ fd„|j        D ¦   «         }t          j        |¦  «        }|S )a  
        Tokenizes images into discrete tokens with VQGAN module. Converts
        obtained image tokens into BPE tokens and wraps with "boi" and "eoi"
        special tokens.

        Args:
            pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
                The tensors corresponding to the input images.
            image_sizes (`torch.LongTensor` of shape `(batch_size, 2)`):
                The sizes of the images in the batch, being (height, width) for each image.
        T)Úreturn_dictc                 óh   •— g | ].}‰j                              |¦  «                             ¦   «         ‘Œ/S r.   ©rs  r  rÒ  ©r  Útokensr„   s     €r0   r	  z.Emu3Model.get_image_tokens.<locals>.<listcomp>X  óC   ø€ ð 
ð 
ð 
ØJPˆDÔ#×3Ò3°FÑ;Ô;×CÒCÑEÔEð
ð 
ð 
r/   )rq  rÐ  r&   r+   r7   )r„   r”  rÅ  Úvqmodel_outputsÚbpe_tokens_listÚ
bpe_tokenss   `     r0   Úget_image_tokenszEmu3Model.get_image_tokensK  se   ø€ ð 15´×0CÒ0CÀLÐR]ÐkoÐ0CÑ0pÔ0pˆð
ð 
ð 
ð 
ØTcÔTpð
ñ 
ô 
ˆõ ”Y˜Ñ/Ô/ˆ
ØÐr/   zbTokenizes images into discrete tokens with VQGAN module and embeds them with text embeddings layerr   rZ   c                 ó  ‡ —  ‰ j         j        ||fddi|¤Ž}ˆ fd„|D ¦   «         }ˆ fd„|j        D ¦   «         }t          j        |¦  «        } ‰                      ¦   «         |¦  «        }t          j        ||¦  «        }	|	|_        |S )z­
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)):
            The tensors corresponding to the input images.
        rz  Tc                 óZ   •— g | ]'\  }}|‰j         j        z  |‰j         j        z  d z   z  ‘Œ(S ra  )rq  r¿  )r  rÝ   rÞ   r„   s      €r0   r	  z0Emu3Model.get_image_features.<locals>.<listcomp>l  sL   ø€ ð 
ð 
ð 
á�˜ð �t”|Ô9Ñ9¸eÀtÄ|ÔGiÑ>iÐlmÑ>mÑnð
ð 
ð 
r/   c                 óh   •— g | ].}‰j                              |¦  «                             ¦   «         ‘Œ/S r.   r|  r}  s     €r0   r	  z0Emu3Model.get_image_features.<locals>.<listcomp>p  r  r/   )rq  rÐ  r&   r+   r7   ru  ÚsplitÚpooler_output)
r„   r”  rÅ  rZ   r€  Úsplit_sizesr�  r‚  Úimage_embeddingsÚimage_featuress
   `         r0   Úget_image_featureszEmu3Model.get_image_features^  sÖ   ø€ ð 1D°´Ô0CØ˜+ð1
ð 1
Ø37ð1
Ø;Að1
ð 1
ˆð
ð 
ð 
ð 
à!,ð
ñ 
ô 
ˆð
ð 
ð 
ð 
ØTcÔTpð
ñ 
ô 
ˆõ ”Y˜Ñ/Ô/ˆ
Ø6˜4×4Ò4Ñ6Ô6°zÑBÔBÐÝœÐ%5°{ÑCÔCˆØ(6ˆÔ%àÐr/   r&   rÝ   rÞ   c                 óº   — |dd…dd…f                               d||dz   ¦  «        }| j                             |¦  «        }| j                             |¦  «        }|S )aå  
        Decodes generated image tokens from language model to continuous pixel values
        with VQGAN module via upsampling.

        Args:
            image_tokens (`torch.LongTensor` of shape `(batch_size, num_of_tokens)`):
                The tensors corresponding to the input images.
            height (`int`):
                Height of the generated image before upsampling.
            width (`int`):
                Width of the generated image before upsampling.
        Néýÿÿÿr2   r    )r‰   rs  r  rq  rÖ  )r„   r&   rÝ   rÞ   Ú	sequencesr¥  s         r0   Údecode_image_tokenszEmu3Model.decode_image_tokensz  s`   € ð !    C R C Ô(×-Ò-¨b°&¸%À!¹)ÑDÔDˆ	ØÔ.×>Ò>¸yÑIÔIˆØ”×#Ò# LÑ1Ô1ˆØˆr/   rM  rN  r‹  c                 ó   — |€e| |                       ¦   «         t          j        | j        j        t          j        |j        ¬¦  «        ¦  «        k    }|                     d¦  «        }n|| j        j        k    }|                     ¦   «         }|j	        d         |j	        d         z  }| 
                    d¦  «                             |j        ¦  «        }t          ||j	        d         z  |                     ¦   «         k    d|› d|› �¦  «         |S )zï
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        Nr5  r2   r   r    z6Image features and image tokens do not match, tokens: z, features: )ru  r+   Útensorrs  rç  Úlongr  ÚallrØ   r6   r>   rf   r   Únumel)r„   rM  rN  r‹  Úspecial_image_maskÚn_image_tokensÚn_image_featuress          r0   Úget_placeholder_maskzEmu3Model.get_placeholder_mask�  s  € ð ÐØ!.Ð2M°$×2KÒ2KÑ2MÔ2MÝ”˜TÔ4ÔCÍ5Ì:Ð^kÔ^rÐsÑsÔsñ3ô 3ò "Ðð "4×!7Ò!7¸Ñ!;Ô!;ÐÐà!*¨dÔ.EÔ.TÒ!TÐà+×/Ò/Ñ1Ô1ˆØ)Ô/°Ô2°^Ô5IÈ!Ô5LÑLÐØ/×9Ò9¸"Ñ=Ô=×@Ò@ÀÔAUÑVÔVÐÝØ˜]Ô0°Ô4Ñ4¸×8LÒ8LÑ8NÔ8NÒNØsÀ^ÐsÐsÐaqÐsÐsñ	
ô 	
ð 	
ð "Ð!r/   NrW   rÄ   r‡   rÅ   c	           	      ó\  — |du |duz  rt          d¦  «        ‚|€ |                      ¦   «         |¦  «        }|�_|                      ||¦  «        j        }
t	          j        |
d¬¦  «        }
|                      |||
¬¦  «        }|                     ||
¦  «        } | j        d|||||dœ|	¤Ž}|S )ap  
        image_sizes (`torch.LongTensor` of shape `(batch_size, 2)`):
            The sizes of the images in the batch, being (height, width) for each image. Image sizes can be obtained using
            [`AutoImageProcessor`]. See [`Emu3ImageProcessor.__call__`] for details ([]`Emu3Processor`] uses
            [`Emu3ImageProcessor`] for processing images).
        NzaYou cannot specify both input_ids and inputs_embeds at the same time, and must specify either oner   r4   )rN  r‹  )rW   rÄ   r‡   rN  rÅ   r.   )	rF  ru  rŒ  rˆ  r+   r7   r™  Úmasked_scatterro  )r„   rM  r”  rÅ  rW   rÄ   r‡   rN  rÅ   rZ   r‹  r–  rd  s                r0   r‘   zEmu3Model.forward¥  sþ   € ð( ˜Ð -°tÐ";Ñ<ð 	ÝØsñô ð ð Ð Ø7˜D×5Ò5Ñ7Ô7¸	ÑBÔBˆMàÐ#Ø!×4Ò4°\À;ÑOÔOÔ]ˆNÝ"œY ~¸1Ð=Ñ=Ô=ˆNØ!%×!:Ò!:Ø¨À~ð ";ñ "ô "Ðð *×8Ò8Ð9KÈ^Ñ\Ô\ˆMð "�$”/ð 
Ø)Ø%Ø+Ø'Øð
ð 
ð ð
ð 
ˆð ˆr/   )NNNNNNNN)r'   r(   r)   rw   ru  rw  r+   r]  r,   rƒ  r   r   r   r   r”   r%   rŒ  rà  r’   r�  r™  r“   r	   rÉ   r   r‘   r•   r–   s   @r0   rk  rk  ;  sa  ø€ € € € € ðð ð ð ð ð6ð 6ð 6ð4ð 4ð 4ð¨UÔ->ð ÈUÔM]ð ÐbgÔbrð ð ð ð ð& Ø€^Øyðñ ô ðØ!Ô-ðØ<AÔ<LðØX^Ð_qÔXrðà	Ð%Ñ	%ðð ð ñô ñ Ôðð0 €U„]�_„_ð°Ô0@ð È#ð ÐVYð ð ð ñ „_ðð$"ØÔ)ð"Ø:?Ô:Kð"Ø]bÔ]nð"ð "ð "ð "ð0 Øð .2Ø15Ø+/Ø.2Ø04Ø(,Ø26Ø!%ð,ð ,àÔ# dÑ*ð,ð Ô'¨$Ñ.ð,ð ”\ DÑ(ð	,ð
 œ tÑ+ð,ð Ô&¨Ñ-ð,ð  ™ð,ð Ô(¨4Ñ/ð,ð ˜$‘;ð,ð Ð+Ô,ð,ð 
Ð'Ñ	'ð,ð ,ð ,ñ „^ñ Ôð,ð ,ð ,ð ,ð ,r/   rk  c                   ót  ‡ — e Zd ZdZddiZˆ fd„Zd„ Zd„ Zdej	        fd„Z
ed	„ ¦   «         Zd
„ Zee	 	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dedz  dej        dz  dedz  dej        dz  deej        z  dee         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚEmu3ForConditionalGenerationr  rU  z$model.text_model.embed_tokens.weightc                 óú   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        j        |j        j        d¬¦  «        | _	        |  
                    ¦   «          d S r[  )rv   rw   rk  r  rb   r}   rn  ry   rF  rV  rÄ  r¶   s     €r0   rw   z%Emu3ForConditionalGeneration.__init__Ú  se   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý˜vÑ&Ô&ˆŒ
Ý”y Ô!3Ô!?ÀÔASÔA^ÐejÐkÑkÔkˆŒà�ŠÑÔÐÐÐr/   c                 ó4   — | j                              ¦   «         S r¸   )r  ru  r©   s    r0   ru  z1Emu3ForConditionalGeneration.get_input_embeddingsá  s   € ØŒz×.Ò.Ñ0Ô0Ð0r/   c                 ó:   — | j                              |¦  «         d S r¸   )r  rw  rx  s     r0   rw  z1Emu3ForConditionalGeneration.set_input_embeddingsä  s   € ØŒ
×'Ò'¨Ñ.Ô.Ð.Ð.Ð.r/   rI   c                 ó   — | j         S r¸   )rV  r©   s    r0   Úget_output_embeddingsz2Emu3ForConditionalGeneration.get_output_embeddingsç  s
   € ØŒ|Ðr/   c                 ó   — | j         j        S r¸   )r  rs  r©   s    r0   rs  z/Emu3ForConditionalGeneration.vocabulary_mappingê  s   € àŒzÔ,Ð,r/   c                 ó&   —  | j         j        di |¤ŽS r@  )r  r�  )r„   rZ   s     r0   r�  z0Emu3ForConditionalGeneration.decode_image_tokensî  s   € Ø-ˆtŒzÔ-Ð7Ð7°Ð7Ð7Ð7r/   Nr   rM  r”  rÅ  rW   rÄ   r‡   rN  rÅ   r\  r]  rZ   c                 ó`  —  | j         d||||||||dœ|¤Ž}|d         }t          |
t          ¦  «        rt          |
 d¦  «        n|
}|                      |dd…|dd…f         ¦  «        }d}|	�  | j        d||	| j        j        j        dœ|¤Ž}t          |||j
        |j        |j        ¬¦  «        S )aÕ  
        image_sizes (`torch.LongTensor` of shape `(batch_size, 2)`):
            The sizes of the images in the batch, being (height, width) for each image. Image sizes can be obtained using
            [`AutoImageProcessor`]. See [`Emu3ImageProcessor.__call__`] for details ([]`Emu3Processor`] uses
            [`Emu3ImageProcessor`] for processing images).
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import Emu3Processor, Emu3ForConditionalGeneration
        >>> import torch
        >>> import httpx
        >>> from io import BytesIO
        >>> from PIL import Image

        >>> model = Emu3ForConditionalGeneration.from_pretrained("BAAI/Emu3-Chat-hf", dtype=torch.bfloat16)
        >>> processor = Emu3Processor.from_pretrained("BAAI/Emu3-Chat-hf")

        >>> conversation = [
        ...     {
        ...     "role": "system",
        ...     "content": [
        ...         {"type": "text", "text": "You are a helpful assistant."},
        ...         ],
        ...     },
        ...     {
        ...     "role": "user",
        ...     "content": [
        ...         {"type": "image"},
        ...         {"type": "text", "text": "Please describe the image."},
        ...         ],
        ...     },
        ... ]

        >>> prompt = processor.apply_chat_template(conversation, add_generation_prompt=True)
        >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> inputs = processor(images=[image], text=[prompt], return_tensors="pt").to(model.device, torch.bfloat16)

        >>> generated_ids = model.generate(**inputs, max_new_tokens=100, do_sample=False)
        >>> processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        ```)rM  r”  rÅ  rW   rÄ   r‡   rN  rÅ   r   Nr_  r`  r.   )r  r°  r’   rb  rV  rc  rp   rn  rF  r   r‡   rG   r§  )r„   rM  r”  rÅ  rW   rÄ   r‡   rN  rÅ   r\  r]  rZ   rd  rG   re  rX  ra  s                    r0   r‘   z$Emu3ForConditionalGeneration.forwardñ  s  € ð@ �$”*ð 

ØØ%Ø#Ø)Ø%Ø+Ø'Øð

ð 

ð ð

ð 

ˆð   œ
ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%ð Ø f¸¼Ô9PÔ9[ðð Ø_eðð ˆDõ &ØØØ#Ô3Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r/   )
NNNNNNNNNr   )r'   r(   r)   Úoutput_modalitiesrf  rw   ru  rw  rb   rn  r¢  Úpropertyrs  r�  r   r   r+   r,   r]  r“   r	   rÉ   r’   r   r   r”   r   r‘   r•   r–   s   @r0   r�  r�  Ö  sÛ  ø€ € € € € Ø)ÐØ*Ð,RÐSÐðð ð ð ð ð1ð 1ð 1ð/ð /ð /ð r¤yð ð ð ð ð ð-ð -ñ „Xð-ð8ð 8ð 8ð Øð .2Ø15Ø+/Ø.2Ø04Ø(,Ø26Ø!%Ø*.Ø-.ð[
ð [
àÔ# dÑ*ð[
ð Ô'¨$Ñ.ð[
ð ”\ DÑ(ð	[
ð
 œ tÑ+ð[
ð Ô&¨Ñ-ð[
ð  ™ð[
ð Ô(¨4Ñ/ð[
ð ˜$‘;ð[
ð Ô  4Ñ'ð[
ð ˜eœlÑ*ð[
ð Ð+Ô,ð[
ð 
Ð'Ñ	'ð[
ð [
ð [
ñ „^ñ Ôð[
ð [
ð [
ð [
ð [
r/   r�  )r�  rT  rA  r  r£  rk  ra  )rR   )arŒ  Úcollections.abcr   Údataclassesr   Ú	functoolsr   r+   Útorch.nnrb   Útorch.nn.functionalrc   rõ   Ú r   r±  Úactivationsr   Úcache_utilsr	   r
   Ú
generationr   Úintegrationsr   r   Úmasking_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_emu3r!   r"   r#   r%   r;   rF   r“   r’   rQ   rn  r«   rl   rn   r™   r­   r»   rË   rå   rø   r  r  r#  r-  r1  r<  rC  r  rN  rT  r_  r{  rƒ  r™  r£  râ  r  r  rA  rT  rk  r�  Ú__all__r.   r/   r0   ú<module>r¾     sÍ  ðð, €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !Ø %Ð %Ð %Ð %Ð %Ð %à €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )Ø LÐ LÐ LÐ LÐ LÐ LÐ LÐ LØ /Ð /Ð /Ð /Ð /Ð /Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø kÐ kÐ kÐ kÐ kÐ kÐ kÐ kÐ kÐ kØ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aØ 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø KÐ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KÐ Kð Ø
ð1ð 1ð 1ð 1ð 1Ð5ñ 1ô 1ñ „ñ „ð1ð(ð (ð (ð ÐÐ-Ñ.Ô.ðð ð ñ /Ô.ðð2	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð& ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð2 ÐÐ)Ñ*Ô*ð@)ð @)ð @)ð @)ð @)�B”Iñ @)ô @)ñ +Ô*ð@)ðF Ð˜YÑ'Ô'ðJð Jð Jð Jð J�"”)ñ Jô Jñ (Ô'ðJð(ð ð ð ð ˆbŒiñ ô ð ð (ð (ð (ð (ð (Ð1ñ (ô (ð (ðV$ð $ð $ð $ð $˜rœyñ $ô $ð $ðD	ð 	ð 	ð 	ð 	 R¤Yñ 	ô 	ð 	ðð ð ð ð  2¤9ñ ô ð ðð ð ð ð �b”iñ ô ð ð:!ð !ð !ð !ð !˜2œ9ñ !ô !ð !ðHð ð ð ð  ¤	ñ ô ð ð.ð ð ð ð  "¤)ñ ô ð ð&.(ð .(ð .(ð .(ð .( 2¤9ñ .(ô .(ð .(ðb<(ð <(ð <(ð <(ð <(˜2œ9ñ <(ô <(ð <(ð~;)ð ;)ð ;)ð ;)ð ;)˜bœiñ ;)ô ;)ð ;)ð|Vð Vð Vð Vð V˜œñ Vô Vð Vðð ð ð ð ˜2œ9ñ ô ð ðD8ð 8ð 8ð 8ð 8˜œñ 8ô 8ð 8ðv7ð 7ð 7ð 7ð 7�r”yñ 7ô 7ð 7ðtCð Cð Cð Cð C�r”yñ Cô Cð CðLCð Cð Cð Cð C�r”yñ Cô Cð CðL €ððñ ô ðy2ð y2ð y2ð y2ð y2�ñ y2ô y2ñô ðy2ðx3%ð 3%ð 3%ð 3%ð 3%ñ 3%ô 3%ð 3%ðl ðð ð ð ð ˜/ñ ô ñ „ðð*6<ð 6<ð 6<ð 6<ð 6<˜"œ)ñ 6<ô 6<ð 6<ðr ðH
ð H
ð H
ð H
ð H
Ð'ñ H
ô H
ñ „ðH
ðV ðI
ð I
ð I
ð I
ð I
Ð)¨?ñ I
ô I
ñ „ðI
ðXXð Xð Xð Xð XÐ#ñ Xô Xð Xðvx
ð x
ð x
ð x
ð x
Ð#6¸ñ x
ô x
ð x
ðvð ð €€€r/   