§
    9²¸jà  ã                   óX  — d dl mZ d dlmZ d dlZd dlmc mZ d dlmZ ddl	m
Z ddlmZ ddlmZ dd	lmZmZmZmZ dd
lmZ ddlmZ ddlmZmZmZ ddlmZmZ ddl m!Z! ddl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z( ddl)m*Z* ddl+m,Z, ddl-m.Z. ddl/m0Z0m1Z1m2Z2  e&j3        e4¦  «        Z5e$ G d„ de¦  «        ¦   «         Z6 e$d¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z7 e$d¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z8 e$d¬¦  «        e G d „ d!e¦  «        ¦   «         ¦   «         Z9 G d"„ d#ej:        ¦  «        Z;d$ej<        d%e=d&ej<        fd'„Z>	 d]d)ej:        d*ej<        d+ej<        d,ej<        d-ej<        dz  d.e?d/e?d0e!e#         fd1„Z@ G d2„ d3ej:        ¦  «        ZA G d4„ d5ej:        ¦  «        ZB G d6„ d7e¦  «        ZC G d8„ d9ej:        ¦  «        ZDe$ G d:„ d;e6¦  «        ¦   «         ZE G d<„ d=ej:        ¦  «        ZF G d>„ d?ej:        ¦  «        ZG G d@„ dAej:        ¦  «        ZH G dB„ dCej:        ¦  «        ZI G dD„ dEej:        ¦  «        ZJ G dF„ dGej:        ¦  «        ZK G dH„ dIej:        ¦  «        ZL G dJ„ dKej:        ¦  «        ZM G dL„ dMej:        ¦  «        ZNe$e G dN„ dOe¦  «        ¦   «         ¦   «         ZO e$dP¬¦  «         G dQ„ dRe6¦  «        ¦   «         ZP G dS„ dTej:        ¦  «        ZQ G dU„ dVej:        ¦  «        ZR e$dW¬¦  «         G dX„ dYe6¦  «        ¦   «         ZS G dZ„ d[e6e¦  «        ZTg d\¢ZUdS )^é    )ÚCallable)Ú	dataclassN)Únné   )Úinitialization)ÚACT2FN)ÚCache)Ú%ClassifierFreeGuidanceLogitsProcessorÚGenerationMixinÚGenerationModeÚLogitsProcessorList)ÚGenerateDecoderOnlyOutput)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚBaseModelOutputWithPoolingÚModelOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚloggingÚtorch_compilable_checkÚ	torch_int)Úmerge_with_config_defaults)Úcapture_outputsé   )Ú	AutoModelé   )ÚJanusConfigÚJanusVisionConfigÚJanusVQVAEConfigc                   óP   ‡ — e Zd ZU eed<   dZdZdZdgZddgZ	dZ
dZdZˆ fd„Zˆ xZS )	ÚJanusPreTrainedModelÚconfigÚmodel©ÚimageÚtextTÚJanusVisionEncoderLayerÚpast_key_valuesÚcausal_maskc                 ó  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rQt	          j        |j        t          j        |j        j	        d         ¦  «         
                    d¦  «        ¦  «         d S d S )Néÿÿÿÿ©r    r/   )ÚsuperÚ_init_weightsÚ
isinstanceÚJanusVisionEmbeddingsÚinitÚcopy_Úposition_idsÚtorchÚarangeÚshapeÚexpand)ÚselfÚmoduleÚ	__class__s     €úx/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/janus/modeling_janus.pyr2   z"JanusPreTrainedModel._init_weights<   s{   ø€ Ý‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ3Ñ4Ô4ð 	iÝŒJ�vÔ*­E¬L¸Ô9LÔ9RÐSUÔ9VÑ,WÔ,W×,^Ò,^Ð_fÑ,gÔ,gÑhÔhÐhÐhÐhð	ið 	ió    )Ú__name__Ú
__module__Ú__qualname__r!   Ú__annotations__Úbase_model_prefixÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_can_compile_fullgraphr2   Ú__classcell__©r>   s   @r?   r%   r%   /   s†   ø€ € € € € € àÐÐÑØÐØ(ÐØ&*Ð#Ø2Ð3ÐØ#4°mÐ"DÐØÐØ€Nà!Ððið ið ið ið ið ið ið ið ir@   r%   z9
    Base class for Janus VQ-VAE mode model outputs.
    )Úcustom_introc                   óP   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dS )ÚJanusVQVAEOutputzÿ
    decoded_pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
        Reconstructed pixel values after encoding and decoding the input.
    embedding_loss (`torch.FloatTensor`):
        Embedding loss.
    NÚdecoded_pixel_valuesÚembedding_loss)	rA   rB   rC   Ú__doc__rR   r8   ÚFloatTensorrD   rS   © r@   r?   rQ   rQ   B   sO   € € € € € € ðð ð 6:Ð˜%Ô+¨dÑ2Ð9Ð9Ñ9Ø/3€N�EÔ%¨Ñ,Ð3Ð3Ñ3Ð3Ð3r@   rQ   zy
    Base class for Janus model's outputs that may also contain a past key/values (to speed up sequential decoding).
    c                   óÄ   — e Zd ZU dZdZej        dz  ed<   dZe	dz  ed<   dZ
eej                 dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dS )ÚJanusBaseModelOutputWithPasta…  
    last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
        Sequence of hidden-states at the output of the last layer of the model.

        If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,
        hidden_size)` is output.
    image_hidden_states (`tuple(torch.FloatTensor)`, *optional*):
        Tuple of `torch.FloatTensor` (one for the output of the image embeddings, `(batch_size, num_images,
        sequence_length, hidden_size)`.

        image_hidden_states of the model produced by the vision encoder, and optionally by the perceiver
    NÚlast_hidden_stater,   Úhidden_statesÚ
attentionsÚimage_hidden_states)rA   rB   rC   rT   rY   r8   rU   rD   r,   r	   rZ   Útupler[   r\   rV   r@   r?   rX   rX   T   s£   € € € € € € ðð ð 37Ð�uÔ(¨4Ñ/Ð6Ð6Ñ6Ø$(€O�U˜T‘\Ð(Ð(Ñ(Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ø;?Ð˜˜uÔ0Ô1°DÑ8Ð?Ð?Ñ?Ð?Ð?r@   rX   zQ
    Base class for Janus causal language model (or autoregressive) outputs.
    c                   óâ   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dS )	ÚJanusCausalLMOutputWithPastae  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`tuple(torch.FloatTensor)`, *optional*):
        Tuple of `torch.FloatTensor` (one for the output of the image embeddings, `(batch_size, num_images,
        sequence_length, hidden_size)`.

        image_hidden_states of the model produced by the vision encoder, and optionally by the perceiver
    NÚlossÚlogitsr,   rZ   r[   r\   )rA   rB   rC   rT   r`   r8   rU   rD   ra   r,   r	   rZ   r]   r[   r\   rV   r@   r?   r_   r_   o   sº   € € € € € € ðð ð" &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø$(€O�U˜T‘\Ð(Ð(Ñ(Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ø;?Ð˜˜uÔ0Ô1°DÑ8Ð?Ð?Ñ?Ð?Ð?r@   r_   c                   óz   ‡ — e Zd Zdefˆ fd„Zdej        dededej        fd„Zdd	ej        d
e	dej        fd„Z
ˆ xZS )r4   r&   c                 ó  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        |j        | _        t          j        |j	        | j        | j        | j        d¬¦  «        | _
        | j        | j        z  dz  | _        | j        | _        t          j        | j        | j        ¦  «        | _        t          j        t!          j        | j        ¦  «                             d¦  «        d¬¦  «        | _        d S )NÚvalid)Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚpaddingr   r0   F)Ú
persistent)r1   Ú__init__r&   Úhidden_sizeÚ	embed_dimÚ
image_sizeÚ
patch_sizer   ÚConv2dÚnum_channelsÚpatch_embeddingÚnum_patchesÚnum_positionsÚ	EmbeddingÚposition_embeddingÚBufferr8   r9   r;   r7   ©r<   r&   r>   s     €r?   rk   zJanusVisionEmbeddings.__init__�   sä   ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ+ˆŒØ Ô+ˆŒØ Ô+ˆŒå!œyØÔ+ØœØœØ”?Øð 
ñ  
ô  
ˆÔð !œO¨t¬Ñ>À1ÑDˆÔØ!Ô-ˆÔÝ"$¤,¨tÔ/AÀ4Ä>Ñ"RÔ"RˆÔÝœI¥e¤l°4Ô3EÑ&FÔ&F×&MÒ&MÈgÑ&VÔ&VÐchÐiÑiÔiˆÔÐÐr@   Ú
embeddingsÚheightÚwidthÚreturnc                 ó~  — |j         d         }| j        j        j         d         }t          j                             ¦   «         s&||k    r ||k    r|                      | j        ¦  «        S | j        j                             d¦  «        }|j         d         }|| j        z  }|| j        z  }	t          |dz  ¦  «        }
| 
                    d|
|
|¦  «        }|                     dddd¦  «        }t          j                             |||	fdd¬	¦  «        }|                     dddd¦  «                             dd|¦  «        }|S )
a  
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing and no class embeddings.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r    r   r/   g      à?r   r   ÚbicubicF)ÚsizeÚmodeÚalign_corners)r:   rv   Úweightr8   ÚjitÚ
is_tracingr7   Ú	unsqueezero   r   ÚreshapeÚpermuter   Ú
functionalÚinterpolateÚview)r<   ry   rz   r{   rs   rt   Úpatch_pos_embedÚdimÚ
new_heightÚ	new_widthÚsqrt_num_positionss              r?   Úinterpolate_pos_encodingz.JanusVisionEmbeddings.interpolate_pos_encoding¤   sL  € ð !Ô& qÔ)ˆØÔ/Ô6Ô<¸QÔ?ˆõ Œy×#Ò#Ñ%Ô%ð 	>¨+¸Ò*FÐ*FÈ6ÐUZÊ?È?Ø×*Ò*¨4Ô+<Ñ=Ô=Ð=àÔ1Ô8×BÒBÀ1ÑEÔEˆàÔ˜rÔ"ˆà˜tœÑ.ˆ
Ø˜Tœ_Ñ,ˆ	å& }°cÑ'9Ñ:Ô:ÐØ)×1Ò1°!Ð5GÐI[Ð]`ÑaÔaˆØ)×1Ò1°!°Q¸¸1Ñ=Ô=ˆåœ-×3Ò3ØØ˜iÐ(ØØð	 4ñ 
ô 
ˆð *×1Ò1°!°Q¸¸1Ñ=Ô=×BÒBÀ1ÀbÈ#ÑNÔNˆØÐr@   FÚpixel_valuesr�   c                 óV  — |j         \  }}}}| j        j        j        }|                      |                     |¬¦  «        ¦  «        }|                     d¦  «                             dd¦  «        }|r|                      |||¦  «        }	n|                      | j	        ¦  «        }	||	z   }|S )N)Údtyper   r    )
r:   rr   r‚   r“   ÚtoÚflattenÚ	transposer�   rv   r7   )
r<   r‘   r�   Ú_rz   r{   Útarget_dtypeÚpatch_embedsry   Ú
pos_embedss
             r?   ÚforwardzJanusVisionEmbeddings.forwardÊ   s­   € Ø*Ô0Ñˆˆ1ˆf�eØÔ+Ô2Ô8ˆØ×+Ò+¨L¯OªOÀ,¨OÑ,OÔ,OÑPÔPˆØ!×)Ò)¨!Ñ,Ô,×6Ò6°q¸!Ñ<Ô<ˆ
à#ð 	DØ×6Ò6°zÀ6È5ÑQÔQˆJˆJà×0Ò0°Ô1BÑCÔCˆJà *Ñ,ˆ
àÐr@   )F)rA   rB   rC   r"   rk   r8   ÚTensorÚintr�   Úboolr›   rM   rN   s   @r?   r4   r4   �   s¸   ø€ € € € € ðjÐ0ð jð jð jð jð jð jð($°5´<ð $Èð $ÐUXð $Ð]bÔ]ið $ð $ð $ð $ðLð  E¤Lð ÈDð Ð]bÔ]ið ð ð ð ð ð ð ð r@   r4   rZ   Ún_repr|   c                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r    N)r:   r;   r†   )rZ   rŸ   ÚbatchÚnum_key_value_headsÚslenÚhead_dims         r?   Ú	repeat_kvr¥   Ú   s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTr@   ç        r=   ÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 ó  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        |z  }
|�|
|z   }
t
          j                             |
dt          j        ¬¦  «         	                    |j
        ¦  «        }
t
          j                             |
|| j        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «                             ¦   «         }||
fS )Nr   r   r/   )rŒ   r“   )ÚpÚtrainingr    )r¥   Únum_key_value_groupsr8   Úmatmulr–   r   rˆ   ÚsoftmaxÚfloat32r”   r“   r¬   r°   Ú
contiguous)r=   r§   r¨   r©   rª   r«   r¬   r­   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r?   Úeager_attention_forwardrº   æ   sé   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r@   c                   óf   ‡ — e Zd ZdZdefˆ fd„Z	 d	dej        dej        dz  dee	         fd„Z
ˆ xZS )
ÚJanusVisionAttentionz(Attention Class for Janus Vision Encoderr&   c                 ó6  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        | j        | j        z  | _        | j        | j        z  | j        k    r t          d| j        › d| j        › d�¦  «        ‚| j        dz  | _	        |j
        | _
        |j        }|j        }d| _        d| _        t          j        | j        | j        | j        z  |j        ¬¦  «        | _        t          j        | j        | j        | j        z  |j        ¬¦  «        | _        t          j        | j        | j        | j        z  |j        ¬¦  «        | _        t          j        | j        | j        ¦  «        | _        |dk    rt          j        |¦  «        nt          j        ¦   «         | _        |rt          j        | j        ¦  «        nt          j        ¦   «         | _        |rt          j        | j        ¦  «        nt          j        ¦   «         | _        d S )	Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).ç      à¿Fr    ©Úbiasr   )r1   rk   r&   rl   rm   Únum_attention_headsÚ	num_headsr¤   Ú
ValueErrorÚscaleÚattention_dropoutÚprojection_dropoutÚuse_qk_normÚ	is_causalr±   r   ÚLinearÚattention_biasÚq_projÚk_projÚv_projÚprojection_layerÚDropoutÚIdentityÚ	LayerNormÚq_normÚk_norm)r<   r&   Úproj_dropoutÚqk_normr>   s       €r?   rk   zJanusVisionAttention.__init__  sÑ  ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ+ˆŒØÔ3ˆŒØœ¨$¬.Ñ8ˆŒØŒ=˜4œ>Ñ)¨T¬^Ò;Ð;Ýð'ÈdÌnð 'ð 'Ø”Nð'ð 'ð 'ñô ð ð ”] DÑ(ˆŒ
Ø!'Ô!9ˆÔØÔ0ˆØÔ$ˆØˆŒð %&ˆÔ!å”i ¤°´ÀÄÑ0NÐU[ÔUjÐkÑkÔkˆŒÝ”i ¤°´ÀÄÑ0NÐU[ÔUjÐkÑkÔkˆŒÝ”i ¤°´ÀÄÑ0NÐU[ÔUjÐkÑkÔkˆŒÝ "¤	¨$¬.¸$¼.Ñ IÔ IˆÔØ>JÈQÒ>NÐ>N¥"¤*¨\Ñ":Ô":Ð":ÕTVÔT_ÑTaÔTaˆÔà6=ÐP•b”l 4¤>Ñ2Ô2Ð2Å2Ä;Á=Ä=ˆŒØ6=ÐP•b”l 4¤>Ñ2Ô2Ð2Å2Ä;Á=Ä=ˆŒˆˆr@   NrZ   rª   r­   c                 ó   — |                      ¦   «         \  }}}|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }	|                     d| j        | j        ¦  «        }|                      |¦  «        }|                     d| j        | j        ¦  «        }|                      |¦  «        }|                     ||| j        | j        ¦  «         	                    dd¦  «        }|                     ||| j        | j        ¦  «         	                    dd¦  «        }|	 
                    ||| j        | j        ¦  «         	                    dd¦  «        }	t          j        | j        j        t          ¦  «        }
 |
| |||	|f| j        sdn| j        | j        | j        dœ|¤Ž\  }}|                     ||| j        ¦  «        }|                      |¦  «        }|                      |¦  «        }||fS )Nr/   r    r   r¦   )r¬   r«   rÈ   )r   rË   rÌ   rÍ   r†   rÂ   r¤   rÒ   rÓ   r–   rŠ   r   Úget_interfacer&   Ú_attn_implementationrº   r°   rÅ   rÄ   rÈ   rm   rÎ   rÆ   )r<   rZ   rª   r­   Ú
batch_sizeÚseq_lenr—   Úquery_statesr¶   r·   Úattention_interfacer¹   r¸   Úoutputs                 r?   r›   zJanusVisionAttention.forward  sù  € ð "/×!3Ò!3Ñ!5Ô!5Ñˆ
�G˜Qà—{’{ =Ñ1Ô1ˆØ—[’[ Ñ/Ô/ˆ
Ø—{’{ =Ñ1Ô1ˆà#×+Ò+¨B°´ÀÄÑNÔNˆØ—{’{ <Ñ0Ô0ˆà×'Ò'¨¨D¬N¸D¼MÑJÔJˆ
Ø—[’[ Ñ,Ô,ˆ
à#×+Ò+¨J¸ÀÄÐQUÔQ^Ñ_Ô_×iÒiÐjkÐmnÑoÔoˆØ×'Ò'¨
°G¸T¼^ÈTÌ]Ñ[Ô[×eÒeÐfgÐijÑkÔkˆ
Ø#×(Ò(¨°W¸d¼nÈdÌmÑ\Ô\×fÒfÐghÐjkÑlÔlˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð  $œ}ÐH�C�C°$Ô2HØ”JØ”nð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð "×)Ò)¨*°g¸t¼~ÑNÔNˆà×&Ò& {Ñ3Ô3ˆØ×(Ò(¨Ñ0Ô0ˆØ�|Ð#Ð#r@   ©N)rA   rB   rC   rT   r"   rk   r8   rœ   r   r   r›   rM   rN   s   @r?   r¼   r¼   ÿ   sœ   ø€ € € € € Ø2Ð2ðQÐ0ð Qð Qð Qð Qð Qð Qð@ /3ð)$ð )$à”|ð)$ð œ tÑ+ð)$ð Ð+Ô,ð	)$ð )$ð )$ð )$ð )$ð )$ð )$ð )$r@   r¼   c                   óH   ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Zˆ xZS )ÚJanusVisionMLPr&   c                 óÎ  •— t          ¦   «                              ¦   «          || _        t          |j        |j        z  ¦  «        | _        t          |j                 | _	        t          j        |j        | j        ¦  «        | _        t          j        | j        |j        ¦  «        | _        t          j        |j        ¦  «        | _        t          j        |j        ¦  «        | _        d S rÞ   )r1   rk   r&   r�   rl   Ú	mlp_ratioÚintermediate_sizer   Ú
hidden_actÚactivation_fnr   rÉ   Úfc1Úfc2rÏ   Úhidden_dropout_rateÚdropout1Údropout2rx   s     €r?   rk   zJanusVisionMLP.__init__L  s©   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ!$ VÔ%7¸&Ô:JÑ%JÑ!KÔ!KˆÔÝ# FÔ$5Ô6ˆÔÝ”9˜VÔ/°Ô1GÑHÔHˆŒÝ”9˜TÔ3°VÔ5GÑHÔHˆŒÝœ
 6Ô#=Ñ>Ô>ˆŒÝœ
 6Ô#=Ñ>Ô>ˆŒˆˆr@   rZ   r|   c                 óØ   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S rÞ   )ræ   rå   ré   rç   rê   ©r<   rZ   s     r?   r›   zJanusVisionMLP.forwardV  s_   € ØŸš Ñ/Ô/ˆØ×*Ò*¨=Ñ9Ô9ˆØŸš mÑ4Ô4ˆØŸš Ñ/Ô/ˆØŸš mÑ4Ô4ˆØÐr@   )	rA   rB   rC   r"   rk   r8   rœ   r›   rM   rN   s   @r?   rà   rà   K  sk   ø€ € € € € ð?Ð0ð ?ð ?ð ?ð ?ð ?ð ?ð U¤\ð °e´lð ð ð ð ð ð ð ð r@   rà   c            	       óv   ‡ — e Zd Zdefˆ fd„Zedej        dej        dee	         dej
        fd„¦   «         Zˆ xZS )r+   r&   c                 óR  •— t          ¦   «                              ¦   «          |j        | _        t	          j        | j        |j        ¬¦  «        | _        t          |¦  «        | _	        t	          j        | j        |j        ¬¦  «        | _
        t          |¦  «        | _        || _        d S ©N)Úeps)r1   rk   rl   rm   r   rÑ   Úlayer_norm_epsÚlayer_norm1r¼   Ú	self_attnÚlayer_norm2rà   Úmlpr&   rx   s     €r?   rk   z JanusVisionEncoderLayer.__init__`  s„   ø€ Ý‰Œ×ÒÑÔÐØÔ+ˆŒÝœ<¨¬¸FÔ<QÐRÑRÔRˆÔÝ-¨fÑ5Ô5ˆŒÝœ<¨¬¸FÔ<QÐRÑRÔRˆÔÝ! &Ñ)Ô)ˆŒØˆŒˆˆr@   rZ   rª   r­   r|   c                 óÆ   — |}|                       |¦  «        } | j        d||dœ|¤Ž\  }}||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|S )N)rZ   rª   rV   )rò   ró   rô   rõ   )r<   rZ   rª   r­   Úresidualr—   s         r?   r›   zJanusVisionEncoderLayer.forwardi  s“   € ð !ˆà×(Ò(¨Ñ7Ô7ˆØ)˜4œ>ð 
Ø'Ø)ð
ð 
ð ð
ð 
Ñˆ�qð
 ! =Ñ0ˆà ˆØ×(Ò(¨Ñ7Ô7ˆØŸš Ñ/Ô/ˆØ  =Ñ0ˆàÐr@   )rA   rB   rC   r"   rk   r   r8   rœ   r   r   rU   r›   rM   rN   s   @r?   r+   r+   _  s˜   ø€ € € € € ðÐ0ð ð ð ð ð ð ð ðà”|ðð œðð Ð+Ô,ð	ð
 
Ô	ðð ð ñ „^ðð ð ð ð r@   r+   c                   ól   ‡ — e Zd ZdZdefˆ fd„Ze	 d	dej        dz  de	e
         defd„¦   «         Zˆ xZS )
ÚJanusVisionEncoderz»
    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
    [`JanusVisionEncoderLayer`].

    Args:
        config: JanusVisionConfig
    r&   c                 óÔ   •‡— t          ¦   «                              ¦   «          ‰| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        d| _        d S )Nc                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS rV   )r+   ©Ú.0r—   r&   s     €r?   ú
<listcomp>z/JanusVisionEncoder.__init__.<locals>.<listcomp>Ž  s"   ø€ Ð$nÐ$nÐ$nÈÕ%<¸VÑ%DÔ%DÐ$nÐ$nÐ$nr@   F)	r1   rk   r&   r   Ú
ModuleListÚrangeÚnum_hidden_layersÚlayersÚgradient_checkpointingrx   s    `€r?   rk   zJanusVisionEncoder.__init__‹  sa   øø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”mÐ$nÐ$nÐ$nÐ$nÍeÐTZÔTlÑNmÔNmÐ$nÑ$nÔ$nÑoÔoˆŒØ&+ˆÔ#Ð#Ð#r@   Nrª   r­   r|   c                 óN   — |}| j         D ]} |||fi |¤Ž}Œt          |¬¦  «        S )N)rY   )r  r   )r<   Úinputs_embedsrª   r­   rZ   Úencoder_layers         r?   r›   zJanusVisionEncoder.forward’  sU   € ð &ˆØ!œ[ð 	ð 	ˆMØ)˜MØØðð ð ðð ˆMˆMõ °Ð?Ñ?Ô?Ð?r@   rÞ   )rA   rB   rC   rT   r"   rk   r   r8   rœ   r   r   r   r›   rM   rN   s   @r?   rù   rù   ‚  s°   ø€ € € € € ðð ð,Ð0ð ,ð ,ð ,ð ,ð ,ð ,ð ð /3ð@ð @ð œ tÑ+ð@ð Ð+Ô,ð	@ð
 
ð@ð @ð @ñ „^ð@ð @ð @ð @ð @r@   rù   c                   óÌ   ‡ — e Zd ZU dZdZeed<   eedœZ	defˆ fd„Z
e ed¬¦  «        e	 	 ddej        dz  d	ed
ee         deez  fd„¦   «         ¦   «         ¦   «         Zd„ Zˆ xZS )ÚJanusVisionModelr‘   )r)   r&   ©rZ   r[   c                 ó  •— t          ¦   «                              |¦  «         || _        |j        }t	          |¦  «        | _        t          |¦  «        | _        t          j	        ||j
        ¬¦  «        | _        |                      ¦   «          d S rï   )r1   rk   r&   rl   r4   ry   rù   Úencoderr   rÑ   rñ   Úpost_layernormÚ	post_init)r<   r&   rm   r>   s      €r?   rk   zJanusVisionModel.__init__®  sx   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒØÔ&ˆ	å/°Ñ7Ô7ˆŒÝ)¨&Ñ1Ô1ˆŒÝ œl¨9¸&Ô:OÐPÑPÔPˆÔà�ŠÑÔÐÐÐr@   F)Útie_last_hidden_statesNr�   r­   r|   c                 ó  — |€t          d¦  «        ‚|                      ||¬¦  «        } | j        dd|i|¤Ž}|j        }|                      |¦  «        }|d d …dd d …f         }|                      |¦  «        }t          ||¬¦  «        S )Nz You have to specify pixel_values)r�   r  r   )rY   Úpooler_outputrV   )rÃ   ry   r  rY   r  r   )r<   r‘   r�   r­   rZ   Úencoder_outputsrY   Úpooled_outputs           r?   r›   zJanusVisionModel.forward¹  s¿   € ð ÐÝÐ?Ñ@Ô@Ð@àŸš¨ÐOg˜ÑhÔhˆà+7¨4¬<ð ,
ð ,
Ø'ð,
àð,
ð ,
ˆð
 ,Ô=ÐØ ×/Ò/Ð0AÑBÔBÐà)¨!¨!¨!¨Q°°°¨'Ô2ˆØ×+Ò+¨MÑ:Ô:ˆå)Ø/Ø'ð
ñ 
ô 
ð 	
r@   c                 ó   — | j         S rÞ   )ry   )r<   s    r?   Úget_input_embeddingsz%JanusVisionModel.get_input_embeddings×  s
   € ØŒÐr@   ©NF)rA   rB   rC   Úmain_input_namerF   r"   rD   r+   r¼   Ú_can_record_outputsrk   r   r   r   r8   rU   rž   r   r   r]   r   r›   r  rM   rN   s   @r?   r  r  ¤  s
  ø€ € € € € € à$€OØ!ÐØÐÐÑà0Ø*ðð Ðð
	Ð0ð 	ð 	ð 	ð 	ð 	ð 	ð  Ø€_¨EÐ2Ñ2Ô2Øð 26Ø).ð
ð 
àÔ'¨$Ñ.ð
ð #'ð
ð Ð+Ô,ð	
ð
 
Ð+Ñ	+ð
ð 
ð 
ñ „^ñ 3Ô2ñ  Ôð
ð6ð ð ð ð ð ð r@   r  c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚJanusVisionAlignerMLPr&   c                 ó0  •‡— t          ¦   «                              ¦   «          t          j        ‰j        ‰j        ¦  «        | _        t          j        ˆfd„t          d‰j	        ¦  «        D ¦   «         ¦  «        | _
        t          ‰j                 | _        d S )Nc                 óN   •— g | ]!}t          j        ‰j        ‰j        ¦  «        ‘Œ"S rV   ©r   rÉ   Úprojection_dimrü   s     €r?   rþ   z2JanusVisionAlignerMLP.__init__.<locals>.<listcomp>á  s+   ø€ ÐeÐeÐeÈ�RŒY�vÔ,¨fÔ.CÑDÔDÐeÐeÐer@   r    )r1   rk   r   rÉ   rl   r  ræ   rÿ   r   ÚdepthÚhidden_layersr   rä   rå   rx   s    `€r?   rk   zJanusVisionAlignerMLP.__init__Ü  s…   øø€ Ý‰Œ×ÒÑÔÐå”9˜VÔ/°Ô1FÑGÔGˆŒÝœ]ØeÐeÐeÐeÍeÐTUÐW]ÔWcÑNdÔNdÐeÑeÔeñ
ô 
ˆÔõ $ FÔ$5Ô6ˆÔÐÐr@   c                 ó„   — |                       |¦  «        }| j        D ]"}|                      |¦  «        } ||¦  «        }Œ#|S rÞ   ©ræ   r  rå   ©r<   rZ   Úlayers      r?   r›   zJanusVisionAlignerMLP.forwardå  óO   € ØŸš Ñ/Ô/ˆØÔ'ð 	1ð 	1ˆEØ ×.Ò.¨}Ñ=Ô=ˆMØ!˜E -Ñ0Ô0ˆMˆMØÐr@   )rA   rB   rC   r"   rk   r›   rM   rN   s   @r?   r  r  Û  sT   ø€ € € € € ð7Ð0ð 7ð 7ð 7ð 7ð 7ð 7ðð ð ð ð ð ð r@   r  c                   ób   ‡ — e Zd ZdZdefˆ fd„Zdej        fd„Zdej	        dej
        fd„Zˆ xZS )	ÚJanusVQVAEVectorQuantizeraã  
    A module for vector quantization using learned embedding vectors.

    This module implements the quantization process similar to the one described in
    the VQ-VAE (Vector Quantized Variational AutoEncoder) paper. It quantizes continuous
    input vectors into discrete codebook vectors, which are learned during training.
    Current implementation improves over previous ones by avoiding costly matrix multiplications
    and allowing for post-hoc remapping of indices.
    r&   c                 ó  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        t          |dd¦  «        | _        t          j        | j        | j        ¦  «        | _	        |j
        gdz  | _        d S )NÚbetag      Ð?r   )r1   rk   Únum_embeddingsrm   Úembedding_dimÚgetattrr(  r   ru   Ú	embeddingrs   Úquant_state_dimsrx   s     €r?   rk   z"JanusVQVAEVectorQuantizer.__init__ø  st   ø€ Ý‰Œ×ÒÑÔÐØ$Ô3ˆÔØ#Ô-ˆÔÝ˜F F¨DÑ1Ô1ˆŒ	åœ dÔ&9¸4Ô;MÑNÔNˆŒØ!'Ô!3Ð 4°qÑ 8ˆÔÐÐr@   Úhidden_statec           
      óR  — |                      dddd¦  «                             ¦   «         }|                     d| j        ¦  «        }t	          j        |dz  dd¬¦  «        t	          j        | j        j        dz  d¬¦  «        z   dt	          j        d	|| j        j         	                    dd¦  «        ¦  «        z  z
  }t	          j
        |d¬¦  «        }|                      |¦  «                             |j        ¦  «        }t	          j        |                     ¦   «         |z
  dz  ¦  «        | j        t	          j        ||                     ¦   «         z
  dz  ¦  «        z  z   }|||z
                       ¦   «         z   }|                      dddd¦  «                             ¦   «         }|||fS )
Nr   r   r   r    r/   T)rŒ   Úkeepdim©rŒ   z	bd,dn->bn)r‡   rµ   rŠ   r*  r8   Úsumr,  r‚   Úeinsumr–   Úargminr:   ÚmeanÚdetachr(  )r<   r.  Úhidden_state_flattenedÚ	distancesÚmin_encoding_indicesÚhidden_state_quantr`   s          r?   r›   z!JanusVQVAEVectorQuantizer.forward  s¨  € Ø#×+Ò+¨A¨q°!°QÑ7Ô7×BÒBÑDÔDˆØ!-×!2Ò!2°2°tÔ7IÑ!JÔ!JÐõ ŒIÐ,¨aÑ/°QÀÐEÑEÔEÝŒi˜œÔ-¨qÑ0°aÐ8Ñ8Ô8ñ9à•%”,˜{Ð,BÀDÄNÔDY×DcÒDcÐdeÐghÑDiÔDiÑjÔjÑjñkð 	õ  %œ|¨I¸1Ð=Ñ=Ô=ÐØ!Ÿ^š^Ð,@ÑAÔA×FÒFÀ|ÔGYÑZÔZÐõ ŒzÐ-×4Ò4Ñ6Ô6¸ÑEÈ!ÑKÑLÔLÈtÌyÕ[`Ô[eØ ,×"5Ò"5Ñ"7Ô"7Ñ7¸AÑ=ñ\
ô \
ñ P
ñ 
ˆð
 *Ð-?À,Ñ-N×,VÒ,VÑ,XÔ,XÑXÐð 0×7Ò7¸¸1¸aÀÑCÔC×NÒNÑPÔPÐà! 4Ð)=Ð=Ð=r@   Úimage_tokensr|   c                 ó:  — |j         d         }| j        j        j         d         }|                      |¦  «        }t          j        |dd¬¦  «        }|                     |g| j        ¢|‘R ¦  «        }|                     dddd¦  «                             ¦   «         }|S )Nr   r/   r   )r¯   rŒ   r   r    )	r:   r,  r‚   ÚFÚ	normalizerŠ   r-  r‡   rµ   )r<   r;  rÙ   Úemb_dimr:  s        r?   Úget_codebook_entryz,JanusVQVAEVectorQuantizer.get_codebook_entry  s£   € Ø!Ô'¨Ô*ˆ
Ø”~Ô,Ô2°2Ô6ˆð "Ÿ^š^¨LÑ9Ô9Ðåœ[Ð);¸qÀbÐIÑIÔIÐð 0×4Ò4°jÐ5bÀ4ÔCXÐ5bÐZaÐ5bÐ5bÑcÔcÐØ/×7Ò7¸¸1¸aÀÑCÔC×NÒNÑPÔPÐà!Ð!r@   )rA   rB   rC   rT   r#   rk   r8   rœ   r›   Ú
LongTensorrU   r@  rM   rN   s   @r?   r&  r&  í  s”   ø€ € € € € ðð ð9Ð/ð 9ð 9ð 9ð 9ð 9ð 9ð> E¤Lð >ð >ð >ð >ð6"¨uÔ/?ð "ÀEÔDUð "ð "ð "ð "ð "ð "ð "ð "r@   r&  c                   ó*   ‡ — e Zd Z	 	 dˆ fd„	Zd„ Zˆ xZS )ÚJanusVQVAEResnetBlockNFc                 óê  •— t          ¦   «                              ¦   «          || _        |€|n|| _        || _        t
          j                             d|dd¬¦  «        | _        t
          j         	                    ||ddd¬¦  «        | _
        t
          j                             d|dd¬¦  «        | _        t
          j                             |j        ¦  «        | _        t
          j         	                    ||ddd¬¦  «        | _        | j        | j        k    r]| j        r+t
          j         	                    ||ddd¬¦  «        | _        d S t
          j         	                    ||ddd¬¦  «        | _        d S d S )	Né    ç�íµ ÷Æ°>T©Ú
num_groupsrq   rð   Úaffiner   r    ©rg   rh   ri   r   )r1   rk   re   rf   Úuse_conv_shortcutr8   r   Ú	GroupNormÚnorm1rp   Úconv1Únorm2rÏ   r¬   Úconv2Úconv_shortcutÚnin_shortcut)r<   r&   re   rf   rQ  r>   s        €r?   rk   zJanusVQVAEResnetBlock.__init__-  sR  ø€ õ 	‰Œ×ÒÑÔÐØ&ˆÔØ+7Ð+?˜K˜KÀ\ˆÔØ!.ˆÔå”X×'Ò'°2ÀKÐUYÐbfÐ'ÑgÔgˆŒ
Ý”X—_’_ [°,ÈAÐVWÐab�_ÑcÔcˆŒ
Ý”X×'Ò'°2ÀLÐVZÐcgÐ'ÑhÔhˆŒ
Ý”x×'Ò'¨¬Ñ7Ô7ˆŒÝ”X—_’_ \°<ÈQÐWXÐbc�_ÑdÔdˆŒ
ØÔ˜tÔ0Ò0Ð0ØÔ%ð sÝ%*¤X§_¢_°[À,Ð\]ÐfgÐqr _Ñ%sÔ%s�Ô"Ð"Ð"å$)¤H§O¢O°KÀÐ[\ÐefÐpq OÑ$rÔ$r�Ô!Ð!Ð!ð	 1Ð0r@   c                 óÂ  — |}|                       |¦  «        }|t          j        |¦  «        z  }|                      |¦  «        }|                      |¦  «        }|t          j        |¦  «        z  }|                      |¦  «        }|                      |¦  «        }| j        | j        k    r2| j	        r|  
                    |¦  «        }n|                      |¦  «        }||z   S rÞ   )rM  r8   ÚsigmoidrN  rO  r¬   rP  re   rf   rK  rQ  rR  )r<   rZ   r÷   s      r?   r›   zJanusVQVAEResnetBlock.forwardD  sÓ   € Ø ˆØŸ
š
 =Ñ1Ô1ˆØ�œ }Ñ5Ô5Ñ5ˆØŸ
š
 =Ñ1Ô1ˆàŸ
š
 =Ñ1Ô1ˆØ�œ }Ñ5Ô5Ñ5ˆØŸš ]Ñ3Ô3ˆØŸ
š
 =Ñ1Ô1ˆàÔ˜tÔ0Ò0Ð0ØÔ%ð 7Ø×-Ò-¨hÑ7Ô7��à×,Ò,¨XÑ6Ô6�à˜-Ñ'Ð'r@   r  ©rA   rB   rC   rk   r›   rM   rN   s   @r?   rC  rC  ,  sZ   ø€ € € € € ð
 Øðsð sð sð sð sð sð.(ð (ð (ð (ð (ð (ð (r@   rC  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚJanusVQVAEAttnBlockc                 óî  •— t          ¦   «                              ¦   «          || _        t          j                             d|dd¬¦  «        | _        t          j                             ||ddd¬¦  «        | _        t          j                             ||ddd¬¦  «        | _	        t          j                             ||ddd¬¦  «        | _
        t          j                             ||ddd¬¦  «        | _        d S )NrE  rF  TrG  r    r   rJ  )r1   rk   re   r8   r   rL  Únormrp   ÚqÚkÚvÚproj_out©r<   re   r>   s     €r?   rk   zJanusVQVAEAttnBlock.__init__Y  sË   ø€ Ý‰Œ×ÒÑÔÐØ&ˆÔå”H×&Ò&°"À;ÐTXÐaeÐ&ÑfÔfˆŒ	Ý”—’ ¨kÀqÐQRÐ\]�Ñ^Ô^ˆŒÝ”—’ ¨kÀqÐQRÐ\]�Ñ^Ô^ˆŒÝ”—’ ¨kÀqÐQRÐ\]�Ñ^Ô^ˆŒÝœŸš¨°[ÈaÐXYÐcd˜ÑeÔeˆŒˆˆr@   c                 óÄ  — |}|                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|j        \  }}}}	|                     ||||	z  ¦  «                             ddd¦  «        }|                     ||||	z  ¦  «        }t          j        ||¦  «        }
|
t          |¦  «        dz  z  }
t          j        |
d¬¦  «        }
|                     ||||	z  ¦  «        }|
                     ddd¦  «        }
t          j        ||
¦  «                             ||||	¦  «        }|                      |¦  «        }||z   S )Nr   r   r    r¾   r1  )rY  rZ  r[  r\  r:   r†   r‡   r8   Úbmmr�   r=  r³   r]  )r<   rZ   r÷   rÛ   r¶   r·   rÙ   Úchannelsrz   r{   r¸   r¹   s               r?   r›   zJanusVQVAEAttnBlock.forwardc  s[  € Ø ˆØŸ	š	 -Ñ0Ô0ˆØ—v’v˜mÑ,Ô,ˆØ—V’V˜MÑ*Ô*ˆ
Ø—v’v˜mÑ,Ô,ˆð /;Ô.@Ñ+ˆ
�H˜f eØ#×+Ò+¨J¸À&È5Á.ÑQÔQ×YÒYÐZ[Ð]^Ð`aÑbÔbˆØ×'Ò'¨
°H¸fÀu¹nÑMÔMˆ
Ý”y ¨zÑ:Ô:ˆØ#¥s¨8¡}¤}¸Ñ'>Ñ?ˆÝ”y °1Ð5Ñ5Ô5ˆð $×+Ò+¨J¸À&È5Á.ÑQÔQˆØ#×+Ò+¨A¨q°!Ñ4Ô4ˆÝ”i ¨lÑ;Ô;×CÒCÀJÐPXÐZ`ÐbgÑhÔhˆà—m’m KÑ0Ô0ˆØ˜+Ñ%Ð%r@   rU  rN   s   @r?   rW  rW  X  sL   ø€ € € € € ðfð fð fð fð fð&ð &ð &ð &ð &ð &ð &r@   rW  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚJanusVQVAEConvDownsamplec                 ó„   •— t          ¦   «                              ¦   «          t          j        ||ddd¬¦  «        | _        d S )Nr   r   r   rJ  )r1   rk   r   rp   Úconvr^  s     €r?   rk   z!JanusVQVAEConvDownsample.__init__|  s:   ø€ Ý‰Œ×ÒÑÔÐÝ”I˜k¨;ÀAÈaÐYZÐ[Ñ[Ô[ˆŒ	ˆ	ˆ	r@   c                 ó`   — t          j        |ddd¬¦  «        }|                      |¦  «        }|S )N)r   r    r   r    Úconstantr   )Úpadr€   r©   )r=  rh  re  rì   s     r?   r›   z JanusVQVAEConvDownsample.forward€  s2   € åœ˜m°ÀJÐVWÐXÑXÔXˆØŸ	š	 -Ñ0Ô0ˆØÐr@   rU  rN   s   @r?   rc  rc  {  sL   ø€ € € € € ð\ð \ð \ð \ð \ðð ð ð ð ð ð r@   rc  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚJanusVQVAEConvUpsamplec                 óš   •— t          ¦   «                              ¦   «          t          j                             ||ddd¬¦  «        | _        d S )Nr   r    rJ  )r1   rk   r8   r   rp   re  r^  s     €r?   rk   zJanusVQVAEConvUpsample.__init__ˆ  s>   ø€ Ý‰Œ×ÒÑÔÐÝ”H—O’O K°È!ÐTUÐ_`�OÑaÔaˆŒ	ˆ	ˆ	r@   c                 ó^   — t          j        |dd¬¦  «        }|                      |¦  «        }|S )Ng       @Únearest)Úscale_factorr€   )r=  r‰   re  rì   s     r?   r›   zJanusVQVAEConvUpsample.forwardŒ  s/   € Ýœ mÀ#ÈIÐVÑVÔVˆØŸ	š	 -Ñ0Ô0ˆØÐr@   rU  rN   s   @r?   rj  rj  ‡  sL   ø€ € € € € ðbð bð bð bð bðð ð ð ð ð ð r@   rj  c                   óL   ‡ — e Zd Zdedefˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚJanusVQVAEMidBlockr&   ra  c                 óÌ   •— t          ¦   «                              ¦   «          t          |||¬¦  «        | _        t	          |¦  «        | _        t          |||¬¦  «        | _        d S )N©r&   re   rf   )r1   rk   rC  Úblock_1rW  Úattn_1Úblock_2)r<   r&   ra  r>   s      €r?   rk   zJanusVQVAEMidBlock.__init__“  sl   ø€ Ý‰Œ×ÒÑÔÐÝ,ØØ Ø!ð
ñ 
ô 
ˆŒõ
 *¨(Ñ3Ô3ˆŒÝ,ØØ Ø!ð
ñ 
ô 
ˆŒˆˆr@   rZ   r|   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S rÞ   )rs  rt  ru  rì   s     r?   r›   zJanusVQVAEMidBlock.forward¡  s;   € ØŸš ]Ñ3Ô3ˆØŸš MÑ2Ô2ˆØŸš ]Ñ3Ô3ˆØÐr@   )
rA   rB   rC   r#   r�   rk   r8   rœ   r›   rM   rN   s   @r?   rp  rp  ’  sr   ø€ € € € € ð
Ð/ð 
¸3ð 
ð 
ð 
ð 
ð 
ð 
ð U¤\ð °e´lð ð ð ð ð ð ð ð r@   rp  c                   ó4   ‡ — e Zd Zˆ fd„Zdej        fd„Zˆ xZS )ÚJanusVQVAEEncoderc           	      ó¦  •— t          ¦   «                              ¦   «          t          |j        ¦  «        | _        |j        | _        |j        }|j        }|j        }|j	        }|j        }t          j                             ||ddd¬¦  «        | _        dt          |¦  «        z   }|| _        t          j        ¦   «         | _        t%          | j        ¦  «        D �]
}t          j        ¦   «         }	t          j        ¦   «         }
|||         z  }|||         z  }t%          | j        ¦  «        D ]Y}|	                     t)          |||¬¦  «        ¦  «         |}|| j        dz
  k    r"|
                     t+          |¦  «        ¦  «         ŒZt          j        ¦   «         }|	|_        |
|_        || j        dz
  k    rt3          |¦  «        |_        | j                             |¦  «         �Œt7          ||¦  «        | _        t          j                             d|dd¬	¦  «        | _        t          j                             ||rd
|z  n|ddd¬¦  «        | _        d S )Nr   r    rJ  )r    rr  rE  rF  TrG  r   ) r1   rk   ÚlenÚchannel_multiplierÚnum_resolutionsÚnum_res_blocksÚbase_channelsre   Údouble_latentÚlatent_channelsr8   r   rp   Úconv_inr]   Úin_channel_multiplierrÿ   Údownr   ÚappendrC  rW  ÚModuleÚblockÚattnrc  Ú
downsamplerp  ÚmidrL  Únorm_outÚconv_out)r<   r&   r~  re   r  r€  r{  r‚  Úi_levelr†  r‡  Úblock_inÚ	block_outÚi_blockrƒ  r>   s                  €r?   rk   zJanusVQVAEEncoder.__init__©  s@  ø€ Ý‰Œ×ÒÑÔÐå" 6Ô#<Ñ=Ô=ˆÔØ$Ô3ˆÔØÔ,ˆØÔ(ˆØÔ,ˆØ Ô0ˆØ#Ô6Ðå”x—’ {°MÈqÐYZÐde�ÑfÔfˆŒà $¥uÐ-?Ñ'@Ô'@Ñ @ÐØ%:ˆÔ"Ý”M‘O”OˆŒ	Ý˜TÔ1Ñ2Ô2ð 	#ñ 	#ˆGÝ”M‘O”OˆEÝ”=‘?”?ˆDØ$Ð'<¸WÔ'EÑEˆHØ%Ð(:¸7Ô(CÑCˆIÝ  Ô!4Ñ5Ô5ð 
?ð 
?�Ø—’Ý)Ø%Ø$,Ø%.ðñ ô ñô ð ð %�Ø˜dÔ2°QÑ6Ò6Ð6Ø—K’KÕ 3°HÑ =Ô =Ñ>Ô>Ð>øå”9‘;”;ˆDØˆDŒJØˆDŒIØ˜$Ô.°Ñ2Ò2Ð2Ý":¸8Ñ"DÔ"D�”ØŒI×Ò˜TÑ"Ô"Ð"Ñ"å% f¨hÑ7Ô7ˆŒåœ×*Ò*°bÀxÐUYÐbfÐ*ÑgÔgˆŒÝœŸšØØ#0ÐEˆA�ÑÐ°oØØØð (ñ 
ô 
ˆŒˆˆr@   r‘   c                 óØ  — |                       |¦  «        g}t          | j        ¦  «        D ]à}t          | j        ¦  «        D ]‚} | j        |         j        |         |d         ¦  «        }t          | j        |         j        ¦  «        dk    r! | j        |         j        |         |¦  «        }|                     |¦  «         Œƒ|| j        dz
  k    r9|                     | j        |          	                    |d         ¦  «        ¦  «         Œá|d         }|  
                    |¦  «        }|                      |¦  «        }|t          j        |¦  «        z  }|                      |¦  «        }|S )Nr/   r   r    )r�  r   r|  r}  rƒ  r†  rz  r‡  r„  rˆ  r‰  rŠ  r8   rT  r‹  )r<   r‘   rZ   rŒ  r�  r.  rY   s          r?   r›   zJanusVQVAEEncoder.forwardÜ  so  € àŸš lÑ3Ô3Ð4ˆÝ˜TÔ1Ñ2Ô2ð 		Wð 		WˆGÝ  Ô!4Ñ5Ô5ð 3ð 3�Ø@˜tœy¨Ô1Ô7¸Ô@Ø! "Ô%ñ ô  �õ �t”y Ô)Ô.Ñ/Ô/°!Ò3Ð3Ø#C 4¤9¨WÔ#5Ô#:¸7Ô#CÀLÑ#QÔ#Q�LØ×$Ò$ \Ñ2Ô2Ð2Ð2Ø˜$Ô.°Ñ2Ò2Ð2Ø×$Ò$ T¤Y¨wÔ%7×%BÒ%BÀ=ÐQSÔCTÑ%UÔ%UÑVÔVÐVøð *¨"Ô-ÐØ ŸHšHÐ%6Ñ7Ô7Ðð !ŸMšMÐ*;Ñ<Ô<ÐØ�Uœ]Ð+<Ñ=Ô=Ñ=ÐØ ŸMšMÐ*;Ñ<Ô<ÐØ Ð r@   )rA   rB   rC   rk   r8   rA  r›   rM   rN   s   @r?   rx  rx  ¨  sW   ø€ € € € € ð1
ð 1
ð 1
ð 1
ð 1
ðf! EÔ$4ð !ð !ð !ð !ð !ð !ð !ð !r@   rx  c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚJanusVQVAEDecoderc           	      óz  •— t          ¦   «                              ¦   «          t          |j        ¦  «        | _        |j        | _        |j        }|j        }|j        }||j        | j        dz
           z  }t          j
                             ||ddd¬¦  «        | _        t          ||¦  «        | _        t          j        ¦   «         | _        t#          t%          | j        ¦  «        ¦  «        D ]þ}t          j        ¦   «         }t          j        ¦   «         }||j        |         z  }	t%          | j        dz   ¦  «        D ]Y}
|                     t)          |||	¬¦  «        ¦  «         |	}|| j        dz
  k    r"|                     t+          |¦  «        ¦  «         ŒZt          j        ¦   «         }||_        ||_        |dk    rt3          |¦  «        |_        | j                             |¦  «         Œÿt          j
                             d|dd¬	¦  «        | _        t          j
                             ||ddd¬¦  «        | _        d S )
Nr    r   rJ  rr  r   rE  rF  TrG  )r1   rk   rz  r{  r|  r}  r~  r€  rf   r8   r   rp   r�  rp  r‰  rÿ   ÚupÚreversedr   r„  rC  rW  r…  r†  r‡  rj  ÚupsamplerL  rŠ  r‹  )r<   r&   r~  r€  rf   r�  rŒ  r†  r‡  rŽ  r�  r”  r>   s               €r?   rk   zJanusVQVAEDecoder.__init__ö  s  ø€ Ý‰Œ×ÒÑÔÐå" 6Ô#<Ñ=Ô=ˆÔØ$Ô3ˆÔØÔ,ˆØ Ô0ˆØÔ*ˆð ! 6Ô#<¸TÔ=QÐTUÑ=UÔ#VÑVˆõ ”x—’ ¸ÈaÐXYÐcd�ÑeÔeˆŒõ & f¨hÑ7Ô7ˆŒõ ”-‘/”/ˆŒÝ¥ dÔ&:Ñ ;Ô ;Ñ<Ô<ð 	ð 	ˆGÝ”M‘O”OˆEÝ”=‘?”?ˆDØ%¨Ô(AÀ'Ô(JÑJˆIÝ  Ô!4°qÑ!8Ñ9Ô9ð 
?ð 
?�Ø—’Ý)Ø%Ø$,Ø%.ðñ ô ñô ð ð %�Ø˜dÔ2°QÑ6Ò6Ð6Ø—K’KÕ 3°HÑ =Ô =Ñ>Ô>Ð>øÝ”‘”ˆBØˆBŒHØˆBŒGØ˜!Š|ˆ|Ý4°XÑ>Ô>�”ØŒG�NŠN˜2ÑÔÐÐõ œ×*Ò*°bÀxÐUYÐbfÐ*ÑgÔgˆŒÝœŸš¨°,ÈAÐVWÐab˜ÑcÔcˆŒˆˆr@   r.  r|   c                 ód  — |                       |¦  «        }|                      |¦  «        }t          | j        ¦  «        D ]¯}t          | j        dz   ¦  «        D ]g} | j        |         j        |         |¦  «        }t          | j        |         j        ¦  «        dk    r! | j        |         j        |         |¦  «        }Œh|| j        dz
  k    r | j        |          	                    |¦  «        }Œ°|  
                    |¦  «        }|t          j        |¦  «        z  }|                      |¦  «        }|S )Nr    r   )r�  r‰  r   r|  r}  r”  r†  rz  r‡  r–  rŠ  r8   rT  r‹  )r<   r.  rŒ  r�  s       r?   r›   zJanusVQVAEDecoder.forward$  s)  € Ø—|’| LÑ1Ô1ˆð —x’x Ñ-Ô-ˆõ ˜TÔ1Ñ2Ô2ð 	Gð 	GˆGÝ  Ô!4°qÑ!8Ñ9Ô9ð Pð P�Ø>˜tœw wÔ/Ô5°gÔ>¸|ÑLÔL�Ý�t”w˜wÔ'Ô,Ñ-Ô-°Ò1Ð1Ø#A 4¤7¨7Ô#3Ô#8¸Ô#AÀ,Ñ#OÔ#O�LøØ˜$Ô.°Ñ2Ò2Ð2Ø#œw wÔ/×8Ò8¸ÑFÔF�øà—}’} \Ñ2Ô2ˆØ�œ lÑ3Ô3Ñ3ˆØ—}’} \Ñ2Ô2ˆØÐr@   )rA   rB   rC   rk   r8   rU   r›   rM   rN   s   @r?   r’  r’  õ  sf   ø€ € € € € ð,dð ,dð ,dð ,dð ,dð\ EÔ$5ð ¸%Ô:Kð ð ð ð ð ð ð ð r@   r’  c                   ón   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	ej        dz  ed<   dS )ÚJanusVQVAEModelOutputa­  
    quantized_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
        Quantized last hidden state from the VQ-VAE model.
    image_tokens (`torch.FloatTensor` of shape `(batch_size, config.vocab_size`):
        Indices of the image tokens predicted by the VQ-VAE model.
    embedding_loss (`torch.FloatTensor`):
        The embedding loss computed during quantization.
    NÚquantized_last_hidden_stater;  rS   )
rA   rB   rC   rT   rš  r8   rU   rD   r;  rS   rV   r@   r?   r™  r™  9  sh   € € € € € € ðð ð =AÐ Ô!2°TÑ!9Ð@Ð@Ñ@Ø-1€L�%Ô# dÑ*Ð1Ð1Ñ1Ø/3€N�EÔ%¨Ñ,Ð3Ð3Ñ3Ð3Ð3r@   r™  aS  
    The VQ-VAE model used in Janus for encoding/decoding images into discrete tokens.
    This model follows the "Make-a-scene: Scene-based text-to-image generation with human priors" paper from
    [ Oran Gafni, Adam Polyak, Oron Ashual, Shelly Sheynin, Devi Parikh, and Yaniv
    Taigman](https://huggingface.co/papers/2203.13131).
    c                   ó  ‡ — e Zd ZU eed<   g d¢ZeedœZdZ	defˆ fd„Z
eedej        dee         defd„¦   «         ¦   «         Zd	ej        dej        fd
„Zeedej        deej        ej        f         fd„¦   «         ¦   «         Zˆ xZS )Ú
JanusVQVAEr&   )rW  rC  r&  r	  r‘   c                 óà  •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          |¦  «        | _        t          j                             |j	        |j
        d¦  «        | _        t          j                             |j
        |j	        d¦  «        | _        |                      ¦   «          t          |¦  «        | _        d| _        |                      ¦   «          d S )Nr    F)r1   rk   rx  r  r&  Úquantizer8   r   rp   r€  rm   Ú
quant_convÚpost_quant_convÚevalr’  Údecoderr  r  rx   s     €r?   rk   zJanusVQVAE.__init___  s´   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å(¨Ñ0Ô0ˆŒÝ1°&Ñ9Ô9ˆŒÝœ(Ÿ/š/¨&Ô*@À&ÔBRÐTUÑVÔVˆŒÝ$œxŸš¨vÔ/?ÀÔAWÐYZÑ[Ô[ˆÔØ�	Š	‰ŒˆÝ(¨Ñ0Ô0ˆŒØ&+ˆÔ#Ø�ŠÑÔÐÐÐr@   r­   r|   c                 ó®   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        \  }}}t          ||||¬¦  «        S )N)rY   rš  r;  rS   )r  rŸ  rž  r™  )r<   r‘   r­   rZ   Úconv_hidden_statesrš  Úemb_lossÚindicess           r?   ÚencodezJanusVQVAE.encodek  sd   € ð Ÿš \Ñ2Ô2ˆØ!Ÿ_š_¨]Ñ;Ô;ÐØ9=¿ºÐGYÑ9ZÔ9ZÑ6Ð# X¨wÝ$Ø+Ø(CØ Ø#ð	
ñ 
ô 
ð 	
r@   r;  c                 ór  — |j         d         | j        j        d         | j        j        d         z  k    r>t          d| j        j        d         | j        j        d         z  › d|j         › d�¦  «        ‚| j                             |¦  «        }|                      |¦  «        }|                      |¦  «        }|S )aG  
        Decodes quantized token IDs into pixel values.
        Args:
            image_tokens (torch.LongTensor): Batch of token IDs.
        Returns:
            pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
                Pixel values decoded from the token IDs.
        r    r   z4Expected `image_tokens` to have shape `(batch_size, z)`, but got shape `z`.)r:   rž  r-  rÃ   r@  r   r¢  )r<   r;  Úcodebook_entryrZ   r‘   s        r?   ÚdecodezJanusVQVAE.decodex  s×   € ð Ô˜aÔ  D¤MÔ$BÀ1Ô$EÈÌÔHfÐghÔHiÑ$iÒiÐiÝð9ÀtÄ}ÔGeÐfgÔGhÐkoÔkxô  lJð  KLô  lMñ  HMð 9ð 9Ø".Ô"4ð9ð 9ð 9ñô ð ð œ×9Ò9¸,ÑGÔGˆØ×,Ò,¨^Ñ<Ô<ˆØ—|’| MÑ2Ô2ˆØÐr@   c                 óÂ   — |j         d         } | j        |fddi|¤Ž}|                      |j                             |d¦  «        ¦  «        }t          ||j        ¦  «        S )Nr   Úreturn_dictTr/   )r:   r§  rª  r;  rŠ   rQ   rS   )r<   r‘   r­   rÙ   Úencode_outputsrR   s         r?   r›   zJanusVQVAE.forward‹  si   € ð "Ô'¨Ô*ˆ
Ø$˜œ \ÐNÐN¸tÐNÀvÐNÐNˆØ#Ÿ{š{¨>Ô+F×+KÒ+KÈJÐXZÑ+[Ô+[Ñ\Ô\ÐåÐ 4°nÔ6SÑTÔTÐTr@   )rA   rB   rC   r#   rD   rH   rC  rW  r  r  rk   r   r   r8   rA  r   r   r™  r§  rU   rª  r   r   r]   r›   rM   rN   s   @r?   rœ  rœ  J  sO  ø€ € € € € € ð ÐÐÑðð ð Ðð /Ø)ðð Ðð %€Oð
Ð/ð 
ð 
ð 
ð 
ð 
ð 
ð  Øð	
 5Ô#3ð 	
¸vÐFXÔ?Yð 	
Ð^sð 	
ð 	
ð 	
ñ „_ñ  Ôð	
ð 5Ô#3ð ¸Ô8Ið ð ð ð ð& Øð	UàÔ'ð	Uð 
ˆuÔ  %Ô"3Ð3Ô	4ð		Uð 	Uð 	Uñ „^ñ Ôð	Uð 	Uð 	Uð 	Uð 	Ur@   rœ  c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚJanusVQVAEAlignerMLPr&   c                 ó0  •‡— t          ¦   «                              ¦   «          t          j        ‰j        ‰j        ¦  «        | _        t          j        ˆfd„t          d‰j	        ¦  «        D ¦   «         ¦  «        | _
        t          ‰j                 | _        d S )Nc                 óN   •— g | ]!}t          j        ‰j        ‰j        ¦  «        ‘Œ"S rV   r  rü   s     €r?   rþ   z1JanusVQVAEAlignerMLP.__init__.<locals>.<listcomp>Ÿ  s+   ø€ ÐqÐqÐqÈ�RŒY�vÔ,¨fÔ.CÑDÔDÐqÐqÐqr@   r    )r1   rk   r   rÉ   rm   r  ræ   rÿ   r   r  r  r   rä   rå   rx   s    `€r?   rk   zJanusVQVAEAlignerMLP.__init__š  s…   øø€ Ý‰Œ×ÒÑÔÐå”9˜VÔ-¨vÔ/DÑEÔEˆŒÝœ]ØqÐqÐqÐqÍeÐTUÐW]ÔWoÑNpÔNpÐqÑqÔqñ
ô 
ˆÔõ $ FÔ$5Ô6ˆÔÐÐr@   c                 ó„   — |                       |¦  «        }| j        D ]"}|                      |¦  «        } ||¦  «        }Œ#|S rÞ   r!  r"  s      r?   r›   zJanusVQVAEAlignerMLP.forward£  r$  r@   )rA   rB   rC   r#   rk   r›   rM   rN   s   @r?   r¯  r¯  ™  sT   ø€ € € € € ð7Ð/ð 7ð 7ð 7ð 7ð 7ð 7ðð ð ð ð ð ð r@   r¯  c                   óL   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        fd„Z	ˆ xZ
S )ÚJanusVQVAEHeadzOHead used for sampling tokens in image generation, replacing the usual lm head.r&   c                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          |j                 | _	        t          j        |j        |j
        ¦  «        | _        d S rÞ   )r1   rk   r   rÉ   Úimage_token_embed_dimr  r]  r   rä   rå   r)  Úvision_headrx   s     €r?   rk   zJanusVQVAEHead.__init__®  sb   ø€ Ý‰Œ×ÒÑÔÐÝœ	 &Ô">ÀÔ@UÑVÔVˆŒÝ# FÔ$5Ô6ˆÔÝœ9 VÔ%:¸FÔ<QÑRÔRˆÔÐÐr@   rZ   r|   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S rÞ   )r]  rå   r·  rì   s     r?   r›   zJanusVQVAEHead.forward´  s?   € ØŸš mÑ4Ô4ˆØ×*Ò*¨=Ñ9Ô9ˆØ×(Ò(¨Ñ7Ô7ˆØÐr@   )rA   rB   rC   rT   r#   rk   r8   rœ   Útensorr›   rM   rN   s   @r?   r´  r´  «  sx   ø€ € € € € ØYÐYðSÐ/ð Sð Sð Sð Sð Sð Sð U¤\ð °e´lð ð ð ð ð ð ð ð r@   r´  zl
    The Janus model which consists of a siglip vision backbone, a Llama language model and a VQ model.
    c                   óp  ‡ — e Zd Zdefˆ fd„Zeedej        de	e
         deez  fd„¦   «         ¦   «         Zdej        dej        d	ej        fd
„Zee	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dedz  dej        dz  dedz  deej        z  defd„¦   «         ¦   «         Zˆ xZS )Ú
JanusModelr&   c                 ó€  •— t          ¦   «                              |¦  «         || _        t                               |j        ¦  «        | _        t          | j        j        ¦  «        | _        t                               |j
        ¦  «        | _        t          j        | j        j        j        | j        j        j        ¦  «        | _        t#          | j        j        ¦  «        | _        t'          | j        j        ¦  «        | _        t+          j        |j        ¬¦  «        | _        d| _        |                      ¦   «          d S )N)r&   F)r1   rk   r&   r  Ú_from_configÚvision_configÚvision_modelr  Úalignerrœ  Ú	vq_configÚvqmodelr   ru   r)  rm   Úgeneration_embeddingsr¯  Úgeneration_alignerr´  Úgeneration_headr   Úfrom_configÚtext_configÚlanguage_modelr  r  rx   s     €r?   rk   zJanusModel.__init__Á  sñ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒå,×9Ò9¸&Ô:NÑOÔOˆÔÝ,¨TÔ->Ô-EÑFÔFˆŒå!×.Ò.¨vÔ/?Ñ@Ô@ˆŒõ &(¤\°$´,Ô2EÔ2TÐVZÔVbÔViÔVsÑ%tÔ%tˆÔ"Ý"6°t´|Ô7JÑ"KÔ"KˆÔÝ-¨d¬lÔ.AÑBÔBˆÔå'Ô3¸6Ô;MÐNÑNÔNˆÔà&+ˆÔ#à�ŠÑÔÐÐÐr@   r‘   r­   r|   c                 ód   —  | j         |fddi|¤Ž}|                      |j        ¦  «        |_        |S )Nr¬  T)r¿  rÀ  rY   r  )r<   r‘   r­   Úvision_outputss       r?   Úget_image_featureszJanusModel.get_image_featuresÖ  s@   € ð
 +˜Ô*¨<ÐTÐTÀTÐTÈVÐTÐTˆØ'+§|¢|°NÔ4TÑ'UÔ'UˆÔ$àÐr@   Ú	input_idsr  Úimage_featuresc           	      ó"  — |€f| |                       ¦   «         t          j        d| j        j        t          j        |j        ¬¦  «        ¦  «        k    }|                     d¦  «        }n|| j        j        k    }|                     ¦   «         }|j	        d         |j	        d         z  }| 
                    d¦  «                             |j        ¦  «        }t          ||j	        d         z  |                     ¦   «         k    d|› d|› �¦  «         |S )	zï
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        NrV   ©r“   Údevicer/   r   r    z6Image features and image tokens do not match, tokens: z, features: )r  r8   Úfullr&   Úimage_token_idÚlongrÐ  Úallr2  r:   r…   r”   r   Únumel)r<   rÌ  r  rÍ  Úspecial_image_maskÚn_image_tokensÚn_image_featuress          r?   Úget_placeholder_maskzJanusModel.get_placeholder_maskà  s  € ð ÐØ!.Ð2M°$×2KÒ2KÑ2MÔ2MÝ”
˜2˜tœ{Ô9ÅÄÐTaÔThÐiÑiÔiñ3ô 3ò "Ðð "4×!7Ò!7¸Ñ!;Ô!;ÐÐà!*¨d¬kÔ.HÒ!HÐà+×/Ò/Ñ1Ô1ˆØ)Ô/°Ô2°^Ô5IÈ!Ô5LÑLÐØ/×9Ò9¸"Ñ=Ô=×@Ò@ÀÔAUÑVÔVÐÝØ˜]Ô0°Ô4Ñ4¸×8LÒ8LÑ8NÔ8NÒNØsÀ^ÐsÐsÐaqÐsÐsñ	
ô 	
ð 	
ð "Ð!r@   Nr   rª   r7   r,   Ú	use_cacheÚlogits_to_keepc	           
      ó
  — |d u |d uz  rt          d¦  «        ‚|€ |                      ¦   «         |¦  «        }|�‹|                      |d¬¦  «        j        }
|
                     d|j        d         ¦  «        }|                     |j        |j        ¦  «        }|  	                    |||¬¦  «        }| 
                    ||¦  «        } | j        d||||||dœ|	¤Ž}t          |j        |j        |j        |j        |�|
nd ¬¦  «        S )	NzaYou cannot specify both input_ids and inputs_embeds at the same time, and must specify either oneT)r¬  r/   )r  rÍ  )r  rª   r7   r,   rÚ  rÛ  )rY   r,   rZ   r[   r\   rV   )rÃ   r  rË  r  r†   r:   r”   rÐ  r“   rÙ  Úmasked_scatterrÈ  rX   rY   r,   rZ   r[   )r<   rÌ  r‘   rª   r7   r,   r  rÚ  rÛ  r­   Úimage_embedsrÍ  Úimage_attention_maskÚ	lm_outputs                 r?   r›   zJanusModel.forwardø  s]  € ð ˜Ð -°tÐ";Ñ<ð 	ÝØsñô ð ð Ð Ø7˜D×5Ò5Ñ7Ô7¸	ÑBÔBˆMàÐ#Ø×2Ò2°<ÈTÐ2ÑRÔRÔ`ˆLØ)×1Ò1°"°mÔ6IÈ"Ô6MÑNÔNˆNØ+×.Ò.¨}Ô/CÀ]ÔEXÑYÔYˆNØ#'×#<Ò#<Ø¨À~ð $=ñ $ô $Ð ð *×8Ò8Ð9MÈ~Ñ^Ô^ˆMà'�DÔ'ð 
Ø'Ø)Ø%Ø+ØØ)ð
ð 
ð ð
ð 
ˆ	õ ,Ø'Ô9Ø%Ô5Ø#Ô1Ø Ô+Ø0<Ð0H  Èdð
ñ 
ô 
ð 	
r@   )NNNNNNNr   )rA   rB   rC   r!   rk   r   r   r8   rU   r   r   r]   r   rË  rA  rÙ  rœ   r	   rž   r�   rX   r›   rM   rN   s   @r?   r»  r»  »  s£  ø€ € € € € ð˜{ð ð ð ð ð ð ð* ØðØ!Ô-ðØ9?Ð@RÔ9Sðà	Ð+Ñ	+ðð ð ñ „^ñ Ôðð"ØÔ)ð"Ø:?Ô:Kð"Ø]bÔ]nð"ð "ð "ð "ð0 Øð .2Ø15Ø.2Ø04Ø(,Ø26Ø!%Ø-.ð,
ð ,
àÔ# dÑ*ð,
ð Ô'¨$Ñ.ð,
ð œ tÑ+ð	,
ð
 Ô&¨Ñ-ð,
ð  ™ð,
ð Ô(¨4Ñ/ð,
ð ˜$‘;ð,
ð ˜eœlÑ*ð,
ð 
&ð,
ð ,
ð ,
ñ „^ñ Ôð,
ð ,
ð ,
ð ,
ð ,
r@   r»  c                   óÎ  ‡ — e Zd ZddiZdZdZdefˆ fd„Zdej	        dej	        fd	„Z
ee	 	 	 	 	 	 	 	 	 ddej        d
z  dej        d
z  dej	        d
z  dej        d
z  ded
z  dej        d
z  dej        d
z  ded
z  deej	        z  dee         defd„¦   «         ¦   «         Zdej	        fd„Z ej        ¦   «         	 	 	 ddej	        d
z  dej        d
z  ded
z  fˆ fd„¦   «         Zˆ xZS )ÚJanusForConditionalGenerationzlm_head.weightz(model.language_model.embed_tokens.weightr(   Tr&   c                 ó  •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        t          j        |j        j        |j        j	        d¬¦  «        | _
        |                      ¦   «          d S )NFr¿   )r1   rk   r&   r»  r'   r   rÉ   rÇ  rl   Ú
vocab_sizeÚlm_headr  rx   s     €r?   rk   z&JanusForConditionalGeneration.__init__.  sn   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒÝ Ñ'Ô'ˆŒ
Ý”y Ô!3Ô!?ÀÔASÔA^ÐejÐkÑkÔkˆŒð 	�ŠÑÔÐÐÐr@   Úinputsr|   c                 ón   — | j                              |¦  «        }| j                              |¦  «        }|S rÞ   )r'   rÃ  rÄ  )r<   ræ  r.  s      r?   Ú'prepare_embeddings_for_image_generationzEJanusForConditionalGeneration.prepare_embeddings_for_image_generation7  s2   € Ø”z×7Ò7¸Ñ?Ô?ˆØ”z×4Ò4°\ÑBÔBˆØÐr@   Nr   rÌ  r‘   rª   r7   r,   r  ÚlabelsrÚ  rÛ  r­   c
                 óh  —  | j         d|||||||dœ|
¤Ž}|j        }t          |	t          ¦  «        rt	          |	 d¦  «        n|	}|                      |dd…|dd…f         ¦  «        }d}|�  | j        d||| j        j        j	        dœ|
¤Ž}t          |||j        |j        |j        |j        ¬¦  «        S )a­  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        )rÌ  r‘   rª   r7   r,   r  rÚ  N)ra   ré  rä  )r`   ra   r,   rZ   r[   r\   rV   )r'   rY   r3   r�   Úslicerå  Úloss_functionr&   rÇ  rä  r_   r,   rZ   r[   r\   )r<   rÌ  r‘   rª   r7   r,   r  ré  rÚ  rÛ  r­   ÚoutputsrZ   Úslice_indicesra   r`   s                   r?   r›   z%JanusForConditionalGeneration.forward<  s  € ð* �$”*ð 	
ØØ%Ø)Ø%Ø+Ø'Øð	
ð 	
ð ð	
ð 	
ˆð  Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%ð Ø f¸¼Ô9PÔ9[ðð Ø_eðð ˆDõ +ØØØ#Ô3Ø!Ô/ØÔ)Ø 'Ô ;ð
ñ 
ô 
ð 	
r@   r;  c                 ót   — | j         j                             |¦  «        }|                     dddd¦  «        }|S )a,  
        Decodes generated image tokens from language model to continuous pixel values
        with VQGAN module via upsampling.
        Args:
            image_tokens (`torch.LongTensor` of shape `(batch_size, num_of_tokens)`):
                The tensors corresponding to the input images.
        r   r   r   r    )r'   rÂ  rª  r‡   )r<   r;  Údecoded_images      r?   Údecode_image_tokensz1JanusForConditionalGeneration.decode_image_tokenso  s;   € ð œ
Ô*×1Ò1°,Ñ?Ô?ˆØ%×-Ò-¨a°°A°qÑ9Ô9ˆØÐr@   Úlogits_processorc           	      óâ
  •— |                      dd¦  «        } | j        |                      dd ¦  «        fi |¤Ž\  }}|dk    r t          ¦   «         j        d|||d dœ|¤ŽS |                     ¦   «         t
          j        t
          j        fvrt          d¦  «        ‚| 	                    ¦   «          |  
                    |                     ¦   «         ¦  «         |�|nt          ¦   «         }d|d<   |j        €!t                               d¦  «         d	|_        |j        |d
<   |                      ||j        |¦  «        \  }}	}|j        |j        }}
t)          |j        ¦  «        dk    rt          d|j        › d�¦  «        ‚|d u}|                      |||j        ¬¦  «         |j        r9|j        dk    r.|                     t1          |j        ¦  «        ¦  «         d |_        |                      ||j        d         |d ||¬¦  «        } | j        d|||j        dœ|¤Ž\  }}| j        j        j        j        }|j        \  }}|                      dd¦  «        }|                      dd ¦  «        }|                      dd¦  «        }||d<   ||d …d d …f         |j        k    ||d …d d …f         |j!        d         k    z  }||d …d d …f          "                    ||j#        ¦  «          |  $                    ¦   «         |¦  «        }| %                    dd ¦  «        €<|  &                    |j'        pd|dz  tQ          |j)        ||z   ¦  «        |¬¦  «        |d<   tU          j+        ||f|
|¬¦  «        }|j,        }|j-        }|j.        }|j/        }|j0        }|r|rdnd }|r|rdnd }|r|rdnd }|r|rdnd }tc          |¦  «        D �]Z} | j2        d||ddœ|¤Ž}d|v r#|d          3                    |j        ¦  «        |d<    | j        j4        di |¤||dœ¤Ž}|  5                    ||¦  «        }|j6        d d …dd d …f          7                    ¦   «         } | j         8                    | ¦  «        }! |||!¦  «        }"|j9        r@tU          j:        |"d¬¦  «        }#tU          j;        |#d¬¦  «         <                    d¦  «        }$ntU          j=        |"d¬¦  «        }$|$|d d …|f<   tU          j>        |$|$g¦  «        }$|$ ?                    d¦  «        }$|  @                    |$¦  «        }�Œ\|r:|r||!fz  }|r||  A                    ¦   «         fz  }|r
||jB        z  }|r
||jC        z  }|rt‰          ||!||||jE        ¬¦  «        S |S )NÚgeneration_moder*   Úgeneration_config)ræ  rª   rõ  Úguidance_scalez’Got incompatible mode for Image Generation, should be one of greedy or sampling. Ensure that beam search is de-activated by setting `num_beams=1`.TrÚ  zU`guidance_scale` is required for CFG but not provided. Setting to default value of 5.é   rö  r   z;Expected input ids of shape (batch_size, seq_len), but got z3Passing `inputs embeds` is not supported currently.)rÐ  r    )rõ  Úinput_ids_seq_lengthÚencoder_input_idsÚprefix_allowed_tokens_fnrò  rÐ  )rÌ  rª   Úexpand_sizerª   Úboi_token_idr,   Ústatic)Úcache_implementationrÙ   Úmax_cache_lenÚmodel_kwargsrÏ  rV   )r  rÌ  Úis_first_iteration)Úoutput_attentionsÚoutput_hidden_statesr/   r1  )Únum_samples)Ú	sequencesÚscoresra   r[   rZ   r,   )FÚpopÚ_prepare_generation_configr1   ÚgenerateÚget_generation_moder   ÚSAMPLEÚGREEDY_SEARCHrÃ   ÚvalidateÚ_validate_model_kwargsÚcopyr   rö  ÚloggerÚwarningÚ_prepare_model_inputsÚbos_token_idr“   rÐ  rz  r:   Ú_prepare_special_tokensr„  r
   Ú_get_logits_processorÚ_expand_inputs_for_generationÚnum_return_sequencesr'   r¿  r&   Únum_image_tokensÚrepeatÚgeneration_kwargsÚmasked_fill_Úpad_token_idr  ÚgetÚ_prepare_static_cacherþ  ÚmaxÚ
max_lengthr8   Úzerosr  r  Úoutput_scoresÚoutput_logitsÚreturn_dict_in_generater   Úprepare_inputs_for_generationr”   rÈ  Ú#_update_model_kwargs_for_generationrY   ÚclonerÅ  Ú	do_sampler³   ÚmultinomialÚsqueezeÚargmaxÚcatr…   rè  Úfloatr[   rZ   r   r,   )&r<   ræ  rª   rò  r­   rô  rõ  r   rÌ  Úmodel_input_namer“   rÐ  Úkwargs_has_attention_maskr  rÙ   rÚ   Úinput_tokensÚmaskr  Úgenerated_tokensr  r  r"  r#  r$  Ú
raw_scoresÚ
raw_logitsÚdecoder_hidden_statesÚdecoder_attentionsÚiÚmodel_inputsrí  r.  r  Únext_token_scoresÚprobsÚ
next_tokenr>   s&                                        €r?   r	  z&JanusForConditionalGeneration.generate{  s4  ø€ ð !Ÿ*š*Ð%6¸Ñ?Ô?ˆØ*I¨$Ô*IØ�JŠJÐ*¨DÑ1Ô1ð+
ð +
Ø5;ð+
ð +
Ñ'Ð˜<ð
 ˜fÒ$Ð$à#•5‘7”7Ô#ð ØØ-Ø"3Ø#ð	ð ð
 ðð ð ð ×0Ò0Ñ2Ô2½>Ô;PÕR`ÔRnÐ:oÐoÐoÝðTñô ð ð 	×"Ò"Ñ$Ô$Ð$Ø×#Ò# L×$5Ò$5Ñ$7Ô$7Ñ8Ô8Ð8ð 0@Ð/KÐ+Ð+ÕQdÑQfÔQfÐð %)ˆ�[Ñ!àÔ+Ð3Ý�NŠNÐrÑsÔsÐsØ/0ÐÔ,Ø):Ô)IˆÐ%Ñ&ð 59×4NÒ4NØÐ%Ô2°Lñ5
ô 5
Ñ1ˆ	Ð# \ð "œ¨Ô)9ˆvˆåˆyŒÑÔ 1Ò$Ð$ÝðFÈiÌoð Fð Fð Fñô ð ð %3¸$Ð$>Ð!Ø×$Ò$Ð%6Ð8QÐZcÔZjÐ$ÑkÔkÐkð Ô+ð 	4Ð0AÔ0PÐSTÒ0TÐ0TØ×#Ò#Õ$IÐJ[ÔJjÑ$kÔ$kÑlÔlÐlØ/3ÐÔ,ð  ×5Ò5Ø/Ø!*¤°Ô!3Ø'Ø%)Ø-Øð 6ñ 
ô 
Ðð #E $Ô"Dð #
ØØ)Ø)Ô>ð#
ð #
ð ð	#
ð #
Ñˆ	�<ð  œ:Ô2Ô9ÔJÐØ'œoÑˆ
�Gà ×'Ò'¨¨1Ñ-Ô-ˆØ%×)Ò)Ð*:¸DÑAÔAˆØ'×.Ò.¨q°!Ñ4Ô4ˆØ)7ˆÐ%Ñ&ð ˜Z˜[˜[¨!¨!¨!˜^Ô,Ð0AÔ0NÒNØ˜˜˜ a a a˜Ô(Ð,=Ô,OÐP^Ô,_Ò_ñ
ˆð 	�Z�[�[ ! ! !�^Ô$×1Ò1°$Ð8IÔ8VÑWÔWÐWà3˜×1Ò1Ñ3Ô3°LÑAÔAˆà×ÒÐ-¨tÑ4Ô4Ð<à.2×.HÒ.HØ%6Ô%KÐ%WÈxà%¨™>å!Ð"3Ô">Ð@PÐSZÑ@ZÑ[Ô[Ø)ð /Iñ /ô /ˆLÐ*Ñ+õ !œ;¨
Ð4DÐ'EÈUÐ[aÐbÑbÔbÐð .Ô?ÐØ0ÔEÐØ)Ô7ˆØ)Ô7ˆØ"3Ô"KÐà3ÐP¸ÐP�R�RÈDˆ
Ø3ÐP¸ÐP�R�RÈDˆ
Ø'>Ð bÐCWÐ b  Ð^bÐØ$;Ð\Ð@QÐ\˜R˜RÐX\ÐåÐ'Ñ(Ô(ð (	Uñ (	UˆAð >˜4Ô=ð Ø+°|ÐX\ðð Ø`lðð ˆLð   <Ð/Ð/Ø1=Ð>NÔ1O×1RÒ1RÐS`ÔSgÑ1hÔ1h�Ð-Ñ.à/�d”jÔ/ð ð Øðà"3Ø%9ðð ð ð ˆGð  ×CÒCÀGÈ\ÑZÔZˆLØ"Ô4°Q°Q°Q¸¸A¸A¸A°XÔ>×DÒDÑFÔFˆLð ”Z×/Ò/°Ñ=Ô=ˆFØ 0Ð 0°¸FÑ CÔ CÐð !Ô*ð EÝœÐ&7¸RÐ@Ñ@Ô@�Ý"Ô.¨uÀ!ÐDÑDÔD×LÒLÈRÑPÔP�
�
å"œ\Ð*;ÀÐDÑDÔD�
à%/Ð˜Q˜Q˜Q ˜TÑ"õ œ J°
Ð#;Ñ<Ô<ˆJØ#×-Ò-¨bÑ1Ô1ˆJà ×HÒHÈÑTÔTˆM‰Mà"ð 	?Øð (Ø˜v˜iÑ'�
Øð 6Ø˜|×1Ò1Ñ3Ô3Ð5Ñ5�
Ø ð 9Ø" gÔ&8Ñ8Ð"Ø#ð ?Ø%¨Ô)>Ñ>Ð%à"ð 
	$Ý,Ø*ØØ!Ø-Ø3Ø 'Ô 7ðñ ô ð ð $Ð#r@   )	NNNNNNNNr   )NNN)rA   rB   rC   Ú_tied_weights_keysÚoutput_modalitiesrL   r!   rk   r8   rœ   rè  r   r   rA  rU   r	   rž   r�   r   r   r_   r›   rñ  Úno_gradr   r	  rM   rN   s   @r?   râ  râ  )  s  ø€ € € € € Ø*Ð,VÐWÐØ)ÐØ!Ðð˜{ð ð ð ð ð ð ð¸e¼lð ÈuÌ|ð ð ð ð ð
 Øð .2Ø15Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ð/
ð /
àÔ# dÑ*ð/
ð Ô'¨$Ñ.ð/
ð œ tÑ+ð	/
ð
 Ô&¨Ñ-ð/
ð  ™ð/
ð Ô(¨4Ñ/ð/
ð Ô  4Ñ'ð/
ð ˜$‘;ð/
ð ˜eœlÑ*ð/
ð Ð+Ô,ð/
ð 
%ð/
ð /
ð /
ñ „^ñ Ôð/
ðb
°´ð 
ð 
ð 
ð 
ð €U„]�_„_ð '+Ø26Ø7;ð	$ð $à”˜tÑ#ð$ð Ô(¨4Ñ/ð$ð .°Ñ4ð	$ð $ð $ð $ð $ñ „_ð$ð $ð $ð $ð $r@   râ  )r%   râ  r»  rœ  r  )r¦   )VÚcollections.abcr   Údataclassesr   r8   Útorch.nn.functionalr   rˆ   r=  Ú r   r5   Úactivationsr   Úcache_utilsr	   Ú
generationr
   r   r   r   Úgeneration.utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   r   Úutils.genericr   Úutils.output_capturingr   Úautor   Úconfiguration_janusr!   r"   r#   Ú
get_loggerrA   r  r%   rQ   rX   r_   r…  r4   rœ   r�   r¥   r-  rº   r¼   rà   r+   rù   r  r  r&  rC  rW  rc  rj  rp  rx  r’  r™  rœ  r¯  r´  r»  râ  Ú__all__rV   r@   r?   ú<module>rR     s<  ðð* %Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø  Ð  Ð  Ð  Ð  Ð  Ø uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø XÐ XÐ XÐ XÐ XÐ XÐ XÐ XÐ XÐ XØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uØ 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø Ð Ð Ð Ð Ð Ø QÐ QÐ QÐ QÐ QÐ QÐ QÐ QÐ QÐ Qð 
ˆÔ	˜HÑ	%Ô	%€ð ðið ið ið ið i˜?ñ iô iñ „ðið$ €ððñ ô ð
 ð	4ð 	4ð 	4ð 	4ð 	4�{ñ 	4ô 	4ñ „ñô ð	4ð €ððñ ô ð
 ð@ð @ð @ð @ð @ ;ñ @ô @ñ „ñô ð@ð* €ððñ ô ð
 ð@ð @ð @ð @ð @ +ñ @ô @ñ „ñô ð@ð4Hð Hð Hð Hð H˜BœIñ Hô Hð HðV	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð& ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð2I$ð I$ð I$ð I$ð I$˜2œ9ñ I$ô I$ð I$ðXð ð ð ð �R”Yñ ô ð ð( ð  ð  ð  ð  Ð8ñ  ô  ð  ðF@ð @ð @ð @ð @˜œñ @ô @ð @ðD ð3ð 3ð 3ð 3ð 3Ð+ñ 3ô 3ñ „ð3ðlð ð ð ð ˜BœIñ ô ð ð$<"ð <"ð <"ð <"ð <" ¤	ñ <"ô <"ð <"ð~)(ð )(ð )(ð )(ð )(˜BœIñ )(ô )(ð )(ðX &ð  &ð  &ð  &ð  &˜"œ)ñ  &ô  &ð  &ðF	ð 	ð 	ð 	ð 	˜rœyñ 	ô 	ð 	ðð ð ð ð ˜RœYñ ô ð ðð ð ð ð ˜œñ ô ð ð,J!ð J!ð J!ð J!ð J!˜œ	ñ J!ô J!ð J!ðZAð Að Að Að A˜œ	ñ Aô Að AðH Ø
ð4ð 4ð 4ð 4ð 4Ð6ñ 4ô 4ñ „ñ „ð4ð €ððñ ô ðDUð DUð DUð DUð DUÐ%ñ DUô DUñô ðDUðNð ð ð ð ˜2œ9ñ ô ð ð$ð ð ð ð �R”Yñ ô ð ð  €ððñ ô ð
f
ð f
ð f
ð f
ð f
Ð%ñ f
ô f
ñô ð
f
ðRR$ð R$ð R$ð R$ð R$Ð$8¸/ñ R$ô R$ð R$ðj tÐ
sÐ
s€€€r@   