§
    9²¸j’  ã                   óh  — d dl mZ d dlZd dlmZ ddlmZ ddlmZ ddl	m
Z
mZmZ ddlmZmZ dd	lmZmZ dd
lmZ ddlmZ ddlmZmZmZmZ ddlmZmZ ddlm Z m!Z! ddl"m#Z# ddl$m%Z%m&Z&m'Z'm(Z(m)Z) ddl*m+Z+ ddl,m-Z-m.Z. ddl/m0Z0 ddl1m2Z2m3Z3m4Z4 ddl5m6Z6  e)j7        e8¦  «        Z9e& G d„ de!¦  «        ¦   «         Z: G d„ dej;        ¦  «        Z< G d„ dej;        ¦  «        Z= ed¦  «         G d„ dej;        ¦  «        ¦   «         Z> G d „ d!ej;        ¦  «        Z?d"„ Z@ ed#¦  «        dGd$„¦   «         ZAd%ejB        d&eCd'ejB        fd(„ZD	 dHd*ej;        d+ejB        d,ejB        d-ejB        d.ejB        dz  d/eEd0eEd1e#e%         fd2„ZF eeA¦  «         G d3„ d4ej;        ¦  «        ¦   «         ZG G d5„ d6ej;        ¦  «        ZH G d7„ d8e¦  «        ZI G d9„ d:e:¦  «        ZJ G d;„ d<e¦  «        ZK G d=„ d>e:¦  «        ZL e&d?¬@¦  «         G dA„ dBe:¦  «        ¦   «         ZM e&dC¬@¦  «         G dD„ dEe:e6¦  «        ¦   «         ZNg dF¢ZOdS )Ié    )ÚCallableN)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚis_torchdynamo_compilingÚlogging)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )Ú	DiaConfigÚDiaDecoderConfigÚDiaEncoderConfig)ÚDiaGenerationMixinc                   óN   ‡ — e Zd ZU eed<   dZdZdZdZdZ	dZ
dZddgZˆ fd„Zˆ xZS )ÚDiaPreTrainedModelÚconfigÚmodelTÚ	input_idsÚDiaEncoderLayerÚDiaDecoderLayerc                 ó  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rSt	          j        | j        j        t          j        ¬¦  «        | j        j	        z  }t          j        |j        |¦  «         d S d S )N©Údtype)ÚsuperÚ_init_weightsÚ
isinstanceÚDiaMultiChannelEmbeddingÚtorchÚaranger*   Únum_channelsÚlongÚ
vocab_sizeÚinitÚcopy_Úoffsets)ÚselfÚmoduler=   Ú	__class__s      €út/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/dia/modeling_dia.pyr3   z DiaPreTrainedModel._init_weightsA   sy   ø€ Ý‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ6Ñ7Ô7ð 	0Ý”l 4¤;Ô#;Å5Ä:ÐNÑNÔNÐQUÔQ\ÔQgÑgˆGÝŒJ�v”~ wÑ/Ô/Ð/Ð/Ð/ð	0ð 	0ó    )Ú__name__Ú
__module__Ú__qualname__r$   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚmain_input_nameÚ_no_split_modulesr3   Ú__classcell__©r@   s   @rA   r)   r)   5   sz   ø€ € € € € € àÐÐÑØÐØ&*Ð#ØÐØ€NØÐØ!ÐØ!€OØ*Ð,=Ð>Ðð0ð 0ð 0ð 0ð 0ð 0ð 0ð 0ð 0rB   r)   c                   óL   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        fd„Zˆ xZ	S )r5   a  In order to efficiently compute the audio embedding from the 9 different channels,
    we vectorize the embedding process by using a single embedding layer and an offset.
    Example:
    - num_embeds = 4
    - vocab_size = 8
    - num_channels = 3
    We would have offsets = [0, 8, 16]
    If audio_codes = [0, 1, 2, 3], [1, 3, 4, 7], [5, 6, 7, 8],
    then tokens = audio_codes + offsets
                = [0, 1, 2, 3, 9, 11, 12, 15, 21, 22, 23, 24]
    This allows us to use a single embedding layer for all channels.
    r*   c                 ó`  •— t          ¦   «                              ¦   «          t          j        |j        |j        z  |j        ¦  «        | _        |j        | _        |j        | _        t          j	        |j        t          j
        ¬¦  «        |j        z  }t          j        |d¬¦  «        | _        d S )Nr0   F©Ú
persistent)r2   Ú__init__r   Ú	Embeddingr:   r8   Úhidden_sizeÚembedr6   r7   r9   ÚBufferr=   )r>   r*   r=   r@   s      €rA   rU   z!DiaMultiChannelEmbedding.__init__V   s‹   ø€ Ý‰Œ×ÒÑÔÐÝ”\ &Ô"3°fÔ6IÑ"IÈ6ÔK]Ñ^Ô^ˆŒ
Ø!Ô-ˆÔØ"Ô/ˆÔÝ”,˜vÔ2½%¼*ÐEÑEÔEÈÔHYÑYˆÝ”y °UÐ;Ñ;Ô;ˆŒˆˆrB   Úaudio_codesÚreturnc                 óX  — || j                              |j        ¦  «        z                        d|j        d         |j        d         z  ¦  «        }|                      |¦  «                             |j        d         |j        d         d| j        ¦  «        }|                     d¬¦  «        S )Néÿÿÿÿr#   é   r   ©Údim)r=   ÚtoÚdeviceÚviewÚshaperX   rW   Úsum)r>   rZ   ÚtokensÚembedss       rA   Úforwardz DiaMultiChannelEmbedding.forward^   s•   € Ø ¤§¢°Ô0BÑ CÔ CÑC×IÒIØ�Ô! !Ô$ {Ô'8¸Ô';Ñ;ñ
ô 
ˆð —’˜FÑ#Ô#×(Ò(¨¬°a¬¸+Ô:KÈAÔ:NÐPRÐTXÔTdÑeÔeˆØ�zŠz˜aˆzÑ Ô Ð rB   )
rC   rD   rE   Ú__doc__r%   rU   r6   ÚTensorrh   rO   rP   s   @rA   r5   r5   H   su   ø€ € € € € ðð ð<Ð/ð <ð <ð <ð <ð <ð <ð! 5¤<ð !°E´Lð !ð !ð !ð !ð !ð !ð !ð !rB   r5   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚDiaMLPc                 ó"  •— t          ¦   «                              ¦   «          || _        t          j        |j        d|j        z  d¬¦  «        | _        t          j        |j        |j        d¬¦  «        | _        t          |j
                 | _        d S )Nr^   F©Úbias)r2   rU   r*   r   ÚLinearrW   Úintermediate_sizeÚgate_up_projÚ	down_projr   Ú
hidden_actÚactivation_fn©r>   r*   r@   s     €rA   rU   zDiaMLP.__init__g   sz   ø€ Ý‰Œ×ÒÑÔÐàˆŒÝœI fÔ&8¸!¸fÔ>VÑ:VÐ]bÐcÑcÔcˆÔÝœ 6Ô#;¸VÔ=OÐV[Ð\Ñ\Ô\ˆŒÝ# FÔ$5Ô6ˆÔÐÐrB   Úhidden_statesr[   c                 óº   — |                       |¦  «        }|                     dd¬¦  «        \  }}||                      |¦  «        z  }|                      |¦  «        S )Nr^   r]   r_   )rr   Úchunkru   rs   )r>   rw   Ú	up_statesÚgates       rA   rh   zDiaMLP.forwardo   sX   € Ø×%Ò% mÑ4Ô4ˆ	à#Ÿ/š/¨!°˜/Ñ4Ô4‰ˆˆiØ × 2Ò 2°4Ñ 8Ô 8Ñ8ˆ	à�~Š~˜iÑ(Ô(Ð(rB   )rC   rD   rE   rU   r6   ÚFloatTensorrh   rO   rP   s   @rA   rl   rl   f   s`   ø€ € € € € ð7ð 7ð 7ð 7ð 7ð) UÔ%6ð )¸5Ô;Lð )ð )ð )ð )ð )ð )ð )ð )rB   rl   ÚRMSNormc                   óT   ‡ — e Zd Zd	deddfˆ fd„Zdej        dej        fd„Zd„ Zˆ xZ	S )
Ú
DiaRMSNormç�íµ ÷Æ°>Úepsr[   Nc                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )z9
        DiaRMSNorm is equivalent to T5LayerNorm
        N)r2   rU   r   Ú	Parameterr6   ÚonesÚweightÚvariance_epsilon)r>   rW   r�   r@   s      €rA   rU   zDiaRMSNorm.__init__z   sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐrB   rw   c                 ó  — |j         }|                     t          j        ¦  «        }|                     d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        |                     |¦  «        z  S )Nr^   r]   T)Úkeepdim)	r1   ra   r6   Úfloat32ÚpowÚmeanÚrsqrtr†   r…   )r>   rw   Úinput_dtypeÚvariances       rA   rh   zDiaRMSNorm.forward‚   s|   € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØ ×$Ò$ QÑ'Ô'×,Ò,¨R¸Ð,Ñ>Ô>ˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆØŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:rB   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)Útupler…   rd   r†   )r>   s    rA   Ú
extra_reprzDiaRMSNorm.extra_repr‰   s&   € Ý˜œÔ)Ñ*Ô*ÐIÐI°$Ô2GÐIÐIÐIrB   )r€   )
rC   rD   rE   ÚfloatrU   r6   rj   rh   r‘   rO   rP   s   @rA   r   r   x   sŒ   ø€ € € € € ð$ð $¨ð $¸$ð $ð $ð $ð $ð $ð $ð; U¤\ð ;°e´lð ;ð ;ð ;ð ;ðJð Jð Jð Jð Jð Jð JrB   r   c            	       óî   ‡ — e Zd Z edd¬¦  «        d
defˆ fd„¦   «         Ze edd¬¦  «        d
dedeej	        e
f         fd„¦   «         ¦   «         Z ej        ¦   «         ed	„ ¦   «         ¦   «         Zˆ xZS )ÚDiaRotaryEmbeddingrb   z5.18)ÚversionNr*   c                 ó¾  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        | j        j        d         | _        | j        }| j        dk    rt          | j                 } || j        |¦  «        \  }| _
        t          j        |d¬¦  «        | _        t          j        |                     ¦   «         d¬¦  «        | _        d S )NÚ	rope_typeÚdefaultFrS   )r2   rU   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr*   Úrope_parametersr—   Úcompute_default_rope_parametersr   Úattention_scalingr   rY   Úinv_freqÚcloneÚoriginal_inv_freq)r>   r*   rb   Úrope_init_fnrŸ   r@   s        €rA   rU   zDiaRotaryEmbedding.__init__Ž   sÀ   ø€ å‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!àˆŒàœÔ4°[ÔAˆŒØ!%Ô!EˆØŒ>˜YÒ&Ð&Ý.¨t¬~Ô>ˆLØ+7¨<¸¼ÀVÑ+LÔ+LÑ(ˆ�$Ô(åœ	 (°uÐ=Ñ=Ô=ˆŒÝ!#¤¨8¯>ª>Ñ+;Ô+;ÈÐ!NÑ!NÔ!NˆÔÐÐrB   r[   c                 óä   — | j         d         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «        |z  z  z  }|                     |¦  «        |fS )a¾  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNg      ð?r   r^   r0   )rœ   ÚgetattrrW   Únum_attention_headsr6   r7   r’   ra   )r*   rb   ÚkwargsÚbaser`   Úattention_factorrŸ   s          rA   r�   z2DiaRotaryEmbedding.compute_default_rope_parametersŸ   sz   € ð Ô% lÔ3ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐà˜$¥5¤<°°3¸ÅÄÐ#MÑ#MÔ#MÐPSÑ#SÑTÑUˆØ�{Š{˜6Ñ"Ô"Ð$4Ð4Ð4rB   c                 óú  — | j         d d d …d f                              |j        d         dd¦  «                             t          j        |j        ¬¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  ||z                       dd	¦  «        }t	          j        ||fd¬
¦  «        }|                     ¦   «         | j        z  }|                     ¦   «         | j        z  }	d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬¦  «        |	                     |j        ¬¦  «        fS )Nr   r]   r#   )r1   rb   ÚmpsÚcpuF)Údevice_typeÚenabledr^   r_   r0   )rŸ   Úexpandrd   ra   r6   r’   rb   r4   ÚtypeÚstrr    Ú	transposeÚcatÚcosrž   Úsinr1   )
r>   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedr®   ÚfreqsÚembrµ   r¶   s
             rA   rh   zDiaRotaryEmbedding.forward³   s›  € ð ŒM˜$    4˜-Ô(×/Ò/°Ô0BÀ1Ô0EÀrÈ1ÑMÔM×PÒPÕW\ÔWbÐklÔksÐPÑtÔtð 	ð !-¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆå¨¸UÐCÑCÔCð 	5ð 	5Ø&Ð)>Ñ>×IÒIÈ!ÈQÑOÔOˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCð		5ð 	5ð 	5ñ 	5ô 	5ð 	5ð 	5ð 	5ð 	5ð 	5ð 	5øøøð 	5ð 	5ð 	5ð 	5ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   ÃA*D<Ä<E ÅE ©N)rC   rD   rE   r   r$   rU   Ústaticmethodr�   r6   rj   r’   r�   Úno_gradr   rh   rO   rP   s   @rA   r”   r”   �   sö   ø€ € € € € Ø€_�X vÐ.Ñ.Ô.ðOð O˜yð Oð Oð Oð Oð Oñ /Ô.ðOð  Ø€_�X vÐ.Ñ.Ô.ð5ð 5°	ð 5ÐUZÐ[`Ô[gÐinÐ[nÔUoð 5ð 5ð 5ñ /Ô.ñ „\ð5ð$ €U„]�_„_Øð<ð <ñ Ôñ „_ð<ð <ð <ð <ð <rB   r”   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..Nr]   r^   r_   )rd   r6   r´   )r·   Úx1Úx2s      rA   Úrotate_halfrÃ   Æ   s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'rB   Úrotary_pos_embc                 ó¾   — |                      |¦  «        }|                      |¦  «        }| |z  t          | ¦  «        |z  z   }||z  t          |¦  «        |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezerÃ   )ÚqÚkrµ   r¶   Úunsqueeze_dimÚq_embedÚk_embeds          rA   Úapply_rotary_pos_embrÌ   Í   sc   € ð& �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�GÐÐrB   rw   Ún_repr[   c                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r#   N)rd   r°   Úreshape)rw   rÍ   ÚbatchÚnum_key_value_headsÚslenr¥   s         rA   Ú	repeat_kvrÓ   ç   s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTrB   ç        r?   ÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutr¨   c                 ó  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        |z  }
|�|
|z   }
t
          j                             |
dt          j        ¬¦  «         	                    |j
        ¦  «        }
t
          j                             |
|| j        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «                             ¦   «         }||
fS )Nr^   r   r]   )r`   r1   )ÚpÚtrainingr#   )rÓ   Únum_key_value_groupsr6   Úmatmulr³   r   Ú
functionalÚsoftmaxr‰   ra   r1   rÚ   rÝ   Ú
contiguous)r?   rÕ   rÖ   r×   rØ   rÙ   rÚ   r¨   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               rA   Úeager_attention_forwardrç   ó   sé   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$rB   c                   óÚ   ‡ — e Zd ZdZddeez  dedefˆ fd„Z	 	 	 dde	j
        d	ee	j
        e	j
        f         dz  d
e	j
        dz  dedz  dee         dee	j
        e	j
        f         fd„Zˆ xZS )ÚDiaSelfAttentionú=Multi-headed attention from 'Attention Is All You Need' paperFr*   Ú	layer_idxÚ	is_causalc                 óÖ  •— t          ¦   «                              ¦   «          || _        || _        |j        | _        | j        j        | _        | j        j        p| j        | _        | j        | j        z  | _        t          |d|j        | j        z  ¦  «        | _
        d| _        d| _        || _        t          j        | j        | j        | j
        z  d¬¦  «        | _        t          j        | j        | j        | j
        z  d¬¦  «        | _        t          j        | j        | j        | j
        z  d¬¦  «        | _        t          j        | j        | j
        z  | j        d¬¦  «        | _        d S )Nr¥   r#   rÔ   Frn   )r2   rU   r*   rë   rW   r§   Ú	num_headsrÑ   rÞ   r¦   r¥   rÙ   Úattention_dropoutrì   r   rp   Úq_projÚk_projÚv_projÚo_proj)r>   r*   rë   rì   r@   s       €rA   rU   zDiaSelfAttention.__init__  s1  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒØ!Ô-ˆÔØœÔ8ˆŒØ#'¤;Ô#BÐ#TÀdÄnˆÔ Ø$(¤N°dÔ6NÑ$NˆÔ!Ý ¨
°FÔ4FÈ$Ì.Ñ4XÑYÔYˆŒØˆŒØ!$ˆÔØ"ˆŒå”i Ô 0°$´.À4Ä=Ñ2PÐW\Ð]Ñ]Ô]ˆŒÝ”i Ô 0°$Ô2JÈTÌ]Ñ2ZÐafÐgÑgÔgˆŒÝ”i Ô 0°$Ô2JÈTÌ]Ñ2ZÐafÐgÑgÔgˆŒÝ”i ¤°´Ñ >ÀÔ@PÐW\Ð]Ñ]Ô]ˆŒˆˆrB   Nrw   Úposition_embeddingsrØ   Úpast_key_valuesr¨   r[   c                 ó"  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }	|                      |¦  «                             |¦  «                             dd¦  «        }
|\  }}t          ||	||¦  «        \  }}	|�|                     |	|
| j	        ¦  «        \  }	}
t          j        | j        j        t          ¦  «        } || ||	|
|f| j        sdn| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Nr]   r#   r^   rÔ   )rÚ   rÙ   )rd   r¥   rð   rc   r³   rñ   rò   rÌ   Úupdaterë   r   Úget_interfacer*   Ú_attn_implementationrç   rÝ   rï   rÙ   rÏ   râ   ró   )r>   rw   rô   rØ   rõ   r¨   Úinput_shapeÚhidden_shapeÚquery_statesrã   rä   rµ   r¶   Úattention_interfaceræ   rå   s                   rA   rh   zDiaSelfAttention.forward"  sÀ  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(rB   )F)NNN)rC   rD   rE   ri   r&   r%   ÚintÚboolrU   r6   rj   r�   r   r   r   rh   rO   rP   s   @rA   ré   ré     s  ø€ € € € € àGÐGð^ð ^Ð/Ð2BÑBð ^Èsð ^Ð_cð ^ð ^ð ^ð ^ð ^ð ^ð* IMØ.2Ø(,ð&)ð &)à”|ð&)ð # 5¤<°´Ð#=Ô>ÀÑEð&)ð œ tÑ+ð	&)ð
  ™ð&)ð Ð+Ô,ð&)ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð&)ð &)ð &)ð &)ð &)ð &)ð &)ð &)rB   ré   c                   ó²   ‡ — e Zd ZdZdedefˆ fd„Z	 	 ddej        dej        dej        dz  d	e	dz  d
e
e         deej        ej        dz  f         fd„Zˆ xZS )ÚDiaCrossAttentionrê   r*   rë   c                 ó²  •— t          ¦   «                              ¦   «          || _        || _        |j        | _        |j        | _        | j        j        | _        | j        j        | _	        | j        | j	        z  | _
        |j        | _        d| _        d| _        d| _        t!          j        | j        | j        | j        z  d¬¦  «        | _        t!          j        | j        | j	        | j        z  d¬¦  «        | _        t!          j        | j        | j	        | j        z  d¬¦  «        | _        t!          j        | j        | j        z  | j        d¬¦  «        | _        d S )Nr#   rÔ   Frn   )r2   rU   r*   rë   rW   Úcross_hidden_sizeÚcross_num_attention_headsrî   Úcross_num_key_value_headsrÑ   rÞ   Úcross_head_dimr¥   rÙ   rï   rì   r   rp   rð   rñ   rò   ró   ©r>   r*   rë   r@   s      €rA   rU   zDiaCrossAttention.__init__N  s&  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒØ!Ô-ˆÔØ!'Ô!9ˆÔØœÔ>ˆŒØ#'¤;Ô#HˆÔ Ø$(¤N°dÔ6NÑ$NˆÔ!ØÔ-ˆŒØˆŒØ!$ˆÔØˆŒå”i Ô 0°$´.À4Ä=Ñ2PÐW\Ð]Ñ]Ô]ˆŒÝ”i Ô 6¸Ô8PÐSWÔS`Ñ8`ÐglÐmÑmÔmˆŒÝ”i Ô 6¸Ô8PÐSWÔS`Ñ8`ÐglÐmÑmÔmˆŒÝ”i ¤°´Ñ >ÀÔ@PÐW\Ð]Ñ]Ô]ˆŒˆˆrB   Nrw   Úcross_attention_statesrØ   rõ   r¨   r[   c                 ó  — |j         d d…         }g |¢d‘| j        ‘R }g |j         d d…         ¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }	|�|j                             | j        ¦  «        nd}
|�;|
r9|j        j	        | j                 j
        }|j        j	        | j                 j        }n­|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }|�3|j                             ||| j        ¦  «        \  }}d|j        | j        <   t          j        | j        j        t&          ¦  «        } || |	|||fd| j        i|¤Ž\  }}|                     g |¢d‘R ¦  «                             ¦   «         }|                      |¦  «        }||fS )Nr]   r#   r^   FTrÙ   )rd   r¥   rð   rc   r³   Ú
is_updatedÚgetrë   Úcross_attention_cacheÚlayersÚkeysÚvaluesrñ   rò   r÷   r   rø   r*   rù   rç   rÙ   rÏ   râ   ró   )r>   rw   r  rØ   rõ   r¨   rú   rû   Úcross_shaperü   r
  rã   rä   rý   ræ   rå   s                   rA   rh   zDiaCrossAttention.forwarda  s.  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆØMÐ.Ô4°S°b°SÔ9ÐM¸2ÐM¸t¼}ÐMÐMˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆàGVÐGb�_Ô/×3Ò3°D´NÑCÔCÐCÐhmˆ
ØÐ&¨:Ð&à(Ô>ÔEÀdÄnÔUÔZˆJØ*Ô@ÔGÈÌÔWÔ^ˆLˆLàŸšÐ%;Ñ<Ô<×AÒAÀ+ÑNÔN×XÒXÐYZÐ\]Ñ^Ô^ˆJØŸ;š;Ð'=Ñ>Ô>×CÒCÀKÑPÔP×ZÒZÐ[\Ð^_Ñ`Ô`ˆLàÐ*à+:Ô+P×+WÒ+WØØ Ø”Nñ,ô ,Ñ(�
˜Lð >B�Ô*¨4¬>Ñ:å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð%
ð %
ð ”Lð%
ð ð%
ð %
Ñ!ˆ�\ð "×)Ò)Ð*<¨KÐ*<¸Ð*<Ð*<Ñ=Ô=×HÒHÑJÔJˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(rB   ©NN)rC   rD   rE   ri   r%   rþ   rU   r6   rj   r
   r   r   r�   rh   rO   rP   s   @rA   r  r  K  sÞ   ø€ € € € € ØGÐGð^Ð/ð ^¸Cð ^ð ^ð ^ð ^ð ^ð ^ð. /3Ø6:ð1)ð 1)à”|ð1)ð !&¤ð1)ð œ tÑ+ð	1)ð
 -¨tÑ3ð1)ð Ð-Ô.ð1)ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð1)ð 1)ð 1)ð 1)ð 1)ð 1)ð 1)ð 1)rB   r  c                   ó¤   ‡ — e Zd Zdedefˆ fd„Z	 	 ddej        deej        ej        f         dz  dej        dz  de	e
         d	ej        f
d
„Zˆ xZS )r-   r*   rë   c                 ó  •— t          ¦   «                              ¦   «          t          |j        |j        ¬¦  «        | _        t          ||d¬¦  «        | _        t          |j        |j        ¬¦  «        | _        t          |¦  «        | _
        d S )N©r�   F©rì   )r2   rU   r   rW   Únorm_epsÚpre_sa_normré   Úself_attentionÚpost_sa_normrl   Úmlpr  s      €rA   rU   zDiaEncoderLayer.__init__–  su   ø€ Ý‰Œ×ÒÑÔÐÝ% fÔ&8¸f¼oÐNÑNÔNˆÔÝ.¨v°yÈEÐRÑRÔRˆÔÝ& vÔ'9¸v¼ÐOÑOÔOˆÔÝ˜&‘>”>ˆŒˆˆrB   Nrw   rô   rØ   r¨   r[   c                 óÈ   — |}|                       |¦  «        } | j        |f||dœ|¤Ž\  }}||z   }|}|                      |¦  «        }|                      |¦  «        }	||	z   }|S )N)rô   rØ   )r  r  r  r  )
r>   rw   rô   rØ   r¨   ÚresidualÚnormed_statesÚself_attn_outputÚ_Úmlp_outs
             rA   rh   zDiaEncoderLayer.forward�  s™   € ð !ˆØ×(Ò(¨Ñ7Ô7ˆØ1˜dÔ1Øð
à 3Ø)ð
ð 
ð ð	
ð 
ÑÐ˜!ð !Ð#3Ñ3ˆà ˆØ×)Ò)¨-Ñ8Ô8ˆØ—(’(˜=Ñ)Ô)ˆØ  7Ñ*ˆàÐrB   r  )rC   rD   rE   r&   rþ   rU   r6   rj   r�   r   r   rh   rO   rP   s   @rA   r-   r-   •  sÃ   ø€ € € € € ð"Ð/ð "¸Cð "ð "ð "ð "ð "ð "ð IMØ.2ð	ð à”|ðð # 5¤<°´Ð#=Ô>ÀÑEðð œ tÑ+ð	ð
 Ð-Ô.ðð 
Œðð ð ð ð ð ð ð rB   r-   c                   ó    ‡ — e Zd ZeedœZdefˆ fd„Zee	e
	 d
dej        dej        dz  dee         defd	„¦   «         ¦   «         ¦   «         Zˆ xZS )Ú
DiaEncoder)rw   Ú
attentionsr*   c                 ó¢  •‡— t          ¦   «                              ‰¦  «         ‰| _        t          j        ‰j        ‰j        ¦  «        | _        t          j        ˆfd„t          ‰j
        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        t          ‰¬¦  «        | _        |                      ¦   «          d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS © )r-   ©Ú.0rë   r*   s     €rA   ú
<listcomp>z'DiaEncoder.__init__.<locals>.<listcomp>Â  ó#   ø€ ÐaÐaÐa°I�_˜V YÑ/Ô/ÐaÐaÐarB   r  ©r*   )r2   rU   r*   r   rV   r:   rW   Ú	embeddingÚ
ModuleListÚrangeÚnum_hidden_layersr  r   r  Únormr”   Ú
rotary_embÚ	post_initrv   s    `€rA   rU   zDiaEncoder.__init__¼  s´   øø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒåœ fÔ&7¸Ô9KÑLÔLˆŒÝ”mØaÐaÐaÐaÅÀvÔG_ÑA`ÔA`ÐaÑaÔañ
ô 
ˆŒõ ˜vÔ1°v´ÐGÑGÔGˆŒ	Ý,°FÐ;Ñ;Ô;ˆŒà�ŠÑÔÐÐÐrB   Nr,   rØ   r¨   r[   c                 ó`  — |                       |¦  «        }t          j        |j        d         |j        ¬¦  «        d d d …f         }t          | j        ||¬¦  «        }|                      ||¬¦  «        }| j        D ]} ||f|||dœ|¤Ž}Œ|  	                    |¦  «        }t          |¬¦  «        S )Nr]   ©rb   )r*   Úinputs_embedsrØ   ©r¸   )rØ   r¸   rô   )Úlast_hidden_state)r,  r6   r7   rd   rb   r   r*   r1  r  r0  r   )r>   r,   rØ   r¨   rw   r¸   rô   Úencoder_layers           rA   rh   zDiaEncoder.forwardÉ  sæ   € ð Ÿš yÑ1Ô1ˆõ
 ”| I¤O°BÔ$7À	Ô@PÐQÑQÔQÐRVÐXYÐXYÐXYÐRYÔZˆå2Ø”;Ø'Ø)ð
ñ 
ô 
ˆð
 #Ÿošo¨mÈ,˜oÑWÔWÐà!œ[ð 	ð 	ˆMØ)˜MØðà-Ø)Ø$7ð	ð ð
 ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆå°Ð?Ñ?Ô?Ð?rB   r½   )rC   rD   rE   r-   ré   Ú_can_record_outputsr&   rU   r!   r"   r   r6   rj   r   r   r   rh   rO   rP   s   @rA   r"  r"  ¶  sÙ   ø€ € € € € à(Ø&ðð Ðð
Ð/ð ð ð ð ð ð ð  ØØð /3ð@ð @à”<ð@ð œ tÑ+ð@ð Ð+Ô,ð	@ð
 
ð@ð @ð @ñ „^ñ „_ñ  Ôð@ð @ð @ð @ð @rB   r"  c                   óþ   ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 ddej        deej        ej        f         dz  dej        dz  dej        dz  d	ej        dz  d
e	dz  deej        ej        dz  ej        dz  f         fd„Z
ˆ xZS )r.   r*   rë   c                 ó   •— t          ¦   «                              ¦   «          |j        | _        t	          ||d¬¦  «        | _        t          ||¦  «        | _        t          |j        |j	        ¬¦  «        | _
        t          |j        |j	        ¬¦  «        | _        t          |j        |j	        ¬¦  «        | _        t          |¦  «        | _        d S )NTr  r  )r2   rU   rW   Ú	embed_dimré   r  r  Úcross_attentionr   r  r  Úpre_ca_normÚpre_mlp_normrl   r  r  s      €rA   rU   zDiaDecoderLayer.__init__ï  s«   ø€ Ý‰Œ×ÒÑÔÐØÔ+ˆŒÝ.¨v°yÈDÐQÑQÔQˆÔÝ0°¸ÑCÔCˆÔÝ% fÔ&8¸f¼oÐNÑNÔNˆÔÝ% fÔ&8¸f¼oÐNÑNÔNˆÔÝ& vÔ'9¸v¼ÐOÑOÔOˆÔÝ˜&‘>”>ˆŒˆˆrB   Nrw   rô   rØ   Úencoder_hidden_statesÚencoder_attention_maskrõ   r[   c                 óf  — |}t          |t          ¦  «        r|j        }|}	|                      |¦  «        }
 | j        |
|||fi |¤Ž\  }}|	|z   }|}	|                      |¦  «        }
 | j        |
|f||dœ|¤Ž\  }}|	|z   }|}	|                      |¦  «        }
|                      |
¦  «        }|	|z   }|S )N)rØ   rõ   )	r4   r
   Úself_attention_cacher  r  r>  r=  r?  r  )r>   rw   rô   rØ   r@  rA  rõ   r¨   Úself_attn_cacher  r  r  r  Úcross_statesr   s                  rA   rh   zDiaDecoderLayer.forwardù  s  € ð *ˆÝ�oÕ':Ñ;Ô;ð 	CØ-ÔBˆOà ˆØ×(Ò(¨Ñ7Ô7ˆØ1˜dÔ1ØØØð ð
ð 
ð ð
ð 
ÑÐ˜!ð !Ð#3Ñ3ˆà ˆØ×(Ò(¨Ñ7Ô7ˆØ.˜$Ô.ØØ!ð
ð 2Ø+ð	
ð 
ð
 ð
ð 
‰ˆ�að ! <Ñ/ˆà ˆØ×)Ò)¨-Ñ8Ô8ˆØ—(’(˜=Ñ)Ô)ˆØ  7Ñ*ˆàÐrB   ©NNNNN)rC   rD   rE   r%   rþ   rU   r6   rj   r�   r
   rh   rO   rP   s   @rA   r.   r.   î  s  ø€ € € € € ð"Ð/ð "¸Cð "ð "ð "ð "ð "ð "ð IMØ.2Ø59Ø6:Ø6:ð+ð +à”|ð+ð # 5¤<°´Ð#=Ô>ÀÑEð+ð œ tÑ+ð	+ð
  %œ|¨dÑ2ð+ð !&¤¨tÑ 3ð+ð -¨tÑ3ð+ð 
ˆuŒ|˜Uœ\¨DÑ0°%´,ÀÑ2EÐEÔ	Fð+ð +ð +ð +ð +ð +ð +ð +rB   r.   c                   óú   ‡ — e Zd ZdZeeedœZdefˆ fd„Z	e
ee	 	 	 	 	 ddej        dej        dz  dej        dz  d	ej        dz  d
ej        dz  dedz  dee         deez  fd„¦   «         ¦   «         ¦   «         Zˆ xZS )Ú
DiaDecoderz-Transformer Decoder Stack using DenseGeneral.)rw   r#  Úcross_attentionsr*   c                 ó¤  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t	          ‰¦  «        | _        t          j        ˆfd„t          ‰j	        ¦  «        D ¦   «         ¦  «        | _
        t          ‰j        ‰j        ¬¦  «        | _        t          ‰¬¦  «        | _        |                      ¦   «          d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r&  )r.   r'  s     €rA   r)  z'DiaDecoder.__init__.<locals>.<listcomp>6  r*  rB   r  r+  )r2   rU   r8   r:   r5   Ú
embeddingsr   r-  r.  r/  r  r   rW   r  r0  r”   r1  r2  rv   s    `€rA   rU   zDiaDecoder.__init__0  s¸   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø"Ô/ˆÔØ Ô+ˆŒÝ2°6Ñ:Ô:ˆŒÝ”mØaÐaÐaÐaÅÀvÔG_ÑA`ÔA`ÐaÑaÔañ
ô 
ˆŒõ ˜vÔ1°v´ÐGÑGÔGˆŒ	Ý,°FÐ;Ñ;Ô;ˆŒà�ŠÑÔÐÐÐrB   Nr,   r¸   rØ   r@  rA  rõ   r¨   r[   c                 ót  — |                      ¦   «         dd…         \  }}	|�|                     ¦   «         nd}
|€3t          j        |	|j        ¬¦  «        |
z   }|                     d¦  «        }|                      |¦  «        }|€/t          ¦   «         s!|
|	z   }t          j        |||j        ¬¦  «        }t          | j
        |||¬¦  «        }t          | j
        |||¬¦  «        }|                      ||¬¦  «        }| j        D ]} |||||f|||dœ|¤Ž}Œ|                      |¦  «        }t          ||¬	¦  «        S )
a  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length, num_codebooks)`):
            The original `decoder_input_ids` in 3D shape to facilitate more efficient computations.

            [What are input IDs?](../glossary#input-ids)
        Nr]   r   r4  )r*   r5  rØ   rõ   )r*   r5  rØ   r@  r6  )rA  rõ   r¸   )r7  rõ   )ÚsizeÚget_seq_lengthr6   r7   rb   rÆ   rL  r   r„   r   r*   r   r1  r  r0  r   )r>   r,   r¸   rØ   r@  rA  rõ   r¨   Ú
batch_sizeÚ
seq_lengthÚpast_key_values_lengthrw   Úmask_seq_lengthrô   Úlayers                  rA   rh   zDiaDecoder.forward=  sœ  € ð( "+§¢Ñ!1Ô!1°#°2°#Ô!6Ñˆ
�JØETÐE` ×!?Ò!?Ñ!AÔ!AÐ!AÐfgÐàÐÝ œ<¨
¸9Ô;KÐLÑLÔLÐOeÑeˆLØ'×1Ò1°!Ñ4Ô4ˆLð Ÿš¨	Ñ2Ô2ˆàÐ!Õ*BÑ*DÔ*DÐ!à4°zÑAˆOÝ"œZ¨
°OÈIÔL\Ð]Ñ]Ô]ˆNå+Ø”;Ø'Ø)Ø+ð	
ñ 
ô 
ˆõ ";Ø”;Ø'Ø1Ø"7ð	"
ñ "
ô "
Ðð #Ÿošo¨mÈ,˜oÑWÔWÐà”[ð 	ð 	ˆEØ!˜EØð $ØØ%ðð (>Ø /Ø)ðð ð ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆå8Ø+Ø+ð
ñ 
ô 
ð 	
rB   rF  )rC   rD   rE   ri   r.   ré   r  r9  r%   rU   r!   r"   r   r6   rj   Ú
LongTensorr|   r
   r   r   r   r�   rh   rO   rP   s   @rA   rH  rH  '  s>  ø€ € € € € Ø7Ð7ð )Ø&Ø-ðð ÐðÐ/ð ð ð ð ð ð ð  ØØð 15Ø.2Ø:>Ø:>Ø6:ðA
ð A
à”<ðA
ð Ô&¨Ñ-ðA
ð œ tÑ+ð	A
ð
  %Ô0°4Ñ7ðA
ð !&Ô 0°4Ñ 7ðA
ð -¨tÑ3ðA
ð Ð+Ô,ðA
ð 
3°UÑ	:ðA
ð A
ð A
ñ „^ñ „_ñ  ÔðA
ð A
ð A
ð A
ð A
rB   rH  z[
    The bare Dia model outputting raw hidden-states without any specific head on top.
    )Úcustom_introc                   óð   ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	e	e
z  dz  d
edz  dedz  de
ez  fd„¦   «         ¦   «         Zˆ xZS )ÚDiaModelr*   c                 óä   •— t          ¦   «                              |¦  «         || _        t          |j        ¦  «        | _        t          |j        ¦  «        | _        |  	                    ¦   «          d S r½   )
r2   rU   r*   r"  Úencoder_configÚencoderrH  Údecoder_configÚdecoderr2  rv   s     €rA   rU   zDiaModel.__init__Š  s\   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒÝ! &Ô"7Ñ8Ô8ˆŒÝ! &Ô"7Ñ8Ô8ˆŒØ�ŠÑÔÐÐÐrB   Nr,   rØ   Údecoder_input_idsÚdecoder_position_idsÚdecoder_attention_maskÚencoder_outputsrõ   Ú	use_cacher[   c	                 ó®  — |€|€t          d¦  «        ‚| j        r%| j        r|rt                               d¦  «         d}|r8|€6t          t          | j        ¬¦  «        t          | j        ¬¦  «        ¦  «        }|€ | j        d||dœ|	¤Ž}nct          |t          ¦  «        sNt          |d         t          |¦  «        dk    r|d         ndt          |¦  «        d	k    r|d	         nd¬
¦  «        }|d         j        d         d| j        j        j        }}}
|€.t          j        |
d|f| j        j        j        | j        ¬¦  «        }|j        d	k    r+|                     |
||¦  «                             dd	¦  «        } | j        d||||d         |||dœ|	¤Ž}t/          |j        |j        |j        |j        |j        |d         |j        |j        ¬¦  «        S )a\  
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, target_sequence_length)
        or (batch_size, target_sequence_length, num_codebooks)`, *optional*):
            1. (batch_size * num_codebooks, target_sequence_length): corresponds to the general use case where
            the audio input codebooks are flattened into the batch dimension. This also aligns with the flat-
            tened audio logits which are used to calculate the loss.

            2. (batch_size, sequence_length, num_codebooks): corresponds to the internally used shape of
            Dia to calculate embeddings and subsequent steps more efficiently.

            If no `decoder_input_ids` are provided, it will create a tensor of `bos_token_id` with shape
            `(batch_size, 1, num_codebooks)`. Indices can be obtained using the [`DiaProcessor`]. See
            [`DiaProcessor.__call__`] for more details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
            Indices of positions of each input sequence tokens in the position embeddings.
            Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`.

            [What are position IDs?](../glossary#position-ids)
        NzXYou should either provide text ids or the cached text encodings. Neither has been found.zZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fr+  )r,   rØ   r   r#   r^   )r7  rw   r#  r]   )rN  Ú
fill_valuerb   )r,   r¸   rØ   r@  rA  rõ   rb  )r7  rõ   Údecoder_hidden_statesÚdecoder_attentionsrI  Úencoder_last_hidden_stater@  Úencoder_attentionsr&  )Ú
ValueErrorÚis_gradient_checkpointingrÝ   ÚloggerÚwarning_oncer
   r	   r*   r[  r4   r   Úlenrd   r\  r8   r6   ÚfullÚbos_token_idrb   ÚndimrÏ   r³   r]  r   r7  rõ   rw   r#  rI  )r>   r,   rØ   r^  r_  r`  ra  rõ   rb  r¨   ÚbszÚseq_lenÚchannelsÚdecoder_outputss                 rA   rh   zDiaModel.forward‘  sd  € ðH Ð Ð!8ÝØjñô ð ð Ô)ð 	"¨d¬mð 	"Øð "Ý×#Ò#Øpñô ð ð "�	àð 	v˜Ð0Ý1µ,ÀdÄkÐ2RÑ2RÔ2RÕT`ÐhlÔhsÐTtÑTtÔTtÑuÔuˆOàÐ"Ø*˜dœlð Ø#Ø-ðð ð ðð ˆOˆOõ ˜O­_Ñ=Ô=ð 	Ý-Ø"1°!Ô"4Ý47¸Ñ4HÔ4HÈ1Ò4LÐ4L˜o¨aÔ0Ð0ÐRVÝ14°_Ñ1EÔ1EÈÒ1IÐ1I˜?¨1Ô-Ð-Ètðñ ô ˆOð #2°!Ô"4Ô":¸1Ô"=¸rÀ4Ä;ÔC]ÔCj�hˆWˆØÐ$Ý %¤
Ø˜1˜hÐ'°D´KÔ4NÔ4[ÐdhÔdoð!ñ !ô !Ðð Ô! QÒ&Ð&Ø 1× 9Ò 9¸#¸xÈÑ QÔ Q× [Ò [Ð\]Ð_`Ñ aÔ aÐà&˜$œ,ð 	
Ø'Ø-Ø1Ø"1°!Ô"4Ø#1Ø+Øð	
ð 	
ð ð	
ð 	
ˆõ "Ø-Ô?Ø+Ô;Ø"1Ô"?Ø.Ô9Ø,Ô=Ø&5°aÔ&8Ø"1Ô"?Ø.Ô9ð	
ñ 	
ô 	
ð 		
rB   )NNNNNNNN)rC   rD   rE   r$   rU   r   r   r6   rU  r   r�   r
   rÿ   r   rh   rO   rP   s   @rA   rX  rX  „  s9  ø€ € € € € ð˜yð ð ð ð ð ð ð Øð .2Ø26Ø59Ø8<Ø:>Ø:>Ø6:Ø!%ð]
ð ]
àÔ# dÑ*ð]
ð Ô(¨4Ñ/ð]
ð !Ô+¨dÑ2ð	]
ð
 $Ô.°Ñ5ð]
ð !&Ô 0°4Ñ 7ð]
ð )¨5Ñ0°4Ñ7ð]
ð -¨tÑ3ð]
ð ˜$‘;ð]
ð 
Ð#Ñ	#ð]
ð ]
ð ]
ñ Ôñ „^ð]
ð ]
ð ]
ð ]
ð ]
rB   rX  zl
    The Dia model consisting of a (byte) text encoder and audio decoder with a prediction head on top.
    c                   ó  ‡ — e Zd ZdZdZdefˆ fd„Zee	 	 	 	 	 	 	 	 	 dde	j
        dz  de	j
        dz  de	j
        dz  d	e	j
        dz  d
e	j
        dz  deez  dz  dedz  dedz  de	j
        dz  deez  fd„¦   «         ¦   «         Zˆ xZS )ÚDiaForConditionalGenerationr+   )Úaudior*   c                 ó`  •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        |j        j        | _        |j        j        | _        t          j	        |j        j
        | j        | j        z  d¬¦  «        | _        d| _        |                      ¦   «          d S )NFrn   ÚForMaskedLM)r2   rU   r*   rX  r+   r\  r8   r:   r   rp   rW   Úlogits_denseÚ	loss_typer2  rv   s     €rA   rU   z$DiaForConditionalGeneration.__init__ü  sž   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒÝ˜fÑ%Ô%ˆŒ
à"Ô1Ô>ˆÔØ Ô/Ô:ˆŒÝœIØÔ!Ô-°Ô0AÀDÄOÑ0SÐ[`ð
ñ 
ô 
ˆÔð 'ˆŒð 	�ŠÑÔÐÐÐrB   Nr,   rØ   r^  r_  r`  ra  rõ   rb  Úlabelsr[   c
                 óþ  —  | j         d	||||||||dœ|
¤Ž}|d         }|j        d         }|                      |¦  «                             |d| j        | j        f¦  «                             dd¦  «                             ¦   «                              || j        z  d| j        ¦  «        }d}|	� | j        d	||	| j        dœ|
¤Ž}t          |||j
        |j        |j        |j        |j        |j        |j        ¬¦	  «	        S )
a   
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, target_sequence_length)
        or (batch_size, target_sequence_length, num_codebooks)`, *optional*):
            1. (batch_size * num_codebooks, target_sequence_length): corresponds to the general use case where
            the audio input codebooks are flattened into the batch dimension. This also aligns with the flat-
            tened audio logits which are used to calculate the loss.

            2. (batch_size, sequence_length, num_codebooks): corresponds to the internally used shape of
            Dia to calculate embeddings and subsequent steps more efficiently.

            If no `decoder_input_ids` are provided, it will create a tensor of `bos_token_id` with shape
            `(batch_size, 1, num_codebooks)`. Indices can be obtained using the [`DiaProcessor`]. See
            [`DiaProcessor.__call__`] for more details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
            Indices of positions of each input sequence tokens in the position embeddings.
            Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`.

            [What are position IDs?](../glossary#position-ids)
        labels (`torch.LongTensor` of shape `(batch_size * num_codebooks,)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in
            `[0, ..., config.decoder_config.vocab_size - 1]` or -100. Tokens with indices set to `-100`
            are ignored (masked).
        )r,   rØ   r^  r_  r`  ra  rõ   rb  r   r]   r#   r^   N)Úlogitsr|  r:   )	Úlossr~  rõ   re  rf  rI  rg  r@  rh  r&  )r+   rd   rz  rc   r8   r:   r³   râ   Úloss_functionr   rõ   re  rf  rI  rg  r@  rh  )r>   r,   rØ   r^  r_  r`  ra  rõ   rb  r|  r¨   Úoutputsr7  rP  Úaudio_logitsr  s                   rA   rh   z#DiaForConditionalGeneration.forward  s<  € ðR �$”*ð 

ØØ)Ø/Ø!5Ø#9Ø+Ø+Øð

ð 

ð ð

ð 

ˆð $ AœJÐØ&Ô,¨QÔ/ˆ
ð ×ÒÐ/Ñ0Ô0ßŠT�:˜r 4Ô#4°d´oÐFÑGÔGßŠY�q˜!‰_Œ_ßŠZ‰\Œ\ßŠT�*˜tÔ0Ñ0°"°d´oÑFÔFð 	ð ˆØÐØ%�4Ô%Ðo¨\À&ÐUYÔUdÐoÐoÐhnÐoÐoˆDåØØØ#Ô3Ø")Ô"?Ø&Ô9Ø$Ô5Ø&-Ô&GØ")Ô"?Ø&Ô9ð

ñ 

ô 

ð 
	
rB   )	NNNNNNNNN)rC   rD   rE   rG   Úoutput_modalitiesr$   rU   r   r   r6   rU  r   r�   r
   rÿ   r   rh   rO   rP   s   @rA   rv  rv  ó  s\  ø€ € € € € ð  ÐØ"Ðð˜yð ð ð ð ð ð ð Øð .2Ø26Ø59Ø8<Ø:>Ø:>Ø6:Ø!%Ø*.ðL
ð L
àÔ# dÑ*ðL
ð Ô(¨4Ñ/ðL
ð !Ô+¨dÑ2ð	L
ð
 $Ô.°Ñ5ðL
ð !&Ô 0°4Ñ 7ðL
ð )¨5Ñ0°4Ñ7ðL
ð -¨tÑ3ðL
ð ˜$‘;ðL
ð Ô  4Ñ'ðL
ð 
�Ñ	 ðL
ð L
ð L
ñ Ôñ „^ðL
ð L
ð L
ð L
ð L
rB   rv  )rX  r)   rv  )r#   )rÔ   )PÚcollections.abcr   r6   r   Ú r   r;   Úactivationsr   Úcache_utilsr   r	   r
   Úintegrationsr   r   Úmasking_utilsr   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   Úutils.deprecationr   Úutils.genericr    r!   Úutils.output_capturingr"   Úconfiguration_diar$   r%   r&   Úgeneration_diar'   Ú
get_loggerrC   rk  r)   ÚModuler5   rl   r   r”   rÃ   rÌ   rj   rþ   rÓ   r’   rç   ré   r  r-   r"  r.   rH  rX  rv  Ú__all__r&  rB   rA   ú<module>r™     sí  ðð* %Ð $Ð $Ð $Ð $Ð $à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CØ LÐ LÐ LÐ LÐ LÐ LÐ LÐ LØ JÐ JÐ JÐ JÐ JÐ JÐ JÐ JØ BÐ BÐ BÐ BÐ BÐ BØ 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð LÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lØ 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø LÐ LÐ LÐ LÐ LÐ LÐ LÐ LÐ LÐ LØ .Ð .Ð .Ð .Ð .Ð .ð 
ˆÔ	˜HÑ	%Ô	%€ð ð0ð 0ð 0ð 0ð 0˜ñ 0ô 0ñ „ð0ð$!ð !ð !ð !ð !˜rœyñ !ô !ð !ð<)ð )ð )ð )ð )ˆRŒYñ )ô )ð )ð$ Ð˜YÑ'Ô'ðJð Jð Jð Jð J�”ñ Jô Jñ (Ô'ðJð(6<ð 6<ð 6<ð 6<ð 6<˜œñ 6<ô 6<ð 6<ðr(ð (ð (ð ÐÐ-Ñ.Ô.ðð ð ñ /Ô.ðð2	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð& ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð2 ÐÐ)Ñ*Ô*ð;)ð ;)ð ;)ð ;)ð ;)�r”yñ ;)ô ;)ñ +Ô*ð;)ð|G)ð G)ð G)ð G)ð G)˜œ	ñ G)ô G)ð G)ðTð ð ð ð Ð0ñ ô ð ðB5@ð 5@ð 5@ð 5@ð 5@Ð#ñ 5@ô 5@ð 5@ðp6ð 6ð 6ð 6ð 6Ð0ñ 6ô 6ð 6ðrZ
ð Z
ð Z
ð Z
ð Z
Ð#ñ Z
ô Z
ð Z
ðz €ððñ ô ð
g
ð g
ð g
ð g
ð g
Ð!ñ g
ô g
ñô ð
g
ðT €ððñ ô ð
a
ð a
ð a
ð a
ð a
Ð"4Ð6Hñ a
ô a
ñô ð
a
ðH LÐ
KÐ
K€€€rB   