§
    9²¸j@X  ã                   óp  — d dl Z d dlmZ d dlZd dlmZ d dlmc mZ ddl	m
Z ddlmZ ddlmZ ddlmZmZ ddlmZ dd	lmZ dd
lmZmZmZmZ ddlmZ ddlmZ ddl m!Z!  ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z" ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z# ed¦  «         G d„ dej$        ¦  «        ¦   «         Z% G d„ dej$        ¦  «        Z& G d„ dej$        ¦  «        Z' G d„ dej$        ¦  «        Z( G d „ d!ej$        ¦  «        Z) G d"„ d#ej$        ¦  «        Z* G d$„ d%ej$        ¦  «        Z+ G d&„ d'ej$        ¦  «        Z,e G d(„ d)e¦  «        ¦   «         Z- ed*¬¦  «         G d+„ d,e-¦  «        ¦   «         Z. ed-¬¦  «         G d.„ d/e-e!¦  «        ¦   «         Z/g d0¢Z0dS )1é    N)Ú	dataclassé   )Úinitialization)ÚACT2FN)Úuse_kernel_forward_from_hub)ÚBaseModelOutputWithPastÚBaseModelOutputWithPooling)ÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚtorch_compilable_checké   )Ú	AutoModelé   )ÚVibeVoiceConfig)ÚVibeVoiceGenerationMixinzDBase class for VibeVoice outputs, with hidden states and attentions.©Úcustom_introc                   ó2   — e Zd ZU dZdZej        dz  ed<   dS )Ú VibeVoiceBaseModelOutputWithPastzÉ
    audio_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, feature_size)`, *optional*):
        Extracted audio features that can be used for conditioning the language model.
    NÚaudio_features)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚtorchÚFloatTensorÚ__annotations__© ó    ú€/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/vibevoice/modeling_vibevoice.pyr   r   (   s6   € € € € € € ðð ð
 04€N�EÔ%¨Ñ,Ð3Ð3Ñ3Ð3Ð3r"   r   z(VibeVoice causal language model outputs.c                   ón   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	ej        dz  ed<   dS )ÚVibeVoiceCausalLMOutputWithPasta.  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    audio_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, feature_size)`, *optional*):
        Extracted audio features that can be used for conditioning the language model.
    NÚlossÚlogitsr   )
r   r   r   r   r&   r   r   r    r'   r   r!   r"   r#   r%   r%   3   sf   € € € € € € ðð ð &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø/3€N�EÔ%¨Ñ,Ð3Ð3Ñ3Ð3Ð3r"   r%   ÚRMSNormc                   óT   ‡ — e Zd Zd	deddfˆ fd„Zdej        dej        fd„Zd„ Zˆ xZ	S )
ÚVibeVoiceRMSNormç�íµ ÷Æ°>ÚepsÚreturnNc                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )z?
        VibeVoiceRMSNorm is equivalent to T5LayerNorm
        N)ÚsuperÚ__init__ÚnnÚ	Parameterr   ÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizer,   Ú	__class__s      €r#   r0   zVibeVoiceRMSNorm.__init__F   sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐr"   Úhidden_statesc                 ó  — |j         }|                     t          j        ¦  «        }|                     d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        |                     |¦  «        z  S )Nr   éÿÿÿÿT)Úkeepdim)	ÚdtypeÚtor   Úfloat32ÚpowÚmeanÚrsqrtr5   r4   )r6   r9   Úinput_dtypeÚvariances       r#   ÚforwardzVibeVoiceRMSNorm.forwardN   s|   € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØ ×$Ò$ QÑ'Ô'×,Ò,¨R¸Ð,Ñ>Ô>ˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆØŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:r"   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)Útupler4   Úshaper5   )r6   s    r#   Ú
extra_reprzVibeVoiceRMSNorm.extra_reprU   s&   € Ý˜œÔ)Ñ*Ô*ÐIÐI°$Ô2GÐIÐIÐIr"   )r+   )
r   r   r   Úfloatr0   r   ÚTensorrE   rI   Ú__classcell__©r8   s   @r#   r*   r*   D   sŒ   ø€ € € € € ð$ð $¨ð $¸$ð $ð $ð $ð $ð $ð $ð; U¤\ð ;°e´lð ;ð ;ð ;ð ;ðJð Jð Jð Jð Jð Jð Jr"   r*   c                   ó:   ‡ — e Zd Zˆ fd„Zed„ ¦   «         Zd„ Zˆ xZS )Ú)VibeVoiceDiffusionHeadSinusoidalEmbeddingc                 ó²   •— t          ¦   «                              ¦   «          || _        t          j        |                      |¦  «        d¬¦  «        | _        d S )NF)Ú
persistent)r/   r0   Úconfigr1   ÚBufferÚcompute_default_freqÚfreq©r6   rR   r8   s     €r#   r0   z2VibeVoiceDiffusionHeadSinusoidalEmbedding.__init__Z   sH   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”I˜d×7Ò7¸Ñ?Ô?ÈEÐRÑRÔRˆŒ	ˆ	ˆ	r"   c                 ó¶   — | j         dz  }t          j        t          j        | j        ¦  «         t          j        |t          j        ¬¦  «        z  |z  ¦  «        S )Nr   )r=   )Úfrequency_embedding_sizer   ÚexpÚmathÚlogÚdiffusion_max_periodÚaranger?   )rR   Údims     r#   rT   z>VibeVoiceDiffusionHeadSinusoidalEmbedding.compute_default_freq_   sM   € àÔ-°Ñ2ˆÝŒy�$œ( 6Ô#>Ñ?Ô?Ð?Å%Ä,ÈsÕZ_ÔZgÐBhÑBhÔBhÑhÐknÑnÑoÔoÐor"   c                 ó  — |d d …d f                               ¦   «         | j        d                               |j        ¦  «        z  }t	          j        t	          j        |¦  «        t	          j        |¦  «        gd¬¦  «        }|S )Nr;   ©r^   )rJ   rU   r>   Údevicer   ÚcatÚcosÚsin)r6   Ú	timestepsrU   Ú	embeddings       r#   rE   z1VibeVoiceDiffusionHeadSinusoidalEmbedding.forwardd   sk   € Ø˜˜˜˜D˜Ô!×'Ò'Ñ)Ô)¨D¬I°d¬O×,>Ò,>¸yÔ?OÑ,PÔ,PÑPˆÝ”I�uœy¨™œµ´	¸$±´Ð@ÀbÐIÑIÔIˆ	ØÐr"   )r   r   r   r0   ÚstaticmethodrT   rE   rL   rM   s   @r#   rO   rO   Y   sj   ø€ € € € € ðSð Sð Sð Sð Sð
 ðpð pñ „\ðpðð ð ð ð ð ð r"   rO   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚVibeVoiceDiffusionHeadMLPc                 ó  •— t          ¦   «                              ¦   «          || _        t          |j                 | _        t          j        |j        |j	        d¬¦  «        | _
        t          j        |j	        |j	        d¬¦  «        | _        d S ©NF©Úbias)r/   r0   rR   r   Ú
hidden_actÚactivation_fnr1   ÚLinearrX   r7   Úfc1Úfc2rV   s     €r#   r0   z"VibeVoiceDiffusionHeadMLP.__init__k   sq   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ# FÔ$5Ô6ˆÔÝ”9˜VÔ<¸fÔ>PÐW\Ð]Ñ]Ô]ˆŒÝ”9˜VÔ/°Ô1CÈ%ÐPÑPÔPˆŒˆˆr"   r9   r-   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S ©N)rq   ro   rr   )r6   r9   s     r#   rE   z!VibeVoiceDiffusionHeadMLP.forwardr   s=   € ØŸš Ñ/Ô/ˆØ×*Ò*¨=Ñ9Ô9ˆØŸš Ñ/Ô/ˆØÐr"   )r   r   r   r0   r   rK   rE   rL   rM   s   @r#   ri   ri   j   sc   ø€ € € € € ðQð Qð Qð Qð Qð U¤\ð °e´lð ð ð ð ð ð ð ð r"   ri   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚVibeVoiceMLPc                 ó¶  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        t          j        | j        | j        |j        ¬¦  «        | _        t          j        | j        | j        |j        ¬¦  «        | _	        t          j        | j        | j        |j        ¬¦  «        | _
        t          |j                 | _        d S )Nrl   )r/   r0   rR   r7   Úintermediate_sizer1   rp   Úmlp_biasÚ	gate_projÚup_projÚ	down_projr   rn   Úact_fnrV   s     €r#   r0   zVibeVoiceMLP.__init__z   s¯   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ!'Ô!9ˆÔÝœ 4Ô#3°TÔ5KÐRXÔRaÐbÑbÔbˆŒÝ”y Ô!1°4Ô3IÐPVÔP_Ð`Ñ`Ô`ˆŒÝœ 4Ô#9¸4Ô;KÐRXÔRaÐbÑbÔbˆŒÝ˜VÔ.Ô/ˆŒˆˆr"   c                 ó¨   — |                       |                      |                      |¦  «        ¦  «        |                      |¦  «        z  ¦  «        }|S rt   )r|   r}   rz   r{   )r6   Úxr|   s      r#   rE   zVibeVoiceMLP.forward„   sA   € Ø—N’N 4§;¢;¨t¯~ª~¸aÑ/@Ô/@Ñ#AÔ#AÀDÇLÂLÐQRÁOÄOÑ#SÑTÔTˆ	ØÐr"   ©r   r   r   r0   rE   rL   rM   s   @r#   rv   rv   y   sG   ø€ € € € € ð0ð 0ð 0ð 0ð 0ðð ð ð ð ð ð r"   rv   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )Ú"VibeVoiceDiffusionHeadAdaLayerNormc                 óH  •— t          ¦   «                              ¦   «          d| _        t          |¦  «        | _        t          |j        |j        ¬¦  «        | _        t          |j
                 | _        t          j        |j        |j        | j        z  d¬¦  «        | _        d S )Nr   ©r,   Frl   )r/   r0   Ú
num_chunksrv   Úffnr*   r7   Úrms_norm_epsÚnormr   rn   r}   r1   rp   ÚlinearrV   s     €r#   r0   z+VibeVoiceDiffusionHeadAdaLayerNorm.__init__Š   s‚   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ Ñ'Ô'ˆŒÝ$ VÔ%7¸VÔ=PÐQÑQÔQˆŒ	Ý˜VÔ.Ô/ˆŒÝ”i Ô 2°FÔ4FÈÌÑ4XÐ_dÐeÑeÔeˆŒˆˆr"   c                 ó  — |                       |                      |¦  «        ¦  «                             | j        d¬¦  «        \  }}}|                      |¦  «        d|z   z  |z   }|||                      |¦  «        z  z   }|S )Nr;   r`   r   )r‰   r}   Úchunkr…   rˆ   r†   )r6   r9   Ú	conditionÚ	shift_ffnÚ	scale_ffnÚgate_ffnÚmodulated_hidden_statess          r#   rE   z*VibeVoiceDiffusionHeadAdaLayerNorm.forward’   s€   € Ø)-¯ª°T·[²[ÀÑ5KÔ5KÑ)LÔ)L×)RÒ)RÐSWÔSbÐhjÐ)RÑ)kÔ)kÑ&ˆ	�9˜hØ"&§)¢)¨MÑ":Ô":¸aÀ)¹mÑ"LÈyÑ"XÐØ%¨°4·8²8Ð<SÑ3TÔ3TÑ(TÑTˆØÐr"   r€   rM   s   @r#   r‚   r‚   ‰   sL   ø€ € € € € ðfð fð fð fð fðð ð ð ð ð ð r"   r‚   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )Ú VibeVoiceDiffusionHeadFinalLayerc                 ó:  •— t          ¦   «                              ¦   «          d| _        |j        | _        t	          j        |j        | j        |j        z  d¬¦  «        | _        t          |j	                 | _
        t	          j        |j        |d¬¦  «        | _        d S )Nr   Frl   )r/   r0   r…   r‡   r1   rp   r7   Úlinear_1r   rn   r}   Úlinear_2)r6   rR   Úoutput_sizer8   s      €r#   r0   z)VibeVoiceDiffusionHeadFinalLayer.__init__š   s   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"Ô/ˆÔÝœ	 &Ô"4°d´oÈÔHZÑ6ZÐafÐgÑgÔgˆŒÝ˜VÔ.Ô/ˆŒÝœ	 &Ô"4°kÈÐNÑNÔNˆŒˆˆr"   c                 ó   — |                       |                      |¦  «        ¦  «                             | j        d¬¦  «        \  }}t	          j        ||j        d         f| j        ¬¦  «        }|d|z   z  |z   }|                      |¦  «        }|S )Nr;   r`   r„   r   )	r”   r}   r‹   r…   ÚFÚrms_normrH   r‡   r•   )r6   r9   rŒ   ÚshiftÚscaleÚnormeds         r#   rE   z(VibeVoiceDiffusionHeadFinalLayer.forward¢   s‡   € Ø—}’} T§[¢[°Ñ%;Ô%;Ñ<Ô<×BÒBÀ4Ä?ÐXZÐBÑ[Ô[‰ˆˆuÝ”˜M¨MÔ,?ÀÔ,CÐ+EÈ4ÔK\Ð]Ñ]Ô]ˆØ ! e¡)Ñ,¨uÑ4ˆØŸš mÑ4Ô4ˆØÐr"   r€   rM   s   @r#   r’   r’   ™   sL   ø€ € € € € ðOð Oð Oð Oð Oðð ð ð ð ð ð r"   r’   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚVibeVoiceDiffusionHeadc                 óÖ  •‡— t          ¦   «                              ¦   «          t          j        ‰j        ‰j        d¬¦  «        | _        t          j        ‰j        ‰j        d¬¦  «        | _        t          ‰¦  «        | _	        t          ‰¦  «        | _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t!          ‰‰j        ¬¦  «        | _        d S )NFrl   c                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r!   )r‚   )Ú.0Ú_rR   s     €r#   ú
<listcomp>z3VibeVoiceDiffusionHead.__init__.<locals>.<listcomp>²   s"   ø€ ÐaÐaÐa¸AÕ/°Ñ7Ô7ÐaÐaÐar"   )r–   )r/   r0   r1   rp   Úlatent_sizer7   Únoisy_images_projÚ	cond_projrO   Útimestep_embeddingri   Útimestep_projÚ
ModuleListÚrangeÚnum_hidden_layersÚlayersr’   Úfinal_layerrV   s    `€r#   r0   zVibeVoiceDiffusionHead.__init__«   sÏ   øø€ Ý‰Œ×ÒÑÔÐÝ!#¤¨6Ô+=¸vÔ?QÐX]Ð!^Ñ!^Ô!^ˆÔÝœ 6Ô#5°vÔ7IÐPUÐVÑVÔVˆŒÝ"KÈFÑ"SÔ"SˆÔÝ6°vÑ>Ô>ˆÔÝ”mØaÐaÐaÐaÅÀvÔG_ÑA`ÔA`ÐaÑaÔañ
ô 
ˆŒõ <¸FÐPVÔPbÐcÑcÔcˆÔÐÐr"   c                 ó@  — |                       |¦  «        }|                      |¦  «                             |j        ¦  «        }|                      |¦  «        }|                      |¦  «        }||z   }| j        D ]} |||¦  «        }Œ|                      ||¦  «        }|S )z\
        Forward pass of the prediction head. Returns the predicted noise/velocity.
        )r¥   r§   r>   r=   r¨   r¦   r¬   r­   )r6   Únoisy_imagesre   rŒ   r9   Úembedded_timestepsÚlayers          r#   rE   zVibeVoiceDiffusionHead.forward¶   s«   € ð ×.Ò.¨|Ñ<Ô<ˆØ!×4Ò4°YÑ?Ô?×BÒBÀ9Ä?ÑSÔSÐØ!×/Ò/Ð0BÑCÔCÐØ—N’N 9Ñ-Ô-ˆ	ØÐ 2Ñ2ˆ	Ø”[ð 	<ð 	<ˆEØ!˜E -°Ñ;Ô;ˆMˆMØ×(Ò(¨¸	ÑBÔBˆØÐr"   r€   rM   s   @r#   rž   rž   ª   sL   ø€ € € € € ð	dð 	dð 	dð 	dð 	dðð ð ð ð ð ð r"   rž   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚVibeVoiceMultiModalProjectorc                 óÜ   •— t          ¦   «                              ¦   «          t          j        ||¦  «        | _        t          |d¬¦  «        | _        t          j        ||¦  «        | _        d S )Nr+   r„   )r/   r0   r1   rp   r”   r*   Úactr•   )r6   Ú	input_dimÚ
output_dimr8   s      €r#   r0   z%VibeVoiceMultiModalProjector.__init__Æ   sW   ø€ Ý‰Œ×ÒÑÔÐÝœ	 )¨ZÑ8Ô8ˆŒÝ# J°DÐ9Ñ9Ô9ˆŒÝœ	 *¨jÑ9Ô9ˆŒˆˆr"   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S rt   )r”   rµ   r•   )r6   r   r9   s      r#   rE   z$VibeVoiceMultiModalProjector.forwardÌ   s;   € ØŸš nÑ5Ô5ˆØŸš Ñ/Ô/ˆØŸš mÑ4Ô4ˆØÐr"   r€   rM   s   @r#   r³   r³   Å   sG   ø€ € € € € ð:ð :ð :ð :ð :ðð ð ð ð ð ð r"   r³   c                   óZ   ‡ — e Zd ZU eed<   dZdZdZdgZdgZ	dZ
dZdZdZdZdZˆ fd„Zˆ xZS )ÚVibeVoicePreTrainedModelrR   Úmodel)ÚaudioÚtextTrž   Úpast_key_valuesc                 ó„  •— t          ¦   «                              |¦  «         t          |d¦  «        rt          j        |j        d¦  «         t          |d¦  «        rt          j        |j        d¦  «         t          |t          ¦  «        r4t          j	        |j
        |                     |j        ¦  «        ¦  «         d S d S )NÚlatent_scaling_factorç      ð?Úlatent_bias_factorç        )r/   Ú_init_weightsÚhasattrÚinitÚ	constant_rÀ   rÂ   Ú
isinstancerO   Úcopy_rU   rT   rR   )r6   Úmoduler8   s     €r#   rÄ   z&VibeVoicePreTrainedModel._init_weightsâ   sµ   ø€ Ý‰Œ×Ò˜fÑ%Ô%Ð%Ý�6Ð2Ñ3Ô3ð 	>ÝŒN˜6Ô7¸Ñ=Ô=Ð=Ý�6Ð/Ñ0Ô0ð 	;ÝŒN˜6Ô4°cÑ:Ô:Ð:Ý�fÕGÑHÔHð 	PÝŒJ�v”{ F×$?Ò$?ÀÄÑ$NÔ$NÑOÔOÐOÐOÐOð	Pð 	Pr"   )r   r   r   r   r    Úbase_model_prefixÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_cache_classÚ_supports_attention_backendÚ_can_compile_fullgraphrÄ   rL   rM   s   @r#   rº   rº   Ó   s–   ø€ € € € € € àÐÐÑØÐØ(ÐØ&*Ð#Ø1Ð2ÐØ#4Ð"5ÐØÐØ€NØÐØ ÐØ"&ÐØ!ÐðPð Pð Pð Pð Pð Pð Pð Pð Pr"   rº   zw
    The VibeVoice model which consists of audio tokenizers and an LLM backbone, without a language modeling head.
    c                   óX  ‡ — e Zd Zˆ fd„Ze ed¬¦  «        dej        dej        de	e
         deez  fd„¦   «         ¦   «         Zd	ej        d
ej        dej        fd„Zee	 	 	 	 dd	ej        d
ej        dz  dej        dz  dej        dz  de	e
         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚVibeVoiceModelc                 ó¨  •— t          ¦   «                              |¦  «         t          j        |j        ¦  «        | _        t          j        |j        ¦  «        | _        t          |j        j	        |j        j	        ¦  «        | _
        t          j        |j        ¦  «        | _        t          |j        j	        |j        j	        ¦  «        | _        t          |j        ¦  «        | _        t#          j        t'          j        d¦  «        ¦  «        | _        t#          j        t'          j        d¦  «        ¦  «        | _        |                      ¦   «          d S )NrÁ   rÃ   )r/   r0   r   Úfrom_configÚaudio_configÚaudio_towerÚtext_configÚlanguage_modelr³   r7   Úmulti_modal_projectorÚsemantic_model_configÚsemantic_tokenizer_encoderÚsemantic_connectorrž   Údiffusion_head_configÚdiffusion_headr1   r2   r   ÚtensorrÀ   rÂ   Ú	post_initrV   s     €r#   r0   zVibeVoiceModel.__init__ò   s  ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý$Ô0°Ô1DÑEÔEˆÔÝ'Ô3°FÔ4FÑGÔGˆÔÝ%AØÔÔ+¨VÔ-?Ô-Kñ&
ô &
ˆÔ"õ +4Ô*?ÀÔ@\Ñ*]Ô*]ˆÔ'Ý">ØÔ(Ô4°fÔ6HÔ6Tñ#
ô #
ˆÔõ 5°VÔ5QÑRÔRˆÔÝ%'¤\µ%´,¸sÑ2CÔ2CÑ%DÔ%DˆÔ"Ý"$¤,­u¬|¸CÑ/@Ô/@Ñ"AÔ"AˆÔØ�ŠÑÔÐÐÐr"   zÁThis method is used to get the audio embeddings (that replace placeholder audio tokens in the input sequence) and the acoustic features (used as diffusion target) from the input audio waveform.r   Úinput_valuesÚpadding_maskÚkwargsr-   c                 ó
  — t          j        ¦   «         5  | j                             |d¬¦  «        j        }ddd¦  «         n# 1 swxY w Y   || j                             |j        ¦  «        z  }|| j                             |j        ¦  «        z  }t          j	        | 
                    d¬¦  «        | j        j        j        z  ¦  «                             t           j        ¦  «        }t          j        |                     ¦   «         |j        ¬¦  «        |dd…df         k     }|                      |¦  «        }t%          ||                     |j        ¦  «                 ||                     |j        ¦  «                 ¬¦  «        S )zü
        input_values (`torch.FloatTensor`):
            Float values of (normalized) audio waveform.
        padding_mask (`torch.Tensor` of shape `(batch_size, padded_audio_length)`):
            Padding mask to remove padded parts of audio.
        T)ÚsampleNr;   r`   )ra   )Úlast_hidden_stateÚpooler_output)r   Úno_gradrÛ   ÚencodeÚlatentsrÂ   r>   ra   rÀ   ÚceilÚsumrR   rÚ   Ú
hop_lengthÚint64r]   ÚmaxrÞ   r	   )r6   ræ   rç   rè   Úacoustic_featuresÚnum_audio_tokensrì   s          r#   Úget_audio_featuresz!VibeVoiceModel.get_audio_features  s®  € õ& Œ]‰_Œ_ð 	[ð 	[Ø $Ô 0× 7Ò 7¸ÈTÐ 7Ñ RÔ RÔ ZÐð	[ð 	[ð 	[ñ 	[ô 	[ð 	[ð 	[ð 	[ð 	[ð 	[ð 	[øøøð 	[ð 	[ð 	[ð 	[à˜TÔ4×7Ò7Ð8IÔ8PÑQÔQÑQÐØ˜TÔ7×:Ò:Ð;LÔ;SÑTÔTÑTÐõ !œ: l×&6Ò&6¸2Ð&6Ñ&>Ô&>ÀÄÔAYÔAdÑ&dÑeÔe×hÒhÕinÔitÑuÔuÐÝ”|Ð$4×$8Ò$8Ñ$:Ô$:ÐCSÔCZÐ[Ñ[Ô[Ð^nÐopÐopÐopÐrvÐovÔ^wÒwˆà×2Ò2Ð3DÑEÔEˆÝ)Ø/°·²Ð@QÔ@XÑ0YÔ0YÔZØ'¨¯ª¸Ô8LÑ(MÔ(MÔNð
ñ 
ô 
ð 	
s   ”"AÁAÁ	AÚ	input_idsÚinputs_embedsr   c           	      ó@  — |€f| |                       ¦   «         t          j        d| j        j        t          j        |j        ¬¦  «        ¦  «        k    }|                     d¦  «        }n|| j        j        k    }|                     ¦   «         }|j	        d         }| 
                    d¦  «                             |¦  «                             |j        ¦  «        }t          ||                              ¦   «         |                     ¦   «         k    d|› d|› �¦  «         |S )zï
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        Nr!   ©r=   ra   r;   r   z6Audio features and audio tokens do not match, tokens: z, features: )Úget_input_embeddingsr   ÚfullrR   Úaudio_token_idÚlongra   Úallrñ   rH   Ú	unsqueezeÚ	expand_asr>   r   Únumel)r6   rø   rù   r   Úspecial_audio_maskÚn_audio_tokensÚn_audio_featuress          r#   Úget_placeholder_maskz#VibeVoiceModel.get_placeholder_mask$  s!  € ð ÐØ!.Ð2M°$×2KÒ2KÑ2MÔ2MÝ”
˜2˜tœ{Ô9ÅÄÐTaÔThÐiÑiÔiñ3ô 3ò "Ðð "4×!7Ò!7¸Ñ!;Ô!;ÐÐà!*¨d¬kÔ.HÒ!HÐà+×/Ò/Ñ1Ô1ˆØ)Ô/°Ô2ÐØ/×9Ò9¸"Ñ=Ô=×GÒGÈÑVÔV×YÒYÐZgÔZnÑoÔoÐÝØÐ,Ô-×3Ò3Ñ5Ô5¸×9MÒ9MÑ9OÔ9OÒOØsÀ^ÐsÐsÐaqÐsÐsñ	
ô 	
ð 	
ð "Ð!r"   Nc                 óV  — |€ |                       ¦   «         |¦  «        }d}|�j|                      ||¦  «        }|j        }|j        }|                      |||¬¦  «        }	|                     |	|                     |j        ¦  «        ¦  «        } | j        dd|i|¤Ž}
t          dd|i|
¤ŽS )z—
        padding_mask (`torch.Tensor` of shape `(batch_size, padded_audio_length)`):
            Padding mask to remove padded parts of audio.
        N)rù   r   rù   r   r!   )
rü   r÷   rì   rë   r  Úmasked_scatterr>   ra   rÝ   r   )r6   rø   rù   ræ   rç   rè   r   Úaudio_outputsÚaudio_embedsr  Úoutputss              r#   rE   zVibeVoiceModel.forward<  sØ   € ð Ð Ø7˜D×5Ò5Ñ7Ô7¸	ÑBÔBˆMàˆØÐ#Ø ×3Ò3°LÀ,ÑOÔOˆMØ(Ô6ˆLØ*Ô<ˆNð "&×!:Ò!:Ø¨À|ð ";ñ "ô "Ðð *×8Ò8Ð9KÈ\Ï_Ê_Ð]jÔ]qÑMrÔMrÑsÔsˆMà%�$Ô%ÐLÐL°MÐLÀVÐLÐLˆÝ/ÐYÐY¸~ÐYÐQXÐYÐYÐYr"   )NNNN)r   r   r   r0   r   r   r   r   rK   r   r   rG   r	   r÷   Ú
LongTensorr  Ú
BoolTensorr   rE   rL   rM   s   @r#   r×   r×   ì   s—  ø€ € € € € ðð ð ð ð ð  Ø€^ðnðñ ô ð
àÔ'ð
ð ”lð
ð Ð+Ô,ð	
ð
 
Ð+Ñ	+ð
ð 
ð 
ñ	ô ñ Ôð

ð:"ØÔ)ð"Ø:?Ô:Kð"Ø]bÔ]nð"ð "ð "ð "ð0 Øð '+Ø26Ø15Ø04ðZð ZàÔ#ðZð Ô(¨4Ñ/ðZð Ô'¨$Ñ.ð	Zð
 Ô&¨Ñ-ðZð Ð+Ô,ðZð 
Ð(Ñ	(ðZð Zð Zñ „^ñ ÔðZð Zð Zð Zð Zr"   r×   zv
    The VibeVoice model, which consists of a language model, audio tokenizers, connectors, and a diffusion head.
    c                   ó`  ‡ — e Zd ZddiZddiZˆ fd„Zedej        de	j
        de	j
        d	ed
edede	j
        fd„¦   «         Zee	 	 	 	 	 	 	 	 	 	 dde	j        de	j
        dz  de	j        dz  deez  de	j
        dz  de	j        dz  de	j        dz  d	edz  d
ededeez  fd„¦   «         ¦   «         Zˆ xZS )Ú!VibeVoiceForConditionalGenerationzlm_head.weightz(model.language_model.embed_tokens.weightÚlm_headÚcolwise_gather_outputc                 óú   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        j        |j        j        d¬¦  «        | _	        |  
                    ¦   «          d S rk   )r/   r0   r×   r»   r1   rp   rÜ   r7   Ú
vocab_sizer  rå   rV   s     €r#   r0   z*VibeVoiceForConditionalGeneration.__init__f  se   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý# FÑ+Ô+ˆŒ
Ý”y Ô!3Ô!?ÀÔASÔA^ÐejÐkÑkÔkˆŒØ�ŠÑÔÐÐÐr"   rã   r   Úcondition_featuresÚnoise_schedulerÚddpm_batch_multiplierÚnum_diffusion_stepsr-   c                 ó`  — |j         \  }}|                     |d¬¦  «        }|                     |d¬¦  «        }	t          j        ||z  |f|j        |j        ¬¦  «        }
t          j        t          j        |¦  «        ||z  d¬¦  «                             |j        ¦  «        }| 	                    ||
|¦  «        } | || 
                    |¦  «        |	¦  «        }|j                             |j        |j        ¬¦  «        }|j                             |j        |j        ¬¦  «        }||                              ¦   «                              d¦  «        }||                              ¦   «                              d¦  «        }||
z  ||z  z
                       |j        ¦  «        }t          j        j                             |                     ¦   «         |                     ¦   «         d¬¦  «        }|dk    r|dk    r|dk    r|||z  |z  z  }n"t          j        d	d
|j        |j        ¬¦  «        }|S )Nr   r`   )ra   r=   T)Úreplacementr;   rñ   )Ú	reductionr!   rÃ   rû   )rH   Úrepeat_interleaver   Úrandnra   r=   Úmultinomialr3   r>   Ú	add_noiseÚtype_asÚalpha_tÚsigma_tÚflattenr  r1   Ú
functionalÚmse_lossrJ   rý   )rã   r   r  r  r  r  Ú	audio_lenr¤   Úaudio_features_repeatedÚcondition_features_repeatedÚnoisere   Únoisy_audio_featuresÚmodel_outputr!  r"  ÚtargetÚdiffusion_losss                     r#   Úcompute_diffusion_lossz8VibeVoiceForConditionalGeneration.compute_diffusion_lossl  sG  € ð "0Ô!5Ñˆ	�;Ø"0×"BÒ"BÐCXÐ^_Ð"BÑ"`Ô"`ÐØ&8×&JÒ&JÐK`ÐfgÐ&JÑ&hÔ&hÐ#õ ”ØÐ.Ñ.°Ð<Ø!Ô(Ø Ô&ð
ñ 
ô 
ˆõ
 Ô%ÝŒJÐ*Ñ+Ô+ØÐ-Ñ-Øð
ñ 
ô 
÷ Š"ˆ^Ô"Ñ
#Ô
#ð	 	ð
  /×8Ò8Ð9PÐRWÐYbÑcÔcÐð &�~Ø  )×"3Ò"3°NÑ"CÔ"CÐE`ñ
ô 
ˆð
 "Ô)×,Ò,°NÔ4IÐQ_ÔQeÐ,ÑfÔfˆØ!Ô)×,Ò,°NÔ4IÐQ_ÔQeÐ,ÑfÔfˆØ˜)Ô$×,Ò,Ñ.Ô.×8Ò8¸Ñ<Ô<ˆØ˜)Ô$×,Ò,Ñ.Ô.×8Ò8¸Ñ<Ô<ˆØ˜E‘/ GÐ.EÑ$EÑE×IÒIÈ,ÔJ]Ñ^Ô^ˆåœÔ,×5Ò5°l×6HÒ6HÑ6JÔ6JÈFÏLÊLÉNÌNÐfkÐ5ÑlÔlˆØ˜Š?ˆ?Ð4°qÒ8Ð8¸YÈº]¸]Ø+¨{Ð=RÑ/RÐU^Ñ/^Ñ_ˆNˆNå"œZ¨¨C°~Ô7KÐTbÔTiÐjÑjÔjˆNØÐr"   Nr   é   é
   rø   rù   ÚlabelsÚlogits_to_keepræ   rç   Úacoustic_loss_maskc                 óÈ  —  | j         d||||dœ|¤Ž}|j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }d}|� | j        d||| j        j        dœ|¤Ž}|�Ð|j	        �É| j        j
        dk    r¹|€|                      | j        ¦  «        }|j	        }||                     |j        ¦  «                                      |j        ¦  «        }|                      | j         j        ||||	|
¦  «        }d| j        j
        z
  |z  | j        j
        |                     |j        ¦  «        z  z   }t#          d||dœ|¤ŽS )aÒ  
        input_values (`torch.FloatTensor`, *optional*):
            Preprocessed audio waveform for voice cloning.
        padding_mask (`torch.BoolTensor`, *optional*):
            Masks indicating valid input frames.
        acoustic_loss_mask (`torch.BoolTensor`, *optional*):
            Mask to compute diffusion loss only on specific acoustic tokens.
        noise_scheduler (*optional*):
            Needed for training to compute noise targets for the diffusion loss. By default, uses the noise scheduler
            configuration specified in the model's `generation_config`.
        ddpm_batch_multiplier (`int`, *optional*, defaults to 4):
            For training, number of noise samples to generate per audio token for diffusion loss computation, which can
            help stabilize training.
        num_diffusion_steps (`int`, *optional*, defaults to 10):
            For training, the number of diffusion steps to use. Defaults to 10 if not provided.
        )rø   rù   ræ   rç   N)r'   r1  r  rÃ   r   )r&   r'   r!   )r»   rë   rÈ   ÚintÚslicer  Úloss_functionrR   r  r   Údiffusion_loss_weightÚ_build_default_noise_schedulerÚgeneration_configr>   ra   r.  rã   r%   )r6   rø   rù   r1  r2  ræ   rç   r3  r  r  r  rè   r  r9   Úslice_indicesr'   r&   r   r  r-  s                       r#   rE   z)VibeVoiceForConditionalGeneration.forward™  s¹  € ðB �$”*ð 
ØØ'Ø%Ø%ð	
ð 
ð
 ð
ð 
ˆð  Ô1ˆÝ8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%Ðp¨V¸FÈtÌ{ÔOeÐpÐpÐioÐpÐpˆDð
 Ð*ØÔ&Ð2Ø”Ô1°CÒ7Ð7àÐ&à"&×"EÒ"EÀdÔF\Ñ"]Ô"]�à$Ô3ˆNØ!.Ð/A×/DÒ/DÀ]ÔEYÑ/ZÔ/ZÔ![×!^Ò!^Ð_mÔ_tÑ!uÔ!uÐØ!×8Ò8Ø”
Ô)ØØ"ØØ%Ø#ñô ˆNð �D”KÔ5Ñ5Øñà”{Ô8¸>×;LÒ;LÈTÌ[Ñ;YÔ;YÑYñZˆDõ /ÐS°DÀÐSÐSÈ7ÐSÐSÐSr"   )
NNNr   NNNNr/  r0  )r   r   r   Ú_tied_weights_keysÚ_tp_planr0   rg   r1   ÚModuler   r   Úobjectr5  r.  r   r   r  r6  r  rG   r%   rE   rL   rM   s   @r#   r  r  ]  sÜ  ø€ € € € € ð +Ð,VÐWÐØÐ2Ð3€Hðð ð ð ð ð ð*Øœ	ð*àÔ)ð*ð "Ô-ð*ð  ð	*ð
  #ð*ð !ð*ð 
Ô	ð*ð *ð *ñ „\ð*ðX Øð '+Ø26Ø*.Ø&'Ø15Ø04Ø6:Ø)-Ø%&Ø#%ðHTð HTàÔ#ðHTð Ô(¨4Ñ/ðHTð Ô  4Ñ'ð	HTð
 ˜e™ðHTð Ô'¨$Ñ.ðHTð Ô&¨Ñ-ðHTð "Ô,¨tÑ3ðHTð   $™ðHTð  #ðHTð !ðHTð 
Ð0Ñ	0ðHTð HTð HTñ „^ñ ÔðHTð HTð HTð HTð HTr"   r  )r  rº   r×   )1rZ   Údataclassesr   r   Útorch.nnr1   Útorch.nn.functionalr$  r˜   Ú r   rÆ   Úactivationsr   Úintegrationsr   Úmodeling_outputsr   r	   Úmodeling_utilsr
   Úprocessing_utilsr   Úutilsr   r   r   r   Úautor   Úconfiguration_vibevoicer   Úgeneration_vibevoicer   r   r%   r>  r*   rO   ri   rv   r‚   r’   rž   r³   rº   r×   r  Ú__all__r!   r"   r#   ú<module>rN     s[  ðð* €€€Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø SÐ SÐ SÐ SÐ SÐ SÐ SÐ SØ -Ð -Ð -Ð -Ð -Ð -Ø &Ð &Ð &Ð &Ð &Ð &Ø aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aØ Ð Ð Ð Ð Ð Ø 4Ð 4Ð 4Ð 4Ð 4Ð 4Ø :Ð :Ð :Ð :Ð :Ð :ð €ÐgÐhÑhÔhØ
ð4ð 4ð 4ð 4ð 4Ð'>ñ 4ô 4ñ „ñ iÔhð4ð €ÐKÐLÑLÔLØ
ð4ð 4ð 4ð 4ð 4Ð&=ñ 4ô 4ñ „ñ MÔLð4ð Ð˜YÑ'Ô'ðJð Jð Jð Jð J�r”yñ Jô Jñ (Ô'ðJð(ð ð ð ð °´	ñ ô ð ð"ð ð ð ð  ¤	ñ ô ð ðð ð ð ð �2”9ñ ô ð ð ð ð ð ð ¨¬ñ ô ð ð ð ð ð ð  r¤yñ ô ð ð"ð ð ð ð ˜RœYñ ô ð ð6ð ð ð ð  2¤9ñ ô ð ð ðPð Pð Pð Pð P˜ñ Pô Pñ „ðPð0 €ððñ ô ð
iZð iZð iZð iZð iZÐ-ñ iZô iZñô ð
iZðX €ððñ ô ð
ATð ATð ATð ATð ATÐ(@ÐBZñ ATô ATñô ð
ATðH ^Ð
]Ð
]€€€r"   