§
    9²¸jàF  ã                   óB  — d dl Z d dlmZ d dlZd dlmZ d dlmc mZ ddl	m
Z ddlmZ ddlmZmZ ddlmZ ddlmZ dd	lmZmZmZ d
dlmZ d
dlmZ d
dlmZ d
dlm Z  d
dl!m"Z"m#Z#m$Z$ ddl%m&Z&  ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z' ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z( G d„ de ¦  «        Z) G d„ dej*        ¦  «        Z+ G d„ de¦  «        Z, G d„ d e¦  «        Z- G d!„ d"ej*        ¦  «        Z. G d#„ d$ej*        ¦  «        Z/ G d%„ d&ej*        ¦  «        Z0 G d'„ d(e#¦  «        Z1e G d)„ d*e$¦  «        ¦   «         Z2 ed+¬¦  «         G d,„ d-e"¦  «        ¦   «         Z3 ed.¬¦  «         G d/„ d0e2e&¦  «        ¦   «         Z4g d1¢Z5dS )2é    N)Ú	dataclassé   )Úinitialization)ÚACT2FN)ÚBaseModelOutputWithPastÚBaseModelOutputWithPooling)ÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleé   )Ú	AutoModel)ÚCLIPMLP)ÚLlamaMLP)ÚQwen2RMSNorm)ÚVoxtralModelÚVoxtralMultiModalProjectorÚVoxtralPreTrainedModelé   )ÚVibeVoiceGenerationMixinzDBase class for VibeVoice outputs, with hidden states and attentions.©Úcustom_introc                   ó2   — e Zd ZU dZdZej        dz  ed<   dS )Ú VibeVoiceBaseModelOutputWithPastzÉ
    audio_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, feature_size)`, *optional*):
        Extracted audio features that can be used for conditioning the language model.
    NÚaudio_features)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚtorchÚFloatTensorÚ__annotations__© ó    ú/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/vibevoice/modular_vibevoice.pyr   r   (   s6   € € € € € € ðð ð
 04€N�EÔ%¨Ñ,Ð3Ð3Ñ3Ð3Ð3r%   r   z(VibeVoice causal language model outputs.c                   ón   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	ej        dz  ed<   dS )ÚVibeVoiceCausalLMOutputWithPasta.  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    audio_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, feature_size)`, *optional*):
        Extracted audio features that can be used for conditioning the language model.
    NÚlossÚlogitsr   )
r   r   r   r    r)   r!   r"   r#   r*   r   r$   r%   r&   r(   r(   3   sf   € € € € € € ðð ð &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø/3€N�EÔ%¨Ñ,Ð3Ð3Ñ3Ð3Ð3r%   r(   c                   ó   — e Zd ZdS )ÚVibeVoiceRMSNormN©r   r   r   r$   r%   r&   r,   r,   D   ó   € € € € € Ø€Dr%   r,   c                   ó:   ‡ — e Zd Zˆ fd„Zed„ ¦   «         Zd„ Zˆ xZS )Ú)VibeVoiceDiffusionHeadSinusoidalEmbeddingc                 ó²   •— t          ¦   «                              ¦   «          || _        t          j        |                      |¦  «        d¬¦  «        | _        d S )NF)Ú
persistent)ÚsuperÚ__init__ÚconfigÚnnÚBufferÚcompute_default_freqÚfreq©Úselfr5   Ú	__class__s     €r&   r4   z2VibeVoiceDiffusionHeadSinusoidalEmbedding.__init__I   sH   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”I˜d×7Ò7¸Ñ?Ô?ÈEÐRÑRÔRˆŒ	ˆ	ˆ	r%   c                 ó¶   — | j         dz  }t          j        t          j        | j        ¦  «         t          j        |t          j        ¬¦  «        z  |z  ¦  «        S )Nr   )Údtype)Úfrequency_embedding_sizer!   ÚexpÚmathÚlogÚdiffusion_max_periodÚarangeÚfloat32)r5   Údims     r&   r8   z>VibeVoiceDiffusionHeadSinusoidalEmbedding.compute_default_freqN   sM   € àÔ-°Ñ2ˆÝŒy�$œ( 6Ô#>Ñ?Ô?Ð?Å%Ä,ÈsÕZ_ÔZgÐBhÑBhÔBhÑhÐknÑnÑoÔoÐor%   c                 ó  — |d d …d f                               ¦   «         | j        d                               |j        ¦  «        z  }t	          j        t	          j        |¦  «        t	          j        |¦  «        gd¬¦  «        }|S )Néÿÿÿÿ©rF   )Úfloatr9   ÚtoÚdevicer!   ÚcatÚcosÚsin)r;   Ú	timestepsr9   Ú	embeddings       r&   Úforwardz1VibeVoiceDiffusionHeadSinusoidalEmbedding.forwardS   sk   € Ø˜˜˜˜D˜Ô!×'Ò'Ñ)Ô)¨D¬I°d¬O×,>Ò,>¸yÔ?OÑ,PÔ,PÑPˆÝ”I�uœy¨™œµ´	¸$±´Ð@ÀbÐIÑIÔIˆ	ØÐr%   )r   r   r   r4   Ústaticmethodr8   rR   Ú__classcell__©r<   s   @r&   r0   r0   H   sj   ø€ € € € € ðSð Sð Sð Sð Sð
 ðpð pñ „\ðpðð ð ð ð ð ð r%   r0   c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚVibeVoiceDiffusionHeadMLPc                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        d¬¦  «        | _        t          |j                 | _	        t          j        |j        |j        d¬¦  «        | _
        d S ©NF©Úbias)r3   r4   r6   ÚLinearr?   Úhidden_sizeÚfc1r   Ú
hidden_actÚactivation_fnÚfc2r:   s     €r&   r4   z"VibeVoiceDiffusionHeadMLP.__init__Z   sj   ø€ Ý‰Œ×ÒÑÔÐÝ”9˜VÔ<¸fÔ>PÐW\Ð]Ñ]Ô]ˆŒÝ# FÔ$5Ô6ˆÔÝ”9˜VÔ/°Ô1CÈ%ÐPÑPÔPˆŒˆˆr%   ©r   r   r   r4   rT   rU   s   @r&   rW   rW   Y   sA   ø€ € € € € ðQð Qð Qð Qð Qð Qð Qð Qð Qr%   rW   c                   ó   — e Zd ZdS )ÚVibeVoiceMLPNr-   r$   r%   r&   rd   rd   a   r.   r%   rd   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )Ú"VibeVoiceDiffusionHeadAdaLayerNormc                 óH  •— t          ¦   «                              ¦   «          d| _        t          |¦  «        | _        t          |j        |j        ¬¦  «        | _        t          |j
                 | _        t          j        |j        |j        | j        z  d¬¦  «        | _        d S )Nr   ©ÚepsFrZ   )r3   r4   Ú
num_chunksrd   Úffnr,   r]   Úrms_norm_epsÚnormr   r_   Úact_fnr6   r\   Úlinearr:   s     €r&   r4   z+VibeVoiceDiffusionHeadAdaLayerNorm.__init__f   s‚   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ Ñ'Ô'ˆŒÝ$ VÔ%7¸VÔ=PÐQÑQÔQˆŒ	Ý˜VÔ.Ô/ˆŒÝ”i Ô 2°FÔ4FÈÌÑ4XÐ_dÐeÑeÔeˆŒˆˆr%   c                 ó  — |                       |                      |¦  «        ¦  «                             | j        d¬¦  «        \  }}}|                      |¦  «        d|z   z  |z   }|||                      |¦  «        z  z   }|S )NrH   rI   r   )ro   rn   Úchunkrj   rm   rk   )r;   Úhidden_statesÚ	conditionÚ	shift_ffnÚ	scale_ffnÚgate_ffnÚmodulated_hidden_statess          r&   rR   z*VibeVoiceDiffusionHeadAdaLayerNorm.forwardn   s€   € Ø)-¯ª°T·[²[ÀÑ5KÔ5KÑ)LÔ)L×)RÒ)RÐSWÔSbÐhjÐ)RÑ)kÔ)kÑ&ˆ	�9˜hØ"&§)¢)¨MÑ":Ô":¸aÀ)¹mÑ"LÈyÑ"XÐØ%¨°4·8²8Ð<SÑ3TÔ3TÑ(TÑTˆØÐr%   ©r   r   r   r4   rR   rT   rU   s   @r&   rf   rf   e   sL   ø€ € € € € ðfð fð fð fð fðð ð ð ð ð ð r%   rf   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )Ú VibeVoiceDiffusionHeadFinalLayerc                 ó:  •— t          ¦   «                              ¦   «          d| _        |j        | _        t	          j        |j        | j        |j        z  d¬¦  «        | _        t          |j	                 | _
        t	          j        |j        |d¬¦  «        | _        d S )Nr   FrZ   )r3   r4   rj   rl   r6   r\   r]   Úlinear_1r   r_   rn   Úlinear_2)r;   r5   Úoutput_sizer<   s      €r&   r4   z)VibeVoiceDiffusionHeadFinalLayer.__init__v   s   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"Ô/ˆÔÝœ	 &Ô"4°d´oÈÔHZÑ6ZÐafÐgÑgÔgˆŒÝ˜VÔ.Ô/ˆŒÝœ	 &Ô"4°kÈÐNÑNÔNˆŒˆˆr%   c                 ó   — |                       |                      |¦  «        ¦  «                             | j        d¬¦  «        \  }}t	          j        ||j        d         f| j        ¬¦  «        }|d|z   z  |z   }|                      |¦  «        }|S )NrH   rI   rh   r   )	r|   rn   rq   rj   ÚFÚrms_normÚshaperl   r}   )r;   rr   rs   ÚshiftÚscaleÚnormeds         r&   rR   z(VibeVoiceDiffusionHeadFinalLayer.forward~   s‡   € Ø—}’} T§[¢[°Ñ%;Ô%;Ñ<Ô<×BÒBÀ4Ä?ÐXZÐBÑ[Ô[‰ˆˆuÝ”˜M¨MÔ,?ÀÔ,CÐ+EÈ4ÔK\Ð]Ñ]Ô]ˆØ ! e¡)Ñ,¨uÑ4ˆØŸš mÑ4Ô4ˆØÐr%   rx   rU   s   @r&   rz   rz   u   sL   ø€ € € € € ðOð Oð Oð Oð Oðð ð ð ð ð ð r%   rz   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚVibeVoiceDiffusionHeadc                 óÖ  •‡— t          ¦   «                              ¦   «          t          j        ‰j        ‰j        d¬¦  «        | _        t          j        ‰j        ‰j        d¬¦  «        | _        t          ‰¦  «        | _	        t          ‰¦  «        | _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t!          ‰‰j        ¬¦  «        | _        d S )NFrZ   c                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r$   )rf   )Ú.0Ú_r5   s     €r&   ú
<listcomp>z3VibeVoiceDiffusionHead.__init__.<locals>.<listcomp>Ž   s"   ø€ ÐaÐaÐa¸AÕ/°Ñ7Ô7ÐaÐaÐar%   )r~   )r3   r4   r6   r\   Úlatent_sizer]   Únoisy_images_projÚ	cond_projr0   Útimestep_embeddingrW   Útimestep_projÚ
ModuleListÚrangeÚnum_hidden_layersÚlayersrz   Úfinal_layerr:   s    `€r&   r4   zVibeVoiceDiffusionHead.__init__‡   sÏ   øø€ Ý‰Œ×ÒÑÔÐÝ!#¤¨6Ô+=¸vÔ?QÐX]Ð!^Ñ!^Ô!^ˆÔÝœ 6Ô#5°vÔ7IÐPUÐVÑVÔVˆŒÝ"KÈFÑ"SÔ"SˆÔÝ6°vÑ>Ô>ˆÔÝ”mØaÐaÐaÐaÅÀvÔG_ÑA`ÔA`ÐaÑaÔañ
ô 
ˆŒõ <¸FÐPVÔPbÐcÑcÔcˆÔÐÐr%   c                 ó@  — |                       |¦  «        }|                      |¦  «                             |j        ¦  «        }|                      |¦  «        }|                      |¦  «        }||z   }| j        D ]} |||¦  «        }Œ|                      ||¦  «        }|S )z\
        Forward pass of the prediction head. Returns the predicted noise/velocity.
        )rŽ   r�   rK   r>   r‘   r�   r•   r–   )r;   Únoisy_imagesrP   rs   rr   Úembedded_timestepsÚlayers          r&   rR   zVibeVoiceDiffusionHead.forward’   s«   € ð ×.Ò.¨|Ñ<Ô<ˆØ!×4Ò4°YÑ?Ô?×BÒBÀ9Ä?ÑSÔSÐØ!×/Ò/Ð0BÑCÔCÐØ—N’N 9Ñ-Ô-ˆ	ØÐ 2Ñ2ˆ	Ø”[ð 	<ð 	<ˆEØ!˜E -°Ñ;Ô;ˆMˆMØ×(Ò(¨¸	ÑBÔBˆØÐr%   rx   rU   s   @r&   r‡   r‡   †   sL   ø€ € € € € ð	dð 	dð 	dð 	dð 	dðð ð ð ð ð ð r%   r‡   c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚVibeVoiceMultiModalProjectorc                 óÜ   •— t          ¦   «                              ¦   «          t          j        ||¦  «        | _        t          |d¬¦  «        | _        t          j        ||¦  «        | _        d S )Ng�íµ ÷Æ°>rh   )r3   r4   r6   r\   r|   r,   Úactr}   )r;   Ú	input_dimÚ
output_dimr<   s      €r&   r4   z%VibeVoiceMultiModalProjector.__init__¢   sW   ø€ Ý‰Œ×ÒÑÔÐÝœ	 )¨ZÑ8Ô8ˆŒÝ# J°DÐ9Ñ9Ô9ˆŒÝœ	 *¨jÑ9Ô9ˆŒˆˆr%   rb   rU   s   @r&   rœ   rœ   ¡   s8   ø€ € € € € ð:ð :ð :ð :ð :ð :ð :ð :ð :r%   rœ   c                   ó   — e Zd ZdgZd„ ZdS )ÚVibeVoicePreTrainedModelr‡   c                 ój  — t          j        | |¦  «         t          |d¦  «        rt          j        |j        d¦  «         t          |d¦  «        rt          j        |j        d¦  «         t          |t          ¦  «        r4t          j	        |j
        |                     |j        ¦  «        ¦  «         d S d S )NÚlatent_scaling_factorç      ð?Úlatent_bias_factorç        )r	   Ú_init_weightsÚhasattrÚinitÚ	constant_r¤   r¦   Ú
isinstancer0   Úcopy_r9   r8   r5   )r;   Úmodules     r&   r¨   z&VibeVoicePreTrainedModel._init_weights­   s¯   € ÝÔ% d¨FÑ3Ô3Ð3Ý�6Ð2Ñ3Ô3ð 	>ÝŒN˜6Ô7¸Ñ=Ô=Ð=Ý�6Ð/Ñ0Ô0ð 	;ÝŒN˜6Ô4°cÑ:Ô:Ð:Ý�fÕGÑHÔHð 	PÝŒJ�v”{ F×$?Ò$?ÀÄÑ$NÔ$NÑOÔOÐOÐOÐOð	Pð 	Pr%   N)r   r   r   Ú_no_split_modulesr¨   r$   r%   r&   r¢   r¢   ©   s1   € € € € € à1Ð2ÐðPð Pð Pð Pð Pr%   r¢   zw
    The VibeVoice model which consists of audio tokenizers and an LLM backbone, without a language modeling head.
    c                   ó  ‡ — e Zd Zˆ fd„Ze ed¬¦  «        dej        dej        de	e
         deez  fd„¦   «         ¦   «         Z	 	 	 	 dd
ej        dej        d	z  dej        d	z  dej        d	z  de	e
         deez  fd„Zˆ xZS )ÚVibeVoiceModelc                 ó0  •— t          ¦   «                              |¦  «         t          |j        j        |j        j        ¦  «        | _        t          j        |j	        ¦  «        | _
        t          |j	        j        |j        j        ¦  «        | _        t          |j        ¦  «        | _        t          j        t#          j        d¦  «        ¦  «        | _        t          j        t#          j        d¦  «        ¦  «        | _        |                      ¦   «          d S )Nr¥   r§   )r3   r4   rœ   Úaudio_configr]   Útext_configÚmulti_modal_projectorr   Úfrom_configÚsemantic_model_configÚsemantic_tokenizer_encoderÚsemantic_connectorr‡   Údiffusion_head_configÚdiffusion_headr6   Ú	Parameterr!   Útensorr¤   r¦   Ú	post_initr:   s     €r&   r4   zVibeVoiceModel.__init__½   sÜ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý%AØÔÔ+¨VÔ-?Ô-Kñ&
ô &
ˆÔ"õ +4Ô*?ÀÔ@\Ñ*]Ô*]ˆÔ'Ý">ØÔ(Ô4°fÔ6HÔ6Tñ#
ô #
ˆÔõ 5°VÔ5QÑRÔRˆÔÝ%'¤\µ%´,¸sÑ2CÔ2CÑ%DÔ%DˆÔ"Ý"$¤,­u¬|¸CÑ/@Ô/@Ñ"AÔ"AˆÔØ�ŠÑÔÐÐÐr%   zÁThis method is used to get the audio embeddings (that replace placeholder audio tokens in the input sequence) and the acoustic features (used as diffusion target) from the input audio waveform.r   Úinput_valuesÚpadding_maskÚkwargsÚreturnc                 ó
  — t          j        ¦   «         5  | j                             |d¬¦  «        j        }ddd¦  «         n# 1 swxY w Y   || j                             |j        ¦  «        z  }|| j                             |j        ¦  «        z  }t          j	        | 
                    d¬¦  «        | j        j        j        z  ¦  «                             t           j        ¦  «        }t          j        |                     ¦   «         |j        ¬¦  «        |dd…df         k     }|                      |¦  «        }t%          ||                     |j        ¦  «                 ||                     |j        ¦  «                 ¬¦  «        S )zü
        input_values (`torch.FloatTensor`):
            Float values of (normalized) audio waveform.
        padding_mask (`torch.Tensor` of shape `(batch_size, padded_audio_length)`):
            Padding mask to remove padded parts of audio.
        T)ÚsampleNrH   rI   )rL   )Úlast_hidden_stateÚpooler_output)r!   Úno_gradÚaudio_towerÚencodeÚlatentsr¦   rK   rL   r¤   ÚceilÚsumr5   r³   Ú
hop_lengthÚint64rD   Úmaxrµ   r   )r;   r¿   rÀ   rÁ   Úacoustic_featuresÚnum_audio_tokensrÆ   s          r&   Úget_audio_featuresz!VibeVoiceModel.get_audio_featuresË   s®  € õ& Œ]‰_Œ_ð 	[ð 	[Ø $Ô 0× 7Ò 7¸ÈTÐ 7Ñ RÔ RÔ ZÐð	[ð 	[ð 	[ñ 	[ô 	[ð 	[ð 	[ð 	[ð 	[ð 	[ð 	[øøøð 	[ð 	[ð 	[ð 	[à˜TÔ4×7Ò7Ð8IÔ8PÑQÔQÑQÐØ˜TÔ7×:Ò:Ð;LÔ;SÑTÔTÑTÐõ !œ: l×&6Ò&6¸2Ð&6Ñ&>Ô&>ÀÄÔAYÔAdÑ&dÑeÔe×hÒhÕinÔitÑuÔuÐÝ”|Ð$4×$8Ò$8Ñ$:Ô$:ÐCSÔCZÐ[Ñ[Ô[Ð^nÐopÐopÐopÐrvÐovÔ^wÒwˆà×2Ò2Ð3DÑEÔEˆÝ)Ø/°·²Ð@QÔ@XÑ0YÔ0YÔZØ'¨¯ª¸Ô8LÑ(MÔ(MÔNð
ñ 
ô 
ð 	
s   ”"AÁAÁ	ANÚ	input_idsÚinputs_embedsc                 óV  — |€ |                       ¦   «         |¦  «        }d}|�j|                      ||¦  «        }|j        }|j        }|                      |||¬¦  «        }	|                     |	|                     |j        ¦  «        ¦  «        } | j        dd|i|¤Ž}
t          dd|i|
¤ŽS )z—
        padding_mask (`torch.Tensor` of shape `(batch_size, padded_audio_length)`):
            Padding mask to remove padded parts of audio.
        N)rÔ   r   rÔ   r   r$   )
Úget_input_embeddingsrÒ   rÆ   rÅ   Úget_placeholder_maskÚmasked_scatterrK   rL   Úlanguage_modelr   )r;   rÓ   rÔ   r¿   rÀ   rÁ   r   Úaudio_outputsÚaudio_embedsÚspecial_audio_maskÚoutputss              r&   rR   zVibeVoiceModel.forwardí   sØ   € ð Ð Ø7˜D×5Ò5Ñ7Ô7¸	ÑBÔBˆMàˆØÐ#Ø ×3Ò3°LÀ,ÑOÔOˆMØ(Ô6ˆLØ*Ô<ˆNð "&×!:Ò!:Ø¨À|ð ";ñ "ô "Ðð *×8Ò8Ð9KÈ\Ï_Ê_Ð]jÔ]qÑMrÔMrÑsÔsˆMà%�$Ô%ÐLÐL°MÐLÀVÐLÐLˆÝ/ÐYÐY¸~ÐYÐQXÐYÐYÐYr%   )NNNN)r   r   r   r4   r   r   r!   r"   ÚTensorr
   r   Útupler   rÒ   Ú
LongTensorÚ
BoolTensorr   rR   rT   rU   s   @r&   r±   r±   ·   sK  ø€ € € € € ðð ð ð ð ð Ø€^ðnðñ ô ð
àÔ'ð
ð ”lð
ð Ð+Ô,ð	
ð
 
Ð+Ñ	+ð
ð 
ð 
ñ	ô ñ Ôð

ð> '+Ø26Ø15Ø04ðZð ZàÔ#ðZð Ô(¨4Ñ/ðZð Ô'¨$Ñ.ð	Zð
 Ô&¨Ñ-ðZð Ð+Ô,ðZð 
Ð(Ñ	(ðZð Zð Zð Zð Zð Zð Zð Zr%   r±   zv
    The VibeVoice model, which consists of a language model, audio tokenizers, connectors, and a diffusion head.
    c                   ó`  ‡ — e Zd ZddiZddiZˆ fd„Zedej        de	j
        de	j
        d	ed
edede	j
        fd„¦   «         Zee	 	 	 	 	 	 	 	 	 	 dde	j        de	j
        dz  de	j        dz  deez  de	j
        dz  de	j        dz  de	j        dz  d	edz  d
ededeez  fd„¦   «         ¦   «         Zˆ xZS )Ú!VibeVoiceForConditionalGenerationzlm_head.weightz(model.language_model.embed_tokens.weightÚlm_headÚcolwise_gather_outputc                 óú   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        j        |j        j        d¬¦  «        | _	        |  
                    ¦   «          d S rY   )r3   r4   r±   Úmodelr6   r\   r´   r]   Ú
vocab_sizerä   r¾   r:   s     €r&   r4   z*VibeVoiceForConditionalGeneration.__init__  se   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý# FÑ+Ô+ˆŒ
Ý”y Ô!3Ô!?ÀÔASÔA^ÐejÐkÑkÔkˆŒØ�ŠÑÔÐÐÐr%   r»   r   Úcondition_featuresÚnoise_schedulerÚddpm_batch_multiplierÚnum_diffusion_stepsrÂ   c                 ó`  — |j         \  }}|                     |d¬¦  «        }|                     |d¬¦  «        }	t          j        ||z  |f|j        |j        ¬¦  «        }
t          j        t          j        |¦  «        ||z  d¬¦  «                             |j        ¦  «        }| 	                    ||
|¦  «        } | || 
                    |¦  «        |	¦  «        }|j                             |j        |j        ¬¦  «        }|j                             |j        |j        ¬¦  «        }||                              ¦   «                              d¦  «        }||                              ¦   «                              d¦  «        }||
z  ||z  z
                       |j        ¦  «        }t          j        j                             |                     ¦   «         |                     ¦   «         d¬¦  «        }|dk    r|dk    r|dk    r|||z  |z  z  }n"t          j        d	d
|j        |j        ¬¦  «        }|S )Nr   rI   )rL   r>   T)ÚreplacementrH   rÌ   )Ú	reductionr$   r§   )r>   rL   )r‚   Úrepeat_interleaver!   ÚrandnrL   r>   ÚmultinomialÚonesrK   Ú	add_noiseÚtype_asÚalpha_tÚsigma_tÚflattenÚ	unsqueezer6   Ú
functionalÚmse_lossrJ   Úfull)r»   r   ré   rê   rë   rì   Ú	audio_lenr�   Úaudio_features_repeatedÚcondition_features_repeatedÚnoiserP   Únoisy_audio_featuresÚmodel_outputrö   r÷   ÚtargetÚdiffusion_losss                     r&   Úcompute_diffusion_lossz8VibeVoiceForConditionalGeneration.compute_diffusion_loss  sG  € ð "0Ô!5Ñˆ	�;Ø"0×"BÒ"BÐCXÐ^_Ð"BÑ"`Ô"`ÐØ&8×&JÒ&JÐK`ÐfgÐ&JÑ&hÔ&hÐ#õ ”ØÐ.Ñ.°Ð<Ø!Ô(Ø Ô&ð
ñ 
ô 
ˆõ
 Ô%ÝŒJÐ*Ñ+Ô+ØÐ-Ñ-Øð
ñ 
ô 
÷ Š"ˆ^Ô"Ñ
#Ô
#ð	 	ð
  /×8Ò8Ð9PÐRWÐYbÑcÔcÐð &�~Ø  )×"3Ò"3°NÑ"CÔ"CÐE`ñ
ô 
ˆð
 "Ô)×,Ò,°NÔ4IÐQ_ÔQeÐ,ÑfÔfˆØ!Ô)×,Ò,°NÔ4IÐQ_ÔQeÐ,ÑfÔfˆØ˜)Ô$×,Ò,Ñ.Ô.×8Ò8¸Ñ<Ô<ˆØ˜)Ô$×,Ò,Ñ.Ô.×8Ò8¸Ñ<Ô<ˆØ˜E‘/ GÐ.EÑ$EÑE×IÒIÈ,ÔJ]Ñ^Ô^ˆåœÔ,×5Ò5°l×6HÒ6HÑ6JÔ6JÈFÏLÊLÉNÌNÐfkÐ5ÑlÔlˆØ˜Š?ˆ?Ð4°qÒ8Ð8¸YÈº]¸]Ø+¨{Ð=RÑ/RÐU^Ñ/^Ñ_ˆNˆNå"œZ¨¨C°~Ô7KÐTbÔTiÐjÑjÔjˆNØÐr%   Nr   é   é
   rÓ   rÔ   ÚlabelsÚlogits_to_keepr¿   rÀ   Úacoustic_loss_maskc                 óÈ  —  | j         d||||dœ|¤Ž}|j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }d}|� | j        d||| j        j        dœ|¤Ž}|�Ð|j	        �É| j        j
        dk    r¹|€|                      | j        ¦  «        }|j	        }||                     |j        ¦  «                                      |j        ¦  «        }|                      | j         j        ||||	|
¦  «        }d| j        j
        z
  |z  | j        j
        |                     |j        ¦  «        z  z   }t#          d||dœ|¤ŽS )aÒ  
        input_values (`torch.FloatTensor`, *optional*):
            Preprocessed audio waveform for voice cloning.
        padding_mask (`torch.BoolTensor`, *optional*):
            Masks indicating valid input frames.
        acoustic_loss_mask (`torch.BoolTensor`, *optional*):
            Mask to compute diffusion loss only on specific acoustic tokens.
        noise_scheduler (*optional*):
            Needed for training to compute noise targets for the diffusion loss. By default, uses the noise scheduler
            configuration specified in the model's `generation_config`.
        ddpm_batch_multiplier (`int`, *optional*, defaults to 4):
            For training, number of noise samples to generate per audio token for diffusion loss computation, which can
            help stabilize training.
        num_diffusion_steps (`int`, *optional*, defaults to 10):
            For training, the number of diffusion steps to use. Defaults to 10 if not provided.
        )rÓ   rÔ   r¿   rÀ   N)r*   r  rè   r§   r   )r)   r*   r$   )rç   rÅ   r¬   ÚintÚslicerä   Úloss_functionr5   rè   r   Údiffusion_loss_weightÚ_build_default_noise_schedulerÚgeneration_configrK   rL   r  r»   r(   )r;   rÓ   rÔ   r  r	  r¿   rÀ   r
  rê   rë   rì   rÁ   rÝ   rr   Úslice_indicesr*   r)   r   ré   r  s                       r&   rR   z)VibeVoiceForConditionalGeneration.forwardH  s¹  € ðB �$”*ð 
ØØ'Ø%Ø%ð	
ð 
ð
 ð
ð 
ˆð  Ô1ˆÝ8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%Ðp¨V¸FÈtÌ{ÔOeÐpÐpÐioÐpÐpˆDð
 Ð*ØÔ&Ð2Ø”Ô1°CÒ7Ð7àÐ&à"&×"EÒ"EÀdÔF\Ñ"]Ô"]�à$Ô3ˆNØ!.Ð/A×/DÒ/DÀ]ÔEYÑ/ZÔ/ZÔ![×!^Ò!^Ð_mÔ_tÑ!uÔ!uÐØ!×8Ò8Ø”
Ô)ØØ"ØØ%Ø#ñô ˆNð �D”KÔ5Ñ5Øñà”{Ô8¸>×;LÒ;LÈTÌ[Ñ;YÔ;YÑYñZˆDõ /ÐS°DÀÐSÐSÈ7ÐSÐSÐSr%   )
NNNr   NNNNr  r  )r   r   r   Ú_tied_weights_keysÚ_tp_planr4   rS   r6   ÚModuler!   r"   Úobjectr  r  r   r   rà   r  rá   rß   r(   rR   rT   rU   s   @r&   rã   rã     sÜ  ø€ € € € € ð +Ð,VÐWÐØÐ2Ð3€Hðð ð ð ð ð ð*Øœ	ð*àÔ)ð*ð "Ô-ð*ð  ð	*ð
  #ð*ð !ð*ð 
Ô	ð*ð *ð *ñ „\ð*ðX Øð '+Ø26Ø*.Ø&'Ø15Ø04Ø6:Ø)-Ø%&Ø#%ðHTð HTàÔ#ðHTð Ô(¨4Ñ/ðHTð Ô  4Ñ'ð	HTð
 ˜e™ðHTð Ô'¨$Ñ.ðHTð Ô&¨Ñ-ðHTð "Ô,¨tÑ3ðHTð   $™ðHTð  #ðHTð !ðHTð 
Ð0Ñ	0ðHTð HTð HTñ „^ñ ÔðHTð HTð HTð HTð HTr%   rã   )rã   r¢   r±   )6rA   Údataclassesr   r!   Útorch.nnr6   Útorch.nn.functionalrú   r€   Ú r   rª   Úactivationsr   Úmodeling_outputsr   r   Úmodeling_utilsr	   Úprocessing_utilsr
   Úutilsr   r   r   Úautor   Úclip.modeling_clipr   Úllama.modeling_llamar   Úqwen2.modeling_qwen2r   Úvoxtral.modeling_voxtralr   r   r   Úgeneration_vibevoicer   r   r(   r,   r  r0   rW   rd   rf   rz   r‡   rœ   r¢   r±   rã   Ú__all__r$   r%   r&   ú<module>r'     s  ðð €€€Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø SÐ SÐ SÐ SÐ SÐ SÐ SÐ SØ -Ð -Ð -Ð -Ð -Ð -Ø &Ð &Ð &Ð &Ð &Ð &Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ Ð Ð Ð Ð Ð Ø (Ð (Ð (Ð (Ð (Ð (Ø +Ð +Ð +Ð +Ð +Ð +Ø /Ð /Ð /Ð /Ð /Ð /ðð ð ð ð ð ð ð ð ð ð
 ;Ð :Ð :Ð :Ð :Ð :ð €ÐgÐhÑhÔhØ
ð4ð 4ð 4ð 4ð 4Ð'>ñ 4ô 4ñ „ñ iÔhð4ð €ÐKÐLÑLÔLØ
ð4ð 4ð 4ð 4ð 4Ð&=ñ 4ô 4ñ „ñ MÔLð4ð	ð 	ð 	ð 	ð 	�|ñ 	ô 	ð 	ðð ð ð ð °´	ñ ô ð ð"Qð Qð Qð Qð Q ñ Qô Qð Qð	ð 	ð 	ð 	ð 	�8ñ 	ô 	ð 	ðð ð ð ð ¨¬ñ ô ð ð ð ð ð ð  r¤yñ ô ð ð"ð ð ð ð ˜RœYñ ô ð ð6:ð :ð :ð :ð :Ð#=ñ :ô :ð :ð ð
Pð 
Pð 
Pð 
Pð 
PÐ5ñ 
Pô 
Pñ „ð
Pð €ððñ ô ð
MZð MZð MZð MZð MZ�\ñ MZô MZñô ð
MZð` €ððñ ô ð
ATð ATð ATð ATð ATÐ(@ÐBZñ ATô ATñô ð
ATðHð ð €€€r%   