§
    9²¸j¹ ã                   óR  — d dl Z d dlZd dlZd dlZd dlmZ d dlmc mZ	 d dlm
Z
 ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZmZmZmZmZmZ ddlm Z m!Z! ddl"m#Z# ddl$m%Z%m&Z&m'Z'm(Z( ddl)m*Z*m+Z+ ddl,m-Z-m.Z. ddl/m0Z0  e(j1        e2¦  «        Z3 G d„ dej4        ¦  «        Z5 G d„ dej4        ¦  «        Z6 G d„ dej4        ¦  «        Z7 G d„ dej4        ¦  «        Z8 G d„ dej4        ¦  «        Z9 G d„ de¦  «        Z: G d„ de¦  «        Z; G d „ d!ej4        ¦  «        Z< G d"„ d#ej4        ¦  «        Z= G d$„ d%ej4        ¦  «        Z>e& G d&„ d'e ¦  «        ¦   «         Z? G d(„ d)e¦  «        Z@ G d*„ d+e¦  «        ZA G d,„ d-e¦  «        ZB G d.„ d/ej4        ¦  «        ZC G d0„ d1ej4        ¦  «        ZD G d2„ d3ej4        ¦  «        ZE	 	 dNd4eFeGeGf         d5eHd6eGd7ejI        dz  d8eGd9ejJ        fd:„ZKeZLe& G d;„ d<e?¦  «        ¦   «         ZM e&d=¬>¦  «         G d?„ d@e?¦  «        ¦   «         ZN e&dA¬>¦  «         G dB„ dCe?¦  «        ¦   «         ZOe& G dD„ dEe?¦  «        ¦   «         ZP G dF„ dGej4        ¦  «        ZQ G dH„ dIej4        ¦  «        ZR e&dJ¬>¦  «         G dK„ dLe?¦  «        ¦   «         ZSg dM¢ZTdS )Oé    N)ÚCrossEntropyLossé   )Úinitialization)ÚACT2FN)Úfilter_output_hidden_states)Úis_deepspeed_zero3_enabled)Úis_fsdp_managed_module)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚCausalLMOutputÚSequenceClassifierOutputÚTokenClassifierOutputÚWav2Vec2BaseModelOutputÚXVectorOutput)ÚPreTrainedModelÚ*get_torch_context_manager_or_global_device)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚis_peft_availableÚlogging)Úcan_return_tupleÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚWavLMConfigc                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚWavLMSamePadLayerc                 ól   •— t          ¦   «                              ¦   «          |dz  dk    rdnd| _        d S ©Né   r   r   )ÚsuperÚ__init__Únum_pad_remove)ÚselfÚnum_conv_pos_embeddingsÚ	__class__s     €úx/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/wavlm/modeling_wavlm.pyr$   zWavLMSamePadLayer.__init__*   s:   ø€ Ý‰Œ×ÒÑÔÐØ#:¸QÑ#>À!Ò#CÐ#C˜a˜aÈˆÔÐÐó    c                 óJ   — | j         dk    r|d d …d d …d | j          …f         }|S ©Nr   )r%   ©r&   Úhidden_statess     r)   ÚforwardzWavLMSamePadLayer.forward.   s;   € ØÔ Ò"Ð"Ø)¨!¨!¨!¨Q¨Q¨QÐ0F°4Ô3FÐ2FÐ0FÐ*FÔGˆMØÐr*   ©Ú__name__Ú
__module__Ú__qualname__r$   r/   Ú__classcell__©r(   s   @r)   r   r   )   sL   ø€ € € € € ðKð Kð Kð Kð Kðð ð ð ð ð ð r*   r   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚWavLMPositionalConvEmbeddingc                 óÊ  •— t          ¦   «                              ¦   «          t          j        |j        |j        |j        |j        dz  |j        ¬¦  «        | _        t          j        j	        }t          t          j        j        d¦  «        rt          j        j        j	        }t          ¦   «         rëdd l}|j                             | j        j        d¬¦  «        5   || j        dd¬¦  «        | _        d d d ¦  «         n# 1 swxY w Y   t          | j        d¦  «        r-| j        j        j        j        }| j        j        j        j        }n| j        j        }| j        j        }|j                             | |¦  «         |j                             | |¦  «         n || j        dd¬¦  «        | _        t-          |j        ¦  «        | _        t0          |j                 | _        d S )	Nr"   )Úkernel_sizeÚpaddingÚgroupsÚweight_normr   )Úmodifier_rankÚweight)ÚnameÚdimÚparametrizations)r#   r$   ÚnnÚConv1dÚhidden_sizer'   Únum_conv_pos_embedding_groupsÚconvÚutilsr<   ÚhasattrrA   r   Ú	deepspeedÚzeroÚGatheredParametersr>   Ú	original0Ú	original1Úweight_gÚweight_vÚregister_external_parameterr   r:   r   Úfeat_extract_activationÚ
activation)r&   Úconfigr<   rI   rN   rO   r(   s         €r)   r$   z%WavLMPositionalConvEmbedding.__init__5   sþ  ø€ Ý‰Œ×ÒÑÔÐÝ”IØÔØÔØÔ6ØÔ2°aÑ7ØÔ7ð
ñ 
ô 
ˆŒ	õ ”hÔ*ˆÝ•2”8Ô,¨mÑ<Ô<ð 	@Ýœ(Ô3Ô?ˆKå%Ñ'Ô'ð 	EØÐÐÐà”×2Ò2°4´9Ô3CÐSTÐ2ÑUÔUð Ið IØ'˜K¨¬	¸ÀaÐHÑHÔH�”	ðIð Ið Iñ Iô Ið Ið Ið Ið Ið Ið Iøøøð Ið Ið Ið Iå�t”yÐ"4Ñ5Ô5ð .Øœ9Ô5Ô<ÔF�Øœ9Ô5Ô<ÔF��àœ9Ô-�Øœ9Ô-�ØŒN×6Ò6°t¸XÑFÔFÐFØŒN×6Ò6°t¸XÑFÔFÐFÐFà#˜ D¤I°HÀ!ÐDÑDÔDˆDŒIå(¨Ô)GÑHÔHˆŒÝ  Ô!?Ô@ˆŒˆˆs   ÃC?Ã?DÄDc                 óÜ   — |                      dd¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      dd¦  «        }|S ©Nr   r"   )Ú	transposerF   r:   rR   r-   s     r)   r/   z$WavLMPositionalConvEmbedding.forwardV   se   € Ø%×/Ò/°°1Ñ5Ô5ˆàŸ	š	 -Ñ0Ô0ˆØŸš ]Ñ3Ô3ˆØŸš¨Ñ6Ô6ˆà%×/Ò/°°1Ñ5Ô5ˆØÐr*   r0   r5   s   @r)   r7   r7   4   sM   ø€ € € € € ðAð Að Að Að AðBð ð ð ð ð ð r*   r7   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚWavLMFeatureProjectionc                 ó.  •— t          ¦   «                              ¦   «          t          j        |j        d         |j        ¬¦  «        | _        t          j        |j        d         |j        ¦  «        | _	        t          j
        |j        ¦  «        | _        d S )Néÿÿÿÿ©Úeps)r#   r$   rB   Ú	LayerNormÚconv_dimÚlayer_norm_epsÚ
layer_normÚLinearrD   Ú
projectionÚDropoutÚfeat_proj_dropoutÚdropout©r&   rS   r(   s     €r)   r$   zWavLMFeatureProjection.__init__b   sn   ø€ Ý‰Œ×ÒÑÔÐÝœ, v¤°rÔ':ÀÔ@UÐVÑVÔVˆŒÝœ) F¤O°BÔ$7¸Ô9KÑLÔLˆŒÝ”z &Ô":Ñ;Ô;ˆŒˆˆr*   c                 óˆ   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }||fS ©N)r`   rb   re   )r&   r.   Únorm_hidden_statess      r)   r/   zWavLMFeatureProjection.forwardh   sC   € à!Ÿ_š_¨]Ñ;Ô;ÐØŸšÐ(:Ñ;Ô;ˆØŸš ]Ñ3Ô3ˆØÐ0Ð0Ð0r*   r0   r5   s   @r)   rX   rX   a   sG   ø€ € € € € ð<ð <ð <ð <ð <ð1ð 1ð 1ð 1ð 1ð 1ð 1r*   rX   c                   óž  ‡ — e Zd ZdZ	 	 	 	 ddededeez  d	ed
edefˆ fd„Z	 	 	 ddej	        dej	        dz  dej	        dz  dede
e         deej	        ej	        dz  eej	                 dz  f         fd„Zdej        dej        ej        z  dej        deej        ej        f         fd„Zdededej        fd„Zdej        dej        fd„Zˆ xZS )ÚWavLMAttentionz=Multi-headed attention from 'Attention Is All You Need' paperç        é@  é   TÚ	embed_dimÚ	num_headsre   Únum_bucketsÚmax_distanceÚhas_relative_position_biasc                 óà  •— t          ¦   «                              ¦   «          || _        || _        || _        ||z  | _        | j        |z  | j        k    rt          d| j        › d|› d�¦  «        ‚| j        dz  | _        t          j	        ||¦  «        | _
        t          j	        ||¦  «        | _        t          j	        ||¦  «        | _        t          j	        ||¦  «        | _        || _        || _        t          j        t#          j        d| j        dd¦  «        ¦  «        | _        t          j	        | j        d¦  «        | _        |r&t          j        | j        | j        ¦  «        | _        d S d S )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).g      à¿r   é   )r#   r$   ro   rp   re   Úhead_dimÚ
ValueErrorÚscalingrB   ra   Úk_projÚv_projÚq_projÚout_projrq   rr   Ú	ParameterÚtorchÚonesÚgru_rel_pos_constÚgru_rel_pos_linearÚ	EmbeddingÚrel_attn_embed)r&   ro   rp   re   rq   rr   rs   r(   s          €r)   r$   zWavLMAttention.__init__s   sa  ø€ õ 	‰Œ×ÒÑÔÐØ"ˆŒØ"ˆŒØˆŒØ! YÑ.ˆŒàŒM˜IÑ%¨$¬.Ò8Ð8Ýð3ÈdÌnð 3ð 3Ø%.ð3ð 3ð 3ñô ð ð ”} dÑ*ˆŒå”i 	¨9Ñ5Ô5ˆŒÝ”i 	¨9Ñ5Ô5ˆŒÝ”i 	¨9Ñ5Ô5ˆŒÝœ	 )¨YÑ7Ô7ˆŒà&ˆÔØ(ˆÔå!#¤­e¬j¸¸D¼NÈAÈqÑ.QÔ.QÑ!RÔ!RˆÔÝ"$¤)¨D¬M¸1Ñ"=Ô"=ˆÔà%ð 	QÝ"$¤,¨tÔ/?ÀÄÑ"PÔ"PˆDÔÐÐð	Qð 	Qr*   Nr   r.   Úattention_maskÚposition_biasÚindexÚkwargsÚreturnc                 óD  — |                      ¦   «         \  }}}|€^|                      ||¦  «        }|                     d¦  «                             |ddd¦  «                             || j        z  ||¦  «        }|                     |j        dd…         | j        dfz   ¦  «        }	|	                     dddd¦  «        }	|                      |	¦  «        }
|
                     |	j        dd…         dz   ¦  «         	                    d¦  «        }
t          j        |
¦  «                             dd¬¦  «        \  }}||| j        z  d	z
  z  d
z   }|                     || j        z  dd¦  «        |z  }|                     d||f¦  «        }|                      |||¦  «        \  }}|||fS )z'Attention layer with relative attentionNr   r   rZ   r"   r   )r"   é   ©r@   ç      ð?g       @)ÚsizeÚcompute_biasÚ	unsqueezeÚrepeatÚviewrp   ÚshapeÚpermuter�   Úsumr~   ÚsigmoidÚchunkr€   Útorch_multi_head_self_attention)r&   r.   r„   r…   r†   r‡   ÚbszÚtgt_lenÚ_Úgated_hidden_statesÚrelative_position_projÚgate_aÚgate_bÚgate_outputÚgated_position_biasÚattn_outputÚattn_weightss                    r)   r/   zWavLMAttention.forward—   sÍ  € ð (×,Ò,Ñ.Ô.‰ˆˆW�að Ð Ø ×-Ò-¨g°wÑ?Ô?ˆMà×'Ò'¨Ñ*Ô*×1Ò1°#°q¸!¸QÑ?Ô?×DÒDÀSÈ4Ì>ÑEYÐ[bÐdkÑlÔlð ð ,×0Ò0°Ô1DÀSÀbÀSÔ1IÈTÌ^Ð]_ÐL`Ñ1`ÑaÔaÐØ1×9Ò9¸!¸QÀÀ1ÑEÔEÐð "&×!8Ò!8Ð9LÑ!MÔ!MÐØ!7×!<Ò!<Ð=PÔ=VÐWZÐXZÐWZÔ=[Ð^dÑ=dÑ!eÔ!e×!iÒ!iÐjlÑ!mÔ!mÐõ œÐ'=Ñ>Ô>×DÒDÀQÈBÐDÑOÔO‰ˆ�Ø ¨Ô)?Ñ ?À#Ñ EÑFÈÑLˆð *×.Ò.¨s°T´^Ñ/CÀRÈÑKÔKÈmÑ[ÐØ1×6Ò6¸¸GÀWÐ7MÑNÔNÐà$(×$HÒ$HØ˜>Ð+>ñ%
ô %
Ñ!ˆ�\ð ˜L¨-Ð7Ð7r*   r    c                 ó¦  — |                      dd¦  «        x}x}}|�|                     d¦  «        nd}dx}}	d}
t          j        |||| j        | j        t          j        dg¦  «        t          j        | j	        j
        | j        j
        | j        j
        f¦  «        ||	|
| j        | j        j        | j        j
        | j        |d|d| j	        j        | j        j        | j        j        ¬¦  «        \  }}|                      dd¦  «        }|�E|dd…df                              |j        dd…         | j        fz   |j        dd…         z   ¦  «        }||fS )zCsimple wrapper around torch's multi_head_attention_forward functionr   r   NFT)Úneed_weightsÚ	attn_maskÚuse_separate_proj_weightÚq_proj_weightÚk_proj_weightÚv_proj_weight)rV   ÚneÚFÚmulti_head_attention_forwardro   rp   r~   ÚemptyÚcatr{   Úbiasry   rz   re   r|   r>   ÚtrainingÚbroadcast_tor’   )r&   r.   r„   r    ÚqueryÚkeyÚvalueÚkey_padding_maskÚbias_kÚbias_vÚadd_zero_attnr¡   r¢   s                r)   r—   z.WavLMAttention.torch_multi_head_self_attentionÀ   sx  € ð ,×5Ò5°a¸Ñ;Ô;Ð;ˆÐ;��eØ3AÐ3M˜>×,Ò,¨QÑ/Ô/Ð/ÐSWÐð Ðˆ�Øˆõ %&Ô$BØØØØŒNØŒNÝŒK˜˜ÑÔÝŒI�t”{Ô'¨¬Ô)9¸4¼;Ô;KÐLÑMÔMØØØØŒLØŒMÔ ØŒMÔØŒMØØØ)Ø%)Øœ+Ô,Øœ+Ô,Øœ+Ô,ð+%
ñ %
ô %
Ñ!ˆ�\ð2 "×+Ò+¨A¨qÑ1Ô1ˆàÐ#ð (¨¨¨¨4¨Ô0×=Ò=ØÔ" 2 A 2Ô&¨$¬.Ð):Ñ:¸\Ô=OÐPQÐPRÐPRÔ=SÑSñô ˆLð ˜LÐ(Ð(r*   Úquery_lengthÚ
key_lengthc                 ó‚  — t          j        |t           j        ¬¦  «        d d …d f         }t          j        |t           j        ¬¦  «        d d d …f         }||z
  }|                      |¦  «        }|                     | j        j        j        ¦  «        }|                      |¦  «        }|                     g d¢¦  «        }|S )N©Údtype)r"   r   r   )	r~   ÚarangeÚlongÚ_relative_positions_bucketÚtorƒ   r>   Údevicer“   )r&   r¹   rº   Úcontext_positionÚmemory_positionÚrelative_positionÚrelative_position_bucketÚvaluess           r)   rŽ   zWavLMAttention.compute_biasö   sµ   € Ý œ<¨½E¼JÐGÑGÔGÈÈÈÈ4ÈÔPÐÝœ, z½¼ÐDÑDÔDÀTÈ1È1È1ÀWÔMˆØ+Ð.>Ñ>ÐØ#'×#BÒ#BÐCTÑ#UÔ#UÐ Ø#;×#>Ò#>¸tÔ?RÔ?YÔ?`Ñ#aÔ#aÐ Ø×$Ò$Ð%=Ñ>Ô>ˆØ—’ 	 	 	Ñ*Ô*ˆØˆr*   Úrelative_positionsc                 ó  — | j         dz  }|dk                         t          j        ¦  «        |z  }t          j        |¦  «        }|dz  }||k     }t          j        |                     ¦   «         |z  ¦  «        }|t          j        | j        |z  ¦  «        z  }|||z
  z  }||z                        t          j        ¦  «        }t          j	        |t          j
        ||dz
  ¦  «        ¦  «        }|t          j        |||¦  «        z  }|S r!   )rq   rÁ   r~   r¿   ÚabsÚlogÚfloatÚmathrr   ÚminÚ	full_likeÚwhere)r&   rÈ   rq   Úrelative_bucketsÚ	max_exactÚis_smallÚrelative_positions_if_largeÚrelative_position_if_larges           r)   rÀ   z)WavLMAttention._relative_positions_bucket   s  € ØÔ&¨!Ñ+ˆà.°Ò2×6Ò6µu´zÑBÔBÀ[ÑPÐÝ"œYÐ'9Ñ:Ô:Ðà 1Ñ$ˆ	Ø%¨	Ò1ˆå&+¤iÐ0B×0HÒ0HÑ0JÔ0JÈYÑ0VÑ&WÔ&WÐ#Ø&AÅDÄHÈTÔM^ÐajÑMjÑDkÔDkÑ&kÐ#Ø&AÀ[ÐS\ÑE\Ñ&]Ð#Ø&/Ð2MÑ&M×%QÒ%QÕRWÔR\Ñ%]Ô%]Ð"Ý%*¤YØ&­¬Ð8RÐT_ÐbcÑTcÑ(dÔ(dñ&
ô &
Ð"ð 	�EœK¨Ð2DÐF`ÑaÔaÑaÐØÐr*   )rl   rm   rn   T©NNr   )r1   r2   r3   Ú__doc__ÚintrÌ   Úboolr$   r~   ÚTensorr   r   Útupler/   ÚFloatTensorÚ
LongTensorÚ
BoolTensorr—   rŽ   rÀ   r4   r5   s   @r)   rk   rk   p   sé  ø€ € € € € ØGÐGð  #ØØØ+/ð"Qð "Qàð"Qð ð"Qð ˜‘ð	"Qð
 ð"Qð ð"Qð %)ð"Qð "Qð "Qð "Qð "Qð "QðN /3Ø-1Øð'8ð '8à”|ð'8ð œ tÑ+ð'8ð ”| dÑ*ð	'8ð
 ð'8ð Ð+Ô,ð'8ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	Mð'8ð '8ð '8ð '8ðR4)àÔ(ð4)ð Ô(¨5Ô+;Ñ;ð4)ð #Ô.ð	4)ð
 
ˆuÔ  %Ô"3Ð3Ô	4ð4)ð 4)ð 4)ð 4)ðl¨ð ¸#ð À%ÔBSð ð ð ð ð ¸UÔ=Nð  ÐSXÔSdð  ð  ð  ð  ð  ð  ð  ð  r*   rk   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚWavLMFeedForwardc                 óÌ  •— t          ¦   «                              ¦   «          t          j        |j        ¦  «        | _        t          j        |j        |j        ¦  «        | _	        t          |j        t          ¦  «        rt          |j                 | _        n|j        | _        t          j        |j        |j        ¦  «        | _        t          j        |j        ¦  «        | _        d S rh   )r#   r$   rB   rc   Úactivation_dropoutÚintermediate_dropoutra   rD   Úintermediate_sizeÚintermediate_denseÚ
isinstanceÚ
hidden_actÚstrr   Úintermediate_act_fnÚoutput_denseÚhidden_dropoutÚoutput_dropoutrf   s     €r)   r$   zWavLMFeedForward.__init__  s°   ø€ Ý‰Œ×ÒÑÔÐÝ$&¤J¨vÔ/HÑ$IÔ$IˆÔ!å"$¤)¨FÔ,>ÀÔ@XÑ"YÔ"YˆÔÝ�fÔ'­Ñ-Ô-ð 	9Ý'-¨fÔ.?Ô'@ˆDÔ$Ð$à'-Ô'8ˆDÔ$åœI fÔ&>ÀÔ@RÑSÔSˆÔÝ œj¨Ô)>Ñ?Ô?ˆÔÐÐr*   c                 óØ   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S rh   )rå   ré   rã   rê   rì   r-   s     r)   r/   zWavLMFeedForward.forward#  sg   € Ø×/Ò/°Ñ>Ô>ˆØ×0Ò0°Ñ?Ô?ˆØ×1Ò1°-Ñ@Ô@ˆà×)Ò)¨-Ñ8Ô8ˆØ×+Ò+¨MÑ:Ô:ˆØÐr*   r0   r5   s   @r)   rà   rà     sL   ø€ € € € € ð@ð @ð @ð @ð @ðð ð ð ð ð ð r*   rà   c                   ó„   ‡ — e Zd Zddedefˆ fd„Z	 	 	 ddej        dej        dz  d	ej        dz  d
ede	e
         f
d„Zˆ xZS )ÚWavLMEncoderLayerTrS   rs   c                 ó¦  •— t          ¦   «                              ¦   «          t          |j        |j        |j        |j        |j        |¬¦  «        | _        t          j
        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t!          |¦  «        | _        t          j        |j        |j        ¬¦  «        | _        d S ©N)ro   rp   re   rq   rr   rs   r[   ©r#   r$   rk   rD   Únum_attention_headsÚattention_dropoutrq   Úmax_bucket_distanceÚ	attentionrB   rc   rë   re   r]   r_   r`   rà   Úfeed_forwardÚfinal_layer_norm©r&   rS   rs   r(   s      €r)   r$   zWavLMEncoderLayer.__init__.  óµ   ø€ Ý‰Œ×ÒÑÔÐÝ'ØÔ(ØÔ0ØÔ,ØÔ*ØÔ3Ø'Að
ñ 
ô 
ˆŒõ ”z &Ô"7Ñ8Ô8ˆŒÝœ, vÔ'9¸vÔ?TÐUÑUÔUˆŒÝ,¨VÑ4Ô4ˆÔÝ "¤¨VÔ-?ÀVÔEZÐ [Ñ [Ô [ˆÔÐÐr*   Nr   r.   r„   r…   r†   r‡   c                 óò   — |} | j         |f|||dœ|¤Ž\  }}}|                      |¦  «        }||z   }|                      |¦  «        }||                      |¦  «        z   }|                      |¦  «        }||fS )N©r„   r…   r†   )rö   re   r`   r÷   rø   )r&   r.   r„   r…   r†   r‡   Úattn_residualrš   s           r)   r/   zWavLMEncoderLayer.forward=  s§   € ð &ˆØ*8¨$¬.Øð+
à)Ø'Øð	+
ð +
ð
 ð+
ð +
Ñ'ˆ�q˜-ð Ÿš ]Ñ3Ô3ˆØ%¨Ñ5ˆàŸš¨Ñ6Ô6ˆà%¨×(9Ò(9¸-Ñ(HÔ(HÑHˆØ×-Ò-¨mÑ<Ô<ˆà˜mÐ+Ð+r*   ©TrÖ   )r1   r2   r3   r   rÙ   r$   r~   rÚ   rØ   r   r   r/   r4   r5   s   @r)   rï   rï   -  sÃ   ø€ € € € € ð\ð \˜{ð \Èð \ð \ð \ð \ð \ð \ð$ /3Ø-1Øð,ð ,à”|ð,ð œ tÑ+ð,ð ”| dÑ*ð	,ð
 ð,ð Ð+Ô,ð,ð ,ð ,ð ,ð ,ð ,ð ,ð ,r*   rï   c            
       ó~   ‡ — e Zd Zddedefˆ fd„Z	 	 ddej        dej        dz  dej        dz  d	ee	         fd
„Z
ˆ xZS )Ú WavLMEncoderLayerStableLayerNormTrS   rs   c                 ó¦  •— t          ¦   «                              ¦   «          t          |j        |j        |j        |j        |j        |¬¦  «        | _        t          j
        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t!          |¦  «        | _        t          j        |j        |j        ¬¦  «        | _        d S rñ   rò   rù   s      €r)   r$   z)WavLMEncoderLayerStableLayerNorm.__init__Y  rú   r*   Nr.   r„   r…   r‡   c                 óì   — |}|                       |¦  «        } | j        |f||dœ|¤Ž\  }}}|                      |¦  «        }||z   }||                      |                      |¦  «        ¦  «        z   }||fS )N©r„   r…   )r`   rö   re   r÷   rø   )r&   r.   r„   r…   r‡   rý   rš   s          r)   r/   z(WavLMEncoderLayerStableLayerNorm.forwardh  s    € ð &ˆØŸš¨Ñ6Ô6ˆØ*8¨$¬.Øð+
à)Ø'ð+
ð +
ð ð	+
ð +
Ñ'ˆ�q˜-ð Ÿš ]Ñ3Ô3ˆØ%¨Ñ5ˆØ%¨×(9Ò(9¸$×:OÒ:OÐP]Ñ:^Ô:^Ñ(_Ô(_Ñ_ˆà˜mÐ+Ð+r*   rþ   ©NN)r1   r2   r3   r   rÙ   r$   r~   rÚ   r   r   r/   r4   r5   s   @r)   r   r   X  s¶   ø€ € € € € ð\ð \˜{ð \Èð \ð \ð \ð \ð \ð \ð$ /3Ø-1ð	,ð ,à”|ð,ð œ tÑ+ð,ð ”| dÑ*ð	,ð
 Ð+Ô,ð,ð ,ð ,ð ,ð ,ð ,ð ,ð ,r*   r   c                   óN   ‡ — e Zd Zˆ fd„Z	 ddej        dz  dee         fd„Zˆ xZ	S )ÚWavLMEncoderc                 ó‚  •‡— t          ¦   «                              ¦   «          ‰| _        t          ‰¦  «        | _        t          j        ‰j        ‰j        ¬¦  «        | _	        t          j
        ‰j        ¦  «        | _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        d| _        d S )Nr[   c                 ó:   •— g | ]}t          ‰|d k    ¬¦  «        ‘ŒS ©r   )rs   )rï   ©Ú.0ÚirS   s     €r)   ú
<listcomp>z)WavLMEncoder.__init__.<locals>.<listcomp>†  s,   ø€ ÐuÐuÐuÐPQÕ˜vÀ1ÈÂ6ÐKÑKÔKÐuÐuÐur*   F©r#   r$   rS   r7   Úpos_conv_embedrB   r]   rD   r_   r`   rc   rë   re   Ú
ModuleListÚrangeÚnum_hidden_layersÚlayersÚgradient_checkpointingrf   s    `€r)   r$   zWavLMEncoder.__init__  s¨   øø€ Ý‰Œ×ÒÑÔÐØˆŒÝ:¸6ÑBÔBˆÔÝœ, vÔ'9¸vÔ?TÐUÑUÔUˆŒÝ”z &Ô"7Ñ8Ô8ˆŒÝ”mØuÐuÐuÐuÕUZÐ[aÔ[sÑUtÔUtÐuÑuÔuñ
ô 
ˆŒð ',ˆÔ#Ð#Ð#r*   Nr„   r‡   c                 ó   — |�;|                      d¦  «                             dd|j        d         ¦  «        }d|| <   |                      |¦  «        }||z   }|                      |¦  «        }|                      |¦  «        }t          ¦   «         pt          | ¦  «        }d }t          | j	        ¦  «        D ]J\  }}	t          j        g ¦  «        }
| j        o|dk    o|
| j        j        k     }|r|r |	|f|||dœ|¤Ž\  }}ŒKt          |¬¦  «        S )NrZ   r   r"   r   rü   ©Úlast_hidden_state)r�   r�   r’   r  r`   re   r   r	   Ú	enumerater  r~   Úrandr°   rS   Ú	layerdropr   ©r&   r.   r„   r‡   Úexpand_attention_maskÚposition_embeddingsÚsynced_gpusr…   r  ÚlayerÚdropout_probabilityÚskip_the_layers               r)   r/   zWavLMEncoder.forwardŠ  sL  € ð Ð%à$2×$<Ò$<¸RÑ$@Ô$@×$GÒ$GÈÈ1ÈmÔNaÐbcÔNdÑ$eÔ$eÐ!Ø45ˆMÐ0Ð0Ñ1à"×1Ò1°-Ñ@Ô@ÐØ%Ð(;Ñ;ˆØŸš¨Ñ6Ô6ˆØŸš ]Ñ3Ô3ˆå0Ñ2Ô2ÐRÕ6LÈTÑ6RÔ6RˆØˆå! $¤+Ñ.Ô.ð 	ð 	‰HˆAˆuå"'¤*¨R¡.¤.Ðà!œ]Ðf¨q°1ªuÐfÐ:MÐPTÔP[ÔPeÒ:eˆNØ!ð  [ð à/4¨uØ!ð0à#1Ø"/Øð	0ð 0ð
 ð0ð 0Ñ,�˜}øõ °Ð?Ñ?Ô?Ð?r*   rh   ©
r1   r2   r3   r$   r~   rÚ   r   r   r/   r4   r5   s   @r)   r  r  ~  s‚   ø€ € € € € ð	,ð 	,ð 	,ð 	,ð 	,ð /3ð"@ð "@ð œ tÑ+ð"@ð Ð+Ô,ð	"@ð "@ð "@ð "@ð "@ð "@ð "@ð "@r*   r  c                   óN   ‡ — e Zd Zˆ fd„Z	 ddej        dz  dee         fd„Zˆ xZ	S )ÚWavLMEncoderStableLayerNormc                 ó‚  •‡— t          ¦   «                              ¦   «          ‰| _        t          ‰¦  «        | _        t          j        ‰j        ‰j        ¬¦  «        | _	        t          j
        ‰j        ¦  «        | _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        d| _        d S )Nr[   c                 ó:   •— g | ]}t          ‰|d k    ¬¦  «        ‘ŒS r	  )r   r
  s     €r)   r  z8WavLMEncoderStableLayerNorm.__init__.<locals>.<listcomp>·  s=   ø€ ð ð ð àõ 1°ÐUVÐZ[ÒU[Ð]Ñ]Ô]ðð ð r*   Fr  rf   s    `€r)   r$   z$WavLMEncoderStableLayerNorm.__init__°  sµ   øø€ Ý‰Œ×ÒÑÔÐØˆŒÝ:¸6ÑBÔBˆÔÝœ, vÔ'9¸vÔ?TÐUÑUÔUˆŒÝ”z &Ô"7Ñ8Ô8ˆŒÝ”mðð ð ð å˜vÔ7Ñ8Ô8ðñ ô ñ
ô 
ˆŒð ',ˆÔ#Ð#Ð#r*   Nr„   r‡   c                 ó  — |�;|                      d¦  «                             dd|j        d         ¦  «        }d|| <   |                      |¦  «        }||z   }|                      |¦  «        }t          ¦   «         pt          | ¦  «        }d }t          | j        ¦  «        D ]I\  }}	t          j
        g ¦  «        }
| j        o|dk    o|
| j        j        k     }|r|r |	|f||dœ|¤Ž\  }}ŒJ|                      |¦  «        }t          |¬¦  «        S )NrZ   r   r"   r   r  r  )r�   r�   r’   r  re   r   r	   r  r  r~   r  r°   rS   r  r`   r   r  s               r)   r/   z#WavLMEncoderStableLayerNorm.forward¾  sK  € ð Ð%à$2×$<Ò$<¸RÑ$@Ô$@×$GÒ$GÈÈ1ÈmÔNaÐbcÔNdÑ$eÔ$eÐ!Ø45ˆMÐ0Ð0Ñ1à"×1Ò1°-Ñ@Ô@ÐØ%Ð(;Ñ;ˆØŸš ]Ñ3Ô3ˆå0Ñ2Ô2ÐRÕ6LÈTÑ6RÔ6RˆØˆå! $¤+Ñ.Ô.ð 	ð 	‰HˆAˆuå"'¤*¨R¡.¤.Ðà!œ]Ðf¨q°1ªuÐfÐ:MÐPTÔP[ÔPeÒ:eˆNØ!ð  [ð ð 05¨uØ!ð0à#1Ø"/ð0ð 0ð ð	0ð 0Ñ,�˜}øð Ÿš¨Ñ6Ô6ˆå°Ð?Ñ?Ô?Ð?r*   rh   r"  r5   s   @r)   r$  r$  ¯  s‚   ø€ € € € € ð,ð ,ð ,ð ,ð ,ð" /3ð#@ð #@ð œ tÑ+ð#@ð Ð+Ô,ð	#@ð #@ð #@ð #@ð #@ð #@ð #@ð #@r*   r$  c                   ó>   ‡ — e Zd ZdZˆ fd„Zed„ ¦   «         Zd„ Zˆ xZS )ÚWavLMGumbelVectorQuantizerz°
    Vector quantization using gumbel softmax. See [CATEGORICAL REPARAMETERIZATION WITH
    GUMBEL-SOFTMAX](https://huggingface.co/papers/1611.01144) for more information.
    c                 óÞ  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | j        z  dk    r t          d|j        › d| j        › d�¦  «        ‚t          j	        t          j        d| j        | j        z  |j        | j        z  ¦  «        ¦  «        | _        t          j        |j        d         | j        | j        z  ¦  «        | _        d| _        d S )Nr   z`config.codevector_dim z5 must be divisible by `config.num_codevector_groups` z for concatenation.r   rZ   r"   )r#   r$   Únum_codevector_groupsÚ
num_groupsÚnum_codevectors_per_groupÚnum_varsÚcodevector_dimrw   rB   r}   r~   rÜ   Úcodevectorsra   r^   Úweight_projÚtemperaturerf   s     €r)   r$   z#WavLMGumbelVectorQuantizer.__init__ê  sï   ø€ Ý‰Œ×ÒÑÔÐØ Ô6ˆŒØÔ8ˆŒàÔ  4¤?Ñ2°aÒ7Ð7Ýð%¨&Ô*?ð %ð %Ø6:´oð%ð %ð %ñô ð õ œ<ÝÔ˜a ¤°4´=Ñ!@À&ÔBWÐ[_Ô[jÑBjÑkÔkñ
ô 
ˆÔõ œ9 V¤_°RÔ%8¸$¼/ÈDÌMÑ:YÑZÔZˆÔð ˆÔÐÐr*   c                 óÎ   — |                       d¬¦  «        }t          j        t          j        t          j        ||¦  «        d¬¦  «         ¦  «                             ¦   «         }|S )Nr   r‹   rZ   )Úmeanr~   Úexpr”   Úxlogy)ÚprobsÚmarginal_probsÚ
perplexitys      r)   Ú_compute_perplexityz.WavLMGumbelVectorQuantizer._compute_perplexityÿ  sU   € àŸš¨˜Ñ*Ô*ˆÝ”Y¥¤	­%¬+°nÀnÑ*UÔ*UÐ[]Ð ^Ñ ^Ô ^Ð^Ñ_Ô_×cÒcÑeÔeˆ
ØÐr*   c                 óú  — |j         \  }}}|                      |¦  «        }|                     ||z  | j        z  d¦  «        }| j        r©t
          j                             |                     ¦   «         | j	        d¬¦  «        }| 
                    |¦  «        }t          j        |                     ||z  | j        d¦  «                             ¦   «         d¬¦  «        }|                      |¦  «        }n‚|                     d¬¦  «        } |j        |j         Ž                      d|                     dd¦  «        d¦  «        }|                     ||z  | j        d¦  «        }|                      |¦  «        }|                     ||z  d¦  «        }|                     d¦  «        | j        z  }	|	                     ||z  | j        | j        d¦  «        }
|
                     d¦  «                             ||d¦  «        }
|
|fS )NrZ   T)ÚtauÚhardr‹   r   rŒ   éþÿÿÿ)r’   r1  r‘   r,  r°   rB   Ú
functionalÚgumbel_softmaxrÌ   r2  Útype_asr~   Úsoftmaxr:  ÚargmaxÚ	new_zerosÚscatter_r�   r0  r.  r”   )r&   r.   Ú
batch_sizeÚsequence_lengthrD   Úcodevector_probsÚcodevector_soft_distr9  Úcodevector_idxÚcodevectors_per_groupr0  s              r)   r/   z"WavLMGumbelVectorQuantizer.forward  s  € Ø3@Ô3FÑ0ˆ
�O [ð ×(Ò(¨Ñ7Ô7ˆØ%×*Ò*¨:¸Ñ+GÈ$Ì/Ñ+YÐ[]Ñ^Ô^ˆàŒ=ð 	Då!œ}×;Ò;¸M×<OÒ<OÑ<QÔ<QÐW[ÔWgÐnrÐ;ÑsÔsÐØ/×7Ò7¸ÑFÔFÐõ $)¤=Ø×"Ò" :°Ñ#?ÀÄÐRTÑUÔU×[Ò[Ñ]Ô]Ðceð$ñ $ô $Ð ð ×1Ò1Ð2FÑGÔGˆJˆJð +×1Ò1°bÐ1Ñ9Ô9ˆNØ6˜}Ô6¸Ô8KÐL×UÒUØ�N×'Ò'¨¨AÑ.Ô.°ñ ô  Ðð  0×4Ò4°ZÀ/Ñ5QÐSWÔSbÐdfÑgÔgÐà×1Ò1Ð2BÑCÔCˆJà+×0Ò0°¸oÑ1MÈrÑRÔRÐà 0× :Ò :¸2Ñ >Ô >ÀÔAQÑ QÐØ+×0Ò0°¸oÑ1MÈtÌÐ`dÔ`mÐoqÑrÔrˆØ!—o’o bÑ)Ô)×.Ò.¨z¸?ÈBÑOÔOˆà˜JÐ&Ð&r*   )	r1   r2   r3   r×   r$   Ústaticmethodr:  r/   r4   r5   s   @r)   r)  r)  ä  sl   ø€ € € € € ðð ð
ð ð ð ð ð* ðð ñ „\ðð
"'ð "'ð "'ð "'ð "'ð "'ð "'r*   r)  c                   óÜ   ‡ — e Zd ZU eed<   dZdZdZdZdZ	dZ
dZeeg eedd¬	¦  «        d
œZ ej        ¦   «         ˆ fd„¦   «         Zddej        ez  dedz  fd„Z	 ddedej        fd„Zˆ xZS )ÚWavLMPreTrainedModelrS   ÚwavlmÚinput_valuesÚaudioTFr   Úencoder)r†   Ú
layer_name)r.   Ú
attentionsc           
      óL  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rZt	          j        |j        j        dd¬¦  «         t	          j        |j        j	        ¦  «         t	          j
        |j        ¦  «         dS t          |t          ¦  «        rwt	          j        |j        j        ddt          j        d|j        j        d         |j        j        z  z  ¦  «        z  ¬¦  «         t	          j        |j        j	        d¦  «         dS t          |t&          ¦  «        rgt          j        d|j        j        z  ¦  «        }t	          j
        |j        j        | |¬¦  «         t	          j
        |j        j	        | |¬¦  «         dS t          |t,          j        ¦  «        rnt	          j        |j        ¦  «         |j	        �Pt          j        |j        |j        |j        d         z  z  ¦  «        }t	          j
        |j	        | |¬¦  «         dS dS dS )zInitialize the weightsrl   r   )r4  Ústdr   r"   )ÚaÚbN)r#   Ú_init_weightsræ   r)  ÚinitÚnormal_r1  r>   Úzeros_r¯   Úuniform_r0  r7   rF   rÍ   Úsqrtr9   Úin_channelsÚ	constant_rX   rb   Úin_featuresrB   rC   Úkaiming_normal_r;   )r&   ÚmoduleÚkr(   s      €r)   rY  z"WavLMPreTrainedModel._init_weights9  s÷  ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%å�fÕ8Ñ9Ô9ð 	6ÝŒL˜Ô+Ô2¸À!ÐDÑDÔDÐDÝŒK˜Ô*Ô/Ñ0Ô0Ð0ÝŒM˜&Ô,Ñ-Ô-Ð-Ð-Ð-Ý˜Õ <Ñ=Ô=ð 	6ÝŒLØ”Ô"ØØ�œ	 ! v¤{Ô'>¸qÔ'AÀFÄKÔD[Ñ'[Ñ"\Ñ]Ô]Ñ]ðñ ô ð õ
 ŒN˜6œ;Ô+¨QÑ/Ô/Ð/Ð/Ð/Ý˜Õ 6Ñ7Ô7ð 		6Ý”	˜!˜fÔ/Ô;Ñ;Ñ<Ô<ˆAÝŒM˜&Ô+Ô2°q°b¸AÐ>Ñ>Ô>Ð>ÝŒM˜&Ô+Ô0°Q°B¸!Ð<Ñ<Ô<Ð<Ð<Ð<Ý˜¥¤	Ñ*Ô*ð 	6ÝÔ  ¤Ñ/Ô/Ð/àŒ{Ð&Ý”I˜fœm¨vÔ/AÀFÔDVÐWXÔDYÑ/YÑZÑ[Ô[�Ý”˜fœk¨a¨R°1Ð5Ñ5Ô5Ð5Ð5Ð5ð	6ð 	6ð 'Ð&r*   NÚinput_lengthsÚadd_adapterc                 ó  — |€| j         j        n|}d„ }t          | j         j        | j         j        ¦  «        D ]\  }} ||||¦  «        }Œ|r3t          | j         j        ¦  «        D ]} ||d| j         j        ¦  «        }Œ|S )zH
        Computes the output length of the convolutional layers
        Nc                 ó<   — t          j        | |z
  |d¬¦  «        dz   S )NÚfloor)Úrounding_moder   )r~   Údiv©Úinput_lengthr9   Ústrides      r)   Ú_conv_out_lengthzOWavLMPreTrainedModel._get_feat_extract_output_lengths.<locals>._conv_out_length[  s&   € õ ”9˜\¨KÑ7¸ÈwÐWÑWÔWÐZ[Ñ[Ð[r*   r   )rS   rf  ÚzipÚconv_kernelÚconv_strider  Únum_adapter_layersÚadapter_stride)r&   re  rf  ro  r9   rn  rš   s          r)   Ú _get_feat_extract_output_lengthsz5WavLMPreTrainedModel._get_feat_extract_output_lengthsT  s´   € ð
 2=Ð1D�d”kÔ-Ð-È+ˆð	\ð 	\ð 	\õ
 $' t¤{Ô'>ÀÄÔ@WÑ#XÔ#Xð 	Qð 	QÑˆK˜Ø,Ð,¨]¸KÈÑPÔPˆMˆMàð 	_Ý˜4œ;Ô9Ñ:Ô:ð _ð _�Ø 0Ð 0°ÀÀ4Ä;ÔC]Ñ ^Ô ^��àÐr*   Úfeature_vector_lengthr„   c                 ó  — |                      d¬¦  «        d d …df         }|                      ||¬¦  «        }|                     t          j        ¦  «        }|j        d         }t          j        ||f|j        |j        ¬¦  «        }d|t          j	        |j        d         |j        ¬¦  «        |dz
  f<   | 
                    dg¦  «                              d¦  «         
                    dg¦  «                             ¦   «         }|S )NrZ   r‹   ©rf  r   )r½   rÂ   r   )rÂ   )Úcumsumru  rÁ   r~   r¿   r’   Úzerosr½   rÂ   r¾   ÚfliprÙ   )r&   rv  r„   rf  Únon_padded_lengthsÚoutput_lengthsrF  s          r)   Ú"_get_feature_vector_attention_maskz7WavLMPreTrainedModel._get_feature_vector_attention_maski  s  € ð
 ,×2Ò2°rÐ2Ñ:Ô:¸1¸1¸1¸b¸5ÔAÐà×>Ò>Ð?QÐ_jÐ>ÑkÔkˆØ'×*Ò*­5¬:Ñ6Ô6ˆà#Ô)¨!Ô,ˆ
åœØÐ.Ð/°~Ô7KÐTbÔTið
ñ 
ô 
ˆð uvˆ�œ ^Ô%9¸!Ô%<À^ÔEZÐ[Ñ[Ô[Ð]kÐnoÑ]oÐpÑqØ'×,Ò,¨b¨TÑ2Ô2×9Ò9¸"Ñ=Ô=×BÒBÀBÀ4ÑHÔH×MÒMÑOÔOˆØÐr*   rh   )r1   r2   r3   r   Ú__annotations__Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnrï   r   r   rk   Ú_can_record_outputsr~   Úno_gradrY  rÝ   rØ   rÙ   ru  r~  r4   r5   s   @r)   rN  rN  *  s  ø€ € € € € € àÐÐÑØÐØ$€OØÐØ&*Ð#Ø ÐØ€NØÐà+Ð-MÐNØ$�n ^¸1ÈÐSÑSÔSðð Ðð
 €U„]�_„_ð6ð 6ð 6ð 6ñ „_ð6ð4ð ¸eÔ>NÐQTÑ>Tð ÐcgÐjnÑcnð ð ð ð ð, Y]ðð Ø%(ðØ:?Ô:Jðð ð ð ð ð ð ð r*   rN  c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚWavLMNoLayerNormConvLayerr   c                 óZ  •— t          ¦   «                              ¦   «          |dk    r|j        |dz
           nd| _        |j        |         | _        t          j        | j        | j        |j        |         |j        |         |j	        ¬¦  «        | _
        t          |j                 | _        d S )Nr   r   ©r9   rn  r¯   )r#   r$   r^   Úin_conv_dimÚout_conv_dimrB   rC   rq  rr  Ú	conv_biasrF   r   rQ   rR   ©r&   rS   Úlayer_idr(   s      €r)   r$   z"WavLMNoLayerNormConvLayer.__init__  s�   ø€ Ý‰Œ×ÒÑÔÐØ<DÀqºL¸L˜6œ?¨8°a©<Ô8Ð8ÈaˆÔØ"œO¨HÔ5ˆÔå”IØÔØÔØÔ*¨8Ô4ØÔ% hÔ/ØÔ!ð
ñ 
ô 
ˆŒ	õ ! Ô!?Ô@ˆŒˆˆr*   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S rh   )rF   rR   r-   s     r)   r/   z!WavLMNoLayerNormConvLayer.forward�  s*   € ØŸ	š	 -Ñ0Ô0ˆØŸš¨Ñ6Ô6ˆØÐr*   ©r   r0   r5   s   @r)   rŠ  rŠ  ~  sR   ø€ € € € € ðAð Að Að Að Að Aðð ð ð ð ð ð r*   rŠ  c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚWavLMLayerNormConvLayerr   c                 óš  •— t          ¦   «                              ¦   «          |dk    r|j        |dz
           nd| _        |j        |         | _        t          j        | j        | j        |j        |         |j        |         |j	        ¬¦  «        | _
        t          j        | j        d¬¦  «        | _        t          |j                 | _        d S )Nr   r   rŒ  T)Úelementwise_affine)r#   r$   r^   r�  rŽ  rB   rC   rq  rr  r�  rF   r]   r`   r   rQ   rR   r�  s      €r)   r$   z WavLMLayerNormConvLayer.__init__”  s¶   ø€ Ý‰Œ×ÒÑÔÐØ<DÀqºL¸L˜6œ?¨8°a©<Ô8Ð8ÈaˆÔØ"œO¨HÔ5ˆÔå”IØÔØÔØÔ*¨8Ô4ØÔ% hÔ/ØÔ!ð
ñ 
ô 
ˆŒ	õ œ, tÔ'8ÈTÐRÑRÔRˆŒÝ  Ô!?Ô@ˆŒˆˆr*   c                 óÜ   — |                       |¦  «        }|                     dd¦  «        }|                      |¦  «        }|                     dd¦  «        }|                      |¦  «        }|S )Nr>  rZ   )rF   rV   r`   rR   r-   s     r)   r/   zWavLMLayerNormConvLayer.forward£  se   € ØŸ	š	 -Ñ0Ô0ˆà%×/Ò/°°BÑ7Ô7ˆØŸš¨Ñ6Ô6ˆØ%×/Ò/°°BÑ7Ô7ˆàŸš¨Ñ6Ô6ˆØÐr*   r“  r0   r5   s   @r)   r•  r•  “  sR   ø€ € € € € ðAð Að Að Að Að Aðð ð ð ð ð ð r*   r•  c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚWavLMGroupNormConvLayerr   c                 ó¦  •— t          ¦   «                              ¦   «          |dk    r|j        |dz
           nd| _        |j        |         | _        t          j        | j        | j        |j        |         |j        |         |j	        ¬¦  «        | _
        t          |j                 | _        t          j        | j        | j        d¬¦  «        | _        d S )Nr   r   rŒ  T)r,  Únum_channelsÚaffine)r#   r$   r^   r�  rŽ  rB   rC   rq  rr  r�  rF   r   rQ   rR   Ú	GroupNormr`   r�  s      €r)   r$   z WavLMGroupNormConvLayer.__init__¯  s½   ø€ Ý‰Œ×ÒÑÔÐØ<DÀqºL¸L˜6œ?¨8°a©<Ô8Ð8ÈaˆÔØ"œO¨HÔ5ˆÔå”IØÔØÔØÔ*¨8Ô4ØÔ% hÔ/ØÔ!ð
ñ 
ô 
ˆŒ	õ ! Ô!?Ô@ˆŒåœ,°$Ô2CÐRVÔRcÐlpÐqÑqÔqˆŒˆˆr*   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S rh   )rF   r`   rR   r-   s     r)   r/   zWavLMGroupNormConvLayer.forward¿  s;   € ØŸ	š	 -Ñ0Ô0ˆØŸš¨Ñ6Ô6ˆØŸš¨Ñ6Ô6ˆØÐr*   r“  r0   r5   s   @r)   rš  rš  ®  sR   ø€ € € € € ðrð rð rð rð rð rð ð ð ð ð ð ð r*   rš  c                   ó.   ‡ — e Zd ZdZˆ fd„Zd„ Zd„ Zˆ xZS )ÚWavLMFeatureEncoderz.Construct the features from raw audio waveformc                 ó¤  •‡— t          ¦   «                              ¦   «          ‰j        dk    r7t          ‰d¬¦  «        gˆfd„t	          ‰j        dz
  ¦  «        D ¦   «         z   }nD‰j        dk    r!ˆfd„t	          ‰j        ¦  «        D ¦   «         }nt          d‰j        › d	�¦  «        ‚t          j        |¦  «        | _	        d
| _
        d| _        d S )NÚgroupr   ©r‘  c                 ó8   •— g | ]}t          ‰|d z   ¬¦  «        ‘ŒS )r   r¤  )rŠ  r
  s     €r)   r  z0WavLMFeatureEncoder.__init__.<locals>.<listcomp>Í  s>   ø€ ð Kð Kð KØFGÕ)¨&¸1¸q¹5ÐAÑAÔAðKð Kð Kr*   r   r  c                 ó2   •— g | ]}t          ‰|¬ ¦  «        ‘ŒS )r¤  )r•  r
  s     €r)   r  z0WavLMFeatureEncoder.__init__.<locals>.<listcomp>Ñ  s'   ø€ ÐvÐvÐvÈ1Õ2°6ÀAÐFÑFÔFÐvÐvÐvr*   z`config.feat_extract_norm` is z), but has to be one of ['group', 'layer']FT)r#   r$   Úfeat_extract_normrš  r  Únum_feat_extract_layersrw   rB   r  Úconv_layersr  Ú_requires_grad)r&   rS   r©  r(   s    ` €r)   r$   zWavLMFeatureEncoder.__init__É  s  øø€ Ý‰Œ×ÒÑÔÐàÔ# wÒ.Ð.Ý2°6ÀAÐFÑFÔFÐGð Kð Kð Kð KÝKPÐQWÔQoÐrsÑQsÑKtÔKtðKñ Kô Kñ ˆKˆKð Ô%¨Ò0Ð0ØvÐvÐvÐvÕPUÐV\ÔVtÑPuÔPuÐvÑvÔvˆKˆKåØt°Ô1IÐtÐtÐtñô ð õ œ=¨Ñ5Ô5ˆÔØ&+ˆÔ#Ø"ˆÔÐÐr*   c                 óP   — |                       ¦   «         D ]	}d|_        Œ
d| _        d S )NF)Ú
parametersÚrequires_gradrª  ©r&   Úparams     r)   Ú_freeze_parametersz&WavLMFeatureEncoder._freeze_parametersÚ  s4   € Ø—_’_Ñ&Ô&ð 	(ð 	(ˆEØ"'ˆEÔÐØ#ˆÔÐÐr*   c                 ór   — |d d …d f         }| j         r| j        rd|_        | j        D ]} ||¦  «        }Œ|S )NT)rª  r°   r­  r©  )r&   rP  r.   Ú
conv_layers       r)   r/   zWavLMFeatureEncoder.forwardß  s[   € Ø$ Q Q Q¨ WÔ-ˆð Ôð 	/ 4¤=ð 	/Ø*.ˆMÔ'àÔ*ð 	6ð 	6ˆJØ&˜J }Ñ5Ô5ˆMˆMàÐr*   )r1   r2   r3   r×   r$   r°  r/   r4   r5   s   @r)   r¡  r¡  Æ  s\   ø€ € € € € Ø8Ð8ð#ð #ð #ð #ð #ð"$ð $ð $ð

ð 
ð 
ð 
ð 
ð 
ð 
r*   r¡  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚWavLMAdapterLayerc                 ó²   •— t          ¦   «                              ¦   «          t          j        |j        d|j        z  |j        |j        d¬¦  «        | _        d S )Nr"   r   )rn  r:   )r#   r$   rB   rC   Úoutput_hidden_sizeÚadapter_kernel_sizert  rF   rf   s     €r)   r$   zWavLMAdapterLayer.__init__í  sU   ø€ Ý‰Œ×ÒÑÔÐÝ”IØÔ%Ø�Ô)Ñ)ØÔ&ØÔ(Øð
ñ 
ô 
ˆŒ	ˆ	ˆ	r*   c                 ór   — |                       |¦  «        }t          j                             |d¬¦  «        }|S )Nr   r‹   )rF   rB   r?  Úglur-   s     r)   r/   zWavLMAdapterLayer.forward÷  s3   € ØŸ	š	 -Ñ0Ô0ˆÝœ×)Ò)¨-¸QÐ)Ñ?Ô?ˆàÐr*   r0   r5   s   @r)   r´  r´  ì  sG   ø€ € € € € ð
ð 
ð 
ð 
ð 
ðð ð ð ð ð ð r*   r´  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚWavLMAdapterc                 ó’  •‡— t          ¦   «                              ¦   «          ‰j        ‰j        k    rCt	          j        ‰j        ‰j        ¦  «        | _        t	          j        ‰j        ¦  «        | _        nd x| _        | _        t	          j	        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        ‰j        | _        d S )Nc              3   ó6   •K  — | ]}t          ‰¦  «        V — Œd S rh   )r´  )r  rš   rS   s     €r)   ú	<genexpr>z(WavLMAdapter.__init__.<locals>.<genexpr>	  s,   øè è € Ð#hÐ#hÀ!Õ$5°fÑ$=Ô$=Ð#hÐ#hÐ#hÐ#hÐ#hÐ#hr*   )r#   r$   r¶  rD   rB   ra   Úprojr]   Úproj_layer_normr  r  rs  r  r  rf   s    `€r)   r$   zWavLMAdapter.__init__ÿ  s¬   øø€ Ý‰Œ×ÒÑÔÐð Ô$¨Ô(:Ò:Ð:Ýœ	 &Ô"4°fÔ6OÑPÔPˆDŒIÝ#%¤<°Ô0IÑ#JÔ#JˆDÔ Ð à/3Ð3ˆDŒI˜Ô,å”mÐ#hÐ#hÐ#hÐ#hÅuÈVÔMfÑGgÔGgÐ#hÑ#hÔ#hÑhÔhˆŒØÔ)ˆŒˆˆr*   c                 óX  — | j         �1| j        �*|                       |¦  «        }|                      |¦  «        }|                     dd¦  «        }| j        D ]=}t          j                             ¦   «         }| j        r|| j        k    r ||¦  «        }Œ>|                     dd¦  «        }|S rU   )r¿  rÀ  rV   r  ÚnpÚrandomr°   r  )r&   r.   r  Úlayerdrop_probs       r)   r/   zWavLMAdapter.forward  s°   € àŒ9Ð  TÔ%9Ð%EØ ŸIšI mÑ4Ô4ˆMØ ×0Ò0°Ñ?Ô?ˆMà%×/Ò/°°1Ñ5Ô5ˆà”[ð 	5ð 	5ˆEÝœY×-Ò-Ñ/Ô/ˆNØ”=ð 5 ^°d´nÒ%DÐ%DØ %  mÑ 4Ô 4�øà%×/Ò/°°1Ñ5Ô5ˆØÐr*   r0   r5   s   @r)   r»  r»  þ  sG   ø€ € € € € ð*ð *ð *ð *ð *ðð ð ð ð ð ð r*   r»  r’   Ú	mask_probÚmask_lengthr„   Ú	min_masksrˆ   c                 ó@  ‡‡‡‡‡— | \  }Š‰dk     rt          d¦  «        ‚‰‰k    rt          d‰› d‰› d�¦  «        ‚t          j                             d¦  «                             ¦   «         Šˆˆˆˆˆfd„}|�9|                     ¦   «                              d¦  «                             ¦   «         nˆfd	„t          |¦  «        D ¦   «         }t          j	        |‰ft          ¬
¦  «        }g }	 |‰¦  «        }
|
dk    r|S |D ]·} ||¦  «        }t          j                             t          j        |‰dz
  z
  ¦  «        |d¬¦  «        }t          |¦  «        dk    r‰dz
  }n|d         }t          j        |t          j        |
|z
  t          j        ¬
¦  «        |z  g¦  «        }|	                     |¦  «         Œ¸t          j        |	¦  «        }	t          j        |	dd…dd…df         ||
‰f¦  «        }	|	                     ||
‰z  ¦  «        }	t          j        ‰¦  «        dddd…f         }t          j        |||
‰f¦  «                             ||
‰z  ¦  «        }|	|z   }	|	                     ¦   «         ‰dz
  k    r‰dz
  |	|	‰dz
  k    <   t          j        ||	dd¦  «         |S )an  
    Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for
    ASR](https://huggingface.co/papers/1904.08779). Note that this method is not optimized to run on TPU and should be run on
    CPU as part of the preprocessing during training.

    Args:
        shape: The shape for which to compute masks. This should be of a tuple of size 2 where
               the first element is the batch size and the second element is the length of the axis to span.
        mask_prob:  The percentage of the whole axis (between 0 and 1) which will be masked. The number of
                    independently generated mask spans of length `mask_length` is computed by
                    `mask_prob*shape[1]/mask_length`. Note that due to overlaps, `mask_prob` is an upper bound and the
                    actual percentage will be smaller.
        mask_length: size of the mask
        min_masks: minimum number of masked spans
        attention_mask: A (right-padded) attention mask which independently shortens the feature axis of
                        each batch dimension.
    r   z&`mask_length` has to be bigger than 0.zO`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: z and `sequence_length`: ú`c                 ó¸   •— t          ‰| z  ‰z  ‰z   ¦  «        }t          |‰¦  «        }|‰z  ‰k    r‰‰z  }| ‰dz
  z
  |k     rt          | ‰dz
  z
  d¦  «        }|S )z;Given input length, compute how many spans should be maskedr   r   )rØ   Úmax)rm  Únum_masked_spanÚepsilonrÆ  rÅ  rÇ  rG  s     €€€€€r)   Úcompute_num_masked_spanz6_compute_mask_indices.<locals>.compute_num_masked_spanC  s~   ø€ å˜i¨,Ñ6¸ÑDÀwÑNÑOÔOˆÝ˜o¨yÑ9Ô9ˆð ˜[Ñ(¨?Ò:Ð:Ø-°Ñ<ˆOð ˜;¨™?Ñ+¨oÒ=Ð=Ý! ,°+À±/Ñ"BÀAÑFÔFˆOàÐr*   NrZ   c                 ó   •— g | ]}‰‘ŒS © rÐ  )r  rš   rG  s     €r)   r  z)_compute_mask_indices.<locals>.<listcomp>V  s   ø€ Ð9Ð9Ð9 !ˆoÐ9Ð9Ð9r*   r¼   r   F)Úreplace)rw   rÂ  rÃ  r  ÚitemÚdetachr”   Útolistr  rz  rÙ   Úchoicer¾   ÚlenÚconcatenater   Úint32ÚappendÚarrayr±   ÚreshaperË  Úput_along_axis)r’   rÅ  rÆ  r„   rÇ  rF  rÎ  re  Úspec_aug_maskÚspec_aug_mask_idxsÚmax_num_masked_spanrm  rÌ  Úspec_aug_mask_idxÚdummy_mask_idxÚoffsetsrÍ  rG  s    `` `           @@r)   Ú_compute_mask_indicesrã    sP  øøøøø€ ð0 #(Ñ€J�à�Q‚€ÝÐAÑBÔBÐBà�_Ò$Ð$Ýð:Ð^ið :ð :Ø'6ð:ð :ð :ñ
ô 
ð 	
õ Œi�nŠn˜QÑÔ×$Ò$Ñ&Ô&€Gðð ð ð ð ð ð ð ð ð$ Ð%ð 	×ÒÑÔ×#Ò# BÑ'Ô'×.Ò.Ñ0Ô0Ð0à9Ð9Ð9Ð9¥u¨ZÑ'8Ô'8Ð9Ñ9Ô9ð õ ”H˜j¨/Ð:Å$ÐGÑGÔG€MØÐà1Ð1°/ÑBÔBÐà˜aÒÐØÐà%ð 5ð 5ˆà1Ð1°,Ñ?Ô?ˆõ œI×,Ò,ÝŒI�l k°A¡oÑ6Ñ7Ô7¸ÐRWð -ñ 
ô 
Ðõ Ð Ñ!Ô! QÒ&Ð&ð -¨qÑ0ˆNˆNà.¨qÔ1ˆNåœNØ¥¤Ð(;¸oÑ(MÕUWÔU]Ð ^Ñ ^Ô ^ÐaoÑ oÐpñ
ô 
Ðð 	×!Ò!Ð"3Ñ4Ô4Ð4Ð4åœÐ"4Ñ5Ô5Ðõ œØ˜1˜1˜1˜a˜a˜a ˜:Ô&¨Ð5HÈ+Ð(Vñô Ðð ,×3Ò3°JÐ@SÐVaÑ@aÑbÔbÐõ Œi˜Ñ$Ô$ T¨4°°° ]Ô3€GÝŒo˜g¨
Ð4GÈÐ'UÑVÔV×^Ò^ØÐ'¨+Ñ5ñô €Gð ,¨gÑ5Ðð ×ÒÑÔ /°AÑ"5Ò5Ð5ØGVÐYZÑGZÐÐ-°À!Ñ0CÒCÑDõ Ô�mÐ%7¸¸BÑ?Ô?Ð?àÐr*   c                   ó  ‡ — e Zd Zdefˆ fd„Zd„ Z	 	 ddej        dej        dz  dej        dz  fd„Z	e
 ed	¬
¦  «        e	 	 ddej        dz  dej        dz  dej        dz  dee         deez  f
d„¦   «         ¦   «         ¦   «         Zˆ xZS )Ú
WavLMModelrS   c                 ó  •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        t          |¦  «        | _        |j        dk    s|j        dk    rBt          j
        t          j        |j        ¦  «                             ¦   «         ¦  «        | _        |j        rt#          |¦  «        | _        nt'          |¦  «        | _        |j        rt+          |¦  «        nd | _        |                      ¦   «          d S )Nrl   )r#   r$   rS   r¡  Úfeature_extractorrX   Úfeature_projectionÚmask_time_probÚmask_feature_probrB   r}   r~   rÚ   rD   r]  Úmasked_spec_embedÚdo_stable_layer_normr$  rR  r  rf  r»  ÚadapterÚ	post_initrf   s     €r)   r$   zWavLMModel.__init__™  sé   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒÝ!4°VÑ!<Ô!<ˆÔÝ"8¸Ñ"@Ô"@ˆÔð Ô  3Ò&Ð&¨&Ô*BÀSÒ*HÐ*HÝ%'¤\µ%´,¸vÔ?QÑ2RÔ2R×2[Ò2[Ñ2]Ô2]Ñ%^Ô%^ˆDÔ"àÔ&ð 	0Ý6°vÑ>Ô>ˆDŒLˆLå'¨Ñ/Ô/ˆDŒLà/5Ô/AÐK•| FÑ+Ô+Ð+ÀtˆŒð 	�ŠÑÔÐÐÐr*   c                 ó8   — | j                              ¦   «          dS ©z¨
        Calling this function will disable the gradient computation for the feature encoder so that its parameter will
        not be updated during training.
        N)rç  r°  ©r&   s    r)   Úfreeze_feature_encoderz!WavLMModel.freeze_feature_encoder­  s   € ð
 	Ô×1Ò1Ñ3Ô3Ð3Ð3Ð3r*   Nr.   Úmask_time_indicesr„   c                 ó  — t          | j        dd¦  «        s|S |                     ¦   «         \  }}}|�#| j                             |j        ¦  «        ||<   n“| j        j        dk    rƒ| j        r|t          ||f| j        j        | j        j	        || j        j
        ¬¦  «        }t          j        ||j        t          j        ¬¦  «        }| j                             |j        ¦  «        ||<   | j        j        dk    r†| j        rt          ||f| j        j        | j        j        | j        j        ¬¦  «        }t          j        ||j        t          j        ¬¦  «        }|dd…df                              d|d¦  «        }d||<   |S )	z¢
        Masks extracted features along time axis and/or along feature axis according to
        [SpecAugment](https://huggingface.co/papers/1904.08779).
        Úapply_spec_augmentTNr   )rÅ  rÆ  r„   rÇ  )rÂ   r½   )rÅ  rÆ  rÇ  rZ   )ÚgetattrrS   r�   rë  rÁ   r½   ré  r°   rã  Úmask_time_lengthÚmask_time_min_masksr~   ÚtensorrÂ   rÙ   rê  Úmask_feature_lengthÚmask_feature_min_masksÚexpand)r&   r.   ró  r„   rF  rG  rD   Úmask_feature_indicess           r)   Ú_mask_hidden_stateszWavLMModel._mask_hidden_states´  s—  € õ �t”{Ð$8¸$Ñ?Ô?ð 	!Ø Ð ð 4A×3EÒ3EÑ3GÔ3GÑ0ˆ
�O [àÐ(à/3Ô/E×/HÒ/HÈÔI\Ñ/]Ô/]ˆMÐ+Ñ,Ð,ØŒ[Ô'¨!Ò+Ð+°´Ð+Ý 5Ø˜_Ð-Øœ+Ô4Ø œKÔ8Ø-Øœ+Ô9ð!ñ !ô !Ðõ !&¤Ð->À}ÔG[ÕchÔcmÐ nÑ nÔ nÐØ/3Ô/E×/HÒ/HÈÔI\Ñ/]Ô/]ˆMÐ+Ñ,àŒ;Ô(¨1Ò,Ð,°´Ð,å#8Ø˜[Ð)Øœ+Ô7Ø œKÔ;Øœ+Ô<ð	$ñ $ô $Ð õ $)¤<Ð0DÈ]ÔMaÕinÔisÐ#tÑ#tÔ#tÐ Ø#7¸¸¸¸4¸Ô#@×#GÒ#GÈÈOÐ]_Ñ#`Ô#`Ð Ø23ˆMÐ.Ñ/àÐr*   F)Útie_last_hidden_statesrP  r‡   rˆ   c                 óŠ  — |                       |¦  «        }|                     dd¦  «        }|�#|                      |j        d         |d¬¦  «        }|                      |¦  «        \  }}|                      |||¬¦  «        } | j        |fd|i|¤Ž}|j        }| j        �|                      |¦  «        }t          ||¬¦  «        S )	a/  
        mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict
            masked extracted features in *config.proj_codevector_dim* space.
        r   r"   NFrx  )ró  r„   r„   )r  Úextract_features)
rç  rV   r~  r’   rè  rþ  rR  r  rí  ÚWavLMBaseModelOutput)r&   rP  r„   ró  r‡   r  r.   Úencoder_outputss           r)   r/   zWavLMModel.forwardâ  s  € ð  ×1Ò1°,Ñ?Ô?ÐØ+×5Ò5°a¸Ñ;Ô;ÐàÐ%à!×DÒDØ Ô& qÔ)¨>Àuð Eñ ô ˆNð +/×*AÒ*AÐBRÑ*SÔ*SÑ'ˆÐ'Ø×0Ò0ØÐ->È~ð 1ñ 
ô 
ˆð '˜$œ,Øð
ð 
à)ð
ð ð
ð 
ˆð (Ô9ˆàŒ<Ð#Ø ŸLšL¨Ñ7Ô7ˆMå#Ø+Ø-ð
ñ 
ô 
ð 	
r*   r  )r1   r2   r3   r   r$   rò  r~   rÜ   rÝ   rþ  r   r   r   rÚ   r   r   rÛ   r  r/   r4   r5   s   @r)   rå  rå  —  sF  ø€ € € € € ð˜{ð ð ð ð ð ð ð(4ð 4ð 4ð 7;Ø26ð	,ð ,àÔ(ð,ð !Ô,¨tÑ3ð,ð Ô(¨4Ñ/ð	,ð ,ð ,ð ,ð\  Ø€_¨EÐ2Ñ2Ô2Øð /3Ø6:ð	(
ð (
à”l TÑ)ð(
ð œ tÑ+ð(
ð !Ô,¨tÑ3ð	(
ð
 Ð+Ô,ð(
ð 
Ð%Ñ	%ð(
ð (
ð (
ñ „^ñ 3Ô2ñ  Ôð(
ð (
ð (
ð (
ð (
r*   rå  zm
    WavLM Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).
    )Úcustom_introc                   óÂ   ‡ — e Zd Zddedz  fˆ fd„Zd„ Zd„ Zd„ Zee		 	 dde
j        dz  de
j        dz  d	e
j        dz  d
ee         deez  f
d„¦   «         ¦   «         Zˆ xZS )ÚWavLMForCTCNÚtarget_langc                 óª  •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        ¦  «        | _        || _        |j	        €t          d| j        › d�¦  «        ‚t          |d¦  «        r|j        r|j        n|j        }t	          j        ||j	        ¦  «        | _        |                      ¦   «          dS )a/  
        target_lang (`str`, *optional*):
            Language id of adapter weights. Adapter weights are stored in the format adapter.<lang>.safetensors or
            adapter.<lang>.bin. Only relevant when using an instance of [`WavLMForCTC`] with adapters. Uses 'eng' by
            default.
        NzYou are trying to instantiate zô with a configuration that does not define the vocabulary size of the language model head. Please instantiate the model as follows: `WavLMForCTC.from_pretrained(..., vocab_size=vocab_size)`. or define `vocab_size` of your model's configuration.rf  )r#   r$   rå  rO  rB   rc   Úfinal_dropoutre   r  Ú
vocab_sizerw   r(   rH   rf  r¶  rD   ra   Úlm_headrî  )r&   rS   r  r¶  r(   s       €r)   r$   zWavLMForCTC.__init__  sÝ   ø€ õ 	‰Œ×Ò˜Ñ Ô Ð å Ñ'Ô'ˆŒ
Ý”z &Ô"6Ñ7Ô7ˆŒà&ˆÔàÔÐ$ÝðH°´ð Hð Hð Hñô ð õ *1°¸Ñ)GÔ)GÐvÈFÔL^ÐvˆFÔ%Ð%ÐdjÔdvð 	õ ”yÐ!3°VÔ5FÑGÔGˆŒð 	�ŠÑÔÐÐÐr*   c                 óT  — t          ¦   «         t          j        d¦  «        k    rdS | j        }|�)t	          | j        dd¦  «        €t          d|› d�¦  «        ‚|€2t	          | j        dd¦  «        �t                               d¦  «         dS |�|  	                    |d¬¦  «         dS dS )	a'  
        This method overwrites [`~PreTrainedModel.tie_weights`] so that adapter weights can be correctly loaded when
        passing `target_lang=...` to `from_pretrained(...)`.

        This method is **not** supposed to be called by the user and is prone to be changed in the future.
        ÚmetaNÚadapter_attn_dimzCannot pass `target_lang`: z- if `config.adapter_attn_dim` is not defined.z)By default `target_lang` is set to 'eng'.T)Ú
force_load)
r   r~   rÂ   r  rö  rS   rw   ÚloggerÚinfoÚload_adapter)r&   r‡   r  s      r)   Útie_weightszWavLMForCTC.tie_weights3  sÃ   € õ 6Ñ7Ô7½5¼<ÈÑ;OÔ;OÒOÐOØˆFð Ô&ˆàÐ"¥w¨t¬{Ð<NÐPTÑ'UÔ'UÐ']ÝÐu¸;ÐuÐuÐuÑvÔvÐvØÐ ¥W¨T¬[Ð:LÈdÑ%SÔ%SÐ%_Ý�KŠKÐCÑDÔDÐDÐDÐDØÐ$Ø×Ò˜k°dÐÑ;Ô;Ð;Ð;Ð;ð %Ð$r*   c                 óB   — | j         j                             ¦   «          dS rð  ©rO  rç  r°  rñ  s    r)   rò  z"WavLMForCTC.freeze_feature_encoderK  ó!   € ð
 	Œ
Ô$×7Ò7Ñ9Ô9Ð9Ð9Ð9r*   c                 óL   — | j                              ¦   «         D ]	}d|_        Œ
dS ©zÒ
        Calling this function will disable the gradient computation for the base model so that its parameters will not
        be updated during training. Only the classification head will be updated.
        FN©rO  r¬  r­  r®  s     r)   Úfreeze_base_modelzWavLMForCTC.freeze_base_modelR  ó6   € ð
 ”Z×*Ò*Ñ,Ô,ð 	(ð 	(ˆEØ"'ˆEÔÐð	(ð 	(r*   rP  r„   Úlabelsr‡   rˆ   c           
      óü  — |�>|                      ¦   «         | j        j        k    rt          d| j        j        › �¦  «        ‚ | j        |fd|i|¤Ž}|j        }|                      |¦  «        }|                      |¦  «        }d}|��Z|�|nt          j	        |t          j
        ¬¦  «        }|                      |                     d¦  «        ¦  «                             t          j
        ¦  «        }	|dk    }
|
                     d¦  «        }|                     |
¦  «        }t          j                             |dt          j        ¬¦  «                             dd¦  «        }t          j        j                             d	¬
¦  «        5  t          j                             |||	|| j        j        | j        j        | j        j        ¬¦  «        }ddd¦  «         n# 1 swxY w Y   t7          |||j        |j        ¬¦  «        S )aà  
        labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*):
            Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to
            the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`.
            All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ...,
            config.vocab_size - 1]`.
        Nz$Label values must be <= vocab_size: r„   r¼   rZ   r   )r@   r½   r   F)Úenabled)ÚblankÚ	reductionÚzero_infinity©ÚlossÚlogitsr.   rT  )rË  rS   r
  rw   rO  r  re   r  r~   Ú	ones_liker¿   ru  r”   rÁ   Úmasked_selectrB   r?  Úlog_softmaxÚfloat32rV   ÚbackendsÚcudnnÚflagsÚctc_lossÚpad_token_idÚctc_loss_reductionÚctc_zero_infinityr   r.   rT  )r&   rP  r„   r  r‡   Úoutputsr.   r$  r#  re  Úlabels_maskÚtarget_lengthsÚflattened_targetsÚ	log_probss                 r)   r/   zWavLMForCTC.forwardZ  s:  € ð  Ð &§*¢*¡,¤,°$´+Ô2HÒ"HÐ"HÝÐ\ÀDÄKÔDZÐ\Ð\Ñ]Ô]Ð]à�$”*Øð
ð 
à)ð
ð ð
ð 
ˆð  Ô1ˆØŸš ]Ñ3Ô3ˆà—’˜mÑ,Ô,ˆàˆØÑð #1Ð"<��Å%Ä/ÐR^ÕfkÔfpÐBqÑBqÔBqð ð !×AÒAÀ.×BTÒBTÐUWÑBXÔBXÑYÔY×\Ò\Õ]bÔ]gÑhÔhˆMð ! Aš+ˆKØ(Ÿ_š_¨RÑ0Ô0ˆNØ &× 4Ò 4°[Ñ AÔ AÐõ œ×1Ò1°&¸bÍÌÐ1ÑVÔV×`Ò`ÐabÐdeÑfÔfˆIå”Ô%×+Ò+°EÐ+Ñ:Ô:ð 	ð 	Ý”}×-Ò-ØØ%Ø!Ø"Øœ+Ô2Ø"œkÔ<Ø"&¤+Ô"?ð .ñ ô �ð	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	õ Ø˜f°GÔ4IÐV]ÔVhð
ñ 
ô 
ð 	
s   ÆAGÇGÇGrh   r  )r1   r2   r3   rè   r$   r  rò  r  r   r   r~   rÚ   r   r   rÛ   r   r/   r4   r5   s   @r)   r  r    s  ø€ € € € € ðð ¨C°$©Jð ð ð ð ð ð ð:<ð <ð <ð0:ð :ð :ð(ð (ð (ð Øð /3Ø&*ð	:
ð :
à”l TÑ)ð:
ð œ tÑ+ð:
ð ”˜tÑ#ð	:
ð
 Ð+Ô,ð:
ð 
�Ñ	ð:
ð :
ð :
ñ „^ñ Ôð:
ð :
ð :
ð :
ð :
r*   r  z”
    WavLM Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like
    SUPERB Keyword Spotting.
    c                   ó¾   ‡ — e Zd Zˆ fd„Zd„ Zd„ Zeee	 	 dde	j
        dz  de	j
        dz  de	j
        dz  dee         d	eez  f
d
„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚWavLMForSequenceClassificationc                 óô  •— t          ¦   «                              |¦  «         t          |d¦  «        r|j        rt	          d¦  «        ‚t          |¦  «        | _        |j        dz   }|j        r.t          j
        t          j        |¦  «        |z  ¦  «        | _        t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¦  «        | _        |                      ¦   «          d S )Nrf  z\Sequence classification does not support the use of WavLM adapters (config.add_adapter=True)r   )r#   r$   rH   rf  rw   rå  rO  r  Úuse_weighted_layer_sumrB   r}   r~   r   Úlayer_weightsra   rD   Úclassifier_proj_sizeÚ	projectorÚ
num_labelsÚ
classifierrî  ©r&   rS   Ú
num_layersr(   s      €r)   r$   z'WavLMForSequenceClassification.__init__   sá   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å�6˜=Ñ)Ô)ð 	¨fÔ.@ð 	ÝØnñô ð õ   Ñ'Ô'ˆŒ
ØÔ-°Ñ1ˆ
ØÔ(ð 	SÝ!#¤­e¬j¸Ñ.DÔ.DÀzÑ.QÑ!RÔ!RˆDÔÝœ 6Ô#5°vÔ7RÑSÔSˆŒÝœ) FÔ$?ÀÔARÑSÔSˆŒð 	�ŠÑÔÐÐÐr*   c                 óB   — | j         j                             ¦   «          dS rð  r  rñ  s    r)   rò  z5WavLMForSequenceClassification.freeze_feature_encoder±  r  r*   c                 óL   — | j                              ¦   «         D ]	}d|_        Œ
dS r  r  r®  s     r)   r  z0WavLMForSequenceClassification.freeze_base_model¸  r  r*   NrP  r„   r  r‡   rˆ   c                 óæ  — | j         j        rd|d<    | j        |fd|i|¤Ž}| j         j        rr|j        }t	          j        |d¬¦  «        }t          j                             | j	        d¬¦  «        }|| 
                    ddd¦  «        z                       d¬¦  «        }n|j        }|                      |¦  «        }|€|                     d¬¦  «        }n�|                      |j        d         |¦  «        }	|	                     d¦  «                             dd|j        d         ¦  «        }
d	||
 <   |                     d¬¦  «        |	                     d¬¦  «         
                    dd¦  «        z  }|                      |¦  «        }d}|�Kt)          ¦   «         } || 
                    d| j         j        ¦  «        | 
                    d¦  «        ¦  «        }t-          |||j        |j        ¬
¦  «        S )á	  
        input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
            Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file
            into an array of type `list[float]`, a `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library
            (`pip install torchcodec`) or the soundfile library (`pip install soundfile`).
            To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and conversion
            into a tensor of type `torch.FloatTensor`. See [`WavLMProcessor.__call__`] for details.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        TÚoutput_hidden_statesr„   r   r‹   rZ   Nr"   rl   r"  )rS   r8  rO  r.   r~   ÚstackrB   r?  rB  r9  r‘   r”   r  r;  r4  r~  r’   r�   r�   r=  r   r<  r   rT  )r&   rP  r„   r  r‡   r0  r.   Únorm_weightsÚpooled_outputÚpadding_maskÚexpand_padding_maskr$  r#  Úloss_fcts                 r)   r/   z&WavLMForSequenceClassification.forwardÀ  s  € ð, Œ;Ô-ð 	2Ø-1ˆFÐ)Ñ*à�$”*Øð
ð 
à)ð
ð ð
ð 
ˆð Œ;Ô-ð 	6Ø#Ô1ˆMÝ!œK¨¸1Ð=Ñ=Ô=ˆMÝœ=×0Ò0°Ô1CÈÐ0ÑLÔLˆLØ*¨\×->Ò->¸rÀ1ÀaÑ-HÔ-HÑH×MÒMÐRSÐMÑTÔTˆMˆMà#Ô5ˆMàŸš }Ñ5Ô5ˆØÐ!Ø)×.Ò.°1Ð.Ñ5Ô5ˆMˆMà×BÒBÀ=ÔCVÐWXÔCYÐ[iÑjÔjˆLØ".×"8Ò"8¸Ñ"<Ô"<×"CÒ"CÀAÀqÈ-ÔJ]Ð^_ÔJ`Ñ"aÔ"aÐØ25ˆMÐ.Ð.Ñ/Ø)×-Ò-°!Ð-Ñ4Ô4°|×7GÒ7GÈAÐ7GÑ7NÔ7N×7SÒ7SÐTVÐXYÑ7ZÔ7ZÑZˆMà—’ Ñ/Ô/ˆàˆØÐÝ'Ñ)Ô)ˆHØ�8˜FŸKšK¨¨D¬KÔ,BÑCÔCÀVÇ[Â[ÐQSÁ_Ä_ÑUÔUˆDå'ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r*   r  )r1   r2   r3   r$   rò  r  r   r   r   r~   rÚ   r   r   rÛ   r   r/   r4   r5   s   @r)   r6  r6  ™  sê   ø€ € € € € ðð ð ð ð ð":ð :ð :ð(ð (ð (ð Ø Øð /3Ø&*ð	9
ð 9
à”l TÑ)ð9
ð œ tÑ+ð9
ð ”˜tÑ#ð	9
ð
 Ð+Ô,ð9
ð 
Ð)Ñ	)ð9
ð 9
ð 9
ñ „^ñ !Ô ñ Ôð9
ð 9
ð 9
ð 9
ð 9
r*   r6  c                   ó¾   ‡ — e Zd Zˆ fd„Zd„ Zd„ Zeee	 	 dde	j
        dz  de	j
        dz  de	j
        dz  dee         d	eez  f
d
„¦   «         ¦   «         ¦   «         Zˆ xZS )Ú WavLMForAudioFrameClassificationc                 óÄ  •— t          ¦   «                              |¦  «         t          |d¦  «        r|j        rt	          d¦  «        ‚t          |¦  «        | _        |j        dz   }|j        r.t          j
        t          j        |¦  «        |z  ¦  «        | _        t          j        |j        |j        ¦  «        | _        |j        | _        |                      ¦   «          d S )Nrf  z_Audio frame classification does not support the use of WavLM adapters (config.add_adapter=True)r   )r#   r$   rH   rf  rw   rå  rO  r  r8  rB   r}   r~   r   r9  ra   rD   r<  r=  rî  r>  s      €r)   r$   z)WavLMForAudioFrameClassification.__init__  sÐ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å�6˜=Ñ)Ô)ð 	¨fÔ.@ð 	ÝØqñô ð õ   Ñ'Ô'ˆŒ
ØÔ-°Ñ1ˆ
ØÔ(ð 	SÝ!#¤­e¬j¸Ñ.DÔ.DÀzÑ.QÑ!RÔ!RˆDÔÝœ) FÔ$6¸Ô8IÑJÔJˆŒØ Ô+ˆŒà�ŠÑÔÐÐÐr*   c                 óB   — | j         j                             ¦   «          dS rð  r  rñ  s    r)   rò  z7WavLMForAudioFrameClassification.freeze_feature_encoder  r  r*   c                 óL   — | j                              ¦   «         D ]	}d|_        Œ
dS r  r  r®  s     r)   r  z2WavLMForAudioFrameClassification.freeze_base_model  r  r*   NrP  r„   r  r‡   rˆ   c           	      óz  — | j         j        rd|d<    | j        |fd|i|¤Ž}| j         j        rr|j        }t	          j        |d¬¦  «        }t          j                             | j	        d¬¦  «        }|| 
                    ddd¦  «        z                       d¬¦  «        }n|j        }|                      |¦  «        }d}	|�`t          ¦   «         }
 |
| 
                    d| j        ¦  «        t	          j        | 
                    d| j        ¦  «        d¬¦  «        ¦  «        }	t#          |	||j        |j        ¬	¦  «        S )
rC  TrD  r„   r   r‹   rZ   N)Úaxisr"  )rS   r8  rO  r.   r~   rE  rB   r?  rB  r9  r‘   r”   r  r=  r   r<  rC  r   rT  )r&   rP  r„   r  r‡   r0  r.   rF  r$  r#  rJ  s              r)   r/   z(WavLMForAudioFrameClassification.forward   s]  € ð, Œ;Ô-ð 	2Ø-1ˆFÐ)Ñ*à�$”*Øð
ð 
à)ð
ð ð
ð 
ˆð Œ;Ô-ð 	6Ø#Ô1ˆMÝ!œK¨¸1Ð=Ñ=Ô=ˆMÝœ=×0Ò0°Ô1CÈÐ0ÑLÔLˆLØ*¨\×->Ò->¸rÀ1ÀaÑ-HÔ-HÑH×MÒMÐRSÐMÑTÔTˆMˆMà#Ô5ˆMà—’ Ñ/Ô/ˆàˆØÐÝ'Ñ)Ô)ˆHØ�8˜FŸKšK¨¨D¬OÑ<Ô<½e¼lÈ6Ï;Ê;ÐWYÐ[_Ô[jÑKkÔKkÐrsÐ>tÑ>tÔ>tÑuÔuˆDå$ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r*   r  )r1   r2   r3   r$   rò  r  r   r   r   r~   rÚ   r   r   rÛ   r   r/   r4   r5   s   @r)   rL  rL  ÿ  sê   ø€ € € € € ðð ð ð ð ð :ð :ð :ð(ð (ð (ð Ø Øð /3Ø&*ð	0
ð 0
à”l TÑ)ð0
ð œ tÑ+ð0
ð ”˜tÑ#ð	0
ð
 Ð+Ô,ð0
ð 
Ð&Ñ	&ð0
ð 0
ð 0
ñ „^ñ !Ô ñ Ôð0
ð 0
ð 0
ð 0
ð 0
r*   rL  c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚAMSoftmaxLossç      >@çš™™™™™Ù?c                 óþ   •— t          ¦   «                              ¦   «          || _        || _        || _        t          j        t          j        ||¦  «        d¬¦  «        | _	        t          j
        ¦   «         | _        d S )NT)r­  )r#   r$   ÚscaleÚmarginr<  rB   r}   r~   Úrandnr>   r   r#  )r&   Ú	input_dimr<  rW  rX  r(   s        €r)   r$   zAMSoftmaxLoss.__init__W  se   ø€ Ý‰Œ×ÒÑÔÐØˆŒ
ØˆŒØ$ˆŒÝ”l¥5¤;¨y¸*Ñ#EÔ#EÐUYÐZÑZÔZˆŒÝÔ'Ñ)Ô)ˆŒ	ˆ	ˆ	r*   c                 óÐ  — |                      ¦   «         }t          j                             | j        d¬¦  «        }t          j                             |d¬¦  «        }t          j        ||¦  «        }|| j        z
  }t          j                             || j	        ¦  «        }| j
        t          j        |                     ¦   «         ||¦  «        z  }|                      ||¦  «        }|S )Nr   r‹   r   )ÚflattenrB   r?  Ú	normalizer>   r~   ÚmmrX  Úone_hotr<  rW  rÐ   rÙ   r#  )	r&   r.   r  r>   Ú	cos_thetaÚpsiÚonehotr$  r#  s	            r)   r/   zAMSoftmaxLoss.forward_  s·   € Ø—’Ñ!Ô!ˆÝ”×(Ò(¨¬¸!Ð(Ñ<Ô<ˆÝœ×/Ò/°À1Ð/ÑEÔEˆÝ”H˜]¨FÑ3Ô3ˆ	Ø˜$œ+Ñ%ˆå”×&Ò& v¨t¬Ñ?Ô?ˆØ”�eœk¨&¯+ª+©-¬-¸¸iÑHÔHÑHˆØ�yŠy˜ Ñ(Ô(ˆàˆr*   )rT  rU  r0   r5   s   @r)   rS  rS  V  sL   ø€ € € € € ð*ð *ð *ð *ð *ð *ðð ð ð ð ð ð r*   rS  c                   óD   ‡ — e Zd Zdˆ fd„	Zdej        dej        fd„Zˆ xZS )Ú	TDNNLayerr   c                 óŒ  •— t          ¦   «                              ¦   «          |dk    r|j        |dz
           n|j        |         | _        |j        |         | _        |j        |         | _        |j        |         | _        t          j
        | j        | j        z  | j        ¦  «        | _        t          j        ¦   «         | _        d S )Nr   r   )r#   r$   Útdnn_dimr�  rŽ  Útdnn_kernelr9   Útdnn_dilationÚdilationrB   ra   ÚkernelÚReLUrR   r�  s      €r)   r$   zTDNNLayer.__init__n  s¡   ø€ Ý‰Œ×ÒÑÔÐØ<DÀqºL¸L˜6œ?¨8°a©<Ô8Ð8ÈfÌoÐ^fÔNgˆÔØ"œO¨HÔ5ˆÔØ!Ô-¨hÔ7ˆÔØÔ,¨XÔ6ˆŒå”i Ô 0°4Ô3CÑ CÀTÔEVÑWÔWˆŒÝœ'™)œ)ˆŒˆˆr*   r.   rˆ   c                 ó
  — t          ¦   «         rddlm} t          ¦   «         r)t          | j        |¦  «        rt          j        d¦  «         |                     dd¦  «        }| j        j         	                    | j
        | j        | j        ¦  «                             dd¦  «        }t          j                             ||| j        j        | j        ¬¦  «        }|                     dd¦  «        }|                      |¦  «        }|S )Nr   )Ú	LoraLayerz‡Detected LoRA on TDNNLayer. LoRA weights won't be applied due to optimization. You should exclude TDNNLayer from LoRA's target modules.r   r"   )ri  )r   Úpeft.tuners.lorarm  ræ   rj  ÚwarningsÚwarnrV   r>   r‘   rŽ  r9   r�  rB   r?  Úconv1dr¯   ri  rR   )r&   r.   rm  r>   s       r)   r/   zTDNNLayer.forwardx  sÿ   € ÝÑÔð 	3Ø2Ð2Ð2Ð2Ð2Ð2åÑÔð 	Ý˜$œ+ yÑ1Ô1ð Ý”ðOñô ð ð &×/Ò/°°1Ñ5Ô5ˆØ”Ô#×(Ò(¨Ô):¸DÔ<LÈdÔN^Ñ_Ô_×iÒiÐjkÐmnÑoÔoˆÝœ×,Ò,¨]¸FÀDÄKÔDTÐ_cÔ_lÐ,ÑmÔmˆØ%×/Ò/°°1Ñ5Ô5ˆàŸš¨Ñ6Ô6ˆØÐr*   r“  )r1   r2   r3   r$   r~   rÚ   r/   r4   r5   s   @r)   rd  rd  m  sc   ø€ € € € € ð$ð $ð $ð $ð $ð $ð U¤\ð °e´lð ð ð ð ð ð ð ð r*   rd  zi
    WavLM Model with an XVector feature extraction head on top for tasks like Speaker Verification.
    c                   óÚ   ‡ — e Zd Zˆ fd„Zd„ Zd„ Zdej        ez  fd„Z	e
ee	 	 ddej        dz  dej        dz  d	ej        dz  d
ee         deez  f
d„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚWavLMForXVectorc                 óÚ  •‡— t          ¦   «                              ‰¦  «         t          ‰¦  «        | _        ‰j        dz   }‰j        r.t          j        t          j	        |¦  «        |z  ¦  «        | _
        t          j        ‰j        ‰j        d         ¦  «        | _        ˆfd„t          t!          ‰j        ¦  «        ¦  «        D ¦   «         }t          j        |¦  «        | _        t          j        ‰j        d         dz  ‰j        ¦  «        | _        t          j        ‰j        ‰j        ¦  «        | _        t-          ‰j        ‰j        ¦  «        | _        |                      ¦   «          d S )Nr   r   c                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS rÐ  )rd  r
  s     €r)   r  z,WavLMForXVector.__init__.<locals>.<listcomp>œ  s#   ø€ ÐQÐQÐQ°•y ¨Ñ+Ô+ÐQÐQÐQr*   rZ   r"   )r#   r$   rå  rO  r  r8  rB   r}   r~   r   r9  ra   rD   rf  r;  r  rÖ  r  ÚtdnnÚxvector_output_dimrç  r=  rS  r<  Ú	objectiverî  )r&   rS   r?  Útdnn_layersr(   s    `  €r)   r$   zWavLMForXVector.__init__“  s'  øø€ Ý‰Œ×Ò˜Ñ Ô Ð å Ñ'Ô'ˆŒ
ØÔ-°Ñ1ˆ
ØÔ(ð 	SÝ!#¤­e¬j¸Ñ.DÔ.DÀzÑ.QÑ!RÔ!RˆDÔÝœ 6Ô#5°v´ÀqÔ7IÑJÔJˆŒàQÐQÐQÐQµU½3¸v¼Ñ;OÔ;OÑ5PÔ5PÐQÑQÔQˆÝ”M +Ñ.Ô.ˆŒ	å!#¤¨6¬?¸2Ô+>ÀÑ+BÀFÔD]Ñ!^Ô!^ˆÔÝœ) FÔ$=¸vÔ?XÑYÔYˆŒå& vÔ'@À&ÔBSÑTÔTˆŒà�ŠÑÔÐÐÐr*   c                 óB   — | j         j                             ¦   «          dS rð  r  rñ  s    r)   rò  z&WavLMForXVector.freeze_feature_encoder¦  r  r*   c                 óL   — | j                              ¦   «         D ]	}d|_        Œ
dS r  r  r®  s     r)   r  z!WavLMForXVector.freeze_base_model­  r  r*   re  c                 óD   — d„ }| j         j        D ]} |||d¦  «        }Œ|S )z?
        Computes the output length of the TDNN layers
        c                 ó   — | |z
  |z  dz   S )Nr   rÐ  rl  s      r)   ro  zBWavLMForXVector._get_tdnn_output_lengths.<locals>._conv_out_lengthº  s   € ð ! ;Ñ.°6Ñ9¸AÑ=Ð=r*   r   )rS   rg  )r&   re  ro  r9   s       r)   Ú_get_tdnn_output_lengthsz(WavLMForXVector._get_tdnn_output_lengthsµ  sE   € ð
	>ð 	>ð 	>ð
  œ;Ô2ð 	Lð 	LˆKØ,Ð,¨]¸KÈÑKÔKˆMˆMàÐr*   NrP  r„   r  r‡   rˆ   c                 ó¾  — | j         j        rd|d<    | j        |fd|i|¤Ž}| j         j        rr|j        }t	          j        |d¬¦  «        }t          j                             | j	        d¬¦  «        }|| 
                    ddd¦  «        z                       d¬¦  «        }n|j        }|                      |¦  «        }| j        D ]} ||¦  «        }Œ|€-|                     d¬¦  «        }	|                     d¬¦  «        }
nå|                      |                     d¬¦  «        ¦  «        }|                      |¦  «        }g }	g }
t'          |¦  «        D ]k\  }}|	                     ||d|…f                              d¬¦  «        ¦  «         |
                     ||d|…f                              d¬¦  «        ¦  «         Œlt	          j        |	¦  «        }	t	          j        |
¦  «        }
t	          j        |	|
gd¬¦  «        }|                      |¦  «        }|                      |¦  «        }d}|�|                      ||¦  «        }t3          ||||j        |j        ¬	¦  «        S )
rC  TrD  r„   r   r‹   rZ   Nr   )r#  r$  Ú
embeddingsr.   rT  )rS   r8  rO  r.   r~   rE  rB   r?  rB  r9  r‘   r”   r  r;  rv  r4  rV  ru  r~  r  rÙ  r®   rç  r=  rx  r   rT  )r&   rP  r„   r  r‡   r0  r.   rF  Ú
tdnn_layerÚmean_featuresÚstd_featuresÚfeat_extract_output_lengthsÚtdnn_output_lengthsr  ÚlengthÚstatistic_poolingÚoutput_embeddingsr$  r#  s                      r)   r/   zWavLMForXVector.forwardÄ  s¦  € ð, Œ;Ô-ð 	2Ø-1ˆFÐ)Ñ*à�$”*Øð
ð 
à)ð
ð ð
ð 
ˆð Œ;Ô-ð 	6Ø#Ô1ˆMÝ!œK¨¸1Ð=Ñ=Ô=ˆMÝœ=×0Ò0°Ô1CÈÐ0ÑLÔLˆLØ*¨\×->Ò->¸rÀ1ÀaÑ-HÔ-HÑH×MÒMÐRSÐMÑTÔTˆMˆMà#Ô5ˆMàŸš }Ñ5Ô5ˆàœ)ð 	6ð 	6ˆJØ&˜J }Ñ5Ô5ˆMˆMð Ð!Ø)×.Ò.°1Ð.Ñ5Ô5ˆMØ(×,Ò,°Ð,Ñ3Ô3ˆLˆLà*.×*OÒ*OÐP^×PbÒPbÐghÐPbÑPiÔPiÑ*jÔ*jÐ'Ø"&×"?Ò"?Ð@[Ñ"\Ô"\ÐØˆMØˆLÝ&Ð':Ñ;Ô;ð Jð J‘	��6Ø×$Ò$ ]°1°g°v°g°:Ô%>×%CÒ%CÈÐ%CÑ%JÔ%JÑKÔKÐKØ×#Ò# M°!°W°f°W°*Ô$=×$AÒ$AÀaÐ$AÑ$HÔ$HÑIÔIÐIÐIÝ!œK¨Ñ6Ô6ˆMÝ œ; |Ñ4Ô4ˆLÝ!œI }°lÐ&CÈÐLÑLÔLÐà ×2Ò2Ð3DÑEÔEÐØ—’Ð!2Ñ3Ô3ˆàˆØÐØ—>’> &¨&Ñ1Ô1ˆDåØØØ(Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r*   r  )r1   r2   r3   r$   rò  r  r~   rÝ   rØ   r~  r   r   r   rÚ   r   r   rÛ   r   r/   r4   r5   s   @r)   rs  rs  �  s  ø€ € € € € ðð ð ð ð ð&:ð :ð :ð(ð (ð (ð°eÔ6FÈÑ6Lð ð ð ð ð Ø Øð /3Ø&*ð	F
ð F
à”l TÑ)ðF
ð œ tÑ+ðF
ð ”˜tÑ#ð	F
ð
 Ð+Ô,ðF
ð 
�Ñ	ðF
ð F
ð F
ñ „^ñ !Ô ñ ÔðF
ð F
ð F
ð F
ð F
r*   rs  )rL  r  r6  rs  rå  rN  r,   )UrÍ   ro  ÚnumpyrÂ  r~   Útorch.nnrB   Útorch.nn.functionalr?  r«   r   Ú r   rZ  Úactivationsr   Úbackbone_utilsr   Úintegrations.deepspeedr   Úintegrations.fsdpr	   Úmodeling_layersr
   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   rG   r   r   r   r   Úutils.genericr   r   Úutils.output_capturingr   r   Úconfiguration_wavlmr   Ú
get_loggerr1   r  ÚModuler   r7   rX   rk   rà   rï   r   r  r$  r)  rN  rŠ  r•  rš  r¡  r´  r»  rÛ   rØ   rÌ   rÝ   Úndarrayrã  r  rå  r  r6  rL  rS  rd  rs  Ú__all__rÐ  r*   r)   ú<module>rœ     sk  ðð €€€Ø €€€à Ð Ð Ð Ø €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø %Ð %Ð %Ð %Ð %Ð %à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø @Ð @Ð @Ð @Ð @Ð @Ø 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ZÐ YÐ YÐ YÐ YÐ YÐ YÐ YØ &Ð &Ð &Ð &Ð &Ð &Ø SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SØ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ ,Ð ,Ð ,Ð ,Ð ,Ð ,ð 
ˆÔ	˜HÑ	%Ô	%€ðð ð ð ð ˜œ	ñ ô ð ð*ð *ð *ð *ð * 2¤9ñ *ô *ð *ðZ1ð 1ð 1ð 1ð 1˜RœYñ 1ô 1ð 1ðb ð b ð b ð b ð b �R”Yñ b ô b ð b ðJð ð ð ð �r”yñ ô ð ð0(,ð (,ð (,ð (,ð (,Ð2ñ (,ô (,ð (,ðV#,ð #,ð #,ð #,ð #,Ð'Añ #,ô #,ð #,ðL.@ð .@ð .@ð .@ð .@�2”9ñ .@ô .@ð .@ðb2@ð 2@ð 2@ð 2@ð 2@ "¤)ñ 2@ô 2@ð 2@ðjC'ð C'ð C'ð C'ð C' ¤ñ C'ô C'ð C'ðL ðPð Pð Pð Pð P˜?ñ Pô Pñ „ðPðfð ð ð ð Ð :ñ ô ð ð*ð ð ð ð Ð8ñ ô ð ð6ð ð ð ð Ð8ñ ô ð ð0#ð #ð #ð #ð #˜"œ)ñ #ô #ð #ðLð ð ð ð ˜œ	ñ ô ð ð$ð ð ð ð �2”9ñ ô ð ðF /3Øðtð tØ��c�Œ?ðtàðtð ðtð Ô$ tÑ+ð	tð
 ðtð „Zðtð tð tð tðn /Ð ð ðu
ð u
ð u
ð u
ð u
Ð%ñ u
ô u
ñ „ðu
ðp €ððñ ô ð
A
ð A
ð A
ð A
ð A
Ð&ñ A
ô A
ñô ð
A
ðH €ððñ ô ð]
ð ]
ð ]
ð ]
ð ]
Ð%9ñ ]
ô ]
ñô ð]
ð@ ðS
ð S
ð S
ð S
ð S
Ð';ñ S
ô S
ñ „ðS
ðlð ð ð ð �B”Iñ ô ð ð.ð ð ð ð �”	ñ ô ð ð@ €ððñ ô ð
{
ð {
ð {
ð {
ð {
Ð*ñ {
ô {
ñô ð
{
ð|ð ð €€€r*   