§
    9²¸j³Ú  ã                   óŽ  — d dl Z d dlmZ d dlmZ d dlZd dlZd dlm	Z	 d dlm
Z
 ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZmZmZm Z m!Z! ddl"m#Z#m$Z$m%Z% ddl&m'Z' ddl(m)Z)m*Z*m+Z+ ddl,m-Z-m.Z. ddl/m0Z0m1Z1 ddl2m3Z3  e+j4        e5¦  «        Z6 e*d¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z7 G d„ de	j8        ¦  «        Z9 G d„ de	j8        ¦  «        Z: G d„ de¦  «        Z; G d „ d!e¦  «        Z< G d"„ d#e¦  «        Z= G d$„ d%e	j8        ¦  «        Z> G d&„ d'e	j8        ¦  «        Z?	 	 dVd)e	j8        d*ej@        d+ej@        d,ej@        d-ej@        dz  d.eAdz  d/eAd0e'e)         fd1„ZB G d2„ d3e	j8        ¦  «        ZC G d4„ d5e	j8        ¦  «        ZD G d6„ d7e¦  «        ZE G d8„ d9e	j8        ¦  «        ZF G d:„ d;e	j8        ¦  «        ZG G d<„ d=e¦  «        ZH G d>„ d?e	j8        ¦  «        ZI G d@„ dAe	j8        ¦  «        ZJe* G dB„ dCe$¦  «        ¦   «         ZK	 	 dWdDeLeMeMf         dEeAdFeMd-ejN        dz  dGeMdHejO        fdI„ZPe!ZQe* G dJ„ dKeK¦  «        ¦   «         ZR e*dL¬¦  «         G dM„ dNeK¦  «        ¦   «         ZS e*dO¬¦  «         G dP„ dQeK¦  «        ¦   «         ZT e*dR¬¦  «         G dS„ dTeK¦  «        ¦   «         ZUg dU¢ZVdS )Xé    N)ÚCallable)Ú	dataclass)ÚCrossEntropyLossé   )Úinitialization)ÚACT2FN)Úfilter_output_hidden_states)Úis_deepspeed_zero3_enabled)Úis_fsdp_managed_module)Úcreate_bidirectional_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚCausalLMOutputÚModelOutputÚSequenceClassifierOutputÚWav2Vec2BaseModelOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModelÚ*get_torch_context_manager_or_global_device)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)Úcan_return_tupleÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚUniSpeechConfigzh
    Output type of [`UniSpeechForPreTrainingOutput`], with potential hidden states and attentions.
    )Úcustom_introc                   óà   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	ej        dz  ed<   dZ
ej        dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dS )	ÚUniSpeechForPreTrainingOutputaÝ  
    loss (*optional*, returned when model is in train mode, `torch.FloatTensor` of shape `(1,)`):
        Total loss as the sum of the contrastive loss (L_m) and the diversity loss (L_d) as stated in the [official
        paper](https://huggingface.co/papers/2006.11477).
    projected_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.proj_codevector_dim)`):
        Hidden-states of the model projected to *config.proj_codevector_dim* that can be used to predict the masked
        projected quantized states.
    projected_quantized_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.proj_codevector_dim)`):
        Quantized extracted feature vectors projected to *config.proj_codevector_dim* representing the positive
        target vectors for contrastive loss.
    codevector_perplexity (`torch.FloatTensor` of shape `(1,)`):
        The perplexity of the codevector distribution, used to measure the diversity of the codebook.
    NÚlossÚprojected_statesÚprojected_quantized_statesÚcodevector_perplexityÚhidden_statesÚ
attentions)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r$   ÚtorchÚFloatTensorÚ__annotations__r%   r&   r'   r(   Útupler)   © ó    ú€/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/unispeech/modeling_unispeech.pyr#   r#   8   s»   € € € € € € ðð ð &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø15Ð�eÔ'¨$Ñ.Ð5Ð5Ñ5Ø;?Ð Ô 1°DÑ 8Ð?Ð?Ñ?Ø6:Ð˜5Ô,¨tÑ3Ð:Ð:Ñ:Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ð6Ð6r3   r#   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚUniSpeechSamePadLayerc                 ól   •— t          ¦   «                              ¦   «          |dz  dk    rdnd| _        d S )Né   r   r   )ÚsuperÚ__init__Únum_pad_remove)ÚselfÚnum_conv_pos_embeddingsÚ	__class__s     €r4   r:   zUniSpeechSamePadLayer.__init__V   s:   ø€ Ý‰Œ×ÒÑÔÐØ#:¸QÑ#>À!Ò#CÐ#C˜a˜aÈˆÔÐÐr3   c                 óJ   — | j         dk    r|d d …d d …d | j          …f         }|S ©Nr   )r;   ©r<   r(   s     r4   ÚforwardzUniSpeechSamePadLayer.forwardZ   s;   € ØÔ Ò"Ð"Ø)¨!¨!¨!¨Q¨Q¨QÐ0F°4Ô3FÐ2FÐ0FÐ*FÔGˆMØÐr3   ©r*   r+   r,   r:   rB   Ú__classcell__©r>   s   @r4   r6   r6   U   sL   ø€ € € € € ðKð Kð Kð Kð Kðð ð ð ð ð ð r3   r6   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )Ú UniSpeechPositionalConvEmbeddingc                 óÊ  •— t          ¦   «                              ¦   «          t          j        |j        |j        |j        |j        dz  |j        ¬¦  «        | _        t          j        j	        }t          t          j        j        d¦  «        rt          j        j        j	        }t          ¦   «         rëdd l}|j                             | j        j        d¬¦  «        5   || j        dd¬¦  «        | _        d d d ¦  «         n# 1 swxY w Y   t          | j        d¦  «        r-| j        j        j        j        }| j        j        j        j        }n| j        j        }| j        j        }|j                             | |¦  «         |j                             | |¦  «         n || j        dd¬¦  «        | _        t-          |j        ¦  «        | _        t0          |j                 | _        d S )	Nr8   )Úkernel_sizeÚpaddingÚgroupsÚweight_normr   )Úmodifier_rankÚweight)ÚnameÚdimÚparametrizations)r9   r:   ÚnnÚConv1dÚhidden_sizer=   Únum_conv_pos_embedding_groupsÚconvÚutilsrL   ÚhasattrrQ   r
   Ú	deepspeedÚzeroÚGatheredParametersrN   Ú	original0Ú	original1Úweight_gÚweight_vÚregister_external_parameterr6   rJ   r   Úfeat_extract_activationÚ
activation)r<   ÚconfigrL   rY   r^   r_   r>   s         €r4   r:   z)UniSpeechPositionalConvEmbedding.__init__a   sþ  ø€ Ý‰Œ×ÒÑÔÐÝ”IØÔØÔØÔ6ØÔ2°aÑ7ØÔ7ð
ñ 
ô 
ˆŒ	õ ”hÔ*ˆÝ•2”8Ô,¨mÑ<Ô<ð 	@Ýœ(Ô3Ô?ˆKå%Ñ'Ô'ð 	EØÐÐÐà”×2Ò2°4´9Ô3CÐSTÐ2ÑUÔUð Ið IØ'˜K¨¬	¸ÀaÐHÑHÔH�”	ðIð Ið Iñ Iô Ið Ið Ið Ið Ið Ið Iøøøð Ið Ið Ið Iå�t”yÐ"4Ñ5Ô5ð .Øœ9Ô5Ô<ÔF�Øœ9Ô5Ô<ÔF��àœ9Ô-�Øœ9Ô-�ØŒN×6Ò6°t¸XÑFÔFÐFØŒN×6Ò6°t¸XÑFÔFÐFÐFà#˜ D¤I°HÀ!ÐDÑDÔDˆDŒIå,¨VÔ-KÑLÔLˆŒÝ  Ô!?Ô@ˆŒˆˆs   ÃC?Ã?DÄDc                 óÜ   — |                      dd¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      dd¦  «        }|S )Nr   r8   )Ú	transposerV   rJ   rb   rA   s     r4   rB   z(UniSpeechPositionalConvEmbedding.forward‚   se   € Ø%×/Ò/°°1Ñ5Ô5ˆàŸ	š	 -Ñ0Ô0ˆØŸš ]Ñ3Ô3ˆØŸš¨Ñ6Ô6ˆà%×/Ò/°°1Ñ5Ô5ˆØÐr3   rC   rE   s   @r4   rG   rG   `   sM   ø€ € € € € ðAð Að Að Að AðBð ð ð ð ð ð r3   rG   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚUniSpeechNoLayerNormConvLayerr   c                 óZ  •— t          ¦   «                              ¦   «          |dk    r|j        |dz
           nd| _        |j        |         | _        t          j        | j        | j        |j        |         |j        |         |j	        ¬¦  «        | _
        t          |j                 | _        d S )Nr   r   ©rI   ÚstrideÚbias)r9   r:   Úconv_dimÚin_conv_dimÚout_conv_dimrR   rS   Úconv_kernelÚconv_strideÚ	conv_biasrV   r   ra   rb   ©r<   rc   Úlayer_idr>   s      €r4   r:   z&UniSpeechNoLayerNormConvLayer.__init__Ž   s�   ø€ Ý‰Œ×ÒÑÔÐØ<DÀqºL¸L˜6œ?¨8°a©<Ô8Ð8ÈaˆÔØ"œO¨HÔ5ˆÔå”IØÔØÔØÔ*¨8Ô4ØÔ% hÔ/ØÔ!ð
ñ 
ô 
ˆŒ	õ ! Ô!?Ô@ˆŒˆˆr3   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S ©N)rV   rb   rA   s     r4   rB   z%UniSpeechNoLayerNormConvLayer.forwardœ   s*   € ØŸ	š	 -Ñ0Ô0ˆØŸš¨Ñ6Ô6ˆØÐr3   ©r   rC   rE   s   @r4   rg   rg   �   sR   ø€ € € € € ðAð Að Að Að Að Aðð ð ð ð ð ð r3   rg   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚUniSpeechLayerNormConvLayerr   c                 óš  •— t          ¦   «                              ¦   «          |dk    r|j        |dz
           nd| _        |j        |         | _        t          j        | j        | j        |j        |         |j        |         |j	        ¬¦  «        | _
        t          j        | j        d¬¦  «        | _        t          |j                 | _        d S )Nr   r   ri   T)Úelementwise_affine)r9   r:   rl   rm   rn   rR   rS   ro   rp   rq   rV   Ú	LayerNormÚ
layer_normr   ra   rb   rr   s      €r4   r:   z$UniSpeechLayerNormConvLayer.__init__£   s¶   ø€ Ý‰Œ×ÒÑÔÐØ<DÀqºL¸L˜6œ?¨8°a©<Ô8Ð8ÈaˆÔØ"œO¨HÔ5ˆÔå”IØÔØÔØÔ*¨8Ô4ØÔ% hÔ/ØÔ!ð
ñ 
ô 
ˆŒ	õ œ, tÔ'8ÈTÐRÑRÔRˆŒÝ  Ô!?Ô@ˆŒˆˆr3   c                 óÜ   — |                       |¦  «        }|                     dd¦  «        }|                      |¦  «        }|                     dd¦  «        }|                      |¦  «        }|S )Néþÿÿÿéÿÿÿÿ)rV   re   r|   rb   rA   s     r4   rB   z#UniSpeechLayerNormConvLayer.forward²   se   € ØŸ	š	 -Ñ0Ô0ˆà%×/Ò/°°BÑ7Ô7ˆØŸš¨Ñ6Ô6ˆØ%×/Ò/°°BÑ7Ô7ˆàŸš¨Ñ6Ô6ˆØÐr3   rv   rC   rE   s   @r4   rx   rx   ¢   sR   ø€ € € € € ðAð Að Að Að Að Aðð ð ð ð ð ð r3   rx   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚUniSpeechGroupNormConvLayerr   c                 ó¦  •— t          ¦   «                              ¦   «          |dk    r|j        |dz
           nd| _        |j        |         | _        t          j        | j        | j        |j        |         |j        |         |j	        ¬¦  «        | _
        t          |j                 | _        t          j        | j        | j        d¬¦  «        | _        d S )Nr   r   ri   T)Ú
num_groupsÚnum_channelsÚaffine)r9   r:   rl   rm   rn   rR   rS   ro   rp   rq   rV   r   ra   rb   Ú	GroupNormr|   rr   s      €r4   r:   z$UniSpeechGroupNormConvLayer.__init__¾   s½   ø€ Ý‰Œ×ÒÑÔÐØ<DÀqºL¸L˜6œ?¨8°a©<Ô8Ð8ÈaˆÔØ"œO¨HÔ5ˆÔå”IØÔØÔØÔ*¨8Ô4ØÔ% hÔ/ØÔ!ð
ñ 
ô 
ˆŒ	õ ! Ô!?Ô@ˆŒåœ,°$Ô2CÐRVÔRcÐlpÐqÑqÔqˆŒˆˆr3   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S ru   )rV   r|   rb   rA   s     r4   rB   z#UniSpeechGroupNormConvLayer.forwardÎ   s;   € ØŸ	š	 -Ñ0Ô0ˆØŸš¨Ñ6Ô6ˆØŸš¨Ñ6Ô6ˆØÐr3   rv   rC   rE   s   @r4   r�   r�   ½   sR   ø€ € € € € ðrð rð rð rð rð rð ð ð ð ð ð ð r3   r�   c                   ó.   ‡ — e Zd ZdZˆ fd„Zd„ Zd„ Zˆ xZS )ÚUniSpeechFeatureEncoderz.Construct the features from raw audio waveformc                 ó¤  •‡— t          ¦   «                              ¦   «          ‰j        dk    r7t          ‰d¬¦  «        gˆfd„t	          ‰j        dz
  ¦  «        D ¦   «         z   }nD‰j        dk    r!ˆfd„t	          ‰j        ¦  «        D ¦   «         }nt          d‰j        › d	�¦  «        ‚t          j        |¦  «        | _	        d
| _
        d| _        d S )NÚgroupr   ©rs   c                 ó8   •— g | ]}t          ‰|d z   ¬¦  «        ‘ŒS )r   rŒ   )rg   ©Ú.0Úirc   s     €r4   ú
<listcomp>z4UniSpeechFeatureEncoder.__init__.<locals>.<listcomp>Ü   s@   ø€ ð Oð Oð Oàõ .¨f¸qÀ1¹uÐEÑEÔEðOð Oð Or3   r   Úlayerc                 ó2   •— g | ]}t          ‰|¬ ¦  «        ‘ŒS )rŒ   )rx   rŽ   s     €r4   r‘   z4UniSpeechFeatureEncoder.__init__.<locals>.<listcomp>á   s4   ø€ ð ð ð ØDEÕ+¨F¸QÐ?Ñ?Ô?ðð ð r3   z`config.feat_extract_norm` is z), but has to be one of ['group', 'layer']FT)r9   r:   Úfeat_extract_normr�   ÚrangeÚnum_feat_extract_layersÚ
ValueErrorrR   Ú
ModuleListÚconv_layersÚgradient_checkpointingÚ_requires_grad)r<   rc   r™   r>   s    ` €r4   r:   z UniSpeechFeatureEncoder.__init__Ø   s  øø€ Ý‰Œ×ÒÑÔÐàÔ# wÒ.Ð.Ý6°vÈÐJÑJÔJÐKð Oð Oð Oð Oå˜vÔ=ÀÑAÑBÔBðOñ Oô Oñ ˆKˆKð Ô%¨Ò0Ð0ðð ð ð ÝINÈvÔOmÑInÔInðñ ô ˆKˆKõ Øt°Ô1IÐtÐtÐtñô ð õ œ=¨Ñ5Ô5ˆÔØ&+ˆÔ#Ø"ˆÔÐÐr3   c                 óP   — |                       ¦   «         D ]	}d|_        Œ
d| _        d S )NF)Ú
parametersÚrequires_gradr›   ©r<   Úparams     r4   Ú_freeze_parametersz*UniSpeechFeatureEncoder._freeze_parametersì   s4   € Ø—_’_Ñ&Ô&ð 	(ð 	(ˆEØ"'ˆEÔÐØ#ˆÔÐÐr3   c                 ór   — |d d …d f         }| j         r| j        rd|_        | j        D ]} ||¦  «        }Œ|S )NT)r›   Útrainingrž   r™   )r<   Úinput_valuesr(   Ú
conv_layers       r4   rB   zUniSpeechFeatureEncoder.forwardñ   s[   € Ø$ Q Q Q¨ WÔ-ˆð Ôð 	/ 4¤=ð 	/Ø*.ˆMÔ'àÔ*ð 	6ð 	6ˆJØ&˜J }Ñ5Ô5ˆMˆMàÐr3   )r*   r+   r,   r-   r:   r¡   rB   rD   rE   s   @r4   r‰   r‰   Õ   s\   ø€ € € € € Ø8Ð8ð#ð #ð #ð #ð #ð($ð $ð $ð

ð 
ð 
ð 
ð 
ð 
ð 
r3   r‰   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚUniSpeechFeatureProjectionc                 ó.  •— t          ¦   «                              ¦   «          t          j        |j        d         |j        ¬¦  «        | _        t          j        |j        d         |j        ¦  «        | _	        t          j
        |j        ¦  «        | _        d S )Nr   ©Úeps)r9   r:   rR   r{   rl   Úlayer_norm_epsr|   ÚLinearrT   Ú
projectionÚDropoutÚfeat_proj_dropoutÚdropout©r<   rc   r>   s     €r4   r:   z#UniSpeechFeatureProjection.__init__ÿ   sn   ø€ Ý‰Œ×ÒÑÔÐÝœ, v¤°rÔ':ÀÔ@UÐVÑVÔVˆŒÝœ) F¤O°BÔ$7¸Ô9KÑLÔLˆŒÝ”z &Ô":Ñ;Ô;ˆŒˆˆr3   c                 óˆ   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }||fS ru   )r|   r­   r°   )r<   r(   Únorm_hidden_statess      r4   rB   z"UniSpeechFeatureProjection.forward  sC   € à!Ÿ_š_¨]Ñ;Ô;ÐØŸšÐ(:Ñ;Ô;ˆØŸš ]Ñ3Ô3ˆØÐ0Ð0Ð0r3   rC   rE   s   @r4   r§   r§   þ   sG   ø€ € € € € ð<ð <ð <ð <ð <ð1ð 1ð 1ð 1ð 1ð 1ð 1r3   r§   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingr°   Úkwargsc                 ó®  — |€|                      d¦  «        dz  }t          j        ||                     dd¦  «        ¦  «        |z  }|�||z   }t          j                             |d¬¦  «        }t          j                             ||| j        ¬¦  «        }t          j        ||¦  «        }	|	                     dd¦  «         	                    ¦   «         }	|	|fS )Nr   ç      à¿r8   r   ©rP   )Úpr£   r   )
Úsizer.   Úmatmulre   rR   Ú
functionalÚsoftmaxr°   r£   Ú
contiguous)
rµ   r¶   r·   r¸   r¹   rº   r°   r»   Úattn_weightsÚattn_outputs
             r4   Úeager_attention_forwardrÇ     sÈ   € ð €Ø—*’*˜R‘.”. DÑ(ˆõ ”<  s§}¢}°Q¸Ñ':Ô':Ñ;Ô;¸gÑE€LàÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2Ð(Ñ>Ô>€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r3   c                   óà   ‡ — e Zd ZdZ	 	 	 	 	 ddededed	ed
edededz  fˆ fd„Z	 	 	 dde	j
        de	j
        dz  de	j
        dz  dedz  dee         dee	j
        e	j
        dz  f         fd„Zˆ xZS )ÚUniSpeechAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr´   FTNÚ	embed_dimÚ	num_headsr°   Ú
is_decoderrk   Ú	is_causalrc   c                 ó
  •— t          ¦   «                              ¦   «          || _        || _        || _        ||z  | _        || _        | j        |z  | j        k    rt          d| j        › d|› d�¦  «        ‚| j        dz  | _        || _	        || _
        t          j        |||¬¦  «        | _        t          j        |||¬¦  «        | _        t          j        |||¬¦  «        | _        t          j        |||¬¦  «        | _        d S )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).r½   )rk   )r9   r:   rÊ   rË   r°   Úhead_dimrc   r—   rº   rÌ   rÍ   rR   r¬   Úk_projÚv_projÚq_projÚout_proj)	r<   rÊ   rË   r°   rÌ   rk   rÍ   rc   r>   s	           €r4   r:   zUniSpeechAttention.__init__,  s  ø€ õ 	‰Œ×ÒÑÔÐØ"ˆŒØ"ˆŒØˆŒØ! YÑ.ˆŒØˆŒàŒM˜IÑ%¨$¬.Ò8Ð8Ýð3ÈdÌnð 3ð 3Ø%.ð3ð 3ð 3ñô ð ð ”} dÑ*ˆŒØ$ˆŒØ"ˆŒå”i 	¨9¸4Ð@Ñ@Ô@ˆŒÝ”i 	¨9¸4Ð@Ñ@Ô@ˆŒÝ”i 	¨9¸4Ð@Ñ@Ô@ˆŒÝœ	 )¨Y¸TÐBÑBÔBˆŒˆˆr3   r(   Úkey_value_statesr¹   Úoutput_attentionsr»   Úreturnc                 óø  — |du}|j         dd…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }	|r|n|}
g |
j         dd…         ¢d‘| j        ‘R }|                      |
¦  «                             |¦  «                             dd¦  «        }|                      |
¦  «                             |¦  «                             dd¦  «        }t          j        | j	        j
        t          ¦  «        } || |	|||f| j        sdn| j        | j        |dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )z#Input shape: Batch x Time x ChannelNr   r   r8   r´   )r°   rº   rÕ   )ÚshaperÏ   rÒ   Úviewre   rÐ   rÑ   r   Úget_interfacerc   Ú_attn_implementationrÇ   r£   r°   rº   ÚreshaperÄ   rÓ   )r<   r(   rÔ   r¹   rÕ   r»   Úis_cross_attentionÚinput_shapeÚhidden_shapeÚquery_statesÚcurrent_statesÚkv_shapeÚ
key_statesÚvalue_statesÚattention_interfacerÆ   rÅ   s                    r4   rB   zUniSpeechAttention.forwardK  s»  € ð .°TÐ9Ðð $Ô)¨#¨2¨#Ô.ˆà8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆð —{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆà-?ÐRÐ)Ð)À]ˆØB�^Ô)¨#¨2¨#Ô.ÐB°ÐB°D´MÐBÐBˆØ—[’[ Ñ0Ô0×5Ò5°hÑ?Ô?×IÒIÈ!ÈQÑOÔOˆ
Ø—{’{ >Ñ2Ô2×7Ò7¸ÑAÔA×KÒKÈAÈqÑQÔQˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð  $œ}Ð>�C�C°$´,Ø”LØ/ð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—m’m KÑ0Ô0ˆà˜LÐ(Ð(r3   )r´   FTFN)NNF)r*   r+   r,   r-   ÚintÚfloatÚboolr    r:   r.   ÚTensorr   r   r1   rB   rD   rE   s   @r4   rÉ   rÉ   )  s<  ø€ € € € € ØGÐGð Ø ØØØ)-ðCð CàðCð ðCð ð	Cð
 ðCð ðCð ðCð   $Ñ&ðCð Cð Cð Cð Cð CðD 15Ø.2Ø).ð0)ð 0)à”|ð0)ð  œ,¨Ñ-ð0)ð œ tÑ+ð	0)ð
   $™;ð0)ð Ð-Ô.ð0)ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð0)ð 0)ð 0)ð 0)ð 0)ð 0)ð 0)ð 0)r3   rÉ   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚUniSpeechFeedForwardc                 óÌ  •— t          ¦   «                              ¦   «          t          j        |j        ¦  «        | _        t          j        |j        |j        ¦  «        | _	        t          |j        t          ¦  «        rt          |j                 | _        n|j        | _        t          j        |j        |j        ¦  «        | _        t          j        |j        ¦  «        | _        d S ru   )r9   r:   rR   r®   Úactivation_dropoutÚintermediate_dropoutr¬   rT   Úintermediate_sizeÚintermediate_denseÚ
isinstanceÚ
hidden_actÚstrr   Úintermediate_act_fnÚoutput_denseÚhidden_dropoutÚoutput_dropoutr±   s     €r4   r:   zUniSpeechFeedForward.__init__  s°   ø€ Ý‰Œ×ÒÑÔÐÝ$&¤J¨vÔ/HÑ$IÔ$IˆÔ!å"$¤)¨FÔ,>ÀÔ@XÑ"YÔ"YˆÔÝ�fÔ'­Ñ-Ô-ð 	9Ý'-¨fÔ.?Ô'@ˆDÔ$Ð$à'-Ô'8ˆDÔ$åœI fÔ&>ÀÔ@RÑSÔSˆÔÝ œj¨Ô)>Ñ?Ô?ˆÔÐÐr3   c                 óØ   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S ru   )rð   rô   rî   rõ   r÷   rA   s     r4   rB   zUniSpeechFeedForward.forwardŒ  sg   € Ø×/Ò/°Ñ>Ô>ˆØ×0Ò0°Ñ?Ô?ˆØ×1Ò1°-Ñ@Ô@ˆà×)Ò)¨-Ñ8Ô8ˆØ×+Ò+¨MÑ:Ô:ˆØÐr3   rC   rE   s   @r4   rë   rë   ~  sL   ø€ € € € € ð@ð @ð @ð @ð @ðð ð ð ð ð ð r3   rë   c                   ó\   ‡ — e Zd Zˆ fd„Z	 ddej        dej        dz  dee         fd„Zˆ xZ	S )ÚUniSpeechEncoderLayerc                 ó�  •— t          ¦   «                              ¦   «          t          |j        |j        |j        d|¬¦  «        | _        t          j        |j	        ¦  «        | _
        t          j        |j        |j        ¬¦  «        | _        t          |¦  «        | _        t          j        |j        |j        ¬¦  «        | _        d S )NF©rÊ   rË   r°   rÌ   rc   r©   )r9   r:   rÉ   rT   Únum_attention_headsÚattention_dropoutÚ	attentionrR   r®   rö   r°   r{   r«   r|   rë   Úfeed_forwardÚfinal_layer_normr±   s     €r4   r:   zUniSpeechEncoderLayer.__init__—  s¬   ø€ Ý‰Œ×ÒÑÔÐÝ+ØÔ(ØÔ0ØÔ,ØØð
ñ 
ô 
ˆŒõ ”z &Ô"7Ñ8Ô8ˆŒÝœ, vÔ'9¸vÔ?TÐUÑUÔUˆŒÝ0°Ñ8Ô8ˆÔÝ "¤¨VÔ-?ÀVÔEZÐ [Ñ [Ô [ˆÔÐÐr3   Nr(   r¹   r»   c                 óè   — |} | j         |fd|i|¤Ž\  }}|                      |¦  «        }||z   }|                      |¦  «        }||                      |¦  «        z   }|                      |¦  «        }|S ©Nr¹   )rÿ   r°   r|   r   r  ©r<   r(   r¹   r»   Úattn_residualÚ_s         r4   rB   zUniSpeechEncoderLayer.forward¦  sŠ   € ð &ˆØ)˜4œ>¨-ÐaÐaÈÐaÐZ`ÐaÐaÑˆ�qØŸš ]Ñ3Ô3ˆØ%¨Ñ5ˆàŸš¨Ñ6Ô6ˆØ%¨×(9Ò(9¸-Ñ(HÔ(HÑHˆØ×-Ò-¨mÑ<Ô<ˆàÐr3   ru   ©
r*   r+   r,   r:   r.   ré   r   r   rB   rD   rE   s   @r4   rú   rú   –  s†   ø€ € € € € ð\ð \ð \ð \ð \ð$ /3ðð à”|ðð œ tÑ+ðð Ð+Ô,ð	ð ð ð ð ð ð ð r3   rú   c                   ó\   ‡ — e Zd Zˆ fd„Z	 ddej        dej        dz  dee         fd„Z	ˆ xZ
S )ÚUniSpeechEncoderc                 ó‚  •‡— t          ¦   «                              ¦   «          ‰| _        t          ‰¦  «        | _        t          j        ‰j        ‰j        ¬¦  «        | _	        t          j
        ‰j        ¦  «        | _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        d| _        d S )Nr©   c                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r2   )rú   ©r�   r  rc   s     €r4   r‘   z-UniSpeechEncoder.__init__.<locals>.<listcomp>¿  s"   ø€ Ð$lÐ$lÐ$lÀqÕ%:¸6Ñ%BÔ%BÐ$lÐ$lÐ$lr3   F©r9   r:   rc   rG   Úpos_conv_embedrR   r{   rT   r«   r|   r®   rö   r°   r˜   r•   Únum_hidden_layersÚlayersrš   r±   s    `€r4   r:   zUniSpeechEncoder.__init__¹  s¡   øø€ Ý‰Œ×ÒÑÔÐØˆŒÝ>¸vÑFÔFˆÔÝœ, vÔ'9¸vÔ?TÐUÑUÔUˆŒÝ”z &Ô"7Ñ8Ô8ˆŒÝ”mÐ$lÐ$lÐ$lÐ$lÍEÐRXÔRjÑLkÔLkÐ$lÑ$lÔ$lÑmÔmˆŒØ&+ˆÔ#Ð#Ð#r3   Nr(   r¹   r»   c                 óD  — |�;|                      d¦  «                             dd|j        d         ¦  «        }d|| <   t          | j        ||¬¦  «        }|                      |¦  «        }||                     |j        ¦  «        z   }|                      |¦  «        }|  	                    |¦  «        }t          ¦   «         pt          | ¦  «        }| j        D ]<}t          j        g ¦  «        }| j        o|| j        j        k     }	|	r|r ||fd|i|¤Ž}Œ=t#          |¬¦  «        S ©Nr   r   r8   r   )rc   Úinputs_embedsr¹   r¹   )Úlast_hidden_state)Ú	unsqueezeÚrepeatrØ   r   rc   r  ÚtoÚdevicer|   r°   r
   r   r  r.   Úrandr£   Ú	layerdropr   ©
r<   r(   r¹   r»   Úexpand_attention_maskÚposition_embeddingsÚsynced_gpusr’   Údropout_probabilityÚskip_the_layers
             r4   rB   zUniSpeechEncoder.forwardÂ  sK  € ð Ð%à$2×$<Ò$<¸RÑ$@Ô$@×$GÒ$GÈÈ1ÈmÔNaÐbcÔNdÑ$eÔ$eÐ!Ø45ˆMÐ0Ð0Ñ1å2Ø”;Ø'Ø)ð
ñ 
ô 
ˆð #×1Ò1°-Ñ@Ô@ÐØ%Ð(;×(>Ò(>¸}Ô?SÑ(TÔ(TÑTˆØŸš¨Ñ6Ô6ˆØŸš ]Ñ3Ô3ˆå0Ñ2Ô2ÐRÕ6LÈTÑ6RÔ6Rˆà”[ð 	^ð 	^ˆEå"'¤*¨R¡.¤.Ðà!œ]ÐZÐ/BÀTÄ[ÔEZÒ/ZˆNØ!ð ^ [ð ^à %  mÐ ]Ð ]ÀNÐ ]ÐV\Ð ]Ð ]�øå°Ð?Ñ?Ô?Ð?r3   ru   )r*   r+   r,   r:   r.   Útensorré   r   r   rB   rD   rE   s   @r4   r	  r	  ¸  s�   ø€ € € € € ð,ð ,ð ,ð ,ð ,ð /3ð!@ð !@à”|ð!@ð œ tÑ+ð!@ð Ð+Ô,ð	!@ð !@ð !@ð !@ð !@ð !@ð !@ð !@r3   r	  c                   ó4   ‡ — e Zd Zˆ fd„Zdej        fd„Zˆ xZS )ÚUniSpeechAttnAdapterLayerc                 ót  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        t          j        | j        ¦  «        | _        t          j	        | j        | j        ¦  «        | _
        t          j        ¦   «         | _        t          j	        | j        | j        ¦  «        | _        dS )zŸ
        Implements adapter modules directly with 3D tensor weight as parameters and without using ModuleList to speed
        up training throughput.
        N)r9   r:   Úadapter_attn_dimÚ	input_dimrT   Ú
hidden_dimrR   r{   Únormr¬   Úlinear_1ÚReLUÚact_fnÚlinear_2r±   s     €r4   r:   z"UniSpeechAttnAdapterLayer.__init__ç  s�   ø€ õ
 	‰Œ×ÒÑÔÐØÔ0ˆŒØ Ô,ˆŒå”L ¤Ñ1Ô1ˆŒ	Ýœ	 $¤/°4´>ÑBÔBˆŒÝ”g‘i”iˆŒÝœ	 $¤.°$´/ÑBÔBˆŒˆˆr3   r(   c                 ó®   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S ru   )r(  r)  r+  r,  rA   s     r4   rB   z!UniSpeechAttnAdapterLayer.forwardõ  sL   € ØŸ	š	 -Ñ0Ô0ˆàŸš mÑ4Ô4ˆØŸš MÑ2Ô2ˆØŸš mÑ4Ô4ˆàÐr3   )r*   r+   r,   r:   r.   r/   rB   rD   rE   s   @r4   r#  r#  æ  s[   ø€ € € € € ðCð Cð Cð Cð Cð UÔ%6ð ð ð ð ð ð ð ð r3   r#  c                   ó\   ‡ — e Zd Zˆ fd„Z	 ddej        dej        dz  dee         fd„Zˆ xZ	S )Ú$UniSpeechEncoderLayerStableLayerNormc                 óì  •— t          ¦   «                              ¦   «          t          |j        |j        |j        d|¬¦  «        | _        t          j        |j	        ¦  «        | _
        t          j        |j        |j        ¬¦  «        | _        t          |¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t#          |dd ¦  «        �t%          |¦  «        | _        d S d | _        d S )NFrü   r©   r%  )r9   r:   rÉ   rT   rý   rþ   rÿ   rR   r®   rö   r°   r{   r«   r|   rë   r   r  Úgetattrr#  Úadapter_layerr±   s     €r4   r:   z-UniSpeechEncoderLayerStableLayerNorm.__init__   sÝ   ø€ Ý‰Œ×ÒÑÔÐÝ+ØÔ(ØÔ0ØÔ,ØØð
ñ 
ô 
ˆŒõ ”z &Ô"7Ñ8Ô8ˆŒÝœ, vÔ'9¸vÔ?TÐUÑUÔUˆŒÝ0°Ñ8Ô8ˆÔÝ "¤¨VÔ-?ÀVÔEZÐ [Ñ [Ô [ˆÔå�6Ð-¨tÑ4Ô4Ð@Ý!:¸6Ñ!BÔ!BˆDÔÐÐà!%ˆDÔÐÐr3   Nr(   r¹   r»   c                 ó"  — |}|                       |¦  «        } | j        |fd|i|¤Ž\  }}|                      |¦  «        }||z   }||                      |                      |¦  «        ¦  «        z   }| j        �||                      |¦  «        z   }|S r  )r|   rÿ   r°   r   r  r2  r  s         r4   rB   z,UniSpeechEncoderLayerStableLayerNorm.forward  s§   € ð &ˆØŸš¨Ñ6Ô6ˆØ)˜4œ>¨-ÐaÐaÈÐaÐZ`ÐaÐaÑˆ�qØŸš ]Ñ3Ô3ˆØ%¨Ñ5ˆØ%¨×(9Ò(9¸$×:OÒ:OÐP]Ñ:^Ô:^Ñ(_Ô(_Ñ_ˆàÔÐ)Ø)¨D×,>Ò,>¸}Ñ,MÔ,MÑMˆMàÐr3   ru   r  rE   s   @r4   r/  r/  ÿ  s�   ø€ € € € € ð&ð &ð &ð &ð &ð, /3ðð à”|ðð œ tÑ+ðð Ð+Ô,ð	ð ð ð ð ð ð ð r3   r/  c                   ó:   ‡ — e Zd Zˆ fd„Z	 ddee         fd„Zˆ xZS )ÚUniSpeechEncoderStableLayerNormc                 ó‚  •‡— t          ¦   «                              ¦   «          ‰| _        t          ‰¦  «        | _        t          j        ‰j        ‰j        ¬¦  «        | _	        t          j
        ‰j        ¦  «        | _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        d| _        d S )Nr©   c                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r2   )r/  r  s     €r4   r‘   z<UniSpeechEncoderStableLayerNorm.__init__.<locals>.<listcomp>.  s"   ø€ ÐcÐcÐc¸aÕ1°&Ñ9Ô9ÐcÐcÐcr3   Fr  r±   s    `€r4   r:   z(UniSpeechEncoderStableLayerNorm.__init__'  s¦   øø€ Ý‰Œ×ÒÑÔÐØˆŒÝ>¸vÑFÔFˆÔÝœ, vÔ'9¸vÔ?TÐUÑUÔUˆŒÝ”z &Ô"7Ñ8Ô8ˆŒÝ”mØcÐcÐcÐcÅ5ÈÔIaÑCbÔCbÐcÑcÔcñ
ô 
ˆŒð ',ˆÔ#Ð#Ð#r3   Nr»   c                 ó  — |�;|                      d¦  «                             dd|j        d         ¦  «        }d|| <   t          | j        ||¬¦  «        }|                      |¦  «        }||z   }|                      |¦  «        }t          ¦   «         pt          | ¦  «        }| j	        D ]<}t          j        g ¦  «        }| j        o|| j        j        k     }	|	r|r ||fd|i|¤Ž}Œ=|                      |¦  «        }t          |¬¦  «        S r  )r  r  rØ   r   rc   r  r°   r
   r   r  r.   r  r£   r  r|   r   r  s
             r4   rB   z'UniSpeechEncoderStableLayerNorm.forward2  s<  € ð Ð%à$2×$<Ò$<¸RÑ$@Ô$@×$GÒ$GÈÈ1ÈmÔNaÐbcÔNdÑ$eÔ$eÐ!Ø45ˆMÐ0Ð0Ñ1å2Ø”;Ø'Ø)ð
ñ 
ô 
ˆð #×1Ò1°-Ñ@Ô@ÐØ%Ð(;Ñ;ˆØŸš ]Ñ3Ô3ˆå0Ñ2Ô2ÐRÕ6LÈTÑ6RÔ6Rˆà”[ð 	^ð 	^ˆEå"'¤*¨R¡.¤.Ðà!œ]ÐZÐ/BÀTÄ[ÔEZÒ/ZˆNØ!ð ^ [ð ^ð !&  mÐ ]Ð ]ÀNÐ ]ÐV\Ð ]Ð ]�øàŸš¨Ñ6Ô6ˆå°Ð?Ñ?Ô?Ð?r3   ru   )r*   r+   r,   r:   r   r   rB   rD   rE   s   @r4   r5  r5  &  sp   ø€ € € € € ð	,ð 	,ð 	,ð 	,ð 	,ð ð#@ð #@ð Ð+Ô,ð	#@ð #@ð #@ð #@ð #@ð #@ð #@ð #@r3   r5  c                   ó>   ‡ — e Zd ZdZˆ fd„Zed„ ¦   «         Zd„ Zˆ xZS )ÚUniSpeechGumbelVectorQuantizerz±
    Vector quantization using gumbel softmax. See `[CATEGORICAL REPARAMETERIZATION WITH
    GUMBEL-SOFTMAX](https://huggingface.co/papers/1611.01144) for more information.
    c                 óÞ  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | j        z  dk    r t          d|j        › d| j        › d�¦  «        ‚t          j	        t          j        d| j        | j        z  |j        | j        z  ¦  «        ¦  «        | _        t          j        |j        d         | j        | j        z  ¦  «        | _        d| _        d S )Nr   z`config.codevector_dim z5 must be divisible by `config.num_codevector_groups` z for concatenationr   r   r8   )r9   r:   Únum_codevector_groupsrƒ   Únum_codevectors_per_groupÚnum_varsÚcodevector_dimr—   rR   Ú	Parameterr.   r/   Úcodevectorsr¬   rl   Úweight_projÚtemperaturer±   s     €r4   r:   z'UniSpeechGumbelVectorQuantizer.__init__^  sõ   ø€ Ý‰Œ×ÒÑÔÐØ Ô6ˆŒØÔ8ˆŒàÔ  4¤?Ñ2°aÒ7Ð7ÝðY¨&Ô*?ð Yð YØ59´_ðYð Yð Yñô ð õ œ<ÝÔ˜a ¤°4´=Ñ!@À&ÔBWÐ[_Ô[jÑBjÑkÔkñ
ô 
ˆÔõ œ9 V¤_°RÔ%8¸$¼/ÈDÌMÑ:YÑZÔZˆÔð ˆÔÐÐr3   c                 óÎ   — |                       d¬¦  «        }t          j        t          j        t          j        ||¦  «        d¬¦  «         ¦  «                             ¦   «         }|S )Nr   r¾   r   )Úmeanr.   ÚexpÚsumÚxlogy)ÚprobsÚmarginal_probsÚ
perplexitys      r4   Ú_compute_perplexityz2UniSpeechGumbelVectorQuantizer._compute_perplexityr  sU   € àŸš¨˜Ñ*Ô*ˆÝ”Y¥¤	­%¬+°nÀnÑ*UÔ*UÐ[]Ð ^Ñ ^Ô ^Ð^Ñ_Ô_×cÒcÑeÔeˆ
ØÐr3   c                 óö  — |j         \  }}}|                      |¦  «        }|                     ||z  | j        z  d¦  «        }| j        r§t
          j                             |                     ¦   «         | j	        d¬¦  «         
                    |¦  «        }t          j        |                     ||z  | j        d¦  «                             ¦   «         d¬¦  «        }|                      |¦  «        }n‚|                     d¬¦  «        } |j        |j         Ž                      d|                     dd¦  «        d¦  «        }|                     ||z  | j        d¦  «        }|                      |¦  «        }|                     ||z  d¦  «        }|                     d¦  «        | j        z  }	|	                     ||z  | j        | j        d¦  «        }
|
                     d¦  «                             ||d¦  «        }
|
|fS )Nr   T)ÚtauÚhardr¾   r   g      ð?r~   )rØ   rB  rÙ   rƒ   r£   rR   rÂ   Úgumbel_softmaxrç   rC  Útype_asr.   rÃ   rL  ÚargmaxÚ	new_zerosÚscatter_r  rA  r>  rG  )r<   r(   Ú
batch_sizeÚsequence_lengthrT   Úcodevector_probsÚcodevector_soft_distrK  Úcodevector_idxÚcodevectors_per_grouprA  s              r4   rB   z&UniSpeechGumbelVectorQuantizer.forwardx  s  € Ø3@Ô3FÑ0ˆ
�O [ð ×(Ò(¨Ñ7Ô7ˆØ%×*Ò*¨:¸Ñ+GÈ$Ì/Ñ+YÐ[]Ñ^Ô^ˆàŒ=ð 	Då!œ}×;Ò;Ø×#Ò#Ñ%Ô%¨4Ô+;À$ð  <ñ  ô  çŠg�mÑ$Ô$ð õ
 $)¤=Ø×"Ò" :°Ñ#?ÀÄÐRTÑUÔU×[Ò[Ñ]Ô]Ðceð$ñ $ô $Ð ð ×1Ò1Ð2FÑGÔGˆJˆJð +×1Ò1°bÐ1Ñ9Ô9ˆNØ6˜}Ô6¸Ô8KÐL×UÒUØ�N×'Ò'¨¨AÑ.Ô.°ñ ô  Ðð  0×4Ò4°ZÀ/Ñ5QÐSWÔSbÐdfÑgÔgÐà×1Ò1Ð2BÑCÔCˆJà+×0Ò0°¸oÑ1MÈrÑRÔRÐà 0× :Ò :¸2Ñ >Ô >ÀÔAQÑ QÐØ+×0Ò0°¸oÑ1MÈtÌÐ`dÔ`mÐoqÑrÔrˆØ!—o’o bÑ)Ô)×.Ò.¨z¸?ÈBÑOÔOˆà˜JÐ&Ð&r3   )	r*   r+   r,   r-   r:   ÚstaticmethodrL  rB   rD   rE   s   @r4   r:  r:  X  sl   ø€ € € € € ðð ð
ð ð ð ð ð( ðð ñ „\ðð
#'ð #'ð #'ð #'ð #'ð #'ð #'r3   r:  c                   óÌ   ‡ — e Zd ZU eed<   dZdZdZdZdZ	dZ
dZeeg eedd¬¦  «        d	œZ ej        ¦   «         ˆ fd
„¦   «         Zdej        ez  fd„Zdedej        fd„Zˆ xZS )ÚUniSpeechPreTrainedModelrc   Ú	unispeechr¤   ÚaudioTr   Úencoder)ÚindexÚ
layer_name)r(   r)   c           
      óL  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rZt	          j        |j        j        dd¬¦  «         t	          j        |j        j	        ¦  «         t	          j
        |j        ¦  «         dS t          |t          ¦  «        rwt	          j        |j        j        ddt          j        d|j        j        d         |j        j        z  z  ¦  «        z  ¬¦  «         t	          j        |j        j	        d¦  «         dS t          |t&          ¦  «        rgt          j        d|j        j        z  ¦  «        }t	          j
        |j        j        | |¬¦  «         t	          j
        |j        j	        | |¬¦  «         dS t          |t,          j        ¦  «        rnt	          j        |j        ¦  «         |j	        �Pt          j        |j        |j        |j        d         z  z  ¦  «        }t	          j
        |j	        | |¬¦  «         dS dS dS )zInitialize the weightsr´   r   )rE  Ústdr   r8   )ÚaÚbN)r9   Ú_init_weightsrñ   r:  ÚinitÚnormal_rB  rN   Úzeros_rk   Úuniform_rA  rG   rV   ÚmathÚsqrtrI   Úin_channelsÚ	constant_r§   r­   Úin_featuresrR   rS   Úkaiming_normal_rK   )r<   rµ   Úkr>   s      €r4   rg  z&UniSpeechPreTrainedModel._init_weights­  s÷  ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%å�fÕ<Ñ=Ô=ð 	6ÝŒL˜Ô+Ô2¸À!ÐDÑDÔDÐDÝŒK˜Ô*Ô/Ñ0Ô0Ð0ÝŒM˜&Ô,Ñ-Ô-Ð-Ð-Ð-Ý˜Õ @ÑAÔAð 	6ÝŒLØ”Ô"ØØ�œ	 ! v¤{Ô'>¸qÔ'AÀFÄKÔD[Ñ'[Ñ"\Ñ]Ô]Ñ]ðñ ô ð õ
 ŒN˜6œ;Ô+¨QÑ/Ô/Ð/Ð/Ð/Ý˜Õ :Ñ;Ô;ð 		6Ý”	˜!˜fÔ/Ô;Ñ;Ñ<Ô<ˆAÝŒM˜&Ô+Ô2°q°b¸AÐ>Ñ>Ô>Ð>ÝŒM˜&Ô+Ô0°Q°B¸!Ð<Ñ<Ô<Ð<Ð<Ð<Ý˜¥¤	Ñ*Ô*ð 	6ÝÔ  ¤Ñ/Ô/Ð/àŒ{Ð&Ý”I˜fœm¨vÔ/AÀFÔDVÐWXÔDYÑ/YÑZÑ[Ô[�Ý”˜fœk¨a¨R°1Ð5Ñ5Ô5Ð5Ð5Ð5ð	6ð 	6ð 'Ð&r3   Úinput_lengthsc                 óz   — d„ }t          | j        j        | j        j        ¦  «        D ]\  }} ||||¦  «        }Œ|S )zH
        Computes the output length of the convolutional layers
        c                 ó<   — t          j        | |z
  |d¬¦  «        dz   S )NÚfloor)Úrounding_moder   )r.   Údiv)Úinput_lengthrI   rj   s      r4   Ú_conv_out_lengthzSUniSpeechPreTrainedModel._get_feat_extract_output_lengths.<locals>._conv_out_lengthÍ  s&   € õ ”9˜\¨KÑ7¸ÈwÐWÑWÔWÐZ[Ñ[Ð[r3   )Úziprc   ro   rp   )r<   rs  rz  rI   rj   s        r4   Ú _get_feat_extract_output_lengthsz9UniSpeechPreTrainedModel._get_feat_extract_output_lengthsÈ  s\   € ð
	\ð 	\ð 	\õ
 $' t¤{Ô'>ÀÄÔ@WÑ#XÔ#Xð 	Qð 	QÑˆK˜Ø,Ð,¨]¸KÈÑPÔPˆMˆMàÐr3   Úfeature_vector_lengthr¹   c                 ó  — |                      d¬¦  «        d d …df         }|                      |¦  «                             t          j        ¦  «        }|j        d         }t          j        ||f|j        |j        ¬¦  «        }d|t          j	        |j        d         |j        ¬¦  «        |dz
  f<   | 
                    dg¦  «                              d¦  «         
                    dg¦  «                             ¦   «         }|S )Nr   r¾   r   )Údtyper  r   )r  )Úcumsumr|  r  r.   ÚlongrØ   Úzerosr  r  ÚarangeÚfliprè   )r<   r}  r¹   Únon_padded_lengthsÚoutput_lengthsrU  s         r4   Ú"_get_feature_vector_attention_maskz;UniSpeechPreTrainedModel._get_feature_vector_attention_mask×  sþ   € ð ,×2Ò2°rÐ2Ñ:Ô:¸1¸1¸1¸b¸5ÔAÐØ×>Ò>Ð?QÑRÔR×UÒUÕV[ÔV`ÑaÔaˆØ#Ô)¨!Ô,ˆ
åœØÐ.Ð/°~Ô7KÐTbÔTið
ñ 
ô 
ˆð uvˆ�œ ^Ô%9¸!Ô%<À^ÔEZÐ[Ñ[Ô[Ð]kÐnoÑ]oÐpÑqØ'×,Ò,¨b¨TÑ2Ô2×9Ò9¸"Ñ=Ô=×BÒBÀBÀ4ÑHÔH×MÒMÑOÔOˆØÐr3   )r*   r+   r,   r    r0   Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnrú   r/  r   rÉ   Ú_can_record_outputsr.   Úno_gradrg  Ú
LongTensorræ   r|  r‡  rD   rE   s   @r4   r]  r]  ž  s÷   ø€ € € € € € àÐÐÑØ#ÐØ$€OØÐØ&*Ð#ØÐØ€NØÐà/Ð1UÐVØ$�nÐ%7¸qÈYÐWÑWÔWðð Ðð
 €U„]�_„_ð6ð 6ð 6ð 6ñ „_ð6ð4¸eÔ>NÐQTÑ>Tð ð ð ð ðÈð Ð]bÔ]mð ð ð ð ð ð ð ð r3   r]  rØ   Ú	mask_probÚmask_lengthÚ	min_masksrÖ   c                 ó@  ‡‡‡‡‡— | \  }Š‰dk     rt          d¦  «        ‚‰‰k    rt          d‰› d‰› d�¦  «        ‚t          j                             d¦  «                             ¦   «         Šˆˆˆˆˆfd„}|�9|                     ¦   «                              d¦  «                             ¦   «         nˆfd	„t          |¦  «        D ¦   «         }t          j	        |‰ft          ¬
¦  «        }g }	 |‰¦  «        }
|
dk    r|S |D ]·} ||¦  «        }t          j                             t          j        |‰dz
  z
  ¦  «        |d¬¦  «        }t          |¦  «        dk    r‰dz
  }n|d         }t          j        |t          j        |
|z
  t          j        ¬
¦  «        |z  g¦  «        }|	                     |¦  «         Œ¸t          j        |	¦  «        }	t          j        |	dd…dd…df         ||
‰f¦  «        }	|	                     ||
‰z  ¦  «        }	t          j        ‰¦  «        dddd…f         }t          j        |||
‰f¦  «                             ||
‰z  ¦  «        }|	|z   }	|	                     ¦   «         ‰dz
  k    r‰dz
  |	|	‰dz
  k    <   t          j        ||	dd¦  «         |S )an  
    Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for
    ASR](https://huggingface.co/papers/1904.08779). Note that this method is not optimized to run on TPU and should be run on
    CPU as part of the preprocessing during training.

    Args:
        shape: The shape for which to compute masks. This should be of a tuple of size 2 where
               the first element is the batch size and the second element is the length of the axis to span.
        mask_prob:  The percentage of the whole axis (between 0 and 1) which will be masked. The number of
                    independently generated mask spans of length `mask_length` is computed by
                    `mask_prob*shape[1]/mask_length`. Note that due to overlaps, `mask_prob` is an upper bound and the
                    actual percentage will be smaller.
        mask_length: size of the mask
        min_masks: minimum number of masked spans
        attention_mask: A (right-padded) attention mask which independently shortens the feature axis of
                        each batch dimension.
    r   z&`mask_length` has to be bigger than 0.zO`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: z and `sequence_length`: ú`c                 ó¸   •— t          ‰| z  ‰z  ‰z   ¦  «        }t          |‰¦  «        }|‰z  ‰k    r‰‰z  }| ‰dz
  z
  |k     rt          | ‰dz
  z
  d¦  «        }|S )z;Given input length, compute how many spans should be maskedr   r   )ræ   Úmax)ry  Únum_masked_spanÚepsilonr“  r’  r”  rV  s     €€€€€r4   Úcompute_num_masked_spanz6_compute_mask_indices.<locals>.compute_num_masked_span  s~   ø€ å˜i¨,Ñ6¸ÑDÀwÑNÑOÔOˆÝ˜o¨yÑ9Ô9ˆð ˜[Ñ(¨?Ò:Ð:Ø-°Ñ<ˆOð ˜;¨™?Ñ+¨oÒ=Ð=Ý! ,°+À±/Ñ"BÀAÑFÔFˆOàÐr3   Nr   c                 ó   •— g | ]}‰‘ŒS r2   r2   )r�   r  rV  s     €r4   r‘   z)_compute_mask_indices.<locals>.<listcomp>   s   ø€ Ð9Ð9Ð9 !ˆoÐ9Ð9Ð9r3   ©r  r   F)Úreplace)r—   ÚnpÚrandomr  ÚitemÚdetachrG  Útolistr•   r‚  rè   Úchoicerƒ  ÚlenÚconcatenateÚonesÚint32ÚappendÚarrayÚbroadcast_torÜ   r˜  Úput_along_axis)rØ   r’  r“  r¹   r”  rU  r›  rs  Úspec_aug_maskÚspec_aug_mask_idxsÚmax_num_masked_spanry  r™  Úspec_aug_mask_idxÚdummy_mask_idxÚoffsetsrš  rV  s    `` `           @@r4   Ú_compute_mask_indicesr³  ç  sP  øøøøø€ ð0 #(Ñ€J�à�Q‚€ÝÐAÑBÔBÐBà�_Ò$Ð$Ýð:Ð^ið :ð :Ø'6ð:ð :ð :ñ
ô 
ð 	
õ Œi�nŠn˜QÑÔ×$Ò$Ñ&Ô&€Gðð ð ð ð ð ð ð ð ð$ Ð%ð 	×ÒÑÔ×#Ò# BÑ'Ô'×.Ò.Ñ0Ô0Ð0à9Ð9Ð9Ð9¥u¨ZÑ'8Ô'8Ð9Ñ9Ô9ð õ ”H˜j¨/Ð:Å$ÐGÑGÔG€MØÐà1Ð1°/ÑBÔBÐà˜aÒÐØÐà%ð 5ð 5ˆà1Ð1°,Ñ?Ô?ˆõ œI×,Ò,ÝŒI�l k°A¡oÑ6Ñ7Ô7¸ÐRWð -ñ 
ô 
Ðõ Ð Ñ!Ô! QÒ&Ð&ð -¨qÑ0ˆNˆNà.¨qÔ1ˆNåœNØ¥¤Ð(;¸oÑ(MÕUWÔU]Ð ^Ñ ^Ô ^ÐaoÑ oÐpñ
ô 
Ðð 	×!Ò!Ð"3Ñ4Ô4Ð4Ð4åœÐ"4Ñ5Ô5Ðõ œØ˜1˜1˜1˜a˜a˜a ˜:Ô&¨Ð5HÈ+Ð(Vñô Ðð ,×3Ò3°JÐ@SÐVaÑ@aÑbÔbÐõ Œi˜Ñ$Ô$ T¨4°°° ]Ô3€GÝŒo˜g¨
Ð4GÈÐ'UÑVÔV×^Ò^ØÐ'¨+Ñ5ñô €Gð ,¨gÑ5Ðð ×ÒÑÔ /°AÑ"5Ò5Ð5ØGVÐYZÑGZÐÐ-°À!Ñ0CÒCÑDõ Ô�mÐ%7¸¸BÑ?Ô?Ð?àÐr3   c                   ó  ‡ — e Zd Zdefˆ fd„Z	 	 ddej        dej        dz  dej        dz  fd„Ze	 e
d¬	¦  «        e	 	 dd
ej        dz  dej        dz  dej        dz  dee         deez  f
d„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚUniSpeechModelrc   c                 óà  •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        t          |¦  «        | _        |j        dk    s|j        dk    rBt          j
        t          j        |j        ¦  «                             ¦   «         ¦  «        | _        |j        rt#          |¦  «        | _        nt'          |¦  «        | _        |                      ¦   «          d S ©Nr´   )r9   r:   rc   r‰   Úfeature_extractorr§   Úfeature_projectionÚmask_time_probÚmask_feature_probrR   r@  r.   ré   rT   rk  Úmasked_spec_embedÚdo_stable_layer_normr5  r`  r	  Ú	post_initr±   s     €r4   r:   zUniSpeechModel.__init__c  sË   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒÝ!8¸Ñ!@Ô!@ˆÔÝ"<¸VÑ"DÔ"DˆÔàÔ  3Ò&Ð&¨&Ô*BÀSÒ*HÐ*HÝ%'¤\µ%´,¸vÔ?QÑ2RÔ2R×2[Ò2[Ñ2]Ô2]Ñ%^Ô%^ˆDÔ"àÔ&ð 	4Ý:¸6ÑBÔBˆDŒLˆLå+¨FÑ3Ô3ˆDŒLð 	�ŠÑÔÐÐÐr3   Nr(   Úmask_time_indicesr¹   c                 ó  — t          | j        dd¦  «        s|S |                     ¦   «         \  }}}|�#| j                             |j        ¦  «        ||<   n“| j        j        dk    rƒ| j        r|t          ||f| j        j        | j        j	        || j        j
        ¬¦  «        }t          j        ||j        t          j        ¬¦  «        }| j                             |j        ¦  «        ||<   | j        j        dk    r†| j        rt          ||f| j        j        | j        j        | j        j        ¬¦  «        }t          j        ||j        t          j        ¬¦  «        }|dd…df                              d|d¦  «        }d||<   |S )	z¢
        Masks extracted features along time axis and/or along feature axis according to
        [SpecAugment](https://huggingface.co/papers/1904.08779).
        Úapply_spec_augmentTNr   )r’  r“  r¹   r”  )r  r  )r’  r“  r”  r   )r1  rc   rÀ   r¼  r  r  rº  r£   r³  Úmask_time_lengthÚmask_time_min_masksr.   r!  r  rè   r»  Úmask_feature_lengthÚmask_feature_min_masksÚexpand)r<   r(   r¿  r¹   rU  rV  rT   Úmask_feature_indicess           r4   Ú_mask_hidden_statesz"UniSpeechModel._mask_hidden_statest  s—  € õ �t”{Ð$8¸$Ñ?Ô?ð 	!Ø Ð ð 4A×3EÒ3EÑ3GÔ3GÑ0ˆ
�O [àÐ(à/3Ô/E×/HÒ/HÈÔI\Ñ/]Ô/]ˆMÐ+Ñ,Ð,ØŒ[Ô'¨!Ò+Ð+°´Ð+Ý 5Ø˜_Ð-Øœ+Ô4Ø œKÔ8Ø-Øœ+Ô9ð!ñ !ô !Ðõ !&¤Ð->À}ÔG[ÕchÔcmÐ nÑ nÔ nÐØ/3Ô/E×/HÒ/HÈÔI\Ñ/]Ô/]ˆMÐ+Ñ,àŒ;Ô(¨1Ò,Ð,°´Ð,å#8Ø˜[Ð)Øœ+Ô7Ø œKÔ;Øœ+Ô<ð	$ñ $ô $Ð õ $)¤<Ð0DÈ]ÔMaÕinÔisÐ#tÑ#tÔ#tÐ Ø#7¸¸¸¸4¸Ô#@×#GÒ#GÈÈOÐ]_Ñ#`Ô#`Ð Ø23ˆMÐ.Ñ/àÐr3   F)Útie_last_hidden_statesr¤   r»   rÖ   c                 óN  — |                       |¦  «        }|                     dd¦  «        }|�!|                      |j        d         |¦  «        }|                      |¦  «        \  }}|                      |||¬¦  «        } | j        |fd|i|¤Ž}|j        }t          ||¬¦  «        S )a/  
        mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict
            masked extracted features in *config.proj_codevector_dim* space.
        r   r8   N)r¿  r¹   r¹   )r  Úextract_features)	r¸  re   r‡  rØ   r¹  rÈ  r`  r  ÚUniSpeechBaseModelOutput)r<   r¤   r¹   r¿  r»   rË  r(   Úencoder_outputss           r4   rB   zUniSpeechModel.forward¢  sæ   € ð  ×1Ò1°,Ñ?Ô?ÐØ+×5Ò5°a¸Ñ;Ô;ÐàÐ%à!×DÒDÐEUÔE[Ð\]ÔE^Ð`nÑoÔoˆNà*.×*AÒ*AÐBRÑ*SÔ*SÑ'ˆÐ'Ø×0Ò0ØÐ->È~ð 1ñ 
ô 
ˆð '˜$œ,Øð
ð 
à)ð
ð ð
ð 
ˆð (Ô9ˆå'Ø+Ø-ð
ñ 
ô 
ð 	
r3   ©NN)r*   r+   r,   r    r:   r.   r/   r‘  rÈ  r   r   r   ré   r   r   r1   rÌ  rB   rD   rE   s   @r4   rµ  rµ  a  s7  ø€ € € € € ð˜ð ð ð ð ð ð ð( 7;Ø26ð	,ð ,àÔ(ð,ð !Ô,¨tÑ3ð,ð Ô(¨4Ñ/ð	,ð ,ð ,ð ,ð\  Ø€_¨EÐ2Ñ2Ô2Øð /3Ø6:ð	#
ð #
à”l TÑ)ð#
ð œ tÑ+ð#
ð !Ô,¨tÑ3ð	#
ð
 Ð+Ô,ð#
ð 
Ð)Ñ	)ð#
ð #
ð #
ñ „^ñ 3Ô2ñ  Ôð#
ð #
ð #
ð #
ð #
r3   rµ  zZ
    UniSpeech Model with a vector-quantization module and ctc loss for pre-training.
    c                   óî   ‡ — e Zd Zdefˆ fd„Zdefd„Zd„ Ze	 dde	j
        de	j
        d	e	j
        defd
„¦   «         Zee	 dde	j        dz  de	j        dz  dee         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚUniSpeechForPreTrainingrc   c                 ó  •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        ¦  «        | _        t          |¦  «        | _	        t	          j
        |j        |j        ¦  «        | _        t	          j
        |j        |j        ¦  «        | _        t	          j
        |j        |j        ¦  «        | _        t	          j        |j        ¦  «        | _        |                      ¦   «          d S ru   )r9   r:   rµ  r^  rR   r®   Úfeat_quantizer_dropoutÚdropout_featuresr:  Ú	quantizerr¬   r?  Úproj_codevector_dimÚ	project_qrT   Úproject_hidÚnum_ctc_classesÚctc_projÚfinal_dropoutr°   r¾  r±   s     €r4   r:   z UniSpeechForPreTraining.__init__Ñ  sÆ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý'¨Ñ/Ô/ˆŒÝ "¤
¨6Ô+HÑ IÔ IˆÔå7¸Ñ?Ô?ˆŒÝœ 6Ô#8¸&Ô:TÑUÔUˆŒÝœ9 VÔ%?ÀÔASÑTÔTˆÔåœ	 &Ô"4°fÔ6LÑMÔMˆŒÝ”z &Ô"6Ñ7Ô7ˆŒð 	�ŠÑÔÐÐÐr3   rC  c                 ó   — || j         _        dS )zb
        Set the Gumbel softmax temperature to a given value. Only necessary for training
        N)rÔ  rC  )r<   rC  s     r4   Úset_gumbel_temperaturez.UniSpeechForPreTraining.set_gumbel_temperatureà  s   € ð &1ˆŒÔ"Ð"Ð"r3   c                 óB   — | j         j                             ¦   «          dS ©z¨
        Calling this function will disable the gradient computation for the feature encoder so that its parameter will
        not be updated during training.
        N©r^  r¸  r¡   ©r<   s    r4   Úfreeze_feature_encoderz.UniSpeechForPreTraining.freeze_feature_encoderæ  ó!   € ð
 	ŒÔ(×;Ò;Ñ=Ô=Ð=Ð=Ð=r3   r   Útarget_featuresÚnegative_featuresÚpredicted_featuresc                 óà   — t          j        | |gd¬¦  «        } t          j        |                     ¦   «         |                      ¦   «         d¬¦  «        }|                     | ¦  «        }||z  }|S )zé
        Compute logits for contrastive loss based using cosine similarity as the distance measure between
        `[positive_feature, negative_features]` and `[predicted_features]`. Additionally, temperature can be applied.
        r   r¾   r   )r.   ÚcatÚcosine_similarityrç   rQ  )rã  rä  rå  rC  Úlogitss        r4   Úcompute_contrastive_logitsz2UniSpeechForPreTraining.compute_contrastive_logitsí  sq   € õ  œ) _Ð6GÐ$HÈaÐPÑPÔPˆåÔ(Ð);×)AÒ)AÑ)CÔ)CÀ_×EZÒEZÑE\ÔE\ÐbdÐeÑeÔeˆØ—’ Ñ0Ô0ˆð ˜+Ñ%ˆØˆr3   Nr¤   r¹   r»   rÖ   c                 óÌ  —  | j         |fd|i|¤Ž}|j        }|                      |j        ¦  «        }|                      |¦  «        \  }}|                      |                     | j        j        j        ¦  «        ¦  «        }|  	                    |¦  «        }t          j        |                     d¦  «        |                     d¦  «        ¦  «                             | j        j        ¦  «        }	|	                     dd¦  «        }	t          j        |	¦  «                             ¦   «                              |j        ¦  «        }
|
                     dd¦  «        }
|
                     d¦  «        }
|                     |
d¦  «        |                     |
 d¦  «        z   }|                      |¦  «        }|                      |¦  «        }d}t1          |||||j        |j        ¬¦  «        S )a›  
        Example:

        ```python
        >>> import torch
        >>> from transformers import AutoFeatureExtractor, UniSpeechForPreTraining

        >>> feature_extractor = AutoFeatureExtractor.from_pretrained("microsoft/unispeech-large-1500h-cv")
        >>> model = UniSpeechForPreTraining.from_pretrained("microsoft/unispeech-large-1500h-cv")
        >>> # TODO: Add full pretraining example
        ```r¹   r   r   r   r´   N)r$   r%   r&   r'   r(   r)   )r^  r  rÓ  rË  rÔ  rÖ  r  rN   r  r×  r.   ÚemptyrÀ   Úfill_rc   Úreplace_probre   Ú	bernoullirè   r  r  Úmasked_fillr°   rÙ  r#   r(   r)   )r<   r¤   r¹   r»   ÚoutputsÚtransformer_featuresrË  Úquantized_featuresr'   Úprob_replace_matrixÚsampled_replace_matrixré  r$   s                r4   rB   zUniSpeechForPreTraining.forward  sô  € ð& !�$”.Øð
ð 
à)ð
ð ð
ð 
ˆð
  'Ô8Ðð  ×0Ò0°Ô1IÑJÔJÐØ48·N²NÐCSÑ4TÔ4TÑ1ÐÐ1ð "Ÿ^š^Ð,>×,AÒ,AÀ$Ä.ÔBWÔB]Ñ,^Ô,^Ñ_Ô_ÐØ!×-Ò-Ð.@ÑAÔAÐå#œkÐ*>×*CÒ*CÀAÑ*FÔ*FÐH\×HaÒHaÐbcÑHdÔHdÑeÔe×kÒkØŒKÔ$ñ
ô 
Ðð 2×;Ò;¸A¸qÑAÔAÐÝ!&¤Ð1DÑ!EÔ!E×!JÒ!JÑ!LÔ!L×!OÒ!OÐPdÔPkÑ!lÔ!lÐØ!7×!AÒ!AÀ!ÀQÑ!GÔ!GÐØ!7×!AÒ!AÀ"Ñ!EÔ!EÐØ%×1Ò1Ð2HÈ#ÑNÔNØ×*Ò*Ð,BÐ+BÀCÑHÔHñ
ˆð
 —’˜fÑ%Ô%ˆØ—’˜vÑ&Ô&ˆð ˆå,ØØ1Ø'9Ø"7Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r3   )r   ru   )r*   r+   r,   r    r:   ræ   rÜ  rá  r[  r.   r/   rê  r   r   ré   r   r   r1   r#   rB   rD   rE   s   @r4   rÐ  rÐ  Ë  s:  ø€ € € € € ð˜ð ð ð ð ð ð ð1°#ð 1ð 1ð 1ð 1ð>ð >ð >ð ð
 ð	ð ØÔ*ðà Ô,ðð "Ô-ðð ð	ð ð ñ „\ðð& Øð /3ð9
ð 9
à”l TÑ)ð9
ð œ tÑ+ð9
ð Ð+Ô,ð	9
ð
 
Ð.Ñ	.ð9
ð 9
ð 9
ñ „^ñ Ôð9
ð 9
ð 9
ð 9
ð 9
r3   rÐ  zq
    UniSpeech Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).
    c                   óÂ   ‡ — e Zd Zddedz  fˆ fd„Zd„ Zd„ Zd„ Zee		 	 dde
j        dz  de
j        dz  d	e
j        dz  d
ee         deez  f
d„¦   «         ¦   «         Zˆ xZS )ÚUniSpeechForCTCNÚtarget_langc                 óª  •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        ¦  «        | _        || _        |j	        €t          d| j        › d�¦  «        ‚t          |d¦  «        r|j        r|j        n|j        }t	          j        ||j	        ¦  «        | _        |                      ¦   «          dS )a3  
        target_lang (`str`, *optional*):
            Language id of adapter weights. Adapter weights are stored in the format adapter.<lang>.safetensors or
            adapter.<lang>.bin. Only relevant when using an instance of [`UniSpeechForCTC`] with adapters. Uses 'eng' by
            default.
        NzYou are trying to instantiate zø with a configuration that does not define the vocabulary size of the language model head. Please instantiate the model as follows: `UniSpeechForCTC.from_pretrained(..., vocab_size=vocab_size)`. or define `vocab_size` of your model's configuration.Úadd_adapter)r9   r:   rµ  r^  rR   r®   rÚ  r°   rø  Ú
vocab_sizer—   r>   rX   rú  Úoutput_hidden_sizerT   r¬   Úlm_headr¾  )r<   rc   rø  rü  r>   s       €r4   r:   zUniSpeechForCTC.__init__E  sÝ   ø€ õ 	‰Œ×Ò˜Ñ Ô Ð å'¨Ñ/Ô/ˆŒÝ”z &Ô"6Ñ7Ô7ˆŒà&ˆÔàÔÐ$ÝðH°´ð Hð Hð Hñô ð õ *1°¸Ñ)GÔ)GÐvÈFÔL^ÐvˆFÔ%Ð%ÐdjÔdvð 	õ ”yÐ!3°VÔ5FÑGÔGˆŒð 	�ŠÑÔÐÐÐr3   c                 óT  — t          ¦   «         t          j        d¦  «        k    rdS | j        }|�)t	          | j        dd¦  «        €t          d|› d�¦  «        ‚|€2t	          | j        dd¦  «        �t                               d¦  «         dS |�|  	                    |d¬¦  «         dS dS )	a'  
        This method overwrites [`~PreTrainedModel.tie_weights`] so that adapter weights can be correctly loaded when
        passing `target_lang=...` to `from_pretrained(...)`.

        This method is **not** supposed to be called by the user and is prone to be changed in the future.
        ÚmetaNr%  zCannot pass `target_lang`: z- if `config.adapter_attn_dim` is not defined.z)By default `target_lang` is set to 'eng'.T)Ú
force_load)
r   r.   r  rø  r1  rc   r—   ÚloggerÚinfoÚload_adapter)r<   r»   rø  s      r4   Útie_weightszUniSpeechForCTC.tie_weightsb  sÃ   € õ 6Ñ7Ô7½5¼<ÈÑ;OÔ;OÒOÐOØˆFð Ô&ˆàÐ"¥w¨t¬{Ð<NÐPTÑ'UÔ'UÐ']ÝÐu¸;ÐuÐuÐuÑvÔvÐvØÐ ¥W¨T¬[Ð:LÈdÑ%SÔ%SÐ%_Ý�KŠKÐCÑDÔDÐDÐDÐDØÐ$Ø×Ò˜k°dÐÑ;Ô;Ð;Ð;Ð;ð %Ð$r3   c                 óB   — | j         j                             ¦   «          dS rÞ  rß  rà  s    r4   rá  z&UniSpeechForCTC.freeze_feature_encoderz  râ  r3   c                 óL   — | j                              ¦   «         D ]	}d|_        Œ
dS ©zÒ
        Calling this function will disable the gradient computation for the base model so that its parameters will not
        be updated during training. Only the classification head will be updated.
        FN©r^  r�   rž   rŸ   s     r4   Úfreeze_base_modelz!UniSpeechForCTC.freeze_base_model�  ó6   € ð
 ”^×.Ò.Ñ0Ô0ð 	(ð 	(ˆEØ"'ˆEÔÐð	(ð 	(r3   r¤   r¹   Úlabelsr»   rÖ   c           
      óü  — |�>|                      ¦   «         | j        j        k    rt          d| j        j        › �¦  «        ‚ | j        |fd|i|¤Ž}|j        }|                      |¦  «        }|                      |¦  «        }d}|��Z|�|nt          j	        |t          j
        ¬¦  «        }|                      |                     d¦  «        ¦  «                             t          j
        ¦  «        }	|dk    }
|
                     d¦  «        }|                     |
¦  «        }t          j                             |dt          j        ¬¦  «                             dd¦  «        }t          j        j                             d	¬
¦  «        5  t          j                             |||	|| j        j        | j        j        | j        j        ¬¦  «        }ddd¦  «         n# 1 swxY w Y   t7          |||j        |j        ¬¦  «        S )aà  
        labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*):
            Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to
            the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`.
            All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ...,
            config.vocab_size - 1]`.
        Nz$Label values must be <= vocab_size: r¹   r�  r   r   )rP   r  r   F)Úenabled)ÚblankÚ	reductionÚzero_infinity©r$   ré  r(   r)   )r˜  rc   rû  r—   r^  r  r°   rý  r.   Ú	ones_liker�  r|  rG  r  Úmasked_selectrR   rÂ   Úlog_softmaxÚfloat32re   ÚbackendsÚcudnnÚflagsÚctc_lossÚpad_token_idÚctc_loss_reductionÚctc_zero_infinityr   r(   r)   )r<   r¤   r¹   r  r»   rñ  r(   ré  r$   rs  Úlabels_maskÚtarget_lengthsÚflattened_targetsÚ	log_probss                 r4   rB   zUniSpeechForCTC.forward‰  s:  € ð  Ð &§*¢*¡,¤,°$´+Ô2HÒ"HÐ"HÝÐ\ÀDÄKÔDZÐ\Ð\Ñ]Ô]Ð]à �$”.Øð
ð 
à)ð
ð ð
ð 
ˆð  Ô1ˆØŸš ]Ñ3Ô3ˆà—’˜mÑ,Ô,ˆàˆØÑð #1Ð"<��Å%Ä/ÐR^ÕfkÔfpÐBqÑBqÔBqð ð !×AÒAÀ.×BTÒBTÐUWÑBXÔBXÑYÔY×\Ò\Õ]bÔ]gÑhÔhˆMð ! Aš+ˆKØ(Ÿ_š_¨RÑ0Ô0ˆNØ &× 4Ò 4°[Ñ AÔ AÐõ œ×1Ò1°&¸bÍÌÐ1ÑVÔV×`Ò`ÐabÐdeÑfÔfˆIå”Ô%×+Ò+°EÐ+Ñ:Ô:ð 	ð 	Ý”}×-Ò-ØØ%Ø!Ø"Øœ+Ô2Ø"œkÔ<Ø"&¤+Ô"?ð .ñ ô �ð	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	õ Ø˜f°GÔ4IÐV]ÔVhð
ñ 
ô 
ð 	
s   ÆAGÇGÇGru   rÎ  )r*   r+   r,   ró   r:   r  rá  r	  r   r   r.   ré   r   r   r1   r   rB   rD   rE   s   @r4   r÷  r÷  ?  s  ø€ € € € € ðð ¨C°$©Jð ð ð ð ð ð ð:<ð <ð <ð0>ð >ð >ð(ð (ð (ð Øð /3Ø&*ð	:
ð :
à”l TÑ)ð:
ð œ tÑ+ð:
ð ”˜tÑ#ð	:
ð
 Ð+Ô,ð:
ð 
�Ñ	ð:
ð :
ð :
ñ „^ñ Ôð:
ð :
ð :
ð :
ð :
r3   r÷  z˜
    UniSpeech Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like
    SUPERB Keyword Spotting.
    c                   ó¾   ‡ — e Zd Zˆ fd„Zd„ Zd„ Zeee	 	 dde	j
        dz  de	j
        dz  de	j
        dz  dee         d	eez  f
d
„¦   «         ¦   «         ¦   «         Zˆ xZS )Ú"UniSpeechForSequenceClassificationc                 óô  •— t          ¦   «                              |¦  «         t          |d¦  «        r|j        rt	          d¦  «        ‚t          |¦  «        | _        |j        dz   }|j        r.t          j
        t          j        |¦  «        |z  ¦  «        | _        t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¦  «        | _        |                      ¦   «          d S )Nrú  z`Sequence classification does not support the use of UniSpeech adapters (config.add_adapter=True)r   )r9   r:   rX   rú  r—   rµ  r^  r  Úuse_weighted_layer_sumrR   r@  r.   r§  Úlayer_weightsr¬   rT   Úclassifier_proj_sizeÚ	projectorÚ
num_labelsÚ
classifierr¾  )r<   rc   Ú
num_layersr>   s      €r4   r:   z+UniSpeechForSequenceClassification.__init__Ï  sá   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å�6˜=Ñ)Ô)ð 	¨fÔ.@ð 	ÝØrñô ð õ (¨Ñ/Ô/ˆŒØÔ-°Ñ1ˆ
ØÔ(ð 	SÝ!#¤­e¬j¸Ñ.DÔ.DÀzÑ.QÑ!RÔ!RˆDÔÝœ 6Ô#5°vÔ7RÑSÔSˆŒÝœ) FÔ$?ÀÔARÑSÔSˆŒð 	�ŠÑÔÐÐÐr3   c                 óB   — | j         j                             ¦   «          dS rÞ  rß  rà  s    r4   rá  z9UniSpeechForSequenceClassification.freeze_feature_encoderà  râ  r3   c                 óL   — | j                              ¦   «         D ]	}d|_        Œ
dS r  r  rŸ   s     r4   r	  z4UniSpeechForSequenceClassification.freeze_base_modelç  r
  r3   Nr¤   r¹   r  r»   rÖ   c                 óæ  — | j         j        rd|d<    | j        |fd|i|¤Ž}| j         j        rr|j        }t	          j        |d¬¦  «        }t          j                             | j	        d¬¦  «        }|| 
                    ddd¦  «        z                       d¬¦  «        }n|j        }|                      |¦  «        }|€|                     d¬¦  «        }n�|                      |j        d         |¦  «        }	|	                     d¦  «                             dd|j        d         ¦  «        }
d	||
 <   |                     d¬¦  «        |	                     d¬¦  «         
                    dd¦  «        z  }|                      |¦  «        }d}|�Kt)          ¦   «         } || 
                    d| j         j        ¦  «        | 
                    d¦  «        ¦  «        }t-          |||j        |j        ¬
¦  «        S )a  
        input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
            Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file
            into an array of type `list[float]`, a `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library
            (`pip install torchcodec`) or the soundfile library (`pip install soundfile`).
            To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and conversion
            into a tensor of type `torch.FloatTensor`. See [`UniSpeechProcessor.__call__`] for details.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        TÚoutput_hidden_statesr¹   r   r¾   r   Nr8   r´   r  )rc   r$  r^  r(   r.   ÚstackrR   rÂ   rÃ   r%  rÙ   rG  r  r'  rE  r‡  rØ   r  r  r)  r   r(  r   r)   )r<   r¤   r¹   r  r»   rñ  r(   Únorm_weightsÚpooled_outputÚpadding_maskÚexpand_padding_maskré  r$   Úloss_fcts                 r4   rB   z*UniSpeechForSequenceClassification.forwardï  s  € ð, Œ;Ô-ð 	2Ø-1ˆFÐ)Ñ*à �$”.Øð
ð 
à)ð
ð ð
ð 
ˆð Œ;Ô-ð 	6Ø#Ô1ˆMÝ!œK¨¸1Ð=Ñ=Ô=ˆMÝœ=×0Ò0°Ô1CÈÐ0ÑLÔLˆLØ*¨\×->Ò->¸rÀ1ÀaÑ-HÔ-HÑH×MÒMÐRSÐMÑTÔTˆMˆMà#Ô5ˆMàŸš }Ñ5Ô5ˆØÐ!Ø)×.Ò.°1Ð.Ñ5Ô5ˆMˆMà×BÒBÀ=ÔCVÐWXÔCYÐ[iÑjÔjˆLØ".×"8Ò"8¸Ñ"<Ô"<×"CÒ"CÀAÀqÈ-ÔJ]Ð^_ÔJ`Ñ"aÔ"aÐØ25ˆMÐ.Ð.Ñ/Ø)×-Ò-°!Ð-Ñ4Ô4°|×7GÒ7GÈAÐ7GÑ7NÔ7N×7SÒ7SÐTVÐXYÑ7ZÔ7ZÑZˆMà—’ Ñ/Ô/ˆàˆØÐÝ'Ñ)Ô)ˆHØ�8˜FŸKšK¨¨D¬KÔ,BÑCÔCÀVÇ[Â[ÐQSÁ_Ä_ÑUÔUˆDå'ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r3   rÎ  )r*   r+   r,   r:   rá  r	  r   r	   r   r.   ré   r   r   r1   r   rB   rD   rE   s   @r4   r"  r"  È  sê   ø€ € € € € ðð ð ð ð ð">ð >ð >ð(ð (ð (ð Ø Øð /3Ø&*ð	9
ð 9
à”l TÑ)ð9
ð œ tÑ+ð9
ð ”˜tÑ#ð	9
ð
 Ð+Ô,ð9
ð 
Ð)Ñ	)ð9
ð 9
ð 9
ñ „^ñ !Ô ñ Ôð9
ð 9
ð 9
ð 9
ð 9
r3   r"  )r÷  rÐ  r"  rµ  r]  r·  r@   )Wrl  Úcollections.abcr   Údataclassesr   ÚnumpyrŸ  r.   Útorch.nnrR   r   Ú r   rh  Úactivationsr   Úbackbone_utilsr	   Úintegrations.deepspeedr
   Úintegrations.fsdpr   Úmasking_utilsr   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   Úmodeling_utilsr   r   r   Úprocessing_utilsr   rW   r   r   r   Úutils.genericr   r   Úutils.output_capturingr   r   Úconfiguration_unispeechr    Ú
get_loggerr*   r  r#   ÚModuler6   rG   rg   rx   r�   r‰   r§   ré   rç   rÇ   rÉ   rë   rú   r	  r#  r/  r5  r:  r]  r1   ræ   r‘  Úndarrayr³  rÌ  rµ  rÐ  r÷  r"  Ú__all__r2   r3   r4   ú<module>rK     s   ðð* €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !à Ð Ð Ð Ø €€€Ø Ð Ð Ð Ð Ð Ø %Ð %Ð %Ð %Ð %Ð %à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø @Ð @Ð @Ð @Ð @Ð @Ø 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø BÐ BÐ BÐ BÐ BÐ BØ 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð ð ð sÐ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rØ &Ð &Ð &Ð &Ð &Ð &Ø @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ 4Ð 4Ð 4Ð 4Ð 4Ð 4ð 
ˆÔ	˜HÑ	%Ô	%€ð €ððñ ô ð
 ð7ð 7ð 7ð 7ð 7 Kñ 7ô 7ñ „ñô ð7ð.ð ð ð ð ˜BœIñ ô ð ð*ð *ð *ð *ð * r¤yñ *ô *ð *ðZð ð ð ð Ð$>ñ ô ð ð*ð ð ð ð Ð"<ñ ô ð ð6ð ð ð ð Ð"<ñ ô ð ð0&ð &ð &ð &ð &˜bœiñ &ô &ð &ðR1ð 1ð 1ð 1ð 1 ¤ñ 1ô 1ð 1ð* !Øð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð �T‰\ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð8R)ð R)ð R)ð R)ð R)˜œñ R)ô R)ð R)ðjð ð ð ð ˜2œ9ñ ô ð ð0ð ð ð ð Ð6ñ ô ð ðD+@ð +@ð +@ð +@ð +@�r”yñ +@ô +@ð +@ð\ð ð ð ð  ¤	ñ ô ð ð2$ð $ð $ð $ð $Ð+Eñ $ô $ð $ðN/@ð /@ð /@ð /@ð /@ b¤iñ /@ô /@ð /@ðdC'ð C'ð C'ð C'ð C' R¤Yñ C'ô C'ð C'ðL ðEð Eð Eð Eð E˜ñ Eô Eñ „ðEðX /3Øðtð tØ��c�Œ?ðtàðtð ðtð Ô$ tÑ+ð	tð
 ðtð „Zðtð tð tð tðn 3Ð ð ðf
ð f
ð f
ð f
ð f
Ð-ñ f
ô f
ñ „ðf
ðR €ððñ ô ð
l
ð l
ð l
ð l
ð l
Ð6ñ l
ô l
ñô ð
l
ð^ €ððñ ô ð
A
ð A
ð A
ð A
ð A
Ð.ñ A
ô A
ñô ð
A
ðH €ððñ ô ð]
ð ]
ð ]
ð ]
ð ]
Ð)Añ ]
ô ]
ñô ð]
ð@ð ð €€€r3   