§
    9²¸jQ@  ã                   ó¸  — d Z ddlZddlmZ ddlZddlmZ ddlmZ	 ddl
mZmZ ddlmZ ddlmZ dd	lmZmZmZ dd
lmZmZ ddlmZmZ ddlmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$ ddl%m&Z&  ej'        e(¦  «        Z) ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z* G d„ de$¦  «        Z+ G d„ de¦  «        Z, G d„ de¦  «        Z- G d„ de¦  «        Z. G d„ de¦  «        Z/ G d„ de"¦  «        Z0e G d „ d!e¦  «        ¦   «         Z1eZ2 G d"„ d#e1e#¦  «        Z3 ed$¬¦  «         G d%„ d&e1¦  «        ¦   «         Z4 G d'„ d(e ¦  «        Z5 G d)„ d*e!¦  «        Z6g d+¢Z7dS ),zPyTorch UniSpeech model.é    N)Ú	dataclassé   )Úinitialization)ÚModelOutputÚWav2Vec2BaseModelOutput)ÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)Úcan_return_tupleÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )	ÚWav2Vec2EncoderÚWav2Vec2EncoderStableLayerNormÚWav2Vec2FeatureEncoderÚWav2Vec2FeatureProjectionÚWav2Vec2ForCTCÚ!Wav2Vec2ForSequenceClassificationÚWav2Vec2GumbelVectorQuantizerÚWav2Vec2ModelÚWav2Vec2PositionalConvEmbeddingé   )ÚUniSpeechConfigzh
    Output type of [`UniSpeechForPreTrainingOutput`], with potential hidden states and attentions.
    )Úcustom_introc                   óà   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	ej        dz  ed<   dZ
ej        dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dS )	ÚUniSpeechForPreTrainingOutputaÝ  
    loss (*optional*, returned when model is in train mode, `torch.FloatTensor` of shape `(1,)`):
        Total loss as the sum of the contrastive loss (L_m) and the diversity loss (L_d) as stated in the [official
        paper](https://huggingface.co/papers/2006.11477).
    projected_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.proj_codevector_dim)`):
        Hidden-states of the model projected to *config.proj_codevector_dim* that can be used to predict the masked
        projected quantized states.
    projected_quantized_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.proj_codevector_dim)`):
        Quantized extracted feature vectors projected to *config.proj_codevector_dim* representing the positive
        target vectors for contrastive loss.
    codevector_perplexity (`torch.FloatTensor` of shape `(1,)`):
        The perplexity of the codevector distribution, used to measure the diversity of the codebook.
    NÚlossÚprojected_statesÚprojected_quantized_statesÚcodevector_perplexityÚhidden_statesÚ
attentions)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r    ÚtorchÚFloatTensorÚ__annotations__r!   r"   r#   r$   Útupler%   © ó    ú/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/unispeech/modular_unispeech.pyr   r   .   s»   € € € € € € ðð ð &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø15Ð�eÔ'¨$Ñ.Ð5Ð5Ñ5Ø;?Ð Ô 1°DÑ 8Ð?Ð?Ñ?Ø6:Ð˜5Ô,¨tÑ3Ð:Ð:Ñ:Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ð6Ð6r/   r   c                   ó   — e Zd ZdS )Ú UniSpeechPositionalConvEmbeddingN©r&   r'   r(   r.   r/   r0   r2   r2   K   ó   € € € € € Ø€Dr/   r2   c                   ó   — e Zd ZdS )ÚUniSpeechFeatureEncoderNr3   r.   r/   r0   r6   r6   O   r4   r/   r6   c                   ó   — e Zd ZdS )ÚUniSpeechFeatureProjectionNr3   r.   r/   r0   r8   r8   S   r4   r/   r8   c                   ó   — e Zd ZdS )ÚUniSpeechEncoderNr3   r.   r/   r0   r:   r:   W   r4   r/   r:   c                   ó   — e Zd ZdS )ÚUniSpeechEncoderStableLayerNormNr3   r.   r/   r0   r<   r<   [   r4   r/   r<   c                   ó*   — e Zd Zed„ ¦   «         Zd„ ZdS )ÚUniSpeechGumbelVectorQuantizerc                 óÎ   — |                       d¬¦  «        }t          j        t          j        t          j        ||¦  «        d¬¦  «         ¦  «                             ¦   «         }|S )Nr   ©Údiméÿÿÿÿ)Úmeanr*   ÚexpÚsumÚxlogy)ÚprobsÚmarginal_probsÚ
perplexitys      r0   Ú_compute_perplexityz2UniSpeechGumbelVectorQuantizer._compute_perplexity`   sU   € àŸš¨˜Ñ*Ô*ˆÝ”Y¥¤	­%¬+°nÀnÑ*UÔ*UÐ[]Ð ^Ñ ^Ô ^Ð^Ñ_Ô_×cÒcÑeÔeˆ
ØÐr/   c                 óö  — |j         \  }}}|                      |¦  «        }|                     ||z  | j        z  d¦  «        }| j        r§t
          j                             |                     ¦   «         | j	        d¬¦  «         
                    |¦  «        }t          j        |                     ||z  | j        d¦  «                             ¦   «         d¬¦  «        }|                      |¦  «        }n‚|                     d¬¦  «        } |j        |j         Ž                      d|                     dd¦  «        d¦  «        }|                     ||z  | j        d¦  «        }|                      |¦  «        }|                     ||z  d¦  «        }|                     d¦  «        | j        z  }	|	                     ||z  | j        | j        d¦  «        }
|
                     d¦  «                             ||d¦  «        }
|
|fS )NrB   T)ÚtauÚhardr@   r   g      ð?éþÿÿÿ)ÚshapeÚweight_projÚviewÚ
num_groupsÚtrainingÚnnÚ
functionalÚgumbel_softmaxÚfloatÚtemperatureÚtype_asr*   ÚsoftmaxrJ   ÚargmaxÚ	new_zerosÚscatter_Ú	unsqueezeÚcodevectorsÚnum_varsrE   )Úselfr$   Ú
batch_sizeÚsequence_lengthÚhidden_sizeÚcodevector_probsÚcodevector_soft_distrI   Úcodevector_idxÚcodevectors_per_groupr_   s              r0   Úforwardz&UniSpeechGumbelVectorQuantizer.forwardf   s  € Ø3@Ô3FÑ0ˆ
�O [ð ×(Ò(¨Ñ7Ô7ˆØ%×*Ò*¨:¸Ñ+GÈ$Ì/Ñ+YÐ[]Ñ^Ô^ˆàŒ=ð 	Då!œ}×;Ò;Ø×#Ò#Ñ%Ô%¨4Ô+;À$ð  <ñ  ô  çŠg�mÑ$Ô$ð õ
 $)¤=Ø×"Ò" :°Ñ#?ÀÄÐRTÑUÔU×[Ò[Ñ]Ô]Ðceð$ñ $ô $Ð ð ×1Ò1Ð2FÑGÔGˆJˆJð +×1Ò1°bÐ1Ñ9Ô9ˆNØ6˜}Ô6¸Ô8KÐL×UÒUØ�N×'Ò'¨¨AÑ.Ô.°ñ ô  Ðð  0×4Ò4°ZÀ/Ñ5QÐSWÔSbÐdfÑgÔgÐà×1Ò1Ð2BÑCÔCˆJà+×0Ò0°¸oÑ1MÈrÑRÔRÐà 0× :Ò :¸2Ñ >Ô >ÀÔAQÑ QÐØ+×0Ò0°¸oÑ1MÈtÌÐ`dÔ`mÐoqÑrÔrˆØ!—o’o bÑ)Ô)×.Ò.¨z¸?ÈBÑOÔOˆà˜JÐ&Ð&r/   N)r&   r'   r(   ÚstaticmethodrJ   ri   r.   r/   r0   r>   r>   _   s<   € € € € € Øðð ñ „\ðð
#'ð #'ð #'ð #'ð #'r/   r>   c                   óÌ   ‡ — e Zd ZU eed<   dZdZdZdZdZ	dZ
dZeeg eedd¬¦  «        d	œZ ej        ¦   «         ˆ fd
„¦   «         Zdej        ez  fd„Zdedej        fd„Zˆ xZS )ÚUniSpeechPreTrainedModelÚconfigÚ	unispeechÚinput_valuesÚaudioTr   Úencoder)ÚindexÚ
layer_name)r$   r%   c           
      óL  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rZt	          j        |j        j        dd¬¦  «         t	          j        |j        j	        ¦  «         t	          j
        |j        ¦  «         dS t          |t          ¦  «        rwt	          j        |j        j        ddt          j        d|j        j        d         |j        j        z  z  ¦  «        z  ¬¦  «         t	          j        |j        j	        d¦  «         dS t          |t&          ¦  «        rgt          j        d|j        j        z  ¦  «        }t	          j
        |j        j        | |¬¦  «         t	          j
        |j        j	        | |¬¦  «         dS t          |t,          j        ¦  «        rnt	          j        |j        ¦  «         |j	        �Pt          j        |j        |j        |j        d         z  z  ¦  «        }t	          j
        |j	        | |¬¦  «         dS dS dS )zInitialize the weightsç        r   )rC   Ústdr   r   )ÚaÚbN)ÚsuperÚ_init_weightsÚ
isinstancer>   ÚinitÚnormal_rP   ÚweightÚzeros_ÚbiasÚuniform_r_   r2   ÚconvÚmathÚsqrtÚkernel_sizeÚin_channelsÚ	constant_r8   Ú
projectionÚin_featuresrT   ÚConv1dÚkaiming_normal_Úgroups)ra   ÚmoduleÚkÚ	__class__s      €r0   rz   z&UniSpeechPreTrainedModel._init_weights›   s÷  ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%å�fÕ<Ñ=Ô=ð 	6ÝŒL˜Ô+Ô2¸À!ÐDÑDÔDÐDÝŒK˜Ô*Ô/Ñ0Ô0Ð0ÝŒM˜&Ô,Ñ-Ô-Ð-Ð-Ð-Ý˜Õ @ÑAÔAð 	6ÝŒLØ”Ô"ØØ�œ	 ! v¤{Ô'>¸qÔ'AÀFÄKÔD[Ñ'[Ñ"\Ñ]Ô]Ñ]ðñ ô ð õ
 ŒN˜6œ;Ô+¨QÑ/Ô/Ð/Ð/Ð/Ý˜Õ :Ñ;Ô;ð 		6Ý”	˜!˜fÔ/Ô;Ñ;Ñ<Ô<ˆAÝŒM˜&Ô+Ô2°q°b¸AÐ>Ñ>Ô>Ð>ÝŒM˜&Ô+Ô0°Q°B¸!Ð<Ñ<Ô<Ð<Ð<Ð<Ý˜¥¤	Ñ*Ô*ð 	6ÝÔ  ¤Ñ/Ô/Ð/àŒ{Ð&Ý”I˜fœm¨vÔ/AÀFÔDVÐWXÔDYÑ/YÑZÑ[Ô[�Ý”˜fœk¨a¨R°1Ð5Ñ5Ô5Ð5Ð5Ð5ð	6ð 	6ð 'Ð&r/   Úinput_lengthsc                 óz   — d„ }t          | j        j        | j        j        ¦  «        D ]\  }} ||||¦  «        }Œ|S )zH
        Computes the output length of the convolutional layers
        c                 ó<   — t          j        | |z
  |d¬¦  «        dz   S )NÚfloor)Úrounding_moder   )r*   Údiv)Úinput_lengthr…   Ústrides      r0   Ú_conv_out_lengthzSUniSpeechPreTrainedModel._get_feat_extract_output_lengths.<locals>._conv_out_length»   s&   € õ ”9˜\¨KÑ7¸ÈwÐWÑWÔWÐZ[Ñ[Ð[r/   )Úziprm   Úconv_kernelÚconv_stride)ra   r�   r˜   r…   r—   s        r0   Ú _get_feat_extract_output_lengthsz9UniSpeechPreTrainedModel._get_feat_extract_output_lengths¶   s\   € ð
	\ð 	\ð 	\õ
 $' t¤{Ô'>ÀÄÔ@WÑ#XÔ#Xð 	Qð 	QÑˆK˜Ø,Ð,¨]¸KÈÑPÔPˆMˆMàÐr/   Úfeature_vector_lengthÚattention_maskc                 ó  — |                      d¬¦  «        d d …df         }|                      |¦  «                             t          j        ¦  «        }|j        d         }t          j        ||f|j        |j        ¬¦  «        }d|t          j	        |j        d         |j        ¬¦  «        |dz
  f<   | 
                    dg¦  «                              d¦  «         
                    dg¦  «                             ¦   «         }|S )NrB   r@   r   )ÚdtypeÚdevicer   )r¡   )Úcumsumrœ   Útor*   ÚlongrO   Úzerosr    r¡   ÚarangeÚflipÚbool)ra   r�   rž   Únon_padded_lengthsÚoutput_lengthsrb   s         r0   Ú"_get_feature_vector_attention_maskz;UniSpeechPreTrainedModel._get_feature_vector_attention_maskÅ   sþ   € ð ,×2Ò2°rÐ2Ñ:Ô:¸1¸1¸1¸b¸5ÔAÐØ×>Ò>Ð?QÑRÔR×UÒUÕV[ÔV`ÑaÔaˆØ#Ô)¨!Ô,ˆ
åœØÐ.Ð/°~Ô7KÐTbÔTið
ñ 
ô 
ˆð uvˆ�œ ^Ô%9¸!Ô%<À^ÔEZÐ[Ñ[Ô[Ð]kÐnoÑ]oÐpÑqØ'×,Ò,¨b¨TÑ2Ô2×9Ò9¸"Ñ=Ô=×BÒBÀBÀ4ÑHÔH×MÒMÑOÔOˆØÐr/   )r&   r'   r(   r   r,   Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚUniSpeechEncoderLayerÚ$UniSpeechEncoderLayerStableLayerNormr   ÚUniSpeechAttentionÚ_can_record_outputsr*   Úno_gradrz   Ú
LongTensorÚintrœ   r«   Ú__classcell__©r�   s   @r0   rl   rl   Œ   s÷   ø€ € € € € € àÐÐÑØ#ÐØ$€OØÐØ&*Ð#ØÐØ€NØÐà/Ð1UÐVØ$�nÐ%7¸qÈYÐWÑWÔWðð Ðð
 €U„]�_„_ð6ð 6ð 6ð 6ñ „_ð6ð4¸eÔ>NÐQTÑ>Tð ð ð ð ðÈð Ð]bÔ]mð ð ð ð ð ð ð ð r/   rl   c                   óÈ   — e Zd Zdefd„Zd„ Ze ed¬¦  «        e	 	 dde	j
        dz  de	j
        dz  d	e	j        dz  d
ee         deez  f
d„¦   «         ¦   «         ¦   «         ZdS )ÚUniSpeechModelrm   c                 óÒ  — t                                | |¦  «         || _        t          |¦  «        | _        t          |¦  «        | _        |j        dk    s|j        dk    rBt          j
        t          j        |j        ¦  «                             ¦   «         ¦  «        | _        |j        rt#          |¦  «        | _        nt'          |¦  «        | _        |                      ¦   «          d S )Nru   )rl   Ú__init__rm   r6   Úfeature_extractorr8   Úfeature_projectionÚmask_time_probÚmask_feature_probrT   Ú	Parameterr*   ÚTensorrd   r�   Úmasked_spec_embedÚdo_stable_layer_normr<   rq   r:   Ú	post_init)ra   rm   s     r0   r¿   zUniSpeechModel.__init__Ù   sÈ   € Ý ×)Ò)¨$°Ñ7Ô7Ð7ØˆŒÝ!8¸Ñ!@Ô!@ˆÔÝ"<¸VÑ"DÔ"DˆÔàÔ  3Ò&Ð&¨&Ô*BÀSÒ*HÐ*HÝ%'¤\µ%´,¸vÔ?QÑ2RÔ2R×2[Ò2[Ñ2]Ô2]Ñ%^Ô%^ˆDÔ"àÔ&ð 	4Ý:¸6ÑBÔBˆDŒLˆLå+¨FÑ3Ô3ˆDŒLð 	�ŠÑÔÐÐÐr/   c                 ó    — t          d¦  «        ‚)NzNot needed for UniSpeech)ÚAttributeError©ra   s    r0   Úfreeze_feature_encoderz%UniSpeechModel.freeze_feature_encoderê   s   € ÝÐ7Ñ8Ô8Ð8r/   F)Útie_last_hidden_statesNro   rž   Úmask_time_indicesÚkwargsÚreturnc                 óN  — |                       |¦  «        }|                     dd¦  «        }|�!|                      |j        d         |¦  «        }|                      |¦  «        \  }}|                      |||¬¦  «        } | j        |fd|i|¤Ž}|j        }t          ||¬¦  «        S )a/  
        mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict
            masked extracted features in *config.proj_codevector_dim* space.
        r   r   N)rÎ   rž   rž   )Úlast_hidden_stateÚextract_features)	rÀ   Ú	transposer«   rO   rÁ   Ú_mask_hidden_statesrq   rÒ   ÚUniSpeechBaseModelOutput)ra   ro   rž   rÎ   rÏ   rÓ   r$   Úencoder_outputss           r0   ri   zUniSpeechModel.forwardí   sæ   € ð  ×1Ò1°,Ñ?Ô?ÐØ+×5Ò5°a¸Ñ;Ô;ÐàÐ%à!×DÒDÐEUÔE[Ð\]ÔE^Ð`nÑoÔoˆNà*.×*AÒ*AÐBRÑ*SÔ*SÑ'ˆÐ'Ø×0Ò0ØÐ->È~ð 1ñ 
ô 
ˆð '˜$œ,Øð
ð 
à)ð
ð ð
ð 
ˆð (Ô9ˆå'Ø+Ø-ð
ñ 
ô 
ð 	
r/   )NN)r&   r'   r(   r   r¿   rÌ   r   r   r   r*   rÅ   r+   r	   r
   r-   rÖ   ri   r.   r/   r0   r½   r½   Ø   sà   € € € € € ð˜ð ð ð ð ð"9ð 9ð 9ð  Ø€_¨EÐ2Ñ2Ô2Øð /3Ø6:ð	#
ð #
à”l TÑ)ð#
ð œ tÑ+ð#
ð !Ô,¨tÑ3ð	#
ð
 Ð+Ô,ð#
ð 
Ð)Ñ	)ð#
ð #
ð #
ñ „^ñ 3Ô2ñ  Ôð#
ð #
ð #
r/   r½   zZ
    UniSpeech Model with a vector-quantization module and ctc loss for pre-training.
    c                   óî   ‡ — e Zd Zdefˆ fd„Zdefd„Zd„ Ze	 dde	j
        de	j
        d	e	j
        defd
„¦   «         Zee	 dde	j        dz  de	j        dz  dee         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚUniSpeechForPreTrainingrm   c                 ó  •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        ¦  «        | _        t          |¦  «        | _	        t	          j
        |j        |j        ¦  «        | _        t	          j
        |j        |j        ¦  «        | _        t	          j
        |j        |j        ¦  «        | _        t	          j        |j        ¦  «        | _        |                      ¦   «          d S ©N)ry   r¿   r½   rn   rT   ÚDropoutÚfeat_quantizer_dropoutÚdropout_featuresr>   Ú	quantizerÚLinearÚcodevector_dimÚproj_codevector_dimÚ	project_qrd   Úproject_hidÚnum_ctc_classesÚctc_projÚfinal_dropoutÚdropoutrÈ   )ra   rm   r�   s     €r0   r¿   z UniSpeechForPreTraining.__init__  sÆ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý'¨Ñ/Ô/ˆŒÝ "¤
¨6Ô+HÑ IÔ IˆÔå7¸Ñ?Ô?ˆŒÝœ 6Ô#8¸&Ô:TÑUÔUˆŒÝœ9 VÔ%?ÀÔASÑTÔTˆÔåœ	 &Ô"4°fÔ6LÑMÔMˆŒÝ”z &Ô"6Ñ7Ô7ˆŒð 	�ŠÑÔÐÐÐr/   rX   c                 ó   — || j         _        dS )zb
        Set the Gumbel softmax temperature to a given value. Only necessary for training
        N)rß   rX   )ra   rX   s     r0   Úset_gumbel_temperaturez.UniSpeechForPreTraining.set_gumbel_temperature+  s   € ð &1ˆŒÔ"Ð"Ð"r/   c                 óB   — | j         j                             ¦   «          dS )z¨
        Calling this function will disable the gradient computation for the feature encoder so that its parameter will
        not be updated during training.
        N)rn   rÀ   Ú_freeze_parametersrË   s    r0   rÌ   z.UniSpeechForPreTraining.freeze_feature_encoder1  s!   € ð
 	ŒÔ(×;Ò;Ñ=Ô=Ð=Ð=Ð=r/   r   Útarget_featuresÚnegative_featuresÚpredicted_featuresc                 óà   — t          j        | |gd¬¦  «        } t          j        |                     ¦   «         |                      ¦   «         d¬¦  «        }|                     | ¦  «        }||z  }|S )zé
        Compute logits for contrastive loss based using cosine similarity as the distance measure between
        `[positive_feature, negative_features]` and `[predicted_features]`. Additionally, temperature can be applied.
        r   r@   rB   )r*   ÚcatÚcosine_similarityrW   rY   )rí   rî   rï   rX   Úlogitss        r0   Úcompute_contrastive_logitsz2UniSpeechForPreTraining.compute_contrastive_logits8  sq   € õ  œ) _Ð6GÐ$HÈaÐPÑPÔPˆåÔ(Ð);×)AÒ)AÑ)CÔ)CÀ_×EZÒEZÑE\ÔE\ÐbdÐeÑeÔeˆØ—’ Ñ0Ô0ˆð ˜+Ñ%ˆØˆr/   Nro   rž   rÏ   rÐ   c                 óÌ  —  | j         |fd|i|¤Ž}|j        }|                      |j        ¦  «        }|                      |¦  «        \  }}|                      |                     | j        j        j        ¦  «        ¦  «        }|  	                    |¦  «        }t          j        |                     d¦  «        |                     d¦  «        ¦  «                             | j        j        ¦  «        }	|	                     dd¦  «        }	t          j        |	¦  «                             ¦   «                              |j        ¦  «        }
|
                     dd¦  «        }
|
                     d¦  «        }
|                     |
d¦  «        |                     |
 d¦  «        z   }|                      |¦  «        }|                      |¦  «        }d}t1          |||||j        |j        ¬¦  «        S )a›  
        Example:

        ```python
        >>> import torch
        >>> from transformers import AutoFeatureExtractor, UniSpeechForPreTraining

        >>> feature_extractor = AutoFeatureExtractor.from_pretrained("microsoft/unispeech-large-1500h-cv")
        >>> model = UniSpeechForPreTraining.from_pretrained("microsoft/unispeech-large-1500h-cv")
        >>> # TODO: Add full pretraining example
        ```rž   r   r   rB   ru   N)r    r!   r"   r#   r$   r%   )rn   rÒ   rÞ   rÓ   rß   rã   r£   r~   r    rä   r*   ÚemptyÚsizeÚfill_rm   Úreplace_probrÔ   Ú	bernoullir¨   r¡   r^   Úmasked_fillrè   ræ   r   r$   r%   )ra   ro   rž   rÏ   ÚoutputsÚtransformer_featuresrÓ   Úquantized_featuresr#   Úprob_replace_matrixÚsampled_replace_matrixró   r    s                r0   ri   zUniSpeechForPreTraining.forwardL  sô  € ð& !�$”.Øð
ð 
à)ð
ð ð
ð 
ˆð
  'Ô8Ðð  ×0Ò0°Ô1IÑJÔJÐØ48·N²NÐCSÑ4TÔ4TÑ1ÐÐ1ð "Ÿ^š^Ð,>×,AÒ,AÀ$Ä.ÔBWÔB]Ñ,^Ô,^Ñ_Ô_ÐØ!×-Ò-Ð.@ÑAÔAÐå#œkÐ*>×*CÒ*CÀAÑ*FÔ*FÐH\×HaÒHaÐbcÑHdÔHdÑeÔe×kÒkØŒKÔ$ñ
ô 
Ðð 2×;Ò;¸A¸qÑAÔAÐÝ!&¤Ð1DÑ!EÔ!E×!JÒ!JÑ!LÔ!L×!OÒ!OÐPdÔPkÑ!lÔ!lÐØ!7×!AÒ!AÀ!ÀQÑ!GÔ!GÐØ!7×!AÒ!AÀ"Ñ!EÔ!EÐØ%×1Ò1Ð2HÈ#ÑNÔNØ×*Ò*Ð,BÐ+BÀCÑHÔHñ
ˆð
 —’˜fÑ%Ô%ˆØ—’˜vÑ&Ô&ˆð ˆå,ØØ1Ø'9Ø"7Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r/   )r   rÛ   )r&   r'   r(   r   r¿   r¹   rê   rÌ   rj   r*   r+   rô   r   r   rÅ   r	   r
   r-   r   ri   rº   r»   s   @r0   rÙ   rÙ     s:  ø€ € € € € ð˜ð ð ð ð ð ð ð1°#ð 1ð 1ð 1ð 1ð>ð >ð >ð ð
 ð	ð ØÔ*ðà Ô,ðð "Ô-ðð ð	ð ð ñ „\ðð& Øð /3ð9
ð 9
à”l TÑ)ð9
ð œ tÑ+ð9
ð Ð+Ô,ð	9
ð
 
Ð.Ñ	.ð9
ð 9
ð 9
ñ „^ñ Ôð9
ð 9
ð 9
ð 9
ð 9
r/   rÙ   c                   ó   — e Zd ZdS )ÚUniSpeechForCTCNr3   r.   r/   r0   r  r  Š  r4   r/   r  c                   ó   — e Zd ZdS )Ú"UniSpeechForSequenceClassificationNr3   r.   r/   r0   r  r  Ž  r4   r/   r  )r  rÙ   r  r½   rl   )8r)   rƒ   Údataclassesr   r*   Útorch.nnrT   Ú r   r|   Úmodeling_outputsr   r   Úmodeling_utilsr   Úprocessing_utilsr	   Úutilsr
   r   r   Úutils.genericr   r   Úutils.output_capturingr   r   Úwav2vec2.modeling_wav2vec2r   r   r   r   r   r   r   r   r   Úconfiguration_unispeechr   Ú
get_loggerr&   Úloggerr   r2   r6   r8   r:   r<   r>   rl   rÖ   r½   rÙ   r  r  Ú__all__r.   r/   r0   ú<module>r     s
  ðð Ð à €€€Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø DÐ DÐ DÐ DÐ DÐ DÐ DÐ DØ -Ð -Ð -Ð -Ð -Ð -Ø &Ð &Ð &Ð &Ð &Ð &Ø @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ EÐ EÐ EÐ EÐ EÐ EÐ EÐ Eð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 5Ð 4Ð 4Ð 4Ð 4Ð 4ð 
ˆÔ	˜HÑ	%Ô	%€ð €ððñ ô ð
 ð7ð 7ð 7ð 7ð 7 Kñ 7ô 7ñ „ñô ð7ð.	ð 	ð 	ð 	ð 	Ð'Fñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð4ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð!:ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	�ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð&Dñ 	ô 	ð 	ð*'ð *'ð *'ð *'ð *'Ð%Bñ *'ô *'ð *'ðZ ðEð Eð Eð Eð E˜ñ Eô Eñ „ðEðP 3Ð ð;
ð ;
ð ;
ð ;
ð ;
Ð-¨}ñ ;
ô ;
ð ;
ð| €ððñ ô ð
l
ð l
ð l
ð l
ð l
Ð6ñ l
ô l
ñô ð
l
ð^	ð 	ð 	ð 	ð 	�nñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð)Jñ 	ô 	ð 	ðð ð €€€r/   