§
    9²¸jƒ»  ã                   ó¸  — d dl mZ d dlZd dlZd dlmZ d dlmZ ddlm	Z
 ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZmZmZ ddlmZmZm Z  ddl!m"Z" ddl#m$Z$m%Z%m&Z& ddl'm(Z(m)Z) ddl*m+Z+m,Z, ddl-m.Z.  e&j/        e0¦  «        Z1 G d„ dej2        ¦  «        Z3 G d„ dej2        ¦  «        Z4 G d„ de¦  «        Z5 G d„ de¦  «        Z6 G d„ de¦  «        Z7 G d„ d ej2        ¦  «        Z8 G d!„ d"ej2        ¦  «        Z9	 	 dMd$ej2        d%ej:        d&ej:        d'ej:        d(ej:        dz  d)e;dz  d*e;d+e"e$         fd,„Z< G d-„ d.ej2        ¦  «        Z= G d/„ d0ej2        ¦  «        Z> G d1„ d2e¦  «        Z? G d3„ d4ej2        ¦  «        Z@ G d5„ d6ej2        ¦  «        ZA G d7„ d8e¦  «        ZB G d9„ d:ej2        ¦  «        ZCe% G d;„ d<e¦  «        ¦   «         ZD	 	 dNd=eEeFeFf         d>e;d?eFd(ejG        dz  d@eFdAejH        fdB„ZIe% G dC„ dDeD¦  «        ¦   «         ZJ e%dE¬F¦  «         G dG„ dHeD¦  «        ¦   «         ZK e%dI¬F¦  «         G dJ„ dKeD¦  «        ¦   «         ZLg dL¢ZMdS )Oé    )ÚCallableN)ÚCrossEntropyLossé   )Úinitialization)ÚACT2FN)Úfilter_output_hidden_states)Úis_deepspeed_zero3_enabled)Úis_fsdp_managed_module)Úcreate_bidirectional_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚCausalLMOutputÚSequenceClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModelÚ*get_torch_context_manager_or_global_device)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)Úcan_return_tupleÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚHubertConfigc                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚHubertPositionalConvEmbeddingc                 ó:  •— t          ¦   «                              ¦   «          t          j        |j        |j        |j        |j        dz  |j        ¬¦  «        | _        d | _        t          |dd¦  «        r t          j
        |j        ¦  «        | _        �nWt          j        j        }t          t          j        j        d¦  «        rt          j        j        j        }t          ¦   «         rëdd l}|j                             | j        j        d¬¦  «        5   || j        dd¬	¦  «        | _        d d d ¦  «         n# 1 swxY w Y   t          | j        d
¦  «        r-| j        j        j        j        }| j        j        j        j        }n| j        j        }| j        j        }|j                             | |¦  «         |j                             | |¦  «         n || j        dd¬	¦  «        | _        t3          |j        ¦  «        | _        t6          |j                 | _        d S )Né   )Úkernel_sizeÚpaddingÚgroupsÚconv_pos_batch_normFÚweight_normr   ©Úmodifier_rankÚweight)ÚnameÚdimÚparametrizations)ÚsuperÚ__init__ÚnnÚConv1dÚhidden_sizeÚnum_conv_pos_embeddingsÚnum_conv_pos_embedding_groupsÚconvÚ
batch_normÚgetattrÚBatchNorm1dÚutilsr&   Úhasattrr,   r	   Ú	deepspeedÚzeroÚGatheredParametersr)   Ú	original0Ú	original1Úweight_gÚweight_vÚregister_external_parameterÚHubertSamePadLayerr#   r   Úfeat_extract_activationÚ
activation)ÚselfÚconfigr&   r:   r?   r@   Ú	__class__s         €úz/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/hubert/modeling_hubert.pyr.   z&HubertPositionalConvEmbedding.__init__1   s1  ø€ Ý‰Œ×ÒÑÔÐÝ”IØÔØÔØÔ6ØÔ2°aÑ7ØÔ7ð
ñ 
ô 
ˆŒ	ð ˆŒÝ�6Ð0°%Ñ8Ô8ð 	IÝ œn¨VÔ-?Ñ@Ô@ˆDŒO‰Oåœ(Ô.ˆKÝ•r”xÔ0°-Ñ@Ô@ð DÝ œhÔ7ÔC�å)Ñ+Ô+ð IØ Ð Ð Ð à”^×6Ò6°t´yÔ7GÐWXÐ6ÑYÔYð Mð MØ + ¨D¬I¸HÈ!Ð LÑ LÔ L�D”IðMð Mð Mñ Mô Mð Mð Mð Mð Mð Mð Møøøð Mð Mð Mð Må˜4œ9Ð&8Ñ9Ô9ð 2Ø#œyÔ9Ô@ÔJ�HØ#œyÔ9Ô@ÔJ�H�Hà#œyÔ1�HØ#œyÔ1�HØ”×:Ò:¸4ÀÑJÔJÐJØ”×:Ò:¸4ÀÑJÔJÐJÐJà'˜K¨¬	¸ÀaÐHÑHÔH�”	å)¨&Ô*HÑIÔIˆŒÝ  Ô!?Ô@ˆŒˆˆs   ÄD7Ä7D;Ä>D;c                 ó  — |                      dd¦  «        }| j        �|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      dd¦  «        }|S )Nr   r!   )Ú	transposer5   r4   r#   rD   ©rE   Úhidden_statess     rH   Úforwardz%HubertPositionalConvEmbedding.forwardV   s~   € Ø%×/Ò/°°1Ñ5Ô5ˆØŒ?Ð&Ø ŸOšO¨MÑ:Ô:ˆMØŸ	š	 -Ñ0Ô0ˆØŸš ]Ñ3Ô3ˆØŸš¨Ñ6Ô6ˆà%×/Ò/°°1Ñ5Ô5ˆØÐó    ©Ú__name__Ú
__module__Ú__qualname__r.   rM   Ú__classcell__©rG   s   @rH   r   r   0   sM   ø€ € € € € ð#Að #Að #Að #Að #AðJ	ð 	ð 	ð 	ð 	ð 	ð 	rN   r   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )rB   c                 ól   •— t          ¦   «                              ¦   «          |dz  dk    rdnd| _        d S )Nr!   r   r   )r-   r.   Únum_pad_remove)rE   r2   rG   s     €rH   r.   zHubertSamePadLayer.__init__c   s:   ø€ Ý‰Œ×ÒÑÔÐØ#:¸QÑ#>À!Ò#CÐ#C˜a˜aÈˆÔÐÐrN   c                 óJ   — | j         dk    r|d d …d d …d | j          …f         }|S ©Nr   )rW   rK   s     rH   rM   zHubertSamePadLayer.forwardg   s;   € ØÔ Ò"Ð"Ø)¨!¨!¨!¨Q¨Q¨QÐ0F°4Ô3FÐ2FÐ0FÐ*FÔGˆMØÐrN   rO   rT   s   @rH   rB   rB   b   sL   ø€ € € € € ðKð Kð Kð Kð Kðð ð ð ð ð ð rN   rB   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚHubertNoLayerNormConvLayerr   c                 óZ  •— t          ¦   «                              ¦   «          |dk    r|j        |dz
           nd| _        |j        |         | _        t          j        | j        | j        |j        |         |j        |         |j	        ¬¦  «        | _
        t          |j                 | _        d S )Nr   r   ©r"   ÚstrideÚbias)r-   r.   Úconv_dimÚin_conv_dimÚout_conv_dimr/   r0   Úconv_kernelÚconv_strideÚ	conv_biasr4   r   rC   rD   ©rE   rF   Úlayer_idrG   s      €rH   r.   z#HubertNoLayerNormConvLayer.__init__n   s�   ø€ Ý‰Œ×ÒÑÔÐØ<DÀqºL¸L˜6œ?¨8°a©<Ô8Ð8ÈaˆÔØ"œO¨HÔ5ˆÔå”IØÔØÔØÔ*¨8Ô4ØÔ% hÔ/ØÔ!ð
ñ 
ô 
ˆŒ	õ ! Ô!?Ô@ˆŒˆˆrN   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S ©N)r4   rD   rK   s     rH   rM   z"HubertNoLayerNormConvLayer.forward|   s*   € ØŸ	š	 -Ñ0Ô0ˆØŸš¨Ñ6Ô6ˆØÐrN   ©r   rO   rT   s   @rH   r[   r[   m   sR   ø€ € € € € ðAð Að Að Að Að Aðð ð ð ð ð ð rN   r[   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚHubertLayerNormConvLayerr   c                 óš  •— t          ¦   «                              ¦   «          |dk    r|j        |dz
           nd| _        |j        |         | _        t          j        | j        | j        |j        |         |j        |         |j	        ¬¦  «        | _
        t          j        | j        d¬¦  «        | _        t          |j                 | _        d S )Nr   r   r]   T)Úelementwise_affine)r-   r.   r`   ra   rb   r/   r0   rc   rd   re   r4   Ú	LayerNormÚ
layer_normr   rC   rD   rf   s      €rH   r.   z!HubertLayerNormConvLayer.__init__ƒ   s¶   ø€ Ý‰Œ×ÒÑÔÐØ<DÀqºL¸L˜6œ?¨8°a©<Ô8Ð8ÈaˆÔØ"œO¨HÔ5ˆÔå”IØÔØÔØÔ*¨8Ô4ØÔ% hÔ/ØÔ!ð
ñ 
ô 
ˆŒ	õ œ, tÔ'8ÈTÐRÑRÔRˆŒÝ  Ô!?Ô@ˆŒˆˆrN   c                 óÜ   — |                       |¦  «        }|                     dd¦  «        }|                      |¦  «        }|                     dd¦  «        }|                      |¦  «        }|S )Néþÿÿÿéÿÿÿÿ)r4   rJ   rp   rD   rK   s     rH   rM   z HubertLayerNormConvLayer.forward’   se   € ØŸ	š	 -Ñ0Ô0ˆà%×/Ò/°°BÑ7Ô7ˆØŸš¨Ñ6Ô6ˆØ%×/Ò/°°BÑ7Ô7ˆàŸš¨Ñ6Ô6ˆØÐrN   rj   rO   rT   s   @rH   rl   rl   ‚   sR   ø€ € € € € ðAð Að Að Að Að Aðð ð ð ð ð ð rN   rl   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚHubertGroupNormConvLayerr   c                 ó¦  •— t          ¦   «                              ¦   «          |dk    r|j        |dz
           nd| _        |j        |         | _        t          j        | j        | j        |j        |         |j        |         |j	        ¬¦  «        | _
        t          |j                 | _        t          j        | j        | j        d¬¦  «        | _        d S )Nr   r   r]   T)Ú
num_groupsÚnum_channelsÚaffine)r-   r.   r`   ra   rb   r/   r0   rc   rd   re   r4   r   rC   rD   Ú	GroupNormrp   rf   s      €rH   r.   z!HubertGroupNormConvLayer.__init__ž   s½   ø€ Ý‰Œ×ÒÑÔÐØ<DÀqºL¸L˜6œ?¨8°a©<Ô8Ð8ÈaˆÔØ"œO¨HÔ5ˆÔå”IØÔØÔØÔ*¨8Ô4ØÔ% hÔ/ØÔ!ð
ñ 
ô 
ˆŒ	õ ! Ô!?Ô@ˆŒåœ,°$Ô2CÐRVÔRcÐlpÐqÑqÔqˆŒˆˆrN   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S ri   )r4   rp   rD   rK   s     rH   rM   z HubertGroupNormConvLayer.forward®   s;   € ØŸ	š	 -Ñ0Ô0ˆØŸš¨Ñ6Ô6ˆØŸš¨Ñ6Ô6ˆØÐrN   rj   rO   rT   s   @rH   ru   ru   �   sR   ø€ € € € € ðrð rð rð rð rð rð ð ð ð ð ð ð rN   ru   c                   ó.   ‡ — e Zd ZdZˆ fd„Zd„ Zd„ Zˆ xZS )ÚHubertFeatureEncoderz.Construct the features from raw audio waveformc                 ó¤  •‡— t          ¦   «                              ¦   «          ‰j        dk    r7t          ‰d¬¦  «        gˆfd„t	          ‰j        dz
  ¦  «        D ¦   «         z   }nD‰j        dk    r!ˆfd„t	          ‰j        ¦  «        D ¦   «         }nt          d‰j        › d	�¦  «        ‚t          j        |¦  «        | _	        d
| _
        d| _        d S )NÚgroupr   ©rg   c                 ó8   •— g | ]}t          ‰|d z   ¬¦  «        ‘ŒS )r   r€   )r[   ©Ú.0ÚirF   s     €rH   ú
<listcomp>z1HubertFeatureEncoder.__init__.<locals>.<listcomp>¼   s>   ø€ ð Lð Lð LØGHÕ*¨6¸AÀ¹EÐBÑBÔBðLð Lð LrN   r   Úlayerc                 ó2   •— g | ]}t          ‰|¬ ¦  «        ‘ŒS )r€   )rl   r‚   s     €rH   r…   z1HubertFeatureEncoder.__init__.<locals>.<listcomp>À   s'   ø€ ÐwÐwÐwÈAÕ3°FÀQÐGÑGÔGÐwÐwÐwrN   z`config.feat_extract_norm` is z), but has to be one of ['group', 'layer']FT)r-   r.   Úfeat_extract_normru   ÚrangeÚnum_feat_extract_layersÚ
ValueErrorr/   Ú
ModuleListÚconv_layersÚgradient_checkpointingÚ_requires_grad)rE   rF   r�   rG   s    ` €rH   r.   zHubertFeatureEncoder.__init__¸   s  øø€ Ý‰Œ×ÒÑÔÐàÔ# wÒ.Ð.Ý3°FÀQÐGÑGÔGÐHð Lð Lð Lð LÝLQÐRXÔRpÐstÑRtÑLuÔLuðLñ Lô Lñ ˆKˆKð Ô%¨Ò0Ð0ØwÐwÐwÐwÕQVÐW]ÔWuÑQvÔQvÐwÑwÔwˆKˆKåØt°Ô1IÐtÐtÐtñô ð õ œ=¨Ñ5Ô5ˆÔØ&+ˆÔ#Ø"ˆÔÐÐrN   c                 óP   — |                       ¦   «         D ]	}d|_        Œ
d| _        d S )NF)Ú
parametersÚrequires_gradr�   ©rE   Úparams     rH   Ú_freeze_parametersz'HubertFeatureEncoder._freeze_parametersÉ   s4   € Ø—_’_Ñ&Ô&ð 	(ð 	(ˆEØ"'ˆEÔÐØ#ˆÔÐÐrN   c                 ór   — |d d …d f         }| j         r| j        rd|_        | j        D ]} ||¦  «        }Œ|S )NT)r�   Útrainingr’   r�   )rE   Úinput_valuesrL   Ú
conv_layers       rH   rM   zHubertFeatureEncoder.forwardÎ   s[   € Ø$ Q Q Q¨ WÔ-ˆð Ôð 	/ 4¤=ð 	/Ø*.ˆMÔ'àÔ*ð 	6ð 	6ˆJØ&˜J }Ñ5Ô5ˆMˆMàÐrN   )rP   rQ   rR   Ú__doc__r.   r•   rM   rS   rT   s   @rH   r}   r}   µ   s\   ø€ € € € € Ø8Ð8ð#ð #ð #ð #ð #ð"$ð $ð $ð

ð 
ð 
ð 
ð 
ð 
ð 
rN   r}   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚHubertFeatureProjectionc                 óT  •— t          ¦   «                              ¦   «          |j        | _        | j        r+t          j        |j        d         |j        ¬¦  «        | _        t          j        |j        d         |j	        ¦  «        | _
        t          j        |j        ¦  «        | _        d S )Nrs   ©Úeps)r-   r.   Úfeat_proj_layer_normr/   ro   r`   Úlayer_norm_epsrp   ÚLinearr1   Ú
projectionÚDropoutÚfeat_proj_dropoutÚdropout©rE   rF   rG   s     €rH   r.   z HubertFeatureProjection.__init__Ü   s…   ø€ Ý‰Œ×ÒÑÔÐØ$*Ô$?ˆÔ!ØÔ$ð 	[Ý œl¨6¬?¸2Ô+>ÀFÔDYÐZÑZÔZˆDŒOÝœ) F¤O°BÔ$7¸Ô9KÑLÔLˆŒÝ”z &Ô":Ñ;Ô;ˆŒˆˆrN   c                 ó’   — | j         r|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S ri   )r    rp   r£   r¦   rK   s     rH   rM   zHubertFeatureProjection.forwardä   sF   € àÔ$ð 	;Ø ŸOšO¨MÑ:Ô:ˆMØŸš¨Ñ6Ô6ˆØŸš ]Ñ3Ô3ˆØÐrN   rO   rT   s   @rH   rœ   rœ   Û   sG   ø€ € € € € ð<ð <ð <ð <ð <ðð ð ð ð ð ð rN   rœ   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingr¦   Úkwargsc                 ó®  — |€|                      d¦  «        dz  }t          j        ||                     dd¦  «        ¦  «        |z  }|�||z   }t          j                             |d¬¦  «        }t          j                             ||| j        ¬¦  «        }t          j        ||¦  «        }	|	                     dd¦  «         	                    ¦   «         }	|	|fS )Nrs   ç      à¿r!   r   ©r+   )Úpr—   r   )
ÚsizeÚtorchÚmatmulrJ   r/   Ú
functionalÚsoftmaxr¦   r—   Ú
contiguous)
rª   r«   r¬   r­   r®   r¯   r¦   r°   Úattn_weightsÚattn_outputs
             rH   Úeager_attention_forwardr½   í   sÈ   € ð €Ø—*’*˜R‘.”. DÑ(ˆõ ”<  s§}¢}°Q¸Ñ':Ô':Ñ;Ô;¸gÑE€LàÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2Ð(Ñ>Ô>€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$rN   c                   óà   ‡ — e Zd ZdZ	 	 	 	 	 ddededed	ed
edededz  fˆ fd„Z	 	 	 dde	j
        de	j
        dz  de	j
        dz  dedz  dee         dee	j
        e	j
        dz  f         fd„Zˆ xZS )ÚHubertAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr©   FTNÚ	embed_dimÚ	num_headsr¦   Ú
is_decoderr_   Ú	is_causalrF   c                 ó
  •— t          ¦   «                              ¦   «          || _        || _        || _        ||z  | _        || _        | j        |z  | j        k    rt          d| j        › d|› d�¦  «        ‚| j        dz  | _        || _	        || _
        t          j        |||¬¦  «        | _        t          j        |||¬¦  «        | _        t          j        |||¬¦  «        | _        t          j        |||¬¦  «        | _        d S )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).r²   )r_   )r-   r.   rÀ   rÁ   r¦   Úhead_dimrF   r‹   r¯   rÂ   rÃ   r/   r¢   Úk_projÚv_projÚq_projÚout_proj)	rE   rÀ   rÁ   r¦   rÂ   r_   rÃ   rF   rG   s	           €rH   r.   zHubertAttention.__init__  s  ø€ õ 	‰Œ×ÒÑÔÐØ"ˆŒØ"ˆŒØˆŒØ! YÑ.ˆŒØˆŒàŒM˜IÑ%¨$¬.Ò8Ð8Ýð3ÈdÌnð 3ð 3Ø%.ð3ð 3ð 3ñô ð ð ”} dÑ*ˆŒØ$ˆŒØ"ˆŒå”i 	¨9¸4Ð@Ñ@Ô@ˆŒÝ”i 	¨9¸4Ð@Ñ@Ô@ˆŒÝ”i 	¨9¸4Ð@Ñ@Ô@ˆŒÝœ	 )¨Y¸TÐBÑBÔBˆŒˆˆrN   rL   Úkey_value_statesr®   Úoutput_attentionsr°   Úreturnc                 óø  — |du}|j         dd…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }	|r|n|}
g |
j         dd…         ¢d‘| j        ‘R }|                      |
¦  «                             |¦  «                             dd¦  «        }|                      |
¦  «                             |¦  «                             dd¦  «        }t          j        | j	        j
        t          ¦  «        } || |	|||f| j        sdn| j        | j        |dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )z#Input shape: Batch x Time x ChannelNrs   r   r!   r©   )r¦   r¯   rË   )ÚshaperÅ   rÈ   ÚviewrJ   rÆ   rÇ   r   Úget_interfacerF   Ú_attn_implementationr½   r—   r¦   r¯   Úreshaperº   rÉ   )rE   rL   rÊ   r®   rË   r°   Úis_cross_attentionÚinput_shapeÚhidden_shapeÚquery_statesÚcurrent_statesÚkv_shapeÚ
key_statesÚvalue_statesÚattention_interfacer¼   r»   s                    rH   rM   zHubertAttention.forward+  s»  € ð .°TÐ9Ðð $Ô)¨#¨2¨#Ô.ˆà8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆð —{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆà-?ÐRÐ)Ð)À]ˆØB�^Ô)¨#¨2¨#Ô.ÐB°ÐB°D´MÐBÐBˆØ—[’[ Ñ0Ô0×5Ò5°hÑ?Ô?×IÒIÈ!ÈQÑOÔOˆ
Ø—{’{ >Ñ2Ô2×7Ò7¸ÑAÔA×KÒKÈAÈqÑQÔQˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð  $œ}Ð>�C�C°$´,Ø”LØ/ð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—m’m KÑ0Ô0ˆà˜LÐ(Ð(rN   )r©   FTFN)NNF)rP   rQ   rR   rš   ÚintÚfloatÚboolr   r.   r¶   ÚTensorr   r   ÚtuplerM   rS   rT   s   @rH   r¿   r¿   	  s<  ø€ € € € € ØGÐGð Ø ØØØ&*ðCð CàðCð ðCð ð	Cð
 ðCð ðCð ðCð ˜tÑ#ðCð Cð Cð Cð Cð CðD 15Ø.2Ø).ð0)ð 0)à”|ð0)ð  œ,¨Ñ-ð0)ð œ tÑ+ð	0)ð
   $™;ð0)ð Ð-Ô.ð0)ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð0)ð 0)ð 0)ð 0)ð 0)ð 0)ð 0)ð 0)rN   r¿   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚHubertFeedForwardc                 óÌ  •— t          ¦   «                              ¦   «          t          j        |j        ¦  «        | _        t          j        |j        |j        ¦  «        | _	        t          |j        t          ¦  «        rt          |j                 | _        n|j        | _        t          j        |j        |j        ¦  «        | _        t          j        |j        ¦  «        | _        d S ri   )r-   r.   r/   r¤   Úactivation_dropoutÚintermediate_dropoutr¢   r1   Úintermediate_sizeÚintermediate_denseÚ
isinstanceÚ
hidden_actÚstrr   Úintermediate_act_fnÚoutput_denseÚhidden_dropoutÚoutput_dropoutr§   s     €rH   r.   zHubertFeedForward.__init___  s°   ø€ Ý‰Œ×ÒÑÔÐÝ$&¤J¨vÔ/HÑ$IÔ$IˆÔ!å"$¤)¨FÔ,>ÀÔ@XÑ"YÔ"YˆÔÝ�fÔ'­Ñ-Ô-ð 	9Ý'-¨fÔ.?Ô'@ˆDÔ$Ð$à'-Ô'8ˆDÔ$åœI fÔ&>ÀÔ@RÑSÔSˆÔÝ œj¨Ô)>Ñ?Ô?ˆÔÐÐrN   c                 óØ   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S ri   )rç   rë   rå   rì   rî   rK   s     rH   rM   zHubertFeedForward.forwardl  sg   € Ø×/Ò/°Ñ>Ô>ˆØ×0Ò0°Ñ?Ô?ˆØ×1Ò1°-Ñ@Ô@ˆà×)Ò)¨-Ñ8Ô8ˆØ×+Ò+¨MÑ:Ô:ˆØÐrN   rO   rT   s   @rH   râ   râ   ^  sL   ø€ € € € € ð@ð @ð @ð @ð @ðð ð ð ð ð ð rN   râ   c                   ó\   ‡ — e Zd Zˆ fd„Z	 ddej        dej        dz  dee         fd„Zˆ xZ	S )ÚHubertEncoderLayerc                 ó�  •— t          ¦   «                              ¦   «          t          |j        |j        |j        d|¬¦  «        | _        t          j        |j	        ¦  «        | _
        t          j        |j        |j        ¬¦  «        | _        t          |¦  «        | _        t          j        |j        |j        ¬¦  «        | _        d S )NF©rÀ   rÁ   r¦   rÂ   rF   rž   )r-   r.   r¿   r1   Únum_attention_headsÚattention_dropoutÚ	attentionr/   r¤   rí   r¦   ro   r¡   rp   râ   Úfeed_forwardÚfinal_layer_normr§   s     €rH   r.   zHubertEncoderLayer.__init__w  s¬   ø€ Ý‰Œ×ÒÑÔÐÝ(ØÔ(ØÔ0ØÔ,ØØð
ñ 
ô 
ˆŒõ ”z &Ô"7Ñ8Ô8ˆŒÝœ, vÔ'9¸vÔ?TÐUÑUÔUˆŒÝ-¨fÑ5Ô5ˆÔÝ "¤¨VÔ-?ÀVÔEZÐ [Ñ [Ô [ˆÔÐÐrN   NrL   r®   r°   c                 óè   — |} | j         |fd|i|¤Ž\  }}|                      |¦  «        }||z   }|                      |¦  «        }||                      |¦  «        z   }|                      |¦  «        }|S ©Nr®   )rö   r¦   rp   r÷   rø   ©rE   rL   r®   r°   Úattn_residualÚ_s         rH   rM   zHubertEncoderLayer.forward†  sŠ   € ð &ˆØ)˜4œ>¨-ÐaÐaÈÐaÐZ`ÐaÐaÑˆ�qØŸš ]Ñ3Ô3ˆØ%¨Ñ5ˆàŸš¨Ñ6Ô6ˆØ%¨×(9Ò(9¸-Ñ(HÔ(HÑHˆØ×-Ò-¨mÑ<Ô<ˆàÐrN   ri   ©
rP   rQ   rR   r.   r¶   rß   r   r   rM   rS   rT   s   @rH   rñ   rñ   v  s†   ø€ € € € € ð\ð \ð \ð \ð \ð$ /3ðð à”|ðð œ tÑ+ðð Ð+Ô,ð	ð ð ð ð ð ð ð rN   rñ   c                   ó\   ‡ — e Zd Zˆ fd„Z	 ddej        dej        dz  dee         fd„Z	ˆ xZ
S )ÚHubertEncoderc                 ó‚  •‡— t          ¦   «                              ¦   «          ‰| _        t          ‰¦  «        | _        t          j        ‰j        ‰j        ¬¦  «        | _	        t          j
        ‰j        ¦  «        | _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        d| _        d S )Nrž   c                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS © )rñ   ©rƒ   rý   rF   s     €rH   r…   z*HubertEncoder.__init__.<locals>.<listcomp>Ÿ  s"   ø€ Ð$iÐ$iÐ$iÀAÕ%7¸Ñ%?Ô%?Ð$iÐ$iÐ$irN   F©r-   r.   rF   r   Úpos_conv_embedr/   ro   r1   r¡   rp   r¤   rí   r¦   rŒ   r‰   Únum_hidden_layersÚlayersrŽ   r§   s    `€rH   r.   zHubertEncoder.__init__™  s    øø€ Ý‰Œ×ÒÑÔÐØˆŒÝ;¸FÑCÔCˆÔÝœ, vÔ'9¸vÔ?TÐUÑUÔUˆŒÝ”z &Ô"7Ñ8Ô8ˆŒÝ”mÐ$iÐ$iÐ$iÐ$iÍÈvÔOgÑIhÔIhÐ$iÑ$iÔ$iÑjÔjˆŒØ&+ˆÔ#Ð#Ð#rN   NrL   r®   r°   c                 óD  — |�;|                      d¦  «                             dd|j        d         ¦  «        }d|| <   t          | j        ||¬¦  «        }|                      |¦  «        }||                     |j        ¦  «        z   }|                      |¦  «        }|  	                    |¦  «        }t          ¦   «         pt          | ¦  «        }| j        D ]<}t          j        g ¦  «        }| j        o|| j        j        k     }	|	r|r ||fd|i|¤Ž}Œ=t#          |¬¦  «        S ©Nrs   r   r!   r   )rF   Úinputs_embedsr®   r®   )Úlast_hidden_state)Ú	unsqueezeÚrepeatrÎ   r   rF   r  ÚtoÚdevicerp   r¦   r	   r
   r  r¶   Úrandr—   Ú	layerdropr   ©
rE   rL   r®   r°   Úexpand_attention_maskÚposition_embeddingsÚsynced_gpusr†   Údropout_probabilityÚskip_the_layers
             rH   rM   zHubertEncoder.forward¢  sK  € ð Ð%à$2×$<Ò$<¸RÑ$@Ô$@×$GÒ$GÈÈ1ÈmÔNaÐbcÔNdÑ$eÔ$eÐ!Ø45ˆMÐ0Ð0Ñ1å2Ø”;Ø'Ø)ð
ñ 
ô 
ˆð #×1Ò1°-Ñ@Ô@ÐØ%Ð(;×(>Ò(>¸}Ô?SÑ(TÔ(TÑTˆØŸš¨Ñ6Ô6ˆØŸš ]Ñ3Ô3ˆå0Ñ2Ô2ÐRÕ6LÈTÑ6RÔ6Rˆà”[ð 	^ð 	^ˆEå"'¤*¨R¡.¤.Ðà!œ]ÐZÐ/BÀTÄ[ÔEZÒ/ZˆNØ!ð ^ [ð ^à %  mÐ ]Ð ]ÀNÐ ]ÐV\Ð ]Ð ]�øå°Ð?Ñ?Ô?Ð?rN   ri   )rP   rQ   rR   r.   r¶   Útensorrß   r   r   rM   rS   rT   s   @rH   r   r   ˜  s�   ø€ € € € € ð,ð ,ð ,ð ,ð ,ð /3ð!@ð !@à”|ð!@ð œ tÑ+ð!@ð Ð+Ô,ð	!@ð !@ð !@ð !@ð !@ð !@ð !@ð !@rN   r   c                   ó4   ‡ — e Zd Zˆ fd„Zdej        fd„Zˆ xZS )ÚHubertAttnAdapterLayerc                 ót  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        t          j        | j        ¦  «        | _        t          j	        | j        | j        ¦  «        | _
        t          j        ¦   «         | _        t          j	        | j        | j        ¦  «        | _        dS )zŸ
        Implements adapter modules directly with 3D tensor weight as parameters and without using ModuleList to speed
        up training throughput.
        N)r-   r.   Úadapter_attn_dimÚ	input_dimr1   Ú
hidden_dimr/   ro   Únormr¢   Úlinear_1ÚReLUÚact_fnÚlinear_2r§   s     €rH   r.   zHubertAttnAdapterLayer.__init__Ç  s�   ø€ õ
 	‰Œ×ÒÑÔÐØÔ0ˆŒØ Ô,ˆŒå”L ¤Ñ1Ô1ˆŒ	Ýœ	 $¤/°4´>ÑBÔBˆŒÝ”g‘i”iˆŒÝœ	 $¤.°$´/ÑBÔBˆŒˆˆrN   rL   c                 ó®   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S ri   )r   r!  r#  r$  rK   s     rH   rM   zHubertAttnAdapterLayer.forwardÕ  sL   € ØŸ	š	 -Ñ0Ô0ˆàŸš mÑ4Ô4ˆØŸš MÑ2Ô2ˆØŸš mÑ4Ô4ˆàÐrN   )rP   rQ   rR   r.   r¶   ÚFloatTensorrM   rS   rT   s   @rH   r  r  Æ  s[   ø€ € € € € ðCð Cð Cð Cð Cð UÔ%6ð ð ð ð ð ð ð ð rN   r  c                   ó\   ‡ — e Zd Zˆ fd„Z	 ddej        dej        dz  dee         fd„Zˆ xZ	S )Ú!HubertEncoderLayerStableLayerNormc                 óì  •— t          ¦   «                              ¦   «          t          |j        |j        |j        d|¬¦  «        | _        t          j        |j	        ¦  «        | _
        t          j        |j        |j        ¬¦  «        | _        t          |¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t#          |dd ¦  «        �t%          |¦  «        | _        d S d | _        d S )NFró   rž   r  )r-   r.   r¿   r1   rô   rõ   rö   r/   r¤   rí   r¦   ro   r¡   rp   râ   r÷   rø   r6   r  Úadapter_layerr§   s     €rH   r.   z*HubertEncoderLayerStableLayerNorm.__init__à  sÝ   ø€ Ý‰Œ×ÒÑÔÐÝ(ØÔ(ØÔ0ØÔ,ØØð
ñ 
ô 
ˆŒõ ”z &Ô"7Ñ8Ô8ˆŒÝœ, vÔ'9¸vÔ?TÐUÑUÔUˆŒÝ-¨fÑ5Ô5ˆÔÝ "¤¨VÔ-?ÀVÔEZÐ [Ñ [Ô [ˆÔå�6Ð-¨tÑ4Ô4Ð@Ý!7¸Ñ!?Ô!?ˆDÔÐÐà!%ˆDÔÐÐrN   NrL   r®   r°   c                 ó"  — |}|                       |¦  «        } | j        |fd|i|¤Ž\  }}|                      |¦  «        }||z   }||                      |                      |¦  «        ¦  «        z   }| j        �||                      |¦  «        z   }|S rú   )rp   rö   r¦   r÷   rø   r*  rû   s         rH   rM   z)HubertEncoderLayerStableLayerNorm.forwardó  s§   € ð &ˆØŸš¨Ñ6Ô6ˆØ)˜4œ>¨-ÐaÐaÈÐaÐZ`ÐaÐaÑˆ�qØŸš ]Ñ3Ô3ˆØ%¨Ñ5ˆØ%¨×(9Ò(9¸$×:OÒ:OÐP]Ñ:^Ô:^Ñ(_Ô(_Ñ_ˆàÔÐ)Ø)¨D×,>Ò,>¸}Ñ,MÔ,MÑMˆMàÐrN   ri   rþ   rT   s   @rH   r(  r(  ß  s�   ø€ € € € € ð&ð &ð &ð &ð &ð, /3ðð à”|ðð œ tÑ+ðð Ð+Ô,ð	ð ð ð ð ð ð ð rN   r(  c                   ó:   ‡ — e Zd Zˆ fd„Z	 ddee         fd„Zˆ xZS )ÚHubertEncoderStableLayerNormc                 ó‚  •‡— t          ¦   «                              ¦   «          ‰| _        t          ‰¦  «        | _        t          j        ‰j        ‰j        ¬¦  «        | _	        t          j
        ‰j        ¦  «        | _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        d| _        d S )Nrž   c                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r  )r(  r  s     €rH   r…   z9HubertEncoderStableLayerNorm.__init__.<locals>.<listcomp>  s"   ø€ Ð`Ð`Ð`¸1Õ.¨vÑ6Ô6Ð`Ð`Ð`rN   Fr  r§   s    `€rH   r.   z%HubertEncoderStableLayerNorm.__init__  s¦   øø€ Ý‰Œ×ÒÑÔÐØˆŒÝ;¸FÑCÔCˆÔÝœ, vÔ'9¸vÔ?TÐUÑUÔUˆŒÝ”z &Ô"7Ñ8Ô8ˆŒÝ”mØ`Ð`Ð`Ð`ÅÀfÔF^Ñ@_Ô@_Ð`Ñ`Ô`ñ
ô 
ˆŒð ',ˆÔ#Ð#Ð#rN   Nr°   c                 ó  — |�;|                      d¦  «                             dd|j        d         ¦  «        }d|| <   t          | j        ||¬¦  «        }|                      |¦  «        }||z   }|                      |¦  «        }t          ¦   «         pt          | ¦  «        }| j	        D ]<}t          j        g ¦  «        }| j        o|| j        j        k     }	|	r|r ||fd|i|¤Ž}Œ=|                      |¦  «        }t          |¬¦  «        S r
  )r  r  rÎ   r   rF   r  r¦   r	   r
   r  r¶   r  r—   r  rp   r   r  s
             rH   rM   z$HubertEncoderStableLayerNorm.forward  s<  € ð Ð%à$2×$<Ò$<¸RÑ$@Ô$@×$GÒ$GÈÈ1ÈmÔNaÐbcÔNdÑ$eÔ$eÐ!Ø45ˆMÐ0Ð0Ñ1å2Ø”;Ø'Ø)ð
ñ 
ô 
ˆð #×1Ò1°-Ñ@Ô@ÐØ%Ð(;Ñ;ˆØŸš ]Ñ3Ô3ˆå0Ñ2Ô2ÐRÕ6LÈTÑ6RÔ6Rˆà”[ð 	^ð 	^ˆEå"'¤*¨R¡.¤.Ðà!œ]ÐZÐ/BÀTÄ[ÔEZÒ/ZˆNØ!ð ^ [ð ^ð !&  mÐ ]Ð ]ÀNÐ ]ÐV\Ð ]Ð ]�øàŸš¨Ñ6Ô6ˆå°Ð?Ñ?Ô?Ð?rN   ri   )rP   rQ   rR   r.   r   r   rM   rS   rT   s   @rH   r-  r-    sp   ø€ € € € € ð	,ð 	,ð 	,ð 	,ð 	,ð ð#@ð #@ð Ð+Ô,ð	#@ð #@ð #@ð #@ð #@ð #@ð #@ð #@rN   r-  c                   óÔ   ‡ — e Zd ZU eed<   dZdZdZddgZdZ	dZ
dZdZeeg eedd	¬
¦  «        dœZ ej        ¦   «         ˆ fd„¦   «         Zdej        ez  fd„Zdedej        fd„Zˆ xZS )ÚHubertPreTrainedModelrF   Úhubertr˜   Úaudiorñ   ÚParametrizedConv1dTr   Úencoder)ÚindexÚ
layer_name)rL   Ú
attentionsc                 óâ  •— t          ¦   «                              |¦  «         t          |t          j        ¦  «        �rt          ¦   «         rÑddl}t          |d¦  «        rjt          |d¦  «        rZ|j         	                    |j
        |j        gd¬¦  «        5  t          j        |j        ¦  «         ddd¦  «         n# 1 swxY w Y   nl|j         	                    |j        d¬¦  «        5  t          j        |j        ¦  «         ddd¦  «         n# 1 swxY w Y   nt          j        |j        ¦  «         |j        �t          j        |j        ¦  «         dS dS t          |t"          ¦  «        r-t          |d¦  «        rt          j        |j        ¦  «         dS dS t          |t(          ¦  «        r<t          |d¦  «        r.t          j        |j        d| j        j        d	z   z  ¦  «         dS dS dS )
zInitialize the weightsr   Nr@   r?   r'   Úmasked_spec_embedÚlayer_weightsg      ð?r   )r-   Ú_init_weightsrè   r/   r0   r	   r:   r9   r;   r<   r@   r?   ÚinitÚkaiming_normal_r)   r_   Úzeros_ÚHubertModelÚuniform_r;  ÚHubertForSequenceClassificationÚ	constant_r<  rF   r  )rE   rª   r:   rG   s      €rH   r=  z#HubertPreTrainedModel._init_weightsH  sv  ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�f�bœiÑ(Ô(ñ 	`Ý)Ñ+Ô+ð 
4Ø Ð Ð Ð å˜6 :Ñ.Ô.ð <µ7¸6À:Ñ3NÔ3Nð <Ø"œ×:Ò:¸F¼OÈVÌ_Ð;]ÐmnÐ:ÑoÔoð <ð <ÝÔ,¨V¬]Ñ;Ô;Ð;ð<ð <ð <ñ <ô <ð <ð <ð <ð <ð <ð <øøøð <ð <ð <ð <øð #œ×:Ò:¸6¼=ÐXYÐ:ÑZÔZð <ð <ÝÔ,¨V¬]Ñ;Ô;Ð;ð<ð <ð <ñ <ô <ð <ð <ð <ð <ð <ð <øøøð <ð <ð <ð <øõ Ô$ V¤]Ñ3Ô3Ð3àŒ{Ð&Ý”˜FœKÑ(Ô(Ð(Ð(Ð(ð 'Ð&å˜¥Ñ,Ô,ð 	`Ý�vÐ2Ñ3Ô3ð 8Ý”˜fÔ6Ñ7Ô7Ð7Ð7Ð7ð8ð 8å˜Õ ?Ñ@Ô@ð 	`Ý�v˜Ñ/Ô/ð `Ý”˜vÔ3°S¸D¼KÔ<YÐ\]Ñ<]Ñ5^Ñ_Ô_Ð_Ð_Ð_ð	`ð 	`ð`ð `s$   ÂB>Â>CÃCÃ+DÄDÄDÚinput_lengthsc                 óz   — d„ }t          | j        j        | j        j        ¦  «        D ]\  }} ||||¦  «        }Œ|S )zH
        Computes the output length of the convolutional layers
        c                 ó<   — t          j        | |z
  |d¬¦  «        dz   S )NÚfloor)Úrounding_moder   )r¶   Údiv)Úinput_lengthr"   r^   s      rH   Ú_conv_out_lengthzPHubertPreTrainedModel._get_feat_extract_output_lengths.<locals>._conv_out_lengthg  s&   € õ ”9˜\¨KÑ7¸ÈwÐWÑWÔWÐZ[Ñ[Ð[rN   )ÚziprF   rc   rd   )rE   rE  rL  r"   r^   s        rH   Ú _get_feat_extract_output_lengthsz6HubertPreTrainedModel._get_feat_extract_output_lengthsb  s\   € ð
	\ð 	\ð 	\õ
 $' t¤{Ô'>ÀÄÔ@WÑ#XÔ#Xð 	Qð 	QÑˆK˜Ø,Ð,¨]¸KÈÑPÔPˆMˆMàÐrN   Úfeature_vector_lengthr®   c                 óê  — |                       |                     d¦  «        ¦  «                             t          j        ¦  «        }|j        d         }t          j        ||f|j        |j        ¬¦  «        }d|t          j	        |j        d         |j        ¬¦  «        |dz
  f<   | 
                    dg¦  «                             d¦  «         
                    dg¦  «                             ¦   «         }|S )Nrs   r   )Údtyper  r   )r  )rN  Úsumr  r¶   ÚlongrÎ   ÚzerosrQ  r  ÚarangeÚflipÚcumsumrÞ   )rE   rO  r®   Úoutput_lengthsÚ
batch_sizes        rH   Ú"_get_feature_vector_attention_maskz8HubertPreTrainedModel._get_feature_vector_attention_maskq  sæ   € Ø×>Ò>¸~×?QÒ?QÐRTÑ?UÔ?UÑVÔV×YÒYÕZ_ÔZdÑeÔeˆØ#Ô)¨!Ô,ˆ
åœØÐ.Ð/°~Ô7KÐTbÔTið
ñ 
ô 
ˆð uvˆ�œ ^Ô%9¸!Ô%<À^ÔEZÐ[Ñ[Ô[Ð]kÐnoÑ]oÐpÑqØ'×,Ò,¨b¨TÑ2Ô2×9Ò9¸"Ñ=Ô=×BÒBÀBÀ4ÑHÔH×MÒMÑOÔOˆØÐrN   )rP   rQ   rR   r   Ú__annotations__Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚ_no_split_modulesÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnrñ   r(  r   r¿   Ú_can_record_outputsr¶   Úno_gradr=  Ú
LongTensorrÜ   rN  rZ  rS   rT   s   @rH   r2  r2  8  s  ø€ € € € € € àÐÐÑØ ÐØ$€OØÐØ-Ð/CÐDÐØ&*Ð#ØÐØ€NØÐà,Ð.OÐPØ$�n _¸AÈ)ÐTÑTÔTðð Ðð
 €U„]�_„_ð`ð `ð `ð `ñ „_ð`ð2¸eÔ>NÐQTÑ>Tð ð ð ð ð
Èð 
Ð]bÔ]mð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
rN   r2  rÎ   Ú	mask_probÚmask_lengthÚ	min_masksrÌ   c                 ó@  ‡‡‡‡‡— | \  }Š‰dk     rt          d¦  «        ‚‰‰k    rt          d‰› d‰› d�¦  «        ‚t          j                             d¦  «                             ¦   «         Šˆˆˆˆˆfd„}|�9|                     ¦   «                              d¦  «                             ¦   «         nˆfd	„t          |¦  «        D ¦   «         }t          j	        |‰ft          ¬
¦  «        }g }	 |‰¦  «        }
|
dk    r|S |D ]·} ||¦  «        }t          j                             t          j        |‰dz
  z
  ¦  «        |d¬¦  «        }t          |¦  «        dk    r‰dz
  }n|d         }t          j        |t          j        |
|z
  t          j        ¬
¦  «        |z  g¦  «        }|	                     |¦  «         Œ¸t          j        |	¦  «        }	t          j        |	dd…dd…df         ||
‰f¦  «        }	|	                     ||
‰z  ¦  «        }	t          j        ‰¦  «        dddd…f         }t          j        |||
‰f¦  «                             ||
‰z  ¦  «        }|	|z   }	|	                     ¦   «         ‰dz
  k    r‰dz
  |	|	‰dz
  k    <   t          j        ||	dd¦  «         |S )an  
    Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for
    ASR](https://huggingface.co/papers/1904.08779). Note that this method is not optimized to run on TPU and should be run on
    CPU as part of the preprocessing during training.

    Args:
        shape: The shape for which to compute masks. This should be of a tuple of size 2 where
               the first element is the batch size and the second element is the length of the axis to span.
        mask_prob:  The percentage of the whole axis (between 0 and 1) which will be masked. The number of
                    independently generated mask spans of length `mask_length` is computed by
                    `mask_prob*shape[1]/mask_length`. Note that due to overlaps, `mask_prob` is an upper bound and the
                    actual percentage will be smaller.
        mask_length: size of the mask
        min_masks: minimum number of masked spans
        attention_mask: A (right-padded) attention mask which independently shortens the feature axis of
                        each batch dimension.
    r   z&`mask_length` has to be bigger than 0.zO`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: z and `sequence_length`: ú`c                 ó¸   •— t          ‰| z  ‰z  ‰z   ¦  «        }t          |‰¦  «        }|‰z  ‰k    r‰‰z  }| ‰dz
  z
  |k     rt          | ‰dz
  z
  d¦  «        }|S )z;Given input length, compute how many spans should be maskedr   r   )rÜ   Úmax)rK  Únum_masked_spanÚepsilonrh  rg  ri  Úsequence_lengths     €€€€€rH   Úcompute_num_masked_spanz6_compute_mask_indices.<locals>.compute_num_masked_span¤  s~   ø€ å˜i¨,Ñ6¸ÑDÀwÑNÑOÔOˆÝ˜o¨yÑ9Ô9ˆð ˜[Ñ(¨?Ò:Ð:Ø-°Ñ<ˆOð ˜;¨™?Ñ+¨oÒ=Ð=Ý! ,°+À±/Ñ"BÀAÑFÔFˆOàÐrN   Nrs   c                 ó   •— g | ]}‰‘ŒS r  r  )rƒ   rý   rp  s     €rH   r…   z)_compute_mask_indices.<locals>.<listcomp>·  s   ø€ Ð9Ð9Ð9 !ˆoÐ9Ð9Ð9rN   ©rQ  r   F)Úreplace)r‹   ÚnpÚrandomr  ÚitemÚdetachrR  Útolistr‰   rT  rÞ   ÚchoicerU  ÚlenÚconcatenateÚonesÚint32ÚappendÚarrayÚbroadcast_torÒ   rm  Úput_along_axis)rÎ   rg  rh  r®   ri  rY  rq  rE  Úspec_aug_maskÚspec_aug_mask_idxsÚmax_num_masked_spanrK  rn  Úspec_aug_mask_idxÚdummy_mask_idxÚoffsetsro  rp  s    `` `           @@rH   Ú_compute_mask_indicesr‰  ~  sP  øøøøø€ ð0 #(Ñ€J�à�Q‚€ÝÐAÑBÔBÐBà�_Ò$Ð$Ýð:Ð^ið :ð :Ø'6ð:ð :ð :ñ
ô 
ð 	
õ Œi�nŠn˜QÑÔ×$Ò$Ñ&Ô&€Gðð ð ð ð ð ð ð ð ð$ Ð%ð 	×ÒÑÔ×#Ò# BÑ'Ô'×.Ò.Ñ0Ô0Ð0à9Ð9Ð9Ð9¥u¨ZÑ'8Ô'8Ð9Ñ9Ô9ð õ ”H˜j¨/Ð:Å$ÐGÑGÔG€MØÐà1Ð1°/ÑBÔBÐà˜aÒÐØÐà%ð 5ð 5ˆà1Ð1°,Ñ?Ô?ˆõ œI×,Ò,ÝŒI�l k°A¡oÑ6Ñ7Ô7¸ÐRWð -ñ 
ô 
Ðõ Ð Ñ!Ô! QÒ&Ð&ð -¨qÑ0ˆNˆNà.¨qÔ1ˆNåœNØ¥¤Ð(;¸oÑ(MÕUWÔU]Ð ^Ñ ^Ô ^ÐaoÑ oÐpñ
ô 
Ðð 	×!Ò!Ð"3Ñ4Ô4Ð4Ð4åœÐ"4Ñ5Ô5Ðõ œØ˜1˜1˜1˜a˜a˜a ˜:Ô&¨Ð5HÈ+Ð(Vñô Ðð ,×3Ò3°JÐ@SÐVaÑ@aÑbÔbÐõ Œi˜Ñ$Ô$ T¨4°°° ]Ô3€GÝŒo˜g¨
Ð4GÈÐ'UÑVÔV×^Ò^ØÐ'¨+Ñ5ñô €Gð ,¨gÑ5Ðð ×ÒÑÔ /°AÑ"5Ò5Ð5ØGVÐYZÑGZÐÐ-°À!Ñ0CÒCÑDõ Ô�mÐ%7¸¸BÑ?Ô?Ð?àÐrN   c                   ó$  ‡ — e Zd Zdefˆ fd„Z	 	 ddej        dej        dz  dej        dz  fd„Ze	 e
d¬	¦  «        e	 	 	 	 	 dd
ej        dz  dej        dz  dej        dz  dedz  dedz  dedz  deez  fd„¦   «         ¦   «         ¦   «         Zˆ xZS )rA  rF   c                 óà  •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        t          |¦  «        | _        |j        dk    s|j        dk    rBt          j
        t          j        |j        ¦  «                             ¦   «         ¦  «        | _        |j        rt#          |¦  «        | _        nt'          |¦  «        | _        |                      ¦   «          d S ©Nr©   )r-   r.   rF   r}   Úfeature_extractorrœ   Úfeature_projectionÚmask_time_probÚmask_feature_probr/   Ú	Parameterr¶   rß   r1   rB  r;  Údo_stable_layer_normr-  r6  r   Ú	post_initr§   s     €rH   r.   zHubertModel.__init__÷  sÍ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒÝ!5°fÑ!=Ô!=ˆÔÝ"9¸&Ñ"AÔ"AˆÔð Ô  3Ò&Ð&¨&Ô*BÀSÒ*HÐ*HÝ%'¤\µ%´,¸vÔ?QÑ2RÔ2R×2[Ò2[Ñ2]Ô2]Ñ%^Ô%^ˆDÔ"àÔ&ð 	1Ý7¸Ñ?Ô?ˆDŒLˆLå(¨Ñ0Ô0ˆDŒLð 	�ŠÑÔÐÐÐrN   NrL   Úmask_time_indicesr®   c                 ó  — t          | j        dd¦  «        s|S |                     ¦   «         \  }}}|�#| j                             |j        ¦  «        ||<   n“| j        j        dk    rƒ| j        r|t          ||f| j        j        | j        j	        || j        j
        ¬¦  «        }t          j        ||j        t          j        ¬¦  «        }| j                             |j        ¦  «        ||<   | j        j        dk    r†| j        rt          ||f| j        j        | j        j        | j        j        ¬¦  «        }t          j        ||j        t          j        ¬¦  «        }|dd…df                              d|d¦  «        }d||<   |S )	z¢
        Masks extracted features along time axis and/or along feature axis according to
        [SpecAugment](https://huggingface.co/papers/1904.08779).
        Úapply_spec_augmentTNr   )rg  rh  r®   ri  )r  rQ  )rg  rh  ri  rs   )r6   rF   rµ   r;  r  rQ  r�  r—   r‰  Úmask_time_lengthÚmask_time_min_masksr¶   r  r  rÞ   r�  Úmask_feature_lengthÚmask_feature_min_masksÚexpand)rE   rL   r”  r®   rY  rp  r1   Úmask_feature_indicess           rH   Ú_mask_hidden_stateszHubertModel._mask_hidden_states	  s—  € õ �t”{Ð$8¸$Ñ?Ô?ð 	!Ø Ð ð 4A×3EÒ3EÑ3GÔ3GÑ0ˆ
�O [àÐ(à/3Ô/E×/HÒ/HÈÔI\Ñ/]Ô/]ˆMÐ+Ñ,Ð,ØŒ[Ô'¨!Ò+Ð+°´Ð+Ý 5Ø˜_Ð-Øœ+Ô4Ø œKÔ8Ø-Øœ+Ô9ð!ñ !ô !Ðõ !&¤Ð->À}ÔG[ÕchÔcmÐ nÑ nÔ nÐØ/3Ô/E×/HÒ/HÈÔI\Ñ/]Ô/]ˆMÐ+Ñ,àŒ;Ô(¨1Ò,Ð,°´Ð,å#8Ø˜[Ð)Øœ+Ô7Ø œKÔ;Øœ+Ô<ð	$ñ $ô $Ð õ $)¤<Ð0DÈ]ÔMaÕinÔisÐ#tÑ#tÔ#tÐ Ø#7¸¸¸¸4¸Ô#@×#GÒ#GÈÈOÐ]_Ñ#`Ô#`Ð Ø23ˆMÐ.Ñ/àÐrN   F)Útie_last_hidden_statesr˜   rË   Úoutput_hidden_statesÚreturn_dictrÌ   c                 óò  — |�|n| j         j        }|�|n| j         j        }|�|n| j         j        }|                      |¦  «        }|                     dd¦  «        }|�!|                      |j        d         |¦  «        }|                      |¦  «        }	|  	                    |	|¬¦  «        }	|  
                    |	||||¬¦  «        }
|
d         }	|s|	f|
dd…         z   S t          |	|
j        |
j        ¬¦  «        S )a1  
        mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict
            masked extracted features in *config.proj_codevector_dim* space.

        Example:

        ```python
        >>> from transformers import AutoProcessor, HubertModel
        >>> from datasets import load_dataset

        >>> processor = AutoProcessor.from_pretrained("facebook/hubert-large-ls960-ft")
        >>> model = HubertModel.from_pretrained("facebook/hubert-large-ls960-ft")


        >>> def map_to_array(example):
        ...     example["speech"] = example["audio"]["array"]
        ...     return example


        >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        >>> ds = ds.map(map_to_array)

        >>> input_values = processor(ds["speech"][0], return_tensors="pt").input_values  # Batch size 1
        >>> hidden_states = model(input_values).last_hidden_state
        ```Nr   r!   )r”  )r®   rË   rŸ  r   r   )r  rL   r9  )rF   rË   rŸ  r   r�  rJ   rZ  rÎ   rŽ  r�  r6  r   rL   r9  )rE   r˜   r®   r”  rË   rŸ  r   r°   Úextract_featuresrL   Úencoder_outputss              rH   rM   zHubertModel.forward7  s@  € ðN 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆà×1Ò1°,Ñ?Ô?ÐØ+×5Ò5°a¸Ñ;Ô;ÐàÐ%à!×DÒDÐEUÔE[Ð\]ÔE^Ð`nÑoÔoˆNà×/Ò/Ð0@ÑAÔAˆØ×0Ò0°ÐRcÐ0ÑdÔdˆàŸ,š,ØØ)Ø/Ø!5Ø#ð 'ñ 
ô 
ˆð (¨Ô*ˆàð 	:Ø!Ð# o°a°b°bÔ&9Ñ9Ð9åØ+Ø)Ô7Ø&Ô1ð
ñ 
ô 
ð 	
rN   ©NN)NNNNN)rP   rQ   rR   r   r.   r¶   r&  rf  r�  r   r   r   rß   rÞ   rà   r   rM   rS   rT   s   @rH   rA  rA  õ  si  ø€ € € € € ð˜|ð ð ð ð ð ð ð* 7;Ø26ð	,ð ,àÔ(ð,ð !Ô,¨tÑ3ð,ð Ô(¨4Ñ/ð	,ð ,ð ,ð ,ð\  Ø€_¨EÐ2Ñ2Ô2Øð /3Ø6:Ø)-Ø,0Ø#'ðE
ð E
à”l TÑ)ðE
ð œ tÑ+ðE
ð !Ô,¨tÑ3ð	E
ð
   $™;ðE
ð # T™kðE
ð ˜D‘[ðE
ð 
�Ñ	 ðE
ð E
ð E
ñ „^ñ 3Ô2ñ  ÔðE
ð E
ð E
ð E
ð E
rN   rA  zn
    Hubert Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).
    )Úcustom_introc                   óÂ   ‡ — e Zd Zddedz  fˆ fd„Zd„ Zd„ Zd„ Zee		 	 dde
j        dz  de
j        dz  d	e
j        dz  d
ee         deez  f
d„¦   «         ¦   «         Zˆ xZS )ÚHubertForCTCNÚtarget_langc                 óª  •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        ¦  «        | _        || _        |j	        €t          d| j        › d�¦  «        ‚t          |d¦  «        r|j        r|j        n|j        }t	          j        ||j	        ¦  «        | _        |                      ¦   «          dS )a0  
        target_lang (`str`, *optional*):
            Language id of adapter weights. Adapter weights are stored in the format adapter.<lang>.safetensors or
            adapter.<lang>.bin. Only relevant when using an instance of [`HubertForCTC`] with adapters. Uses 'eng' by
            default.
        NzYou are trying to instantiate zõ with a configuration that does not define the vocabulary size of the language model head. Please instantiate the model as follows: `HubertForCTC.from_pretrained(..., vocab_size=vocab_size)`. or define `vocab_size` of your model's configuration.Úadd_adapter)r-   r.   rA  r3  r/   r¤   Úfinal_dropoutr¦   r¨  Ú
vocab_sizer‹   rG   r9   rª  Úoutput_hidden_sizer1   r¢   Úlm_headr“  )rE   rF   r¨  r­  rG   s       €rH   r.   zHubertForCTC.__init__ˆ  sÝ   ø€ õ 	‰Œ×Ò˜Ñ Ô Ð å! &Ñ)Ô)ˆŒÝ”z &Ô"6Ñ7Ô7ˆŒà&ˆÔàÔÐ$ÝðH°´ð Hð Hð Hñô ð õ *1°¸Ñ)GÔ)GÐvÈFÔL^ÐvˆFÔ%Ð%ÐdjÔdvð 	õ ”yÐ!3°VÔ5FÑGÔGˆŒð 	�ŠÑÔÐÐÐrN   c                 óT  — t          ¦   «         t          j        d¦  «        k    rdS | j        }|�)t	          | j        dd¦  «        €t          d|› d�¦  «        ‚|€2t	          | j        dd¦  «        �t                               d¦  «         dS |�|  	                    |d¬¦  «         dS dS )	a'  
        This method overwrites [`~PreTrainedModel.tie_weights`] so that adapter weights can be correctly loaded when
        passing `target_lang=...` to `from_pretrained(...)`.

        This method is **not** supposed to be called by the user and is prone to be changed in the future.
        ÚmetaNr  zCannot pass `target_lang`: z- if `config.adapter_attn_dim` is not defined.z)By default `target_lang` is set to 'eng'.T)Ú
force_load)
r   r¶   r  r¨  r6   rF   r‹   ÚloggerÚinfoÚload_adapter)rE   r°   r¨  s      rH   Útie_weightszHubertForCTC.tie_weights¥  sÃ   € õ 6Ñ7Ô7½5¼<ÈÑ;OÔ;OÒOÐOØˆFð Ô&ˆàÐ"¥w¨t¬{Ð<NÐPTÑ'UÔ'UÐ']ÝÐu¸;ÐuÐuÐuÑvÔvÐvØÐ ¥W¨T¬[Ð:LÈdÑ%SÔ%SÐ%_Ý�KŠKÐCÑDÔDÐDÐDÐDØÐ$Ø×Ò˜k°dÐÑ;Ô;Ð;Ð;Ð;ð %Ð$rN   c                 óB   — | j         j                             ¦   «          dS ©z¨
        Calling this function will disable the gradient computation for the feature encoder so that its parameter will
        not be updated during training.
        N©r3  r�  r•   ©rE   s    rH   Úfreeze_feature_encoderz#HubertForCTC.freeze_feature_encoder½  ó!   € ð
 	ŒÔ%×8Ò8Ñ:Ô:Ð:Ð:Ð:rN   c                 óL   — | j                              ¦   «         D ]	}d|_        Œ
dS ©zÒ
        Calling this function will disable the gradient computation for the base model so that its parameters will not
        be updated during training. Only the classification head will be updated.
        FN©r3  r‘   r’   r“   s     rH   Úfreeze_base_modelzHubertForCTC.freeze_base_modelÄ  ó6   € ð
 ”[×+Ò+Ñ-Ô-ð 	(ð 	(ˆEØ"'ˆEÔÐð	(ð 	(rN   r˜   r®   Úlabelsr°   rÌ   c           
      óü  — |�>|                      ¦   «         | j        j        k    rt          d| j        j        › �¦  «        ‚ | j        |fd|i|¤Ž}|j        }|                      |¦  «        }|                      |¦  «        }d}|��Z|�|nt          j	        |t          j
        ¬¦  «        }|                      |                     d¦  «        ¦  «                             t          j
        ¦  «        }	|dk    }
|
                     d¦  «        }|                     |
¦  «        }t          j                             |dt          j        ¬¦  «                             dd¦  «        }t          j        j                             d	¬
¦  «        5  t          j                             |||	|| j        j        | j        j        | j        j        ¬¦  «        }ddd¦  «         n# 1 swxY w Y   t7          |||j        |j        ¬¦  «        S )aà  
        labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*):
            Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to
            the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`.
            All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ...,
            config.vocab_size - 1]`.
        Nz$Label values must be <= vocab_size: r®   rs  rs   r   )r+   rQ  r   F)Úenabled)ÚblankÚ	reductionÚzero_infinity©ÚlossÚlogitsrL   r9  )rm  rF   r¬  r‹   r3  r  r¦   r®  r¶   Ú	ones_likerS  rN  rR  r  Úmasked_selectr/   r¸   Úlog_softmaxÚfloat32rJ   ÚbackendsÚcudnnÚflagsÚctc_lossÚpad_token_idÚctc_loss_reductionÚctc_zero_infinityr   rL   r9  )rE   r˜   r®   rÁ  r°   ÚoutputsrL   rÉ  rÈ  rE  Úlabels_maskÚtarget_lengthsÚflattened_targetsÚ	log_probss                 rH   rM   zHubertForCTC.forwardÌ  s:  € ð  Ð &§*¢*¡,¤,°$´+Ô2HÒ"HÐ"HÝÐ\ÀDÄKÔDZÐ\Ð\Ñ]Ô]Ð]à�$”+Øð
ð 
à)ð
ð ð
ð 
ˆð  Ô1ˆØŸš ]Ñ3Ô3ˆà—’˜mÑ,Ô,ˆàˆØÑð #1Ð"<��Å%Ä/ÐR^ÕfkÔfpÐBqÑBqÔBqð ð !×AÒAÀ.×BTÒBTÐUWÑBXÔBXÑYÔY×\Ò\Õ]bÔ]gÑhÔhˆMð ! Aš+ˆKØ(Ÿ_š_¨RÑ0Ô0ˆNØ &× 4Ò 4°[Ñ AÔ AÐõ œ×1Ò1°&¸bÍÌÐ1ÑVÔV×`Ò`ÐabÐdeÑfÔfˆIå”Ô%×+Ò+°EÐ+Ñ:Ô:ð 	ð 	Ý”}×-Ò-ØØ%Ø!Ø"Øœ+Ô2Ø"œkÔ<Ø"&¤+Ô"?ð .ñ ô �ð	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	õ Ø˜f°GÔ4IÐV]ÔVhð
ñ 
ô 
ð 	
s   ÆAGÇGÇGri   r¤  )rP   rQ   rR   rê   r.   rµ  rº  r¿  r   r   r¶   rß   r   r   rà   r   rM   rS   rT   s   @rH   r§  r§  ‚  s  ø€ € € € € ðð ¨C°$©Jð ð ð ð ð ð ð:<ð <ð <ð0;ð ;ð ;ð(ð (ð (ð Øð /3Ø&*ð	:
ð :
à”l TÑ)ð:
ð œ tÑ+ð:
ð ”˜tÑ#ð	:
ð
 Ð+Ô,ð:
ð 
�Ñ	ð:
ð :
ð :
ñ „^ñ Ôð:
ð :
ð :
ð :
ð :
rN   r§  z•
    Hubert Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like
    SUPERB Keyword Spotting.
    c                   ó¾   ‡ — e Zd Zˆ fd„Zd„ Zd„ Zeee	 	 dde	j
        dz  de	j
        dz  de	j
        dz  dee         d	eez  f
d
„¦   «         ¦   «         ¦   «         Zˆ xZS )rC  c                 óô  •— t          ¦   «                              |¦  «         t          |d¦  «        r|j        rt	          d¦  «        ‚t          |¦  «        | _        |j        dz   }|j        r.t          j
        t          j        |¦  «        |z  ¦  «        | _        t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¦  «        | _        |                      ¦   «          d S )Nrª  z]Sequence classification does not support the use of Hubert adapters (config.add_adapter=True)r   )r-   r.   r9   rª  r‹   rA  r3  r  Úuse_weighted_layer_sumr/   r‘  r¶   r}  r<  r¢   r1   Úclassifier_proj_sizeÚ	projectorÚ
num_labelsÚ
classifierr“  )rE   rF   Ú
num_layersrG   s      €rH   r.   z(HubertForSequenceClassification.__init__  sá   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å�6˜=Ñ)Ô)ð 	¨fÔ.@ð 	ÝØoñô ð õ " &Ñ)Ô)ˆŒØÔ-°Ñ1ˆ
ØÔ(ð 	SÝ!#¤­e¬j¸Ñ.DÔ.DÀzÑ.QÑ!RÔ!RˆDÔÝœ 6Ô#5°vÔ7RÑSÔSˆŒÝœ) FÔ$?ÀÔARÑSÔSˆŒð 	�ŠÑÔÐÐÐrN   c                 óB   — | j         j                             ¦   «          dS r·  r¸  r¹  s    rH   rº  z6HubertForSequenceClassification.freeze_feature_encoder#  r»  rN   c                 óL   — | j                              ¦   «         D ]	}d|_        Œ
dS r½  r¾  r“   s     rH   r¿  z1HubertForSequenceClassification.freeze_base_model*  rÀ  rN   Nr˜   r®   rÁ  r°   rÌ   c                 óæ  — | j         j        rd|d<    | j        |fd|i|¤Ž}| j         j        rr|j        }t	          j        |d¬¦  «        }t          j                             | j	        d¬¦  «        }|| 
                    ddd¦  «        z                       d¬¦  «        }n|j        }|                      |¦  «        }|€|                     d¬¦  «        }n�|                      |j        d         |¦  «        }	|	                     d¦  «                             dd|j        d         ¦  «        }
d	||
 <   |                     d¬¦  «        |	                     d¬¦  «         
                    dd¦  «        z  }|                      |¦  «        }d}|�Kt)          ¦   «         } || 
                    d| j         j        ¦  «        | 
                    d¦  «        ¦  «        }t-          |||j        |j        ¬
¦  «        S )a
  
        input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
            Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file
            into an array of type `list[float]`, a `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library
            (`pip install torchcodec`) or the soundfile library (`pip install soundfile`).
            To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and conversion
            into a tensor of type `torch.FloatTensor`. See [`HubertProcessor.__call__`] for details.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        TrŸ  r®   r   r³   rs   Nr!   r©   rÇ  )rF   rÜ  r3  rL   r¶   Ústackr/   r¸   r¹   r<  rÏ   rR  r  rÞ  ÚmeanrZ  rÎ   r  r  rà  r   rß  r   r9  )rE   r˜   r®   rÁ  r°   rÕ  rL   Únorm_weightsÚpooled_outputÚpadding_maskÚexpand_padding_maskrÉ  rÈ  Úloss_fcts                 rH   rM   z'HubertForSequenceClassification.forward2  s  € ð, Œ;Ô-ð 	2Ø-1ˆFÐ)Ñ*à�$”+Øð
ð 
à)ð
ð ð
ð 
ˆð Œ;Ô-ð 	6Ø#Ô1ˆMÝ!œK¨¸1Ð=Ñ=Ô=ˆMÝœ=×0Ò0°Ô1CÈÐ0ÑLÔLˆLØ*¨\×->Ò->¸rÀ1ÀaÑ-HÔ-HÑH×MÒMÐRSÐMÑTÔTˆMˆMà#Ô5ˆMàŸš }Ñ5Ô5ˆØÐ!Ø)×.Ò.°1Ð.Ñ5Ô5ˆMˆMà×BÒBÀ=ÔCVÐWXÔCYÐ[iÑjÔjˆLØ".×"8Ò"8¸Ñ"<Ô"<×"CÒ"CÀAÀqÈ-ÔJ]Ð^_ÔJ`Ñ"aÔ"aÐØ25ˆMÐ.Ð.Ñ/Ø)×-Ò-°!Ð-Ñ4Ô4°|×7GÒ7GÈAÐ7GÑ7NÔ7N×7SÒ7SÐTVÐXYÑ7ZÔ7ZÑZˆMà—’ Ñ/Ô/ˆàˆØÐÝ'Ñ)Ô)ˆHØ�8˜FŸKšK¨¨D¬KÔ,BÑCÔCÀVÇ[Â[ÐQSÁ_Ä_ÑUÔUˆDå'ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
rN   r¤  )rP   rQ   rR   r.   rº  r¿  r   r   r   r¶   rß   r   r   rà   r   rM   rS   rT   s   @rH   rC  rC    sê   ø€ € € € € ðð ð ð ð ð";ð ;ð ;ð(ð (ð (ð Ø Øð /3Ø&*ð	9
ð 9
à”l TÑ)ð9
ð œ tÑ+ð9
ð ”˜tÑ#ð	9
ð
 Ð+Ô,ð9
ð 
Ð)Ñ	)ð9
ð 9
ð 9
ñ „^ñ !Ô ñ Ôð9
ð 9
ð 9
ð 9
ð 9
rN   rC  )r§  rC  rA  r2  rŒ  rY   )NÚcollections.abcr   Únumpyru  r¶   Útorch.nnr/   r   Ú r   r>  Úactivationsr   Úbackbone_utilsr   Úintegrations.deepspeedr	   Úintegrations.fsdpr
   Úmasking_utilsr   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   Úmodeling_utilsr   r   r   Úprocessing_utilsr   r8   r   r   r   Úutils.genericr   r   Úutils.output_capturingr   r   Úconfiguration_hubertr   Ú
get_loggerrP   r²  ÚModuler   rB   r[   rl   ru   r}   rœ   rß   rÝ   r½   r¿   râ   rñ   r   r  r(  r-  r2  rà   rÜ   rf  Úndarrayr‰  rA  r§  rC  Ú__all__r  rN   rH   ú<module>r     sf  ðð* %Ð $Ð $Ð $Ð $Ð $à Ð Ð Ð Ø €€€Ø Ð Ð Ð Ð Ð Ø %Ð %Ð %Ð %Ð %Ð %à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø @Ð @Ð @Ð @Ð @Ð @Ø 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø BÐ BÐ BÐ BÐ BÐ BØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YØ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rØ &Ð &Ð &Ð &Ð &Ð &Ø @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ .Ð .Ð .Ð .Ð .Ð .ð 
ˆÔ	˜HÑ	%Ô	%€ð/ð /ð /ð /ð / B¤Iñ /ô /ð /ðdð ð ð ð ˜œñ ô ð ðð ð ð ð Ð!;ñ ô ð ð*ð ð ð ð Ð9ñ ô ð ð6ð ð ð ð Ð9ñ ô ð ð0#ð #ð #ð #ð #˜2œ9ñ #ô #ð #ðLð ð ð ð ˜bœiñ ô ð ð0 !Øð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð �T‰\ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð8R)ð R)ð R)ð R)ð R)�b”iñ R)ô R)ð R)ðjð ð ð ð ˜œ	ñ ô ð ð0ð ð ð ð Ð3ñ ô ð ðD+@ð +@ð +@ð +@ð +@�B”Iñ +@ô +@ð +@ð\ð ð ð ð ˜RœYñ ô ð ð2$ð $ð $ð $ð $Ð(Bñ $ô $ð $ðN/@ð /@ð /@ð /@ð /@ 2¤9ñ /@ô /@ð /@ðd ðBð Bð Bð Bð B˜Oñ Bô Bñ „ðBðR /3Øðtð tØ��c�Œ?ðtàðtð ðtð Ô$ tÑ+ð	tð
 ðtð „Zðtð tð tð tðn ðI
ð I
ð I
ð I
ð I
Ð'ñ I
ô I
ñ „ðI
ðX €ððñ ô ð
A
ð A
ð A
ð A
ð A
Ð(ñ A
ô A
ñô ð
A
ðH €ððñ ô ð]
ð ]
ð ]
ð ]
ð ]
Ð&;ñ ]
ô ]
ñô ð]
ð@ fÐ
eÐ
e€€€rN   