§
    9²¸j«¦  ã                   óX  — d dl Z d dlmZ d dlZd dlZd dlmZ d dlmZ ddl	m
Z ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZmZmZ ddlmZm Z m!Z! ddl"m#Z# ddl$m%Z%m&Z&m'Z' ddl(m)Z)m*Z* ddl+m,Z,m-Z- ddl.m/Z/  e'j0        e1¦  «        Z2 G d„ de¦  «        Z3 G d„ de¦  «        Z4 G d„ de¦  «        Z5 G d„ dej6        ¦  «        Z7 G d„ dej6        ¦  «        Z8 G d „ d!ej6        ¦  «        Z9 G d"„ d#ej6        ¦  «        Z:	 	 dHd%ej6        d&ej;        d'ej;        d(ej;        d)ej;        dz  d*e<dz  d+e<d,e#e%         fd-„Z= G d.„ d/ej6        ¦  «        Z> G d0„ d1ej6        ¦  «        Z? G d2„ d3e¦  «        Z@ G d4„ d5ej6        ¦  «        ZAe& G d6„ d7e ¦  «        ¦   «         ZB	 	 dId8eCeDeDf         d9e<d:eDd)ejE        dz  d;eDd<ejF        fd=„ZGe& G d>„ d?eB¦  «        ¦   «         ZH e&d@¬A¦  «         G dB„ dCeB¦  «        ¦   «         ZI e&dD¬A¦  «         G dE„ dFeB¦  «        ¦   «         ZJg dG¢ZKdS )Jé    N)ÚCallable)Únn)ÚCrossEntropyLossé   )Úinitialization)ÚACT2FN)Úfilter_output_hidden_states)Úis_deepspeed_zero3_enabled)Úis_fsdp_managed_module)Úcreate_bidirectional_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚCausalLMOutputÚSequenceClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModelÚ*get_torch_context_manager_or_global_device)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)Úcan_return_tupleÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )Ú	SEWConfigc                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚSEWNoLayerNormConvLayerr   c                 óZ  •— t          ¦   «                              ¦   «          |dk    r|j        |dz
           nd| _        |j        |         | _        t          j        | j        | j        |j        |         |j        |         |j	        ¬¦  «        | _
        t          |j                 | _        d S )Nr   r   ©Úkernel_sizeÚstrideÚbias)ÚsuperÚ__init__Úconv_dimÚin_conv_dimÚout_conv_dimr   ÚConv1dÚconv_kernelÚconv_strideÚ	conv_biasÚconvr   Úfeat_extract_activationÚ
activation©ÚselfÚconfigÚlayer_idÚ	__class__s      €út/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/sew/modeling_sew.pyr'   z SEWNoLayerNormConvLayer.__init__2   s�   ø€ Ý‰Œ×ÒÑÔÐØ<DÀqºL¸L˜6œ?¨8°a©<Ô8Ð8ÈaˆÔØ"œO¨HÔ5ˆÔå”IØÔØÔØÔ*¨8Ô4ØÔ% hÔ/ØÔ!ð
ñ 
ô 
ˆŒ	õ ! Ô!?Ô@ˆŒˆˆó    c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S ©N)r/   r1   ©r3   Úhidden_statess     r7   ÚforwardzSEWNoLayerNormConvLayer.forward@   s*   € ØŸ	š	 -Ñ0Ô0ˆØŸš¨Ñ6Ô6ˆØÐr8   ©r   ©Ú__name__Ú
__module__Ú__qualname__r'   r=   Ú__classcell__©r6   s   @r7   r    r    1   sR   ø€ € € € € ðAð Að Að Að Að Aðð ð ð ð ð ð r8   r    c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚSEWLayerNormConvLayerr   c                 óš  •— t          ¦   «                              ¦   «          |dk    r|j        |dz
           nd| _        |j        |         | _        t          j        | j        | j        |j        |         |j        |         |j	        ¬¦  «        | _
        t          j        | j        d¬¦  «        | _        t          |j                 | _        d S )Nr   r   r"   T)Úelementwise_affine)r&   r'   r(   r)   r*   r   r+   r,   r-   r.   r/   Ú	LayerNormÚ
layer_normr   r0   r1   r2   s      €r7   r'   zSEWLayerNormConvLayer.__init__G   s¶   ø€ Ý‰Œ×ÒÑÔÐØ<DÀqºL¸L˜6œ?¨8°a©<Ô8Ð8ÈaˆÔØ"œO¨HÔ5ˆÔå”IØÔØÔØÔ*¨8Ô4ØÔ% hÔ/ØÔ!ð
ñ 
ô 
ˆŒ	õ œ, tÔ'8ÈTÐRÑRÔRˆŒÝ  Ô!?Ô@ˆŒˆˆr8   c                 óÜ   — |                       |¦  «        }|                     dd¦  «        }|                      |¦  «        }|                     dd¦  «        }|                      |¦  «        }|S )Néþÿÿÿéÿÿÿÿ)r/   Ú	transposerJ   r1   r;   s     r7   r=   zSEWLayerNormConvLayer.forwardV   se   € ØŸ	š	 -Ñ0Ô0ˆà%×/Ò/°°BÑ7Ô7ˆØŸš¨Ñ6Ô6ˆØ%×/Ò/°°BÑ7Ô7ˆàŸš¨Ñ6Ô6ˆØÐr8   r>   r?   rD   s   @r7   rF   rF   F   sR   ø€ € € € € ðAð Að Að Að Að Aðð ð ð ð ð ð r8   rF   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚSEWGroupNormConvLayerr   c                 ó¦  •— t          ¦   «                              ¦   «          |dk    r|j        |dz
           nd| _        |j        |         | _        t          j        | j        | j        |j        |         |j        |         |j	        ¬¦  «        | _
        t          |j                 | _        t          j        | j        | j        d¬¦  «        | _        d S )Nr   r   r"   T)Ú
num_groupsÚnum_channelsÚaffine)r&   r'   r(   r)   r*   r   r+   r,   r-   r.   r/   r   r0   r1   Ú	GroupNormrJ   r2   s      €r7   r'   zSEWGroupNormConvLayer.__init__b   s½   ø€ Ý‰Œ×ÒÑÔÐØ<DÀqºL¸L˜6œ?¨8°a©<Ô8Ð8ÈaˆÔØ"œO¨HÔ5ˆÔå”IØÔØÔØÔ*¨8Ô4ØÔ% hÔ/ØÔ!ð
ñ 
ô 
ˆŒ	õ ! Ô!?Ô@ˆŒåœ,°$Ô2CÐRVÔRcÐlpÐqÑqÔqˆŒˆˆr8   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r:   )r/   rJ   r1   r;   s     r7   r=   zSEWGroupNormConvLayer.forwardr   s;   € ØŸ	š	 -Ñ0Ô0ˆØŸš¨Ñ6Ô6ˆØŸš¨Ñ6Ô6ˆØÐr8   r>   r?   rD   s   @r7   rP   rP   a   sR   ø€ € € € € ðrð rð rð rð rð rð ð ð ð ð ð ð r8   rP   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚSEWPositionalConvEmbeddingc                 óÖ  •— t          ¦   «                              ¦   «          t          j        |j        |j        |j        |j        dz  |j        |j        ¬¦  «        | _        t          j	        j
        }t          t          j	        j        d¦  «        rt          j	        j        j
        }t          ¦   «         rëdd l}|j                             | j        j        d¬¦  «        5   || j        dd¬¦  «        | _        d d d ¦  «         n# 1 swxY w Y   t          | j        d¦  «        r-| j        j        j        j        }| j        j        j        j        }n| j        j        }| j        j        }|j                             | |¦  «         |j                             | |¦  «         n || j        dd¬¦  «        | _        t/          |j        ¦  «        | _        t2          |j                 | _        d S )	Né   )r#   ÚpaddingÚgroupsr$   Úweight_normr   ©Úmodifier_rankÚweight)ÚnameÚdimÚparametrizations)r&   r'   r   r+   Úhidden_sizeÚnum_conv_pos_embeddingsÚnum_conv_pos_embedding_groupsÚsqueeze_factorr/   Úutilsr]   Úhasattrrc   r
   Ú	deepspeedÚzeroÚGatheredParametersr`   Ú	original0Ú	original1Úweight_gÚweight_vÚregister_external_parameterÚSEWSamePadLayerr[   r   r0   r1   )r3   r4   r]   rj   ro   rp   r6   s         €r7   r'   z#SEWPositionalConvEmbedding.__init__z   s  ø€ Ý‰Œ×ÒÑÔÐÝ”IØÔØÔØÔ6ØÔ2°aÑ7ØÔ7ØÔ(ð
ñ 
ô 
ˆŒ	õ ”hÔ*ˆÝ•2”8Ô,¨mÑ<Ô<ð 	@Ýœ(Ô3Ô?ˆKå%Ñ'Ô'ð 	EØÐÐÐà”×2Ò2°4´9Ô3CÐSTÐ2ÑUÔUð Ið IØ'˜K¨¬	¸ÀaÐHÑHÔH�”	ðIð Ið Iñ Iô Ið Ið Ið Ið Ið Ið Iøøøð Ið Ið Ið Iå�t”yÐ"4Ñ5Ô5ð .Øœ9Ô5Ô<ÔF�Øœ9Ô5Ô<ÔF��àœ9Ô-�Øœ9Ô-�ØŒN×6Ò6°t¸XÑFÔFÐFØŒN×6Ò6°t¸XÑFÔFÐFÐFà#˜ D¤I°HÀ!ÐDÑDÔDˆDŒIå& vÔ'EÑFÔFˆŒÝ  Ô!?Ô@ˆŒˆˆs   Ã DÄD	ÄD	c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r:   )r/   r[   r1   r;   s     r7   r=   z"SEWPositionalConvEmbedding.forwardœ   s;   € ØŸ	š	 -Ñ0Ô0ˆØŸš ]Ñ3Ô3ˆØŸš¨Ñ6Ô6ˆàÐr8   r?   rD   s   @r7   rX   rX   y   sM   ø€ € € € € ð Að  Að  Að  Að  AðDð ð ð ð ð ð r8   rX   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )rr   c                 ól   •— t          ¦   «                              ¦   «          |dz  dk    rdnd| _        d S )NrZ   r   r   )r&   r'   Únum_pad_remove)r3   re   r6   s     €r7   r'   zSEWSamePadLayer.__init__¥   s:   ø€ Ý‰Œ×ÒÑÔÐØ#:¸QÑ#>À!Ò#CÐ#C˜a˜aÈˆÔÐÐr8   c                 óJ   — | j         dk    r|d d …d d …d | j          …f         }|S ©Nr   )rv   r;   s     r7   r=   zSEWSamePadLayer.forward©   s;   € ØÔ Ò"Ð"Ø)¨!¨!¨!¨Q¨Q¨QÐ0F°4Ô3FÐ2FÐ0FÐ*FÔGˆMØÐr8   r?   rD   s   @r7   rr   rr   ¤   sL   ø€ € € € € ðKð Kð Kð Kð Kðð ð ð ð ð ð r8   rr   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚSEWUpsamplingc                 óæ   •— t          ¦   «                              ¦   «          t          j        |j        |j        |j        z  ¦  «        | _        t          |j                 | _	        |j        | _        d S r:   )
r&   r'   r   ÚLinearrd   rg   Ú
projectionr   r0   r1   ©r3   r4   r6   s     €r7   r'   zSEWUpsampling.__init__°   sZ   ø€ Ý‰Œ×ÒÑÔÐÝœ) FÔ$6¸Ô8JÈVÔMbÑ8bÑcÔcˆŒÝ  Ô!?Ô@ˆŒØ$Ô3ˆÔÐÐr8   c                 ó0  — |                       |¦  «        }|                      |¦  «        }| j        dk    r`|                     ¦   «         \  }}}|| j        z  }|| j        z  }|                     ||| j        |¦  «        }|                     |||¦  «        }|S )Nr   )r}   r1   rg   ÚsizeÚreshape)r3   r<   ÚbszÚsrc_lenÚsrc_embed_dimÚtgt_lenÚtgt_embed_dims          r7   r=   zSEWUpsampling.forward¶   sž   € ØŸš¨Ñ6Ô6ˆØŸš¨Ñ6Ô6ˆàÔ Ò"Ð"à*7×*<Ò*<Ñ*>Ô*>Ñ'ˆC�˜-Ø Ô 3Ñ3ˆGØ)¨TÔ-@Ñ@ˆMØ)×1Ò1°#°wÀÔ@SÐUbÑcÔcˆMØ)×1Ò1°#°wÀÑNÔNˆMàÐr8   r?   rD   s   @r7   rz   rz   ¯   sG   ø€ € € € € ð4ð 4ð 4ð 4ð 4ðð ð ð ð ð ð r8   rz   c                   ó.   ‡ — e Zd ZdZˆ fd„Zd„ Zd„ Zˆ xZS )ÚSEWFeatureEncoderz.Construct the features from raw audio waveformc                 ó¤  •‡— t          ¦   «                              ¦   «          ‰j        dk    r7t          ‰d¬¦  «        gˆfd„t	          ‰j        dz
  ¦  «        D ¦   «         z   }nD‰j        dk    r!ˆfd„t	          ‰j        ¦  «        D ¦   «         }nt          d‰j        › d	�¦  «        ‚t          j        |¦  «        | _	        d
| _
        d| _        d S )NÚgroupr   ©r5   c                 ó8   •— g | ]}t          ‰|d z   ¬¦  «        ‘ŒS )r   r‹   )r    ©Ú.0Úir4   s     €r7   ú
<listcomp>z.SEWFeatureEncoder.__init__.<locals>.<listcomp>Ì   s>   ø€ ð Ið Ið IØDEÕ'¨¸¸Q¹Ð?Ñ?Ô?ðIð Ið Ir8   r   Úlayerc                 ó2   •— g | ]}t          ‰|¬ ¦  «        ‘ŒS )r‹   )rF   r�   s     €r7   r�   z.SEWFeatureEncoder.__init__.<locals>.<listcomp>Ð   s'   ø€ ÐtÐtÐtÈÕ0°À!ÐDÑDÔDÐtÐtÐtr8   z`config.feat_extract_norm` is z), but has to be one of ['group', 'layer']FT)r&   r'   Úfeat_extract_normrP   ÚrangeÚnum_feat_extract_layersÚ
ValueErrorr   Ú
ModuleListÚconv_layersÚgradient_checkpointingÚ_requires_grad)r3   r4   r˜   r6   s    ` €r7   r'   zSEWFeatureEncoder.__init__È   s  øø€ Ý‰Œ×ÒÑÔÐàÔ# wÒ.Ð.Ý0°À!ÐDÑDÔDÐEð Ið Ið Ið IÝINÈvÔOmÐpqÑOqÑIrÔIrðIñ Iô Iñ ˆKˆKð Ô%¨Ò0Ð0ØtÐtÐtÐtÍeÐTZÔTrÑNsÔNsÐtÑtÔtˆKˆKåØt°Ô1IÐtÐtÐtñô ð õ œ=¨Ñ5Ô5ˆÔØ&+ˆÔ#Ø"ˆÔÐÐr8   c                 óP   — |                       ¦   «         D ]	}d|_        Œ
d| _        d S )NF)Ú
parametersÚrequires_gradrš   ©r3   Úparams     r7   Ú_freeze_parametersz$SEWFeatureEncoder._freeze_parametersÙ   s4   € Ø—_’_Ñ&Ô&ð 	(ð 	(ˆEØ"'ˆEÔÐØ#ˆÔÐÐr8   c                 ór   — |d d …d f         }| j         r| j        rd|_        | j        D ]} ||¦  «        }Œ|S )NT)rš   Útrainingr�   r˜   )r3   Úinput_valuesr<   Ú
conv_layers       r7   r=   zSEWFeatureEncoder.forwardÞ   s[   € Ø$ Q Q Q¨ WÔ-ˆð Ôð 	/ 4¤=ð 	/Ø*.ˆMÔ'àÔ*ð 	6ð 	6ˆJØ&˜J }Ñ5Ô5ˆMˆMàÐr8   )r@   rA   rB   Ú__doc__r'   r    r=   rC   rD   s   @r7   rˆ   rˆ   Å   s\   ø€ € € € € Ø8Ð8ð#ð #ð #ð #ð #ð"$ð $ð $ð

ð 
ð 
ð 
ð 
ð 
ð 
r8   rˆ   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 ó®  — |€|                      d¦  «        dz  }t          j        ||                     dd¦  «        ¦  «        |z  }|�||z   }t          j                             |d¬¦  «        }t          j                             ||| j        ¬¦  «        }t          j        ||¦  «        }	|	                     dd¦  «         	                    ¦   «         }	|	|fS )NrM   ç      à¿rZ   r   ©rb   )Úpr¢   r   )
r€   ÚtorchÚmatmulrN   r   Ú
functionalÚsoftmaxr­   r¢   Ú
contiguous)
r§   r¨   r©   rª   r«   r¬   r­   r®   Úattn_weightsÚattn_outputs
             r7   Úeager_attention_forwardrº   ë   sÈ   € ð €Ø—*’*˜R‘.”. DÑ(ˆõ ”<  s§}¢}°Q¸Ñ':Ô':Ñ;Ô;¸gÑE€LàÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2Ð(Ñ>Ô>€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r8   c                   óà   ‡ — e Zd ZdZ	 	 	 	 	 ddededed	ed
edededz  fˆ fd„Z	 	 	 dde	j
        de	j
        dz  de	j
        dz  dedz  dee         dee	j
        e	j
        dz  f         fd„Zˆ xZS )ÚSEWAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr¦   FTNÚ	embed_dimÚ	num_headsr­   Ú
is_decoderr%   Ú	is_causalr4   c                 ó
  •— t          ¦   «                              ¦   «          || _        || _        || _        ||z  | _        || _        | j        |z  | j        k    rt          d| j        › d|› d�¦  «        ‚| j        dz  | _        || _	        || _
        t          j        |||¬¦  «        | _        t          j        |||¬¦  «        | _        t          j        |||¬¦  «        | _        t          j        |||¬¦  «        | _        d S )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).r°   )r%   )r&   r'   r½   r¾   r­   Úhead_dimr4   r–   r¬   r¿   rÀ   r   r|   Úk_projÚv_projÚq_projÚout_proj)	r3   r½   r¾   r­   r¿   r%   rÀ   r4   r6   s	           €r7   r'   zSEWAttention.__init__
  s  ø€ õ 	‰Œ×ÒÑÔÐØ"ˆŒØ"ˆŒØˆŒØ! YÑ.ˆŒØˆŒàŒM˜IÑ%¨$¬.Ò8Ð8Ýð3ÈdÌnð 3ð 3Ø%.ð3ð 3ð 3ñô ð ð ”} dÑ*ˆŒØ$ˆŒØ"ˆŒå”i 	¨9¸4Ð@Ñ@Ô@ˆŒÝ”i 	¨9¸4Ð@Ñ@Ô@ˆŒÝ”i 	¨9¸4Ð@Ñ@Ô@ˆŒÝœ	 )¨Y¸TÐBÑBÔBˆŒˆˆr8   r<   Úkey_value_statesr«   Úoutput_attentionsr®   Úreturnc                 óø  — |du}|j         dd…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }	|r|n|}
g |
j         dd…         ¢d‘| j        ‘R }|                      |
¦  «                             |¦  «                             dd¦  «        }|                      |
¦  «                             |¦  «                             dd¦  «        }t          j        | j	        j
        t          ¦  «        } || |	|||f| j        sdn| j        | j        |dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )z#Input shape: Batch x Time x ChannelNrM   r   rZ   r¦   )r­   r¬   rÈ   )ÚshaperÂ   rÅ   ÚviewrN   rÃ   rÄ   r   Úget_interfacer4   Ú_attn_implementationrº   r¢   r­   r¬   r�   r·   rÆ   )r3   r<   rÇ   r«   rÈ   r®   Úis_cross_attentionÚinput_shapeÚhidden_shapeÚquery_statesÚcurrent_statesÚkv_shapeÚ
key_statesÚvalue_statesÚattention_interfacer¹   r¸   s                    r7   r=   zSEWAttention.forward)  s»  € ð .°TÐ9Ðð $Ô)¨#¨2¨#Ô.ˆà8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆð —{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆà-?ÐRÐ)Ð)À]ˆØB�^Ô)¨#¨2¨#Ô.ÐB°ÐB°D´MÐBÐBˆØ—[’[ Ñ0Ô0×5Ò5°hÑ?Ô?×IÒIÈ!ÈQÑOÔOˆ
Ø—{’{ >Ñ2Ô2×7Ò7¸ÑAÔA×KÒKÈAÈqÑQÔQˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð  $œ}Ð>�C�C°$´,Ø”LØ/ð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—m’m KÑ0Ô0ˆà˜LÐ(Ð(r8   )r¦   FTFN)NNF)r@   rA   rB   r¥   ÚintÚfloatÚboolr   r'   r³   ÚTensorr   r   Útupler=   rC   rD   s   @r7   r¼   r¼     s<  ø€ € € € € ØGÐGð Ø ØØØ#'ðCð CàðCð ðCð ð	Cð
 ðCð ðCð ðCð ˜DÑ ðCð Cð Cð Cð Cð CðD 15Ø.2Ø).ð0)ð 0)à”|ð0)ð  œ,¨Ñ-ð0)ð œ tÑ+ð	0)ð
   $™;ð0)ð Ð-Ô.ð0)ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð0)ð 0)ð 0)ð 0)ð 0)ð 0)ð 0)ð 0)r8   r¼   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚSEWFeedForwardc                 óÌ  •— t          ¦   «                              ¦   «          t          j        |j        ¦  «        | _        t          j        |j        |j        ¦  «        | _	        t          |j        t          ¦  «        rt          |j                 | _        n|j        | _        t          j        |j        |j        ¦  «        | _        t          j        |j        ¦  «        | _        d S r:   )r&   r'   r   ÚDropoutÚactivation_dropoutÚintermediate_dropoutr|   rd   Úintermediate_sizeÚintermediate_denseÚ
isinstanceÚ
hidden_actÚstrr   Úintermediate_act_fnÚoutput_denseÚhidden_dropoutÚoutput_dropoutr~   s     €r7   r'   zSEWFeedForward.__init__]  s°   ø€ Ý‰Œ×ÒÑÔÐÝ$&¤J¨vÔ/HÑ$IÔ$IˆÔ!å"$¤)¨FÔ,>ÀÔ@XÑ"YÔ"YˆÔÝ�fÔ'­Ñ-Ô-ð 	9Ý'-¨fÔ.?Ô'@ˆDÔ$Ð$à'-Ô'8ˆDÔ$åœI fÔ&>ÀÔ@RÑSÔSˆÔÝ œj¨Ô)>Ñ?Ô?ˆÔÐÐr8   c                 óØ   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r:   )rä   rè   râ   ré   rë   r;   s     r7   r=   zSEWFeedForward.forwardj  sg   € Ø×/Ò/°Ñ>Ô>ˆØ×0Ò0°Ñ?Ô?ˆØ×1Ò1°-Ñ@Ô@ˆà×)Ò)¨-Ñ8Ô8ˆØ×+Ò+¨MÑ:Ô:ˆØÐr8   r?   rD   s   @r7   rÞ   rÞ   \  sL   ø€ € € € € ð@ð @ð @ð @ð @ðð ð ð ð ð ð r8   rÞ   c                   ó\   ‡ — e Zd Zˆ fd„Z	 ddej        dej        dz  dee         fd„Zˆ xZ	S )ÚSEWEncoderLayerc                 ó�  •— t          ¦   «                              ¦   «          t          |j        |j        |j        d|¬¦  «        | _        t          j        |j	        ¦  «        | _
        t          j        |j        |j        ¬¦  «        | _        t          |¦  «        | _        t          j        |j        |j        ¬¦  «        | _        d S )NF)r½   r¾   r­   r¿   r4   ©Úeps)r&   r'   r¼   rd   Únum_attention_headsÚattention_dropoutÚ	attentionr   rà   rê   r­   rI   Úlayer_norm_epsrJ   rÞ   Úfeed_forwardÚfinal_layer_normr~   s     €r7   r'   zSEWEncoderLayer.__init__u  s¬   ø€ Ý‰Œ×ÒÑÔÐÝ%ØÔ(ØÔ0ØÔ,ØØð
ñ 
ô 
ˆŒõ ”z &Ô"7Ñ8Ô8ˆŒÝœ, vÔ'9¸vÔ?TÐUÑUÔUˆŒÝ*¨6Ñ2Ô2ˆÔÝ "¤¨VÔ-?ÀVÔEZÐ [Ñ [Ô [ˆÔÐÐr8   Nr<   r«   r®   c                 óè   — |} | j         |fd|i|¤Ž\  }}|                      |¦  «        }||z   }|                      |¦  «        }||                      |¦  «        z   }|                      |¦  «        }|S )Nr«   )rô   r­   rJ   rö   r÷   )r3   r<   r«   r®   Úattn_residualÚ_s         r7   r=   zSEWEncoderLayer.forward„  sŠ   € ð &ˆØ)˜4œ>¨-ÐaÐaÈÐaÐZ`ÐaÐaÑˆ�qØŸš ]Ñ3Ô3ˆØ%¨Ñ5ˆàŸš¨Ñ6Ô6ˆØ%¨×(9Ò(9¸-Ñ(HÔ(HÑHˆØ×-Ò-¨mÑ<Ô<ˆàÐr8   r:   )
r@   rA   rB   r'   r³   rÛ   r   r   r=   rC   rD   s   @r7   rî   rî   t  s†   ø€ € € € € ð\ð \ð \ð \ð \ð$ /3ðð à”|ðð œ tÑ+ðð Ð+Ô,ð	ð ð ð ð ð ð ð r8   rî   c                   ó:   ‡ — e Zd Zˆ fd„Z	 ddee         fd„Zˆ xZS )Ú
SEWEncoderc                 óò  •‡— t          ¦   «                              ¦   «          ‰| _        t          ‰¦  «        | _        t          j        ‰j        ‰j        ¦  «        | _        t          j	        ‰j
        ‰j        ¬¦  «        | _        t          j        ‰j        ¦  «        | _        t          j        ˆfd„t#          ‰j        ¦  «        D ¦   «         ¦  «        | _        t)          ‰¦  «        | _        d| _        d S )Nrð   c                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS © )rî   )rŽ   rú   r4   s     €r7   r�   z'SEWEncoder.__init__.<locals>.<listcomp>ž  s!   ø€ Ð$fÐ$fÐ$fÀ¥_°VÑ%<Ô%<Ð$fÐ$fÐ$fr8   F)r&   r'   r4   rX   Úpos_conv_embedr   Ú	AvgPool1drg   ÚpoolrI   rd   rõ   rJ   rà   rê   r­   r—   r”   Únum_hidden_layersÚlayersrz   Úupsampler™   r~   s    `€r7   r'   zSEWEncoder.__init__—  sÈ   øø€ Ý‰Œ×ÒÑÔÐØˆŒÝ8¸Ñ@Ô@ˆÔÝ”L Ô!6¸Ô8MÑNÔNˆŒ	Ýœ, vÔ'9¸vÔ?TÐUÑUÔUˆŒÝ”z &Ô"7Ñ8Ô8ˆŒÝ”mÐ$fÐ$fÐ$fÐ$fÅeÈFÔLdÑFeÔFeÐ$fÑ$fÔ$fÑgÔgˆŒÝ% fÑ-Ô-ˆŒØ&+ˆÔ#Ð#Ð#r8   Nr®   c           	      óô  — |�»|                      d¦  «                             dd|j        d         ¦  «        }d|| <   t          j                             |                     ¦   «                               d¦  «        | j        j        | j        j        ¬¦  «         	                    d¦  «         
                    ¦   «         }|j        d         }|                     dd¦  «        }|                      |¦  «        }|                      |¦  «        }t          |                     d¦  «        |                     d¦  «        ¦  «        }|dd |…f         |dd |…f         z   }|                     dd¦  «        }t!          | j        ||¬¦  «        }|                      |¦  «        }|                      |¦  «        }t'          ¦   «         pt)          | ¦  «        }	| j        D ]<}
t-          j        g ¦  «        }| j        o|| j        j        k     }|r|	r |
|fd|i|¤Ž}Œ=|                      |¦  «        }|j        d         |k     r2t          j                             |d	d	d	||j        d         z
  f¦  «        }t9          |¬
¦  «        S )NrM   r   rZ   r¦   )r#   r$   .)r4   Úinputs_embedsr«   r«   r   ©Úlast_hidden_state)Ú	unsqueezeÚrepeatrË   r   rµ   Ú
max_pool1drÙ   r4   rg   ÚsqueezeÚlongrN   r   r  Úminr€   r   rJ   r­   r
   r   r  r³   Úrandr¢   Ú	layerdropr  Úpadr   )r3   r<   r«   r®   Úexpand_attention_maskÚn_input_timestepsÚposition_embeddingsÚpooled_hidden_statesÚ
min_lengthÚsynced_gpusr‘   Údropout_probabilityÚskip_the_layers                r7   r=   zSEWEncoder.forward¢  s‹  € ð Ð%à$2×$<Ò$<¸RÑ$@Ô$@×$GÒ$GÈÈ1ÈmÔNaÐbcÔNdÑ$eÔ$eÐ!Ø47ˆMÐ0Ð0Ñ1õ ”×(Ò(Ø"×(Ò(Ñ*Ô*×4Ò4°QÑ7Ô7Ø $¤Ô :Øœ;Ô5ð )ñ ô ÷
 ’˜‘”ß’‘”ð ð *Ô/°Ô2Ðà%×/Ò/°°1Ñ5Ô5ˆØ"×1Ò1°-Ñ@Ô@ÐØ#Ÿyšy¨Ñ7Ô7ÐÝÐ,×1Ò1°"Ñ5Ô5Ð7K×7PÒ7PÐQSÑ7TÔ7TÑUÔUˆ
Ø,¨S°+°:°+Ð-=Ô>ÐATÐUXÐZeÐ[eÐZeÐUeÔAfÑfˆØ%×/Ò/°°1Ñ5Ô5ˆå2Ø”;¨mÈNð
ñ 
ô 
ˆð Ÿš¨Ñ6Ô6ˆØŸš ]Ñ3Ô3ˆå0Ñ2Ô2ÐRÕ6LÈTÑ6RÔ6Rˆà”[ð 	^ð 	^ˆEå"'¤*¨R¡.¤.Ðà!œ]ÐZÐ/BÀTÄ[ÔEZÒ/ZˆNØ!ð ^ [ð ^à %  mÐ ]Ð ]ÀNÐ ]ÐV\Ð ]Ð ]�øàŸš mÑ4Ô4ˆØÔ˜qÔ!Ð$5Ò5Ð5ÝœM×-Ò-¨m¸aÀÀAÐGXÐ[hÔ[nÐopÔ[qÑGqÐ=rÑsÔsˆMå°Ð?Ñ?Ô?Ð?r8   r:   )r@   rA   rB   r'   r   r   r=   rC   rD   s   @r7   rü   rü   –  sp   ø€ € € € € ð	,ð 	,ð 	,ð 	,ð 	,ð ð7@ð 7@ð Ð+Ô,ð	7@ð 7@ð 7@ð 7@ð 7@ð 7@ð 7@ð 7@r8   rü   c                   óÈ   ‡ — e Zd ZU eed<   dZdZdZdZdZ	dZ
dZe eedd¬¦  «        d	œZ ej        ¦   «         ˆ fd
„¦   «         Zdej        ez  fd„Zdedej        fd„Zˆ xZS )ÚSEWPreTrainedModelr4   Úsewr£   ÚaudioTr   Úencoder)ÚindexÚ
layer_name)r<   Ú
attentionsc           
      ó’  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rwt	          j        |j        j        ddt          j	        d|j        j
        d         |j        j        z  z  ¦  «        z  ¬¦  «         t	          j        |j        j        d¦  «         dS t          |t          j        ¦  «        rþt!          ¦   «         rÕddl}t%          |d¦  «        rlt%          |d¦  «        r\|j                             |j        |j        gd¬¦  «        5  t	          j        |j        ¦  «         ddd¦  «         dS # 1 swxY w Y   dS |j                             |j        d¬¦  «        5  t	          j        |j        ¦  «         ddd¦  «         dS # 1 swxY w Y   dS t	          j        |j        ¦  «         dS dS )	zInitialize the weightsr   rZ   r   )ÚmeanÚstdNrp   ro   r^   )r&   Ú_init_weightsrå   rX   ÚinitÚnormal_r/   r`   ÚmathÚsqrtr#   Úin_channelsÚ	constant_r%   r   r+   r
   rj   ri   rk   rl   rp   ro   Úkaiming_normal_)r3   r§   rj   r6   s      €r7   r&  z SEWPreTrainedModel._init_weightsë  sG  ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ8Ñ9Ô9ð 	4ÝŒLØ”Ô"ØØ�œ	 ! v¤{Ô'>¸qÔ'AÀFÄKÔD[Ñ'[Ñ"\Ñ]Ô]Ñ]ðñ ô ð õ
 ŒN˜6œ;Ô+¨QÑ/Ô/Ð/Ð/Ð/Ý˜¥¤	Ñ*Ô*ð 	4Ý)Ñ+Ô+ð 
4Ø Ð Ð Ð å˜6 :Ñ.Ô.ð <µ7¸6À:Ñ3NÔ3Nð <Ø"œ×:Ò:¸F¼OÈVÌ_Ð;]ÐmnÐ:ÑoÔoð <ð <ÝÔ,¨V¬]Ñ;Ô;Ð;ð<ð <ð <ñ <ô <ð <ð <ð <ð <ð <ð <ð <øøøð <ð <ð <ð <ð <ð <ð #œ×:Ò:¸6¼=ÐXYÐ:ÑZÔZð <ð <ÝÔ,¨V¬]Ñ;Ô;Ð;ð<ð <ð <ñ <ô <ð <ð <ð <ð <ð <ð <ð <øøøð <ð <ð <ð <ð <ð <õ Ô$ V¤]Ñ3Ô3Ð3Ð3Ð3ð	4ð 	4s$   Ä#E
Å
EÅEÅ8FÆF#Æ&F#Úinput_lengthsc                 óz   — d„ }t          | j        j        | j        j        ¦  «        D ]\  }} ||||¦  «        }Œ|S )zH
        Computes the output length of the convolutional layers
        c                 ó<   — t          j        | |z
  |d¬¦  «        dz   S )NÚfloor)Úrounding_moder   )r³   Údiv)Úinput_lengthr#   r$   s      r7   Ú_conv_out_lengthzMSEWPreTrainedModel._get_feat_extract_output_lengths.<locals>._conv_out_length  s&   € õ ”9˜\¨KÑ7¸ÈwÐWÑWÔWÐZ[Ñ[Ð[r8   )Úzipr4   r,   r-   )r3   r.  r5  r#   r$   s        r7   Ú _get_feat_extract_output_lengthsz3SEWPreTrainedModel._get_feat_extract_output_lengths  s\   € ð
	\ð 	\ð 	\õ
 $' t¤{Ô'>ÀÄÔ@WÑ#XÔ#Xð 	Qð 	QÑˆK˜Ø,Ð,¨]¸KÈÑPÔPˆMˆMàÐr8   Úfeature_vector_lengthr«   c                 ó  — |                       |                     d¦  «        ¦  «                             t          j        ¦  «        }t          j        ||j        ¬¦  «        }|                     d¦  «        |                     d¦  «        k     S )NrM   )Údevicer   r   )r7  ÚsumÚtor³   r  Úaranger:  r
  )r3   r8  r«   Úoutput_lengthsÚattention_idss        r7   Ú"_get_feature_vector_attention_maskz5SEWPreTrainedModel._get_feature_vector_attention_mask  sw   € Ø×>Ò>¸~×?QÒ?QÐRTÑ?UÔ?UÑVÔV×YÒYÕZ_ÔZdÑeÔeˆõ œÐ%:À>ÔCXÐYÑYÔYˆØ×&Ò& qÑ)Ô)¨N×,DÒ,DÀQÑ,GÔ,GÒGÐGr8   )r@   rA   rB   r   Ú__annotations__Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnrî   r   r¼   Ú_can_record_outputsr³   Úno_gradr&  Ú
LongTensorrØ   r7  r@  rC   rD   s   @r7   r  r  Ü  sú   ø€ € € € € € àÐÐÑØÐØ$€OØÐØ&*Ð#ØÐØ€NØÐà(Ø$�n \¸ÀyÐQÑQÔQðð Ðð
 €U„]�_„_ð4ð 4ð 4ð 4ñ „_ð4ð.¸eÔ>NÐQTÑ>Tð ð ð ð ðHÈð HÐ]bÔ]mð Hð Hð Hð Hð Hð Hð Hð Hr8   r  rË   Ú	mask_probÚmask_lengthÚ	min_masksrÉ   c                 ó@  ‡‡‡‡‡— | \  }Š‰dk     rt          d¦  «        ‚‰‰k    rt          d‰› d‰› d�¦  «        ‚t          j                             d¦  «                             ¦   «         Šˆˆˆˆˆfd„}|�9|                     ¦   «                              d¦  «                             ¦   «         nˆfd	„t          |¦  «        D ¦   «         }t          j	        |‰ft          ¬
¦  «        }g }	 |‰¦  «        }
|
dk    r|S |D ]·} ||¦  «        }t          j                             t          j        |‰dz
  z
  ¦  «        |d¬¦  «        }t          |¦  «        dk    r‰dz
  }n|d         }t          j        |t          j        |
|z
  t          j        ¬
¦  «        |z  g¦  «        }|	                     |¦  «         Œ¸t          j        |	¦  «        }	t          j        |	dd…dd…df         ||
‰f¦  «        }	|	                     ||
‰z  ¦  «        }	t          j        ‰¦  «        dddd…f         }t          j        |||
‰f¦  «                             ||
‰z  ¦  «        }|	|z   }	|	                     ¦   «         ‰dz
  k    r‰dz
  |	|	‰dz
  k    <   t          j        ||	dd¦  «         |S )an  
    Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for
    ASR](https://huggingface.co/papers/1904.08779). Note that this method is not optimized to run on TPU and should be run on
    CPU as part of the preprocessing during training.

    Args:
        shape: The shape for which to compute masks. This should be of a tuple of size 2 where
               the first element is the batch size and the second element is the length of the axis to span.
        mask_prob:  The percentage of the whole axis (between 0 and 1) which will be masked. The number of
                    independently generated mask spans of length `mask_length` is computed by
                    `mask_prob*shape[1]/mask_length`. Note that due to overlaps, `mask_prob` is an upper bound and the
                    actual percentage will be smaller.
        mask_length: size of the mask
        min_masks: minimum number of masked spans
        attention_mask: A (right-padded) attention mask which independently shortens the feature axis of
                        each batch dimension.
    r   z&`mask_length` has to be bigger than 0.zO`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: z and `sequence_length`: ú`c                 ó¸   •— t          ‰| z  ‰z  ‰z   ¦  «        }t          |‰¦  «        }|‰z  ‰k    r‰‰z  }| ‰dz
  z
  |k     rt          | ‰dz
  z
  d¦  «        }|S )z;Given input length, compute how many spans should be maskedr   r   )rØ   Úmax)r4  Únum_masked_spanÚepsilonrM  rL  rN  Úsequence_lengths     €€€€€r7   Úcompute_num_masked_spanz6_compute_mask_indices.<locals>.compute_num_masked_spanA  s~   ø€ å˜i¨,Ñ6¸ÑDÀwÑNÑOÔOˆÝ˜o¨yÑ9Ô9ˆð ˜[Ñ(¨?Ò:Ð:Ø-°Ñ<ˆOð ˜;¨™?Ñ+¨oÒ=Ð=Ý! ,°+À±/Ñ"BÀAÑFÔFˆOàÐr8   NrM   c                 ó   •— g | ]}‰‘ŒS rÿ   rÿ   )rŽ   rú   rU  s     €r7   r�   z)_compute_mask_indices.<locals>.<listcomp>T  s   ø€ Ð9Ð9Ð9 !ˆoÐ9Ð9Ð9r8   ©Údtyper   F)Úreplace)r–   ÚnpÚrandomr  ÚitemÚdetachr;  Útolistr”   ÚzerosrÚ   Úchoicer=  ÚlenÚconcatenateÚonesÚint32ÚappendÚarrayÚbroadcast_tor�   rR  Úput_along_axis)rË   rL  rM  r«   rN  Ú
batch_sizerV  r.  Úspec_aug_maskÚspec_aug_mask_idxsÚmax_num_masked_spanr4  rS  Úspec_aug_mask_idxÚdummy_mask_idxÚoffsetsrT  rU  s    `` `           @@r7   Ú_compute_mask_indicesrq    sP  øøøøø€ ð0 #(Ñ€J�à�Q‚€ÝÐAÑBÔBÐBà�_Ò$Ð$Ýð:Ð^ið :ð :Ø'6ð:ð :ð :ñ
ô 
ð 	
õ Œi�nŠn˜QÑÔ×$Ò$Ñ&Ô&€Gðð ð ð ð ð ð ð ð ð$ Ð%ð 	×ÒÑÔ×#Ò# BÑ'Ô'×.Ò.Ñ0Ô0Ð0à9Ð9Ð9Ð9¥u¨ZÑ'8Ô'8Ð9Ñ9Ô9ð õ ”H˜j¨/Ð:Å$ÐGÑGÔG€MØÐà1Ð1°/ÑBÔBÐà˜aÒÐØÐà%ð 5ð 5ˆà1Ð1°,Ñ?Ô?ˆõ œI×,Ò,ÝŒI�l k°A¡oÑ6Ñ7Ô7¸ÐRWð -ñ 
ô 
Ðõ Ð Ñ!Ô! QÒ&Ð&ð -¨qÑ0ˆNˆNà.¨qÔ1ˆNåœNØ¥¤Ð(;¸oÑ(MÕUWÔU]Ð ^Ñ ^Ô ^ÐaoÑ oÐpñ
ô 
Ðð 	×!Ò!Ð"3Ñ4Ô4Ð4Ð4åœÐ"4Ñ5Ô5Ðõ œØ˜1˜1˜1˜a˜a˜a ˜:Ô&¨Ð5HÈ+Ð(Vñô Ðð ,×3Ò3°JÐ@SÐVaÑ@aÑbÔbÐõ Œi˜Ñ$Ô$ T¨4°°° ]Ô3€GÝŒo˜g¨
Ð4GÈÐ'UÑVÔV×^Ò^ØÐ'¨+Ñ5ñô €Gð ,¨gÑ5Ðð ×ÒÑÔ /°AÑ"5Ò5Ð5ØGVÐYZÑGZÐÐ-°À!Ñ0CÒCÑDõ Ô�mÐ%7¸¸BÑ?Ô?Ð?àÐr8   c                   ó  ‡ — e Zd Zdefˆ fd„Z	 	 ddej        dej        dz  dej        dz  fd„Ze	 e
d¬	¦  «        e	 	 dd
ej        dz  dej        dz  dej        dz  dee         deez  f
d„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚSEWModelr4   c                 óª  •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        t          j        |j        d         |j        ¬¦  «        | _	        |j        d         |j
        k    | _        | j        r*t          j        |j        d         |j
        ¦  «        | _        t          j        |j        ¦  «        | _        |j        dk    s|j        dk    rBt          j        t)          j        |j
        ¦  «                             ¦   «         ¦  «        | _        t1          |¦  «        | _        |                      ¦   «          d S )NrM   rð   r¦   )r&   r'   r4   rˆ   Úfeature_extractorr   rI   r(   rõ   rJ   rd   Úproject_featuresr|   Úfeature_projectionrà   Úfeat_proj_dropoutÚfeature_dropoutÚmask_time_probÚmask_feature_probÚ	Parameterr³   rÛ   Úuniform_Úmasked_spec_embedrü   r  Ú	post_initr~   s     €r7   r'   zSEWModel.__init__”  s  ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒÝ!2°6Ñ!:Ô!:ˆÔÝœ, v¤°rÔ':ÀÔ@UÐVÑVÔVˆŒà &¤°Ô 3°vÔ7IÒ IˆÔØÔ ð 	YÝ&(¤i°´ÀÔ0CÀVÔEWÑ&XÔ&XˆDÔ#Ý!œz¨&Ô*BÑCÔCˆÔàÔ  3Ò&Ð&¨&Ô*BÀSÒ*HÐ*HÝ%'¤\µ%´,¸vÔ?QÑ2RÔ2R×2[Ò2[Ñ2]Ô2]Ñ%^Ô%^ˆDÔ"å! &Ñ)Ô)ˆŒð 	�ŠÑÔÐÐÐr8   Nr<   Úmask_time_indicesr«   c                 ó  — t          | j        dd¦  «        s|S |                     ¦   «         \  }}}|�#| j                             |j        ¦  «        ||<   n“| j        j        dk    rƒ| j        r|t          ||f| j        j        | j        j	        || j        j
        ¬¦  «        }t          j        ||j        t          j        ¬¦  «        }| j                             |j        ¦  «        ||<   | j        j        dk    r†| j        rt          ||f| j        j        | j        j        | j        j        ¬¦  «        }t          j        ||j        t          j        ¬¦  «        }|dd…df                              d|d¦  «        }d||<   |S )	z¢
        Masks extracted features along time axis and/or along feature axis according to
        [SpecAugment](https://huggingface.co/papers/1904.08779).
        Úapply_spec_augmentTNr   )rL  rM  r«   rN  )r:  rY  )rL  rM  rN  rM   )Úgetattrr4   r€   r~  r<  rY  rz  r¢   rq  Úmask_time_lengthÚmask_time_min_masksr³   Útensorr:  rÚ   r{  Úmask_feature_lengthÚmask_feature_min_masksÚexpand)r3   r<   r€  r«   rj  rU  rd   Úmask_feature_indicess           r7   Ú_mask_hidden_stateszSEWModel._mask_hidden_states¨  s—  € õ �t”{Ð$8¸$Ñ?Ô?ð 	!Ø Ð ð 4A×3EÒ3EÑ3GÔ3GÑ0ˆ
�O [àÐ(à/3Ô/E×/HÒ/HÈÔI\Ñ/]Ô/]ˆMÐ+Ñ,Ð,ØŒ[Ô'¨!Ò+Ð+°´Ð+Ý 5Ø˜_Ð-Øœ+Ô4Ø œKÔ8Ø-Øœ+Ô9ð!ñ !ô !Ðõ !&¤Ð->À}ÔG[ÕchÔcmÐ nÑ nÔ nÐØ/3Ô/E×/HÒ/HÈÔI\Ñ/]Ô/]ˆMÐ+Ñ,àŒ;Ô(¨1Ò,Ð,°´Ð,å#8Ø˜[Ð)Øœ+Ô7Ø œKÔ;Øœ+Ô<ð	$ñ $ô $Ð õ $)¤<Ð0DÈ]ÔMaÕinÔisÐ#tÑ#tÔ#tÐ Ø#7¸¸¸¸4¸Ô#@×#GÒ#GÈÈOÐ]_Ñ#`Ô#`Ð Ø23ˆMÐ.Ñ/àÐr8   F)Útie_last_hidden_statesr£   r®   rÉ   c                 ó¦  — |                       |¦  «        }|                     dd¦  «        }|                      |¦  «        }| j        r|                      |¦  «        }|                      |¦  «        }|�!|                      |j        d         |¦  «        }|                      ||¬¦  «        } | j	        |fd|i|¤Ž}|j
        }t          |¬¦  «        S )a/  
        mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict
            masked extracted features in *config.proj_codevector_dim* space.
        r   rZ   N)r€  r«   r  )ru  rN   rJ   rv  rw  ry  r@  rË   r‹  r  r	  r   )r3   r£   r«   r€  r®   Úextract_featuresr<   Úencoder_outputss           r7   r=   zSEWModel.forwardÖ  sü   € ð  ×1Ò1°,Ñ?Ô?ÐØ+×5Ò5°a¸Ñ;Ô;ÐØŸ?š?Ð+;Ñ<Ô<ÐàÔ ð 	IØ#×6Ò6Ð7GÑHÔHÐØ×,Ò,Ð-=Ñ>Ô>ˆàÐ%à!×DÒDÀ]ÔEXÐYZÔE[Ð]kÑlÔlˆNà×0Ò0°ÐRcÐ0ÑdÔdˆà&˜$œ,Øð
ð 
à)ð
ð ð
ð 
ˆð (Ô9ˆå°Ð?Ñ?Ô?Ð?r8   ©NN)r@   rA   rB   r   r'   r³   ÚFloatTensorrK  r‹  r   r   r   rÛ   r   r   rÜ   r   r=   rC   rD   s   @r7   rs  rs  ’  sD  ø€ € € € € ð˜yð ð ð ð ð ð ð. 7;Ø26ð	,ð ,àÔ(ð,ð !Ô,¨tÑ3ð,ð Ô(¨4Ñ/ð	,ð ,ð ,ð ,ð\  Ø€_¨EÐ2Ñ2Ô2Øð /3Ø6:ð	"@ð "@à”l TÑ)ð"@ð œ tÑ+ð"@ð !Ô,¨tÑ3ð	"@ð
 Ð+Ô,ð"@ð 
�Ñ	 ð"@ð "@ð "@ñ „^ñ 3Ô2ñ  Ôð"@ð "@ð "@ð "@ð "@r8   rs  zk
    SEW Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).
    )Úcustom_introc                   óÂ   ‡ — e Zd Zddedz  fˆ fd„Zd„ Zd„ Zd„ Zee		 	 dde
j        dz  de
j        dz  d	e
j        dz  d
ee         deez  f
d„¦   «         ¦   «         Zˆ xZS )Ú	SEWForCTCNÚtarget_langc                 óª  •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        ¦  «        | _        || _        |j	        €t          d| j        › d�¦  «        ‚t          |d¦  «        r|j        r|j        n|j        }t	          j        ||j	        ¦  «        | _        |                      ¦   «          dS )a-  
        target_lang (`str`, *optional*):
            Language id of adapter weights. Adapter weights are stored in the format adapter.<lang>.safetensors or
            adapter.<lang>.bin. Only relevant when using an instance of [`SEWForCTC`] with adapters. Uses 'eng' by
            default.
        NzYou are trying to instantiate zò with a configuration that does not define the vocabulary size of the language model head. Please instantiate the model as follows: `SEWForCTC.from_pretrained(..., vocab_size=vocab_size)`. or define `vocab_size` of your model's configuration.Úadd_adapter)r&   r'   rs  r  r   rà   Úfinal_dropoutr­   r•  Ú
vocab_sizer–   r6   ri   r—  Úoutput_hidden_sizerd   r|   Úlm_headr  )r3   r4   r•  rš  r6   s       €r7   r'   zSEWForCTC.__init__  sÝ   ø€ õ 	‰Œ×Ò˜Ñ Ô Ð å˜FÑ#Ô#ˆŒÝ”z &Ô"6Ñ7Ô7ˆŒà&ˆÔàÔÐ$ÝðH°´ð Hð Hð Hñô ð õ *1°¸Ñ)GÔ)GÐvÈFÔL^ÐvˆFÔ%Ð%ÐdjÔdvð 	õ ”yÐ!3°VÔ5FÑGÔGˆŒð 	�ŠÑÔÐÐÐr8   c                 óT  — t          ¦   «         t          j        d¦  «        k    rdS | j        }|�)t	          | j        dd¦  «        €t          d|› d�¦  «        ‚|€2t	          | j        dd¦  «        �t                               d¦  «         dS |�|  	                    |d¬¦  «         dS dS )	a'  
        This method overwrites [`~PreTrainedModel.tie_weights`] so that adapter weights can be correctly loaded when
        passing `target_lang=...` to `from_pretrained(...)`.

        This method is **not** supposed to be called by the user and is prone to be changed in the future.
        ÚmetaNÚadapter_attn_dimzCannot pass `target_lang`: z- if `config.adapter_attn_dim` is not defined.z)By default `target_lang` is set to 'eng'.T)Ú
force_load)
r   r³   r:  r•  rƒ  r4   r–   ÚloggerÚinfoÚload_adapter)r3   r®   r•  s      r7   Útie_weightszSEWForCTC.tie_weights!  sÃ   € õ 6Ñ7Ô7½5¼<ÈÑ;OÔ;OÒOÐOØˆFð Ô&ˆàÐ"¥w¨t¬{Ð<NÐPTÑ'UÔ'UÐ']ÝÐu¸;ÐuÐuÐuÑvÔvÐvØÐ ¥W¨T¬[Ð:LÈdÑ%SÔ%SÐ%_Ý�KŠKÐCÑDÔDÐDÐDÐDØÐ$Ø×Ò˜k°dÐÑ;Ô;Ð;Ð;Ð;ð %Ð$r8   c                 óB   — | j         j                             ¦   «          dS ©z¨
        Calling this function will disable the gradient computation for the feature encoder so that its parameter will
        not be updated during training.
        N©r  ru  r    ©r3   s    r7   Úfreeze_feature_encoderz SEWForCTC.freeze_feature_encoder9  ó!   € ð
 	ŒÔ"×5Ò5Ñ7Ô7Ð7Ð7Ð7r8   c                 óL   — | j                              ¦   «         D ]	}d|_        Œ
dS ©zÒ
        Calling this function will disable the gradient computation for the base model so that its parameters will not
        be updated during training. Only the classification head will be updated.
        FN©r  rœ   r�   rž   s     r7   Úfreeze_base_modelzSEWForCTC.freeze_base_model@  ó6   € ð
 ”X×(Ò(Ñ*Ô*ð 	(ð 	(ˆEØ"'ˆEÔÐð	(ð 	(r8   r£   r«   Úlabelsr®   rÉ   c           
      óü  — |�>|                      ¦   «         | j        j        k    rt          d| j        j        › �¦  «        ‚ | j        |fd|i|¤Ž}|j        }|                      |¦  «        }|                      |¦  «        }d}|��Z|�|nt          j	        |t          j
        ¬¦  «        }|                      |                     d¦  «        ¦  «                             t          j
        ¦  «        }	|dk    }
|
                     d¦  «        }|                     |
¦  «        }t          j                             |dt          j        ¬¦  «                             dd¦  «        }t          j        j                             d	¬
¦  «        5  t          j                             |||	|| j        j        | j        j        | j        j        ¬¦  «        }ddd¦  «         n# 1 swxY w Y   t7          |||j        |j        ¬¦  «        S )aà  
        labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*):
            Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to
            the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`.
            All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ...,
            config.vocab_size - 1]`.
        Nz$Label values must be <= vocab_size: r«   rX  rM   r   )rb   rY  r   F)Úenabled)ÚblankÚ	reductionÚzero_infinity©ÚlossÚlogitsr<   r"  )rR  r4   r™  r–   r  r	  r­   r›  r³   Ú	ones_liker  r7  r;  r<  Úmasked_selectr   rµ   Úlog_softmaxÚfloat32rN   ÚbackendsÚcudnnÚflagsÚctc_lossÚpad_token_idÚctc_loss_reductionÚctc_zero_infinityr   r<   r"  )r3   r£   r«   r¯  r®   Úoutputsr<   r·  r¶  r.  Úlabels_maskÚtarget_lengthsÚflattened_targetsÚ	log_probss                 r7   r=   zSEWForCTC.forwardH  s:  € ð  Ð &§*¢*¡,¤,°$´+Ô2HÒ"HÐ"HÝÐ\ÀDÄKÔDZÐ\Ð\Ñ]Ô]Ð]à�$”(Øð
ð 
à)ð
ð ð
ð 
ˆð  Ô1ˆØŸš ]Ñ3Ô3ˆà—’˜mÑ,Ô,ˆàˆØÑð #1Ð"<��Å%Ä/ÐR^ÕfkÔfpÐBqÑBqÔBqð ð !×AÒAÀ.×BTÒBTÐUWÑBXÔBXÑYÔY×\Ò\Õ]bÔ]gÑhÔhˆMð ! Aš+ˆKØ(Ÿ_š_¨RÑ0Ô0ˆNØ &× 4Ò 4°[Ñ AÔ AÐõ œ×1Ò1°&¸bÍÌÐ1ÑVÔV×`Ò`ÐabÐdeÑfÔfˆIå”Ô%×+Ò+°EÐ+Ñ:Ô:ð 	ð 	Ý”}×-Ò-ØØ%Ø!Ø"Øœ+Ô2Ø"œkÔ<Ø"&¤+Ô"?ð .ñ ô �ð	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	õ Ø˜f°GÔ4IÐV]ÔVhð
ñ 
ô 
ð 	
s   ÆAGÇGÇGr:   r�  )r@   rA   rB   rç   r'   r£  r¨  r­  r   r   r³   rÛ   r   r   rÜ   r   r=   rC   rD   s   @r7   r”  r”  þ  s  ø€ € € € € ðð ¨C°$©Jð ð ð ð ð ð ð:<ð <ð <ð08ð 8ð 8ð(ð (ð (ð Øð /3Ø&*ð	:
ð :
à”l TÑ)ð:
ð œ tÑ+ð:
ð ”˜tÑ#ð	:
ð
 Ð+Ô,ð:
ð 
�Ñ	ð:
ð :
ð :
ñ „^ñ Ôð:
ð :
ð :
ð :
ð :
r8   r”  z’
    SEW Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like
    SUPERB Keyword Spotting.
    c                   ó¾   ‡ — e Zd Zˆ fd„Zd„ Zd„ Zeee	 	 dde	j
        dz  de	j
        dz  de	j
        dz  dee         d	eez  f
d
„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚSEWForSequenceClassificationc                 óô  •— t          ¦   «                              |¦  «         t          |d¦  «        r|j        rt	          d¦  «        ‚t          |¦  «        | _        |j        dz   }|j        r.t          j
        t          j        |¦  «        |z  ¦  «        | _        t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¦  «        | _        |                      ¦   «          d S )Nr—  zZSequence classification does not support the use of SEW adapters (config.add_adapter=True)r   )r&   r'   ri   r—  r–   rs  r  r  Úuse_weighted_layer_sumr   r|  r³   rd  Úlayer_weightsr|   rd   Úclassifier_proj_sizeÚ	projectorÚ
num_labelsÚ
classifierr  )r3   r4   Ú
num_layersr6   s      €r7   r'   z%SEWForSequenceClassification.__init__Ž  sá   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å�6˜=Ñ)Ô)ð 	¨fÔ.@ð 	ÝØlñô ð õ ˜FÑ#Ô#ˆŒØÔ-°Ñ1ˆ
ØÔ(ð 	SÝ!#¤­e¬j¸Ñ.DÔ.DÀzÑ.QÑ!RÔ!RˆDÔÝœ 6Ô#5°vÔ7RÑSÔSˆŒÝœ) FÔ$?ÀÔARÑSÔSˆŒð 	�ŠÑÔÐÐÐr8   c                 óB   — | j         j                             ¦   «          dS r¥  r¦  r§  s    r7   r¨  z3SEWForSequenceClassification.freeze_feature_encoderŸ  r©  r8   c                 óL   — | j                              ¦   «         D ]	}d|_        Œ
dS r«  r¬  rž   s     r7   r­  z.SEWForSequenceClassification.freeze_base_model¦  r®  r8   Nr£   r«   r¯  r®   rÉ   c                 óæ  — | j         j        rd|d<    | j        |fd|i|¤Ž}| j         j        rr|j        }t	          j        |d¬¦  «        }t          j                             | j	        d¬¦  «        }|| 
                    ddd¦  «        z                       d¬¦  «        }n|j        }|                      |¦  «        }|€|                     d¬¦  «        }n�|                      |j        d         |¦  «        }	|	                     d¦  «                             dd|j        d         ¦  «        }
d	||
 <   |                     d¬¦  «        |	                     d¬¦  «         
                    dd¦  «        z  }|                      |¦  «        }d}|�Kt)          ¦   «         } || 
                    d| j         j        ¦  «        | 
                    d¦  «        ¦  «        }t-          |||j        |j        ¬
¦  «        S )a  
        input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
            Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file
            into an array of type `list[float]`, a `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library
            (`pip install torchcodec`) or the soundfile library (`pip install soundfile`).
            To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and conversion
            into a tensor of type `torch.FloatTensor`. See [`SEWProcessor.__call__`] for details.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        TÚoutput_hidden_statesr«   r   r±   rM   NrZ   r¦   rµ  )r4   rË  r  r<   r³   Ústackr   rµ   r¶   rÌ  rÌ   r;  r	  rÎ  r$  r@  rË   r
  r  rÐ  r   rÏ  r   r"  )r3   r£   r«   r¯  r®   rÃ  r<   Únorm_weightsÚpooled_outputÚpadding_maskÚexpand_padding_maskr·  r¶  Úloss_fcts                 r7   r=   z$SEWForSequenceClassification.forward®  s  € ð, Œ;Ô-ð 	2Ø-1ˆFÐ)Ñ*à�$”(Øð
ð 
à)ð
ð ð
ð 
ˆð Œ;Ô-ð 	6Ø#Ô1ˆMÝ!œK¨¸1Ð=Ñ=Ô=ˆMÝœ=×0Ò0°Ô1CÈÐ0ÑLÔLˆLØ*¨\×->Ò->¸rÀ1ÀaÑ-HÔ-HÑH×MÒMÐRSÐMÑTÔTˆMˆMà#Ô5ˆMàŸš }Ñ5Ô5ˆØÐ!Ø)×.Ò.°1Ð.Ñ5Ô5ˆMˆMà×BÒBÀ=ÔCVÐWXÔCYÐ[iÑjÔjˆLØ".×"8Ò"8¸Ñ"<Ô"<×"CÒ"CÀAÀqÈ-ÔJ]Ð^_ÔJ`Ñ"aÔ"aÐØ25ˆMÐ.Ð.Ñ/Ø)×-Ò-°!Ð-Ñ4Ô4°|×7GÒ7GÈAÐ7GÑ7NÔ7N×7SÒ7SÐTVÐXYÑ7ZÔ7ZÑZˆMà—’ Ñ/Ô/ˆàˆØÐÝ'Ñ)Ô)ˆHØ�8˜FŸKšK¨¨D¬KÔ,BÑCÔCÀVÇ[Â[ÐQSÁ_Ä_ÑUÔUˆDå'ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r8   r�  )r@   rA   rB   r'   r¨  r­  r   r	   r   r³   rÛ   r   r   rÜ   r   r=   rC   rD   s   @r7   rÉ  rÉ  ‡  sê   ø€ € € € € ðð ð ð ð ð"8ð 8ð 8ð(ð (ð (ð Ø Øð /3Ø&*ð	9
ð 9
à”l TÑ)ð9
ð œ tÑ+ð9
ð ”˜tÑ#ð	9
ð
 Ð+Ô,ð9
ð 
Ð)Ñ	)ð9
ð 9
ð 9
ñ „^ñ !Ô ñ Ôð9
ð 9
ð 9
ð 9
ð 9
r8   rÉ  )r”  rÉ  rs  r  )Nr¦   rx   )Lr)  Úcollections.abcr   Únumpyr[  r³   r   Útorch.nnr   Ú r   r'  Úactivationsr   Úbackbone_utilsr	   Úintegrations.deepspeedr
   Úintegrations.fsdpr   Úmasking_utilsr   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   Úmodeling_utilsr   r   r   Úprocessing_utilsr   rh   r   r   r   Úutils.genericr   r   Úutils.output_capturingr   r   Úconfiguration_sewr   Ú
get_loggerr@   r   r    rF   rP   ÚModulerX   rr   rz   rˆ   rÛ   rÙ   rº   r¼   rÞ   rî   rü   r  rÜ   rØ   rK  Úndarrayrq  rs  r”  rÉ  Ú__all__rÿ   r8   r7   ú<module>rñ     sò  ðð* €€€Ø $Ð $Ð $Ð $Ð $Ð $à Ð Ð Ð Ø €€€Ø Ð Ð Ð Ð Ð Ø %Ð %Ð %Ð %Ð %Ð %à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø @Ð @Ð @Ð @Ð @Ð @Ø 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø BÐ BÐ BÐ BÐ BÐ BØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YØ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rØ &Ð &Ð &Ð &Ð &Ð &Ø @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ (Ð (Ð (Ð (Ð (Ð (ð 
ˆÔ	˜HÑ	%Ô	%€ðð ð ð ð Ð8ñ ô ð ð*ð ð ð ð Ð6ñ ô ð ð6ð ð ð ð Ð6ñ ô ð ð0(ð (ð (ð (ð ( ¤ñ (ô (ð (ðVð ð ð ð �b”iñ ô ð ðð ð ð ð �B”Iñ ô ð ð,#ð #ð #ð #ð #˜œ	ñ #ô #ð #ðX !Øð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð �T‰\ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð8R)ð R)ð R)ð R)ð R)�2”9ñ R)ô R)ð R)ðjð ð ð ð �R”Yñ ô ð ð0ð ð ð ð Ð0ñ ô ð ðDC@ð C@ð C@ð C@ð C@�”ñ C@ô C@ð C@ðL ð;Hð ;Hð ;Hð ;Hð ;H˜ñ ;Hô ;Hñ „ð;HðD /3Øðtð tØ��c�Œ?ðtàðtð ðtð Ô$ tÑ+ð	tð
 ðtð „Zðtð tð tð tðn ðh@ð h@ð h@ð h@ð h@Ð!ñ h@ô h@ñ „ðh@ðV €ððñ ô ð
A
ð A
ð A
ð A
ð A
Ð"ñ A
ô A
ñô ð
A
ðH €ððñ ô ð]
ð ]
ð ]
ð ]
ð ]
Ð#5ñ ]
ô ]
ñô ð]
ð@ ZÐ
YÐ
Y€€€r8   