§
    9²¸jÚ-  ã                   óì  — d Z ddlZddlmZ ddlmZ ddlmZ ddl	m
Z
 ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZmZmZmZmZmZ ddlmZ dZ G d„ dej        ¦  «        Z G d„ de¦  «        Z  G d„ de¦  «        Z! G d„ dej        ¦  «        Z" G d„ de¦  «        Z# G d„ de¦  «        Z$e G d„ de¦  «        ¦   «         Z% G d„ dee%¦  «        Z& G d„ d e¦  «        Z' G d!„ d"e¦  «        Z(g d#¢Z)dS )$zPyTorch Hubert model.é    Né   )Úinitialization)ÚACT2FN)Úis_deepspeed_zero3_enabled)ÚBaseModelOutput)ÚPreTrainedModel)Úauto_docstring)ÚOutputRecorderé   )ÚWav2Vec2EncoderÚWav2Vec2EncoderStableLayerNormÚWav2Vec2FeatureEncoderÚWav2Vec2ForCTCÚ!Wav2Vec2ForSequenceClassificationÚWav2Vec2ModelÚWav2Vec2SamePadLayeré   )ÚHubertConfigc                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚHubertPositionalConvEmbeddingc                 ó:  •— t          ¦   «                              ¦   «          t          j        |j        |j        |j        |j        dz  |j        ¬¦  «        | _        d | _        t          |dd¦  «        r t          j
        |j        ¦  «        | _        �nWt          j        j        }t          t          j        j        d¦  «        rt          j        j        j        }t          ¦   «         rëdd l}|j                             | j        j        d¬¦  «        5   || j        dd¬	¦  «        | _        d d d ¦  «         n# 1 swxY w Y   t          | j        d
¦  «        r-| j        j        j        j        }| j        j        j        j        }n| j        j        }| j        j        }|j                             | |¦  «         |j                             | |¦  «         n || j        dd¬	¦  «        | _        t3          |j        ¦  «        | _        t6          |j                 | _        d S )Nr   )Úkernel_sizeÚpaddingÚgroupsÚconv_pos_batch_normFÚweight_normr   ©Úmodifier_rankÚweight)ÚnameÚdimÚparametrizations)ÚsuperÚ__init__ÚnnÚConv1dÚhidden_sizeÚnum_conv_pos_embeddingsÚnum_conv_pos_embedding_groupsÚconvÚ
batch_normÚgetattrÚBatchNorm1dÚutilsr   Úhasattrr"   r   Ú	deepspeedÚzeroÚGatheredParametersr   Ú	original0Ú	original1Úweight_gÚweight_vÚregister_external_parameterÚHubertSamePadLayerr   r   Úfeat_extract_activationÚ
activation)ÚselfÚconfigr   r0   r5   r6   Ú	__class__s         €úy/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/hubert/modular_hubert.pyr$   z&HubertPositionalConvEmbedding.__init__*   s1  ø€ Ý‰Œ×ÒÑÔÐÝ”IØÔØÔØÔ6ØÔ2°aÑ7ØÔ7ð
ñ 
ô 
ˆŒ	ð ˆŒÝ�6Ð0°%Ñ8Ô8ð 	IÝ œn¨VÔ-?Ñ@Ô@ˆDŒO‰Oåœ(Ô.ˆKÝ•r”xÔ0°-Ñ@Ô@ð DÝ œhÔ7ÔC�å)Ñ+Ô+ð IØ Ð Ð Ð à”^×6Ò6°t´yÔ7GÐWXÐ6ÑYÔYð Mð MØ + ¨D¬I¸HÈ!Ð LÑ LÔ L�D”IðMð Mð Mñ Mô Mð Mð Mð Mð Mð Mð Møøøð Mð Mð Mð Må˜4œ9Ð&8Ñ9Ô9ð 2Ø#œyÔ9Ô@ÔJ�HØ#œyÔ9Ô@ÔJ�H�Hà#œyÔ1�HØ#œyÔ1�HØ”×:Ò:¸4ÀÑJÔJÐJØ”×:Ò:¸4ÀÑJÔJÐJÐJà'˜K¨¬	¸ÀaÐHÑHÔH�”	å)¨&Ô*HÑIÔIˆŒÝ  Ô!?Ô@ˆŒˆˆs   ÄD7Ä7D;Ä>D;c                 ó  — |                      dd¦  «        }| j        �|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      dd¦  «        }|S )Nr   r   )Ú	transposer+   r*   r   r:   ©r;   Úhidden_statess     r>   Úforwardz%HubertPositionalConvEmbedding.forwardO   s~   € Ø%×/Ò/°°1Ñ5Ô5ˆØŒ?Ð&Ø ŸOšO¨MÑ:Ô:ˆMØŸ	š	 -Ñ0Ô0ˆØŸš ]Ñ3Ô3ˆØŸš¨Ñ6Ô6ˆà%×/Ò/°°1Ñ5Ô5ˆØÐó    ©Ú__name__Ú
__module__Ú__qualname__r$   rC   Ú__classcell__©r=   s   @r>   r   r   )   sM   ø€ € € € € ð#Að #Að #Að #Að #AðJ	ð 	ð 	ð 	ð 	ð 	ð 	rD   r   c                   ó   — e Zd ZdS )r8   N©rF   rG   rH   © rD   r>   r8   r8   [   ó   € € € € € Ø€DrD   r8   c                   ó   — e Zd ZdS )ÚHubertFeatureEncoderNrL   rM   rD   r>   rP   rP   _   rN   rD   rP   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚHubertFeatureProjectionc                 óT  •— t          ¦   «                              ¦   «          |j        | _        | j        r+t          j        |j        d         |j        ¬¦  «        | _        t          j        |j        d         |j	        ¦  «        | _
        t          j        |j        ¦  «        | _        d S )Néÿÿÿÿ)Úeps)r#   r$   Úfeat_proj_layer_normr%   Ú	LayerNormÚconv_dimÚlayer_norm_epsÚ
layer_normÚLinearr'   Ú
projectionÚDropoutÚfeat_proj_dropoutÚdropout©r;   r<   r=   s     €r>   r$   z HubertFeatureProjection.__init__d   s…   ø€ Ý‰Œ×ÒÑÔÐØ$*Ô$?ˆÔ!ØÔ$ð 	[Ý œl¨6¬?¸2Ô+>ÀFÔDYÐZÑZÔZˆDŒOÝœ) F¤O°BÔ$7¸Ô9KÑLÔLˆŒÝ”z &Ô":Ñ;Ô;ˆŒˆˆrD   c                 ó’   — | j         r|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S )N)rV   rZ   r\   r_   rA   s     r>   rC   zHubertFeatureProjection.forwardl   sF   € àÔ$ð 	;Ø ŸOšO¨MÑ:Ô:ˆMØŸš¨Ñ6Ô6ˆØŸš ]Ñ3Ô3ˆØÐrD   rE   rJ   s   @r>   rR   rR   c   sG   ø€ € € € € ð<ð <ð <ð <ð <ðð ð ð ð ð ð rD   rR   c                   ó   — e Zd ZdS )ÚHubertEncoderNrL   rM   rD   r>   rc   rc   u   rN   rD   rc   c                   ó   — e Zd ZdS )ÚHubertEncoderStableLayerNormNrL   rM   rD   r>   re   re   y   rN   rD   re   c                   óÔ   ‡ — e Zd ZU eed<   dZdZdZddgZdZ	dZ
dZdZeeg eedd	¬
¦  «        dœZ ej        ¦   «         ˆ fd„¦   «         Zdej        ez  fd„Zdedej        fd„Zˆ xZS )ÚHubertPreTrainedModelr<   ÚhubertÚinput_valuesÚaudioÚHubertEncoderLayerÚParametrizedConv1dTr   Úencoder)ÚindexÚ
layer_name)rB   Ú
attentionsc                 óâ  •— t          ¦   «                              |¦  «         t          |t          j        ¦  «        �rt          ¦   «         rÑddl}t          |d¦  «        rjt          |d¦  «        rZ|j         	                    |j
        |j        gd¬¦  «        5  t          j        |j        ¦  «         ddd¦  «         n# 1 swxY w Y   nl|j         	                    |j        d¬¦  «        5  t          j        |j        ¦  «         ddd¦  «         n# 1 swxY w Y   nt          j        |j        ¦  «         |j        �t          j        |j        ¦  «         dS dS t          |t"          ¦  «        r-t          |d¦  «        rt          j        |j        ¦  «         dS dS t          |t(          ¦  «        r<t          |d¦  «        r.t          j        |j        d| j        j        d	z   z  ¦  «         dS dS dS )
zInitialize the weightsr   Nr6   r5   r   Úmasked_spec_embedÚlayer_weightsg      ð?r   )r#   Ú_init_weightsÚ
isinstancer%   r&   r   r0   r/   r1   r2   r6   r5   ÚinitÚkaiming_normal_r   ÚbiasÚzeros_ÚHubertModelÚuniform_rr   ÚHubertForSequenceClassificationÚ	constant_rs   r<   Únum_hidden_layers)r;   Úmoduler0   r=   s      €r>   rt   z#HubertPreTrainedModel._init_weights�   sv  ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�f�bœiÑ(Ô(ñ 	`Ý)Ñ+Ô+ð 
4Ø Ð Ð Ð å˜6 :Ñ.Ô.ð <µ7¸6À:Ñ3NÔ3Nð <Ø"œ×:Ò:¸F¼OÈVÌ_Ð;]ÐmnÐ:ÑoÔoð <ð <ÝÔ,¨V¬]Ñ;Ô;Ð;ð<ð <ð <ñ <ô <ð <ð <ð <ð <ð <ð <øøøð <ð <ð <ð <øð #œ×:Ò:¸6¼=ÐXYÐ:ÑZÔZð <ð <ÝÔ,¨V¬]Ñ;Ô;Ð;ð<ð <ð <ñ <ô <ð <ð <ð <ð <ð <ð <øøøð <ð <ð <ð <øõ Ô$ V¤]Ñ3Ô3Ð3àŒ{Ð&Ý”˜FœKÑ(Ô(Ð(Ð(Ð(ð 'Ð&å˜¥Ñ,Ô,ð 	`Ý�vÐ2Ñ3Ô3ð 8Ý”˜fÔ6Ñ7Ô7Ð7Ð7Ð7ð8ð 8å˜Õ ?Ñ@Ô@ð 	`Ý�v˜Ñ/Ô/ð `Ý”˜vÔ3°S¸D¼KÔ<YÐ\]Ñ<]Ñ5^Ñ_Ô_Ð_Ð_Ð_ð	`ð 	`ð`ð `s$   ÂB>Â>CÃCÃ+DÄDÄDÚinput_lengthsc                 óz   — d„ }t          | j        j        | j        j        ¦  «        D ]\  }} ||||¦  «        }Œ|S )zH
        Computes the output length of the convolutional layers
        c                 ó<   — t          j        | |z
  |d¬¦  «        dz   S )NÚfloor)Úrounding_moder   )ÚtorchÚdiv)Úinput_lengthr   Ústrides      r>   Ú_conv_out_lengthzPHubertPreTrainedModel._get_feat_extract_output_lengths.<locals>._conv_out_length¬   s&   € õ ”9˜\¨KÑ7¸ÈwÐWÑWÔWÐZ[Ñ[Ð[rD   )Úzipr<   Úconv_kernelÚconv_stride)r;   r€   r‰   r   rˆ   s        r>   Ú _get_feat_extract_output_lengthsz6HubertPreTrainedModel._get_feat_extract_output_lengths§   s\   € ð
	\ð 	\ð 	\õ
 $' t¤{Ô'>ÀÄÔ@WÑ#XÔ#Xð 	Qð 	QÑˆK˜Ø,Ð,¨]¸KÈÑPÔPˆMˆMàÐrD   Úfeature_vector_lengthÚattention_maskc                 óê  — |                       |                     d¦  «        ¦  «                             t          j        ¦  «        }|j        d         }t          j        ||f|j        |j        ¬¦  «        }d|t          j	        |j        d         |j        ¬¦  «        |dz
  f<   | 
                    dg¦  «                             d¦  «         
                    dg¦  «                             ¦   «         }|S )NrT   r   )ÚdtypeÚdevicer   )r’   )r�   ÚsumÚtor…   ÚlongÚshapeÚzerosr‘   r’   ÚarangeÚflipÚcumsumÚbool)r;   rŽ   r�   Úoutput_lengthsÚ
batch_sizes        r>   Ú"_get_feature_vector_attention_maskz8HubertPreTrainedModel._get_feature_vector_attention_mask¶   sæ   € Ø×>Ò>¸~×?QÒ?QÐRTÑ?UÔ?UÑVÔV×YÒYÕZ_ÔZdÑeÔeˆØ#Ô)¨!Ô,ˆ
åœØÐ.Ð/°~Ô7KÐTbÔTið
ñ 
ô 
ˆð uvˆ�œ ^Ô%9¸!Ô%<À^ÔEZÐ[Ñ[Ô[Ð]kÐnoÑ]oÐpÑqØ'×,Ò,¨b¨TÑ2Ô2×9Ò9¸"Ñ=Ô=×BÒBÀBÀ4ÑHÔH×MÒMÑOÔOˆØÐrD   )rF   rG   rH   r   Ú__annotations__Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚ_no_split_modulesÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnrk   Ú!HubertEncoderLayerStableLayerNormr
   ÚHubertAttentionÚ_can_record_outputsr…   Úno_gradrt   Ú
LongTensorÚintr�   rž   rI   rJ   s   @r>   rg   rg   }   s  ø€ € € € € € àÐÐÑØ ÐØ$€OØÐØ-Ð/CÐDÐØ&*Ð#ØÐØ€NØÐà,Ð.OÐPØ$�n _¸AÈ)ÐTÑTÔTðð Ðð
 €U„]�_„_ð`ð `ð `ð `ñ „_ð`ð2¸eÔ>NÐQTÑ>Tð ð ð ð ð
Èð 
Ð]bÔ]mð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
rD   rg   c                   ó¢   ‡ — e Zd Zdefˆ fd„Zd„ Z	 	 	 	 	 ddej        dz  dej        dz  dej        dz  de	dz  d	e	dz  d
e	dz  de
ez  fd„Zˆ xZS )rz   r<   c                 óä  •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        t          |¦  «        | _        |j        dk    s|j        dk    rBt          j
        t          j        |j        ¦  «                             ¦   «         ¦  «        | _        |j        rt#          |¦  «        | _        nt'          |¦  «        | _        |                      ¦   «          | `d S )Ng        )r#   r$   r<   rP   Úfeature_extractorrR   Úfeature_projectionÚmask_time_probÚmask_feature_probr%   Ú	Parameterr…   ÚTensorr'   r{   rr   Údo_stable_layer_normre   rm   rc   Ú	post_initÚadapterr`   s     €r>   r$   zHubertModel.__init__Ä   sÎ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒÝ!5°fÑ!=Ô!=ˆÔÝ"9¸&Ñ"AÔ"AˆÔàÔ  3Ò&Ð&¨&Ô*BÀSÒ*HÐ*HÝ%'¤\µ%´,¸vÔ?QÑ2RÔ2R×2[Ò2[Ñ2]Ô2]Ñ%^Ô%^ˆDÔ"àÔ&ð 	1Ý7¸Ñ?Ô?ˆDŒLˆLå(¨Ñ0Ô0ˆDŒLð 	�ŠÑÔÐàˆLˆLˆLrD   c                 ó    — t          d¦  «        ‚)NzNot needed for Hubert)ÚAttributeError)r;   s    r>   Úfreeze_feature_encoderz"HubertModel.freeze_feature_encoder×   s   € ÝÐ4Ñ5Ô5Ð5rD   Nri   r�   Úmask_time_indicesÚoutput_attentionsÚoutput_hidden_statesÚreturn_dictÚreturnc                 óò  — |�|n| j         j        }|�|n| j         j        }|�|n| j         j        }|                      |¦  «        }|                     dd¦  «        }|�!|                      |j        d         |¦  «        }|                      |¦  «        }	|  	                    |	|¬¦  «        }	|  
                    |	||||¬¦  «        }
|
d         }	|s|	f|
dd…         z   S t          |	|
j        |
j        ¬¦  «        S )a1  
        mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict
            masked extracted features in *config.proj_codevector_dim* space.

        Example:

        ```python
        >>> from transformers import AutoProcessor, HubertModel
        >>> from datasets import load_dataset

        >>> processor = AutoProcessor.from_pretrained("facebook/hubert-large-ls960-ft")
        >>> model = HubertModel.from_pretrained("facebook/hubert-large-ls960-ft")


        >>> def map_to_array(example):
        ...     example["speech"] = example["audio"]["array"]
        ...     return example


        >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        >>> ds = ds.map(map_to_array)

        >>> input_values = processor(ds["speech"][0], return_tensors="pt").input_values  # Batch size 1
        >>> hidden_states = model(input_values).last_hidden_state
        ```Nr   r   )r¼   )r�   r½   r¾   r¿   r   )Úlast_hidden_staterB   rp   )r<   r½   r¾   r¿   r°   r@   rž   r–   r±   Ú_mask_hidden_statesrm   r   rB   rp   )r;   ri   r�   r¼   r½   r¾   r¿   ÚkwargsÚextract_featuresrB   Úencoder_outputss              r>   rC   zHubertModel.forwardÚ   s@  € ðH 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆà×1Ò1°,Ñ?Ô?ÐØ+×5Ò5°a¸Ñ;Ô;ÐàÐ%à!×DÒDÐEUÔE[Ð\]ÔE^Ð`nÑoÔoˆNà×/Ò/Ð0@ÑAÔAˆØ×0Ò0°ÐRcÐ0ÑdÔdˆàŸ,š,ØØ)Ø/Ø!5Ø#ð 'ñ 
ô 
ˆð (¨Ô*ˆàð 	:Ø!Ð# o°a°b°bÔ&9Ñ9Ð9åØ+Ø)Ô7Ø&Ô1ð
ñ 
ô 
ð 	
rD   )NNNNN)rF   rG   rH   r   r$   r»   r…   rµ   ÚFloatTensorr›   Útupler   rC   rI   rJ   s   @r>   rz   rz   Ã   sø   ø€ € € € € ð˜|ð ð ð ð ð ð ð&6ð 6ð 6ð /3Ø6:Ø)-Ø,0Ø#'ðE
ð E
à”l TÑ)ðE
ð œ tÑ+ðE
ð !Ô,¨tÑ3ð	E
ð
   $™;ðE
ð # T™kðE
ð ˜D‘[ðE
ð 
�Ñ	 ðE
ð E
ð E
ð E
ð E
ð E
ð E
ð E
rD   rz   c                   ó   — e Zd ZdS )ÚHubertForCTCNrL   rM   rD   r>   rÊ   rÊ   "  rN   rD   rÊ   c                   ó   — e Zd ZdS )r|   NrL   rM   rD   r>   r|   r|   &  rN   rD   r|   )rÊ   r|   rz   rg   )*Ú__doc__r…   Útorch.nnr%   Ú r   rv   Úactivationsr   Úintegrations.deepspeedr   Úmodeling_outputsr   Úmodeling_utilsr   r.   r	   Úutils.output_capturingr
   Úwav2vec2.modeling_wav2vec2r   r   r   r   r   r   r   Úconfiguration_hubertr   Ú_HIDDEN_STATES_START_POSITIONÚModuler   r8   rP   rR   rc   re   rg   rz   rÊ   r|   Ú__all__rM   rD   r>   ú<module>rÙ      sü  ðð Ð à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø @Ð @Ð @Ð @Ð @Ð @Ø /Ð /Ð /Ð /Ð /Ð /Ø -Ð -Ð -Ð -Ð -Ð -Ø #Ð #Ð #Ð #Ð #Ð #Ø 4Ð 4Ð 4Ð 4Ð 4Ð 4ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð /Ð .Ð .Ð .Ð .Ð .ð !"Ð ð/ð /ð /ð /ð / B¤Iñ /ô /ð /ðd	ð 	ð 	ð 	ð 	Ð-ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð1ñ 	ô 	ð 	ðð ð ð ð ˜bœiñ ô ð ð$	ð 	ð 	ð 	ð 	�Oñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð#Añ 	ô 	ð 	ð ðBð Bð Bð Bð B˜Oñ Bô Bñ „ðBðJ\
ð \
ð \
ð \
ð \
�-Ð!6ñ \
ô \
ð \
ð~	ð 	ð 	ð 	ð 	�>ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð&Gñ 	ô 	ð 	ð fÐ
eÐ
e€€€rD   