§
    9²¸jù¥  ã                   óè  — d dl mZ d dlmZ d dlZd dlmZ ddlmZ ddl	m
Z
mZmZ ddlmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZmZmZmZmZ ddlmZmZ ddlm Z m!Z! ddl"m#Z# ddl$m%Z%m&Z&m'Z' ddl(m)Z) ddl*m+Z+m,Z, ddl-m.Z.m/Z/ ddl0m1Z1  e&d¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z2 G d„ dej3        ¦  «        Z4 G d„ dej3        ¦  «        Z5 G d„ dej3        ¦  «        Z6dej7        d e8d!ej7        fd"„Z9	 dEd$ej3        d%ej7        d&ej7        d'ej7        d(ej7        dz  d)e:d*e:d+e#e%         fd,„Z;d-„ Z<dFd.„Z= G d/„ d0ej3        ¦  «        Z> G d1„ d2e¦  «        Z? G d3„ d4e¦  «        Z@e& G d5„ d6e!¦  «        ¦   «         ZA G d7„ d8eA¦  «        ZBe& G d9„ d:eA¦  «        ¦   «         ZCe& G d;„ d<eA¦  «        ¦   «         ZDd=ej7        d>e8d?e8fd@„ZE e&dA¬¦  «         G dB„ dCeAe¦  «        ¦   «         ZFg dD¢ZGdS )Gé    )ÚCallable)Ú	dataclassNé   )ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚBaseModelOutputWithPastÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚMoonshineConfigz™
    Extends [~modeling_outputs.BaseModelOutput] to include the output attention mask since sequence length is not preserved in the model's forward.
    )Úcustom_introc                   ó2   — e Zd ZU dZdZej        dz  ed<   dS )ÚMoonshineEncoderModelOutputa–  
    attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
        Mask to avoid performing attention on padding token indices after sequence compression. Returned because the
        sequence length may differ from the input sequence length. Mask values selected in `[0, 1]`:

        - 1 for tokens that are **not masked**,
        - 0 for tokens that are **masked**.
    NÚattention_mask)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r&   ÚtorchÚTensorÚ__annotations__© ó    ú€/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/moonshine/modeling_moonshine.pyr%   r%   2   s5   € € € € € € ðð ð +/€N�E”L 4Ñ'Ð.Ð.Ñ.Ð.Ð.r/   r%   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚMoonshineEncoderMLPc                 ó
  •— t          ¦   «                              ¦   «          || _        t          |         | _        t          j        |j        |j        ¦  «        | _	        t          j        |j        |j        ¦  «        | _
        d S ©N©ÚsuperÚ__init__Úconfigr   Úactivation_fnÚnnÚLinearÚhidden_sizeÚintermediate_sizeÚfc1Úfc2©Úselfr8   Ú
hidden_actÚ	__class__s      €r0   r7   zMoonshineEncoderMLP.__init__F   sc   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ# JÔ/ˆÔÝ”9˜VÔ/°Ô1IÑJÔJˆŒÝ”9˜VÔ5°vÔ7IÑJÔJˆŒˆˆr/   Úhidden_statesÚreturnc                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r4   )r>   r9   r?   )rA   rD   s     r0   ÚforwardzMoonshineEncoderMLP.forwardM   s=   € ØŸš Ñ/Ô/ˆØ×*Ò*¨=Ñ9Ô9ˆØŸš Ñ/Ô/ˆØÐr/   ©r'   r(   r)   r7   r+   r,   rG   Ú__classcell__©rC   s   @r0   r2   r2   E   sc   ø€ € € € € ðKð Kð Kð Kð Kð U¤\ð °e´lð ð ð ð ð ð ð ð r/   r2   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚMoonshineDecoderMLPc                 ó  •— t          ¦   «                              ¦   «          || _        t          |         | _        t          j        |j        |j        dz  ¦  «        | _	        t          j        |j        |j        ¦  «        | _
        d S )Né   r5   r@   s      €r0   r7   zMoonshineDecoderMLP.__init__U   sh   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ# JÔ/ˆÔÝ”9˜VÔ/°Ô1IÈAÑ1MÑNÔNˆŒÝ”9˜VÔ5°vÔ7IÑJÔJˆŒˆˆr/   rD   rE   c                 ó¾   — |                       |¦  «        }|                     dd¬¦  «        \  }}|                      |¦  «        |z  }|                      |¦  «        }|S )NrN   éÿÿÿÿ©Údim)r>   Úchunkr9   r?   )rA   rD   Úgates      r0   rG   zMoonshineDecoderMLP.forward\   s_   € ØŸš Ñ/Ô/ˆØ+×1Ò1°!¸Ð1Ñ<Ô<Ñˆ�tØ×*Ò*¨4Ñ0Ô0°=Ñ@ˆØŸš Ñ/Ô/ˆØÐr/   rH   rJ   s   @r0   rL   rL   T   sc   ø€ € € € € ðKð Kð Kð Kð Kð U¤\ð °e´lð ð ð ð ð ð ð ð r/   rL   c            	       óî   ‡ — e Zd Z edd¬¦  «        d
defˆ fd„¦   «         Ze edd¬¦  «        d
dedeej	        e
f         fd„¦   «         ¦   «         Z ej        ¦   «         ed	„ ¦   «         ¦   «         Zˆ xZS )ÚMoonshineRotaryEmbeddingÚdevicez5.18)ÚversionNr8   c                 ó¾  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        | j        j        d         | _        | j        }| j        dk    rt          | j                 } || j        |¦  «        \  }| _
        t          j        |d¬¦  «        | _        t          j        |                     ¦   «         d¬¦  «        | _        d S )NÚ	rope_typeÚdefaultF)Ú
persistent)r6   r7   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr8   Úrope_parametersrZ   Úcompute_default_rope_parametersr   Úattention_scalingr:   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)rA   r8   rW   Úrope_init_fnrd   rC   s        €r0   r7   z!MoonshineRotaryEmbedding.__init__e   sÀ   ø€ å‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!àˆŒàœÔ4°[ÔAˆŒØ!%Ô!EˆØŒ>˜YÒ&Ð&Ý.¨t¬~Ô>ˆLØ+7¨<¸¼ÀVÑ+LÔ+LÑ(ˆ�$Ô(åœ	 (°uÐ=Ñ=Ô=ˆŒÝ!#¤¨8¯>ª>Ñ+;Ô+;ÈÐ!NÑ!NÔ!NˆÔÐÐr/   rE   c                 ó>  — | j         d         }| j                              dd¦  «        }t          | dd¦  «        p| j        | j        z  }t          ||z  ¦  «        }d}d|t          j        d|dt          j        ¬¦  «        |z  z  z  }| 	                    |¦  «        |fS )	a¾  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚpartial_rotary_factorg      ð?Úhead_dimNr   rN   ©Údtype)
r`   ÚgetÚgetattrr<   Únum_attention_headsÚintr+   ÚarangeÚfloatÚto)	r8   rW   ÚkwargsÚbaserj   rk   rR   Úattention_factorrd   s	            r0   ra   z8MoonshineRotaryEmbedding.compute_default_rope_parametersv   s¨   € ð Ô% lÔ3ˆØ &Ô 6× :Ò :Ð;RÐTWÑ XÔ XÐÝ˜6 :¨tÑ4Ô4Ðh¸Ô8JÈfÔNhÑ8hˆÝ�(Ð2Ñ2Ñ3Ô3ˆàÐà˜$¥5¤<°°3¸ÅÄÐ#MÑ#MÔ#MÐPSÑ#SÑTÑUˆØ�{Š{˜6Ñ"Ô"Ð$4Ð4Ð4r/   c                 óú  — | j         d d d …d f                              |j        d         dd¦  «                             t          j        |j        ¬¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  ||z                       dd	¦  «        }t	          j        ||fd¬
¦  «        }|                     ¦   «         | j        z  }|                     ¦   «         | j        z  }	d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬¦  «        |	                     |j        ¬¦  «        fS )Nr   rP   r!   )rm   rW   ÚmpsÚcpuF)Údevice_typeÚenabledrN   rQ   rl   )rd   ÚexpandÚshapert   r+   rs   rW   Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosrb   Úsinrm   )
rA   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedr{   ÚfreqsÚembr„   r…   s
             r0   rG   z MoonshineRotaryEmbedding.forwardŒ   s›  € ð ŒM˜$    4˜-Ô(×/Ò/°Ô0BÀ1Ô0EÀrÈ1ÑMÔM×PÒPÕW\ÔWbÐklÔksÐPÑtÔtð 	ð !-¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆå¨¸UÐCÑCÔCð 	5ð 	5Ø&Ð)>Ñ>×IÒIÈ!ÈQÑOÔOˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCð		5ð 	5ð 	5ñ 	5ô 	5ð 	5ð 	5ð 	5ð 	5ð 	5ð 	5øøøð 	5ð 	5ð 	5ð 	5ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   ÃA*D<Ä<E ÅE r4   )r'   r(   r)   r   r"   r7   ÚstaticmethodÚtupler+   r,   rs   ra   Úno_gradr   rG   rI   rJ   s   @r0   rV   rV   d   sö   ø€ € € € € Ø€_�X vÐ.Ñ.Ô.ðOð O˜ð Oð Oð Oð Oð Oñ /Ô.ðOð  Ø€_�X vÐ.Ñ.Ô.ð5ð 5°ð 5Ð[`ÐafÔamÐotÐatÔ[uð 5ð 5ð 5ñ /Ô.ñ „\ð5ð( €U„]�_„_Øð<ð <ñ Ôñ „_ð<ð <ð <ð <ð <r/   rV   rD   Ún_reprE   c                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r!   N)r~   r}   Úreshape)rD   r�   ÚbatchÚnum_key_value_headsÚslenrk   s         r0   Ú	repeat_kvr•   Ÿ   s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTr/   ç        ÚmoduleÚqueryÚkeyÚvaluer&   ÚscalingÚdropoutru   c                 ó  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        |z  }
|�|
|z   }
t
          j                             |
dt          j        ¬¦  «         	                    |j
        ¦  «        }
t
          j                             |
|| j        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «                             ¦   «         }||
fS )NrN   r   rP   )rR   rm   )ÚpÚtrainingr!   )r•   Únum_key_value_groupsr+   Úmatmulr‚   r:   Ú
functionalÚsoftmaxÚfloat32rt   rm   rœ   rŸ   Ú
contiguous)r—   r˜   r™   rš   r&   r›   rœ   ru   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r0   Úeager_attention_forwardrª   «   sé   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r/   c                 óŽ   — | dddd…f         }| dddd…f         }t          j        | |fd¬¦  «                             d¦  «        S )	z*Rotates half the hidden dims of the input..r   NrN   r!   rP   rQ   éþÿÿÿ)r+   ÚstackÚflatten)r†   Úx1Úx2s      r0   Úrotate_halfr±   Ä   sQ   € à	
ˆ3���1�ˆ9Œ€BØ	
ˆ3���1�ˆ9Œ€BÝŒ;˜˜˜R�y bÐ)Ñ)Ô)×1Ò1°"Ñ5Ô5Ð5r/   c                 óT  — |                      |¦  «        }|                      |¦  «        }|dd|j        d         dz  …f                              dd¬¦  «        }|dd|j        d         dz  …f                              dd¬¦  «        }|j        d         }| dd|…f         | d|d…f         }}|dd|…f         |d|d…f         }	}||z  t          |¦  «        |z  z   }
||z  t          |¦  «        |z  z   }t	          j        |
|gd¬¦  «        }
t	          j        ||	gd¬¦  «        }|
|fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    .NrP   rN   rQ   )Ú	unsqueezer~   Úrepeat_interleaver±   r+   rƒ   )ÚqÚkr„   r…   Úunsqueeze_dimÚ
rotary_dimÚq_rotÚq_passÚk_rotÚk_passÚq_embedÚk_embeds               r0   Úapply_rotary_pos_embr¿   Ë   s\  € ð$ �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€Cð ˆcÐ'�S”Y˜r”] aÑ'Ð'Ð'Ô
(×
:Ò
:¸1À"Ð
:Ñ
EÔ
E€CØ
ˆcÐ'�S”Y˜r”] aÑ'Ð'Ð'Ô
(×
:Ò
:¸1À"Ð
:Ñ
EÔ
E€Cð ”˜2”€JØ�c˜;˜J˜;Ð&Ô'¨¨3°
°°Ð+;Ô)<ˆ6€EØ�c˜;˜J˜;Ð&Ô'¨¨3°
°°Ð+;Ô)<ˆ6€Eð �s‰{�{¨5Ñ1Ô1°CÑ7Ñ8€GØ�s‰{�{¨5Ñ1Ô1°CÑ7Ñ8€Gõ Œi˜ &Ð)¨rÐ2Ñ2Ô2€GÝŒi˜ &Ð)¨rÐ2Ñ2Ô2€GØ�GÐÐr/   c                   ó  ‡ — e Zd ZdZdededededef
ˆ fd„Z	 	 	 	 dd	ej	        d
e
ej	        ej	        f         dz  dej	        dz  dedz  dej	        dz  dee         de
ej	        ej	        dz  e
ej	                 dz  f         fd„Zˆ xZS )ÚMoonshineAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr8   Ú	layer_idxÚ	is_causalrp   r“   c                 óZ  •— t          ¦   «                              ¦   «          |                     ||dœ¦  «         || _        || _        t          |d|j        |j        z  ¦  «        | _        |j        |j	        z  | _
        | j        dz  | _        |j        | _        || _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        |j	        | j        z  |j        ¬¦  «        | _        t          j        |j        |j	        | j        z  |j        ¬¦  «        | _        t          j        |j        | j        z  |j        d¬¦  «        | _        | j        j        �0| j        j        }|| j        |z   dz
  |z  z  }|| j        z
  | _        d S d| _        d S )N)rp   r“   rk   g      à¿©ÚbiasFr!   r   )r6   r7   Úupdater8   rÂ   ro   r<   rp   rk   r“   r    r›   Úattention_dropoutrÃ   r:   r;   Úattention_biasÚq_projÚk_projÚv_projÚo_projÚpad_head_dim_to_multiple_ofÚhead_dim_padding)	rA   r8   rÂ   rÃ   rp   r“   Útarget_multipleÚtarget_head_dimrC   s	           €r0   r7   zMoonshineAttention.__init__ö   s©  ø€ õ 	‰Œ×ÒÑÔÐØ�ŠÐ.AÐZmÐnÐnÑoÔoÐoØˆŒØ"ˆŒÝ ¨
°FÔ4FÈ&ÔJdÑ4dÑeÔeˆŒØ$*Ô$>À&ÔB\Ñ$\ˆÔ!Ø”} dÑ*ˆŒØ!'Ô!9ˆÔØ"ˆŒå”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”i Ô :¸T¼]Ñ JÈFÔL^ÐejÐkÑkÔkˆŒð Œ;Ô2Ð>Ø"œkÔEˆOØ-°$´-À/Ñ2QÐTUÑ2UÐZiÑ1iÑjˆOØ$3°d´mÑ$CˆDÔ!Ð!Ð!à$%ˆDÔ!Ð!Ð!r/   NrD   Úposition_embeddingsr&   Úpast_key_valuesÚkey_value_statesru   rE   c                 óN  — |j         d d…         \  }}|                      |¦  «                             ||| j        j        | j        ¦  «                             dd¦  «        }	|d u}
|�?|j                             | j	        ¦  «        }|
rd|j        | j	        <   |j
        }n|j        }|�|n|}|
r3|r1|r/|j        | j	                 j        }|j        | j	                 j        }n¿|                      |¦  «                             |d| j        j        | j        ¦  «                             dd¦  «        }|                      |¦  «                             |d| j        j        | j        ¦  «                             dd¦  «        }|
r!|�|                     ||| j	        ¦  «        \  }}|
s;|\  }}t%          |	|||¦  «        \  }	}|�|                     ||| j	        ¦  «        \  }}t'          j        | j        j        t,          ¦  «        }| j        o	|d u o|dk    }| j        dk    r„t2          j        j                             |	d| j        f¦  «        }	t2          j        j                             |d| j        f¦  «        }t2          j        j                             |d| j        f¦  «        } || |	|||f| j        sdn| j        | j        |dœ|¤Ž\  }}| j        dk    r|dd | j         …f         }|                      ||d¦  «         !                    ¦   «         }|  "                    |¦  «        }||fS )	NrP   r!   rN   Tr   r–   )rœ   r›   rÃ   .)#r~   rÊ   Úviewr8   r“   rk   r‚   Ú
is_updatedrn   rÂ   Úcross_attention_cacheÚself_attention_cacheÚlayersÚkeysÚvaluesrË   rÌ   rÇ   r¿   r   Úget_interfaceÚ_attn_implementationrª   rÃ   rÏ   r+   r:   r¢   ÚpadrŸ   rÈ   r›   r‘   r¥   rÍ   )rA   rD   rÒ   r&   rÓ   rÔ   ru   ÚbszÚq_lenÚquery_statesÚis_cross_attentionr×   Úcurrent_statesr¦   r§   r„   r…   Úattention_interfacerÃ   r©   r¨   s                        r0   rG   zMoonshineAttention.forward  sm  € ð #Ô(¨¨"¨Ô-‰
ˆˆUð �KŠK˜Ñ&Ô&×+Ò+¨C°¸¼Ô8WÐY]ÔYfÑgÔg×qÒqÐrsÐuvÑwÔwð 	ð .°TÐ9ÐØÐ&Ø(Ô3×7Ò7¸¼ÑGÔGˆJØ!ð Gà=A�Ô*¨4¬>Ñ:Ø"1Ô"G��à"1Ô"F�ð .>Ð-IÐ)Ð)È}ˆØð 	l /ð 	l°jð 	lØ(Ô/°´Ô?ÔDˆJØ*Ô1°$´.ÔAÔHˆLˆLð —’˜NÑ+Ô+ß’�c˜2˜tœ{Ô>ÀÄÑNÔNß’˜1˜a‘”ð ð —’˜NÑ+Ô+ß’�c˜2˜tœ{Ô>ÀÄÑNÔNß’˜1˜a‘”ð ð
 "ð l oÐ&AØ+:×+AÒ+AÀ*ÈlÐ\`Ô\jÑ+kÔ+kÑ(�
˜Là!ð 	lØ*‰HˆC�Ý';¸LÈ*ÐVYÐ[^Ñ'_Ô'_Ñ$ˆL˜*àÐ*Ø+:×+AÒ+AÀ*ÈlÐ\`Ô\jÑ+kÔ+kÑ(�
˜Lå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð ”NÐK ~¸Ð'=ÐKÀ%È!Â)ˆ	àÔ  1Ò$Ð$Ý œ8Ô.×2Ò2°<À!ÀTÔEZÐA[Ñ\Ô\ˆLÝœÔ,×0Ò0°¸aÀÔAVÐ=WÑXÔXˆJÝ œ8Ô.×2Ò2°<À!ÀTÔEZÐA[Ñ\Ô\ˆLà$7Ð$7ØØØØØð
%
ð  $œ}ÐH�C�C°$Ô2HØ”LØð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð Ô  1Ò$Ð$Ø% cÐ+C¨dÔ.CÐ-CÐ+CÐ&CÔDˆKà!×)Ò)¨#¨u°bÑ9Ô9×DÒDÑFÔFˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r/   )NNNN)r'   r(   r)   r*   r"   rq   Úboolr7   r+   r,   r�   r   r   r   rG   rI   rJ   s   @r0   rÁ   rÁ   ó   s?  ø€ € € € € ØGÐGð#&àð#&ð ð#&ð ð	#&ð
 !ð#&ð !ð#&ð #&ð #&ð #&ð #&ð #&ðP IMØ.2Ø(,Ø04ðO)ð O)à”|ðO)ð # 5¤<°´Ð#=Ô>ÀÑEðO)ð œ tÑ+ð	O)ð
  ™ðO)ð  œ,¨Ñ-ðO)ð Ð-Ô.ðO)ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	MðO)ð O)ð O)ð O)ð O)ð O)ð O)ð O)r/   rÁ   c                   óÒ   ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 ddej        dej        dz  dej        dz  d	e	dz  d
e
dz  deej        ej        f         dz  dee         dej        fd„Zˆ xZS )ÚMoonshineEncoderLayerr8   rÂ   c                 óZ  •— t          ¦   «                              ¦   «          |j        | _        t          ||d|j        |j        ¬¦  «        | _        t          ||j        ¦  «        | _	        t          j        |j        d¬¦  «        | _        t          j        |j        d¬¦  «        | _        d S )NF©r8   rÂ   rÃ   rp   r“   rÅ   )r6   r7   r<   rÁ   Úencoder_num_attention_headsÚencoder_num_key_value_headsÚ	self_attnr2   Úencoder_hidden_actÚmlpr:   Ú	LayerNormÚinput_layernormÚpost_attention_layernorm©rA   r8   rÂ   rC   s      €r0   r7   zMoonshineEncoderLayer.__init__n  sŸ   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔå+ØØØØ &Ô BØ &Ô Bð
ñ 
ô 
ˆŒõ ' v¨vÔ/HÑIÔIˆŒÝ!œ|¨FÔ,>ÀUÐKÑKÔKˆÔÝ(*¬°VÔ5GÈeÐ(TÑ(TÔ(TˆÔ%Ð%Ð%r/   NFrD   r&   r‡   rÓ   Ú	use_cacherÒ   ru   rE   c           
      óÎ   — |}|                       |¦  «        } | j        d||||||dœ|¤Ž\  }}	||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|S )N©rD   r&   r‡   rÓ   rô   rÒ   r.   )rñ   rí   rò   rï   )
rA   rD   r&   r‡   rÓ   rô   rÒ   ru   ÚresidualÚ_s
             r0   rG   zMoonshineEncoderLayer.forward~  s¡   € ð !ˆØ×,Ò,¨]Ñ;Ô;ˆà)˜4œ>ð 
Ø'Ø)Ø%Ø+ØØ 3ð
ð 
ð ð
ð 
Ñˆ�qð ! =Ñ0ˆð !ˆØ×5Ò5°mÑDÔDˆØŸš Ñ/Ô/ˆØ  =Ñ0ˆØÐr/   )NNNFN)r'   r(   r)   r"   rq   r7   r+   r,   Ú
LongTensorr   ræ   r�   r   r   rG   rI   rJ   s   @r0   rè   rè   m  sÿ   ø€ € € € € ðU˜ð U¸3ð Uð Uð Uð Uð Uð Uð& /3Ø04Ø(,Ø!&ØHLðð à”|ðð œ tÑ+ðð Ô&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð # 5¤<°´Ð#=Ô>ÀÑEðð Ð+Ô,ðð 
Œðð ð ð ð ð ð ð r/   rè   c                   ó†  ‡ — e Zd Zddededz  fˆ fd„Z	 	 	 	 	 	 	 	 	 ddej        dej        dz  dej        dz  d	ej        dz  d
ej        dz  dej        dz  de	dz  de
dz  deej        ej        f         dz  deej        ej        f         dz  dee         deej        eej        ej        f         dz  f         fd„Zˆ xZS )ÚMoonshineDecoderLayerNr8   rÂ   c                 óà  •— t          ¦   «                              ¦   «          |j        | _        t          ||d|j        |j        ¬¦  «        | _        t          ||d|j        |j        ¬¦  «        | _        t          ||j	        ¦  «        | _
        t          j        |j        d¬¦  «        | _        t          j        |j        d¬¦  «        | _        t          j        |j        d¬¦  «        | _        d S )NTrê   FrÅ   )r6   r7   r<   rÁ   rp   r“   rí   Úencoder_attnrL   rB   rï   r:   rð   rñ   rò   Úfinal_layernormró   s      €r0   r7   zMoonshineDecoderLayer.__init__Ÿ  sç   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔå+ØØØØ &Ô :Ø &Ô :ð
ñ 
ô 
ˆŒõ /ØØØØ &Ô :Ø &Ô :ð
ñ 
ô 
ˆÔõ ' v¨vÔ/@ÑAÔAˆŒÝ!œ|¨FÔ,>ÀUÐKÑKÔKˆÔÝ(*¬°VÔ5GÈeÐ(TÑ(TÔ(TˆÔ%Ý!œ|¨FÔ,>ÀUÐKÑKÔKˆÔÐÐr/   FrD   r&   Úencoder_hidden_statesÚencoder_attention_maskr‡   Úencoder_position_idsrÓ   rô   rÒ   Úencoder_position_embeddingsru   rE   c           
      óD  — |}|                       |¦  «        } | j        d||||||	dœ|¤Ž\  }}||z   }|�9|}|                      |¦  «        }|                      |||||¬¦  «        \  }}||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|S )Nrö   )rD   rÔ   r&   rÓ   rô   r.   )rñ   rí   rò   rý   rþ   rï   )rA   rD   r&   rÿ   r   r‡   r  rÓ   rô   rÒ   r  ru   r÷   rø   s                 r0   rG   zMoonshineDecoderLayer.forward·  s÷   € ð !ˆØ×,Ò,¨]Ñ;Ô;ˆà)˜4œ>ð 
Ø'Ø)Ø%Ø+ØØ 3ð
ð 
ð ð
ð 
Ñˆ�qð ! =Ñ0ˆà Ð,Ø$ˆHØ ×9Ò9¸-ÑHÔHˆMØ#×0Ò0Ø+Ø!6Ø5Ø /Ø#ð  1ñ  ô  ÑˆM˜1ð % }Ñ4ˆMà ˆØ×,Ò,¨]Ñ;Ô;ˆØŸš Ñ/Ô/ˆØ  =Ñ0ˆØÐr/   r4   )	NNNNNNFNN)r'   r(   r)   r"   rq   r7   r+   r,   rù   r   ræ   r�   r   r   ÚFloatTensorrG   rI   rJ   s   @r0   rû   rû   ž  s‡  ø€ € € € € ðLð L˜ð L¸3À¹:ð Lð Lð Lð Lð Lð Lð6 /3Ø59Ø6:Ø04Ø8<Ø(,Ø!&ØHLØPTð,ð ,à”|ð,ð œ tÑ+ð,ð  %œ|¨dÑ2ð	,ð
 !&¤¨tÑ 3ð,ð Ô&¨Ñ-ð,ð $Ô.°Ñ5ð,ð  ™ð,ð ˜$‘;ð,ð # 5¤<°´Ð#=Ô>ÀÑEð,ð &+¨5¬<¸¼Ð+EÔ%FÈÑ%Mð,ð Ð+Ô,ð,ð 
ˆuÔ  %¨Ô(9¸5Ô;LÐ(LÔ"MÐPTÑ"TÐTÔ	Uð,ð ,ð ,ð ,ð ,ð ,ð ,ð ,r/   rû   c                   óT   — e Zd ZU eed<   dZdZdZdZddgZ	dZ
dZdZdej        fd	„Zd
S )ÚMoonshinePreTrainedModelr8   ÚmodelÚinput_valuesÚaudioTrè   rû   Úinput_lengthsc                 ó–   — t          |dz
  dz  dz   ¦  «        }t          |dz
  dz  dz   ¦  «        }t          |dz
  dz  dz   ¦  «        }|S )zH
        Computes the output length of the convolutional layers
        é   é@   r!   é   r   rN   )rq   )rA   r
  Úoutput_conv1_lengthÚoutput_conv2_lengthÚoutput_conv3_lengths        r0   Ú _get_feat_extract_output_lengthsz9MoonshinePreTrainedModel._get_feat_extract_output_lengthsô  sc   € õ " =°3Ñ#6¸"Ñ"<¸qÑ"@ÑAÔAÐÝ!Ð#6¸Ñ#:¸aÑ"?À!Ñ"CÑDÔDÐÝ!Ð#6¸Ñ#:¸aÑ"?À!Ñ"CÑDÔDÐà"Ð"r/   N)r'   r(   r)   r"   r-   Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_flash_attnÚ_supports_sdpaÚ_can_compile_fullgraphr+   rù   r  r.   r/   r0   r  r  æ  st   € € € € € € àÐÐÑØÐØ$€OØÐØ&*Ð#Ø0Ð2IÐJÐØÐØ€Nà!Ðð#¸eÔ>Nð #ð #ð #ð #ð #ð #r/   r  c                   óÊ   ‡ — e Zd ZdZdZeedœZdefˆ fd„Z	de
j        fd„Zde
j        fd	„Zee	 ddej        dej        d
z  dee         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚMoonshineEncoderz£
    Transformer encoder consisting of *config.num_hidden_layers* layers. Each layer is a [`MoonshineEncoderLayer`]

    Args:
        config: MoonshineConfig
    r  )Ú
attentionsrD   r8   c                 ó`  •‡— t          ¦   «                              ‰¦  «         ‰| _        ‰j        }t	          j        d|ddd¬¦  «        | _        t	          j        |d|z  dd¬	¦  «        | _        t	          j        d|z  |dd¬	¦  «        | _        t	          j	        d|d
¬¦  «        | _
        t	          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t	          j        |d¬¦  «        | _        t#          ‰¬¦  «        | _        d| _        |                      ¦   «          d S )Nr!   r  r  F)Úkernel_sizeÚstriderÆ   rN   r  r   )r  r   gñhãˆµøä>)Ú
num_groupsÚnum_channelsÚepsc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r.   )rè   ©Ú.0Úidxr8   s     €r0   ú
<listcomp>z-MoonshineEncoder.__init__.<locals>.<listcomp>  s$   ø€ ÐcÐcÐc°CÕ" 6¨3Ñ/Ô/ÐcÐcÐcr/   rÅ   ©r8   )r6   r7   r8   r<   r:   ÚConv1dÚconv1Úconv2Úconv3Ú	GroupNormÚ	groupnormÚ
ModuleListÚrangeÚencoder_num_hidden_layersrÚ   rð   Ú
layer_normrV   Ú
rotary_embÚgradient_checkpointingÚ	post_init)rA   r8   Ú	embed_dimrC   s    ` €r0   r7   zMoonshineEncoder.__init__  s  øø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒØÔ&ˆ	å”Y˜q )¸ÀRÈeÐTÑTÔTˆŒ
Ý”Y˜y¨!¨i©-ÀQÈqÐQÑQÔQˆŒ
Ý”Y˜q 9™}¨iÀQÈqÐQÑQÔQˆŒ
Ýœ°ÀÐPTÐUÑUÔUˆŒå”mØcÐcÐcÐc½5ÀÔAaÑ;bÔ;bÐcÑcÔcñ
ô 
ˆŒõ œ, y°uÐ=Ñ=Ô=ˆŒÝ2¸&ÐAÑAÔAˆŒØ&+ˆÔ#Ø�ŠÑÔÐÐÐr/   rE   c                 ó   — | j         S r4   ©r+  ©rA   s    r0   Úget_input_embeddingsz%MoonshineEncoder.get_input_embeddings  s
   € ØŒzÐr/   rš   c                 ó   — || _         d S r4   r9  ©rA   rš   s     r0   Úset_input_embeddingsz%MoonshineEncoder.set_input_embeddings"  s   € ØˆŒ
ˆ
ˆ
r/   Nr&   ru   c                 ó¦  — |                      d¦  «        }t          j                             |                      |¦  «        ¦  «        }|                      |¦  «        }t          j                             |                      |¦  «        ¦  «        }t          j                             |                      |¦  «        ¦  «        }| 	                    ddd¦  «        }d}|�;|  
                    |j        d         ¦  «        }d}|ddd|…f         dd|…f         }|}t          | j        |||¬¦  «        }t          j        d|j        d         |j        ¬	¦  «                              d¦  «        }|                      ||¬
¦  «        }	| j        D ]}
 |
|f|||	dœ|¤Ž}Œ|                      |¦  «        }t)          ||�|                     ¦   «         nd¬¦  «        S )a.  
        Args:
            input_values (`torch.FloatTensor` of shape `(batch_size, audio_length)`):
                Float values of the raw speech waveform. Raw speech waveform can be
                obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]`, a
                `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library (`pip install torchcodec`) or
                the soundfile library (`pip install soundfile`). To prepare the array into
                `input_values`, the [`AutoFeatureExtractor`] should be used for padding
                and conversion into a tensor of type `torch.FloatTensor`.
            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding indices in `input_values`. Mask values selected in `[0, 1]`:
                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.
                [What are attention masks?](../glossary#attention-mask)
        r!   r   rN   NrP   i€  .©r8   Úinputs_embedsr&   rÿ   ©rW   ©r‡   )r&   r‡   rÒ   )Úlast_hidden_stater&   )r³   r:   r¢   Útanhr+  r/  Úgelur,  r-  Úpermuter  r~   r   r8   r+   rr   rW   r4  rÚ   r3  r%   rq   )rA   r  r&   ru   rD   Úoutput_attention_maskÚmask_lenÚdownsample_strider‡   rÒ   Úencoder_layers              r0   rG   zMoonshineEncoder.forward%  sô  € ð. $×-Ò-¨aÑ0Ô0ˆÝœ×*Ò*¨4¯:ª:°lÑ+CÔ+CÑDÔDˆØŸš }Ñ5Ô5ˆÝœ×*Ò*¨4¯:ª:°mÑ+DÔ+DÑEÔEˆÝœ×*Ò*¨4¯:ª:°mÑ+DÔ+DÑEÔEˆØ%×-Ò-¨a°°AÑ6Ô6ˆð !%ÐØÐ%Ø×<Ò<¸^Ô=QÐRTÔ=UÑVÔVˆHØ *ÐØ+¨CÐ1DÐ1DÐ3DÐ1DÐ,DÔEÀcÈ9ÈHÈ9ÀnÔUˆNØ$2Ð!å2Ø”;Ø'Ø)Ø"/ð	
ñ 
ô 
ˆõ ”| A }Ô':¸1Ô'=ÀmÔFZÐ[Ñ[Ô[×eÒeÐfgÑhÔhˆØ"Ÿošo¨mÈ,˜oÑWÔWÐà!œ[ð 	ð 	ˆMØ)˜MØðà-Ø)Ø$7ð	ð ð
 ðð ˆMˆMð Ÿš¨Ñ6Ô6ˆå*Ø+Ø:OÐ:[Ð0×4Ò4Ñ6Ô6Ð6Ðaeð
ñ 
ô 
ð 	
r/   r4   )r'   r(   r)   r*   r  rÁ   rè   Ú_can_record_outputsr"   r7   r:   ÚModuler;  r>  r   r    r+   r  r,   r   r   r�   r   rG   rI   rJ   s   @r0   r  r  ÿ  s  ø€ € € € € ðð ð %€Oà(Ø.ðð Ðð
˜ð ð ð ð ð ð ð$ b¤ið ð ð ð ð¨"¬)ð ð ð ð ð  Øð /3ð<
ð <
àÔ'ð<
ð œ tÑ+ð<
ð Ð+Ô,ð	<
ð
 
Ð(Ñ	(ð<
ð <
ð <
ñ „_ñ  Ôð<
ð <
ð <
ð <
ð <
r/   r  c                   óD  ‡ — e Zd ZdZ eedd¬¦  «        e eedd¬¦  «        dœZdefˆ fd„Z	e
e	 	 	 	 	 	 	 	 ddej        d	z  d
ej        d	z  dej        d	z  ded	z  dej        d	z  ded	z  dej        d	z  dej        d	z  dee         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚMoonshineDecoderÚ	input_idsr!   rí   )ÚindexÚ
layer_namerý   )r  rD   Úcross_attentionsr8   c                 óÞ  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          j        ‰j        ‰j        | j        ¦  «        | _        t          j	        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          j        ‰j        d¬¦  «        | _        t          ‰¬¦  «        | _        d| _        |                      ¦   «          d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r.   )rû   r%  s     €r0   r(  z-MoonshineDecoder.__init__.<locals>.<listcomp>u  s$   ø€ Ð$sÐ$sÐ$sÈCÕ%:¸6À3Ñ%GÔ%GÐ$sÐ$sÐ$sr/   FrÅ   r)  )r6   r7   Úpad_token_idÚpadding_idxÚ
vocab_sizer:   Ú	Embeddingr<   Úembed_tokensr0  r1  Únum_hidden_layersrÚ   rð   ÚnormrV   r4  r5  r6  ©rA   r8   rC   s    `€r0   r7   zMoonshineDecoder.__init__o  sÏ   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒåœL¨Ô):¸FÔ<NÐPTÔP`ÑaÔaˆÔÝ”mÐ$sÐ$sÐ$sÐ$sÕSXÐY_ÔYqÑSrÔSrÐ$sÑ$sÔ$sÑtÔtˆŒÝ”L Ô!3¸%Ð@Ñ@Ô@ˆŒ	Ý2¸&ÐAÑAÔAˆŒØ&+ˆÔ#ð 	�ŠÑÔÐÐÐr/   Nr&   r‡   rÓ   rA  rô   rÿ   r   ru   rE   c	           
      ó¢  — |du |duz  rt          d¦  «        ‚|€|                      |¦  «        }|r8|€6t          t          | j        ¬¦  «        t          | j        ¬¦  «        ¦  «        }|€V|�|                     ¦   «         nd}
t          j        |j        d         |j	        ¬¦  «        |
z   }| 
                    d¦  «        }t          | j        ||||¬¦  «        }t          | j        |||¬¦  «        }|}|                      ||¬	¦  «        }| j        D ]} ||||f|||||d
œ|	¤Ž}Œ|                      |¦  «        }t!          ||r|nd¬¦  «        S )a¤  
        encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):
            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention
            of the decoder.
        encoder_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding indices in `encoder_hidden_states`. Mask values selected in `[0, 1]`:
            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.
            [What are attention masks?](../glossary#attention-mask)
        Nz:You must specify exactly one of input_ids or inputs_embedsr)  r   r!   rB  )r8   rA  r&   rÓ   r‡   r@  rC  )r   r‡   rÓ   rô   rÒ   )rD  rÓ   )Ú
ValueErrorrZ  r	   r   r8   Úget_seq_lengthr+   rr   r~   rW   r³   r   r   r4  rÚ   r\  r   )rA   rP  r&   r‡   rÓ   rA  rô   rÿ   r   ru   Úpast_seen_tokensÚcausal_maskrD   rÒ   Údecoder_layers                  r0   rG   zMoonshineDecoder.forward}  sË  € ð0 ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàð 	v˜Ð0Ý1µ,ÀdÄkÐ2RÑ2RÔ2RÕT`ÐhlÔhsÐTtÑTtÔTtÑuÔuˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå(Ø”;Ø'Ø)Ø+Ø%ð
ñ 
ô 
ˆõ ";Ø”;Ø'Ø1Ø"7ð	"
ñ "
ô "
Ðð &ˆØ"Ÿošo¨mÈ,˜oÑWÔWÐà!œ[ð 	ð 	ˆMØ)˜MØØØ%ð
ð (>Ø)Ø /Ø#Ø$7ð
ð 
ð ð
ð 
ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆå8Ø+Ø/8ÐB˜O˜O¸dð
ñ 
ô 
ð 	
r/   )NNNNNNNN)r'   r(   r)   r  r   rÁ   rû   rL  r"   r7   r   r    r+   rù   r,   r   r  ræ   r   r   r�   r   rG   rI   rJ   s   @r0   rO  rO  f  s…  ø€ € € € € à!€Oà$�nÐ%7¸qÈ[ÐYÑYÔYØ.Ø*˜NÐ+=ÀQÐSaÐbÑbÔbðð Ðð˜ð ð ð ð ð ð ð  Øð .2Ø.2Ø04Ø(,Ø26Ø!%Ø:>Ø6:ðG
ð G
àÔ# dÑ*ðG
ð œ tÑ+ðG
ð Ô&¨Ñ-ð	G
ð
  ™ðG
ð Ô(¨4Ñ/ðG
ð ˜$‘;ðG
ð  %Ô0°4Ñ7ðG
ð !&¤¨tÑ 3ðG
ð Ð+Ô,ðG
ð 
Ð(Ñ	(ðG
ð G
ð G
ñ „_ñ  ÔðG
ð G
ð G
ð G
ð G
r/   rO  c                   ó\  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zd„ Zd„ Ze	e
	 	 	 	 	 	 	 	 	 ddej        dz  d	ej        dz  d
ej        dz  dej        dz  deeej                          dz  dedz  deej                 dz  deej                 dz  dedz  dee         defd„¦   «         ¦   «         Zˆ xZS )ÚMoonshineModelr8   c                 óÂ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          |¦  «        | _        |                      ¦   «          d S r4   )r6   r7   r  ÚencoderrO  Údecoderr6  r]  s     €r0   r7   zMoonshineModel.__init__Ë  sO   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å'¨Ñ/Ô/ˆŒÝ'¨Ñ/Ô/ˆŒà�ŠÑÔÐÐÐr/   c                 ó   — | j         j        S r4   ©rh  rZ  r:  s    r0   r;  z#MoonshineModel.get_input_embeddingsÓ  s   € ØŒ|Ô(Ð(r/   c                 ó   — || j         _        d S r4   rj  r=  s     r0   r>  z#MoonshineModel.set_input_embeddingsÖ  s   € Ø$)ˆŒÔ!Ð!Ð!r/   c                 ó8   — | j                              ¦   «          dS )z«
        Calling this function will disable the gradient computation for the Moonshine encoder so that its parameters will
        not be updated during training.
        N)rg  Ú_freeze_parametersr:  s    r0   Úfreeze_encoderzMoonshineModel.freeze_encoderÙ  s   € ð
 	Œ×'Ò'Ñ)Ô)Ð)Ð)Ð)r/   c                 ó    — t          d¦  «        ‚)z¢
        Masks extracted features along time axis and/or along feature axis according to
        [SpecAugment](https://huggingface.co/papers/1904.08779).
        zNot needed for Moonshine)ÚAttributeErrorr:  s    r0   Ú_mask_input_featuresz#MoonshineModel._mask_input_featuresà  s   € õ
 Ð7Ñ8Ô8Ð8r/   Nr  r&   Údecoder_input_idsÚdecoder_attention_maskÚencoder_outputsrÓ   Údecoder_inputs_embedsÚdecoder_position_idsrô   ru   rE   c
                 óä   — |€ | j         |fd|i|
¤Ž} | j        d|||j        |j        ||||	dœ|
¤Ž}t	          |j        |j        |j        |j        |j        |j        |j        |j        ¬¦  «        S )a
  
        input_values (`torch.FloatTensor` of shape `(batch_size, audio_length)`):
            Float values of the raw speech waveform. Raw speech waveform can be
            obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]`, a
            `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library (`pip install torchcodec`) or
            the soundfile library (`pip install soundfile`). To prepare the array into
            `input_values`, the [`AutoFeatureExtractor`] should be used for padding
            and conversion into a tensor of type `torch.FloatTensor`.
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
            Indices of positions of each input sequence tokens in the position embeddings.
            Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`

        Example:

        ```python
        >>> import torch
        >>> from transformers import AutoFeatureExtractor, MoonshineModel
        >>> from datasets import load_dataset

        >>> model = MoonshineModel.from_pretrained("UsefulSensors/moonshine-tiny")
        >>> feature_extractor = AutoFeatureExtractor.from_pretrained("UsefulSensors/moonshine-tiny")
        >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        >>> inputs = feature_extractor(ds[0]["audio"]["array"], return_tensors="pt")
        >>> input_values = inputs.input_values
        >>> decoder_input_ids = torch.tensor([[1, 1]]) * model.config.decoder_start_token_id
        >>> last_hidden_state = model(input_values, decoder_input_ids=decoder_input_ids).last_hidden_state
        >>> list(last_hidden_state.shape)
        [1, 2, 288]
        ```
        Nr&   )rP  r&   rÿ   r   rÓ   rA  r‡   rô   )rD  rÓ   Údecoder_hidden_statesÚdecoder_attentionsrS  Úencoder_last_hidden_staterÿ   Úencoder_attentionsr.   )	rg  rh  rD  r&   r   rÓ   rD   r  rS  )rA   r  r&   rr  rs  rt  rÓ   ru  rv  rô   ru   Údecoder_outputss               r0   rG   zMoonshineModel.forwardç  s¾   € ðZ Ð"Ø/;¨t¬|¸LÐ/rÐ/rÐYgÐ/rÐkqÐ/rÐ/rˆOàEQÀTÄ\ð 
F
Ø'Ø1Ø"1Ô"CØ#2Ô#AØ+Ø/Ø-Øð
F
ð 
F
ð ð
F
ð 
F
ˆõ "Ø-Ô?Ø+Ô;Ø"1Ô"?Ø.Ô9Ø,Ô=Ø&5Ô&GØ"1Ô"?Ø.Ô9ð	
ñ 	
ô 	
ð 		
r/   )	NNNNNNNNN)r'   r(   r)   r"   r7   r;  r>  rn  rq  r   r   r+   r  rù   r�   r	   ræ   r   r   r   rG   rI   rJ   s   @r0   re  re  É  s¨  ø€ € € € € ð˜ð ð ð ð ð ð ð)ð )ð )ð*ð *ð *ð*ð *ð *ð9ð 9ð 9ð Øð 26Ø26Ø59Ø:>ØBFØ6:ØAEØ?CØ!%ðC
ð C
àÔ'¨$Ñ.ðC
ð Ô(¨4Ñ/ðC
ð !Ô+¨dÑ2ð	C
ð
 !&Ô 0°4Ñ 7ðC
ð ˜u UÔ%6Ô7Ô8¸4Ñ?ðC
ð -¨tÑ3ðC
ð  % UÔ%6Ô7¸$Ñ>ðC
ð $ EÔ$4Ô5¸Ñ<ðC
ð ˜$‘;ðC
ð Ð+Ô,ðC
ð 
ðC
ð C
ð C
ñ „^ñ ÔðC
ð C
ð C
ð C
ð C
r/   re  rP  rV  Údecoder_start_token_idc                 óô   — |                       | j        ¦  «        }| dd…dd…f                              ¦   «         |dd…dd…f<   ||dd…df<   |€t          d¦  «        ‚|                     |dk    |¦  «         |S )z1
    Shift input ids one token to the right.
    NrP   r!   r   z1self.model.config.pad_token_id has to be defined.iœÿÿÿ)Ú	new_zerosr~   re   r_  Úmasked_fill_)rP  rV  r}  Úshifted_input_idss       r0   Úshift_tokens_rightr‚  /  s˜   € ð "×+Ò+¨I¬OÑ<Ô<ÐØ(¨¨¨¨C¨R¨C¨Ô0×6Ò6Ñ8Ô8Ð�a�a�a˜˜˜�eÑØ4Ð�a�a�a˜�dÑàÐÝÐLÑMÔMÐMà×"Ò"Ð#4¸Ò#<¸lÑKÔKÐKàÐr/   zj
    The Moonshine Model with a language modeling head. Can be used for automatic speech recognition.
    c                   ó„  ‡ — e Zd ZddiZdefˆ fd„Zd„ Zd„ Zdej	        fd„Z
ee	 	 	 	 	 	 	 	 	 	 dd
ej        d	z  dej        d	z  dej        d	z  dej        d	z  deeej                          d	z  ded	z  deej                 d	z  deej                 d	z  ded	z  dej        d	z  dee         defd„¦   «         ¦   «         Zˆ xZS )Ú!MoonshineForConditionalGenerationzproj_out.weightz!model.decoder.embed_tokens.weightr8   c                 óæ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        |j        d¬¦  «        | _        |  	                    ¦   «          d S )NFrÅ   )
r6   r7   re  r  r:   r;   r<   rX  Úproj_outr6  r]  s     €r0   r7   z*MoonshineForConditionalGeneration.__init__G  s`   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý# FÑ+Ô+ˆŒ
Ýœ	 &Ô"4°fÔ6GÈeÐTÑTÔTˆŒð 	�ŠÑÔÐÐÐr/   c                 ó   — | j         S r4   ©r†  r:  s    r0   Úget_output_embeddingsz7MoonshineForConditionalGeneration.get_output_embeddingsO  s
   € ØŒ}Ðr/   c                 ó   — || _         d S r4   rˆ  )rA   Únew_embeddingss     r0   Úset_output_embeddingsz7MoonshineForConditionalGeneration.set_output_embeddingsR  s   € Ø&ˆŒˆˆr/   rE   c                 ó4   — | j                              ¦   «         S r4   )r  r;  r:  s    r0   r;  z6MoonshineForConditionalGeneration.get_input_embeddingsU  s   € ØŒz×.Ò.Ñ0Ô0Ð0r/   Nr  r&   rr  rs  rt  rÓ   ru  rv  rô   Úlabelsru   c                 ó¤  — |                      dd¦  «        }|
�)|€'|€%t          |
| j        j        | j        j        ¦  «        } | j        |f||||||||	dœ|¤Ž}|                      |j        ¦  «        }d}|
�  | j        d|d| j        j	        |�|n|
dœ|¤Ž}t          |||j        |j        |j        |j        |j        |j        |j        ¬¦	  «	        S )a0  
        input_values (`torch.FloatTensor` of shape `(batch_size, audio_length)`):
            Float values of the raw speech waveform. Raw speech waveform can be
            obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]`, a
            `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library (`pip install torchcodec`) or
            the soundfile library (`pip install soundfile`). To prepare the array into
            `input_values`, the [`AutoFeatureExtractor`] should be used for padding
            and conversion into a tensor of type `torch.FloatTensor`.
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
            Indices of positions of each input sequence tokens in the position embeddings.
            Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`

        Example:

        ```python
        >>> import torch
        >>> from transformers import AutoProcessor, MoonshineForConditionalGeneration
        >>> from datasets import load_dataset

        >>> processor = AutoProcessor.from_pretrained("UsefulSensors/moonshine-tiny")
        >>> model = MoonshineForConditionalGeneration.from_pretrained("UsefulSensors/moonshine-tiny")

        >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")

        >>> inputs = processor(ds[0]["audio"]["array"], return_tensors="pt")
        >>> input_values = inputs.input_values

        >>> generated_ids = model.generate(input_values, max_new_tokens=100)

        >>> transcription = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        >>> transcription
        'Mr. Quilter is the apostle of the middle classes, and we are glad to welcome his gospel.'
        ```Úshift_labelsN)r&   rr  rt  rs  rÓ   ru  rv  rô   )ÚlogitsrŽ  rX  r�  )	Úlossr‘  rÓ   rx  ry  rS  rz  rÿ   r{  r.   )Úpopr‚  r8   rV  r}  r  r†  rD  Úloss_functionrX  r   rÓ   rx  ry  rS  rz  rÿ   r{  )rA   r  r&   rr  rs  rt  rÓ   ru  rv  rô   rŽ  ru   r�  Úoutputsr‘  r’  s                   r0   rG   z)MoonshineForConditionalGeneration.forwardX  s=  € ðh —z’z .°$Ñ7Ô7ˆàÐØ Ð(Ð-BÐ-JÝ$6Ø˜DœKÔ4°d´kÔ6Xñ%ô %Ð!ð '1 d¤jØð'
à)Ø/Ø+Ø#9Ø+Ø"7Ø!5Øð'
ð '
ð ð'
ð '
ˆð —’˜wÔ8Ñ9Ô9ˆàˆØÐð
 &�4Ô%ð ØØØœ;Ô1Ø-9Ð-E˜\˜\È6ð	ð ð
 ðð ˆDõ ØØØ#Ô3Ø")Ô"?Ø&Ô9Ø$Ô5Ø&-Ô&GØ")Ô"?Ø&Ô9ð

ñ 

ô 

ð 
	
r/   )
NNNNNNNNNN)r'   r(   r)   Ú_tied_weights_keysr"   r7   r‰  rŒ  r:   rM  r;  r   r   r+   r  rù   r�   r	   ræ   r   r   r   rG   rI   rJ   s   @r0   r„  r„  ?  sË  ø€ € € € € ð ,Ð-PÐQÐð˜ð ð ð ð ð ð ðð ð ð'ð 'ð 'ð1 b¤ið 1ð 1ð 1ð 1ð Øð 26Ø26Ø59Ø:>ØBFØ6:ØAEØ?CØ!%Ø*.ð`
ð `
àÔ'¨$Ñ.ð`
ð Ô(¨4Ñ/ð`
ð !Ô+¨dÑ2ð	`
ð
 !&Ô 0°4Ñ 7ð`
ð ˜u UÔ%6Ô7Ô8¸4Ñ?ð`
ð -¨tÑ3ð`
ð  % UÔ%6Ô7¸$Ñ>ð`
ð $ EÔ$4Ô5¸Ñ<ð`
ð ˜$‘;ð`
ð Ô  4Ñ'ð`
ð Ð+Ô,ð`
ð 
ð`
ð `
ð `
ñ „^ñ Ôð`
ð `
ð `
ð `
ð `
r/   r„  )re  r  r„  )r–   )r!   )HÚcollections.abcr   Údataclassesr   r+   Útorch.nnr:   Úactivationsr   Úcache_utilsr   r   r	   Ú
generationr
   Úmasking_utilsr   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   r    Úconfiguration_moonshiner"   r%   rM  r2   rL   rV   r,   rq   r•   rs   rª   r±   r¿   rÁ   rè   rû   r  r  rO  re  r‚  r„  Ú__all__r.   r/   r0   ú<module>rª     s�  ðð* %Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð à !Ð !Ð !Ð !Ð !Ð !Ø CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CØ )Ð )Ð )Ð )Ð )Ð )Ø JÐ JÐ JÐ JÐ JÐ JÐ JÐ JØ BÐ BÐ BÐ BÐ BÐ BØ 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð ð ð LÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ 4Ð 4Ð 4Ð 4Ð 4Ð 4ð €ððñ ô ð
 ð
/ð 
/ð 
/ð 
/ð 
/ /ñ 
/ô 
/ñ „ñô ð
/ðð ð ð ð ˜"œ)ñ ô ð ðð ð ð ð ˜"œ)ñ ô ð ð 8<ð 8<ð 8<ð 8<ð 8<˜rœyñ 8<ô 8<ð 8<ðv	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð& ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð26ð 6ð 6ð%ð %ð %ð %ðPw)ð w)ð w)ð w)ð w)˜œñ w)ô w)ð w)ðt.ð .ð .ð .ð .Ð6ñ .ô .ð .ðbEð Eð Eð Eð EÐ6ñ Eô Eð EðP ð#ð #ð #ð #ð #˜ñ #ô #ñ „ð#ð0d
ð d
ð d
ð d
ð d
Ð/ñ d
ô d
ð d
ðN ð_
ð _
ð _
ð _
ð _
Ð/ñ _
ô _
ñ „ð_
ðD ðb
ð b
ð b
ð b
ð b
Ð-ñ b
ô b
ñ „ðb
ðJ %¤,ð ¸cð Ð[^ð ð ð ð ð  €ððñ ô ð
v
ð v
ð v
ð v
ð v
Ð(@À/ñ v
ô v
ñô ð
v
ðr ^Ð
]Ð
]€€€r/   