§
    9²¸j   ã                   ó�   — d Z ddlZddlmZ ddlmZmZ ddlmZ  ed¬¦  «        e G d	„ d
e¦  «        ¦   «         ¦   «         Z	d
gZ
dS )zZamba model configurationé    N)Ústricté   )ÚPreTrainedConfigÚremap_legacy_layer_types)Úauto_docstringzZyphra/Zamba-7B-v1)Ú
checkpointc                   óh  ‡ — e Zd ZU dZdZdgZdddœZdZee	d<   d	Z
ee	d
<   dZee	d<   dZedz  e	d<   dZee	d<   dZee	d<   dZee	d<   dZedz  e	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   d	Zee	d <   d!Zee	d"<   d#Zedz  e	d$<   d!Zedz  e	d%<   dZeee         z  dz  e	d&<   d'Z ee	d(<   d)Z!eez  e	d*<   d+Z"ee	d,<   d-Z#ee	d.<   d	Z$ee	d/<   dZ%ee	d0<   d-Z&ee	d1<   dZ'ee	d2<   d3Z(eez  e	d4<   d5Z)ee	d6<   d7Z*ee	d8<   d9Z+ee	d:<   d	Z,ee	d;<   d<Z-ee	d=<   dZ.ee         dz  e	d<   ˆ fd>„Z/d?„ Z0ˆ xZ1S )@ÚZambaConfiga‹  
    attention_hidden_size (`int`, *optional*):
        Dimension of the hidden representations of the inputs to the Attention layer.
    attention_head_dim (`int`, *optional*):
        Dimension of the attention head in the Transformer decoder.
    n_mamba_heads (`int`, *optional*, defaults to 2):
        Number of mamba heads for each mamba layer.
    hidden_mamba_act (`str` or `function`, *optional*, defaults to `"silu"`):
        The non-linear activation function (function or string) in the mamba layer.
    num_logits_to_keep (`int` or `None`, *optional*, defaults to 1):
        Number of prompt logits to calculate during generation. If `None`, all logits will be calculated. If an
        integer value, only last `num_logits_to_keep` logits will be calculated. Default is 1 because only the
        logits of the last prompt token are needed for generation. For long sequences, the logits for the entire
        sequence may use a lot of memory so, setting `num_logits_to_keep=1` will reduce memory footprint
        significantly.
    attn_layer_period (`int`, *optional*, defaults to 6):
        Once in this many layers, we will have a shared attention layer
    attn_layer_offset (`int`, *optional*, defaults to 4):
        Offset of the shared attention layer
    use_mamba_kernels (`bool`, *optional*, defaults to `True`):
        Flag indicating whether or not to use the fast mamba kernels. These are available only if `mamba-ssm` and
        `causal-conv1d` are installed, and the mamba modules are running on a CUDA device. Raises ValueError if
        `True` and kernels are not available
    mamba_dt_rank (`Union[int,str]`, *optional*, defaults to `"auto"`):
        Rank of the mamba discretization projection matrix. `"auto"` means that it will default to `math.ceil(self.hidden_size / 16)`
    layers_block_type (`str`, *optional*):
        Alias for `layer_types` which is kept for BC purposes.
    ÚzambaÚpast_key_valuesÚlayers_block_typeÚattention_head_dim)Úlayer_typesÚhead_dimi }  Ú
vocab_sizeTÚtie_word_embeddingsi€  Úhidden_sizeNÚattention_hidden_sizei :  Úintermediate_sizeéL   Únum_hidden_layersé   Únum_attention_headsÚnum_key_value_headsé   Ún_mamba_headsÚgeluÚ
hidden_actÚsiluÚhidden_mamba_actg{®Gáz”?Úinitializer_rangegñhãˆµøä>Úrms_norm_epsÚ	use_cacheé   Únum_logits_to_keepr   Úpad_token_idÚbos_token_idÚeos_token_idi   Úmax_position_embeddingsg        Úattention_dropouté   Úattn_layer_periodé   Úattn_layer_offsetÚuse_mamba_kernelsÚmamba_d_stateÚmamba_d_convÚmamba_expandÚautoÚmamba_dt_rankgü©ñÒMbP?Útime_step_mingš™™™™™¹?Útime_step_maxg-Cëâ6?Útime_step_floorÚmamba_conv_biasFÚmamba_proj_biasc                 ó¦  •‡ — ‰ j         p	d‰ j        z  ‰ _         ‰ j        pd‰ j        z  ‰ j        z  ‰ _        ‰ j        dk    rt          j        ‰ j        dz  ¦  «        n‰ j        ‰ _        ‰ j        €.g d¢ˆ fd„t          ‰ j	        dz
  ¦  «        D ¦   «         z   ‰ _        nt          ‰ j        ¦  «        ‰ _         t          ¦   «         j        di |¤Ž d S )Nr   r3   r   )Úlinear_attentionr;   Úhybridc                 ó>   •— g | ]}|‰j         z  ‰j        k    rd nd‘ŒS )r<   r;   )r,   r.   )Ú.0ÚiÚselfs     €ú}/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/zamba/configuration_zamba.pyú
<listcomp>z-ZambaConfig.__post_init__.<locals>.<listcomp>i   sC   ø€ ð ð ð àð  Ô 6Ñ6¸$Ô:PÒPÐP��ÐVhðð ð ó    r   © )r   r   r   r   r4   ÚmathÚceilr   Úranger   r   ÚsuperÚ__post_init__)r@   ÚkwargsÚ	__class__s   ` €rA   rI   zZambaConfig.__post_init___   s  øø€ Ø%)Ô%?Ð%WÀ1ÀtÔGWÑCWˆÔ"Ø"&Ô"9Ð"m¸QÀÔAQÑ=QÐUYÔUmÑ=mˆÔØAEÔASÐW]ÒA]ÐA]�TœY tÔ'7¸"Ñ'<Ñ=Ô=Ð=ÐcgÔcuˆÔàÔ!Ð)ð&ð &ð &ðð ð ð å˜tÔ5¸Ñ9Ñ:Ô:ðñ ô ñ	&ˆDÔ"Ð"õ &>¸dÔ>TÑ%UÔ%UˆDÔ"à�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'rC   c                 óZ   — | j         | j        z  | j        z  dk    rt          d¦  «        ‚dS )zOPart of `@strict`-powered validation. Validates the architecture of the config.r   z;`intermediate_size` should be divisible by `n_mamba_heads`.N)r2   r   r   Ú
ValueError)r@   s    rA   Úvalidate_architecturez!ZambaConfig.validate_architecturer   s8   € àÔ Ô 0Ñ0°DÔ4FÑFÈ!ÒKÐKÝÐZÑ[Ô[Ð[ð LÐKrC   )2Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚattribute_mapr   ÚintÚ__annotations__r   Úboolr   r   r   r   r   r   r   r   r   Ústrr    r!   Úfloatr"   r#   r%   r&   r'   r(   Úlistr)   r*   r,   r.   r/   r0   r1   r2   r4   r5   r6   r7   r8   r9   r   rI   rN   Ú__classcell__)rK   s   @rA   r
   r
      sÖ  ø€ € € € € € ðð ð: €JØ#4Ð"5ÐØ$7ÐEYÐZÐZ€Mà€J�ÐÐÑØ $Ð˜Ð$Ð$Ñ$Ø€K�ÐÐÑØ(,Ð˜3 ™:Ð,Ð,Ñ,Ø"Ð�sÐ"Ð"Ñ"ØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø%)Ð˜˜d™
Ð)Ð)Ñ)Ø!Ð˜Ð!Ð!Ñ!Ø€M�3ÐÐÑØ€J�ÐÐÑØ"Ð�cÐ"Ð"Ñ"Ø#Ð�uÐ#Ð#Ñ#Ø€L�%ÐÐÑØ€IˆtÐÐÑØÐ˜ÐÐÑØ €L�#˜‘*Ð Ð Ñ Ø €L�#˜‘*Ð Ð Ñ Ø+,€L�#˜˜Sœ	‘/ DÑ(Ð,Ð,Ñ,Ø#'Ð˜SÐ'Ð'Ñ'Ø%(Ð�u˜s‘{Ð(Ð(Ñ(ØÐ�sÐÐÑØÐ�sÐÐÑØ"Ð�tÐ"Ð"Ñ"Ø€M�3ÐÐÑØ€L�#ÐÐÑØ€L�#ÐÐÑØ%€M�3˜‘9Ð%Ð%Ñ%Ø €M�5Ð Ð Ñ Ø€M�5ÐÐÑØ!€O�UÐ!Ð!Ñ!Ø €O�TÐ Ð Ñ Ø!€O�TÐ!Ð!Ñ!Ø*.Ð�t˜C”y 4Ñ'Ð.Ð.Ñ.ð(ð (ð (ð (ð (ð&\ð \ð \ð \ð \ð \ð \rC   r
   )rR   rE   Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   r   Úutilsr   r
   Ú__all__rD   rC   rA   ú<module>ra      sÀ   ðð  Ð à €€€à .Ð .Ð .Ð .Ð .Ð .à MÐ MÐ MÐ MÐ MÐ MÐ MÐ MØ #Ð #Ð #Ð #Ð #Ð #ð €Ð/Ð0Ñ0Ô0Øð[\ð [\ð [\ð [\ð [\Ð"ñ [\ô [\ñ „ñ 1Ô0ð[\ð| ˆ/€€€rC   