§
    9²¸j=  ã                   óŒ   — d Z ddlZddlmZ ddlmZ ddlmZ  ed¬¦  «        e G d	„ d
e¦  «        ¦   «         ¦   «         Zd
gZ	dS )zJamba model configurationé    N)Ústricté   )ÚPreTrainedConfig)Úauto_docstringzai21labs/Jamba-v0.1)Ú
checkpointc                   ó|  ‡ — e Zd ZU dZdZdgZddiZdZee	d<   dZ
ee	d	<   d
Zee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZedz  e	d <   d!Zedz  e	d"<   d#Zeee         z  dz  e	d$<   d%Zee	d&<   d'Zeez  e	d(<   d#Zee	d)<   d*Z ee	d<   d#Z!ee	d+<   d!Z"ee	d,<   dZ#ee	d-<   d.Z$ee	d/<   dZ%ee	d0<   d*Z&ee	d1<   d.Z'ee	d2<   d#Z(ee	d3<   d4Z)eez  e	d5<   dZ*ee	d6<   dZ+ee	d7<   dZ,ee	d8<   dZ-ee	d9<   ˆ fd:„Z.e/d;„ ¦   «         Z0e/d<„ ¦   «         Z1e/d=„ ¦   «         Z2d>„ Z3ˆ xZ4S )?ÚJambaConfiga‚  
    expert_layer_period (`int`, *optional*, defaults to 2):
        Once in this many layers, we will have an expert layer
    expert_layer_offset (`int`, *optional*, defaults to 1):
        The first layer index that contains an expert mlp layer
    attn_layer_period (`int`, *optional*, defaults to 8):
        Once in this many layers, we will have a vanilla attention layer
    attn_layer_offset (`int`, *optional*, defaults to 4):
        The first layer index that contains a vanilla attention mlp layer
    use_mamba_kernels (`bool`, *optional*, defaults to `True`):
        Flag indicating whether or not to use the fast mamba kernels. These are available only if `mamba-ssm` and
        `causal-conv1d` are installed, and the mamba modules are running on a CUDA device. Raises ValueError if
        `True` and kernels are not available
    mamba_dt_rank (`Union[int,str]`, *optional*, defaults to `"auto"`):
        Rank of the mamba discretization projection matrix. `"auto"` means that it will default to `math.ceil(self.hidden_size / 16)`
    use_mambapy (`bool`, *optional*, defaults to `False`):
        Determines the fallback strategy during training if the CUDA-based official implementation of Mamba is not available. If `True`,
        the mamba.py implementation is used. If `False`, the naive and slower implementation is used. Consider switching to the naive
        version if memory is limited.
    use_associative_scan (`bool`, *optional*, defaults to `True`):
        Whether to use PyTorch's `torch._higher_order_ops.associative_scan` for the parallel scan instead of the naive
        sequential implementation. The associative scan is only active during `torch.compile` tracing and
        requires torch >= 2.9.0. Both paths are tested to produce numerically identical results (see
        `test_associative_scan_matches_sequential`). Set to `False` to fall back to the sequential loop.
    ÚjambaÚpast_key_valuesÚnum_local_expertsÚnum_expertsi   Ú
vocab_sizeFÚtie_word_embeddingsi   Úhidden_sizei 8  Úintermediate_sizeé    Únum_hidden_layersÚnum_attention_headsé   Únum_key_value_headsÚsiluÚ
hidden_actg{®Gáz”?Úinitializer_rangeg�íµ ÷Æ°>Úrms_norm_epsTÚ	use_cacheÚoutput_router_logitsgü©ñÒMbP?Úrouter_aux_loss_coefr   NÚpad_token_idé   Úbos_token_idé   Úeos_token_idi   Úmax_position_embeddingsg        Úattention_dropoutÚnum_experts_per_toké   Úexpert_layer_periodÚexpert_layer_offsetÚattn_layer_periodé   Úattn_layer_offsetÚuse_mamba_kernelsÚmamba_d_stateÚmamba_d_convÚmamba_expandÚautoÚmamba_dt_rankÚmamba_conv_biasÚmamba_proj_biasÚuse_mambapyÚuse_associative_scanc                 óÆ   •— | j         €| j        | _         | j        dk    rt          j        | j        dz  ¦  «        n| j        | _         t          ¦   «         j        di |¤Ž d S )Nr0   r&   © )r   r   r1   ÚmathÚceilr   ÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s     €ú}/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/jamba/configuration_jamba.pyr;   zJambaConfig.__post_init__]   sk   ø€ ØÔ#Ð+Ø'+Ô'?ˆDÔ$àAEÔASÐW]ÒA]ÐA]�TœY tÔ'7¸"Ñ'<Ñ=Ô=Ð=ÐcgÔcuˆÔØ�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'ó    c                 óD   ‡ — ˆ fd„t          ‰ j        ¦  «        D ¦   «         S )Nc                 ó>   •— g | ]}|‰j         z  ‰j        k    rd nd‘ŒS )Ú	attentionÚmamba)r)   r+   ©Ú.0Úir<   s     €r?   ú
<listcomp>z1JambaConfig.layers_block_type.<locals>.<listcomp>f   sC   ø€ ð 
ð 
ð 
àð ˜tÔ5Ñ5¸Ô9OÒOÐOˆKˆKÐU\ð
ð 
ð 
r@   ©Úranger   ©r<   s   `r?   Úlayers_block_typezJambaConfig.layers_block_typed   ó9   ø€ ð
ð 
ð 
ð 
å˜4Ô1Ñ2Ô2ð
ñ 
ô 
ð 	
r@   c                 ó$   — d„ | j         D ¦   «         S )Nc                 ó"   — g | ]}|d k    rdnd‘ŒS )rD   Úlinear_attentionÚfull_attentionr7   )rF   Úxs     r?   rH   z+JambaConfig.layer_types.<locals>.<listcomp>n   s'   € ÐiÐiÐiÈQ a¨7¢l lÐ"Ð"Ð8HÐiÐiÐir@   )rL   rK   s    r?   Úlayer_typeszJambaConfig.layer_typesk   s   € ð jÐiÐRVÔRhÐiÑiÔiÐir@   c                 óD   ‡ — ˆ fd„t          ‰ j        ¦  «        D ¦   «         S )Nc                 óH   •— g | ]}|‰j         z  ‰j        k    r‰j        nd ‘ŒS )r   )r'   r(   r   rE   s     €r?   rH   z2JambaConfig.layers_num_experts.<locals>.<listcomp>r   sG   ø€ ð 
ð 
ð 
àð !" DÔ$<Ñ <ÀÔ@XÒ XÐ XˆDÔÐÐ^_ð
ð 
ð 
r@   rI   rK   s   `r?   Úlayers_num_expertszJambaConfig.layers_num_expertsp   rM   r@   c                 óÆ   — | j         | j        k    r t          d| j         › d| j        › d�¦  «        ‚| j        | j        k    r t          d| j        › d| j        › d�¦  «        ‚dS )zOPart of `@strict`-powered validation. Validates the architecture of the config.zattention layer offset (z/) must be smaller than attention layer period (ú)zexpert layer offset (z,) must be smaller than expert layer period (N)r+   r)   Ú
ValueErrorr(   r'   rK   s    r?   Úvalidate_architecturez!JambaConfig.validate_architecturew   s¹   € àÔ! TÔ%;Ò;Ð;Ýð L¨4Ô+Að  Lð  LÐrvô  sIð  Lð  Lð  Lñô ð ð Ô# tÔ'?Ò?Ð?Ýð J¨Ô(@ð  Jð  JÐnrô  oGð  Jð  Jð  Jñô ð ð @Ð?r@   )5Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚattribute_mapr   ÚintÚ__annotations__r   Úboolr   r   r   r   r   r   Ústrr   Úfloatr   r   r   r   r   r    r"   Úlistr#   r$   r%   r   r'   r(   r)   r+   r,   r-   r.   r/   r1   r2   r3   r4   r5   r;   ÚpropertyrL   rS   rV   rZ   Ú__classcell__)r>   s   @r?   r	   r	      s  ø€ € € € € € ðð ð4 €JØ#4Ð"5Ðà˜]ð€Mð €J�ÐÐÑØ %Ð˜Ð%Ð%Ñ%Ø€K�ÐÐÑØ"Ð�sÐ"Ð"Ñ"ØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø Ð˜Ð Ð Ñ Ø€J�ÐÐÑØ#Ð�uÐ#Ð#Ñ#Ø€L�%ÐÐÑØ€IˆtÐÐÑØ!&Ð˜$Ð&Ð&Ñ&Ø"'Ð˜%Ð'Ð'Ñ'Ø €L�#˜‘*Ð Ð Ñ Ø €L�#˜‘*Ð Ð Ñ Ø+,€L�#˜˜Sœ	‘/ DÑ(Ð,Ð,Ñ,Ø#)Ð˜SÐ)Ð)Ñ)Ø%(Ð�u˜s‘{Ð(Ð(Ñ(Ø Ð˜Ð Ð Ñ Ø€K�ÐÐÑØ Ð˜Ð Ð Ñ Ø Ð˜Ð Ð Ñ ØÐ�sÐÐÑØÐ�sÐÐÑØ"Ð�tÐ"Ð"Ñ"Ø€M�3ÐÐÑØ€L�#ÐÐÑØ€L�#ÐÐÑØ%€M�3˜‘9Ð%Ð%Ñ%Ø €O�TÐ Ð Ñ Ø!€O�TÐ!Ð!Ñ!Ø€K�ÐÐÑØ!%Ð˜$Ð%Ð%Ñ%ð(ð (ð (ð (ð (ð ð
ð 
ñ „Xð
ð ðjð jñ „Xðjð ð
ð 
ñ „Xð
ð
ð 
ð 
ð 
ð 
ð 
ð 
r@   r	   )
r^   r8   Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úutilsr   r	   Ú__all__r7   r@   r?   ú<module>rn      s²   ðð  Ð à €€€à .Ð .Ð .Ð .Ð .Ð .à 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø #Ð #Ð #Ð #Ð #Ð #ð €Ð0Ð1Ñ1Ô1Øðgð gð gð gð gÐ"ñ gô gñ „ñ 2Ô1ðgðT ˆ/€€€r@   