§
    9²¸jù¶  ã                   óN  — d Z ddlZddlmZ ddlZddlmc mZ ddlmZ ddl	m
Z
mZmZ ddlmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZmZ ddlmZ ddlmZmZ ddlm Z  ddl!m"Z"m#Z#m$Z$ ddl%m&Z&m'Z' ddl(m)Z) ddl*m+Z+m,Z,m-Z-m.Z. ddl/m0Z0 ddl1m2Z2m3Z3m4Z4 ddl5m6Z6 ddl7m8Z8  e.j9        e:¦  «        Z; G d„ dej<        ¦  «        Z=dej>        de?dej>        fd„Z@	 d\dej<        d ej>        d!ej>        d"ej>        d#ej>        dz  d$eAd%eAfd&„ZB G d'„ d(ej<        ¦  «        ZCd)„ ZD ed*d+¦  «        	 	 d]dej>        d,ej>        d-ejE        d.ejE        dz  d/eFdz  f
d0„¦   «         ZG ed1d+¦  «        	 	 d]dej>        d-ejE        d.ejE        dz  d/eFdz  fd2„¦   «         ZH ed3d4¦  «        	 	 	 	 d^d6ej>        dej>        d7ej>        d8ej>        d9ej>        d:ej>        d;ej>        dz  d<ej>        dz  d=eId>ej>        dz  fd?„¦   «         ZJ ed@d4¦  «        	 	 	 	 	 	 	 d_dej>        d7ej>        d8ej>        d9ej>        d:ej>        d;ej>        dz  d>ej>        dz  dAej>        dz  dBeIdCeIdDeIdEeIfdF„¦   «         ZK eeKeJeHeGg¦  «         G dG„ dHej<        ¦  «        ¦   «         ZL G dI„ dJej<        ¦  «        ZM G dK„ dLej<        ¦  «        ZN G dM„ dNe ¦  «        ZO G dO„ dPe ¦  «        ZPe, G dQ„ dRe'¦  «        ¦   «         ZQe, G dS„ dTeQ¦  «        ¦   «         ZR G dU„ dVeQe¦  «        ZS e,dW¬X¦  «         G dY„ dZeQ¦  «        ¦   «         ZTg d[¢ZUdS )`zPyTorch Zamba model.é    N)ÚCallable)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Ú&use_kernel_func_from_hub_with_fallbackÚuse_kernelized_func)Úforce_accelerate_hooks)Úcreate_causal_maskÚcreate_recurrent_attention_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPastÚ SequenceClassifierOutputWithPast)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmerge_with_config_defaults)Úis_mambapy_availableÚis_torch_greater_or_equalÚ
is_tracing)Úcapture_outputsé   )ÚZambaConfigc                   óT   ‡ — e Zd Zd	deddfˆ fd„Zdej        dej        fd„Zd„ Zˆ xZ	S )
ÚZambaRMSNormç�íµ ÷Æ°>ÚepsÚreturnNc                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )z;
        ZambaRMSNorm is equivalent to T5LayerNorm
        N)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizer(   Ú	__class__s      €úx/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/zamba/modeling_zamba.pyr,   zZambaRMSNorm.__init__8   sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐó    Úhidden_statesc                 ó  — |j         }|                     t          j        ¦  «        }|                     d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        |                     |¦  «        z  S )Né   éÿÿÿÿT)Úkeepdim)	ÚdtypeÚtor.   Úfloat32ÚpowÚmeanÚrsqrtr1   r0   )r2   r7   Úinput_dtypeÚvariances       r5   ÚforwardzZambaRMSNorm.forward@   s|   € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØ ×$Ò$ QÑ'Ô'×,Ò,¨R¸Ð,Ñ>Ô>ˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆØŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:r6   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)Útupler0   Úshaper1   )r2   s    r5   Ú
extra_reprzZambaRMSNorm.extra_reprG   s&   € Ý˜œÔ)Ñ*Ô*ÐIÐI°$Ô2GÐIÐIÐIr6   )r'   )
Ú__name__Ú
__module__Ú__qualname__Úfloatr,   r.   ÚTensorrD   rH   Ú__classcell__©r4   s   @r5   r&   r&   7   sŒ   ø€ € € € € ð$ð $¨ð $¸$ð $ð $ð $ð $ð $ð $ð; U¤\ð ;°e´lð ;ð ;ð ;ð ;ðJð Jð Jð Jð Jð Jð Jr6   r&   r7   Ún_repr)   c                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r#   N)rG   ÚexpandÚreshape)r7   rP   ÚbatchÚnum_key_value_headsÚslenÚhead_dims         r5   Ú	repeat_kvrX   L   s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTr6   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutc                 ó  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        |z  }
|�|
|z   }
t
          j                             |
dt          j        ¬¦  «         	                    |j
        ¦  «        }
t
          j                             |
|| j        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «                             ¦   «         }||
fS )Nr9   r   r:   )Údimr<   )ÚpÚtrainingr#   )rX   Únum_key_value_groupsr.   ÚmatmulÚ	transposer   Ú
functionalÚsoftmaxr>   r=   r<   r`   rd   Ú
contiguous)rZ   r[   r\   r]   r^   r_   r`   ÚkwargsÚ
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r5   Úeager_attention_forwardrp   X   sé   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r6   c                   óÄ   ‡ — e Zd ZdZdedefˆ fd„Z	 ddej        dedej        dz  de	dz  d	e
e         d
eej        ej        dz  eej                 dz  f         fd„Zˆ xZS )ÚZambaAttentionaA  
    Multi-headed attention from 'Attention Is All You Need' paper. Modified to use sliding window attention: Longformer
    and "Generating Long Sequences with Sparse Transformers".

    Adapted from transformers.models.mistral.modeling_mistral.MistralAttention:
    The input dimension here is attention_hidden_size = 2 * hidden_size, and head_dim = attention_hidden_size // num_heads.
    The extra factor of 2 comes from the input being the concatenation of original_hidden_states with the output of the previous (mamba) layer
    (see fig. 2 in https://huggingface.co/papers/2405.16712).
    Additionally, replaced
    attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) with
    attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim/2)
    ÚconfigÚ	layer_idxc                 óŽ  •— t          ¦   «                              ¦   «          || _        || _        |j        | _        |j        | _        |j        |j        z  | _	        |j
        | _
        | j        dz  dz  | _        d| _        |j        | _        t          j        |j        |j        | j        z  d¬¦  «        | _        t          j        |j        |j        | j        z  d¬¦  «        | _        t          j        |j        |j        | j        z  d¬¦  «        | _        t          j        |j        | j        z  |j        d¬¦  «        | _        d S )Nr9   ç      à¿TF©Úbias)r+   r,   rs   rt   Úattention_hidden_sizeÚattention_head_dimrW   Únum_attention_headsrU   re   Úmax_position_embeddingsr_   Ú	is_causalÚattention_dropoutr   ÚLinearÚq_projÚk_projÚv_projr3   Úo_proj©r2   rs   rt   r4   s      €r5   r,   zZambaAttention.__init__   s!  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒà%+Ô%AˆÔ"ØÔ1ˆŒØ$*Ô$>À&ÔB\Ñ$\ˆÔ!Ø'-Ô'EˆÔ$Øœ¨Ñ)¨dÑ2ˆŒØˆŒØ!'Ô!9ˆÔå”i Ô <¸fÔ>XÐ[_Ô[hÑ>hÐotÐuÑuÔuˆŒÝ”i Ô <¸fÔ>XÐ[_Ô[hÑ>hÐotÐuÑuÔuˆŒÝ”i Ô <¸fÔ>XÐ[_Ô[hÑ>hÐotÐuÑuÔuˆŒÝ”i Ô :¸T¼]Ñ JÈFÔL^ÐejÐkÑkÔkˆŒˆˆr6   Nr7   r^   Úpast_key_valuesrk   r)   c                 óä  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }	|                      |¦  «                             |¦  «                             dd¦  «        }
|�|                     |	|
|¦  «        \  }	}
t          j	        | j
        j        t          ¦  «        } || ||	|
|f| j        sdn| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Nr:   r#   r9   rY   )r`   r_   )rG   rW   r€   Úviewrg   r�   r‚   Úupdater   Úget_interfacers   Ú_attn_implementationrp   rd   r~   r_   rS   rj   rƒ   )r2   r7   rt   r^   r…   rk   Úinput_shapeÚhidden_shapeÚquery_statesrl   rm   Úattention_interfacero   rn   s                 r5   rD   zZambaAttention.forward‘   sš  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆàÐ&Ø'6×'=Ò'=¸jÈ,ÐXaÑ'bÔ'bÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r6   ©N)rI   rJ   rK   Ú__doc__r$   Úintr,   r.   rM   r   r   r   rF   rD   rN   rO   s   @r5   rr   rr   q   sé   ø€ € € € € ðð ðl˜{ð l°sð lð lð lð lð lð lð. )-ð#)ð #)à”|ð#)ð ð#)ð œ tÑ+ð	#)ð
  ™ð#)ð Ð+Ô,ð#)ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	Mð#)ð #)ð #)ð #)ð #)ð #)ð #)ð #)r6   rr   c                 ób   — |�,| j         }| |dd…dd…df         z                       |¦  «        } | S )zm
    Tunes out the hidden states for padding tokens, see https://github.com/state-spaces/mamba/issues/66
    N)r<   r=   )r7   r^   r<   s      r5   Úapply_mask_to_padding_statesr“   ·   sC   € ð
 Ð!ØÔ#ˆØ&¨¸¸¸¸1¸1¸1¸d¸
Ô)CÑC×GÒGÈÑNÔNˆàÐr6   Úcausal_conv1d_updateÚcausal_conv1dÚ
conv_stater0   rx   Ú
activationc                 óº  — | j         \  }}}|j         d         }t          j        || gd¬¦  «                             |j        ¦  «        }	|                     |	d d …d d …| d …f         ¦  «         t          j        |	|                     d¦  «        |d|¬¦  «        }
|
d d …d d …| d …f         }
|�t          |         |
¦  «        }
|
                     | j        ¦  «        S )Nr:   ©rb   r#   r   )ÚpaddingÚgroups)
rG   r.   Úcatr=   r<   Úcopy_ÚFÚconv1dÚ	unsqueezer
   )r7   r–   r0   rx   r—   Ú_r3   Úseq_lenÚ	state_lenÚhidden_states_newÚouts              r5   r”   r”   Ã   sî   € ð ,Ô1Ñ€A€{�GØÔ  Ô$€Iåœ	 :¨}Ð"=À2ÐFÑFÔF×IÒIÈ&Ì,ÑWÔWÐØ×ÒÐ& q q q¨!¨!¨!¨i¨Z¨[¨[Ð'8Ô9Ñ:Ô:Ð:Ý
Œ(Ð$ f×&6Ò&6°qÑ&9Ô&9¸4ÈÐS^Ð
_Ñ
_Ô
_€CØ
ˆaˆaˆa����W�H�I�IˆoÔ
€CØÐÝ�ZÔ  Ñ%Ô%ˆØ�6Š6�-Ô%Ñ&Ô&Ð&r6   Úcausal_conv1d_fnc                 ó@  — | j         \  }}}|j         d         dz
  }t          j        |                      |j        ¦  «        |                     d¦  «        |||¬¦  «        d d …d d …d |…f         }	|�t          |         |	¦  «        }	|	                     | j        ¦  «        S )Nr:   r#   )r0   rx   rš   r›   )rG   rž   rŸ   r=   r<   r    r
   )
r7   r0   rx   r—   rk   r¡   r3   r¢   rš   r¥   s
             r5   r¦   r¦   ×   s²   € ð ,Ô1Ñ€A€{�GØŒl˜2Ô Ñ"€Gå
Œ(Ø×Ò˜œÑ&Ô&Ø×Ò Ñ"Ô"ØØØðñ ô ð €a€aˆˆˆˆHˆWˆH€nô€Cð ÐÝ�ZÔ  Ñ%Ô%ˆØ�6Š6�-Ô%Ñ&Ô&Ð&r6   Úselective_state_updateÚ	mamba_ssmFÚstateÚdtÚAÚBÚCÚDÚdt_biasÚdt_softplusÚzc
                 ó„  — |j         }|�||                     |j         ¦  «        z   }|rt          j        |¦  «        }t	          j        |                     ¦   «         d         |                     ¦   «         z  ¦  «                             | j        ¬¦  «        }|                     ¦   «         d         |                     ¦   «         d d …d d d …f         z  }||                     ¦   «         d         z  }|                      ¦   «         |z  |z   }|                      |                     | j         ¦  «        ¦  «         t	          j	        |                     |j         ¦  «        | 
                    d¦  «        ¦  «                             d¦  «        }|�|||z  z   }|	�|t          j        |	¦  «        z  }|                     |¦  «        S )N©.N©Údevicer:   )r<   r=   rž   Úsoftplusr.   ÚexprL   r¶   r�   rf   r    ÚsqueezeÚsilu)rª   r7   r«   r¬   r­   r®   r¯   r°   r±   r²   rk   rB   ÚdAÚdBÚdBxÚ	ssm_stater¥   s                    r5   Úmamba_selective_state_updater¿   î   s|  € ð"  Ô%€KàÐØ�'—*’*˜RœXÑ&Ô&Ñ&ˆØð ÝŒZ˜‰^Œ^ˆõ 
Œ�2—8’8‘:”:˜iÔ(¨1¯7ª7©9¬9Ñ4Ñ	5Ô	5×	8Ò	8ÀÄÐ	8Ñ	MÔ	M€Bð 
�Š‰Œ�IÔ	 §¢¡¤¨1¨1¨1¨d°A°A°A¨:Ô!6Ñ	6€Bà
ˆ}×"Ò"Ñ$Ô$ YÔ/Ñ
/€Cð —’‘” Ñ" SÑ(€IØ	‡K‚K�	—’˜Uœ[Ñ)Ô)Ñ*Ô*Ð*õ Œ,�y—|’| A¤GÑ,Ô,¨a¯kªk¸"©o¬oÑ
>Ô
>×
FÒ
FÀrÑ
JÔ
J€Cð 	€}Ø�M AÑ%Ñ%ˆà€}Ø•A”F˜1‘I”I‰oˆà�6Š6�+ÑÔÐr6   Úselective_scan_fnÚ
delta_biasÚdelta_softplusÚreturn_last_stateÚuse_mambapyÚuse_associative_scanc           	      ó4  — t          d¦  «        rddlm} nd }t          ¦   «         rddlm} nd }| j        \  }}}| j        }|�#||                     |j        ¦  «        d         z   }|rt          j
        |¦  «        }|                     dd¦  «        }|                     dd¦  «        }t          j        |d d d …d d d …f         |d d …d d …d d …d f         z  ¦  «        }|d d …d d …d d …d f         |d d …d d d …d d …f                              ¦   «         z  }|| d d …d d …d d …d f                              ¦   «         z  }|
rƒ|�� ||                     dd¦  «        |                     dd¦  «        ¦  «        }||                     d¦  «        z                       d	¦  «                             dd¦  «        }|d d …df         }�n£|rÂ|�Àt#          | ¦  «        r±d
„ }|j        j        dv rdnd} ||||fd|¬¦  «        \  }}t          j        |                     dd¦  «                             |¦  «        |                     d¦  «        ¦  «        }|                     d¦  «                             dd¦  «        }|d d …d d …df         }nßt          j        |||j        d         || j        ¬¦  «        }g }t-          |¦  «        D ]Ž}|d d …d d …|f         |z  |d d …d d …|f         z   }t          j        |                     |¦  «        |d d …|d d …f                              d¦  «        ¦  «        }|                     |d d …d d …df         ¦  «         Œ�t          j        |d¬¦  «        }|�|| |d d d …d f         z  z   }|�|t          j        |¦  «        z  }|	r||fS |S )Nz2.9.0r   )Úassociative_scan)Úpscanr´   r#   r9   r:   r   c                 ó0   — | \  }}|\  }}||z  ||z  |z   fS r�   © )ÚleftÚrightÚa_leftÚb_leftÚa_rightÚb_rights         r5   Ú
combine_fnz(mamba_selective_scan.<locals>.combine_fnV  s/   € Ø!‰NˆF�FØ$ÑˆG�WØ˜GÑ# W¨vÑ%5¸Ñ%?Ð?Ð?r6   )ÚcudaÚxpuÚ	pointwiseÚgeneric)rb   Úcombine_mode)r<   r¶   r™   )r    Ú(torch._higher_order_ops.associative_scanrÇ   r   Úmambapy.pscanrÈ   rG   r<   r=   rž   r·   rg   r.   r¸   rL   r    r¹   r!   r¶   Útyperf   ÚzerosÚrangeÚappendÚstackrº   )r7   r«   r¬   r­   r®   r¯   r²   rÁ   rÂ   rÃ   rÄ   rÅ   rk   rÇ   rÈ   Ú
batch_sizeÚintermediate_sizer¢   rB   Ú
discrete_AÚ
discrete_BÚdeltaB_uÚ
all_statesÚscan_outputr¾   rÑ   rÖ   r¡   Úscan_outputsÚindexs                                 r5   Úmamba_selective_scanrç     sO  € õ( ! Ñ)Ô)ð  ØMÐMÐMÐMÐMÐMÐMàÐåÑÔð Ø'Ð'Ð'Ð'Ð'Ð'Ð'àˆà-:Ô-@Ñ*€JÐ! 7ØÔ%€KàÐØ�*—-’- ¤Ñ)Ô)¨)Ô4Ñ4ˆØð ÝŒZ˜‰^Œ^ˆð 	
�Š�A�qÑÔ€AØ	�Š�A�qÑÔ€Aõ ”˜1˜T 1 1 1 d¨A¨A¨AÐ-Ô.°°A°A°A°q°q°q¸!¸!¸!¸T°MÔ1BÑBÑCÔC€JØ�A�A�A�q�q�q˜!˜!˜!˜T�MÔ" Q q q q¨$°°°°1°1°1 }Ô%5×%;Ò%;Ñ%=Ô%=Ñ=€JØ˜M¨!¨!¨!¨Q¨Q¨Q°°°°4¨-Ô8×>Ò>Ñ@Ô@Ñ@€Hàð -8�uÐ(Ø�U˜:×/Ò/°°1Ñ5Ô5°x×7IÒ7IÈ!ÈQÑ7OÔ7OÑPÔPˆ
à! A§K¢K°¡O¤OÑ3×<Ò<¸QÑ?Ô?×IÒIÈ!ÈQÑOÔOˆØ˜q˜q˜q "˜uÔ%ˆ	‰	à	ð '8Ð"2Ð">Å:ÈmÑC\ÔC\Ð">ð	@ð 	@ð 	@ð
 '1Ô&7Ô&<ÀÐ&OÐ&O�{�{ÐU^ˆØ(Ð(ØØ˜Ð"ØØ%ð	
ñ 
ô 
‰ˆˆ:õ ”l :×#7Ò#7¸¸1Ñ#=Ô#=×#@Ò#@ÀÑ#MÔ#MÈqÏ{Ê{Ð[]ÉÌÑ_Ô_ˆØ!×)Ò)¨"Ñ-Ô-×7Ò7¸¸1Ñ=Ô=ˆØ˜q˜q˜q ! ! ! R˜xÔ(ˆ	ˆ	õ ”KØØØŒG�BŒKØØ Ô'ð
ñ 
ô 
ˆ	ð ˆÝ˜7‘^”^ð 	6ð 	6ˆEà" 1 1 1 a a a¨ ;Ô/°)Ñ;¸hÀqÀqÀqÈ!È!È!ÈUÀ{Ô>SÑSˆIõ  œ, y§|¢|°KÑ'@Ô'@À!ÀAÀAÀAÀuÈaÈaÈaÀKÄ.×BZÒBZÐ[]ÑB^ÔB^Ñ_Ô_ˆKØ×Ò ¨A¨A¨A¨q¨q¨q°!¨GÔ 4Ñ5Ô5Ð5Ð5Ý”k ,°BÐ7Ñ7Ô7ˆà€}Ø! M°A°d¸A¸A¸A¸t°mÔ4DÑ$DÑDˆà€}Ø!¥A¤F¨1¡I¤IÑ-ˆàð &Ø˜IÐ%Ð%àÐr6   c            	       ó„   ‡ — e Zd ZdZdefˆ fd„Z ed¦  «        	 	 d
dej        de	dz  dej
        dz  fd	„¦   «         Zˆ xZS )ÚZambaMambaMixeruE  
    Compute âˆ†, A, B, C, and D the state space parameters and compute the `contextualized_states`.
    A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
    âˆ†, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
    and is why Mamba is called **selective** state spaces)

    This module differs from `transformers.models.mamba.modeling_mamba.MambaMixer` in two ways:
    - Added multi-head: the output of `self.in_proj` is split into `self.n_mamba_heads` heads, and each head
    undergoes an independent forward pass, identical to the original `MambaMixer`, up until the pre-activations of
    `self.out_proj`. The pre-activations, coming from different mamba heads, are then concatenated and fed into `self.out_proj`.
    rs   c           	      ód  •— t          ¦   «                              ¦   «          || _        || _        |j        | _        |j        | _        |j        | _        |j	        |j        z  | _
        |j        | _        |j        | _        | j
        | j        z  | _        |j        | _        |j        | _        t'          j        | j
        | j
        | j        | j        | j
        | j        dz
  ¬¦  «        | _        |j        | _        t0          |j                 | _        |j        | _        t'          j        | j        | j
        dz  | j        ¬¦  «        | _        t'          j        t?          j         | j        | j        | j        dz  z   | j        ¦  «        ¦  «        | _!        t'          j        t?          j         | j        | j        | j        ¦  «        dz
  dz  | j        dz  z  ¦  «        | _"        t'          j        t?          j         | j        | j        ¦  «        ¦  «        | _#        t?          j$        d| j        dz   t>          j%        ¬¦  «        d d d …f         }| &                    | j
        d¦  «         '                    ¦   «         }t'          j        t?          j(        |¦  «         )                    | j        | j        d¦  «        ¦  «        | _*        t'          j        t?          j+        | j        | j        ¦  «        ¦  «        | _,        t'          j        | j
        | j        | j        ¬¦  «        | _-        |j.        |         | _/        d S )Nr#   )Úin_channelsÚout_channelsrx   Úkernel_sizer›   rš   r9   rw   g      à?©r<   r:   )0r+   r,   rs   rt   r3   Úmamba_d_stateÚssm_state_sizeÚmamba_d_convÚconv_kernel_sizeÚmamba_expandrß   Úmamba_dt_rankÚtime_step_rankÚn_mamba_headsÚmamba_head_dimÚmamba_conv_biasÚuse_conv_biasÚmamba_proj_biasÚuse_biasr   ÚConv1drŸ   Úhidden_mamba_actr—   r
   ÚactÚuse_mamba_kernelsÚuse_fast_kernelsr   Úin_projr-   r.   rÚ   Úx_proj_weightÚdt_proj_weightÚdt_proj_biasÚaranger>   rR   rj   ÚlogrS   ÚA_logr/   r¯   Úout_projÚlayer_typesÚ
layer_type)r2   rs   rt   r¬   r4   s       €r5   r,   zZambaMambaMixer.__init__—  s¿  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒØ!Ô-ˆÔØ$Ô2ˆÔØ &Ô 3ˆÔØ!'Ô!4°vÔ7IÑ!IˆÔØ$Ô2ˆÔØ#Ô1ˆÔØ"Ô4¸Ô8JÑJˆÔØ#Ô3ˆÔØÔ.ˆŒÝ”iØÔ.ØÔ/ØÔ#ØÔ-ØÔ)ØÔ)¨AÑ-ð
ñ 
ô 
ˆŒð !Ô1ˆŒÝ˜&Ô1Ô2ˆŒà &Ô 8ˆÔõ ”y Ô!1°4Ô3IÈAÑ3MÐTXÔTaÐbÑbÔbˆŒõ  œ\ÝŒKØÔ"ØÔ# dÔ&9¸AÑ&=Ñ=ØÔ#ñô ñ
ô 
ˆÔõ !œlÝŒ[˜Ô+¨TÔ-@À$ÔBUÑVÔVÐY\Ñ\ØñàÔ! 3Ñ&ñ'ñ
ô 
ˆÔõ
 œL­¬°TÔ5GÈÔI\Ñ)]Ô)]Ñ^Ô^ˆÔõ ŒL˜˜DÔ/°!Ñ3½5¼=ÐIÑIÔIÈ$ÐPQÐPQÐPQÈ'ÔRˆØ�HŠH�TÔ+¨RÑ0Ô0×;Ò;Ñ=Ô=ˆÝ”\¥%¤)¨A¡,¤,×"6Ò"6°tÔ7IÈ4ÔK^Ð`bÑ"cÔ"cÑdÔdˆŒ
Ý”�eœj¨Ô);¸TÔ=PÑQÔQÑRÔRˆŒÝœ	 $Ô"8¸$Ô:JÐQUÔQ^Ð_Ñ_Ô_ˆŒà Ô,¨YÔ7ˆŒˆˆr6   rŸ   Nr7   Úcache_paramsr^   c                 óH  — |j         \  }}}|j        }|d uo|                     | j        ¦  «        }	t	          ||¦  «        }|                      |¦  «                             dd¦  «        }
|
                     |dd|¦  «                             dd¬¦  «        \  }}| 	                    d¦  «         
                    ¦   «         }|                     || j        | j        |¦  «                             dd¦  «        }|	r:|j        | j                 j        d         }|j        | j                 j        d         }|	r^|dk    rX|j        | j                 j        sAt%          ||| j        j         	                    d¦  «        | j        j        | j        ¬¦  «        }n‹|�"|                     || j        | j        ¬¦  «        }t3          || j        j         	                    d¦  «        | j        j        | j        |                     d¦  «        ¬	¦  «        }|�|d d …d d …| d …f         }t	          |                     dd¦  «        |¦  «        }|                     dd¦  «        }|                     || j        | j        |¦  «                             dd¦  «        }| j        d d …d f         |z                       dd
¦  «        }t9          j        || j        | j        | j        gd¬¦  «        \  }}}| j         d d …d f         |                     dd
¦  «        z  }| j!        �| j!         "                    ¦   «         nd }t9          j#        | j$         "                    ¦   «         ¦  «         }|	rÎ|dk    rÈg }tK          | j        ¦  «        D ]™}tM          |d d …|f         ||ddf         ||ddf         ||         ||d d …df         ||d d …df         | j'        |         ||ddf         |�||         nd d¬¦
  «
         (                    d¦  «        }| )                    |¦  «         Œšt9          j*        |d¬¦  «        }�n=|d u}g }g }tK          | j        ¦  «        D ]×}tW          ||         ||         ||         ||                              dd¦  «        ||                              dd¦  «        | j'        |          "                    ¦   «         ||         |�||         nd d|dd¬¦  «        }|r0|\  }}| )                    |¦  «         | )                    |¦  «         ŒÂ| )                    |¦  «         ŒØt9          j*        |d¬¦  «        }|r1t9          j,        |d¬¦  «        }| -                    || j        ¦  «         |  .                    |                     dd¦  «         /                    |¦  «        ¦  «        }|S )Nr#   r9   r:   r™   r   )r—   )rò   Úseq_idx)r—   r  éþÿÿÿ.T)r²   r°   r±   F)r¯   r²   rÁ   rÂ   rÃ   rÄ   rÅ   )0rG   r<   Úhas_previous_statert   r“   r  rg   r‡   Úchunkr¹   rj   rS   rö   r÷   ÚlayersÚconv_statesÚrecurrent_statesÚrecord_pastr”   rŸ   r0   rx   r—   Úupdate_conv_staterò   r¦   Úgetr  r.   Úsplitrõ   rð   r  r  rL   r¸   r  rÛ   r¿   r¯   r    rÜ   rœ   rç   rÝ   Úupdate_recurrent_stater  r=   )r2   r7   r  r^   rk   rÞ   r¢   r¡   r<   Úuse_precomputed_statesÚprojected_statesÚhidden_states_B_CÚgater–   Úrecurrent_stateÚssm_parametersÚ	time_stepr­   r®   Útime_proj_biasr¬   rå   Úhead_idxrä   Úoutput_final_stateÚfinal_statesÚscan_resultÚhead_outputÚfinal_stateÚcontextualized_statess                                 r5   rD   zZambaMambaMixer.forwardÏ  s-  € ð "/Ô!4Ñˆ
�G˜QØÔ#ˆØ!-°TÐ!9Ð!m¸l×>]Ò>]Ð^bÔ^lÑ>mÔ>mÐõ 5°]ÀNÑSÔSˆØŸ<š<¨Ñ6Ô6×@Ò@ÀÀAÑFÔFÐð #3×"7Ò"7¸
ÀBÈÈ7Ñ"SÔ"S×"YÒ"YÐZ[ÐabÐ"YÑ"cÔ"cÑÐ˜4Ø-×5Ò5°aÑ8Ô8×CÒCÑEÔEÐØ�|Š|˜J¨Ô(:¸DÔ<OÐQXÑYÔY×cÒcÐdeÐghÑiÔiˆà!ð 	VØ%Ô,¨T¬^Ô<ÔHÈÔKˆJØ*Ô1°$´.ÔAÔRÐSTÔUˆOð "ð 	G g°¢l l¸<Ô;NÈtÌ~Ô;^Ô;j lÝ 4Ø!ØØ”Ô"×*Ò*¨1Ñ-Ô-Ø”Ô Øœ?ð!ñ !ô !ÐÐð Ð'Ø$0×$BÒ$BØ%Ø”NØ%)Ô%:ð %Cñ %ô %Ð!õ !1Ø!Ø”Ô"×*Ò*¨1Ñ-Ô-Ø”Ô Øœ?ØŸ
š
 9Ñ-Ô-ð!ñ !ô !Ðð Ð'Ø$5°a°a°a¸¸¸¸W¸H¸I¸I°oÔ$FÐ!õ 9Ø×'Ò'¨¨1Ñ-Ô-Øñ
ô 
Ðð .×7Ò7¸¸1Ñ=Ô=ÐØ-×5Ò5ØØÔØÔØñ	
ô 
÷
 Š)�A�q‰/Œ/ð 	ð Ô,¨Q¨Q¨Q°¨WÔ5Ð8IÑI×TÒTÐUWÐY[Ñ\Ô\ˆÝœ+ØØÔ  $Ô"5°tÔ7JÐKØð
ñ 
ô 
‰ˆ	�1�að Ô'¨¨¨¨4¨Ô0°9×3FÒ3FÀrÈ2Ñ3NÔ3NÑNˆ	Ø6:Ô6GÐ6S˜Ô*×0Ò0Ñ2Ô2Ð2ÐY]ˆÝŒY�t”z×'Ò'Ñ)Ô)Ñ*Ô*Ð*ˆð "ð 6	Q g°¢l lØˆLå! $Ô"4Ñ5Ô5ð 1ð 1�Ý:Ø# A A A x KÔ0Ø% h°°QÐ&6Ô7Ø˜h¨¨QÐ.Ô/Ø�h”KØ�h    1�nÔ%Ø�h    1�nÔ%Ø”F˜8Ô$Ø˜8 S¨!Ð+Ô,Ø8FÐ8R˜N¨8Ô4Ð4ÐX\Ø $ðñ ô ÷ ’)˜B‘-”-ð ð ×#Ò# KÑ0Ô0Ð0Ð0åœ) L°aÐ8Ñ8Ô8ˆK‰Kð ".°TÐ!9ÐØˆLØˆLå! $Ô"4Ñ5Ô5ð 5ð 5�Ý2Ø% hÔ/Ø˜hÔ'Ø�h”KØ�h”K×)Ò)¨!¨QÑ/Ô/Ø�h”K×)Ò)¨!¨QÑ/Ô/Ø”f˜XÔ&×,Ò,Ñ.Ô.Ø˜8”nØ;IÐ;U˜~¨hÔ7Ð7Ð[_Ø#'Ø&8à %Ø).ðñ ô �ð  &ð 5Ø/:Ñ,�K Ø ×'Ò'¨Ñ4Ô4Ð4Ø ×'Ò'¨Ñ4Ô4Ð4Ð4à ×'Ò'¨Ñ4Ô4Ð4Ð4åœ) L°aÐ8Ñ8Ô8ˆKà!ð QÝ#œk¨,¸AÐ>Ñ>Ô>�Ø×3Ò3°KÀÄÑPÔPÐPð !%§¢¨k×.CÒ.CÀAÀqÑ.IÔ.I×.LÒ.LÈUÑ.SÔ.SÑ TÔ TÐØ$Ð$r6   ©NN)rI   rJ   rK   r�   r$   r,   r   r.   rM   r   Ú
LongTensorrD   rN   rO   s   @r5   ré   ré   ‰  s¾   ø€ € € € € ð
ð 
ð68˜{ð 68ð 68ð 68ð 68ð 68ð 68ðp Ð˜HÑ%Ô%ð &*Ø26ð	I%ð I%à”|ðI%ð ˜d‘lðI%ð Ô(¨4Ñ/ð	I%ð I%ð I%ñ &Ô%ðI%ð I%ð I%ð I%ð I%r6   ré   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚZambaMLPc                 ó˜  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _	        t          |j                 | _        d S ©NFrw   )r+   r,   rs   r3   rß   r   r   Ú	gate_projÚup_projÚ	down_projr
   Ú
hidden_actÚact_fn©r2   rs   r4   s     €r5   r,   zZambaMLP.__init__^  s¦   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ!'Ô!9ˆÔÝœ 4Ô#3°TÔ5KÐRWÐXÑXÔXˆŒÝ”y Ô!1°4Ô3IÐPUÐVÑVÔVˆŒÝœ 4Ô#9¸4Ô;KÐRWÐXÑXÔXˆŒÝ˜VÔ.Ô/ˆŒˆˆr6   c                 ó¨   — |                       |                      |                      |¦  «        ¦  «        |                      |¦  «        z  ¦  «        }|S r�   )r0  r2  r.  r/  )r2   Úxr0  s      r5   rD   zZambaMLP.forwardh  sA   € Ø—N’N 4§;¢;¨t¯~ª~¸aÑ/@Ô/@Ñ#AÔ#AÀDÇLÂLÐQRÁOÄOÑ#SÑTÔTˆ	ØÐr6   )rI   rJ   rK   r,   rD   rN   rO   s   @r5   r+  r+  ]  sG   ø€ € € € € ð0ð 0ð 0ð 0ð 0ðð ð ð ð ð ð r6   r+  c                   óà   ‡ — e Zd Zddededz  fˆ fd„Z	 	 	 ddej        dej        dedej        dz  d	edz  d
e	dz  de
e         deej        eej        ej        f         dz  f         fd„Zˆ xZS )ÚZambaAttentionDecoderLayerNrs   rt   c                 ó  •— t          ¦   «                              ¦   «          t          ||¦  «        | _        t	          |¦  «        | _        t          |j        |j        ¬¦  «        | _	        t          |j
        |j        ¬¦  «        | _        d S )N©r(   )r+   r,   rr   Ú	self_attnr+  Úfeed_forwardr&   ry   Úrms_norm_epsÚinput_layernormr3   Úpre_ff_layernormr„   s      €r5   r,   z#ZambaAttentionDecoderLayer.__init__n  sv   ø€ Ý‰Œ×ÒÑÔÐÝ'¨°	Ñ:Ô:ˆŒå$ VÑ,Ô,ˆÔÝ+¨FÔ,HÈfÔNaÐbÑbÔbˆÔÝ ,¨VÔ-?ÀVÔEXÐ YÑ YÔ YˆÔÐÐr6   Fr7   Úoriginal_hidden_statesr^   r…   Ú	use_cacherk   r)   c           	      óà   — t          j        ||gd¬¦  «        }|                      |¦  «        } | j        d|||||dœ|¤Ž\  }}|                      |¦  «        }|                      |¦  «        }|S )ac  
        Args:
            hidden_states (`torch.FloatTensor`): output of previous Mamba layer of shape `(batch, seq_len, embed_dim)`
            original_hidden_states (`torch.FloatTensor`): word embedding output of shape `(batch, seq_len, embed_dim)`.
                This is concatenated with `hidden_states` (which is the output of the previous (mamba) layer). The
                concatenated tensor is then used as input of the pre-attention RMSNorm
                (see fig. 2 in https://huggingface.co/papers/2405.16712).
            layer_idx (`int`): layer_idx in the forward pass. Used to distinguish Zamba's tied transformer layers.
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, sequence_length)` where padding elements are indicated by 0.
            past_key_values (`Cache`, *optional*): cached past key and value projection states
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
        r:   r™   )r7   rt   r^   r…   r@  rÊ   )r.   Úconcatenater=  r:  r>  r;  )	r2   r7   r?  rt   r^   r…   r@  rk   r¡   s	            r5   rD   z"ZambaAttentionDecoderLayer.forwardv  sœ   € õ2 Ô)¨=Ð:PÐ*QÐWYÐZÑZÔZˆØ×,Ò,¨]Ñ;Ô;ˆØ)˜4œ>ð 
Ø'ØØ)Ø+Øð
ð 
ð ð
ð 
Ñˆ�qð ×-Ò-¨mÑ<Ô<ˆØ×)Ò)¨-Ñ8Ô8ˆàÐr6   r�   )NNF)rI   rJ   rK   r$   r‘   r,   r.   rM   r   Úboolr   r   rF   ÚFloatTensorrD   rN   rO   s   @r5   r7  r7  m  s  ø€ € € € € ðZð Z˜{ð Z°s¸T±zð Zð Zð Zð Zð Zð Zð /3Ø(,Ø!&ð'ð 'à”|ð'ð !&¤ð'ð ð	'ð
 œ tÑ+ð'ð  ™ð'ð ˜$‘;ð'ð Ð+Ô,ð'ð 
ˆuÔ  %¨Ô(9¸5Ô;LÐ(LÔ"MÐPTÑ"TÐTÔ	Uð'ð 'ð 'ð 'ð 'ð 'ð 'ð 'r6   r7  c                   óð   ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 	 	 	 ddej        dej        dz  dedz  dej        dz  d	ej        dz  d
edz  de	dz  dej
        dz  dej        dz  dee         dej        fd„Zˆ xZS )ÚZambaMambaDecoderLayerrs   rt   c                 óÂ   •— t          ¦   «                              ¦   «          t          ||¬¦  «        | _        t	          |j        |j        ¬¦  «        | _        || _        d S )N)rs   rt   r9  )	r+   r,   ré   Úmambar&   r3   r<  r=  rt   r„   s      €r5   r,   zZambaMambaDecoderLayer.__init__¡  sS   ø€ Ý‰Œ×ÒÑÔÐÝ$¨F¸iÐHÑHÔHˆŒ
Ý+¨FÔ,>ÀFÔDWÐXÑXÔXˆÔØ"ˆŒˆˆr6   NFr7   r?  r^   Úcausal_maskr…   r@  Úposition_idsÚtransformer_hidden_statesrk   r)   c
                 ór   — |}|	�||	z   n|}|                       |¦  «        } | j        d|||dœ|
¤Ž}||z   }|S )aX  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, sequence_length)` where padding elements are indicated by 0.
            past_key_values (`Cache`, *optional*): cached past key and value projection states
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
        N)r7   r  r^   rÊ   )r=  rH  )r2   r7   r?  rt   r^   rI  r…   r@  rJ  rK  rk   Úresiduals               r5   rD   zZambaMambaDecoderLayer.forward§  s|   € ð0 !ˆð
 :SÐ9^ˆMÐ5Ñ5Ð5Ðdqð 	ð ×,Ò,¨]Ñ;Ô;ˆà"˜œ
ð 
Ø'Ø(Ø)ð
ð 
ð ð	
ð 
ˆð ! =Ñ0ˆàÐr6   )NNNNNFNN)rI   rJ   rK   r$   r‘   r,   r.   rM   r   rC  r)  r   r   rD   rN   rO   s   @r5   rF  rF     s$  ø€ € € € € ð#˜{ð #°sð #ð #ð #ð #ð #ð #ð 7;Ø $Ø.2Ø+/Ø(,Ø!&Ø04Ø9=ð*ð *à”|ð*ð !&¤¨tÑ 3ð*ð ˜‘:ð	*ð
 œ tÑ+ð*ð ”\ DÑ(ð*ð  ™ð*ð ˜$‘;ð*ð Ô&¨Ñ-ð*ð $)¤<°$Ñ#6ð*ð Ð+Ô,ð*ð 
Œð*ð *ð *ð *ð *ð *ð *ð *r6   rF  c                   ó  ‡ — e Zd Zdedej        defˆ fd„Z	 	 	 	 	 	 ddej	        dej	        dz  d	e
dz  d
ej	        dz  dej	        dz  dedz  dedz  dee         deej        eej        ej        f         dz  f         fd„Zˆ xZS )ÚZambaHybridLayerÚshared_transfÚlinearrH  c                 ór   •— t          ¦   «                              ¦   «          || _        || _        || _        d S r�   )r+   r,   rP  rQ  Úmamba_decoder)r2   rP  rQ  rH  r4   s       €r5   r,   zZambaHybridLayer.__init__Õ  s6   ø€ Ý‰Œ×ÒÑÔÐØ*ˆÔØˆŒØ"ˆÔÐÐr6   NFr7   r?  rt   r^   rI  r…   r@  rk   r)   c           	      ó~   —  | j         |f|||||dœ|¤Ž}	|                      |	¦  «        }	 | j        |f|	|||dœ|¤Ž}|S )aF  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            original_hidden_states (`torch.FloatTensor`): word embedding output that will be concatenated with
            hidden activations to form the input of the shared transformer layer.
            layer_idx (`int`): layer number.
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, sequence_length)` where padding elements are indicated by 0.
            past_key_values (`Cache`, *optional*): cached past key and value projection states
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
        )r?  rt   r^   r…   r@  )rK  r^   r…   r@  )rP  rQ  rS  )
r2   r7   r?  rt   r^   rI  r…   r@  rk   rK  s
             r5   rD   zZambaHybridLayer.forwardÛ  s“   € ð2 %7 DÔ$6Øð%
à#9ØØ&Ø+Øð%
ð %
ð ð%
ð %
Ð!ð %)§K¢KÐ0IÑ$JÔ$JÐ!à*˜Ô*Øð
à&?Ø)Ø+Øð
ð 
ð ð
ð 
ˆð Ðr6   )NNNNNF)rI   rJ   rK   r7  r   r   rF  r,   r.   rM   r‘   r   rC  r   r   rF   rD  rD   rN   rO   s   @r5   rO  rO  Ô  s'  ø€ € € € € ð#Ð&@ð #È"Ì)ð #Ð\rð #ð #ð #ð #ð #ð #ð 7;Ø $Ø.2Ø+/Ø(,Ø!&ð-ð -à”|ð-ð !&¤¨tÑ 3ð-ð ˜‘:ð	-ð
 œ tÑ+ð-ð ”\ DÑ(ð-ð  ™ð-ð ˜$‘;ð-ð Ð+Ô,ð-ð 
ˆuÔ  %¨Ô(9¸5Ô;LÐ(LÔ"MÐPTÑ"TÐTÔ	Uð-ð -ð -ð -ð -ð -ð -ð -r6   rO  c                   ó€   ‡ — e Zd ZU eed<   dZdZddgZdgZdZ	dZ
dZeedœZ ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )	ÚZambaPreTrainedModelrs   ÚmodelTrO  rF  r…   )r7   Ú
attentionsc                 óÖ  •— | j         j        }t          ¦   «                              |¦  «         t	          |t
          ¦  «        �r$t          j        |j        d|¬¦  «         | j         j	        dz  }t          j
        |j        | |¦  «         | j         j        | j         j        z  | j         j        z  }t          j        t          j        | j         j        |¦  «        t%          j        | j         j        ¦  «        t%          j        | j         j        ¦  «        z
  z  t%          j        | j         j        ¦  «        z   ¦  «                             | j         j        ¬¦  «        }|t          j        t          j        | ¦  «         ¦  «        z   }t          j        |j        |¦  «         t          j        d|j        dz   t          j        ¬¦  «        d d d …f         }|                     |j        d¦  «                              ¦   «         }t          j        |j!        t          j        |¦  «         "                    |j        |j#        d¦  «        ¦  «         t          j$        |j%        ¦  «         d S d S )NrY   )r@   Ústdrv   )Úminr#   rî   r:   )&rs   Úinitializer_ranger+   Ú_init_weightsÚ
isinstanceré   ÚinitÚnormal_r  rô   Úuniform_r  ró   r3   rö   r.   r¸   ÚrandÚmathr  Útime_step_maxÚtime_step_minÚclampÚtime_step_floorÚexpm1r�   r  r  rð   r>   rR   rß   rj   r  rS   r÷   Úones_r¯   )	r2   rZ   rZ  Údt_init_stdr÷   r«   Úinv_dtr¬   r4   s	           €r5   r]  z"ZambaPreTrainedModel._init_weights  s   ø€ àŒkÔ+ˆÝ‰Œ×Ò˜fÑ%Ô%Ð%Ý�f�oÑ.Ô.ñ 	!ÝŒL˜Ô-°C¸SÐAÑAÔAÐAØœ+Ô3°TÑ9ˆKÝŒM˜&Ô/°+°¸{ÑKÔKÐKà!œ[Ô5¸¼Ô8OÑOÐSWÔS^ÔSlÑlˆNÝ”Ý”
˜4œ;Ô4°nÑEÔEÝ”8˜DœKÔ5Ñ6Ô6½¼À$Ä+ÔB[Ñ9\Ô9\Ñ\ñ^å”(˜4œ;Ô4Ñ5Ô5ñ6ñô ÷ Še˜œÔ3ˆeÑ4Ô4ð	 ð �%œ)¥U¤[°"°Ñ%5Ô%5Ð$5Ñ6Ô6Ñ6ˆFÝŒJ�vÔ*¨FÑ3Ô3Ð3å”˜Q Ô 5¸Ñ 9ÅÄÐOÑOÔOÐPTÐVWÐVWÐVWÐPWÔXˆAØ—’˜Ô1°2Ñ6Ô6×AÒAÑCÔCˆAÝŒJ�v”|¥U¤Y¨q¡\¤\×%9Ò%9¸&Ô:NÐPVÔPeÐgiÑ%jÔ%jÑkÔkÐkÝŒJ�v”xÑ Ô Ð Ð Ð ð%	!ð 	!r6   )rI   rJ   rK   r$   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_is_statefulrF  rr   Ú_can_record_outputsr.   Úno_gradr]  rN   rO   s   @r5   rV  rV    sž   ø€ € € € € € àÐÐÑØÐØ&*Ð#Ø+Ð-EÐFÐØ#4Ð"5ÐØÐØ€NØ€Là/Ø$ðð Ðð
 €U„]�_„_ð!ð !ð !ð !ñ „_ð!ð !ð !ð !ð !r6   rV  c                   óì   ‡ — e Zd ZdZdefˆ fd„Zeee	 	 	 	 	 	 dde	j
        dz  de	j        dz  de	j
        dz  dedz  d	e	j        dz  d
edz  dee         deez  fd„¦   «         ¦   «         ¦   «         Zˆ xZS )Ú
ZambaModelz›
    Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`ZambaDecoderLayer`]

    Args:
        config: ZambaConfig
    rs   c                 ó  •— t          ¦   «                              |¦  «         |j        | _        |j        | _        t          j        |j        |j        | j        ¦  «        | _        |j	        | _	        g }d | _
        t          | j	        ¦  «        D ]¦\  }}t          ||¬¦  «        }|dk    rut          j        | j        j        | j        j        d¬¦  «        }|                     t!          t#          |¦  «        ||¦  «        ¦  «         | j
        €d|› d�d|› d�i| _
        Œ‘|                     |¦  «         Œ§t          j        |¦  «        | _        t)          |j        |j        ¬	¦  «        | _        d| _        |                      ¦   «          d S )
N)rt   ÚhybridFrw   z
layers.(?!z\.)\d+.shared_transfzlayers.z.shared_transfr9  )r+   r,   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr3   Úembed_tokensÚlayers_block_typeÚ_tied_weights_keysÚ	enumeraterF  r   rs   rÜ   rO  r7  Ú
ModuleListr  r&   r<  Úfinal_layernormÚgradient_checkpointingÚ	post_init)r2   rs   r  Úlayer_idr
  rH  rQ  r4   s          €r5   r,   zZambaModel.__init__<  s�  ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒåœL¨Ô):¸FÔ<NÐPTÔP`ÑaÔaˆÔØ!'Ô!9ˆÔØˆØ"&ˆÔÝ$-¨dÔ.DÑ$EÔ$Eð 
	%ð 
	%Ñ ˆH�jÝ*¨6¸XÐFÑFÔFˆEØ˜XÒ%Ð%Ýœ 4¤;Ô#:¸D¼KÔ<SÐZ_Ð`Ñ`Ô`�Ø—’Õ.Õ/IÈ&Ñ/QÔ/QÐSYÐ[`ÑaÔaÑbÔbÐbØÔ*Ð2àD hÐDÐDÐDÐFhÐPXÐFhÐFhÐFhð/�DÔ+øð —’˜eÑ$Ô$Ð$Ð$Ý”m FÑ+Ô+ˆŒå+¨FÔ,>ÀFÔDWÐXÑXÔXˆÔà&+ˆÔ#à�ŠÑÔÐÐÐr6   NÚ	input_idsr^   rJ  r…   Úinputs_embedsr@  rk   r)   c           	      ó¨  — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|}t          j        |¦  «        }	|r|€t	          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}
t          j        |j        d         |j	        ¬¦  «        |
z   }| 
                    d¦  «        }t          |x}t          ¦  «        s&| j        ||||dœ}t          di |¤Žt          di |¤Ždœ}t          | j        ¦  «        D ]!\  }} |||	||d         |d	         f||d
œ|¤Ž}Œ"|                      |¦  «        }t%          ||r|nd ¬¦  «        S )NzaYou cannot specify both input_ids and inputs_embeds at the same time, and must specify either one)rs   r   r#   rµ   )rs   rˆ  r^   r…   rJ  )Úfull_attentionÚlinear_attentionr‹  rŠ  )r…   r@  )Úlast_hidden_stater…   rÊ   )Ú
ValueErrorr~  r.   Úcloner   rs   Úget_seq_lengthr  rG   r¶   r    r^  Údictr   r   r�  r  rƒ  r   )r2   r‡  r^   rJ  r…   rˆ  r@  rk   r7   r?  Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsrt   Úlayers                  r5   rD   zZambaModel.forwardX  sç  € ð ˜Ð -°tÐ";Ñ<ð 	ÝØsñô ð ð Ð Ø ×-Ò-¨iÑ8Ô8ˆMà%ˆå!&¤¨]Ñ!;Ô!;Ðð ð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå°Ð?Ð-ÅÑFÔFð 	àœ+Ø!.Ø"0Ø#2Ø ,ðð ˆKõ #5Ð"CÐ"C°{Ð"CÐ"CÝ$CÐ$RÐ$RÀkÐ$RÐ$Rð#ð #Ðõ
 !*¨$¬+Ñ 6Ô 6ð 
	ð 
	ÑˆI�uØ!˜EØØ&ØØ#Ð$6Ô7Ø#Ð$4Ô5ð	ð !0Ø#ð	ð 	ð ð	ð 	ˆMˆMð ×,Ò,¨]Ñ;Ô;ˆå&Ø+Ø/8ÐB˜O˜O¸dð
ñ 
ô 
ð 	
r6   )NNNNNN)rI   rJ   rK   r�   r$   r,   r   r"   r   r.   r)  rM   r   rD  rC  r   r   rF   r   rD   rN   rO   s   @r5   rw  rw  3  s  ø€ € € € € ðð ð˜{ð ð ð ð ð ð ð8  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ð=
ð =
àÔ# dÑ*ð=
ð œ tÑ+ð=
ð Ô&¨Ñ-ð	=
ð
  ™ð=
ð Ô(¨4Ñ/ð=
ð ˜$‘;ð=
ð Ð+Ô,ð=
ð 
Ð(Ñ	(ð=
ð =
ð =
ñ „^ñ „_ñ  Ôð=
ð =
ð =
ð =
ð =
r6   rw  c                   ó  ‡ — e Zd ZddiZdefˆ fd„Zee	 	 	 	 	 	 	 	 ddej	        dz  dej
        dz  d	ej	        dz  d
edz  dej        dz  dej	        dz  dedz  deej
        z  dee         deez  fd„¦   «         ¦   «         Zˆ fd„Zˆ xZS )ÚZambaForCausalLMzlm_head.weightzmodel.embed_tokens.weightrs   c                 óþ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        | _        t          j        |j        |j        d¬¦  «        | _        |  	                    ¦   «          d S r-  )
r+   r,   rw  rW  r|  r   r   r3   Úlm_headr…  r3  s     €r5   r,   zZambaForCausalLM.__init__Ÿ  sj   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý Ñ'Ô'ˆŒ
Ø Ô+ˆŒÝ”y Ô!3°VÔ5FÈUÐSÑSÔSˆŒð 	�ŠÑÔÐÐÐr6   Nr   r‡  r^   rJ  r…   rˆ  Úlabelsr@  Úlogits_to_keeprk   r)   c	           
      óD  —  | j         d||||||dœ|	¤Ž}
|
j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }d}|� | j        ||| j        fi |	¤Ž}t          |||
j	        |
j
        |
j        ¬¦  «        S )ah  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, ZambaForCausalLM

        >>> model = ZambaForCausalLM.from_pretrained("Zyphra/Zamba-7B-v1")
        >>> tokenizer = AutoTokenizer.from_pretrained("Zyphra/Zamba-7B-v1")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```)r‡  r^   rJ  r…   rˆ  r@  N©ÚlossÚlogitsr…   r7   rX  rÊ   )rW  rŒ  r^  r‘   Úslicer˜  Úloss_functionr|  r   r…   r7   rX  )r2   r‡  r^   rJ  r…   rˆ  r™  r@  rš  rk   Úoutputsr7   Úslice_indicesrž  r�  s                  r5   rD   zZambaForCausalLM.forward¨  sþ   € ðH ,6¨4¬:ð ,
ØØ)Ø%Ø+Ø'Øð,
ð ,
ð ð,
ð ,
ˆð  Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%ØØØ”ðð ð ð	ð ˆDõ &ØØØ#Ô3Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r6   c                 óZ   •— | j         j        |d<    t          ¦   «         j        |fi |¤Ž}|S )Nrš  )rs   Únum_logits_to_keepr+   Úprepare_inputs_for_generation)r2   r‡  rk   Úmodel_inputsr4   s       €r5   r¥  z.ZambaForCausalLM.prepare_inputs_for_generationì  s7   ø€ Ø#'¤;Ô#AˆÐÑ Ø<•u‘w”wÔ<¸YÐQÐQÈ&ÐQÐQˆØÐr6   )NNNNNNNr   )rI   rJ   rK   r€  r$   r,   r   r   r.   r)  rM   r   rD  rC  r‘   r   r   rF   r   rD   r¥  rN   rO   s   @r5   r–  r–  œ  sg  ø€ € € € € Ø*Ð,GÐHÐð˜{ð ð ð ð ð ð ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ð@
ð @
àÔ# dÑ*ð@
ð œ tÑ+ð@
ð Ô&¨Ñ-ð	@
ð
  ™ð@
ð Ô(¨4Ñ/ð@
ð Ô  4Ñ'ð@
ð ˜$‘;ð@
ð ˜eœlÑ*ð@
ð Ð+Ô,ð@
ð 
Ð'Ñ	'ð@
ð @
ð @
ñ „^ñ Ôð@
ðDð ð ð ð ð ð ð ð r6   r–  aÊ  
    The Zamba Model with a sequence classification head on top (linear layer).

    [`ZambaForSequenceClassification`] uses the last token in order to do the classification, as other causal models
    (e.g. GPT-2) do.

    Since it does classification on the last token, it requires to know the position of the last token. If a
    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
    each row of the batch).
    )Úcustom_introc                   óè   ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  de	dz  dej
        dz  dej        dz  d	edz  d
ee         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚZambaForSequenceClassificationc                 óþ   •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          j        |j        | j        d¬¦  «        | _        |  	                    ¦   «          d S r-  )
r+   r,   Ú
num_labelsrw  rW  r   r   r3   Úscorer…  r3  s     €r5   r,   z'ZambaForSequenceClassification.__init__  si   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒÝ Ñ'Ô'ˆŒ
Ý”Y˜vÔ1°4´?ÈÐOÑOÔOˆŒ
ð 	�ŠÑÔÐÐÐr6   Nr‡  r^   rJ  r…   rˆ  r™  r@  rk   r)   c           	      óÊ  —  | j         |f|||||dœ|¤Ž}	|	j        }
|                      |
¦  «        }|�|j        d         }n|j        d         }| j        j        €|dk    rt          d¦  «        ‚| j        j        €d}n¨|�}|| j        j        k                         |j        t          j
        ¦  «        }t          j        |j        d         |j        t          j
        ¬¦  «        }||z                       d¦  «        }n)d}t                               | j        j        › d�¦  «         |t          j        ||j        ¬	¦  «        |f         }d}|��t|                     |j        ¦  «        }| j        j        €f| j        dk    rd
| j        _        nN| j        dk    r7|j        t          j        k    s|j        t          j        k    rd| j        _        nd| j        _        | j        j        d
k    rWt-          ¦   «         }| j        dk    r1 ||                     ¦   «         |                     ¦   «         ¦  «        }nŽ |||¦  «        }n�| j        j        dk    rGt1          ¦   «         } ||                     d| j        ¦  «        |                     d¦  «        ¦  «        }n*| j        j        dk    rt5          ¦   «         } |||¦  «        }t7          |||	j        |	j        |	j        ¬¦  «        S )a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        )r^   rJ  r…   rˆ  r@  Nr   r#   z=Cannot handle batch sizes > 1 if no padding token is defined.r:   )r¶   r<   zŠ will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`rµ   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationrœ  )rW  rŒ  r¬  rG   rs   rz  r�  r=   r¶   r.   Úint32r  ÚargmaxÚloggerÚwarning_oncer4   rI   Úproblem_typer«  r<   Úlongr‘   r   r¹   r   r‡   r   r   r…   r7   rX  )r2   r‡  r^   rJ  r…   rˆ  r™  r@  rk   Útransformer_outputsr7   rž  rÞ   Úlast_non_pad_tokenÚnon_pad_maskÚtoken_indicesÚpooled_logitsr�  Úloss_fcts                      r5   rD   z&ZambaForSequenceClassification.forward
  s  € ð& 8B°t´zØð8
à)Ø%Ø+Ø'Øð8
ð 8
ð ð8
ð 8
Ðð ,Ô=ˆØ—’˜MÑ*Ô*ˆàÐ Ø"œ¨Ô+ˆJˆJà&Ô,¨QÔ/ˆJàŒ;Ô#Ð+°
¸a²°ÝÐ\Ñ]Ô]Ð]ØŒ;Ô#Ð+Ø!#ÐÐØÐ"à%¨¬Ô)AÒA×EÒEÀfÄmÕUZÔU`ÑaÔaˆLÝ!œL¨¬¸Ô)<ÀVÄ]ÕZ_ÔZeÐfÑfÔfˆMØ"/°,Ñ">×!FÒ!FÀrÑ!JÔ!JÐÐà!#ÐÝ×ÒØ”>Ô*ð Zð Zð Zñô ð ð
 �uœ|¨J¸v¼}ÐMÑMÔMÐOaÐaÔbˆàˆØÑØ—Y’Y˜vœ}Ñ-Ô-ˆFØŒ{Ô'Ð/Ø”? aÒ'Ð'Ø/;�D”KÔ,Ð,Ø”_ qÒ(Ð(¨f¬l½e¼jÒ.HÐ.HÈFÌLÕ\aÔ\eÒLeÐLeØ/L�D”KÔ,Ð,à/K�D”KÔ,àŒ{Ô'¨<Ò7Ð7Ý"™9œ9�Ø”? aÒ'Ð'Ø#˜8 M×$9Ò$9Ñ$;Ô$;¸V¿^º^Ñ=MÔ=MÑNÔN�D�Dà#˜8 M°6Ñ:Ô:�D�DØ”Ô)Ð-JÒJÐJÝ+Ñ-Ô-�Ø�x × 2Ò 2°2°t´Ñ GÔ GÈÏÊÐUWÉÌÑYÔY��Ø”Ô)Ð-IÒIÐIÝ,Ñ.Ô.�Ø�x ¨vÑ6Ô6�å/ØØ Ø/Ô?Ø-Ô;Ø*Ô5ð
ñ 
ô 
ð 	
r6   )NNNNNNN)rI   rJ   rK   r,   r   r   r.   r)  rM   r   rD  rC  r   r   rF   r   rD   rN   rO   s   @r5   r©  r©  ò  s$  ø€ € € € € ðð ð ð ð ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%ðR
ð R
àÔ# dÑ*ðR
ð œ tÑ+ðR
ð Ô&¨Ñ-ð	R
ð
  ™ðR
ð Ô(¨4Ñ/ðR
ð Ô  4Ñ'ðR
ð ˜$‘;ðR
ð Ð+Ô,ðR
ð 
Ð1Ñ	1ðR
ð R
ð R
ñ „^ñ ÔðR
ð R
ð R
ð R
ð R
r6   r©  )r–  r©  rw  rV  )rY   r(  )NNFN)NNNFFFF)Vr�   rc  Úcollections.abcr   r.   Útorch.nn.functionalr   rh   rž   Útorch.nnr   r   r   Ú r	   r_  Úactivationsr
   Úcache_utilsr   r   Ú
generationr   Úintegrationsr   r   Úintegrations.accelerater   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   Úutils.import_utilsr   r    r!   Úutils.output_capturingr"   Úconfiguration_zambar$   Ú
get_loggerrI   r³  ÚModuler&   rM   r‘   rX   rL   rp   rr   r“   r-   Ústrr”   r¦   rC  r¿   rç   ré   r+  r7  rF  rO  rV  rw  r–  r©  Ú__all__rÊ   r6   r5   ú<module>rÔ     sõ  ðð& Ð à €€€Ø $Ð $Ð $Ð $Ð $Ð $à €€€Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aà &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )Ø WÐ WÐ WÐ WÐ WÐ WÐ WÐ WØ =Ð =Ð =Ð =Ð =Ð =Ø PÐ PÐ PÐ PÐ PÐ PÐ PÐ PØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø qÐ qÐ qÐ qÐ qÐ qÐ qÐ qÐ qÐ qØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RØ 7Ð 7Ð 7Ð 7Ð 7Ð 7ðð ð ð ð ð ð ð ð ð ð
 6Ð 5Ð 5Ð 5Ð 5Ð 5Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,ð 
ˆÔ	˜HÑ	%Ô	%€ðJð Jð Jð Jð J�2”9ñ Jô Jð Jð*	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð& ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð %ð %ð %ð2C)ð C)ð C)ð C)ð C)�R”Yñ C)ô C)ð C)ðL	ð 	ð 	ð (Ð'Ð(>ÀÑPÔPð
 !%Ø!ð'ð 'Ø”<ð'à”ð'ð ŒLð'ð Œ,˜Ñ
ð	'ð
 �d‘
ð'ð 'ð 'ñ QÔPð'ð& (Ð'Ð(:¸OÑLÔLð !%Ø!ð	'ð 'Ø”<ð'àŒLð'ð Œ,˜Ñ
ð'ð �d‘
ð	'ð 'ð 'ñ MÔLð'ð, (Ð'ØØñô ð "Ø#'ØØ!ð*ð *ØŒ<ð*à”<ð*ð 	Œð*ð „|ð	*ð
 „|ð*ð „|ð*ð „|�dÑð*ð Œ\˜DÑ ð*ð ð*ð „|�dÑð*ð *ð *ñ	ô ð*ðZ (Ð'ØØñô ð "Ø!Ø&*Ø Ø#ØØ!&ðcð cØ”<ðcàŒðcð „|ðcð „|ð	cð
 „|ðcð „|�dÑðcð „|�dÑðcð ”˜tÑ#ðcð ðcð ðcð ðcð ðcð cð cñ	ô ðcðL ÐÐ*Ð,HÐJZÐ\pÐqÑrÔrðO%ð O%ð O%ð O%ð O%�b”iñ O%ô O%ñ sÔrðO%ðfð ð ð ð ˆrŒyñ ô ð ð 0ð 0ð 0ð 0ð 0 ¤ñ 0ô 0ð 0ðf1ð 1ð 1ð 1ð 1Ð7ñ 1ô 1ð 1ðh4ð 4ð 4ð 4ð 4Ð1ñ 4ô 4ð 4ðn ð$!ð $!ð $!ð $!ð $!˜?ñ $!ô $!ñ „ð$!ðN ðd
ð d
ð d
ð d
ð d
Ð%ñ d
ô d
ñ „ðd
ðPSð Sð Sð Sð SÐ+¨_ñ Sô Sð Sðl €ððñ ô ð^
ð ^
ð ^
ð ^
ð ^
Ð%9ñ ^
ô ^
ñô ð^
ðB gÐ
fÐ
f€€€r6   