§
    9²¸jÅw  ã            $       ó®  — d Z ddlZddlmZ ddlZddlmc mZ ddlmZ ddl	m
Z ddlmZ ddlmZmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZ ddlmZ ddlmZmZmZ ddl m!Z!m"Z"m#Z#m$Z$ ddl%m&Z&  ej'        e(¦  «        Z)d„ Z* edd¦  «        	 	 dUdej+        dej+        dej,        dej,        dz  de-dz  f
d„¦   «         Z. edd¦  «        	 	 dUdej+        dej,        dej,        dz  de-dz  fd„¦   «         Z/ edd¦  «        	 	 	 	 	 	 	 	 	 dVd!ej+        d"ej+        d#ej+        dz  d$ej+        d%ej+        d&ej+        d'ej+        dz  d(ej+        d)ej+        dz  d*ej+        dz  d+ej+        dz  d,ej+        dz  d-e0d.ej+        dz  d/ej+        dz  d0ej+        dz  d1e1f"d2„¦   «         Z2 ed3d¦  «        	 	 	 	 dWd5ej+        dej+        d6ej+        d(ej+        d)ej+        d*ej+        d+ej+        dz  d7ej+        dz  d8e0d9ej+        dz  fd:„¦   «         Z3 ed;d¦  «        	 	 	 	 	 	 	 dXdej+        d6ej+        d(ej+        d)ej+        d*ej+        d+ej+        dz  d9ej+        dz  d,ej+        dz  d-e0d<e0d=e0d>e0fd?„¦   «         Z4 ee2e4e3e/e.g¦  «         G d@„ dAej5        ¦  «        ¦   «         Z6 G dB„ dCej5        ¦  «        Z7 G dD„ dEe¦  «        Z8e G dF„ dGe¦  «        ¦   «         Z9 edH¬I¦  «        e G dJ„ dKe¦  «        ¦   «         ¦   «         Z: edL¬I¦  «        e G dM„ dNe¦  «        ¦   «         ¦   «         Z;e G dO„ dPe9¦  «        ¦   «         Z< edQ¬I¦  «         G dR„ dSe9e¦  «        ¦   «         Z=g dT¢Z>dS )YzPyTorch MAMBA model.é    N)Ú	dataclass)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Ú&use_kernel_func_from_hub_with_fallbackÚuse_kernelized_func)Úforce_accelerate_hooks)ÚGradientCheckpointingLayer)ÚPreTrainedModel)ÚModelOutputÚauto_docstringÚlogging)Úis_mambapy_availableÚis_torch_greater_or_equalÚis_torchdynamo_compilingÚis_torchdynamo_exportingé   )ÚMambaConfigc                 ób   — |�,| j         }| |dd…dd…df         z                       |¦  «        } | S )zm
    Tunes out the hidden states for padding tokens, see https://github.com/state-spaces/mamba/issues/66
    N)ÚdtypeÚto)Úhidden_statesÚattention_maskr   s      úx/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/mamba/modeling_mamba.pyÚapply_mask_to_padding_statesr   0   sC   € ð
 Ð!ØÔ#ˆØ&¨¸¸¸¸1¸1¸1¸d¸
Ô)CÑC×GÒGÈÑNÔNˆàÐó    Úcausal_conv1d_updateÚcausal_conv1dr   Ú
conv_stateÚweightÚbiasÚ
activationc                 óº  — | j         \  }}}|j         d         }t          j        || gd¬¦  «                             |j        ¦  «        }	|                     |	d d …d d …| d …f         ¦  «         t          j        |	|                     d¦  «        |d|¬¦  «        }
|
d d …d d …| d …f         }
|�t          |         |
¦  «        }
|
                     | j        ¦  «        S )Néÿÿÿÿ©Údimr   r   )ÚpaddingÚgroups)
ÚshapeÚtorchÚcatr   r   Úcopy_ÚFÚconv1dÚ	unsqueezer   )r   r#   r$   r%   r&   Ú_Úhidden_sizeÚseq_lenÚ	state_lenÚhidden_states_newÚouts              r   r!   r!   <   sî   € ð ,Ô1Ñ€A€{�GØÔ  Ô$€Iåœ	 :¨}Ð"=À2ÐFÑFÔF×IÒIÈ&Ì,ÑWÔWÐØ×ÒÐ& q q q¨!¨!¨!¨i¨Z¨[¨[Ð'8Ô9Ñ:Ô:Ð:Ý
Œ(Ð$ f×&6Ò&6°qÑ&9Ô&9¸4ÈÐS^Ð
_Ñ
_Ô
_€CØ
ˆaˆaˆa����W�H�I�IˆoÔ
€CØÐÝ�ZÔ  Ñ%Ô%ˆØ�6Š6�-Ô%Ñ&Ô&Ð&r    Úcausal_conv1d_fnc                 ó@  — | j         \  }}}|j         d         dz
  }t          j        |                      |j        ¦  «        |                     d¦  «        |||¬¦  «        d d …d d …d |…f         }	|�t          |         |	¦  «        }	|	                     | j        ¦  «        S )Nr(   r   )r$   r%   r+   r,   )r-   r1   r2   r   r   r3   r   )
r   r$   r%   r&   Úkwargsr4   r5   r6   r+   r9   s
             r   r:   r:   P   s²   € ð ,Ô1Ñ€A€{�GØŒl˜2Ô Ñ"€Gå
Œ(Ø×Ò˜œÑ&Ô&Ø×Ò Ñ"Ô"ØØØðñ ô ð €a€aˆˆˆˆHˆWˆH€nô€Cð ÐÝ�ZÔ  Ñ%Ô%ˆØ�6Š6�-Ô%Ñ&Ô&Ð&r    Úmamba_inner_fnÚ	mamba_ssmTç�íµ ÷Æ°>ÚxzÚconv1d_weightÚconv1d_biasÚx_proj_weightÚdelta_proj_weightÚout_proj_weightÚout_proj_biasÚAÚBÚCÚDÚ
delta_biasÚdelta_softplusÚb_rms_weightÚc_rms_weightÚdt_rms_weightÚb_c_dt_rms_epsc                 ó   — d S ©N© )r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   r<   s                     r   r=   r=   g   s	   € ð* ˆ4r    Úselective_state_updateFÚstateÚdtÚdt_biasÚdt_softplusÚzc
                 ó„  — |j         }|�||                     |j         ¦  «        z   }|rt          j        |¦  «        }t	          j        |                     ¦   «         d         |                     ¦   «         z  ¦  «                             | j        ¬¦  «        }|                     ¦   «         d         |                     ¦   «         d d …d d d …f         z  }||                     ¦   «         d         z  }|                      ¦   «         |z  |z   }|                      |                     | j         ¦  «        ¦  «         t	          j	        |                     |j         ¦  «        | 
                    d¦  «        ¦  «                             d¦  «        }|�|||z  z   }|	�|t          j        |	¦  «        z  }|                     |¦  «        S )N©.N)Údevicer(   )r   r   r1   Úsoftplusr.   ÚexpÚfloatr\   r0   Úmatmulr3   ÚsqueezeÚsilu)rU   r   rV   rG   rH   rI   rJ   rW   rX   rY   r<   Úinput_dtypeÚdAÚdBÚdBxÚ	ssm_stater9   s                    r   Úmamba_selective_state_updaterh      s|  € ð  Ô%€KàÐØ�'—*’*˜RœXÑ&Ô&Ñ&ˆØð ÝŒZ˜‰^Œ^ˆõ 
Œ�2—8’8‘:”:˜iÔ(¨1¯7ª7©9¬9Ñ4Ñ	5Ô	5×	8Ò	8ÀÄÐ	8Ñ	MÔ	M€Bð 
�Š‰Œ�IÔ	 §¢¡¤¨1¨1¨1¨d°A°A°A¨:Ô!6Ñ	6€Bà
ˆ}×"Ò"Ñ$Ô$ YÔ/Ñ
/€Cð —’‘” Ñ" SÑ(€IØ	‡K‚K�	—’˜Uœ[Ñ)Ô)Ñ*Ô*Ð*õ Œ,�y—|’| A¤GÑ,Ô,¨a¯kªk¸"©o¬oÑ
>Ô
>×
FÒ
FÀrÑ
JÔ
J€Cð 	€}Ø�M AÑ%Ñ%ˆà€}Ø•A”F˜1‘I”I‰oˆà�6Š6�+ÑÔÐr    Úselective_scan_fnÚreturn_last_stateÚuse_mambapyÚuse_associative_scanc           	      óN  — t          d¦  «        rddlm} nd }t          ¦   «         rddlm} nd }| j        \  }}}| j        }|�#||                     |j        ¦  «        d         z   }|rt          j
        |¦  «        }|                     dd¦  «        }|                     dd¦  «        }t          j        |d d d …d d d …f         |d d …d d …d d …d f         z  ¦  «        }|d d …d d …d d …d f         |d d …d d d …d d …f                              ¦   «         z  }|| d d …d d …d d …d f                              ¦   «         z  }|
rƒ|�� ||                     dd¦  «        |                     dd¦  «        ¦  «        }||                     d¦  «        z                       d	¦  «                             dd¦  «        }|d d …df         }�n°|rÏ|�Ít#          ¦   «         r¿t%          ¦   «         s±d
„ }|j        j        dv rdnd} ||||fd|¬¦  «        \  }}t          j        |                     dd¦  «                             |¦  «        |                     d¦  «        ¦  «        }|                     d¦  «                             dd¦  «        }|d d …d d …df         }nßt          j        |||j        d         || j        ¬¦  «        }g }t/          |¦  «        D ]Ž}|d d …d d …|f         |z  |d d …d d …|f         z   }t          j        |                     |¦  «        |d d …|d d …f                              d¦  «        ¦  «        }|                     |d d …d d …df         ¦  «         Œ�t          j        |d¬¦  «        }|�|| |d d d …d f         z  z   }|�|t          j        |¦  «        z  }|	r||fS |S )Nz2.9.0r   )Úassociative_scan)Úpscanr[   r   é   r(   r   c                 ó0   — | \  }}|\  }}||z  ||z  |z   fS rR   rS   )ÚleftÚrightÚa_leftÚb_leftÚa_rightÚb_rights         r   Ú
combine_fnz(mamba_selective_scan.<locals>.combine_fnæ   s/   € Ø!‰NˆF�FØ$ÑˆG�WØ˜GÑ# W¨vÑ%5¸Ñ%?Ð?Ð?r    )ÚcudaÚxpuÚ	pointwiseÚgeneric)r*   Úcombine_mode©r   r\   r)   )r   Ú(torch._higher_order_ops.associative_scanrn   r   Úmambapy.pscanro   r-   r   r   r1   r]   Ú	transposer.   r^   r_   r3   ra   r   r   r\   Útyper`   ÚzerosÚrangeÚappendÚstackrb   )r   rV   rG   rH   rI   rJ   rY   rK   rL   rj   rk   rl   r<   rn   ro   Ú
batch_sizeÚintermediate_sizer6   rc   Ú
discrete_AÚ
discrete_BÚdeltaB_uÚ
all_statesÚscan_outputrg   rx   r}   r4   Úscan_outputsÚindexs                                 r   Úmamba_selective_scanr�   ­   s`  € õ" ! Ñ)Ô)ð  ØMÐMÐMÐMÐMÐMÐMàÐåÑÔð Ø'Ð'Ð'Ð'Ð'Ð'Ð'àˆà-:Ô-@Ñ*€JÐ! 7ØÔ%€KàÐØ�*—-’- ¤Ñ)Ô)¨)Ô4Ñ4ˆØð ÝŒZ˜‰^Œ^ˆð 	
�Š�A�qÑÔ€AØ	�Š�A�qÑÔ€Aõ ”˜1˜T 1 1 1 d¨A¨A¨AÐ-Ô.°°A°A°A°q°q°q¸!¸!¸!¸T°MÔ1BÑBÑCÔC€JØ�A�A�A�q�q�q˜!˜!˜!˜T�MÔ" Q q q q¨$°°°°1°1°1 }Ô%5×%;Ò%;Ñ%=Ô%=Ñ=€JØ˜M¨!¨!¨!¨Q¨Q¨Q°°°°4¨-Ô8×>Ò>Ñ@Ô@Ñ@€Hàð 28�uÐ(Ø�U˜:×/Ò/°°1Ñ5Ô5°x×7IÒ7IÈ!ÈQÑ7OÔ7OÑPÔPˆ
à! A§K¢K°¡O¤OÑ3×<Ò<¸QÑ?Ô?×IÒIÈ!ÈQÑOÔOˆØ˜q˜q˜q "˜uÔ%ˆ	‰	ð 	ð,8àÐ(å%Ñ'Ô'ð )å0HÑ0JÔ0Jð )ð
	@ð 	@ð 	@ð
 '1Ô&7Ô&<ÀÐ&OÐ&O�{�{ÐU^ˆØ(Ð(ØØ˜Ð"ØØ%ð	
ñ 
ô 
‰ˆˆ:õ ”l :×#7Ò#7¸¸1Ñ#=Ô#=×#@Ò#@ÀÑ#MÔ#MÈqÏ{Ê{Ð[]ÉÌÑ_Ô_ˆØ!×)Ò)¨"Ñ-Ô-×7Ò7¸¸1Ñ=Ô=ˆØ˜q˜q˜q ! ! ! R˜xÔ(ˆ	ˆ	õ ”KØØØŒG�BŒKØØ Ô'ð
ñ 
ô 
ˆ	ð ˆÝ˜7‘^”^ð 	6ð 	6ˆEà" 1 1 1 a a a¨ ;Ô/°)Ñ;¸hÀqÀqÀqÈ!È!È!ÈUÀ{Ô>SÑSˆIõ  œ, y§|¢|°KÑ'@Ô'@À!ÀAÀAÀAÀuÈaÈaÈaÀKÄ.×BZÒBZÐ[]ÑB^ÔB^Ñ_Ô_ˆKØ×Ò ¨A¨A¨A¨q¨q¨q°!¨GÔ 4Ñ5Ô5Ð5Ð5Ý”k ,°BÐ7Ñ7Ô7ˆà€}Ø! M°A°d¸A¸A¸A¸t°mÔ4DÑ$DÑDˆà€}Ø!¥A¤F¨1¡I¤IÑ-ˆàð &Ø˜IÐ%Ð%àÐr    c            	       óÂ   ‡ — e Zd ZdZddededefˆ fd„Z ej	        ¦   «         d„ ¦   «         Z
 edd	g¦  «        	 	 ddej        ded
z  dej        d
z  fd„¦   «         Zˆ xZS )Ú
MambaMixeruƒ  
    Compute âˆ†, A, B, C, and D the state space parameters and compute the `contextualized_states`.
    A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
    âˆ†, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
    and is why Mamba is called **selective** state spaces)
    TÚconfigÚ	layer_idxÚinitialize_mixer_weightsc           	      óÌ  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        |j        | _        |j        | _        t          |j
        ¦  «        | _
        || _        |j        | _        t          j        | j        | j        |j        |j        | j        |j        dz
  ¬¦  «        | _        |j        | _        t$          |j                 | _        |j        | _        |j        | _        t          j        | j        | j        dz  |j        ¬¦  «        | _        t          j        | j        | j
        | j        dz  z   d¬¦  «        | _        t          j        | j
        | j        d¬¦  «        | _        t          j        t9          j        | j        | j        ¦  «        ¦  «        | _        t          j        t9          j        | j        ¦  «        ¦  «        | _        |r.| j        j         j!        j"        dk    r|  #                    ¦   «          t          j        | j        | j        |j        ¬¦  «        | _$        |j        | _        |j%        |         | _&        d S )Nr   )Úin_channelsÚout_channelsr%   Úkernel_sizer,   r+   rp   ©r%   FTÚmeta)'ÚsuperÚ__init__r“   r5   Ú
state_sizeÚssm_state_sizeÚconv_kernelÚconv_kernel_sizerˆ   ÚintÚtime_step_rankr”   Úuse_conv_biasr   ÚConv1dr2   Ú
hidden_actr&   r   Úactrk   rl   ÚLinearÚuse_biasÚin_projÚx_projÚdt_projÚ	Parameterr.   ÚemptyÚA_logrJ   r$   r\   r‚   Úinit_mamba_weightsÚout_projÚlayer_typesÚ
layer_type)Úselfr“   r”   r•   Ú	__class__s       €r   r�   zMambaMixer.__init__$  s  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ$Ô/ˆÔØ &Ô 2ˆÔØ!'Ô!9ˆÔÝ! &Ô"7Ñ8Ô8ˆÔØ"ˆŒØ#Ô1ˆÔÝ”iØÔ.ØÔ/ØÔ%ØÔ*ØÔ)ØÔ&¨Ñ*ð
ñ 
ô 
ˆŒð !Ô+ˆŒÝ˜&Ô+Ô,ˆŒà!Ô-ˆÔØ$*Ô$?ˆÔ!õ ”y Ô!1°4Ô3IÈAÑ3MÐTZÔTcÐdÑdÔdˆŒå”i Ô 6¸Ô8KÈdÔNaÐdeÑNeÑ8eÐlqÐrÑrÔrˆŒå”y Ô!4°dÔ6LÐSWÐXÑXÔXˆŒõ ”\¥%¤+¨dÔ.DÀdÔFYÑ"ZÔ"ZÑ[Ô[ˆŒ
Ý”�eœk¨$Ô*@ÑAÔAÑBÔBˆŒØ#ð 	&¨¬Ô(;Ô(BÔ(GÈ6Ò(QÐ(QØ×#Ò#Ñ%Ô%Ð%Ýœ	 $Ô"8¸$Ô:JÐQWÔQ`ÐaÑaÔaˆŒØœˆŒà Ô,¨YÔ7ˆŒˆˆr    c                 óx  — t          j        d| j        dz   t           j        | j        j        ¬¦  «        d d d …f         }|                     | j        d¦  «                             ¦   «         }t          j
        | j        t          j        |¦  «        ¦  «         t          j        | j        ¦  «         | j        j        dz  | j        j        z  }| j        j        dk    r t          j        | j        j        |¦  «         n1| j        j        dk    r!t          j        | j        j        | |¦  «         t          j        t          j        | j        | j        j        j        t           j        ¬¦  «        t3          j        | j        j        ¦  «        t3          j        | j        j        ¦  «        z
  z  t3          j        | j        j        ¦  «        z   ¦  «                             | j        j        ¬¦  «        }|t          j        t          j        | ¦  «         ¦  «        z   }t          j
        | j        j        |¦  «         d S )	Nr   r~   r(   g      à¿ÚconstantÚrandom)r\   r   )Úmin)r.   ÚarangerŸ   Úfloat32r¯   r\   Úexpandrˆ   Ú
contiguousÚinitr0   ÚlogÚones_rJ   r“   r£   Útime_step_scaleÚtime_step_init_schemeÚ	constant_r¬   r$   Úuniform_r^   Úrandr%   ÚmathÚtime_step_maxÚtime_step_minÚclampÚtime_step_floorÚexpm1)r´   rG   Údt_init_stdrV   Úinv_dts        r   r°   zMambaMixer.init_mamba_weightsO  sÍ  € åŒL˜˜DÔ/°!Ñ3½5¼=ÐQUÔQ[ÔQbÐcÑcÔcÐdhÐjkÐjkÐjkÐdkÔlˆØ�HŠH�TÔ+¨RÑ0Ô0×;Ò;Ñ=Ô=ˆÝŒ
�4”:�uœy¨™|œ|Ñ,Ô,Ð,ÝŒ
�4”6ÑÔÐà”kÔ0°$Ñ6¸¼Ô9TÑTˆØŒ;Ô,°
Ò:Ð:ÝŒN˜4œ<Ô.°Ñ<Ô<Ð<Ð<ØŒ[Ô.°(Ò:Ð:ÝŒM˜$œ,Ô-°¨|¸[ÑIÔIÐIåŒYÝŒJ�tÔ-°d´lÔ6GÔ6NÕV[ÔVcÐdÑdÔdÝŒx˜œÔ1Ñ2Ô2µT´X¸d¼kÔ>WÑ5XÔ5XÑXñZåŒh�t”{Ô0Ñ1Ô1ñ2ñ
ô 
÷ Š%�D”KÔ/ˆ%Ñ
0Ô
0ð	 	ð •e”i¥¤¨b¨SÑ!1Ô!1Ð 1Ñ2Ô2Ñ2ˆÝŒ
�4”<Ô$ fÑ-Ô-Ð-Ð-Ð-r    r2   r¬   Nr   Úcache_paramsr   c                 ó
	  — |j         d         }|j        }|d uo|                     | j        ¦  «        }t	          ||¦  «        }|                      |¦  «                             dd¦  «        }t          j        | j	         
                    ¦   «         ¦  «         }	| j        rµ|€³t          || j        j        | j        r| j        j        nd | j        j        | j        j        | j        j        | j        r| j        j         
                    ¦   «         nd |	d d | j         
                    ¦   «         | j        j         
                    ¦   «         d¬¦  «        }
|
�|
S |                     dd¬¦  «        \  }}|r:|j        | j                 j        d         }|j        | j                 j        d         }|r^|dk    rX|j        | j                 j        sAt7          ||| j        j                             d¦  «        | j        j        | j        ¬¦  «        }nu|�"|                     || j        | j        ¬¦  «        }tA          || j        j                             d¦  «        | j        j        fd	| j        i|¤Ž}|�|d d …d d …| d …f         }t	          |                     dd¦  «        |¦  «        }t          j!        |                      |¦  «        | j"        | j#        | j#        gd
¬¦  «        \  }}}| j        j        |                     dd¦  «        z  }| j        j        �| j        j         
                    ¦   «         nd }|rr|dk    rltI          ||                     dd¦  «        d         |d         |	|d d …df         |d d …df         | j        |d         |d¬¦
  «
         %                    d
¦  «        }n¡|d u}tM          |                     dd¦  «        ||	|                     dd¦  «        |                     dd¦  «        | j         
                    ¦   «         ||d|| j'        | j(        ¬¦  «        }|r!|\  }}| )                    || j        ¦  «         n|}|                      |                     dd¦  «         *                    |¦  «        ¦  «        }|S )Nr   rp   T)rK   rL   r)   r   )r&   )r¡   r&   r(   ).r   )rY   rW   rX   )rJ   rY   rK   rL   rj   rk   rl   )+r-   r   Úhas_previous_stater”   r   rª   r�   r.   r^   r¯   r_   Útrainingr=   r2   r$   r¤   r%   r«   r¬   r±   r©   rJ   ÚchunkÚlayersÚconv_statesÚrecurrent_statesÚrecord_pastr!   ra   r&   Úupdate_conv_stater¡   r:   Úsplitr£   rŸ   rh   r3   r�   rk   rl   Úupdate_recurrent_stater   )r´   r   rÎ   r   r<   r6   r   Úuse_precomputed_statesÚprojected_statesrG   Úfused_outputÚhidden_states_B_CÚgater#   Úrecurrent_stateÚ	time_steprH   rI   Útime_proj_biasr�   Úoutput_final_stateÚscan_resultÚfinal_stateÚcontextualized_statess                           r   ÚforwardzMambaMixer.forwarde  sÐ  € ð  Ô% aÔ(ˆØÔ#ˆØ!-°TÐ!9Ð!m¸l×>]Ò>]Ð^bÔ^lÑ>mÔ>mÐõ 5°]ÀNÑSÔSˆØŸ<š<¨Ñ6Ô6×@Ò@ÀÀAÑFÔFÐåŒY�t”z×'Ò'Ñ)Ô)Ñ*Ô*Ð*ˆØŒ=ð 	$˜\Ð1Ý)Ø Ø”Ô"Ø$(Ô$6Ð@�”Ô Ð ¸DØ”Ô"Ø”Ô#Ø”Ô$Ø.2¬mÐE�”Ô"×(Ò(Ñ*Ô*Ð*ÀØØØØ”—’‘”Øœ<Ô,×2Ò2Ñ4Ô4Ø#ðñ ô ˆLð" Ð'Ø#Ð#à"2×"8Ò"8¸ÀÐ"8Ñ"BÔ"BÑÐ˜4à!ð 	VØ%Ô,¨T¬^Ô<ÔHÈÔKˆJØ*Ô1°$´.ÔAÔRÐSTÔUˆOð "ð 	G g°¢l l¸<Ô;NÈtÌ~Ô;^Ô;j lÝ 4Ø!ØØ”Ô"×*Ò*¨1Ñ-Ô-Ø”Ô Øœ?ð!ñ !ô !ÐÐð Ð'Ø$0×$BÒ$BØ%Ø”NØ%)Ô%:ð %Cñ %ô %Ð!õ !1Ø!Ø”Ô"×*Ò*¨1Ñ-Ô-Ø”Ô ð!ð !ð  œ?ð	!ð
 ð!ð !Ðð Ð'Ø$5°a°a°a¸¸¸¸W¸H¸I¸I°oÔ$FÐ!õ 9Ð9J×9TÒ9TÐUVÐXYÑ9ZÔ9ZÐ\jÑkÔkÐÝœ+Ø�KŠKÐ)Ñ*Ô*ØÔ  $Ô"5°tÔ7JÐKØð
ñ 
ô 
‰ˆ	�1�að ”LÔ'¨)×*=Ò*=¸aÀÑ*CÔ*CÑCˆ	Ø6:´lÔ6GÐ6S˜œÔ*×0Ò0Ñ2Ô2Ð2ÐY]ˆð "ð $	* g°¢l lÝ6ØØ!×+Ò+¨A¨qÑ1Ô1°&Ô9Ø˜&Ô!ØØ�!�!�!�Q�$”Ø�!�!�!�Q�$”Ø”Ø�v”,Ø&Ø ðñ ô ÷ Ši˜‰mŒmð ˆKð ".°TÐ!9ÐÝ.Ø!×+Ò+¨A¨qÑ1Ô1ØØØ—’˜A˜qÑ!Ô!Ø—’˜A˜qÑ!Ô!Ø”&—,’,‘.”.ØØ)Ø#Ø"4Ø Ô,Ø%)Ô%>ðñ ô ˆKð "ð *Ø+6Ñ(�˜[Ø×3Ò3°KÀÄÑPÔPÐPÐPà)�ð !%§¢¨k×.CÒ.CÀAÀqÑ.IÔ.I×.LÒ.LÈUÑ.SÔ.SÑ TÔ TÐØ$Ð$r    )T©NN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r¢   Úboolr�   r.   Úno_gradr°   r   ÚTensorr   Ú
LongTensorræ   Ú__classcell__©rµ   s   @r   r’   r’     sú   ø€ € € € € ðð ð)8ð )8˜{ð )8°sð )8ÐVZð )8ð )8ð )8ð )8ð )8ð )8ðV €U„]�_„_ð.ð .ñ „_ð.ð* Ð˜X yÐ1Ñ2Ô2ð &*Ø26ð	{%ð {%à”|ð{%ð ˜d‘lð{%ð Ô(¨4Ñ/ð	{%ð {%ð {%ñ 3Ô2ð{%ð {%ð {%ð {%ð {%r    r’   c                   ó,   ‡ — e Zd Zdˆ fd„	Zd„ Zd„ Zˆ xZS )ÚMambaRMSNormr?   c                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )zL
        MambaRMSNorm is equivalent to T5LayerNorm and LlamaRMSNorm
        N)rœ   r�   r   r­   r.   Úonesr$   Úvariance_epsilon)r´   r5   Úepsrµ   s      €r   r�   zMambaRMSNorm.__init__å  sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐr    c                 ó  — |j         }|                     t          j        ¦  «        }|                     d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        |                     |¦  «        z  S )Nrp   r(   T)Úkeepdim)	r   r   r.   r»   ÚpowÚmeanÚrsqrtrö   r$   )r´   r   rc   Úvariances       r   ræ   zMambaRMSNorm.forwardí  s|   € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØ ×$Ò$ QÑ'Ô'×,Ò,¨R¸Ð,Ñ>Ô>ˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆØŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:r    c                 ó:   — | j         j        d         › d| j        › �S )Nr   z, eps=)r$   r-   rö   ©r´   s    r   Ú
extra_reprzMambaRMSNorm.extra_reprô  s#   € Ø”+Ô# AÔ&ÐEÐE¨dÔ.CÐEÐEÐEr    )r?   )rè   ré   rê   r�   ræ   r   rð   rñ   s   @r   ró   ró   ä  sb   ø€ € € € € ð$ð $ð $ð $ð $ð $ð;ð ;ð ;ðFð Fð Fð Fð Fð Fð Fr    ró   c                   óJ   ‡ — e Zd Zˆ fd„Z	 	 ddedz  dej        dz  fd„Zˆ xZS )Ú
MambaBlockc                 óê   •— t          ¦   «                              ¦   «          || _        || _        |j        | _        t          |j        |j        ¬¦  «        | _        t          ||d¬¦  «        | _
        d S )N©r÷   F)r”   r•   )rœ   r�   r“   r”   Úresidual_in_fp32ró   r5   Úlayer_norm_epsilonÚnormr’   Úmixer)r´   r“   r”   rµ   s      €r   r�   zMambaBlock.__init__ù  sg   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒØ &Ô 7ˆÔÝ  Ô!3¸Ô9RÐSÑSÔSˆŒ	Ý °)ÐV[Ð\Ñ\Ô\ˆŒ
ˆ
ˆ
r    NrÎ   r   c                 ó   — |}|                       |                     | j         j        j        ¬¦  «        ¦  «        }| j        r|                     t
          j        ¦  «        }|                      |||¬¦  «        }||z   }|S )N)r   ©rÎ   r   )r  r   r$   r   r  r.   r»   r  )r´   r   rÎ   r   r<   Úresiduals         r   ræ   zMambaBlock.forward  sy   € ð !ˆØŸ	š	 -×"2Ò"2¸¼Ô9IÔ9OÐ"2Ñ"PÔ"PÑQÔQˆØÔ ð 	2Ø—{’{¥5¤=Ñ1Ô1ˆHàŸ
š
 =¸|Ð\j˜
ÑkÔkˆØ  =Ñ0ˆØÐr    rç   )	rè   ré   rê   r�   r   r.   rï   ræ   rð   rñ   s   @r   r  r  ø  s~   ø€ € € € € ð]ð ]ð ]ð ]ð ]ð &*Ø26ð	ð ð ˜d‘lðð Ô(¨4Ñ/ð	ð ð ð ð ð ð ð r    r  c                   óh   ‡ — e Zd ZU eed<   dZddgZdZdZ e	j
        ¦   «         ˆ fd„¦   «         Zˆ xZS )ÚMambaPreTrainedModelr“   Úbackboner  r’   Tc                 ó2  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rÝ|                     ¦   «          t          j        |j        j        t          j
        d¦  «        ¬¦  «         |j        j        �t          j        |j        j        ¦  «         t          j        |j        j        t          j
        d¦  «        ¬¦  «         | j        j        r1|j        j        }|t          j
        | j        j        ¦  «        z  }dS dS dS )zInitialize the weights.é   )ÚaN)rœ   Ú_init_weightsÚ
isinstancer’   r°   r¾   Úkaiming_uniform_r2   r$   rÆ   Úsqrtr%   Úzeros_r±   r“   Úrescale_prenorm_residualÚnum_hidden_layers)r´   ÚmoduleÚprµ   s      €r   r  z"MambaPreTrainedModel._init_weights  só   ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�f�jÑ)Ô)ð 	>ð ×%Ò%Ñ'Ô'Ð'åÔ! &¤-Ô"6½$¼)ÀA¹,¼,ÐGÑGÔGÐGØŒ}Ô!Ð-Ý”˜FœMÔ.Ñ/Ô/Ð/ÝÔ! &¤/Ô"8½D¼IÀa¹L¼LÐIÑIÔIÐIàŒ{Ô3ð >ð ”OÔ*�Ø•T”Y˜tœ{Ô<Ñ=Ô=Ñ=���ð-	>ð 	>ð>ð >r    )rè   ré   rê   r   Ú__annotations__Úbase_model_prefixÚ_no_split_modulesÚsupports_gradient_checkpointingÚ_is_statefulr.   rí   r  rð   rñ   s   @r   r  r    st   ø€ € € € € € àÐÐÑØ"ÐØ% |Ð4ÐØ&*Ð#Ø€Là€U„]�_„_ð>ð >ð >ð >ñ „_ð>ð >ð >ð >ð >r    r  z,
    Class for the MAMBA model outputs.
    )Úcustom_introc                   óp   — e Zd ZU dZdZej        dz  ed<   dZe	dz  ed<   dZ
eej                 dz  ed<   dS )ÚMambaOutputa4  
    cache_params (`Cache`):
        The state of the model at the last time step. Can be used in a forward method with the next `input_ids` to
        avoid providing the old `input_ids`.

        Includes both the State space model state matrices after the selective scan, and the Convolutional states
    NÚlast_hidden_staterÎ   r   )rè   ré   rê   rë   r#  r.   ÚFloatTensorr  rÎ   r   r   ÚtuplerS   r    r   r"  r"  7  sh   € € € € € € ðð ð 37Ð�uÔ(¨4Ñ/Ð6Ð6Ñ6Ø!%€L�%˜$‘,Ð%Ð%Ñ%Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ð9Ð9r    r"  zK
    Base class for causal language model (or autoregressive) outputs.
    c                   óŽ   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
dz  ed<   dZeej                 dz  ed<   dS )ÚMambaCausalLMOutputa™  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    cache_params (`Cache`):
        The state of the model at the last time step. Can be used in a forward method with the next `input_ids` to
        avoid providing the old `input_ids`.

        Includes both the State space model state matrices after the selective scan, and the Convolutional states
    NÚlossÚlogitsrÎ   r   )rè   ré   rê   rë   r(  r.   r$  r  r)  rÎ   r   r   r%  rS   r    r   r'  r'  K  s   € € € € € € ð
ð 
ð &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø!%€L�%˜$‘,Ð%Ð%Ñ%Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ð9Ð9r    r'  c                   óÆ   ‡ — e Zd Zˆ fd„Zd„ Zd„ Zd„ Ze	 	 	 	 	 	 	 ddej	        dz  dej	        dz  de
dz  d	edz  d
edz  dedz  dej	        dz  deez  fd„¦   «         Zˆ xZS )Ú
MambaModelc                 ó¬  •‡— t          ¦   «                              ‰¦  «         t          j        ‰j        ‰j        ¦  «        | _        t          j        ˆfd„t          ‰j	        ¦  «        D ¦   «         ¦  «        | _
        d| _        t          ‰j        ‰j        ¬¦  «        | _        |                      | j        ¦  «         |                      ¦   «          d S )Nc                 ó2   •— g | ]}t          ‰|¬ ¦  «        ‘ŒS ))r”   )r  )Ú.0Úidxr“   s     €r   ú
<listcomp>z'MambaModel.__init__.<locals>.<listcomp>j  s&   ø€ Ð$rÐ$rÐ$rÈ3¥Z°À#Ð%FÑ%FÔ%FÐ$rÐ$rÐ$rr    Fr  )rœ   r�   r   Ú	EmbeddingÚ
vocab_sizer5   Ú
embeddingsÚ
ModuleListr„   r  rÓ   Úgradient_checkpointingró   r  Únorm_fÚ"_register_load_state_dict_pre_hookÚ	load_hookÚ	post_init©r´   r“   rµ   s    `€r   r�   zMambaModel.__init__f  s¶   øø€ Ý‰Œ×Ò˜Ñ Ô Ð åœ, vÔ'8¸&Ô:LÑMÔMˆŒÝ”mÐ$rÐ$rÐ$rÐ$rÕRWÐX^ÔXpÑRqÔRqÐ$rÑ$rÔ$rÑsÔsˆŒà&+ˆÔ#Ý" 6Ô#5¸6Ô;TÐUÑUÔUˆŒà×/Ò/°´Ñ?Ô?Ð?Ø�ŠÑÔÐÐÐr    c                 óv   — |D ]5}d|v r/|                      |¦  «        ||                     dd¦  «        <    d S Œ6d S )Nz
embedding.zembeddings.)ÚpopÚreplace)r´   Ú
state_dictÚprefixÚargsÚks        r   r8  zMambaModel.load_hookr  sW   € Øð 	ð 	ˆAØ˜qÐ Ð ØEOÇ^Â^ÐTUÑEVÔEV�
˜1Ÿ9š9 \°=ÑAÔAÑBØ��ð !ð	ð 	r    c                 ó   — | j         S rR   ©r3  rÿ   s    r   Úget_input_embeddingszMambaModel.get_input_embeddingsx  s
   € ØŒÐr    c                 ó   — || _         d S rR   rC  ©r´   Únew_embeddingss     r   Úset_input_embeddingszMambaModel.set_input_embeddings{  s   € Ø(ˆŒˆˆr    NÚ	input_idsÚinputs_embedsrÎ   Ú	use_cacheÚoutput_hidden_statesÚreturn_dictr   Úreturnc                 ó  — |�|n| j         j        }|�|n| j        s| j         j        nd}|�|n| j         j        }|du |duz  rt          d¦  «        ‚|€|                      |¦  «        }| j        r| j        r|rd}|r|€t          | j         ¬¦  «        }|}	|rdnd}
| j	        D ]} ||	||¬¦  «        }	|r|
|	fz   }
Œ|  
                    |	¦  «        }	|r|
|	fz   }
|st          d„ |	||
fD ¦   «         ¦  «        S t          |	|r|nd|
¬¦  «        S )	a¤  
        cache_params (`Cache`, *optional*):
            If passed along, the model uses the previous state in all the blocks (which will give the output for the
            `input_ids` provided as if the model add `state_input_ids + input_ids` as context).
        use_cache (`bool`, *optional*):
            If set to `True`, the `cache_params` is returned and can be used to quickly generate the next logits.
        NFz:You must specify exactly one of input_ids or inputs_embeds)r“   rS   r
  c              3   ó   K  — | ]}|®|V — Œ	d S rR   rS   )r.  Úvs     r   ú	<genexpr>z%MambaModel.forward.<locals>.<genexpr>µ  s(   è è € ÐfÐf˜qÐXYÐXe˜ÐXeÐXeÐXeÐXeÐfÐfr    )r#  rÎ   r   )r“   rL  rÑ   rK  rM  Ú
ValueErrorr3  r5  r	   rÓ   r6  r%  r"  )r´   rI  rJ  rÎ   rK  rL  rM  r   r<   r   Úall_hidden_statesÚmixer_blocks               r   ræ   zMambaModel.forward~  s¯  € ð( %9Ð$DÐ Ð È$Ì+ÔJjð 	ð "+Ð!6�I�IÐZ^ÔZgÐ=r¸T¼[Ô=RÐ=RÐmrˆ	Ø%0Ð%<�k�kÀ$Ä+ÔBYˆà˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ŸOšO¨IÑ6Ô6ˆMàÔ&ð 	¨4¬=ð 	¸Yð 	ØˆIàð 	<˜Ð-Ý'¨t¬{Ð;Ñ;Ô;ˆLà%ˆØ"6Ð@˜B˜B¸DÐØœ;ð 	Ið 	IˆKØ'˜KØØ)Ø-ðñ ô ˆMð $ð IØ$5¸Ð8HÑ$HÐ!øàŸš MÑ2Ô2ˆàð 	EØ 1°]Ð4DÑ DÐàð 	gÝÐfÐf ]°LÐBSÐ$TÐfÑfÔfÑfÔfÐfåØ+Ø)2Ð<˜˜¸Ø+ð
ñ 
ô 
ð 	
r    )NNNNNNN)rè   ré   rê   r�   r8  rD  rH  r   r.   rï   r   rì   r%  r"  ræ   rð   rñ   s   @r   r+  r+  d  s  ø€ € € € € ð
ð 
ð 
ð 
ð 
ðð ð ðð ð ð)ð )ð )ð ð .2Ø15Ø%)Ø!%Ø,0Ø#'Ø26ð<
ð <
àÔ# dÑ*ð<
ð Ô'¨$Ñ.ð<
ð ˜d‘lð	<
ð
 ˜$‘;ð<
ð # T™kð<
ð ˜D‘[ð<
ð Ô(¨4Ñ/ð<
ð 
�Ñ	ð<
ð <
ð <
ñ „^ð<
ð <
ð <
ð <
ð <
r    r+  zˆ
    The MAMBA Model transformer with a language modeling head on top (linear layer with weights tied to the input
    embeddings).
    c                   ó4  ‡ — e Zd ZddiZˆ fd„Zd„ Zd„ Z	 	 	 	 	 ddedz  d	ej	        dz  d
e
dz  fˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej	        dz  d	ej	        dz  dej        dz  dedz  dej	        dz  de
dz  de
dz  de
dz  deej        z  deez  fd„¦   «         Zˆ xZS )ÚMambaForCausalLMzlm_head.weightzbackbone.embeddings.weightc                 óæ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        |j        d¬¦  «        | _        |  	                    ¦   «          d S )NFrš   )
rœ   r�   r+  r  r   r¨   r5   r2  Úlm_headr9  r:  s     €r   r�   zMambaForCausalLM.__init__Ç  s^   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý" 6Ñ*Ô*ˆŒÝ”y Ô!3°VÔ5FÈUÐSÑSÔSˆŒà�ŠÑÔÐÐÐr    c                 ó4   — | j                              ¦   «         S rR   )r  rD  rÿ   s    r   rD  z%MambaForCausalLM.get_input_embeddingsÎ  s   € ØŒ}×1Ò1Ñ3Ô3Ð3r    c                 ó6   — | j                              |¦  «        S rR   )r  rH  rF  s     r   rH  z%MambaForCausalLM.set_input_embeddingsÑ  s   € ØŒ}×1Ò1°.ÑAÔAÐAr    NFrÎ   r   Úis_first_iterationc           	      óZ   •—  t          ¦   «         j        |f|||||dœ|¤Ž}|r|sd |d<   |S )N)rJ  rK  rÎ   r   r\  r   )rœ   Úprepare_inputs_for_generation)
r´   rI  rJ  rK  rÎ   r   r\  r<   Úmodel_inputsrµ   s
            €r   r^  z.MambaForCausalLM.prepare_inputs_for_generationÔ  sf   ø€ ð =•u‘w”wÔ<Øð
à'ØØ%Ø)Ø1ð
ð 
ð ð
ð 
ˆð ð 	2Ð/ð 	2Ø-1ˆLÐ)Ñ*àÐr    r   rI  rJ  ÚlabelsrL  rM  rK  Úlogits_to_keeprN  c
           	      ó  — |�|n| j         j        }|                      |||||||¬¦  «        }|d         }t          |	t          ¦  «        rt          |	 d¦  «        n|	}|                      |dd…|dd…f                              | j        j        j	        ¦  «        ¦  «         
                    ¦   «         }d}|� | j        d||| j         j        dœ|
¤Ž}|s|f|dd…         z   }|�|f|z   n|S t          |||j        |j        ¬¦  «        S )aN  
        cache_params (`Cache`, *optional*):
            If passed along, the model uses the previous state in all the blocks (which will give the output for the
            `input_ids` provided as if the model add `state_input_ids + input_ids` as context).
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
        use_cache (`bool`, *optional*):
            If set to `True`, the `cache_params` is returned and can be used to quickly generate the next logits.
        N)rÎ   rJ  rL  rM  rK  r   r   )r)  r`  r2  r   )r(  r)  rÎ   r   rS   )r“   rM  r  r  r¢   ÚslicerY  r   r$   r   r_   Úloss_functionr2  r'  rÎ   r   )r´   rI  r   rJ  rÎ   r`  rL  rM  rK  ra  r<   Úmamba_outputsr   Úslice_indicesr)  r(  Úoutputs                    r   ræ   zMambaForCausalLM.forwardí  sN  € ð2 &1Ð%<�k�kÀ$Ä+ÔBYˆàŸšØØ%Ø'Ø!5Ø#ØØ)ð &ñ 
ô 
ˆð & aÔ(ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@×CÒCÀDÄLÔDWÔD]Ñ^Ô^Ñ_Ô_×eÒeÑgÔgˆàˆØÐØ%�4Ô%Ðp¨V¸FÈtÌ{ÔOeÐpÐpÐioÐpÐpˆDàð 	FØ�Y ¨q¨r¨rÔ!2Ñ2ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå"ØØØ&Ô3Ø'Ô5ð	
ñ 
ô 
ð 	
r    )NNNNF)	NNNNNNNNr   )rè   ré   rê   Ú_tied_weights_keysr�   rD  rH  r   r.   rï   rì   r^  r   r$  r¢   rî   r%  r'  ræ   rð   rñ   s   @r   rW  rW  ¾  sª  ø€ € € € € ð +Ð,HÐIÐðð ð ð ð ð4ð 4ð 4ðBð Bð Bð ØØ%)Ø26Ø*/ðð ð
 ˜d‘lðð Ô(¨4Ñ/ðð ! 4™Kðð ð ð ð ð ð2 ð .2Ø26Ø26Ø%)Ø*.Ø,0Ø#'Ø!%Ø-.ð6
ð 6
àÔ# dÑ*ð6
ð Ô(¨4Ñ/ð6
ð Ô(¨4Ñ/ð	6
ð
 ˜d‘lð6
ð Ô  4Ñ'ð6
ð # T™kð6
ð ˜D‘[ð6
ð ˜$‘;ð6
ð ˜eœlÑ*ð6
ð 
Ð$Ñ	$ð6
ð 6
ð 6
ñ „^ð6
ð 6
ð 6
ð 6
ð 6
r    rW  )rW  r+  r  rç   )	NNNNTNNNr?   )NNFN)NNNFFFF)?rë   rÆ   Údataclassesr   r.   Útorch.nn.functionalr   Ú
functionalr1   Ú r   r¾   Úactivationsr   Úcache_utilsr   r	   Ú
generationr
   Úintegrationsr   r   Úintegrations.accelerater   Úmodeling_layersr   Úmodeling_utilsr   Úutilsr   r   r   Úutils.import_utilsr   r   r   r   Úconfiguration_mambar   Ú
get_loggerrè   Úloggerr   rî   r­   Ústrr!   r:   rì   r_   r=   rh   r�   ÚModuler’   ró   r  r  r"  r'  r+  rW  Ú__all__rS   r    r   ú<module>r|     sà  ðð Ð à €€€Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )Ø WÐ WÐ WÐ WÐ WÐ WÐ WÐ WØ =Ð =Ð =Ð =Ð =Ð =Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø -Ð -Ð -Ð -Ð -Ð -ðð ð ð ð ð ð ð ð ð ð
ð ð ð ð ð ð ð ð ð ð ð ð -Ð ,Ð ,Ð ,Ð ,Ð ,ð 
ˆÔ	˜HÑ	%Ô	%€ð	ð 	ð 	ð (Ð'Ð(>ÀÑPÔPð
 !%Ø!ð'ð 'Ø”<ð'à”ð'ð ŒLð'ð Œ,˜Ñ
ð	'ð
 �d‘
ð'ð 'ð 'ñ QÔPð'ð& (Ð'Ð(:¸OÑLÔLð !%Ø!ð	'ð 'Ø”<ð'àŒLð'ð Œ,˜Ñ
ð'ð �d‘
ð	'ð 'ð 'ñ MÔLð'ð, (Ð'Ð(8¸+ÑFÔFð "Ø!Ø!Ø&*ØØ(,Ø(,Ø)-Ø ð#ð ØŒðà”<ðð ” Ñ$ðð ”<ð	ð
 ”|ðð ”\ðð ”< $Ñ&ðð „|ðð „|�dÑðð „|�dÑðð „|�dÑðð ”˜tÑ#ðð ðð ”, Ñ%ðð ”, Ñ%ðð  ”< $Ñ&ð!ð" ð#ð ð ñ GÔFðð. (Ð'Ð(@À+ÑNÔNð "Ø#'ØØ!ð*ð *ØŒ<ð*à”<ð*ð 	Œð*ð „|ð	*ð
 „|ð*ð „|ð*ð „|�dÑð*ð Œ\˜DÑ ð*ð ð*ð „|�dÑð*ð *ð *ñ OÔNð*ðZ (Ð'Ð(;¸[ÑIÔIð "Ø!Ø&*Ø Ø#ØØ!&ðhð hØ”<ðhàŒðhð „|ðhð „|ð	hð
 „|ðhð „|�dÑðhð „|�dÑðhð ”˜tÑ#ðhð ðhð ðhð ðhð ðhð hð hñ JÔIðhðV ÐØÐ)Ð+GÐIYÐ[oÐpñô ðE%ð E%ð E%ð E%ð E%�”ñ E%ô E%ñô ðE%ðPFð Fð Fð Fð F�2”9ñ Fô Fð Fð(ð ð ð ð Ð+ñ ô ð ð4 ð!>ð !>ð !>ð !>ð !>˜?ñ !>ô !>ñ „ð!>ðH €ððñ ô ð
 ð:ð :ð :ð :ð :�+ñ :ô :ñ „ñô ð:ð €ððñ ô ð
 ð:ð :ð :ð :ð :˜+ñ :ô :ñ „ñô ð:ð& ðV
ð V
ð V
ð V
ð V
Ð%ñ V
ô V
ñ „ðV
ðr €ððñ ô ð`
ð `
ð `
ð `
ð `
Ð+¨_ñ `
ô `
ñô ð`
ðF EÐ
DÐ
D€€€r    