§
    ·¯¸jÛ3  ã                   óÂ   — d Z ddlmZmZmZ ddlZddlmZ ddlm	Z	 ddl
mZmZ ddlmZ ddlmZmZ dd	lmZ  G d
„ dej        ¦  «        Z G d„ dej        ¦  «        ZdS )a:   Attention Pool 2D

Implementations of 2D spatial feature pooling using multi-head attention instead of average pool.

Based on idea in CLIP by OpenAI, licensed Apache 2.0
https://github.com/openai/CLIP/blob/3b473b0e682c091a9e53623eebc1ca1657385717/clip/model.py

Hacked together by / Copyright 2021 Ross Wightman
é    )ÚOptionalÚUnionÚTupleNé   )Úuse_fused_attn)Úget_device_dtypeÚ	to_2tuple)Úresample_abs_pos_embed)Úapply_rot_embed_catÚcreate_rope_embed)Útrunc_normal_c                   óf  ‡ — e Zd ZU dZej        j        e         ed<   	 	 	 	 	 	 	 	 	 	 	 	 	 d#de	de
e	         dee	ee	e	f         f         de
e	         de
e	         de
e	         dedededededefˆ fd„Zd$defd„Z	 	 	 	 d%de
e	         de
e         fd„Zdej        de	de	dej        fd „Zd$d!efd"„Zˆ xZS )&ÚRotAttentionPool2da�   Attention based 2D feature pooling w/ rotary (relative) pos embedding.
    This is a multi-head attention based replacement for (spatial) average pooling in NN architectures.

    Adapted from the AttentionPool2d in CLIP w/ rotary embedding instead of learned embed.
    https://github.com/openai/CLIP/blob/3b473b0e682c091a9e53623eebc1ca1657385717/clip/model.py

    NOTE: While this impl does not require a fixed feature size, performance at differeing resolutions from
    train varies widely and falls off dramatically. I'm not sure if there is a way around this... -RW

    Setting out_features=0 disables the output projection (pre_logits mode).
    Ú
fused_attnNé   é@   TFÚtokenç        ÚcatÚin_featuresÚout_featuresÚref_feat_sizeÚ	embed_dimÚhead_dimÚ	num_headsÚqkv_biasÚqkv_separateÚ	pool_typeÚclass_tokenÚ	drop_rateÚ	rope_typec           
      óä  •— ||dœ}t          ¦   «                              ¦   «          |	dv sJ ‚|p|x| _        }|| _        |€|| _        n|dk    r|| _        n|| _        t          |¦  «        }|�||z  dk    sJ ‚||z  }n||z  dk    sJ ‚||z  }|| _        || _        |	                     ¦   «         | _	        | j        dz  | _
        t          ¦   «         | _        || _        |
r+t          j        t!          j        d|fi |¤Ž¦  «        | _        nd | _        |rVt          j        ||fd|i|¤Ž| _        t          j        ||fd|i|¤Ž| _        t          j        ||fd|i|¤Ž| _        d | _        nt          j        ||dz  fd|i|¤Ž| _        t          j        |¦  «        | _        |dk    rt          j        || j        fi |¤Žnt          j        ¦   «         | _        t9          d
|||d|dd	œ|¤Ž| _        d S )N©ÚdeviceÚdtype©Ú r   r   ç      à¿r   Úbiasé   F)r!   Údimr   Ú	in_pixelsÚref_feat_shapeÚrotate_half© )ÚsuperÚ__init__r   r   r   r	   r   r   Úlowerr   Úscaler   r   r!   ÚnnÚ	ParameterÚtorchÚzerosÚ	cls_tokenÚLinearÚqÚkÚvÚqkvÚDropoutÚdropÚIdentityÚprojr   Ú	pos_embed)Úselfr   r   r   r   r   r   r   r   r   r   r    r!   r$   r%   ÚddÚ	__class__s                   €úl/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/layers/attention_pool2d.pyr1   zRotAttentionPool2d.__init__$   sg  ø€ ð" ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ˜MÐ)Ð)Ð)Ð)Ø%.Ð%=°+Ð=ˆŒ˜Ø&ˆÔØÐØ +ˆDÔÐØ˜AÒÐØ ,ˆDÔÐà )ˆDÔÝ! -Ñ0Ô0ˆØÐ Ø˜yÑ(¨AÒ-Ð-Ð-Ð-Ø  IÑ-ˆHˆHà˜xÑ'¨1Ò,Ð,Ð,Ð,Ø! XÑ-ˆIØ"ˆŒØ ˆŒØ"ŸšÑ*Ô*ˆŒØ”] dÑ*ˆŒ
Ý(Ñ*Ô*ˆŒØ"ˆŒàð 	"Ýœ\­%¬+°a¸Ð*IÐ*IÀbÐ*IÐ*IÑJÔJˆDŒNˆNà!ˆDŒNàð 	RÝ”Y˜{¨IÐKÐK¸HÐKÈÐKÐKˆDŒFÝ”Y˜{¨IÐKÐK¸HÐKÈÐKÐKˆDŒFÝ”Y˜{¨IÐKÐK¸HÐKÈÐKÐKˆDŒFØˆDŒHˆHå”y ¨i¸!©mÐQÐQÀ(ÐQÈbÐQÐQˆDŒHÝ”J˜yÑ)Ô)ˆŒ	ØEQÐUVÒEVÐEV•B”I˜i¨Ô):ÐAÐA¸bÐAÐAÐAÕ\^Ô\gÑ\iÔ\iˆŒ	å*ð 
ØØØØØ(Øð
ð 
ð ð
ð 
ˆŒˆˆó    Úzero_init_lastc                 ó€  — | j         €ã| j        j        }t          | j        j        |dz  ¬¦  «         t
          j                             | j        j        ¦  «         t          | j	        j        |dz  ¬¦  «         t
          j                             | j	        j        ¦  «         t          | j
        j        |dz  ¬¦  «         t
          j                             | j
        j        ¦  «         d S | j         j        }t          | j         j        |dz  ¬¦  «         t
          j                             | j         j        ¦  «         d S ©Nr(   )Ústd)r=   r:   r   r   Úweightr4   ÚinitÚzeros_r)   r;   r<   ©rC   rH   r   s      rF   Úinit_weightszRotAttentionPool2d.init_weightsg   sú   € ØŒ8ÐØœ&Ô,ˆKÝ˜$œ&œ-¨[¸DÑ-@ÐAÑAÔAÐAÝŒG�NŠN˜4œ6œ;Ñ'Ô'Ð'Ý˜$œ&œ-¨[¸DÑ-@ÐAÑAÔAÐAÝŒG�NŠN˜4œ6œ;Ñ'Ô'Ð'Ý˜$œ&œ-¨[¸DÑ-@ÐAÑAÔAÐAÝŒG�NŠN˜4œ6œ;Ñ'Ô'Ð'Ð'Ð'àœ(Ô.ˆKÝ˜$œ(œ/¨{¸dÑ/BÐCÑCÔCÐCÝŒG�NŠN˜4œ8œ=Ñ)Ô)Ð)Ð)Ð)rG   Únum_classesc                 ó"  — t          | ||¬¦  «        }|�|dv sJ ‚|| _        |�k|dk    rt          j        | j        |fi |¤Žnt          j        ¦   «         | _        | j                             | j        ¦  «         |dk    r|n| j        | _	        d S d S ©Nr#   r&   r   ©
r   r   r4   r9   r   r@   rA   ÚtrainÚtrainingr   ©rC   rQ   r   r$   r%   rD   s         rF   ÚresetzRotAttentionPool2d.resetu   ó«   € õ ˜d¨6¸Ð?Ñ?Ô?ˆØÐ Ø Ð-Ð-Ð-Ð-Ø&ˆDŒNØÐ"ØHSÐVWÊÈ�œ	 $¤.°+ÐDÐDÀÐDÐDÐDÕ]_Ô]hÑ]jÔ]jˆDŒIØŒI�OŠO˜DœMÑ*Ô*Ð*Ø/:¸Qª¨  ÀDÄNˆDÔÐÐð #Ð"rG   ÚxÚHÚWÚreturnc                 óÀ   — | j         dk    r|d d …df         }nE|d d …dd …f                              |j        d         ||d¦  «                             dddd¦  «        }|S ©Nr   r   r   éÿÿÿÿr*   é   ©r   ÚreshapeÚshapeÚpermute©rC   rZ   r[   r\   s       rF   Ú_poolzRotAttentionPool2d._pool†   ól   € ØŒ>˜WÒ$Ð$Ø�!�!�!�Q�$”ˆAˆAð �!�!�!�Q�R�R�%”× Ò  ¤¨¤¨Q°°2Ñ6Ô6×>Ò>¸qÀ!ÀQÈÑJÔJˆAØˆrG   Ú
pre_logitsc                 óì  — |j         \  }}}}||z  }|                     d¦  «                             dd¦  «        }| j        €.t	          j        |                     dd¬¦  «        |gd¬¦  «        }n=t	          j        | j                             |j         d         dd¦  «        |gd¬¦  «        }| j        €å|  	                    |¦  «         
                    ||dz   | j        | j        ¦  «                             dd¦  «        }|                      |¦  «         
                    ||dz   | j        | j        ¦  «                             dd¦  «        }	|                      |¦  «         
                    ||dz   | j        | j        ¦  «                             dd¦  «        }
ni|                      |¦  «         
                    ||dz   d| j        | j        ¦  «                             ddddd	¦  «        }|                     d¦  «        \  }}	}
| j                             ||f¦  «        }t'          |t(          ¦  «        rt	          j        |d¬¦  «        }t	          j        |d d …d d …d d…d d …f         t+          |d d …d d …dd …d d …f         |¦  «        gd¬¦  «                             |
¦  «        }t	          j        |	d d …d d …d d…d d …f         t+          |	d d …d d …dd …d d …f         |¦  «        gd¬¦  «                             |
¦  «        }	| j        r"t0          j                             ||	|
¦  «        }n>|| j        z  }||	                     d
d¦  «        z  }|                     d¬¦  «        }||
z  }|                     dd¦  «         
                    ||dz   d¦  «        }|                      |¦  «        }|r|                      |||¦  «        }|S |                      |¦  «        }|                      |||¦  «        }|S )Nra   r   T©Úkeepdim©r+   r   r`   r*   é   éþÿÿÿ) rd   ÚflattenÚ	transposer8   r6   r   ÚmeanÚexpandr=   r:   rc   r   r   r;   r<   re   ÚunbindrB   Ú	get_embedÚ
isinstanceÚtupler   Útype_asr   r4   Ú
functionalÚscaled_dot_product_attentionr3   Úsoftmaxr?   rg   rA   )rC   rZ   ri   ÚBÚ_r[   r\   ÚNr:   r;   r<   ÚropeÚattns                rF   ÚforwardzRotAttentionPool2d.forwardŽ   s»  € Ø”W‰
ˆˆ1ˆa�Ø�‰EˆØ�IŠI�a‰LŒL×"Ò" 1 aÑ(Ô(ˆØŒ>Ð!Ý”	˜1Ÿ6š6 !¨T˜6Ñ2Ô2°AÐ6¸AÐ>Ñ>Ô>ˆAˆAå”	˜4œ>×0Ò0°´¸´¸RÀÑDÔDÀaÐHÈaÐPÑPÔPˆAØŒ8ÐØ—’�q‘	”	×!Ò! ! Q¨¡U¨D¬N¸D¼MÑJÔJ×TÒTÐUVÐXYÑZÔZˆAØ—’�q‘	”	×!Ò! ! Q¨¡U¨D¬N¸D¼MÑJÔJ×TÒTÐUVÐXYÑZÔZˆAØ—’�q‘	”	×!Ò! ! Q¨¡U¨D¬N¸D¼MÑJÔJ×TÒTÐUVÐXYÑZÔZˆAˆAà—’˜‘”×#Ò# A q¨1¡u¨a°´ÀÄÑOÔO×WÒWÐXYÐ[\Ð^_ÐabÐdeÑfÔfˆAØ—h’h˜q‘k”k‰GˆAˆq�!àŒ~×'Ò'¨¨A¨Ñ/Ô/ˆÝ�d�EÑ"Ô"ð 	+å”9˜T rÐ*Ñ*Ô*ˆDÝŒI�q˜˜˜˜A˜A˜A˜r ˜r 1 1 1˜”~Õ':¸1¸Q¸Q¸QÀÀÀÀ1À2À2ÀqÀqÀq¸[¼>È4Ñ'PÔ'PÐQÐWXÐYÑYÔY×aÒaÐbcÑdÔdˆÝŒI�q˜˜˜˜A˜A˜A˜r ˜r 1 1 1˜”~Õ':¸1¸Q¸Q¸QÀÀÀÀ1À2À2ÀqÀqÀq¸[¼>È4Ñ'PÔ'PÐQÐWXÐYÑYÔY×aÒaÐbcÑdÔdˆàŒ?ð 	Ý”×:Ò:¸1¸aÀÑCÔCˆAˆAà�D”J‘ˆAØ�q—{’{ 2 rÑ*Ô*Ñ*ˆDØ—<’< B�<Ñ'Ô'ˆDØ�q‘ˆAØ�KŠK˜˜1ÑÔ×%Ò% a¨¨Q©°Ñ3Ô3ˆØ�IŠI�a‰LŒLˆØð 	Ø—
’
˜1˜a Ñ#Ô#ˆAØˆHØ�IŠI�a‰LŒLˆØ�JŠJ�q˜!˜QÑÔˆØˆrG   )Nr   Nr   NTFr   Fr   r   NN©F©NNNN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r6   ÚjitÚFinalÚboolÚ__annotations__Úintr   r   r   ÚstrÚfloatr1   rP   rX   ÚTensorrg   r�   Ú__classcell__©rE   s   @rF   r   r      sö  ø€ € € € € € ð
ð 
ð ”	” Ô%Ð%Ð%Ñ%ð
 +/Ø9:Ø'+Ø&(Ø'+Ø!Ø!&Ø$Ø %Ø!Ø"ØØðA
ð A
àðA
ð # 3œ-ðA
ð !  e¨C°¨H¤oÐ!5Ô6ð	A
ð
   ”}ðA
ð ˜s”mðA
ð   ”}ðA
ð ðA
ð ðA
ð ðA
ð ðA
ð ðA
ð ðA
ð A
ð A
ð A
ð A
ð A
ðF*ð *¨4ð *ð *ð *ð *ð  *.Ø'+ØØðSð Sà! #œðSð   ”}ðSð Sð Sð Sð"�u”|ð ¨ð °ð ¸¼ð ð ð ð ð%ð % Tð %ð %ð %ð %ð %ð %ð %ð %rG   r   c                   ó`  ‡ — e Zd ZU dZej        j        e         ed<   	 	 	 	 	 	 	 	 	 	 	 	 d!d
e	de
e	ee	e	f         f         dee	         dee	         dee	         dee	         dededededefˆ fd„Zd"defd„Z	 	 	 	 d#dee	         dee         fd„Zdej        de	de	dej        fd„Zd"defd „Zˆ xZS )$ÚAttentionPool2da÷   Attention based 2D feature pooling w/ learned (absolute) pos embedding.
    This is a multi-head attention based replacement for (spatial) average pooling in NN architectures.

    It was based on impl in CLIP by OpenAI
    https://github.com/openai/CLIP/blob/3b473b0e682c091a9e53623eebc1ca1657385717/clip/model.py

    NOTE: This requires feature size upon construction and well prevent adaptive sizing of the network.

    Setting out_features=0 disables the output projection (pre_logits mode).
    r   r   Nr   TFr   r   r   Ú	feat_sizer   r   r   r   r   r   r   r   r    c                 ó‚  •— ||dœ}t          ¦   «                              ¦   «          |	dv sJ ‚|p|x| _        }|| _        |€|| _        n|dk    r|| _        n|| _        |�||z  dk    sJ ‚||z  }n||z  dk    sJ ‚||z  }t          |¦  «        | _        | j        d         | j        d         z  | _        || _        || _	        |	| _
        | j	        dz  | _        t          ¦   «         | _        |
r+t          j        t!          j        d|fi |¤Ž¦  «        | _        nd | _        |rVt          j        ||fd|i|¤Ž| _        t          j        ||fd|i|¤Ž| _        t          j        ||fd|i|¤Ž| _        d | _        n2d x| _        x| _        | _        t          j        ||dz  fd|i|¤Ž| _        t          j        |¦  «        | _        |dk    rt          j        || j        fi |¤Žnt          j        ¦   «         | _        t          j        t!          j        | j        dz   |fi |¤Ž¦  «        | _        |                      ¦   «          d S )Nr#   r&   r   r   r(   r)   r*   )r0   r1   r   r   r   r	   r”   Úseq_lenr   r   r   r3   r   r   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rB   rP   )rC   r   r”   r   r   r   r   r   r   r   r   r    r$   r%   rD   rE   s                  €rF   r1   zAttentionPool2d.__init__Ã   s‘  ø€ ð  ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ˜MÐ)Ð)Ð)Ð)Ø%.Ð%=°+Ð=ˆŒ˜Ø&ˆÔØÐØ +ˆDÔÐØ˜AÒÐØ ,ˆDÔÐà )ˆDÔØÐ Ø˜yÑ(¨AÒ-Ð-Ð-Ð-Ø  IÑ-ˆHˆHà˜xÑ'¨1Ò,Ð,Ð,Ð,Ø! XÑ-ˆIÝ" 9Ñ-Ô-ˆŒØ”~ aÔ(¨4¬>¸!Ô+<Ñ<ˆŒØ"ˆŒØ ˆŒØ"ˆŒØ”] dÑ*ˆŒ
Ý(Ñ*Ô*ˆŒàð 	"Ýœ\­%¬+°a¸Ð*IÐ*IÀbÐ*IÐ*IÑJÔJˆDŒNˆNà!ˆDŒNàð 	RÝ”Y˜{¨IÐKÐK¸HÐKÈÐKÐKˆDŒFÝ”Y˜{¨IÐKÐK¸HÐKÈÐKÐKˆDŒFÝ”Y˜{¨IÐKÐK¸HÐKÈÐKÐKˆDŒFØˆDŒHˆHà'+Ð+ˆDŒFÐ+�T”V˜dœfÝ”y ¨i¸!©mÐQÐQÀ(ÐQÈbÐQÐQˆDŒHÝ”J˜yÑ)Ô)ˆŒ	ØEQÐUVÒEVÐEV•B”I˜i¨Ô):ÐAÐA¸bÐAÐAÐAÕ\^Ô\gÑ\iÔ\iˆŒ	Ýœ¥e¤k°$´,ÀÑ2BÀKÐ&VÐ&VÐSUÐ&VÐ&VÑWÔWˆŒà×ÒÑÔÐÐÐrG   rH   c                 ó°  — | j         €â| j        j        }t          | j        j        |dz  ¬¦  «         t
          j                             | j        j        ¦  «         t          | j	        j        |dz  ¬¦  «         t
          j                             | j	        j        ¦  «         t          | j
        j        |dz  ¬¦  «         t
          j                             | j
        j        ¦  «         nS| j         j        }t          | j         j        |dz  ¬¦  «         t
          j                             | j         j        ¦  «         t          | j        |dz  ¬¦  «         d S rJ   )r=   r:   r   r   rL   r4   rM   rN   r)   r;   r<   rB   rO   s      rF   rP   zAttentionPool2d.init_weightsÿ   s  € ØŒ8ÐØœ&Ô,ˆKÝ˜$œ&œ-¨[¸DÑ-@ÐAÑAÔAÐAÝŒG�NŠN˜4œ6œ;Ñ'Ô'Ð'Ý˜$œ&œ-¨[¸DÑ-@ÐAÑAÔAÐAÝŒG�NŠN˜4œ6œ;Ñ'Ô'Ð'Ý˜$œ&œ-¨[¸DÑ-@ÐAÑAÔAÐAÝŒG�NŠN˜4œ6œ;Ñ'Ô'Ð'Ð'àœ(Ô.ˆKÝ˜$œ(œ/¨{¸dÑ/BÐCÑCÔCÐCÝŒG�NŠN˜4œ8œ=Ñ)Ô)Ð)Ý�d”n¨+¸Ñ*=Ð>Ñ>Ô>Ð>Ð>Ð>rG   rQ   c                 ó"  — t          | ||¬¦  «        }|�|dv sJ ‚|| _        |�k|dk    rt          j        | j        |fi |¤Žnt          j        ¦   «         | _        | j                             | j        ¦  «         |dk    r|n| j        | _	        d S d S rS   rT   rW   s         rF   rX   zAttentionPool2d.reset  rY   rG   rZ   r[   r\   r]   c                 óÀ   — | j         dk    r|d d …df         }nE|d d …dd …f                              |j        d         ||d¦  «                             dddd¦  «        }|S r_   rb   rf   s       rF   rg   zAttentionPool2d._pool  rh   rG   ri   c                 óF  — |j         \  }}}}||z  }|                     d¦  «                             dd¦  «        }| j        €.t	          j        |                     dd¬¦  «        |gd¬¦  «        }n=t	          j        | j                             |j         d         dd¦  «        |gd¬¦  «        }t          | j	         
                    d¦  «        ||fd¬¦  «        }||z   }| j        €å|                      |¦  «                             ||dz   | j        | j        ¦  «                             dd¦  «        }	|                      |¦  «                             ||dz   | j        | j        ¦  «                             dd¦  «        }
|                      |¦  «                             ||dz   | j        | j        ¦  «                             dd¦  «        }nf|                      |¦  «                             |dd	| j        | j        ¦  «                             ddd	dd
¦  «        }|                     d¦  «        \  }	}
}| j        r"t*          j                             |	|
|¦  «        }n>|	| j        z  }	|	|
                     dd¦  «        z  }|                     d¬¦  «        }||z  }|                     dd¦  «                             ||dz   d¦  «        }|                      |¦  «        }|r|                      |||¦  «        }|S |                      |¦  «        }|                      |||¦  «        }|S )Nra   r   Trk   rm   r   r`   )Únum_prefix_tokensr*   rn   ro   )rd   rp   rq   r8   r6   r   rr   rs   r
   rB   Ú	unsqueezer=   r:   rc   r   r   r;   r<   re   rt   r   r4   ry   rz   r3   r{   r?   rg   rA   )rC   rZ   ri   r|   r}   r[   r\   r~   rB   r:   r;   r<   r€   s                rF   r�   zAttentionPool2d.forward'  sÒ  € Ø”W‰
ˆˆ1ˆa�Ø�‰EˆØ�IŠI�a‰LŒL×"Ò" 1 aÑ(Ô(ˆØŒ>Ð!Ý”	˜1Ÿ6š6 !¨T˜6Ñ2Ô2°AÐ6¸AÐ>Ñ>Ô>ˆAˆAå”	˜4œ>×0Ò0°´¸´¸RÀÑDÔDÀaÐHÈaÐPÑPÔPˆAÝ*¨4¬>×+CÒ+CÀAÑ+FÔ+FÈÈAÈÐbcÐdÑdÔdˆ	Ø�	‰MˆàŒ8ÐØ—’�q‘	”	×!Ò! ! Q¨¡U¨D¬N¸D¼MÑJÔJ×TÒTÐUVÐXYÑZÔZˆAØ—’�q‘	”	×!Ò! ! Q¨¡U¨D¬N¸D¼MÑJÔJ×TÒTÐUVÐXYÑZÔZˆAØ—’�q‘	”	×!Ò! ! Q¨¡U¨D¬N¸D¼MÑJÔJ×TÒTÐUVÐXYÑZÔZˆAˆAà—’˜‘”×#Ò# A r¨1¨d¬n¸d¼mÑLÔL×TÒTÐUVÐXYÐ[\Ð^_ÐabÑcÔcˆAØ—h’h˜q‘k”k‰GˆAˆq�!àŒ?ð 	Ý”×:Ò:¸1¸aÀÑCÔCˆAˆAà�D”J‘ˆAØ�q—{’{ 2 rÑ*Ô*Ñ*ˆDØ—<’< B�<Ñ'Ô'ˆDØ�q‘ˆAØ�KŠK˜˜1ÑÔ×%Ò% a¨¨Q©°Ñ3Ô3ˆØ�IŠI�a‰LŒLˆØð 	Ø—
’
˜1˜a Ñ#Ô#ˆAØˆHØ�IŠI�a‰LŒLˆØ�JŠJ�q˜!˜QÑÔˆØˆrG   )r   NNr   NTFr   Fr   NNr‚   rƒ   )r„   r…   r†   r‡   r6   rˆ   r‰   rŠ   r‹   rŒ   r   r   r   r�   rŽ   r1   rP   rX   r�   rg   r�   r�   r‘   s   @rF   r“   r“   ¶   sÖ  ø€ € € € € € ð	ð 	ð ”	” Ô%Ð%Ð%Ñ%ð
 67Ø*.Ø'+Ø&(Ø'+Ø!Ø!&Ø$Ø %Ø!ØØð:ð :àð:ð ˜S %¨¨S¨¤/Ð1Ô2ð:ð # 3œ-ð	:ð
   ”}ð:ð ˜s”mð:ð   ”}ð:ð ð:ð ð:ð ð:ð ð:ð ð:ð :ð :ð :ð :ð :ðx?ð ?¨4ð ?ð ?ð ?ð ?ð" *.Ø'+ØØðSð Sà! #œðSð   ”}ðSð Sð Sð Sð"�u”|ð ¨ð °ð ¸¼ð ð ð ð ð!ð ! Tð !ð !ð !ð !ð !ð !ð !ð !rG   r“   )r‡   Útypingr   r   r   r6   Útorch.nnr4   Úconfigr   Úhelpersr   r	   rB   r
   Úpos_embed_sincosr   r   Úweight_initr   ÚModuler   r“   r/   rG   rF   ú<module>r¤      s%  ððð ð *Ð )Ð )Ð )Ð )Ð )Ð )Ð )Ð )Ð )à €€€Ø Ð Ð Ð Ð Ð à "Ð "Ð "Ð "Ð "Ð "Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø -Ð -Ð -Ð -Ð -Ð -Ø DÐ DÐ DÐ DÐ DÐ DÐ DÐ DØ &Ð &Ð &Ð &Ð &Ð &ð]ð ]ð ]ð ]ð ]˜œñ ]ô ]ð ]ð@Rð Rð Rð Rð R�b”iñ Rô Rð Rð Rð RrG   