§
    ·¯¸joe  ã            
       ó°  — d Z ddlZddlZddlZddlmZ ddlmZm	Z	m
Z
mZmZ ddlZddlmc mZ ddlmZ ddlmZmZ ddlmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z" dd	l#m$Z$ dd
l%m&Z& ddl'm(Z( ddl)m*Z*m+Z+ ddl,m-Z-m.Z.m/Z/ dgZ0 ej1        e2¦  «        Z3 G d„ dej4        ¦  «        Z5 G d„ dej4        ¦  «        Z6 G d„ dej4        ¦  «        Z7de8fd„Z9e(de8fd„¦   «         Z: G d„ dej4        ¦  «        Z; G d„ dej4        ¦  «        Z<d4dej4        de=de>fd „Z?d!„ Z@d"„ ZAd5d$„ZBd6d%„ZC e. eC¦   «          eC¦   «          eC¦   «          eCd&¬'¦  «         eCd&¬'¦  «         eCd&¬'¦  «        d(œ¦  «        ZDe-d5d)e<fd*„¦   «         ZEe-d5d)e<fd+„¦   «         ZFe-d5d)e<fd,„¦   «         ZGe-d5d)e<fd-„¦   «         ZHe-d5d)e<fd.„¦   «         ZIe-d5d)e<fd/„¦   «         ZJ e/e2d0d1d2d3œ¦  «         dS )7a   Nested Transformer (NesT) in PyTorch

A PyTorch implement of Aggregating Nested Transformers as described in:

'Aggregating Nested Transformers'
    - https://arxiv.org/abs/2105.12723

The official Jax code is released and available at https://github.com/google-research/nested-transformer. The weights
have been converted with convert/convert_nest_flax.py

Acknowledgments:
* The paper authors for sharing their research, code, and model weights
* Ross Wightman's existing code off which I based this

Copyright 2021 Alexander Soare
é    N)Úpartial)ÚListÚOptionalÚTupleÚTypeÚUnion)Únn©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)Úget_device_dtypeÚ
PatchEmbedÚMlpÚDropPathÚcalculate_drop_path_ratesÚcreate_classifierÚtrunc_normal_Ú_assertÚcreate_conv2dÚcreate_pool2dÚ	to_ntupleÚuse_fused_attnÚ	LayerNormé   )Úbuild_model_with_cfg)Úfeature_take_indices)Úregister_notrace_function)Úcheckpoint_seqÚnamed_apply)Úregister_modelÚgenerate_default_cfgsÚregister_model_deprecationsÚNestc                   óx   ‡ — e Zd ZU dZej        j        e         ed<   	 	 	 	 	 	 dde	de	d	ed
e
de
f
ˆ fd„Zd„ Zˆ xZS )Ú	Attentionzš
    This is much like `.vision_transformer.Attention` but uses *localised* self attention by accepting an input with
     an extra "image block" dim
    Ú
fused_attné   Fç        NÚdimÚ	num_headsÚqkv_biasÚ	attn_dropÚ	proj_dropc                 ór  •— ||dœ}t          ¦   «                              ¦   «          || _        ||z  }	|	dz  | _        t	          ¦   «         | _        t          j        |d|z  fd|i|¤Ž| _        t          j	        |¦  «        | _
        t          j        ||fi |¤Ž| _        t          j	        |¦  «        | _        d S )N©ÚdeviceÚdtypeg      à¿é   Úbias)ÚsuperÚ__init__r*   Úscaler   r&   r	   ÚLinearÚqkvÚDropoutr,   Úprojr-   )Úselfr)   r*   r+   r,   r-   r0   r1   ÚddÚhead_dimÚ	__class__s             €ú`/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/models/nest.pyr5   zAttention.__init__>   s·   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ"ˆŒØ˜)Ñ#ˆØ Ñ%ˆŒ
Ý(Ñ*Ô*ˆŒå”9˜S ! C¡%Ð=Ð=¨hÐ=¸"Ð=Ð=ˆŒÝœ IÑ.Ô.ˆŒÝ”I˜c 3Ð-Ð-¨"Ð-Ð-ˆŒ	Ýœ IÑ.Ô.ˆŒˆˆó    c           	      ó²  — |j         \  }}}}|                      |¦  «                             |||d| j        || j        z  ¦  «                             dddddd¦  «        }|                     d¦  «        \  }}}	| j        r,t          j        |||	| j	        r| j
        j        nd¬¦  «        }nS|| j        z  }||                     d	d
¦  «        z  }
|
                     d
¬¦  «        }
|  
                    |
¦  «        }
|
|	z  }|                     ddddd¦  «                             ||||¦  «        }|                      |¦  «        }|                      |¦  «        }|S )zm
        x is shape: B (batch_size), T (image blocks), N (seq length per image block), C (embed dim)
        r2   r   é   r   é   é   r(   )Ú	dropout_péþÿÿÿéÿÿÿÿ)r)   )Úshaper8   Úreshaper*   ÚpermuteÚunbindr&   ÚFÚscaled_dot_product_attentionÚtrainingr,   Úpr6   Ú	transposeÚsoftmaxr:   r-   )r;   ÚxÚBÚTÚNÚCr8   ÚqÚkÚvÚattns              r?   ÚforwardzAttention.forwardT   sS  € ð ”W‰
ˆˆ1ˆa�à�hŠh�q‰kŒk×!Ò! ! Q¨¨1¨d¬n¸aÀ4Ä>Ñ>QÑRÔR×ZÒZÐ[\Ð^_ÐabÐdeÐghÐjkÑlÔlˆØ—*’*˜Q‘-”-‰ˆˆ1ˆaàŒ?ð 	ÝÔ.¨q°!°QÐVZÔVcÐBkÀ$Ä.ÔBRÐBRÐikÐlÑlÔlˆAˆAà�D”J‘ˆAØ�q—{’{ 2 rÑ*Ô*Ñ*ˆDØ—<’< B�<Ñ'Ô'ˆDØ—>’> $Ñ'Ô'ˆDØ�q‘ˆAð �IŠI�a˜˜A˜q !Ñ$Ô$×,Ò,¨Q°°1°aÑ8Ô8ˆØ�IŠI�a‰LŒLˆØ�NŠN˜1ÑÔˆØˆr@   )r'   Fr(   r(   NN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚtorchÚjitÚFinalÚboolÚ__annotations__ÚintÚfloatr5   r[   Ú__classcell__©r>   s   @r?   r%   r%   7   s¸   ø€ € € € € € ðð ð ”	” Ô%Ð%Ð%Ñ%ð
 Ø"Ø!Ø!ØØð/ð /àð/ð ð/ð ð	/ð
 ð/ð ð/ð /ð /ð /ð /ð /ð,ð ð ð ð ð ð r@   r%   c                   ó¢   ‡ — e Zd ZdZdddddej        ej        ddf	dededed	e	d
ededede
ej                 de
ej                 fˆ fd„Zd„ Zˆ xZS )ÚTransformerLayerzü
    This is much like `.vision_transformer.Block` but:
        - Called TransformerLayer here to allow for "block" as defined in the paper ("non-overlapping image blocks")
        - Uses modified Attention layer that handles the "block" dimension
    ç      @Fr(   Nr)   r*   Ú	mlp_ratior+   r-   r,   Ú	drop_pathÚ	act_layerÚ
norm_layerc                 ó¼  •— |
|dœ}t          ¦   «                              ¦   «           |	|fi |¤Ž| _        t          |f||||dœ|¤Ž| _        |dk    rt          |¦  «        nt          j        ¦   «         | _         |	|fi |¤Ž| _	        t          ||z  ¦  «        }t          d||||dœ|¤Ž| _        |dk    rt          |¦  «        nt          j        ¦   «         | _        d S )Nr/   )r*   r+   r,   r-   r(   )Úin_featuresÚhidden_featuresrn   Údrop© )r4   r5   Únorm1r%   rZ   r   r	   ÚIdentityÚ
drop_path1Únorm2re   r   ÚmlpÚ
drop_path2)r;   r)   r*   rl   r+   r-   r,   rm   rn   ro   r0   r1   r<   Úmlp_hidden_dimr>   s                 €r?   r5   zTransformerLayer.__init__s   s  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ�Z Ð*Ð* rÐ*Ð*ˆŒ
ÝØð
àØØØð
ð 
ð ð
ð 
ˆŒ	ð 2;¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒØ�Z Ð*Ð* rÐ*Ð*ˆŒ
Ý˜S 9™_Ñ-Ô-ˆÝð 
ØØ*ØØð	
ð 
ð
 ð
ð 
ˆŒð 2;¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒˆˆr@   c                 ó  — |                       |¦  «        }||                      |                      |¦  «        ¦  «        z   }||                      |                      |                      |¦  «        ¦  «        ¦  «        z   }|S ©N)ru   rw   rZ   rz   ry   rx   )r;   rR   Úys      r?   r[   zTransformerLayer.forward˜   sb   € Ø�JŠJ�q‰MŒMˆØ�—’ §	¢	¨!¡¤Ñ-Ô-Ñ-ˆØ�—’ §¢¨¯ª°A©¬Ñ 7Ô 7Ñ8Ô8Ñ8ˆØˆr@   )r\   r]   r^   r_   r	   ÚGELUr   re   rf   rc   r   ÚModuler5   r[   rg   rh   s   @r?   rj   rj   m   sñ   ø€ € € € € ðð ð  "Ø"Ø!Ø!Ø!Ø)+¬Ø*,¬,ØØð#Sð #Sàð#Sð ð#Sð ð	#Sð
 ð#Sð ð#Sð ð#Sð ð#Sð ˜BœI”ð#Sð ˜RœYœð#Sð #Sð #Sð #Sð #Sð #SðJð ð ð ð ð ð r@   rj   c            	       óT   ‡ — e Zd Z	 	 	 d	dededeej                 defˆ fd„Zd„ Z	ˆ xZ
S )
ÚConvPoolÚ NÚin_channelsÚout_channelsro   Úpad_typec                 óÌ   •— ||dœ}t          ¦   «                              ¦   «          t          ||fd|ddœ|¤Ž| _         ||fi |¤Ž| _        t          ddd|¬¦  «        | _        d S )Nr/   r2   T)Úkernel_sizeÚpaddingr3   ÚmaxrC   )rˆ   Ústrider‰   )r4   r5   r   ÚconvÚnormr   Úpool)	r;   r„   r…   ro   r†   r0   r1   r<   r>   s	           €r?   r5   zConvPool.__init__    s‚   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ! +¨|ÐnÈÐT\ÐcgÐnÐnÐkmÐnÐnˆŒ	Ø�J˜|Ð2Ð2¨rÐ2Ð2ˆŒ	Ý! %°Q¸qÈ(ÐSÑSÔSˆŒ	ˆ	ˆ	r@   c                 ód  — t          |j        d         dz  dk    d¦  «         t          |j        d         dz  dk    d¦  «         |                      |¦  «        }|                      |                     dddd¦  «        ¦  «                             dddd¦  «        }|                      |¦  «        }|S )z:
        x is expected to have shape (B, C, H, W)
        rF   rC   r   z1BlockAggregation requires even input spatial dimsrG   r2   r   )r   rH   rŒ   r�   rJ   rŽ   ©r;   rR   s     r?   r[   zConvPool.forward¯   sž   € õ 	�”˜”˜a‘ 1Ò$Ð&YÑZÔZÐZÝ�”˜”˜a‘ 1Ò$Ð&YÑZÔZÐZØ�IŠI�a‰LŒLˆà�IŠI�a—i’i  1 a¨Ñ+Ô+Ñ,Ô,×4Ò4°Q¸¸1¸aÑ@Ô@ˆØ�IŠI�a‰LŒLˆØˆr@   )rƒ   NN)r\   r]   r^   re   r   r	   r€   Ústrr5   r[   rg   rh   s   @r?   r‚   r‚   Ÿ   s“   ø€ € € € € ð ØØðTð TàðTð ðTð ˜RœYœð	Tð
 ðTð Tð Tð Tð Tð Tð
ð 
ð 
ð 
ð 
ð 
ð 
r@   r‚   Ú
block_sizec                 ó   — | j         \  }}}}t          ||z  dk    d¦  «         t          ||z  dk    d¦  «         ||z  }||z  }|                      ||||||¦  «        } |                      dd¦  «                             |||z  d|¦  «        } | S )zšimage to blocks
    Args:
        x (Tensor): with shape (B, H, W, C)
        block_size (int): edge length of a single square block in units of H, W
    r   z,`block_size` must divide input height evenlyz+`block_size` must divide input width evenlyrC   r2   rG   )rH   r   rI   rP   )rR   r’   rS   ÚHÚWrV   Úgrid_heightÚ
grid_widths           r?   Úblockifyr˜   ¼   s¥   € ð ”'�J€A€qˆ!ˆQÝˆA�
‰N˜aÒÐ!OÑPÔPÐPÝˆA�
‰N˜aÒÐ!NÑOÔOÐOØ�z‘/€KØ�j‘€JØ	�	Š	�!�[ *¨j¸*ÀaÑHÔH€AØ	�Š�A�qÑÔ×!Ò! ! [°:Ñ%=¸rÀ1ÑEÔE€AØ€Hr@   c                 óú   — | j         \  }}}}t          t          j        |¦  «        ¦  «        }||z  x}}|                      ||||||¦  «        } |                      dd¦  «                             ||||¦  «        } | S )zßblocks to image
    Args:
        x (Tensor): with shape (B, T, N, C) where T is number of blocks and N is sequence size per block
        block_size (int): edge length of a single square block in units of desired H, W
    rC   r2   )rH   re   ÚmathÚsqrtrI   rP   )	rR   r’   rS   rT   Ú_rV   Ú	grid_sizeÚheightÚwidths	            r?   Ú
deblockifyr    Ì   s~   € ð ”�J€A€qˆ!ˆQÝ•D”I˜a‘L”LÑ!Ô!€IØ Ñ+Ð+€FˆUØ	�	Š	�!�Y 	¨:°zÀ1ÑEÔE€AØ	�Š�A�qÑÔ×!Ò! ! V¨U°AÑ6Ô6€AØ€Hr@   c                   óæ   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 ddeded	ed
edededee         dededededeee                  dee	e
j                          dee	e
j                          defˆ fd„Zd„ Zˆ xZS )Ú	NestLevelz7 Single hierarchical level of a Nested Transformer
    Nrk   Tr(   rƒ   Ú
num_blocksr’   Ú
seq_lengthr*   ÚdepthÚ	embed_dimÚprev_embed_dimrl   r+   r-   r,   rm   ro   rn   r†   c                 óþ  •‡‡‡‡	‡
‡‡‡‡‡— ||dœŠt          ¦   «                              ¦   «          || _        d| _        t	          j        t          j        d||‰fi ‰¤Ž¦  «        | _        |�t          |‰f‰|dœ‰¤Ž| _
        nt	          j        ¦   «         | _
        t          ‰¦  «        rt          ‰¦  «        |k    s
J d¦   «         ‚t	          j        ˆˆˆˆˆˆˆˆˆ
ˆ	f
d„t          |¦  «        D ¦   «         Ž | _        d S )Nr/   Fr   )ro   r†   zDMust provide as many drop path rates as there are transformer layersc                 óP   •
— g | ]"}t          d‰‰	‰‰‰
‰‰r‰|         nd ‰‰dœ	‰¤Ž‘Œ#S )N)	r)   r*   rl   r+   r-   r,   rm   ro   rn   rt   )rj   )Ú.0Úirn   r,   r<   rm   r¦   rl   ro   r*   r-   r+   s     €€€€€€€€€€r?   ú
<listcomp>z&NestLevel.__init__.<locals>.<listcomp>  sq   ø€ ð 3#ð 3#ð 3#ð õ ð ØØ#Ø#Ø!Ø#Ø#Ø*3Ð=˜) Aœ,˜,¸Ø%Ø#ðð ð ðð ð3#ð 3#ð 3#r@   )r4   r5   r’   Úgrad_checkpointingr	   Ú	Parameterr`   ÚzerosÚ	pos_embedr‚   rŽ   rv   ÚlenÚ
SequentialÚrangeÚtransformer_encoder)r;   r£   r’   r¤   r*   r¥   r¦   r§   rl   r+   r-   r,   rm   ro   rn   r†   r0   r1   r<   r>   s       ` ` ```````   @€r?   r5   zNestLevel.__init__Þ   sC  øøøøøøøøøøø€ ð( ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ$ˆŒØ"'ˆÔåœ¥e¤k°!°ZÀÈYÐ&]Ð&]ÐZ\Ð&]Ð&]Ñ^Ô^ˆŒàÐ%Ý  °ÐkÀzÐ\dÐkÐkÐhjÐkÐkˆDŒIˆIåœ™œˆDŒIõ ˆy‰>Œ>ð 	sÝ�y‘>”> UÒ*Ð*Ð*Ð,rÑ*Ô*Ð*Ý#%¤=ð 3#ð 3#ð 3#ð 3#ð 3#ð 3#ð 3#ð 3#ð 3#ð 3#ð 3#ð 3#ð 3#õ ˜5‘\”\ð3#ñ 3#ô 3#ð $$ˆÔ Ð Ð r@   c                 ó”  — |                       |¦  «        }|                     dddd¦  «        }t          || j        ¦  «        }|| j        z   }| j        r4t          j                             ¦   «         st          | j
        |¦  «        }n|  
                    |¦  «        }t          || j        ¦  «        }|                     dddd¦  «        S )z+
        expects x as (B, C, H, W)
        r   rC   r2   r   )rŽ   rJ   r˜   r’   r°   r­   r`   ra   Úis_scriptingr   r´   r    r�   s     r?   r[   zNestLevel.forward  s·   € ð �IŠI�a‰LŒLˆØ�IŠI�a˜˜A˜qÑ!Ô!ˆÝ�Q˜œÑ(Ô(ˆØ�”ÑˆØÔ"ð 	,­5¬9×+AÒ+AÑ+CÔ+Cð 	,Ý˜tÔ7¸Ñ;Ô;ˆAˆAà×(Ò(¨Ñ+Ô+ˆAÝ�q˜$œ/Ñ*Ô*ˆà�yŠy˜˜A˜q !Ñ$Ô$Ð$r@   )Nrk   Tr(   r(   NNNrƒ   NN)r\   r]   r^   r_   re   r   rf   rc   r   r   r	   r€   r‘   r5   r[   rg   rh   s   @r?   r¢   r¢   Û   s6  ø€ € € € € ðð ð -1Ø!Ø!Ø!Ø!Ø/3Ø48Ø37ØØØð%0$ð 0$àð0$ð ð0$ð ð	0$ð
 ð0$ð ð0$ð ð0$ð % SœMð0$ð ð0$ð ð0$ð ð0$ð ð0$ð    U¤Ô,ð0$ð !  b¤i¤Ô1ð0$ð    R¤Y¤Ô0ð0$ð  ð!0$ð 0$ð 0$ð 0$ð 0$ð 0$ðd%ð %ð %ð %ð %ð %ð %r@   r¢   c            '       ó  ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d=dededededeedf         deedf         deedf         dedededededededee	e
j                          d ee	e
j                          d!ed"ed#ef&ˆ fd$„Zej        j        d>d%„¦   «         Zej        j        d&„ ¦   «         Zej        j        d?d(„¦   «         Zej        j        d@d)„¦   «         Zej        j        d*e
j        fd+„¦   «         ZdAded#efd,„Z	 	 	 	 	 dBd.ej        d/eeeee         f                  d0ed1ed2ed3ed*eeej                 eej        eej                 f         f         fd4„Z	 	 	 dCd/eeee         f         d6ed7efd8„Zd9„ Zd?d:efd;„Zd<„ Zˆ xZS )Dr#   z… Nested Transformer (NesT)

    A PyTorch impl of : `Aggregating Nested Transformers`
        - https://arxiv.org/abs/2105.12723
    éà   r2   rB   ©é€   é   i   ©rB   r'   é   ©rC   rC   é   éè  rk   Tr(   ç      à?Nrƒ   ÚavgÚimg_sizeÚin_chansÚ
patch_sizeÚ
num_levelsÚ
embed_dims.r*   ÚdepthsÚnum_classesrl   r+   Ú	drop_rateÚproj_drop_rateÚattn_drop_rateÚdrop_path_ratero   rn   r†   Úweight_initÚglobal_poolc                 ót  •— t          ¦   «                              ¦   «          ||dœ}dD ]V}t          ¦   «         |         }t          |t          j        j        ¦  «        r!t          |¦  «        |k    sJ d|› d�¦   «         ‚ŒW t          |¦  «        |¦  «        } t          |¦  «        |¦  «        } t          |¦  «        |¦  «        }|| _	        || _
        |d         x| _        | _        g | _        |pt          }|pt          j        }|| _        || _        t          |t          j        j        ¦  «        r$|d         |d         k    s
J d¦   «         ‚|d         }||z  dk    s
J d	¦   «         ‚|| _        d
t)          j        |dt(          j        ¬¦  «        z                       d¦  «                             ¦   «         | _        ||z  t5          j        | j        d         ¦  «        z  dk    s
J d¦   «         ‚t9          ||z  t5          j        | j        d         ¦  «        z  ¦  «        | _        t=          d||||d         ddœ|¤Ž| _        | j        j         | _         | j         | j        d         z  | _!        g }tE          ||d¬¦  «        }d}d
}tG          t          | j        ¦  «        ¦  «        D ]‹}||         }| $                    tK          | j        |         | j        | j!        ||         ||         ||f|	|
||||         |||dœ|¤Ž¦  «         | xj        tM          ||d|› �¬¦  «        gz  c_        |}|dz  }ŒŒt          j'        |Ž | _(         ||d         fi |¤Ž| _)        tU          | j        | j	        fd|i|¤Ž\  }}|| _+        t          j,        |¦  «        | _-        || _.        |  /                    |¦  «         dS )a‰  
        Args:
            img_size (int, tuple): input image size
            in_chans (int): number of input channels
            patch_size (int): patch size
            num_levels (int): number of block hierarchies (T_d in the paper)
            embed_dims (int, tuple): embedding dimensions of each level
            num_heads (int, tuple): number of attention heads for each level
            depths (int, tuple): number of transformer layers for each level
            num_classes (int): number of classes for classification head
            mlp_ratio (int): ratio of mlp hidden dim to embedding dim for MLP of transformer layers
            qkv_bias (bool): enable bias for qkv if True
            drop_rate (float): dropout rate for MLP of transformer layers, MSA final projection layer, and classifier
            attn_drop_rate (float): attention dropout rate
            drop_path_rate (float): stochastic depth rate
            norm_layer: (nn.Module): normalization layer for transformer layers
            act_layer: (nn.Module): activation layer in MLP of transformer layers
            pad_type: str: Type of padding to use '' for PyTorch symmetric, 'same' for TF SAME
            weight_init: (str): weight init scheme
            global_pool: (str): type of pooling operation to apply to final feature map

        Notes:
            - Default values follow NesT-B from the original Jax code.
            - `embed_dims`, `num_heads`, `depths` should be ints or tuples with length `num_levels`.
            - For those following the paper, Table A1 may have errors!
                - https://github.com/google-research/nested-transformer/issues/2
        r/   ©rÇ   r*   rÈ   zRequire `len(z) == num_levels`rG   r   r   z Model only handles square inputsz*`patch_size` must divide `img_size` evenlyrB   ÚcpuzUFirst level blocks don't fit evenly. Check `img_size`, `patch_size`, and `num_levels`F)rÃ   rÅ   rÄ   r¦   ÚflattenT)Ú	stagewiseN)rl   r+   r-   r,   rm   ro   rn   r†   zlevels.)Únum_chsÚ	reductionÚmodulerC   Ú	pool_typert   )0r4   r5   ÚlocalsÚ
isinstanceÚcollectionsÚabcÚSequencer±   r   rÉ   rÄ   Únum_featuresÚhead_hidden_sizeÚfeature_infor   r	   r   rÊ   rÆ   rÅ   r`   ÚarangeÚlongÚflipÚtolistr£   rš   r›   re   r’   r   Úpatch_embedÚnum_patchesr¤   r   r³   Úappendr¢   Údictr²   Úlevelsr�   r   rÏ   r9   Ú	head_dropÚheadÚinit_weights)!r;   rÃ   rÄ   rÅ   rÆ   rÇ   r*   rÈ   rÉ   rl   r+   rÊ   rË   rÌ   rÍ   ro   rn   r†   rÎ   rÏ   r0   r1   r<   Ú
param_nameÚparam_valueré   Údp_ratesÚprev_dimÚcurr_strider«   r)   rë   r>   s!                                   €r?   r5   zNest.__init__(  s,  ø€ õf 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆØ?ð 	dð 	dˆJÝ ™(œ( :Ô.ˆKÝ˜+¥{¤Ô'?Ñ@Ô@ð dÝ˜;Ñ'Ô'¨:Ò5Ð5Ð5Ð7cÀzÐ7cÐ7cÐ7cÑ5Ô5Ð5øà*•Y˜zÑ*Ô*¨:Ñ6Ô6ˆ
Ø)•I˜jÑ)Ô)¨)Ñ4Ô4ˆ	Ø&•˜:Ñ&Ô& vÑ.Ô.ˆØ&ˆÔØ ˆŒØ4>¸r´NÐBˆÔ˜DÔ1ØˆÔØÐ,¥9ˆ
ØÐ(¥¤ˆ	Ø"ˆŒØ$ˆŒÝ�h¥¤Ô 8Ñ9Ô9ð 	#Ø˜A”; (¨1¤+Ò-Ð-Ð-Ð/QÑ-Ô-Ð-Ø ”{ˆHØ˜*Ñ$¨Ò)Ð)Ð)Ð+WÑ)Ô)Ð)Ø$ˆŒð ¥¤¨ZÀÍUÌZÐ XÑ XÔ XÑX×^Ò^Ð_`ÑaÔa×hÒhÑjÔjˆŒØ˜JÑ&­$¬)°D´OÀAÔ4FÑ*GÔ*GÑGÈ1ÒLÐLÐLØdñ MÔLÐLõ ˜x¨:Ñ5½$¼)ÀDÄOÐTUÔDVÑ:WÔ:WÑWÑXÔXˆŒõ &ð 
ØØ!ØØ  ”mØð
ð 
ð ð
ð 
ˆÔð  Ô+Ô7ˆÔØÔ*¨d¬o¸aÔ.@Ñ@ˆŒð ˆÝ,¨^¸VÈtÐTÑTÔTˆØˆØˆÝ•s˜4œ?Ñ+Ô+Ñ,Ô,ð 	ð 	ˆAØ˜Q”-ˆCØ�MŠM�)Ø” Ô"Ø”Ø”Ø˜!”Ø�q”	ØØðð $Ø!Ø(Ø(Ø" 1œ+Ø%Ø#Ø!ðð ð  ð!ð ñ ô ð ð$ ÐÔ¥$¨s¸kÐR_Ð\]ÐR_ÐR_Ð"`Ñ"`Ô"`Ð!aÑaÐÔØˆHØ˜1ÑˆKˆKÝ”m VÐ,ˆŒð �J˜z¨"œ~Ð4Ð4°Ð4Ð4ˆŒ	õ .¨dÔ.?ÀÔAQÐoÐoÐ]hÐoÐlnÐoÐoÑˆ�TØ&ˆÔÝœ IÑ.Ô.ˆŒØˆŒ	à×Ò˜+Ñ&Ô&Ð&Ð&Ð&r@   c                 óÞ   — |dv sJ ‚d|v rt          j        | j        ¦  «         nd}| j        D ]}t	          |j        ddd¬¦  «         Œt          t          t          |¬¦  «        | ¦  «         d S )	N)Únlhbrƒ   ró   r(   ç{®Gáz”?rF   rC   ©ÚstdÚaÚb)Ú	head_bias)	rš   ÚlogrÉ   ré   r   r°   r   r   Ú_init_nest_weights)r;   Úmoderù   Úlevels       r?   rì   zNest.init_weights²  s†   € à�|Ð#Ð#Ð#Ð#Ø39¸T°>°>•T”X˜dÔ.Ñ/Ô/Ð/Ð/Àrˆ	Ø”[ð 	?ð 	?ˆEÝ˜%œ/¨s°b¸AÐ>Ñ>Ô>Ð>Ð>Ý•GÕ.¸)ÐDÑDÔDÀdÑKÔKÐKÐKÐKr@   c                 óX   — d„ t          t          | j        ¦  «        ¦  «        D ¦   «         S )Nc                 ó   — h | ]}d |› d�’Œ	S )zlevel.z
.pos_embedrt   )rª   r«   s     r?   ú	<setcomp>z'Nest.no_weight_decay.<locals>.<setcomp>¼  s$   € ÐHÐHÐH¨1Ð&˜Ð&Ð&Ð&ÐHÐHÐHr@   )r³   r±   ré   ©r;   s    r?   Úno_weight_decayzNest.no_weight_decayº  s(   € àHÐHµµc¸$¼+Ñ6FÔ6FÑ0GÔ0GÐHÑHÔHÐHr@   Fc                 ó:   — t          d|rdndd fddg¬¦  «        }|S )Nz^patch_embedz^levels\.(\d+)z*^levels\.(\d+)\.transformer_encoder\.(\d+))z"^levels\.(\d+)\.(?:pool|pos_embed))r   )z^norm)iŸ† )ÚstemÚblocks)rè   )r;   ÚcoarseÚmatchers      r?   Úgroup_matcherzNest.group_matcher¾  s@   € åØ à&,Ð_Ð"Ð"Ð2_ÐaeÐfØ=Ø$ðð
ñ 
ô 
ˆð ˆr@   c                 ó(   — | j         D ]	}||_        Œ
d S r}   )ré   r­   )r;   ÚenableÚls      r?   Úset_grad_checkpointingzNest.set_grad_checkpointingÊ  s(   € à”ð 	*ð 	*ˆAØ#)ˆAÔ Ð ð	*ð 	*r@   Úreturnc                 ó   — | j         S r}   )rë   r  s    r?   Úget_classifierzNest.get_classifierÏ  s
   € àŒyÐr@   c                 óü   — t          | ¦  «        }|| _        t          | j        | j        fd|i|¤Ž\  | _        | _        | j                             | j        ¦  «         | j                             | j        ¦  «         d S )NrØ   )r   rÉ   r   rÞ   rÏ   rë   ÚtrainrN   )r;   rÉ   rÏ   r<   s       r?   Úreset_classifierzNest.reset_classifierÓ  sŠ   € Ý˜dÑ#Ô#ˆØ&ˆÔÝ&7ØÔ˜tÔ/ð'Nð 'NØ;Fð'NØJLð'Nð 'NÑ#ˆÔ˜$œ)àÔ×Ò˜tœ}Ñ-Ô-Ð-ØŒ	�Š˜œÑ&Ô&Ð&Ð&Ð&r@   ÚNCHWrR   Úindicesr�   Ú
stop_earlyÚ
output_fmtÚintermediates_onlyc           	      ó  — |dv s
J d¦   «         ‚g }t          t          | j        ¦  «        |¦  «        \  }}	|                      |¦  «        }t          | j        ¦  «        dz
  }
t
          j                             ¦   «         s|s| j        }n| j        d|	dz   …         }t          |¦  «        D ]ˆ\  }} ||¦  «        }||v rt|r]||
k    rW|  	                    | 
                    dddd¦  «        ¦  «         
                    dddd¦  «        }|                     |¦  «         Œs|                     |¦  «         Œ‰|r|S ||
k    rA|  	                    | 
                    dddd¦  «        ¦  «         
                    dddd¦  «        }||fS )aï   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )r  zOutput shape must be NCHW.r   Nr   rC   r2   )r   r±   ré   rå   r£   r`   ra   r¶   Ú	enumerater�   rJ   rç   )r;   rR   r  r�   r  r  r  ÚintermediatesÚtake_indicesÚ	max_indexÚlast_idxÚstagesÚfeat_idxÚstageÚx_inters                  r?   Úforward_intermediateszNest.forward_intermediatesÛ  s¨  € ð* ˜YÐ&Ð&Ð&Ð(DÑ&Ô&Ð&ØˆÝ"6µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�ið ×Ò˜QÑÔˆÝ�t”Ñ'Ô'¨!Ñ+ˆÝŒ9×!Ò!Ñ#Ô#ð 	1¨:ð 	1Ø”[ˆFˆFà”[  )¨a¡- Ô0ˆFå(¨Ñ0Ô0ð 	,ð 	,‰OˆH�eØ��a‘”ˆAØ˜<Ð'Ð'Øð ,˜H¨Ò0Ð0Ø"Ÿiši¨¯	ª	°!°Q¸¸1Ñ(=Ô(=Ñ>Ô>×FÒFÀqÈ!ÈQÐPQÑRÔR�GØ!×(Ò(¨Ñ1Ô1Ð1Ð1à!×(Ò(¨Ñ+Ô+Ð+øàð 	!Ø Ð à�xÒÐà—	’	˜!Ÿ)š) A q¨!¨QÑ/Ô/Ñ0Ô0×8Ò8¸¸A¸qÀ!ÑDÔDˆAà�-ÐÐr@   r   Ú
prune_normÚ
prune_headc                 óâ   — t          t          | j        ¦  «        |¦  «        \  }}| j        d|dz   …         | _        |rt          j        ¦   «         | _        |r|                      dd¦  «         |S )z@ Prune layers not required for specified intermediates.
        Nr   r   rƒ   )r   r±   ré   r	   rv   r�   r  )r;   r  r#  r$  r  r  s         r?   Úprune_intermediate_layerszNest.prune_intermediate_layers  sq   € õ #7µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�iØ”k . 9¨q¡= .Ô1ˆŒØð 	&Ýœ™œˆDŒIØð 	)Ø×!Ò! ! RÑ(Ô(Ð(ØÐr@   c                 óÜ   — |                       |¦  «        }|                      |¦  «        }|                      |                     dddd¦  «        ¦  «                             dddd¦  «        }|S )Nr   rC   r2   r   )rå   ré   r�   rJ   r�   s     r?   Úforward_featureszNest.forward_features  s`   € Ø×Ò˜QÑÔˆØ�KŠK˜‰NŒNˆà�IŠI�a—i’i  1 a¨Ñ+Ô+Ñ,Ô,×4Ò4°Q¸¸1¸aÑ@Ô@ˆØˆr@   Ú
pre_logitsc                 óˆ   — |                       |¦  «        }|                      |¦  «        }|r|n|                      |¦  «        S r}   )rÏ   rê   rë   )r;   rR   r)  s      r?   Úforward_headzNest.forward_head%  s?   € Ø×Ò˜QÑÔˆØ�NŠN˜1ÑÔˆØÐ0ˆqˆq D§I¢I¨a¡L¤LÐ0r@   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S r}   )r(  r+  r�   s     r?   r[   zNest.forward*  s-   € Ø×!Ò! !Ñ$Ô$ˆØ×Ò˜aÑ Ô ˆØˆr@   )r¸   r2   rB   r2   r¹   r¼   r¾   rÀ   rk   Tr(   r(   r(   rÁ   NNrƒ   rƒ   rÂ   NN©rƒ   ©F)T)rÂ   )NFFr  F)r   FT) r\   r]   r^   r_   re   r   rf   rc   r   r   r	   r€   r‘   r5   r`   ra   Úignorerì   r  r  r  r  r  ÚTensorr   r   r"  r&  r(  r+  r[   rg   rh   s   @r?   r#   r#   !  s´  ø€ € € € € ðð ð  ØØØØ*9Ø)3Ø&0Ø#Ø!Ø!Ø!Ø$&Ø$&Ø$'Ø48Ø37ØØ!Ø$ØØð-H'ð H'àðH'ð ðH'ð ð	H'ð
 ðH'ð ˜c 3˜hœðH'ð ˜S #˜X”ðH'ð ˜#˜s˜(”OðH'ð ðH'ð ðH'ð ðH'ð ðH'ð "ðH'ð "ðH'ð "ðH'ð  !  b¤i¤Ô1ð!H'ð"    R¤Y¤Ô0ð#H'ð$ ð%H'ð& ð'H'ð( ð)H'ð H'ð H'ð H'ð H'ð H'ðT „YÔðLð Lð Lñ ÔðLð „YÔðIð Iñ ÔðIð „YÔð	ð 	ð 	ñ Ôð	ð „YÔð*ð *ð *ñ Ôð*ð „YÔð ¤	ð ð ð ñ Ôðð'ð '¨Cð '¸cð 'ð 'ð 'ð 'ð 8<ØØ$Ø$Ø',ð1 ð 1 àŒ|ð1 ð ˜e C¨¨c¬ NÔ3Ô4ð1 ð ð	1 ð
 ð1 ð ð1 ð !%ð1 ð 
ˆt�E”LÔ! 5¨¬°t¸E¼LÔ7IÐ)IÔ#JÐJÔ	Kð1 ð 1 ð 1 ð 1 ðj ./Ø$Ø#ð	ð à˜3  S¤	˜>Ô*ðð ðð ð	ð ð ð ð ð ð ð1ð 1¨$ð 1ð 1ð 1ð 1ð
ð ð ð ð ð ð r@   rƒ   r(   r×   Únamerù   c                 ó2  — t          | t          j        ¦  «        r›|                     d¦  «        r?t	          | j        ddd¬¦  «         t          j                             | j        |¦  «         dS t	          | j        ddd¬¦  «         | j        �&t          j         	                    | j        ¦  «         dS dS t          | t          j
        ¦  «        rEt	          | j        ddd¬¦  «         | j        �(t          j         	                    | j        ¦  «         dS dS dS )zn NesT weight initialization
    Can replicate Jax implementation. Otherwise follows vision_transformer.py
    rë   rô   rF   rC   rõ   N)rÚ   r	   r7   Ú
startswithr   ÚweightÚinitÚ	constant_r3   Úzeros_ÚConv2d)r×   r1  rù   s      r?   rû   rû   0  s  € õ �&�"œ)Ñ$Ô$ð (Ø�?Š?˜6Ñ"Ô"ð 	,Ý˜&œ-¨S°B¸!Ð<Ñ<Ô<Ð<ÝŒG×Ò˜fœk¨9Ñ5Ô5Ð5Ð5Ð5å˜&œ-¨S°B¸!Ð<Ñ<Ô<Ð<ØŒ{Ð&Ý”—’˜vœ{Ñ+Ô+Ð+Ð+Ð+ð 'Ð&å	�F�BœIÑ	&Ô	&ð (Ý�f”m¨°°aÐ8Ñ8Ô8Ð8ØŒ;Ð"ÝŒG�NŠN˜6œ;Ñ'Ô'Ð'Ð'Ð'ð(ð (à"Ð"r@   c                 ó   — t                                d| j        |j        ¦  «         | j        d         }|j        dd…         \  }}t          t	          j        ||z  ¦  «        ¦  «        }t          | t          t	          j        |¦  «        ¦  «        ¦  «                             dddd¦  «        } t          j	        | ||gdd¬¦  «        } t          |                      dddd¦  «        t          t	          j        |¦  «        ¦  «        ¦  «        } | S )	zª
    Rescale the grid of position embeddings when loading from state_dict
    Expected shape of position embeddings is (1, T, N, C), and considers only square images
    z$Resized position embedding: %s to %srC   r   r2   r   ÚbicubicF)Úsizerü   Úalign_corners)Ú_loggerÚinforH   re   rš   r›   r    rJ   rL   Úinterpolater˜   )ÚposembÚ
posemb_newÚseq_length_oldÚnum_blocks_newÚseq_length_newÚsize_news         r?   Úresize_pos_embedrF  B  sé   € õ
 ‡L‚LÐ7¸¼ÀzÔGWÑXÔXÐXØ”\ !”_€NØ%/Ô%5°a¸°cÔ%:Ñ"€N�NÝ•4”9˜^¨NÑ:Ñ;Ô;Ñ<Ô<€Hå˜¥¥D¤I¨nÑ$=Ô$=Ñ >Ô >Ñ?Ô?×GÒGÈÈ1ÈaÐQRÑSÔS€FÝŒ]˜6¨°8Ð(<À9Ð\aÐbÑbÔb€Få�f—n’n Q¨¨1¨aÑ0Ô0µ#µd´iÀÑ6OÔ6OÑ2PÔ2PÑQÔQ€FØ€Mr@   c                 óâ   — d„ |                       ¦   «         D ¦   «         }|D ]M}| |         j        t          ||¦  «        j        k    r't          | |         t          ||¦  «        ¦  «        | |<   ŒN| S )z4 resize positional embeddings of pretrained weights c                 ó<   — g | ]}|                      d ¦  «        ¯|‘ŒS )Ú
pos_embed_)r3  )rª   rX   s     r?   r¬   z(checkpoint_filter_fn.<locals>.<listcomp>U  s)   € ÐQÐQÐQ˜A°a·l²lÀ<Ñ6PÔ6PÐQ�aÐQÐQÐQr@   )ÚkeysrH   ÚgetattrrF  )Ú
state_dictÚmodelÚpos_embed_keysrX   s       r?   Úcheckpoint_filter_fnrO  S  sw   € àQÐQ §¢Ñ!2Ô!2ÐQÑQÔQ€NØð Oð OˆØ�aŒ=Ô¥'¨%°Ñ"3Ô"3Ô"9Ò9Ð9Ý,¨Z¸¬]½GÀEÈ1Ñ<MÔ<MÑNÔNˆJ�q‰MøØÐr@   Fc                 ó\   — t          t          | |ft          dd¬¦  «        t          dœ|¤Ž}|S )N)r   r   rC   T)Úout_indicesÚflatten_sequential)Úfeature_cfgÚpretrained_filter_fn)r   r#   rè   rO  )ÚvariantÚ
pretrainedÚkwargsrM  s       r?   Ú_create_nestrX  \  sJ   € Ý ÝØØðõ  YÀ4ÐHÑHÔHÝ1ðð ð ðð €Eð €Lr@   c                 ó<   — | ddddgdddt           t          ddd	d
œ|¥S )NrÀ   )r2   r¸   r¸   é   g      ì?r:  Tzpatch_embed.projrë   z
apache-2.0)ÚurlrÉ   Ú
input_sizeÚ	pool_sizeÚcrop_pctÚinterpolationÚfixed_input_sizeÚmeanrö   Ú
first_convÚ
classifierÚlicenser
   )r[  rW  s     r?   Ú_cfgre  i  s>   € àØ¨=ÈÈBÀxØ¨9È$Ý%Õ.BØ(¸Øðð ð ðð r@   ztimm/)Ú	hf_hub_id)znest_base.untrainedznest_small.untrainedznest_tiny.untrainedznest_base_jx.goog_in1kznest_small_jx.goog_in1kznest_tiny_jx.goog_in1kr  c                 óB   — t          dddddœ|¤Ž}t          dd| i|¤Ž}|S )	ú Nest-B @ 224x224
    r¹   r¼   r¾   rÑ   Ú	nest_baserV  rt   )ri  ©rè   rX  ©rV  rW  Úmodel_kwargsrM  s       r?   ri  ri  €  sN   € õ ð WØ"¨jÀðWð WØOUðWð W€LåÐLÐL°ÐL¸|ÐLÐL€EØ€Lr@   c                 óB   — t          dddddœ|¤Ž}t          dd| i|¤Ž}|S )	ú Nest-S @ 224x224
    ©é`   éÀ   i€  ©r2   é   é   r¾   rÑ   Ú
nest_smallrV  rt   )ru  rj  rk  s       r?   ru  ru  Š  s?   € õ Ðe >¸ZÐPZÐeÐeÐ^dÐeÐe€LÝÐMÐM°*ÐMÀÐMÐM€EØ€Lr@   c                 óB   — t          dddddœ|¤Ž}t          dd| i|¤Ž}|S )	ú Nest-T @ 224x224
    ro  rr  ©rC   rC   r'   rÑ   Ú	nest_tinyrV  rt   )ry  rj  rk  s       r?   ry  ry  “  s?   € õ Ðd >¸ZÐPYÐdÐdÐ]cÐdÐd€LÝÐLÐL°ÐL¸|ÐLÐL€EØ€Lr@   c                 ón   — |                      dd¦  «         t          d	ddddœ|¤Ž}t          d
d| i|¤Ž}|S )rh  r†   Úsamer¹   r¼   r¾   rÑ   Únest_base_jxrV  rt   )r|  ©Ú
setdefaultrè   rX  rk  s       r?   r|  r|  œ  sd   € ð ×Ò�j &Ñ)Ô)Ð)Ýð WØ"¨jÀðWð WØOUðWð W€LåÐOÐO°JÐOÀ,ÐOÐO€EØ€Lr@   c                 ón   — |                      dd¦  «         t          d	ddddœ|¤Ž}t          d
d| i|¤Ž}|S )rn  r†   r{  ro  rr  r¾   rÑ   Únest_small_jxrV  rt   )r€  r}  rk  s       r?   r€  r€  §  sU   € ð ×Ò�j &Ñ)Ô)Ð)ÝÐe >¸ZÐPZÐeÐeÐ^dÐeÐe€LÝÐPÐP°ZÐPÀ<ÐPÐP€EØ€Lr@   c                 ón   — |                      dd¦  «         t          d	ddddœ|¤Ž}t          d
d| i|¤Ž}|S )rw  r†   r{  ro  rr  rx  rÑ   Únest_tiny_jxrV  rt   )r‚  r}  rk  s       r?   r‚  r‚  ±  sU   € ð ×Ò�j &Ñ)Ô)Ð)ÝÐd >¸ZÐPYÐdÐdÐ]cÐdÐd€LÝÐOÐO°JÐOÀ,ÐOÐO€EØ€Lr@   r|  r€  r‚  )Újx_nest_baseÚjx_nest_smallÚjx_nest_tiny)rƒ   r(   r.  r-  )Kr_   Úcollections.abcrÛ   Úloggingrš   Ú	functoolsr   Útypingr   r   r   r   r   r`   Útorch.nn.functionalr	   Ú
functionalrL   Ú	timm.datar   r   Útimm.layersr   r   r   r   r   r   r   r   r   r   r   r   r   Ú_builderr   Ú	_featuresr   Ú_features_fxr   Ú_manipulater   r   Ú	_registryr    r!   r"   Ú__all__Ú	getLoggerr\   r=  r€   r%   rj   r‚   re   r˜   r    r¢   r#   r‘   rf   rû   rF  rO  rX  re  Údefault_cfgsri  ru  ry  r|  r€  r‚  rt   r@   r?   ú<module>r–     s&  ððð ð" Ð Ð Ð Ø €€€Ø €€€Ø Ð Ð Ð Ð Ð Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5à €€€Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð +Ð *Ð *Ð *Ð *Ð *Ø +Ð +Ð +Ð +Ð +Ð +Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø 4Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4Ø YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ Yàˆ(€à
ˆ'Ô
˜HÑ
%Ô
%€ð3ð 3ð 3ð 3ð 3�”	ñ 3ô 3ð 3ðl/ð /ð /ð /ð /�r”yñ /ô /ð /ðdð ð ð ð ˆrŒyñ ô ð ð:˜Cð ð ð ð ð  ð˜cð ð ð ñ ÔððC%ð C%ð C%ð C%ð C%�”	ñ C%ô C%ð C%ðLLð Lð Lð Lð Lˆ2Œ9ñ Lô Lð Lð^(ð (˜rœyð (°ð (ÀUð (ð (ð (ð (ð$ð ð ð"ð ð ð
ð 
ð 
ð 
ð	ð 	ð 	ð 	ð %Ð$Ø˜4™6œ6Ø ˜D™FœFØ˜4™6œ6à"˜d¨WÐ5Ñ5Ô5Ø#˜t¨gÐ6Ñ6Ô6Ø"˜d¨WÐ5Ñ5Ô5ð&ð &ñ ô €ð ðð ¨Tð ð ð ñ „ðð ðð ¨dð ð ð ñ „ðð ðð ¨Tð ð ð ñ „ðð ðð °ð ð ð ñ „ðð ðð °ð ð ð ñ „ðð ðð °ð ð ð ñ „ðð Ð ˜HØ"Ø$Ø"ð'ð 'ñ ô ð ð ð r@   