§
    ·¯¸j¸­  ã                   ó¦
  — d Z ddlZddlmZ ddlmZmZmZmZm	Z	m
Z
 ddlZddlmZ ddlmZmZ ddlmZmZmZmZmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddl m!Z!m"Z"m#Z# ddl$m%Z% dgZ&e G d„ dej'        ¦  «        ¦   «         Z(d¨d„Z) G d„ dej'        ¦  «        Z* G d„ dej'        ¦  «        Z+ G d„ dej'        ¦  «        Z, G d„ dej'        ¦  «        Z- G d„ dej'        ¦  «        Z. G d„ dej'        ¦  «        Z/d„ Z0d©d„Z1dªd!„Z2 e"i d" e2d#d$¬%¦  «        “d& e2d#d'¬%¦  «        “d( e2d#d)d*¬+¦  «        “d, e2d#d-¬%¦  «        “d. e2d#d/¬%¦  «        “d0 e2d#d1d*¬+¦  «        “d2 e2d#d3¬%¦  «        “d4 e2d#d5¬%¦  «        “d6 e2d#d7d*¬+¦  «        “d8 e2d#d9¬%¦  «        “d: e2d#d;¬%¦  «        “d< e2d#d=d*¬+¦  «        “d> e2d#d?¬%¦  «        “d@ e2d#dA¬%¦  «        “dB e2d#dCd*¬+¦  «        “dD e2d#dE¬%¦  «        “dF e2d#dG¬%¦  «        “i dH e2d#dId*¬+¦  «        “dJ e2d#dK¬%¦  «        “dL e2d#dM¬%¦  «        “dN e2d#dOd*¬+¦  «        “dP e2d#dQ¬%¦  «        “dR e2d#dS¬%¦  «        “dT e2d#dUd*¬+¦  «        “dV e2d#dW¬%¦  «        “dX e2d#dY¬%¦  «        “dZ e2d#d[d*¬+¦  «        “d\ e2d#d]¬%¦  «        “d^ e2d#d_¬%¦  «        “d` e2d#dad*¬+¦  «        “db e2d#dc¬%¦  «        “dd e2d#de¬%¦  «        “df e2d#dgd*¬+¦  «        “dh e2d#di¬%¦  «        “¥ e2d#dj¬%¦  «         e2d#dkd*¬+¦  «         e2d#dl¬%¦  «         e2d#dm¬%¦  «         e2d#dnd*¬+¦  «         e2d#do¬%¦  «         e2d#dp¬%¦  «         e2d#dqd*¬+¦  «        drœ¥¦  «        Z3e!d«dse/fdt„¦   «         Z4e!d«dse/fdu„¦   «         Z5e!d«dse/fdv„¦   «         Z6e!d«dse/fdw„¦   «         Z7e!d«dse/fdx„¦   «         Z8e!d«dse/fdy„¦   «         Z9e!d«dse/fdz„¦   «         Z:e!d«dse/fd{„¦   «         Z;e!d«dse/fd|„¦   «         Z<e!d«dse/fd}„¦   «         Z=e!d«dse/fd~„¦   «         Z>e!d«dse/fd„¦   «         Z?e!d«dse/fd€„¦   «         Z@e!d«dse/fd�„¦   «         ZAe!d«dse/fd‚„¦   «         ZBe!d«dse/fdƒ„¦   «         ZCe!d«dse/fd„„¦   «         ZDe!d«dse/fd…„¦   «         ZEe!d«dse/fd†„¦   «         ZFe!d«dse/fd‡„¦   «         ZGe!d«dse/fdˆ„¦   «         ZHe!d«dse/fd‰„¦   «         ZIe!d«dse/fdŠ„¦   «         ZJe!d«dse/fd‹„¦   «         ZKe!d«dse/fdŒ„¦   «         ZLe!d«dse/fd�„¦   «         ZMe!d«dse/fdŽ„¦   «         ZNe!d«dse/fd�„¦   «         ZO e#ePi d�d&“d‘d(“d’d.“d“d0“d”d4“d•d6“d–d:“d—d<“d˜d@“d™dB“dšdF“d›dH“dœdL“d�dN“dždR“dŸdT“d dX“dZd^d`dddfd¡d¢d£d¤d¥d¦d§œ¥¦  «         dS )¬a]   Cross-Covariance Image Transformer (XCiT) in PyTorch

Paper:
    - https://arxiv.org/abs/2106.09681

Same as the official implementation, with some minor adaptations, original copyright below
    - https://github.com/facebookresearch/xcit/blob/master/xcit.py

Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman
é    N)Úpartial)ÚListÚOptionalÚTupleÚUnionÚTypeÚAny©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)ÚDropPathÚtrunc_normal_Ú	to_2tupleÚuse_fused_attnÚMlpÚget_device_dtypeé   )Úbuild_model_with_cfg)Úfeature_take_indices)Úregister_notrace_module)Ú
checkpoint)Úregister_modelÚgenerate_default_cfgsÚregister_model_deprecations)Ú	ClassAttnÚXcitc                   óP   ‡ — e Zd ZdZ	 	 	 	 	 ddededefˆ fd	„Zd
ededefd„Zˆ xZS )ÚPositionalEncodingFourierzä
    Positional encoding relying on a fourier kernel matching the one used in the "Attention is all you Need" paper.
    Based on the official XCiT code
        - https://github.com/facebookresearch/xcit/blob/master/xcit.py
    é    é   é'  NÚ
hidden_dimÚdimÚtemperaturec                 óì   •— ||dœ}t          ¦   «                              ¦   «          t          j        |dz  |fddi|¤Ž| _        dt
          j        z  | _        || _        || _	        || _
        d| _        d S )N©ÚdeviceÚdtypeé   Úkernel_sizer   ç�íµ ÷Æ°>)ÚsuperÚ__init__ÚnnÚConv2dÚtoken_projectionÚmathÚpiÚscaler$   r"   r#   Úeps)Úselfr"   r#   r$   r'   r(   ÚddÚ	__class__s          €ú`/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/models/xcit.pyr-   z"PositionalEncodingFourier.__init__)   s|   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ "¤	¨*°q©.¸#Ð SÐ SÈ1Ð SÐPRÐ SÐ SˆÔØ�œ‘[ˆŒ
Ø&ˆÔØ$ˆŒØˆŒØˆŒˆˆó    ÚBÚHÚWc           
      óô  — | j         j        j        }| j         j        j        }t	          j        d|dz   |¬¦  «                             t          j        ¦  «                             d¦  «         	                    dd|¦  «        }t	          j        d|dz   |¬¦  «                             t          j        ¦  «         	                    d|d¦  «        }||d d …dd …d d …f         | j
        z   z  | j        z  }||d d …d d …dd …f         | j
        z   z  | j        z  }t	          j        | j        |¬¦  «                             t          j        ¦  «        }| j        dt	          j        |dd¬¦  «        z  | j        z  z  }|d d …d d …d d …d f         |z  }	|d d …d d …d d …d f         |z  }
t	          j        |	d d …d d …d d …dd d…f                              ¦   «         |	d d …d d …d d …dd d…f                              ¦   «         gd¬	¦  «                             d
¦  «        }	t	          j        |
d d …d d …d d …dd d…f                              ¦   «         |
d d …d d …d d …dd d…f                              ¦   «         gd¬	¦  «                             d
¦  «        }
t	          j        |
|	fd
¬	¦  «                             dd
dd¦  «        }|                       |                     |¦  «        ¦  «        }| 	                    |ddd¦  «        S )Nr   )r'   éÿÿÿÿr)   Úfloor)Úrounding_moder   é   ©r#   é   )r0   Úweightr'   r(   ÚtorchÚarangeÚtoÚfloat32Ú	unsqueezeÚrepeatr4   r3   r"   r$   ÚdivÚstackÚsinÚcosÚflattenÚcatÚpermute)r5   r:   r;   r<   r'   r(   Úy_embedÚx_embedÚdim_tÚpos_xÚpos_yÚposs               r8   Úforwardz!PositionalEncodingFourier.forward:   s   € ØÔ&Ô-Ô4ˆØÔ%Ô,Ô2ˆÝ”,˜q ! a¡%°Ð7Ñ7Ô7×:Ò:½5¼=ÑIÔI×SÒSÐTUÑVÔV×]Ò]Ð^_ÐabÐdeÑfÔfˆÝ”,˜q ! a¡%°Ð7Ñ7Ô7×:Ò:½5¼=ÑIÔI×PÒPÐQRÐTUÐWXÑYÔYˆØ˜W Q Q Q¨¨¨¨Q¨Q¨Q YÔ/°$´(Ñ:Ñ;¸d¼jÑHˆØ˜W Q Q Q¨¨¨¨2¨3¨3 YÔ/°$´(Ñ:Ñ;¸d¼jÑHˆÝ”˜Tœ_°VÐ<Ñ<Ô<×?Ò?ÅÄÑNÔNˆØÔ  Q­¬°5¸!È7Ð)SÑ)SÔ)SÑ%SÐVZÔVeÑ%eÑfˆØ˜˜˜˜1˜1˜1˜a˜a˜a ˜Ô&¨Ñ.ˆØ˜˜˜˜1˜1˜1˜a˜a˜a ˜Ô&¨Ñ.ˆÝ”˜U 1 1 1 a a a¨¨¨¨A¨D¨q¨D =Ô1×5Ò5Ñ7Ô7¸¸q¸q¸qÀ!À!À!ÀQÀQÀQÈÈÈ1È¸}Ô9M×9QÒ9QÑ9SÔ9SÐTÐZ[Ð\Ñ\Ô\×dÒdÐefÑgÔgˆÝ”˜U 1 1 1 a a a¨¨¨¨A¨D¨q¨D =Ô1×5Ò5Ñ7Ô7¸¸q¸q¸qÀ!À!À!ÀQÀQÀQÈÈÈ1È¸}Ô9M×9QÒ9QÑ9SÔ9SÐTÐZ[Ð\Ñ\Ô\×dÒdÐefÑgÔgˆÝŒi˜ ˜¨AÐ.Ñ.Ô.×6Ò6°q¸!¸QÀÑBÔBˆØ×#Ò# C§F¢F¨5¡M¤MÑ2Ô2ˆØ�zŠz˜!˜Q  1Ñ%Ô%Ð%r9   )r   r    r!   NN)	Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚintÚfloatr-   rX   Ú__classcell__©r7   s   @r8   r   r   !   s¢   ø€ € € € € ðð ð !ØØ!&ØØðð àðð ðð ð	ð ð ð ð ð ð"&˜ð & ð &¨ð &ð &ð &ð &ð &ð &ð &ð &r9   r   c           
      ó˜   — ||dœ}t           j                             t          j        | |fd|dddœ|¤Žt          j        |fi |¤Ž¦  «        S )z3x3 convolution + batch normr&   rC   r   F)r*   ÚstrideÚpaddingÚbias)rE   r.   Ú
Sequentialr/   ÚBatchNorm2d)Ú	in_planesÚ
out_planesrb   r'   r(   r6   s         r8   Úconv3x3ri   L   sb   € à UÐ	+Ð	+€BÝŒ8×ÒÝ
Œ	�)˜ZÐc°Q¸vÈqÐW\ÐcÐcÐ`bÐcÐcÝ
Œ�zÐ(Ð( RÐ(Ð(ñô ð r9   c                   ón   ‡ — e Zd ZdZddddej        ddfdeded	ed
edeej                 f
ˆ fd„Z	d„ Z
ˆ xZS )ÚConvPatchEmbedz<Image to Patch Embedding using multiple convolutional layerséà   é   rC   r    NÚimg_sizeÚ
patch_sizeÚin_chansÚ	embed_dimÚ	act_layerc                 óÎ  •— ||dœ}t          ¦   «                              ¦   «          t          |¦  «        }|d         |z  |d         |z  z  }	|| _        || _        |	| _        |dk    rŠt          j                             t          ||dz  dfi |¤Ž |¦   «         t          |dz  |dz  dfi |¤Ž |¦   «         t          |dz  |dz  dfi |¤Ž |¦   «         t          |dz  |dfi |¤Ž¦  «        | _
        d S |dk    rmt          j                             t          ||dz  dfi |¤Ž |¦   «         t          |dz  |dz  dfi |¤Ž |¦   «         t          |dz  |dfi |¤Ž¦  «        | _
        d S d‚)	Nr&   r   r   rm   é   r)   rA   z=For convolutional projection, patch size has to be in [8, 16])r,   r-   r   rn   ro   Únum_patchesrE   r.   re   ri   Úproj)r5   rn   ro   rp   rq   rr   r'   r(   r6   ru   r7   s             €r8   r-   zConvPatchEmbed.__init__X   s¶  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ˜XÑ&Ô&ˆØ ”{ jÑ0°X¸a´[ÀJÑ5NÑOˆØ ˆŒØ$ˆŒØ&ˆÔà˜ÒÐÝœ×+Ò+Ý˜ )¨q¡.°!Ð:Ð:°rÐ:Ð:Ø�	‘”Ý˜	 Q™¨	°Q©¸Ð@Ð@¸RÐ@Ð@Ø�	‘”Ý˜	 Q™¨	°Q©¸Ð@Ð@¸RÐ@Ð@Ø�	‘”Ý˜	 Q™¨	°1Ð;Ð;¸Ð;Ð;ñô ˆDŒIˆIˆIð ˜1Š_ˆ_Ýœ×+Ò+Ý˜ )¨q¡.°!Ð:Ð:°rÐ:Ð:Ø�	‘”Ý˜	 Q™¨	°Q©¸Ð@Ð@¸RÐ@Ð@Ø�	‘”Ý˜	 Q™¨	°1Ð;Ð;¸Ð;Ð;ñô ˆDŒIˆIˆIð RÐRr9   c                 ó¾   — |                       |¦  «        }|j        d         |j        d         }}|                     d¦  «                             dd¦  «        }|||ffS )Nr)   rC   r   )rv   ÚshaperO   Ú	transpose)r5   ÚxÚHpÚWps       r8   rX   zConvPatchEmbed.forward   sS   € Ø�IŠI�a‰LŒLˆØ”˜”˜QœW QœZˆBˆØ�IŠI�a‰LŒL×"Ò" 1 aÑ(Ô(ˆØ�2�r�(ˆ{Ðr9   )rY   rZ   r[   r\   r.   ÚGELUr]   r   ÚModuler-   rX   r_   r`   s   @r8   rk   rk   U   s³   ø€ € € € € ØFÐFð  Ø ØØ Ø)+¬ØØð%Sð %Sàð%Sð ð%Sð ð	%Sð
 ð%Sð ˜BœI”ð%Sð %Sð %Sð %Sð %Sð %SðNð ð ð ð ð ð r9   rk   c            	       ó|   ‡ — e Zd ZdZdej        dddfdedee         deej	                 defˆ fd„Z
d	ed
efd„Zˆ xZS )ÚLPIa  
    Local Patch Interaction module that allows explicit communication between tokens in 3x3 windows to augment the
    implicit communication performed by the block diagonal scatter attention. Implemented using 2 layers of separable
    3x3 convolutions with GeLU and BatchNorm2d
    NrC   Úin_featuresÚout_featuresrr   r*   c                 ó4  •— t          ¦   «                              ¦   «          ||dœ}|p|}|dz  }t          j        j        ||f|||dœ|¤Ž| _         |¦   «         | _        t          j        |fi |¤Ž| _        t          j        j        ||f|||dœ|¤Ž| _	        d S )Nr&   r)   )r*   rc   Úgroups)
r,   r-   rE   r.   r/   Úconv1Úactrf   ÚbnÚconv2)
r5   r�   r‚   rr   r*   r'   r(   r6   rc   r7   s
            €r8   r-   zLPI.__init__�   sØ   ø€ õ 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆØ#Ð2 {ˆØ Ñ"ˆå”X”_Ø˜ðjØ2=ÀwÐWbðjð jØfhðjð jˆŒ
à�9‘;”;ˆŒÝ”. Ð3Ð3°Ð3Ð3ˆŒÝ”X”_Ø˜ðlØ3>ÈÐXdðlð lØhjðlð lˆŒ
ˆ
ˆ
r9   r;   r<   c                 óv  — |j         \  }}}|                     ddd¦  «                             ||||¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                     |||¦  «                             ddd¦  «        }|S )Nr   r)   r   )rx   rQ   Úreshaper…   r†   r‡   rˆ   )r5   rz   r;   r<   r:   ÚNÚCs          r8   rX   zLPI.forward¢   s    € Ø”'‰ˆˆ1ˆaØ�IŠI�a˜˜AÑÔ×&Ò& q¨!¨Q°Ñ2Ô2ˆØ�JŠJ�q‰MŒMˆØ�HŠH�Q‰KŒKˆØ�GŠG�A‰JŒJˆØ�JŠJ�q‰MŒMˆØ�IŠI�a˜˜AÑÔ×&Ò& q¨!¨QÑ/Ô/ˆØˆr9   )rY   rZ   r[   r\   r.   r}   r]   r   r   r~   r-   rX   r_   r`   s   @r8   r€   r€   †   s¼   ø€ € € € € ðð ð +/Ø)+¬Ø ØØðlð làðlð # 3œ-ðlð ˜BœI”ð	lð
 ðlð lð lð lð lð lð*˜Cð  Cð ð ð ð ð ð ð ð r9   r€   c                   óº   ‡ — e Zd ZdZdddddej        ej        ddddfdeded	ed
e	dededede
ej                 de
ej                 dee         de	fˆ fd„Zd„ Zˆ xZS )ÚClassAttentionBlockzAClass Attention Layer as in CaiT https://arxiv.org/abs/2103.17239ç      @Fç        ç      ð?Nr#   Ú	num_headsÚ	mlp_ratioÚqkv_biasÚ	proj_dropÚ	attn_dropÚ	drop_pathrr   Ú
norm_layerÚetaÚtokens_normc                 óš  •— ||dœ}t          ¦   «                              ¦   «           |	|fi |¤Ž| _        t          |f||||dœ|¤Ž| _        |dk    rt          |¦  «        nt          j        ¦   «         | _         |	|fi |¤Ž| _	        t          d|t          ||z  ¦  «        ||dœ|¤Ž| _        |dk    rt          |¦  «        nt          j        ¦   «         | _        |
�Yt          j        |
t          j        |fi |¤Žz  ¦  «        | _        t          j        |
t          j        |fi |¤Žz  ¦  «        | _        nd\  | _        | _        || _        d S )Nr&   ©r’   r”   r–   r•   r�   ©r�   Úhidden_featuresrr   Údrop)r‘   r‘   © )r,   r-   Únorm1r   Úattnr   r.   ÚIdentityÚ
drop_path1Únorm2r   r]   ÚmlpÚ
drop_path2Ú	ParameterrE   ÚonesÚgamma1Úgamma2rš   )r5   r#   r’   r“   r”   r•   r–   r—   rr   r˜   r™   rš   r'   r(   r6   r7   s                  €r8   r-   zClassAttentionBlock.__init__°   sˆ  ø€ ð  ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ�Z Ð*Ð* rÐ*Ð*ˆŒ
ÝØð
àØØØð
ð 
ð ð
ð 
ˆŒ	ð 2;¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒà�Z Ð*Ð* rÐ*Ð*ˆŒ
Ýð 
ØÝ  i¡Ñ0Ô0ØØð	
ð 
ð
 ð
ð 
ˆŒð 2;¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒàˆ?Ýœ, s­U¬Z¸Ð-BÐ-B¸rÐ-BÐ-BÑ'BÑCÔCˆDŒKÝœ, s­U¬Z¸Ð-BÐ-B¸rÐ-BÐ-BÑ'BÑCÔCˆDŒKˆKà'/Ñ$ˆDŒK˜œð 'ˆÔÐÐr9   c                 óp  — |                       |¦  «        }t          j        |                      |¦  «        |d d …dd …f         gd¬¦  «        }||                      | j        |z  ¦  «        z   }| j        r|                      |¦  «        }nCt          j        |                      |d d …dd…f         ¦  «        |d d …dd …f         gd¬¦  «        }|}|d d …dd…f         }| j        |  	                    |¦  «        z  }t          j        ||d d …dd …f         gd¬¦  «        }||  
                    |¦  «        z   }|S )Nr   rB   r   )r¡   rE   rP   r¢   r¤   rª   rš   r¥   r«   r¦   r§   )r5   rz   Úx_norm1Úx_attnÚx_resÚ	cls_tokens         r8   rX   zClassAttentionBlock.forwardà   s;  € Ø—*’*˜Q‘-”-ˆÝ”˜DŸIšI gÑ.Ô.°¸¸¸¸1¸2¸2¸´Ð?ÀQÐGÑGÔGˆØ�—’ ¤¨fÑ 4Ñ5Ô5Ñ5ˆàÔð 	DØ—
’
˜1‘”ˆAˆAå”	˜4Ÿ:š: a¨¨¨¨1¨Q¨3¨¤iÑ0Ô0°!°A°A°A°q°r°r°E´(Ð;ÀÐCÑCÔCˆAØˆØ�a�a�a˜˜1˜�f”Iˆ	Ø”K $§(¢(¨9Ñ"5Ô"5Ñ5ˆ	ÝŒI�y ! A A A q r r E¤(Ð+°Ð3Ñ3Ô3ˆØ�D—O’O AÑ&Ô&Ñ&ˆØˆr9   )rY   rZ   r[   r\   r.   r}   Ú	LayerNormr]   r^   Úboolr   r~   r   r-   rX   r_   r`   s   @r8   rŽ   rŽ   ­   sû   ø€ € € € € ØKÐKð  "Ø"Ø!Ø!Ø!Ø)+¬Ø*,¬,Ø#%Ø %ØØð.'ð .'àð.'ð ð.'ð ð	.'ð
 ð.'ð ð.'ð ð.'ð ð.'ð ˜BœI”ð.'ð ˜RœYœð.'ð ˜%”ð.'ð ð.'ð .'ð .'ð .'ð .'ð .'ð`ð ð ð ð ð ð r9   rŽ   c                   ó    ‡ — e Zd ZU ej        j        e         ed<   	 	 	 	 	 	 	 ddededed	e	d
e	f
ˆ fd„Z
d„ Zej        j        d„ ¦   «         Zˆ xZS )ÚXCAÚ
fused_attnrt   Fr�   Nr#   r’   r”   r–   r•   c                 ó®  •— ||dœ}t          ¦   «                              ¦   «          || _        t          d¬¦  «        | _        t          j        t          j        |ddfi |¤Ž¦  «        | _	        t          j
        ||dz  fd|i|¤Ž| _        t          j        |¦  «        | _        t          j
        ||fi |¤Ž| _        t          j        |¦  «        | _        d S )Nr&   T)Úexperimentalr   rC   rd   )r,   r-   r’   r   rµ   r.   r¨   rE   r©   r$   ÚLinearÚqkvÚDropoutr–   rv   r•   )
r5   r#   r’   r”   r–   r•   r'   r(   r6   r7   s
            €r8   r-   zXCA.__init__ø   sÎ   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ"ˆŒÝ(°dÐ;Ñ;Ô;ˆŒÝœ<­¬
°9¸aÀÐ(IÐ(IÀbÐ(IÐ(IÑJÔJˆÔÝ”9˜S #¨¡'Ð?Ð?°Ð?¸BÐ?Ð?ˆŒÝœ IÑ.Ô.ˆŒÝ”I˜c 3Ð-Ð-¨"Ð-Ð-ˆŒ	Ýœ IÑ.Ô.ˆŒˆˆr9   c                 óÞ  — |j         \  }}}|                      |¦  «                             ||d| j        || j        z  ¦  «                             ddddd¦  «        }|                     d¦  «        \  }}}| j        r}t          j        j	         
                    |d¬¦  «        | j        z  }t          j        j	         
                    |d¬¦  «        }t          j        j	                             |||d¬	¦  «        }n�t          j        j	         
                    |d¬¦  «        }t          j        j	         
                    |d¬¦  «        }||                     d
d¦  «        z  | j        z  }	|	                     d¬¦  «        }	|                      |	¦  «        }	|	|z  }|                     dddd¦  «                             |||¦  «        }|                      |¦  «        }|                      |¦  «        }|S )NrC   r)   r   rA   r   r>   rB   r‘   )r3   éþÿÿÿ)rx   r¹   rŠ   r’   rQ   Úunbindrµ   rE   r.   Ú
functionalÚ	normalizer$   Úscaled_dot_product_attentionry   Úsoftmaxr–   rv   r•   )
r5   rz   r:   r‹   rŒ   r¹   ÚqÚkÚvr¢   s
             r8   rX   zXCA.forward  s´  € Ø”'‰ˆˆ1ˆaà�hŠh�q‰kŒk×!Ò! ! Q¨¨4¬>¸1ÀÄÑ;NÑOÔO×WÒWÐXYÐ[\Ð^_ÐabÐdeÑfÔfˆØ—*’*˜Q‘-”-‰ˆˆ1ˆaàŒ?ð 	Ý”Ô#×-Ò-¨a°RÐ-Ñ8Ô8¸4Ô;KÑKˆAÝ”Ô#×-Ò-¨a°RÐ-Ñ8Ô8ˆAÝ”Ô#×@Ò@ÀÀAÀqÐPSÐ@ÑTÔTˆAˆAõ ”Ô#×-Ò-¨a°RÐ-Ñ8Ô8ˆAÝ”Ô#×-Ò-¨a°RÐ-Ñ8Ô8ˆAØ˜Ÿš B¨Ñ+Ô+Ñ+¨tÔ/?Ñ?ˆDØ—<’< B�<Ñ'Ô'ˆDØ—>’> $Ñ'Ô'ˆDØ�q‘ˆAà�IŠI�a˜˜A˜qÑ!Ô!×)Ò)¨!¨Q°Ñ2Ô2ˆØ�IŠI�a‰LŒLˆØ�NŠN˜1ÑÔˆØˆr9   c                 ó   — dhS )Nr$   r    ©r5   s    r8   Úno_weight_decayzXCA.no_weight_decay$  s
   € àˆÐr9   )rt   Fr�   r�   NN)rY   rZ   r[   rE   ÚjitÚFinalr²   Ú__annotations__r]   r^   r-   rX   ÚignorerÇ   r_   r`   s   @r8   r´   r´   ñ   sÒ   ø€ € € € € € Ø”	” Ô%Ð%Ð%Ñ%ðð Ø"Ø!Ø!ØØð/ð /àð/ð ð/ð ð	/ð
 ð/ð ð/ð /ð /ð /ð /ð /ð(ð ð ð0 „YÔðð ñ Ôðð ð ð ð r9   r´   c                   ó®   ‡ — e Zd Zdddddej        ej        dddf
dededed	ed
ededede	ej
                 de	ej
                 defˆ fd„Zdedefd„Zˆ xZS )ÚXCABlockr�   Fr�   r‘   Nr#   r’   r“   r”   r•   r–   r—   rr   r˜   r™   c                 ó^  •— ||dœ}t          ¦   «                              ¦   «           |	|fi |¤Ž| _        t          |f||||dœ|¤Ž| _        |dk    rt          |¦  «        nt          j        ¦   «         | _         |	|fi |¤Ž| _	        t          d||dœ|¤Ž| _        |dk    rt          |¦  «        nt          j        ¦   «         | _         |	|fi |¤Ž| _        t          d|t          ||z  ¦  «        ||dœ|¤Ž| _        |dk    rt          |¦  «        nt          j        ¦   «         | _        t          j        |
t'          j        |fi |¤Žz  ¦  «        | _        t          j        |
t'          j        |fi |¤Žz  ¦  «        | _        t          j        |
t'          j        |fi |¤Žz  ¦  «        | _        d S )Nr&   rœ   r�   )r�   rr   r�   r    )r,   r-   r¡   r´   r¢   r   r.   r£   r¤   Únorm3r€   Úlocal_mpÚ
drop_path3r¥   r   r]   r¦   r§   r¨   rE   r©   rª   Úgamma3r«   )r5   r#   r’   r“   r”   r•   r–   r—   rr   r˜   r™   r'   r(   r6   r7   s                 €r8   r-   zXCABlock.__init__*  sæ  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ�Z Ð*Ð* rÐ*Ð*ˆŒ
ÝØð
àØØØð
ð 
ð ð
ð 
ˆŒ	ð 2;¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒà�Z Ð*Ð* rÐ*Ð*ˆŒ
ÝÐG¨°yÐGÐGÀBÐGÐGˆŒØ1:¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒà�Z Ð*Ð* rÐ*Ð*ˆŒ
Ýð 
ØÝ  i¡Ñ0Ô0ØØð	
ð 
ð
 ð
ð 
ˆŒð 2;¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒå”l 3­¬°CÐ)>Ð)>¸2Ð)>Ð)>Ñ#>Ñ?Ô?ˆŒÝ”l 3­¬°CÐ)>Ð)>¸2Ð)>Ð)>Ñ#>Ñ?Ô?ˆŒÝ”l 3­¬°CÐ)>Ð)>¸2Ð)>Ð)>Ñ#>Ñ?Ô?ˆŒˆˆr9   r;   r<   c           	      ó®  — ||                       | j        |                      |                      |¦  «        ¦  «        z  ¦  «        z   }||                      | j        |                      |                      |¦  «        ||¦  «        z  ¦  «        z   }||                      | j	        |  
                    |                      |¦  «        ¦  «        z  ¦  «        z   }|S ©N)r¤   rª   r¢   r¡   rÑ   rÒ   rÐ   rÏ   r§   r«   r¦   r¥   )r5   rz   r;   r<   s       r8   rX   zXCABlock.forwardX  s¦   € Ø�—’ ¤¨d¯iªi¸¿
º
À1¹¼Ñ.FÔ.FÑ FÑGÔGÑGˆð �—’ ¤¨d¯mªm¸D¿JºJÀq¹M¼MÈ1ÈaÑ.PÔ.PÑ PÑQÔQÑQˆØ�—’ ¤¨d¯hªh°t·z²zÀ!±}´}Ñ.EÔ.EÑ EÑFÔFÑFˆØˆr9   )rY   rZ   r[   r.   r}   r±   r]   r^   r²   r   r~   r-   rX   r_   r`   s   @r8   rÍ   rÍ   )  s  ø€ € € € € ð
  "Ø"Ø!Ø!Ø!Ø)+¬Ø*,¬,ØØØð,@ð ,@àð,@ð ð,@ð ð	,@ð
 ð,@ð ð,@ð ð,@ð ð,@ð ˜BœI”ð,@ð ˜RœYœð,@ð ð,@ð ,@ð ,@ð ,@ð ,@ð ,@ð\˜Cð  Cð ð ð ð ð ð ð ð r9   rÍ   c            +       óö  ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d=deeeeef         f         dedededededededede	dededededede
eej                          d e
eej                          d!ed"e	d#ed$e	f*ˆ fd%„Zd&„ Zej        j        d'„ ¦   «         Zej        j        d>d(„¦   «         Zej        j        d?d)„¦   «         Zej        j        d*ej        fd+„¦   «         Zd@dede
e         fd,„Z	 	 	 	 	 dAd.ej        d/e
eeee         f                  d0e	d1e	d2ed3e	d*eeej                 eej        eej                 f         f         fd4„Z	 	 	 dBd/eeee         f         d6e	d7e	fd8„Zd9„ Zd>d:e	fd;„Zd<„ Zˆ xZS )Cr   z 
    Based on timm and DeiT code bases
    https://github.com/rwightman/pytorch-image-models/tree/master/timm
    https://github.com/facebookresearch/deit/
    rl   rm   rC   éè  Útokenr    é   r�   Tr�   Nr)   r‘   Frn   ro   rp   Únum_classesÚglobal_poolrq   Údepthr’   r“   r”   Ú	drop_rateÚpos_drop_rateÚproj_drop_rateÚattn_drop_rateÚdrop_path_raterr   r˜   Úcls_attn_layersÚuse_pos_embedr™   rš   c                 óâ  •‡‡‡	‡
‡‡‡‡‡‡‡‡‡‡— t          ¦   «                              ¦   «          ||dœŠ|dv sJ ‚t          |¦  «        }|d         |z  dk    r|d         |z  dk    s
J d¦   «         ‚‰pt          t          j        d¬¦  «        Š‰pt          j        Š|| _        || _        ‰x| _	        x| _
        | _        || _        d| _        t          d|||‰‰dœ‰¤Ž| _        |Št	          j        t#          j        d	d	‰fi ‰¤Ž¦  «        | _        |rt)          dd
‰i‰¤Ž| _        nd| _        t	          j        |¬¦  «        | _        t	          j        ˆˆˆˆˆˆˆ	ˆˆˆˆ
fd„t3          |¦  «        D ¦   «         ¦  «        | _        ˆˆfd„t3          |¦  «        D ¦   «         | _        t	          j        ˆˆˆˆˆˆˆ	ˆˆˆ
ˆfd„t3          |¦  «        D ¦   «         ¦  «        | _         ‰‰fi ‰¤Ž| _        t	          j        ‰¦  «        | _        |dk    rt	          j        | j	        |fi ‰¤Žnt	          j         ¦   «         | _!        tE          | j        d¬¦  «         |  #                    | j$        ¦  «         dS )a¦  
        Args:
            img_size (int, tuple): input image size
            patch_size (int): patch size
            in_chans (int): number of input channels
            num_classes (int): number of classes for classification head
            embed_dim (int): embedding dimension
            depth (int): depth of transformer
            num_heads (int): number of attention heads
            mlp_ratio (int): ratio of mlp hidden dim to embedding dim
            qkv_bias (bool): enable bias for qkv if True
            drop_rate (float): dropout rate after positional embedding, and in XCA/CA projection + MLP
            pos_drop_rate: position embedding dropout rate
            proj_drop_rate (float): projection dropout rate
            attn_drop_rate (float): attention dropout rate
            drop_path_rate (float): stochastic depth rate (constant across all layers)
            norm_layer: (nn.Module): normalization layer
            cls_attn_layers: (int) Depth of Class attention layers
            use_pos_embed: (bool) whether to use positional encoding
            eta: (float) layerscale initialization value
            tokens_norm: (bool) Whether to normalize all tokens or just the cls_token in the CA

        Notes:
            - Although `layer_norm` is user specifiable, there are hard-coded `BatchNorm2d`s in the local patch
              interaction (class LPI) and the patch embedding (class ConvPatchEmbed)
        r&   ©Ú Úavgr×   r   z2`patch_size` should divide image dimensions evenlyr+   )r4   F)rn   ro   rp   rq   rr   r   r#   N)Úpc                 ó>   •— g | ]}t          d‰‰
‰‰‰‰‰‰‰	‰d œ
‰¤Ž‘ŒS ))
r#   r’   r“   r”   r•   r–   r—   rr   r˜   r™   r    )rÍ   )Ú.0Ú_rr   rß   r6   rà   rq   r™   r“   r˜   r’   rÞ   r”   s     €€€€€€€€€€€r8   ú
<listcomp>z!Xcit.__init__.<locals>.<listcomp>¼  sg   ø€ ð %#ð %#ð %#ð õ ð ØØ#Ø#Ø!Ø(Ø(Ø(Ø#Ø%Øðð ð ðð ð%#ð %#ð %#r9   c                 ó:   •— g | ]}t          ‰‰d |› �¬¦  «        ‘ŒS )zblocks.)Únum_chsÚ	reductionÚmodule©Údict)ré   Úirq   Úrs     €€r8   rë   z!Xcit.__init__.<locals>.<listcomp>Ë  s0   ø€ ÐnÐnÐnÐ\]�T¨)¸qÈÐSTÈÈÐWÑWÔWÐnÐnÐnr9   c                 ó>   •— g | ]}t          d‰‰
‰‰‰‰‰‰	‰‰d œ
‰¤Ž‘ŒS ))
r#   r’   r“   r”   r•   r–   rr   r˜   r™   rš   r    )rŽ   )ré   rê   rr   rß   r6   rÜ   rq   r™   r“   r˜   r’   r”   rš   s     €€€€€€€€€€€r8   rë   z!Xcit.__init__.<locals>.<listcomp>Í  sg   ø€ ð .-ð .-ð .-ð õ  ð ØØ#Ø#Ø!Ø#Ø(Ø#Ø%ØØ'ðð ð ðð ð.-ð .-ð .-r9   ç{®Gáz”?©Ústdr    )%r,   r-   r   r   r.   r±   r}   rÙ   rp   Únum_featuresÚhead_hidden_sizerq   rÚ   Úgrad_checkpointingrk   Úpatch_embedr¨   rE   Úzerosr°   r   Ú	pos_embedrº   Úpos_dropÚ
ModuleListÚrangeÚblocksÚfeature_infoÚcls_attn_blocksÚnormÚ	head_dropr¸   r£   Úheadr   ÚapplyÚ_init_weights)r5   rn   ro   rp   rÙ   rÚ   rq   rÛ   r’   r“   r”   rÜ   rÝ   rÞ   rß   rà   rr   r˜   rá   râ   r™   rš   r'   r(   r6   ró   r7   s         ` ```` `````  ``  @@€r8   r-   zXcit.__init__h  s6  øøøøøøøøøøøøøøø€ õh 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆØÐ2Ð2Ð2Ð2Ð2Ý˜XÑ&Ô&ˆØ˜”˜jÑ(¨AÒ-Ð-°H¸Q´KÀ*Ñ4LÐPQÒ4QÐ4QÐ4QØ@ñ 5RÔ4QÐRàÐB¥7­2¬<¸TÐ#BÑ#BÔ#Bˆ
ØÐ(¥¤ˆ	à&ˆÔØ ˆŒØENÐNˆÔÐN˜DÔ1°D´NØ&ˆÔØ"'ˆÔå)ð 
ØØ!ØØØð
ð 
ð ð
ð 
ˆÔð ˆåœ¥e¤k°!°Q¸	Ð&HÐ&HÀRÐ&HÐ&HÑIÔIˆŒØð 	"Ý6ÐKÐK¸9ÐKÈÐKÐKˆDŒNˆNà!ˆDŒNÝœ
 ]Ð3Ñ3Ô3ˆŒå”mð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#õ ˜5‘\”\ð%#ñ %#ô %#ñ $ô $ˆŒð oÐnÐnÐnÐnÕafÐglÑamÔamÐnÑnÔnˆÔå!œ}ð .-ð .-ð .-ð .-ð .-ð .-ð .-ð .-ð .-ð .-ð .-ð .-ð .-ð .-õ ˜?Ñ+Ô+ð.-ñ .-ô .-ñ  .ô  .ˆÔð" �J˜yÐ/Ð/¨BÐ/Ð/ˆŒ	Ýœ IÑ.Ô.ˆŒØGRÐUVÂÀ•B”I˜dÔ/°ÐCÐCÀÐCÐCÐCÕ\^Ô\gÑ\iÔ\iˆŒ	õ 	�d”n¨#Ð.Ñ.Ô.Ð.Ø�
Š
�4Ô%Ñ&Ô&Ð&Ð&Ð&r9   c                 óþ   — t          |t          j        ¦  «        r^t          |j        d¬¦  «         t          |t          j        ¦  «        r0|j        �+t          j                             |j        d¦  «         d S d S d S d S )Nrõ   rö   r   )Ú
isinstancer.   r¸   r   rD   rd   ÚinitÚ	constant_)r5   Úms     r8   r  zXcit._init_weightsæ  s€   € Ý�a�œÑ#Ô#ð 	-Ý˜!œ(¨Ð,Ñ,Ô,Ð,Ý˜!�RœYÑ'Ô'ð -¨A¬FÐ,>Ý”×!Ò! !¤&¨!Ñ,Ô,Ð,Ð,Ð,ð	-ð 	-ð-ð -Ð,>Ð,>r9   c                 ó
   — ddhS )Nrý   r°   r    rÆ   s    r8   rÇ   zXcit.no_weight_decayì  s   € à˜[Ð)Ð)r9   c                 ó*   — t          ddddg¬¦  «        S )Nz ^cls_token|pos_embed|patch_embedz^blocks\.(\d+))z^cls_attn_blocks\.(\d+)N)z^norm)iŸ† )Ústemr  r  rð   )r5   Úcoarses     r8   Úgroup_matcherzXcit.group_matcherð  s(   € åØ4Ø$Ø?ÐAUÐVð
ñ 
ô 
ð 	
r9   c                 ó   — || _         d S rÔ   )rú   )r5   Úenables     r8   Úset_grad_checkpointingzXcit.set_grad_checkpointingø  s   € à"(ˆÔÐÐr9   Úreturnc                 ó   — | j         S rÔ   )r  rÆ   s    r8   Úget_classifierzXcit.get_classifierü  s
   € àŒyÐr9   c                 óú   — t          | ¦  «        }|| _        |�|dv sJ ‚|| _        |dk    rt          j        | j        |fi |¤Žnt          j        ¦   «         | _        | j                             | j	        ¦  «         d S )Nrä   r   )
r   rÙ   rÚ   r.   r¸   rø   r£   r  ÚtrainÚtraining)r5   rÙ   rÚ   r6   s       r8   Úreset_classifierzXcit.reset_classifier   s‰   € Ý˜dÑ#Ô#ˆØ&ˆÔØÐ"ØÐ"6Ð6Ð6Ð6Ð6Ø*ˆDÔØGRÐUVÂÀ•B”I˜dÔ/°ÐCÐCÀÐCÐCÐCÕ\^Ô\gÑ\iÔ\iˆŒ	ØŒ	�Š˜œÑ&Ô&Ð&Ð&Ð&r9   ÚNCHWrz   Úindicesr  Ú
stop_earlyÚ
output_fmtÚintermediates_onlyc                 ó†  ‡‡‡— |dv s
J d¦   «         ‚|dk    }g }t          t          | j        ¦  «        |¦  «        \  }	}
|j        \  Š}}}|                      |¦  «        \  }\  ŠŠ| j        �Q|                      ‰‰‰¦  «                             ‰d|j        d         ¦  «                             ddd¦  «        }||z   }|                      |¦  «        }t          j
                             ¦   «         s|s| j        }n| j        d|
dz   …         }t          |¦  «        D ]z\  }}| j        r1t          j
                             ¦   «         st          ||‰‰¦  «        }n ||‰‰¦  «        }||	v r,|                     |r|                      |¦  «        n|¦  «         Œ{|rˆˆˆfd	„|D ¦   «         }|r|S t          j        | j                             ‰dd¦  «        |fd¬
¦  «        }| j        D ]C}| j        r/t          j
                             ¦   «         st          ||¦  «        }Œ8 ||¦  «        }ŒD|                      |¦  «        }||fS )aè   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to all intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )r  ÚNLCz)Output format must be one of NCHW or NLC.r  Nr>   r   r   r)   c                 ó�   •— g | ]B}|                      ‰‰‰d ¦  «                             dddd¦  «                             ¦   «         ‘ŒCS )r>   r   rC   r   r)   )rŠ   rQ   Ú
contiguous)ré   Úyr:   r{   r|   s     €€€r8   rë   z.Xcit.forward_intermediates.<locals>.<listcomp><  sL   ø€ ÐnÐnÐnÐ[\˜QŸYšY q¨"¨b°"Ñ5Ô5×=Ò=¸aÀÀAÀqÑIÔI×TÒTÑVÔVÐnÐnÐnr9   rB   )r   Úlenr  rx   rû   rý   rŠ   rQ   rþ   rE   rÈ   Úis_scriptingÚ	enumeraterú   r   Úappendr  rP   r°   Úexpandr  )r5   rz   r  r  r  r   r!  rŠ   ÚintermediatesÚtake_indicesÚ	max_indexrê   ÚheightÚwidthÚpos_encodingr  rò   Úblkr:   r{   r|   s                     @@@r8   Úforward_intermediateszXcit.forward_intermediates	  s„  øøø€ ð* ˜_Ð,Ð,Ð,Ð.YÑ,Ô,Ð,Ø Ò&ˆØˆÝ"6µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�ið  œgÑˆˆ1ˆf�eØ×&Ò& qÑ)Ô)‰ˆ‰8ˆB�ØŒ>Ð%àŸ>š>¨!¨R°Ñ4Ô4×<Ò<¸QÀÀAÄGÈAÄJÑOÔO×WÒWÐXYÐ[\Ð^_Ñ`Ô`ˆLØ�LÑ ˆAØ�MŠM˜!ÑÔˆåŒ9×!Ò!Ñ#Ô#ð 	1¨:ð 	1Ø”[ˆFˆFà”[  )¨a¡- Ô0ˆFÝ Ñ'Ô'ð 	Bð 	B‰FˆAˆsØÔ&ð #­u¬y×/EÒ/EÑ/GÔ/Gð #Ý˜s A r¨2Ñ.Ô.��à�C˜˜2˜r‘N”N�Ø�LÐ Ð à×$Ò$°TÐ%@ T§Y¢Y¨q¡\¤\ \¸qÑAÔAÐAøð ð 	oànÐnÐnÐnÐnÐnÐ`mÐnÑnÔnˆMàð 	!Ø Ð õ ŒI�t”~×,Ò,¨Q°°BÑ7Ô7¸Ð;ÀÐCÑCÔCˆØÔ'ð 	ð 	ˆCØÔ&ð ­u¬y×/EÒ/EÑ/GÔ/Gð Ý˜s AÑ&Ô&��à�C˜‘F”F��à�IŠI�a‰LŒLˆà�-ÐÐr9   r   Ú
prune_normÚ
prune_headc                 ó  — t          t          | j        ¦  «        |¦  «        \  }}| j        d|dz   …         | _        |rt          j        ¦   «         | _        |r.t          j        ¦   «         | _        |                      dd¦  «         |S )z@ Prune layers not required for specified intermediates.
        Nr   r   rå   )	r   r'  r  r.   r£   r  rÿ   r  r  )r5   r  r4  r5  r-  r.  s         r8   Úprune_intermediate_layerszXcit.prune_intermediate_layersM  s   € õ #7µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�iØ”k . 9¨q¡= .Ô1ˆŒØð 	&Ýœ™œˆDŒIØð 	)Ý#%¤=¡?¤?ˆDÔ Ø×!Ò! ! RÑ(Ô(Ð(ØÐr9   c                 óò  — |j         d         }|                      |¦  «        \  }\  }}| j        �Q|                      |||¦  «                             |d|j         d         ¦  «                             ddd¦  «        }||z   }|                      |¦  «        }| j        D ]G}| j        r1t          j	         
                    ¦   «         st          ||||¦  «        }Œ: ||||¦  «        }ŒHt          j        | j                             |dd¦  «        |fd¬¦  «        }| j        D ]C}| j        r/t          j	         
                    ¦   «         st          ||¦  «        }Œ8 ||¦  «        }ŒD|                      |¦  «        }|S )Nr   r>   r   r)   rB   )rx   rû   rý   rŠ   rQ   rþ   r  rú   rE   rÈ   r(  r   rP   r°   r+  r  r  )r5   rz   r:   r{   r|   r1  r2  s          r8   Úforward_featureszXcit.forward_features^  sy  € ØŒG�AŒJˆà×&Ò& qÑ)Ô)‰ˆ‰8ˆB�àŒ>Ð%àŸ>š>¨!¨R°Ñ4Ô4×<Ò<¸QÀÀAÄGÈAÄJÑOÔO×WÒWÐXYÐ[\Ð^_Ñ`Ô`ˆLØ�LÑ ˆAØ�MŠM˜!ÑÔˆà”;ð 	#ð 	#ˆCØÔ&ð #­u¬y×/EÒ/EÑ/GÔ/Gð #Ý˜s A r¨2Ñ.Ô.��à�C˜˜2˜r‘N”N��åŒI�t”~×,Ò,¨Q°°BÑ7Ô7¸Ð;ÀÐCÑCÔCˆàÔ'ð 	ð 	ˆCØÔ&ð ­u¬y×/EÒ/EÑ/GÔ/Gð Ý˜s AÑ&Ô&��à�C˜‘F”F��à�IŠI�a‰LŒLˆØˆr9   Ú
pre_logitsc                 óÞ   — | j         r9| j         dk    r"|d d …dd …f                              d¬¦  «        n|d d …df         }|                      |¦  «        }|r|n|                      |¦  «        S )Nræ   r   rB   r   )rÚ   Úmeanr  r  )r5   rz   r:  s      r8   Úforward_headzXcit.forward_headz  sy   € ØÔð 	OØ(,Ô(8¸EÒ(AÐ(A��!�!�!�Q�R�R�%”—’ !�Ñ$Ô$Ð$ÀqÈÈÈÈAÈÄwˆAØ�NŠN˜1ÑÔˆØÐ0ˆqˆq D§I¢I¨a¡L¤LÐ0r9   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S rÔ   )r9  r=  )r5   rz   s     r8   rX   zXcit.forward€  s-   € Ø×!Ò! !Ñ$Ô$ˆØ×Ò˜aÑ Ô ˆØˆr9   )rl   rm   rC   rÖ   r×   r    rØ   rØ   r�   Tr�   r�   r�   r�   r�   NNr)   Tr‘   FNN©F)TrÔ   )NFFr  F)r   FT) rY   rZ   r[   r\   r   r]   r   Ústrr^   r²   r   r   r.   r~   r-   r  rE   rÈ   rË   rÇ   r  r  r  r  ÚTensorr   r3  r7  r9  r=  rX   r_   r`   s   @r8   r   r   a  s¼  ø€ € € € € ðð ð 58Ø ØØ#Ø&Ø ØØØ!Ø!Ø!Ø#%Ø$&Ø$&Ø$&Ø37Ø48Ø#$Ø"&ØØ %ØØð1|'ð |'à˜C  s¨C x¤Ð0Ô1ð|'ð ð|'ð ð	|'ð
 ð|'ð ð|'ð ð|'ð ð|'ð ð|'ð ð|'ð ð|'ð ð|'ð !ð|'ð "ð|'ð "ð|'ð  "ð!|'ð"    R¤Y¤Ô0ð#|'ð$ !  b¤i¤Ô1ð%|'ð& !ð'|'ð(  ð)|'ð* ð+|'ð, ð-|'ð |'ð |'ð |'ð |'ð |'ð|-ð -ð -ð „YÔð*ð *ñ Ôð*ð „YÔð
ð 
ð 
ñ Ôð
ð „YÔð)ð )ð )ñ Ôð)ð „YÔð ¤	ð ð ð ñ Ôðð'ð '¨Cð '¸hÀs¼mð 'ð 'ð 'ð 'ð 8<ØØ$Ø$Ø',ðB ð B àŒ|ðB ð ˜e C¨¨c¬ NÔ3Ô4ðB ð ð	B ð
 ðB ð ðB ð !%ðB ð 
ˆt�E”LÔ! 5¨¬°t¸E¼LÔ7IÐ)IÔ#JÐJÔ	KðB ð B ð B ð B ðL ./Ø$Ø#ð	ð à˜3  S¤	˜>Ô*ðð ðð ð	ð ð ð ð"ð ð ð81ð 1¨$ð 1ð 1ð 1ð 1ðð ð ð ð ð ð r9   c           	      óÀ  — d| v r| d         } t          |dd ¦  «        d u}d„ | D ¦   «         }|D ]4}|r-|                      |¦  «        | |                     dd¦  «        <   Œ1| |= Œ5d| v r÷d|                     ¦   «         v rát	          |j        ¦  «        }t          |¦  «        D ]½}|                      d|› d	�¦  «        }|                     d
d|j        d         ¦  «        }t          d¦  «        D ]\  }}	||         | d|› d|	› d�<   Œ|                      d|› d�d ¦  «        }
|
�=|
                     d
d¦  «        }
t          d¦  «        D ]\  }}	|
|         | d|› d|	› d�<   ŒŒ¾| S )NÚmodelrý   c                 ó<   — g | ]}|                      d ¦  «        ¯|‘ŒS )rý   )Ú
startswith)ré   rÃ   s     r8   rë   z(checkpoint_filter_fn.<locals>.<listcomp>Œ  s)   € ÐIÐIÐI˜A¨q¯|ª|¸KÑ/HÔ/HÐI�aÐIÐIÐIr9   zpos_embeder.z
pos_embed.z!cls_attn_blocks.0.attn.qkv.weightzcls_attn_blocks.0.attn.q.weightzcls_attn_blocks.z.attn.qkv.weightrC   r>   r¹   z.attn.z.weightz.attn.qkv.biasz.bias)
ÚgetattrÚpopÚreplaceÚ
state_dictr'  r  r   rŠ   rx   r)  )rI  rC  râ   Úpos_embed_keysrÃ   Únum_ca_blocksrò   Ú
qkv_weightÚjÚ	subscriptr”   s              r8   Úcheckpoint_filter_fnrO  †  sÛ  € Ø�*ÐÐØ Ô(ˆ
õ ˜E ;°Ñ5Ô5¸TÐA€MØIÐI ÐIÑIÔI€NØð ð ˆØð 	ØBLÇ.Â.ÐQRÑBSÔBSˆJ�q—y’y °Ñ>Ô>Ñ?Ð?à˜1��ð +¨jÐ8Ð8Ð=^Ðbg×brÒbrÑbtÔbtÐ=tÐ=tÝ˜EÔ1Ñ2Ô2ˆÝ�}Ñ%Ô%ð 		[ð 		[ˆAØ#ŸšÐ(N¸1Ð(NÐ(NÐ(NÑOÔOˆJØ#×+Ò+¨A¨r°:Ô3CÀBÔ3GÑHÔHˆJÝ )¨%Ñ 0Ô 0ð [ð [‘��9ØMWÐXYÌ]�
ÐI¨aÐIÐI°yÐIÐIÐIÑJÐJØ!—~’~Ð&J¸Ð&JÐ&JÐ&JÈDÑQÔQˆHØÐ#Ø#×+Ò+¨A¨rÑ2Ô2�Ý$-¨eÑ$4Ô$4ð [ð [‘L�A�yØOWÐXYÌ{�JÐK°!ÐKÐK¸9ÐKÐKÐKÑLÐLøØÐr9   Fc                 óˆ   — |                      dd¦  «        }t          t          | |ft          t	          |d¬¦  «        dœ|¤Ž}|S )NÚout_indicesrC   Úgetter)rQ  Úfeature_cls)Úpretrained_filter_fnÚfeature_cfg)rG  r   r   rO  rñ   )ÚvariantÚ
pretrainedÚdefault_cfgÚkwargsrQ  rC  s         r8   Ú_create_xcitrZ  £  s]   € Ø—*’*˜]¨AÑ.Ô.€KÝ ÝØØðõ 2Ý [¸hÐGÑGÔGðð ð ðð €Eð €Lr9   rå   c                 ó8   — | ddd dddt           t          dddd	œ|¥S )
NrÖ   )rC   rl   rl   r‘   ÚbicubicTzpatch_embed.proj.0.0r  z
apache-2.0)ÚurlrÙ   Ú
input_sizeÚ	pool_sizeÚcrop_pctÚinterpolationÚfixed_input_sizer<  r÷   Ú
first_convÚ
classifierÚlicenser
   )r]  rY  s     r8   Ú_cfgrf  °  s:   € àØ¨=ÀtØ¨)ÈÝ%Õ.BØ,¸FØðð ð $*ðð r9   zxcit_nano_12_p16_224.fb_in1kztimm/z<https://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p16_224.pth)Ú	hf_hub_idr]  z!xcit_nano_12_p16_224.fb_dist_in1kzAhttps://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p16_224_dist.pthz!xcit_nano_12_p16_384.fb_dist_in1kzAhttps://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p16_384_dist.pth)rC   é€  rh  )rg  r]  r^  zxcit_tiny_12_p16_224.fb_in1kz<https://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p16_224.pthz!xcit_tiny_12_p16_224.fb_dist_in1kzAhttps://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p16_224_dist.pthz!xcit_tiny_12_p16_384.fb_dist_in1kzAhttps://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p16_384_dist.pthzxcit_tiny_24_p16_224.fb_in1kz<https://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p16_224.pthz!xcit_tiny_24_p16_224.fb_dist_in1kzAhttps://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p16_224_dist.pthz!xcit_tiny_24_p16_384.fb_dist_in1kzAhttps://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p16_384_dist.pthzxcit_small_12_p16_224.fb_in1kz=https://dl.fbaipublicfiles.com/xcit/xcit_small_12_p16_224.pthz"xcit_small_12_p16_224.fb_dist_in1kzBhttps://dl.fbaipublicfiles.com/xcit/xcit_small_12_p16_224_dist.pthz"xcit_small_12_p16_384.fb_dist_in1kzBhttps://dl.fbaipublicfiles.com/xcit/xcit_small_12_p16_384_dist.pthzxcit_small_24_p16_224.fb_in1kz=https://dl.fbaipublicfiles.com/xcit/xcit_small_24_p16_224.pthz"xcit_small_24_p16_224.fb_dist_in1kzBhttps://dl.fbaipublicfiles.com/xcit/xcit_small_24_p16_224_dist.pthz"xcit_small_24_p16_384.fb_dist_in1kzBhttps://dl.fbaipublicfiles.com/xcit/xcit_small_24_p16_384_dist.pthzxcit_medium_24_p16_224.fb_in1kz>https://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p16_224.pthz#xcit_medium_24_p16_224.fb_dist_in1kzChttps://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p16_224_dist.pthz#xcit_medium_24_p16_384.fb_dist_in1kzChttps://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p16_384_dist.pthzxcit_large_24_p16_224.fb_in1kz=https://dl.fbaipublicfiles.com/xcit/xcit_large_24_p16_224.pthz"xcit_large_24_p16_224.fb_dist_in1kzBhttps://dl.fbaipublicfiles.com/xcit/xcit_large_24_p16_224_dist.pthz"xcit_large_24_p16_384.fb_dist_in1kzBhttps://dl.fbaipublicfiles.com/xcit/xcit_large_24_p16_384_dist.pthzxcit_nano_12_p8_224.fb_in1kz;https://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p8_224.pthz xcit_nano_12_p8_224.fb_dist_in1kz@https://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p8_224_dist.pthz xcit_nano_12_p8_384.fb_dist_in1kz@https://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p8_384_dist.pthzxcit_tiny_12_p8_224.fb_in1kz;https://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p8_224.pthz xcit_tiny_12_p8_224.fb_dist_in1kz@https://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p8_224_dist.pthz xcit_tiny_12_p8_384.fb_dist_in1kz@https://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p8_384_dist.pthzxcit_tiny_24_p8_224.fb_in1kz;https://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p8_224.pthz xcit_tiny_24_p8_224.fb_dist_in1kz@https://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p8_224_dist.pthz xcit_tiny_24_p8_384.fb_dist_in1kz@https://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p8_384_dist.pthzxcit_small_12_p8_224.fb_in1kz<https://dl.fbaipublicfiles.com/xcit/xcit_small_12_p8_224.pthz!xcit_small_12_p8_224.fb_dist_in1kzAhttps://dl.fbaipublicfiles.com/xcit/xcit_small_12_p8_224_dist.pthz!xcit_small_12_p8_384.fb_dist_in1kzAhttps://dl.fbaipublicfiles.com/xcit/xcit_small_12_p8_384_dist.pthzxcit_small_24_p8_224.fb_in1kz<https://dl.fbaipublicfiles.com/xcit/xcit_small_24_p8_224.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_small_24_p8_224_dist.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_small_24_p8_384_dist.pthz=https://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p8_224.pthzBhttps://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p8_224_dist.pthzBhttps://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p8_384_dist.pthz<https://dl.fbaipublicfiles.com/xcit/xcit_large_24_p8_224.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_large_24_p8_224_dist.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_large_24_p8_384_dist.pth)ú!xcit_small_24_p8_224.fb_dist_in1kú!xcit_small_24_p8_384.fb_dist_in1kzxcit_medium_24_p8_224.fb_in1kú"xcit_medium_24_p8_224.fb_dist_in1kú"xcit_medium_24_p8_384.fb_dist_in1kzxcit_large_24_p8_224.fb_in1kú!xcit_large_24_p8_224.fb_dist_in1kú!xcit_large_24_p8_384.fb_dist_in1kr  c           	      ób   — t          dddddd¬¦  «        }t          d
d	| it          |fi |¤Ž¤Ž}|S )Nrm   é€   rØ   rA   r‘   F©ro   rq   rÛ   r’   r™   rš   Úxcit_nano_12_p16_224rW  )rr  ©rñ   rZ  ©rW  rY  Ú
model_argsrC  s       r8   rr  rr  @  sU   € åØ ¨B¸!ÀÐRWðYñ Yô Y€JåÐeÐe¸JÐeÍ$ÈzÐJdÐJdÐ]cÐJdÐJdÐeÐe€EØ€Lr9   c           	      ód   — t          ddddddd¬¦  «        }t          dd
| it          |fi |¤Ž¤Ž}|S )Nrm   rp  rØ   rA   r‘   Frh  )ro   rq   rÛ   r’   r™   rš   rn   Úxcit_nano_12_p16_384rW  )rw  rs  rt  s       r8   rw  rw  H  sX   € åØ ¨B¸!ÀÐRWÐbeðgñ gô g€JåÐeÐe¸JÐeÍ$ÈzÐJdÐJdÐ]cÐJdÐJdÐeÐe€EØ€Lr9   c           	      ób   — t          dddddd¬¦  «        }t          d
d	| it          |fi |¤Ž¤Ž}|S )Nrm   éÀ   rØ   rA   r‘   Trq  Úxcit_tiny_12_p16_224rW  )rz  rs  rt  s       r8   rz  rz  P  óU   € åØ ¨B¸!ÀÐRVðXñ Xô X€JåÐeÐe¸JÐeÍ$ÈzÐJdÐJdÐ]cÐJdÐJdÐeÐe€EØ€Lr9   c           	      ób   — t          dddddd¬¦  «        }t          d
d	| it          |fi |¤Ž¤Ž}|S )Nrm   ry  rØ   rA   r‘   Trq  Úxcit_tiny_12_p16_384rW  )r}  rs  rt  s       r8   r}  r}  X  r{  r9   c           	      ób   — t          dddddd¬¦  «        }t          d
d	| it          |fi |¤Ž¤Ž}|S )Nrm   rh  rØ   rt   r‘   Trq  Úxcit_small_12_p16_224rW  )r  rs  rt  s       r8   r  r  `  óV   € åØ ¨B¸!ÀÐRVðXñ Xô X€JåÐfÐf¸ZÐfÍ4ÐPZÐKeÐKeÐ^dÐKeÐKeÐfÐf€EØ€Lr9   c           	      ób   — t          dddddd¬¦  «        }t          d
d	| it          |fi |¤Ž¤Ž}|S )Nrm   rh  rØ   rt   r‘   Trq  Úxcit_small_12_p16_384rW  )r‚  rs  rt  s       r8   r‚  r‚  h  r€  r9   c           	      ób   — t          dddddd¬¦  «        }t          d
d	| it          |fi |¤Ž¤Ž}|S )Nrm   ry  é   rA   çñhãˆµøä>Trq  Úxcit_tiny_24_p16_224rW  )r†  rs  rt  s       r8   r†  r†  p  óU   € åØ ¨B¸!ÀÐSWðYñ Yô Y€JåÐeÐe¸JÐeÍ$ÈzÐJdÐJdÐ]cÐJdÐJdÐeÐe€EØ€Lr9   c           	      ób   — t          dddddd¬¦  «        }t          d
d	| it          |fi |¤Ž¤Ž}|S )Nrm   ry  r„  rA   r…  Trq  Úxcit_tiny_24_p16_384rW  )r‰  rs  rt  s       r8   r‰  r‰  x  r‡  r9   c           	      ób   — t          dddddd¬¦  «        }t          d
d	| it          |fi |¤Ž¤Ž}|S )Nrm   rh  r„  rt   r…  Trq  Úxcit_small_24_p16_224rW  )r‹  rs  rt  s       r8   r‹  r‹  €  óV   € åØ ¨B¸!ÀÐSWðYñ Yô Y€JåÐfÐf¸ZÐfÍ4ÐPZÐKeÐKeÐ^dÐKeÐKeÐfÐf€EØ€Lr9   c           	      ób   — t          dddddd¬¦  «        }t          d
d	| it          |fi |¤Ž¤Ž}|S )Nrm   rh  r„  rt   r…  Trq  Úxcit_small_24_p16_384rW  )rŽ  rs  rt  s       r8   rŽ  rŽ  ˆ  rŒ  r9   c           	      ób   — t          dddddd¬¦  «        }t          d
d	| it          |fi |¤Ž¤Ž}|S )Nrm   é   r„  rt   r…  Trq  Úxcit_medium_24_p16_224rW  )r‘  rs  rt  s       r8   r‘  r‘  �  óV   € åØ ¨B¸!ÀÐSWðYñ Yô Y€JåÐgÐg¸jÐgÍDÐQ[ÐLfÐLfÐ_eÐLfÐLfÐgÐg€EØ€Lr9   c           	      ób   — t          dddddd¬¦  «        }t          d
d	| it          |fi |¤Ž¤Ž}|S )Nrm   r�  r„  rt   r…  Trq  Úxcit_medium_24_p16_384rW  )r”  rs  rt  s       r8   r”  r”  ˜  r’  r9   c           	      ób   — t          dddddd¬¦  «        }t          d	d| it          |fi |¤Ž¤Ž}|S )
Nrm   r    r„  r…  Trq  Úxcit_large_24_p16_224rW  )r–  rs  rt  s       r8   r–  r–     óV   € åØ ¨B¸"À$ÐTXðZñ Zô Z€JåÐfÐf¸ZÐfÍ4ÐPZÐKeÐKeÐ^dÐKeÐKeÐfÐf€EØ€Lr9   c           	      ób   — t          dddddd¬¦  «        }t          d	d| it          |fi |¤Ž¤Ž}|S )
Nrm   r    r„  r…  Trq  Úxcit_large_24_p16_384rW  )r™  rs  rt  s       r8   r™  r™  ¨  r—  r9   c           	      ób   — t          dddddd¬¦  «        }t          d
d	| it          |fi |¤Ž¤Ž}|S )Nrt   rp  rØ   rA   r‘   Frq  Úxcit_nano_12_p8_224rW  )r›  rs  rt  s       r8   r›  r›  ±  óU   € åØ ¨2¸ÀÐQVðXñ Xô X€JåÐdÐd¸:ÐdÍÈjÐIcÐIcÐ\bÐIcÐIcÐdÐd€EØ€Lr9   c           	      ób   — t          dddddd¬¦  «        }t          d
d	| it          |fi |¤Ž¤Ž}|S )Nrt   rp  rØ   rA   r‘   Frq  Úxcit_nano_12_p8_384rW  )rž  rs  rt  s       r8   rž  rž  ¹  rœ  r9   c           	      ób   — t          dddddd¬¦  «        }t          d
d	| it          |fi |¤Ž¤Ž}|S )Nrt   ry  rØ   rA   r‘   Trq  Úxcit_tiny_12_p8_224rW  )r   rs  rt  s       r8   r   r   Á  óU   € åØ ¨2¸ÀÐQUðWñ Wô W€JåÐdÐd¸:ÐdÍÈjÐIcÐIcÐ\bÐIcÐIcÐdÐd€EØ€Lr9   c           	      ób   — t          dddddd¬¦  «        }t          d
d	| it          |fi |¤Ž¤Ž}|S )Nrt   ry  rØ   rA   r‘   Trq  Úxcit_tiny_12_p8_384rW  )r£  rs  rt  s       r8   r£  r£  É  r¡  r9   c           	      ób   — t          dddddd¬¦  «        }t          d	d| it          |fi |¤Ž¤Ž}|S )
Nrt   rh  rØ   r‘   Trq  Úxcit_small_12_p8_224rW  )r¥  rs  rt  s       r8   r¥  r¥  Ñ  óU   € åØ ¨2¸ÀÐQUðWñ Wô W€JåÐeÐe¸JÐeÍ$ÈzÐJdÐJdÐ]cÐJdÐJdÐeÐe€EØ€Lr9   c           	      ób   — t          dddddd¬¦  «        }t          d	d| it          |fi |¤Ž¤Ž}|S )
Nrt   rh  rØ   r‘   Trq  Úxcit_small_12_p8_384rW  )r¨  rs  rt  s       r8   r¨  r¨  Ù  r¦  r9   c           	      ób   — t          dddddd¬¦  «        }t          d
d	| it          |fi |¤Ž¤Ž}|S )Nrt   ry  r„  rA   r…  Trq  Úxcit_tiny_24_p8_224rW  )rª  rs  rt  s       r8   rª  rª  á  óU   € åØ ¨2¸ÀÐRVðXñ Xô X€JåÐdÐd¸:ÐdÍÈjÐIcÐIcÐ\bÐIcÐIcÐdÐd€EØ€Lr9   c           	      ób   — t          dddddd¬¦  «        }t          d
d	| it          |fi |¤Ž¤Ž}|S )Nrt   ry  r„  rA   r…  Trq  Úxcit_tiny_24_p8_384rW  )r­  rs  rt  s       r8   r­  r­  é  r«  r9   c           	      ób   — t          dddddd¬¦  «        }t          d	d| it          |fi |¤Ž¤Ž}|S )
Nrt   rh  r„  r…  Trq  Úxcit_small_24_p8_224rW  )r¯  rs  rt  s       r8   r¯  r¯  ñ  óU   € åØ ¨2¸ÀÐRVðXñ Xô X€JåÐeÐe¸JÐeÍ$ÈzÐJdÐJdÐ]cÐJdÐJdÐeÐe€EØ€Lr9   c           	      ób   — t          dddddd¬¦  «        }t          d	d| it          |fi |¤Ž¤Ž}|S )
Nrt   rh  r„  r…  Trq  Úxcit_small_24_p8_384rW  )r²  rs  rt  s       r8   r²  r²  ù  r°  r9   c           	      ób   — t          dddddd¬¦  «        }t          d	d| it          |fi |¤Ž¤Ž}|S )
Nrt   r�  r„  r…  Trq  Úxcit_medium_24_p8_224rW  )r´  rs  rt  s       r8   r´  r´    óV   € åØ ¨2¸ÀÐRVðXñ Xô X€JåÐfÐf¸ZÐfÍ4ÐPZÐKeÐKeÐ^dÐKeÐKeÐfÐf€EØ€Lr9   c           	      ób   — t          dddddd¬¦  «        }t          d	d| it          |fi |¤Ž¤Ž}|S )
Nrt   r�  r„  r…  Trq  Úxcit_medium_24_p8_384rW  )r·  rs  rt  s       r8   r·  r·  	  rµ  r9   c           	      ób   — t          dddddd¬¦  «        }t          d
d	| it          |fi |¤Ž¤Ž}|S )Nrt   r    r„  rm   r…  Trq  Úxcit_large_24_p8_224rW  )r¹  rs  rt  s       r8   r¹  r¹    óU   € åØ ¨2¸ÀÐSWðYñ Yô Y€JåÐeÐe¸JÐeÍ$ÈzÐJdÐJdÐ]cÐJdÐJdÐeÐe€EØ€Lr9   c           	      ób   — t          dddddd¬¦  «        }t          d
d	| it          |fi |¤Ž¤Ž}|S )Nrt   r    r„  rm   r…  Trq  Úxcit_large_24_p8_384rW  )r¼  rs  rt  s       r8   r¼  r¼    rº  r9   Úxcit_nano_12_p16_224_distÚxcit_nano_12_p16_384_distÚxcit_tiny_12_p16_224_distÚxcit_tiny_12_p16_384_distÚxcit_tiny_24_p16_224_distÚxcit_tiny_24_p16_384_distÚxcit_small_12_p16_224_distÚxcit_small_12_p16_384_distÚxcit_small_24_p16_224_distÚxcit_small_24_p16_384_distÚxcit_medium_24_p16_224_distÚxcit_medium_24_p16_384_distÚxcit_large_24_p16_224_distÚxcit_large_24_p16_384_distÚxcit_nano_12_p8_224_distÚxcit_nano_12_p8_384_distÚxcit_tiny_12_p8_224_distri  rj  rk  rl  rm  rn  )Úxcit_tiny_12_p8_384_distÚxcit_tiny_24_p8_224_distÚxcit_tiny_24_p8_384_distÚxcit_small_12_p8_224_distÚxcit_small_12_p8_384_distÚxcit_small_24_p8_224_distÚxcit_small_24_p8_384_distÚxcit_medium_24_p8_224_distÚxcit_medium_24_p8_384_distÚxcit_large_24_p8_224_distÚxcit_large_24_p8_384_dist)r   NN)FN)rå   r?  )Qr\   r1   Ú	functoolsr   Útypingr   r   r   r   r   r	   rE   Útorch.nnr.   Ú	timm.datar   r   Útimm.layersr   r   r   r   r   r   Ú_builderr   Ú	_featuresr   Ú_features_fxr   Ú_manipulater   Ú	_registryr   r   r   Úcaitr   Ú__all__r~   r   ri   rk   r€   rŽ   r´   rÍ   r   rO  rZ  rf  Údefault_cfgsrr  rw  rz  r}  r  r‚  r†  r‰  r‹  rŽ  r‘  r”  r–  r™  r›  rž  r   r£  r¥  r¨  rª  r­  r¯  r²  r´  r·  r¹  r¼  rY   r    r9   r8   ú<module>ræ     s¢  ðð	ð 	ð €€€Ø Ð Ð Ð Ð Ð Ø :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :à €€€Ø Ð Ð Ð Ð Ð à AÐ AÐ AÐ AÐ AÐ AÐ AÐ AØ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aØ *Ð *Ð *Ð *Ð *Ð *Ø +Ð +Ð +Ð +Ð +Ð +Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø #Ð #Ð #Ð #Ð #Ð #Ø YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YØ Ð Ð Ð Ð Ð àˆ(€ð ð'&ð '&ð '&ð '&ð '& ¤	ñ '&ô '&ñ Ôð'&ðTð ð ð ð.ð .ð .ð .ð .�R”Yñ .ô .ð .ðb$ð $ð $ð $ð $ˆ"Œ)ñ $ô $ð $ðNAð Að Að Að A˜"œ)ñ Aô Að AðH5ð 5ð 5ð 5ð 5ˆ"Œ)ñ 5ô 5ð 5ðp5ð 5ð 5ð 5ð 5ˆrŒyñ 5ô 5ð 5ðpbð bð bð bð bˆ2Œ9ñ bô bð bðJ	ð ð ð:
ð 
ð 
ð 
ðð ð ð ð %Ð$ð B&à" D DØØJð%Lñ %Lô %LðB&ð
 (¨¨ØØOð*Qñ *Qô *QðB&ð (¨¨ØØOÐ\ið*kñ *kô *kðB&ð # D DØØJð%Lñ %Lô %LðB&ð (¨¨ØØOð*Qñ *Qô *QðB&ð" (¨¨ØØOÐ\ið*kñ *kô *kð#B&ð( # D DØØJð%Lñ %Lô %Lð)B&ð. (¨¨ØØOð*Qñ *Qô *Qð/B&ð4 (¨¨ØØOÐ\ið*kñ *kô *kð5B&ð: $ T TØØKð&Mñ &Mô &Mð;B&ð@ )¨$¨$ØØPð+Rñ +Rô +RðAB&ðF )¨$¨$ØØPÐ]jð+lñ +lô +lðGB&ðL $ T TØØKð&Mñ &Mô &MðMB&ðR )¨$¨$ØØPð+Rñ +Rô +RðSB&ðX )¨$¨$ØØPÐ]jð+lñ +lô +lðYB&ð^ % d dØØLð'Nñ 'Nô 'Nð_B&ðd *¨4¨4ØØQð,Sñ ,Sô ,SðeB&ð B&ðj *¨4¨4ØØQÐ^kð,mñ ,mô ,mðkB&ðp $ T TØØKð&Mñ &Mô &MðqB&ðv )¨$¨$ØØPð+Rñ +Rô +RðwB&ð| )¨$¨$ØØPÐ]jð+lñ +lô +lð}B&ðF " 4 4ØØIð$Kñ $Kô $KðGB&ðL '¨¨ØØNð)Pñ )Pô )PðMB&ðR '¨¨ØØNÐ[hð)jñ )jô )jðSB&ðX " 4 4ØØIð$Kñ $Kô $KðYB&ð^ '¨¨ØØNð)Pñ )Pô )Pð_B&ðd '¨¨ØØNÐ[hð)jñ )jô )jðeB&ðj " 4 4ØØIð$Kñ $Kô $KðkB&ðp '¨¨ØØNð)Pñ )Pô )PðqB&ðv '¨¨ØØNÐ[hð)jñ )jô )jðwB&ð| # D DØØJð%Lñ %Lô %Lð}B&ðB (¨¨ØØOð*Qñ *Qô *QðCB&ðH (¨¨ØØOÐ\ið*kñ *kô *kðIB&ðN # D DØØJð%Lñ %Lô %LðOB&ð B&ðT *.¨ØØOð*Qñ *Qô *Qð *.¨ØØOÐ\ið*kñ *kô *kð &* TØØKð&Mñ &Mô &Mð +/¨$ØØPð+Rñ +Rô +Rð +/¨$ØØPÐ]jð+lñ +lô +lð %) DØØJð%Lñ %Lô %Lð *.¨ØØOð*Qñ *Qô *Qð *.¨ØØOÐ\ið*kñ *kô *kðB&ð B&ð B&ñ Bô B€ðJ ðð ¸ð ð ð ñ „ðð ðð ¸ð ð ð ñ „ðð ðð ¸ð ð ð ñ „ðð ðð ¸ð ð ð ñ „ðð ðð ¸ð ð ð ñ „ðð ðð ¸ð ð ð ñ „ðð ðð ¸ð ð ð ñ „ðð ðð ¸ð ð ð ñ „ðð ðð ¸ð ð ð ñ „ðð ðð ¸ð ð ð ñ „ðð ðð ¸$ð ð ð ñ „ðð ðð ¸$ð ð ð ñ „ðð ðð ¸ð ð ð ñ „ðð ðð ¸ð ð ð ñ „ðð ðð °tð ð ð ñ „ðð ðð °tð ð ð ñ „ðð ðð °tð ð ð ñ „ðð ðð °tð ð ð ñ „ðð ðð ¸ð ð ð ñ „ðð ðð ¸ð ð ð ñ „ðð ðð °tð ð ð ñ „ðð ðð °tð ð ð ñ „ðð ðð ¸ð ð ð ñ „ðð ðð ¸ð ð ð ñ „ðð ðð ¸ð ð ð ñ „ðð ðð ¸ð ð ð ñ „ðð ðð ¸ð ð ð ñ „ðð ðð ¸ð ð ð ñ „ðð Ð ˜Hð  'àÐ!Dð 'ð  Ð!Dð 'ð  Ð!Dð	 'ð
  Ð!Dð 'ð  Ð!Dð 'ð  Ð!Dð 'ð !Ð"Fð 'ð !Ð"Fð 'ð !Ð"Fð 'ð !Ð"Fð 'ð "Ð#Hð 'ð "Ð#Hð 'ð !Ð"Fð 'ð !Ð"Fð 'ð$ Ð Bð% 'ð& Ð Bð' 'ð( Ð Bð) 'ð* !CØ BØ BØ!DØ!DØ!DØ!DØ"FØ"FØ!DØ!Dð? 'ð  'ð  'ñ  ô  ð  ð  ð  r9   