§
    ·¯¸ja|  ã                   ón  — d Z ddlmZmZmZmZmZmZ ddlZddl	m
Z
 ddlm
c mZ ddlmZmZ ddlmZmZmZmZmZmZmZmZ ddlmZ ddlmZmZ d	gZ G d
„ de
j         ¦  «        Z! G d„ de
j         ¦  «        Z" G d„ de
j         ¦  «        Z# G d„ de
j         ¦  «        Z$ G d„ de
j         ¦  «        Z% G d„ d	e
j         ¦  «        Z&d„ Z'd„ Z(d„ Z)d,d„Z*d-d„Z+ e e+d¬¦  «         e+d¬¦  «         e+d¬¦  «         e+d¬¦  «         e+d¬¦  «         e+d¬¦  «         e+d¬¦  «         e+dddd ¬!¦  «        d"œ¦  «        Z,ed.d#e&fd$„¦   «         Z-ed.d#e&fd%„¦   «         Z.ed.d#e&fd&„¦   «         Z/ed.d#e&fd'„¦   «         Z0ed.d#e&fd(„¦   «         Z1ed.d#e&fd)„¦   «         Z2ed.d#e&fd*„¦   «         Z3ed.d#e&fd+„¦   «         Z4dS )/z×
CoaT architecture.

Paper: Co-Scale Conv-Attentional Image Transformers - https://arxiv.org/abs/2104.06399

Official CoaT code at: https://github.com/mlpc-ucsd/CoaT

Modified from timm/models/vision_transformer.py
é    )ÚListÚOptionalÚTupleÚUnionÚTypeÚAnyN©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)Ú
PatchEmbedÚMlpÚDropPathÚ	to_2tupleÚtrunc_normal_Ú_assertÚ	LayerNormÚget_device_dtypeé   )Úbuild_model_with_cfg)Úregister_modelÚgenerate_default_cfgsÚCoaTc            	       ób   ‡ — e Zd ZdZ	 	 d	dededeeef         fˆ fd„Zdeeef         fd„Z	ˆ xZ
S )
ÚConvRelPosEncz+ Convolutional relative position encoding. NÚhead_chsÚ	num_headsÚwindowc           	      ój  •‡— ||dœ}t          ¦   «                              ¦   «          t          |t          ¦  «        r||i}|| _        n+t          |t
          ¦  «        r|| _        nt          ¦   «         ‚t          j        ¦   «         | _	        g | _
        |                     ¦   «         D ]s\  }}d}	||dz
  |	dz
  z  z   dz  }
t          j        |‰z  |‰z  f||f|
|
f|	|	f|‰z  dœ|¤Ž}| j	                             |¦  «         | j
                             |¦  «         Œtˆfd„| j
        D ¦   «         | _        dS )aj  
        Initialization.
            Ch: Channels per head.
            h: Number of heads.
            window: Window size(s) in convolutional relative positional encoding. It can have two forms:
                1. An integer of window size, which assigns all attention heads with the same window s
                    size in ConvRelPosEnc.
                2. A dict mapping window size to #attention head splits (
                    e.g. {window size 1: #attention head split 1, window size 2: #attention head split 2})
                    It will apply different window size to the attention head splits.
        ©ÚdeviceÚdtyper   é   )Úkernel_sizeÚpaddingÚdilationÚgroupsc                 ó   •— g | ]}|‰z  ‘ŒS © r(   )Ú.0Úxr   s     €ú`/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/models/coat.pyú
<listcomp>z*ConvRelPosEnc.__init__.<locals>.<listcomp>K   s   ø€ ÐFÐFÐF°˜q 8™|ÐFÐFÐFó    N)ÚsuperÚ__init__Ú
isinstanceÚintr   ÚdictÚ
ValueErrorÚnnÚ
ModuleListÚ	conv_listÚhead_splitsÚitemsÚConv2dÚappendÚchannel_splits)Úselfr   r   r   r    r!   ÚddÚ
cur_windowÚcur_head_splitr%   Úpadding_sizeÚcur_convÚ	__class__s    `          €r+   r/   zConvRelPosEnc.__init__   su  øø€ ð& ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐå�f�cÑ"Ô"ð 	à˜iÐ(ˆFØ ˆDŒKˆKÝ˜¥Ñ%Ô%ð 	Ø ˆDŒKˆKå‘,”,Ðåœ™œˆŒØˆÔØ*0¯,ª,©.¬.ð 	4ð 	4Ñ&ˆJ˜ØˆHð '¨*°q©.¸XÈ¹\Ñ)JÑJÈqÑPˆLÝ”yØ Ñ)Ø Ñ)ðð (¨Ð4Ø% |Ð4Ø" HÐ-Ø%¨Ñ0ðð ð ðð ˆHð ŒN×!Ò! (Ñ+Ô+Ð+ØÔ×#Ò# NÑ3Ô3Ð3Ð3ØFÐFÐFÐF°TÔ5EÐFÑFÔFˆÔÐÐr-   Úsizec                 ó€  — |j         \  }}}}|\  }}	t          |d||	z  z   k    d¦  «         |d d …d d …dd …d d …f         }
|d d …d d …dd …d d …f         }|                     dd¦  «                             |||z  ||	¦  «        }t	          j        || j        d¬¦  «        }g }t          | j        ¦  «        D ])\  }}| 	                     |||         ¦  «        ¦  «         Œ*t	          j
        |d¬¦  «        }|                     |||||	z  ¦  «                             dd¦  «        }|
|z  }t          j        |d¦  «        }|S )Nr   Ú éÿÿÿÿéþÿÿÿ©Údim)r   r   r   r   r   r   )Úshaper   Ú	transposeÚreshapeÚtorchÚsplitr;   Ú	enumerater6   r:   ÚcatÚFÚpad)r<   ÚqÚvrC   ÚBr   ÚNÚCÚHÚWÚq_imgÚv_imgÚ
v_img_listÚconv_v_img_listÚiÚconvÚ
conv_v_imgÚEV_hats                     r+   ÚforwardzConvRelPosEnc.forwardM   sh  € ØœWÑˆˆ9�a˜Ø‰ˆˆ1Ý��Q˜˜Q™‘Y’ Ñ#Ô#Ð#ð �!�!�!�Q�Q�Q˜˜˜˜A˜A˜A�+”ˆØ�!�!�!�Q�Q�Q˜˜˜˜A˜A˜A�+”ˆà—’  BÑ'Ô'×/Ò/°°9¸q±=À!ÀQÑGÔGˆÝ”[ ¨Ô(;ÀÐCÑCÔCˆ
ØˆÝ  ¤Ñ0Ô0ð 	8ð 	8‰GˆAˆtØ×"Ò" 4 4¨
°1¬Ñ#6Ô#6Ñ7Ô7Ð7Ð7Ý”Y˜°AÐ6Ñ6Ô6ˆ
Ø×'Ò'¨¨9°a¸¸Q¹Ñ?Ô?×IÒIÈ"ÈbÑQÔQˆ
à˜Ñ#ˆÝ”�vÐ1Ñ2Ô2ˆØˆr-   )NN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r1   r   r2   r/   r   rb   Ú__classcell__©rB   s   @r+   r   r      s£   ø€ € € € € Ø5Ð5ð Øð1Gð 1Gàð1Gð ð1Gð ˜#˜t˜)Ô$ð	1Gð 1Gð 1Gð 1Gð 1Gð 1Gðf %¨¨S¨¤/ð ð ð ð ð ð ð ð r-   r   c                   ót   ‡ — e Zd ZdZ	 	 	 	 	 	 	 ddededed	ed
edee         fˆ fd„Z	de
eef         fd„Zˆ xZS )ÚFactorAttnConvRelPosEnczK Factorized attention with convolutional relative position encoding class. é   Fç        NrI   r   Úqkv_biasÚ	attn_dropÚ	proj_dropÚshared_crpec	                 óZ  •— ||dœ}	t          ¦   «                              ¦   «          || _        ||z  }
|
dz  | _        t	          j        ||dz  fd|i|	¤Ž| _        t	          j        |¦  «        | _        t	          j        ||fi |	¤Ž| _	        t	          j        |¦  «        | _
        || _        d S )Nr   g      à¿é   Úbias)r.   r/   r   Úscaler4   ÚLinearÚqkvÚDropoutrn   Úprojro   Úcrpe)r<   rI   r   rm   rn   ro   rp   r    r!   r=   Úhead_dimrB   s              €r+   r/   z FactorAttnConvRelPosEnc.__init__e   s³   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ"ˆŒØ˜)Ñ#ˆØ Ñ%ˆŒ
å”9˜S #¨¡'Ð?Ð?°Ð?¸BÐ?Ð?ˆŒÝœ IÑ.Ô.ˆŒÝ”I˜c 3Ð-Ð-¨"Ð-Ð-ˆŒ	Ýœ IÑ.Ô.ˆŒð  ˆŒ	ˆ	ˆ	r-   rC   c                 óJ  — |j         \  }}}|                      |¦  «                             ||d| j        || j        z  ¦  «                             ddddd¦  «        }|                     d¦  «        \  }}}	|                     d¬¦  «        }
|
                     dd¦  «        |	z  }||z  }|                      ||	|¬	¦  «        }| j	        |z  |z   }|                     dd¦  «                             |||¦  «        }|  
                    |¦  «        }|                      |¦  «        }|S )
Nrr   r"   r   r   é   rH   rF   rG   ©rC   )rJ   rv   rL   r   ÚpermuteÚunbindÚsoftmaxrK   ry   rt   rx   ro   )r<   r*   rC   rU   rV   rW   rv   rS   ÚkrT   Ú	k_softmaxÚ
factor_attry   s                r+   rb   zFactorAttnConvRelPosEnc.forward~   s  € Ø”'‰ˆˆ1ˆað �hŠh�q‰kŒk×!Ò! ! Q¨¨4¬>¸1ÀÄÑ;NÑOÔO×WÒWÐXYÐ[\Ð^_ÐabÐdeÑfÔfˆØ—*’*˜Q‘-”-‰ˆˆ1ˆað —I’I !�IÑ$Ô$ˆ	Ø×(Ò(¨¨RÑ0Ô0°1Ñ4ˆ
Ø˜‘^ˆ
ð �yŠy˜˜A DˆyÑ)Ô)ˆð ŒJ˜Ñ# dÑ*ˆØ�KŠK˜˜1ÑÔ×%Ò% a¨¨AÑ.Ô.ˆð �IŠI�a‰LŒLˆØ�NŠN˜1ÑÔˆàˆr-   )rk   Frl   rl   NNN)rc   rd   re   rf   r1   ÚboolÚfloatr   r   r/   r   rb   rg   rh   s   @r+   rj   rj   c   sÀ   ø€ € € € € ØUÐUð Ø"Ø!Ø!Ø)-ØØð ð  àð ð ð ð ð	 ð
 ð ð ð ð " #œð ð  ð  ð  ð  ð  ð2˜u S¨# Xœð ð ð ð ð ð ð ð r-   rj   c                   óP   ‡ — e Zd ZdZ	 	 	 d	dedefˆ fd„Zdeeef         fd„Zˆ xZS )
Ú
ConvPosEnczy Convolutional Position Encoding.
        Note: This module is similar to the conditional position encoding in CPVT.
    rr   NrI   r�   c                 ó’   •— ||dœ}t          ¦   «                              ¦   «          t          j        |||d|dz  fd|i|¤Ž| _        d S )Nr   r   r"   r&   )r.   r/   r4   r9   rx   )r<   rI   r�   r    r!   r=   rB   s         €r+   r/   zConvPosEnc.__init__œ   sV   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ”I˜c 3¨¨1¨a°©dÐEÐE¸3ÐEÀ"ÐEÐEˆŒ	ˆ	ˆ	r-   rC   c                 óœ  — |j         \  }}}|\  }}t          |d||z  z   k    d¦  «         |d d …d d…f         |d d …dd …f         }	}|	                     dd¦  «                             ||||¦  «        }
|                      |
¦  «        |
z   }|                     d¦  «                             dd¦  «        }t          j        ||fd¬¦  «        }|S )Nr   rE   r"   rH   )rJ   r   rK   Úviewrx   ÚflattenrM   rP   )r<   r*   rC   rU   rV   rW   rX   rY   Ú	cls_tokenÚ
img_tokensÚfeats              r+   rb   zConvPosEnc.forward§   sÜ   € Ø”'‰ˆˆ1ˆaØ‰ˆˆ1Ý��Q˜˜Q™‘Y’ Ñ#Ô#Ð#ð !" ! ! ! R a R %¤¨!¨A¨A¨A¨q¨r¨r¨E¬(�:ˆ	ð ×#Ò# A qÑ)Ô)×.Ò.¨q°!°Q¸Ñ:Ô:ˆØ�IŠI�d‰OŒO˜dÑ"ˆØ�IŠI�a‰LŒL×"Ò" 1 aÑ(Ô(ˆõ ŒI�y !�n¨!Ð,Ñ,Ô,ˆàˆr-   )rr   NN)	rc   rd   re   rf   r1   r/   r   rb   rg   rh   s   @r+   r‡   r‡   ˜   s•   ø€ € € € € ðð ð ØØð	Fð 	Fàð	Fð ð	Fð 	Fð 	Fð 	Fð 	Fð 	Fð˜u S¨# Xœð ð ð ð ð ð ð ð r-   r‡   c                   óÜ   ‡ — e Zd ZdZdddddej        ej        ddddfdededed	e	d
ededede
ej                 de
ej                 dee         dee         fˆ fd„Zdeeef         fd„Zˆ xZS )ÚSerialBlockz„ Serial block class.
        Note: In this implementation, each serial block only contains a conv-attention and a FFN (MLP) module. ç      @Frl   NrI   r   Ú	mlp_ratiorm   ro   rn   Ú	drop_pathÚ	act_layerÚ
norm_layerÚ
shared_cperp   c           	      ór  •— ||dœ}t          ¦   «                              ¦   «          |
| _         |	|fi |¤Ž| _        t	          |f|||||dœ|¤Ž| _        |dk    rt          |¦  «        nt          j        ¦   «         | _	         |	|fi |¤Ž| _
        t          ||z  ¦  «        }t          d||||dœ|¤Ž| _        d S )Nr   ©r   rm   rn   ro   rp   rl   ©Úin_featuresÚhidden_featuresr”   Údropr(   )r.   r/   ÚcpeÚnorm1rj   Úfactoratt_crper   r4   ÚIdentityr“   Únorm2r1   r   Úmlp)r<   rI   r   r’   rm   ro   rn   r“   r”   r•   r–   rp   r    r!   r=   Úmlp_hidden_dimrB   s                   €r+   r/   zSerialBlock.__init__½   s  ø€ ð  ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐð ˆŒà�Z Ð*Ð* rÐ*Ð*ˆŒ
Ý5Øð
àØØØØ#ð
ð 
ð ð
ð 
ˆÔð 1:¸B²°� )Ñ,Ô,Ð,ÅBÄKÁMÄMˆŒð  �Z Ð*Ð* rÐ*Ð*ˆŒ
Ý˜S 9™_Ñ-Ô-ˆÝð 
ØØ*ØØð	
ð 
ð
 ð
ð 
ˆŒˆˆr-   rC   c                 ó<  — |                       ||¦  «        }|                      |¦  «        }|                      ||¦  «        }||                      |¦  «        z   }|                      |¦  «        }|                      |¦  «        }||                      |¦  «        z   }|S ©N)r�   rž   rŸ   r“   r¡   r¢   )r<   r*   rC   Úcurs       r+   rb   zSerialBlock.forwardê   sŠ   € à�HŠH�Q˜ÑÔˆØ�jŠj˜‰mŒmˆØ×!Ò! # tÑ,Ô,ˆØ�—’˜sÑ#Ô#Ñ#ˆð �jŠj˜‰mŒmˆØ�hŠh�s‰mŒmˆØ�—’˜sÑ#Ô#Ñ#ˆàˆr-   )rc   rd   re   rf   r4   ÚGELUr   r1   r…   r„   r   ÚModuler   r   r/   r   rb   rg   rh   s   @r+   r�   r�   º   s  ø€ € € € € ðsð sð  "Ø"Ø!Ø!Ø!Ø)+¬Ø*,¬,Ø(,Ø)-ØØð+
ð +
àð+
ð ð+
ð ð	+
ð
 ð+
ð ð+
ð ð+
ð ð+
ð ˜BœI”ð+
ð ˜RœYœð+
ð ! œð+
ð " #œð+
ð +
ð +
ð +
ð +
ð +
ðZ˜u S¨# Xœð ð ð ð ð ð ð ð r-   r�   c                   óZ  ‡ — e Zd ZdZdddddej        ej        dddf
dee         dedee	         de
d	e	d
e	de	deej                 deej                 deee                  fˆ fd„Zde	deeef         fd„Zde	deeef         fd„Zde	deeef         fd„Zdeeeef                  fd„Zˆ xZS )ÚParallelBlockz Parallel block class. NFrl   Údimsr   Ú
mlp_ratiosrm   ro   rn   r“   r”   r•   Úshared_crpesc           	      ó˜  •— ||dœ}t          ¦   «                              ¦   «          |€g } |	|d         fi |¤Ž| _         |	|d         fi |¤Ž| _         |	|d         fi |¤Ž| _        t          |d         f|||||
d         dœ|¤Ž| _        t          |d         f|||||
d         dœ|¤Ž| _        t          |d         f|||||
d         dœ|¤Ž| _        |dk    rt          |¦  «        nt          j        ¦   «         | _         |	|d         fi |¤Ž| _         |	|d         fi |¤Ž| _         |	|d         fi |¤Ž| _        |d         |d         cxk    r|d         k    sn J ‚|d         |d         cxk    r|d         k    sn J ‚t!          |d         |d         z  ¦  «        }t#          d|d         |||dœ|¤Žx| _        x| _        | _        d S )	Nr   r   r"   rr   r˜   rl   r™   r(   )r.   r/   Únorm12Únorm13Únorm14rj   Úfactoratt_crpe2Úfactoratt_crpe3Úfactoratt_crpe4r   r4   r    r“   Únorm22Únorm23Únorm24r1   r   Úmlp2Úmlp3Úmlp4)r<   r«   r   r¬   rm   ro   rn   r“   r”   r•   r­   r    r!   r=   r£   rB   s                  €r+   r/   zParallelBlock.__init__û   s”  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØÐØˆJð !�j  a¤Ð/Ð/¨BÐ/Ð/ˆŒØ �j  a¤Ð/Ð/¨BÐ/Ð/ˆŒØ �j  a¤Ð/Ð/¨BÐ/Ð/ˆŒÝ6Ø�ŒGð 
àØØØØ$ Qœð 
ð  
ð ð 
ð  
ˆÔõ  7Ø�ŒGð 
àØØØØ$ Qœð 
ð  
ð ð 
ð  
ˆÔõ  7Ø�ŒGð 
àØØØØ$ Qœð 
ð  
ð ð 
ð  
ˆÔð 1:¸B²°� )Ñ,Ô,Ð,ÅBÄKÁMÄMˆŒð !�j  a¤Ð/Ð/¨BÐ/Ð/ˆŒØ �j  a¤Ð/Ð/¨BÐ/Ð/ˆŒØ �j  a¤Ð/Ð/¨BÐ/Ð/ˆŒà�AŒw˜$˜qœ'Ð,Ð,Ò,Ð, T¨!¤WÒ,Ð,Ð,Ð,Ð,Ð,Ø˜!Œ} 
¨1¤Ð>Ð>Ò>Ð>°¸A´Ò>Ð>Ð>Ð>Ð>Ð>Ý˜T !œW z°!¤}Ñ4Ñ5Ô5ˆÝ,/ð -
Ø˜QœØ*ØØð	-
ð -
ð
 ð-
ð -
ð 	
ˆŒ	ð 	
�D”I ¤	 	 	r-   ÚfactorrC   c                 ó2   — |                       |||¬¦  «        S )z Feature map up-sampling. ©Úscale_factorrC   ©Úinterpolate©r<   r*   r»   rC   s       r+   ÚupsamplezParallelBlock.upsample@  s   € à×Ò °¸TÐÑBÔBÐBr-   c                 ó8   — |                       |d|z  |¬¦  «        S )z Feature map down-sampling. ç      ð?r½   r¿   rÁ   s       r+   Ú
downsamplezParallelBlock.downsampleD  s!   € à×Ò °°F±
ÀÐÑFÔFÐFr-   r¾   c                 ó®  — |j         \  }}}|\  }}t          |d||z  z   k    d¦  «         |dd…dd…dd…f         }	|dd…dd…dd…f         }
|
                     dd¦  «                             ||||¦  «        }
t	          j        |
|ddd¬¦  «        }
|
                     ||d¦  «                             dd¦  «        }
t          j        |	|
fd¬	¦  «        }|S )
z Feature map interpolation. r   rE   Nr"   FÚbilinear)r¾   Úrecompute_scale_factorÚmodeÚalign_cornersrF   rH   )rJ   r   rK   rL   rQ   rÀ   rM   rP   )r<   r*   r¾   rC   rU   rV   rW   rX   rY   rŒ   r�   Úouts               r+   rÀ   zParallelBlock.interpolateH  sÿ   € à”'‰ˆˆ1ˆaØ‰ˆˆ1Ý��Q˜˜Q™‘Y’ Ñ#Ô#Ð#à�a�a�a˜˜!˜˜Q˜Q˜Q�h”Kˆ	Ø�q�q�q˜!˜"˜"˜a˜a˜a�x”[ˆ
à×)Ò)¨!¨QÑ/Ô/×7Ò7¸¸1¸aÀÑCÔCˆ
Ý”]ØØ%Ø#(ØØð
ñ 
ô 
ˆ
ð  ×'Ò'¨¨1¨bÑ1Ô1×;Ò;¸A¸qÑAÔAˆ
åŒi˜ JÐ/°QÐ7Ñ7Ô7ˆàˆ
r-   Úsizesc                 ó�  — |\  }}}}	|                       |¦  «        }
|                      |¦  «        }|                      |¦  «        }|                      |
|¬¦  «        }
|                      ||¬¦  «        }|                      ||	¬¦  «        }|                      |d|¬¦  «        }|                      |d|	¬¦  «        }|                      |d|	¬¦  «        }|                      |
d|¬¦  «        }|                      |d|¬¦  «        }|                      |
d|¬¦  «        }|
|z   |z   }
||z   |z   }||z   |z   }||                      |
¦  «        z   }||                      |¦  «        z   }||                      |¦  «        z   }|  	                    |¦  «        }
|  
                    |¦  «        }|                      |¦  «        }|                      |
¦  «        }
|                      |¦  «        }|                      |¦  «        }||                      |
¦  «        z   }||                      |¦  «        z   }||                      |¦  «        z   }||||fS )Nr}   g       @)r»   rC   r‘   )r¯   r°   r±   r²   r³   r´   rÂ   rÅ   r“   rµ   r¶   r·   r¸   r¹   rº   )r<   Úx1Úx2Úx3Úx4rÌ   Ú_ÚS2ÚS3ÚS4Úcur2Úcur3Úcur4Úupsample3_2Úupsample4_3Úupsample4_2Údownsample2_3Údownsample3_4Údownsample2_4s                      r+   rb   zParallelBlock.forward_  s)  € Ø‰ˆˆ2ˆr�2Ø�{Š{˜2‰ŒˆØ�{Š{˜2‰ŒˆØ�{Š{˜2‰ŒˆØ×#Ò# D¨rÐ#Ñ2Ô2ˆØ×#Ò# D¨rÐ#Ñ2Ô2ˆØ×#Ò# D¨rÐ#Ñ2Ô2ˆØ—m’m D°¸"�mÑ=Ô=ˆØ—m’m D°¸"�mÑ=Ô=ˆØ—m’m D°¸"�mÑ=Ô=ˆØŸš¨°R¸b˜ÑAÔAˆØŸš¨°R¸b˜ÑAÔAˆØŸš¨°R¸b˜ÑAÔAˆØ�kÑ! KÑ/ˆØ�kÑ! MÑ1ˆØ�mÑ# mÑ3ˆØ�$—.’. Ñ&Ô&Ñ&ˆØ�$—.’. Ñ&Ô&Ñ&ˆØ�$—.’. Ñ&Ô&Ñ&ˆð �{Š{˜2‰ŒˆØ�{Š{˜2‰ŒˆØ�{Š{˜2‰ŒˆØ�yŠy˜‰ŒˆØ�yŠy˜‰ŒˆØ�yŠy˜‰ŒˆØ�$—.’. Ñ&Ô&Ñ&ˆØ�$—.’. Ñ&Ô&Ñ&ˆØ�$—.’. Ñ&Ô&Ñ&ˆà�2�r˜2ˆ~Ðr-   )rc   rd   re   rf   r4   r§   r   r   r1   r…   r„   r   r¨   r   r   r/   r   rÂ   rÅ   rÀ   rb   rg   rh   s   @r+   rª   rª   ù   s¯  ø€ € € € € Ø!Ð!ð
 '+Ø"Ø!Ø!Ø!Ø)+¬Ø*,¬,Ø04ØØðC
ð C
à�s”)ðC
ð ðC
ð ˜Uœð	C
ð
 ðC
ð ðC
ð ðC
ð ðC
ð ˜BœI”ðC
ð ˜RœYœðC
ð # 4¨¤9Ô-ðC
ð C
ð C
ð C
ð C
ð C
ðJC %ð C¨u°S¸#°X¬ð Cð Cð Cð CðG Eð G°°s¸C°x´ð Gð Gð Gð Gð¨5ð ¸¸cÀ3¸h¼ð ð ð ð ð. ¨T°%¸¸S¸´/Ô-Bð  ð  ð  ð  ð  ð  ð  ð  r-   rª   c            '       óP  ‡ — e Zd ZdZdddddddd	d
dddddeddddddfdededededeeeeef         deeeeef         dededeeeeef         dededededede	e
j                 ded eee                  d!ee         d"ef&ˆ fd#„Zd$„ Zej        j        d%„ ¦   «         Zej        j        d0d&„¦   «         Zej        j        d1d'„¦   «         Zej        j        d(e
j        fd)„¦   «         Zd2ded"ee         fd*„Zd+„ Zd1d,eej        eej                 f         d-efd.„Zd(ej        fd/„Zˆ xZS )3r   z CoaT class. éà   é   rr   éè  ©é@   é€   é@  é   ©rr   r|   é   rr   r   rk   )r|   r|   r|   r|   Trl   FNÚtokenÚimg_sizeÚ
patch_sizeÚin_chansÚnum_classesÚ
embed_dimsÚserial_depthsÚparallel_depthr   r¬   rm   Ú	drop_rateÚproj_drop_rateÚattn_drop_rateÚdrop_path_rater•   Úreturn_interm_layersÚout_featuresÚcrpe_windowÚglobal_poolc           	      ó  •‡ ‡‡	‡‡— t          ¦   «                              ¦   «          ||dœŠ|dv sJ ‚|pddddœ}|‰ _        |‰ _        ‰‰ _        ‰d         x‰ _        ‰ _        |‰ _        |‰ _        |‰ _	        t          |¦  «        }t          d|||‰d         t          j        dœ‰¤Ž‰ _        t          dd	„ |D ¦   «         d‰d         ‰d
         t          j        dœ‰¤Ž‰ _        t          dd„ |D ¦   «         d‰d
         ‰d         t          j        dœ‰¤Ž‰ _        t          dd„ |D ¦   «         d‰d         ‰d         t          j        dœ‰¤Ž‰ _        t          j        t'          j        d
d
‰d         fi ‰¤Ž¦  «        ‰ _        t          j        t'          j        d
d
‰d
         fi ‰¤Ž¦  «        ‰ _        t          j        t'          j        d
d
‰d         fi ‰¤Ž¦  «        ‰ _        t          j        t'          j        d
d
‰d         fi ‰¤Ž¦  «        ‰ _        t3          d‰d         ddœ‰¤Ž‰ _        t3          d‰d
         ddœ‰¤Ž‰ _        t3          d‰d         ddœ‰¤Ž‰ _        t3          d‰d         ddœ‰¤Ž‰ _        t=          d‰d         |z  ||dœ‰¤Ž‰ _        t=          d‰d
         |z  ||dœ‰¤Ž‰ _         t=          d‰d         |z  ||dœ‰¤Ž‰ _!        t=          d‰d         |z  ||dœ‰¤Ž‰ _"        |}tG          ||
||||¬¦  «        Št          j$        ˆˆˆ	ˆ ˆfd„tK          |d         ¦  «        D ¦   «         ¦  «        ‰ _&        t          j$        ˆˆˆ	ˆ ˆfd„tK          |d
         ¦  «        D ¦   «         ¦  «        ‰ _'        t          j$        ˆˆˆ	ˆ ˆfd„tK          |d         ¦  «        D ¦   «         ¦  «        ‰ _(        t          j$        ˆˆˆ	ˆ ˆfd„tK          |d         ¦  «        D ¦   «         ¦  «        ‰ _)        |‰ _*        ‰ j*        dk    r7t          j$        ˆˆˆ	ˆ ˆfd„tK          |¦  «        D ¦   «         ¦  «        ‰ _+        nd ‰ _+        ‰ j        �sE‰ j+        �) |‰d
         fi ‰¤Ž‰ _,         |‰d         fi ‰¤Ž‰ _-        nd x‰ _,        ‰ _-         |‰d         fi ‰¤Ž‰ _.        ‰ j*        dk    r’‰d
         ‰d         cxk    r‰d         k    sn J ‚t'          j        j/        ddd
d
dœ‰¤Ž‰ _0        t          j1        |¦  «        ‰ _2        |dk    rt          j3        ‰ j        |fi ‰¤Žnt          j4        ¦   «         ‰ _5        nVd ‰ _0        t          j1        |¦  «        ‰ _2        |dk    rt          j3        ‰ j        |fi ‰¤Žnt          j4        ¦   «         ‰ _5        tm          ‰ j        d¬¦  «         tm          ‰ j        d¬¦  «         tm          ‰ j        d¬¦  «         tm          ‰ j        d¬¦  «         ‰  7                    ‰ j8        ¦  «         d S )Nr   ©rê   Úavgr"   rr   )rr   é   é   rF   r   )rë   rì   rí   Ú	embed_dimr•   c                 ó   — g | ]}|d z  ‘ŒS )r|   r(   ©r)   r*   s     r+   r,   z!CoaT.__init__.<locals>.<listcomp>®  ó   € Ð/Ð/Ð/ �a˜1‘fÐ/Ð/Ð/r-   r   c                 ó   — g | ]}|d z  ‘ŒS )rk   r(   r  s     r+   r,   z!CoaT.__init__.<locals>.<listcomp>±  r  r-   c                 ó   — g | ]}|d z  ‘ŒS )rá   r(   r  s     r+   r,   z!CoaT.__init__.<locals>.<listcomp>´  s   € Ð0Ð0Ð0 !�a˜2‘gÐ0Ð0Ð0r-   )rI   r�   )r   r   r   )r   rm   ro   rn   r“   r•   c           
      ób   •— g | ]+}t          d‰d          ‰d          ‰j        ‰j        dœ‰¤‰¤Ž‘Œ,S )r   ©rI   r’   r–   rp   r(   )r�   Úcpe1Úcrpe1©r)   rÒ   r=   rï   r¬   r<   Úskwargss     €€€€€r+   r,   z!CoaT.__init__.<locals>.<listcomp>Ô  ók   ø€ ð 	-.ð 	-.ð 	-.ð õ ð Ø˜q”MØ$ Qœ-Øœ9Ø œJð	ð ð
 ðð ðð ð	-.ð 	-.ð 	-.r-   c           
      ób   •— g | ]+}t          d‰d          ‰d          ‰j        ‰j        dœ‰¤‰¤Ž‘Œ,S )r   r  r(   )r�   Úcpe2Úcrpe2r	  s     €€€€€r+   r,   z!CoaT.__init__.<locals>.<listcomp>á  r  r-   c           
      ób   •— g | ]+}t          d‰d          ‰d          ‰j        ‰j        dœ‰¤‰¤Ž‘Œ,S )r"   r  r(   )r�   Úcpe3Úcrpe3r	  s     €€€€€r+   r,   z!CoaT.__init__.<locals>.<listcomp>î  r  r-   c           
      ób   •— g | ]+}t          d‰d          ‰d          ‰j        ‰j        dœ‰¤‰¤Ž‘Œ,S )rr   r  r(   )r�   Úcpe4Úcrpe4r	  s     €€€€€r+   r,   z!CoaT.__init__.<locals>.<listcomp>û  r  r-   c                 ód   •— g | ],}t          d‰‰‰j        ‰j        ‰j        ‰j        fd œ‰¤‰¤Ž‘Œ-S ))r«   r¬   r­   r(   )rª   r  r  r  r  r	  s     €€€€€r+   r,   z!CoaT.__init__.<locals>.<listcomp>
  sm   ø€ ð 20ð 20ð 20ð õ ð Ø#Ø)Ø"&¤*¨d¬j¸$¼*ÀdÄjÐ!Qðð ð ð	ð
 ðð ð20ð 20ð 20r-   )Úin_channelsÚout_channelsr#   ç{®Gáz”?©Ústdr(   )9r.   r/   rö   r÷   rï   Únum_featuresÚhead_hidden_sizerî   rí   rù   r   r   r4   r   Úpatch_embed1Úpatch_embed2Úpatch_embed3Úpatch_embed4Ú	ParameterrM   ÚzerosÚ
cls_token1Ú
cls_token2Ú
cls_token3Ú
cls_token4r‡   r  r  r  r  r   r  r  r  r  r2   r5   ÚrangeÚserial_blocks1Úserial_blocks2Úserial_blocks3Úserial_blocks4rñ   Úparallel_blocksr¡   Únorm3Únorm4ÚConv1dÚ	aggregaterw   Ú	head_dropru   r    Úheadr   ÚapplyÚ_init_weights)r<   rë   rì   rí   rî   rï   rð   rñ   r   r¬   rm   rò   ró   rô   rõ   r•   rö   r÷   rø   rù   r    r!   Údprr=   r
  rB   s   `    `   `             @@€r+   r/   zCoaT.__init__„  s¸  øøøøøø€ õ0 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆØÐ.Ð.Ð.Ð.Ð.Ø!Ð7¨¨q°QÐ%7Ð%7ˆØ$8ˆÔ!Ø(ˆÔØ$ˆŒØ4>¸r´NÐBˆÔ˜DÔ1Ø&ˆÔØ ˆŒØ&ˆÔõ ˜XÑ&Ô&ˆÝ&ð DØ¨*¸xØ  ”mµ´ðDð Dà@BðDð DˆÔõ 'ð DØ/Ð/ hÐ/Ñ/Ô/¸AÈ
ÐSTÌØ  ”mµ´ðDð Dà@BðDð DˆÔõ 'ð DØ/Ð/ hÐ/Ñ/Ô/¸AÈ
ÐSTÌØ  ”mµ´ðDð Dà@BðDð DˆÔõ 'ð DØ0Ð0 xÐ0Ñ0Ô0¸QÈÐTUÌØ  ”mµ´ðDð Dà@BðDð DˆÔõ
 œ,¥u¤{°1°a¸ÀA¼Ð'MÐ'MÈ"Ð'MÐ'MÑNÔNˆŒÝœ,¥u¤{°1°a¸ÀA¼Ð'MÐ'MÈ"Ð'MÐ'MÑNÔNˆŒÝœ,¥u¤{°1°a¸ÀA¼Ð'MÐ'MÈ"Ð'MÐ'MÑNÔNˆŒÝœ,¥u¤{°1°a¸ÀA¼Ð'MÐ'MÈ"Ð'MÐ'MÑNÔNˆŒõ Ð< :¨a¤=°AÐ<Ð<¸Ð<Ð<ˆŒ	ÝÐ< :¨a¤=°AÐ<Ð<¸Ð<Ð<ˆŒ	ÝÐ< :¨a¤=°AÐ<Ð<¸Ð<Ð<ˆŒ	ÝÐ< :¨a¤=°AÐ<Ð<¸Ð<Ð<ˆŒ	õ #Ðv¨J°q¬M¸YÑ,FÐR[ÐdoÐvÐvÐsuÐvÐvˆŒ
Ý"Ðv¨J°q¬M¸YÑ,FÐR[ÐdoÐvÐvÐsuÐvÐvˆŒ
Ý"Ðv¨J°q¬M¸YÑ,FÐR[ÐdoÐvÐvÐsuÐvÐvˆŒ
Ý"Ðv¨J°q¬M¸YÑ,FÐR[ÐdoÐvÐvÐsuÐvÐvˆŒ
àˆÝØØØ$Ø$ØØ!ð
ñ 
ô 
ˆõ !œmð 	-.ð 	-.ð 	-.ð 	-.ð 	-.ð 	-.ð 	-.ð 	-.õ ˜=¨Ô+Ñ,Ô,ð	-.ñ 	-.ô 	-.ñ 

ô 

ˆÔõ !œmð 	-.ð 	-.ð 	-.ð 	-.ð 	-.ð 	-.ð 	-.ð 	-.õ ˜=¨Ô+Ñ,Ô,ð	-.ñ 	-.ô 	-.ñ 

ô 

ˆÔõ !œmð 	-.ð 	-.ð 	-.ð 	-.ð 	-.ð 	-.ð 	-.ð 	-.õ ˜=¨Ô+Ñ,Ô,ð	-.ñ 	-.ô 	-.ñ 

ô 

ˆÔõ !œmð 	-.ð 	-.ð 	-.ð 	-.ð 	-.ð 	-.ð 	-.ð 	-.õ ˜=¨Ô+Ñ,Ô,ð	-.ñ 	-.ô 	-.ñ 

ô 

ˆÔð -ˆÔØÔ Ò"Ð"Ý#%¤=ð 20ð 20ð 20ð 20ð 20ð 20ð 20ð 20õ ˜~Ñ.Ô.ð20ñ 20ô 20ñ 	$ô 	$ˆDÔ Ð ð $(ˆDÔ ð Ô(ñ 	rØÔ#Ð/Ø'˜Z¨
°1¬Ð<Ð<¸Ð<Ð<�”
Ø'˜Z¨
°1¬Ð<Ð<¸Ð<Ð<�”
�
à*.Ð.�”
˜TœZØ#˜ J¨q¤MÐ8Ð8°RÐ8Ð8ˆDŒJàÔ" QÒ&Ð&à! !”}¨
°1¬ÐFÐFÒFÐF¸ÀA¼ÒFÐFÐFÐFÐFÐFÝ!&¤¤Ð!d¸QÈQÐ\]Ð!dÐ!dÐacÐ!dÐ!d�”Ý!#¤¨IÑ!6Ô!6�”ØOZÐ]^ÊÈ�BœI dÔ&7¸ÐKÐKÈÐKÐKÐKÕdfÔdoÑdqÔdq�”	�	ð "&�”Ý!#¤¨IÑ!6Ô!6�”ØOZÐ]^ÊÈ�BœI dÔ&7¸ÐKÐKÈÐKÐKÐKÕdfÔdoÑdqÔdq�”	õ 	�d”o¨3Ð/Ñ/Ô/Ð/Ý�d”o¨3Ð/Ñ/Ô/Ð/Ý�d”o¨3Ð/Ñ/Ô/Ð/Ý�d”o¨3Ð/Ñ/Ô/Ð/Ø�
Š
�4Ô%Ñ&Ô&Ð&Ð&Ð&r-   c                 óÊ  — t          |t          j        ¦  «        rbt          |j        d¬¦  «         t          |t          j        ¦  «        r.|j        �)t          j                             |j        d¦  «         d S d S d S t          |t          j        ¦  «        rLt          j                             |j        d¦  «         t          j                             |j        d¦  «         d S d S )Nr  r  r   rÄ   )	r0   r4   ru   r   Úweightrs   ÚinitÚ	constant_r   )r<   Úms     r+   r4  zCoaT._init_weights3  sÎ   € Ý�a�œÑ#Ô#ð 	-Ý˜!œ(¨Ð,Ñ,Ô,Ð,Ý˜!�RœYÑ'Ô'ð -¨A¬FÐ,>Ý”×!Ò! !¤&¨!Ñ,Ô,Ð,Ð,Ð,ð-ð -Ð,>Ð,>å˜�2œ<Ñ(Ô(ð 	-ÝŒG×Ò˜aœf aÑ(Ô(Ð(ÝŒG×Ò˜aœh¨Ñ,Ô,Ð,Ð,Ð,ð	-ð 	-r-   c                 ó
   — h d£S )N>   r#  r$  r%  r&  r(   ©r<   s    r+   Úno_weight_decayzCoaT.no_weight_decay<  s   € àGÐGÐGÐGr-   c                 ó   — |r
J d¦   «         ‚d S )Nz$gradient checkpointing not supportedr(   )r<   Úenables     r+   Úset_grad_checkpointingzCoaT.set_grad_checkpointing@  s   € àÐAÐAÐAÑAÔAˆzÐAÐAr-   c                 ó:   — t          ddddddddd	d
g¬¦	  «	        }|S )Nz#^cls_token1|patch_embed1|crpe1|cpe1z^serial_blocks1\.(\d+)z#^cls_token2|patch_embed2|crpe2|cpe2z^serial_blocks2\.(\d+)z#^cls_token3|patch_embed3|crpe3|cpe3z^serial_blocks3\.(\d+)z#^cls_token4|patch_embed4|crpe4|cpe4z^serial_blocks4\.(\d+))z^parallel_blocks\.(\d+)N)z^norm|aggregate)iŸ† )	Ústem1r(  Ústem2r)  Ústem3r*  Ústem4r+  r,  )r2   )r<   ÚcoarseÚmatchers      r+   Úgroup_matcherzCoaT.group_matcherD  s@   € åØ8Ø4Ø8Ø4Ø8Ø4Ø8Ø4à2Ø.ðð
ñ 
ô 
ˆð ˆr-   Úreturnc                 ó   — | j         S r¥   )r2  r<  s    r+   Úget_classifierzCoaT.get_classifierV  s
   € àŒyÐr-   c                 óú   — t          | ¦  «        }|| _        |�|dv sJ ‚|| _        |dk    rt          j        | j        |fi |¤Žnt          j        ¦   «         | _        | j                             | j	        ¦  «         d S )Nrû   r   )
r   rî   rù   r4   ru   r  r    r2  ÚtrainÚtraining)r<   rî   rù   r=   s       r+   Úreset_classifierzCoaT.reset_classifierZ  s‰   € Ý˜dÑ#Ô#ˆØ&ˆÔØÐ"ØÐ"2Ð2Ð2Ð2Ð2Ø*ˆDÔØGRÐUVÂÀ•B”I˜dÔ/°ÐCÐCÀÐCÐCÐCÕ\^Ô\gÑ\iÔ\iˆŒ	ØŒ	�Š˜œÑ&Ô&Ð&Ð&Ð&r-   c                 ó®
  — |j         d         }|                      |¦  «        }| j        j        \  }}t          || j        ¦  «        }| j        D ]} ||||f¬¦  «        }Œt          |¦  «                             |||d¦  «                             dddd¦  «         	                    ¦   «         }|  
                    |¦  «        }| j
        j        \  }	}
t          || j        ¦  «        }| j        D ]} |||	|
f¬¦  «        }Œt          |¦  «                             ||	|
d¦  «                             dddd¦  «         	                    ¦   «         }|                      |¦  «        }| j        j        \  }}t          || j        ¦  «        }| j        D ]} ||||f¬¦  «        }Œt          |¦  «                             |||d¦  «                             dddd¦  «         	                    ¦   «         }|                      |¦  «        }| j        j        \  }}t          || j        ¦  «        }| j        D ]} ||||f¬¦  «        }Œt          |¦  «                             |||d¦  «                             dddd¦  «         	                    ¦   «         }| j        €xt(          j                             ¦   «         sC| j        r<i }d| j        v r||d<   d| j        v r||d<   d	| j        v r||d	<   d
| j        v r||d
<   |S |                      |¦  «        }|S | j        D ]k}|                      ||	|
f¦  «        |                      |||f¦  «        |                      |||f¦  «        }}} |||||||f|	|
f||f||fg¬¦  «        \  }}}}Œlt(          j                             ¦   «         �sx| j        �rpi }d| j        v rRt          |¦  «                             |||d¦  «                             dddd¦  «         	                    ¦   «         }||d<   d| j        v rRt          |¦  «                             ||	|
d¦  «                             dddd¦  «         	                    ¦   «         }||d<   d	| j        v rRt          |¦  «                             |||d¦  «                             dddd¦  «         	                    ¦   «         }||d	<   d
| j        v rRt          |¦  «                             |||d¦  «                             dddd¦  «         	                    ¦   «         }||d
<   |S |                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|||gS )Nr   r}   rF   rr   r   r"   Úx1_noclsÚx2_noclsÚx3_noclsÚx4_nocls)rÌ   )rJ   r  Ú	grid_sizeÚ
insert_clsr#  r(  Ú
remove_clsrL   r~   Ú
contiguousr  r$  r)  r  r%  r*  r   r&  r+  r,  rM   ÚjitÚis_scriptingrö   r÷   r.  r  r  r  r¡   r-  )r<   Úx0rU   rÎ   ÚH1ÚW1ÚblkrQ  rÏ   ÚH2ÚW2rR  rÐ   ÚH3ÚW3rS  rÑ   ÚH4ÚW4rT  Úfeat_outs                        r+   Úforward_featureszCoaT.forward_featuresc  s™  € ØŒH�QŒKˆð ×Ò˜rÑ"Ô"ˆØÔ"Ô,‰ˆˆBÝ˜˜DœOÑ,Ô,ˆØÔ&ð 	(ð 	(ˆCØ��R˜r 2˜hÐ'Ñ'Ô'ˆBˆBÝ˜b‘>”>×)Ò)¨!¨R°°RÑ8Ô8×@Ò@ÀÀAÀqÈ!ÑLÔL×WÒWÑYÔYˆð ×Ò˜xÑ(Ô(ˆØÔ"Ô,‰ˆˆBÝ˜˜DœOÑ,Ô,ˆØÔ&ð 	(ð 	(ˆCØ��R˜r 2˜hÐ'Ñ'Ô'ˆBˆBÝ˜b‘>”>×)Ò)¨!¨R°°RÑ8Ô8×@Ò@ÀÀAÀqÈ!ÑLÔL×WÒWÑYÔYˆð ×Ò˜xÑ(Ô(ˆØÔ"Ô,‰ˆˆBÝ˜˜DœOÑ,Ô,ˆØÔ&ð 	(ð 	(ˆCØ��R˜r 2˜hÐ'Ñ'Ô'ˆBˆBÝ˜b‘>”>×)Ò)¨!¨R°°RÑ8Ô8×@Ò@ÀÀAÀqÈ!ÑLÔL×WÒWÑYÔYˆð ×Ò˜xÑ(Ô(ˆØÔ"Ô,‰ˆˆBÝ˜˜DœOÑ,Ô,ˆØÔ&ð 	(ð 	(ˆCØ��R˜r 2˜hÐ'Ñ'Ô'ˆBˆBÝ˜b‘>”>×)Ò)¨!¨R°°RÑ8Ô8×@Ò@ÀÀAÀqÈ!ÑLÔL×WÒWÑYÔYˆð ÔÐ'Ý”9×)Ò)Ñ+Ô+ð °Ô0Ið à�Ø Ô!2Ð2Ð2Ø+3�H˜ZÑ(Ø Ô!2Ð2Ð2Ø+3�H˜ZÑ(Ø Ô!2Ð2Ð2Ø+3�H˜ZÑ(Ø Ô!2Ð2Ð2Ø+3�H˜ZÑ(Ø�ð —Z’Z ‘^”^�Ø�	ð Ô'ð 	að 	aˆCØŸš 2¨¨B xÑ0Ô0°$·)²)¸BÀÀRÀÑ2IÔ2IÈ4Ï9Ê9ÐUWÐZ\Ð^`ÐYaÑKbÔKb�B�ˆBØ ˜S  R¨¨R¸¸R¸À2ÀrÀ(ÈRÐQSÈHÐWYÐ[]ÐV^Ð7_Ð`Ñ`Ô`‰NˆB��B˜˜åŒy×%Ò%Ñ'Ô'ñ 	 ¨DÔ,Eñ 	 àˆHØ˜TÔ.Ð.Ð.Ý% b™>œ>×1Ò1°!°R¸¸RÑ@Ô@×HÒHÈÈAÈqÐRSÑTÔT×_Ò_ÑaÔa�Ø'/�˜Ñ$Ø˜TÔ.Ð.Ð.Ý% b™>œ>×1Ò1°!°R¸¸RÑ@Ô@×HÒHÈÈAÈqÐRSÑTÔT×_Ò_ÑaÔa�Ø'/�˜Ñ$Ø˜TÔ.Ð.Ð.Ý% b™>œ>×1Ò1°!°R¸¸RÑ@Ô@×HÒHÈÈAÈqÐRSÑTÔT×_Ò_ÑaÔa�Ø'/�˜Ñ$Ø˜TÔ.Ð.Ð.Ý% b™>œ>×1Ò1°!°R¸¸RÑ@Ô@×HÒHÈÈAÈqÐRSÑTÔT×_Ò_ÑaÔa�Ø'/�˜Ñ$ØˆOà—’˜B‘”ˆBØ—’˜B‘”ˆBØ—’˜B‘”ˆBØ˜˜B�<Ðr-   Úx_featÚ
pre_logitsc                 óø  — t          |t          ¦  «        r| j        €J ‚| j        dk    r!t	          j        d„ |D ¦   «         d¬¦  «        }n t	          j        d„ |D ¦   «         d¬¦  «        }|                      |¦  «                             d¬¦  «        }n9| j        dk    r"|d d …dd …f                              d¬¦  «        n|d d …df         }|  	                    |¦  «        }|r|n|  
                    |¦  «        S )Nrü   c                 óT   — g | ]%}|d d …dd …f                               dd¬¦  «        ‘Œ&S )Nr   T)rI   Úkeepdim)Úmean©r)   Úxls     r+   r,   z%CoaT.forward_head.<locals>.<listcomp>¸  s8   € ÐTÐTÐTÀr˜r ! ! ! Q R R %œyŸ~š~°!¸T˜~ÑBÔBÐTÐTÐTr-   r   rH   c                 ó&   — g | ]}|d d …df         ‘ŒS )Nr   r(   rm  s     r+   r,   z%CoaT.forward_head.<locals>.<listcomp>º  s$   € Ð ;Ð ;Ð ;¨b  A A A q D¤Ð ;Ð ;Ð ;r-   r   )r0   Úlistr0  rù   rM   rP   ÚstackÚsqueezerl  r1  r2  )r<   rg  rh  r*   s       r+   Úforward_headzCoaT.forward_head´  s  € Ý�f�dÑ#Ô#ð 	YØ”>Ð-Ð-Ð-ØÔ 5Ò(Ð(Ý”IÐTÐTÈVÐTÑTÔTÐZ[Ð\Ñ\Ô\��å”KÐ ;Ð ;°FÐ ;Ñ ;Ô ;ÀÐCÑCÔC�Ø—’˜qÑ!Ô!×)Ò)¨aÐ)Ñ0Ô0ˆAˆAà-1Ô-=ÀÒ-FÐ-F��q�q�q˜!˜"˜"�u”×"Ò" qÐ"Ñ)Ô)Ð)ÈFÐSTÐSTÐSTÐVWÐSWÌLˆAØ�NŠN˜1ÑÔˆØÐ0ˆqˆq D§I¢I¨a¡L¤LÐ0r-   c                 óÎ   — t           j                             ¦   «         s| j        r|                      |¦  «        S |                      |¦  «        }|                      |¦  «        }|S r¥   )rM   rY  rZ  rö   rf  rs  )r<   r*   rg  s      r+   rb   zCoaT.forwardÁ  sc   € ÝŒy×%Ò%Ñ'Ô'ð 	¨DÔ,Eð 	à×(Ò(¨Ñ+Ô+Ð+ð ×*Ò*¨1Ñ-Ô-ˆFØ×!Ò! &Ñ)Ô)ˆAØˆHr-   )T©Fr¥   ) rc   rd   re   rf   r   r1   r   r…   r„   r   r4   r¨   r   r   Ústrr2   r/   r4  rM   rY  Úignorer=  r@  rH  rK  rO  rf  r   ÚTensorrs  rb   rg   rh   s   @r+   r   r   ‚  så  ø€ € € € € ØÐð  Ø ØØ#Ø4GØ7CØ"#ØØ<HØ!Ø!Ø$&Ø$&Ø$&Ø*3Ø).Ø04Ø*.Ø&ØØð-m'ð m'àðm'ð ðm'ð ð	m'ð
 ðm'ð ˜c 3¨¨SÐ0Ô1ðm'ð !  c¨3°Ð!3Ô4ðm'ð  ðm'ð ðm'ð ˜e U¨E°5Ð8Ô9ðm'ð ðm'ð ðm'ð "ðm'ð "ðm'ð "ðm'ð  ˜RœYœð!m'ð" #'ð#m'ð$ # 4¨¤9Ô-ð%m'ð& " $œð'm'ð( ð)m'ð m'ð m'ð m'ð m'ð m'ð^-ð -ð -ð „YÔðHð Hñ ÔðHð „YÔðBð Bð Bñ ÔðBð „YÔðð ð ñ Ôðð" „YÔð ¤	ð ð ð ñ Ôðð'ð '¨Cð '¸hÀs¼mð 'ð 'ð 'ð 'ðO ð O ð O ðb1ð 1 5¨¬°t¸E¼LÔ7IÐ)IÔ#Jð 1ÐX\ð 1ð 1ð 1ð 1ð˜EœLð ð ð ð ð ð ð ð r-   c                 óz   — |                      | j        d         dd¦  «        }t          j        || fd¬¦  «        } | S )z Insert CLS token. r   rF   r   rH   )ÚexpandrJ   rM   rP   )r*   rŒ   Ú
cls_tokenss      r+   rV  rV  Ì  s<   € à×!Ò! !¤'¨!¤*¨b°"Ñ5Ô5€JÝŒ	�:˜q�/ qÐ)Ñ)Ô)€AØ€Hr-   c                 ó$   — | dd…dd…dd…f         S )z Remove CLS token. Nr   r(   )r*   s    r+   rW  rW  Ó  s   € àˆQˆQˆQ����A�A�AˆXŒ;Ðr-   c                 ó  — i }|                       d| ¦  «        } |                      ¦   «         D ]Þ\  }}|                     d¦  «        s¾|                     d¦  «        rt          |dd ¦  «        �˜|                     d¦  «        rt          |dd ¦  «        �r|                     d¦  «        rt          |dd ¦  «        �L|                     d¦  «        rt          |dd ¦  «        �&|                     d¦  «        rt          |dd ¦  «        €ŒÙ|||<   Œß|S )NÚmodelrž   r¡   r-  r.  r0  r2  )Úgetr8   Ú
startswithÚgetattr)Ú
state_dictr~  Úout_dictr�   rT   s        r+   Úcheckpoint_filter_fnr„  Ø  s"  € Ø€HØ—’ ¨Ñ4Ô4€JØ× Ò Ñ"Ô"ð 	ð 	‰ˆˆ1à�<Š<˜Ñ Ô ð 	Ø—’˜gÑ&Ô&ð	Ý+2°5¸'À4Ñ+HÔ+HÐ+PØ—’˜gÑ&Ô&ð ,QÝ+2°5¸'À4Ñ+HÔ+HÐ+PØ—’˜gÑ&Ô&ð ,QÝ+2°5¸'À4Ñ+HÔ+HÐ+PØ—’˜kÑ*Ô*ð ,QÝ/6°u¸kÈ4Ñ/PÔ/PÐ/XØ—’˜fÑ%Ô%ð 0YÝ*1°%¸ÀÑ*FÔ*FÐ*NØØˆ�‰ˆØ€Or-   Fc                 ó†   — |                      dd ¦  «        rt          d¦  «        ‚t          t          | |fdt          i|¤Ž}|S )NÚfeatures_onlyz<features_only not implemented for Vision Transformer models.Úpretrained_filter_fn)r  ÚRuntimeErrorr   r   r„  )ÚvariantÚ
pretrainedÚdefault_cfgÚkwargsr~  s        r+   Ú_create_coatr�  è  s`   € Ø‡z‚z�/ 4Ñ(Ô(ð [ÝÐYÑZÔZÐZå ÝØØðð õ 2ð	ð
 ðð €Eð €Lr-   rE   c                 ó8   — | ddd dddt           t          dddd	œ|¥S )
Nrâ   )rr   rà   rà   gÍÌÌÌÌÌì?ÚbicubicTzpatch_embed1.projr2  z
apache-2.0)Úurlrî   Ú
input_sizeÚ	pool_sizeÚcrop_pctÚinterpolationÚfixed_input_sizerl  r  Ú
first_convÚ
classifierÚlicenser	   )r�  rŒ  s     r+   Ú	_cfg_coatr™  ö  s:   € àØ¨=ÀtØ¨ÈÝ%Õ.BØ)¸Øðð ð ðð r-   ztimm/)Ú	hf_hub_id)rr   é€  r›  rÄ   Úsquash)rš  r‘  r“  Ú	crop_mode)zcoat_tiny.in1kzcoat_mini.in1kzcoat_small.in1kzcoat_lite_tiny.in1kzcoat_lite_mini.in1kzcoat_lite_small.in1kzcoat_lite_medium.in1kzcoat_lite_medium_384.in1krI  c           	      óf   — t          dg d¢g d¢d¬¦  «        }t          dd| it          |fi |¤Ž¤Ž}|S )	Nr|   )é˜   rŸ  rŸ  rŸ  ©r"   r"   r"   r"   ré   ©rì   rï   rð   rñ   Ú	coat_tinyrŠ  )r¢  ©r2   r�  ©rŠ  rŒ  Ú	model_cfgr~  s       r+   r¢  r¢    ó\   € åØÐ!5Ð!5Ð!5À\À\À\Ðbcðeñ eô e€IåÐYÐY°ÐY½tÀIÐ?XÐ?XÐQWÐ?XÐ?XÐYÐY€EØ€Lr-   c           	      óf   — t          dg d¢g d¢d¬¦  «        }t          dd| it          |fi |¤Ž¤Ž}|S )	Nr|   )rŸ  éØ   r¨  r¨  r   ré   r¡  Ú	coat_minirŠ  )r©  r£  r¤  s       r+   r©  r©    r¦  r-   c           	      ób   — t          ddg d¢g d¢ddœ|¤Ž}t          d	d| it          |fi |¤Ž¤Ž}|S )
Nr|   )rŸ  ræ   ræ   ræ   r   ré   r¡  Ú
coat_smallrŠ  r(   )r«  r£  r¤  s       r+   r«  r«  !  sk   € åð oØÐ!5Ð!5Ð!5À\À\À\Ðbcðoð oØgmðoð o€IåÐZÐZ°*ÐZÅÀYÐ@YÐ@YÐRXÐ@YÐ@YÐZÐZ€EØ€Lr-   c           	      ój   — t          dg d¢g d¢g d¢¬¦  «        }t          dd| it          |fi |¤Ž¤Ž}|S )	Nr|   )rä   rå   é   ræ   r   ©rk   rk   r|   r|   ©rì   rï   rð   r¬   Úcoat_lite_tinyrŠ  )r°  r£  r¤  s       r+   r°  r°  )  ób   € åØÐ!4Ð!4Ð!4ÀLÀLÀLÐ]iÐ]iÐ]iðkñ kô k€IåÐ^Ð^°jÐ^ÅDÈÐD]ÐD]ÐV\ÐD]ÐD]Ð^Ð^€EØ€Lr-   c           	      ój   — t          dg d¢g d¢g d¢¬¦  «        }t          dd| it          |fi |¤Ž¤Ž}|S )	Nr|   rã   r   r®  r¯  Úcoat_lite_minirŠ  )r³  r£  r¤  s       r+   r³  r³  1  r±  r-   c           	      ój   — t          dg d¢g d¢g d¢¬¦  «        }t          dd| it          |fi |¤Ž¤Ž}|S )	Nr|   rã   rè   r®  r¯  Úcoat_lite_smallrŠ  )rµ  r£  r¤  s       r+   rµ  rµ  9  sb   € åØÐ!4Ð!4Ð!4ÀLÀLÀLÐ]iÐ]iÐ]iðkñ kô k€IåÐ_Ð_°zÐ_ÅTÈ)ÐE^ÐE^ÐW]ÐE^ÐE^Ð_Ð_€EØ€Lr-   c           	      ód   — t          dg d¢g d¢¬¦  «        }t          dd| it          |fi |¤Ž¤Ž}|S )Nr|   ©rå   r­  ræ   rç   ©rr   ré   é
   rk   )rì   rï   rð   Úcoat_lite_mediumrŠ  )rº  r£  r¤  s       r+   rº  rº  A  sY   € åØÐ!5Ð!5Ð!5À]À]À]ðTñ Tô T€IåÐ`Ð`¸
Ð`ÅdÈ9ÐF_ÐF_ÐX^ÐF_ÐF_Ð`Ð`€EØ€Lr-   c           	      óf   — t          ddg d¢g d¢¬¦  «        }t          dd| it          |fi |¤Ž¤Ž}|S )	Nr›  r|   r·  r¸  )rë   rì   rï   rð   Úcoat_lite_medium_384rŠ  )r¼  r£  r¤  s       r+   r¼  r¼  I  s^   € åØ Ð/CÐ/CÐ/CÐS`ÐS`ÐS`ðbñ bô b€IåÐdÐd¸JÐdÍ$ÈyÐJcÐJcÐ\bÐJcÐJcÐdÐd€EØ€Lr-   )FN)rE   ru  )5rf   Útypingr   r   r   r   r   r   rM   Útorch.nnr4   Útorch.nn.functionalÚ
functionalrQ   Ú	timm.datar
   r   Útimm.layersr   r   r   r   r   r   r   r   Ú_builderr   Ú	_registryr   r   Ú__all__r¨   r   rj   r‡   r�   rª   r   rV  rW  r„  r�  r™  Údefault_cfgsr¢  r©  r«  r°  r³  rµ  rº  r¼  r(   r-   r+   ú<module>rÇ     s�  ððð ð ;Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :à €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð à AÐ AÐ AÐ AÐ AÐ AÐ AÐ AØ qÐ qÐ qÐ qÐ qÐ qÐ qÐ qÐ qÐ qÐ qÐ qÐ qÐ qÐ qÐ qÐ qÐ qÐ qÐ qØ *Ð *Ð *Ð *Ð *Ð *Ø <Ð <Ð <Ð <Ð <Ð <Ð <Ð <àˆ(€ðHð Hð Hð Hð H�B”Iñ Hô Hð HðV2ð 2ð 2ð 2ð 2˜bœiñ 2ô 2ð 2ðjð ð ð ð �”ñ ô ð ðD<ð <ð <ð <ð <�"”)ñ <ô <ð <ð~Fð Fð Fð Fð F�B”Iñ Fô Fð FðRGð Gð Gð Gð Gˆ2Œ9ñ Gô Gð GðT
ð ð ðð ð ð
ð ð ð ð ð ð ð	ð 	ð 	ð 	ð %Ð$Ø�i¨'Ð2Ñ2Ô2Ø�i¨'Ð2Ñ2Ô2Ø �y¨7Ð3Ñ3Ô3Ø$˜9¨wÐ7Ñ7Ô7Ø$˜9¨wÐ7Ñ7Ô7Ø%˜I°Ð8Ñ8Ô8Ø&˜Y°Ð9Ñ9Ô9Ø!* ØØ ¨3¸(ð"ñ "ô "ð&ð &ñ ô €ð ðð ¨Tð ð ð ñ „ðð ðð ¨Tð ð ð ñ „ðð ðð ¨dð ð ð ñ „ðð ðð °$ð ð ð ñ „ðð ðð °$ð ð ð ñ „ðð ðð °4ð ð ð ñ „ðð ðð °Dð ð ð ñ „ðð ðð ¸ð ð ð ñ „ðð ð r-   