§
    ·¯¸jqa  ã            	       ó¤  — d Z ddlZddlmZ ddlmZmZmZmZm	Z	m
Z
 ddlZddlmZ ddlmZmZ ddlmZmZmZmZmZmZmZmZmZmZmZmZmZmZ ddl m!Z! dd	l"m#Z# dd
l$m%Z% ddl&m'Z'm(Z( ddl)m*Z*m+Z+ dgZ, G d„ dej-        ¦  «        Z. G d„ dej-        ¦  «        Z/ G d„ dej-        ¦  «        Z0 G d„ dej-        ¦  «        Z1 G d„ dej-        ¦  «        Z2dee3e3f         fd„Z4e%dee3e3f         dee3e3f         fd„¦   «         Z5 G d„ dej-        ¦  «        Z6 G d„ dej-        ¦  «        Z7 G d „ dej-        ¦  «        Z8d2d"„Z9d3d$„Z: e+ e:d%¬&¦  «         e:d'¬&¦  «         e:d(¬&¦  «         e:d)¬&¦  «         e:d*¬&¦  «        d+œ¦  «        Z;e*d2d,e8fd-„¦   «         Z<e*d2d,e8fd.„¦   «         Z=e*d2d,e8fd/„¦   «         Z>e*d2d,e8fd0„¦   «         Z?e*d2d,e8fd1„¦   «         Z@dS )4a)   Global Context ViT

From scratch implementation of GCViT in the style of timm swin_transformer_v2_cr.py

Global Context Vision Transformers -https://arxiv.org/abs/2206.09959

@article{hatamizadeh2022global,
  title={Global Context Vision Transformers},
  author={Hatamizadeh, Ali and Yin, Hongxu and Kautz, Jan and Molchanov, Pavlo},
  journal={arXiv preprint arXiv:2206.09959},
  year={2022}
}

Free of any code related to NVIDIA GCVit impl at https://github.com/NVlabs/GCVit.
The license for this code release is Apache 2.0 with no commercial restrictions.

However, weight files adapted from NVIDIA GCVit impl ARE under a non-commercial share-alike license
(https://creativecommons.org/licenses/by-nc-sa/4.0/) until I have a chance to train new ones...

Hacked together by / Copyright 2022, Ross Wightman
é    N)Úpartial)ÚCallableÚListÚOptionalÚTupleÚTypeÚUnion©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)Úget_device_dtypeÚDropPathÚcalculate_drop_path_ratesÚ	to_2tupleÚ	to_ntupleÚMlpÚClassifierHeadÚLayerNorm2dÚ
LayerScaleÚget_attnÚget_act_layerÚget_norm_layerÚ
RelPosBiasÚ_asserté   )Úbuild_model_with_cfg)Úfeature_take_indices)Úregister_notrace_function)Únamed_applyÚ
checkpoint)Úregister_modelÚgenerate_default_cfgsÚGlobalContextVitc                   ó~   ‡ — e Zd ZdZddddej        ddfdedee         ded	e	d
e
deej                 fˆ fd„Zd„ Zˆ xZS )ÚMbConvBlockzR A depthwise separable / fused mbconv style residual block with SE, `no norm.
    Ng      ð?ÚseFÚin_chsÚout_chsÚexpand_ratioÚ
attn_layerÚbiasÚ	act_layerc	                 ó¾  •— ||dœ}	t          ¦   «                              ¦   «          t          dd|i|	¤Ž}
t          |t          ¦  «        r|dk    s|dk    r
d|
d<   d|
d<   t          |¦  «        }|p|}t          ||z  ¦  «        }t          j        ||d	d
d
f||dœ|	¤Ž| _	         |¦   «         | _
         ||fi |
¤Ž| _        t          j        ||d
d
dfd|i|	¤Ž| _        d S )N©ÚdeviceÚdtyper,   r&   Úecag      Ð?Úrd_ratioFr+   é   r   )Úgroupsr+   r   © )ÚsuperÚ__init__ÚdictÚ
isinstanceÚstrr   ÚintÚnnÚConv2dÚconv_dwÚactr&   Úconv_pw)Úselfr'   r(   r)   r*   r+   r,   r/   r0   ÚddÚattn_kwargsÚmid_chsÚ	__class__s               €úa/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/models/gcvit.pyr7   zMbConvBlock.__init__:   s  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝÐ5Ð5 YÐ5°"Ð5Ð5ˆÝ�j¥#Ñ&Ô&ð 	(¨:¸Ò+=Ð+=ÀÈuÒATÐATØ&*ˆK˜
Ñ#Ø"'ˆK˜ÑÝ˜jÑ)Ô)ˆ
ØÐ#˜VˆÝ�l VÑ+Ñ,Ô,ˆå”y ¨°!°Q¸ÐZÀ&ÈtÐZÐZÐWYÐZÐZˆŒØ�9‘;”;ˆŒØ�*˜WÐ4Ð4¨Ð4Ð4ˆŒÝ”y ¨'°1°a¸ÐLÐLÀÐLÈÐLÐLˆŒˆˆó    c                 ó¼   — |}|                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }||z   }|S ©N)r>   r?   r&   r@   )rA   ÚxÚshortcuts      rF   ÚforwardzMbConvBlock.forwardT   sQ   € ØˆØ�LŠL˜‰OŒOˆØ�HŠH�Q‰KŒKˆØ�GŠG�A‰JŒJˆØ�LŠL˜‰OŒOˆØ�‰LˆØˆrG   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r<   ÚGELUr;   r   Úfloatr:   Úboolr   ÚModuler7   rL   Ú__classcell__©rE   s   @rF   r%   r%   7   sÅ   ø€ € € € € ðð ð
 &*Ø"%Ø"ØØ)+¬ØØðMð MàðMð ˜c”]ðMð  ð	Mð
 ðMð ðMð ˜BœI”ðMð Mð Mð Mð Mð Mð4ð ð ð ð ð ð rG   r%   c                   óŠ   ‡ — e Zd Zddej        eddfdedee         dede	ej
                 de	ej
                 f
ˆ fd„Zd	„ Zˆ xZS )
ÚDownsample2dNÚconvÚdimÚdim_outÚ	reductionr,   Ú
norm_layerc                 óü  •— ||dœ}t          ¦   «                              ¦   «          |p|}|�	 ||fi |¤Žnt          j        ¦   «         | _        t          |fd|i|¤Ž| _        |dv sJ ‚|dk    rt          j        ||dddfdd	i|¤Ž| _        nM|d
k    r%||k    sJ ‚t          j	        ddd¬¦  «        | _        n"||k    sJ ‚t          j
        d¬¦  «        | _        |�	 ||fi |¤Žnt          j        ¦   «         | _        d S )Nr.   r,   )rY   ÚmaxÚavgrY   r3   é   r   r+   Fr_   ©Úkernel_sizeÚstrideÚpadding©rc   )r6   r7   r<   ÚIdentityÚnorm1r%   Ú
conv_blockr=   r\   Ú	MaxPool2dÚ	AvgPool2dÚnorm2)
rA   rZ   r[   r\   r,   r]   r/   r0   rB   rE   s
            €rF   r7   zDownsample2d.__init___   s;  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ�.˜Sˆà.8Ð.D�Z�Z Ð*Ð* rÐ*Ð*Ð*Í"Ì+É-Ì-ˆŒ
Ý% cÐEÐE°YÐEÀ"ÐEÐEˆŒØÐ2Ð2Ð2Ð2Ð2Ø˜ÒÐÝœY s¨G°Q¸¸1ÐOÐOÀ5ÐOÈBÐOÐOˆDŒNˆNØ˜%ÒÐØ˜'’>�>�>�>Ýœ\°aÀÈ1ÐMÑMÔMˆDŒNˆNà˜'’>�>�>�>Ýœ\°aÐ8Ñ8Ô8ˆDŒNØ2<Ð2H�Z�Z Ð.Ð.¨2Ð.Ð.Ð.ÍbÌkÉmÌmˆŒ
ˆ
ˆ
rG   c                 ó®   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S rI   )rh   ri   r\   rl   ©rA   rJ   s     rF   rL   zDownsample2d.forwardz   sG   € Ø�JŠJ�q‰MŒMˆØ�OŠO˜AÑÔˆØ�NŠN˜1ÑÔˆØ�JŠJ�q‰MŒMˆØˆrG   )rM   rN   rO   r<   rQ   r   r;   r   r:   r   rT   r7   rL   rU   rV   s   @rF   rX   rX   ^   s³   ø€ € € € € ð &*Ø#Ø)+¬Ø*5ØØð\ð \àð\ð ˜c”]ð\ð ð	\ð
 ˜BœI”ð\ð ˜RœYœð\ð \ð \ð \ð \ð \ð6ð ð ð ð ð ð rG   rX   c            
       ób   ‡ — e Zd Zddej        ddfdedededeej                 fˆ fd„Z	d	„ Z
ˆ xZS )
ÚFeatureBlockr   r_   NrZ   Úlevelsr\   r,   c           	      ó  •— ||dœ}t          ¦   «                              ¦   «          |}t          d|¦  «        }|dk    rt          t          j        d¬¦  «        }	nt          t          j        ddd¬¦  «        }	t	          j        ¦   «         | _        t          |¦  «        D ]`}
| j         
                    d|
dz   › �t          |fd	|i|¤Ž¦  «         |r.| j         
                    d
|
dz   › � |	¦   «         ¦  «         |dz  }Œad S )Nr.   r   r`   ra   rf   r3   rb   rY   r,   Úpool)r6   r7   r_   r   r<   rk   rj   Ú
SequentialÚblocksÚrangeÚ
add_moduler%   )rA   rZ   rq   r\   r,   r/   r0   rB   Ú
reductionsÚpool_fnÚirE   s              €rF   r7   zFeatureBlock.__init__ƒ   s  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØˆ
Ý�Q˜‘”ˆØ˜ÒÐÝ�bœl¸Ð:Ñ:Ô:ˆGˆGå�bœl¸À!ÈQÐOÑOÔOˆGÝ”m‘o”oˆŒÝ�v‘”ð 	 ð 	 ˆAØŒK×"Ò" <¨!¨A©# < <µ¸SÐ1\Ð1\ÈIÐ1\ÐY[Ð1\Ð1\Ñ]Ô]Ð]Øð  Ø”×&Ò& |¨a°©c | |°W°W±Y´YÑ?Ô?Ð?Ø˜a‘�
øð		 ð 	 rG   c                 ó,   — |                       |¦  «        S rI   )ru   rn   s     rF   rL   zFeatureBlock.forward›   s   € Ø�{Š{˜1‰~Œ~ÐrG   )rM   rN   rO   r<   rQ   r;   r:   r   rT   r7   rL   rU   rV   s   @rF   rp   rp   ‚   s�   ø€ € € € € ð Ø"Ø)+¬ØØð ð  àð ð ð ð ð	 ð
 ˜BœI”ð ð  ð  ð  ð  ð  ð0ð ð ð ð ð ð rG   rp   c            
       óz   ‡ — e Zd Zddej        eddfdededeej                 deej                 fˆ fd„Z	d	„ Z
ˆ xZS )
ÚStemr3   é`   Nr'   r(   r,   r]   c                 ó´   •— t          ¦   «                              ¦   «          ||dœ}t          j        ||fddddœ|¤Ž| _        t          |f||dœ|¤Ž| _        d S )Nr.   r3   ra   r   rb   )r,   r]   )r6   r7   r<   r=   Úconv1rX   Údown)	rA   r'   r(   r,   r]   r/   r0   rB   rE   s	           €rF   r7   zStem.__init__    sq   ø€ õ 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆÝ”Y˜v wÐY¸AÀaÐQRÐYÐYÐVXÐYÐYˆŒ
Ý  Ð[°IÈ*Ð[Ð[ÐXZÐ[Ð[ˆŒ	ˆ	ˆ	rG   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S rI   )r€   r�   rn   s     rF   rL   zStem.forward®   s%   € Ø�JŠJ�q‰MŒMˆØ�IŠI�a‰LŒLˆØˆrG   )rM   rN   rO   r<   rQ   r   r;   r   rT   r7   rL   rU   rV   s   @rF   r}   r}   Ÿ   s¤   ø€ € € € € ð ØØ)+¬Ø*5ØØð\ð \àð\ð ð\ð ˜BœI”ð	\ð
 ˜RœYœð\ð \ð \ð \ð \ð \ðð ð ð ð ð ð rG   r}   c                   ó~   ‡ — e Zd Z	 	 	 	 	 	 ddededeeef         deded	ed
efˆ fd„Zddee	j
                 fd„Zˆ xZS )ÚWindowAttentionGlobalTç        NrZ   Ú	num_headsÚwindow_sizeÚ
use_globalÚqkv_biasÚ	attn_dropÚ	proj_dropc
                 ó  •— ||	dœ}
t          ¦   «                              ¦   «          t          |¦  «        }|| _        || _        ||z  | _        | j        dz  | _        || _        t          d||dœ|
¤Ž| _	        | j        rt          j        ||dz  fd|i|
¤Ž| _        nt          j        ||dz  fd|i|
¤Ž| _        t          j        |¦  «        | _        t          j        ||fi |
¤Ž| _        t          j        |¦  «        | _        d S )Nr.   g      à¿)r‡   r†   ra   r+   r3   r5   )r6   r7   r   r‡   r†   Úhead_dimÚscalerˆ   r   Úrel_posr<   ÚLinearÚqkvÚDropoutrŠ   Úprojr‹   )rA   rZ   r†   r‡   rˆ   r‰   rŠ   r‹   r/   r0   rB   rE   s              €rF   r7   zWindowAttentionGlobal.__init__¶   s  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ Ñ,Ô,ˆØ&ˆÔØ"ˆŒØ˜yÑ(ˆŒØ”] dÑ*ˆŒ
Ø$ˆŒå!ÐU¨kÀYÐUÐUÐRTÐUÐUˆŒØŒ?ð 	DÝ”y  c¨A¡gÐCÐC°HÐCÀÐCÐCˆDŒHˆHå”y  c¨A¡gÐCÐC°HÐCÀÐCÐCˆDŒHÝœ IÑ.Ô.ˆŒÝ”I˜c 3Ð-Ð-¨"Ð-Ð-ˆŒ	Ýœ IÑ.Ô.ˆŒˆˆrG   Úq_globalc                 ót  — |j         \  }}}| j        rò|�ðt          |j         d         |j         d         k    d¦  «         |                      |¦  «        }|                     ||d| j        | j        ¦  «                             ddddd¦  «        }|                     d¦  «        \  }}| 	                    ||j         d         z  ddd¦  «        }	|	                     ||| j        | j        ¦  «                             dddd¦  «        }	nf|                      |¦  «                             ||d| j        | j        ¦  «                             ddddd¦  «        }
|
                     d¦  «        \  }	}}|	| j
        z  }	|	|                     dd¦  «                             ¦   «         z  }|                      |¦  «        }|                     d¬	¦  «        }|                      |¦  «        }||z                       dd¦  «                             |||¦  «        }|                      |¦  «        }|                      |¦  «        }|S )
Néÿÿÿÿz*x and q_global seq lengths should be equalra   r   r3   r   é   éþÿÿÿ)rZ   )Úshaperˆ   r   r‘   Úreshaper†   r�   ÚpermuteÚunbindÚrepeatrŽ   Ú	transposeÚ
contiguousr�   ÚsoftmaxrŠ   r“   r‹   )rA   rJ   r”   ÚBÚNÚCÚkvÚkÚvÚqr‘   Úattns               rF   rL   zWindowAttentionGlobal.forwardÔ   sû  € Ø”'‰ˆˆ1ˆaØŒ?ð 	$˜xÐ3Ý�A”G˜B”K 8¤>°"Ô#5Ò5Ð7cÑdÔdÐdà—’˜!‘”ˆBØ—’˜A˜q ! T¤^°T´]ÑCÔC×KÒKÈAÈqÐRSÐUVÐXYÑZÔZˆBØ—9’9˜Q‘<”<‰DˆAˆqà—’  X¤^°AÔ%6Ñ 6¸¸1¸aÑ@Ô@ˆAØ—	’	˜!˜Q ¤°´Ñ>Ô>×FÒFÀqÈ!ÈQÐPQÑRÔRˆAˆAà—(’(˜1‘+”+×%Ò% a¨¨A¨t¬~¸t¼}ÑMÔM×UÒUÐVWÐYZÐ\]Ð_`ÐbcÑdÔdˆCØ—j’j ‘m”m‰GˆAˆq�!Ø�”
‰Nˆà�1—;’;˜r 2Ñ&Ô&×1Ò1Ñ3Ô3Ñ3ˆØ�|Š|˜DÑ!Ô!ˆØ�|Š| ˆ|Ñ#Ô#ˆØ�~Š~˜dÑ#Ô#ˆà�A‰X× Ò   AÑ&Ô&×.Ò.¨q°!°QÑ7Ô7ˆØ�IŠI�a‰LŒLˆØ�NŠN˜1ÑÔˆØˆrG   )TTr…   r…   NNrI   )rM   rN   rO   r;   r   rS   rR   r7   r   ÚtorchÚTensorrL   rU   rV   s   @rF   r„   r„   ´   sÉ   ø€ € € € € ð  $Ø!Ø!Ø!ØØð/ð /àð/ð ð/ð ˜s C˜xœð	/ð
 ð/ð ð/ð ð/ð ð/ð /ð /ð /ð /ð /ð<ð  8¨E¬LÔ#9ð ð ð ð ð ð ð ð rG   r„   r‡   c                 ó*  — | j         \  }}}}|                      |||d         z  |d         ||d         z  |d         |¦  «        } |                      dddddd¦  «                             ¦   «                              d|d         |d         |¦  «        }|S )Nr   r   r3   ra   r—   é   r–   ©r™   Úviewr›   rŸ   )rJ   r‡   r¡   ÚHÚWr£   Úwindowss          rF   Úwindow_partitionr²   ï   s”   € Ø”�J€A€qˆ!ˆQØ	�Šˆq�!�{ 1”~Ñ% {°1¤~°q¸KÈ¼NÑ7JÈKÐXYÌNÐ\]Ñ^Ô^€AØ�iŠi˜˜1˜a  A qÑ)Ô)×4Ò4Ñ6Ô6×;Ò;¸BÀÈAÄÐP[Ð\]ÔP^Ð`aÑbÔb€GØ€NrG   Úimg_sizec                 ó  — |\  }}| j         d         }|                      d||d         z  ||d         z  |d         |d         |¦  «        }|                     dddddd¦  «                             ¦   «                              d|||¦  «        }|S )Nr–   r   r   r3   ra   r—   r¬   r­   )r±   r‡   r³   r¯   r°   r£   rJ   s          rF   Úwindow_reverserµ   ö   s�   € à�D€A€qØŒ�bÔ€AØ�Š�R˜˜k¨!œnÑ,¨a°;¸q´>Ñ.AÀ;ÈqÄ>ÐS^Ð_`ÔSaÐcdÑeÔe€AØ	�	Š	�!�Q˜˜1˜a Ñ#Ô#×.Ò.Ñ0Ô0×5Ò5°b¸!¸QÀÑBÔB€AØ€HrG   c                   ó  ‡ — e Zd Zddddddddeej        ej        ddfdedeeef         ded	ed
e	de
de
dee	         de	de	de	dedeej                 deej                 fˆ fd„Zddeej                 fd„Zddeej                 fd„Zˆ xZS )ÚGlobalContextVitBlocké   ç      @TNr…   rZ   Ú	feat_sizer†   r‡   Ú	mlp_ratiorˆ   r‰   Úlayer_scaler‹   rŠ   Ú	drop_pathr*   r,   r]   c           
      ó  •— ||dœ}t          ¦   «                              ¦   «          t          |¦  «        }t          |¦  «        }|| _        t	          |d         |d         z  |d         |d         z  z  ¦  «        | _         ||fi |¤Ž| _         ||f|||||
|	dœ|¤Ž| _        |�t          ||fi |¤Žnt          j
        ¦   «         | _        |dk    rt          |¦  «        nt          j
        ¦   «         | _         ||fi |¤Ž| _        t          d|t	          ||z  ¦  «        ||	dœ|¤Ž| _        |�t          ||fi |¤Žnt          j
        ¦   «         | _        |dk    rt          |¦  «        nt          j
        ¦   «         | _        d S )Nr.   r   r   )r†   r‡   rˆ   r‰   rŠ   r‹   r…   )Úin_featuresÚhidden_featuresr,   Údropr5   )r6   r7   r   r‡   r;   Únum_windowsrh   r¨   r   r<   rg   Úls1r   Ú
drop_path1rl   r   ÚmlpÚls2Ú
drop_path2)rA   rZ   rº   r†   r‡   r»   rˆ   r‰   r¼   r‹   rŠ   r½   r*   r,   r]   r/   r0   rB   rE   s                     €rF   r7   zGlobalContextVitBlock.__init__   s¾  ø€ ð& ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ˜iÑ(Ô(ˆ	Ý Ñ,Ô,ˆØ&ˆÔÝ 	¨!¤°¸A´Ñ >À9ÈQÄ<ÐS^Ð_`ÔSaÑCaÑbÑcÔcˆÔà�Z Ð*Ð* rÐ*Ð*ˆŒ
Ø�JØð	
àØ#Ø!ØØØð	
ð 	
ð ð	
ð 	
ˆŒ	ð :EÐ9P•:˜c ;Ð5Ð5°"Ð5Ð5Ð5ÕVXÔVaÑVcÔVcˆŒØ1:¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒà�Z Ð*Ð* rÐ*Ð*ˆŒ
ÝÐx 3½¸CÀ)¹OÑ8LÔ8LÐXaÐhqÐxÐxÐuwÐxÐxˆŒØ9DÐ9P•:˜c ;Ð5Ð5°"Ð5Ð5Ð5ÕVXÔVaÑVcÔVcˆŒØ1:¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒˆˆrG   r”   c                 ó  — |j         \  }}}}t          || j        ¦  «        }|                     d| j        d         | j        d         z  |¦  «        }|                      ||¦  «        }t          || j        ||f¦  «        }|S )Nr–   r   r   )r™   r²   r‡   r®   r¨   rµ   )	rA   rJ   r”   r¡   r¯   r°   r£   Úx_winÚattn_wins	            rF   Ú_window_attnz"GlobalContextVitBlock._window_attn-  s}   € Ø”W‰
ˆˆ1ˆa�Ý   DÔ$4Ñ5Ô5ˆØ—
’
˜2˜tÔ/°Ô2°TÔ5EÀaÔ5HÑHÈ!ÑLÔLˆØ—9’9˜U HÑ-Ô-ˆÝ˜8 TÔ%5¸¸1°vÑ>Ô>ˆØˆrG   c           
      óL  — ||                       |                      |                      |                      |¦  «        |¦  «        ¦  «        ¦  «        z   }||                      |                      |                      |                      |¦  «        ¦  «        ¦  «        ¦  «        z   }|S rI   )rÄ   rÃ   rË   rh   rÇ   rÆ   rÅ   rl   )rA   rJ   r”   s      rF   rL   zGlobalContextVitBlock.forward5  s{   € Ø�—’ §¢¨×):Ò):¸4¿:º:Àa¹=¼=È(Ñ)SÔ)SÑ TÔ TÑUÔUÑUˆØ�—’ §¢¨¯ª°$·*²*¸Q±-´-Ñ)@Ô)@Ñ AÔ AÑBÔBÑBˆØˆrG   rI   )rM   rN   rO   r„   r<   rQ   Ú	LayerNormr;   r   rR   rS   r   r   r   rT   r7   r©   rª   rË   rL   rU   rV   s   @rF   r·   r·   ÿ   su  ø€ € € € € ð  !Ø!Ø#Ø!Ø+/Ø!Ø!Ø!Ø#8Ø)+¬Ø*,¬,ØØð#+Sð +Sàð+Sð ˜S #˜X”ð+Sð ð	+Sð
 ð+Sð ð+Sð ð+Sð ð+Sð " %œð+Sð ð+Sð ð+Sð ð+Sð !ð+Sð ˜BœI”ð+Sð ˜RœYœð+Sð +Sð +Sð +Sð +Sð +SðZð ¨°´Ô(>ð ð ð ð ðð  8¨E¬LÔ#9ð ð ð ð ð ð ð ð rG   r·   c            $       ó&  ‡ — e Zd Zdddddddddej        ej        eddfdededed	eeef         d
eeef         de	de	de	de
de	dee
         de
de
deee
         e
f         deej                 deej                 deej                 f"ˆ fd„Zd„ Zˆ xZS )ÚGlobalContextVitStageTFr¹   Nr…   rZ   Údepthr†   rº   r‡   Ú
downsampleÚglobal_normÚ
stage_normr»   r‰   r¼   r‹   rŠ   r½   r,   r]   Únorm_layer_clc                 ó  •‡‡‡‡‡	‡
‡‡‡‡‡‡‡— ||dœŠt          ¦   «                              ¦   «          |r4t          d‰‰dz  |dœ‰¤Ž| _        ‰dz  Š‰d         dz  ‰d         dz  fŠnt	          j        ¦   «         | _        ‰| _        t          ‰¦  «        Št          t          j
        t          ‰¦  «        t          ‰¦  «        z  ¦  «        ¦  «        }t          ‰|fi ‰¤Ž| _        |r	 ‰‰fi ‰¤Žnt	          j        ¦   «         | _        t	          j        ˆˆˆˆˆˆˆˆ	ˆˆˆˆ
ˆfd„t!          |¦  «        D ¦   «         ¦  «        | _        |r	 ‰‰fi ‰¤Žnt	          j        ¦   «         | _        ‰| _        ‰| _        d| _        d S )	Nr.   ra   )rZ   r[   r]   r   r   c                 óŒ   •— g | ]@}t          d‰‰‰‰‰	‰|d z  dk    ‰‰‰t          ‰t          ¦  «        r‰|         n‰‰‰
dœ‰¤Ž‘ŒAS )ra   r   )rZ   r†   rº   r‡   r»   r‰   rˆ   r¼   r‹   rŠ   r½   r,   r]   r5   )r·   r9   Úlist)Ú.0rz   r,   rŠ   rB   rZ   r½   rº   r¼   r»   rÔ   r†   r‹   r‰   r‡   s     €€€€€€€€€€€€€rF   ú
<listcomp>z2GlobalContextVitStage.__init__.<locals>.<listcomp>f  s�   ø€ ð %
ð %
ð %
ð" õ! "ð ØØ#Ø#Ø'Ø#Ø!Ø ™E QšJØ'Ø#Ø#Ý*4°YÅÑ*EÔ*EÐT˜) Aœ,˜,È9Ø#Ø(ðð ð ðð ð%
ð %
ð %
rG   Fr5   )r6   r7   rX   rÑ   r<   rg   rº   r   r;   ÚmathÚlog2Úminrp   Úglobal_blockrÒ   Ú
ModuleListrv   ru   ÚnormrZ   Úgrad_checkpointing)rA   rZ   rÐ   r†   rº   r‡   rÑ   rÒ   rÓ   r»   r‰   r¼   r‹   rŠ   r½   r,   r]   rÔ   r/   r0   Úfeat_levelsrB   rE   s    ` ```   ``````` `   @€rF   r7   zGlobalContextVitStage.__init__<  së  øøøøøøøøøøøøøø€ ð, ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØð 
	,Ý*ð ØØ˜a™Ø%ðð ð ð	ð ˆDŒOð ˜‘'ˆCØ" 1œ¨Ñ*¨I°a¬L¸AÑ,=Ð>ˆIˆIå œk™mœmˆDŒOØ"ˆŒÝ Ñ,Ô,ˆå�$œ)¥C¨	¡N¤NµS¸Ñ5EÔ5EÑ$EÑFÔFÑGÔGˆÝ(¨¨kÐ@Ð@¸RÐ@Ð@ˆÔØ7BÐU˜=˜=¨Ð3Ð3°Ð3Ð3Ð3ÍÌÉÌˆÔå”mð %
ð %
ð %
ð %
ð %
ð %
ð %
ð %
ð %
ð %
ð %
ð %
ð %
ð %
ð %
ð %
õ" ˜5‘\”\ð#%
ñ %
ô %
ñ ô ˆŒð& 1;ÐM�M�M #Ð,Ð,¨Ð,Ð,Ð,ÅÄÁÄˆŒ	ØˆŒØ"ˆŒØ"'ˆÔÐÐrG   c                 óø  — |                       |¦  «        }|                      |¦  «        }|                     dddd¦  «        }|                      |                     dddd¦  «        ¦  «        }| j        D ]E}| j        r0t          j                             ¦   «         st          |||¦  «        }Œ9 |||¦  «        }ŒF|  
                    |¦  «        }|                     dddd¦  «                             ¦   «         }|S )Nr   ra   r3   r   )rÑ   rÝ   r›   rÒ   ru   rà   r©   ÚjitÚis_scriptingr    rß   rŸ   )rA   rJ   Úglobal_queryÚblks       rF   rL   zGlobalContextVitStage.forward~  sô   € à�OŠO˜AÑÔˆØ×(Ò(¨Ñ+Ô+ˆð �IŠI�a˜˜A˜qÑ!Ô!ˆØ×'Ò'¨×(<Ò(<¸QÀÀ1ÀaÑ(HÔ(HÑIÔIˆØ”;ð 	)ð 	)ˆCØÔ&ð )­u¬y×/EÒ/EÑ/GÔ/Gð )Ý˜s A |Ñ4Ô4��à�C˜˜<Ñ(Ô(��Ø�IŠI�a‰LŒLˆØ�IŠI�a˜˜A˜qÑ!Ô!×,Ò,Ñ.Ô.ˆØˆrG   )rM   rN   rO   r<   rQ   rÍ   r   r;   r   rS   rR   r   r	   r   r   rT   r7   rL   rU   rV   s   @rF   rÏ   rÏ   ;  sv  ø€ € € € € ð  $Ø %Ø$Ø!Ø!Ø+/Ø!Ø!Ø36Ø)+¬Ø*,¬,Ø-8ØØð)@(ð @(àð@(ð ð@(ð ð	@(ð
 ˜S #˜X”ð@(ð ˜s C˜xœð@(ð ð@(ð ð@(ð ð@(ð ð@(ð ð@(ð " %œð@(ð ð@(ð ð@(ð ˜T %œ[¨%Ð/Ô0ð@(ð  ˜BœI”ð!@(ð" ˜RœYœð#@(ð$   ¤	œ?ð%@(ð @(ð @(ð @(ð @(ð @(ðDð ð ð ð ð ð rG   rÏ   c            +       óT  ‡ — e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dBdedededeeeeef         f         deedf         deeeeedf         f                  dedeedf         deedf         dede	dee         ded ed!ed"ed#ed$ed%ed&ed'ef*ˆ fd(„Z
dCd*„Zej        j        d+„ ¦   «         Zej        j        dDd-„¦   «         Zej        j        dEd.„¦   «         Zej        j        d/ej        fd0„¦   «         ZdFdedee         fd1„Z	 	 	 	 	 dGd3ej        d4eeeee         f                  d5e	d6e	d7ed8e	d/eeej                 eej        eej                 f         f         fd9„Z	 	 	 dHd4eeee         f         d;e	d<e	fd=„Zd3ej        d/ej        fd>„ZdDd?e	fd@„Zd3ej        d/ej        fdA„Zˆ xZS )Ir#   r3   éè  r`   éà   ©é    rë   é   rë   Né@   ©r3   r—   é   r¬   ©ra   r—   é   rì   ç      @Tr…   Ú ÚgeluÚlayernorm2dÚ	layernormçñhãˆµøä>Úin_chansÚnum_classesÚglobal_poolr³   Úwindow_ratio.r‡   Ú	embed_dimÚdepthsr†   r»   r‰   r¼   Ú	drop_rateÚproj_drop_rateÚattn_drop_rateÚdrop_path_rateÚweight_initr,   r]   rÔ   Únorm_epsc                 óV  •‡— t          ¦   «                              ¦   «          ||dœ}t          |¦  «        }t          t	          |¦  «        |¬¦  «        }t          t	          |¦  «        |¬¦  «        }g | _        t          ‰¦  «        Št          d„ ‰D ¦   «         ¦  «        }|| _        || _	        || _
        || _        t          |¦  «        }t          |d|dz
  z  z  ¦  «        x| _        | _        |� t!          |¦  «        |¦  «        }n5|€J ‚t          ˆfd„ t!          |¦  «        |¦  «        D ¦   «         ¦  «        }t#          d ||||dœ|¤Ž| _        t'          ||d¬	¦  «        }g }t)          |¦  «        D ]Ø}||dz
  k    }dt+          |dz
  d
¦  «        z  }|                     t/          d i d||z  “d||         “d|	|         “d|d
         |z  |d         |z  f“d||         “d|d
k    “d|“d|
“d|“d|“d|“d|“d||         “d|“d|“d|“|¤Ž¦  «         | xj        t1          |d         j        d|dz   z  d|› �¬¦  «        gz  c_        ŒÙt5          j        |Ž | _        t;          | j        |f||dœ|¤Ž| _        |r&t?          t          | j         |¬¦  «        | ¦  «         d S d S )!Nr.   )Úepsc              3   ó    K  — | ]	}|d z  V — Œ
dS )r—   Nr5   )rØ   Úds     rF   ú	<genexpr>z,GlobalContextVit.__init__.<locals>.<genexpr>²  s&   è è € Ð3Ð3 Q˜!˜q™&Ð3Ð3Ð3Ð3Ð3Ð3rG   ra   r   c                 ó<   •— g | ]}‰d          |z  ‰d         |z  f‘ŒS )r   r   r5   )rØ   Úrr³   s     €rF   rÙ   z-GlobalContextVit.__init__.<locals>.<listcomp>½  s0   ø€ Ð sÐ sÐ sÈ! (¨1¤+°Ñ"2°H¸Q´KÀ1Ñ4DÐ!EÐ sÐ sÐ srG   )r'   r(   r,   r]   T)Ú	stagewiser   rZ   rÐ   r†   rº   r‡   rÑ   rÓ   r»   r‰   r¼   r‹   rŠ   r½   r,   r]   rÔ   r–   zstages.)Únum_chsr\   Úmodule©Ú	pool_typerþ   )Úschemer5   )!r6   r7   r   r   r   Úfeature_infor   Útuplerú   rù   rø   rþ   Úlenr;   Únum_featuresÚhead_hidden_sizer   r}   Ústemr   rv   r_   ÚappendrÏ   r8   rZ   r<   rt   Ústagesr   Úheadr   Ú_init_weights)!rA   rø   rù   rú   r³   rû   r‡   rü   rý   r†   r»   r‰   r¼   rþ   rÿ   r   r  r  r,   r]   rÔ   r  r/   r0   rB   rº   Ú
num_stagesÚdprr  rz   Ú
last_stageÚstage_scalerE   s!       `                           €rF   r7   zGlobalContextVit.__init__‘  s§  øø€ õ4 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆÝ! )Ñ,Ô,ˆ	Ý�^¨JÑ7Ô7¸XÐFÑFÔFˆ
Ý¥¨}Ñ =Ô =À8ÐLÑLÔLˆØˆÔÝ˜XÑ&Ô&ˆÝÐ3Ð3¨(Ð3Ñ3Ô3Ñ3Ô3ˆ	Ø&ˆÔØ&ˆÔØ ˆŒØ"ˆŒÝ˜‘[”[ˆ
Ý47¸	ÀAÈ*ÐWXÉ.ÑDYÑ8YÑ4ZÔ4ZÐZˆÔ˜DÔ1ØÐ"Ø/�) JÑ/Ô/°Ñ<Ô<ˆKˆKàÐ+Ð+Ð+ÝÐ sÐ sÐ sÐ sÐOdÍyÐYcÑOdÔOdÐeqÑOrÔOrÐ sÑ sÔ sÑtÔtˆKåð 
ØØØØ!ð	
ð 
ð
 ð
ð 
ˆŒ	õ (¨¸È$ÐOÑOÔOˆØˆÝ�zÑ"Ô"ð 	jð 	jˆAØ˜j¨1™nÒ,ˆJØ�s 1 q¡5¨!™}œ}Ñ,ˆKØ�MŠMÕ/ð ð ð Ø Ñ+Ð+ðà˜Q”i�iðð $ Aœ,˜,ðð % Qœ<¨;Ñ6¸	À!¼ÈÑ8SÐTÐTð	ð
 (¨œN˜Nðð  š6˜6ðð &˜:ðð $˜)ðð "˜ðð (˜Kðð )˜.ðð )˜.ðð ˜aœ&˜&ðð $˜)ðð &˜:ðð  ,˜mØð#ð ñ ô ð ð& ÐÔ¥$¨v°b¬z¬~ÈÈQÈqÉSÉÐZgÐdeÐZgÐZgÐ"hÑ"hÔ"hÐ!iÑiÐÔÐÝ”m VÐ,ˆŒõ # 4Ô#4°kÐtÈ[ÐdmÐtÐtÐqsÐtÐtˆŒ	àð 	OÝ� Ô 2¸;ÐGÑGÔGÈÑNÔNÐNÐNÐNð	Oð 	OrG   Úvitc                 ó&  — |dk    r›t          |t          j        ¦  «        r}t          j                             |j        ¦  «         |j        �Td|v r(t          j                             |j        d¬¦  «         d S t          j                             |j        ¦  «         d S d S d S t          |t          j        ¦  «        rSt          j                             |j        d¬¦  «         |j        �(t          j                             |j        ¦  «         d S d S d S )Nr  rÅ   g�íµ ÷Æ°>)Ústdg{®Gáz”?)	r9   r<   r�   ÚinitÚxavier_uniform_Úweightr+   Únormal_Úzeros_)rA   r  Únamer  s       rF   r  zGlobalContextVit._init_weightsè  sø   € à�UŠ?ˆ?Ý˜&¥"¤)Ñ,Ô,ð 4Ý”×'Ò'¨¬Ñ6Ô6Ð6Ø”;Ð*Ø �}�}ÝœŸš¨¬¸˜Ñ>Ô>Ð>Ð>Ð>åœŸš v¤{Ñ3Ô3Ð3Ð3Ð3ð4ð 4à*Ð*õ ˜&¥"¤)Ñ,Ô,ð 0Ý”—’ ¤°3�Ñ7Ô7Ð7Ø”;Ð*Ý”G—N’N 6¤;Ñ/Ô/Ð/Ð/Ð/ð0ð 0à*Ð*rG   c                 ó>   — d„ |                       ¦   «         D ¦   «         S )Nc                 óP   ‡— h | ]"\  Š}t          ˆfd „dD ¦   «         ¦  «        ¯ ‰’Œ#S )c              3   ó    •K  — | ]}|‰v V — Œ	d S rI   r5   )rØ   Únr¥   s     €rF   r  z=GlobalContextVit.no_weight_decay.<locals>.<setcomp>.<genexpr>ü  s'   øè è € ÐSÐS˜a�1˜�6ÐSÐSÐSÐSÐSÐSrG   )Úrelative_position_bias_tablezrel_pos.mlp)Úany)rØ   Ú_r¥   s     @rF   ú	<setcomp>z3GlobalContextVit.no_weight_decay.<locals>.<setcomp>ú  sX   ø€ ð Uð Uð UÙ�!�QÝÐSÐSÐSÐSÐ#RÐSÑSÔSÑSÔSðUØðUð Uð UrG   )Únamed_parameters©rA   s    rF   Úno_weight_decayz GlobalContextVit.no_weight_decayø  s5   € ðUð UØ×/Ò/Ñ1Ô1ðUñ Uô Uð 	UrG   Fc                 ó(   — t          dd¬¦  «        }|S )Nz^stemz^stages\.(\d+))r  ru   )r8   )rA   ÚcoarseÚmatchers      rF   Úgroup_matcherzGlobalContextVit.group_matcherþ  s#   € åØØ$ð
ñ 
ô 
ˆð ˆrG   c                 ó(   — | j         D ]	}||_        Œ
d S rI   )r  rà   )rA   ÚenableÚss      rF   Úset_grad_checkpointingz'GlobalContextVit.set_grad_checkpointing  s(   € à”ð 	*ð 	*ˆAØ#)ˆAÔ Ð ð	*ð 	*rG   Úreturnc                 ó   — | j         j        S rI   )r  Úfcr1  s    rF   Úget_classifierzGlobalContextVit.get_classifier  s   € àŒyŒ|ÐrG   c                 óÜ   — t          | ||¬¦  «        }|| _        |€| j        j        j        }t          | j        |f|| j        dœ|¤Ž| _        | j                             | j	        ¦  «         d S )Nr.   r  )
r   rù   r  rú   r  r   r  rþ   ÚtrainÚtraining)rA   rù   rú   r/   r0   rB   s         rF   Úreset_classifierz!GlobalContextVit.reset_classifier  su   € Ý˜d¨6¸Ð?Ñ?Ô?ˆØ&ˆÔØÐØœ)Ô/Ô9ˆKÝ" 4Ô#4°kÐyÈ[ÐdhÔdrÐyÐyÐvxÐyÐyˆŒ	ØŒ	�Š˜œÑ&Ô&Ð&Ð&Ð&rG   ÚNCHWrJ   Úindicesrß   Ú
stop_earlyÚ
output_fmtÚintermediates_onlyc                 óŒ  — |dv s
J d¦   «         ‚g }t          t          | j        ¦  «        |¦  «        \  }}	|                      |¦  «        }t          j                             ¦   «         s|s| j        }
n| j        d|	dz   …         }
t          |
¦  «        D ])\  }} ||¦  «        }||v r|                     |¦  «         Œ*|r|S ||fS )aï   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )rC  zOutput shape must be NCHW.Nr   )	r   r  r  r  r©   rã   rä   Ú	enumerater  )rA   rJ   rD  rß   rE  rF  rG  ÚintermediatesÚtake_indicesÚ	max_indexr  Úfeat_idxÚstages                rF   Úforward_intermediatesz&GlobalContextVit.forward_intermediates  sí   € ð* ˜YÐ&Ð&Ð&Ð(DÑ&Ô&Ð&ØˆÝ"6µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�ið �IŠI�a‰LŒLˆÝŒ9×!Ò!Ñ#Ô#ð 	1¨:ð 	1Ø”[ˆFˆFà”[  )¨a¡- Ô0ˆFå(¨Ñ0Ô0ð 	(ð 	(‰OˆH�eØ��a‘”ˆAØ˜<Ð'Ð'Ø×$Ò$ QÑ'Ô'Ð'øàð 	!Ø Ð à�-ÐÐrG   r   Ú
prune_normÚ
prune_headc                 ó®   — t          t          | j        ¦  «        |¦  «        \  }}| j        d|dz   …         | _        |r|                      dd¦  «         |S )z@ Prune layers not required for specified intermediates.
        Nr   r   ró   )r   r  r  rB  )rA   rD  rP  rQ  rK  rL  s         rF   Úprune_intermediate_layersz*GlobalContextVit.prune_intermediate_layersA  s\   € õ #7µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�iØ”k . 9¨q¡= .Ô1ˆŒØð 	)Ø×!Ò! ! RÑ(Ô(Ð(ØÐrG   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S rI   )r  r  rn   s     rF   Úforward_featuresz!GlobalContextVit.forward_featuresO  s%   € Ø�IŠI�a‰LŒLˆØ�KŠK˜‰NŒNˆØˆrG   Ú
pre_logitsc                 ó^   — |r|                       ||¬¦  «        n|                       |¦  «        S )N)rV  )r  )rA   rJ   rV  s      rF   Úforward_headzGlobalContextVit.forward_headT  s-   € Ø6@ÐRˆt�yŠy˜ zˆyÑ2Ô2Ð2ÀdÇiÂiÐPQÁlÄlÐRrG   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S rI   )rU  rX  rn   s     rF   rL   zGlobalContextVit.forwardW  s-   € Ø×!Ò! !Ñ$Ô$ˆØ×Ò˜aÑ Ô ˆØˆrG   )r3   rè   r`   ré   rê   Nrí   rî   rð   rò   TNr…   r…   r…   r…   ró   rô   rõ   rö   r÷   NN)r  ©F)T)NNN)NFFrC  F)r   FT)rM   rN   rO   r;   r:   r	   r   r   rR   rS   r7   r  r©   rã   Úignorer2  r6  r:  r<   rT   r>  rB  rª   r   rO  rS  rU  rX  rL   rU   rV   s   @rF   r#   r#   �  s  ø€ € € € € ð Ø#Ø$Ø47Ø,<ØAEØØ&3Ø)6Ø"Ø!Ø+/Ø!Ø$&Ø$&Ø$&Ø!Ø#Ø+Ø!,Ø"ØØð1UOð UOàðUOð ðUOð ð	UOð
 ˜C  s¨C x¤Ð0Ô1ðUOð    S œ/ðUOð " %¨¨U°3¸°8¬_Ð(<Ô"=Ô>ðUOð ðUOð ˜#˜s˜(”OðUOð ˜S #˜X”ðUOð ðUOð ðUOð " %œðUOð ðUOð "ðUOð  "ð!UOð" "ð#UOð$ ð%UOð& ð'UOð( ð)UOð* ð+UOð, ð-UOð UOð UOð UOð UOð UOðn0ð 0ð 0ð 0ð  „YÔðUð Uñ ÔðUð
 „YÔðð ð ñ Ôðð „YÔð*ð *ð *ñ Ôð*ð „YÔð ¤	ð ð ð ñ Ôðð'ð '¨Cð '¸hÀs¼mð 'ð 'ð 'ð 'ð 8<ØØ$Ø$Ø',ð( ð ( àŒ|ð( ð ˜e C¨¨c¬ NÔ3Ô4ð( ð ð	( ð
 ð( ð ð( ð !%ð( ð 
ˆt�E”LÔ! 5¨¬°t¸E¼LÔ7IÐ)IÔ#JÐJÔ	Kð( ð ( ð ( ð ( ðX ./Ø$Ø#ð	ð à˜3  S¤	˜>Ô*ðð ðð ð	ð ð ð ð %¤,ð °5´<ð ð ð ð ð
Sð S¨$ð Sð Sð Sð Sð˜œð ¨%¬,ð ð ð ð ð ð ð ð rG   Fc                 óP   — t          t          | |fdt          dd¬¦  «        i|¤Ž}|S )NÚfeature_cfg)r   r   ra   r3   T)Úout_indicesÚflatten_sequential)r   r#   r8   )ÚvariantÚ
pretrainedÚkwargsÚmodels       rF   Ú_create_gcvitrd  ]  sC   € Ý Ý˜' :ðð å \ÀdÐKÑKÔKðð ðð €Eð
 €LrG   ró   c                 ó8   — | dddddt           t          dddd	d
œ|¥S )Nrè   )r3   ré   ré   )r¸   r¸   g      ì?Úbicubicz
stem.conv1zhead.fcTz
apache-2.0)Úurlrù   Ú
input_sizeÚ	pool_sizeÚcrop_pctÚinterpolationÚmeanr!  Ú
first_convÚ
classifierÚfixed_input_sizeÚlicenser
   )rg  rb  s     rF   Ú_cfgrq  f  s;   € à 4°}ÐSYØ¨IÝ%Õ.BØ"°)Ø Øðð ð ðð rG   z}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_xxtiny_224_nvidia-d1d86009.pth)rg  z|https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_xtiny_224_nvidia-274b92b7.pthz{https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_tiny_224_nvidia-ac783954.pthz|https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_small_224_nvidia-4e98afa2.pthz{https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_base_224_nvidia-f009139b.pth)zgcvit_xxtiny.in1kzgcvit_xtiny.in1kzgcvit_tiny.in1kzgcvit_small.in1kzgcvit_base.in1kr;  c                 ó<   — t          ddddœ|¤Ž}t          dd| i|¤ŽS )N)ra   ra   é   ra   rð   ©rý   r†   Úgcvit_xxtinyra  r5   )ru  ©r8   rd  ©ra  rb  Úmodel_kwargss      rF   ru  ru  €  sF   € åð ØØðð ð ðð €Lõ ÐOÐO°JÐOÀ,ÐOÐOÐOrG   c                 ó<   — t          ddddœ|¤Ž}t          dd| i|¤ŽS )N)r3   r—   rs  r¬   rð   rt  Úgcvit_xtinyra  r5   )rz  rv  rw  s      rF   rz  rz  ‰  sF   € åð ØØðð ð ðð €Lõ ÐNÐN°:ÐNÀÐNÐNÐNrG   c                 ó<   — t          ddddœ|¤Ž}t          dd| i|¤ŽS )Nrî   rð   rt  Ú
gcvit_tinyra  r5   )r|  rv  rw  s      rF   r|  r|  ’  sF   € åð ØØðð ð ðð €Lõ ÐMÐM°*ÐMÀÐMÐMÐMrG   c           	      óB   — t          d	ddddddœ|¤Ž}t          d
d| i|¤ŽS )Nrî   )r3   rs  é   é   r~   ra   r÷   ©rý   r†   rü   r»   r¼   Úgcvit_smallra  r5   )r�  rv  rw  s      rF   r�  r�  ›  sO   € åð ØØ ØØØðð ð ðð €Lõ ÐNÐN°:ÐNÀÐNÐNÐNrG   c           	      óB   — t          d	ddddddœ|¤Ž}t          d
d| i|¤ŽS )Nrî   )r—   rñ   rì   rë   é€   ra   r÷   r€  Ú
gcvit_basera  r5   )r„  rv  rw  s      rF   r„  r„  §  sO   € åð ØØ ØØØðð ð ðð €Lõ ÐMÐM°*ÐMÀÐMÐMÐMrG   rZ  )ró   )ArP   rÚ   Ú	functoolsr   Útypingr   r   r   r   r   r	   r©   Útorch.nnr<   Ú	timm.datar   r   Útimm.layersr   r   r   r   r   r   r   r   r   r   r   r   r   r   Ú_builderr   Ú	_featuresr   Ú_features_fxr   Ú_manipulater   r    Ú	_registryr!   r"   Ú__all__rT   r%   rX   rp   r}   r„   r;   r²   rµ   r·   rÏ   r#   rd  rq  Údefault_cfgsru  rz  r|  r�  r„  r5   rG   rF   ú<module>r‘     sY  ððð ð* €€€Ø Ð Ð Ð Ð Ð Ø ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?à €€€Ø Ð Ð Ð Ð Ð à AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð  +Ð *Ð *Ð *Ð *Ð *Ø +Ð +Ð +Ð +Ð +Ð +Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø <Ð <Ð <Ð <Ð <Ð <Ð <Ð <àÐ
€ð$ð $ð $ð $ð $�"”)ñ $ô $ð $ðN!ð !ð !ð !ð !�2”9ñ !ô !ð !ðHð ð ð ð �2”9ñ ô ð ð:ð ð ð ð ˆ2Œ9ñ ô ð ð*8ð 8ð 8ð 8ð 8˜BœIñ 8ô 8ð 8ðv U¨3°¨8¤_ð ð ð ð ð ð¨¨s°C¨x¬ð ÀEÈ#ÈsÈ(ÄOð ð ð ñ Ôðð9ð 9ð 9ð 9ð 9˜BœIñ 9ô 9ð 9ðxRð Rð Rð Rð R˜BœIñ Rô Rð RðjJð Jð Jð Jð J�r”yñ Jô Jð JðZð ð ð ð	ð 	ð 	ð 	ð %Ð$Ø˜ð LðMñ Mô Mà˜ð KðLñ Lô Là�tð JðKñ Kô Kà˜ð KðLñ Lô Là�tð JðKñ Kô Kð&ð &ñ ô €ð ðPð PÐ0@ð Pð Pð Pñ „ðPð ðOð OÐ/?ð Oð Oð Oñ „ðOð ðNð NÐ.>ð Nð Nð Nñ „ðNð ðOð OÐ/?ð Oð Oð Oñ „ðOð ðNð NÐ.>ð Nð Nð Nñ „ðNð Nð NrG   