§
    ·¯¸jmJ  ã                   ó¢  — d Z ddlmZ ddlmZmZmZmZ ddlZddl	m
Z
 ddlmZmZ ddlmZmZmZmZmZ ddlmZ dd	lmZmZmZ dd
lmZmZ ddlmZmZ ddl m!Z!  G d„ de
j"        ¦  «        Z#d„ Z$dgd„Z%dhd„Z&	 	 didee'ej(        f         de!de'de)dee'ej(        f         f
d„Z*djd„Z+dkd„Z, ei d  e,d!d"dd#¬$¦  «        “d% e,d&d"d#d'd(d¬)¦  «        “d* e,d+d"d¬,¦  «        “d- e,d.d"d'd(d¬/¦  «        “d0 e,¦   «         “d1 e,d2d"d'd(¬3¦  «        “d4 e,d5d"d¬,¦  «        “d6 e,d7d"d'd(d¬/¦  «        “d8 e,d9d"d:d;d#d¬<¦  «        “d= e,d>d"d:d;d¬?¦  «        “d@ e,d"dd;¬A¦  «        “dB e,dCd"d:d;d¬?¦  «        “dD e,eedE¬F¦  «        “dG e,eedE¬F¦  «        “dH e,eedE¬F¦  «        “dI e,eedE¬F¦  «        “dJ e,dKdLdMdNdOdPdQ¬R¦  «        “ e,dSdTdNdMdOdPdQ¬U¦  «         e,d"dNdOdPdQdM¬V¦  «        dWœ¥¦  «        Z-edlde!fdX„¦   «         Z.edlde!fdY„¦   «         Z/edlde!fdZ„¦   «         Z0edlde!fd[„¦   «         Z1edlde!fd\„¦   «         Z2edlde!fd]„¦   «         Z3edlde!fd^„¦   «         Z4edlde!fd_„¦   «         Z5edlde!fd`„¦   «         Z6edlde!fda„¦   «         Z7edlde!fdb„¦   «         Z8edlde!fdc„¦   «         Z9edlde!fdd„¦   «         Z:edlde!fde„¦   «         Z; ee<d8d=d@d@dBd1dfœ¦  «         dS )ma   Hybrid Vision Transformer (ViT) in PyTorch

A PyTorch implement of the Hybrid Vision Transformers as described in:

'An Image Is Worth 16 x 16 Words: Transformers for Image Recognition at Scale'
    - https://arxiv.org/abs/2010.11929

`How to train your ViT? Data, Augmentation, and Regularization in Vision Transformers`
    - https://arxiv.org/abs/2106.10270

NOTE These hybrid model definitions depend on code in vision_transformer.py.
They were moved here to keep file sizes sane.

Hacked together by / Copyright 2020, Ross Wightman
é    )Úpartial)ÚDictÚTupleÚTypeÚUnionN)ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)ÚStdConv2dSameÚ	StdConv2dÚConvNormActÚ	to_ntupleÚHybridEmbedé   )Úbuild_model_with_cfg)Úgenerate_default_cfgsÚregister_modelÚregister_model_deprecations)Ú	resnet26dÚ	resnet50d)ÚResNetV2Úcreate_resnetv2_stem)ÚVisionTransformerc                   ó  ‡ — e Zd Zddddddej        ej        ddf
dededeeeed	f         f         d
eeeed	f         f         deeeed	f         f         dee	eeed	f         f         de
ej                 de
ej                 fˆ fd„Zˆ xZS )ÚConvStemé   é@   )é   r   r   Ú NÚin_chansÚdepthÚchannels.Úkernel_sizeÚstrideÚpaddingÚ
norm_layerÚ	act_layerc                 óÈ  •‡— |	|
dœ}t          ¦   «                              ¦   «          t          ‰t          ¦  «        r1t	          ˆfd„t          |¦  «        D ¦   «         d d d…         ¦  «        Š t          |¦  «        |¦  «        } t          |¦  «        |¦  «        }|t          |¦  «        cxk    r't          |¦  «        cxk    rt          ‰¦  «        k    sn J ‚|}t          t          ‰¦  «        ¦  «        D ]f}|t          ‰¦  «        dz
  k    }|                      |› t          |‰|         f||         ||         ||         || | ||dœ|¤Ž¦  «         ‰|         }Œgd S )N©ÚdeviceÚdtypec                 ó    •— g | ]
}‰d |z  z  ‘ŒS )r   © )Ú.0Úir!   s     €úu/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/models/vision_transformer_hybrid.pyú
<listcomp>z%ConvStem.__init__.<locals>.<listcomp>2   s"   ø€ ÐEÐEÐE°1˜h¨!¨Q©$Ñ.ÐEÐEÐEó    éÿÿÿÿr   )r"   r#   r$   ÚbiasÚ
apply_normÚ	apply_actr%   r&   )
ÚsuperÚ__init__Ú
isinstanceÚintÚtupleÚranger   ÚlenÚ
add_moduler   )Úselfr   r    r!   r"   r#   r$   r%   r&   r)   r*   ÚddÚin_chsr.   Ú	last_convÚ	__class__s      `           €r/   r7   zConvStem.__init__!   s   øø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ�h¥Ñ$Ô$ð 	MåÐEÐEÐEÐE½¸e¹¼ÐEÑEÔEÀdÀdÈÀdÔKÑLÔLˆHà&•i Ñ&Ô& {Ñ3Ô3ˆØ"•)˜EÑ"Ô" 7Ñ+Ô+ˆØ�˜F™œÐHÐHÒHÐH¥s¨;Ñ'7Ô'7ÐHÐHÒHÐH½3¸x¹=¼=ÒHÐHÐHÐHÐHÐHàˆÝ•s˜8‘}”}Ñ%Ô%ð 	!ð 	!ˆAØ�S ™]œ]¨QÑ.Ò.ˆIØ�OŠO˜q˜F¥KØØ˜”ð%ð (¨œNØ˜a”yØ œ
ØØ(˜=Ø'˜-Ø%Ø#ð%ð %ð ð%ð %ñ ô ð ð ˜a”[ˆFˆFð	!ð 	!r1   )Ú__name__Ú
__module__Ú__qualname__ÚnnÚBatchNorm2dÚReLUr9   r   r   Ústrr   ÚModuler7   Ú__classcell__)rB   s   @r/   r   r       s
  ø€ € € € € ð ØØ46Ø78Ø2;Ø8:Ø*,¬.Ø)+¬ØØð'!ð '!àð'!ð ð'!ð ˜C  s¨C x¤Ð0Ô1ð	'!ð
 ˜s E¨#¨s¨(¤OÐ3Ô4ð'!ð ˜#˜u S¨# XœÐ.Ô/ð'!ð ˜3  U¨3°¨8¤_Ð4Ô5ð'!ð ˜RœYœð'!ð ˜BœI”ð'!ð '!ð '!ð '!ð '!ð '!ð '!ð '!ð '!ð '!r1   r   c                  ó\   — |                       dd ¦  «        |                       dd ¦  «        dœS )Nr)   r*   r(   )Úget)Úkwargss    r/   Ú_dd_from_kwargsrO   K   s,   € Ø—j’j ¨4Ñ0Ô0¸6¿:º:ÀgÈtÑ;TÔ;TÐUÐUÐUr1   ©r   é   é	   c                 ó|  — |                      dd¦  «        }|rdnd}|rt          t          d¬¦  «        nt          t          d¬¦  «        }t	          | ¦  «        r3t          d| dd|                      dd	¦  «        d
||dœt          di |¤Ž¤Ž}n/t          |                      dd	¦  «        f|d
|dœt          di |¤Ž¤Ž}|S )z ResNet-V2 backbone helperÚpadding_sameTÚsamer   g:Œ0âŽyE>)Úepsr   r   r   F)ÚlayersÚnum_classesÚglobal_poolr   ÚpreactÚ	stem_typeÚ
conv_layer)r[   rZ   r\   r,   )rM   r   r
   r   r<   r   rO   r   )rW   rN   rT   r[   r\   Úbackbones         r/   Ú	_resnetv2r^   O   s  € à—:’:˜n¨dÑ3Ô3€LØ&Ð.��¨B€IØ5AÐc•�¨DÐ1Ñ1Ô1Ð1ÅwÍyÐ^bÐGcÑGcÔGc€JÝ
ˆ6�{„{ð 
Ýð 	
ØØØØ—Z’Z 
¨AÑ.Ô.ØØØ!ð	
ð 	
õ Ð'Ð' Ð'Ð'ð	
ð 	
ˆˆõ (Ø�JŠJ�z 1Ñ%Ô%ð
àØØ!ð	
ð 
õ
 Ð'Ð' Ð'Ð'ð
ð 
ˆð €Or1   úimage_encoder.model.c                 ó   — i }|                       ¦   «         D �]µ\  }}|                     |¦  «        sŒ|                     |d¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd	¦  «        }|                     d
d¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|dk    rd}|                     d¦  «        }d|v rU|                     dd¦  «        }|                     dd¦  «        }|j        }t          j        |j        d         ¦  «        ||<   |||<   �Œ·|S ) Nr   z
patch_emb.zpatch_embed.backbone.z
block.convÚconvz
block.normÚbnzpost_transformer_norm.znorm.zpre_norm_mha.0Únorm1zpre_norm_mha.1Úattnzpre_norm_ffn.0Únorm2zpre_norm_ffn.1zmlp.fc1zpre_norm_ffn.4zmlp.fc2z	qkv_proj.zqkv.z	out_proj.zproj.ztransformer.zblocks.zpos_embed.pos_embed.pos_embedÚ	pos_embedr   zclassifier.projz	head.biaszhead.weight)ÚitemsÚ
startswithÚreplaceÚsqueezeÚTÚtorchÚzerosÚshape)Ú
state_dictÚmodelÚprefixÚoutÚkÚvÚbias_ks          r/   Ú_convert_mobilecliprv   j   s¾  € Ø
€CØ× Ò Ñ"Ô"ð ñ ‰ˆˆ1Ø�|Š|˜FÑ#Ô#ð 	ØØ�IŠI�f˜bÑ!Ô!ˆØ�IŠI�lÐ$;Ñ<Ô<ˆØ�IŠI�l FÑ+Ô+ˆØ�IŠI�l DÑ)Ô)ˆØ�IŠIÐ.°Ñ8Ô8ˆØ�IŠIÐ&¨Ñ0Ô0ˆØ�IŠIÐ&¨Ñ/Ô/ˆØ�IŠIÐ&¨Ñ0Ô0ˆØ�IŠIÐ&¨	Ñ2Ô2ˆØ�IŠIÐ&¨	Ñ2Ô2ˆØ�IŠI�k 6Ñ*Ô*ˆØ�IŠI�k 7Ñ+Ô+ˆØ�IŠI�n iÑ0Ô0ˆØÐ/Ò/Ð/ØˆAØ—	’	˜!‘”ˆAØ Ð!Ð!Ø—Y’YÐ0°+Ñ>Ô>ˆFØ—	’	Ð+¨]Ñ;Ô;ˆAØ”ˆAÝœ+ a¤g¨a¤jÑ1Ô1ˆC�‰KØˆˆA‰‰Ø€Jr1   ÚbicubicTro   rp   ÚinterpolationÚ	antialiasÚreturnc                 óT   — ddl m} d| v rt          | |¦  «        }  || |||¬¦  «        S )Nr   )Úcheckpoint_filter_fnz1image_encoder.model.patch_emb.0.block.conv.weight)rx   ry   )Úvision_transformerr|   rv   )ro   rp   rx   ry   Ú
_filter_fns        r/   r|   r|   ˆ   sK   € ð GÐFÐFÐFÐFÐFà:¸jÐHÐHÝ(¨°UÑ;Ô;ˆ
àˆ:�j %°}ÐPYÐZÑZÔZÐZr1   Fc                 ó  — |                      dd¦  «        }|pi }t          t          fd|i|¤Ž}|                     d|¦  «         |                     dd¦  «         t	          t
          | |ft          t          |d¬¦  «        d	œ|¤ŽS )
NÚout_indicesr   r]   Úembed_layerÚ
patch_sizer   Úgetter)r€   Úfeature_cls)Úpretrained_filter_fnÚfeature_cfg)Úpopr   r   Ú
setdefaultr   r   r|   Údict)Úvariantr]   Ú
embed_argsÚ
pretrainedrN   r€   r�   s          r/   Ú!_create_vision_transformer_hybridr�   –   s©   € Ø—*’*˜]¨AÑ.Ô.€KØÐ!˜r€JÝ�+ÐGÐG°ÐG¸JÐGÐG€KØ
×Ò�m [Ñ1Ô1Ð1Ø
×Ò�l AÑ&Ô&Ð&ÝÝØØðõ 2Ý [¸hÐGÑGÔGðð ð ðð ð r1   r   c                 ó$   — | ddd dddddddd	d
œ|¥S )Niè  )r   éà   r�   çÍÌÌÌÌÌì?rw   T)ç      à?r‘   r‘   zpatch_embed.backbone.stem.convÚheadz
apache-2.0)ÚurlrX   Ú
input_sizeÚ	pool_sizeÚcrop_pctrx   Úfixed_input_sizeÚmeanÚstdÚ
first_convÚ
classifierÚlicenser,   )r“   rN   s     r/   Ú_cfgr�   ¦   s9   € àØ¨=ÀtØ¨ÈØ¨Ø6ÀfØðð ð ðð r1   z*vit_tiny_r_s16_p8_224.augreg_in21k_ft_in1kz—https://storage.googleapis.com/vit_models/augreg/R_Ti_16-i21k-300ep-lr_0.001-aug_none-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_224.npzztimm/zpatch_embed.backbone.conv)r“   Ú	hf_hub_idÚcustom_loadrš   z*vit_tiny_r_s16_p8_384.augreg_in21k_ft_in1kz—https://storage.googleapis.com/vit_models/augreg/R_Ti_16-i21k-300ep-lr_0.001-aug_none-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_384.npz)r   é€  r    ç      ð?)r“   rž   rš   r”   r–   rŸ   z*vit_small_r26_s32_224.augreg_in21k_ft_in1kzšhttps://storage.googleapis.com/vit_models/augreg/R26_S_32-i21k-300ep-lr_0.001-aug_light0-wd_0.03-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.03-res_224.npz)r“   rž   rŸ   z*vit_small_r26_s32_384.augreg_in21k_ft_in1kz›https://storage.googleapis.com/vit_models/augreg/R26_S_32-i21k-300ep-lr_0.001-aug_medium2-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_384.npz)r“   rž   r”   r–   rŸ   zvit_base_r26_s32_224.untrainedz'vit_base_r50_s16_384.orig_in21k_ft_in1kzthttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-vitjx/jx_vit_base_resnet50_384-9fd3c705.pth)r“   rž   r”   r–   z*vit_large_r50_s32_224.augreg_in21k_ft_in1kzšhttps://storage.googleapis.com/vit_models/augreg/R50_L_32-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.01-res_224.npzz*vit_large_r50_s32_384.augreg_in21k_ft_in1kzšhttps://storage.googleapis.com/vit_models/augreg/R50_L_32-i21k-300ep-lr_0.001-aug_medium2-wd_0.1-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.01-res_384.npzz"vit_tiny_r_s16_p8_224.augreg_in21kzohttps://storage.googleapis.com/vit_models/augreg/R_Ti_16-i21k-300ep-lr_0.001-aug_none-wd_0.03-do_0.0-sd_0.0.npziSU  r�   )r“   rž   rX   r–   rš   rŸ   z"vit_small_r26_s32_224.augreg_in21kzshttps://storage.googleapis.com/vit_models/augreg/R26_S_32-i21k-300ep-lr_0.001-aug_medium2-wd_0.03-do_0.0-sd_0.0.npz)r“   rž   rX   r–   rŸ   zvit_base_r50_s16_224.orig_in21k)rž   rX   r–   z"vit_large_r50_s32_224.augreg_in21kzrhttps://storage.googleapis.com/vit_models/augreg/R50_L_32-i21k-300ep-lr_0.001-aug_medium2-wd_0.1-do_0.0-sd_0.0.npzz!vit_small_resnet26d_224.untrainedzpatch_embed.backbone.conv1.0)r˜   r™   rš   z%vit_small_resnet50d_s16_224.untrainedz vit_base_resnet26d_224.untrainedz vit_base_resnet50d_224.untrainedzvit_base_mci_224.apple_mclip_ltzapple/mobileclip_b_lt_timmzYhttps://docs-assets.developer.apple.com/ml-research/datasets/mobileclip/mobileclip_blt.ptz
apple-amlri   )ç        r¢   r¢   )r¡   r¡   r¡   zpatch_embed.backbone.0.conv)rž   r“   rœ   rX   r˜   r™   rš   zapple/mobileclip_b_timmzWhttps://docs-assets.developer.apple.com/ml-research/datasets/mobileclip/mobileclip_b.pt)rž   r“   rX   rœ   r˜   r™   rš   )rž   rX   r˜   r™   rš   rœ   )zvit_base_mci_224.apple_mclipz%vit_base_mci_224.apple_mclip2_dfndr2bc           	      ó~   — t          dddi|¤Ž}t          dddd¬¦  «        }t          	 d
|| d	œt          |fi |¤Ž¤Ž}|S )z3 R+ViT-Ti/S16 w/ 8x8 patch hybrid @ 224 x 224.
    rW   r,   é   éÀ   é   r   ©r‚   Ú	embed_dimr    Ú	num_headsÚvit_tiny_r_s16_p8_224©r]   rŒ   )rª   ©r^   r‰   r�   ©rŒ   rN   r]   Ú
model_argsrp   s        r/   rª   rª   	  ów   € õ Ð-Ð- Ð- fÐ-Ð-€HÝ ¨c¸ÀqÐIÑIÔI€JÝ-ØðiØ*2¸zðið iÝMQÐR\ÐMgÐMgÐ`fÐMgÐMgðið i€Eà€Lr1   c           	      ó~   — t          dddi|¤Ž}t          dddd¬¦  «        }t          	 d
|| d	œt          |fi |¤Ž¤Ž}|S )z3 R+ViT-Ti/S16 w/ 8x8 patch hybrid @ 384 x 384.
    rW   r,   r¤   r¥   r¦   r   r§   Úvit_tiny_r_s16_p8_384r«   )r±   r¬   r­   s        r/   r±   r±     r¯   r1   c           	      óx   — t          di |¤Ž}t          ddd¬¦  «        }t          	 d	|| dœt          |fi |¤Ž¤Ž}|S )
ú R26+ViT-S/S32 hybrid.
    ©r   r   r   r   r    r¦   é   ©r¨   r    r©   Úvit_small_r26_s32_224r«   ©r´   )r·   r¬   r­   s        r/   r·   r·     óp   € õ Ð0Ð0¨Ð0Ð0€HÝ ¨2¸Ð;Ñ;Ô;€JÝ-ØðiØ*2¸zðið iÝMQÐR\ÐMgÐMgÐ`fÐMgÐMgðið i€Eà€Lr1   c           	      óx   — t          di |¤Ž}t          ddd¬¦  «        }t          	 d	|| dœt          |fi |¤Ž¤Ž}|S )
r³   r´   r    r¦   rµ   r¶   Úvit_small_r26_s32_384r«   r¸   )r»   r¬   r­   s        r/   r»   r»   *  r¹   r1   c           	      óx   — t          di |¤Ž}t          ddd¬¦  «        }t          	 d|| dœt          |fi |¤Ž¤Ž}|S )	z R26+ViT-B/S32 hybrid.
    r´   é   r¦   r¶   Úvit_base_r26_s32_224r«   r¸   )r¾   r¬   r­   s        r/   r¾   r¾   5  sp   € õ Ð0Ð0¨Ð0Ð0€HÝ ¨2¸Ð<Ñ<Ô<€JÝ-ØðhØ)1¸jðhð hÝLPÐQ[ÐLfÐLfÐ_eÐLfÐLfðhð h€Eà€Lr1   c           	      óx   — t          di |¤Ž}t          ddd¬¦  «        }t          	 d|| dœt          |fi |¤Ž¤Ž}|S )	zR R50+ViT-B/S16 hybrid from original paper (https://arxiv.org/abs/2010.11929).
    rP   r½   r¦   r¶   Úvit_base_r50_s16_224r«   ©rP   )rÀ   r¬   r­   s        r/   rÀ   rÀ   @  sp   € õ Ð-Ð- fÐ-Ð-€HÝ ¨2¸Ð<Ñ<Ô<€JÝ-ØðhØ)1¸jðhð hÝLPÐQ[ÐLfÐLfÐ_eÐLfÐLfðhð h€Eà€Lr1   c           	      óx   — t          di |¤Ž}t          ddd¬¦  «        }t          	 d|| dœt          |fi |¤Ž¤Ž}|S )	zÈ R50+ViT-B/16 hybrid from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k @ 384x384, source https://github.com/google-research/vision_transformer.
    rP   r½   r¦   r¶   Úvit_base_r50_s16_384r«   rÁ   )rÃ   r¬   r­   s        r/   rÃ   rÃ   K  sp   € õ
 Ð-Ð- fÐ-Ð-€HÝ ¨2¸Ð<Ñ<Ô<€JÝ-ØðhØ)1¸jðhð hÝLPÐQ[ÐLfÐLfÐ_eÐLfÐLfðhð h€Eà€Lr1   c           	      óx   — t          di |¤Ž}t          ddd¬¦  «        }t          	 d	|| dœt          |fi |¤Ž¤Ž}|S )
ú R50+ViT-L/S32 hybrid.
    ©r   rQ   rµ   r   é   é   é   r¶   Úvit_large_r50_s32_224r«   ©rÆ   )rÊ   r¬   r­   s        r/   rÊ   rÊ   W  óp   € õ Ð0Ð0¨Ð0Ð0€HÝ ¨B¸"Ð=Ñ=Ô=€JÝ-ØðiØ*2¸zðið iÝMQÐR\ÐMgÐMgÐ`fÐMgÐMgðið i€Eà€Lr1   c           	      óx   — t          di |¤Ž}t          ddd¬¦  «        }t          	 d	|| dœt          |fi |¤Ž¤Ž}|S )
rÅ   rÆ   rÇ   rÈ   rÉ   r¶   Úvit_large_r50_s32_384r«   rË   )rÎ   r¬   r­   s        r/   rÎ   rÎ   b  rÌ   r1   c           	      óÂ   — t          d| |                     dd¦  «        ddgdœt          di |¤Ž¤Ž}t          dddd¬¦  «        }t	          	 d|| d
œt          |fi |¤Ž¤Ž}|S )zL Custom ViT small hybrid w/ ResNet26D stride 32. No pretrained weights.
    r   r   TrQ   ©rŒ   r   Úfeatures_onlyr€   r½   r¤   ©r¨   r    r©   Ú	mlp_ratioÚvit_small_resnet26d_224r«   r,   )rÔ   ©r   rM   rO   r‰   r�   r­   s        r/   rÔ   rÔ   m  s®   € õ ð ØØ—’˜J¨Ñ*Ô*ØØ�Cð	ð õ
 Ð
#Ð
#˜FÐ
#Ð
#ðð €Hõ  ¨1¸ÀQÐGÑGÔG€JÝ-Ø!ðkØ,4Àðkð kÝOSÐT^ÐOiÐOiÐbhÐOiÐOiðkð k€Eà€Lr1   c           	      óÂ   — t          d
| |                     dd¦  «        ddgdœt          d
i |¤Ž¤Ž}t          dddd¬¦  «        }t	          	 d|| d	œt          |fi |¤Ž¤Ž}|S )zV Custom ViT small hybrid w/ ResNet50D 3-stages, stride 16. No pretrained weights.
    r   r   TrÐ   r½   r¤   rÒ   Úvit_small_resnet50d_s16_224r«   r,   )r×   ©r   rM   rO   r‰   r�   r­   s        r/   r×   r×   ~  s®   € õ ð ØØ—’˜J¨Ñ*Ô*ØØ�Cð	ð õ
 Ð
#Ð
#˜FÐ
#Ð
#ðð €Hõ  ¨1¸ÀQÐGÑGÔG€JÝ-Ø%ðoØ08ÀZðoð oÝSWÐXbÐSmÐSmÐflÐSmÐSmðoð o€Eà€Lr1   c           	      óÀ   — t          d| |                     dd¦  «        ddgdœt          di |¤Ž¤Ž}t          ddd¬¦  «        }t	          	 d|| d
œt          |fi |¤Ž¤Ž}|S )zK Custom ViT base hybrid w/ ResNet26D stride 32. No pretrained weights.
    r   r   TrQ   rÐ   r½   r¦   r¶   Úvit_base_resnet26d_224r«   r,   )rÚ   rÕ   r­   s        r/   rÚ   rÚ   �  ó¬   € õ ð ØØ—’˜J¨Ñ*Ô*ØØ�Cð	ð õ
 Ð
#Ð
#˜FÐ
#Ð
#ðð €Hõ  ¨2¸Ð<Ñ<Ô<€JÝ-Ø ðjØ+3À
ðjð jÝNRÐS]ÐNhÐNhÐagÐNhÐNhðjð j€Eà€Lr1   c           	      óÀ   — t          d| |                     dd¦  «        ddgdœt          di |¤Ž¤Ž}t          ddd¬¦  «        }t	          	 d|| d
œt          |fi |¤Ž¤Ž}|S )úK Custom ViT base hybrid w/ ResNet50D stride 32. No pretrained weights.
    r   r   TrQ   rÐ   r½   r¦   r¶   Úvit_base_resnet50d_224r«   r,   )rÞ   rØ   r­   s        r/   rÞ   rÞ      rÛ   r1   c                 óö   — t          ddddd|                     dd¦  «        t          j        dœt	          di |¤Ž¤Ž}t          dddd	¬
¦  «        }t          	 d|t          d¬¦  «        | dœt          |fi |¤Ž¤Ž}|S )rÝ   )r¥   r¥   r½   )rQ   r   r   r   r   r   )r!   r#   r"   r$   r   r&   r½   r¦   T)r¨   r    r©   Úno_embed_classÚvit_base_mci_224F)Úproj)r]   r‹   rŒ   r,   )rá   )r   rM   rF   ÚGELUrO   r‰   r�   r­   s        r/   rá   rá   ±  s½   € õ ð Ø&ØØØØ—’˜J¨Ñ*Ô*Ý”'ðð õ Ð
#Ð
#˜FÐ
#Ð
#ðð €Hõ  ¨2¸ÈDÐQÑQÔQ€JÝ-ØðØ%-½$ÀEÐ:JÑ:JÔ:JØðð å!% jÐ!;Ð!;°FÐ!;Ð!;ðð €Eð €Lr1   )Úvit_tiny_r_s16_p8_224_in21kÚvit_small_r26_s32_224_in21kÚvit_base_r50_s16_224_in21kÚvit_base_resnet50_224_in21kÚvit_large_r50_s32_224_in21kÚvit_base_resnet50_384rÁ   )r_   )rw   T)NF)r   )F)=Ú__doc__Ú	functoolsr   Útypingr   r   r   r   rl   Útorch.nnrF   Ú	timm.datar   r	   Útimm.layersr
   r   r   r   r   Ú_builderr   Ú	_registryr   r   r   Úresnetr   r   Úresnetv2r   r   r}   r   Ú
Sequentialr   rO   r^   rv   rI   ÚTensorÚboolr|   r�   r�   Údefault_cfgsrª   r±   r·   r»   r¾   rÀ   rÃ   rÊ   rÎ   rÔ   r×   rÚ   rÞ   rá   rC   r,   r1   r/   ú<module>rø      s@  ððð ð Ð Ð Ð Ð Ð Ø +Ð +Ð +Ð +Ð +Ð +Ð +Ð +Ð +Ð +Ð +Ð +à €€€Ø Ð Ð Ð Ð Ð à AÐ AÐ AÐ AÐ AÐ AÐ AÐ AØ UÐ UÐ UÐ UÐ UÐ UÐ UÐ UÐ UÐ UÐ UÐ UÐ UÐ Uà *Ð *Ð *Ð *Ð *Ð *Ø YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YØ (Ð (Ð (Ð (Ð (Ð (Ð (Ð (Ø 4Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1ð(!ð (!ð (!ð (!ð (!ˆrŒ}ñ (!ô (!ð (!ðVVð Vð Vðð ð ð ð6ð ð ð ðB 'Øð	[ð [Ø˜˜eœlÐ*Ô+ð[à ð[ð ð[ð ð	[ð
 
ˆ#ˆuŒ|Ð
Ôð[ð [ð [ð [ðð ð ð ð 	ð 	ð 	ð 	ð %Ð$ð T&à0°$°$ð fØØØ.ð	30ñ 30ô 30ðT&ð 1°$°$ð fØØ.¸=ÐSVÐdhð3jñ 3jô 3jðT&ð 1°$°$ð iØØð3ñ 3ô 3ðT&ð  1°$°$ð jØØ ¨3¸Dð3Bñ 3Bô 3Bð!T&ð( % d d¡f¤fð)T&ð* .¨t¨tð CØØ ¨3ð00ñ 00ô 00ð+T&ð2 1°$°$ð iØØð3ñ 3ô 3ð3T&ð< 1°$°$ð iØØ ¨3¸Dð3ñ 3ô 3ð=T&ðJ )¨$¨$Ø}ØØ CÐ4OÐ]að+cñ +cô +cðKT&ðR )¨$¨$ð BØØ C°Tð+;ñ +;ô +;ðST&ðZ & t tàØ ð(%ñ (%ô (%ð[T&ðb )¨$¨$ð AØØ C°Tð+;ñ +;ô +;ðcT&ðn (¨¨Ø"Ð(<ÐIgð*iñ *iô *iðoT&ðr ,¨T¨TØ"Ð(<ÐIgð.iñ .iô .iðsT&ðv '¨¨Ø"Ð(<ÐIgð)iñ )iô )iðwT&ðz '¨¨Ø"Ð(<ÐIgð)iñ )iô )ið{T&ð@ & t tØ.ØgØØØ˜|Ð8Uð(ñ (ô (ðAT&ðN %) DØ+ØeØØØ˜|Ð8Uð%ñ %ô %ð .2¨TØØØ˜|Ð8UØð	.ñ .ô .ð]T&ð T&ð T&ñ Tô T€ðn ðð Ð9Jð ð ð ñ „ðð ðð Ð9Jð ð ð ñ „ðð ðð Ð9Jð ð ð ñ „ðð ðð Ð9Jð ð ð ñ „ðð ðð Ð8Ið ð ð ñ „ðð ðð Ð8Ið ð ð ñ „ðð ðð Ð8Ið ð ð ñ „ðð ðð Ð9Jð ð ð ñ „ðð ðð Ð9Jð ð ð ñ „ðð ðð Ð;Lð ð ð ñ „ðð  ðð Ð?Pð ð ð ñ „ðð  ðð Ð:Kð ð ð ñ „ðð  ðð Ð:Kð ð ð ñ „ðð  ðð Ð4Eð ð ð ñ „ðð( Ð ˜HØ#GØ#GØ"CØ#DØ#GØFð'ð 'ñ ô ð ð ð r1   