§
    ·¯¸j!‹ ã                   óF#  — d Z ddlZddlmZ ddlmZmZmZ ddlm	Z	 ddl
mZmZmZmZmZmZmZmZ ddlZddlmZ ddlmZ dd	lmZmZ dd
lmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4 ddl5m6Z6 ddl7m8Z8 ddl9m:Z: ddl;m<Z<m=Z= ddl>m?Z?m@Z@ g d¢ZAe G d„ d¦  «        ¦   «         ZBe G d„ d¦  «        ¦   «         ZCe G d„ d¦  «        ¦   «         ZD G d„ dejE        ¦  «        ZF G d„ dejE        ¦  «        ZG G d„ dejE        ¦  «        ZH�d‰dejE        d eId!eId"dfd#„ZJ G d$„ d%ejE        ¦  «        ZK�d‰dejE        d eId!eId"dfd&„ZLd'eeM         d(eMd"eMfd)„ZN G d*„ d+ejE        ¦  «        ZO G d,„ d-ejE        ¦  «        ZPd.ejQ        d/eeM         d"ejQ        fd0„ZRe:d1ejQ        d/eeM         d2eeM         d"ejQ        fd3„¦   «         ZSd.ejQ        d4eeM         d"ejQ        fd5„ZTe:d1ejQ        d4eeM         d2eeM         d"ejQ        fd6„¦   «         ZUd7eBd/eeMeMf         d"ee         fd8„ZV G d9„ d:ejE        ¦  «        ZW G d;„ d<ejE        ¦  «        ZXd.ejQ        d/eeM         d"ejQ        fd=„ZYe:d1ejQ        d/eeM         d2eeM         d"ejQ        fd>„¦   «         ZZd.ejQ        d4eeM         d"ejQ        fd?„Z[e:d1ejQ        d4eeM         d2eeM         d"ejQ        fd@„¦   «         Z\ G dA„ dBejE        ¦  «        Z] G dC„ dDejE        ¦  «        Z^ G dE„ dFejE        ¦  «        Z_ G dG„ dHejE        ¦  «        Z` G dI„ dJejE        ¦  «        Zad7eBd2eeMeMf         d"eBfdK„Zbd7eDdLed"eDfdM„Zc G dN„ dOejE        ¦  «        Zd	 	 	 	 	 	 	 	 	 	 	 	 �dŠdYeIdZeId[eed\eed]eId^eId_eed`eIdaeIdbeef         dceIddeMd"eeIef         fde„Zg	 	 	 	 	 	 	 	 	 	 	 	 �d‹dYeIdZeId[eediefd^eId`eIdaeId/eeeMeMf                  djeMdbeef         dceIddeMd"eeIef         fdk„Zh	 	 	 	 	 	 	 	 	 	 	 �dŒdYeIdZeId^eIdneId`eIdaeId/eeeMeMf                  doeedbeefeefeff         f         dceIddeMd"eeIef         fdp„Zid"eeIef         fdq„Zj ek�d�i dr eD�d�dsdtdudvœ ehdTdw¬x¦  «        ¤Ž“dy eD�d�dsdzdudvœ ehdPdTdw¬{¦  «        ¤Ž“d| eD�d�d}d~dudvœ egdTdR¬¦  «        ¤Ž“d€ eD�d�d}d�dudvœ egdfdTdR¬‚¦  «        ¤Ž“dƒ eD�d�d„d�d…dvœ egdfd†¬‡¦  «        ¤Ž“dˆ eD�d�d‰d�dŠdvœ egdfd†dl¬‹¦  «        ¤Ž“dŒ eD�d�d}d~dudvœ egdfdTdRd�¬Ž¦  «        ¤Ž“d� eD�d�dsdzdudvœ ehdTdwdmd�¬‘¦  «        ¤Ž“d’ eD�d�d}d~dudvœ egdfdm¬“¦  «        ¤Ž“d” eD�d�d}d�dudvœ egd•dTdRdmd�¬–¦  «        ¤Ž“d— eD�d�d}d�dudvœ egdfdmdX¬˜¦  «        ¤Ž“d™ eD�d�d„d�d…dvœ egdfd†dldm¬š¦  «        ¤Ž“d› eD�d�d‰d�dŠdvœ egdfd†dldm¬š¦  «        ¤Ž“dœ eD�d�dsdzdud�džœ eg¦   «         ¤Ž“dŸ eD�d�dsdzdud d¡œ eidWd¢¬£¦  «        ¤Ž“d¤ eDd}dtd¥d¦¬§¦  «        “d¨ eDd}d�d¥d¦¬§¦  «        “d© eDd„d�dªd«¬§¦  «        “d¬ eDd‰d�d­d®¬§¦  «        “d¯ eDd‰d°d­d®¬§¦  «        “d± eDd²d°d­d³¬§¦  «        “d´ eD�d�dµd¶d·d¸d¹œ eh¦   «         ¤Ž“dº eD�d�dsd»d·dud¹œ eh¦   «         ¤Ž“d¼ eD�d�dsd¶d·dud¹œ eh¦   «         ¤Ž“d½ eD�d�dsd¶d¾dud¹œ eh¦   «         ¤Ž“d¿ eD�d�dµd¶d·d¸d¹œ ehdm¬À¦  «        ¤Ž“dÁ eD�d�dsd»d·dud¹œ ehdm¬À¦  «        ¤Ž“dÂ eD�d�dsd¶d·dud¹œ ehdm¬À¦  «        ¤Ž“dÃ eD�d�d}d¶d·dud¹œ ehdmdl¬£¦  «        ¤Ž“dÄ eD�d�d}d�d·dud¦dÅœ ehdm¬À¦  «        ¤Ž“dÆ eD�d�dsd»d·dud dÇœ ei¦   «         ¤Ž“dÈ eD�d�dsd¶d·dud¹œ ei¦   «         ¤Ž“dÉ eD�d�d}d¶d·dÊd¹œ ei¦   «         ¤Ž“dË eD�d�d}d»d·dÊd dÇœ eidTdW¬Ì¦  «        ¤Ž“dÍ eD�d�d„dÎd·d…d¹œ eidT¬Ï¦  «        ¤Ž“dÐ eD�d�dÑdÒd·dÓdÔdÅœ eidT¬Ï¦  «        ¤Ž“dÕ eD�d�dsd¶d·d¥dTdXdÖœ ej¦   «         ¤Ž“d× eD�d�d}d¶d·d¥dTd¦dÖœ ej¦   «         ¤Ž“dØ eD�d�d}d�d·d¥dTd¦dÖœ ej¦   «         ¤Ž“dÙ eD�d�d„d�d·dªdTd«dÖœ ej¦   «         ¤Ž“dÚ eD�d�d‰d�d·d­dTd®dÖœ ej¦   «         ¤Ž“ŽZldÛeeIejQ        f         dÜejE        d"eeIejQ        f         fdÝ„Zm�dŽdÞeIdßeeI         dàeedLed"edf
dá„Zn�d‰dâeIdLed"eeIef         fdã„Zo e?i dä eod¬å¦  «        “dæ eodçdèdé¬ê¦  «        “dë eodçdì¬í¦  «        “dî eodçdï¬í¦  «        “dð eodç¬ñ¦  «        “dò eodç¬ñ¦  «        “dó eodç¬ñ¦  «        “dô eodçdõdöd÷dø¬ù¦  «        “dú eodçdûeedü¬ý¦  «        “dþ eodçdÿdé¬ê¦  «        “�d  eod¬å¦  «        “�d eodç�d¬í¦  «        “�d eodç�d¬í¦  «        “�d eod¬å¦  «        “�d eod¬å¦  «        “�d eodç�ddé¬ê¦  «        “�d	 eodç�d
�¬¦  «        “i �d eodç�d
�¬¦  «        “�d eodç�d
�¬¦  «        “�d eodç�d
�¬¦  «        “�d eod¬å¦  «        “�d eod¬å¦  «        “�d eod¬å¦  «        “�d eod¬å¦  «        “�d eod¬å¦  «        “�d eod¬å¦  «        “�d eod�d�d�¬¦  «        “�d eodç�d�d�d�¬¦  «        “�d eodç�d¬í¦  «        “�d eod�d�d�¬¦  «        “�d eod�d�d�¬¦  «        “�d  eodç�d!�d�d�¬¦  «        “�d" eodç�d#�d�d�¬¦  «        “�d$ eodç�d%�d�d�¬¦  «        “¥i �d& eodç�d'dé¬ê¦  «        “�d( eod�d�d�¬¦  «        “�d) eodç¬ñ¦  «        “�d* eodçdõdöd÷dø¬ù¦  «        “�d+ eodç�d
�¬¦  «        “�d, eodç�d-�d�d�¬¦  «        “�d. eod�d�d�¬¦  «        “�d/ eodç�d0�d�d�¬¦  «        “�d1 eodç�d�d�¬2¦  «        “�d3 eodç¬ñ¦  «        “�d4 eodçdõdöd÷dø¬ù¦  «        “�d5 eod¬å¦  «        “�d6 eodç�d
�¬¦  «        “�d7 eodçee�¬8¦  «        “�d9 eodçdõdöd÷dø¬ù¦  «        “�d: eodç�d;�d<d÷dø¬ù¦  «        “�d= eodçee�¬8¦  «        “¥i �d> eodçdõdöd÷dø¬ù¦  «        “�d? eodç�d;�d<d÷dø¬ù¦  «        “�d@ eodçee�¬8¦  «        “�dA eodçdõdöd÷dø¬ù¦  «        “�dB eodç�d;�d<d÷dø¬ù¦  «        “�dC eodçee�¬8¦  «        “�dD eodçdõdöd÷dø¬ù¦  «        “�dE eodç�d;�d<d÷dø¬ù¦  «        “�dF eodç�dG�¬¦  «        “�dH eodçdõdöd÷dø¬ù¦  «        “�dI eodç�d;�d<d÷dø¬ù¦  «        “�dJ eodç�dG�¬¦  «        “�dK eodçdõdöd÷dø¬ù¦  «        “�dL eodç�d;d÷dø�¬M¦  «        “�dN eodç�dG�¬¦  «        “�dO eodçdõdöd÷dø¬ù¦  «        “�dP eodç�d;�d<d÷dø¬ù¦  «        “¥¦  «        Zpe@�d�dàeedLed"edf�dQ„¦   «         Zqe@�d�dàeedLed"edf�dR„¦   «         Zre@�d�dàeedLed"edf�dS„¦   «         Zse@�d�dàeedLed"edf�dT„¦   «         Zte@�d�dàeedLed"edf�dU„¦   «         Zue@�d�dàeedLed"edf�dV„¦   «         Zve@�d�dàeedLed"edf�dW„¦   «         Zwe@�d�dàeedLed"edf�dX„¦   «         Zxe@�d�dàeedLed"edf�dY„¦   «         Zye@�d�dàeedLed"edf�dZ„¦   «         Zze@�d�dàeedLed"edf�d[„¦   «         Z{e@�d�dàeedLed"edf�d\„¦   «         Z|e@�d�dàeedLed"edf�d]„¦   «         Z}e@�d�dàeedLed"edf�d^„¦   «         Z~e@�d�dàeedLed"edf�d_„¦   «         Ze@�d�dàeedLed"edf�d`„¦   «         Z€e@�d�dàeedLed"edf�da„¦   «         Z�e@�d�dàeedLed"edf�db„¦   «         Z‚e@�d�dàeedLed"edf�dc„¦   «         Zƒe@�d�dàeedLed"edf�dd„¦   «         Z„e@�d�dàeedLed"edf�de„¦   «         Z…e@�d�dàeedLed"edf�df„¦   «         Z†e@�d�dàeedLed"edf�dg„¦   «         Z‡e@�d�dàeedLed"edf�dh„¦   «         Zˆe@�d�dàeedLed"edf�di„¦   «         Z‰e@�d�dàeedLed"edf�dj„¦   «         ZŠe@�d�dàeedLed"edf�dk„¦   «         Z‹e@�d�dàeedLed"edf�dl„¦   «         ZŒe@�d�dàeedLed"edf�dm„¦   «         Z�e@�d�dàeedLed"edf�dn„¦   «         ZŽe@�d�dàeedLed"edf�do„¦   «         Z�e@�d�dàeedLed"edf�dp„¦   «         Z�e@�d�dàeedLed"edf�dq„¦   «         Z‘e@�d�dàeedLed"edf�dr„¦   «         Z’e@�d�dàeedLed"edf�ds„¦   «         Z“e@�d�dàeedLed"edf�dt„¦   «         Z”e@�d�dàeedLed"edf�du„¦   «         Z•e@�d�dàeedLed"edf�dv„¦   «         Z–e@�d�dàeedLed"edf�dw„¦   «         Z—e@�d�dàeedLed"edf�dx„¦   «         Z˜e@�d�dàeedLed"edf�dy„¦   «         Z™e@�d�dàeedLed"edf�dz„¦   «         Zše@�d�dàeedLed"edf�d{„¦   «         Z›e@�d�dàeedLed"edf�d|„¦   «         Zœe@�d�dàeedLed"edf�d}„¦   «         Z�e@�d�dàeedLed"edf�d~„¦   «         Zže@�d�dàeedLed"edf�d„¦   «         ZŸe@�d�dàeedLed"edf�d€„¦   «         Z e@�d�dàeedLed"edf�d�„¦   «         Z¡e@�d�dàeedLed"edf�d‚„¦   «         Z¢e@�d�dàeedLed"edf�dƒ„¦   «         Z£e@�d�dàeedLed"edf�d„„¦   «         Z¤e@�d�dàeedLed"edf�d…„¦   «         Z¥e@�d�dàeedLed"edf�d†„¦   «         Z¦e@�d�dàeedLed"edf�d‡„¦   «         Z§e@�d�dàeedLed"edf�dˆ„¦   «         Z¨dS (�  a   MaxVit and CoAtNet Vision Transformer - CNN Hybrids in PyTorch

This is a from-scratch implementation of both CoAtNet and MaxVit in PyTorch.

99% of the implementation was done from papers, however last minute some adjustments were made
based on the (as yet unfinished?) public code release https://github.com/google-research/maxvit

There are multiple sets of models defined for both architectures. Typically, names with a
 `_rw` suffix are my own original configs prior to referencing https://github.com/google-research/maxvit.
These configs work well and appear to be a bit faster / lower resource than the paper.

The models without extra prefix / suffix' (coatnet_0_224, maxvit_tiny_224, etc), are intended to
match paper, BUT, without any official pretrained weights it's difficult to confirm a 100% match.

Papers:

MaxViT: Multi-Axis Vision Transformer - https://arxiv.org/abs/2204.01697
@article{tu2022maxvit,
  title={MaxViT: Multi-Axis Vision Transformer},
  author={Tu, Zhengzhong and Talebi, Hossein and Zhang, Han and Yang, Feng and Milanfar, Peyman and Bovik, Alan and Li, Yinxiao},
  journal={ECCV},
  year={2022},
}

CoAtNet: Marrying Convolution and Attention for All Data Sizes - https://arxiv.org/abs/2106.04803
@article{DBLP:journals/corr/abs-2106-04803,
  author    = {Zihang Dai and Hanxiao Liu and Quoc V. Le and Mingxing Tan},
  title     = {CoAtNet: Marrying Convolution and Attention for All Data Sizes},
  journal   = {CoRR},
  volume    = {abs/2106.04803},
  year      = {2021}
}

Hacked together by / Copyright 2022, Ross Wightman
é    N)ÚOrderedDict)Ú	dataclassÚreplaceÚfield)Úpartial)ÚAnyÚCallableÚDictÚListÚOptionalÚSetÚTupleÚUnion)Únn)ÚFinal)ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)Úget_device_dtypeÚMlpÚConvMlpÚDropPathÚcalculate_drop_path_ratesÚ	LayerNormÚ
LayerScaleÚLayerScale2dÚClassifierHeadÚNormMlpClassifierHeadÚcreate_attnÚget_act_layerÚget_norm_layerÚget_norm_act_layerÚcreate_conv2dÚcreate_pool2dÚtrunc_normal_tf_Ú	to_2tupleÚextend_tupleÚmake_divisibleÚ_assertÚ	RelPosMlpÚ
RelPosBiasÚRelPosBiasTfÚuse_fused_attnÚresize_rel_pos_bias_tableé   )Úbuild_model_with_cfg)Úfeature_take_indices)Úregister_notrace_function)Únamed_applyÚcheckpoint_seq)Úgenerate_default_cfgsÚregister_model)Ú
MaxxVitCfgÚMaxxVitConvCfgÚMaxxVitTransformerCfgÚMaxxVitc                   ó„  — e Zd ZU dZdZeed<   dZeed<   dZ	e
ed<   dZeed<   dZeed	<   dZeed
<   dZe
ed<   dZe
ed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeeeef                  ed<   dZeeeef                  ed<   dZeed<   dZeed<   dZee
         ed<   dZeed<   dZeed<   d Zeed!<   d"Ze
ed#<   d$„ Z dS )%r8   z-Configuration for MaxxVit transformer blocks.é    Údim_headTÚ
head_firstç      @Úexpand_ratioÚexpand_firstÚshortcut_biasÚ	attn_biasç        Ú	attn_dropÚ	proj_dropÚavg2Ú	pool_typeÚbiasÚrel_pos_typeé   Úrel_pos_dimÚpartition_ratioNÚwindow_sizeÚ	grid_sizeFÚno_block_attnÚuse_nchw_attnÚinit_valuesÚgeluÚ	act_layerÚlayernorm2dÚ
norm_layerÚ	layernormÚnorm_layer_clç�íµ ÷Æ°>Únorm_epsc                 ó´   — | j         �t          | j         ¦  «        | _         | j        �.t          | j        ¦  «        | _        | j         €| j        | _         d S d S d S ©N)rN   r%   rM   ©Úselfs    úc/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/models/maxxvit.pyÚ__post_init__z#MaxxVitTransformerCfg.__post_init__n   s[   € ØŒ>Ð%Ý& t¤~Ñ6Ô6ˆDŒNØÔÐ'Ý(¨Ô)9Ñ:Ô:ˆDÔØŒ~Ð%Ø!%Ô!1�”��ð (Ð'à%Ð%ó    )!Ú__name__Ú
__module__Ú__qualname__Ú__doc__r<   ÚintÚ__annotations__r=   Úboolr?   Úfloatr@   rA   rB   rD   rE   rG   ÚstrrI   rK   rL   rM   r   r   rN   rO   rP   rQ   rS   rU   rW   rY   r_   © r`   r^   r8   r8   U   s™  € € € € € € à7Ð7Ø€HˆcÐÐÑØ€J�ÐÐÑØ€L�%ÐÐÑØ€L�$ÐÐÑØ€M�4ÐÐÑØ€IˆtÐÐÑØ€IˆuÐÐÑØ€IˆuÐÐÑØ€IˆsÐÐÑØ€L�#ÐÐÑØ€K�ÐÐÑØ€O�SÐÐÑØ-1€K�˜%  S œ/Ô*Ð1Ð1Ñ1Ø+/€Iˆx˜˜c 3˜hœÔ(Ð/Ð/Ñ/Ø€M�4ÐÐÑØ€M�4ÐÐÑØ#'€K�˜%”Ð'Ð'Ñ'Ø€IˆsÐÐÑØ#€J�Ð#Ð#Ñ#Ø$€M�3Ð$Ð$Ñ$Ø€HˆeÐÐÑð2ð 2ð 2ð 2ð 2r`   r8   c                   óJ  — e Zd ZU dZdZeed<   dZeed<   dZ	e
ed<   dZeed	<   d
Zeed<   dZe
ed<   dZe
ed<   dZeed<   dZeed<   dZeed<   dZeed<   dZe
ed<   dZeed<   dZeed<   dZeed<   dZee         ed<   dZeed <   dZeed!<   dZeed"<   d#Zee         ed$<   d%„ Zd#S )&r7   z-Configuration for MaxxVit convolution blocks.ÚmbconvÚ
block_typer>   r?   TÚexpand_outputé   Úkernel_sizer.   Ú
group_sizeFÚpre_norm_actÚoutput_biasÚdwÚstride_moderF   rG   Údownsample_pool_typeÚ ÚpaddingÚ
attn_earlyÚseÚ
attn_layerÚsiluÚattn_act_layerç      Ð?Ú
attn_ratiorX   rQ   rR   rS   rU   rW   NrY   c                 óÀ   — | j         dv sJ ‚| j         dk    }| j        s|rdnd| _        | j        s	|sd| _        | j        €|rdnd| _        | j        p| j        | _        d S )N)rl   Úconvnextrl   Úbatchnorm2drT   rV   çñhãˆµøä>rX   )rm   rU   rW   rY   rv   rG   )r]   Ú
use_mbconvs     r^   r_   zMaxxVitConvCfg.__post_init__�   s‡   € àŒÐ"8Ð8Ð8Ð8Ð8Ø”_¨Ò0ˆ
ØŒð 	MØ/9ÐL˜m˜m¸}ˆDŒOØÔ!ð 	-¨*ð 	-Ø!,ˆDÔØŒ=Ð Ø$.Ð8˜D˜D°DˆDŒMØ$(Ô$=Ð$OÀÄˆÔ!Ð!Ð!r`   )ra   rb   rc   rd   rm   ri   rf   r?   rh   rn   rg   rp   re   rq   rr   rs   ru   rG   rv   rx   ry   r{   r}   r   rQ   r   rS   rU   rW   rY   r_   rj   r`   r^   r7   r7   w   sy  € € € € € € à7Ð7Ø€J�ÐÐÑØ€L�%ÐÐÑØ€M�4ÐÐÑØ€K�ÐÐÑØ€J�ÐÐÑØ€L�$ÐÐÑØ€K�ÐÐÑØ€K�ÐÐÑØ€IˆsÐÐÑØ &Ð˜#Ð&Ð&Ñ&Ø€GˆSÐÐÑØ€J�ÐÐÑØ€J�ÐÐÑØ €N�CÐ Ð Ñ Ø€J�ÐÐÑØ#'€K�˜%”Ð'Ð'Ñ'Ø€IˆsÐÐÑØ€J�ÐÐÑØ€M�3ÐÐÑØ $€Hˆh�uŒoÐ$Ð$Ñ$ð
Pð 
Pð 
Pð 
Pð 
Pr`   r7   c                   ó6  — e Zd ZU dZdZeedf         ed<   dZeedf         ed<   dZ	ee
eeedf         f         df         ed<   d	Ze
eeeef         f         ed
<   dZeed<    ee¬¦  «        Zeed<    ee¬¦  «        Zeed<   dZee         ed<   dZeed<   dS )r6   z!Configuration for MaxxVit models.©é`   éÀ   é€  é   .Ú	embed_dim©é   ro   é   r�   Údepths)ÚCr�   ÚTr‘   rm   é@   Ú
stem_widthFÚ	stem_bias)Údefault_factoryÚconv_cfgÚtransformer_cfgNÚhead_hidden_sizeÚvit_effÚweight_init)ra   rb   rc   rd   r‹   r   re   rf   r�   rm   r   ri   r“   r”   rg   r   r7   r–   r8   r—   r˜   r   rš   rj   r`   r^   r6   r6   œ   s  € € € € € € à+Ð+Ø!4€Iˆu�S˜#�XŒÐ4Ð4Ñ4Ø*€FˆE�#�s�(ŒOÐ*Ð*Ñ*Ø:N€J��e˜C  s¨C x¤Ð0Ô1°3Ð6Ô7ÐNÐNÑNØ.0€J��c˜5  c œ?Ð*Ô+Ð0Ð0Ñ0Ø€IˆtÐÐÑØ$˜u°^ÐDÑDÔD€HˆnÐDÐDÑDØ-2¨UÐCXÐ-YÑ-YÔ-Y€OÐ*ÐYÐYÑYØ&*Ð�h˜s”mÐ*Ð*Ñ*Ø €K�Ð Ð Ñ Ð Ð r`   r6   c                   óÎ   ‡ — e Zd ZU dZee         ed<   	 	 	 	 	 	 	 	 	 	 ddedee         d	ed
edededee	         de
de
fˆ fd„Zddej        deej                 dej        fd„Zˆ xZS )ÚAttention2dz)Multi-head attention for 2D NCHW tensors.Ú
fused_attnNr;   TrC   ÚdimÚdim_outr<   rH   r@   r=   Úrel_pos_clsrD   rE   c                 óÖ  •— |
|dœ}t          ¦   «                              ¦   «          |p|}|r|n|}||z  | _        || _        || _        |dz  | _        t          ¦   «         | _        t          j	        ||dz  dfd|i|¤Ž| _
        |r |dd| j        i|¤Žnd| _        t          j        |¦  «        | _        t          j	        ||dfd|i|¤Ž| _        t          j        |	¦  «        | _        dS )	á  
        Args:
            dim: Input dimension.
            dim_out: Output dimension (defaults to input dimension).
            dim_head: Dimension per attention head.
            bias: Whether to use bias in qkv and projection.
            expand_first: Whether to expand channels before or after qkv.
            head_first: Whether heads are first in tensor layout.
            rel_pos_cls: Relative position class to use.
            attn_drop: Attention dropout rate.
            proj_drop: Projection dropout rate.
        ©ÚdeviceÚdtypeç      à¿ro   r.   rH   Ú	num_headsNrj   )ÚsuperÚ__init__r§   r<   r=   Úscaler,   r�   r   ÚConv2dÚqkvÚrel_posÚDropoutrD   ÚprojrE   ©r]   rž   rŸ   r<   rH   r@   r=   r    rD   rE   r¤   r¥   ÚddÚdim_attnÚ	__class__s                 €r^   r©   zAttention2d.__init__®   s	  ø€ ð4 ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ�.˜SˆØ*Ð3�7�7°ˆØ! XÑ-ˆŒØ ˆŒØ$ˆŒØ Ñ%ˆŒ
Ý(Ñ*Ô*ˆŒå”9˜S (¨Q¡,°ÐCÐC¸ÐCÀÐCÐCˆŒØFQÐ[�{�{ÐBÐB¨T¬^ÐB¸rÐBÐBÐBÐW[ˆŒÝœ IÑ.Ô.ˆŒÝ”I˜h¨°ÐDÐD¸ÐDÀÐDÐDˆŒ	Ýœ IÑ.Ô.ˆŒˆˆr`   ÚxÚshared_rel_posÚreturnc                 ó  — |j         \  }}}}| j        rR|                      |¦  «                             || j        | j        dz  d¦  «                             dd¬¦  «        \  }}}	nM|                      |¦  «                             |d| j        | j        d¦  «                             d¦  «        \  }}}	| j	        �r d }
| j
        �| j
                             ¦   «         }
n|�|}
t          j        j                             |                     dd¦  «                             ¦   «         |                     dd¦  «                             ¦   «         |	                     dd¦  «                             ¦   «         |
| j        r| j        j        nd¬¦  «                             dd¦  «                             |d||¦  «        }n¡|| j        z  }|                     dd¦  «        |z  }| j
        �|  
                    |¦  «        }n|�||z   }|                     d¬¦  «        }|                      |¦  «        }|	|                     dd¦  «        z                       |d||¦  «        }|                      |¦  «        }|                      |¦  «        }|S )	Nro   éÿÿÿÿr�   ©rž   r.   éþÿÿÿrC   ©Ú	attn_maskÚ	dropout_p)Úshaper=   r¬   Úviewr§   r<   ÚchunkÚreshapeÚunbindr�   r­   Úget_biasÚtorchr   Ú
functionalÚscaled_dot_product_attentionÚ	transposeÚ
contiguousÚtrainingrD   Úprª   Úsoftmaxr¯   rE   )r]   r´   rµ   ÚBr�   ÚHÚWÚqÚkÚvrB   Úattns               r^   ÚforwardzAttention2d.forwardØ   sU  € Ø”W‰
ˆˆ1ˆa�àŒ?ð 	]Ø—h’h˜q‘k”k×&Ò& q¨$¬.¸$¼-È!Ñ:KÈRÑPÔP×VÒVÐWXÐ^_ÐVÑ`Ô`‰GˆAˆq�!�!à—h’h˜q‘k”k×)Ò)¨!¨Q°´ÀÄÈrÑRÔR×YÒYÐZ[Ñ\Ô\‰GˆAˆq�!àŒ?ñ 	?ØˆIØŒ|Ð'Ø œL×1Ò1Ñ3Ô3�	�	ØÐ+Ø*�	å”Ô#×@Ò@Ø—’˜B Ñ#Ô#×.Ò.Ñ0Ô0Ø—’˜B Ñ#Ô#×.Ò.Ñ0Ô0Ø—’˜B Ñ#Ô#×.Ò.Ñ0Ô0Ø#Ø.2¬mÐC˜$œ.Ô*Ð*Àð Añ ô ÷ Ši˜˜BÑÔ§¢¨¨2¨q°!Ñ 4Ô 4ð ˆAð �D”J‘ˆAØ—;’;˜r 2Ñ&Ô&¨Ñ*ˆDØŒ|Ð'Ø—|’| DÑ)Ô)��ØÐ+Ø˜nÑ,�Ø—<’< B�<Ñ'Ô'ˆDØ—>’> $Ñ'Ô'ˆDØ�T—^’^ B¨Ñ+Ô+Ñ+×1Ò1°!°R¸¸AÑ>Ô>ˆAà�IŠI�a‰LŒLˆØ�NŠN˜1ÑÔˆØˆr`   ©
Nr;   TTTNrC   rC   NNr[   ©ra   rb   rc   rd   r   rg   rf   re   r   r	   rh   r©   rÄ   ÚTensorrÓ   Ú__classcell__©r³   s   @r^   rœ   rœ   ª   s  ø€ € € € € € Ø3Ð3Ø�d”ÐÐÑð
 &*ØØØ!%Ø#Ø.2Ø!Ø!ØØð(/ð (/àð(/ð ˜c”]ð(/ð ð	(/ð
 ð(/ð ð(/ð ð(/ð " (Ô+ð(/ð ð(/ð ð(/ð (/ð (/ð (/ð (/ð (/ðT#ð #˜œð #°xÀÄÔ7Mð #ÐY^ÔYeð #ð #ð #ð #ð #ð #ð #ð #r`   rœ   c                   óÎ   ‡ — e Zd ZU dZee         ed<   	 	 	 	 	 	 	 	 	 	 ddedee         d	ed
edededee	         de
de
fˆ fd„Zddej        deej                 dej        fd„Zˆ xZS )ÚAttentionClz/Channels-last multi-head attention (B, ..., C).r�   Nr;   TrC   rž   rŸ   r<   rH   r@   r=   r    rD   rE   c                 ó  •— |
|dœ}t          ¦   «                              ¦   «          |p|}|r||k    r|n|}||z  dk    s
J d¦   «         ‚||z  | _        || _        || _        |dz  | _        t          ¦   «         | _        t          j	        ||dz  fd|i|¤Ž| _
        |r |d	d| j        i|¤Žnd| _        t          j        |¦  «        | _        t          j	        ||fd|i|¤Ž| _        t          j        |	¦  «        | _        dS )
r¢   r£   r   z(attn dim should be divisible by head_dimr¦   ro   rH   r§   Nrj   )r¨   r©   r§   r<   r=   rª   r,   r�   r   ÚLinearr¬   r­   r®   rD   r¯   rE   r°   s                 €r^   r©   zAttentionCl.__init__  s,  ø€ ð4 ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ�.˜SˆØ*ÐE¨w¸ª}¨}�7�7À#ˆØ˜(Ñ" aÒ'Ð'Ð'Ð)SÑ'Ô'Ð'Ø! XÑ-ˆŒØ ˆŒØ$ˆŒØ Ñ%ˆŒ
Ý(Ñ*Ô*ˆŒå”9˜S (¨Q¡,Ð@Ð@°TÐ@¸RÐ@Ð@ˆŒØFQÐ[�{�{ÐBÐB¨T¬^ÐB¸rÐBÐBÐBÐW[ˆŒÝœ IÑ.Ô.ˆŒÝ”I˜h¨ÐAÐA°dÐA¸bÐAÐAˆŒ	Ýœ IÑ.Ô.ˆŒˆˆr`   r´   rµ   r¶   c                 óN  — |j         d         }|j         d d…         }| j        rf|                      |¦  «                             |d| j        | j        dz  ¦  «                             dd¦  «                             dd¬¦  «        \  }}}na|                      |¦  «                             |dd| j        | j        ¦  «                             dd¦  «         	                    d¦  «        \  }}}| j
        rdd }| j        �| j                             ¦   «         }n|�|}t          j        j                             ||||| j        r| j        j        nd¬¦  «        }ny|| j        z  }||                     d	d¦  «        z  }	| j        �|                      |	|¬
¦  «        }	n|�|	|z   }	|	                     d¬¦  «        }	|                      |	¦  «        }	|	|z  }|                     dd¦  «                             |dz   ¦  «        }|                      |¦  «        }|                      |¦  «        }|S )Nr   r¸   ro   r.   r�   r¹   rC   r»   rº   ©rµ   )r¸   )r¾   r=   r¬   r¿   r§   r<   rÇ   rÀ   rÁ   rÂ   r�   r­   rÃ   rÄ   r   rÅ   rÆ   rÉ   rD   rÊ   rª   rË   r¯   rE   )
r]   r´   rµ   rÌ   Úrestore_shaperÏ   rÐ   rÑ   rB   rÒ   s
             r^   rÓ   zAttentionCl.forward-  s  € ØŒG�AŒJˆØœ   œˆàŒ?ð 	mØ—h’h˜q‘k”k×&Ò& q¨"¨d¬n¸d¼mÈaÑ>OÑPÔP×ZÒZÐ[\Ð^_Ñ`Ô`×fÒfÐghÐnoÐfÑpÔp‰GˆAˆq�!�!à—h’h˜q‘k”k×)Ò)¨!¨R°°D´NÀDÄMÑRÔR×\Ò\Ð]^Ð`aÑbÔb×iÒiÐjkÑlÔl‰GˆAˆq�!àŒ?ð 	ØˆIØŒ|Ð'Ø œL×1Ò1Ñ3Ô3�	�	ØÐ+Ø*�	å”Ô#×@Ò@Ø�1�aØ#Ø.2¬mÐC˜$œ.Ô*Ð*Àð Añ ô ˆAˆAð �D”J‘ˆAØ�q—{’{ 2 rÑ*Ô*Ñ*ˆDØŒ|Ð'Ø—|’| D¸�|ÑHÔH��ØÐ+Ø˜nÑ,�Ø—<’< B�<Ñ'Ô'ˆDØ—>’> $Ñ'Ô'ˆDØ�q‘ˆAà�KŠK˜˜1ÑÔ×%Ò% m°eÑ&;Ñ<Ô<ˆØ�IŠI�a‰LŒLˆØ�NŠN˜1ÑÔˆØˆr`   rÔ   r[   rÕ   rØ   s   @r^   rÚ   rÚ   þ   s  ø€ € € € € € Ø9Ð9Ø�d”ÐÐÑð
 &*ØØØ!%Ø#Ø.2Ø!Ø!ØØð)/ð )/àð)/ð ˜c”]ð)/ð ð	)/ð
 ð)/ð ð)/ð ð)/ð " (Ô+ð)/ð ð)/ð ð)/ð )/ð )/ð )/ð )/ð )/ðV#ð #˜œð #°xÀÄÔ7Mð #ÐY^ÔYeð #ð #ð #ð #ð #ð #ð #ð #r`   rÚ   c                   óh   ‡ — e Zd ZdZ	 	 	 	 	 ddededed	ed
ef
ˆ fd„Zdej	        dej	        fd„Z
ˆ xZS )ÚDownsample2da5  A downsample pooling module supporting several maxpool and avgpool modes.

    * 'max' - MaxPool2d w/ kernel_size 3, stride 2, padding 1
    * 'max2' - MaxPool2d w/ kernel_size = stride = 2
    * 'avg' - AvgPool2d w/ kernel_size 3, stride 2, padding 1
    * 'avg2' - AvgPool2d w/ kernel_size = stride = 2
    rF   rw   TNrž   rŸ   rG   rx   rH   c                 óÊ  •— t          ¦   «                              ¦   «          |dv sJ ‚|dk    rt          ddd|pd¬¦  «        | _        n[|dk    rt          dd|pd¬	¦  «        | _        n;|d
k    rt          d
ddd|pd¬¦  «        | _        nt          d
d|pd¬	¦  «        | _        ||k    r!t	          j        ||d|||¬¦  «        | _        dS t	          j        ¦   «         | _        dS )zî
        Args:
            dim: Input dimension.
            dim_out: Output dimension.
            pool_type: Type of pooling operation.
            padding: Padding mode.
            bias: Whether to use bias in expansion conv.
        )ÚmaxÚmax2ÚavgrF   rã   ro   r�   r.   )rp   Ústriderx   rä   r   )rx   rå   F)rp   ræ   Úcount_include_padrx   )rH   r¤   r¥   N)r¨   r©   r#   Úpoolr   r«   ÚexpandÚIdentity)	r]   rž   rŸ   rG   rx   rH   r¤   r¥   r³   s	           €r^   r©   zDownsample2d.__init__\  s  ø€ õ$ 	‰Œ×ÒÑÔÐØÐ:Ð:Ð:Ð:Ð:Ø˜ÒÐÝ% e¸À1ÈgÈlÐYZÐ[Ñ[Ô[ˆDŒIˆIØ˜&Ò Ð Ý% e¨Q¸¸À1ÐEÑEÔEˆDŒIˆIØ˜%ÒÐÝ%Ø 1¨QÀ%ÐQXÐQ]Ð\]ð_ñ _ô _ˆDŒIˆIõ & e¨Q¸¸À1ÐEÑEÔEˆDŒIà�'Š>ˆ>Ýœ) C¨°!¸$ÀvÐUZÐ[Ñ[Ô[ˆDŒKˆKˆKåœ+™-œ-ˆDŒKˆKˆKr`   r´   r¶   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S r[   )rè   ré   ©r]   r´   s     r^   rÓ   zDownsample2d.forward  s%   € Ø�IŠI�a‰LŒLˆØ�KŠK˜‰NŒNˆØˆr`   )rF   rw   TNN)ra   rb   rc   rd   re   ri   rg   r©   rÄ   rÖ   rÓ   r×   rØ   s   @r^   rá   rá   S  s´   ø€ € € € € ðð ð $ØØØØð!(ð !(àð!(ð ð!(ð ð	!(ð
 ð!(ð ð!(ð !(ð !(ð !(ð !(ð !(ðF˜œð ¨%¬,ð ð ð ð ð ð ð ð r`   rá   rw   ÚmoduleÚnameÚschemer¶   c                 óP  — t          | t          j        t          j        f¦  «        �r||dk    rUt          j                             | j        d¬¦  «         | j        �&t          j                             | j        ¦  «         dS dS |dk    rEt          | j        d¬¦  «         | j        �&t          j                             | j        ¦  «         dS dS |dk    rSt          j         
                    | j        ¦  «         | j        �&t          j                             | j        ¦  «         dS dS t          j                             | j        ¦  «         | j        �Td|v r(t          j                             | j        d¬¦  «         dS t          j                             | j        ¦  «         dS dS dS )	z&Initialize transformer module weights.Únormalç{®Gáz”?©ÚstdNÚtrunc_normalÚxavier_normalÚmlprX   )Ú
isinstancer   r«   rÜ   ÚinitÚnormal_ÚweightrH   Úzeros_r$   Úxavier_normal_Úxavier_uniform_)rí   rî   rï   s      r^   Ú_init_transformerrÿ   …  s‚  € å�&�2œ9¥b¤iÐ0Ñ1Ô1ñ 0Ø�XÒÐÝŒG�OŠO˜FœM¨sˆOÑ3Ô3Ð3ØŒ{Ð&Ý”—’˜vœ{Ñ+Ô+Ð+Ð+Ð+ð 'Ð&à�~Ò%Ð%Ý˜Vœ]°Ð4Ñ4Ô4Ð4ØŒ{Ð&Ý”—’˜vœ{Ñ+Ô+Ð+Ð+Ð+ð 'Ð&à�Ò&Ð&ÝŒG×"Ò" 6¤=Ñ1Ô1Ð1ØŒ{Ð&Ý”—’˜vœ{Ñ+Ô+Ð+Ð+Ð+ð 'Ð&õ ŒG×#Ò# F¤MÑ2Ô2Ð2ØŒ{Ð&Ø˜D�=�=Ý”G—O’O F¤K°T�OÑ:Ô:Ð:Ð:Ð:å”G—N’N 6¤;Ñ/Ô/Ð/Ð/Ð/ð)0ð 0ð  'Ð&r`   c                   ó¸   ‡ — e Zd ZdZdd e¦   «         dddfdedededee         d	ed
efˆ fd„Z	dde
ddfd„Zddej        deej                 dej        fd„Zˆ xZS )ÚTransformerBlock2daY  Transformer block with 2D downsampling.

    '2D' NCHW tensor layout

    Some gains can be seen on GPU using a 1D / CL block, BUT w/ the need to switch back/forth to NCHW
    for spatial pooling, the benefit is minimal so ended up using just this variant for CoAt configs.

    This impl was faster on TPU w/ PT XLA than the 1D experiment.
    r.   NrC   rž   rŸ   ræ   r    ÚcfgÚ	drop_pathc	                 ób  •— ||dœ}	t          ¦   «                              ¦   «          t          t          |j        ¦  «        |j        ¬¦  «        }
t          |j        ¦  «        }|dk    rgt          ||f|j	        |j
        dœ|	¤Ž| _        t          j        t          d |
|fi |	¤Žfdt          ||fd|j	        i|	¤Žfg¦  «        ¦  «        | _        n.||k    sJ ‚t          j        ¦   «         | _         |
|fi |	¤Ž| _        t#          ||f|j        |j        |j        ||j        |j        dœ|	¤Ž| _        |j        rt3          |fd	|j        i|	¤Žnt          j        ¦   «         | _        |d
k    rt7          |¦  «        nt          j        ¦   «         | _         |
|fi |	¤Ž| _        t=          d|t?          ||j         z  ¦  «        ||j        dœ|	¤Ž| _!        |j        rt3          |fd	|j        i|	¤Žnt          j        ¦   «         | _"        |d
k    rt7          |¦  «        nt          j        ¦   «         | _#        dS )a  
        Args:
            dim: Input dimension.
            dim_out: Output dimension.
            stride: Stride for downsampling.
            rel_pos_cls: Relative position class.
            cfg: Transformer block configuration.
            drop_path: Drop path rate.
        r£   ©Úepsr�   )rG   rH   ÚnormÚdownrG   )r<   r@   rH   r    rD   rE   rQ   rC   ©Úin_featuresÚhidden_featuresrS   ÚdropNrj   )$r¨   r©   r   r    rU   rY   r   rS   rá   rG   rA   Úshortcutr   Ú
Sequentialr   Únorm1rê   rœ   r<   r@   rB   rD   rE   rÒ   rQ   r   Úls1r   Ú
drop_path1Únorm2r   re   r?   r÷   Úls2Ú
drop_path2)r]   rž   rŸ   ræ   r    r  r  r¤   r¥   r±   rU   rS   r³   s               €r^   r©   zTransformerBlock2d.__init__©  sy  ø€ ð( ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ�^¨C¬NÑ;Ô;ÀÄÐNÑNÔNˆ
Ý! #¤-Ñ0Ô0ˆ	à�QŠ;ˆ;Ý(¨¨gÐmÀÄÐUXÔUfÐmÐmÐjlÐmÐmˆDŒMÝœ¥{Ø˜˜ CÐ.Ð.¨2Ð.Ð.Ð/Ø� c¨3ÐNÐN¸#¼-ÐNÈ2ÐNÐNÐOð4ñ (ô (ñ ô ˆDŒJˆJð
 ˜'’>�>�>�>ÝœK™MœMˆDŒMØ#˜ CÐ.Ð.¨2Ð.Ð.ˆDŒJåØØð

ð ”\ØÔ)Ø”Ø#Ø”mØ”mð

ð 

ð ð

ð 

ˆŒ	ð PSÌÐq•< ÐKÐK°S´_ÐKÈÐKÐKÐKÕdfÔdoÑdqÔdqˆŒØ1:¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒà�Z Ð.Ð.¨2Ð.Ð.ˆŒ
Ýð 
ØÝ ¨#Ô*:Ñ :Ñ;Ô;ØØ”ð	
ð 
ð
 ð
ð 
ˆŒð PSÌÐq•< ÐKÐK°S´_ÐKÈÐKÐKÐKÕdfÔdoÑdqÔdqˆŒØ1:¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒˆˆr`   rw   rï   r¶   c                 óN   — t          t          t          |¬¦  «        | ¦  «         d S ©N©rï   )r2   r   rÿ   ©r]   rï   s     r^   Úinit_weightszTransformerBlock2d.init_weightsæ  s&   € Ý•GÕ-°fÐ=Ñ=Ô=¸tÑDÔDÐDÐDÐDr`   r´   rµ   c           
      ót  — |                       |¦  «        |                      |                      |                      |                      |¦  «        |¬¦  «        ¦  «        ¦  «        z   }||                      |                      |                      |                      |¦  «        ¦  «        ¦  «        ¦  «        z   }|S )NrÞ   )	r  r  r  rÒ   r  r  r  r÷   r  )r]   r´   rµ   s      r^   rÓ   zTransformerBlock2d.forwardé  sˆ   € Ø�MŠM˜!ÑÔ˜tŸš¨t¯xªx¸¿	º	À$Ç*Â*ÈQÁ-Ä-Ð`n¸	Ñ8oÔ8oÑ/pÔ/pÑqÔqÑqˆØ�—’ §¢¨¯ª°$·*²*¸Q±-´-Ñ)@Ô)@Ñ AÔ AÑBÔBÑBˆØˆr`   ©rw   r[   )ra   rb   rc   rd   r8   re   r   r	   rh   r©   ri   r  rÄ   rÖ   rÓ   r×   rØ   s   @r^   r  r  ž  s  ø€ € € € € ðð ð Ø.2Ø)>Ð)>Ñ)@Ô)@Ø!ØØð;Sð ;Sàð;Sð ð;Sð ð	;Sð
 " (Ô+ð;Sð 'ð;Sð ð;Sð ;Sð ;Sð ;Sð ;Sð ;SðzEð E 3ð E°ð Eð Eð Eð Eðð ˜œð °xÀÄÔ7Mð ÐY^ÔYeð ð ð ð ð ð ð ð r`   r  c                 óh  — t          | t          j        ¦  «        �r”|dk    rUt          j                             | j        d¬¦  «         | j        �&t          j                             | j        ¦  «         dS dS |dk    rEt          | j        d¬¦  «         | j        �&t          j                             | j        ¦  «         dS dS |dk    rSt          j         	                    | j        ¦  «         | j        �&t          j                             | j        ¦  «         dS dS | j
        d         | j
        d         z  | j        z  }|| j        z  }t          j                             | j        dt          j        d	|z  ¦  «        ¦  «         | j        �(t          j                             | j        ¦  «         dS dS dS )
z&Initialize convolution module weights.rñ   rò   ró   Nrõ   rö   r   r.   g       @)rø   r   r«   rù   rú   rû   rH   rü   r$   rý   rp   Úout_channelsÚgroupsÚmathÚsqrt)rí   rî   rï   Úfan_outs       r^   Ú
_init_convr"  ï  s�  € å�&�"œ)Ñ$Ô$ñ ,Ø�XÒÐÝŒG�OŠO˜FœM¨sˆOÑ3Ô3Ð3ØŒ{Ð&Ý”—’˜vœ{Ñ+Ô+Ð+Ð+Ð+ð 'Ð&à�~Ò%Ð%Ý˜Vœ]°Ð4Ñ4Ô4Ð4ØŒ{Ð&Ý”—’˜vœ{Ñ+Ô+Ð+Ð+Ð+ð 'Ð&à�Ò&Ð&ÝŒG×"Ò" 6¤=Ñ1Ô1Ð1ØŒ{Ð&Ý”—’˜vœ{Ñ+Ô+Ð+Ð+Ð+ð 'Ð&ð Ô(¨Ô+¨fÔ.@ÀÔ.CÑCÀfÔFYÑYˆGØ˜œÑ%ˆGÝŒG�OŠO˜FœM¨1­d¬i¸¸g¹Ñ.FÔ.FÑGÔGÐGØŒ{Ð&Ý”—’˜vœ{Ñ+Ô+Ð+Ð+Ð+ð',ð ,ð$ 'Ð&r`   rq   Úchannelsc                 ó*   — | sdS || z  dk    sJ ‚|| z  S )z3Calculate number of groups for grouped convolution.r.   r   rj   )rq   r#  s     r^   Ú
num_groupsr%    s2   € àð &Øˆqð ˜*Ñ$¨Ò)Ð)Ð)Ð)Ø˜:Ñ%Ð%r`   c                   ó    ‡ — e Zd ZdZdd e¦   «         dddfdededed	eeef         d
edefˆ fd„Zdde	ddfd„Z
dej        dej        fd„Zˆ xZS )ÚMbConvBlockzGPre-Norm Conv Block - 1x1 - kxk - 1x1, w/ inverted bottleneck (expand).r.   ©r.   r.   rC   NÚin_chsÚout_chsræ   Údilationr  r  c	                 ó€  •— ||dœ}	t          ¦   «                              ¦   «          t          t          |j        |j        ¦  «        |j        ¬¦  «        }
t          |j        r|n||j	        z  ¦  «        }t          |j        |¦  «        }|dk    r't          ||f|j        |j        |j        dœ|	¤Ž| _        nt#          j        ¦   «         | _        |j        dv sJ ‚d\  }}}|j        dk    r||d         }}n |j        d	k    r||d         }}n
||d
         }} |
|fd|j        i|	¤Ž| _        |dk    r!t          ||f|j        |j        dœ|	¤Ž| _        nt#          j        ¦   «         | _        t1          ||dfd|i|	¤Ž| _         |
|fi |	¤Ž| _        t1          |||j        f||||j        dœ|	¤Ž| _        i }t;          |j        t>          ¦  «        rC|j        dk    s|j        dk    r-|j         |d<   tC          |j"        |j        r|n|z  ¦  «        |d<   |j#        r0tI          |j        |fi |¤|	¤Ž| _%         |
|fi |	¤Ž| _&        d| _'        n/d| _%         |
|fi |	¤Ž| _&        tI          |j        |fi |¤|	¤Ž| _'        t1          ||dfd|j        i|	¤Ž| _(        |dk    rtS          |¦  «        nt#          j        ¦   «         | _*        dS )a  
        Args:
            in_chs: Input channels.
            out_chs: Output channels.
            stride: Stride for conv.
            dilation: Dilation for conv.
            cfg: Convolution block configuration.
            drop_path: Drop path rate.
        r£   r  r�   )rG   rH   rx   )rè   Ú1x1rt   )r.   r.   r.   rè   r.   r-  r   Ú	apply_act)rG   rx   ræ   )ræ   r+  r  rx   rz   ÚecarS   Úrd_channelsNrH   rC   )+r¨   r©   r   r!   rU   rS   rY   r'   rn   r?   r%  rq   rá   rG   rs   rx   r  r   rê   ru   rr   Úpre_normrv   r  r"   Ú	conv1_1x1r  rp   Ú	conv2_kxkrø   r{   ri   r}   re   r   ry   r   Úse_earlyr  rz   Ú	conv3_1x1r   r  )r]   r)  r*  ræ   r+  r  r  r¤   r¥   r±   Únorm_act_layerÚmid_chsr  Ústride_poolÚstride_1Ústride_2Ú
dilation_2Úattn_kwargsr³   s                     €r^   r©   zMbConvBlock.__init__  sU  ø€ ð( ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ Õ!3°C´NÀCÄMÑ!RÔ!RÐX[ÔXdÐeÑeÔeˆÝ ¨SÔ->Ð"J ' 'ÀFÈcÔN^Ñ!^Ñ_Ô_ˆÝ˜CœN¨GÑ4Ô4ˆà�QŠ;ˆ;Ý(Ø˜ðkØ+.¬=¸s¼ÐX[ÔXcðkð kØgiðkð kˆDŒMˆMõ œK™MœMˆDŒMàŒÐ"7Ð7Ð7Ð7Ð7Ø*1Ñ'ˆ�X˜xØŒ?˜fÒ$Ð$à&,¨h°q¬k˜ˆKˆKàŒ_ Ò%Ð%à#)¨8°A¬;�jˆHˆHà#)¨8°A¬;�jˆHà&˜ vÐPÐP¸Ô9IÐPÈRÐPÐPˆŒØ˜Š?ˆ?Ý$ V¨VÐs¸sÔ?WÐadÔalÐsÐsÐprÐsÐsˆDŒIˆIåœ™œˆDŒIÝ& v¨w¸ÐQÐQÀ(ÐQÈbÐQÐQˆŒØ#�^ GÐ2Ð2¨rÐ2Ð2ˆŒ
å&ØØØŒOð	
ð ØØØ”Kð	
ð 	
ð ð	
ð 	
ˆŒð ˆÝ�c”n¥cÑ*Ô*ð 	oØŒ~ Ò%Ð%¨¬¸5Ò)@Ð)@Ø+.Ô+=�˜KÑ(Ý-0°´ÈcÔN_ÐClÀ7À7ÐelÑ1mÑ-nÔ-n�˜MÑ*ð Œ>ð 	PÝ'¨¬¸ÐUÐUÀ;ÐUÐRTÐUÐUˆDŒMØ'˜¨Ð6Ð6°2Ð6Ð6ˆDŒJØˆDŒGˆGà ˆDŒMØ'˜¨Ð6Ð6°2Ð6Ð6ˆDŒJÝ! #¤.°'ÐOÐO¸[ÐOÈBÐOÐOˆDŒGå& w°¸ÐWÐWÀÄÐWÐTVÐWÐWˆŒØ09¸B²°� )Ñ,Ô,Ð,ÅBÄKÁMÄMˆŒˆˆr`   rw   rï   r¶   c                 óN   — t          t          t          |¬¦  «        | ¦  «         d S r  ©r2   r   r"  r  s     r^   r  zMbConvBlock.init_weightsf  ó%   € Ý•G�J¨vÐ6Ñ6Ô6¸Ñ=Ô=Ð=Ð=Ð=r`   r´   c                 óö  — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }| j        �|                      |¦  «        }|                      |¦  «        }| j        �|                      |¦  «        }|  	                    |¦  «        }|  
                    |¦  «        |z   }|S r[   )r  r1  r  r2  r  r3  r4  r  rz   r5  r  ©r]   r´   r  s      r^   rÓ   zMbConvBlock.forwardi  sÕ   € Ø—=’= Ñ#Ô#ˆØ�MŠM˜!ÑÔˆØ�IŠI�a‰LŒLˆð �NŠN˜1ÑÔˆØ�JŠJ�q‰MŒMˆð �NŠN˜1ÑÔˆØŒ=Ð$Ø—’˜aÑ Ô ˆAØ�JŠJ�q‰MŒMˆØŒ7ÐØ—’˜‘
”
ˆAð �NŠN˜1ÑÔˆØ�NŠN˜1ÑÔ Ñ(ˆØˆr`   r  )ra   rb   rc   rd   r7   re   r   rh   r©   ri   r  rÄ   rÖ   rÓ   r×   rØ   s   @r^   r'  r'    s  ø€ € € € € ØQÐQð Ø(.Ø"0 .Ñ"2Ô"2Ø!ØØðPRð PRàðPRð ðPRð ð	PRð
 ˜C ˜H”oðPRð  ðPRð ðPRð PRð PRð PRð PRð PRðd>ð > 3ð >°ð >ð >ð >ð >ð˜œð ¨%¬,ð ð ð ð ð ð ð ð r`   r'  c                   ó¨   ‡ — e Zd ZdZdddd e¦   «         ddddf	ded	ee         d
ededeeef         dedede	fˆ fd„Z
dej        dej        fd„Zˆ xZS )ÚConvNeXtBlockzConvNeXt Block.Né   r.   r(  TrC   r)  r*  rp   ræ   r+  r  Úconv_mlpr  c           	      ón  •— |	|
dœ}t          ¦   «                              ¦   «          |p|}t          |j        ¦  «        }|r0t	          t          |j        ¦  «        |j        ¬¦  «        }t          }nd|j        v sJ ‚t          }t          }|| _        |dk    rt          ||fi |¤Ž| _        n?||k    r!t          j        ||fd|j        dœ|¤Ž| _        nt          j        ¦   «         | _        |j        dv sJ ‚d\  }}|j        d	k    r|}n|}|dk    rt          ||fd
|j        i|¤Ž| _        nt          j        ¦   «         | _        t+          ||f|||d         d|j        dœ|¤Ž| _         ||fi |¤Ž| _         ||t1          |j        |z  ¦  «        f|j        |dœ|¤Ž| _        |r3|j        rt9          ||j        fi |¤Žnt          j        ¦   «         | _        n2|j        rt=          ||j        fi |¤Žnt          j        ¦   «         | _        |dk    rt?          |¦  «        nt          j        ¦   «         | _         dS )ay  
        Args:
            in_chs: Input channels.
            out_chs: Output channels.
            kernel_size: Kernel size for depthwise conv.
            stride: Stride for conv.
            dilation: Dilation for conv.
            cfg: Convolution block configuration.
            conv_mlp: Whether to use convolutional MLP.
            drop_path: Drop path rate.
        r£   r  rV   r�   r.   )rp   rH   )rè   rt   r(  rè   rG   T)rp   ræ   r+  Ú	depthwiserH   )rH   rS   rC   N)!r¨   r©   r   rS   r   r    rU   rY   r   r   r   Úuse_conv_mlprá   r  r   r«   rs   rê   ru   rv   r  r"   Úconv_dwr  re   r?   r÷   rQ   r   Úlsr   r   r  )r]   r)  r*  rp   ræ   r+  r  rE  r  r¤   r¥   r±   rS   rU   Ú	mlp_layerr8  Ú	stride_dwr³   s                    €r^   r©   zConvNeXtBlock.__init__ƒ  s¡  ø€ ð0 ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØÐ#˜VˆÝ! #¤-Ñ0Ô0ˆ	Øð 	Ý ¥°´Ñ!?Ô!?ÀSÄ\ÐRÑRÔRˆJÝˆIˆIà #¤.Ð0Ð0Ð0Ð0Ý"ˆJÝˆIØ$ˆÔà�QŠ;ˆ;Ý(¨°Ð?Ð?¸BÐ?Ð?ˆDŒMˆMØ�wÒÐÝœI f¨gÐaÀ1È3Ì?ÐaÐaÐ^`ÐaÐaˆDŒMˆMåœK™MœMˆDŒMàŒ .Ð0Ð0Ð0Ð0Ø!%Ñˆ�YàŒ?˜fÒ$Ð$Ø ˆKˆKàˆIà˜!ÒÐÝ$ V¨VÐ^Ð^¸sÔ?WÐ^Ð[]Ð^Ð^ˆDŒIˆIåœ™œˆDŒIå$ØØð	
ð $ØØ˜a”[ØØ”ð	
ð 	
ð ð	
ð 	
ˆŒð �J˜wÐ-Ð-¨"Ð-Ð-ˆŒ	Ø�9ØÝ�Ô  7Ñ*Ñ+Ô+ð
ð ”Øð	
ð 
ð
 ð
ð 
ˆŒð ð 	gØFIÄoÐh•l 7¨C¬OÐBÐB¸rÐBÐBÐBÕ[]Ô[fÑ[hÔ[hˆDŒGˆGàDGÄOÐf•j ¨#¬/Ð@Ð@¸RÐ@Ð@Ð@ÕY[ÔYdÑYfÔYfˆDŒGØ09¸B²°� )Ñ,Ô,Ð,ÅBÄKÁMÄMˆŒˆˆr`   r´   r¶   c                 ó   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }| j        r@|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }no|                     dddd¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                     dddd¦  «        }|                      |¦  «        |z   }|S ©Nr   r�   ro   r.   )	r  r  rI  rH  r  r÷   rJ  Úpermuter  rA  s      r^   rÓ   zConvNeXtBlock.forwardÔ  sá   € Ø—=’= Ñ#Ô#ˆØ�IŠI�a‰LŒLˆØ�LŠL˜‰OŒOˆØÔð 		&Ø—	’	˜!‘”ˆAØ—’˜‘”ˆAØ—’˜‘
”
ˆAˆAà—	’	˜!˜Q  1Ñ%Ô%ˆAØ—	’	˜!‘”ˆAØ—’˜‘”ˆAØ—’˜‘
”
ˆAØ—	’	˜!˜Q  1Ñ%Ô%ˆAà�NŠN˜1ÑÔ Ñ(ˆØˆr`   )ra   rb   rc   rd   r7   re   r   r   rg   rh   r©   rÄ   rÖ   rÓ   r×   rØ   s   @r^   rC  rC  €  s  ø€ € € € € ØÐð
 &*Ø ØØ(.Ø"0 .Ñ"2Ô"2Ø!Ø!ØØðORð ORàðORð ˜c”]ðORð ð	ORð
 ðORð ˜C ˜H”oðORð  ðORð ðORð ðORð ORð ORð ORð ORð ORðb˜œð ¨%¬,ð ð ð ð ð ð ð ð r`   rC  r´   rM   c                 óÒ  — | j         \  }}}}t          ||d         z  dk    d|› d|d         › d�¦  «         t          ||d         z  dk    d|› d|d         › d�¦  «         |                      |||d         z  |d         ||d         z  |d         |¦  «        } |                      ddddd	d
¦  «                             ¦   «                              d|d         |d         |¦  «        }|S )z'Partition into non-overlapping windows.r   úheight (ú) must be divisible by window (ú)r.   úwidth (ro   r�   é   rŽ   r¸   ©r¾   r(   r¿   rO  rÈ   )r´   rM   rÌ   rÍ   rÎ   r�   Úwindowss          r^   Úwindow_partitionrX  ç  s  € à”�J€A€qˆ!ˆQÝˆA�˜A”Ñ !Ò#Ð%c°Ð%cÐ%cÐR]Ð^_ÔR`Ð%cÐ%cÐ%cÑdÔdÐdÝˆA�˜A”Ñ !Ò#Ð%b¨qÐ%bÐ%bÐQ\Ð]^ÔQ_Ð%bÐ%bÐ%bÑcÔcÐcØ	�Šˆq�!�{ 1”~Ñ% {°1¤~°q¸KÈ¼NÑ7JÈKÐXYÌNÐ\]Ñ^Ô^€AØ�iŠi˜˜1˜a  A qÑ)Ô)×4Ò4Ñ6Ô6×;Ò;¸BÀÈAÄÐP[Ð\]ÔP^Ð`aÑbÔb€GØ€Nr`   rW  Úimg_sizec                 ó  — |\  }}| j         d         }|                      d||d         z  ||d         z  |d         |d         |¦  «        }|                     dddddd¦  «                             ¦   «                              d|||¦  «        }|S )zReverse window partition.r¸   r   r.   ro   r�   rU  rŽ   ©r¾   r¿   rO  rÈ   ©rW  rM   rY  rÍ   rÎ   r�   r´   s          r^   Úwindow_reverser]  ñ  s’   € ð �D€A€qØŒ�bÔ€AØ�Š�R˜˜k¨!œnÑ,¨a°;¸q´>Ñ.AÀ;ÈqÄ>ÐS^Ð_`ÔSaÐcdÑeÔe€AØ	�	Š	�!�Q˜˜1˜a Ñ#Ô#×.Ò.Ñ0Ô0×5Ò5°b¸!¸QÀÑBÔB€AØ€Hr`   rN   c           	      óÎ  — | j         \  }}}}t          ||d         z  dk    d|› d|d         › �¦  «         t          ||d         z  dk    d|› d|d         › �¦  «         |                      ||d         ||d         z  |d         ||d         z  |¦  «        } |                      dddddd	¦  «                             ¦   «                              d
|d         |d         |¦  «        }|S )z6Partition into overlapping windows with grid striding.r   úheight ú must be divisible by grid r.   úwidth r�   rU  ro   rŽ   r¸   rV  )r´   rN   rÌ   rÍ   rÎ   r�   rW  s          r^   Úgrid_partitionrb  û  sú   € à”�J€A€qˆ!ˆQÝˆA�	˜!”Ñ Ò!Ð#Y¨QÐ#YÐ#YÈ9ÐUVÌ<Ð#YÐ#YÑZÔZÐZÝˆA�	˜!”Ñ Ò!Ð#X¨AÐ#XÐ#XÈ)ÐTUÌ,Ð#XÐ#XÑYÔYÐYØ	�Šˆq�)˜A”,  Y¨q¤\Ñ 1°9¸Q´<ÀÀiÐPQÄlÑARÐTUÑVÔV€AØ�iŠi˜˜1˜a  A qÑ)Ô)×4Ò4Ñ6Ô6×;Ò;¸BÀ	È!ÄÈiÐXYÌlÐ\]Ñ^Ô^€GØ€Nr`   c                 ó  — |\  }}| j         d         }|                      d||d         z  ||d         z  |d         |d         |¦  «        }|                     dddddd¦  «                             ¦   «                              d|||¦  «        }|S )zReverse grid partition.r¸   r   r.   ro   rU  r�   rŽ   r[  ©rW  rN   rY  rÍ   rÎ   r�   r´   s          r^   Úgrid_reversere    s�   € ð �D€A€qØŒ�bÔ€AØ�Š�R˜˜i¨œlÑ*¨A°¸1´Ñ,=¸yÈ¼|ÈYÐWXÌ\Ð[\Ñ]Ô]€AØ	�	Š	�!�Q˜˜1˜a Ñ#Ô#×.Ò.Ñ0Ô0×5Ò5°b¸!¸QÀÑBÔB€AØ€Hr`   r  c                 óà   — d}| j         dk    rt          t          || j        ¬¦  «        }nC| j         dk    rt          t          |¬¦  «        }n!| j         dk    rt          t
          |¬¦  «        }|S )z,Get relative position class based on config.Nr÷   )rM   Ú
hidden_dimrH   )rM   Úbias_tf)rI   r   r)   rK   r*   r+   )r  rM   r    s      r^   Úget_rel_pos_clsri    su   € à€KØ
Ô˜5Ò Ð Ý�i°[ÈSÌ_Ð]Ñ]Ô]ˆˆØ	Ô	˜VÒ	#Ð	#Ý�j°kÐBÑBÔBˆˆØ	Ô	˜YÒ	&Ð	&Ý�l¸ÐDÑDÔDˆØÐr`   c            	       ó\   ‡ — e Zd ZdZd e¦   «         dddfdedededefˆ fd	„Zd
„ Z	d„ Z
ˆ xZS )ÚPartitionAttentionClzRGrid or Block partition + Attn + FFN.

    NxC 'channels last' tensor layout.
    ÚblockrC   Nrž   Úpartition_typer  r  c           
      óÖ  •— ||dœ}t          ¦   «                              ¦   «          t          t          |j        ¦  «        |j        ¬¦  «        }t          |j        ¦  «        }	|dk    | _        t          | j        r|j
        n|j        ¦  «        | _        t          || j        ¦  «        }
 ||fi |¤Ž| _        t          ||f|j        |j        |j        |
|j        |j        dœ|¤Ž| _        |j        rt/          |fd|j        i|¤Žnt1          j        ¦   «         | _        |dk    rt7          |¦  «        nt1          j        ¦   «         | _         ||fi |¤Ž| _        t=          d|t?          ||j         z  ¦  «        |	|j        dœ|¤Ž| _!        |j        rt/          |fd|j        i|¤Žnt1          j        ¦   «         | _"        |dk    rt7          |¦  «        nt1          j        ¦   «         | _#        d S )	Nr£   r  rl  ©r<   rH   r=   r    rD   rE   rQ   rC   r	  rj   )$r¨   r©   r   r    rW   rY   r   rS   Úpartition_blockr%   rM   rN   Úpartition_sizeri  r  rÚ   r<   rB   r=   rD   rE   rÒ   rQ   r   r   rê   r  r   r  r  r   re   r?   r÷   r  r  ©r]   rž   rm  r  r  r¤   r¥   r±   rU   rS   r    r³   s              €r^   r©   zPartitionAttentionCl.__init__!  s  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ�^¨CÔ,=Ñ>Ô>ÀCÄLÐQÑQÔQˆ
Ý! #¤-Ñ0Ô0ˆ	à-°Ò8ˆÔÝ'¸4Ô;OÐ(b¨¬¨ÐUXÔUbÑcÔcˆÔÝ% c¨4Ô+>Ñ?Ô?ˆà�Z Ð*Ð* rÐ*Ð*ˆŒ
ÝØØð

ð ”\Ø”Ø”~Ø#Ø”mØ”mð

ð 

ð ð

ð 

ˆŒ	ð JMÌÐk•:˜cÐEÐE¨s¬ÐEÀ"ÐEÐEÐEÕ^`Ô^iÑ^kÔ^kˆŒØ1:¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒà�Z Ð*Ð* rÐ*Ð*ˆŒ
Ýð 
ØÝ  cÔ&6Ñ 6Ñ7Ô7ØØ”ð	
ð 
ð
 ð
ð 
ˆŒð JMÌÐk•:˜cÐEÐE¨s¬ÐEÀ"ÐEÐEÐEÕ^`Ô^iÑ^kÔ^kˆŒØ1:¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒˆˆr`   c                 ó  — |j         dd…         }| j        rt          || j        ¦  «        }nt	          || j        ¦  «        }|                      |¦  «        }| j        rt          || j        |¦  «        }nt          || j        |¦  «        }|S )Nr.   ro   )r¾   rp  rX  rq  rb  rÒ   r]  re  ©r]   r´   rY  Úpartitioneds       r^   Ú_partition_attnz$PartitionAttentionCl._partition_attnM  s�   € Ø”7˜1˜Q˜3”<ˆØÔð 	AÝ*¨1¨dÔ.AÑBÔBˆKˆKå(¨¨DÔ,?Ñ@Ô@ˆKà—i’i Ñ,Ô,ˆàÔð 	IÝ˜{¨DÔ,?ÀÑJÔJˆAˆAå˜[¨$Ô*=¸xÑHÔHˆAØˆr`   c           
      óJ  — ||                       |                      |                      |                      |¦  «        ¦  «        ¦  «        ¦  «        z   }||                      |                      |                      |                      |¦  «        ¦  «        ¦  «        ¦  «        z   }|S r[   ©r  r  rv  r  r  r  r÷   r  rì   s     r^   rÓ   zPartitionAttentionCl.forward\  óy   € Ø�—’ §¢¨×)=Ò)=¸d¿jºjÈ¹m¼mÑ)LÔ)LÑ MÔ MÑNÔNÑNˆØ�—’ §¢¨¯ª°$·*²*¸Q±-´-Ñ)@Ô)@Ñ AÔ AÑBÔBÑBˆØˆr`   )ra   rb   rc   rd   r8   re   ri   rh   r©   rv  rÓ   r×   rØ   s   @r^   rk  rk    s¶   ø€ € € € € ðð ð #*Ø)>Ð)>Ñ)@Ô)@Ø!ØØð*Sð *Sàð*Sð  ð*Sð 'ð	*Sð
 ð*Sð *Sð *Sð *Sð *Sð *SðXð ð ðð ð ð ð ð ð r`   rk  c                   ó’   ‡ — e Zd ZdZ e¦   «         dddfdededefˆ fd„Zdej	        d	ej	        fd
„Z
dej	        d	ej	        fd„Zˆ xZS )ÚParallelPartitionAttentionzQExperimental. Grid and Block partition + single FFN.

    NxC tensor layout.
    rC   Nrž   r  r  c           
      óR  •— ||dœ}t          ¦   «                              ¦   «          |dz  dk    sJ ‚t          t          |j        ¦  «        |j        ¬¦  «        }t          |j        ¦  «        }|j        |j	        k    sJ ‚t          |j        ¦  «        | _        t          || j        ¦  «        }	 ||fi |¤Ž| _        t          ||dz  f|j        |j        |j        |	|j        |j        dœ|¤Ž| _        t          ||dz  f|j        |j        |j        |	|j        |j        dœ|¤Ž| _        |j        rt/          |fd|j        i|¤Žnt1          j        ¦   «         | _        |dk    rt7          |¦  «        nt1          j        ¦   «         | _         ||fi |¤Ž| _        t=          d
|t?          ||j         z  ¦  «        |||j        dœ|¤Ž| _!        |j        rt/          |fd|j        i|¤Žnt1          j        ¦   «         | _"        |dk    rt7          |¦  «        nt1          j        ¦   «         | _#        d	S )z’
        Args:
            dim: Input dimension.
            cfg: Transformer block configuration.
            drop_path: Drop path rate.
        r£   r�   r   r  ro  rQ   rC   )r
  r  Úout_featuresrS   r  Nrj   )$r¨   r©   r   r    rW   rY   r   rS   rM   rN   r%   rq  ri  r  rÚ   r<   rB   r=   rD   rE   Ú
attn_blockÚ	attn_gridrQ   r   r   rê   r  r   r  r  r   re   r?   r÷   r  r  )r]   rž   r  r  r¤   r¥   r±   rU   rS   r    r³   s             €r^   r©   z#ParallelPartitionAttention.__init__h  sl  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ�Q‰w˜!Š|ˆ|ˆ|ˆ|Ý�^¨CÔ,=Ñ>Ô>ÀCÄLÐQÑQÔQˆ
Ý! #¤-Ñ0Ô0ˆ	àŒ #¤-Ò/Ð/Ð/Ð/Ý'¨¬Ñ8Ô8ˆÔÝ% c¨4Ô+>Ñ?Ô?ˆà�Z Ð*Ð* rÐ*Ð*ˆŒ
Ý%ØØ�1‰Hð

ð ”\Ø”Ø”~Ø#Ø”mØ”mð

ð 

ð ð

ð 

ˆŒõ %ØØ�1‰Hð

ð ”\Ø”Ø”~Ø#Ø”mØ”mð

ð 

ð ð

ð 

ˆŒð JMÌÐk•:˜cÐEÐE¨s¬ÐEÀ"ÐEÐEÐEÕ^`Ô^iÑ^kÔ^kˆŒØ1:¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒà�Z Ð*Ð* rÐ*Ð*ˆŒ
Ýð 
ØÝ  cÔ&6Ñ 6Ñ7Ô7ØØØ”ð
ð 
ð ð
ð 
ˆŒð JMÌÐk•:˜cÐEÐE¨s¬ÐEÀ"ÐEÐEÐEÕ^`Ô^iÑ^kÔ^kˆŒØ1:¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒˆˆr`   r´   r¶   c                 óP  — |j         dd…         }t          || j        ¦  «        }|                      |¦  «        }t	          || j        |¦  «        }t          || j        ¦  «        }|                      |¦  «        }t          || j        |¦  «        }t          j	        ||gd¬¦  «        S )Nr.   ro   r¸   r¹   )
r¾   rX  rq  r~  r]  rb  r  re  rÄ   Úcat)r]   r´   rY  Úpartitioned_blockÚx_windowÚpartitioned_gridÚx_grids          r^   rv  z*ParallelPartitionAttention._partition_attn¦  s¡   € Ø”7˜1˜Q˜3”<ˆå,¨Q°Ô0CÑDÔDÐØ ŸOšOÐ,=Ñ>Ô>ÐÝ!Ð"3°TÔ5HÈ(ÑSÔSˆå)¨!¨TÔ-@ÑAÔAÐØŸ>š>Ð*:Ñ;Ô;ÐÝÐ.°Ô0CÀXÑNÔNˆåŒy˜( FÐ+°Ð4Ñ4Ô4Ð4r`   c           
      óJ  — ||                       |                      |                      |                      |¦  «        ¦  «        ¦  «        ¦  «        z   }||                      |                      |                      |                      |¦  «        ¦  «        ¦  «        ¦  «        z   }|S r[   rx  rì   s     r^   rÓ   z"ParallelPartitionAttention.forward³  ry  r`   )ra   rb   rc   rd   r8   re   rh   r©   rÄ   rÖ   rv  rÓ   r×   rØ   s   @r^   r{  r{  b  sÖ   ø€ € € € € ðð ð *?Ð)>Ñ)@Ô)@Ø!ØØð<Sð <Sàð<Sð 'ð<Sð ð	<Sð <Sð <Sð <Sð <Sð <Sð|5 ¤ð 5°%´,ð 5ð 5ð 5ð 5ð˜œð ¨%¬,ð ð ð ð ð ð ð ð r`   r{  c           	      óÒ  — | j         \  }}}}t          ||d         z  dk    d|› d|d         › d�¦  «         t          ||d         z  dk    d|› d|d         › d�¦  «         |                      ||||d         z  |d         ||d         z  |d         ¦  «        } |                      ddddd	d
¦  «                             ¦   «                              d||d         |d         ¦  «        }|S )z#Partition windows for NCHW tensors.r   rQ  rR  rS  r.   rT  r�   rU  ro   rŽ   r¸   rV  )r´   rM   rÌ   r�   rÍ   rÎ   rW  s          r^   Úwindow_partition_nchwrˆ  ¹  s  € à”�J€A€qˆ!ˆQÝˆA�˜A”Ñ !Ò#Ð%c°Ð%cÐ%cÐR]Ð^_ÔR`Ð%cÐ%cÐ%cÑdÔdÐdÝˆA�˜A”Ñ !Ò#Ð%b¨qÐ%bÐ%bÐQ\Ð]^ÔQ_Ð%bÐ%bÐ%bÑcÔcÐcØ	�Šˆq�!�Q˜+ aœ.Ñ(¨+°a¬.¸!¸{È1¼~Ñ:MÈ{Ð[\Ì~Ñ^Ô^€AØ�iŠi˜˜1˜a  A qÑ)Ô)×4Ò4Ñ6Ô6×;Ò;¸BÀÀ;ÈqÄ>ÐS^Ð_`ÔSaÑbÔb€GØ€Nr`   c           	      ó  — |\  }}| j         d         }|                      d||d         z  ||d         z  ||d         |d         ¦  «        }|                     dddddd¦  «                             ¦   «                              d|||¦  «        }|S )z*Reverse window partition for NCHW tensors.r.   r¸   r   ro   rU  r�   rŽ   r[  r\  s          r^   Úwindow_reverse_nchwrŠ  Ã  s’   € ð �D€A€qØŒ�aÔ€AØ�Š�R˜˜k¨!œnÑ,¨a°;¸q´>Ñ.AÀ1ÀkÐRSÄnÐVaÐbcÔVdÑeÔe€AØ	�	Š	�!�Q˜˜1˜a Ñ#Ô#×.Ò.Ñ0Ô0×5Ò5°b¸!¸QÀÑBÔB€AØ€Hr`   c           
      óÎ  — | j         \  }}}}t          ||d         z  dk    d|› d|d         › �¦  «         t          ||d         z  dk    d|› d|d         › �¦  «         |                      |||d         ||d         z  |d         ||d         z  ¦  «        } |                      dddddd	¦  «                             ¦   «                              d
||d         |d         ¦  «        }|S )z Grid partition for NCHW tensors.r   r_  r`  r.   ra  ro   rŽ   r�   rU  r¸   rV  )r´   rN   rÌ   r�   rÍ   rÎ   rW  s          r^   Úgrid_partition_nchwrŒ  Í  sú   € à”�J€A€qˆ!ˆQÝˆA�	˜!”Ñ Ò!Ð#Y¨QÐ#YÐ#YÈ9ÐUVÌ<Ð#YÐ#YÑZÔZÐZÝˆA�	˜!”Ñ Ò!Ð#X¨AÐ#XÐ#XÈ)ÐTUÌ,Ð#XÐ#XÑYÔYÐYØ	�Šˆq�!�Y˜q”\ 1¨	°!¬Ñ#4°iÀ´lÀAÈÐSTÌÑDUÑVÔV€AØ�iŠi˜˜1˜a  A qÑ)Ô)×4Ò4Ñ6Ô6×;Ò;¸BÀÀ9ÈQÄ<ÐQZÐ[\ÔQ]Ñ^Ô^€GØ€Nr`   c           	      ó  — |\  }}| j         d         }|                      d||d         z  ||d         z  ||d         |d         ¦  «        }|                     dddddd¦  «                             ¦   «                              d|||¦  «        }|S )z(Reverse grid partition for NCHW tensors.r.   r¸   r   ro   rU  rŽ   r�   r[  rd  s          r^   Úgrid_reverse_nchwrŽ  ×  s‘   € ð �D€A€qØŒ�aÔ€AØ�Š�R˜˜i¨œlÑ*¨A°¸1´Ñ,=¸qÀ)ÈAÄ,ÐPYÐZ[ÔP\Ñ]Ô]€AØ	�	Š	�!�Q˜˜1˜a Ñ#Ô#×.Ò.Ñ0Ô0×5Ò5°b¸!¸QÀÑBÔB€AØ€Hr`   c            	       ó˜   ‡ — e Zd ZdZd e¦   «         dddfdedededefˆ fd	„Zd
e	j
        de	j
        fd„Zd
e	j
        de	j
        fd„Zˆ xZS )ÚPartitionAttention2dzHGrid or Block partition + Attn + FFN.

    '2D' NCHW tensor layout.
    rl  rC   Nrž   rm  r  r  c           
      óÖ  •— ||dœ}t          ¦   «                              ¦   «          t          t          |j        ¦  «        |j        ¬¦  «        }t          |j        ¦  «        }	|dk    | _        t          | j        r|j
        n|j        ¦  «        | _        t          || j        ¦  «        }
 ||fi |¤Ž| _        t          ||f|j        |j        |j        |
|j        |j        dœ|¤Ž| _        |j        rt/          |fd|j        i|¤Žnt1          j        ¦   «         | _        |dk    rt7          |¦  «        nt1          j        ¦   «         | _         ||fi |¤Ž| _        t=          d	|t?          ||j         z  ¦  «        |	|j        dœ|¤Ž| _!        |j        rt/          |fd|j        i|¤Žnt1          j        ¦   «         | _"        |dk    rt7          |¦  «        nt1          j        ¦   «         | _#        dS )
zÒ
        Args:
            dim: Input dimension.
            partition_type: Partition type ('block' or 'grid').
            cfg: Transformer block configuration.
            drop_path: Drop path rate.
        r£   r  rl  ro  rQ   rC   r	  Nrj   )$r¨   r©   r   r    rU   rY   r   rS   rp  r%   rM   rN   rq  ri  r  rœ   r<   rB   r=   rD   rE   rÒ   rQ   r   r   rê   r  r   r  r  r   re   r?   r÷   r  r  rr  s              €r^   r©   zPartitionAttention2d.__init__ç  s  ø€ ð  ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ�^¨C¬NÑ;Ô;ÀÄÐNÑNÔNˆ
Ý! #¤-Ñ0Ô0ˆ	à-°Ò8ˆÔÝ'¸4Ô;OÐ(b¨¬¨ÐUXÔUbÑcÔcˆÔÝ% c¨4Ô+>Ñ?Ô?ˆà�Z Ð*Ð* rÐ*Ð*ˆŒ
ÝØØð

ð ”\Ø”Ø”~Ø#Ø”mØ”mð

ð 

ð ð

ð 

ˆŒ	ð LOÌ?Ðm•< ÐGÐG°´ÐGÀBÐGÐGÐGÕ`bÔ`kÑ`mÔ`mˆŒØ1:¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒà�Z Ð*Ð* rÐ*Ð*ˆŒ
Ýð 
ØÝ  cÔ&6Ñ 6Ñ7Ô7ØØ”ð	
ð 
ð
 ð
ð 
ˆŒð LOÌ?Ðm•< ÐGÐG°´ÐGÀBÐGÐGÐGÕ`bÔ`kÑ`mÔ`mˆŒØ1:¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒˆˆr`   r´   r¶   c                 ó  — |j         dd …         }| j        rt          || j        ¦  «        }nt	          || j        ¦  «        }|                      |¦  «        }| j        rt          || j        |¦  «        }nt          || j        |¦  «        }|S )Nrº   )r¾   rp  rˆ  rq  rŒ  rÒ   rŠ  rŽ  rt  s       r^   rv  z$PartitionAttention2d._partition_attn  s�   € Ø”7˜2˜3˜3”<ˆØÔð 	FÝ/°°4Ô3FÑGÔGˆKˆKå-¨a°Ô1DÑEÔEˆKà—i’i Ñ,Ô,ˆàÔð 	NÝ# K°Ô1DÀhÑOÔOˆAˆAå! +¨tÔ/BÀHÑMÔMˆAØˆr`   c           
      óJ  — ||                       |                      |                      |                      |¦  «        ¦  «        ¦  «        ¦  «        z   }||                      |                      |                      |                      |¦  «        ¦  «        ¦  «        ¦  «        z   }|S r[   rx  rì   s     r^   rÓ   zPartitionAttention2d.forward)  ry  r`   )ra   rb   rc   rd   r8   re   ri   rh   r©   rÄ   rÖ   rv  rÓ   r×   rØ   s   @r^   r�  r�  á  sä   ø€ € € € € ðð ð #*Ø)>Ð)>Ñ)@Ô)@Ø!ØØð1Sð 1Sàð1Sð  ð1Sð 'ð	1Sð
 ð1Sð 1Sð 1Sð 1Sð 1Sð 1Sðf ¤ð °%´,ð ð ð ð ð˜œð ¨%¬,ð ð ð ð ð ð ð ð r`   r�  c                   óx   ‡ — e Zd ZdZd e¦   «          e¦   «         dddfdedededed	ed
efˆ fd„Zdd„Z	d„ Z
ˆ xZS )ÚMaxxVitBlockz;MaxVit conv, window partition + FFN , grid partition + FFN.r.   rC   Nrž   rŸ   ræ   r–   r—   r  c	                 óV  •— ||dœ}	t          ¦   «                              ¦   «          |j        | _        |j        dk    rt
          nt          }
 |
||f|||dœ|	¤Ž| _        t          d|||dœ|	¤Ž}| j        rt          nt          }|j        rdn |di |¤Ž| _         |dddi|¤Ž| _        dS )	a^  Initialize MaxxVitBlock.

        Args:
            dim: Input channel dimension.
            dim_out: Output channel dimension.
            stride: Stride for downsampling.
            conv_cfg: Configuration for convolutional blocks.
            transformer_cfg: Configuration for transformer blocks.
            drop_path: Drop path rate.
        r£   r�   ©ræ   r  r  ©rž   r  r  Nrm  Úgridrj   )r¨   r©   rP   Ú	nchw_attnrm   rC  r'  ÚconvÚdictr�  rk  rO   r~  r  )r]   rž   rŸ   ræ   r–   r—   r  r¤   r¥   r±   Úconv_clsr<  Úpartition_layerr³   s                €r^   r©   zMaxxVitBlock.__init__2  sâ   ø€ ð* ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ(Ô6ˆŒà$,Ô$7¸:Ò$EÐ$E•=�=Í;ˆØ�H˜S 'Ðb°&¸hÐR[ÐbÐbÐ_aÐbÐbˆŒ	åÐW˜w¨OÀyÐWÐWÐTVÐWÐWˆØ26´.ÐZÕ.Ð.ÕFZˆØ"1Ô"?Ðc˜$˜$À_À_ÐEcÐEcÐWbÐEcÐEcˆŒØ(˜ÐNÐN¸ÐNÀ+ÐNÐNˆŒˆˆr`   rw   c                 ó
  — | j         �)t          t          t          |¬¦  «        | j         ¦  «         t          t          t          |¬¦  «        | j        ¦  «         t          t          t
          |¬¦  «        | j        ¦  «         d S r  )r~  r2   r   rÿ   r  r"  r›  r  s     r^   r  zMaxxVitBlock.init_weightsS  so   € ØŒ?Ð&Ý�Õ 1¸&ÐAÑAÔAÀ4Ä?ÑSÔSÐSÝ•GÕ-°fÐ=Ñ=Ô=¸t¼~ÑNÔNÐNÝ•G�J¨vÐ6Ñ6Ô6¸¼	ÑBÔBÐBÐBÐBr`   c                 ó  — |                       |¦  «        }| j        s|                     dddd¦  «        }| j        �|                      |¦  «        }|                      |¦  «        }| j        s|                     dddd¦  «        }|S rN  )r›  rš  rO  r~  r  rì   s     r^   rÓ   zMaxxVitBlock.forwardY  s‚   € à�IŠI�a‰LŒLˆàŒ~ð 	&Ø—	’	˜!˜Q  1Ñ%Ô%ˆAØŒ?Ð&Ø—’ Ñ"Ô"ˆAØ�NŠN˜1ÑÔˆØŒ~ð 	&Ø—	’	˜!˜Q  1Ñ%Ô%ˆAØˆr`   r  )ra   rb   rc   rd   r7   r8   re   rh   r©   r  rÓ   r×   rØ   s   @r^   r•  r•  /  sÜ   ø€ € € € € ØEÐEð Ø'5 ~Ñ'7Ô'7Ø5JÐ5JÑ5LÔ5LØ!ØØðOð OàðOð ðOð ð	Oð
 %ðOð 3ðOð ðOð Oð Oð Oð Oð OðBCð Cð Cð Cðð ð ð ð ð ð r`   r•  c                   ó¦   ‡ — e Zd ZdZdd e¦   «          e¦   «         dddfdededed	ed
ededefˆ fd„Zdde	ddfd„Z
dej        dej        fd„Zˆ xZS )ÚParallelMaxxVitBlockzYMaxVit block with parallel cat(window + grid), one FF.

    Experimental timm block.
    r.   r�   rC   Nrž   rŸ   ræ   Únum_convr–   r—   r  c
                 óT  •— ||	dœ}
t          ¦   «                              ¦   «          |j        dk    rt          nt          }|dk    r: |||f|||dœ|
¤Žg}| |||f||dœ|
¤Žg|dz
  z  z  }t          j        |Ž | _        n |||f|||dœ|
¤Ž| _        t          d|||dœ|
¤Ž| _	        dS )	aa  
        Args:
            dim: Input dimension.
            dim_out: Output dimension.
            stride: Stride for first conv block.
            num_conv: Number of convolution blocks.
            conv_cfg: Convolution block configuration.
            transformer_cfg: Transformer block configuration.
            drop_path: Drop path rate.
        r£   r�   r.   r—  )r  r  r˜  Nrj   )
r¨   r©   rm   rC  r'  r   r  r›  r{  rÒ   )r]   rž   rŸ   ræ   r£  r–   r—   r  r¤   r¥   r±   r�  Úconvsr³   s                €r^   r©   zParallelMaxxVitBlock.__init__m  sý   ø€ ð, ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐà$,Ô$7¸:Ò$EÐ$E•=�=Í;ˆØ�aŠ<ˆ<Ø�X˜c 7Ðc°6¸xÐS\ÐcÐcÐ`bÐcÐcÐdˆEØ�h�h˜w¨ÐY°XÈÐYÐYÐVXÐYÐYÐZÐ^fÐijÑ^jÑkÑkˆEÝœ uÐ-ˆDŒIˆIà ˜  gÐf°fÀ(ÐV_ÐfÐfÐceÐfÐfˆDŒIÝ.Ðk°7ÀÐ[dÐkÐkÐhjÐkÐkˆŒ	ˆ	ˆ	r`   rw   rï   r¶   c                 óª   — t          t          t          |¬¦  «        | j        ¦  «         t          t          t          |¬¦  «        | j        ¦  «         d S r  )r2   r   rÿ   rÒ   r"  r›  r  s     r^   r  z!ParallelMaxxVitBlock.init_weights�  sG   € Ý•GÕ-°fÐ=Ñ=Ô=¸t¼yÑIÔIÐIÝ•G�J¨vÐ6Ñ6Ô6¸¼	ÑBÔBÐBÐBÐBr`   r´   c                 óº   — |                       |¦  «        }|                     dddd¦  «        }|                      |¦  «        }|                     dddd¦  «        }|S rN  )r›  rO  rÒ   rì   s     r^   rÓ   zParallelMaxxVitBlock.forward“  sS   € Ø�IŠI�a‰LŒLˆØ�IŠI�a˜˜A˜qÑ!Ô!ˆØ�IŠI�a‰LŒLˆØ�IŠI�a˜˜A˜qÑ!Ô!ˆØˆr`   r  )ra   rb   rc   rd   r7   r8   re   rh   r©   ri   r  rÄ   rÖ   rÓ   r×   rØ   s   @r^   r¢  r¢  g  s  ø€ € € € € ðð ð ØØ'5 ~Ñ'7Ô'7Ø5JÐ5JÑ5LÔ5LØ!ØØð lð  làð lð ð lð ð	 lð
 ð lð %ð lð 3ð lð ð lð  lð  lð  lð  lð  lðDCð C 3ð C°ð Cð Cð Cð Cð˜œð ¨%¬,ð ð ð ð ð ð ð ð r`   r¢  c                   óè   ‡ — e Zd ZdZdddd e¦   «          e¦   «         dddf	ded	ed
ededeeef         dee	ee	         f         dededee
ee
         f         fˆ fd„Zdej        dej        fd„Zˆ xZS )ÚMaxxVitStagezEMaxxVit stage consisting of mixed convolution and transformer blocks.r�   rU  )é   rª  r�   rC   Nr)  r*  ræ   ÚdepthÚ	feat_sizeÚblock_typesr—   r–   r  c           
      ó^  •— |
|dœ}t          ¦   «                              ¦   «          d| _        t          ||¦  «        }g }t	          |¦  «        D ]Ê\  }}|dk    r|nd}|dv sJ ‚|dk    r2|j        dk    rt          nt          }| |||f|||	|         dœ|¤Žgz  }n{|d	k    r.t          ||¦  «        }|t          ||f||||	|         d
œ|¤Žgz  }nG|dk    r|t          ||f||||	|         dœ|¤Žgz  }n#|dk    r|t          ||f||||	|         dœ|¤Žgz  }|}ŒËt          j        |Ž | _        dS )aÒ  
        Args:
            in_chs: Input channels.
            out_chs: Output channels.
            stride: Stride for first block.
            depth: Number of blocks in stage.
            feat_size: Feature map size.
            block_types: Block types ('C' for conv, 'T' for transformer, etc).
            transformer_cfg: Transformer block configuration.
            conv_cfg: Convolution block configuration.
            drop_path: Drop path rate(s).
        r£   Fr   r.   )r�   r‘   ÚMÚPMr�   r�   r—  r‘   )ræ   r    r  r  r¯  )ræ   r–   r—   r  r°  N)r¨   r©   Úgrad_checkpointingr&   Ú	enumeraterm   rC  r'  ri  r  r•  r¢  r   r  Úblocks)r]   r)  r*  ræ   r«  r¬  r­  r—   r–   r  r¤   r¥   r±   r³  ÚiÚtÚblock_strider�  r    r³   s                      €r^   r©   zMaxxVitStage.__init__ž  s  ø€ ð4 ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ"'ˆÔå" ;°Ñ6Ô6ˆØˆÝ˜kÑ*Ô*ð ,	ð ,	‰DˆAˆqØ%&¨!¢V V˜6˜6°ˆLØÐ-Ð-Ð-Ð-Ð-Ø�CŠxˆxØ,4Ô,?À:Ò,MÐ,M�=˜=ÕS^�Ø˜8˜8ØØðð (Ø Ø'¨œlðð ð ðð ð ñ ��ð �c’�Ý-¨o¸yÑIÔI�ØÕ-ØØðð (Ø +Ø'Ø'¨œlðð ð ðð ð ñ ��ð �c’�Ø�<ØØðð (Ø%Ø$3Ø'¨œlðð ð ðð ð ñ ��ð �d’�ØÕ/ØØðð (Ø%Ø$3Ø'¨œlðð ð ðð ð ñ �ð ˆFˆFÝ”m VÐ,ˆŒˆˆr`   r´   r¶   c                 ó¦   — | j         r4t          j                             ¦   «         st	          | j        |¦  «        }n|                      |¦  «        }|S r[   )r±  rÄ   ÚjitÚis_scriptingr3   r³  rì   s     r^   rÓ   zMaxxVitStage.forwardí  sI   € ØÔ"ð 	­5¬9×+AÒ+AÑ+CÔ+Cð 	Ý˜tœ{¨AÑ.Ô.ˆAˆAà—’˜A‘”ˆAØˆr`   )ra   rb   rc   rd   r8   r7   re   r   r   ri   rh   r   r©   rÄ   rÖ   rÓ   r×   rØ   s   @r^   r©  r©  ›  s(  ø€ € € € € ØOÐOð ØØ)1Ø25Ø5JÐ5JÑ5LÔ5LØ'5 ~Ñ'7Ô'7Ø35ØØðM-ð M-àðM-ð ðM-ð ð	M-ð
 ðM-ð ˜S #˜X”ðM-ð ˜s E¨#¤J˜Ô/ðM-ð 3ðM-ð %ðM-ð ˜U D¨¤KÐ/Ô0ðM-ð M-ð M-ð M-ð M-ð M-ð^˜œð ¨%¬,ð ð ð ð ð ð ð ð r`   r©  c                   óŒ   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 dd	ed
ededededededefˆ fd„Zddeddfd„Z	de
j        de
j        fd„Zˆ xZS )ÚStemz"Stem layer for feature extraction.ro   rw   FrR   r‚   rƒ   Nr)  r*  rp   rx   rH   rS   rU   rY   c                 óº  •— |	|
dœ}t          ¦   «                              ¦   «          t          |t          t          f¦  «        st          |¦  «        }t          t          ||¦  «        |¬¦  «        }|d         | _        d| _	        t          ||d         |fd||dœ|¤Ž| _         ||d         fi |¤Ž| _        t          |d         |d         |fd||dœ|¤Ž| _        dS )	ae  
        Args:
            in_chs: Input channels.
            out_chs: Output channels.
            kernel_size: Kernel size for convolutions.
            padding: Padding mode.
            bias: Whether to use bias.
            act_layer: Activation layer.
            norm_layer: Normalization layer.
            norm_eps: Normalization epsilon.
        r£   r  r¸   r�   r   )ræ   rx   rH   r.   N)r¨   r©   rø   ÚlistÚtupler%   r   r!   r*  ræ   r"   Úconv1r  Úconv2)r]   r)  r*  rp   rx   rH   rS   rU   rY   r¤   r¥   r±   r6  r³   s                €r^   r©   zStem.__init__ø  sö   ø€ ð0 ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ˜'¥D­% =Ñ1Ô1ð 	)Ý Ñ(Ô(ˆGå Õ!3°JÀ	Ñ!JÔ!JÐPXÐYÑYÔYˆØ˜r”{ˆŒØˆŒå" 6¨7°1¬:°{ÐoÈ1ÐV]ÐdhÐoÐoÐlnÐoÐoˆŒ
Ø#�^ G¨A¤JÐ5Ð5°"Ð5Ð5ˆŒ
Ý" 7¨1¤:¨w°q¬z¸;ÐsÈqÐZaÐhlÐsÐsÐprÐsÐsˆŒ
ˆ
ˆ
r`   rï   r¶   c                 óN   — t          t          t          |¬¦  «        | ¦  «         d S r  r>  r  s     r^   r  zStem.init_weights  r?  r`   r´   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r[   )r¿  r  rÀ  rì   s     r^   rÓ   zStem.forward   s4   € Ø�JŠJ�q‰MŒMˆØ�JŠJ�q‰MŒMˆØ�JŠJ�q‰MŒMˆØˆr`   )ro   rw   FrR   r‚   rƒ   NNr  )ra   rb   rc   rd   re   ri   rg   rh   r©   r  rÄ   rÖ   rÓ   r×   rØ   s   @r^   r»  r»  õ  s  ø€ € € € € Ø,Ð,ð  !ØØØ#Ø+Ø"ØØð#tð #tàð#tð ð#tð ð	#tð
 ð#tð ð#tð ð#tð ð#tð ð#tð #tð #tð #tð #tð #tðJ>ð > 3ð >°ð >ð >ð >ð >ð˜œð ¨%¬,ð ð ð ð ð ð ð ð r`   r»  c                 óŽ   — | j         �| j        sJ ‚| S |d         | j        z  |d         | j        z  f}t          | ||¬¦  «        } | S )z>Configure window size based on image size and partition ratio.Nr   r.   )rM   rN   )rM   rN   rL   r   )r  rY  rq  s      r^   Úcfg_window_sizerÄ  '  sW   € à
„Ð"ØŒ}ÐÐˆ}Øˆ
Ø˜a”[ CÔ$7Ñ7¸À!¼ÈÔH[Ñ9[Ð[€NÝ
�# >¸^Ð
LÑ
LÔ
L€CØ€Jr`   Úkwargsc           	      ól  — i }i }i }|                      ¦   «         D ]h\  }}|                     d¦  «        r|||                     dd¦  «        <   Œ4|                     d¦  «        r|||                     dd¦  «        <   Œc|||<   Œit          | ft          | j        fi |¤Žt          | j        fi |¤Ždœ|¤Ž} | S )z-Overlay keyword arguments onto configuration.Útransformer_rw   Úconv_)r—   r–   )ÚitemsÚ
startswithr   r—   r–   )r  rÅ  Útransformer_kwargsÚconv_kwargsÚbase_kwargsrÐ   rÑ   s          r^   Ú_overlay_kwargsrÎ  1  sð   € àÐØ€KØ€KØ—’‘”ð ð ‰ˆˆ1Ø�<Š<˜Ñ'Ô'ð 	Ø@AÐ˜qŸyšy¨¸Ñ<Ô<Ñ=Ð=Ø�\Š\˜'Ñ"Ô"ð 	Ø23ˆK˜Ÿ	š	 '¨2Ñ.Ô.Ñ/Ð/àˆK˜‰NˆNÝ
Øðå Ô 3ÐJÐJÐ7IÐJÐJÝ˜œÐ5Ð5¨Ð5Ð5ðð ð ð	ð €Cð €Jr`   c                   ó$  ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 d1ded	eeeeef         f         d
ededede	de	de
fˆ fd„Zd2dej        dededdfd„Zej        j        dee         fd„¦   «         Zej        j        d3dedeee
f         fd„¦   «         Zej        j        d4deddfd„¦   «         Zej        j        dej        fd„¦   «         Zd5dedee         ddfd„Z	 	 	 	 	 d6d!ej        d"eeeee         f                  d#ed$ed%ed&edeeej                 eej        eej                 f         f         fd'„Z	 	 	 d7d"eeee         f         d)ed*edeed+f         fd,„Zd!ej        dej        fd-„Zd3d!ej        d.edej        fd/„Z d!ej        dej        fd0„Z!ˆ xZ"S )8r9   z{CoaTNet + MaxVit base model.

    Highly configurable for different block compositions, tensor layouts, pooling types.
    éà   ro   éè  rå   rC   Nr  rY  Úin_chansÚnum_classesÚglobal_poolÚ	drop_rateÚdrop_path_raterÅ  c
                 ó¨  •‡— t          ¦   «                              ¦   «          ||	dœ}t          |¦  «        }|
rt          |fi |
¤Ž}t	          |j        |¦  «        }|| _        || _        || _        |j	        d         x| _
        | _	        || _        d| _        g | _        t          d||j        |j        j        |j        |j        j        |j        j        |j        j        dœ|¤Ž| _        | j        j        }| xj        t1          | j        j        dd¬¦  «        gz  c_        t5          d„ t7          |t          |¦  «        ¦  «        D ¦   «         ¦  «        }t9          |j	        ¦  «        }t9          |j        ¦  «        |k    sJ ‚t=          ||j        d	¬
¦  «        }| j        j        }g }t?          |¦  «        D ]‘}dŠ|j	        |         }t5          ˆfd„|D ¦   «         ¦  «        }|tA          ||f|j        |         |j!        |         |j        ||||         dœ|¤Žgz  }|‰z  }|}| xj        t1          ||d|› �¬¦  «        gz  c_        Œ’tE          j#        |Ž | _$        tK          tM          |j        j        ¦  «        |j        j        ¬¦  «        }|j'        rGtE          j(        ¦   «         | _)        |j'        | _'        tU          | j
        |f| j'        |||dœ|¤Ž| _+        n:| j
        | _'         || j
        fi |¤Ž| _)        tY          | j
        |f||dœ|¤Ž| _+        |j-        dv sJ ‚|j-        r+t]          tK          | j/        |j-        ¬¦  «        | ¦  «         dS dS )a˜  
        Args:
            cfg: Model configuration.
            img_size: Input image size.
            in_chans: Number of input channels.
            num_classes: Number of classification classes.
            global_pool: Global pooling type.
            drop_rate: Dropout rate.
            drop_path_rate: Drop path rate.
            **kwargs: Additional keyword arguments to overlay on config.
        r£   r¸   F)r)  r*  rx   rH   rS   rU   rY   r�   Ústem)Únum_chsÚ	reductionrí   c                 ó   — g | ]
\  }}||z  ‘ŒS rj   rj   )Ú.0r´  Úss      r^   ú
<listcomp>z$MaxxVit.__init__.<locals>.<listcomp>~  s    € ÐOÐOÐO¡d a¨˜1 ™6ÐOÐOÐOr`   T)Ú	stagewisec                 ó&   •— g | ]}|d z
  ‰z  d z   ‘ŒS )r.   rj   )rÜ  ÚrÚstage_strides     €r^   rÞ  z$MaxxVit.__init__.<locals>.<listcomp>ˆ  s'   ø€ ÐNÐNÐN¸q  A¡¨,Ñ6¸Ñ:ÐNÐNÐNr`   )r«  r­  r–   r—   r¬  r  zstages.r  )Úhidden_sizerG   rÕ  rU   )rG   rÕ  )rw   rñ   rõ   rö   r™   r  Nrj   )0r¨   r©   r%   rÎ  rÄ  r—   rÓ  rÒ  rÔ  r‹   Únum_featuresrÕ  r±  Úfeature_infor»  r“   r–   rx   r”   rS   rU   rY   rØ  ræ   rœ  r*  r¾  ÚzipÚlenr�   r   Úranger©  rm   r   r  Ústagesr   r    r˜   rê   r  r   Úheadr   rš   r2   Ú_init_weights)r]   r  rY  rÒ  rÓ  rÔ  rÕ  rÖ  r¤   r¥   rÅ  r±   r—   ræ   r¬  Ú
num_stagesÚdprr)  ré  r´  r*  Úfinal_norm_layerrâ  r³   s                         @€r^   r©   zMaxxVit.__init__L  sé  øø€ õ0 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆÝ˜XÑ&Ô&ˆØð 	1Ý! #Ð0Ð0¨Ð0Ð0ˆCÝ)¨#Ô*=¸xÑHÔHˆØ&ˆÔØ ˆŒØ&ˆÔØ-0¬]¸2Ô->Ð>ˆÔ˜DœNØ"ˆŒØ"'ˆÔØˆÔåð 	
ØØ”NØ”LÔ(Ø”Ø”lÔ,Ø”|Ô.Ø”\Ô*ð	
ð 	
ð ð	
ð 	
ˆŒ	ð ”Ô!ˆØÐÔ�d¨4¬9Ô+<ÈÐRXÐYÑYÔYÐZÑZÐÔÝÐOÐO­c°(½IÀfÑ<MÔ<MÑ.NÔ.NÐOÑOÔOÑPÔPˆ	å˜œÑ'Ô'ˆ
Ý�3”:‰Œ *Ò,Ð,Ð,Ð,Ý'¨¸¼
ÈdÐSÑSÔSˆØ”Ô"ˆØˆÝ�zÑ"Ô"ð 	að 	aˆAØˆLØ”m AÔ&ˆGÝÐNÐNÐNÐNÀIÐNÑNÔNÑOÔOˆIØ•|ØØð
ð ”j ”mØœN¨1Ô-ØœØ /Ø#Ø˜aœ&ð
ð 
ð ð
ð 
ð 
ñ 
ˆFð �lÑ"ˆFØˆFØÐÔ¥$¨wÀ&ÐQ^Ð[\ÐQ^ÐQ^Ð"_Ñ"_Ô"_Ð!`Ñ`ÐÔÐÝ”m VÐ,ˆŒå"¥>°#Ô2EÔ2PÑ#QÔ#QÐWZÔWjÔWsÐtÑtÔtÐØÔð 	Ýœ™œˆDŒIØ$'Ô$8ˆDÔ!Ý-ØÔ!Øðð !Ô1Ø%Ø#Ø+ðð ð ðð ˆDŒIˆIð %)Ô$5ˆDÔ!Ø(Ð(¨Ô):ÐAÐA¸bÐAÐAˆDŒIÝ&ØÔ!Øðð &Ø#ð	ð ð
 ðð ˆDŒIð ŒÐ"\Ð\Ð\Ð\Ð\ØŒ?ð 	SÝ� Ô 2¸3¼?ÐKÑKÔKÈTÑRÔRÐRÐRÐRð	Sð 	Sr`   rw   rí   rî   rï   r¶   c                 ó¢   — t          |d¦  «        r>	 |                     |¬¦  «         d S # t          $ r |                     ¦   «          Y d S w xY wd S )Nr  r  )Úhasattrr  Ú	TypeError)r]   rí   rî   rï   s       r^   rë  zMaxxVit._init_weights·  sv   € Ý�6˜>Ñ*Ô*ð 	&ð&Ø×#Ò#¨6Ð#Ñ2Ô2Ð2Ð2Ð2øÝð &ð &ð &Ø×#Ò#Ñ%Ô%Ð%Ð%Ð%Ð%ð&øøøð	&ð 	&s   ’* ªAÁAc                 ó>   — d„ |                       ¦   «         D ¦   «         S )Nc                 óP   ‡— h | ]"\  Š}t          ˆfd „dD ¦   «         ¦  «        ¯ ‰’Œ#S )c              3   ó    •K  — | ]}|‰v V — Œ	d S r[   rj   )rÜ  ÚnrÐ   s     €r^   ú	<genexpr>z4MaxxVit.no_weight_decay.<locals>.<setcomp>.<genexpr>Â  s'   øè è € ÐSÐS˜a�1˜�6ÐSÐSÐSÐSÐSÐSr`   )Úrelative_position_bias_tablezrel_pos.mlp)Úany)rÜ  Ú_rÐ   s     @r^   ú	<setcomp>z*MaxxVit.no_weight_decay.<locals>.<setcomp>À  sX   ø€ ð Uð Uð UÙ�!�QÝÐSÐSÐSÐSÐ#RÐSÑSÔSÑSÔSðUØðUð Uð Ur`   )Únamed_parametersr\   s    r^   Úno_weight_decayzMaxxVit.no_weight_decay¾  s5   € ðUð UØ×/Ò/Ñ1Ô1ðUñ Uô Uð 	Ur`   FÚcoarsec                 ó,   — t          dddg¬¦  «        }|S )Nz^stem)z^stages\.(\d+)N)z^norm)iŸ† )rØ  r³  )rœ  )r]   rý  Úmatchers      r^   Úgroup_matcherzMaxxVit.group_matcherÄ  s)   € åØØ-Ð/CÐDð
ñ 
ô 
ˆð ˆr`   TÚenablec                 ó(   — | j         D ]	}||_        Œ
d S r[   )ré  r±  )r]   r  rÝ  s      r^   Úset_grad_checkpointingzMaxxVit.set_grad_checkpointingÌ  s(   € à”ð 	*ð 	*ˆAØ#)ˆAÔ Ð ð	*ð 	*r`   c                 ó   — | j         j        S r[   )rê  Úfcr\   s    r^   Úget_classifierzMaxxVit.get_classifierÑ  s   € àŒyŒ|Ðr`   c                 óZ   — t          | ¦  «        }|| _         | j        j        ||fi |¤Ž d S r[   )r   rÓ  rê  Úreset)r]   rÓ  rÔ  r±   s       r^   Úreset_classifierzMaxxVit.reset_classifierÕ  s;   € Ý˜dÑ#Ô#ˆØ&ˆÔØˆŒ	Œ˜ [Ð7Ð7°BÐ7Ð7Ð7Ð7Ð7r`   ÚNCHWr´   Úindicesr  Ú
stop_earlyÚ
output_fmtÚintermediates_onlyc                 óJ  — |dv s
J d¦   «         ‚g }t          t          | j        ¦  «        dz   |¦  «        \  }}	d}
|                      |¦  «        }|
|v r|                     |¦  «         t          | j        ¦  «        }t
          j                             ¦   «         s|s| j        }n| j        d|	…         }|D ]K}|
dz  }
 ||¦  «        }|
|v r5|r|
|k    r|                      |¦  «        }n|}|                     |¦  «         ŒL|r|S |
|k    r|                      |¦  «        }||fS )aï   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )r
  zOutput shape must be NCHW.r.   r   N)	r0   rç  ré  rØ  ÚappendrÄ   r¸  r¹  r  )r]   r´   r  r  r  r  r  ÚintermediatesÚtake_indicesÚ	max_indexÚfeat_idxÚlast_idxré  ÚstageÚx_inters                  r^   Úforward_intermediateszMaxxVit.forward_intermediatesÚ  s`  € ð* ˜YÐ&Ð&Ð&Ð(DÑ&Ô&Ð&ØˆÝ"6µs¸4¼;Ñ7GÔ7GÈ!Ñ7KÈWÑ"UÔ"UÑˆ�ið ˆØ�IŠI�a‰LŒLˆØ�|Ð#Ð#Ø× Ò  Ñ#Ô#Ð#å�t”{Ñ#Ô#ˆÝŒ9×!Ò!Ñ#Ô#ð 	-¨:ð 	-Ø”[ˆFˆFà”[  ) Ô,ˆFØð 	.ð 	.ˆEØ˜‰MˆHØ��a‘”ˆAØ˜<Ð'Ð'Øð  ˜H¨Ò0Ð0Ø"Ÿiši¨™lœl�G�Gà�GØ×$Ò$ WÑ-Ô-Ð-øàð 	!Ø Ð à�xÒÐØ—	’	˜!‘”ˆAà�-ÐÐr`   r.   Ú
prune_normÚ
prune_head.c                 óì   — t          t          | j        ¦  «        dz   |¦  «        \  }}| j        d|…         | _        |rt          j        ¦   «         | _        |r|                      dd¦  «        | _        |S )z6Prune layers not required for specified intermediates.r.   Nr   rw   )r0   rç  ré  r   rê   r  r	  rê  )r]   r  r  r  r  r  s         r^   Úprune_intermediate_layersz!MaxxVit.prune_intermediate_layers  ss   € õ #7µs¸4¼;Ñ7GÔ7GÈ!Ñ7KÈWÑ"UÔ"UÑˆ�iØ”k * 9 *Ô-ˆŒØð 	&Ýœ™œˆDŒIØð 	5Ø×-Ò-¨a°Ñ4Ô4ˆDŒIØÐr`   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r[   )rØ  ré  r  rì   s     r^   Úforward_featureszMaxxVit.forward_features  s4   € Ø�IŠI�a‰LŒLˆØ�KŠK˜‰NŒNˆØ�IŠI�a‰LŒLˆØˆr`   Ú
pre_logitsc                 ó^   — |r|                       ||¬¦  «        n|                       |¦  «        S )N)r  )rê  )r]   r´   r  s      r^   Úforward_headzMaxxVit.forward_head%  s-   € Ø6@ÐRˆt�yŠy˜ zˆyÑ2Ô2Ð2ÀdÇiÂiÐPQÁlÄlÐRr`   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S r[   )r  r!  rì   s     r^   rÓ   zMaxxVit.forward(  s-   € Ø×!Ò! !Ñ$Ô$ˆØ×Ò˜aÑ Ô ˆØˆr`   )rÐ  ro   rÑ  rå   rC   rC   NNr  ©F)Tr[   )NFFr
  F)r.   FT)#ra   rb   rc   rd   r6   r   re   r   ri   rh   r   r©   r   ÚModulerë  rÄ   r¸  Úignorer   rü  rg   r
   r   r  r  r   r	  rÖ   r   r  r  r  r!  rÓ   r×   rØ   s   @r^   r9   r9   F  sº  ø€ € € € € ðð ð 58ØØ#Ø$Ø!Ø$&ØØðiSð iSàðiSð ˜C  s¨C x¤Ð0Ô1ðiSð ð	iSð
 ðiSð ðiSð ðiSð "ðiSð ðiSð iSð iSð iSð iSð iSðV&ð & B¤Ið &°Sð &À#ð &Ètð &ð &ð &ð &ð „YÔðU  S¤ð Uð Uð Uñ ÔðUð
 „YÔðð  Dð °T¸#¸s¸(´^ð ð ð ñ Ôðð „YÔð*ð *¨Tð *¸Tð *ð *ð *ñ Ôð*ð „YÔð ¤	ð ð ð ñ Ôðð8ð 8¨Cð 8¸hÀs¼mð 8ÐW[ð 8ð 8ð 8ð 8ð 8<ØØ$Ø$Ø',ð4 ð 4 àŒ|ð4 ð ˜e C¨¨c¬ NÔ3Ô4ð4 ð ð	4 ð
 ð4 ð ð4 ð !%ð4 ð 
ˆt�E”LÔ! 5¨¬°t¸E¼LÔ7IÐ)IÔ#JÐJÔ	Kð4 ð 4 ð 4 ð 4 ðp ./Ø$Ø#ð	ð à˜3  S¤	˜>Ô*ðð ðð ð	ð
 
ˆs�CˆxŒðð ð ð ð %¤,ð °5´<ð ð ð ð ðSð S˜eœlð S¸ð SÈÌð Sð Sð Sð Sð˜œð ¨%¬,ð ð ð ð ð ð ð ð r`   r9   rè   rF   FÚreluTrT   rV   rH   rJ   ru   rG   Úconv_output_biasÚconv_attn_earlyÚconv_attn_act_layerÚconv_norm_layerÚtransformer_shortcut_biasÚtransformer_norm_layerÚtransformer_norm_layer_clrQ   rI   rK   c                 óz   — t          t          | |dd|||d|¬¦	  «	        t          d|||	|||
|¬¦  «        ¬¦  «        S )as  RW variant configuration for CoAtNet models.

    These models were created and trained before seeing https://github.com/google-research/maxvit

    Common differences for initial timm models:
      - pre-norm layer in MZBConv included an activation after norm
      - mbconv expansion calculated from input instead of output chs
      - mbconv shortcut and final 1x1 conv did not have a bias
      - SE act layer was relu, not silu
      - mbconv uses silu in timm, not gelu
      - expansion in attention block done via output proj, not input proj

    Variable differences (evolved over training initial models):
      - avg pool with kernel_size=2 favoured downsampling (instead of maxpool for coat)
      - SE attention was between conv2 and norm/act
      - default to avg pool for mbconv downsample instead of 1x1 or dw conv
      - transformer block shortcut has no bias
    TFr|   )	ru   rG   rr   rn   rs   ry   r}   rS   rU   )r@   rA   rG   rQ   rU   rW   rI   rK   ©r–   r—   ©rœ  r7   r8   )ru   rG   r'  r(  r)  r*  r+  r,  r-  rQ   rI   rK   s               r^   Ú_rw_coat_cfgr1  .  su   € õ@ ÝØ#ØØØØ(Ø&Ø.ØØ&ð

ñ 

ô 

õ .ØØ3ØØ#Ø-Ø3Ø%Ø#ð	
ñ 	
ô 	
ðñ ô ð r`   rt   ç      °?r;   Úconv_attn_ratior<   c                 óx   — t          t          | |d||d|¬¦  «        t          d||||	|||
|¬¦	  «	        ¬¦  «        S )aµ  RW variant configuration for MaxViT models.

    These models were created and trained before seeing https://github.com/google-research/maxvit

    Differences of initial timm models:
      - mbconv expansion calculated from input instead of output chs
      - mbconv shortcut and final 1x1 conv did not have a bias
      - mbconv uses silu in timm, not gelu
      - expansion in attention block done via output proj, not input proj
    Fr|   )ru   rG   rn   rs   r   rS   rU   )	r@   rG   r<   rM   rQ   rU   rW   rI   rK   r/  r0  )ru   rG   r'  r3  r*  r,  r-  rM   r<   rQ   rI   rK   s               r^   Ú_rw_max_cfgr5  g  sq   € õ0 ÝØ#ØØØ(Ø&ØØ&ð
ñ 
ô 
õ .ØØØØ#Ø#Ø-Ø3Ø%Ø#ð

ñ 

ô 

ðñ ô ð r`   rX   r÷   Úconv_norm_layer_clrO   c                 ó®   — t          |¦  «        }t          t          d| |d|d         ||¬¦  «        t          d||||d         |||	|
¬¦	  «	        ¬¦  «        S )z=Configuration for experimental ConvNeXt-based MaxxViT models.r�   Fr   )rm   ru   rG   rn   rQ   rU   rW   r.   )	r@   rG   rM   rO   rQ   rU   rW   rI   rK   r/  )r%   rœ  r7   r8   )ru   rG   r*  r6  r,  r-  rM   rO   rQ   rI   rK   s              r^   Ú	_next_cfgr8  —  s†   € õ ˜KÑ(Ô(€KÝÝØ!Ø#ØØØ# AœØ&Ø,ð
ñ 
ô 
õ .ØØØ#Ø'Ø# AœØ-Ø3Ø%Ø#ð

ñ 

ô 

ðñ ô ð r`   c            	      óf   — t          t          ddd¬¦  «        t          dddd¬¦  «        ¬	¦  «        S )
z0Configuration matching TensorFlow MaxViT models.gü©ñÒMbP?Ú	gelu_tanhÚsame)rY   rS   rx   rƒ   Frh  )rY   rS   r=   rI   r/  r0  rj   r`   r^   Ú_tf_cfgr<  ¾  sT   € åÝØØ!Øð
ñ 
ô 
õ
 .ØØ!ØØ"ð	
ñ 
ô 
ðñ ô ð r`   Úcoatnet_pico_rw)r’   é€   é   rJ   rŒ   )r;   r’   )r‹   r�   r“   r~   )r'  r3  Úcoatnet_nano_rw)ro   rU  é   ro   )ru   r'  r3  Úcoatnet_0_rwr†   )r�   ro   rD  r�   )r(  r+  Úcoatnet_1_rw)r�   rA  rª  r�   )ru   r(  r+  Úcoatnet_2_rw)r>  r?  rJ   é   )r’   r>  r|   )ru   r)  Úcoatnet_3_rw)rˆ   r‰   rŠ   é   )r‡   rˆ   )ru   r)  rQ   Úcoatnet_bn_0_rwr‚   )ru   r(  r+  r,  Úcoatnet_rmlp_nano_rwr‰   )r'  r3  rI   rK   Úcoatnet_rmlp_0_rw)ru   rI   Úcoatnet_rmlp_1_rwrã   )rG   r(  r+  rI   rK   Úcoatnet_rmlp_1_rw2)ru   rI   rK   Úcoatnet_rmlp_2_rw)ru   r)  rQ   rI   Úcoatnet_rmlp_3_rwÚcoatnet_nano_cc)r�   r�   ©r�   r‘   rP  )r‹   r�   r“   rm   Úcoatnext_nano_rwrñ   )r‹   r�   r“   rš   )rƒ   N)rI   rQ   Ú	coatnet_0r’   rŠ   )r‹   r�   r“   r˜   Ú	coatnet_1Ú	coatnet_2r>  rE  Ú	coatnet_3rˆ   rG  Ú	coatnet_4)r�   é   é   r�   Ú	coatnet_5)r?  rJ   é   é   r[  Úmaxvit_pico_rw)r;   r’   r>  r?  )r�   r�   rŽ   r�   )r¯  r¯  r¯  r¯  )é   r;   )r‹   r�   rm   r“   Úmaxvit_nano_rw)r.   r�   ro   r.   Úmaxvit_tiny_rwÚmaxvit_tiny_pm)r°  r°  r°  r°  Úmaxvit_rmlp_pico_rw)rI   Úmaxvit_rmlp_nano_rwÚmaxvit_rmlp_tiny_rwÚmaxvit_rmlp_small_rwÚmaxvit_rmlp_base_rw)r‹   r�   rm   r“   r˜   Úmaxxvit_rmlp_nano_rw)r‹   r�   rm   r“   rš   Úmaxxvit_rmlp_tiny_rwÚmaxxvit_rmlp_small_rw)é0   r‡   Úmaxxvitv2_nano_rw)rO   rI   Úmaxxvitv2_rmlp_base_rw)r�   rA  rW  r�   )rO   Úmaxxvitv2_rmlp_large_rw)é    i@  i€  rZ  )r�   rA  é   r�   )éP   rm  rZ  Úmaxvit_tiny_tf)r‹   r�   rm   r“   r”   r˜   Úmaxvit_small_tfÚmaxvit_base_tfÚmaxvit_large_tfÚmaxvit_xlarge_tfÚ
state_dictÚmodelc                 óf  — |                      ¦   «         }i }|                      ¦   «         D �]\  }}|                     d¦  «        ro|                     |dd…         ¦  «        }|j        |j        j        k    s|j        d         |j        d         k    r!t          ||j        |j        j        ¬¦  «        }||v rq|j        ||         j        k    r[| 	                    ¦   «         ||          	                    ¦   «         k    r+|j        dv sJ ‚| 
                    ||         j        ¦  «        }|||<   �Œ|S )z/Filter checkpoint state dict for compatibility.r÷  Niãÿÿÿr   r.   )Únew_window_sizeÚnew_bias_shape)r�   rU  )ru  rÉ  ÚendswithÚget_submoduler¾   r÷  rM   r-   ÚndimÚnumelrÁ   )ru  rv  Úmodel_state_dictÚout_dictrÐ   rÑ   Úms          r^   Úcheckpoint_filter_fnr�  @  sA  € à×'Ò'Ñ)Ô)ÐØ€HØ× Ò Ñ"Ô"ð ñ ‰ˆˆ1Ø�:Š:Ð4Ñ5Ô5ð 	Ø×#Ò# A d s d¤GÑ,Ô,ˆAØŒw˜!Ô8Ô>Ò>Ð>À!Ä-ÐPQÔBRÐVWÔVcÐdeÔVfÒBfÐBfÝ-ØØ$%¤MØ#$Ô#AÔ#Gðñ ô �ð Ð Ð Ð  Q¤VÐ/?ÀÔ/BÔ/GÒ%GÐ%GÈAÏGÊGÉIÌIÐYiÐjkÔYl×YrÒYrÑYtÔYtÒLtÐLtà”6˜VÐ#Ð#Ð#Ð#Ø—	’	Ð*¨1Ô-Ô3Ñ4Ô4ˆAØˆ�‰‰Ø€Or`   ÚvariantÚcfg_variantÚ
pretrainedc                 óê   — |€<| t           v r| }n0d                     |                      d¦  «        dd…         ¦  «        }t          t          | |ft           |         t          d¬¦  «        t          dœ|¤ŽS )zCreate a MaxxVit model variant.Nrù  r¸   T)Úflatten_sequential)Ú	model_cfgÚfeature_cfgÚpretrained_filter_fn)Ú
model_cfgsÚjoinÚsplitr/   r9   rœ  r�  )r‚  rƒ  r„  rÅ  s       r^   Ú_create_maxxvitr�  V  s†   € àÐØ•jÐ Ð Ø!ˆKˆKàŸ(š( 7§=¢=°Ñ#5Ô#5°c°r°cÔ#:Ñ;Ô;ˆKÝÝ�˜*ðå˜[Ô)Ý¨DÐ1Ñ1Ô1Ý1ð	ð ð
 ðð ð r`   Úurlc                 ó$   — | dddddddddd	d
dœ|¥S )z$Create a default configuration dict.rÑ  )ro   rÐ  rÐ  )rD  rD  çffffffî?Úbicubic)ç      à?r’  r’  z
stem.conv1zhead.fcTz
apache-2.0)rŽ  rÓ  Ú
input_sizeÚ	pool_sizeÚcrop_pctÚinterpolationÚmeanrô   Ú
first_convÚ
classifierÚfixed_input_sizeÚlicenserj   )rŽ  rÅ  s     r^   Ú_cfgrœ  e  s<   € ð  4°}ÐSYØ¨9Ø¨Ø"°)Ø Øðð ð $*ðð r`   zcoatnet_pico_rw_224.untrained)rŽ  zcoatnet_nano_rw_224.sw_in1kztimm/zyhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_nano_rw_224_sw-f53093b4.pthgÍÌÌÌÌÌì?)Ú	hf_hub_idrŽ  r•  zcoatnet_0_rw_224.sw_in1kzvhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_0_rw_224_sw-a6439706.pth)r�  rŽ  zcoatnet_1_rw_224.sw_in1kzvhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_1_rw_224_sw-5cae1ea8.pthz!coatnet_2_rw_224.sw_in12k_ft_in1k)r�  z'coatnet_rmlp_1_rw2_224.sw_in12k_ft_in1kz&coatnet_rmlp_2_rw_224.sw_in12k_ft_in1kz&coatnet_rmlp_2_rw_384.sw_in12k_ft_in1k)ro   r‰   r‰   )rW  rW  g      ð?Úsquash)r�  r“  r”  r•  Ú	crop_modezcoatnet_bn_0_rw_224.sw_in1kzyhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_bn_0_rw_224_sw-c228e218.pthr�  )r�  rŽ  r—  rô   r•  z coatnet_rmlp_nano_rw_224.sw_in1kz~https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_rmlp_nano_rw_224_sw-bd1d51b3.pthzcoatnet_rmlp_0_rw_224.untrainedzcoatnet_rmlp_1_rw_224.sw_in1kz{https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_rmlp_1_rw_224_sw-9051e6c3.pthzcoatnet_rmlp_2_rw_224.sw_in1kz{https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_rmlp_2_rw_224_sw-5ccfac55.pthzcoatnet_rmlp_3_rw_224.untrainedzcoatnet_nano_cc_224.untrainedzcoatnext_nano_rw_224.sw_in1kzzhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnext_nano_rw_224_ad-22cb71c2.pthzcoatnet_2_rw_224.sw_in12ki-.  )r�  rÓ  zcoatnet_3_rw_224.sw_in12kzcoatnet_rmlp_1_rw2_224.sw_in12kzcoatnet_rmlp_2_rw_224.sw_in12kzcoatnet_0_224.untrainedzcoatnet_1_224.untrainedzcoatnet_2_224.untrainedzcoatnet_3_224.untrainedzcoatnet_4_224.untrainedzcoatnet_5_224.untrainedzmaxvit_pico_rw_256.untrained)ro   r?  r?  )é   r   )rŽ  r“  r”  zmaxvit_nano_rw_256.sw_in1kzxhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_nano_rw_256_sw-fb127241.pth)r�  rŽ  r“  r”  zmaxvit_tiny_rw_224.sw_in1kzxhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_tiny_rw_224_sw-7d0dffeb.pthzmaxvit_tiny_rw_256.untrainedzmaxvit_tiny_pm_256.untrainedzmaxvit_rmlp_pico_rw_256.sw_in1kz}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_rmlp_pico_rw_256_sw-8d82f2c6.pthzmaxvit_rmlp_nano_rw_256.sw_in1kz}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_rmlp_nano_rw_256_sw-c17bb0d6.pthzmaxvit_rmlp_tiny_rw_256.sw_in1kz}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_rmlp_tiny_rw_256_sw-bbef0ff5.pthz maxvit_rmlp_small_rw_224.sw_in1kz~https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_rmlp_small_rw_224_sw-6ef0ae4f.pthz"maxvit_rmlp_small_rw_256.untrainedz(maxvit_rmlp_base_rw_224.sw_in12k_ft_in1kz(maxvit_rmlp_base_rw_384.sw_in12k_ft_in1kz maxvit_rmlp_base_rw_224.sw_in12kz maxxvit_rmlp_nano_rw_256.sw_in1kz~https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxxvit_rmlp_nano_rw_256_sw-0325d459.pthz"maxxvit_rmlp_tiny_rw_256.untrainedz!maxxvit_rmlp_small_rw_256.sw_in1kzhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxxvit_rmlp_small_rw_256_sw-37e217ff.pthzmaxxvitv2_nano_rw_256.sw_in1k)r�  r“  r”  z+maxxvitv2_rmlp_base_rw_224.sw_in12k_ft_in1kz+maxxvitv2_rmlp_base_rw_384.sw_in12k_ft_in1kz%maxxvitv2_rmlp_large_rw_224.untrainedz#maxxvitv2_rmlp_base_rw_224.sw_in12kzmaxvit_tiny_tf_224.in1k)r�  r—  rô   zmaxvit_tiny_tf_384.in1kzmaxvit_tiny_tf_512.in1k)ro   rJ   rJ   )rn  rn  zmaxvit_small_tf_224.in1kzmaxvit_small_tf_384.in1kzmaxvit_small_tf_512.in1kzmaxvit_base_tf_224.in1kzmaxvit_base_tf_384.in1kzmaxvit_base_tf_512.in1kzmaxvit_large_tf_224.in1kzmaxvit_large_tf_384.in1kzmaxvit_large_tf_512.in1kzmaxvit_base_tf_224.in21kiSU  z maxvit_base_tf_384.in21k_ft_in1kz maxvit_base_tf_512.in21k_ft_in1kzmaxvit_large_tf_224.in21kz!maxvit_large_tf_384.in21k_ft_in1kz!maxvit_large_tf_512.in21k_ft_in1k)r�  r“  r•  rŸ  zmaxvit_xlarge_tf_224.in21kz"maxvit_xlarge_tf_384.in21k_ft_in1kz"maxvit_xlarge_tf_512.in21k_ft_in1kc                 ó   — t          dd| i|¤ŽS )z)CoatNet Pico model with RW configuration.Úcoatnet_pico_rw_224r„  )r¢  ©r�  ©r„  rÅ  s     r^   r¢  r¢  L	  ó   € õ ÐRÐR¸ZÐRÈ6ÐRÐRÐRr`   c                 ó   — t          dd| i|¤ŽS )z)CoatNet Nano model with RW configuration.Úcoatnet_nano_rw_224r„  )r§  r£  r¤  s     r^   r§  r§  R	  r¥  r`   c                 ó   — t          dd| i|¤ŽS )z&CoatNet-0 model with RW configuration.Úcoatnet_0_rw_224r„  )r©  r£  r¤  s     r^   r©  r©  X	  ó   € õ ÐOÐO¸*ÐOÈÐOÐOÐOr`   c                 ó   — t          dd| i|¤ŽS )z&CoatNet-1 model with RW configuration.Úcoatnet_1_rw_224r„  )r¬  r£  r¤  s     r^   r¬  r¬  ^	  rª  r`   c                 ó   — t          dd| i|¤ŽS )z&CoatNet-2 model with RW configuration.Úcoatnet_2_rw_224r„  )r®  r£  r¤  s     r^   r®  r®  d	  rª  r`   c                 ó   — t          dd| i|¤ŽS )z&CoatNet-3 model with RW configuration.Úcoatnet_3_rw_224r„  )r°  r£  r¤  s     r^   r°  r°  j	  rª  r`   c                 ó   — t          dd| i|¤ŽS )z4CoatNet-0 model with BatchNorm and RW configuration.Úcoatnet_bn_0_rw_224r„  )r²  r£  r¤  s     r^   r²  r²  p	  r¥  r`   c                 ó   — t          dd| i|¤ŽS )z.CoatNet Nano model with Relative Position MLP.Úcoatnet_rmlp_nano_rw_224r„  )r´  r£  r¤  s     r^   r´  r´  v	  ó   € õ ÐWÐWÀ*ÐWÐPVÐWÐWÐWr`   c                 ó   — t          dd| i|¤ŽS )z+CoatNet-0 model with Relative Position MLP.Úcoatnet_rmlp_0_rw_224r„  )r·  r£  r¤  s     r^   r·  r·  |	  ó   € õ ÐTÐT¸zÐTÈVÐTÐTÐTr`   c                 ó   — t          dd| i|¤ŽS )z+CoatNet-1 model with Relative Position MLP.Úcoatnet_rmlp_1_rw_224r„  )rº  r£  r¤  s     r^   rº  rº  ‚	  r¸  r`   c                 ó   — t          dd| i|¤ŽS )z.CoatNet-1 model with Relative Position MLP v2.Úcoatnet_rmlp_1_rw2_224r„  )r¼  r£  r¤  s     r^   r¼  r¼  ˆ	  s   € õ ÐUÐUÀ
ÐUÈfÐUÐUÐUr`   c                 ó   — t          dd| i|¤ŽS )z+CoatNet-2 model with Relative Position MLP.Úcoatnet_rmlp_2_rw_224r„  )r¾  r£  r¤  s     r^   r¾  r¾  Ž	  r¸  r`   c                 ó   — t          dd| i|¤ŽS )z6CoatNet-2 model with Relative Position MLP at 384x384.Úcoatnet_rmlp_2_rw_384r„  )rÀ  r£  r¤  s     r^   rÀ  rÀ  ”	  r¸  r`   c                 ó   — t          dd| i|¤ŽS )z+CoatNet-3 model with Relative Position MLP.Úcoatnet_rmlp_3_rw_224r„  )rÂ  r£  r¤  s     r^   rÂ  rÂ  š	  r¸  r`   c                 ó   — t          dd| i|¤ŽS )z(CoatNet Nano model with ConvNeXt blocks.Úcoatnet_nano_cc_224r„  )rÄ  r£  r¤  s     r^   rÄ  rÄ   	  r¥  r`   c                 ó   — t          dd| i|¤ŽS )z*CoAtNeXt Nano model with RW configuration.Úcoatnext_nano_rw_224r„  )rÆ  r£  r¤  s     r^   rÆ  rÆ  ¦	  s   € õ ÐSÐS¸jÐSÈFÐSÐSÐSr`   c                 ó   — t          dd| i|¤ŽS )zCoatNet-0 model.Úcoatnet_0_224r„  )rÈ  r£  r¤  s     r^   rÈ  rÈ  ¬	  ó   € õ ÐLÐL°zÐLÀVÐLÐLÐLr`   c                 ó   — t          dd| i|¤ŽS )zCoatNet-1 model.Úcoatnet_1_224r„  )rË  r£  r¤  s     r^   rË  rË  ²	  rÉ  r`   c                 ó   — t          dd| i|¤ŽS )zCoatNet-2 model.Úcoatnet_2_224r„  )rÍ  r£  r¤  s     r^   rÍ  rÍ  ¸	  rÉ  r`   c                 ó   — t          dd| i|¤ŽS )zCoatNet-3 model.Úcoatnet_3_224r„  )rÏ  r£  r¤  s     r^   rÏ  rÏ  ¾	  rÉ  r`   c                 ó   — t          dd| i|¤ŽS )zCoatNet-4 model.Úcoatnet_4_224r„  )rÑ  r£  r¤  s     r^   rÑ  rÑ  Ä	  rÉ  r`   c                 ó   — t          dd| i|¤ŽS )zCoatNet-5 model.Úcoatnet_5_224r„  )rÓ  r£  r¤  s     r^   rÓ  rÓ  Ê	  rÉ  r`   c                 ó   — t          dd| i|¤ŽS )z(MaxViT Pico model with RW configuration.Úmaxvit_pico_rw_256r„  )rÕ  r£  r¤  s     r^   rÕ  rÕ  Ð	  ó   € õ ÐQÐQ¸JÐQÈ&ÐQÐQÐQr`   c                 ó   — t          dd| i|¤ŽS )z(MaxViT Nano model with RW configuration.Úmaxvit_nano_rw_256r„  )rØ  r£  r¤  s     r^   rØ  rØ  Ö	  rÖ  r`   c                 ó   — t          dd| i|¤ŽS )z(MaxViT Tiny model with RW configuration.Úmaxvit_tiny_rw_224r„  )rÚ  r£  r¤  s     r^   rÚ  rÚ  Ü	  rÖ  r`   c                 ó   — t          dd| i|¤ŽS )z3MaxViT Tiny model with RW configuration at 256x256.Úmaxvit_tiny_rw_256r„  )rÜ  r£  r¤  s     r^   rÜ  rÜ  â	  rÖ  r`   c                 ó   — t          dd| i|¤ŽS )z3MaxViT Relative Position MLP Pico RW 256x256 model.Úmaxvit_rmlp_pico_rw_256r„  )rÞ  r£  r¤  s     r^   rÞ  rÞ  è	  ó   € õ ÐVÐVÀÐVÈvÐVÐVÐVr`   c                 ó   — t          dd| i|¤ŽS )z3MaxViT Relative Position MLP Nano RW 256x256 model.Úmaxvit_rmlp_nano_rw_256r„  )rá  r£  r¤  s     r^   rá  rá  î	  rß  r`   c                 ó   — t          dd| i|¤ŽS )z3MaxViT Relative Position MLP Tiny RW 256x256 model.Úmaxvit_rmlp_tiny_rw_256r„  )rã  r£  r¤  s     r^   rã  rã  ô	  rß  r`   c                 ó   — t          dd| i|¤ŽS )z4MaxViT Relative Position MLP Small RW 224x224 model.Úmaxvit_rmlp_small_rw_224r„  )rå  r£  r¤  s     r^   rå  rå  ú	  rµ  r`   c                 ó   — t          dd| i|¤ŽS )z9MaxViT Small model with Relative Position MLP at 256x256.Úmaxvit_rmlp_small_rw_256r„  )rç  r£  r¤  s     r^   rç  rç   
  rµ  r`   c                 ó   — t          dd| i|¤ŽS )z-MaxViT Base model with Relative Position MLP.Úmaxvit_rmlp_base_rw_224r„  )ré  r£  r¤  s     r^   ré  ré  
  rß  r`   c                 ó   — t          dd| i|¤ŽS )z8MaxViT Base model with Relative Position MLP at 384x384.Úmaxvit_rmlp_base_rw_384r„  )rë  r£  r¤  s     r^   rë  rë  
  rß  r`   c                 ó   — t          dd| i|¤ŽS )z'MaxViT Tiny model with parallel blocks.Úmaxvit_tiny_pm_256r„  )rí  r£  r¤  s     r^   rí  rí  
  rÖ  r`   c                 ó   — t          dd| i|¤ŽS )z4MaxxViT Relative Position MLP Nano RW 256x256 model.Úmaxxvit_rmlp_nano_rw_256r„  )rï  r£  r¤  s     r^   rï  rï  
  rµ  r`   c                 ó   — t          dd| i|¤ŽS )z.MaxxViT Tiny model with Relative Position MLP.Úmaxxvit_rmlp_tiny_rw_256r„  )rñ  r£  r¤  s     r^   rñ  rñ  
  rµ  r`   c                 ó   — t          dd| i|¤ŽS )z/MaxxViT Small model with Relative Position MLP.Úmaxxvit_rmlp_small_rw_256r„  )ró  r£  r¤  s     r^   ró  ró  $
  s   € õ ÐXÐXÀ:ÐXÐQWÐXÐXÐXr`   c                 ó   — t          dd| i|¤ŽS )zMaxxViT-V2 Nano model.Úmaxxvitv2_nano_rw_256r„  )rõ  r£  r¤  s     r^   rõ  rõ  *
  r¸  r`   c                 ó   — t          dd| i|¤ŽS )z1MaxxViT-V2 Base model with Relative Position MLP.Úmaxxvitv2_rmlp_base_rw_224r„  )r÷  r£  r¤  s     r^   r÷  r÷  0
  ó   € õ ÐYÐYÀJÐYÐRXÐYÐYÐYr`   c                 ó   — t          dd| i|¤ŽS )z<MaxxViT-V2 Base model with Relative Position MLP at 384x384.Úmaxxvitv2_rmlp_base_rw_384r„  )rú  r£  r¤  s     r^   rú  rú  6
  rø  r`   c                 ó   — t          dd| i|¤ŽS )z2MaxxViT-V2 Large model with Relative Position MLP.Úmaxxvitv2_rmlp_large_rw_224r„  )rü  r£  r¤  s     r^   rü  rü  <
  s   € õ ÐZÐZÀZÐZÐSYÐZÐZÐZr`   c                 ó   — t          dd| i|¤ŽS )z"MaxViT Tiny model from TensorFlow.Úmaxvit_tiny_tf_224rp  r„  )rþ  rp  r£  r¤  s     r^   rþ  rþ  B
  ó   € õ ÐcÐcÈjÐcÐ\bÐcÐcÐcr`   c                 ó   — t          dd| i|¤ŽS )z-MaxViT Tiny model from TensorFlow at 384x384.Úmaxvit_tiny_tf_384rp  r„  )r  rp  r£  r¤  s     r^   r  r  H
  rÿ  r`   c                 ó   — t          dd| i|¤ŽS )z-MaxViT Tiny model from TensorFlow at 512x512.Úmaxvit_tiny_tf_512rp  r„  )r  rp  r£  r¤  s     r^   r  r  N
  rÿ  r`   c                 ó   — t          dd| i|¤ŽS )z#MaxViT Small model from TensorFlow.Úmaxvit_small_tf_224rq  r„  )r  rq  r£  r¤  s     r^   r  r  T
  ó   € õ ÐeÐeÐPZÐeÐ^dÐeÐeÐer`   c                 ó   — t          dd| i|¤ŽS )z.MaxViT Small model from TensorFlow at 384x384.Úmaxvit_small_tf_384rq  r„  )r  rq  r£  r¤  s     r^   r  r  Z
  r  r`   c                 ó   — t          dd| i|¤ŽS )z.MaxViT Small model from TensorFlow at 512x512.Úmaxvit_small_tf_512rq  r„  )r
  rq  r£  r¤  s     r^   r
  r
  `
  r  r`   c                 ó   — t          dd| i|¤ŽS )z"MaxViT Base model from TensorFlow.Úmaxvit_base_tf_224rr  r„  )r  rr  r£  r¤  s     r^   r  r  f
  rÿ  r`   c                 ó   — t          dd| i|¤ŽS )z-MaxViT Base model from TensorFlow at 384x384.Úmaxvit_base_tf_384rr  r„  )r  rr  r£  r¤  s     r^   r  r  l
  rÿ  r`   c                 ó   — t          dd| i|¤ŽS )z-MaxViT Base model from TensorFlow at 512x512.Úmaxvit_base_tf_512rr  r„  )r  rr  r£  r¤  s     r^   r  r  r
  rÿ  r`   c                 ó   — t          dd| i|¤ŽS )z#MaxViT Large model from TensorFlow.Úmaxvit_large_tf_224rs  r„  )r  rs  r£  r¤  s     r^   r  r  x
  r  r`   c                 ó   — t          dd| i|¤ŽS )z.MaxViT Large model from TensorFlow at 384x384.Úmaxvit_large_tf_384rs  r„  )r  rs  r£  r¤  s     r^   r  r  ~
  r  r`   c                 ó   — t          dd| i|¤ŽS )z.MaxViT Large model from TensorFlow at 512x512.Úmaxvit_large_tf_512rs  r„  )r  rs  r£  r¤  s     r^   r  r  „
  r  r`   c                 ó   — t          dd| i|¤ŽS )z$MaxViT XLarge model from TensorFlow.Úmaxvit_xlarge_tf_224rt  r„  )r  rt  r£  r¤  s     r^   r  r  Š
  ó   € õ ÐgÐgÐR\ÐgÐ`fÐgÐgÐgr`   c                 ó   — t          dd| i|¤ŽS )z/MaxViT XLarge model from TensorFlow at 384x384.Úmaxvit_xlarge_tf_384rt  r„  )r  rt  r£  r¤  s     r^   r  r  �
  r  r`   c                 ó   — t          dd| i|¤ŽS )z/MaxViT XLarge model from TensorFlow at 512x512.Úmaxvit_xlarge_tf_512rt  r„  )r  rt  r£  r¤  s     r^   r  r  –
  r  r`   r  )rè   rF   FFr&  rw   TrT   rV   NrH   rJ   )rt   rF   Fr2  rw   rT   rV   Nr;   NrH   rJ   )rt   rF   rT   rV   rT   rV   NFrX   r÷   rJ   rj   )NFr#  )©rd   r  Úcollectionsr   Údataclassesr   r   r   Ú	functoolsr   Útypingr   r	   r
   r   r   r   r   r   rÄ   r   Ú	torch.jitr   Ú	timm.datar   r   Útimm.layersr   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   Ú_builderr/   Ú	_featuresr0   Ú_features_fxr1   Ú_manipulater2   r3   Ú	_registryr4   r5   Ú__all__r8   r7   r6   r$  rœ   rÚ   rá   ri   rÿ   r  r"  re   r%  r'  rC  rÖ   rX  r]  rb  re  ri  rk  r{  rˆ  rŠ  rŒ  rŽ  r�  r•  r¢  r©  r»  rÄ  rÎ  r9   rg   rh   r1  r5  r8  r<  rœ  rŠ  r�  r�  rœ  Údefault_cfgsr¢  r§  r©  r¬  r®  r°  r²  r´  r·  rº  r¼  r¾  rÀ  rÂ  rÄ  rÆ  rÈ  rË  rÍ  rÏ  rÑ  rÓ  rÕ  rØ  rÚ  rÜ  rÞ  rá  rã  rå  rç  ré  rë  rí  rï  rñ  ró  rõ  r÷  rú  rü  rþ  r  r  r  r  r
  r  r  r  r  r  r  r  r  r  rj   r`   r^   ú<module>r,     sz1  ðð"ð "ðH €€€Ø #Ð #Ð #Ð #Ð #Ð #Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø Ð Ð Ð Ð Ð Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ Ià €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð8 +Ð *Ð *Ð *Ð *Ð *Ø +Ð +Ð +Ð +Ð +Ð +Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø 4Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4Ø <Ð <Ð <Ð <Ð <Ð <Ð <Ð <à
NÐ
NÐ
N€ð ð2ð 2ð 2ð 2ð 2ñ 2ô 2ñ „ð2ðB ð!Pð !Pð !Pð !Pð !Pñ !Pô !Pñ „ð!PðH ð
!ð 
!ð 
!ð 
!ð 
!ñ 
!ô 
!ñ „ð
!ðQð Qð Qð Qð Q�"”)ñ Qô Qð QðhRð Rð Rð Rð R�"”)ñ Rô Rð Rðj/ð /ð /ð /ð /�2”9ñ /ô /ð /ñd0ð 0˜bœið 0¨sð 0¸Cð 0Èð 0ð 0ð 0ð 0ð2Nð Nð Nð Nð N˜œñ Nô Nð Nñb,ð ,�r”yð ,¨ð ,°Sð ,À$ð ,ð ,ð ,ð ,ð0&˜8 Cœ=ð &°Cð &¸Cð &ð &ð &ð &ðlð lð lð lð l�"”)ñ lô lð lð^dð dð dð dð d�B”Iñ dô dð dðN˜œð °4¸´9ð ÀÄð ð ð ð ð ð˜EœLð °t¸C´yð ÈDÐQTÌIð ÐZ_ÔZfð ð ð ñ Ôðð�e”lð ¨t°C¬yð ¸U¼\ð ð ð ð ð ð˜%œ,ð °4¸´9ð ÈÈSÌ	ð ÐV[ÔVbð ð ð ñ Ôðð	Ð.ð 	¸UÀ3ÈÀ8¼_ð 	ÐQYÐZbÔQcð 	ð 	ð 	ð 	ðDð Dð Dð Dð D˜2œ9ñ Dô Dð DðNTð Tð Tð Tð T ¤ñ Tô Tð Tðn˜Uœ\ð ¸¸S¼	ð ÀeÄlð ð ð ð ð ð ¤ð ¸DÀ¼Ið ÐQUÐVYÔQZð Ð_dÔ_kð ð ð ñ Ôðð˜5œ<ð °D¸´Ið À%Ä,ð ð ð ð ð ð˜uœ|ð ¸¸S¼	ð ÈTÐRUÌYð Ð[`Ô[gð ð ð ñ ÔððKð Kð Kð Kð K˜2œ9ñ Kô Kð Kð\5ð 5ð 5ð 5ð 5�2”9ñ 5ô 5ð 5ðp1ð 1ð 1ð 1ð 1˜2œ9ñ 1ô 1ð 1ðhWð Wð Wð Wð W�2”9ñ Wô Wð Wðt/ð /ð /ð /ð /ˆ2Œ9ñ /ô /ð /ðdÐ.ð ¸%ÀÀSÀ¼/ð ÐNcð ð ð ð ð˜ð ¨sð °zð ð ð ð ð*eð eð eð eð eˆbŒiñ eô eð eðR "ØØ!&Ø %Ø#)Ø!Ø*.Ø&3Ø)4Ø'+Ø"Øñ6ð 6Øð6àð6ð ð6ð ð	6ð
 !ð6ð ð6ð $(ð6ð !$ð6ð $'ð6ð ˜e”_ð6ð ð6ð ð6ð 
ˆ#ˆsˆ(„^ð6ð 6ð 6ð 6ðt  ØØ!&Ø!'Ø!Ø&3Ø)4Ø15ØØ'+Ø"Øñ-ð -Øð-àð-ð ð-ð ð	-ð
 ð-ð !$ð-ð $'ð-ð ˜e C¨ HœoÔ.ð-ð ð-ð ˜e”_ð-ð ð-ð ð-ð 
ˆ#ˆsˆ(„^ð-ð -ð -ð -ðb  ØØ,Ø"-Ø&3Ø)4Ø15Ø#Ø9=Ø!Øñ$ð $Øð$àð$ð ð$ð  ð	$ð
 !$ð$ð $'ð$ð ˜e C¨ HœoÔ.ð$ð ð$ð ˜5 %¨¨u¨Ô"5Ð5Ô6ð$ð ð$ð ð$ð 
ˆ#ˆsˆ(„^ð$ð $ð $ð $ðN��c˜3�h”ð ð ð ð ð" ˆTñ nð nð nà�Jñ Ø%ØØðð ð ˆ+Ø!Ø ð
ñ 
ô 
ð	ð ð ðnð �Jñ 	Ø%ØØð	ð 	ð ˆ+ØØ!Ø ð
ñ 
ô 
ð		ð 	ð 	ðnð* �ñ Ø%ØØðð ð ˆ,Ø Ø&+ð
ñ 
ô 
ð	ð ð ð+nð< �ñ 	Ø%ØØð	ð 	ð ˆ,ØØ Ø&+ð
ñ 
ô 
ð		ð 	ð 	ð=nðP �ñ 	Ø'ØØð	ð 	ð ˆ,ØØ &ð
ñ 
ô 
ð		ð 	ð 	ðQnðd �ñ 	Ø'ØØð	ð 	ð ˆ,ØØ &Øð
ñ 
ô 
ð		ð 	ð 	ðenð| �Jñ 
Ø%ØØð
ð 
ð ˆ,ØØ Ø&+Ø#0ð	
ñ 
ô 
ð	
ð 
ð 
ð}nðR $˜ñ 
Ø%ØØð
ð 
ð ˆ+Ø!Ø ØØð	
ñ 
ô 
ð	
ð 
ð 
ðSnðh !�jñ Ø%ØØðð ð ˆ,ØØð
ñ 
ô 
ð	ð ð ðinðz !�jñ Ø%ØØðð ð ˆ,ØØ Ø&+ØØð
ñ 
ô 
ð	ð ð ð{nðR "�zñ 	Ø%ØØð	ð 	ð ˆ,ØØØð
ñ 
ô 
ð		ð 	ð 	ðSnðf !�jñ 
Ø'ØØð
ð 
ð ˆ,ØØ &ØØð	
ñ 
ô 
ð	
ð 
ð 
ðgnð| !�jñ 
Ø'ØØð
ð 
ð ˆ,ØØ &ØØð	
ñ 
ô 
ð	
ð 
ð 
ð}nðT �Jñ Ø%ØØØ5ð	ð ð
 ˆ,‰.Œ.ðð ð ðUnðb  �Zñ 	Ø%ØØØð		ð 	ð
 ˆ)ØØ$ð
ñ 
ô 
ð	ð 	ð 	ðcnðz ˆjØ%ØØØð	ñ ô ð ð{nðF ˆjØ%ØØØð	ñ ô ð ðGnðR ˆjØ'ØØØð	ñ ô ð ðSnð^ ˆjØ'ØØØð	ñ ô ð ð_nðj ˆjØ'ØØØð	ñ ô ð ðknðv ˆjØ(ØØØð	ñ ô ð ðwnðF �:ñ Ø$ØØØð	ð ð
 ˆ+‰-Œ-ðð ð ðGnðT �:ñ Ø%ØØØð	ð ð
 ˆ+‰-Œ-ðð ð ðUnðb �:ñ Ø%ØØØð	ð ð
 ˆ+‰-Œ-ðð ð ðcnðp �:ñ Ø%ØØØð	ð ð
 ˆ+‰-Œ-ðð ð ðqnð@ #˜
ñ Ø$ØØØð	ð ð
 ˆ+ 5Ð
)Ñ
)Ô
)ðð ð ðAnðN #˜
ñ Ø%ØØØð	ð ð
 ˆ+ 5Ð
)Ñ
)Ô
)ðð ð ðOnð\ #˜
ñ Ø%ØØØð	ð ð
 ˆ+ 5Ð
)Ñ
)Ô
)ðð ð ð]nðj $˜ñ 	Ø%ØØØð		ð 	ð
 ˆ+ØØð
ñ 
ô 
ð	ð 	ð 	ðknð~ #˜
ñ 	Ø%ØØØØð	ð 	ð ˆ+Øð
ñ 
ô 
ð	ð 	ð 	ðnðT $˜ñ Ø%ØØØØðð ð ˆ)‰+Œ+ðð ð ðUnðd $˜ñ Ø%ØØØð	ð ð
 ˆ)‰+Œ+ðð ð ðenðr %˜*ñ Ø%ØØØð	ð ð
 ˆ)‰+Œ+ðð ð ðsnðB	 !�jñ 
Ø%ØØØØð
ð 
ð ˆ)ØØð
ñ 
ô 
ð
ð 
ð 
ðC	nðX	 &˜:ñ Ø'ØØØð	ð ð
 ˆ)Øð
ñ 
ô 
ðð ð ðY	nðj	 '˜Jñ 	Ø'ØØØØð	ð 	ð ˆ)Øð
ñ 
ô 
ð	ð 	ð 	ðk	nðB
 �:ñ Ø%ØØØØØðð ð ˆ'‰)Œ)ðð ð ðC
nðT
 �Jñ Ø%ØØØØØðð ð ˆ'‰)Œ)ðð ð ðU
nðf
 �:ñ Ø%ØØØØØðð ð ˆ'‰)Œ)ðð ð ðg
nðx
 �Jñ Ø'ØØØØØðð ð ˆ'‰)Œ)ðð ð ðy
nðJ  �Zñ Ø'ØØØØØðð ð ˆ'‰)Œ)ðð ð ðKn€
ðb T¨#¨u¬|Ð*;Ô%<ð ÀRÄYð ÐSWÐX[Ð]bÔ]iÐXiÔSjð ð ð ð ñ,ð ˜Sð ¨x¸¬}ð ÐQUð Ðilð Ðqxð ð ð ð ñ	ð 	ˆcð 	 #ð 	¨$¨s°C¨x¬.ð 	ð 	ð 	ð 	ð %Ð$ð X&à# T T¨b \¡\¤\ðX&ð " 4 4Øð HØð$ñ $ô $ðX&ð   Øð Eð!Fñ !Fô !FðX&ð   Øð Eð!ñ !ô !ðX&ð  (¨¨Øð*ñ *ô *ð!X&ð* .¨t¨tØð0ñ 0ô 0ð+X&ð. -¨d¨dØð/ñ /ô /ð/X&ð2 -¨d¨dØØ ¨H¸sÈhð/Xñ /Xô /Xð3X&ð< " 4 4Øð HØ"Ð(<Øð	$ñ $ô $ð=X&ðF '¨¨Øð MØð)ñ )ô )ðGX&ñN & t t° |¡|¤|ðOX&ñP $ T TØñ Jð&Kñ &Kô &KðQX&ñV $ T TØñ Jð&Kñ &Kô &KðWX&ñ\ & t t° |¡|¤|ð]X&ñ^ $ T T¨b \¡\¤\ð_X&ñ` # D DØñ IØð%ñ %ô %ðaX&ñl    ØÙñ"ñ "ô "ðmX&ð X&ñr    ØÙñ"ñ "ô "ðsX&ñx & t tØÙñ(ñ (ô (ðyX&ñ~ % d dØÙñ'ñ 'ô 'ðX&ñH ˜t˜t¨˜|™|œ|ðIX&ñJ ˜t˜t¨˜|™|œ|ðKX&ñL ˜t˜t¨˜|™|œ|ðMX&ñN ˜t˜t¨˜|™|œ|ðOX&ñP ˜t˜t¨˜|™|œ|ðQX&ñR ˜t˜t¨˜|™|œ|ðSX&ñX # D D¨R¹MÑU[Ñ$\Ñ$\Ô$\ðYX&ñZ ! $ $Øñ GÙ ©Fñ#4ñ #4ô #4ð[X&ñb ! $ $Øñ Gð#Hñ #Hô #HðcX&ñh # D DØÙ ©Fñ%4ñ %4ô %4ðiX&ñn # D D¨R¹MÑU[Ñ$\Ñ$\Ô$\ðoX&ñt & t tØñ LÙ ©Fñ(4ñ (4ô (4ðuX&ñ| & t tØñ LÙ ©Fñ(4ñ (4ô (4ð}X&ñD & t tØñ LÙ ©Fñ(4ñ (4ô (4ðEX&ð X&ð X&ñL '¨¨Øñ MØð)ñ )ô )ðMX&ñV )¨$¨$ØÙ ©Fñ+4ñ +4ô +4ðWX&ñ` /°°Øð1ñ 1ô 1ðaX&ñf /°°ØØ ¨H¸sÈhð1Xñ 1Xô 1XðgX&ñp '¨¨ØÙñ)ñ )ô )ðqX&ñ| '¨¨Øñ MÙ ©Fñ)4ñ )4ô )4ð}X&ñD )¨$¨$°2Á-Ñ[aÑ*bÑ*bÔ*bðEX&ñF (¨¨Øñ NÙ ©Fñ*4ñ *4ô *4ðGX&ñR $ T TØÙ ©Fñ&4ñ &4ô &4ðSX&ñX 2°4°4Øð4ñ 4ô 4ðYX&ñ\ 2°4°4ØØ ¨H¸sÈhð4Xñ 4Xô 4Xð]X&ñb ,¨T¨T°b¨\©\¬\ðcX&ñf *¨4¨4ØÙñ,ñ ,ô ,ðgX&ñp ˜t˜tØØ"Ð(<ñ >ñ  >ô  >ðqX&ñv ˜t˜tØØ ¨H¸sÈhð Xñ  Xô  XðwX&ñ| ˜t˜tØÙ ©H¸sÈhð Xñ  Xô  Xð}X&ñB   ØØ"Ð(<ñ!>ñ !>ô !>ðCX&ð X&ð X&ñH   ØØ ¨H¸sÈhð!Xñ !Xô !XðIX&ñN   ØÙ ©H¸sÈhð!Xñ !Xô !XðOX&ñT ˜t˜tØØ"Ð(<ñ >ñ  >ô  >ðUX&ñZ ˜t˜tØØ ¨H¸sÈhð Xñ  Xô  Xð[X&ñ` ˜t˜tØÙ ©H¸sÈhð Xñ  Xô  XðaX&ñf   ØØ"Ð(<ñ!>ñ !>ô !>ðgX&ñl   ØØ ¨H¸sÈhð!Xñ !Xô !XðmX&ñr   ØÙ ©H¸sÈhð!Xñ !Xô !XðsX&ñz   ØÙñ!ñ !ô !ð{X&ñ@ '¨¨ØØ ¨H¸sÈhð)Xñ )Xô )XðAX&ñF '¨¨ØÙ ©H¸sÈhð)Xñ )Xô )XðGX&ñL    ØÙñ"ñ "ô "ðMX&ñR (¨¨ØØ ¨H¸sÈhð*Xñ *Xô *XðSX&ñX (¨¨ØÙ ¨3¸(ñ*Dñ *Dô *DðYX&ñ^ ! $ $ØÙñ#ñ #ô #ð_X&ñd )¨$¨$ØØ ¨H¸sÈhð+Xñ +Xô +XðeX&ñj )¨$¨$ØÙ ©H¸sÈhð+Xñ +Xô +XðkX&ð X&ñ Xô X€ðv ñSð S Dð S¸Cð SÀGð Sñ Sð Sñ „ðSð
 ñSð S Dð S¸Cð SÀGð Sñ Sð Sñ „ðSð
 ñPð P ð P¸ð PÀð Pñ Pð Pñ „ðPð
 ñPð P ð P¸ð PÀð Pñ Pð Pñ „ðPð
 ñPð P ð P¸ð PÀð Pñ Pð Pñ „ðPð
 ñPð P ð P¸ð PÀð Pñ Pð Pñ „ðPð
 ñSð S Dð S¸Cð SÀGð Sñ Sð Sñ „ðSð
 ñXð X¨ð XÀð XÈð Xñ Xð Xñ „ðXð
 ñUð U dð U¸cð UÀgð Uñ Uð Uñ „ðUð
 ñUð U dð U¸cð UÀgð Uñ Uð Uñ „ðUð
 ñVð V tð V¸sð VÀwð Vñ Vð Vñ „ðVð
 ñUð U dð U¸cð UÀgð Uñ Uð Uñ „ðUð
 ñUð U dð U¸cð UÀgð Uñ Uð Uñ „ðUð
 ñUð U dð U¸cð UÀgð Uñ Uð Uñ „ðUð
 ñSð S Dð S¸Cð SÀGð Sñ Sð Sñ „ðSð
 ñTð T Tð T¸Sð TÀWð Tñ Tð Tñ „ðTð
 ñMð M˜dð M°cð M¸gð Mñ Mð Mñ „ðMð
 ñMð M˜dð M°cð M¸gð Mñ Mð Mñ „ðMð
 ñMð M˜dð M°cð M¸gð Mñ Mð Mñ „ðMð
 ñMð M˜dð M°cð M¸gð Mñ Mð Mñ „ðMð
 ñMð M˜dð M°cð M¸gð Mñ Mð Mñ „ðMð
 ñMð M˜dð M°cð M¸gð Mñ Mð Mñ „ðMð
 ñRð R 4ð R¸3ð RÀ7ð Rñ Rð Rñ „ðRð
 ñRð R 4ð R¸3ð RÀ7ð Rñ Rð Rñ „ðRð
 ñRð R 4ð R¸3ð RÀ7ð Rñ Rð Rñ „ðRð
 ñRð R 4ð R¸3ð RÀ7ð Rñ Rð Rñ „ðRð
 ñWð W¨ð WÀð WÈð Wñ Wð Wñ „ðWð
 ñWð W¨ð WÀð WÈð Wñ Wð Wñ „ðWð
 ñWð W¨ð WÀð WÈð Wñ Wð Wñ „ðWð
 ñXð X¨ð XÀð XÈð Xñ Xð Xñ „ðXð
 ñXð X¨ð XÀð XÈð Xñ Xð Xñ „ðXð
 ñWð W¨ð WÀð WÈð Wñ Wð Wñ „ðWð
 ñWð W¨ð WÀð WÈð Wñ Wð Wñ „ðWð
 ñRð R 4ð R¸3ð RÀ7ð Rñ Rð Rñ „ðRð
 ñXð X¨ð XÀð XÈð Xñ Xð Xñ „ðXð
 ñXð X¨ð XÀð XÈð Xñ Xð Xñ „ðXð
 ñYð Y¨$ð YÀ#ð YÈ'ð Yñ Yð Yñ „ðYð
 ñUð U dð U¸cð UÀgð Uñ Uð Uñ „ðUð
 ñZð Z¨4ð ZÀ3ð ZÈ7ð Zñ Zð Zñ „ðZð
 ñZð Z¨4ð ZÀ3ð ZÈ7ð Zñ Zð Zñ „ðZð
 ñ[ð [¨Dð [ÀCð [ÈGð [ñ [ð [ñ „ð[ð
 ñdð d 4ð d¸3ð dÀ7ð dñ dð dñ „ðdð
 ñdð d 4ð d¸3ð dÀ7ð dñ dð dñ „ðdð
 ñdð d 4ð d¸3ð dÀ7ð dñ dð dñ „ðdð
 ñfð f Dð f¸Cð fÀGð fñ fð fñ „ðfð
 ñfð f Dð f¸Cð fÀGð fñ fð fñ „ðfð
 ñfð f Dð f¸Cð fÀGð fñ fð fñ „ðfð
 ñdð d 4ð d¸3ð dÀ7ð dñ dð dñ „ðdð
 ñdð d 4ð d¸3ð dÀ7ð dñ dð dñ „ðdð
 ñdð d 4ð d¸3ð dÀ7ð dñ dð dñ „ðdð
 ñfð f Dð f¸Cð fÀGð fñ fð fñ „ðfð
 ñfð f Dð f¸Cð fÀGð fñ fð fñ „ðfð
 ñfð f Dð f¸Cð fÀGð fñ fð fñ „ðfð
 ñhð h Tð h¸Sð hÀWð hñ hð hñ „ðhð
 ñhð h Tð h¸Sð hÀWð hñ hð hñ „ðhð
 ñhð h Tð h¸Sð hÀWð hñ hð hñ „ðhð hð hr`   