§
    ·¯¸j¢ƒ  ã                   óø  — d Z ddlZddlmZ ddlmZmZmZmZm	Z	m
Z
mZ ddlZddlmZ ddlmZmZ ddlmZmZmZmZmZmZmZmZ ddlmZ dd	lmZ dd
lm Z m!Z!m"Z" ddl#m$Z$m%Z%m&Z& ddgZ' G d„ dej(        ¦  «        Z) G d„ dej(        ¦  «        Z* G d„ dej(        ¦  «        Z+ G d„ dej(        ¦  «        Z, G d„ dej(        ¦  «        Z- G d„ dej(        ¦  «        Z.dldej(        de/de0de1ddf
d„Z2d „ Z3dmde.fd!„Z4dndee/ef         fd#„Z5 e$i d$ e5¦   «         “d% e5¦   «         “d& e5¦   «         “d' e5d(d)¬*¦  «        “d+ e5d(d,d-¬.¦  «        “d/ e5¦   «         “d0 e5d(d1¬*¦  «        “d2 e5d(d3d-¬.¦  «        “d4 e5d(d5d6d7d8d9d:¬;¦  «        “d< e5d(d=d6d7d8d9¬>¦  «        “d? e5ee¬@¦  «        “dA e5d(dBee¬C¦  «        “dD e5d(dEee¬C¦  «        “dF e5d(dGee¬C¦  «        “dH e5d(dIee¬C¦  «        “dJ e5d(dKee¬C¦  «        “dL e5d(dMee¬C¦  «        “ e5d(dNee¬C¦  «         e5d(dOee¬C¦  «         e5d(dPee¬C¦  «         e5d(dQee¬C¦  «         e5d(dRee¬C¦  «         e5d(dSee¬C¦  «         e5¦   «          e5d(dT¬*¦  «         e5¦   «         dUœ	¥¦  «        Z6e%dmde.fdV„¦   «         Z7e%dmde.fdW„¦   «         Z8e%dmde.fdX„¦   «         Z9e%dmde.fdY„¦   «         Z:e%dmde.fdZ„¦   «         Z;e%dmde.fd[„¦   «         Z<e%dmde.fd\„¦   «         Z=e%dmde.fd]„¦   «         Z>e%dmde.fd^„¦   «         Z?e%dmde.fd_„¦   «         Z@e%dmde.fd`„¦   «         ZAe%dmde.fda„¦   «         ZBe%dmde.fdb„¦   «         ZCe%dmde.fdc„¦   «         ZDe%dmde.fdd„¦   «         ZE e&eFd'd0d<d4dLdedfdgdhdidjdkœ¦  «         dS )ouj   MLP-Mixer, ResMLP, and gMLP in PyTorch

This impl originally based on MLP-Mixer paper.

Official JAX impl: https://github.com/google-research/vision_transformer/blob/linen/vit_jax/models_mixer.py

Paper: 'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601

@article{tolstikhin2021,
  title={MLP-Mixer: An all-MLP Architecture for Vision},
  author={Tolstikhin, Ilya and Houlsby, Neil and Kolesnikov, Alexander and Beyer, Lucas and Zhai, Xiaohua and Unterthiner,
        Thomas and Yung, Jessica and Keysers, Daniel and Uszkoreit, Jakob and Lucic, Mario and Dosovitskiy, Alexey},
  journal={arXiv preprint arXiv:2105.01601},
  year={2021}
}

Also supporting ResMlp, and a preliminary (not verified) implementations of gMLP

Code: https://github.com/facebookresearch/deit
Paper: `ResMLP: Feedforward networks for image classification...` - https://arxiv.org/abs/2105.03404
@misc{touvron2021resmlp,
      title={ResMLP: Feedforward networks for image classification with data-efficient training},
      author={Hugo Touvron and Piotr Bojanowski and Mathilde Caron and Matthieu Cord and Alaaeldin El-Nouby and
        Edouard Grave and Armand Joulin and Gabriel Synnaeve and Jakob Verbeek and HervÃ© JÃ©gou},
      year={2021},
      eprint={2105.03404},
}

Paper: `Pay Attention to MLPs` - https://arxiv.org/abs/2105.08050
@misc{liu2021pay,
      title={Pay Attention to MLPs},
      author={Hanxiao Liu and Zihang Dai and David R. So and Quoc V. Le},
      year={2021},
      eprint={2105.08050},
}

A thank you to paper authors for releasing code and weights.

Hacked together by / Copyright 2021 Ross Wightman
é    N)Úpartial)ÚAnyÚDictÚListÚOptionalÚTypeÚUnionÚTuple)ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)Ú
PatchEmbedÚMlpÚGluMlpÚGatedMlpÚDropPathÚlecun_normal_Ú	to_2tupleÚget_device_dtypeé   )Úbuild_model_with_cfg)Úfeature_take_indices)Únamed_applyÚ
checkpointÚcheckpoint_seq)Úgenerate_default_cfgsÚregister_modelÚregister_model_deprecationsÚ
MixerBlockÚMlpMixerc                   ó
  ‡ — e Zd ZdZde eej        d¬¦  «        ej        ddddfde	de	d	e
eeeef         f         d
eej                 deej                 deej                 dededdfˆ fd„Zdej        dej        fd„Zˆ xZS )r   z—Residual Block w/ token mixing and channel MLPs.

    Based on: 'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601
    ©ç      à?ç      @ç�íµ ÷Æ°>©Úepsç        NÚdimÚseq_lenÚ	mlp_ratioÚ	mlp_layerÚ
norm_layerÚ	act_layerÚdropÚ	drop_pathÚreturnc                 ój  •‡— |	|
dœ}t          ¦   «                              ¦   «          ˆfd„t          |¦  «        D ¦   «         \  }} |‰fi |¤Ž| _         |||f||dœ|¤Ž| _        |dk    rt          |¦  «        nt          j        ¦   «         | _         |‰fi |¤Ž| _	         |‰|f||dœ|¤Ž| _
        dS )a’  Initialize MixerBlock.

        Args:
            dim: Dimension of input features.
            seq_len: Sequence length.
            mlp_ratio: Expansion ratios for token mixing and channel MLPs.
            mlp_layer: MLP layer class.
            norm_layer: Normalization layer.
            act_layer: Activation layer.
            drop: Dropout rate.
            drop_path: Drop path rate.
        ©ÚdeviceÚdtypec                 ó4   •— g | ]}t          |‰z  ¦  «        ‘ŒS © )Úint)Ú.0Úxr(   s     €úe/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/models/mlp_mixer.pyú
<listcomp>z'MixerBlock.__init__.<locals>.<listcomp>[   s#   ø€ Ð#OÐ#OÐ#O°Q¥C¨¨C©¡L¤LÐ#OÐ#OÐ#Oó    ©r-   r.   r'   N)ÚsuperÚ__init__r   Únorm1Ú
mlp_tokensr   ÚnnÚIdentityr/   Únorm2Úmlp_channels)Úselfr(   r)   r*   r+   r,   r-   r.   r/   r3   r4   ÚddÚ
tokens_dimÚchannels_dimÚ	__class__s    `            €r:   r?   zMixerBlock.__init__@   sò   øø€ ð2 ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ#OÐ#OÐ#OÐ#O½)ÀIÑ:NÔ:NÐ#OÑ#OÔ#OÑ ˆ
�LØ�Z Ð*Ð* rÐ*Ð*ˆŒ
Ø#˜) G¨ZÐ^À9ÐSWÐ^Ð^Ð[]Ð^Ð^ˆŒØ09¸B²°� )Ñ,Ô,Ð,ÅBÄKÁMÄMˆŒØ�Z Ð*Ð* rÐ*Ð*ˆŒ
Ø%˜I c¨<Ð^À9ÐSWÐ^Ð^Ð[]Ð^Ð^ˆÔÐÐr<   r9   c           	      óN  — ||                       |                      |                      |¦  «                             dd¦  «        ¦  «                             dd¦  «        ¦  «        z   }||                       |                      |                      |¦  «        ¦  «        ¦  «        z   }|S ©úForward pass.r   é   )r/   rA   r@   Ú	transposerE   rD   ©rF   r9   s     r:   ÚforwardzMixerBlock.forwardb   sƒ   € à�—’˜tŸš¨t¯zªz¸!©}¬}×/FÒ/FÀqÈ!Ñ/LÔ/LÑMÔM×WÒWÐXYÐ[\Ñ]Ô]Ñ^Ô^Ñ^ˆØ�—’˜t×0Ò0°·²¸A±´Ñ?Ô?Ñ@Ô@Ñ@ˆØˆr<   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   rB   Ú	LayerNormÚGELUr7   r	   Úfloatr
   r   ÚModuler?   ÚtorchÚTensorrQ   Ú__classcell__©rJ   s   @r:   r   r   ;   s.  ø€ € € € € ðð ð <FØ),Ø*1¨'°"´,ÀDÐ*IÑ*IÔ*IØ)+¬ØØ!ØØð _ð  _àð _ð ð _ð ˜U E¨%°¨,Ô$7Ð7Ô8ð	 _ð
 ˜BœI”ð _ð ˜RœYœð _ð ˜BœI”ð _ð ð _ð ð _ð 
ð _ð  _ð  _ð  _ð  _ð  _ðD˜œð ¨%¬,ð ð ð ð ð ð ð ð r<   c                   óR   ‡ — e Zd ZdZddeddfˆ fd„Zdej        dej        fd„Zˆ xZ	S )	ÚAffinezAffine transformation layer.Nr(   r0   c                 ó  •— ||dœ}t          ¦   «                              ¦   «          t          j        t	          j        dd|ffi |¤Ž¦  «        | _        t          j        t	          j        dd|ffi |¤Ž¦  «        | _        dS )zXInitialize Affine layer.

        Args:
            dim: Dimension of features.
        r2   r   N)	r>   r?   rB   Ú	ParameterrZ   ÚonesÚalphaÚzerosÚbeta)rF   r(   r3   r4   rG   rJ   s        €r:   r?   zAffine.__init__l   s~   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ”\¥%¤*¨a°°C¨[Ð"?Ð"?¸BÐ"?Ð"?Ñ@Ô@ˆŒ
Ý”L¥¤¨a°°C¨[Ð!?Ð!?¸BÐ!?Ð!?Ñ@Ô@ˆŒ	ˆ	ˆ	r<   r9   c                 óB   — t          j        | j        | j        |¦  «        S )zApply affine transformation.)rZ   Úaddcmulre   rc   rP   s     r:   rQ   zAffine.forwardw   s   € åŒ}˜TœY¨¬
°AÑ6Ô6Ð6r<   )NN)
rR   rS   rT   rU   r7   r?   rZ   r[   rQ   r\   r]   s   @r:   r_   r_   i   s…   ø€ € € € € Ø&Ð&ð	Að 	A˜Cð 	A¸Tð 	Að 	Að 	Að 	Að 	Að 	Að7˜œð 7¨%¬,ð 7ð 7ð 7ð 7ð 7ð 7ð 7ð 7r<   r_   c                   óÐ   ‡ — e Zd ZdZdeeej        dddddf	dedede	d	e
ej                 d
e
ej                 de
ej                 de	de	de	ddfˆ fd„Zdej        dej        fd„Zˆ xZS )ÚResBlockz¥Residual MLP block w/ LayerScale and Affine 'norm'.

    Based on: `ResMLP: Feedforward networks for image classification...` - https://arxiv.org/abs/2105.03404
    é   g-Cëâ6?r'   Nr(   r)   r*   r+   r,   r-   Úinit_valuesr.   r/   r0   c                 ó  •— |
|dœ}t          ¦   «                              ¦   «          t          ||z  ¦  «        } ||fi |¤Ž| _        t	          j        ||fi |¤Ž| _        |	dk    rt          |	¦  «        nt	          j        ¦   «         | _	         ||fi |¤Ž| _
         |||f||dœ|¤Ž| _        t	          j        |t          j        |fi |¤Žz  ¦  «        | _        t	          j        |t          j        |fi |¤Žz  ¦  «        | _        dS )a²  Initialize ResBlock.

        Args:
            dim: Dimension of input features.
            seq_len: Sequence length.
            mlp_ratio: Channel MLP expansion ratio.
            mlp_layer: MLP layer class.
            norm_layer: Normalization layer.
            act_layer: Activation layer.
            init_values: Initial values for layer scale.
            drop: Dropout rate.
            drop_path: Drop path rate.
        r2   r'   r=   N)r>   r?   r7   r@   rB   ÚLinearÚlinear_tokensr   rC   r/   rD   rE   ra   rZ   rb   Úls1Úls2)rF   r(   r)   r*   r+   r,   r-   rk   r.   r/   r3   r4   rG   Úchannel_dimrJ   s                 €r:   r?   zResBlock.__init__�   s"  ø€ ð6 ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ˜# 	™/Ñ*Ô*ˆØ�Z Ð*Ð* rÐ*Ð*ˆŒ
ÝœY w°Ð>Ð>¸2Ð>Ð>ˆÔØ09¸B²°� )Ñ,Ô,Ð,ÅBÄKÁMÄMˆŒØ�Z Ð*Ð* rÐ*Ð*ˆŒ
Ø%˜I c¨;Ð]À)ÐRVÐ]Ð]ÐZ\Ð]Ð]ˆÔÝ”< ­e¬j¸Ð.CÐ.CÀÐ.CÐ.CÑ CÑDÔDˆŒÝ”< ­e¬j¸Ð.CÐ.CÀÐ.CÐ.CÑ CÑDÔDˆŒˆˆr<   r9   c           
      ón  — ||                       | j        |                      |                      |¦  «                             dd¦  «        ¦  «                             dd¦  «        z  ¦  «        z   }||                       | j        |                      |                      |¦  «        ¦  «        z  ¦  «        z   }|S rL   )r/   ro   rn   r@   rO   rp   rE   rD   rP   s     r:   rQ   zResBlock.forward§   s•   € à�—’˜tœx¨$×*<Ò*<¸T¿ZºZÈ¹]¼]×=TÒ=TÐUVÐXYÑ=ZÔ=ZÑ*[Ô*[×*eÒ*eÐfgÐijÑ*kÔ*kÑkÑlÔlÑlˆØ�—’˜tœx¨$×*;Ò*;¸D¿JºJÀq¹M¼MÑ*JÔ*JÑJÑKÔKÑKˆØˆr<   )rR   rS   rT   rU   r   r_   rB   rW   r7   rX   r   rY   r?   rZ   r[   rQ   r\   r]   s   @r:   ri   ri   |   s  ø€ € € € € ðð ð  !Ø),Ø*0Ø)+¬Ø!%ØØ!ØØð$Eð $Eàð$Eð ð$Eð ð	$Eð
 ˜BœI”ð$Eð ˜RœYœð$Eð ˜BœI”ð$Eð ð$Eð ð$Eð ð$Eð 
ð$Eð $Eð $Eð $Eð $Eð $EðL˜œð ¨%¬,ð ð ð ð ð ð ð ð r<   ri   c            	       óˆ   ‡ — e Zd ZdZej        ddfdededeej                 ddfˆ fd„Z	dd„Z
d	ej        dej        fd
„Zˆ xZS )ÚSpatialGatingUnitzcSpatial Gating Unit.

    Based on: `Pay Attention to MLPs` - https://arxiv.org/abs/2105.08050
    Nr(   r)   r,   r0   c                 ó¨   •— ||dœ}t          ¦   «                              ¦   «          |dz  } ||fi |¤Ž| _        t          j        ||fi |¤Ž| _        dS )z¸Initialize Spatial Gating Unit.

        Args:
            dim: Dimension of input features.
            seq_len: Sequence length.
            norm_layer: Normalization layer.
        r2   rN   N)r>   r?   ÚnormrB   rm   Úproj)	rF   r(   r)   r,   r3   r4   rG   Úgate_dimrJ   s	           €r:   r?   zSpatialGatingUnit.__init__³   si   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ˜!‘8ˆØ�J˜xÐ.Ð.¨2Ð.Ð.ˆŒ	Ý”I˜g wÐ5Ð5°"Ð5Ð5ˆŒ	ˆ	ˆ	r<   c                 ó®   — t           j                             | j        j        d¬¦  «         t           j                             | j        j        ¦  «         dS )z'Initialize weights for projection gate.r$   ©ÚstdN)rB   ÚinitÚnormal_rw   ÚweightÚones_Úbias©rF   s    r:   Úinit_weightszSpatialGatingUnit.init_weightsÈ   s?   € õ 	Œ�Š˜œ	Ô(¨dˆÑ3Ô3Ð3Ý
Œ�Š�d”i”nÑ%Ô%Ð%Ð%Ð%r<   r9   c                 óä   — |                      dd¬¦  «        \  }}|                      |¦  «        }|                      |                     dd¦  «        ¦  «        }||                     dd¦  «        z  S )zApply spatial gating.rN   éÿÿÿÿ©r(   éþÿÿÿ)Úchunkrv   rw   rO   )rF   r9   ÚuÚvs       r:   rQ   zSpatialGatingUnit.forwardÎ   sb   € à�wŠw�q˜bˆwÑ!Ô!‰ˆˆ1Ø�IŠI�a‰LŒLˆØ�IŠI�a—k’k " bÑ)Ô)Ñ*Ô*ˆØ�1—;’;˜r 2Ñ&Ô&Ñ&Ð&r<   )r0   N)rR   rS   rT   rU   rB   rV   r7   r   rY   r?   r‚   rZ   r[   rQ   r\   r]   s   @r:   rt   rt   ®   s¿   ø€ € € € € ðð ð +-¬,ØØð6ð 6àð6ð ð6ð ˜RœYœð	6ð 
ð6ð 6ð 6ð 6ð 6ð 6ð*&ð &ð &ð &ð'˜œð '¨%¬,ð 'ð 'ð 'ð 'ð 'ð 'ð 'ð 'r<   rt   c                   óê   ‡ — e Zd ZdZde eej        d¬¦  «        ej        ddddfde	de	d	e
d
eej                 deej                 deej                 de
de
ddfˆ fd„Zdej        dej        fd„Zˆ xZS )ÚSpatialGatingBlockzpResidual Block w/ Spatial Gating.

    Based on: `Pay Attention to MLPs` - https://arxiv.org/abs/2105.08050
    rj   r$   r%   r'   Nr(   r)   r*   r+   r,   r-   r.   r/   r0   c                 ó:  •— |	|
dœ}t          ¦   «                              ¦   «          t          ||z  ¦  «        } ||fi |¤Ž| _        t	          t
          fd|i|¤Ž} |||f|||dœ|¤Ž| _        |dk    rt          |¦  «        nt          j	        ¦   «         | _
        dS )aƒ  Initialize SpatialGatingBlock.

        Args:
            dim: Dimension of input features.
            seq_len: Sequence length.
            mlp_ratio: Channel MLP expansion ratio.
            mlp_layer: MLP layer class.
            norm_layer: Normalization layer.
            act_layer: Activation layer.
            drop: Dropout rate.
            drop_path: Drop path rate.
        r2   r)   )r-   Ú
gate_layerr.   r'   N)r>   r?   r7   rv   r   rt   rE   r   rB   rC   r/   )rF   r(   r)   r*   r+   r,   r-   r.   r/   r3   r4   rG   rq   ÚsgurJ   s                 €r:   r?   zSpatialGatingBlock.__init__Û   sÎ   ø€ ð2 ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ˜# 	™/Ñ*Ô*ˆØ�J˜sÐ)Ð) bÐ)Ð)ˆŒ	ÝÕ'Ð?Ð?°Ð?¸BÐ?Ð?ˆØ%˜IØØð
ð  ØØð
ð 
ð ð
ð 
ˆÔð 1:¸B²°� )Ñ,Ô,Ð,ÅBÄKÁMÄMˆŒˆˆr<   r9   c                 ó‚   — ||                       |                      |                      |¦  «        ¦  «        ¦  «        z   }|S ©rM   )r/   rE   rv   rP   s     r:   rQ   zSpatialGatingBlock.forward  s5   € à�—’˜t×0Ò0°·²¸1±´Ñ>Ô>Ñ?Ô?Ñ?ˆØˆr<   )rR   rS   rT   rU   r   r   rB   rV   rW   r7   rX   r   rY   r?   rZ   r[   rQ   r\   r]   s   @r:   r‹   r‹   Ö   s  ø€ € € € € ðð ð  !Ø)1Ø*1¨'°"´,ÀDÐ*IÑ*IÔ*IØ)+¬ØØ!ØØð&Rð &Ràð&Rð ð&Rð ð	&Rð
 ˜BœI”ð&Rð ˜RœYœð&Rð ˜BœI”ð&Rð ð&Rð ð&Rð 
ð&Rð &Rð &Rð &Rð &Rð &RðP˜œð ¨%¬,ð ð ð ð ð ð ð ð r<   r‹   c            %       ó¶  ‡ — e Zd ZdZdddddddee eej        d	¬
¦  «        ej	        ddddddddfde
de
de
de
de
de
deeeeef         f         deej                 deej                 deej                 deej                 dedededededed df$ˆ fd!„Zej        j        d:ded dfd"„¦   «         Zej        j        d:d#ed eeef         fd$„¦   «         Zej        j        d;d&ed dfd'„¦   «         Zej        j        d ej        fd(„¦   «         Zd<de
dee         d dfd)„Z	 	 	 	 	 d=d+ej        d,eee
ee
         f                  d-ed.ed/ed0ed eeej                 eej        eej                 f         f         fd1„Z 	 	 	 d>d,ee
ee
         f         d3ed4ed ee
         fd5„Z!d+ej        d ej        fd6„Z"d:d+ej        d7ed ej        fd8„Z#d+ej        d ej        fd9„Z$ˆ xZ%S )?r   z„MLP-Mixer model architecture.

    Based on: 'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601
    éè  éà   é   é   é   é   r!   r$   r%   r'   FÚavgNÚnum_classesÚimg_sizeÚin_chansÚ
patch_sizeÚ
num_blocksÚ	embed_dimr*   Úblock_layerr+   r,   r-   Ú	drop_rateÚproj_drop_rateÚdrop_path_rateÚnlhbÚ	stem_normÚglobal_poolr0   c                 óò  •‡ ‡‡‡‡	‡
‡‡‡‡‡— t          ¦   «                              ¦   «          ||dœŠ|‰ _        |‰ _        |‰ _        ‰x‰ _        x‰ _        ‰ _        d‰ _        t          d
|||‰|r‰
nddœ‰¤Ž‰ _
        t          ‰ j
        d¦  «        r‰ j
                             ¦   «         n|Št          j        ˆˆˆˆˆˆ	ˆˆ
ˆˆ f
d„t          |¦  «        D ¦   «         Ž ‰ _        ˆˆfd„t          |¦  «        D ¦   «         ‰ _         ‰
‰fi ‰¤Ž‰ _        t          j        |¦  «        ‰ _        |dk    rt          j        ‰‰ j        fi ‰¤Žnt          j        ¦   «         ‰ _        ‰                      |¬	¦  «         dS )aB  Initialize MLP-Mixer.

        Args:
            num_classes: Number of classes for classification.
            img_size: Input image size.
            in_chans: Number of input channels.
            patch_size: Patch size.
            num_blocks: Number of mixer blocks.
            embed_dim: Embedding dimension.
            mlp_ratio: MLP expansion ratio(s).
            block_layer: Block layer class.
            mlp_layer: MLP layer class.
            norm_layer: Normalization layer.
            act_layer: Activation layer.
            drop_rate: Head dropout rate.
            proj_drop_rate: Projection dropout rate.
            drop_path_rate: Drop path rate.
            nlhb: Use negative log bias initialization.
            stem_norm: Apply normalization to stem.
            global_pool: Global pooling type.
        r2   FN)rš   rœ   r›   rž   r,   Ú
feat_ratioc                 óF   •
— g | ]} ‰‰‰j         j        ‰f‰‰	‰‰
‰d œ‰¤Ž‘ŒS ))r+   r,   r-   r.   r/   )ÚstemÚnum_patches)r8   Ú_r-   rŸ   rG   r¢   rž   r+   r*   r,   r¡   rF   s     €€€€€€€€€€r:   r;   z%MlpMixer.__init__.<locals>.<listcomp>L  sj   ø€ ð &(ð &(ð &(ð ð ˆKØØ”	Ô%Øð
ð $Ø%Ø#Ø#Ø(ð
ð 
ð ð
ð 
ð&(ð &(ð &(r<   c                 ó:   •— g | ]}t          d |› �‰‰¬¦  «        ‘ŒS )zblocks.)ÚmoduleÚnum_chsÚ	reduction©Údict)r8   Úirž   r¯   s     €€r:   r;   z%MlpMixer.__init__.<locals>.<listcomp>Y  sA   ø€ ð kð kð kØST�D˜ !˜˜¨yÀIÐNÑNÔNðkð kð kr<   r   )r£   r6   )r>   r?   r™   r›   r¥   Únum_featuresÚhead_hidden_sizerž   Úgrad_checkpointingr   r©   Úhasattrr§   rB   Ú
SequentialÚrangeÚblocksÚfeature_inforv   ÚDropoutÚ	head_droprm   rC   Úheadr‚   )rF   r™   rš   r›   rœ   r�   rž   r*   rŸ   r+   r,   r-   r    r¡   r¢   r£   r¤   r¥   r3   r4   rG   r¯   rJ   s   `     `````` ``     @@€r:   r?   zMlpMixer.__init__  sü  øøøøøøøøøøøø€ õV 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆØ&ˆÔØ ˆŒØ&ˆÔØENÐNˆÔÐN˜DÔ1°D´NØ"'ˆÔåð 
ØØ!ØØØ%.Ð8�z�z°Dð
ð 
ð ð
ð 
ˆŒ	õ /6°d´iÀÑ.NÔ.NÐ^�D”I×(Ò(Ñ*Ô*Ð*ÐT^ˆ	å”mð &(ð &(ð &(ð &(ð &(ð &(ð &(ð &(ð &(ð &(ð &(ð &(ð &(õ ˜:Ñ&Ô&ð&(ñ &(ô &(ð )ˆŒðkð kð kð kð kÝX]Ð^hÑXiÔXiðkñ kô kˆÔà�J˜yÐ/Ð/¨BÐ/Ð/ˆŒ	Ýœ IÑ.Ô.ˆŒØDOÐRSÂOÀO•B”I˜i¨Ô)9Ð@Ð@¸RÐ@Ð@Ð@ÕY[ÔYdÑYfÔYfˆŒ	à×Ò˜tÐÑ$Ô$Ð$Ð$Ð$r<   c                 óŒ   — |rt          j        | j        ¦  «         nd}t          t	          t
          |¬¦  «        | ¬¦  «         dS )zrInitialize model weights.

        Args:
            nlhb: Use negative log bias initialization for head.
        r'   )Ú	head_bias)r­   N)ÚmathÚlogr™   r   r   Ú_init_weights)rF   r£   r¿   s      r:   r‚   zMlpMixer.init_weightsa  sI   € ð 48Ð?•T”X˜dÔ.Ñ/Ô/Ð/Ð/¸Rˆ	Ý•G�M°YÐ?Ñ?Ô?ÈÐMÑMÔMÐMÐMÐMr<   Úcoarsec                 ó(   — t          dddg¬¦  «        S )z¾Create regex patterns for parameter grouping.

        Args:
            coarse: Use coarse grouping.

        Returns:
            Dictionary mapping group names to regex patterns.
        z^stem)z^blocks\.(\d+)N)z^norm)iŸ† )r©   r¹   r°   )rF   rÃ   s     r:   Úgroup_matcherzMlpMixer.group_matcherk  s'   € õ ØØ-Ð/CÐDð
ñ 
ô 
ð 	
r<   TÚenablec                 ó   — || _         dS )zEnable or disable gradient checkpointing.

        Args:
            enable: Whether to enable gradient checkpointing.
        N)rµ   )rF   rÆ   s     r:   Úset_grad_checkpointingzMlpMixer.set_grad_checkpointingz  s   € ð #)ˆÔÐÐr<   c                 ó   — | j         S )zGet the classifier module.)r½   r�   s    r:   Úget_classifierzMlpMixer.get_classifierƒ  s   € ð ŒyÐr<   c                 óú   — t          | ¦  «        }|| _        |�|dv sJ ‚|| _        |dk    rt          j        | j        |fi |¤Žnt          j        ¦   «         | _        | j                             | j	        ¦  «         dS )zŸReset the classifier head.

        Args:
            num_classes: Number of classes for new classifier.
            global_pool: Global pooling type.
        N)Ú r˜   r   )
r   r™   r¥   rB   rm   rž   rC   r½   ÚtrainÚtraining)rF   r™   r¥   rG   s       r:   Úreset_classifierzMlpMixer.reset_classifierˆ  s‰   € õ ˜dÑ#Ô#ˆØ&ˆÔØÐ"Ø +Ð-Ð-Ð-Ð-Ø*ˆDÔØDOÐRSÂOÀO•B”I˜dœn¨kÐ@Ð@¸RÐ@Ð@Ð@ÕY[ÔYdÑYfÔYfˆŒ	ØŒ	�Š˜œÑ&Ô&Ð&Ð&Ð&r<   ÚNCHWr9   Úindicesrv   Ú
stop_earlyÚ
output_fmtÚintermediates_onlyc                 óÜ  ‡‡‡— |dv s
J d¦   «         ‚|dk    }g }t          t          | j        ¦  «        |¦  «        \  }	}
|j        \  Š}}}|                      |¦  «        }t
          j                             ¦   «         s|s| j        }n| j        d|
dz   …         }t          |¦  «        D ]v\  }}| j	        r/t
          j                             ¦   «         st          ||¦  «        }n ||¦  «        }||	v r,|                     |r|                      |¦  «        n|¦  «         Œw|r/| j                             ||f¦  «        \  ŠŠˆˆˆfd„|D ¦   «         }|r|S |                      |¦  «        }||fS )aV  Forward features that returns intermediates.

        Args:
            x: Input image tensor.
            indices: Take last n blocks if int, all if None, select matching indices if sequence.
            norm: Apply norm layer to all intermediates.
            stop_early: Stop iterating over blocks when last desired intermediate hit.
            output_fmt: Shape of intermediate feature outputs ('NCHW' or 'NLC').
            intermediates_only: Only return intermediate features.

        Returns:
            List of intermediate features or tuple of (final features, intermediates).
        )rÐ   ÚNLCz)Output format must be one of NCHW or NLC.rÐ   Nr   c                 ó�   •— g | ]B}|                      ‰‰‰d ¦  «                             dddd¦  «                             ¦   «         ‘ŒCS )r„   r   r”   r   rN   )ÚreshapeÚpermuteÚ
contiguous)r8   ÚyÚBÚHÚWs     €€€r:   r;   z2MlpMixer.forward_intermediates.<locals>.<listcomp>Ç  sL   ø€ ÐlÐlÐlÐYZ˜QŸYšY q¨!¨Q°Ñ3Ô3×;Ò;¸A¸qÀ!ÀQÑGÔG×RÒRÑTÔTÐlÐlÐlr<   )r   Úlenr¹   Úshaper©   rZ   ÚjitÚis_scriptingÚ	enumeraterµ   r   Úappendrv   Údynamic_feat_size)rF   r9   rÑ   rv   rÒ   rÓ   rÔ   rØ   ÚintermediatesÚtake_indicesÚ	max_indexr«   ÚheightÚwidthr¹   r²   ÚblkrÜ   rÝ   rÞ   s                    @@@r:   Úforward_intermediateszMlpMixer.forward_intermediates—  s§  øøø€ ð, ˜_Ð,Ð,Ð,Ð.YÑ,Ô,Ð,Ø Ò&ˆØˆÝ"6µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�ið  œgÑˆˆ1ˆf�eØ�IŠI�a‰LŒLˆåŒ9×!Ò!Ñ#Ô#ð 	1¨:ð 	1Ø”[ˆFˆFà”[  )¨a¡- Ô0ˆFÝ Ñ'Ô'ð 	Bð 	B‰FˆAˆsØÔ&ð ­u¬y×/EÒ/EÑ/GÔ/Gð Ý˜s AÑ&Ô&��à�C˜‘F”F�Ø�LÐ Ð à×$Ò$°TÐ%@ T§Y¢Y¨q¡\¤\ \¸qÑAÔAÐAøð ð 	mà”9×.Ò.°¸¨Ñ?Ô?‰DˆAˆqØlÐlÐlÐlÐlÐlÐ^kÐlÑlÔlˆMàð 	!Ø Ð à�IŠI�a‰LŒLˆà�-ÐÐr<   r   Ú
prune_normÚ
prune_headc                 óâ   — t          t          | j        ¦  «        |¦  «        \  }}| j        d|dz   …         | _        |rt          j        ¦   «         | _        |r|                      dd¦  «         |S )aE  Prune layers not required for specified intermediates.

        Args:
            indices: Indices of intermediate layers to keep.
            prune_norm: Whether to prune normalization layer.
            prune_head: Whether to prune the classifier head.

        Returns:
            List of indices that were kept.
        Nr   r   rÌ   )r   rß   r¹   rB   rC   rv   rÏ   )rF   rÑ   rí   rî   rç   rè   s         r:   Úprune_intermediate_layersz"MlpMixer.prune_intermediate_layersÐ  sq   € õ  #7µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�iØ”k . 9¨q¡= .Ô1ˆŒØð 	&Ýœ™œˆDŒIØð 	)Ø×!Ò! ! RÑ(Ô(Ð(ØÐr<   c                 óú   — |                       |¦  «        }| j        r4t          j                             ¦   «         st          | j        |¦  «        }n|                      |¦  «        }|                      |¦  «        }|S )z/Forward pass through feature extraction layers.)r©   rµ   rZ   rá   râ   r   r¹   rv   rP   s     r:   Úforward_featureszMlpMixer.forward_featuresè  sg   € à�IŠI�a‰LŒLˆØÔ"ð 	­5¬9×+AÒ+AÑ+CÔ+Cð 	Ý˜tœ{¨AÑ.Ô.ˆAˆAà—’˜A‘”ˆAØ�IŠI�a‰LŒLˆØˆr<   Ú
pre_logitsc                 ó    — | j         dk    r|                     d¬¦  «        }|                      |¦  «        }|r|n|                      |¦  «        S )zÊForward pass through classifier head.

        Args:
            x: Feature tensor.
            pre_logits: Return features before final classifier.

        Returns:
            Output tensor.
        r˜   r   r…   )r¥   Úmeanr¼   r½   )rF   r9   ró   s      r:   Úforward_headzMlpMixer.forward_headò  sM   € ð Ô˜uÒ$Ð$Ø—’˜1�‘”ˆAØ�NŠN˜1ÑÔˆØÐ0ˆqˆq D§I¢I¨a¡L¤LÐ0r<   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S r�   )rò   rö   rP   s     r:   rQ   zMlpMixer.forward  s-   € à×!Ò! !Ñ$Ô$ˆØ×Ò˜aÑ Ô ˆØˆr<   ©F)T)N)NFFrÐ   F)r   FT)&rR   rS   rT   rU   r   r   r   rB   rV   rW   r7   r	   rX   r
   r   rY   ÚboolÚstrr?   rZ   rá   Úignorer‚   r   r   rÅ   rÈ   rÊ   r   rÏ   r[   r   rì   rð   rò   rö   rQ   r\   r]   s   @r:   r   r   	  s-  ø€ € € € € ðð ð  $ØØØ ØØ Ø;EØ+5Ø),Ø*1¨'°"´,ÀDÐ*IÑ*IÔ*IØ)+¬Ø!Ø$&Ø$&ØØ#Ø$ØØð)P%ð P%àðP%ð ðP%ð ð	P%ð
 ðP%ð ðP%ð ðP%ð ˜U E¨%°¨,Ô$7Ð7Ô8ðP%ð ˜bœiœðP%ð ˜BœI”ðP%ð ˜RœYœðP%ð ˜BœI”ðP%ð ðP%ð "ðP%ð "ðP%ð  ð!P%ð" ð#P%ð$ ð%P%ð* 
ð+P%ð P%ð P%ð P%ð P%ð P%ðd „YÔðNð N ð N°$ð Nð Nð Nñ ÔðNð „YÔð
ð 
 Dð 
°T¸#¸s¸(´^ð 
ð 
ð 
ñ Ôð
ð „YÔð)ð )¨Tð )¸Tð )ð )ð )ñ Ôð)ð „YÔð ¤	ð ð ð ñ Ôðð'ð '¨Cð '¸hÀs¼mð 'ÐW[ð 'ð 'ð 'ð 'ð$ 8<ØØ$Ø$Ø',ð7 ð 7 àŒ|ð7 ð ˜e C¨¨c¬ NÔ3Ô4ð7 ð ð	7 ð
 ð7 ð ð7 ð !%ð7 ð 
ˆt�E”LÔ! 5¨¬°t¸E¼LÔ7IÐ)IÔ#JÐJÔ	Kð7 ð 7 ð 7 ð 7 ðv ./Ø$Ø#ð	ð à˜3  S¤	˜>Ô*ðð ðð ð	ð
 
ˆcŒðð ð ð ð0 %¤,ð °5´<ð ð ð ð ð1ð 1˜eœlð 1¸ð 1ÈÌð 1ð 1ð 1ð 1ð˜œð ¨%¬,ð ð ð ð ð ð ð ð r<   r'   Fr­   Únamer¿   Úflaxr0   c                 ó€  — t          | t          j        ¦  «        �r$|                     d¦  «        rKt          j                             | j        ¦  «         t          j                             | j        |¦  «         dS |rCt          | j        ¦  «         | j        �&t          j                             | j        ¦  «         dS dS t          j         
                    | j        ¦  «         | j        �Rd|v r(t          j                             | j        d¬¦  «         dS t          j                             | j        ¦  «         dS dS t          | t          j        ¦  «        rCt          | j        ¦  «         | j        �&t          j                             | j        ¦  «         dS dS t          | t          j        t          j        t          j        f¦  «        rJt          j                             | j        ¦  «         t          j                             | j        ¦  «         dS t#          | d¦  «        r|                      ¦   «          dS dS )zèMixer weight initialization (trying to match Flax defaults).

    Args:
        module: Module to initialize.
        name: Module name.
        head_bias: Bias value for head layer.
        flax: Use Flax-style initialization.
    r½   NÚmlpr$   rz   r‚   )Ú
isinstancerB   rm   Ú
startswithr|   Úzeros_r~   Ú	constant_r€   r   Úxavier_uniform_r}   ÚConv2drV   ÚBatchNorm2dÚ	GroupNormr   r¶   r‚   )r­   rü   r¿   rý   s       r:   rÂ   rÂ     sî  € õ �&�"œ)Ñ$Ô$ñ Ø�?Š?˜6Ñ"Ô"ð 	4ÝŒG�NŠN˜6œ=Ñ)Ô)Ð)ÝŒG×Ò˜fœk¨9Ñ5Ô5Ð5Ð5Ð5àð 4å˜fœmÑ,Ô,Ð,Ø”;Ð*Ý”G—N’N 6¤;Ñ/Ô/Ð/Ð/Ð/ð +Ð*õ ”×'Ò'¨¬Ñ6Ô6Ð6Ø”;Ð*Ø �}�}ÝœŸš¨¬¸˜Ñ>Ô>Ð>Ð>Ð>åœŸš v¤{Ñ3Ô3Ð3Ð3Ð3ð	 +Ð*õ
 
�F�BœIÑ	&Ô	&ð 
Ý�f”mÑ$Ô$Ð$ØŒ;Ð"ÝŒG�NŠN˜6œ;Ñ'Ô'Ð'Ð'Ð'ð #Ð"å	�F�Rœ\­2¬>½2¼<ÐHÑ	IÔ	Ið Ý
Œ�Š�f”mÑ$Ô$Ð$Ý
Œ�Š�v”{Ñ#Ô#Ð#Ð#Ð#Ý	�˜Ñ	(Ô	(ð ð 	×ÒÑÔÐÐÐðð r<   c                 ó†  — d| v r¼i }|                       ¦   «         D ]£\  }}|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd	¦  «        }|                     d
¦  «        s|                     d¦  «        r|                     ddd¦  «        }|||<   Œ¤|S | S )z Remap checkpoints if needed zpatch_embed.proj.weightzpatch_embed.zstem.zattn.zlinear_tokens.zmlp.zmlp_channels.Úgamma_Úlsz.alphaz.betar   r„   )ÚitemsÚreplaceÚendswithrØ   )Ú
state_dictÚmodelÚout_dictÚkr‰   s        r:   Úcheckpoint_filter_fnr  0  sÐ   € à  JÐ.Ð.àˆØ×$Ò$Ñ&Ô&ð 	ð 	‰DˆAˆqØ—	’	˜.¨'Ñ2Ô2ˆAØ—	’	˜'Ð#3Ñ4Ô4ˆAØ—	’	˜& /Ñ2Ô2ˆAØ—	’	˜( DÑ)Ô)ˆAØ�zŠz˜(Ñ#Ô#ð ( q§z¢z°'Ñ':Ô':ð (Ø—I’I˜a  BÑ'Ô'�ØˆH�Q‰KˆKØˆØÐr<   c                 óˆ   — |                      dd¦  «        }t          t          | |ft          t	          |d¬¦  «        dœ|¤Ž}|S )NÚout_indicesr”   Úgetter)r  Úfeature_cls)Úpretrained_filter_fnÚfeature_cfg)Úpopr   r   r  r±   )ÚvariantÚ
pretrainedÚkwargsr  r  s        r:   Ú_create_mixerr  A  s]   € Ø—*’*˜]¨AÑ.Ô.€KÝ ÝØØðõ 2Ý [¸hÐGÑGÔGðð ð ðð €Eð €Lr<   rÌ   c                 ó$   — | ddd dddddddd	d
œ|¥S )Nr’   )r”   r“   r“   ç      ì?ÚbicubicT)r"   r"   r"   z	stem.projr½   z
apache-2.0)Úurlr™   Ú
input_sizeÚ	pool_sizeÚcrop_pctÚinterpolationÚfixed_input_sizerõ   r{   Ú
first_convÚ
classifierÚlicenser6   )r!  r  s     r:   Ú_cfgr*  N  s9   € àØ¨=ÀtØ¨IÈ4Ø¨Ø!°Øðð ð ðð r<   zmixer_s32_224.untrainedzmixer_s16_224.untrainedzmixer_b32_224.untrainedz mixer_b16_224.goog_in21k_ft_in1kztimm/zlhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-vitjx/jx_mixer_b16_224-76587d61.pth)Ú	hf_hub_idr!  zmixer_b16_224.goog_in21kzrhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-vitjx/jx_mixer_b16_224_in21k-617b3de2.pthiSU  )r+  r!  r™   zmixer_l32_224.untrainedz mixer_l16_224.goog_in21k_ft_in1kzlhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-vitjx/jx_mixer_l16_224-92f9adc4.pthzmixer_l16_224.goog_in21kzrhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-vitjx/jx_mixer_l16_224_in21k-846aa33c.pthzmixer_b16_224.miil_in21kzvhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tresnet/mixer_b16_224_miil_in21k-2a558a71.pth)r'   r'   r'   )ç      ð?r,  r,  r  ÚbilineariÕ+  )r+  r!  rõ   r{   r$  r%  r™   z mixer_b16_224.miil_in21k_ft_in1kzphttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tresnet/mixer_b16_224_miil-9229a591.pth)r+  r!  rõ   r{   r$  r%  zgmixer_12_224.untrained)rõ   r{   zgmixer_24_224.ra3_in1kzohttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/gmixer_24_224_raa-7daf7ae6.pth)r+  r!  rõ   r{   zresmlp_12_224.fb_in1kz9https://dl.fbaipublicfiles.com/deit/resmlp_12_no_dist.pthzresmlp_24_224.fb_in1kz9https://dl.fbaipublicfiles.com/deit/resmlp_24_no_dist.pthzresmlp_36_224.fb_in1kz9https://dl.fbaipublicfiles.com/deit/resmlp_36_no_dist.pthzresmlp_big_24_224.fb_in1kz:https://dl.fbaipublicfiles.com/deit/resmlpB_24_no_dist.pthzresmlp_12_224.fb_distilled_in1kz6https://dl.fbaipublicfiles.com/deit/resmlp_12_dist.pthz6https://dl.fbaipublicfiles.com/deit/resmlp_24_dist.pthz6https://dl.fbaipublicfiles.com/deit/resmlp_36_dist.pthz7https://dl.fbaipublicfiles.com/deit/resmlpB_24_dist.pthz6https://dl.fbaipublicfiles.com/deit/resmlpB_24_22k.pthz6https://dl.fbaipublicfiles.com/deit/resmlp_12_dino.pthz6https://dl.fbaipublicfiles.com/deit/resmlp_24_dino.pthznhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/gmlp_s16_224_raa-10536d42.pth)	úresmlp_24_224.fb_distilled_in1kúresmlp_36_224.fb_distilled_in1kú#resmlp_big_24_224.fb_distilled_in1kú"resmlp_big_24_224.fb_in22k_ft_in1kzresmlp_12_224.fb_dinozresmlp_24_224.fb_dinozgmlp_ti16_224.untrainedzgmlp_s16_224.ra3_in1kzgmlp_b16_224.untrainedc                 óB   — t          dddddœ|¤Ž}t          dd| i|¤Ž}|S )	zv Mixer-S/32 224x224
    Paper: 'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601
    é    r–   r—   ©rœ   r�   rž   Úmixer_s32_224r  r6   )r5  ©r±   r  ©r  r  Ú
model_argsr  s       r:   r5  r5  ¾  ó=   € õ
 ÐK °¸SÐKÐKÀFÐKÐK€JÝÐOÐO°jÐOÀJÐOÐO€EØ€Lr<   c                 óB   — t          dddddœ|¤Ž}t          dd| i|¤Ž}|S )	zw Mixer-S/16 224x224
    Paper:  'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601
    r•   r–   r—   r4  Úmixer_s16_224r  r6   )r;  r6  r7  s       r:   r;  r;  È  r9  r<   c                 óB   — t          dddddœ|¤Ž}t          dd| i|¤Ž}|S )	zw Mixer-B/32 224x224
    Paper:  'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601
    r3  é   é   r4  Úmixer_b32_224r  r6   )r?  r6  r7  s       r:   r?  r?  Ò  ó=   € õ
 ÐL °¸cÐLÐLÀVÐLÐL€JÝÐOÐO°jÐOÀJÐOÐO€EØ€Lr<   c                 óB   — t          dddddœ|¤Ž}t          dd| i|¤Ž}|S )	z˜ Mixer-B/16 224x224. ImageNet-1k pretrained weights.
    Paper:  'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601
    r•   r=  r>  r4  Úmixer_b16_224r  r6   )rB  r6  r7  s       r:   rB  rB  Ü  r@  r<   c                 óB   — t          dddddœ|¤Ž}t          dd| i|¤Ž}|S )	zx Mixer-L/32 224x224.
    Paper:  'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601
    r3  é   é   r4  Úmixer_l32_224r  r6   )rF  r6  r7  s       r:   rF  rF  æ  ó=   € õ
 ÐM °¸dÐMÐMÀfÐMÐM€JÝÐOÐO°jÐOÀJÐOÐO€EØ€Lr<   c                 óB   — t          dddddœ|¤Ž}t          dd| i|¤Ž}|S )	z˜ Mixer-L/16 224x224. ImageNet-1k pretrained weights.
    Paper:  'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601
    r•   rD  rE  r4  Úmixer_l16_224r  r6   )rI  r6  r7  s       r:   rI  rI  ð  rG  r<   c           
      óf   — t          dddddt          t          j        dœ|¤Ž}t	          d	d| i|¤Ž}|S )
zV Glu-Mixer-12 224x224
    Experiment by Ross Wightman, adding SwiGLU to MLP-Mixer
    r•   r=  é€  ©r,  r#   ©rœ   r�   rž   r*   r+   r-   Úgmixer_12_224r  r6   )rN  ©r±   r   rB   ÚSiLUr  r7  s       r:   rN  rN  ú  óT   € õ
 ð 7Ø "°¸zÝ¥B¤Gð7ð 7à/5ð7ð 7€Jõ ÐOÐO°jÐOÀJÐOÐO€EØ€Lr<   c           
      óf   — t          dddddt          t          j        dœ|¤Ž}t	          d	d| i|¤Ž}|S )
zV Glu-Mixer-24 224x224
    Experiment by Ross Wightman, adding SwiGLU to MLP-Mixer
    r•   rD  rK  rL  rM  Úgmixer_24_224r  r6   )rS  rO  r7  s       r:   rS  rS    rQ  r<   c           
      ó\   — t          dddddt          t          dœ|¤Ž}t          d	d| i|¤Ž}|S )
zx ResMLP-12
    Paper: `ResMLP: Feedforward networks for image classification...` - https://arxiv.org/abs/2105.03404
    r•   r=  rK  rj   ©rœ   r�   rž   r*   rŸ   r,   Úresmlp_12_224r  r6   )rV  )r±   ri   r_   r  r7  s       r:   rV  rV    sU   € õ
 ð uØ "°¸qÍhÕciðuð uØmsðuð u€JåÐOÐO°jÐOÀJÐOÐO€EØ€Lr<   c                 óz   — t          d
ddddt          t          d¬¦  «        t          dœ|¤Ž}t	          dd	| i|¤Ž}|S )zx ResMLP-24
    Paper: `ResMLP: Feedforward networks for image classification...` - https://arxiv.org/abs/2105.03404
    r•   rD  rK  rj   gñhãˆµøä>©rk   rU  Úresmlp_24_224r  r6   )rY  ©r±   r   ri   r_   r  r7  s       r:   rY  rY    ód   € õ
 ð VØ "°¸qÝ�H°$Ð7Ñ7Ô7ÅFðVð VàNTðVð V€Jõ ÐOÐO°jÐOÀJÐOÐO€EØ€Lr<   c                 óz   — t          d
ddddt          t          d¬¦  «        t          dœ|¤Ž}t	          dd	| i|¤Ž}|S )zx ResMLP-36
    Paper: `ResMLP: Feedforward networks for image classification...` - https://arxiv.org/abs/2105.03404
    r•   é$   rK  rj   r$   rX  rU  Úresmlp_36_224r  r6   )r^  rZ  r7  s       r:   r^  r^  )  r[  r<   c                 óz   — t          d
ddddt          t          d¬¦  «        t          dœ|¤Ž}t	          dd	| i|¤Ž}|S )zz ResMLP-B-24
    Paper: `ResMLP: Feedforward networks for image classification...` - https://arxiv.org/abs/2105.03404
    r–   rD  r>  rj   r$   rX  rU  Úresmlp_big_24_224r  r6   )r`  rZ  r7  s       r:   r`  r`  5  sd   € õ
 ð VØ ¨s¸aÝ�H°$Ð7Ñ7Ô7ÅFðVð VàNTðVð V€Jõ ÐSÐS¸*ÐSÈ
ÐSÐS€EØ€Lr<   c           
      ó\   — t          dddddt          t          dœ|¤Ž}t          d	d| i|¤Ž}|S )
zU gMLP-Tiny
    Paper: `Pay Attention to MLPs` - https://arxiv.org/abs/2105.08050
    r•   é   é€   é   ©rœ   r�   rž   r*   rŸ   r+   Úgmlp_ti16_224r  r6   )rf  ©r±   r‹   r   r  r7  s       r:   rf  rf  A  sS   € õ
 ð &Ø "°¸qÕN`Ýð&ð &à$ð&ð &€Jõ ÐOÐO°jÐOÀJÐOÐO€EØ€Lr<   c           
      ó\   — t          dddddt          t          dœ|¤Ž}t          d	d| i|¤Ž}|S )
zV gMLP-Small
    Paper: `Pay Attention to MLPs` - https://arxiv.org/abs/2105.08050
    r•   rb  é   rd  re  Úgmlp_s16_224r  r6   )rj  rg  r7  s       r:   rj  rj  M  óS   € õ
 ð &Ø "°¸qÕN`Ýð&ð &à$ð&ð &€Jõ ÐNÐN°ZÐNÀ:ÐNÐN€EØ€Lr<   c           
      ó\   — t          dddddt          t          dœ|¤Ž}t          d	d| i|¤Ž}|S )
zU gMLP-Base
    Paper: `Pay Attention to MLPs` - https://arxiv.org/abs/2105.08050
    r•   rb  r—   rd  re  Úgmlp_b16_224r  r6   )rm  rg  r7  s       r:   rm  rm  Y  rk  r<   r.  r/  r0  r1  rV  rY  )Úmixer_b16_224_in21kÚmixer_l16_224_in21kÚmixer_b16_224_miilÚmixer_b16_224_miil_in21kÚresmlp_12_distilled_224Úresmlp_24_distilled_224Úresmlp_36_distilled_224Úresmlp_big_24_distilled_224Úresmlp_big_24_224_in22ft1kÚresmlp_12_224_dinoÚresmlp_24_224_dino)r'   Frø   )rÌ   )GrU   rÀ   Ú	functoolsr   Útypingr   r   r   r   r   r	   r
   rZ   Útorch.nnrB   Ú	timm.datar   r   Útimm.layersr   r   r   r   r   r   r   r   Ú_builderr   Ú	_featuresr   Ú_manipulater   r   r   Ú	_registryr   r   r   Ú__all__rY   r   r_   ri   rt   r‹   r   rú   rX   rù   rÂ   r  r  r*  Údefault_cfgsr5  r;  r?  rB  rF  rI  rN  rS  rV  rY  r^  r`  rf  rj  rm  rR   r6   r<   r:   ú<module>r„     sc	  ðð'ð 'ðP €€€Ø Ð Ð Ð Ð Ð Ø @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @à €€€Ø Ð Ð Ð Ð Ð à AÐ AÐ AÐ AÐ AÐ AÐ AÐ AØ oÐ oÐ oÐ oÐ oÐ oÐ oÐ oÐ oÐ oÐ oÐ oÐ oÐ oÐ oÐ oÐ oÐ oÐ oÐ oà *Ð *Ð *Ð *Ð *Ð *Ø +Ð +Ð +Ð +Ð +Ð +Ø @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ø YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ Yà˜Ð
$€ð+ð +ð +ð +ð +�”ñ +ô +ð +ð\7ð 7ð 7ð 7ð 7ˆRŒYñ 7ô 7ð 7ð&/ð /ð /ð /ð /ˆrŒyñ /ô /ð /ðd%'ð %'ð %'ð %'ð %'˜œ	ñ %'ô %'ð %'ðP0ð 0ð 0ð 0ð 0˜œñ 0ô 0ð 0ðf|ð |ð |ð |ð |ˆrŒyñ |ô |ð |ð~%ð %˜"œ)ð %¨3ð %¸5ð %ÈTð %Ð^bð %ð %ð %ð %ðPð ð ð"
ð 
¸(ð 
ð 
ð 
ð 
ð	ð 	˜d 3¨ 8œnð 	ð 	ð 	ð 	ð %Ð$ð a&Ø˜t˜t™vœvða&à˜t˜t™vœvða&ð ˜t˜t™vœvða&ð '¨¨ØØzð)ñ )ô )ð	a&ð   Øð AØð!ñ !ô !ða&ð ˜t˜t™vœvða&ð '¨¨ØØzð)ñ )ô )ða&ð$   Øð AØð!ñ !ô !ð%a&ð2   Øð EØ˜|°eÈ:Ðchð!ñ !ô !ð3a&ð< '¨¨ØØ~Ø˜|°eÈ:ð)ñ )ô )ð=a&ðH ˜t˜tÐ)>ÐDXÐYÑYÔYðIa&ðJ ˜d˜dØØ}Ø"Ð(<ð>ñ >ô >ðKa&ðT ˜T˜TØØGØ"Ð(<ð>ñ >ô >ðUa&ð\ ˜T˜TØØGà"Ð(<ð	>ñ >ô >ð]a&ðf ˜T˜TØØGØ"Ð(<ð>ñ >ô >ðga&ðn    ØØHØ"Ð(<ð">ñ ">ô ">ðoa&ðx & t tØØDØ"Ð(<ð(>ñ (>ô (>ðya&ð@ (, tØØDØ"Ð(<ð(>ñ (>ô (>ð (, tØØDØ"Ð(<ð(>ñ (>ô (>ð ,0¨4ØØEØ"Ð(<ð,>ñ ,>ô ,>ð
 +/¨$ØØDØ"Ð(<ð+>ñ +>ô +>ð
 "˜TØØDØ"Ð(<ð>ñ >ô >ð "˜TØØDØ"Ð(<ð>ñ >ô >ð
  $˜t™vœvØ!˜TØØ|ðñ ô ð #˜d™fœfðAa&ð a&ð a&ñ aô a€ðH ðð °ð ð ð ñ „ðð ðð °ð ð ð ñ „ðð ðð °ð ð ð ñ „ðð ðð °ð ð ð ñ „ðð ðð °ð ð ð ñ „ðð ðð °ð ð ð ñ „ðð ðð °ð ð ð ñ „ðð ðð °ð ð ð ñ „ðð ðð °ð ð ð ñ „ðð ðð °ð ð ð ñ „ðð ðð °ð ð ð ñ „ðð ðð °Xð ð ð ñ „ðð ðð °ð ð ð ñ „ðð ðð °ð ð ð ñ „ðð ðð °ð ð ð ñ „ðð Ð ˜HØ=Ø=Ø<Ø :Ø@Ø@Ø@Ø#HØ"FØ)Ø)ð'ð 'ñ ô ð ð ð r<   