§
    ·¯¸jlH  ã                   óT  — d Z ddlZddlZddlmZ ddlmZmZmZm	Z	m
Z
mZmZ ddlZddlmZ ddlmZmZ ddlmZmZmZmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZ dgZ  G d„ dej!        ¦  «        Z" G d„ dej#        ¦  «        Z$ G d„ dej#        ¦  «        Z% G d„ dej#        ¦  «        Z& G d„ dej#        ¦  «        Z'd„ Z(d*d„Z)d+d„Z* e e*d¬¦  «         e*d¬¦  «         e*d¬¦  «         e*d¬¦  «         e*dd¬¦  «         e*dd¬¦  «         e*dd¬¦  «         e*dd¬¦  «        d œ¦  «        Z+ed*d!e'fd"„¦   «         Z,ed*d!e'fd#„¦   «         Z-ed*d!e'fd$„¦   «         Z.ed*d!e'fd%„¦   «         Z/ed*d!e'fd&„¦   «         Z0ed*d!e'fd'„¦   «         Z1ed*d!e'fd(„¦   «         Z2ed*d!e'fd)„¦   «         Z3dS ),a€   Pooling-based Vision Transformer (PiT) in PyTorch

A PyTorch implement of Pooling-based Vision Transformers as described in
'Rethinking Spatial Dimensions of Vision Transformers' - https://arxiv.org/abs/2103.16302

This code was adapted from the original version at https://github.com/naver-ai/pit, original copyright below.

Modifications for timm by / Copyright 2020 Ross Wightman
é    N)Úpartial)ÚListÚOptionalÚSequenceÚTupleÚUnionÚTypeÚAny)Únn©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)Útrunc_normal_Ú	to_2tupleÚcalculate_drop_path_ratesÚget_device_dtypeé   )Úbuild_model_with_cfg)Úfeature_take_indices)Úregister_modelÚgenerate_default_cfgs)ÚBlockÚPoolingVisionTransformerc                   ój   — e Zd ZdZdeej        ej        f         deej        ej        f         fd„ZdS )ÚSequentialTuplezI This module exists to work around torchscript typing issues list -> listÚxÚreturnc                 ó&   — | D ]} ||¦  «        }Œ|S ©N© )Úselfr   Úmodules      ú_/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/models/pit.pyÚforwardzSequentialTuple.forward#   s%   € Øð 	ð 	ˆFØ��q‘	”	ˆAˆAØˆó    N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚtorchÚTensorr$   r    r%   r#   r   r   !   sZ   € € € € € ØSÐSð˜˜uœ|¨U¬\Ð9Ô:ð ¸uÀUÄ\ÐSXÔS_ÐE_Ô?`ð ð ð ð ð ð r%   r   c                   óò   ‡ — e Zd Z	 	 	 	 	 	 	 ddededededee         ded	ed
eee                  deee	j
                          fˆ fd„Zdeej        ej        f         deej        ej        f         fd„Zˆ xZS )ÚTransformerNç        Úbase_dimÚdepthÚheadsÚ	mlp_ratioÚpoolÚ	proj_dropÚ	attn_dropÚdrop_path_probÚ
norm_layerc           	      ó"  •‡‡‡‡‡‡‡— |
|dœŠt          ¦   «                              ¦   «          |‰z  Š|| _        |	r	 |	‰fi ‰¤Žnt          j        ¦   «         | _        t          j        ˆˆˆˆˆˆˆfd„t          |¦  «        D ¦   «         Ž | _        d S )N©ÚdeviceÚdtypec                 óx   •— g | ]6}t          d‰‰‰d ‰‰‰|         t          t          j        d¬¦  «        dœ‰¤Ž‘Œ7S )Tç�íµ ÷Æ°>)Úeps)ÚdimÚ	num_headsr2   Úqkv_biasr4   r5   Ú	drop_pathr7   r    )r   r   r   Ú	LayerNorm)	Ú.0Úir5   Úddr6   Ú	embed_dimr1   r2   r4   s	     €€€€€€€r#   ú
<listcomp>z(Transformer.__init__.<locals>.<listcomp>>   su   ø€ ð &#ð &#ð &#ð õ ð 
ØØØ#ØØ#Ø#Ø(¨Ô+Ý"¥2¤<°TÐ:Ñ:Ô:ð
ð 
ð ð
ð 
ð&#ð &#ð &#r%   )	ÚsuperÚ__init__r3   r   ÚIdentityÚnormÚ
SequentialÚrangeÚblocks)r!   r/   r0   r1   r2   r3   r4   r5   r6   r7   r:   r;   rF   rG   Ú	__class__s      `` ```   @@€r#   rJ   zTransformer.__init__*   sÇ   øøøøøøøø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ˜uÑ$ˆ	àˆŒ	Ø3=ÐP�J�J˜yÐ/Ð/¨BÐ/Ð/Ð/Å2Ä;Á=Ä=ˆŒ	Ý”mð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#õ ˜5‘\”\ð&#ñ &#ô &#ð $ˆŒˆˆr%   r   r   c                 óì  — |\  }}|j         d         }| j        �|                      ||¦  «        \  }}|j         \  }}}}|                     d¦  «                             dd¦  «        }t	          j        ||fd¬¦  «        }|                      |¦  «        }|                      |¦  «        }|d d …d |…f         }|d d …|d …f         }|                     dd¦  «                             ||||¦  «        }||fS )Nr   é   )r?   )	Úshaper3   ÚflattenÚ	transposer*   ÚcatrL   rO   Úreshape)r!   r   Ú
cls_tokensÚtoken_lengthÚBÚCÚHÚWs           r#   r$   zTransformer.forwardL   sú   € Ø‰ˆˆ:Ø!Ô'¨Ô*ˆØŒ9Ð Ø ŸIšI a¨Ñ4Ô4‰MˆAˆzà”W‰
ˆˆ1ˆa�Ø�IŠI�a‰LŒL×"Ò" 1 aÑ(Ô(ˆÝŒI�z 1�o¨1Ð-Ñ-Ô-ˆà�IŠI�a‰LŒLˆØ�KŠK˜‰NŒNˆà�q�q�q˜-˜<˜-Ð'Ô(ˆ
Øˆaˆaˆa���ÐÔˆØ�KŠK˜˜1ÑÔ×%Ò% a¨¨A¨qÑ1Ô1ˆà�*ˆ}Ðr%   )Nr.   r.   NNNN)r&   r'   r(   ÚintÚfloatr   r
   r   r	   r   ÚModulerJ   r   r*   r+   r$   Ú__classcell__©rP   s   @r#   r-   r-   )   s
  ø€ € € € € ð #'Ø!Ø!Ø48Ø48ØØð $ð  $àð $ð ð $ð ð	 $ð
 ð $ð ˜3”-ð $ð ð $ð ð $ð % T¨%¤[Ô1ð $ð !  b¤i¤Ô1ð $ð  $ð  $ð  $ð  $ð  $ðD˜˜uœ|¨U¬\Ð9Ô:ð ¸uÀUÄ\ÐSXÔS_ÐE_Ô?`ð ð ð ð ð ð ð ð r%   r-   c            	       óh   ‡ — e Zd Z	 	 	 d
dedededefˆ fd„Zdeej        ej        f         fd	„Z	ˆ xZ
S )ÚPoolingÚzerosNÚ
in_featureÚout_featureÚstrideÚpadding_modec           	      óÊ   •— ||dœ}t          ¦   «                              ¦   «          t          j        ||f|dz   |dz  |||dœ|¤Ž| _        t          j        ||fi |¤Ž| _        d S )Nr9   r   rR   )Úkernel_sizeÚpaddingrh   ri   Úgroups)rI   rJ   r   ÚConv2dÚconvÚLinearÚfc)	r!   rf   rg   rh   ri   r:   r;   rF   rP   s	           €r#   rJ   zPooling.__init__a   s�   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐå”IØØð	
ð  ™
Ø˜a‘KØØ%Øð	
ð 	
ð ð	
ð 	
ˆŒ	õ ”)˜J¨Ð:Ð:°rÐ:Ð:ˆŒˆˆr%   r   c                 ó^   — |                       |¦  «        }|                      |¦  «        }||fS r   )ro   rq   )r!   r   Ú	cls_tokens      r#   r$   zPooling.forwardy   s,   € Ø�IŠI�a‰LŒLˆØ—G’G˜IÑ&Ô&ˆ	Ø�)ˆ|Ðr%   )re   NN)r&   r'   r(   r^   ÚstrrJ   r   r*   r+   r$   ra   rb   s   @r#   rd   rd   `   sœ   ø€ € € € € ð !(ØØð;ð ;àð;ð ð;ð ð	;ð
 ð;ð ;ð ;ð ;ð ;ð ;ð0 u¨U¬\¸5¼<Ð-GÔ'Hð ð ð ð ð ð ð ð r%   rd   c                   óL   ‡ — e Zd Z	 	 	 	 	 	 ddededed	ed
edefˆ fd„Zd„ Zˆ xZS )ÚConvEmbeddingéà   é   é   r   NÚin_channelsÚout_channelsÚimg_sizeÚ
patch_sizerh   rl   c	                 óþ  •— ||dœ}	t          ¦   «                              ¦   «          |}t          |¦  «        | _        t          |¦  «        | _        t          j        | j        d         d|z  z   | j        d         z
  |z  dz   ¦  «        | _        t          j        | j        d         d|z  z   | j        d         z
  |z  dz   ¦  «        | _        | j        | j        f| _	        t          j        ||f|||ddœ|	¤Ž| _        d S )Nr9   r   rR   r   T)rk   rh   rl   Úbias)rI   rJ   r   r|   r}   ÚmathÚfloorÚheightÚwidthÚ	grid_sizer   rn   ro   )r!   rz   r{   r|   r}   rh   rl   r:   r;   rF   rP   s             €r#   rJ   zConvEmbedding.__init__€   s  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØˆÝ! (Ñ+Ô+ˆŒÝ# JÑ/Ô/ˆŒÝ”j $¤-°Ô"2°Q¸±[Ñ"@À4Ä?ÐSTÔCUÑ"UÐY_Ñ!_ÐbcÑ!cÑdÔdˆŒÝ”Z ¤¨qÔ!1°A¸±KÑ!?À$Ä/ÐRSÔBTÑ!TÐX^Ñ ^ÐabÑ bÑcÔcˆŒ
Øœ+ t¤zÐ2ˆŒå”IØØð
ð #ØØØð
ð 
ð ð
ð 
ˆŒ	ˆ	ˆ	r%   c                 ó0   — |                       |¦  «        }|S r   )ro   ©r!   r   s     r#   r$   zConvEmbedding.forwardž   s   € Ø�IŠI�a‰LŒLˆØˆr%   )rw   rx   ry   r   NN)r&   r'   r(   r^   rJ   r$   ra   rb   s   @r#   rv   rv      sŸ   ø€ € € € € ð
  Ø ØØØØð
ð 
àð
ð ð
ð ð	
ð
 ð
ð ð
ð ð
ð 
ð 
ð 
ð 
ð 
ð<ð ð ð ð ð ð r%   rv   c            #       óˆ  ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d:dededededee         dee         dee         dededededededededed ef"ˆ fd!„Z	d"„ Z
ej        j        d#„ ¦   «         Zej        j        d;d%„¦   «         Zej        j        d;d&„¦   «         Zd'ej        fd(„Zd<dedee         fd)„Z	 	 	 	 	 d=d+ej        d,eeeee         f                  d-ed.ed/ed0ed'eeej                 eej        eej                 f         f         fd1„Z	 	 	 d>d,eeee         f         d3ed4efd5„Zd6„ Zd?d7ed'ej        fd8„Zd9„ Zˆ xZS )@r   z¤ Pooling-based Vision Transformer

    A PyTorch implement of 'Rethinking Spatial Dimensions of Vision Transformers'
        - https://arxiv.org/abs/2103.16302
    rw   rx   ry   Úoverlap©é0   rŠ   rŠ   ©rR   é   é   ©rR   r�   ry   r�   éè  é   ÚtokenFr.   Nr|   r}   rh   Ú	stem_typeÚ	base_dimsr0   r1   r2   Únum_classesÚin_chansÚglobal_poolÚ	distilledÚ	drop_rateÚpos_drop_drateÚproj_drop_rateÚattn_drop_rateÚdrop_path_ratec           
      óü  •— t          ¦   «                              ¦   «          ||dœ}|dv sJ ‚|| _        || _        |d         |d         z  }|	| _        |
| _        || _        |rdnd| _        g | _        t          |
||||fi |¤Ž| _
        t          j        t          j        d|| j
        j        | j
        j        fi |¤Ž¦  «        | _        t          j        t          j        d| j        |fi |¤Ž¦  «        | _        t          j        |¬¦  «        | _        g }t+          ||d¬¦  «        }|}t-          t/          |¦  «        ¦  «        D ]‹}d }||         ||         z  }|dk    rt1          ||fd	di|¤Ž}|t3          ||         ||         ||         |f|||||         d
œ|¤Žgz  }|}| xj        t5          ||dz
  d|z  z  d|› �¬¦  «        gz  c_        ŒŒt7          |Ž | _        t          j        |d         |d         z  fddi|¤Ž| _        |x| _        x| _         | _!        t          j        |¦  «        | _"        |	dk    rt          j#        | j!        |	fi |¤Žnt          j$        ¦   «         | _%        d | _&        |r;|	dk    rt          j#        | j!        | j        fi |¤Žnt          j$        ¦   «         | _&        d| _'        tQ          | j        d¬¦  «         tQ          | j        d¬¦  «         |  )                    | j*        ¦  «         d S )Nr9   )r‘   r   rR   r   )ÚpT)Ú	stagewiserh   )r3   r4   r5   r6   útransformers.)Únum_chsÚ	reductionr"   éÿÿÿÿr>   r=   Fg{®Gáz”?)Ústd)+rI   rJ   r“   r1   r”   r•   r–   Ú
num_tokensÚfeature_inforv   Úpatch_embedr   Ú	Parameterr*   Úrandnr‚   rƒ   Ú	pos_embedrs   ÚDropoutÚpos_dropr   rN   Úlenrd   r-   Údictr   ÚtransformersrC   rL   Únum_featuresÚhead_hidden_sizerG   Ú	head_droprp   rK   ÚheadÚ	head_distÚdistilled_trainingr   ÚapplyÚ_init_weights)r!   r|   r}   rh   r’   r“   r0   r1   r2   r”   r•   r–   r—   r˜   r™   rš   r›   rœ   r:   r;   rF   rG   r¯   ÚdprÚprev_dimrE   r3   rP   s                              €r#   rJ   z!PoolingVisionTransformer.__init__©   s}  ø€ õ, 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆØ˜jÐ(Ð(Ð(Ð(à"ˆŒØˆŒ
Ø˜a”L 5¨¤8Ñ+ˆ	Ø&ˆÔØ ˆŒØ&ˆÔØ(Ð/˜!˜!¨aˆŒØˆÔå(¨°9¸hÈ
ÐTZÐaÐaÐ^`ÐaÐaˆÔÝœ¥e¤k°!°YÀÔ@PÔ@WÐY]ÔYiÔYoÐ&vÐ&vÐsuÐ&vÐ&vÑwÔwˆŒÝœ¥e¤k°!°T´_ÀiÐ&VÐ&VÐSUÐ&VÐ&VÑWÔWˆŒÝœ
 ^Ð4Ñ4Ô4ˆŒàˆå'¨¸ÈÐNÑNÔNˆØˆÝ•s˜5‘z”zÑ"Ô"ð 	uð 	uˆAØˆDØ! !œ u¨Q¤xÑ/ˆIØ�1ŠuˆuÝØØðð ð ðð ð	ð �ð �[Ø˜!”Ø�a”Ø�a”Øð	
ð
 Ø(Ø(Ø" 1œvð
ð 
ð ð
ð 
ð 
ñ 
ˆLð !ˆHØÐÔ¥$¨xÀFÈQÁJÐRSÐUVÑRVÑCVÐ_rÐopÐ_rÐ_rÐ"sÑ"sÔ"sÐ!tÑtÐÔÐå+¨\Ð:ˆÔÝ”L ¨2¤°°r´Ñ!:ÐKÐKÀÐKÈÐKÐKˆŒ	ØENÐNˆÔÐN˜DÔ1°D´Nõ œ IÑ.Ô.ˆŒØDOÐRSÂOÀO•B”I˜dœn¨kÐ@Ð@¸RÐ@Ð@Ð@ÕY[ÔYdÑYfÔYfˆŒ	ØˆŒØð 	uØR]Ð`aÒRaÐRa�RœY t¤~°tÔ7GÐNÐNÈ2ÐNÐNÐNÕgiÔgrÑgtÔgtˆDŒNØ"'ˆÔå�d”n¨#Ð.Ñ.Ô.Ð.Ý�d”n¨#Ð.Ñ.Ô.Ð.Ø�
Š
�4Ô%Ñ&Ô&Ð&Ð&Ð&r%   c                 óÒ   — t          |t          j        ¦  «        rLt          j                             |j        d¦  «         t          j                             |j        d¦  «         d S d S )Nr   g      ð?)Ú
isinstancer   rC   ÚinitÚ	constant_r   Úweight)r!   Úms     r#   r·   z&PoolingVisionTransformer._init_weightsý   sZ   € Ý�a�œÑ&Ô&ð 	-ÝŒG×Ò˜aœf aÑ(Ô(Ð(ÝŒG×Ò˜aœh¨Ñ,Ô,Ð,Ð,Ð,ð	-ð 	-r%   c                 ó
   — ddhS )Nrª   rs   r    ©r!   s    r#   Úno_weight_decayz(PoolingVisionTransformer.no_weight_decay  s   € à˜[Ð)Ð)r%   Tc                 ó   — || _         d S r   )rµ   ©r!   Úenables     r#   Úset_distilled_trainingz/PoolingVisionTransformer.set_distilled_training  s   € à"(ˆÔÐÐr%   c                 ó   — |r
J d¦   «         ‚d S )Nz$gradient checkpointing not supportedr    rÄ   s     r#   Úset_grad_checkpointingz/PoolingVisionTransformer.set_grad_checkpointing
  s   € àÐAÐAÐAÑAÔAˆzÐAÐAr%   r   c                 ó:   — | j         �| j        | j         fS | j        S r   )r´   r³   rÁ   s    r#   Úget_classifierz'PoolingVisionTransformer.get_classifier  s!   € ØŒ>Ð%Ø”9˜dœnÐ,Ð,à”9Ðr%   c                 ó´  — t          | ¦  «        }|| _        |�|| _        |dk    rt          j        | j        |fi |¤Žnt          j        ¦   «         | _        | j                             | j	        ¦  «         | j
        �\|dk    rt          j        | j        | j        fi |¤Žnt          j        ¦   «         | _
        | j
                             | j	        ¦  «         d S d S )Nr   )r   r”   r–   r   rp   rG   rK   r³   ÚtrainÚtrainingr´   )r!   r”   r–   rF   s       r#   Úreset_classifierz)PoolingVisionTransformer.reset_classifier  sØ   € Ý˜dÑ#Ô#ˆØ&ˆÔØÐ"Ø*ˆDÔØDOÐRSÂOÀO•B”I˜dœn¨kÐ@Ð@¸RÐ@Ð@Ð@ÕY[ÔYdÑYfÔYfˆŒ	ØŒ	�Š˜œÑ&Ô&Ð&ØŒ>Ð%ØR]Ð`aÒRaÐRa�RœY t¤~°tÔ7GÐNÐNÈ2ÐNÐNÐNÕgiÔgrÑgtÔgtˆDŒNØŒN× Ò  ¤Ñ/Ô/Ð/Ð/Ð/ð &Ð%r%   ÚNCHWr   ÚindicesrL   Ú
stop_earlyÚ
output_fmtÚintermediates_onlyc                 ó‚  — |dv s
J d¦   «         ‚g }t          t          | j        ¦  «        |¦  «        \  }}	|                      |¦  «        }|                      || j        z   ¦  «        }| j                             |j        d         dd¦  «        }
t          | j        ¦  «        dz
  }t          j
                             ¦   «         s|s| j        }n| j        d|	dz   …         }t          |¦  «        D ].\  }} |||
f¦  «        \  }}
||v r|                     |¦  «         Œ/|r|S ||k    r|                      |
¦  «        }
|
|fS )aï   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )rÏ   zOutput shape must be NCHW.r   r£   r   N)r   r­   r¯   r§   r¬   rª   rs   ÚexpandrS   r*   ÚjitÚis_scriptingÚ	enumerateÚappendrL   )r!   r   rÐ   rL   rÑ   rÒ   rÓ   ÚintermediatesÚtake_indicesÚ	max_indexrX   Úlast_idxÚstagesÚfeat_idxÚstages                  r#   Úforward_intermediatesz.PoolingVisionTransformer.forward_intermediates  sf  € ð* ˜YÐ&Ð&Ð&Ð(DÑ&Ô&Ð&ØˆÝ"6µs¸4Ô;LÑ7MÔ7MÈwÑ"WÔ"WÑˆ�ið ×Ò˜QÑÔˆØ�MŠM˜!˜dœnÑ,Ñ-Ô-ˆØ”^×*Ò*¨1¬7°1¬:°r¸2Ñ>Ô>ˆ
å�tÔ(Ñ)Ô)¨AÑ-ˆÝŒ9×!Ò!Ñ#Ô#ð 	7¨:ð 	7ØÔ&ˆFˆFàÔ& ~¨	°A© ~Ô6ˆFå(¨Ñ0Ô0ð 	(ð 	(‰OˆH�eØ!˜E 1 j /Ñ2Ô2‰MˆAˆzØ˜<Ð'Ð'Ø×$Ò$ QÑ'Ô'Ð'øàð 	!Ø Ð à�xÒÐØŸš :Ñ.Ô.ˆJà˜=Ð(Ð(r%   r   Ú
prune_normÚ
prune_headc                 óâ   — t          t          | j        ¦  «        |¦  «        \  }}| j        d|dz   …         | _        |rt          j        ¦   «         | _        |r|                      dd¦  «         |S )z@ Prune layers not required for specified intermediates.
        Nr   r   Ú )r   r­   r¯   r   rK   rL   rÎ   )r!   rÐ   râ   rã   rÛ   rÜ   s         r#   Úprune_intermediate_layersz2PoolingVisionTransformer.prune_intermediate_layersP  st   € õ #7µs¸4Ô;LÑ7MÔ7MÈwÑ"WÔ"WÑˆ�iØ Ô-¨n¨y¸1©}¨nÔ=ˆÔØð 	&Ýœ™œˆDŒIØð 	)Ø×!Ò! ! RÑ(Ô(Ð(ØÐr%   c                 ó  — |                       |¦  «        }|                      || j        z   ¦  «        }| j                             |j        d         dd¦  «        }|                      ||f¦  «        \  }}|                      |¦  «        }|S )Nr   r£   )r§   r¬   rª   rs   rÕ   rS   r¯   rL   )r!   r   rX   s      r#   Úforward_featuresz)PoolingVisionTransformer.forward_features`  s~   € Ø×Ò˜QÑÔˆØ�MŠM˜!˜dœnÑ,Ñ-Ô-ˆØ”^×*Ò*¨1¬7°1¬:°r¸2Ñ>Ô>ˆ
Ø×)Ò)¨1¨j¨/Ñ:Ô:‰ˆˆ:Ø—Y’Y˜zÑ*Ô*ˆ
ØÐr%   Ú
pre_logitsc                 ó   — | j         �³| j        dk    sJ ‚|d d …df         |d d …df         }}|                      |¦  «        }|                      |¦  «        }|s*|                      |¦  «        }|                       |¦  «        }| j        r)| j        r"t          j                             ¦   «         s||fS ||z   dz  S | j        dk    r|d d …df         }|                      |¦  «        }|s|                      |¦  «        }|S )Nr‘   r   r   rR   )	r´   r–   r²   r³   rµ   rÍ   r*   rÖ   r×   )r!   r   ré   Úx_dists       r#   Úforward_headz%PoolingVisionTransformer.forward_headh  s  € ØŒ>Ð%ØÔ# wÒ.Ð.Ð.Ð.Ø˜!˜!˜!˜Q˜$œ  1 1 1 a 4¤ˆvˆAØ—’˜qÑ!Ô!ˆAØ—^’^ FÑ+Ô+ˆFØð 0Ø—I’I˜a‘L”L�ØŸš¨Ñ/Ô/�ØÔ&ð (¨4¬=ð (ÅÄ×AWÒAWÑAYÔAYð (à˜&�yÐ ð ˜F™
 aÑ'Ð'àÔ 7Ò*Ð*Ø�a�a�a˜�d”G�Ø—’˜qÑ!Ô!ˆAØð !Ø—I’I˜a‘L”L�ØˆHr%   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S r   )rè   rì   r†   s     r#   r$   z PoolingVisionTransformer.forward  s-   € Ø×!Ò! !Ñ$Ô$ˆØ×Ò˜aÑ Ô ˆØˆr%   )rw   rx   ry   rˆ   r‰   r‹   rŽ   r�   r�   r�   r‘   Fr.   r.   r.   r.   r.   NN)Tr   )NFFrÏ   F)r   FT©F) r&   r'   r(   r)   r^   rt   r   r_   ÚboolrJ   r·   r*   rÖ   ÚignorerÂ   rÆ   rÈ   r   r`   rÊ   r   rÎ   r+   r   r   r   rá   ræ   rè   rì   r$   ra   rb   s   @r#   r   r   £   sW  ø€ € € € € ðð ð  Ø ØØ&Ø'3Ø#,Ø#,Ø Ø#ØØ&Ø#Ø!Ø$&Ø$&Ø$&Ø$&ØØð)R'ð R'àðR'ð ðR'ð ð	R'ð
 ðR'ð   ”}ðR'ð ˜C”=ðR'ð ˜C”=ðR'ð ðR'ð ðR'ð ðR'ð ðR'ð ðR'ð ðR'ð "ðR'ð  "ð!R'ð" "ð#R'ð$ "ð%R'ð R'ð R'ð R'ð R'ð R'ðh-ð -ð -ð
 „YÔð*ð *ñ Ôð*ð „YÔð)ð )ð )ñ Ôð)ð „YÔðBð Bð Bñ ÔðBð ¤	ð ð ð ð ð	0ð 	0¨Cð 	0¸hÀs¼mð 	0ð 	0ð 	0ð 	0ð 8<ØØ$Ø$Ø',ð/)ð /)àŒ|ð/)ð ˜e C¨¨c¬ NÔ3Ô4ð/)ð ð	/)ð
 ð/)ð ð/)ð !%ð/)ð 
ˆt�E”LÔ! 5¨¬°t¸E¼LÔ7IÐ)IÔ#JÐJÔ	Kð/)ð /)ð /)ð /)ðf ./Ø$Ø#ð	ð à˜3  S¤	˜>Ô*ðð ðð ð	ð ð ð ð ð ð ðð ¨$ð ¸5¼<ð ð ð ð ð.ð ð ð ð ð ð r%   c                 óž   — i }t          j        d¦  «        }|                      ¦   «         D ]!\  }}|                     d„ |¦  «        }|||<   Œ"|S )z preprocess checkpoints zpools\.(\d)\.c                 óT   — dt          |                      d¦  «        ¦  «        dz   › d�S )Nr    r   z.pool.)r^   Úgroup)Úexps    r#   ú<lambda>z&checkpoint_filter_fn.<locals>.<lambda>Ž  s)   € Ð%RµS¸¿ºÀ1¹¼Ñ5FÔ5FÈÑ5JÐ%RÐ%RÐ%R€ r%   )ÚreÚcompileÚitemsÚsub)Ú
state_dictÚmodelÚout_dictÚp_blocksÚkÚvs         r#   Úcheckpoint_filter_fnr   …  s`   € à€HÝŒzÐ*Ñ+Ô+€HØ× Ò Ñ"Ô"ð ð ‰ˆˆ1ð
 �LŠLÐRÐRÐTUÑVÔVˆØˆ�‰ˆØ€Or%   Fc                 óÀ   — t          t          d¦  «        ¦  «        }|                     d|¦  «        }t          t          | |ft
          t          d|¬¦  «        dœ|¤Ž}|S )Nr�   Úout_indicesÚhook)Úfeature_clsr  )Úpretrained_filter_fnÚfeature_cfg)ÚtuplerN   Úpopr   r   r   r®   )ÚvariantÚ
pretrainedÚkwargsÚdefault_out_indicesr  rû   s         r#   Ú_create_pitr  “  sp   € Ý¥ a¡¤™/œ/ÐØ—*’*˜]Ð,?Ñ@Ô@€Kå Ý ØØðõ 2Ý V¸ÐEÑEÔEðð ð ðð €Eð €Lr%   rå   c                 ó8   — | ddd dddt           t          dddd	œ|¥S )
Nr�   )r�   rw   rw   gÍÌÌÌÌÌì?ÚbicubicTzpatch_embed.convr³   z
apache-2.0)Úurlr”   Ú
input_sizeÚ	pool_sizeÚcrop_pctÚinterpolationÚfixed_input_sizeÚmeanr¤   Ú
first_convÚ
classifierÚlicenser   )r  r  s     r#   Ú_cfgr  ¢  s:   € àØ¨=ÀtØ¨ÈÝ%Õ.BØ(¸Øðð ð ðð r%   ztimm/)Ú	hf_hub_id)r³   r´   )r  r  )zpit_ti_224.in1kzpit_xs_224.in1kzpit_s_224.in1kzpit_b_224.in1kzpit_ti_distilled_224.in1kzpit_xs_distilled_224.in1kzpit_s_distilled_224.in1kzpit_b_distilled_224.in1kr   c           	      ój   — t          ddg d¢g d¢g d¢d¬¦  «        }t          d| fi t          |fi |¤Ž¤ŽS )	Né   é   ©é@   r   r   ©r�   rŒ   r�   ©r�   ry   rx   r�   ©r}   rh   r“   r0   r1   r2   Ú	pit_b_224©r®   r  ©r
  r  Ú
model_argss      r#   r$  r$  Ã  ó^   € åØØØ�,�,ØˆiˆiØˆjˆjØðñ ô €Jõ �{ JÐMÐMµ$°zÐ2LÐ2LÀVÐ2LÐ2LÐMÐMÐMr%   c           	      ój   — t          ddg d¢g d¢g d¢d¬¦  «        }t          d| fi t          |fi |¤Ž¤ŽS )	Nrx   ry   r‰   r‹   ©r�   rŒ   é   r�   r#  Ú	pit_s_224r%  r&  s      r#   r,  r,  Ð  r(  r%   c           	      ój   — t          ddg d¢g d¢g d¢d¬¦  «        }t          d| fi t          |fi |¤Ž¤ŽS )	Nrx   ry   r‰   r‹   rŽ   r�   r#  Ú
pit_xs_224r%  r&  s      r#   r.  r.  Ý  ó^   € åØØØ�,�,ØˆiˆiØˆiˆiØðñ ô €Jõ �| ZÐNÐNµ4¸
Ð3MÐ3MÀfÐ3MÐ3MÐNÐNÐNr%   c           	      ój   — t          ddg d¢g d¢g d¢d¬¦  «        }t          d| fi t          |fi |¤Ž¤ŽS )	Nrx   ry   ©é    r2  r2  r‹   rŽ   r�   r#  Ú
pit_ti_224r%  r&  s      r#   r3  r3  ê  r/  r%   c           	      ól   — t          ddg d¢g d¢g d¢dd¬¦  «        }t          d	| fi t          |fi |¤Ž¤ŽS )
Nr  r  r  r!  r"  r�   T©r}   rh   r“   r0   r1   r2   r—   Úpit_b_distilled_224r%  r&  s      r#   r6  r6  ÷  ób   € åØØØ�,�,ØˆiˆiØˆjˆjØØðñ ô €Jõ Ð,¨jÐWÐW½DÀÐ<VÐ<VÈvÐ<VÐ<VÐWÐWÐWr%   c           	      ól   — t          ddg d¢g d¢g d¢dd¬¦  «        }t          d	| fi t          |fi |¤Ž¤ŽS )
Nrx   ry   r‰   r‹   r*  r�   Tr5  Úpit_s_distilled_224r%  r&  s      r#   r9  r9    r7  r%   c           	      ól   — t          ddg d¢g d¢g d¢dd¬¦  «        }t          d	| fi t          |fi |¤Ž¤ŽS )
Nrx   ry   r‰   r‹   rŽ   r�   Tr5  Úpit_xs_distilled_224r%  r&  s      r#   r;  r;    óc   € åØØØ�,�,ØˆiˆiØˆiˆiØØðñ ô €Jõ Ð-¨zÐXÐX½TÀ*Ð=WÐ=WÐPVÐ=WÐ=WÐXÐXÐXr%   c           	      ól   — t          ddg d¢g d¢g d¢dd¬¦  «        }t          d	| fi t          |fi |¤Ž¤ŽS )
Nrx   ry   r1  r‹   rŽ   r�   Tr5  Úpit_ti_distilled_224r%  r&  s      r#   r>  r>  !  r<  r%   rî   )rå   )4r)   r€   rö   Ú	functoolsr   Útypingr   r   r   r   r   r	   r
   r*   r   Ú	timm.datar   r   Útimm.layersr   r   r   r   Ú_builderr   Ú	_featuresr   Ú	_registryr   r   Úvision_transformerr   Ú__all__rM   r   r`   r-   rd   rv   r   r   r  r  Údefault_cfgsr$  r,  r.  r3  r6  r9  r;  r>  r    r%   r#   ú<module>rI     s¨  ððð ð €€€Ø 	€	€	€	Ø Ð Ð Ð Ð Ð Ø DÐ DÐ DÐ DÐ DÐ DÐ DÐ DÐ DÐ DÐ DÐ DÐ DÐ DÐ DÐ DÐ DÐ Dà €€€Ø Ð Ð Ð Ð Ð à AÐ AÐ AÐ AÐ AÐ AÐ AÐ AØ ]Ð ]Ð ]Ð ]Ð ]Ð ]Ð ]Ð ]Ð ]Ð ]Ð ]Ð ]Ø *Ð *Ð *Ð *Ð *Ð *Ø +Ð +Ð +Ð +Ð +Ð +Ø <Ð <Ð <Ð <Ð <Ð <Ð <Ð <Ø %Ð %Ð %Ð %Ð %Ð %ð &Ð
&€ðð ð ð ð �b”mñ ô ð ð4ð 4ð 4ð 4ð 4�"”)ñ 4ô 4ð 4ðnð ð ð ð ˆbŒiñ ô ð ð>!ð !ð !ð !ð !�B”Iñ !ô !ð !ðH_ð _ð _ð _ð _˜rœyñ _ô _ð _ðDð ð ðð ð ð ð	ð 	ð 	ð 	ð %Ð$à�t gÐ.Ñ.Ô.Ø�t gÐ.Ñ.Ô.Ø�d WÐ-Ñ-Ô-Ø�d WÐ-Ñ-Ô-Ø!% ØØ(ð"*ñ "*ô "*ð "& ØØ(ð"*ñ "*ô "*ð !% ØØ(ð!*ñ !*ô !*ð !% ØØ(ð!*ñ !*ô !*ð&ð &ñ ô €ð* ð	Nð 	NÐ-Eð 	Nð 	Nð 	Nñ „ð	Nð ð	Nð 	NÐ-Eð 	Nð 	Nð 	Nñ „ð	Nð ð	Oð 	OÐ.Fð 	Oð 	Oð 	Oñ „ð	Oð ð	Oð 	OÐ.Fð 	Oð 	Oð 	Oñ „ð	Oð ð
Xð 
XÐ7Oð 
Xð 
Xð 
Xñ „ð
Xð ð
Xð 
XÐ7Oð 
Xð 
Xð 
Xñ „ð
Xð ð
Yð 
YÐ8Pð 
Yð 
Yð 
Yñ „ð
Yð ð
Yð 
YÐ8Pð 
Yð 
Yð 
Yñ „ð
Yð 
Yð 
Yr%   