§
    ·¯¸j0w  ã                   ó¬  — d Z ddlZddlZddlmZ ddlmZmZmZm	Z	m
Z
 	 ddlmZ n# e$ r	 ddlmZ Y nw xY wddlZddlmZ ddlmZ ddlmZmZ ddlmZmZmZmZmZmZmZmZmZm Z  d	d
l!m"Z" d	dl#m$Z$ d	dl%m&Z&m'Z' d	dl(m)Z)m*Z* d	dl+m,Z, dgZ- ej.        e/¦  «        Z0 G d„ dej1        ¦  «        Z2 G d„ dej1        ¦  «        Z3 G d„ dej1        ¦  «        Z4 G d„ dej1        ¦  «        Z5d9d„Z6d:d„Z7 e) e7ddd¬¦  «         e7dd¬ ¦  «         e7d!d¬"¦  «         e7d#d¬"¦  «         e7d$d¬"¦  «         e7d%d¬"¦  «         e7d&d¬"¦  «         e7d'd¬"¦  «         e7¦   «          e7d(d¬"¦  «         e7¦   «          e7d)d¬"¦  «         e7¦   «         d*œ¦  «        Z8e*d9d+e5fd,„¦   «         Z9e*d9d+e5fd-„¦   «         Z:e*d9d+e5fd.„¦   «         Z;e*d9d+e5fd/„¦   «         Z<e*d9d+e5fd0„¦   «         Z=e*d9d+e5fd1„¦   «         Z>e*d9d+e5fd2„¦   «         Z?e*d9d+e5fd3„¦   «         Z@e*d9d+e5fd4„¦   «         ZAe*d9d+e5fd5„¦   «         ZBe*d9d+e5fd6„¦   «         ZCe*d9d+e5fd7„¦   «         ZDe*d9d+e5fd8„¦   «         ZEdS );z¦ Relative Position Vision Transformer (ViT) in PyTorch

NOTE: these models are experimental / WIP, expect changes

Hacked together by / Copyright 2022, Ross Wightman
é    N)Úpartial)ÚListÚOptionalÚTupleÚTypeÚUnion)ÚLiteral)ÚFinal©ÚIMAGENET_INCEPTION_MEANÚIMAGENET_INCEPTION_STD)
Úget_device_dtypeÚ
PatchEmbedÚMlpÚ
LayerScaleÚDropPathÚcalculate_drop_path_ratesÚ	RelPosMlpÚ
RelPosBiasÚuse_fused_attnÚ	LayerTypeé   )Úbuild_model_with_cfg)Úfeature_take_indices)Únamed_applyÚ
checkpoint)Úgenerate_default_cfgsÚregister_model)Úget_init_weights_vitÚVisionTransformerRelPosc                   óÒ   ‡ — e Zd ZU ee         ed<   ddddddej        ddf	dededed	ed
e	e
ej                          dedede
ej                 fˆ fd„Zdde	ej                 fd„Zˆ xZS )ÚRelPosAttentionÚ
fused_attné   FNç        ÚdimÚ	num_headsÚqkv_biasÚqk_normÚrel_pos_clsÚ	attn_dropÚ	proj_dropÚ
norm_layerc                 ór  •— |	|
dœ}t          ¦   «                              ¦   «          ||z  dk    s
J d¦   «         ‚|| _        ||z  | _        | j        dz  | _        t          ¦   «         | _        t          j        ||dz  fd|i|¤Ž| _	        |r || j        fi |¤Žnt          j
        ¦   «         | _        |r || j        fi |¤Žnt          j
        ¦   «         | _        |r
 |dd|i|¤Žnd | _        t          j        |¦  «        | _        t          j        ||fi |¤Ž| _        t          j        |¦  «        | _        d S )	N©ÚdeviceÚdtyper   z$dim should be divisible by num_headsg      à¿é   Úbiasr'   © )ÚsuperÚ__init__r'   Úhead_dimÚscaler   r#   ÚnnÚLinearÚqkvÚIdentityÚq_normÚk_normÚrel_posÚDropoutr+   Úprojr,   )Úselfr&   r'   r(   r)   r*   r+   r,   r-   r0   r1   ÚddÚ	__class__s               €úu/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/models/vision_transformer_relpos.pyr6   zRelPosAttention.__init__0   sV  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ�Y‰ !Ò#Ð#Ð#Ð%KÑ#Ô#Ð#Ø"ˆŒØ˜yÑ(ˆŒØ”] dÑ*ˆŒ
Ý(Ñ*Ô*ˆŒå”9˜S #¨¡'Ð?Ð?°Ð?¸BÐ?Ð?ˆŒØ9@ÐS�j�j ¤Ð5Ð5°"Ð5Ð5Ð5ÅbÄkÁmÄmˆŒØ9@ÐS�j�j ¤Ð5Ð5°"Ð5Ð5Ð5ÅbÄkÁmÄmˆŒØALÐV�{�{Ð=Ð=¨YÐ=¸"Ð=Ð=Ð=ÐRVˆŒÝœ IÑ.Ô.ˆŒÝ”I˜c 3Ð-Ð-¨"Ð-Ð-ˆŒ	Ýœ IÑ.Ô.ˆŒˆˆó    Úshared_rel_posc                 ó°  — |j         \  }}}|                      |¦  «                             ||d| j        | j        ¦  «                             ddddd¦  «        }|                     d¦  «        \  }}}	|                      |¦  «        }|                      |¦  «        }| j	        re| j
        �| j
                             ¦   «         }
n|�|}
nd }
t          j        j                             |||	|
| j        r| j        j        nd¬¦  «        }ny|| j        z  }||                     dd	¦  «        z  }| j
        �|  
                    ||¬
¦  «        }n|�||z   }|                     d	¬¦  «        }|                      |¦  «        }||	z  }|                     dd¦  «                             |||¦  «        }|                      |¦  «        }|                      |¦  «        }|S )Nr2   é   r   r   é   r%   )Ú	attn_maskÚ	dropout_péþÿÿÿéÿÿÿÿ©rG   ©r&   )Úshaper;   Úreshaper'   r7   ÚpermuteÚunbindr=   r>   r#   r?   Úget_biasÚtorchr9   Ú
functionalÚscaled_dot_product_attentionÚtrainingr+   Úpr8   Ú	transposeÚsoftmaxrA   r,   )rB   ÚxrG   ÚBÚNÚCr;   ÚqÚkÚvÚ	attn_biasÚattns               rE   ÚforwardzRelPosAttention.forwardM   sÊ  € Ø”'‰ˆˆ1ˆaØ�hŠh�q‰kŒk×!Ò! ! Q¨¨4¬>¸4¼=ÑIÔI×QÒQÐRSÐUVÐXYÐ[\Ð^_Ñ`Ô`ˆØ—*’*˜Q‘-”-‰ˆˆ1ˆaØ�KŠK˜‰NŒNˆØ�KŠK˜‰NŒNˆàŒ?ð 	ØŒ|Ð'Ø œL×1Ò1Ñ3Ô3�	�	ØÐ+Ø*�	�	à �	å”Ô#×@Ò@Ø�1�aØ#Ø.2¬mÐC˜$œ.Ô*Ð*Àð Añ ô ˆAˆAð �D”J‘ˆAØ�q—{’{ 2 rÑ*Ô*Ñ*ˆDØŒ|Ð'Ø—|’| D¸�|ÑHÔH��ØÐ+Ø˜nÑ,�Ø—<’< B�<Ñ'Ô'ˆDØ—>’> $Ñ'Ô'ˆDØ�q‘ˆAà�KŠK˜˜1ÑÔ×%Ò% a¨¨AÑ.Ô.ˆØ�IŠI�a‰LŒLˆØ�NŠN˜1ÑÔˆØˆrF   ©N)Ú__name__Ú
__module__Ú__qualname__r
   ÚboolÚ__annotations__r9   Ú	LayerNormÚintr   r   ÚModuleÚfloatr6   rV   ÚTensorrf   Ú__classcell__©rD   s   @rE   r"   r"   -   sù   ø€ € € € € € Ø�d”ÐÐÑð
 Ø"Ø!Ø59Ø!Ø!Ø*,¬,ØØð/ð /àð/ð ð/ð ð	/ð
 ð/ð " $ r¤y¤/Ô2ð/ð ð/ð ð/ð ˜RœYœð/ð /ð /ð /ð /ð /ð:"ð "¨°%´,Ô)?ð "ð "ð "ð "ð "ð "ð "ð "rF   r"   c                   óü   ‡ — e Zd Zddddddddej        ej        ddfdedededed	ed
e	e
ej                          de	e         dededede
ej                 de
ej                 fˆ fd„Zdde	ej                 fd„Zˆ xZS )ÚRelPosBlockç      @FNr%   r&   r'   Ú	mlp_ratior(   r)   r*   Úinit_valuesr,   r+   Ú	drop_pathÚ	act_layerr-   c           
      ób  •— ||dœ}t          ¦   «                              ¦   «           ||fi |¤Ž| _        t          ||f||||	||dœ|¤Ž| _        |rt          |fd|i|¤Žnt          j        ¦   «         | _        |
dk    rt          |
¦  «        nt          j        ¦   «         | _
         ||fi |¤Ž| _        t          d|t          ||z  ¦  «        ||dœ|¤Ž| _        |rt          |fd|i|¤Žnt          j        ¦   «         | _        |
dk    rt          |
¦  «        nt          j        ¦   «         | _        d S )Nr/   ©r(   r)   r*   r+   r,   r-   rx   r%   ©Úin_featuresÚhidden_featuresrz   Údropr4   )r5   r6   Únorm1r"   re   r   r9   r<   Úls1r   Ú
drop_path1Únorm2r   rn   ÚmlpÚls2Ú
drop_path2©rB   r&   r'   rw   r(   r)   r*   rx   r,   r+   ry   rz   r-   r0   r1   rC   rD   s                   €rE   r6   zRelPosBlock.__init__t   s†  ø€ ð" ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ�Z Ð*Ð* rÐ*Ð*ˆŒ
Ý#ØØð

ð ØØ#ØØØ!ð

ð 

ð ð

ð 

ˆŒ	ð FQÐc•:˜cÐAÐA¨{ÐA¸bÐAÐAÐAÕVXÔVaÑVcÔVcˆŒà1:¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒà�Z Ð*Ð* rÐ*Ð*ˆŒ
Ýð 
ØÝ  i¡Ñ0Ô0ØØð	
ð 
ð
 ð
ð 
ˆŒð FQÐc•:˜cÐAÐA¨{ÐA¸bÐAÐAÐAÕVXÔVaÑVcÔVcˆŒØ1:¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒˆˆrF   rG   c           
      óN  — ||                       |                      |                      |                      |¦  «        |¬¦  «        ¦  «        ¦  «        z   }||                      |                      |                      |                      |¦  «        ¦  «        ¦  «        ¦  «        z   }|S ©NrO   )rƒ   r‚   re   r�   r‡   r†   r…   r„   ©rB   r]   rG   s      rE   rf   zRelPosBlock.forward¢   s|   € Ø�—’ §¢¨¯ª°4·:²:¸a±=´=ÐQ_¨Ñ)`Ô)`Ñ aÔ aÑbÔbÑbˆØ�—’ §¢¨¯ª°$·*²*¸Q±-´-Ñ)@Ô)@Ñ AÔ AÑBÔBÑBˆØˆrF   rg   )rh   ri   rj   r9   ÚGELUrm   rn   rp   rk   r   r   ro   r6   rV   rq   rf   rr   rs   s   @rE   ru   ru   r   s8  ø€ € € € € ð  "Ø"Ø!Ø59Ø+/Ø!Ø!Ø!Ø)+¬Ø*,¬,ØØð,Sð ,Sàð,Sð ð,Sð ð	,Sð
 ð,Sð ð,Sð " $ r¤y¤/Ô2ð,Sð " %œð,Sð ð,Sð ð,Sð ð,Sð ˜BœI”ð,Sð ˜RœYœð,Sð ,Sð ,Sð ,Sð ,Sð ,Sð\ð ¨°%´,Ô)?ð ð ð ð ð ð ð ð rF   ru   c                   ó  ‡ — e Zd Zddddddddej        ej        ddfdedededed	ed
e	e
ej                          de	e         dededede
ej                 de
ej                 fˆ fd„Zd„ Zdde	ej                 fd„Zˆ xZS )ÚResPostRelPosBlockrv   FNr%   r&   r'   rw   r(   r)   r*   rx   r,   r+   ry   rz   r-   c           
      óô  •— ||dœ}t          ¦   «                              ¦   «          || _        t          ||f||||	||dœ|¤Ž| _         ||fi |¤Ž| _        |
dk    rt          |
¦  «        nt          j        ¦   «         | _	        t          d|t          ||z  ¦  «        ||dœ|¤Ž| _         ||fi |¤Ž| _        |
dk    rt          |
¦  «        nt          j        ¦   «         | _        |                      ¦   «          d S )Nr/   r|   r%   r}   r4   )r5   r6   rx   r"   re   r�   r   r9   r<   rƒ   r   rn   r…   r„   r‡   Úinit_weightsrˆ   s                   €rE   r6   zResPostRelPosBlock.__init__ª   s>  ø€ ð" ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ&ˆÔå#ØØð

ð ØØ#ØØØ!ð

ð 

ð ð

ð 

ˆŒ	ð  �Z Ð*Ð* rÐ*Ð*ˆŒ
Ø1:¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒåð 
ØÝ  i¡Ñ0Ô0ØØð	
ð 
ð
 ð
ð 
ˆŒð  �Z Ð*Ð* rÐ*Ð*ˆŒ
Ø1:¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒà×ÒÑÔÐÐÐrF   c                 óÔ   — | j         �`t          j                             | j        j        | j         ¦  «         t          j                             | j        j        | j         ¦  «         d S d S rg   )rx   r9   ÚinitÚ	constant_r�   Úweightr„   ©rB   s    rE   r�   zResPostRelPosBlock.init_weightsÙ   sY   € àÔÐ'ÝŒG×Ò˜dœjÔ/°Ô1AÑBÔBÐBÝŒG×Ò˜dœjÔ/°Ô1AÑBÔBÐBÐBÐBð (Ð'rF   rG   c           	      ó  — ||                       |                      |                      ||¬¦  «        ¦  «        ¦  «        z   }||                      |                      |                      |¦  «        ¦  «        ¦  «        z   }|S rŠ   )rƒ   r�   re   r‡   r„   r…   r‹   s      rE   rf   zResPostRelPosBlock.forwardß   se   € Ø�—’ §
¢
¨4¯9ª9°QÀ~¨9Ñ+VÔ+VÑ WÔ WÑXÔXÑXˆØ�—’ §
¢
¨4¯8ª8°A©;¬;Ñ 7Ô 7Ñ8Ô8Ñ8ˆØˆrF   rg   )rh   ri   rj   r9   rŒ   rm   rn   rp   rk   r   r   ro   r6   r�   rV   rq   rf   rr   rs   s   @rE   rŽ   rŽ   ¨   s7  ø€ € € € € ð  "Ø"Ø!Ø59Ø+/Ø!Ø!Ø!Ø)+¬Ø*,¬,ØØð-ð -àð-ð ð-ð ð	-ð
 ð-ð ð-ð " $ r¤y¤/Ô2ð-ð " %œð-ð ð-ð ð-ð ð-ð ˜BœI”ð-ð ˜RœYœð-ð -ð -ð -ð -ð -ð^Cð Cð Cðð ¨°%´,Ô)?ð ð ð ð ð ð ð ð rF   rŽ   c            8       óˆ  ‡ — e Zd ZdZddddddddd	d
dddddddddddddeddeddfdeeeeef         f         deeeeef         f         dedede	d         dededede
dededee
         deded ed!ee         d"ed#e
d$e
d%e
d&e
d'e	d(         d)ed*eej                 d+ee         d,ee         d-eej                 f6ˆ fd.„ZdKd0„ZdLd2ed3ed/dfd4„Zej        j        d5„ ¦   «         Zej        j        dMd6„¦   «         Zej        j        dNd7„¦   «         Zej        j        d/ej        fd8„¦   «         ZdOdedee         fd9„Z	 	 	 	 	 	 dPd;ej        d<eeeee         f                  d=ed>ed?ed@edAed/eeej                 eej        eej                 f         f         fdB„Z	 	 	 dQd<eeee         f         dDedEefdF„Z dG„ Z!dMdHefdI„Z"dJ„ Z#ˆ xZ$S )Rr    ah   Vision Transformer w/ Relative Position Bias

    Differing from classic vit, this impl
      * uses relative position index (swin v1 / beit) or relative log coord + mlp (swin v2) pos embed
      * defaults to no class token (can be enabled)
      * defaults to global avg pool for head (can be changed)
      * layer-scale (residual branch gain) enabled
    éà   é   r2   éè  Úavgé   é   rv   TFç�íµ ÷Æ°>r…   Nr%   ÚresetÚimg_sizeÚ
patch_sizeÚin_chansÚnum_classesÚglobal_pool)Ú r›   ÚtokenÚmapÚ	embed_dimÚdepthr'   rw   r(   r)   rx   Úclass_tokenÚfc_normÚrel_pos_typeÚrel_pos_dimrG   Ú	drop_rateÚproj_drop_rateÚattn_drop_rateÚdrop_path_rateÚweight_init)ÚskiprŸ   ÚjaxÚmocor¥   Úfix_initÚembed_layerr-   rz   Úblock_fnc                 óÈ  •‡‡‡	‡
‡‡‡‡‡‡‡‡ ‡!‡"‡#— t          ¦   «                              ¦   «          ||dœŠ |dv sJ ‚|s|dk    sJ ‚‰pt          t          j        d¬¦  «        Š‰pt          j        Š|| _        || _        || _        ‰x| _	        x| _
        | _        |rdnd| _        d| _         |d|||‰d	œ‰ ¤Ž| _        | j        j        }t!          | j        d
¦  «        r| j                             ¦   «         n|Š"t%          || j        ¬¦  «        }|                     d¦  «        r#|r||d<   d|v rd|d<   t          t(          fi |¤ŽŠ#nt          t*          fi |¤ŽŠ#d| _        |r ‰#dd‰i‰ ¤Ž| _        dŠ#|r+t          j        t1          j        d| j        ‰fi ‰ ¤Ž¦  «        nd| _        t7          ||¦  «        Š!t          j        ˆˆˆˆ ˆ!ˆˆˆ	ˆˆˆˆˆ
ˆ#fd„t;          |¦  «        D ¦   «         ¦  «        | _        ˆˆ"fd„t;          |¦  «        D ¦   «         | _        |s	 ‰‰fi ‰ ¤Žnt          j         ¦   «         | _!        |r	 ‰‰fi ‰ ¤Žnt          j         ¦   «         | _"        t          j#        |¦  «        | _$        |dk    rt          j%        | j        |fi ‰ ¤Žnt          j         ¦   «         | _&        |dk    rdn|| _'        || _(        |dk    r|  )                    d¬¦  «         dS dS )aE  
        Args:
            img_size: input image size
            patch_size: patch size
            in_chans: number of input channels
            num_classes: number of classes for classification head
            global_pool: type of global pooling for final sequence (default: 'avg')
            embed_dim: embedding dimension
            depth: depth of transformer
            num_heads: number of attention heads
            mlp_ratio: ratio of mlp hidden dim to embedding dim
            qkv_bias: enable bias for qkv if True
            qk_norm: Enable normalization of query and key in attention
            init_values: layer-scale init values
            class_token: use class token (default: False)
            fc_norm: use pre classifier norm instead of pre-pool
            rel_pos_type: type of relative position
            shared_rel_pos: share relative pos across all blocks
            drop_rate: dropout rate
            proj_drop_rate: projection dropout rate
            attn_drop_rate: attention dropout rate
            drop_path_rate: stochastic depth rate
            weight_init: weight init scheme
            fix_init: apply weight initialization fix (scaling w/ layer index)
            embed_layer: patch embedding layer
            norm_layer: normalization layer
            act_layer: MLP activation layer
        r/   ©r¥   r›   r¦   r¦   rž   )Úepsr   r   F)r    r¡   r¢   r¨   Ú
feat_ratio)Úwindow_sizeÚprefix_tokensr…   Ú
hidden_dimÚswinÚmodeNr'   c                 óF   •— g | ]} ‰d‰‰‰	‰‰‰‰‰‰‰|         ‰
‰d œ‰¤Ž‘ŒS ))r&   r'   rw   r(   r)   r*   rx   r,   r+   ry   r-   rz   r4   r4   )Ú.0Úirz   r°   r¸   rC   Údprr¨   rx   rw   r-   r'   r¯   r)   r(   r*   s     €€€€€€€€€€€€€€rE   ú
<listcomp>z4VisionTransformerRelPos.__init__.<locals>.<listcomp>U  ss   ø€ ð %#ð %#ð %#ð  ð ˆHð ØØ#Ø#Ø!ØØ'Ø'Ø(Ø(Ø˜aœ&Ø%Ø#ðð ð ðð ð%#ð %#ð %#rF   c                 ó:   •— g | ]}t          d |› �‰‰¬¦  «        ‘ŒS )zblocks.)ÚmoduleÚnum_chsÚ	reduction©Údict)rÃ   rÄ   r¨   Úrs     €€rE   rÆ   z4VisionTransformerRelPos.__init__.<locals>.<listcomp>f  sA   ø€ ð ^ð ^ð ^ØKL�D˜ !˜˜¨yÀAÐFÑFÔFð^ð ^ð ^rF   r³   rŸ   ©Úneeds_resetr4   )*r5   r6   r   r9   rm   rŒ   r£   r¢   r¤   Únum_featuresÚhead_hidden_sizer¨   Únum_prefix_tokensÚgrad_checkpointingÚpatch_embedÚ	grid_sizeÚhasattrr¼   rÌ   Ú
startswithr   r   rG   Ú	ParameterrV   ÚzerosÚ	cls_tokenr   Ú
ModuleListÚrangeÚblocksÚfeature_infor<   Únormr«   r@   Ú	head_dropr:   ÚheadÚweight_init_moder¶   r�   )%rB   r    r¡   r¢   r£   r¤   r¨   r©   r'   rw   r(   r)   rx   rª   r«   r¬   r­   rG   r®   r¯   r°   r±   r²   r¶   r·   r-   rz   r¸   r0   r1   Ú	feat_sizeÚrel_pos_argsrC   rÅ   rÍ   r*   rD   s%         ` `````      ``    ```    @@@@€rE   r6   z VisionTransformerRelPos.__init__ï   sØ  øøøøøøøøøøøøøøøø€ õx 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆØÐ2Ð2Ð2Ð2Ð2ØÐ4˜k¨WÒ4Ð4Ð4Ð4ØÐB¥7­2¬<¸TÐ#BÑ#BÔ#Bˆ
ØÐ(¥¤ˆ	à&ˆÔØ ˆŒØ&ˆÔØENÐNˆÔÐN˜DÔ1°D´NØ&1Ð!8  °qˆÔØ"'ˆÔà&˜;ð 
ØØ!ØØð	
ð 
ð
 ð
ð 
ˆÔð Ô$Ô.ˆ	Ý-4°TÔ5EÀ|Ñ-TÔ-TÐdˆDÔ×'Ò'Ñ)Ô)Ð)ÐZdˆå¨	ÀÔAWÐXÑXÔXˆØ×"Ò" 5Ñ)Ô)ð 	>Øð 9Ø-8�˜\Ñ*Ø˜Ð%Ð%Ø'-�˜VÑ$Ý!¥)Ð<Ð<¨|Ð<Ð<ˆKˆKå!¥*Ð=Ð=°Ð=Ð=ˆKØ"ˆÔØð 	Ø"- +Ð"HÐ"H¸	Ð"HÀRÐ"HÐ"HˆDÔàˆKàbmÐw�œ¥e¤k°!°TÔ5KÈYÐ&]Ð&]ÐZ\Ð&]Ð&]Ñ^Ô^Ð^ÐswˆŒå'¨¸Ñ>Ô>ˆÝ”mð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#õ  ˜5‘\”\ð!%#ñ %#ô %#ñ $ô $ˆŒð"^ð ^ð ^ð ^ð ^ÝPUÐV[ÑP\ÔP\ð^ñ ^ô ^ˆÔà7>ÐQ�J�J˜yÐ/Ð/¨BÐ/Ð/Ð/ÅBÄKÁMÄMˆŒ	ð 7>ÐP�z�z )Ð2Ð2¨rÐ2Ð2Ð2Å2Ä;Á=Ä=ˆŒÝœ IÑ.Ô.ˆŒØDOÐRSÂOÀO•B”I˜dœn¨kÐ@Ð@¸RÐ@Ð@Ð@ÕY[ÔYdÑYfÔYfˆŒ	à+6¸&Ò+@Ð+@  ÀkˆÔØ ˆŒà˜&Ò Ð Ø×Ò¨%ÐÑ0Ô0Ð0Ð0Ð0ð !Ð rF   Úreturnc                 óX  — t          j        ¦   «         5  t          | j        ¦  «        D ]g\  }}t	          j        d|dz   z  ¦  «        }|j        j        j         	                    |¦  «         |j
        j        j         	                    |¦  «         Œh	 ddd¦  «         dS # 1 swxY w Y   dS )z9Apply weight initialization fix (scaling w/ layer index).g       @r   N)rV   Úno_gradÚ	enumeraterÝ   ÚmathÚsqrtre   rA   r”   Údiv_r…   Úfc2)rB   Úlayer_idÚlayerr8   s       rE   Úfix_init_weightz'VisionTransformerRelPos.fix_init_weightu  så   € åŒ]‰_Œ_ð 	1ð 	1Ý#,¨T¬[Ñ#9Ô#9ð 1ð 1‘�˜%Ýœ	 #¨°A©Ñ"6Ñ7Ô7�Ø”
”Ô&×+Ò+¨EÑ2Ô2Ð2Ø”	”Ô$×)Ò)¨%Ñ0Ô0Ð0Ð0ð1ð	1ð 	1ð 	1ñ 	1ô 	1ð 	1ð 	1ð 	1ð 	1ð 	1ð 	1ð 	1øøøð 	1ð 	1ð 	1ð 	1ð 	1ð 	1s   ”A=BÂB#Â&B#r¥   rÁ   rÏ   c                 ó8  — |p| j         }|dv sJ ‚d|v rt          j        | j        ¦  «         nd}| j        �&t
          j                             | j        d¬¦  «         t          t          |||¬¦  «        | ¦  «         | j
        r|                      ¦   «          dS dS )aP  Initialize model weights.

        Args:
            mode: Weight initialization mode ('jax', 'jax_nlhb', 'moco', or '').
            needs_reset: If True, call reset_parameters() on modules (default for after to_empty()).
                If False, skip reset_parameters() (for __init__ where modules already self-initialized).
        )r´   Újax_nlhbrµ   rŸ   r¥   Únlhbr%   Nrž   )ÚstdrÎ   )râ   ré   Úlogr£   rÚ   r9   r’   Únormal_r   r   r¶   rï   )rB   rÁ   rÏ   Ú	head_biass       rE   r�   z$VisionTransformerRelPos.init_weights}  s´   € ð Ð,�tÔ,ˆØÐ?Ð?Ð?Ð?Ð?Ø39¸T°>°>•T”X˜dÔ.Ñ/Ô/Ð/Ð/Àrˆ	ØŒ>Ð%ÝŒG�OŠO˜DœN°ˆOÑ5Ô5Ð5åÕ(¨¨yÀkÐRÑRÔRÐTXÑYÔYÐYàŒ=ð 	#Ø× Ò Ñ"Ô"Ð"Ð"Ð"ð	#ð 	#rF   c                 ó   — dhS )NrÚ   r4   r•   s    rE   Úno_weight_decayz'VisionTransformerRelPos.no_weight_decay�  s
   € àˆ}ÐrF   c                 ó(   — t          dddg¬¦  «        S )Nz^cls_token|patch_embed)z^blocks\.(\d+)N)z^norm)iŸ† )ÚstemrÝ   rË   )rB   Úcoarses     rE   Úgroup_matcherz%VisionTransformerRelPos.group_matcher”  s%   € åØ*Ø-Ð/CÐDð
ñ 
ô 
ð 	
rF   c                 ó   — || _         d S rg   )rÓ   )rB   Úenables     rE   Úset_grad_checkpointingz.VisionTransformerRelPos.set_grad_checkpointing›  s   € à"(ˆÔÐÐrF   c                 ó   — | j         S rg   )rá   r•   s    rE   Úget_classifierz&VisionTransformerRelPos.get_classifierŸ  s
   € àŒyÐrF   c                 ó   — t          | ||¬¦  «        }|| _        |�|dv sJ ‚|| _        |dk    rt          j        | j        |fi |¤Žnt          j        ¦   «         | _        | j                             | j	        ¦  «         d S )Nr/   rº   r   )
r   r£   r¤   r9   r:   r¨   r<   rá   ÚtrainrY   )rB   r£   r¤   r0   r1   rC   s         rE   Úreset_classifierz(VisionTransformerRelPos.reset_classifier£  s�   € Ý˜d¨6¸Ð?Ñ?Ô?ˆØ&ˆÔØÐ"ØÐ"6Ð6Ð6Ð6Ð6Ø*ˆDÔØDOÐRSÂOÀO•B”I˜dœn¨kÐ@Ð@¸RÐ@Ð@Ð@ÕY[ÔYdÑYfÔYfˆŒ	ØŒ	�Š˜œÑ&Ô&Ð&Ð&Ð&rF   ÚNCHWr]   ÚindicesÚreturn_prefix_tokensrß   Ú
stop_earlyÚ
output_fmtÚintermediates_onlyc                 ór  ‡ ‡‡‡— |dv s
J d¦   «         ‚|dk    }g }	t          t          ‰ j        ¦  «        |¦  «        \  }
}|j        \  Š}}}‰                      |¦  «        }‰ j        �=t          j        ‰ j                             |j        d         dd¦  «        |fd¬¦  «        }‰ j	        �‰ j	         
                    ¦   «         nd}t          j                             ¦   «         s|s‰ j        }n‰ j        d|dz   …         }t          |¦  «        D ]z\  }}‰ j        r1t          j                             ¦   «         st          |||¬	¦  «        }n |||¬	¦  «        }||
v r,|	                     |r‰                      |¦  «        n|¦  «         Œ{‰ j        rˆ fd
„|	D ¦   «         }ˆ fd„|	D ¦   «         }	|r/‰ j                             ||f¦  «        \  ŠŠˆˆˆfd„|	D ¦   «         }	t          j                             ¦   «         s|rt)          t+          |	|¦  «        ¦  «        }	|r|	S ‰                      |¦  «        }||	fS )a=   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            return_prefix_tokens: Return both prefix and spatial intermediate tokens
            norm: Apply norm layer to all intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )r  ÚNLCz)Output format must be one of NCHW or NLC.r  Nr   rN   r   rP   rO   c                 ó6   •— g | ]}|d d …d‰j         …f         ‘ŒS )Nr   ©rÒ   ©rÃ   ÚyrB   s     €rE   rÆ   zAVisionTransformerRelPos.forward_intermediates.<locals>.<listcomp>ß  s/   ø€ ÐSÐSÐSÀ˜Q˜q˜q˜q ! DÔ$:Ð":Ð:Ô;ÐSÐSÐSrF   c                 ó6   •— g | ]}|d d …‰j         d …f         ‘ŒS rg   r  r  s     €rE   rÆ   zAVisionTransformerRelPos.forward_intermediates.<locals>.<listcomp>à  s0   ø€ ÐRÐRÐR¸q˜Q˜q˜q˜q $Ô"8Ð"9Ð"9Ð9Ô:ÐRÐRÐRrF   c                 ó�   •— g | ]B}|                      ‰‰‰d ¦  «                             dddd¦  «                             ¦   «         ‘ŒCS )rN   r   r2   r   rI   )rR   rS   Ú
contiguous)rÃ   r  r^   ÚHÚWs     €€€rE   rÆ   zAVisionTransformerRelPos.forward_intermediates.<locals>.<listcomp>ä  sL   ø€ ÐlÐlÐlÐYZ˜QŸYšY q¨!¨Q°Ñ3Ô3×;Ò;¸A¸qÀ!ÀQÑGÔG×RÒRÑTÔTÐlÐlÐlrF   )r   ÚlenrÝ   rQ   rÔ   rÚ   rV   ÚcatÚexpandrG   rU   ÚjitÚis_scriptingrè   rÓ   r   Úappendrß   rÒ   Údynamic_feat_sizeÚlistÚzip)rB   r]   r  r  rß   r  r	  r
  rR   ÚintermediatesÚtake_indicesÚ	max_indexÚ_ÚheightÚwidthrG   rÝ   rÄ   Úblkr¾   r^   r  r  s   `                   @@@rE   Úforward_intermediatesz-VisionTransformerRelPos.forward_intermediates¬  s‹  øøøø€ ð. ˜_Ð,Ð,Ð,Ð.YÑ,Ô,Ð,Ø Ò&ˆØˆÝ"6µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�ið  œgÑˆˆ1ˆf�eØ×Ò˜QÑÔˆØŒ>Ð%Ý”	˜4œ>×0Ò0°´¸´¸RÀÑDÔDÀaÐHÈaÐPÑPÔPˆAà;?Ô;NÐ;Z˜Ô,×5Ò5Ñ7Ô7Ð7Ð`dˆÝŒ9×!Ò!Ñ#Ô#ð 	1¨:ð 	1Ø”[ˆFˆFà”[  )¨a¡- Ô0ˆFÝ Ñ'Ô'ð 	Bð 	B‰FˆAˆsØÔ&ð :­u¬y×/EÒ/EÑ/GÔ/Gð :Ý˜s A°nÐEÑEÔE��à�C˜¨.Ð9Ñ9Ô9�Ø�LÐ Ð à×$Ò$°TÐ%@ T§Y¢Y¨q¡\¤\ \¸qÑAÔAÐAøð Ô!ð 	SàSÐSÐSÐSÀ]ÐSÑSÔSˆMØRÐRÐRÐRÀMÐRÑRÔRˆMØð 	màÔ#×5Ò5°v¸u°oÑFÔF‰DˆAˆqØlÐlÐlÐlÐlÐlÐ^kÐlÑlÔlˆMÝŒy×%Ò%Ñ'Ô'ð 	DÐ,@ð 	Då ¥ ]°MÑ!BÔ!BÑCÔCˆMàð 	!Ø Ð à�IŠI�a‰LŒLˆà�-ÐÐrF   r   Ú
prune_normÚ
prune_headc                 ó  — t          t          | j        ¦  «        |¦  «        \  }}| j        d|dz   …         | _        |rt          j        ¦   «         | _        |r.t          j        ¦   «         | _        |                      dd¦  «         |S )z@ Prune layers not required for specified intermediates.
        Nr   r   r¥   )r   r  rÝ   r9   r<   rß   r«   r  )rB   r  r'  r(  r   r!  s         rE   Úprune_intermediate_layersz1VisionTransformerRelPos.prune_intermediate_layersð  s~   € õ #7µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�iØ”k . 9¨q¡= .Ô1ˆŒØð 	&Ýœ™œˆDŒIØð 	)Ýœ;™=œ=ˆDŒLØ×!Ò! ! RÑ(Ô(Ð(ØÐrF   c                 óÄ  — |                       |¦  «        }| j        �=t          j        | j                             |j        d         dd¦  «        |fd¬¦  «        }| j        �| j                             ¦   «         nd }| j        D ]G}| j	        r1t          j
                             ¦   «         st          |||¬¦  «        }Œ: |||¬¦  «        }ŒH|                      |¦  «        }|S )Nr   rN   r   rP   rO   )rÔ   rÚ   rV   r  r  rQ   rG   rU   rÝ   rÓ   r  r  r   rß   )rB   r]   rG   r%  s       rE   Úforward_featuresz(VisionTransformerRelPos.forward_features  sã   € Ø×Ò˜QÑÔˆØŒ>Ð%Ý”	˜4œ>×0Ò0°´¸´¸RÀÑDÔDÀaÐHÈaÐPÑPÔPˆAà;?Ô;NÐ;Z˜Ô,×5Ò5Ñ7Ô7Ð7Ð`dˆØ”;ð 	:ð 	:ˆCØÔ&ð :­u¬y×/EÒ/EÑ/GÔ/Gð :Ý˜s A°nÐEÑEÔE��à�C˜¨.Ð9Ñ9Ô9��Ø�IŠI�a‰LŒLˆØˆrF   Ú
pre_logitsc                 ó  — | j         r>| j         dk    r'|d d …| j        d …f                              d¬¦  «        n|d d …df         }|                      |¦  «        }|                      |¦  «        }|r|n|                      |¦  «        S )Nr›   r   rP   r   )r¤   rÒ   Úmeanr«   rà   rá   )rB   r]   r-  s      rE   Úforward_headz$VisionTransformerRelPos.forward_head  sš   € ØÔð 	dØ=AÔ=MÐQVÒ=VÐ=V��!�!�!�TÔ+Ð,Ð,Ð,Ô-×2Ò2°qÐ2Ñ9Ô9Ð9Ð\]Ð^_Ð^_Ð^_ÐabÐ^bÔ\cˆAØ�LŠL˜‰OŒOˆØ�NŠN˜1ÑÔˆØÐ0ˆqˆq D§I¢I¨a¡L¤LÐ0rF   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S rg   )r,  r0  )rB   r]   s     rE   rf   zVisionTransformerRelPos.forward  s-   € Ø×!Ò! !Ñ$Ô$ˆØ×Ò˜aÑ Ô ˆØˆrF   )rå   N)r¥   T©F)T)NNN)NFFFr  F)r   FT)%rh   ri   rj   Ú__doc__r   ru   r   rn   r   r	   rp   rk   r   Ústrr   r9   ro   r   r6   rï   r�   rV   r  Úignorerø   rü   rÿ   r  r  rq   r   r&  r*  r,  r0  rf   rr   rs   s   @rE   r    r    å   sw  ø€ € € € € ðð ð 58Ø68ØØ#Ø>CØ ØØØ!Ø!Ø!Ø+/Ø %Ø!Ø %Ø)-Ø#(Ø!Ø$&Ø$&Ø$&ØGNØ"Ø+5Ø.2Ø-1Ø(3ØØð=D1ð D1à˜C  s¨C x¤Ð0Ô1ðD1ð ˜c 5¨¨c¨¤?Ð2Ô3ðD1ð ð	D1ð
 ðD1ð !Ð!:Ô;ðD1ð ðD1ð ðD1ð ðD1ð ðD1ð ðD1ð ðD1ð " %œðD1ð ðD1ð ðD1ð  ð!D1ð" " #œð#D1ð$ !ð%D1ð& ð'D1ð( "ð)D1ð* "ð+D1ð, "ð-D1ð. !Ð!CÔDð/D1ð0 ð1D1ð2 ˜bœiœð3D1ð4 ! Ô+ð5D1ð6   	Ô*ð7D1ð8 ˜2œ9”oð9D1ð D1ð D1ð D1ð D1ð D1ðL1ð 1ð 1ð 1ð#ð # ð #¸ð #Èð #ð #ð #ð #ð& „YÔðð ñ Ôðð „YÔð
ð 
ð 
ñ Ôð
ð „YÔð)ð )ð )ñ Ôð)ð „YÔð ¤	ð ð ð ñ Ôðð'ð '¨Cð '¸hÀs¼mð 'ð 'ð 'ð 'ð 8<Ø).ØØ$Ø$Ø',ðB ð B àŒ|ðB ð ˜e C¨¨c¬ NÔ3Ô4ðB ð #'ð	B ð
 ðB ð ðB ð ðB ð !%ðB ð 
ˆt�E”LÔ! 5¨¬°t¸E¼LÔ7IÐ)IÔ#JÐJÔ	KðB ð B ð B ð B ðL ./Ø$Ø#ð	ð à˜3  S¤	˜>Ô*ðð ðð ð	ð ð ð ð"ð ð ð1ð 1¨$ð 1ð 1ð 1ð 1ðð ð ð ð ð ð rF   Fc                 ó|   — |                      dd¦  «        }t          t          | |fdt          |d¬¦  «        i|¤Ž}|S )NÚout_indicesr2   Úfeature_cfgÚgetter)r7  Úfeature_cls)Úpopr   r    rÌ   )ÚvariantÚ
pretrainedÚkwargsr7  Úmodels        rE   Ú!_create_vision_transformer_relposr@    sV   € Ø—*’*˜]¨AÑ.Ô.€KÝ Ý ¨*ðð å [¸hÐGÑGÔGðð ðð €Eð
 €LrF   r¥   c                 ó8   — | ddd dddt           t          dddd	œ|¥S )
Nrš   )r2   r˜   r˜   gÍÌÌÌÌÌì?ÚbicubicTzpatch_embed.projrá   z
apache-2.0)Úurlr£   Ú
input_sizeÚ	pool_sizeÚcrop_pctÚinterpolationÚfixed_input_sizer/  ró   Ú
first_convÚ
classifierÚlicenser   )rC  r>  s     rE   Ú_cfgrL  &  s:   € àØ¨=ÀtØ¨ÈÝ'Õ0FØ(¸Øðð ð ðð rF   z‰https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/vit_replos_base_patch32_plus_rpn_256-sw-dd486f51.pthztimm/)r2   é   rM  )rC  Ú	hf_hub_idrD  )r2   éð   rO  )rC  rD  z�https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/vit_relpos_small_patch16_224-sw-ec2778b4.pth)rC  rN  z‚https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/vit_relpos_medium_patch16_224-sw-11c174af.pthz€https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/vit_relpos_base_patch16_224-sw-49049aed.pthz‚https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/vit_srelpos_small_patch16_224-sw-6cdb8849.pthzƒhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/vit_srelpos_medium_patch16_224-sw-ad702b8c.pthz†https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/vit_relpos_medium_patch16_cls_224-sw-cfe8e259.pthz‡https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/vit_relpos_base_patch16_gapcls_224-sw-1a341d6c.pthz†https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/vit_relpos_medium_patch16_rpn_224-sw-5d2befd8.pth)z,vit_relpos_base_patch32_plus_rpn_256.sw_in1kz*vit_relpos_base_patch16_plus_240.untrainedz$vit_relpos_small_patch16_224.sw_in1kz%vit_relpos_medium_patch16_224.sw_in1kz#vit_relpos_base_patch16_224.sw_in1kz%vit_srelpos_small_patch16_224.sw_in1kz&vit_srelpos_medium_patch16_224.sw_in1kz)vit_relpos_medium_patch16_cls_224.sw_in1kz)vit_relpos_base_patch16_cls_224.untrainedz*vit_relpos_base_patch16_clsgap_224.sw_in1kz*vit_relpos_small_patch16_rpn_224.untrainedz)vit_relpos_medium_patch16_rpn_224.sw_in1kz)vit_relpos_base_patch16_rpn_224.untrainedrå   c           	      ól   — t          ddddt          ¬¦  «        }t          	 dd| it          |fi |¤Ž¤Ž}|S )	z` ViT-Base (ViT-B/32+) w/ relative log-coord position and residual post-norm, no class token
    é    é€  r�   é   )r¡   r¨   r©   r'   r¸   Ú$vit_relpos_base_patch32_plus_rpn_256r=  )rT  ©rÌ   rŽ   r@  ©r=  r>  Ú
model_argsr?  s       rE   rT  rT  Z  s_   € õ  ¨s¸"ÈÕUgÐhÑhÔh€JÝ-Ø.ðeð eØ;EðeÝIMÈjÐIcÐIcÐ\bÐIcÐIcðeð e€Eà€LrF   c           	      ó`   — t          dddd¬¦  «        }t          	 dd| it          |fi |¤Ž¤Ž}|S )	zI ViT-Base (ViT-B/16+) w/ relative log-coord position, no class token
    r™   rR  r�   rS  )r¡   r¨   r©   r'   Ú vit_relpos_base_patch16_plus_240r=  )rY  ©rÌ   r@  rV  s       rE   rY  rY  d  s\   € õ  ¨s¸"ÈÐKÑKÔK€JÝ-Ø*ðað aØ7AðaÝEIÈ*ÐE_ÐE_ÐX^ÐE_ÐE_ðað a€Eà€LrF   c           	      ód   — t          dddddd¬¦  «        }t          	 d
d	| it          |fi |¤Ž¤Ž}|S )úH ViT-Base (ViT-B/16) w/ relative log-coord position, no class token
    r™   é€  r�   é   FT©r¡   r¨   r©   r'   r(   r«   Úvit_relpos_small_patch16_224r=  )r`  rZ  rV  s       rE   r`  r`  n  sb   € õ  ¨s¸"ÈÐTYÐcgÐhÑhÔh€JÝ-Ø&ð]ð ]Ø3=ð]ÝAEÀjÐA[ÐA[ÐTZÐA[ÐA[ð]ð ]€Eà€LrF   c           	      ód   — t          dddddd¬¦  «        }t          	 d
d	| it          |fi |¤Ž¤Ž}|S )r\  r™   é   r�   r$   FTr_  Úvit_relpos_medium_patch16_224r=  )rc  rZ  rV  s       rE   rc  rc  x  sk   € õ Ø ¨B¸!ÀeÐUYð[ñ [ô [€Jå-Ø'ð^ð ^Ø4>ð^ÝBFÀzÐB\ÐB\ÐU[ÐB\ÐB\ð^ð ^€Eà€LrF   c           	      ód   — t          dddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤Ž}|S )
r\  r™   rœ   r�   FTr_  Úvit_relpos_base_patch16_224r=  )re  rZ  rV  s       rE   re  re  ƒ  sk   € õ Ø ¨B¸"ÀuÐVZð\ñ \ô \€Jå-Ø%ð\ð \Ø2<ð\Ý@DÀZÐ@ZÐ@ZÐSYÐ@ZÐ@Zð\ð \€Eà€LrF   c           
      óh   — t          dddddddd¬¦  «        }t          	 d
d	| it          |fi |¤Ž¤Ž}|S )úO ViT-Base (ViT-B/16) w/ shared relative log-coord position, no class token
    r™   r]  r�   r^  FT©r¡   r¨   r©   r'   r(   r«   r­   rG   Úvit_srelpos_small_patch16_224r=  )ri  rZ  rV  s       rE   ri  ri  Ž  so   € õ Ø ¨B¸!ÀeÐUZØ¨ð.ñ .ô .€Jõ .Ø'ð^ð ^Ø4>ð^ÝBFÀzÐB\ÐB\ÐU[ÐB\ÐB\ð^ð ^€Eà€LrF   c           
      óh   — t          dddddddd¬¦  «        }t          	 d
d	| it          |fi |¤Ž¤Ž}|S )rg  r™   rb  r�   r$   FTrh  Úvit_srelpos_medium_patch16_224r=  )rk  rZ  rV  s       rE   rk  rk  š  so   € õ Ø ¨B¸!ÀeÐUZØ¨ð.ñ .ô .€Jõ .Ø(ð_ð _Ø5?ð_ÝCGÈ
ÐC]ÐC]ÐV\ÐC]ÐC]ð_ð _€Eà€LrF   c                 ój   — t          ddddddddd¬	¦	  «	        }t          	 dd| it          |fi |¤Ž¤Ž}|S )zM ViT-Base (ViT-M/16) w/ relative log-coord position, class token present
    r™   rb  r�   r$   FrM  Tr¦   )	r¡   r¨   r©   r'   r(   r«   r­   rª   r¤   Ú!vit_relpos_medium_patch16_cls_224r=  )rm  rZ  rV  s       rE   rm  rm  ¦  st   € õ Ø ¨B¸!ÀeÐUZØ T°wð@ñ @ô @€Jõ .Ø+ðbð bØ8BðbÝFJÈ:ÐF`ÐF`ÐY_ÐF`ÐF`ðbð b€Eà€LrF   c           	      óf   — t          ddddddd¬¦  «        }t          	 d
d	| it          |fi |¤Ž¤Ž}|S )zM ViT-Base (ViT-B/16) w/ relative log-coord position, class token present
    r™   rœ   r�   FTr¦   )r¡   r¨   r©   r'   r(   rª   r¤   Úvit_relpos_base_patch16_cls_224r=  )ro  rZ  rV  s       rE   ro  ro  ²  sn   € õ Ø ¨B¸"ÀuÐZ^Ðlsðuñ uô u€Jå-Ø)ð`ð `Ø6@ð`ÝDHÈÐD^ÐD^ÐW]ÐD^ÐD^ð`ð `€Eà€LrF   c           	      óf   — t          ddddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤Ž}|S )
a   ViT-Base (ViT-B/16) w/ relative log-coord position, class token present
    NOTE this config is a bit of a mistake, class token was enabled but global avg-pool w/ fc-norm was not disabled
    Leaving here for comparisons w/ a future re-train as it performs quite well.
    r™   rœ   r�   FT)r¡   r¨   r©   r'   r(   r«   rª   Ú"vit_relpos_base_patch16_clsgap_224r=  )rq  rZ  rV  s       rE   rq  rq  ½  sn   € õ Ø ¨B¸"ÀuÐVZÐhlðnñ nô n€Jå-Ø,ðcð cØ9CðcÝGKÈJÐGaÐGaÐZ`ÐGaÐGaðcð c€Eà€LrF   c           	      ón   — t          dddddt          ¬¦  «        }t          	 d	d| it          |fi |¤Ž¤Ž}|S )
ú_ ViT-Base (ViT-B/16) w/ relative log-coord position and residual post-norm, no class token
    r™   r]  r�   r^  F©r¡   r¨   r©   r'   r(   r¸   Ú vit_relpos_small_patch16_rpn_224r=  )ru  rU  rV  s       rE   ru  ru  Ê  sk   € õ Ø ¨B¸!ÀeÕVhðjñ jô j€Jå-Ø*ðað aØ7AðaÝEIÈ*ÐE_ÐE_ÐX^ÐE_ÐE_ðað a€Eà€LrF   c           	      ón   — t          dddddt          ¬¦  «        }t          	 d	d| it          |fi |¤Ž¤Ž}|S )
rs  r™   rb  r�   r$   Frt  Ú!vit_relpos_medium_patch16_rpn_224r=  )rw  rU  rV  s       rE   rw  rw  Õ  sk   € õ Ø ¨B¸!ÀeÕVhðjñ jô j€Jå-Ø+ðbð bØ8BðbÝFJÈ:ÐF`ÐF`ÐY_ÐF`ÐF`ðbð b€Eà€LrF   c           	      ón   — t          dddddt          ¬¦  «        }t          	 dd| it          |fi |¤Ž¤Ž}|S )	rs  r™   rœ   r�   Frt  Úvit_relpos_base_patch16_rpn_224r=  )ry  rU  rV  s       rE   ry  ry  à  sk   € õ Ø ¨B¸"ÀuÕWiðkñ kô k€Jå-Ø)ð`ð `Ø6@ð`ÝDHÈÐD^ÐD^ÐW]ÐD^ÐD^ð`ð `€Eà€LrF   r2  )r¥   )Fr3  Úloggingré   Ú	functoolsr   Útypingr   r   r   r   r   r	   ÚImportErrorÚtyping_extensionsrV   Útorch.nnr9   Ú	torch.jitr
   Ú	timm.datar   r   Útimm.layersr   r   r   r   r   r   r   r   r   r   Ú_builderr   Ú	_featuresr   Ú_manipulater   r   Ú	_registryr   r   Úvision_transformerr   Ú__all__Ú	getLoggerrh   Ú_loggerro   r"   ru   rŽ   r    r@  rL  Údefault_cfgsrT  rY  r`  rc  re  ri  rk  rm  ro  rq  ru  rw  ry  r4   rF   rE   ú<module>rŒ     s“  ððð ð €€€Ø €€€Ø Ð Ð Ð Ð Ð Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5ð*ØÐÐÐÐÐÐøØð *ð *ð *Ø)Ð)Ð)Ð)Ð)Ð)Ð)Ð)ð*øøøð €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à EÐ EÐ EÐ EÐ EÐ EÐ EÐ Eðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð +Ð *Ð *Ð *Ð *Ð *Ø +Ð +Ð +Ð +Ð +Ð +Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø <Ð <Ð <Ð <Ð <Ð <Ð <Ð <Ø 4Ð 4Ð 4Ð 4Ð 4Ð 4à$Ð
%€à
ˆ'Ô
˜HÑ
%Ô
%€ðBð Bð Bð Bð B�b”iñ Bô Bð BðJ3ð 3ð 3ð 3ð 3�"”)ñ 3ô 3ð 3ðl:ð :ð :ð :ð :˜œñ :ô :ð :ðztð tð tð tð t˜bœiñ tô tð tðn	ð ð ð ð	ð 	ð 	ð 	ð %Ð$Ø48°Dð XØØ ð5"ñ 5"ô 5"ð 37°$¸2È-Ð2XÑ2XÔ2Xà,0¨Dð PØð-ñ -ô -ð .2¨Tð QØð.ñ .ô .ð ,0¨4ð OØð,ñ ,ô ,ð .2¨Tð QØð.ñ .ô .ð /3¨dð RØð/ñ /ô /ð 26°ð UØð2ñ 2ô 2ð 26°±´Ø26°$ð VØð3ñ 3ô 3ð 37°$±&´&Ø15°ð UØð2ñ 2ô 2ð 26°±´ðI%&ð %&ñ %ô %€ðP ðð ÐH_ð ð ð ñ „ðð ðð ÐD[ð ð ð ñ „ðð ðð Ð@Wð ð ð ñ „ðð ðð ÐAXð ð ð ñ „ðð ðð Ð?Vð ð ð ñ „ðð ðð ÐAXð ð ð ñ „ðð ðð ÐBYð ð ð ñ „ðð ðð ÐE\ð ð ð ñ „ðð ðð ÐCZð ð ð ñ „ðð ð	ð 	ÐF]ð 	ð 	ð 	ñ „ð	ð ðð ÐD[ð ð ð ñ „ðð ðð ÐE\ð ð ð ñ „ðð ðð ÐCZð ð ð ñ „ðð ð s    ' §5´5