§
    ·¯¸j°¨  ã                   ó¢  — d Z ddlZddlmZ ddlmZmZmZmZm	Z	m
Z
mZmZmZ ddlZddlmZ ddlmc mZ ddlmZmZ ddlmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$ ddl%m&Z& dd	l'm(Z( dd
l)m*Z* ddl+m,Z,m-Z- dgZ.d9dee/e/f         dej0        fd„Z1 G d„ dej2        ¦  «        Z3 G d„ dej2        ¦  «        Z4 G d„ dej2        ¦  «        Z5 G d„ dej2        ¦  «        Z6d:de7dee7ef         fd„Z8 e, e8d¬¦  «         e8ddd¬¦  «         e8dd¬ ¦  «         e8d¬¦  «         e8ddd¬¦  «         e8dd!d¬¦  «         e8dd¬ ¦  «         e8dee¬"¦  «         e8dee¬"¦  «         e8ddee¬#¦  «         e8dd$ee¬%¦  «         e8dd$ee¬%¦  «         e8ddee¬#¦  «        d&œ¦  «        Z9d;d)ee7ej0        f         d*ej2        d+e7d,e:dee7ej0        f         f
d-„Z;d<d/e7d0e:de6fd1„Z<e-d<d0e:de6fd2„¦   «         Z=e-d<d0e:de6fd3„¦   «         Z>e-d<d0e:de6fd4„¦   «         Z?e-d<d0e:de6fd5„¦   «         Z@e-d<d0e:de6fd6„¦   «         ZAe-d<d0e:de6fd7„¦   «         ZBe-d<d0e:de6fd8„¦   «         ZCdS )=aé   BEiT: BERT Pre-Training of Image Transformers (https://arxiv.org/abs/2106.08254)

Model from official source: https://github.com/microsoft/unilm/tree/master/beit

@inproceedings{beit,
title={{BEiT}: {BERT} Pre-Training of Image Transformers},
author={Hangbo Bao and Li Dong and Songhao Piao and Furu Wei},
booktitle={International Conference on Learning Representations},
year={2022},
url={https://openreview.net/forum?id=p-BhZSz59o4}
}

BEiT-v2 from https://github.com/microsoft/unilm/tree/master/beit2

@article{beitv2,
title={{BEiT v2}: Masked Image Modeling with Vector-Quantized Visual Tokenizers},
author={Zhiliang Peng and Li Dong and Hangbo Bao and Qixiang Ye and Furu Wei},
year={2022},
eprint={2208.06366},
archivePrefix={arXiv},
primaryClass={cs.CV}
}

At this point only the 1k fine-tuned classification weights and model configs have been added,
see original source above for pre-training models and procedure.

Modifications by / Copyright 2021 Ross Wightman, original copyrights below
é    N)Úpartial)	ÚAnyÚCallableÚDictÚListÚOptionalÚSetÚTupleÚTypeÚUnion)ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)Úget_device_dtypeÚ
PatchEmbedÚMlpÚSwiGLUÚ	LayerNormÚDropPathÚcalculate_drop_path_ratesÚtrunc_normal_Úuse_fused_attnÚresample_patch_embedÚresample_abs_pos_embedÚresize_rel_pos_bias_tableÚndgridé   )Úbuild_model_with_cfg)Úfeature_take_indices)Ú
checkpoint)Úgenerate_default_cfgsÚregister_modelÚBeitÚwindow_sizeÚreturnc           
      ó‚  — d| d         z  dz
  d| d         z  dz
  z  dz   }| d         | d         z  }t          j        t          t          j        | d         |t           j        ¬¦  «        t          j        | d         |t           j        ¬¦  «        ¦  «        ¦  «        }t          j        |d¦  «        }|dd…dd…df         |dd…ddd…f         z
  }|                     ddd¦  «                             ¦   «         }|dd…dd…dfxx         | d         dz
  z  cc<   |dd…dd…dfxx         | d         dz
  z  cc<   |dd…dd…dfxx         d| d         z  dz
  z  cc<   t          j        |dz   fdz  ||j	        ¬¦  «        }| 
                    d¦  «        |dd…dd…f<   |dz
  |ddd…f<   |dz
  |dd…df<   |dz
  |d	<   |S )
aº  Generate relative position index for window-based attention.

    Creates a lookup table for relative position indices between all pairs of positions
    within a window, including special handling for cls token interactions.

    Args:
        window_size: Height and width of the attention window.

    Returns:
        Relative position index tensor of shape (window_area+1, window_area+1)
        where +1 accounts for the cls token.
    é   r   r   é   ©ÚdeviceÚdtypeN)Úsizer)   r*   éÿÿÿÿ)r   r   )ÚtorchÚstackr   ÚarangeÚlongÚflattenÚpermuteÚ
contiguousÚzerosr*   Úsum)r#   r)   Únum_relative_distanceÚwindow_areaÚcoordsÚcoords_flattenÚrelative_coordsÚrelative_position_indexs           ú`/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/models/beit.pyÚgen_relative_position_indexr=   J   s,  € ð  ¨Q¤Ñ/°!Ñ3¸¸KÈ¼NÑ8JÈQÑ8NÑOÐRSÑSÐð ˜a”. ;¨q¤>Ñ1€KÝŒ[�ÝŒ�[ ”^¨F½%¼*ÐEÑEÔEÝŒ�[ ”^¨F½%¼*ÐEÑEÔEñô ñ ô €Fõ ”] 6¨1Ñ-Ô-€NØ$ Q Q Q¨¨¨¨4 ZÔ0°>À!À!À!ÀTÈ1È1È1À*Ô3MÑM€OØ%×-Ò-¨a°°AÑ6Ô6×AÒAÑCÔC€OØ�A�A�A�q�q�q˜!�GÐÐÔ ¨A¤°Ñ 2Ñ2ÐÐÑØ�A�A�A�q�q�q˜!�GÐÐÔ ¨A¤°Ñ 2Ñ2ÐÐÑØ�A�A�A�q�q�q˜!�GÐÐÔ  K°¤NÑ 2°QÑ 6Ñ6ÐÐÑÝ#œk°¸a±Ð/AÀAÑ/EÈfÐ\kÔ\qÐrÑrÔrÐØ&5×&9Ò&9¸"Ñ&=Ô&=Ð˜A˜B˜B   ˜FÑ#Ø%:¸QÑ%>Ð˜A˜q˜r˜r˜EÑ"Ø%:¸QÑ%>Ð˜A˜B˜B ˜EÑ"Ø$9¸AÑ$=Ð˜DÑ!Ø"Ð"ó    c                   ó  ‡ — e Zd ZU dZej        j        e         ed<   	 	 	 	 	 	 	 	 	 dde	de	d	ed
ede
de
deee	e	f                  dee	         fˆ fd„Zdej        fd„Zddej        deej                 dej        fd„Zdd„Zdd„Zdd„Zˆ xZS )Ú	AttentionzïMulti-head attention module with optional relative position bias.

    Implements multi-head self-attention with support for relative position bias
    and fused attention operations. Can use either standard or custom head dimensions.
    Ú
fused_attné   Fç        NÚdimÚ	num_headsÚqkv_biasÚqkv_bias_separateÚ	attn_dropÚ	proj_dropr#   Úattn_head_dimc                 ón  •— |	|
dœ}t          ¦   «                              ¦   «          || _        ||z  }|�|}|| j        z  }|dz  | _        t	          ¦   «         | _        || _        t          j        ||dz  fddi|¤Ž| _	        |r{t          j
        t          j        |fi |¤Ž¦  «        | _        |                      dt          j        |fi |¤Žd¬¦  «         t          j
        t          j        |fi |¤Ž¦  «        | _        nd| _        d| _        d| _        |r¬|| _        d	|d
         z  dz
  d	|d         z  dz
  z  dz   | _        |d
         |d         z  }t          j
        t          j        | j        |fi |¤Ž¦  «        | _        |                      dt          j        |dz   |dz   f|	t          j        ¬¦  «        d¬¦  «         nd| _        d| _        d| _        t          j        |¦  «        | _        t          j        ||fi |¤Ž| _        t          j        |¦  «        | _        |                      ¦   «          dS )ak  Initialize attention module.

        Args:
            dim: Input feature dimension.
            num_heads: Number of attention heads.
            qkv_bias: If True, add learnable bias to query, key, value projections.
            qkv_bias_separate: If True, use separate bias for q, k, v projections.
            attn_drop: Dropout rate for attention weights.
            proj_drop: Dropout rate for output projection.
            window_size: Window size for relative position bias. If None, no relative position bias.
            attn_head_dim: Dimension per attention head. If None, uses dim // num_heads.
        r(   Ng      à¿r'   ÚbiasFÚk_bias©Ú
persistentr&   r   r   r;   )ÚsuperÚ__init__rE   Úscaler   rA   rG   ÚnnÚLinearÚqkvÚ	Parameterr-   ÚemptyÚq_biasÚregister_bufferÚv_biasrM   r#   r6   Úrelative_position_bias_tabler0   r;   ÚDropoutrH   ÚprojrI   Úreset_parameters)ÚselfrD   rE   rF   rG   rH   rI   r#   rJ   r)   r*   ÚddÚhead_dimÚall_head_dimr7   Ú	__class__s                  €r<   rQ   zAttention.__init__u   sj  ø€ ð2 ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ"ˆŒØ˜)Ñ#ˆØÐ$Ø$ˆHØ $¤.Ñ0ˆØ Ñ%ˆŒ
Ý(Ñ*Ô*ˆŒØ!2ˆÔå”9˜S ,°Ñ"2ÐEÐE¸ÐEÀ"ÐEÐEˆŒØð 	Ýœ,¥u¤{°<Ð'FÐ'FÀ2Ð'FÐ'FÑGÔGˆDŒKØ× Ò  ­5¬;°|Ð+JÐ+JÀrÐ+JÐ+JÐW\Ð Ñ]Ô]Ð]Ýœ,¥u¤{°<Ð'FÐ'FÀ2Ð'FÐ'FÑGÔGˆDŒKˆKàˆDŒKØˆDŒKØˆDŒKàð 	0Ø*ˆDÔØ*+¨k¸!¬nÑ*<¸qÑ*@ÀQÈÐUVÌÑEWÐZ[ÑE[Ñ)\Ð_`Ñ)`ˆDÔ&Ø% aœ.¨;°q¬>Ñ9ˆKÝ02´Ý”˜DÔ6¸	ÐHÐHÀRÐHÐHñ1Jô 1JˆDÔ-à× Ò Ø)Ý”˜[¨1™_¨k¸A©oÐ>ÀvÕUZÔU_Ð`Ñ`Ô`Ø ð !ñ ô ð ð ð  $ˆDÔØ04ˆDÔ-Ø+/ˆDÔ(åœ IÑ.Ô.ˆŒÝ”I˜l¨CÐ6Ð6°2Ð6Ð6ˆŒ	Ýœ IÑ.Ô.ˆŒð 	×ÒÑÔÐÐÐr>   r$   c                 ób  — | j         | j                             d¦  «                                      | j        d         | j        d         z  dz   | j        d         | j        d         z  dz   d¦  «        }|                     ddd¦  «                             ¦   «         }|                     d¦  «        S )z°Get relative position bias for the attention window.

        Returns:
            Relative position bias tensor of shape (1, num_heads, window_area+1, window_area+1).
        r,   r   r   r&   )r[   r;   Úviewr#   r2   r3   Ú	unsqueeze©r_   Úrelative_position_biass     r<   Ú_get_rel_pos_biaszAttention._get_rel_pos_biasº   s­   € ð "&Ô!BØÔ(×-Ò-¨bÑ1Ô1ô"3ß37²4ØÔ˜QÔ $Ô"2°1Ô"5Ñ5¸Ñ9ØÔ˜QÔ $Ô"2°1Ô"5Ñ5¸Ñ9¸2ñ4?ô 4?ð 	ð "8×!?Ò!?ÀÀ1ÀaÑ!HÔ!H×!SÒ!SÑ!UÔ!UÐØ%×/Ò/°Ñ2Ô2Ð2r>   ÚxÚshared_rel_pos_biasc                 ó  — |j         \  }}}| j        €|                      |¦  «        }nit          j        | j        | j        | j        f¦  «        }| j        r|                      |¦  «        }||z  }n!t          j	        || j        j
        |¬¦  «        }|                     ||d| j        d¦  «                             ddddd¦  «        }|                     d¦  «        \  }}	}
| j        rVd}| j        �|                      ¦   «         }|�||z   }n|�|}t          j        ||	|
|| j        r| j        j        nd	¬
¦  «        }nx|| j        z  }||	                     dd¦  «        z  }| j        �||                      ¦   «         z   }|�||z   }|                     d¬¦  «        }|                      |¦  «        }||
z  }|                     dd¦  «                             |||¦  «        }|                      |¦  «        }|                      |¦  «        }|S )a-  Forward pass of attention module.

        Args:
            x: Input tensor of shape (batch_size, num_tokens, dim).
            shared_rel_pos_bias: Optional shared relative position bias from parent module.

        Returns:
            Output tensor of shape (batch_size, num_tokens, dim).
        N)ÚweightrL   r'   r,   r&   r   r   é   rC   )Ú	attn_maskÚ	dropout_péþÿÿÿ©rD   )ÚshaperX   rU   r-   ÚcatrM   rZ   rG   ÚFÚlinearrm   ÚreshaperE   r2   ÚunbindrA   r[   ri   Úscaled_dot_product_attentionÚtrainingrH   ÚprR   Ú	transposeÚsoftmaxr]   rI   )r_   rj   rk   ÚBÚNÚCrU   rF   ÚqÚkÚvÚrel_pos_biasÚattns                r<   ÚforwardzAttention.forwardÇ   s  € ð ”'‰ˆˆ1ˆaàŒ;ÐØ—(’(˜1‘+”+ˆCˆCå”y $¤+¨t¬{¸D¼KÐ!HÑIÔIˆHØÔ%ð IØ—h’h˜q‘k”k�Ø�x‘��å”h˜q¨¬¬¸xÐHÑHÔH�Ø�kŠk˜!˜Q  4¤>°2Ñ6Ô6×>Ò>¸qÀ!ÀQÈÈ1ÑMÔMˆØ—*’*˜Q‘-”-‰ˆˆ1ˆaàŒ?ð 	ØˆLØÔ0Ð<Ø#×5Ò5Ñ7Ô7�Ø&Ð2Ø#/Ð2EÑ#E�LøØ$Ð0Ø2�åÔ.Ø�1�aØ&Ø.2¬mÐC˜$œ.Ô*Ð*Àðñ ô ˆAˆAð �D”J‘ˆAØ˜Ÿš B¨Ñ+Ô+Ñ+ˆDàÔ0Ð<Ø˜d×4Ò4Ñ6Ô6Ñ6�Ø"Ð.ØÐ1Ñ1�à—<’< B�<Ñ'Ô'ˆDØ—>’> $Ñ'Ô'ˆDØ�q‘ˆAà�KŠK˜˜1ÑÔ×%Ò% a¨¨AÑ.Ô.ˆØ�IŠI�a‰LŒLˆØ�NŠN˜1ÑÔˆØˆr>   c                 ó"  — | j         �Ht          j                             | j         ¦  «         t          j                             | j        ¦  «         | j        �$t          j                             | j        ¦  «         |                      ¦   «          dS ©z"Initialize parameters and buffers.N)rX   rS   ÚinitÚzeros_rZ   r[   Ú_init_buffers©r_   s    r<   r^   zAttention.reset_parametersÿ   sn   € àŒ;Ð"ÝŒG�NŠN˜4œ;Ñ'Ô'Ð'ÝŒG�NŠN˜4œ;Ñ'Ô'Ð'ØÔ,Ð8ÝŒG�NŠN˜4Ô<Ñ=Ô=Ð=Ø×ÒÑÔÐÐÐr>   c                 óÈ   — | j         �| j                              ¦   «          | j        �:| j                             t	          | j        | j        j        ¬¦  «        ¦  «         dS dS )ú.Compute and fill non-persistent buffer values.N©r)   )rM   Úzero_r;   Úcopy_r=   r#   r)   rŒ   s    r<   r‹   zAttention._init_buffers  sm   € àŒ;Ð"ØŒK×ÒÑÔÐØÔ'Ð3ØÔ(×.Ò.Ý+¨DÔ,<ÀTÔEaÔEhÐiÑiÔiñô ð ð ð ð 4Ð3r>   c                 ó.   — |                       ¦   «          dS ©z"Initialize non-persistent buffers.N©r‹   rŒ   s    r<   Úinit_non_persistent_buffersz%Attention.init_non_persistent_buffers  ó   € à×ÒÑÔÐÐÐr>   )	rB   FFrC   rC   NNNN©N©r$   N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r-   ÚjitÚFinalÚboolÚ__annotations__ÚintÚfloatr   r
   rQ   ÚTensorri   r†   r^   r‹   r•   Ú__classcell__©rc   s   @r<   r@   r@   m   s†  ø€ € € € € € ðð ð
 ”	” Ô%Ð%Ð%Ñ%ð
 Ø"Ø&+Ø!Ø!Ø59Ø+/ØØðC ð C àðC ð ðC ð ð	C ð
  $ðC ð ðC ð ðC ð " %¨¨S¨¤/Ô2ðC ð $ Cœ=ðC ð C ð C ð C ð C ð C ðJ3 5¤<ð 3ð 3ð 3ð 3ð6ð 6˜œð 6¸HÀUÄ\Ô<Rð 6Ð^cÔ^jð 6ð 6ð 6ð 6ðpð ð ð ðð ð ð ðð ð ð ð ð ð ð r>   r@   c                   ó,  ‡ — e Zd ZdZddddddddej        eddddfdededed	e	d
edede	de	de	de
e	         deej                 deej                 de
eeef                  de
e         fˆ fd„Zdd„Zddej        de
ej                 dej        fd„Zˆ xZS )ÚBlockzóTransformer block with attention and MLP.

    Standard transformer block consisting of multi-head self-attention and MLP
    with residual connections and layer normalization. Supports layer scale and
    stochastic depth regularization.
    Fç      @rC   NrD   rE   rF   Ú	mlp_ratioÚ	scale_mlpÚ
swiglu_mlprI   rH   Ú	drop_pathÚinit_valuesÚ	act_layerÚ
norm_layerr#   rJ   c           
      ó  •— ||dœ}t          ¦   «                              ¦   «           ||fi |¤Ž| _        t          |f||||||dœ|¤Ž| _        |	dk    rt          |	¦  «        nt          j        ¦   «         | _         ||fi |¤Ž| _	        |r+t          d|t          ||z  ¦  «        |r|nd|dœ|¤Ž| _        n+t          d|t          ||z  ¦  «        ||r|nd|dœ|¤Ž| _        |	dk    rt          |	¦  «        nt          j        ¦   «         | _        |
| _        |
rSt          j        t#          j        |fi |¤Ž¦  «        | _        t          j        t#          j        |fi |¤Ž¦  «        | _        nd\  | _        | _        |                      ¦   «          dS )	a‘  Initialize transformer block.

        Args:
            dim: Input feature dimension.
            num_heads: Number of attention heads.
            qkv_bias: If True, add learnable bias to query, key, value projections.
            mlp_ratio: Ratio of MLP hidden dimension to input dimension.
            scale_mlp: If True, apply layer normalization in MLP.
            swiglu_mlp: If True, use SwiGLU activation in MLP.
            proj_drop: Dropout rate for projections.
            attn_drop: Dropout rate for attention.
            drop_path: Drop path rate for stochastic depth.
            init_values: Initial values for layer scale. If None, no layer scale.
            act_layer: Activation function class.
            norm_layer: Normalization layer class.
            window_size: Window size for relative position bias in attention.
            attn_head_dim: Dimension per attention head.
        r(   )rE   rF   rH   rI   r#   rJ   rC   N)Úin_featuresÚhidden_featuresr¯   Údrop)r±   r²   r®   r¯   r³   ©NN© )rP   rQ   Únorm1r@   r…   r   rS   ÚIdentityÚ
drop_path1Únorm2r   r¡   Úmlpr   Ú
drop_path2r­   rV   r-   rW   Úgamma_1Úgamma_2r^   )r_   rD   rE   rF   r©   rª   r«   rI   rH   r¬   r­   r®   r¯   r#   rJ   r)   r*   r`   rc   s                     €r<   rQ   zBlock.__init__  sø  ø€ ðJ ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ�Z Ð*Ð* rÐ*Ð*ˆŒ
ÝØð	
àØØØØ#Ø'ð	
ð 	
ð ð	
ð 	
ˆŒ	ð 2;¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒà�Z Ð*Ð* rÐ*Ð*ˆŒ
Øð 	Ýð ØÝ # C¨)¡OÑ 4Ô 4Ø)2Ð<˜:˜:¸Øð	ð ð
 ðð ˆDŒHˆHõ ð ØÝ # C¨)¡OÑ 4Ô 4Ø#Ø)2Ð<˜:˜:¸Øðð ð ðð ˆDŒHð 2;¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒà&ˆÔØð 	4Ýœ<­¬°CÐ(>Ð(>¸2Ð(>Ð(>Ñ?Ô?ˆDŒLÝœ<­¬°CÐ(>Ð(>¸2Ð(>Ð(>Ñ?Ô?ˆDŒLˆLà)3Ñ&ˆDŒL˜$œ,ð 	×ÒÑÔÐÐÐr>   r$   c                 óÀ   — | j         �Vt          j                             | j         | j        ¦  «         t          j                             | j        | j        ¦  «         dS dS )zInitialize parameters.N)r¼   rS   r‰   Ú	constant_r­   r½   rŒ   s    r<   r^   zBlock.reset_parametersq  sR   € àŒ<Ð#ÝŒG×Ò˜dœl¨DÔ,<Ñ=Ô=Ð=ÝŒG×Ò˜dœl¨DÔ,<Ñ=Ô=Ð=Ð=Ð=ð $Ð#r>   rj   rk   c           	      ó.  — | j         €||                      |                      |                      |¦  «        |¬¦  «        ¦  «        z   }||                      |                      |                      |¦  «        ¦  «        ¦  «        z   }nŽ||                      | j         |                      |                      |¦  «        |¬¦  «        z  ¦  «        z   }||                      | j        |                      |                      |¦  «        ¦  «        z  ¦  «        z   }|S )a  Forward pass of transformer block.

        Args:
            x: Input tensor of shape (batch_size, num_tokens, dim).
            shared_rel_pos_bias: Optional shared relative position bias.

        Returns:
            Output tensor of shape (batch_size, num_tokens, dim).
        N©rk   )r¼   r¸   r…   r¶   r»   rº   r¹   r½   )r_   rj   rk   s      r<   r†   zBlock.forwardw  sÛ   € ð Œ<ÐØ�D—O’O D§I¢I¨d¯jªj¸©m¬mÐQd IÑ$eÔ$eÑfÔfÑfˆAØ�D—O’O D§H¢H¨T¯ZªZ¸©]¬]Ñ$;Ô$;Ñ<Ô<Ñ<ˆAˆAà�D—O’O D¤L°4·9²9¸T¿ZºZÈ¹]¼]Ð`s°9Ñ3tÔ3tÑ$tÑuÔuÑuˆAØ�D—O’O D¤L°4·8²8¸D¿JºJÀq¹M¼MÑ3JÔ3JÑ$JÑKÔKÑKˆAØˆr>   r˜   r—   )r™   rš   r›   rœ   rS   ÚGELUr   r¡   rŸ   r¢   r   r   ÚModuler
   rQ   r^   r-   r£   r†   r¤   r¥   s   @r<   r§   r§     sŠ  ø€ € € € € ðð ð #Ø!Ø#Ø$Ø!Ø!Ø!Ø+/Ø)+¬Ø*3Ø59Ø+/ØØð#Q ð Q àðQ ð ðQ ð ð	Q ð
 ðQ ð ðQ ð ðQ ð ðQ ð ðQ ð ðQ ð " %œðQ ð ˜BœI”ðQ ð ˜RœYœðQ ð " %¨¨S¨¤/Ô2ðQ ð $ Cœ=ðQ ð Q ð Q ð Q ð Q ð Q ðf>ð >ð >ð >ðð ˜œð ¸HÀUÄ\Ô<Rð Ð^cÔ^jð ð ð ð ð ð ð ð r>   r§   c                   ól   ‡ — e Zd ZdZddeeef         defˆ fd„Zdd„Zdd„Zdd	„Z	de
j        fd
„Zˆ xZS )ÚRelativePositionBiaszÊRelative position bias module for window-based attention.

    Generates learnable relative position biases for all pairs of positions
    within a window, including special handling for cls token.
    Nr#   rE   c                 óÚ  •— ||dœ}t          ¦   «                              ¦   «          || _        |d         |d         z  | _        d|d         z  dz
  d|d         z  dz
  z  dz   }t	          j        t          j        ||fi |¤Ž¦  «        | _        |  	                    dt          j        | j        dz   | j        dz   f|t          j
        ¬¦  «        d¬¦  «         |                      ¦   «          d	S )
z¶Initialize relative position bias module.

        Args:
            window_size: Height and width of the attention window.
            num_heads: Number of attention heads.
        r(   r   r   r&   r'   r;   FrN   N)rP   rQ   r#   r7   rS   rV   r-   rW   r[   rY   r0   r^   )r_   r#   rE   r)   r*   r`   r6   rc   s          €r<   rQ   zRelativePositionBias.__init__‘  s  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ&ˆÔØ& qœ>¨K¸¬NÑ:ˆÔØ!" [°¤^Ñ!3°aÑ!7¸AÀÈAÄÑ<NÐQRÑ<RÑ SÐVWÑ WÐÝ,.¬L½¼ÐEZÐ\eÐ9lÐ9lÐikÐ9lÐ9lÑ,mÔ,mˆÔ)Ø×ÒØ%ÝŒK˜Ô)¨AÑ-¨tÔ/?À!Ñ/CÐDÈVÕ[`Ô[eÐfÑfÔfØð 	ñ 	
ô 	
ð 	
ð 	×ÒÑÔÐÐÐr>   r$   c                 óv   — t           j                             | j        ¦  «         |                      ¦   «          dS rˆ   )rS   r‰   rŠ   r[   r‹   rŒ   s    r<   r^   z%RelativePositionBias.reset_parameters§  s1   € å
Œ�Š�tÔ8Ñ9Ô9Ð9Ø×ÒÑÔÐÐÐr>   c                 óv   — | j                              t          | j        | j         j        ¬¦  «        ¦  «         dS )rŽ   r�   N)r;   r‘   r=   r#   r)   rŒ   s    r<   r‹   z"RelativePositionBias._init_buffers¬  s@   € àÔ$×*Ò*Ý'¨Ô(8ÀÔA]ÔAdÐeÑeÔeñ	
ô 	
ð 	
ð 	
ð 	
r>   c                 ó.   — |                       ¦   «          dS r“   r”   rŒ   s    r<   r•   z0RelativePositionBias.init_non_persistent_buffers²  r–   r>   c                 óè   — | j         | j                             d¦  «                                      | j        dz   | j        dz   d¦  «        }|                     ddd¦  «                             ¦   «         S )z™Generate relative position bias.

        Returns:
            Relative position bias tensor of shape (num_heads, window_area+1, window_area+1).
        r,   r   r&   r   )r[   r;   re   r7   r2   r3   rg   s     r<   r†   zRelativePositionBias.forward¶  so   € ð "&Ô!BÀ4ÔC_×CdÒCdÐegÑChÔChÔ!i×!nÒ!nØÔ˜qÑ  $Ô"2°QÑ"6¸ñ"<ô "<Ðà%×-Ò-¨a°°AÑ6Ô6×AÒAÑCÔCÐCr>   r´   r˜   )r™   rš   r›   rœ   r
   r¡   rQ   r^   r‹   r•   r-   r£   r†   r¤   r¥   s   @r<   rÅ   rÅ   Š  sÄ   ø€ € € € € ðð ð ð   E¨#¨s¨(¤Oð  Àð  ð  ð  ð  ð  ð  ð,ð ð ð ð

ð 
ð 
ð 
ðð ð ð ðD˜œð Dð Dð Dð Dð Dð Dð Dð Dr>   rÅ   c            /       óÐ  ‡ — e Zd ZdZddddddddd	d
dddddddedd	dddddfdeeeeef         f         deeeeef         f         dedededededede	de
de	de	de
de
de
de
de
d eej                 d!ee
         d"e	d#e	d$e	d%e
f.ˆ fd&„ZdEd'e	d(dfd)„ZdFd*„ZdEd+ej        d'e	fd,„Zej        j        d(ee         fd-„¦   «         Zej        j        dEd.e	fd/„¦   «         Zej        j        dGd0e	d(eeef         fd1„¦   «         Zej        j        d(ej        fd2„¦   «         ZdHdedee         fd3„Z	 	 	 	 	 	 dId5ej        d6eeeee         f                  d7e	d8e	d9e	d:ed;e	d(eeej                 eej        eej                 f         f         fd<„Z 	 	 	 dJd6eeee         f         d>e	d?e	d(ee         fd@„Z!d5ej        d(ej        fdA„Z"dGd5ej        dBe	d(ej        fdC„Z#d5ej        d(ej        fdD„Z$ˆ xZ%S )Kr"   a  BEiT: BERT Pre-Training of Image Transformers.

    Vision Transformer model with support for relative position bias and
    shared relative position bias across layers. Implements both BEiT v1 and v2
    architectures with flexible configuration options.
    éà   é   r'   éè  Úavgé   é   Tr¨   FrC   Ngü©ñÒMbP?Úimg_sizeÚ
patch_sizeÚin_chansÚnum_classesÚglobal_poolÚ	embed_dimÚdepthrE   rF   r©   r«   rª   Ú	drop_rateÚpos_drop_rateÚproj_drop_rateÚattn_drop_rateÚdrop_path_rater¯   r­   Úuse_abs_pos_embÚuse_rel_pos_biasÚuse_shared_rel_pos_biasÚhead_init_scalec                 ó  •‡ ‡‡‡	‡
‡‡‡‡‡‡‡‡‡‡— ||dœŠt          ¦   «                              ¦   «          |‰ _        |‰ _        |‰ _        ‰x‰ _        x‰ _        ‰ _        d‰ _        d‰ _	        t          d|||‰dœ‰¤Ž‰ _        ‰ j        j        }t          ‰ j        d¦  «        r‰ j                             ¦   «         n|Št          j        t#          j        dd‰fi ‰¤Ž¦  «        ‰ _        |r)t          j        t#          j        d|dz   ‰fi ‰¤Ž¦  «        nd‰ _        t          j        |¬¦  «        ‰ _        |rt/          d‰ j        j        ‰dœ‰¤Ž‰ _        nd‰ _        t5          ||¦  «        Št          j        ˆˆˆˆˆˆ
ˆˆˆˆ	ˆˆ ˆˆfd	„t9          |¦  «        D ¦   «         ¦  «        ‰ _        ˆˆfd
„t9          |¦  «        D ¦   «         ‰ _        ‰ j        dk    }|rt          j        ¦   «         n ‰‰fi ‰¤Ž‰ _         |r	 ‰‰fi ‰¤Žnt          j        ¦   «         ‰ _!        t          j        |¦  «        ‰ _"        |dk    rt          j#        ‰|fi ‰¤Žnt          j        ¦   «         ‰ _$        |‰ _%        ‰  &                    d¬¦  «         dS )a¤  Initialize BEiT model.

        Args:
            img_size: Input image size.
            patch_size: Patch size for patch embedding.
            in_chans: Number of input image channels.
            num_classes: Number of classes for classification head.
            global_pool: Type of global pooling ('avg' or '').
            embed_dim: Embedding dimension.
            depth: Number of transformer blocks.
            num_heads: Number of attention heads.
            qkv_bias: If True, add learnable bias to query, key, value projections.
            mlp_ratio: Ratio of MLP hidden dimension to embedding dimension.
            swiglu_mlp: If True, use SwiGLU activation in MLP.
            scale_mlp: If True, apply layer normalization in MLP.
            drop_rate: Dropout rate.
            pos_drop_rate: Dropout rate for position embeddings.
            proj_drop_rate: Dropout rate for projections.
            attn_drop_rate: Dropout rate for attention.
            drop_path_rate: Stochastic depth rate.
            norm_layer: Normalization layer class.
            init_values: Initial values for layer scale.
            use_abs_pos_emb: If True, use absolute position embeddings.
            use_rel_pos_bias: If True, use relative position bias in attention.
            use_shared_rel_pos_bias: If True, share relative position bias across layers.
            head_init_scale: Scale factor for head initialization.
        r(   r   F)rÒ   rÓ   rÔ   r×   Ú
feat_ratioN)r{   )r#   rE   c                 ój   •— g | ]/}t          d‰‰	‰‰‰‰‰
‰‰|         ‰‰‰r‰j        j        nd dœ‰¤Ž‘Œ0S )N)rD   rE   rF   r©   rª   r«   rI   rH   r¬   r¯   r­   r#   rµ   )r§   Úpatch_embedÚ	grid_size)Ú.0ÚirÜ   r`   Údprr×   r­   r©   r¯   rE   rÛ   rF   rª   r_   r«   rß   s     €€€€€€€€€€€€€€r<   ú
<listcomp>z!Beit.__init__.<locals>.<listcomp>"  s‚   ø€ ð %#ð %#ð %#ð  õ ð ØØ#Ø!Ø#Ø#Ø%Ø(Ø(Ø˜aœ&Ø%Ø'Ø:JÐT˜DÔ,Ô6Ð6ÐPTðð ð ðð ð%#ð %#ð %#r>   c                 ó:   •— g | ]}t          d |› �‰‰¬¦  «        ‘ŒS )zblocks.)ÚmoduleÚnum_chsÚ	reduction©Údict)rç   rè   r×   Úrs     €€r<   rê   z!Beit.__init__.<locals>.<listcomp>3  sA   ø€ ð ^ð ^ð ^ØKL�D˜ !˜˜¨yÀAÐFÑFÔFð^ð ^ð ^r>   rÏ   r   ©Úneeds_resetrµ   )'rP   rQ   rÕ   rÔ   rÖ   Únum_featuresÚhead_hidden_sizer×   Únum_prefix_tokensÚgrad_checkpointingr   rå   Únum_patchesÚhasattrrã   rS   rV   r-   rW   Ú	cls_tokenÚ	pos_embedr\   Úpos_droprÅ   ræ   r„   r   Ú
ModuleListÚrangeÚblocksÚfeature_infor·   ÚnormÚfc_normÚ	head_droprT   Úheadrá   Úinit_weights) r_   rÒ   rÓ   rÔ   rÕ   rÖ   r×   rØ   rE   rF   r©   r«   rª   rÙ   rÚ   rÛ   rÜ   rÝ   r¯   r­   rÞ   rß   rà   rá   r)   r*   rø   Úuse_fc_normr`   ré   rñ   rc   s    `     ` `````  `` `` `      @@@€r<   rQ   zBeit.__init__É  s6  øøøøøøøøøøøøøøøø€ ðn ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ&ˆÔØ ˆŒØ&ˆÔØENÐNˆÔÐN˜DÔ1°D´NØ!"ˆÔØ"'ˆÔå%ð 
ØØ!ØØð	
ð 
ð
 ð
ð 
ˆÔð Ô&Ô2ˆÝ-4°TÔ5EÀ|Ñ-TÔ-TÐdˆDÔ×'Ò'Ñ)Ô)Ð)ÐZdˆåœ¥e¤k°!°Q¸	Ð&HÐ&HÀRÐ&HÐ&HÑIÔIˆŒà[jÐt�œ¥e¤k°!°[À1±_ÀiÐ&VÐ&VÐSUÐ&VÐ&VÑWÔWÐWÐptˆŒÝœ
 ]Ð3Ñ3Ô3ˆŒà"ð 	%Ý 4ð !Ø Ô,Ô6Ø#ð!ð !ð ð!ð !ˆDÔÐð !%ˆDÔå'¨¸Ñ>Ô>ˆÝ”mð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#õ  ˜5‘\”\ð!%#ñ %#ô %#ñ $ô $ˆŒð"^ð ^ð ^ð ^ð ^ÝPUÐV[ÑP\ÔP\ð^ñ ^ô ^ˆÔð Ô&¨%Ò/ˆØ%0ÐQ•B”K‘M”M�M°j°jÀÐ6QÐ6QÈbÐ6QÐ6QˆŒ	Ø6AÐT�z�z )Ð2Ð2¨rÐ2Ð2Ð2ÅrÄ{Á}Ä}ˆŒÝœ IÑ.Ô.ˆŒØ?JÈQº¸•B”I˜i¨Ð;Ð;¸Ð;Ð;Ð;ÕTVÔT_ÑTaÔTaˆŒ	Ø.ˆÔð 	×Ò eÐÑ,Ô,Ð,Ð,Ð,r>   ró   r$   c                 óX  — |                       t          | j        |¬¦  «        ¦  «         | j        �t	          | j        d¬¦  «         t	          | j        d¬¦  «         |                      ¦   «          | j        r°t          | j	        t          j        ¦  «        r“t	          | j	        j        d¬¦  «         t          j        ¦   «         5  | j	        j                             | j        ¦  «         | j	        j                             | j        ¦  «         ddd¦  «         dS # 1 swxY w Y   dS dS dS )zÙInitialize model weights.

        Args:
            needs_reset: If True, call reset_parameters() on modules that have it.
                Set to False when modules have already self-initialized in __init__.
        rò   Nç{®Gáz”?©Ústd)Úapplyr   Ú_init_weightsrû   r   rú   Úfix_init_weightrá   Ú
isinstancer  rS   rT   rm   r-   Úno_gradÚmul_rL   )r_   ró   s     r<   r  zBeit.init_weights@  s]  € ð 	�
Š
•7˜4Ô-¸;ÐGÑGÔGÑHÔHÐHØŒ>Ð%Ý˜$œ.¨cÐ2Ñ2Ô2Ð2Ý�d”n¨#Ð.Ñ.Ô.Ð.à×ÒÑÔÐàÔð 	:¥J¨t¬y½"¼)Ñ$DÔ$Dð 	:Ý˜$œ)Ô*°Ð4Ñ4Ô4Ð4Ý”‘”ð :ð :Ø”	Ô ×%Ò% dÔ&:Ñ;Ô;Ð;Ø”	”×#Ò# DÔ$8Ñ9Ô9Ð9ð:ð :ð :ñ :ô :ð :ð :ð :ð :ð :ð :ð :øøøð :ð :ð :ð :ð :ð :ð	:ð 	:ð 	:ð 	:s   ÃA	DÄDÄ"Dc                 óX  — t          j        ¦   «         5  t          | j        ¦  «        D ]g\  }}t	          j        d|dz   z  ¦  «        }|j        j        j         	                    |¦  «         |j
        j        j         	                    |¦  «         Œh	 ddd¦  «         dS # 1 swxY w Y   dS )z¤Fix initialization weights according to BEiT paper.

        Rescales attention and MLP weights based on layer depth to improve
        training stability.
        g       @r   N)r-   r  Ú	enumeraterÿ   ÚmathÚsqrtr…   r]   rm   Údiv_rº   Úfc2)r_   Úlayer_idÚlayerrR   s       r<   r  zBeit.fix_init_weightT  sç   € õ Œ]‰_Œ_ð 	1ð 	1Ý#,¨T¬[Ñ#9Ô#9ð 1ð 1‘�˜%Ýœ	 #¨°A©Ñ"6Ñ7Ô7�Ø”
”Ô&×+Ò+¨EÑ2Ô2Ð2Ø”	”Ô$×)Ò)¨%Ñ0Ô0Ð0Ð0ð1ð	1ð 	1ð 	1ñ 	1ô 	1ð 	1ð 	1ð 	1ð 	1ð 	1ð 	1ð 	1øøøð 	1ð 	1ð 	1ð 	1ð 	1ð 	1s   ”A=BÂB#Â&B#Úmc                 ó  — t          |t          j        ¦  «        rFt          |j        d¬¦  «         |j        �'t          j                             |j        d¦  «         dS dS |r&t          |d¦  «        r| 	                    ¦   «          dS dS dS )z©Initialize model weights.

        Args:
            m: Module to initialize.
            needs_reset: If True, call reset_parameters() on modules that have it.
        r  r	  Nr   r^   )
r  rS   rT   r   rm   rL   r‰   r¿   rù   r^   )r_   r  ró   s      r<   r  zBeit._init_weights`  s£   € õ �a�œÑ#Ô#ð 	!Ý˜!œ(¨Ð,Ñ,Ô,Ð,ØŒvÐ!Ý”×!Ò! !¤&¨!Ñ,Ô,Ð,Ð,Ð,ð "Ð!àð 	!�W QÐ(:Ñ;Ô;ð 	!Ø×ÒÑ Ô Ð Ð Ð ð	!ð 	!ð 	!ð 	!r>   c                 ót   — ddh}|                       ¦   «         D ]\  }}d|v r|                     |¦  «         Œ|S )z‘Get parameter names that should not use weight decay.

        Returns:
            Set of parameter names to exclude from weight decay.
        rû   rú   r[   )Únamed_parametersÚadd)r_   ÚnwdÚnÚ_s       r<   Úno_weight_decayzBeit.no_weight_decayn  sM   € ð ˜KÐ(ˆØ×)Ò)Ñ+Ô+ð 	ð 	‰DˆAˆqØ-°Ð2Ð2Ø—’˜‘
”
�
øØˆ
r>   Úenablec                 ó   — || _         dS )z}Enable or disable gradient checkpointing.

        Args:
            enable: If True, enable gradient checkpointing.
        N)r÷   )r_   r"  s     r<   Úset_grad_checkpointingzBeit.set_grad_checkpointing{  s   € ð #)ˆÔÐÐr>   Úcoarsec                 ó,   — t          dddg¬¦  «        }|S )zØCreate parameter group matcher for optimizer parameter groups.

        Args:
            coarse: If True, use coarse grouping.

        Returns:
            Dictionary mapping group names to regex patterns.
        z-^cls_token|pos_embed|patch_embed|rel_pos_bias)z^blocks\.(\d+)N)z^norm)iŸ† )Ústemrÿ   rï   )r_   r%  Úmatchers      r<   Úgroup_matcherzBeit.group_matcher„  s+   € õ ØAØ-Ð/CÐDð
ñ 
ô 
ˆð ˆr>   c                 ó   — | j         S )z_Get the classifier head.

        Returns:
            The classification head module.
        )r  rŒ   s    r<   Úget_classifierzBeit.get_classifier”  s   € ð ŒyÐr>   c                 óî   — t          | ¦  «        }|| _        |�|| _        |dk    rt          j        | j        |fi |¤Žnt          j        ¦   «         | _        | j                             | j	        ¦  «         dS )z�Reset the classification head.

        Args:
            num_classes: Number of classes for new head.
            global_pool: Global pooling type.
        Nr   )
r   rÕ   rÖ   rS   rT   r×   r·   r  Útrainrz   )r_   rÕ   rÖ   r`   s       r<   Úreset_classifierzBeit.reset_classifier�  sx   € õ ˜dÑ#Ô#ˆØ&ˆÔØÐ"Ø*ˆDÔØDOÐRSÂOÀO•B”I˜dœn¨kÐ@Ð@¸RÐ@Ð@Ð@ÕY[ÔYdÑYfÔYfˆŒ	ØŒ	�Š˜œÑ&Ô&Ð&Ð&Ð&r>   ÚNCHWrj   ÚindicesÚreturn_prefix_tokensr  Ú
stop_earlyÚ
output_fmtÚintermediates_onlyc                 ó¦  ‡ ‡‡‡— |dv s
J d¦   «         ‚|dk    }g }	t          t          ‰ j        ¦  «        |¦  «        \  }
}|j        \  Š}}}‰                      |¦  «        }t          j        ‰ j                             |j        d         dd¦  «        |fd¬¦  «        }‰ j	        �
|‰ j	        z   }‰  
                    |¦  «        }‰ j        �‰                      ¦   «         nd}t
          j                             ¦   «         s|s‰ j        }n‰ j        d|dz   …         }t          |¦  «        D ]z\  }}‰ j        r1t
          j                             ¦   «         st!          |||¬	¦  «        }n |||¬	¦  «        }||
v r,|	                     |r‰                      |¦  «        n|¦  «         Œ{‰ j        rˆ fd
„|	D ¦   «         }ˆ fd„|	D ¦   «         }	|r/‰ j                             ||f¦  «        \  ŠŠˆˆˆfd„|	D ¦   «         }	t
          j                             ¦   «         s|rt+          t-          |	|¦  «        ¦  «        }	|r|	S ‰                      |¦  «        }||	fS )a
  Forward pass that returns intermediate feature maps.

        Args:
            x: Input image tensor of shape (batch_size, channels, height, width).
            indices: Block indices to return features from. If int, returns last n blocks.
            return_prefix_tokens: If True, return both prefix and spatial tokens.
            norm: If True, apply normalization to intermediate features.
            stop_early: If True, stop at last selected intermediate.
            output_fmt: Output format ('NCHW' or 'NLC').
            intermediates_only: If True, only return intermediate features.

        Returns:
            If intermediates_only is True, returns list of intermediate tensors.
            Otherwise, returns tuple of (final_features, intermediates).
        )r/  ÚNLCz)Output format must be one of NCHW or NLC.r/  r   r,   r   rr   NrÁ   c                 ó6   •— g | ]}|d d …d‰j         …f         ‘ŒS )Nr   ©rö   ©rç   Úyr_   s     €r<   rê   z.Beit.forward_intermediates.<locals>.<listcomp>â  s/   ø€ ÐSÐSÐSÀ˜Q˜q˜q˜q ! DÔ$:Ð":Ð:Ô;ÐSÐSÐSr>   c                 ó6   •— g | ]}|d d …‰j         d …f         ‘ŒS r—   r8  r9  s     €r<   rê   z.Beit.forward_intermediates.<locals>.<listcomp>ã  s0   ø€ ÐRÐRÐR¸q˜Q˜q˜q˜q $Ô"8Ð"9Ð"9Ð9Ô:ÐRÐRÐRr>   c                 ó�   •— g | ]B}|                      ‰‰‰d ¦  «                             dddd¦  «                             ¦   «         ‘ŒCS )r,   r   r'   r   r&   )rw   r2   r3   )rç   r:  r~   ÚHÚWs     €€€r<   rê   z.Beit.forward_intermediates.<locals>.<listcomp>ç  sL   ø€ ÐlÐlÐlÐYZ˜QŸYšY q¨!¨Q°Ñ3Ô3×;Ò;¸A¸qÀ!ÀQÑGÔG×RÒRÑTÔTÐlÐlÐlr>   )r   Úlenrÿ   rs   rå   r-   rt   rú   Úexpandrû   rü   r„   r�   Úis_scriptingr  r÷   r   Úappendr  rö   Údynamic_feat_sizeÚlistÚzip)r_   rj   r0  r1  r  r2  r3  r4  rw   ÚintermediatesÚtake_indicesÚ	max_indexr   ÚheightÚwidthr„   rÿ   rè   ÚblkÚprefix_tokensr~   r=  r>  s   `                   @@@r<   Úforward_intermediateszBeit.forward_intermediates«  s¥  øøøø€ ð2 ˜_Ð,Ð,Ð,Ð.YÑ,Ô,Ð,Ø Ò&ˆØˆÝ"6µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�ið  œgÑˆˆ1ˆf�eØ×Ò˜QÑÔˆÝŒI�t”~×,Ò,¨Q¬W°Q¬Z¸¸RÑ@Ô@À!ÐDÈ!ÐLÑLÔLˆØŒ>Ð%Ø�D”NÑ"ˆAØ�MŠM˜!ÑÔˆà.2Ô.?Ð.K�t×(Ò(Ñ*Ô*Ð*ÐQUˆÝŒ9×!Ò!Ñ#Ô#ð 	1¨:ð 	1Ø”[ˆFˆFà”[  )¨a¡- Ô0ˆFÝ Ñ'Ô'ð 	Bð 	B‰FˆAˆsØÔ&ð =­u¬y×/EÒ/EÑ/GÔ/Gð =Ý˜s A¸<ÐHÑHÔH��à�C˜¨|Ð<Ñ<Ô<�Ø�LÐ Ð à×$Ò$°TÐ%@ T§Y¢Y¨q¡\¤\ \¸qÑAÔAÐAøð Ô!ð 	SàSÐSÐSÐSÀ]ÐSÑSÔSˆMØRÐRÐRÐRÀMÐRÑRÔRˆMØð 	màÔ#×5Ò5°v¸u°oÑFÔF‰DˆAˆqØlÐlÐlÐlÐlÐlÐ^kÐlÑlÔlˆMÝŒy×%Ò%Ñ'Ô'ð 	DÐ,@ð 	Då ¥ ]°MÑ!BÔ!BÑCÔCˆMàð 	!Ø Ð à�IŠI�a‰LŒLˆà�-ÐÐr>   r   Ú
prune_normÚ
prune_headc                 ó  — t          t          | j        ¦  «        |¦  «        \  }}| j        d|dz   …         | _        |rt          j        ¦   «         | _        |r.t          j        ¦   «         | _        |                      dd¦  «         |S )a=  Prune layers not required for specified intermediate outputs.

        Args:
            indices: Indices of blocks to keep.
            prune_norm: If True, remove final normalization.
            prune_head: If True, remove classification head.

        Returns:
            List of indices that were kept.
        Nr   r   Ú )r   r?  rÿ   rS   r·   r  r  r.  )r_   r0  rN  rO  rG  rH  s         r<   Úprune_intermediate_layerszBeit.prune_intermediate_layersó  s~   € õ  #7µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�iØ”k . 9¨q¡= .Ô1ˆŒØð 	&Ýœ™œˆDŒIØð 	)Ýœ;™=œ=ˆDŒLØ×!Ò! ! RÑ(Ô(Ð(ØÐr>   c                 óø  — |                       |¦  «        }t          j        | j                             |j        d         dd¦  «        |fd¬¦  «        }| j        �
|| j        z   }|                      |¦  «        }| j        �|                      ¦   «         nd}| j	        D ]G}| j
        r1t          j                             ¦   «         st          |||¬¦  «        }Œ: |||¬¦  «        }ŒH|                      |¦  «        }|S )zîForward pass through feature extraction layers.

        Args:
            x: Input tensor of shape (batch_size, channels, height, width).

        Returns:
            Feature tensor of shape (batch_size, num_tokens, embed_dim).
        r   r,   r   rr   NrÁ   )rå   r-   rt   rú   r@  rs   rû   rü   r„   rÿ   r÷   r�   rA  r   r  )r_   rj   r„   rK  s       r<   Úforward_featureszBeit.forward_features  sÿ   € ð ×Ò˜QÑÔˆÝŒI�t”~×,Ò,¨Q¬W°Q¬Z¸¸RÑ@Ô@À!ÐDÈ!ÐLÑLÔLˆØŒ>Ð%Ø�D”NÑ"ˆAØ�MŠM˜!ÑÔˆà.2Ô.?Ð.K�t×(Ò(Ñ*Ô*Ð*ÐQUˆØ”;ð 	=ð 	=ˆCØÔ&ð =­u¬y×/EÒ/EÑ/GÔ/Gð =Ý˜s A¸<ÐHÑHÔH��à�C˜¨|Ð<Ñ<Ô<��Ø�IŠI�a‰LŒLˆØˆr>   Ú
pre_logitsc                 ó  — | j         r>| j         dk    r'|dd…| j        d…f                              d¬¦  «        n|dd…df         }|                      |¦  «        }|                      |¦  «        }|r|n|                      |¦  «        S )a7  Forward pass through classification head.

        Args:
            x: Feature tensor of shape (batch_size, num_tokens, embed_dim).
            pre_logits: If True, return features before final linear layer.

        Returns:
            Logits tensor of shape (batch_size, num_classes) or pre-logits.
        rÏ   Nr   rr   r   )rÖ   rö   Úmeanr  r  r  )r_   rj   rU  s      r<   Úforward_headzBeit.forward_head$  sœ   € ð Ôð 	dØ=AÔ=MÐQVÒ=VÐ=V��!�!�!�TÔ+Ð,Ð,Ð,Ô-×2Ò2°qÐ2Ñ9Ô9Ð9Ð\]Ð^_Ð^_Ð^_ÐabÐ^bÔ\cˆAØ�LŠL˜‰OŒOˆØ�NŠN˜1ÑÔˆØÐ0ˆqˆq D§I¢I¨a¡L¤LÐ0r>   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S )zÓForward pass through the model.

        Args:
            x: Input tensor of shape (batch_size, channels, height, width).

        Returns:
            Logits tensor of shape (batch_size, num_classes).
        )rT  rX  )r_   rj   s     r<   r†   zBeit.forward4  s/   € ð ×!Ò! !Ñ$Ô$ˆØ×Ò˜aÑ Ô ˆØˆr>   )Tr˜   ©Fr—   )NFFFr/  F)r   FT)&r™   rš   r›   rœ   r   r   r¡   r
   ÚstrrŸ   r¢   r   rS   rÃ   r   rQ   r  r  r  r-   r�   Úignorer	   r!  r$  r   r   r)  r+  r.  r£   r   rM  rR  rT  rX  r†   r¤   r¥   s   @r<   r"   r"   Á  sÆ  ø€ € € € € ðð ð 58Ø68ØØ#Ø$Ø ØØØ!Ø!Ø$Ø#Ø!Ø#%Ø$&Ø$&Ø$&Ø*3Ø+/Ø$(Ø%*Ø,1Ø%*ØØð5u-ð u-à˜C  s¨C x¤Ð0Ô1ðu-ð ˜c 5¨¨c¨¤?Ð2Ô3ðu-ð ð	u-ð
 ðu-ð ðu-ð ðu-ð ðu-ð ðu-ð ðu-ð ðu-ð ðu-ð ðu-ð ðu-ð !ðu-ð  "ð!u-ð" "ð#u-ð$ "ð%u-ð& ˜RœYœð'u-ð( " %œð)u-ð* "ð+u-ð, #ð-u-ð. &*ð/u-ð0 #ð1u-ð u-ð u-ð u-ð u-ð u-ðn:ð :¨ð :¸ð :ð :ð :ð :ð(
1ð 
1ð 
1ð 
1ð!ð !˜rœyð !°tð !ð !ð !ð !ð „YÔð
  S¤ð 
ð 
ð 
ñ Ôð
ð „YÔð)ð )¨Tð )ð )ð )ñ Ôð)ð „YÔðð  Dð °T¸#¸s¸(´^ð ð ð ñ Ôðð „YÔð ¤	ð ð ð ñ Ôðð'ð '¨Cð '¸hÀs¼mð 'ð 'ð 'ð 'ð" 8<Ø).ØØ$Ø$Ø',ðF ð F àŒ|ðF ð ˜e C¨¨c¬ NÔ3Ô4ðF ð #'ð	F ð
 ðF ð ðF ð ðF ð !%ðF ð 
ˆt�E”LÔ! 5¨¬°t¸E¼LÔ7IÐ)IÔ#JÐJÔ	KðF ð F ð F ð F ðT ./Ø$Ø#ð	ð à˜3  S¤	˜>Ô*ðð ðð ð	ð
 
ˆcŒðð ð ð ð2 %¤,ð °5´<ð ð ð ð ð01ð 1˜eœlð 1¸ð 1ÈÌð 1ð 1ð 1ð 1ð ˜œð ¨%¬,ð ð ð ð ð ð ð ð r>   rQ  Úurlc                 ó$   — | dddddddddd	d
dœ|¥S )zÑCreate a default configuration dictionary for BEiT models.

    Args:
        url: Model weights URL.
        **kwargs: Additional configuration parameters.

    Returns:
        Configuration dictionary.
    rÎ   )r'   rÌ   rÌ   NgÍÌÌÌÌÌì?ÚbicubicT)ç      à?r`  r`  zpatch_embed.projr  z
apache-2.0)r]  rÕ   Ú
input_sizeÚ	pool_sizeÚcrop_pctÚinterpolationÚfixed_input_sizerW  r
  Ú
first_convÚ
classifierÚlicenserµ   )r]  Úkwargss     r<   Ú_cfgrj  B  s;   € ð Ø¨=ÀtØ¨ÈØ¨Ø(¸Øðð ð ðð r>   ztimm/)Ú	hf_hub_id)r'   é€  rl  g      ð?)rk  ra  rc  iQU  )rk  rÕ   )r'   é   rm  )rk  rW  r
  )rk  rÕ   rW  r
  gffffffî?)rk  rc  rW  r
  )z)beit_base_patch16_224.in22k_ft_in22k_in1kz)beit_base_patch16_384.in22k_ft_in22k_in1kz$beit_base_patch16_224.in22k_ft_in22kz*beit_large_patch16_224.in22k_ft_in22k_in1kz*beit_large_patch16_384.in22k_ft_in22k_in1kz*beit_large_patch16_512.in22k_ft_in22k_in1kz%beit_large_patch16_224.in22k_ft_in22kz*beitv2_base_patch16_224.in1k_ft_in22k_in1kz$beitv2_base_patch16_224.in1k_ft_in1kz%beitv2_base_patch16_224.in1k_ft_in22kz+beitv2_large_patch16_224.in1k_ft_in22k_in1kz%beitv2_large_patch16_224.in1k_ft_in1kz&beitv2_large_patch16_224.in1k_ft_in22kr_  TÚ
state_dictÚmodelrd  Ú	antialiasc           	      óø  — |                       d| ¦  «        } |                       d| ¦  «        } i }|                      ¦   «         D �]5\  }}d|v rŒd|v rT|j        j        j        j        \  }}}	}
|j        d         |
k    s|j        d         |	k    rt          ||	|
f||d¬¦  «        }nÍ|d	k    rC|j        d
         |j        j        d
         k    r"d
}t          ||j        j	        |||d¬¦  «        }n„| 
                    d¦  «        ro|                     |dd…         ¦  «        }|j        |j        j        k    s|j        d         |j        d
         k    r!t          ||j        |j        j        ¬¦  «        }|||<   �Œ7|S )aÞ  Filter and process checkpoint state dict for loading.

    Handles resizing of patch embeddings, position embeddings, and relative position
    bias tables when model size differs from checkpoint.

    Args:
        state_dict: Checkpoint state dictionary.
        model: Target model to load weights into.
        interpolation: Interpolation method for resizing.
        antialias: If True, use antialiasing when resizing.

    Returns:
        Filtered state dictionary.
    ro  rì   r;   zpatch_embed.proj.weightr,   rq   T)rd  rp  Úverboserû   r   )Únew_sizerö   rd  rp  rr  r[   Niãÿÿÿr   )Únew_window_sizeÚnew_bias_shape)ÚgetÚitemsrå   r]   rm   rs   r   rû   r   ræ   ÚendswithÚget_submoduler[   r#   r   )rn  ro  rd  rp  Úout_dictr‚   rƒ   ÚOÚIr=  r>  rö   r  s                r<   Úcheckpoint_filter_fnr}  ™  s¾  € ð —’ ¨Ñ4Ô4€JØ—’ ¨*Ñ5Ô5€Jð €HØ× Ò Ñ"Ô"ð  ñ  ‰ˆˆ1Ø$¨Ð)Ð)ØØ$¨Ð)Ð)ØÔ*Ô/Ô6Ô<‰JˆAˆq�!�QØŒw�rŒ{˜aÒÐ 1¤7¨2¤;°!Ò#3Ð#3Ý(ØØ˜�FØ"/Ø'Ø ðñ ô �øð �+ÒÐ !¤'¨!¤*°´Ô0EÀaÔ0HÒ"HÐ"Hà !ÐÝ&ØØÔ*Ô4Ø"3Ø+Ø#Øðñ ô ˆAˆAð �ZŠZÐ6Ñ7Ô7ð 	Ø×#Ò# A d s d¤GÑ,Ô,ˆAØŒw˜!Ô8Ô>Ò>Ð>À!Ä-ÐPQÔBRÐVWÔVcÐdeÔVfÒBfÐBfÝ-ØØ$%¤MØ#$Ô#AÔ#Gðñ ô �ð
 ˆ�‰‰Ø€Or>   FÚvariantÚ
pretrainedc                 óˆ   — |                      dd¦  «        }t          t          | |ft          t	          |d¬¦  «        dœ|¤Ž}|S )zØCreate a BEiT model.

    Args:
        variant: Model variant name.
        pretrained: If True, load pretrained weights.
        **kwargs: Additional model arguments.

    Returns:
        BEiT model instance.
    Úout_indicesr'   Úgetter)r�  Úfeature_cls)Úpretrained_filter_fnÚfeature_cfg)Úpopr   r"   r}  rð   )r~  r  ri  r�  ro  s        r<   Ú_create_beitr‡  Ñ  s[   € ð —*’*˜]¨AÑ.Ô.€KÝ Ýˆg�zðå1Ý [¸hÐGÑGÔGðð ð ð	ð €Eð €Lr>   c           
      óf   — t          dddddddd¬¦  «        }t          dd
| it          |fi |¤Ž¤Ž}|S )z0BEiT base model @ 224x224 with patch size 16x16.rÍ   rÐ   rÑ   rn   FTçš™™™™™¹?©rÓ   r×   rØ   rE   r©   rÞ   rß   r­   Úbeit_base_patch16_224r  )r‹  ©rð   r‡  ©r  ri  Ú
model_argsro  s       r<   r‹  r‹  æ  s^   € õ Ø ¨B¸"ÈØ°À#ðGñ Gô G€Jõ ÐfÐf¸ZÐfÍ4ÐPZÐKeÐKeÐ^dÐKeÐKeÐfÐf€EØ€Lr>   c           
      óf   — t          dddddddd¬¦  «        }t          dd
| it          |fi |¤Ž¤Ž}|S )z0BEiT base model @ 384x384 with patch size 16x16.rl  rÍ   rÐ   rÑ   FTr‰  ©rÒ   rÓ   r×   rØ   rE   rÞ   rß   r­   Úbeit_base_patch16_384r  )r‘  rŒ  r�  s       r<   r‘  r‘  ð  s^   € õ Ø ¨s¸"ÈØ°À#ðGñ Gô G€Jõ ÐfÐf¸ZÐfÍ4ÐPZÐKeÐKeÐ^dÐKeÐKeÐfÐf€EØ€Lr>   c           	      ód   — t          ddddddd¬¦  «        }t          d
d	| it          |fi |¤Ž¤Ž}|S )z1BEiT large model @ 224x224 with patch size 16x16.rÍ   é   é   FTçñhãˆµøä>©rÓ   r×   rØ   rE   rÞ   rß   r­   Úbeit_large_patch16_224r  )r—  rŒ  r�  s       r<   r—  r—  ú  s\   € õ Ø ¨R¸2Ø°À$ðHñ Hô H€Jõ ÐgÐg¸jÐgÍDÐQ[ÐLfÐLfÐ_eÐLfÐLfÐgÐg€EØ€Lr>   c           
      óf   — t          dddddddd¬¦  «        }t          dd
| it          |fi |¤Ž¤Ž}|S )z1BEiT large model @ 384x384 with patch size 16x16.rl  rÍ   r“  r”  FTr•  r�  Úbeit_large_patch16_384r  )r™  rŒ  r�  s       r<   r™  r™    ó^   € õ Ø ¨t¸2ÈØ°À$ðHñ Hô H€Jõ ÐgÐg¸jÐgÍDÐQ[ÐLfÐLfÐ_eÐLfÐLfÐgÐg€EØ€Lr>   c           
      óf   — t          dddddddd¬¦  «        }t          dd
| it          |fi |¤Ž¤Ž}|S )z1BEiT large model @ 512x512 with patch size 16x16.rm  rÍ   r“  r”  FTr•  r�  Úbeit_large_patch16_512r  )rœ  rŒ  r�  s       r<   rœ  rœ    rš  r>   c           
      óf   — t          dddddddd¬¦  «        }t          dd
| it          |fi |¤Ž¤Ž}|S )z3BEiT v2 base model @ 224x224 with patch size 16x16.rÍ   rÐ   rÑ   rn   FTr•  rŠ  Úbeitv2_base_patch16_224r  )rž  rŒ  r�  s       r<   rž  rž    s^   € õ Ø ¨B¸"ÈØ°À$ðHñ Hô H€Jõ ÐhÐh¸zÐhÍTÐR\ÐMgÐMgÐ`fÐMgÐMgÐhÐh€EØ€Lr>   c           	      ód   — t          ddddddd¬¦  «        }t          d
d	| it          |fi |¤Ž¤Ž}|S )z4BEiT v2 large model @ 224x224 with patch size 16x16.rÍ   r“  r”  FTr•  r–  Úbeitv2_large_patch16_224r  )r   rŒ  r�  s       r<   r   r   "  s\   € õ Ø ¨R¸2Ø°À$ðHñ Hô H€Jõ ÐiÐiÀ
ÐiÍdÐS]ÐNhÐNhÐagÐNhÐNhÐiÐi€EØ€Lr>   r—   )rQ  )r_  TrZ  )Drœ   r  Ú	functoolsr   Útypingr   r   r   r   r   r	   r
   r   r   r-   Útorch.nnrS   Útorch.nn.functionalÚ
functionalru   Ú	timm.datar   r   Útimm.layersr   r   r   r   r   r   r   r   r   r   r   r   r   Ú_builderr   Ú	_featuresr   Ú_manipulater   Ú	_registryr    r!   Ú__all__r¡   r£   r=   rÃ   r@   r§   rÅ   r"   r[  rj  Údefault_cfgsrŸ   r}  r‡  r‹  r‘  r—  r™  rœ  rž  r   rµ   r>   r<   ú<module>r®     s™  ððð ðP €€€Ø Ð Ð Ð Ð Ð Ø OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ Oà €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð à AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð  +Ð *Ð *Ð *Ð *Ð *Ø +Ð +Ð +Ð +Ð +Ð +Ø #Ð #Ð #Ð #Ð #Ð #Ø <Ð <Ð <Ð <Ð <Ð <Ð <Ð <àˆ(€ð #ð  #¨U°3¸°8¬_ð  #ÈeÌlð  #ð  #ð  #ð  #ðFfð fð fð fð f�”	ñ fô fð fðRqð qð qð qð qˆBŒIñ qô qð qðh4Dð 4Dð 4Dð 4Dð 4D˜2œ9ñ 4Dô 4Dð 4Dðn~ð ~ð ~ð ~ð ~ˆ2Œ9ñ ~ô ~ð ~ðBð ˆcð  T¨#¨s¨(¤^ð ð ð ð ð* %Ð$Ø15°àð2ñ 2ô 2ð 26°àØ ¨3ð2ñ 2ô 2ð
 -1¨DàØð-ñ -ô -ð
 37°$àð3ñ 3ô 3ð 37°$àØ ¨3ð3ñ 3ô 3ð
 37°$àØ ¨3ð3ñ 3ô 3ð
 .2¨TàØð.ñ .ô .ð 37°$àØ"Ð(<ð3ñ 3ô 3ð
 -1¨DàØ"Ð(<ð-ñ -ô -ð
 .2¨TàØÐ 5Ð;Oð.ñ .ô .ð
 48°4àØÐ1Ð7Kð4ñ 4ô 4ð
 .2¨TàØÐ1Ð7Kð.ñ .ô .ð
 /3¨dàØÐ 5Ð;Oð/ñ /ô /ðu?&ð ?&ñ ?ô ?€ðD5ð 5 T¨#¨u¬|Ð*;Ô%<ð 5ÀRÄYð 5Ð_bð 5Ð{ð 5ð  LPð  QTð  V[ô  Vbð  Qbô  Lcð 5ð 5ð 5ð 5ðpð ˜#ð ¨4ð Àdð ð ð ð ð* ðð  dð Àð ð ð ñ „ðð ðð  dð Àð ð ð ñ „ðð ðð  tð À$ð ð ð ñ „ðð ðð  tð À$ð ð ð ñ „ðð ðð  tð À$ð ð ð ñ „ðð ðð ¨ð À4ð ð ð ñ „ðð ðð ¨ð ÀDð ð ð ñ „ðð ð r>   