§
    ­¸j ù  ã                  ó  — d Z ddlmZ ddlZddlZddlmZmZ ddl	Z	ddl	m
Z
mZ ddlmZmZmZ ddlmZmZ ddlmZ dd	lmZ dd
lmZmZmZmZ ddlmZ ddlmZ ddl m!Z!m"Z"m#Z#m$Z$m%Z%  e#j&        e'¦  «        Z(dVd„Z)dWd„Z*dXd „Z+d!Z, G d"„ d!ee¦  «        Z- G d#„ d$ej.        ¦  «        Z/ G d%„ d&ej.        ¦  «        Z0 G d'„ d(ej.        ¦  «        Z1 G d)„ d*e1¦  «        Z2 G d+„ d,ej.        ¦  «        Z3 G d-„ d.ej.        ¦  «        Z4 G d/„ d0e4¦  «        Z5 G d1„ d2ej.        ¦  «        Z6dYdZd;„Z7 G d<„ d=ej.        ¦  «        Z8 G d>„ d?ej.        ¦  «        Z9 G d@„ dAej.        ¦  «        Z:e4e5dBœZ; G dC„ dDej.        ¦  «        Z< G dE„ dFej.        ¦  «        Z= G dG„ dHe¦  «        Z>dIZ?dJZ@ e!dKe?¦  «         G dL„ dMe>¦  «        ¦   «         ZAdNZB e!dOe?¦  «         G dP„ dQe>¦  «        ¦   «         ZC e!dRe?¦  «         G dS„ dTe>e¦  «        ¦   «         ZDg dU¢ZEdS )[aº  DINOv2-with-Registers backbone with windowed self-attention.

This module is a local copy of the HuggingFace Transformers DINOv2-with-Registers implementation, extended with windowed
attention support for RF-DETR.  It targets the transformers v5 API (``transformers>=5.0.0``).

Transformers v5 API changes vs v4
----------------------------------
``head_mask`` removed:
    The ``head_mask`` parameter that appeared on every ``forward()`` in v4 has been
    dropped in v5.  It defaulted to ``None`` throughout the call chain and callers universally passed ``None``, so
    removing it produces **identical numerics**. Permanent head pruning is still available via ``model._prune_heads()``.

``BackboneMixin._init_transformers_backbone`` signature:
    In v4 this method accepted ``(self, config)``.  In v5 it accepts only ``(self)``;
    the config is accessed via ``self.config`` internally.

Helper functions copied locally:
    ``get_aligned_output_features_output_indices`` and ``find_pruneable_heads_and_indices`` were removed
    from the transformers v5 public API.  Private copies (``_get_aligned_output_features_output_indices``
    and ``_find_pruneable_heads_and_indices``) are kept in this module.
é    )ÚannotationsN)ÚAnyÚcast)ÚTensorÚnn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELoss)ÚBackboneConfigMixinÚBackboneMixin)ÚACT2FN)ÚPretrainedConfig)ÚBackboneOutputÚBaseModelOutputÚBaseModelOutputWithPoolingÚImageClassifierOutput)ÚPreTrainedModel)Úprune_linear_layer)Úadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚloggingÚreplace_return_docstringsÚ	torch_intÚheadsúset[int]Ún_headsÚintÚ	head_sizeÚalready_pruned_headsÚreturnútuple[set[int], Tensor]c                ó�  ‡— t          j        ||¦  «        }t          | ¦  «        |z
  } | D ]%Š‰t          ˆfd„|D ¦   «         ¦  «        z  Šd|‰<   Œ&|                     d¦  «                             ¦   «                              d¦  «        }t          j        t          |¦  «        ¦  «        |          	                    ¦   «         }| |fS )al  Return the set of pruneable heads and their index mask for weight pruning.

    Copied from transformers.pytorch_utils.find_pruneable_heads_and_indices (removed from public API in transformers
    v5.0).
    Source: https://github.com/huggingface/transformers/blob/v4.49.0/src/transformers/pytorch_utils.py#L127
    MAINTENANCE: if this function is moved to another module or deleted, update the
    "Copyright 2022 The HuggingFace Team" line in the file header accordingly.

    Args:
        heads: Indices of heads to prune.
        n_heads: Total number of heads in the layer.
        head_size: Size of each attention head.
        already_pruned_heads: Heads that have already been pruned.

    Returns:
        A tuple of (heads, index) where heads is the adjusted set of head indices and index is a LongTensor boolean mask
        selecting the remaining weights.
    c              3  ó,   •K  — | ]}|‰k     rd ndV — ŒdS )é   r   N© )Ú.0ÚhÚheads     €ú€/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/rfdetr/models/backbone/dinov2_with_windowed_attn.pyú	<genexpr>z4_find_pruneable_heads_and_indices.<locals>.<genexpr>\   s/   øè è € ÐGÐG¨Q˜˜Tš˜�A�A qÐGÐGÐGÐGÐGÐGó    r   éÿÿÿÿr$   )
ÚtorchÚonesÚsetÚsumÚviewÚ
contiguousÚeqÚarangeÚlenÚlong)r   r   r   r   ÚmaskÚindexr(   s         @r)   Ú!_find_pruneable_heads_and_indicesr9   D   sÂ   ø€ õ* Œ:�g˜yÑ)Ô)€DÝ�‰JŒJÐ-Ñ-€EØð ð ˆØ•ÐGÐGÐGÐGÐ2FÐGÑGÔGÑGÔGÑGˆØˆˆT‰
ˆ
Ø�9Š9�R‰=Œ=×#Ò#Ñ%Ô%×(Ò(¨Ñ+Ô+€DÝŒL�˜T™œÑ#Ô# DÔ)×.Ò.Ñ0Ô0€EØ�%ˆ<Ðr+   Úout_featuresúlist[str] | NoneÚout_indicesú"list[int] | tuple[int, ...] | NoneÚstage_namesú	list[str]ú;tuple[list[str] | None, list[int] | tuple[int, ...] | None]c                ó˜   ‡— |€| €t          ‰¦  «        dz
  g}‰d         g} n%|€| �ˆfd„| D ¦   «         }n| €|�ˆfd„|D ¦   «         } | |fS )Nr$   r,   c                ó:   •— g | ]}‰                      |¦  «        ‘ŒS r%   )r8   )r&   Úlayerr>   s     €r)   ú
<listcomp>z9_align_output_features_output_indices.<locals>.<listcomp>l   s'   ø€ ÐJÐJÐJ°E�{×(Ò(¨Ñ/Ô/ÐJÐJÐJr+   c                ó    •— g | ]
}‰|         ‘ŒS r%   r%   )r&   Úidxr>   s     €r)   rD   z9_align_output_features_output_indices.<locals>.<listcomp>n   s   ø€ Ð@Ð@Ð@¨S˜ CÔ(Ð@Ð@Ð@r+   )r5   ©r:   r<   r>   s     `r)   Ú%_align_output_features_output_indicesrH   c   sˆ   ø€ ð
 Ð˜|Ð3Ý˜;Ñ'Ô'¨!Ñ+Ð,ˆØ# BœÐ(ˆˆØ	Ð	 Ð!9ØJÐJÐJÐJ¸\ÐJÑJÔJˆˆØ	Ð	 +Ð"9Ø@Ð@Ð@Ð@°KÐ@Ñ@Ô@ˆØ˜Ð$Ð$r+   útuple[list[str], list[int]]c                ó¦   — |�t          |¦  «        nd}t          | ||¬¦  «        \  } }| €J ‚|€J ‚| t          t           t                   |¦  «        fS )a¸  Align out_features and out_indices against stage_names, filling in defaults when either is None.

    Copied from transformers.utils.backbone_utils.get_aligned_output_features_output_indices (removed from public API in
    transformers v5.0).
    Source: https://github.com/huggingface/transformers/blob/v4.49.0/src/transformers/utils/backbone_utils.py#L30
    MAINTENANCE: if this function is moved to another module or deleted, update the
    "Copyright 2023 The HuggingFace Inc. team" line in the file header accordingly.

    Args:
        out_features: Names of the backbone stages to return features from, or None to derive from out_indices.
        out_indices: Integer indices of the stages to return features from, or None to derive from out_features.
        stage_names: Ordered list of all stage names defined by the backbone config.

    Returns:
        A tuple of (out_features, out_indices) with both fields populated consistently.
    NrG   )ÚlistrH   r   r   rG   s      r)   Ú+_get_aligned_output_features_output_indicesrL   r   sn   € ð* (3Ð'>•$�{Ñ#Ô#Ð#ÀD€KÝ EØ!¨{Èð!ñ !ô !Ñ€L�+ð
 Ð#Ð#Ð#ØÐ"Ð"Ð"Ø��d¥3œi¨Ñ5Ô5Ð5Ð5r+   Ú!WindowedDinov2WithRegistersConfigc                  óZ   ‡ — e Zd ZdZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d6d7ˆ fd5„Zˆ xZS )8rM   aS  
    This is the configuration class to store the configuration of a [`Dinov2WithRegistersModel`].
    It is used to instantiate a Dinov2WithRegisters model according to the specified arguments,
    defining the model architecture. Instantiating a configuration with the defaults will yield a
    similar configuration to that of the DINOv2 with Registers
    [facebook/dinov2-with-registers-base](https://huggingface.co/facebook/dinov2-with-registers-base) architecture.

    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PretrainedConfig`] for more information.

    Args:
        hidden_size (`int`, *optional*, defaults to 768):
            Dimensionality of the encoder layers and the pooler layer.
        num_hidden_layers (`int`, *optional*, defaults to 12):
            Number of hidden layers in the Transformer encoder.
        num_attention_heads (`int`, *optional*, defaults to 12):
            Number of attention heads for each attention layer in the Transformer encoder.
        mlp_ratio (`int`, *optional*, defaults to 4):
            Ratio of the hidden size of the MLPs relative to the `hidden_size`.
        hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
            `"relu"`, `"selu"` and `"gelu_new"` are supported.
        hidden_dropout_prob (`float`, *optional*, defaults to 0.0):
            The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
        attention_probs_dropout_prob (`float`, *optional*, defaults to 0.0):
            The dropout ratio for the attention probabilities.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
        layer_norm_eps (`float`, *optional*, defaults to 1e-06):
            The epsilon used by the layer normalization layers.
        image_size (`int`, *optional*, defaults to 224):
            The size (resolution) of each image.
        patch_size (`int`, *optional*, defaults to 16):
            The size (resolution) of each patch.
        num_channels (`int`, *optional*, defaults to 3):
            The number of input channels.
        qkv_bias (`bool`, *optional*, defaults to `True`):
            Whether to add a bias to the queries, keys and values.
        layerscale_value (`float`, *optional*, defaults to 1.0):
           Initial value to use for layer scale.
        drop_path_rate (`float`, *optional*, defaults to 0.0):
            Stochastic depth rate per sample (when applied in the main path of residual layers).
        use_swiglu_ffn (`bool`, *optional*, defaults to `False`):
            Whether to use the SwiGLU feedforward neural network.
        num_register_tokens (`int`, *optional*, defaults to 4):
            Number of register tokens to use.
        out_features (`List[str]`, *optional*):
            If used as backbone, list of features to output. Can be any of `"stem"`, `"stage1"`, `"stage2"`, etc.
            (depending on how many stages the model has). If unset and `out_indices` is set, will default to the
            corresponding stages. If unset and `out_indices` is unset, will default to the last stage. Must be in the
            same order as defined in the `stage_names` attribute.
        out_indices (`List[int]`, *optional*):
            If used as backbone, list of indices of features to output. Can be any of 0, 1, 2, etc. (depending on how
            many stages the model has). If unset and `out_features` is set, will default to the corresponding stages.
            If unset and `out_features` is unset, will default to the last stage. Must be in the
            same order as defined in the `stage_names` attribute.
        apply_layernorm (`bool`, *optional*, defaults to `True`):
            Whether to apply layer normalization to the feature maps in case the model is used as backbone.
        reshape_hidden_states (`bool`, *optional*, defaults to `True`):
            Whether to reshape the feature maps to 4D tensors of shape `(batch_size, hidden_size, height, width)` in
            case the model is used as backbone. If `False`, the feature maps will be 3D tensors of shape `(batch_size,
            seq_len, hidden_size)`.

    Example:

    >>> from rfdetr.models.backbone.dinov2_with_windowed_attn import WindowedDinov2WithRegistersConfig

    >>> # Initializing a tiny configuration suitable for doctests
    >>> configuration = WindowedDinov2WithRegistersConfig(
    ...     image_size=32,
    ...     patch_size=16,
    ...     hidden_size=32,
    ...     num_hidden_layers=2,
    ...     num_attention_heads=4,
    ...     num_register_tokens=2,
    ... )

    >>> configuration.hidden_size
    32

    Údinov2_with_registersé   é   é   Úgeluç        ç{®Gáz”?ç�íµ ÷Æ°>éà   é   é   Tç      ð?FNr$   Úhidden_sizer   Únum_hidden_layersÚnum_attention_headsÚ	mlp_ratioÚ
hidden_actÚstrÚhidden_dropout_probÚfloatÚattention_probs_dropout_probÚinitializer_rangeÚlayer_norm_epsÚ
image_sizeÚ
patch_sizeÚnum_channelsÚqkv_biasÚboolÚlayerscale_valueÚdrop_path_rateÚuse_swiglu_ffnÚnum_register_tokensr:   r;   r<   r=   Úapply_layernormÚreshape_hidden_statesÚnum_windowsÚwindow_block_indexesúlist[int] | NoneÚgradient_checkpointingÚkwargsr   r    ÚNonec                ó>  •—  t          ¦   «         j        di |¤Ž || _        || _        || _        || _        || _        || _        || _        || _	        |	| _
        |
| _        || _        || _        || _        || _        || _        || _        || _        dgd„ t'          d|dz   ¦  «        D ¦   «         z   | _        t+          ||| j        ¬¦  «        \  | _        | _        || _        || _        || _        |€t7          t'          |¦  «        ¦  «        n|| _        || _        d S )NÚstemc                ó   — g | ]}d |› �‘ŒS )Ústager%   )r&   rF   s     r)   rD   z>WindowedDinov2WithRegistersConfig.__init__.<locals>.<listcomp>  s   € Ð&`Ð&`Ð&`¸ }¨s } }Ð&`Ð&`Ð&`r+   r$   rG   r%   )ÚsuperÚ__init__r[   r\   r]   r^   r_   ra   rc   rd   re   rf   rg   rh   ri   rk   rl   rm   rn   Úranger>   rL   Ú_out_featuresÚ_out_indicesro   rp   rq   rK   rr   rt   )Úselfr[   r\   r]   r^   r_   ra   rc   rd   re   rf   rg   rh   ri   rk   rl   rm   rn   r:   r<   ro   rp   rq   rr   rt   ru   Ú	__class__s                             €r)   r|   z*WindowedDinov2WithRegistersConfig.__init__ë   sN  ø€ ð8 	�‰ŒÔÐ"Ð"˜6Ð"Ð"Ð"à&ˆÔØ!2ˆÔØ#6ˆÔ Ø"ˆŒØ$ˆŒØ#6ˆÔ Ø,HˆÔ)Ø!2ˆÔØ,ˆÔØ$ˆŒØ$ˆŒØ(ˆÔØ ˆŒØ 0ˆÔØ,ˆÔØ,ˆÔØ#6ˆÔ Ø"˜8Ð&`Ð&`ÅÀaÐIZÐ]^ÑI^Ñ@_Ô@_Ð&`Ñ&`Ô&`Ñ`ˆÔÝ0[Ø%°;ÈDÔL\ð1
ñ 1
ô 1
Ñ-ˆÔ˜DÔ-ð  /ˆÔØ%:ˆÔ"Ø&ˆÔà.BÐ.J�D•Ð(Ñ)Ô)Ñ*Ô*Ð*ÐPdð 	Ô!ð '=ˆÔ#Ð#Ð#r+   )rP   rQ   rQ   rR   rS   rT   rT   rU   rV   rW   rX   rY   TrZ   rT   FrR   NNTTr$   NF)4r[   r   r\   r   r]   r   r^   r   r_   r`   ra   rb   rc   rb   rd   rb   re   rb   rf   r   rg   r   rh   r   ri   rj   rk   rb   rl   rb   rm   rj   rn   r   r:   r;   r<   r=   ro   rj   rp   rj   rq   r   rr   rs   rt   rj   ru   r   r    rv   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typer|   Ú__classcell__©r�   s   @r)   rM   rM   –   s    ø€ € € € € ðPð Pðd )€Jð Ø!#Ø#%ØØ Ø%(Ø.1Ø#'Ø $ØØØØØ"%Ø #Ø$Ø#$Ø)-Ø:>Ø $Ø&*ØØ15Ø',ð39=ð 9=ð 9=ð 9=ð 9=ð 9=ð 9=ð 9=ð 9=ð 9=ð 9=r+   c                  ó,   ‡ — e Zd ZdZd
ˆ fd„Zdd	„Zˆ xZS )Ú"Dinov2WithRegistersPatchEmbeddingszâThis class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.Úconfigú#'WindowedDinov2WithRegistersConfig'r    rv   c                ó  •— t          ¦   «                              ¦   «          |j        |j        }}|j        |j        }}t          dt          |t          j	        j
        ¦  «        r|n||f¦  «        }t          dt          |t          j	        j
        ¦  «        r|n||f¦  «        }|d         |d         z  |d         |d         z  z  }|| _        || _        || _        || _        t          j        ||||¬¦  «        | _        d S )Nztuple[int, int]r$   r   )Úkernel_sizeÚstride)r{   r|   rf   rg   rh   r[   r   Ú
isinstanceÚcollectionsÚabcÚIterableÚnum_patchesr   ÚConv2dÚ
projection)
r€   r‹   Úimage_size_cfgÚpatch_size_cfgrh   r[   rf   rg   r”   r�   s
            €r)   r|   z+Dinov2WithRegistersPatchEmbeddings.__init__,  s  ø€ Ý‰Œ×ÒÑÔÐØ)/Ô):¸FÔ<M˜ˆØ$*Ô$7¸Ô9K�kˆåØå˜.­+¬/Ô*BÑCÔCð2ˆNˆNà  .Ð1ñ	
ô 
ˆ
õ Øå˜.­+¬/Ô*BÑCÔCð2ˆNˆNà  .Ð1ñ	
ô 
ˆ
ð " !”}¨
°1¬Ñ5¸*ÀQ¼-È:ÐVWÌ=Ñ:XÑYˆØ$ˆŒØ$ˆŒØ(ˆÔØ&ˆÔåœ) L°+È:Ð^hÐiÑiÔiˆŒˆˆr+   Úpixel_valuesr   c                ó
  — |j         d         }|| j        k    rt          d| j        › d|› d�¦  «        ‚|                      |¦  «                             d¦  «                             dd¦  «        }t          t          |¦  «        S )Nr$   zoMake sure that the channel dimension of the pixel values match with the one set in the configuration. Expected z	 but got ú.é   )Úshaperh   Ú
ValueErrorr–   ÚflattenÚ	transposer   r   )r€   r™   rh   Ú
embeddingss       r)   Úforwardz*Dinov2WithRegistersPatchEmbeddings.forwardE  s�   € Ø#Ô)¨!Ô,ˆØ˜4Ô,Ò,Ð,ÝðIØ!Ô.ðIð IØ9EðIð Ið Iñô ð ð —_’_ \Ñ2Ô2×:Ò:¸1Ñ=Ô=×GÒGÈÈ1ÑMÔMˆ
Ý•F˜JÑ'Ô'Ð'r+   ©r‹   rŒ   r    rv   )r™   r   r    r   ©r‚   rƒ   r„   r…   r|   r¢   r‡   rˆ   s   @r)   rŠ   rŠ   '  sa   ø€ € € € € ðð ðjð jð jð jð jð jð2(ð (ð (ð (ð (ð (ð (ð (r+   rŠ   c                  óZ   ‡ — e Zd ZdZdˆ fd„Zej        j        dd„¦   «         Zddd„Z	ˆ xZ
S )Ú%WindowedDinov2WithRegistersEmbeddingszTConstruct the CLS token, mask token, register tokens, position and patch embeddings.r‹   rM   r    rv   c                ó¢  •— t          ¦   «                              ¦   «          t          j        t	          j        dd|j        ¦  «        ¦  «        | _        t          j        t	          j        d|j        ¦  «        ¦  «        | _	        |j
        dk    r2t          j        t	          j        d|j
        |j        ¦  «        ¦  «        nd | _        t          |¦  «        | _        | j        j        }t          j        t	          j        d|dz   |j        ¦  «        ¦  «        | _        t          j        |j        ¦  «        | _        |j        | _        || _        d S )Nr$   r   )r{   r|   r   Ú	Parameterr-   Úrandnr[   Ú	cls_tokenÚzerosÚ
mask_tokenrn   Úregister_tokensrŠ   Úpatch_embeddingsr”   Úposition_embeddingsÚDropoutra   Údropoutrg   r‹   )r€   r‹   r”   r�   s      €r)   r|   z.WindowedDinov2WithRegistersEmbeddings.__init__S  s  ø€ Ý‰Œ×ÒÑÔÐåœ¥e¤k°!°Q¸Ô8JÑ&KÔ&KÑLÔLˆŒÝœ,¥u¤{°1°fÔ6HÑ'IÔ'IÑJÔJˆŒð Ô)¨AÒ-Ð-õ ŒL�œ Q¨Ô(BÀFÔDVÑWÔWÑXÔXÐXàð 	Ôõ
 !CÀ6Ñ JÔ JˆÔØÔ+Ô7ˆÝ#%¤<µ´¸A¸{ÈQ¹ÐPVÔPbÑ0cÔ0cÑ#dÔ#dˆÔ Ý”z &Ô"<Ñ=Ô=ˆŒØ Ô+ˆŒØˆŒˆˆr+   r¡   r   Úheightr   Úwidthc                ó†  — |j         d         dz
  }| j        j         d         dz
  }t          j                             ¦   «         s&||k    r ||k    rt          t          | j        ¦  «        S | j        dd…df         }| j        dd…dd…f         }|j         d         }|| j        j        z  }|| j        j        z  }t          |dz  ¦  «        }	| 
                    d|	|	|¦  «        }|                     dddd¦  «        }|j        }
t          j                             |                     t          j        ¬¦  «        t          |¦  «        t          |¦  «        fd	d
|j        j        dk    ¬¦  «                             |
¬¦  «        }t          j                             ¦   «         sKt)          |¦  «        |j         d         k    st)          |¦  «        |j         d         k    rt+          d¦  «        ‚|                     dddd¦  «                             dd|¦  «        }t          j        |                     d¦  «        |fd¬¦  «        S )aF  This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher
        resolution images. This implementation supports torch.jit tracing while maintaining backwards compatibility with
        the original implementation.

        Keep this operation eager: PyTorch's antialiased bicubic backward rejects symbolic
        sizes during AOT tracing. This boundary preserves interpolation and its gradients
        while allowing the surrounding model to compile. Remove it once dynamic forward/
        backward parity passes on the supported PyTorch versions without this boundary.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/main/vision_transformer.py
        - https://github.com/facebookresearch/dinov2/blob/main/dinov2/models/vision_transformer.py
        r$   Nr   r,   g      à?rY   rœ   ©ÚdtypeÚbicubicFÚmps)ÚsizeÚmodeÚalign_cornersÚ	antialiaséþÿÿÿzHWidth or height does not match with the interpolated position embeddings©Údim)r�   r¯   r-   ÚjitÚ
is_tracingr   r   r‹   rg   r   ÚreshapeÚpermuter¶   r   Ú
functionalÚinterpolateÚtoÚfloat32ÚdeviceÚtyper   rž   r1   ÚcatÚ	unsqueeze)r€   r¡   r²   r³   r”   Únum_positionsÚclass_pos_embedÚpatch_pos_embedr¿   Úsqrt_num_positionsÚtarget_dtypes              r)   Úinterpolate_pos_encodingz>WindowedDinov2WithRegistersEmbeddings.interpolate_pos_encodingd  sE  € ð !Ô& qÔ)¨AÑ-ˆØÔ0Ô6°qÔ9¸AÑ=ˆõ ”	×$Ò$Ñ&Ô&ð	:à˜}Ò,Ð,Ø˜%’�å� Ô 8Ñ9Ô9Ð9ð Ô2°1°1°1°a°4Ô8ˆØÔ2°1°1°1°a°b°b°5Ô9ˆØÔ˜rÔ"ˆð ˜4œ;Ô1Ñ1ˆØ˜œÔ/Ñ/ˆõ ' }°cÑ'9Ñ:Ô:ÐØ)×1Ò1°!Ð5GÐI[Ð]`ÑaÔaˆØ)×1Ò1°!°Q¸¸1Ñ=Ô=ˆð 'Ô,ˆõ œ-×3Ò3Ø×Ò¥U¤]ÐÑ3Ô3å˜&Ñ!Ô!Ý˜%Ñ Ô ðð ØØ%Ô,Ô1°UÒ:ð 4ñ 	
ô 	
÷ Š"�<ˆ"Ñ
 Ô
 ð 	õ Œy×#Ò#Ñ%Ô%ð 	mÝ�6‰{Œ{˜oÔ3°BÔ7Ò7Ð7½3¸u¹:¼:ÈÔI^Ð_aÔIbÒ;bÐ;bÝ Ð!kÑlÔlÐlð *×1Ò1°!°Q¸¸1Ñ=Ô=×BÒBÀ1ÀbÈ#ÑNÔNˆõ Œy˜/×3Ò3°AÑ6Ô6¸ÐHÈaÐPÑPÔPÐPr+   Nr™   Úbool_masked_posúTensor | Nonec                óì  — |j         \  }}}}| j        | j        j        z  }||z  dk    s	||z  dk    r.t	          d| j        › d| j        j        › d|› d|› d|› d�¦  «        ‚| j        j        j        j        }|                      | 	                    |¬¦  «        ¦  «        }	|�Yt          j        |                     d
¦  «        | j         	                    |	j        ¦  «                             d¦  «        |	¦  «        }	| j                             |d
d
¦  «        }
t          j        |
|	fd¬¦  «        }	|	|                      |	||¦  «        z   }	| j        j        dk    �r|| j        j        z  }|| j        j        z  }|	d	d	…d	d…f         }|	d	d	…dd	…f         }|                     |||d
¦  «        }|| j        j        z  }|| j        j        z  }| j        j        }|                     ||z  |||d
¦  «        }|                     ddddd¦  «        }|                     ||dz  z  ||z  d
¦  «        }|                     |dz  dd¦  «        }t          j        ||fd¬¦  «        }	| j        j        dk    r_| j        €J ‚t          j        |	d	d	…d	d…f         | j                             |	j         d         d
d
¦  «        |	d	d	…dd	…f         fd¬¦  «        }	|                      |	¦  «        }	t3          t4          |	¦  «        S )a"  Compute windowed patch embeddings for the given pixel values.

        Args:
            pixel_values: Image tensor of shape ``(B, C, H, W)``. Both ``H`` and
                ``W`` must be divisible by ``patch_size * num_windows``.
            bool_masked_pos: Optional boolean mask of shape ``(B, num_patches)``.
                Masked positions are replaced with the learnable ``mask_token``.

        Returns:
            Patch embedding tensor. When ``num_windows > 1`` the batch dimension is expanded to ``B * num_windows ** 2``
            and the sequence length corresponds to patches within a single window (plus CLS token and any register
            tokens).

        Raises:
            ValueError: If ``H`` or ``W`` is not divisible by
                ``patch_size * num_windows``.
        r   zHInput spatial dimensions must be divisible by patch_size * num_windows (z * z = z), but got height=z, width=r›   rµ   Nr,   r$   r¾   rœ   rY   rR   )r�   rg   r‹   rq   rž   r®   r–   Úweightr¶   rÆ   r-   ÚwhererË   r¬   rª   ÚexpandrÊ   rÑ   r1   rÂ   rÃ   Úrepeatrn   r­   r±   r   r   )r€   r™   rÒ   Ú
batch_sizeÚ_r²   r³   ÚdivisorrÐ   r¡   Ú
cls_tokensÚnum_h_patchesÚnum_w_patchesÚcls_token_with_pos_embedÚpixel_tokens_with_pos_embedÚnum_w_patches_per_windowÚnum_h_patches_per_windowrq   Úwindowed_pixel_tokensÚ!windowed_cls_token_with_pos_embeds                       r)   r¢   z-WindowedDinov2WithRegistersEmbeddings.forward¦  s‡  € ð$ (4Ô'9Ñ$ˆ
�A�v˜uØ”/ D¤KÔ$;Ñ;ˆØ�GÑ˜qÒ Ð  E¨G¡O°qÒ$8Ð$8Ýð;Ø”Oð;ð ;Ø(,¬Ô(?ð;ð ;ØDKð;ð ;à"(ð;ð ;à27ð;ð ;ð ;ñô ð ð
 Ô,Ô7Ô>ÔDˆØ×*Ò*¨<¯?ª?À¨?Ñ+NÔ+NÑOÔOˆ
àÐ&ÝœØ×)Ò)¨"Ñ-Ô-¨t¬×/AÒ/AÀ*ÔBRÑ/SÔ/S×/]Ò/]Ð^_Ñ/`Ô/`Ðblñô ˆJð
 ”^×*Ò*¨:°r¸2Ñ>Ô>ˆ
Ý”Y 
¨JÐ7¸QÐ?Ñ?Ô?ˆ
ð   $×"?Ò"?À
ÈFÐTYÑ"ZÔ"ZÑZˆ
àŒ;Ô" QÒ&Ñ&à" d¤kÔ&<Ñ<ˆMØ! T¤[Ô%;Ñ;ˆMØ'1°!°!°!°R°a°R°%Ô'8Ð$Ø*4°Q°Q°Q¸¸¸°UÔ*;Ð'Ø*E×*JÒ*JÈ:ÐWdÐfsÐuwÑ*xÔ*xÐ'Ø'4¸¼Ô8OÑ'OÐ$Ø'4¸¼Ô8OÑ'OÐ$Øœ+Ô1ˆKØ$?×$GÒ$GØ˜[Ñ(Ð*BÀKÐQiÐkmñ%ô %Ð!ð %:×$AÒ$AÀ!ÀQÈÈ1ÈaÑ$PÔ$PÐ!Ø$9×$AÒ$AØ˜[¨!™^Ñ+Ð-EÐH`Ñ-`Ðbdñ%ô %Ð!ð 1I×0OÒ0OÐP[Ð]^ÑP^Ð`aÐcdÑ0eÔ0eÐ-ÝœÐ$EÐG\Ð#]ÐcdÐeÑeÔeˆJð Œ;Ô*¨QÒ.Ð.ØÔ'Ð3Ð3Ð3ÝœØ˜A˜A˜A˜r ˜r˜EÔ" DÔ$8×$?Ò$?À
Ô@PÐQRÔ@SÐUWÐY[Ñ$\Ô$\Ð^hÐijÐijÐijÐlmÐlnÐlnÐinÔ^oÐpÐvwðñ ô ˆJð —\’\ *Ñ-Ô-ˆ
å•F˜JÑ'Ô'Ð'r+   ©r‹   rM   r    rv   )r¡   r   r²   r   r³   r   r    r   ©N)r™   r   rÒ   rÓ   r    r   )r‚   rƒ   r„   r…   r|   r-   ÚcompilerÚdisablerÑ   r¢   r‡   rˆ   s   @r)   r¦   r¦   P  s�   ø€ € € € € Ø^Ð^ðð ð ð ð ð ð" „^Ôð?Qð ?Qð ?Qñ Ôð?QðBF(ð F(ð F(ð F(ð F(ð F(ð F(ð F(ð F(r+   r¦   c                  ó4   ‡ — e Zd Zdˆ fd„Zdd„Z	 ddd„Zˆ xZS )Ú Dinov2WithRegistersSelfAttentionr‹   rM   r    rv   c                ó�  •— t          ¦   «                              ¦   «          |j        |j        z  dk    r1t	          |d¦  «        s!t          d|j        f› d|j        › d�¦  «        ‚|j        | _        t          |j        |j        z  ¦  «        | _        | j        | j        z  | _        t          j
        |j        | j        |j        ¬¦  «        | _        t          j
        |j        | j        |j        ¬¦  «        | _        t          j
        |j        | j        |j        ¬¦  «        | _        t          j        |j        ¦  «        | _        d S )Nr   Úembedding_sizezThe hidden size z4 is not a multiple of the number of attention heads r›   ©Úbias)r{   r|   r[   r]   Úhasattrrž   r   Úattention_head_sizeÚall_head_sizer   ÚLinearri   ÚqueryÚkeyÚvaluer°   rc   r±   ©r€   r‹   r�   s     €r)   r|   z)Dinov2WithRegistersSelfAttention.__init__ð  s1  ø€ Ý‰Œ×ÒÑÔÐØÔ Ô :Ñ:¸aÒ?Ð?ÍÐPVÐXhÑHiÔHiÐ?Ýð7 FÔ$6Ð#8ð 7ð 7ØÔ3ð7ð 7ð 7ñô ð ð
 $*Ô#=ˆÔ Ý#& vÔ'9¸FÔ<VÑ'VÑ#WÔ#WˆÔ Ø!Ô5¸Ô8PÑPˆÔå”Y˜vÔ1°4Ô3EÈFÌOÐ\Ñ\Ô\ˆŒ
Ý”9˜VÔ/°Ô1CÈ&Ì/ÐZÑZÔZˆŒÝ”Y˜vÔ1°4Ô3EÈFÌOÐ\Ñ\Ô\ˆŒ
å”z &Ô"EÑFÔFˆŒˆˆr+   Úxr   c                ó²   — |                      ¦   «         d d…         | j        | j        fz   }|                     |¦  «        }|                     dddd¦  «        S )Nr,   r   rœ   r$   rY   )r¹   r]   rð   r1   rÃ   )r€   r÷   Únew_x_shapes      r)   Útranspose_for_scoresz5Dinov2WithRegistersSelfAttention.transpose_for_scores  sP   € Ø—f’f‘h”h˜s ˜s”m tÔ'?ÀÔAYÐ&ZÑZˆØ�FŠF�;ÑÔˆØ�yŠy˜˜A˜q !Ñ$Ô$Ð$r+   FÚhidden_statesÚoutput_attentionsrj   ú,tuple[Tensor, Tensor | None] | tuple[Tensor]c                óô  — |                       |¦  «        }|                      |                      |¦  «        ¦  «        }|                      |                      |¦  «        ¦  «        }|                      |¦  «        }t	          j        ||                     dd¦  «        ¦  «        }|t          j        | j	        ¦  «        z  }t          j                             |d¬¦  «        }|                      |¦  «        }t	          j        ||¦  «        }	|	                     dddd¦  «                             ¦   «         }	|	                     ¦   «         d d…         | j        fz   }
|	                     |
¦  «        }	|r|	|fn|	f}|S )Nr,   r½   r¾   r   rœ   r$   rY   )ró   rú   rô   rõ   r-   Úmatmulr    ÚmathÚsqrtrð   r   rÄ   Úsoftmaxr±   rÃ   r2   r¹   rñ   r1   )r€   rû   rü   Úmixed_query_layerÚ	key_layerÚvalue_layerÚquery_layerÚattention_scoresÚattention_probsÚcontext_layerÚnew_context_layer_shapeÚoutputss               r)   r¢   z(Dinov2WithRegistersSelfAttention.forward  s`  € ð !ŸJšJ }Ñ5Ô5Ðà×-Ò-¨d¯hªh°}Ñ.EÔ.EÑFÔFˆ	Ø×/Ò/°·
²
¸=Ñ0IÔ0IÑJÔJˆØ×/Ò/Ð0AÑBÔBˆõ !œ<¨°Y×5HÒ5HÈÈRÑ5PÔ5PÑQÔQÐà+­d¬i¸Ô8PÑ.QÔ.QÑQÐõ œ-×/Ò/Ð0@ÀbÐ/ÑIÔIˆð Ÿ,š, Ñ7Ô7ˆåœ _°kÑBÔBˆà%×-Ò-¨a°°A°qÑ9Ô9×DÒDÑFÔFˆØ"/×"4Ò"4Ñ"6Ô"6°s¸°sÔ";¸tÔ?QÐ>SÑ"SÐØ%×*Ò*Ð+BÑCÔCˆà6GÐ]�= /Ð2Ð2ÈmÐM]ˆàˆr+   rå   )r÷   r   r    r   ©F©rû   r   rü   rj   r    rý   )r‚   rƒ   r„   r|   rú   r¢   r‡   rˆ   s   @r)   rê   rê   ï  sw   ø€ € € € € ðGð Gð Gð Gð Gð Gð$%ð %ð %ð %ð @Eð!ð !ð !ð !ð !ð !ð !ð !ð !r+   rê   c                  ó0   ‡ — e Zd Zdˆ fd„Z	 ddˆ fd„Zˆ xZS )Ú$Dinov2WithRegistersSdpaSelfAttentionr‹   rM   r    rv   c                ób   •— t          ¦   «                              |¦  «         |j        | _        d S ræ   )r{   r|   rc   rö   s     €r)   r|   z-Dinov2WithRegistersSdpaSelfAttention.__init__,  s,   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø,2Ô,OˆÔ)Ð)Ð)r+   Frû   r   rü   rj   rý   c           	     ó¸  •— |r=t                                d¦  «         t          ¦   «                              ||¬¦  «        S |                      |¦  «        }|                      |                      |¦  «        ¦  «        }|                      |                      |¦  «        ¦  «        }|                      |¦  «        }t          j	        j
                             |||d | j        r| j        nddd ¬¦  «        }|                     dddd	¦  «                             ¦   «         }|                     ¦   «         d d
…         | j        fz   }|                     |¦  «        }|d fS )Nah  Dinov2WithRegistersModel is using Dinov2WithRegistersSdpaSelfAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation. To avoid this fallback, call `model.set_attn_implementation('eager')` or pass `attn_implementation="eager"` when instantiating the model.)rû   rü   rT   F)Ú	is_causalÚscaler   rœ   r$   rY   r½   )ÚloggerÚwarning_oncer{   r¢   ró   rú   rô   rõ   r-   r   rÄ   Úscaled_dot_product_attentionÚtrainingrc   rÃ   r2   r¹   rñ   r1   )
r€   rû   rü   r  r  r  r  r	  r
  r�   s
            €r)   r¢   z,Dinov2WithRegistersSdpaSelfAttention.forward0  s]  ø€ ð ð 	eÝ×ÒðVñô ð õ ‘7”7—?’?°ÐRc�?ÑdÔdÐdà ŸJšJ }Ñ5Ô5Ðà×-Ò-¨d¯hªh°}Ñ.EÔ.EÑFÔFˆ	Ø×/Ò/°·
²
¸=Ñ0IÔ0IÑJÔJˆØ×/Ò/Ð0AÑBÔBˆåœÔ+×HÒHØØØØØ15´ÐGˆDÔ-Ð-ÀCØØð Iñ 
ô 
ˆð &×-Ò-¨a°°A°qÑ9Ô9×DÒDÑFÔFˆØ"/×"4Ò"4Ñ"6Ô"6°s¸°sÔ";¸tÔ?QÐ>SÑ"SÐØ%×*Ò*Ð+BÑCÔCˆà˜dÐ"Ð"r+   rå   r  r  ©r‚   rƒ   r„   r|   r¢   r‡   rˆ   s   @r)   r  r  +  sm   ø€ € € € € ðPð Pð Pð Pð Pð Pð
 @Eð!#ð !#ð !#ð !#ð !#ð !#ð !#ð !#ð !#ð !#ð !#r+   r  c                  ó,   ‡ — e Zd ZdZdˆ fd„Zdd
„Zˆ xZS )ÚDinov2WithRegistersSelfOutputz¦The residual connection is defined in Dinov2WithRegistersLayer instead of here (as is the case with other
    models), due to the layernorm applied before each block.r‹   rM   r    rv   c                óÌ   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        ¦  «        | _        d S ræ   )	r{   r|   r   rò   r[   Údenser°   ra   r±   rö   s     €r)   r|   z&Dinov2WithRegistersSelfOutput.__init__X  sJ   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ý”z &Ô"<Ñ=Ô=ˆŒˆˆr+   rû   r   Úinput_tensorc                óZ   — |                       |¦  «        }|                      |¦  «        }|S ræ   )r  r±   )r€   rû   r  s      r)   r¢   z%Dinov2WithRegistersSelfOutput.forward]  s*   € ØŸ
š
 =Ñ1Ô1ˆØŸš ]Ñ3Ô3ˆàÐr+   rå   )rû   r   r  r   r    r   r¤   rˆ   s   @r)   r  r  T  s]   ø€ € € € € ð@ð @ð>ð >ð >ð >ð >ð >ð
ð ð ð ð ð ð ð r+   r  c                  ó4   ‡ — e Zd Zdˆ fd„Zdd„Z	 ddd„Zˆ xZS )ÚDinov2WithRegistersAttentionr‹   rM   r    rv   c                ó¾   •— t          ¦   «                              ¦   «          t          |¦  «        | _        t	          |¦  «        | _        t          ¦   «         | _        d S ræ   )r{   r|   rê   Ú	attentionr  Úoutputr/   Úpruned_headsrö   s     €r)   r|   z%Dinov2WithRegistersAttention.__init__e  sI   ø€ Ý‰Œ×ÒÑÔÐÝ9¸&ÑAÔAˆŒÝ3°FÑ;Ô;ˆŒÝ&)¡e¤eˆÔÐÐr+   r   r   c                ó´  — t          |¦  «        dk    rd S t          || j        j        | j        j        | j        ¦  «        \  }}t          t          j        |¦  «        }t          | j        j
        |¦  «        | j        _
        t          | j        j        |¦  «        | j        _        t          | j        j        |¦  «        | j        _        t          | j        j        |d¬¦  «        | j        _        | j        j        t          |¦  «        z
  | j        _        | j        j        | j        j        z  | j        _        | j                             |¦  «        | _        d S )Nr   r$   r¾   )r5   r9   r"  r]   rð   r$  r   r-   Ú
LongTensorr   ró   rô   rõ   r#  r  rñ   Úunion)r€   r   Úindex_tensorr8   s       r)   Úprune_headsz(Dinov2WithRegistersAttention.prune_headsk  s  € Ýˆu‰:Œ:˜Š?ˆ?ØˆFÝ?Ø�4”>Ô5°t´~Ô7YÐ[_Ô[lñ
ô 
Ñˆˆ|õ •UÔ% |Ñ4Ô4ˆõ  2°$´.Ô2FÈÑNÔNˆŒÔÝ/°´Ô0BÀEÑJÔJˆŒÔÝ1°$´.Ô2FÈÑNÔNˆŒÔÝ.¨t¬{Ô/@À%ÈQÐOÑOÔOˆŒÔð .2¬^Ô-OÕRUÐV[ÑR\ÔR\Ñ-\ˆŒÔ*Ø'+¤~Ô'IÈDÌNÔLnÑ'nˆŒÔ$Ø Ô-×3Ò3°EÑ:Ô:ˆÔÐÐr+   Frû   r   rü   rj   rý   c                ó¢   — |                       ||¦  «        }|                      |d         |¦  «        }|f|dd …         z   }t          d|¦  «        S )Nr   r$   rý   )r"  r#  r   )r€   rû   rü   Úself_outputsÚattention_outputr  s         r)   r¢   z$Dinov2WithRegistersAttention.forward~  sW   € ð
 —~’~ mÐ5FÑGÔGˆàŸ;š; |°A¤¸ÑFÔFÐà#Ð%¨°Q°R°RÔ(8Ñ8ˆÝÐBÀGÑLÔLÐLr+   rå   )r   r   r    rv   r  r  )r‚   rƒ   r„   r|   r)  r¢   r‡   rˆ   s   @r)   r   r   d  sx   ø€ € € € € ð,ð ,ð ,ð ,ð ,ð ,ð;ð ;ð ;ð ;ð, #(ð
Mð 
Mð 
Mð 
Mð 
Mð 
Mð 
Mð 
Mð 
Mr+   r   c                  ó    ‡ — e Zd Zdˆ fd„Zˆ xZS )Ú Dinov2WithRegistersSdpaAttentionr‹   rM   r    rv   c                ór   •— t          ¦   «                              |¦  «         t          |¦  «        | _        d S ræ   )r{   r|   r  r"  rö   s     €r)   r|   z)Dinov2WithRegistersSdpaAttention.__init__Œ  s.   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý=¸fÑEÔEˆŒˆˆr+   rå   )r‚   rƒ   r„   r|   r‡   rˆ   s   @r)   r.  r.  ‹  sG   ø€ € € € € ðFð Fð Fð Fð Fð Fð Fð Fð Fð Fr+   r.  c                  ó(   ‡ — e Zd Zd	ˆ fd„Zd
d„Zˆ xZS )ÚDinov2WithRegistersLayerScaler‹   rŒ   r    rv   c                ó¸   •— t          ¦   «                              ¦   «          t          j        |j        t          j        |j        ¦  «        z  ¦  «        | _        d S ræ   )	r{   r|   r   r¨   rk   r-   r.   r[   Úlambda1rö   s     €r)   r|   z&Dinov2WithRegistersLayerScale.__init__’  sC   ø€ Ý‰Œ×ÒÑÔÐÝ”| FÔ$;½e¼jÈÔI[Ñ>\Ô>\Ñ$\Ñ]Ô]ˆŒˆˆr+   Úhidden_stater   c                ó   — || j         z  S ræ   )r3  ©r€   r4  s     r)   r¢   z%Dinov2WithRegistersLayerScale.forward–  s   € Ø˜dœlÑ*Ð*r+   r£   ©r4  r   r    r   r  rˆ   s   @r)   r1  r1  ‘  sW   ø€ € € € € ð^ð ^ð ^ð ^ð ^ð ^ð+ð +ð +ð +ð +ð +ð +ð +r+   r1  rT   FÚinputr   Ú	drop_probrb   r  rj   c                óü   — |dk    s|s| S d|z
  }| j         d         fd| j        dz
  z  z   }|t          j        || j        | j        ¬¦  «        z   }|                     ¦   «          |                      |¦  «        |z  }|S )aA  Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).

    Comment by Ross Wightman: This is the same as the DropConnect impl I created for EfficientNet, etc networks,
    however, the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper... See
    discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for changing the
    layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use 'survival rate' as the
    argument.
    rT   r$   r   )r$   )r¶   rÈ   )r�   Úndimr-   Úrandr¶   rÈ   Úfloor_Údiv)r8  r9  r  Ú	keep_probr�   Úrandom_tensorr#  s          r)   Ú	drop_pathrA  š  sŒ   € ð �CÒÐ˜xÐØˆØ�I‘€IØŒ[˜Œ^Ð ¨¬
°Q©Ñ 7Ñ7€EØ¥¤
¨5¸¼ÈEÌLÐ YÑ YÔ YÑY€MØ×ÒÑÔÐØ�YŠY�yÑ!Ô! MÑ1€FØ€Mr+   c                  ó6   ‡ — e Zd ZdZddˆ fd„Zdd
„Zdd„Zˆ xZS )ÚDinov2WithRegistersDropPathzXDrop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).Nr9  úfloat | Noner    rv   c                ó^   •— t          ¦   «                              ¦   «          |�|nd| _        d S )NrT   )r{   r|   r9  )r€   r9  r�   s     €r)   r|   z$Dinov2WithRegistersDropPath.__init__°  s-   ø€ Ý‰Œ×ÒÑÔÐØ&/Ð&;˜˜ÀˆŒˆˆr+   rû   r   c                ó8   — t          || j        | j        ¦  «        S ræ   )rA  r9  r  )r€   rû   s     r)   r¢   z#Dinov2WithRegistersDropPath.forward´  s   € Ý˜¨¬¸¼ÑFÔFÐFr+   r`   c                ó   — d| j         › �S )Nzp=)r9  ©r€   s    r)   Ú
extra_reprz&Dinov2WithRegistersDropPath.extra_repr·  s   € Ø$�D”NÐ$Ð$Ð$r+   ræ   )r9  rD  r    rv   )rû   r   r    r   )r    r`   )r‚   rƒ   r„   r…   r|   r¢   rI  r‡   rˆ   s   @r)   rC  rC  ­  s{   ø€ € € € € ØbÐbðEð Eð Eð Eð Eð Eð EðGð Gð Gð Gð%ð %ð %ð %ð %ð %ð %ð %r+   rC  c                  ó(   ‡ — e Zd Zd	ˆ fd„Zd
d„Zˆ xZS )ÚDinov2WithRegistersMLPr‹   rŒ   r    rv   c                ó~  •— t          ¦   «                              ¦   «          |j        x}}t          |j        |j        z  ¦  «        }t          j        ||d¬¦  «        | _        t          |j	        t          ¦  «        rt          |j	                 | _        n|j	        | _        t          j        ||d¬¦  «        | _        d S )NTrí   )r{   r|   r[   r   r^   r   rò   Úfc1r�   r_   r`   r   Ú
activationÚfc2©r€   r‹   Úin_featuresr:   Úhidden_featuresr�   s        €r)   r|   zDinov2WithRegistersMLP.__init__¼  s¢   ø€ Ý‰Œ×ÒÑÔÐØ%+Ô%7Ð7ˆ�lÝ˜fÔ0°6Ô3CÑCÑDÔDˆÝ”9˜[¨/ÀÐEÑEÔEˆŒÝ�fÔ'­Ñ-Ô-ð 	0Ý$ VÔ%6Ô7ˆDŒOˆOà$Ô/ˆDŒOÝ”9˜_¨lÀÐFÑFÔFˆŒˆˆr+   r4  r   c                ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S ræ   )rM  rN  rO  r6  s     r)   r¢   zDinov2WithRegistersMLP.forwardÇ  s;   € Ø—x’x Ñ-Ô-ˆØ—’ |Ñ4Ô4ˆØ—x’x Ñ-Ô-ˆØÐr+   r£   r7  r  rˆ   s   @r)   rK  rK  »  sW   ø€ € € € € ð	Gð 	Gð 	Gð 	Gð 	Gð 	Gðð ð ð ð ð ð ð r+   rK  c                  ó(   ‡ — e Zd Zd	ˆ fd„Zd
d„Zˆ xZS )ÚDinov2WithRegistersSwiGLUFFNr‹   rŒ   r    rv   c                óD  •— t          ¦   «                              ¦   «          |j        x}}t          |j        |j        z  ¦  «        }t          |dz  dz  ¦  «        dz   dz  dz  }t          j        |d|z  d¬¦  «        | _        t          j        ||d¬¦  «        | _        d S )Nrœ   rY   é   é   Trí   )	r{   r|   r[   r   r^   r   rò   Ú
weights_inÚweights_outrP  s        €r)   r|   z%Dinov2WithRegistersSwiGLUFFN.__init__Ï  s    ø€ Ý‰Œ×ÒÑÔÐØ%+Ô%7Ð7ˆ�lÝ˜fÔ0°6Ô3CÑCÑDÔDˆÝ˜°Ñ2°QÑ6Ñ7Ô7¸!Ñ;ÀÑAÀAÑEˆåœ) K°°_Ñ1DÈ4ÐPÑPÔPˆŒÝœ9 _°lÈÐNÑNÔNˆÔÐÐr+   r4  r   c                óô   — |                       |¦  «        }|                     dd¬¦  «        \  }}t          j                             |¦  «        |z  }t          t          |                      |¦  «        ¦  «        S )Nrœ   r,   r¾   )rY  Úchunkr   rÄ   Úsilur   r   rZ  )r€   r4  Úx1Úx2Úhiddens        r)   r¢   z$Dinov2WithRegistersSwiGLUFFN.forwardØ  sg   € Ø—’ |Ñ4Ô4ˆØ×#Ò# A¨2Ð#Ñ.Ô.‰ˆˆBÝ”×#Ò# BÑ'Ô'¨"Ñ,ˆÝ•F˜D×,Ò,¨VÑ4Ô4Ñ5Ô5Ð5r+   r£   r7  r  rˆ   s   @r)   rU  rU  Î  sW   ø€ € € € € ðOð Oð Oð Oð Oð Oð6ð 6ð 6ð 6ð 6ð 6ð 6ð 6r+   rU  )ÚeagerÚsdpac                  ó2   ‡ — e Zd ZdZdˆ fd„Z	 	 ddd„Zˆ xZS )Ú WindowedDinov2WithRegistersLayerzCThis corresponds to the Block class in the original implementation.r‹   rM   r    rv   c                óT  •— t          ¦   «                              ¦   «          |j        | _        t          j        |j        |j        ¬¦  «        | _        t          |j	                 |¦  «        | _
        t          |¦  «        | _        |j        dk    rt          |j        ¦  «        nt          j        ¦   «         | _        t          j        |j        |j        ¬¦  «        | _        |  |j        rt'          |¦  «        | _        nt+          |¦  «        | _        t          |¦  «        | _        d S )N©ÚepsrT   )r{   r|   rq   r   Ú	LayerNormr[   re   Únorm1Ú'DINOV2_WITH_REGISTERS_ATTENTION_CLASSESÚ_attn_implementationr"  r1  Úlayer_scale1rl   rC  ÚIdentityrA  Únorm2rm   rU  ÚmlprK  Úlayer_scale2rö   s     €r)   r|   z)WindowedDinov2WithRegistersLayer.__init__è  s   ø€ Ý‰Œ×ÒÑÔÐà!Ô-ˆÔå”\ &Ô"4¸&Ô:OÐPÑPÔPˆŒ
Ý@ÀÔA\Ô]Ð^dÑeÔeˆŒÝ9¸&ÑAÔAˆÔàBHÔBWÐZ]ÒB]ÐB]Õ'¨Ô(=Ñ>Ô>Ð>ÕceÔcnÑcpÔcpð 	Œõ ”\ &Ô"4¸&Ô:OÐPÑPÔPˆŒ
àÐØÔ ð 	6Ý3°FÑ;Ô;ˆDŒHˆHå-¨fÑ5Ô5ˆDŒHÝ9¸&ÑAÔAˆÔÐÐr+   Frû   r   rü   rj   Úrun_full_attentionrý   c                óš  — |r
J d¦   «         ‚|}|r2|j         \  }}}| j        dz  }|                     ||z  ||z  |¦  «        }|                      |                      |¦  «        |¬¦  «        }	|	d         }
|r2|j         \  }}}| j        dz  }|
                     ||z  ||z  |¦  «        }
|                      |
¦  «        }
|	dd …         }|                      |
¦  «        |z   }|                      |¦  «        }|                      |¦  «        }|  	                    |¦  «        }|                      |¦  «        |z   }|f|z   }t          d|¦  «        S )Nz9output_attentions is not supported for windowed attentionrœ   )rü   r   r$   rý   )r�   rq   r1   r"  ri  rl  rA  rn  ro  rp  r   )r€   rû   rü   rq  ÚshortcutÚbatch_windowsÚtokens_per_windowÚchannelsÚnum_windows_squaredÚself_attention_outputsr,  r  Úlayer_outputs                r)   r¢   z(WindowedDinov2WithRegistersLayer.forwardý  s£  € ð %ÐaÐaÐ&aÑaÔaÐ$Ø ˆØð 	à9FÔ9LÑ6ˆMÐ,¨hØ"&Ô"2°AÑ"5ÐØ)×.Ò.ØÐ!4Ñ4Ð6IÐL]Ñ6]Ð_gñô ˆMð "&§¢Ø�JŠJ�}Ñ%Ô%Ø/ð "0ñ "
ô "
Ðð 2°!Ô4Ðàð 	à9FÔ9LÑ6ˆMÐ,¨hØ"&Ô"2°AÑ"5Ðà/×4Ò4ØÐ 3Ñ3Ð5FÐJ]Ñ5]Ð_gñ ô  Ðð  ×,Ò,Ð-=Ñ>Ô>ÐØ(¨¨¨Ô,ˆð ŸšÐ'7Ñ8Ô8¸8ÑCˆð —z’z -Ñ0Ô0ˆØ—x’x Ñ-Ô-ˆØ×(Ò(¨Ñ6Ô6ˆð —~’~ lÑ3Ô3°mÑCˆà�/ GÑ+ˆåÐBÀGÑLÔLÐLr+   rå   )FF)rû   r   rü   rj   rq  rj   r    rý   r¤   rˆ   s   @r)   rd  rd  å  ss   ø€ € € € € ØMÐMðBð Bð Bð Bð Bð Bð0 #(Ø#(ð	/Mð /Mð /Mð /Mð /Mð /Mð /Mð /Mð /Mr+   rd  c                  ó0   ‡ — e Zd Zdˆ fd„Z	 	 	 ddd„Zˆ xZS )Ú"WindowedDinov2WithRegistersEncoderr‹   rM   r    rv   c                óÞ   •‡— t          ¦   «                              ¦   «          ‰| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        ‰j        | _        d S )Nc                ó.   •— g | ]}t          ‰¦  «        ‘ŒS r%   )rd  ©r&   rÚ   r‹   s     €r)   rD   z?WindowedDinov2WithRegistersEncoder.__init__.<locals>.<listcomp>3  s#   ø€ Ð#vÐ#vÐ#vÐQRÕ$DÀVÑ$LÔ$LÐ#vÐ#vÐ#vr+   )	r{   r|   r‹   r   Ú
ModuleListr}   r\   rC   rt   rö   s    `€r)   r|   z+WindowedDinov2WithRegistersEncoder.__init__0  se   øø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”]Ð#vÐ#vÐ#vÐ#vÕV[Ð\bÔ\tÑVuÔVuÐ#vÑ#vÔ#vÑwÔwˆŒ
Ø&,Ô&CˆÔ#Ð#Ð#r+   FTrû   r   rü   rj   Úoutput_hidden_statesÚreturn_dictú!tuple[Any, ...] | BaseModelOutputc                óˆ  — |rdnd }|rdnd }| j         j        d         }|                     d¦  «        rt          |dd …         ¦  «        nd }t	          | j        ¦  «        D ]|\  }	}
|r
|€J ‚||fz   }|�|	|k    r nb|	| j         j        v}| j        r%| j        r|  	                    |
j
        |||¦  «        }n |
|||¦  «        }|d         }|r|€J ‚||d         fz   }Œ}|r
|€J ‚||fz   }|st          d„ |||fD ¦   «         ¦  «        S t          t          d|¦  «        t          d	|¦  «        t          d	|¦  «        ¬
¦  «        S )Nr%   r,   rz   é   r   r$   c              3  ó   K  — | ]}|®|V — Œ	d S ræ   r%   )r&   Úvs     r)   r*   z=WindowedDinov2WithRegistersEncoder.forward.<locals>.<genexpr>d  s(   è è € ÐmÐm˜qÐ_`Ð_l˜Ð_lÐ_lÐ_lÐ_lÐmÐmr+   ztorch.FloatTensorz$tuple[torch.FloatTensor, ...] | None)Úlast_hidden_staterû   Ú
attentions)r‹   r:   Ú
startswithr   Ú	enumeraterC   rr   rt   r  Ú_gradient_checkpointing_funcÚ__call__Útupler   r   )r€   rû   rü   r€  r�  Úall_hidden_statesÚall_self_attentionsÚlast_featureÚearly_stop_layerÚiÚlayer_modulerq  Úlayer_outputss                r)   r¢   z*WindowedDinov2WithRegistersEncoder.forward6  sç  € ð >RÐ7[°r°rÐW[ÐØ?PÐ9Z¸¸ÐVZÐà”{Ô/°Ô3ˆà4@×4KÒ4KÈGÑ4TÔ4TÐ^�3˜|¨A¨B¨BÔ/Ñ0Ô0Ð0ÐZ^Ðå(¨¬Ñ4Ô4ð 	Pð 	P‰OˆAˆ|Ø#ð IØ(Ð4Ð4Ð4Ø$5¸Ð8HÑ$HÐ!àÐ+°Ð4DÒ0DÐ0Dà�à!"¨$¬+Ô*JÐ!JÐàÔ*ð c¨t¬}ð cØ $× AÒ AØ Ô)Ø!Ø%Ø&ñ	!ô !��ð !- ¨]Ð<MÐOaÑ bÔ b�à)¨!Ô,ˆMà ð PØ*Ð6Ð6Ð6Ø&9¸]È1Ô=MÐ<OÑ&OÐ#øàð 	EØ$Ð0Ð0Ð0Ø 1°]Ð4DÑ DÐàð 	nÝÐmÐm ]Ð4EÐGZÐ$[ÐmÑmÔmÑmÔmÐmÝÝ"Ð#6¸ÑFÔFÝÐEÐGXÑYÔYÝÐBÐDWÑXÔXð
ñ 
ô 
ð 	
r+   rå   )FFT)
rû   r   rü   rj   r€  rj   r�  rj   r    r‚  r  rˆ   s   @r)   r{  r{  /  sg   ø€ € € € € ðDð Dð Dð Dð Dð Dð #(Ø%*Ø ð3
ð 3
ð 3
ð 3
ð 3
ð 3
ð 3
ð 3
ð 3
r+   r{  c                  ó4   — e Zd ZdZeZdZdZdZdgZ	dZ
dd
„ZdS )Ú*WindowedDinov2WithRegistersPreTrainedModelz|An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.rO   r™   TrU  Úmoduleú$nn.Linear | nn.Conv2d | nn.LayerNormr    rv   c                ó`  — t          |t          j        t          j        f¦  «        r£t          j                             |j        j                             t          j
        ¦  «        d| j        j        ¬¦  «                             |j        j        ¦  «        |j        _        |j        � |j        j                             ¦   «          dS dS t          |t          j        ¦  «        r?|j        j                             ¦   «          |j        j                             d¦  «         dS t          |t$          ¦  «        röt          j                             |j        j                             t          j
        ¦  «        d| j        j        ¬¦  «                             |j        j        ¦  «        |j        _        t          j                             |j        j                             t          j
        ¦  «        d| j        j        ¬¦  «                             |j        j        ¦  «        |j        _        dS dS )zInitialize the weights.rT   )ÚmeanÚstdNrZ   )r�   r   rò   r•   ÚinitÚtrunc_normal_rÕ   ÚdatarÆ   r-   rÇ   r‹   rd   r¶   rî   Úzero_rh  Úfill_r¦   r¯   rª   )r€   r—  s     r)   Ú_init_weightsz8WindowedDinov2WithRegistersPreTrainedModel._init_weightsw  sÑ  € å�f�rœy­"¬)Ð4Ñ5Ô5ð 	)õ "$¤×!6Ò!6Ø”Ô"×%Ò%¥e¤mÑ4Ô4¸3ÀDÄKÔDað "7ñ "ô "çŠb�”Ô$Ñ%Ô%ð ŒMÔð Œ{Ð&Ø”Ô ×&Ò&Ñ(Ô(Ð(Ð(Ð(ð 'Ð&å˜¥¤Ñ-Ô-ð 	)ØŒKÔ×"Ò"Ñ$Ô$Ð$ØŒMÔ×$Ò$ SÑ)Ô)Ð)Ð)Ð)Ý˜Õ EÑFÔFð 	)Ý.0¬g×.CÒ.CØÔ*Ô/×2Ò2µ5´=ÑAÔAØØ”KÔ1ð /Dñ /ô /÷ Šb�Ô+Ô1Ñ2Ô2ð	 Ô&Ô+õ %'¤G×$9Ò$9ØÔ Ô%×(Ò(­¬Ñ7Ô7ØØ”KÔ1ð %:ñ %ô %÷ Šb�Ô!Ô'Ñ(Ô(ð	 ÔÔ!Ð!Ð!ð	)ð 	)r+   N)r—  r˜  r    rv   )r‚   rƒ   r„   r…   rM   Úconfig_classÚbase_model_prefixÚmain_input_nameÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_sdpar¡  r%   r+   r)   r–  r–  l  sX   € € € € € ðð ð 5€LØ/ÐØ$€OØ&*Ð#Ø7Ð8ÐØ€Nð)ð )ð )ð )ð )ð )r+   r–  aU  
    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it
    as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`Dinov2WithRegistersConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
aô  
    Args:
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See
            [`BitImageProcessor.preprocess`] for details.

        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, sequence_length)`):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). Only relevant for
            pre-training.

        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
zmThe bare Dinov2WithRegisters Model transformer outputting raw hidden-states without any specific head on top.c                  ó”   ‡ — e Zd Zdˆ fd„Zdd„Zdd
„Zdd„Z ee¦  «         e	e
e¬¦  «        	 	 	 	 	 ddd„¦   «         ¦   «         Zˆ xZS )Ú WindowedDinov2WithRegistersModelr‹   rM   r    rv   c                ó  •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        t          |¦  «        | _        t          j        |j	        |j
        ¬¦  «        | _        |                      ¦   «          d S )Nrf  )r{   r|   r‹   r¦   r¡   r{  Úencoderr   rh  r[   re   Ú	layernormÚ	post_initrö   s     €r)   r|   z)WindowedDinov2WithRegistersModel.__init__·  st   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒå?ÀÑGÔGˆŒÝ9¸&ÑAÔAˆŒåœ fÔ&8¸fÔ>SÐTÑTÔTˆŒð 	�ŠÑÔÐÐÐr+   rŠ   c                ó   — | j         j        S ræ   ©r¡   r®   rH  s    r)   Úget_input_embeddingsz5WindowedDinov2WithRegistersModel.get_input_embeddingsÃ  ó   € ØŒÔ/Ð/r+   Úheads_to_pruneúdict[int, list[int]]c                óÎ   — |                      ¦   «         D ]O\  }}t          t          | j        j        |         ¦  «        j                             t          |¦  «        ¦  «         ŒPdS )z�Prunes heads of the model.

        heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)Úitemsr   rd  r«  rC   r"  r)  r/   )r€   r²  rC   r   s       r)   Ú_prune_headsz-WindowedDinov2WithRegistersModel._prune_headsÆ  sj   € ð +×0Ò0Ñ2Ô2ð 	pð 	p‰LˆE�5ÝÕ1°4´<Ô3EÀeÔ3LÑMÔMÔW×cÒcÕdgÐhmÑdnÔdnÑoÔoÐoÐoð	pð 	pr+   Úattn_implementationr`   c                óˆ  — |t           vr(t          d|›dt          t           ¦  «        › d�¦  «        ‚|| j        _        | j        j        D ]w}t          t          |¦  «        }t          |         t          t          | j        ¦  «        ¦  «        }| 
                    |j                             ¦   «         ¦  «         ||_        ŒxdS )uD  Switch the attention implementation without reloading the model.

        This is useful when you want to change the attention implementation after the model has been instantiated â€” for
        example, to use ``"eager"`` (manual) attention when inspecting attention weights, without having to reconstruct
        the entire model from scratch.

        Args:
            attn_implementation: One of ``"eager"`` (manual attention) or ``"sdpa"``
                (:func:`torch.nn.functional.scaled_dot_product_attention`).

        Raises:
            ValueError: If *attn_implementation* is not a supported key.

        Example::

            >>> from rfdetr.models.backbone.dinov2_with_windowed_attn import (
            ...     WindowedDinov2WithRegistersConfig,
            ...     WindowedDinov2WithRegistersModel,
            ... )
            >>> config = WindowedDinov2WithRegistersConfig(
            ...     image_size=32,
            ...     patch_size=16,
            ...     hidden_size=32,
            ...     num_hidden_layers=1,
            ...     num_attention_heads=4,
            ...     num_register_tokens=2,
            ... )
            >>> model = WindowedDinov2WithRegistersModel(config)
            >>> model.set_attn_implementation("eager")
            >>> model.config._attn_implementation
            'eager'
        zUnknown attn_implementation z. Choose from r›   N)rj  rž   Úsortedr‹   rk  r«  rC   r   rd  rM   Úload_state_dictr"  Ú
state_dict)r€   r·  r“  rC   Únew_attns        r)   Úset_attn_implementationz8WindowedDinov2WithRegistersModel.set_attn_implementationÏ  sâ   € ðB Õ&MÐMÐMÝðRÐ/Bð Rð RÝ%Õ&MÑNÔNðRð Rð Rñô ð ð ,?ˆŒÔ(Ø œLÔ.ð 
	'ð 
	'ˆLÝÕ9¸<ÑHÔHˆEÝ>Ð?RÔSÝÕ6¸¼ÑDÔDñô ˆHð ×$Ò$ U¤_×%?Ò%?Ñ%AÔ%AÑBÔBÐBØ&ˆEŒOˆOð
	'ð 
	'r+   ©Úoutput_typer¢  Nr™   rÓ   rÒ   rü   úbool | Noner€  r�  ú,tuple[Any, ...] | BaseModelOutputWithPoolingc                ó¸  — |�|n| j         j        }|�|n| j         j        }|�|n| j         j        }|€t	          d¦  «        ‚|                      ||¬¦  «        }|                      ||||¬¦  «        }|d         }|                      |¦  «        }|dd…ddd…f         }	|s||	f}
t          d|
|dd…         z   ¦  «        S t          ||	|j
        |j        ¬¦  «        S )	aý  
        Returns:

        Examples:

        >>> import torch
        >>> from rfdetr.models.backbone.dinov2_with_windowed_attn import (
        ...     WindowedDinov2WithRegistersConfig,
        ...     WindowedDinov2WithRegistersModel,
        ... )
        >>> config = WindowedDinov2WithRegistersConfig(
        ...     image_size=32,
        ...     patch_size=16,
        ...     hidden_size=32,
        ...     num_hidden_layers=2,
        ...     num_attention_heads=4,
        ...     num_register_tokens=2,
        ... )
        >>> model = WindowedDinov2WithRegistersModel(config)
        >>> pixel_values = torch.randn(1, 3, 32, 32)
        >>> outputs = model(pixel_values)
        >>> list(outputs.last_hidden_state.shape)
        [1, 7, 32]
        Nz You have to specify pixel_values)rÒ   ©rü   r€  r�  r   útuple[Any, ...]r$   )r‡  Úpooler_outputrû   rˆ  )r‹   rü   r€  r�  rž   r¡   r«  r¬  r   r   rû   rˆ  )r€   r™   rÒ   rü   r€  r�  Úembedding_outputÚencoder_outputsÚsequence_outputÚpooled_outputÚhead_outputss              r)   r¢   z(WindowedDinov2WithRegistersModel.forward  s*  € ðL 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆàÐÝÐ?Ñ@Ô@Ð@àŸ?š?¨<È˜?ÑYÔYÐàŸ,š,ØØ/Ø!5Ø#ð	 'ñ 
ô 
ˆð *¨!Ô,ˆØŸ.š.¨Ñ9Ô9ˆØ'¨¨¨¨1¨a¨a¨a¨Ô0ˆàð 	OØ+¨]Ð;ˆLÝÐ)¨<¸/È!È"È"Ô:MÑ+MÑNÔNÐNå)Ø-Ø'Ø)Ô7Ø&Ô1ð	
ñ 
ô 
ð 	
r+   rå   ©r    rŠ   )r²  r³  r    rv   )r·  r`   r    rv   ©NNNNN)r™   rÓ   rÒ   rÓ   rü   rÀ  r€  rÀ  r�  rÀ  r    rÁ  )r‚   rƒ   r„   r|   r°  r¶  r½  r   Ú+DINOV2_WITH_REGISTERS_BASE_INPUTS_DOCSTRINGr   r   Ú_CONFIG_FOR_DOCr¢   r‡   rˆ   s   @r)   r©  r©  ²  sí   ø€ € € € € ð

ð 
ð 
ð 
ð 
ð 
ð0ð 0ð 0ð 0ðpð pð pð pð1'ð 1'ð 1'ð 1'ðf +Ð*Ø3ñô ð ÐØ.¸_ðñ ô ð
 '+Ø)-Ø)-Ø,0Ø#'ð>
ð >
ð >
ð >
ñô ñô ð>
ð >
ð >
ð >
ð >
r+   r©  a  
    Args:
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See
            [`BitImageProcessor.preprocess`] for details.

        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
z¦Dinov2WithRegisters Model transformer with an image classification head on top (a linear layer on top of the final hidden state of the [CLS] token) e.g. for ImageNet.c                  ó|   ‡ — e Zd Zdˆ fd„Z ee¦  «         eee¬¦  «        	 	 	 	 	 ddd„¦   «         ¦   «         Z	ˆ xZ
S )Ú1WindowedDinov2WithRegistersForImageClassificationr‹   rM   r    rv   c                óN  •— t          ¦   «                              |¦  «         |j        }|€t          d¦  «        ‚|| _        t	          |¦  «        | _        |dk    rt          j        |j        dz  |¦  «        nt          j	        ¦   «         | _
        |                      ¦   «          d S )Nz=config.num_labels must be an integer for image classificationr   rœ   )r{   r|   Ú
num_labelsrž   r©  rO   r   rò   r[   rm  Ú
classifierr­  )r€   r‹   rÒ  r�   s      €r)   r|   z:WindowedDinov2WithRegistersForImageClassification.__init__d  sž   ø€ Ý‰Œ×Ò˜Ñ Ô Ð àÔ&ˆ
ØÐÝÐ\Ñ]Ô]Ð]Ø$ˆŒÝ%EÀfÑ%MÔ%MˆÔ"ð LVÐXYÊ>È>�"œ) FÔ$6¸Ñ$:¸JÑGÔGÐGÕ_aÔ_jÑ_lÔ_lˆŒð 	�ŠÑÔÐÐÐr+   r¾  Nr™   rÓ   Úlabelsrü   rÀ  r€  r�  ú'tuple[Any, ...] | ImageClassifierOutputc                ó–  — |�|n| j         j        }|                      ||||¬¦  «        }|d         }|dd…df         }|dd…dd…f         }	t          j        ||	                     d¬¦  «        gd¬¦  «        }
|                      |
¦  «        }d}|��t|                     |j        ¦  «        }| j         j	        €f| j
        dk    rd| j         _	        nN| j
        dk    r7|j        t          j        k    s|j        t          j        k    rd| j         _	        nd| j         _	        | j         j	        dk    rWt          ¦   «         }| j
        dk    r1 ||                     ¦   «         |                     ¦   «         ¦  «        }nŽ |||¦  «        }n�| j         j	        dk    rGt!          ¦   «         } ||                     d	| j
        ¦  «        |                     d	¦  «        ¦  «        }n*| j         j	        dk    rt%          ¦   «         } |||¦  «        }|s&|f|d
d…         z   }t'          d|�|g|¢R n|¦  «        S t)          |||j        |j        ¬¦  «        S )a§  Labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):

            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).

        Returns:

        Example:

        >>> import torch
        >>> from rfdetr.models.backbone.dinov2_with_windowed_attn import (
        ...     WindowedDinov2WithRegistersConfig,
        ...     WindowedDinov2WithRegistersForImageClassification,
        ... )
        >>> config = WindowedDinov2WithRegistersConfig(
        ...     image_size=32,
        ...     patch_size=16,
        ...     hidden_size=32,
        ...     num_hidden_layers=2,
        ...     num_attention_heads=4,
        ...     num_register_tokens=2,
        ...     num_labels=3,
        ... )
        >>> model = WindowedDinov2WithRegistersForImageClassification(config)
        >>> pixel_values = torch.randn(1, 3, 32, 32)
        >>> outputs = model(pixel_values)
        >>> list(outputs.logits.shape)
        [1, 3]
        NrÃ  r   r$   r¾   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr,   rœ   rÄ  )ÚlossÚlogitsrû   rˆ  )r‹   r�  rO   r-   rÊ   rš  rÓ  rÆ   rÈ   Úproblem_typerÒ  r¶   r6   r   r
   Úsqueezer	   r1   r   r   r   rû   rˆ  )r€   r™   rÔ  rü   r€  r�  r  rÈ  rª   Úpatch_tokensÚlinear_inputrÛ  rÚ  Úloss_fctr#  s                  r)   r¢   z9WindowedDinov2WithRegistersForImageClassification.forwards  sr  € ðZ &1Ð%<�k�kÀ$Ä+ÔBYˆà×,Ò,ØØ/Ø!5Ø#ð	 -ñ 
ô 
ˆð " !œ*ˆà# A A A q DÔ)ˆ	Ø& q q q¨!¨"¨" uÔ-ˆå”y )¨\×->Ò->À1Ð->Ñ-EÔ-EÐ!FÈAÐNÑNÔNˆà—’ Ñ.Ô.ˆàˆØÑà—Y’Y˜vœ}Ñ-Ô-ˆFØŒ{Ô'Ð/Ø”? aÒ'Ð'Ø/;�D”KÔ,Ð,Ø”_ qÒ(Ð(¨f¬l½e¼jÒ.HÐ.HÈFÌLÕ\aÔ\eÒLeÐLeØ/L�D”KÔ,Ð,à/K�D”KÔ,ð Œ{Ô'¨<Ò7Ð7Ý"™9œ9�Ø”? aÒ'Ð'Ø#˜8 F§N¢NÑ$4Ô$4°f·n²nÑ6FÔ6FÑGÔG�D�Dà#˜8 F¨FÑ3Ô3�D�DØ”Ô)Ð-JÒJÐJÝ+Ñ-Ô-�Ø�x §¢¨B°´Ñ @Ô @À&Ç+Â+ÈbÁ/Ä/ÑRÔR��Ø”Ô)Ð-IÒIÐIÝ,Ñ.Ô.�Ø�x ¨Ñ/Ô/�àð 	\Ø�Y ¨¨¨¤Ñ,ˆFÝÐ)¸dÐ>N¨D¨?°6¨?¨?¨?ÐTZÑ[Ô[Ð[å$ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r+   rå   rÌ  )r™   rÓ   rÔ  rÓ   rü   rÀ  r€  rÀ  r�  rÀ  r    rÕ  )r‚   rƒ   r„   r|   r   Ú&DINOV2_WITH_REGISTERS_INPUTS_DOCSTRINGr   r   rÎ  r¢   r‡   rˆ   s   @r)   rÐ  rÐ  Z  s¶   ø€ € € € € ðð ð ð ð ð ð +Ð*Ø.ñô ð ÐØ)Ø$ðñ ô ð '+Ø $Ø)-Ø,0Ø#'ð[
ð [
ð [
ð [
ñ	ô ñô ð[
ð [
ð [
ð [
ð [
r+   rÐ  zRDinov2WithRegisters backbone, to be used with frameworks like DETR and MaskFormer.c                  ó€   ‡ — e Zd Zdˆ fd„Zdd„Z ee¦  «         eee	¬¦  «        	 	 	 ddd„¦   «         ¦   «         Z
ˆ xZS )Ú#WindowedDinov2WithRegistersBackboner‹   rM   r    rv   c                óž  •‡— t          ¦   «                              ‰¦  «         |                      ¦   «          ˆfd„t          ‰j        dz   ¦  «        D ¦   «         | _        t          ‰¦  «        | _        t          ‰¦  «        | _	        t          j        ‰j        ‰j        ¬¦  «        | _        ‰j        | _        |                      ¦   «          d S )Nc                ó   •— g | ]	}‰j         ‘Œ
S r%   )r[   r~  s     €r)   rD   z@WindowedDinov2WithRegistersBackbone.__init__.<locals>.<listcomp>ã  s   ø€ Ð]Ð]Ð]°A˜VÔ/Ð]Ð]Ð]r+   r$   rf  )r{   r|   Ú_init_transformers_backboner}   r\   Únum_featuresr¦   r¡   r{  r«  r   rh  r[   re   r¬  rn   r­  rö   s    `€r)   r|   z,WindowedDinov2WithRegistersBackbone.__init__à  s¸   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø×(Ò(Ñ*Ô*Ð*Ø]Ð]Ð]Ð]½¸vÔ?WÐZ[Ñ?[Ñ9\Ô9\Ð]Ñ]Ô]ˆÔÝ?ÀÑGÔGˆŒÝ9¸&ÑAÔAˆŒåœ fÔ&8¸fÔ>SÐTÑTÔTˆŒà#)Ô#=ˆÔ ð 	�ŠÑÔÐÐÐr+   rŠ   c                ó   — | j         j        S ræ   r¯  rH  s    r)   r°  z8WindowedDinov2WithRegistersBackbone.get_input_embeddingsî  r±  r+   r¾  Nr™   r   r€  rÀ  rü   r�  ú tuple[Any, ...] | BackboneOutputc                óÀ  — |�|n| j         j        }|�|n| j         j        }|�|n| j         j        }|                      |¦  «        }|                      |d||¬¦  «        }|r|j        n|d         }d}t          | j        |¦  «        D �]o\  }	}
|	| j	        v �r_| j         j
        r|                      |
¦  «        }
| j         j        �r+|
dd…| j        dz   d…f         }
|j        \  }}}}| j         j        }||z  }||z  }| j         j        dk    r¡| j         j        dz  }|
j        \  }}}|| j         j        z  }|| j         j        z  }|
                     ||z  ||z  |¦  «        }
|
                     ||z  | j         j        z  | j         j        |||¦  «        }
|
                     ddddd	¦  «        }
|
                     |||d
¦  «        }
|
                     dddd¦  «                             ¦   «         }
||
fz  }�Œq|s/|r|f|dd…         z   }n|f|dd…         z   }t)          d|¦  «        S t+          t)          t,          |¦  «        |r|j        nd|r|j        nd¬¦  «        S )aV  
        Returns:

        Examples:

        >>> import torch
        >>> from rfdetr.models.backbone.dinov2_with_windowed_attn import (
        ...     WindowedDinov2WithRegistersBackbone,
        ...     WindowedDinov2WithRegistersConfig,
        ... )
        >>> config = WindowedDinov2WithRegistersConfig(
        ...     image_size=32,
        ...     patch_size=16,
        ...     hidden_size=32,
        ...     num_hidden_layers=2,
        ...     num_attention_heads=4,
        ...     num_register_tokens=2,
        ...     out_indices=[2],
        ... )
        >>> model = WindowedDinov2WithRegistersBackbone(config)
        >>> pixel_values = torch.randn(1, 3, 32, 32)
        >>> outputs = model(pixel_values)
        >>> len(outputs.feature_maps)
        1
        >>> list(outputs.feature_maps[0].shape)
        [1, 32, 2, 2]

        NT)r€  rü   r�  r$   r%   rœ   r   rY   rR   r,   rÄ  )Úfeature_mapsrû   rˆ  )r‹   r�  r€  rü   r¡   r«  rû   Úzipr>   r:   ro   r¬  rp   rn   r�   rg   rq   rÂ   rÃ   r2   r   r   r   rˆ  )r€   r™   r€  rü   r�  rÆ  r  rû   rë  rz   r4  rÙ   rÚ   r²   r³   rg   rÝ   rÞ   rw  rt  ru  rv  râ   rá   r#  s                            r)   r¢   z+WindowedDinov2WithRegistersBackbone.forwardñ  sü  € ðR &1Ð%<�k�kÀ$Ä+ÔBYˆà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐàŸ?š?¨<Ñ8Ô8Ðà—,’,Ø°4ÐK\Ðjuð ñ 
ô 
ˆð 2=ÐL˜Ô-Ð-À'È!Ä*ˆà13ˆÝ#& tÔ'7¸Ñ#GÔ#Gð %	0ñ %	0ÑˆE�<Ø˜Ô)Ð)Ñ)Ø”;Ô.ð @Ø#'§>¢>°,Ñ#?Ô#?�LØ”;Ô4ñ QØ#/°°°°4Ô3KÈaÑ3OÐ3QÐ3QÐ0QÔ#R�Lð 4@Ô3EÑ0�J  6¨5Ø!%¤Ô!7�Jà$*¨jÑ$8�MØ$)¨ZÑ$7�Mà”{Ô.°Ò2Ð2à.2¬kÔ.EÀqÑ.HÐ+ØEQÔEWÑB˜Ð'8¸(Ø3@ÀDÄKÔD[Ñ3[Ð0Ø3@ÀDÄKÔD[Ñ3[Ð0Ø'3×';Ò';Ø)Ð-@Ñ@Ø/Ð2CÑCØ$ñ(ô (˜ð
 (4×';Ò';Ø*Ð.AÑAÀTÄ[ÔE\Ñ\Ø œKÔ3Ø4Ø4Ø$ñ(ô (˜ð (4×';Ò';¸A¸qÀ!ÀQÈÑ'JÔ'J˜à#/×#7Ò#7¸
ÀMÐS`ÐbdÑ#eÔ#e�LØ#/×#7Ò#7¸¸1¸aÀÑ#CÔ#C×#NÒ#NÑ#PÔ#P�Là  Ñ/�ùàð 	3Ø#ð 7Ø&˜¨7°1°2°2¬;Ñ6��à&˜¨7°1°2°2¬;Ñ6�ÝÐ)¨6Ñ2Ô2Ð2åõ �c <Ñ0Ô0Ø3GÐQ˜'Ô/Ð/ÈTØ->ÐH�wÔ)Ð)ÀDð
ñ 
ô 
ð 	
r+   rå   rË  )NNN)
r™   r   r€  rÀ  rü   rÀ  r�  rÀ  r    ré  )r‚   rƒ   r„   r|   r°  r   rá  r   r   rÎ  r¢   r‡   rˆ   s   @r)   rã  rã  Ø  sÃ   ø€ € € € € ðð ð ð ð ð ð0ð 0ð 0ð 0ð +Ð*Ø.ñô ð ÐØ"°ðñ ô ð -1Ø)-Ø#'ðf
ð f
ð f
ð f
ñô ñô ðf
ð f
ð f
ð f
ð f
r+   rã  )r–  r©  rÐ  rã  )
r   r   r   r   r   r   r   r   r    r!   )r:   r;   r<   r=   r>   r?   r    r@   )r:   r;   r<   r=   r>   r?   r    rI   )rT   F)r8  r   r9  rb   r  rj   r    r   )Fr…   Ú
__future__r   Úcollections.abcr‘   r   Útypingr   r   r-   r   r   Útorch.nnr   r	   r
   Útransformersr   r   Útransformers.activationsr   Ú transformers.configuration_utilsr   Útransformers.modeling_outputsr   r   r   r   Útransformers.modeling_utilsr   Útransformers.pytorch_utilsr   Útransformers.utilsr   r   r   r   r   Ú
get_loggerr‚   r  r9   rH   rL   rÎ  rM   ÚModulerŠ   r¦   rê   r  r  r   r.  r1  rA  rC  rK  rU  rj  rd  r{  r–  Ú%DINOV2_WITH_REGISTERS_START_DOCSTRINGrÍ  r©  rá  rÐ  rã  Ú__all__r%   r+   r)   ú<module>rü     s  ððð ð, #Ð "Ð "Ð "Ð "Ð "à Ð Ð Ð Ø €€€Ø Ð Ð Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð Ð Ð Ø AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aðð ð ð ð ð ð ð ð ,Ð +Ð +Ð +Ð +Ð +Ø =Ð =Ð =Ð =Ð =Ð =ðð ð ð ð ð ð ð ð ð ð ð ð 8Ð 7Ð 7Ð 7Ð 7Ð 7Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð ð ð 
ˆÔ	˜HÑ	%Ô	%€ðð ð ð ð>%ð %ð %ð %ð6ð 6ð 6ð 6ðB 6€ðN=ð N=ð N=ð N=ð N=Ð(;Ð=Mñ N=ô N=ð N=ðb&(ð &(ð &(ð &(ð &(¨¬ñ &(ô &(ð &(ðR\(ð \(ð \(ð \(ð \(¨B¬Iñ \(ô \(ð \(ð~9ð 9ð 9ð 9ð 9 r¤yñ 9ô 9ð 9ðx&#ð &#ð &#ð &#ð &#Ð+Kñ &#ô &#ð &#ðRð ð ð ð  B¤Iñ ô ð ð $Mð $Mð $Mð $Mð $M 2¤9ñ $Mô $Mð $MðNFð Fð Fð Fð FÐ'Cñ Fô Fð Fð+ð +ð +ð +ð + B¤Iñ +ô +ð +ðð ð ð ð ð&%ð %ð %ð %ð % "¤)ñ %ô %ð %ðð ð ð ð ˜RœYñ ô ð ð&6ð 6ð 6ð 6ð 6 2¤9ñ 6ô 6ð 6ð$ *Ø,ð+ð +Ð 'ðGMð GMð GMð GMð GM r¤yñ GMô GMð GMðT:
ð :
ð :
ð :
ð :
¨¬ñ :
ô :
ð :
ðz#)ð #)ð #)ð #)ð #)°ñ #)ô #)ð #)ðL	)Ð %ð/Ð +ð* ÐØsØ)ñô ðP
ð P
ð P
ð P
ð P
Ð'Qñ P
ô P
ñ	ô ðP
ðf*Ð &ð" Ðð	bð *ñô ðt
ð t
ð t
ð t
ð t
Ø.ñt
ô t
ñô ðt
ðn ÐØXØ)ñô ðA
ð A
ð A
ð A
ð A
Ø.ØñA
ô A
ñ	ô ðA
ðHð ð €€€r+   