§
    ·¯¸jbÁ  ã                   óœ  — d Z ddlZddlZddlmZmZmZmZmZm	Z	m
Z
mZmZ ddlZddlmZ ddlmZmZ ddlmZmZmZmZmZmZmZmZmZmZmZmZm Z  ddl!m"Z" ddl#m$Z$ dd	l%m&Z& dd
l'm(Z(m)Z) ddl*m+Z+m,Z,m-Z- ddl.m/Z/ dgZ0 ej1        e2¦  «        Z3ee4e
e4e4f         f         Z5dej6        de
e4e4f         dej6        fd„Z7e&dej6        de
e4e4f         de4de4dej6        f
d„¦   «         Z8ddde4de4dej6        fd„Z9 G d„ dej:        ¦  «        Z; G d„ dej:        ¦  «        Z< G d„ dej:        ¦  «        Z= G d„ d ej:        ¦  «        Z> G d!„ dej:        ¦  «        Z?d"e@d#ej:        deeAej6        f         fd$„ZBded&eAd'eCde?fd(„ZDdfd*eAdeeAef         fd+„ZE e+i d, eEd-d.¬/¦  «        “d0 eEd-d1¬/¦  «        “d2 eEd-d3d4d5d6¬7¦  «        “d8 eEd-d9¬/¦  «        “d: eEd-d;d4d5d6¬7¦  «        “d< eEd-d=¬/¦  «        “d> eEd-d?¬/¦  «        “d@ eEd-dA¬/¦  «        “dB eEd-dCd4d5d6¬7¦  «        “dD eEd-dE¬/¦  «        “dF eEd-dGdH¬I¦  «        “dJ eEd-dKdH¬I¦  «        “dL eEd-dMdH¬I¦  «        “dN eEd-dOd4d5d6dH¬P¦  «        “dQ eEd-dRdH¬I¦  «        “dS eEd-dTd4d5d6dH¬P¦  «        “dU eEd-dV¬/¦  «        “ eEd-dW¬/¦  «         eEd-dX¬/¦  «        dYœ¥¦  «        ZFe,dede?fdZ„¦   «         ZGe,dede?fd[„¦   «         ZHe,dede?fd\„¦   «         ZIe,dede?fd]„¦   «         ZJe,dede?fd^„¦   «         ZKe,dede?fd_„¦   «         ZLe,dede?fd`„¦   «         ZMe,dede?fda„¦   «         ZNe,dede?fdb„¦   «         ZO e-e2dLdNdQdSdcœ¦  «         dS )gaß   Swin Transformer
A PyTorch impl of : `Swin Transformer: Hierarchical Vision Transformer using Shifted Windows`
    - https://arxiv.org/pdf/2103.14030

Code/weights from https://github.com/microsoft/Swin-Transformer, original copyright/license info below

S3 (AutoFormerV2, https://arxiv.org/abs/2111.14725) Swin weights from
    - https://github.com/microsoft/Cream/tree/main/AutoFormerV2

Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman
é    N)	ÚAnyÚDictÚCallableÚListÚOptionalÚSetÚTupleÚUnionÚType©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)Ú
PatchEmbedÚMlpÚDropPathÚcalculate_drop_path_ratesÚClassifierHeadÚ	to_2tupleÚ	to_ntupleÚtrunc_normal_Úuse_fused_attnÚresize_rel_pos_bias_tableÚresample_patch_embedÚndgridÚget_device_dtypeé   )Úbuild_model_with_cfg)Úfeature_take_indices)Úregister_notrace_function)Úcheckpoint_seqÚnamed_apply)Úgenerate_default_cfgsÚregister_modelÚregister_model_deprecations)Úget_init_weights_vitÚSwinTransformerÚxÚwindow_sizeÚreturnc                 ó*  — | j         \  }}}}|                      |||d         z  |d         ||d         z  |d         |¦  «        } |                      dddddd¦  «                             ¦   «                              d|d         |d         |¦  «        }|S )zóPartition into non-overlapping windows.

    Args:
        x: Input tokens with shape [B, H, W, C].
        window_size: Window size.

    Returns:
        Windows after partition with shape [B * num_windows, window_size, window_size, C].
    r   r   é   é   é   é   éÿÿÿÿ©ÚshapeÚviewÚpermuteÚ
contiguous)r'   r(   ÚBÚHÚWÚCÚwindowss          úl/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/models/swin_transformer.pyÚwindow_partitionr;   *   s–   € ð ”�J€A€qˆ!ˆQØ	�Šˆq�!�{ 1”~Ñ% {°1¤~°q¸KÈ¼NÑ7JÈKÐXYÌNÐ\]Ñ^Ô^€AØ�iŠi˜˜1˜a  A qÑ)Ô)×4Ò4Ñ6Ô6×;Ò;¸BÀÈAÄÐP[Ð\]ÔP^Ð`aÑbÔb€GØ€Nó    r9   r6   r7   c                 ó  — | j         d         }|                      d||d         z  ||d         z  |d         |d         |¦  «        }|                     dddddd¦  «                             ¦   «                              d|||¦  «        }|S )a
  Reverse window partition.

    Args:
        windows: Windows with shape (num_windows*B, window_size, window_size, C).
        window_size: Window size.
        H: Height of image.
        W: Width of image.

    Returns:
        Tensor with shape (B, H, W, C).
    r/   r   r   r+   r,   r-   r.   r0   )r9   r(   r6   r7   r8   r'   s         r:   Úwindow_reverser>   =   s‰   € ð 	Œ�bÔ€AØ�Š�R˜˜k¨!œnÑ,¨a°;¸q´>Ñ.AÀ;ÈqÄ>ÐS^Ð_`ÔSaÐcdÑeÔe€AØ	�	Š	�!�Q˜˜1˜a Ñ#Ô#×.Ò.Ñ0Ô0×5Ò5°b¸!¸QÀÑBÔB€AØ€Hr<   Úwin_hÚwin_wc           
      óH  — t          j        t          t          j        | |t           j        ¬¦  «        t          j        ||t           j        ¬¦  «        ¦  «        ¦  «        }t          j        |d¦  «        }|dd…dd…df         |dd…ddd…f         z
  }|                     ddd¦  «                             ¦   «         }|dd…dd…dfxx         | dz
  z  cc<   |dd…dd…dfxx         |dz
  z  cc<   |dd…dd…dfxx         d|z  dz
  z  cc<   |                     d¦  «        S )zÈGet pair-wise relative position index for each token inside the window.

    Args:
        win_h: Window height.
        win_w: Window width.

    Returns:
        Relative position index tensor.
    ©ÚdeviceÚdtyper   Nr,   r   r/   )	ÚtorchÚstackr   ÚarangeÚlongÚflattenr3   r4   Úsum)r?   r@   rC   ÚcoordsÚcoords_flattenÚrelative_coordss         r:   Úget_relative_position_indexrN   P   sU  € õ Œ[�ÝŒ�U 6µ´Ð<Ñ<Ô<ÝŒ�U 6µ´Ð<Ñ<Ô<ñô ñ ô €Fõ ”] 6¨1Ñ-Ô-€NØ$ Q Q Q¨¨¨¨4 ZÔ0°>À!À!À!ÀTÈ1È1È1À*Ô3MÑM€OØ%×-Ò-¨a°°AÑ6Ô6×AÒAÑCÔC€OØ�A�A�A�q�q�q˜!�GÐÐÔ ¨¡	Ñ)ÐÐÑØ�A�A�A�q�q�q˜!�GÐÐÔ ¨¡	Ñ)ÐÐÑØ�A�A�A�q�q�q˜!�GÐÐÔ  E¡	¨A¡Ñ-ÐÐÑØ×Ò˜rÑ"Ô"Ð"r<   c                   ó  ‡ — e Zd ZU dZej        j        e         ed<   	 	 	 	 	 	 	 dde	de	d	e
e	         d
edededefˆ fd„Zdd„Zdd„Zd
ee	e	f         ddfd„Zdej        fd„Zddej        de
ej                 dej        fd„Zdd„Zˆ xZS )ÚWindowAttentionz‹Window based multi-head self attention (W-MSA) module with relative position bias.

    Supports both shifted and non-shifted windows.
    Ú
fused_attnNé   Tç        ÚdimÚ	num_headsÚhead_dimr(   Úqkv_biasÚ	attn_dropÚ	proj_dropc
           	      ó,  •— ||	dœ}
t          ¦   «                              ¦   «          || _        t          |¦  «        | _        | j        \  }}||z  | _        || _        |p||z  }||z  }|dz  | _        t          d¬¦  «        | _	        t          j        t          j        d|z  dz
  d|z  dz
  z  |fi |
¤Ž¦  «        | _        |                      dt          j        ||z  ||z  |t          j        ¬¦  «        d¬	¦  «         t          j        ||d
z  fd|i|
¤Ž| _        t          j        |¦  «        | _        t          j        ||fi |
¤Ž| _        t          j        |¦  «        | _        t          j        d¬¦  «        | _        |                      ¦   «          dS )a¶  
        Args:
            dim: Number of input channels.
            num_heads: Number of attention heads.
            head_dim: Number of channels per head (dim // num_heads if not set)
            window_size: The height and width of the window.
            qkv_bias:  If True, add a learnable bias to query, key, value.
            attn_drop: Dropout ratio of attention weight.
            proj_drop: Dropout ratio of output.
        rB   g      à¿T)Úexperimentalr,   r   Úrelative_position_indexF©Ú
persistentr+   Úbiasr/   )rT   N)ÚsuperÚ__init__rT   r   r(   Úwindow_arearU   Úscaler   rQ   ÚnnÚ	ParameterrE   ÚemptyÚrelative_position_bias_tableÚregister_bufferrH   ÚLinearÚqkvÚDropoutrX   ÚprojrY   ÚSoftmaxÚsoftmaxÚreset_parameters)ÚselfrT   rU   rV   r(   rW   rX   rY   rC   rD   Úddr?   r@   Úattn_dimÚ	__class__s                 €r:   ra   zWindowAttention.__init__o   s¯  ø€ ð, ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØˆŒÝ$ [Ñ1Ô1ˆÔØÔ'‰ˆˆuØ  5™=ˆÔØ"ˆŒØÐ/˜s iÑ/ˆØ˜iÑ'ˆØ Ñ%ˆŒ
Ý(°dÐ;Ñ;Ô;ˆŒõ -/¬LÝŒK˜˜U™ Q™¨1¨u©9°q©=Ñ9¸9ÐKÐKÈÐKÐKñ-Mô -MˆÔ)ð 	×ÒØ%ÝŒK˜ ™ u¨u¡}¸VÍ5Ì:ÐVÑVÔVØð 	ñ 	
ô 	
ð 	
õ ”9˜S (¨Q¡,ÐDÐD°XÐDÀÐDÐDˆŒÝœ IÑ.Ô.ˆŒÝ”I˜h¨Ð2Ð2¨rÐ2Ð2ˆŒ	Ýœ IÑ.Ô.ˆŒÝ”z bÐ)Ñ)Ô)ˆŒð 	×ÒÑÔÐÐÐr<   r)   c                 óZ   — t          | j        d¬¦  «         |                      ¦   «          dS )ú"Initialize parameters and buffers.g{®Gáz”?)ÚstdN)r   rg   Ú_init_buffers©rp   s    r:   ro   z WindowAttention.reset_parameters¥   s0   € å�dÔ7¸SÐAÑAÔAÐAØ×ÒÑÔÐÐÐr<   c                 ó‚   — | j         \  }}| j                             t          ||| j        j        ¬¦  «        ¦  «         dS )ú.Compute and fill non-persistent buffer values.©rC   N)r(   r\   Úcopy_rN   rC   )rp   r?   r@   s      r:   rw   zWindowAttention._init_buffersª   sK   € àÔ'‰ˆˆuØÔ$×*Ò*Ý'¨¨u¸TÔ=YÔ=`ÐaÑaÔañ	
ô 	
ð 	
ð 	
ð 	
r<   c           	      óÆ  — t          |¦  «        }|| j        k    rdS || _        | j        \  }}||z  | _        t          j        ¦   «         5  d|z  dz
  d|z  dz
  z  | j        f}t          j        t          | j	        | j        |¬¦  «        ¦  «        | _	        |  
                    dt          ||| j	        j        ¬¦  «        d¬¦  «         ddd¦  «         dS # 1 swxY w Y   dS )	zzUpdate window size & interpolate position embeddings
        Args:
            window_size (int): New window size
        Nr,   r   ©Únew_window_sizeÚnew_bias_shaper\   r{   Fr]   )r   r(   rb   rE   Úno_gradrU   rd   re   r   rg   rh   rN   rC   )rp   r(   r?   r@   r€   s        r:   Úset_window_sizezWindowAttention.set_window_size±   sG  € õ
   Ñ,Ô,ˆØ˜$Ô*Ò*Ð*ØˆFØ&ˆÔØÔ'‰ˆˆuØ  5™=ˆÔÝŒ]‰_Œ_ð 	ð 	Ø %™i¨!™m°°E±	¸A±Ñ>ÀÄÐNˆNÝ02´Ý)ØÔ5Ø$(Ô$4Ø#1ðñ ô ñ1ô 1ˆDÔ-ð × Ò Ø)Ý+¨E°5ÀÔAbÔAiÐjÑjÔjØ ð !ñ ô ð ð	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	ð 	ð 	s   ÁA>CÃCÃCc                 ó  — | j         | j                             d¦  «                                      | j        | j        d¦  «        }|                     ddd¦  «                             ¦   «         }|                     d¦  «        S )Nr/   r,   r   r   )rg   r\   r2   rb   r3   r4   Ú	unsqueeze)rp   Úrelative_position_biass     r:   Ú_get_rel_pos_biasz!WindowAttention._get_rel_pos_biasÊ   sv   € Ø!%Ô!BØÔ(×-Ò-¨bÑ1Ô1ô"3ß37²4¸Ô8HÈ$ÔJZÐ\^Ñ3_Ô3_ð 	à!7×!?Ò!?ÀÀ1ÀaÑ!HÔ!H×!SÒ!SÑ!UÔ!UÐØ%×/Ò/°Ñ2Ô2Ð2r<   r'   Úmaskc                 ó¾  — |j         \  }}}|                      |¦  «                             ||d| j        d¦  «                             ddddd¦  «        }|                     d¦  «        \  }}}	| j        r¸|                      ¦   «         }
|�e|j         d         }|                     d|d||¦  «         	                    ||z  d| j        dd¦  «        }|
|                     d| j        ||¦  «        z   }
t          j        j                             |||	|
| j        r| j        j        nd¬	¦  «        }nÜ|| j        z  }||                     d
d¦  «        z  }||                      ¦   «         z   }|�q|j         d         }|                     d|| j        ||¦  «        |                     d¦  «                             d¦  «        z   }|                     d| j        ||¦  «        }|                      |¦  «        }|                      |¦  «        }||	z  }|                     dd¦  «                             ||d¦  «        }|                      |¦  «        }|                      |¦  «        }|S )a  Forward pass.

        Args:
            x: Input features with shape of (num_windows*B, N, C).
            mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None.

        Returns:
            Output features with shape of (num_windows*B, N, C).
        r+   r/   r,   r   r   r-   NrS   )Ú	attn_maskÚ	dropout_péþÿÿÿ)r1   rj   ÚreshaperU   r3   ÚunbindrQ   r†   r2   ÚexpandrE   rd   Ú
functionalÚscaled_dot_product_attentionÚtrainingrX   Úprc   Ú	transposer„   rn   rl   rY   )rp   r'   r‡   ÚB_ÚNr8   rj   ÚqÚkÚvr‰   Únum_winÚattns                r:   ÚforwardzWindowAttention.forwardÐ   s>  € ð ”7‰ˆˆAˆqØ�hŠh�q‰kŒk×!Ò! " a¨¨D¬N¸BÑ?Ô?×GÒGÈÈ1ÈaÐQRÐTUÑVÔVˆØ—*’*˜Q‘-”-‰ˆˆ1ˆaàŒ?ð 	Ø×.Ò.Ñ0Ô0ˆIØÐØœ* Qœ-�Ø—y’y  G¨Q°°1Ñ5Ô5×<Ò<¸RÀ7¹]ÈBÐPTÔP^Ð`bÐdfÑgÔg�Ø%¨¯ª°R¸¼ÈÈAÑ(NÔ(NÑN�	Ý”Ô#×@Ò@Ø�1�aØ#Ø.2¬mÐC˜$œ.Ô*Ð*Àð Añ ô ˆAˆAð �D”J‘ˆAØ�q—{’{ 2 rÑ*Ô*Ñ*ˆDØ˜$×0Ò0Ñ2Ô2Ñ2ˆDØÐØœ* Qœ-�Ø—y’y  W¨d¬n¸aÀÑCÔCÀdÇnÂnÐUVÑFWÔFW×FaÒFaÐbcÑFdÔFdÑd�Ø—y’y  T¤^°Q¸Ñ:Ô:�Ø—<’< Ñ%Ô%ˆDØ—>’> $Ñ'Ô'ˆDØ�q‘ˆAà�KŠK˜˜1ÑÔ×%Ò% b¨!¨RÑ0Ô0ˆØ�IŠI�a‰LŒLˆØ�NŠN˜1ÑÔˆØˆr<   c                 ó.   — |                       ¦   «          dS ©z"Initialize non-persistent buffers.N©rw   rx   s    r:   Úinit_non_persistent_buffersz+WindowAttention.init_non_persistent_buffersú   ó   € à×ÒÑÔÐÐÐr<   )NrR   TrS   rS   NN©r)   N©N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__rE   ÚjitÚFinalÚboolÚ__annotations__Úintr   Ú_int_or_tuple_2_tÚfloatra   ro   rw   r	   r‚   ÚTensorr†   r›   rŸ   Ú__classcell__©rs   s   @r:   rP   rP   h   s‚  ø€ € € € € € ðð ð ”	” Ô%Ð%Ð%Ñ%ð '+Ø-.Ø!Ø!Ø!ØØð4 ð 4 àð4 ð ð4 ð ˜s”mð	4 ð
 +ð4 ð ð4 ð ð4 ð ð4 ð 4 ð 4 ð 4 ð 4 ð 4 ðlð ð ð ð

ð 
ð 
ð 
ð¨5°°c°¬?ð ¸tð ð ð ð ð23 5¤<ð 3ð 3ð 3ð 3ð(ð (˜œð (¨X°e´lÔ-Cð (ÈuÌ|ð (ð (ð (ð (ðTð ð ð ð ð ð ð r<   rP   c                    ó^  ‡ — e Zd ZdZddddddddd	d	d	ej        ej        ddfd
ededede	e         dedede
de
dede
dedededeej                 deej                 fˆ fd„Zd)d„Zd)d„Z	 	 	 d*de	ej                 de	ej                 de	ej                 de	ej                 fd „Z	 d+d!eeeeef         f         d"e	eeeeef         f                  deeeef         eeef         f         fd#„Z	 d+d$eeef         deeef         de	e
         fd%„Zd&„ Zdej        dej        fd'„Zd)d(„Zˆ xZS ),ÚSwinTransformerBlockzkSwin Transformer Block.

    A transformer block with window-based self-attention and shifted windows.
    r-   NrR   r   Fç      @TrS   rT   Úinput_resolutionrU   rV   r(   Ú
shift_sizeÚalways_partitionÚdynamic_maskÚ	mlp_ratiorW   rY   rX   Ú	drop_pathÚ	act_layerÚ
norm_layerc           
      ó  •— ||dœ}t          ¦   «                              ¦   «          || _        || _        t	          |¦  «        | _        || _        || _        |                      ||¦  «        \  | _	        | _
        | j	        d         | j	        d         z  | _        |	| _         ||fi |¤Ž| _        t          |f||| j	        |
||dœ|¤Ž| _        |dk    rt!          |¦  «        nt#          j        ¦   «         | _         ||fi |¤Ž| _        t+          d|t-          ||	z  ¦  «        ||dœ|¤Ž| _        |dk    rt!          |¦  «        nt#          j        ¦   «         | _        |                      ddd	¬
¦  «         |                      ¦   «          dS )aå  
        Args:
            dim: Number of input channels.
            input_resolution: Input resolution.
            window_size: Window size.
            num_heads: Number of attention heads.
            head_dim: Enforce the number of channels per head
            shift_size: Shift size for SW-MSA.
            always_partition: Always partition into full windows and shift
            mlp_ratio: Ratio of mlp hidden dim to embedding dim.
            qkv_bias: If True, add a learnable bias to query, key, value.
            proj_drop: Dropout rate.
            attn_drop: Attention dropout rate.
            drop_path: Stochastic depth rate.
            act_layer: Activation layer.
            norm_layer: Normalization layer.
        rB   r   r   )rU   rV   r(   rW   rX   rY   rS   )Úin_featuresÚhidden_featuresrº   Údropr‰   NFr]   © )r`   ra   rT   r´   r   Útarget_shift_sizer¶   r·   Ú_calc_window_shiftr(   rµ   rb   r¸   Únorm1rP   rš   r   rd   ÚIdentityÚ
drop_path1Únorm2r   r«   ÚmlpÚ
drop_path2rh   ro   )rp   rT   r´   rU   rV   r(   rµ   r¶   r·   r¸   rW   rY   rX   r¹   rº   r»   rC   rD   rq   rs   s                      €r:   ra   zSwinTransformerBlock.__init__  sÇ  ø€ ðJ ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØˆŒØ 0ˆÔÝ!*¨:Ñ!6Ô!6ˆÔØ 0ˆÔØ(ˆÔØ,0×,CÒ,CÀKÐQ[Ñ,\Ô,\Ñ)ˆÔ˜$œ/ØÔ+¨AÔ.°Ô1AÀ!Ô1DÑDˆÔØ"ˆŒà�Z Ð*Ð* rÐ*Ð*ˆŒ
Ý#Øð	
àØØÔ(ØØØð	
ð 	
ð ð	
ð 	
ˆŒ	ð 2;¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒà�Z Ð*Ð* rÐ*Ð*ˆŒ
Ýð 
ØÝ  i¡Ñ0Ô0ØØð	
ð 
ð
 ð
ð 
ˆŒð 2;¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒð 	×Ò˜[¨$¸5ÐÑAÔAÐAð 	×ÒÑÔÐÐÐr<   r)   c                 ó.   — |                       ¦   «          dS )ru   Nrž   rx   s    r:   ro   z%SwinTransformerBlock.reset_parametersR  r    r<   c                 óº   — | j         sS| j        j        j        }| j        j        j        }|                      ||¬¦  «        }|                      d|d¬¦  «         dS dS )rz   rB   r‰   Fr]   N)r·   rÃ   ÚweightrC   rD   Úget_attn_maskrh   )rp   rC   rD   r‰   s       r:   rw   z"SwinTransformerBlock._init_buffersV  sm   € àÔ ð 	KØ”ZÔ&Ô-ˆFØ”JÔ%Ô+ˆEØ×*Ò*°&ÀÐ*ÑFÔFˆIØ× Ò  ¨iÀEÐ ÑJÔJÐJÐJÐJð		Kð 	Kr<   r'   rC   rD   c           	      óD  — t          | j        ¦  «        �r|�)|j        d         |j        d         }}|j        }|j        }n| j        \  }}|}|}t          j        || j        d         z  ¦  «        | j        d         z  }t          j        || j        d         z  ¦  «        | j        d         z  }t          j
        d||df||¬¦  «        }d}d| j        d          f| j        d          | j        d          f| j        d          d ffD ]n}d| j        d          f| j        d          | j        d          f| j        d          d ffD ]0}	||d d …|d         |d         …|	d         |	d         …d d …f<   |dz  }Œ1Œot          || j        ¦  «        }
|
                     d| j        ¦  «        }
|
                     d¦  «        |
                     d¦  «        z
  }|                     |dk    t!          d¦  «        ¦  «                             |dk    t!          d¦  «        ¦  «        }nd }|S )Nr   r,   r   )rD   rC   r/   g      YÀrS   )Úanyrµ   r1   rC   rD   r´   ÚmathÚceilr(   rE   Úzerosr;   r2   rb   r„   Úmasked_fillr­   )rp   r'   rC   rD   r6   r7   Úimg_maskÚcntÚhÚwÚmask_windowsr‰   s               r:   rÌ   z"SwinTransformerBlock.get_attn_mask^  sW  € õ ˆtŒÑÔñ 	àˆ}Ø”w˜q”z 1¤7¨1¤:�1�Øœ�Øœ��àÔ,‘��1Ø�Ø�Ý”	˜!˜dÔ.¨qÔ1Ñ1Ñ2Ô2°TÔ5EÀaÔ5HÑHˆAÝ”	˜!˜dÔ.¨qÔ1Ñ1Ñ2Ô2°TÔ5EÀaÔ5HÑHˆAÝ”{ A q¨!¨Q <°uÀVÐLÑLÔLˆHØˆCà˜Ô)¨!Ô,Ð,Ð-ØÔ& qÔ)Ð)¨D¬O¸AÔ,>Ð+>Ð?Ø”o aÔ(Ð(¨$Ð/ðð ð �ð ˜TÔ-¨aÔ0Ð0Ð1ØÔ*¨1Ô-Ð-°´ÀÔ0BÐ/BÐCØœ/¨!Ô,Ð,¨dÐ3ðð ð �Að
 <?�H˜Q˜Q˜Q  !¤ Q q¤T 	¨1¨Q¬4°°!´¨9°a°a°aÐ7Ñ8Ø˜1‘H�C�Cðõ ,¨H°dÔ6FÑGÔGˆLØ'×,Ò,¨R°Ô1AÑBÔBˆLØ$×.Ò.¨qÑ1Ô1°L×4JÒ4JÈ1Ñ4MÔ4MÑMˆIØ!×-Ò-¨i¸1ªn½eÀF¹m¼mÑLÔL×XÒXÐYbÐfgÒYgÕinÐorÑisÔisÑtÔtˆIˆIàˆIØÐr<   Útarget_window_sizerÁ   c                 ól  — t          |¦  «        }|€-| j        }t          |¦  «        r|d         dz  |d         dz  f}nt          |¦  «        }| j        r||fS d„ t	          | j        |¦  «        D ¦   «         }d„ t	          | j        ||¦  «        D ¦   «         }t          |¦  «        t          |¦  «        fS )Nr   r,   r   c                 ó(   — g | ]\  }}||k    r|n|‘ŒS rÀ   rÀ   )Ú.0ÚrrÖ   s      r:   ú
<listcomp>z;SwinTransformerBlock._calc_window_shift.<locals>.<listcomp>—  s(   € ÐeÐeÐe©d¨a°˜A šF˜F�q�q¨ÐeÐeÐer<   c                 ó*   — g | ]\  }}}||k    rd n|‘ŒS ©r   rÀ   )rÛ   rÜ   rÖ   Úss       r:   rÝ   z;SwinTransformerBlock._calc_window_shift.<locals>.<listcomp>˜  s*   € ÐsÐsÐs©W¨Q°°1˜1 š6˜6�a�a qÐsÐsÐsr<   )r   rÁ   rÎ   r¶   Úzipr´   Útuple)rp   rØ   rÁ   r(   rµ   s        r:   rÂ   z'SwinTransformerBlock._calc_window_shift†  sÝ   € õ
 'Ð'9Ñ:Ô:ÐØÐ$à $Ô 6ÐÝÐ$Ñ%Ô%ð ]Ø%7¸Ô%:¸aÑ%?ÐASÐTUÔAVÐZ[ÑA[Ð$\Ð!øå )Ð*;Ñ <Ô <ÐàÔ ð 	9Ø%Ð'8Ð8Ð8àeÐeµc¸$Ô:OÐQcÑ6dÔ6dÐeÑeÔeˆØsÐs½¸DÔ<QÐS^Ð`qÑ8rÔ8rÐsÑsÔsˆ
Ý�[Ñ!Ô!¥5¨Ñ#4Ô#4Ð4Ð4r<   Ú	feat_sizec                 ó¨  — || _         |�|| _        |                      |¦  «        \  | _        | _        | j        d         | j        d         z  | _        | j                             | j        ¦  «         | j        �| j        j	        nd}| j        �| j        j
        nd}|                      d| j        rdn|                      ||¬¦  «        d¬¦  «         dS )z¸
        Args:
            feat_size: New input resolution
            window_size: New window size
            always_partition: Change always_partition attribute if not None
        Nr   r   r‰   rB   Fr]   )r´   r¶   rÂ   r(   rµ   rb   rš   r‚   r‰   rC   rD   rh   r·   rÌ   )rp   rã   r(   r¶   rC   rD   s         r:   Úset_input_sizez#SwinTransformerBlock.set_input_size›  sé   € ð !*ˆÔØÐ'Ø$4ˆDÔ!Ø,0×,CÒ,CÀKÑ,PÔ,PÑ)ˆÔ˜$œ/ØÔ+¨AÔ.°Ô1AÀ!Ô1DÑDˆÔØŒ	×!Ò! $Ô"2Ñ3Ô3Ð3Ø*.¬.Ð*D�”Ô&Ð&È$ˆØ(,¬Ð(B�”Ô$Ð$ÈˆØ×ÒØØÔ%ÐYˆDˆD¨4×+=Ò+=ÀVÐSXÐ+=Ñ+YÔ+YØð 	ñ 	
ô 	
ð 	
ð 	
ð 	
r<   c           	      óÔ  — |j         \  }}}}t          | j        ¦  «        }|r2t          j        || j        d          | j        d          fd¬¦  «        }n|}| j        d         || j        d         z  z
  | j        d         z  }| j        d         || j        d         z  z
  | j        d         z  }	t          j        j                             |ddd|	d|f¦  «        }|j         \  }
}}}
t          || j        ¦  «        }| 
                    d| j        |¦  «        }t          | dd¦  «        r|                      |¦  «        }n| j        }|                      ||¬¦  «        }| 
                    d| j        d         | j        d         |¦  «        }t!          || j        ||¦  «        }|d d …d |…d |…d d …f                              ¦   «         }|rt          j        || j        d¬¦  «        }n|}|S )	Nr   r   )r   r,   )ÚshiftsÚdimsr/   r·   F)r‡   )r1   rÎ   rµ   rE   Úrollr(   rd   r�   Úpadr;   r2   rb   ÚgetattrrÌ   r‰   rš   r>   r4   )rp   r'   r5   r6   r7   r8   Ú	has_shiftÚ	shifted_xÚpad_hÚpad_wÚ_ÚHpÚWpÚ	x_windowsr‰   Úattn_windowss                   r:   Ú_attnzSwinTransformerBlock._attnµ  s  € Ø”W‰
ˆˆ1ˆa�õ ˜œÑ(Ô(ˆ	Øð 	Ýœ
 1¨t¬¸qÔ/AÐ.AÀDÄOÐTUÔDVÐCVÐ-WÐ^dÐeÑeÔeˆIˆIàˆIð Ô! !Ô$ q¨4Ô+;¸AÔ+>Ñ'>Ñ>À$ÔBRÐSTÔBUÑUˆØÔ! !Ô$ q¨4Ô+;¸AÔ+>Ñ'>Ñ>À$ÔBRÐSTÔBUÑUˆÝ”HÔ'×+Ò+¨I¸¸1¸aÀÈÈ5Ð7QÑRÔRˆ	Ø ”‰ˆˆ2ˆr�1õ % Y°Ô0@ÑAÔAˆ	Ø—N’N 2 tÔ'7¸Ñ;Ô;ˆ	õ �4˜¨Ñ/Ô/ð 	'Ø×*Ò*¨9Ñ5Ô5ˆIˆIàœˆIØ—y’y °�yÑ;Ô;ˆð $×(Ò(¨¨TÔ-=¸aÔ-@À$ÔBRÐSTÔBUÐWXÑYÔYˆÝ" <°Ô1AÀ2ÀrÑJÔJˆ	Ø˜a˜a˜a  !  R a R¨¨¨˜lÔ+×6Ò6Ñ8Ô8ˆ	ð ð 	Ý”
˜9¨T¬_À6ÐJÑJÔJˆAˆAàˆAØˆr<   c                 ót  — |j         \  }}}}||                      |                      |                      |¦  «        ¦  «        ¦  «        z   }|                     |d|¦  «        }||                      |                      |                      |¦  «        ¦  «        ¦  «        z   }|                     ||||¦  «        }|S )z£Forward pass.

        Args:
            x: Input features with shape (B, H, W, C).

        Returns:
            Output features with shape (B, H, W, C).
        r/   )r1   rÅ   rõ   rÃ   rŒ   rÈ   rÇ   rÆ   )rp   r'   r5   r6   r7   r8   s         r:   r›   zSwinTransformerBlock.forwardÜ  sœ   € ð ”W‰
ˆˆ1ˆa�Ø�—’ §
¢
¨4¯:ª:°a©=¬=Ñ 9Ô 9Ñ:Ô:Ñ:ˆØ�IŠI�a˜˜QÑÔˆØ�—’ §¢¨¯ª°A©¬Ñ 7Ô 7Ñ8Ô8Ñ8ˆØ�IŠI�a˜˜A˜qÑ!Ô!ˆØˆr<   c                 ó.   — |                       ¦   «          dS r�   rž   rx   s    r:   rŸ   z0SwinTransformerBlock.init_non_persistent_buffersì  r    r<   r¡   )NNNr¢   )r£   r¤   r¥   r¦   rd   ÚGELUÚ	LayerNormr«   r¬   r   r©   r­   r   ÚModulera   ro   rw   rE   r®   rC   rD   rÌ   r
   r	   rÂ   rå   rõ   r›   rŸ   r¯   r°   s   @r:   r²   r²   ÿ   sä  ø€ € € € € ðð ð Ø&*Ø-.ØØ%*Ø!&Ø!Ø!Ø!Ø!Ø!Ø)+¬Ø*,¬,ØØð%K ð K àðK ð 0ðK ð ð	K ð
 ˜s”mðK ð +ðK ð ðK ð #ðK ð ðK ð ðK ð ðK ð ðK ð ðK ð ðK ð ˜BœI”ðK ð  ˜RœYœð!K ð K ð K ð K ð K ð K ðZð ð ð ðKð Kð Kð Kð )-Ø-1Ø+/ð	&ð &à˜œÔ%ð&ð ˜Uœ\Ô*ð&ð ˜EœKÔ(ð	&ð
 
�%”,Ô	ð&ð &ð &ð &ðV HLð5ð 5à % c¨5°°c°¬?Ð&:Ô ;ð5ð  (¨¨c°5¸¸c¸´?Ð.BÔ(CÔDð5ð 
ˆu�S˜#�XŒ  c¨3 h¤Ð/Ô	0ð	5ð 5ð 5ð 5ð2 04ð	
ð 
à˜S #˜X”ð
ð ˜s C˜xœð
ð ' tœnð	
ð 
ð 
ð 
ð4%ð %ð %ðN˜œð ¨%¬,ð ð ð ð ð ð ð ð ð ð ð ð r<   r²   c                   óŠ   ‡ — e Zd ZdZdej        ddfdedee         deej	                 fˆ fd„Z
dej        dej        fd	„Zˆ xZS )
ÚPatchMergingzVPatch Merging Layer.

    Downsample features by merging 2x2 neighboring patches.
    NrT   Úout_dimr»   c                 óÞ   •— ||dœ}t          ¦   «                              ¦   «          || _        |pd|z  | _         |d|z  fi |¤Ž| _        t          j        d|z  | j        fddi|¤Ž| _        dS )z³
        Args:
            dim: Number of input channels.
            out_dim: Number of output channels (or 2 * dim if None)
            norm_layer: Normalization layer.
        rB   r,   r-   r_   FN)r`   ra   rT   rý   Únormrd   ri   Ú	reduction)rp   rT   rý   r»   rC   rD   rq   rs   s          €r:   ra   zPatchMerging.__init__÷  s†   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØˆŒØÐ) ! c¡'ˆŒØ�J˜q 3™wÐ-Ð-¨"Ð-Ð-ˆŒ	Ýœ 1 s¡7¨D¬LÐKÐK¸uÐKÈÐKÐKˆŒˆˆr<   r'   r)   c                 ó|  — |j         \  }}}}ddd|dz  d|dz  f}t          j                             ||¦  «        }|j         \  }}}}|                     ||dz  d|dz  d|¦  «                             dddddd¦  «                             d¦  «        }|                      |¦  «        }|                      |¦  «        }|S )z¯Forward pass.

        Args:
            x: Input features with shape (B, H, W, C).

        Returns:
            Output features with shape (B, H//2, W//2, out_dim).
        r   r,   r   r+   r-   r.   )	r1   rd   r�   rê   rŒ   r3   rI   rÿ   r   )rp   r'   r5   r6   r7   r8   Ú
pad_valuesrð   s           r:   r›   zPatchMerging.forward  sÂ   € ð ”W‰
ˆˆ1ˆa�à˜˜A˜q 1™u a¨¨Q©Ð/ˆ
ÝŒM×Ò˜a Ñ,Ô,ˆØ”W‰
ˆˆ1ˆa�à�IŠI�a˜˜a™  A¨¡F¨A¨qÑ1Ô1×9Ò9¸!¸QÀÀ1ÀaÈÑKÔK×SÒSÐTUÑVÔVˆØ�IŠI�a‰LŒLˆØ�NŠN˜1ÑÔˆØˆr<   )r£   r¤   r¥   r¦   rd   rù   r«   r   r   rú   ra   rE   r®   r›   r¯   r°   s   @r:   rü   rü   ñ  s²   ø€ € € € € ðð ð &*Ø*,¬,ØØðLð LàðLð ˜c”]ðLð ˜RœYœð	Lð Lð Lð Lð Lð Lð*˜œð ¨%¬,ð ð ð ð ð ð ð ð r<   rü   c            "       ó2  ‡ — e Zd ZdZdddddddddddej        ddfd	ed
edeeef         dededede	e         de
dededededededeee         ef         deej                 f ˆ fd„Z	 ddeeef         dede	e         fd„Zdej        dej        fd„Zˆ xZS ) ÚSwinTransformerStagez|A basic Swin Transformer layer for one stage.

    Contains multiple Swin Transformer blocks and optional downsampling.
    Tr-   NrR   Fr³   rS   rT   rý   r´   ÚdepthÚ
downsamplerU   rV   r(   r¶   r·   r¸   rW   rY   rX   r¹   r»   c                 ó   •‡ ‡‡‡‡‡	‡
‡‡‡‡‡‡‡‡— ||dœŠt          ¦   «                              ¦   «          |‰ _        |‰ _        |rt	          d„ |D ¦   «         ¦  «        n|‰ _        |‰ _        d‰ _        t          ‰¦  «        Št	          d„ ‰D ¦   «         ¦  «        Š|rt          d|‰‰dœ‰¤Ž‰ _
        n |‰k    sJ ‚t          j        ¦   «         ‰ _
        t          j        ˆ	ˆˆˆˆ
ˆˆˆˆˆˆˆˆ ˆˆfd„t          |¦  «        D ¦   «         Ž ‰ _        dS )	aò  
        Args:
            dim: Number of input channels.
            out_dim: Number of output channels.
            input_resolution: Input resolution.
            depth: Number of blocks.
            downsample: Downsample layer at the end of the layer.
            num_heads: Number of attention heads.
            head_dim: Channels per head (dim // num_heads if not set)
            window_size: Local window size.
            mlp_ratio: Ratio of mlp hidden dim to embedding dim.
            qkv_bias: If True, add a learnable bias to query, key, value.
            proj_drop: Projection dropout rate.
            attn_drop: Attention dropout rate.
            drop_path: Stochastic depth rate.
            norm_layer: Normalization layer.
        rB   c              3   ó    K  — | ]	}|d z  V — Œ
dS ©r,   NrÀ   ©rÛ   Úis     r:   ú	<genexpr>z0SwinTransformerStage.__init__.<locals>.<genexpr>Q  s&   è è € Ð&HÐ&H°! q¨A¡vÐ&HÐ&HÐ&HÐ&HÐ&HÐ&Hr<   Fc                 ó   — g | ]}|d z  ‘ŒS ©r,   rÀ   )rÛ   rÖ   s     r:   rÝ   z1SwinTransformerStage.__init__.<locals>.<listcomp>U  s   € Ð8Ð8Ð8 q˜A ™FÐ8Ð8Ð8r<   )rT   rý   r»   c                 ó    •— g | ]J}t          d‰‰j        ‰
‰‰|d z  dk    rdn‰‰‰‰‰‰‰t          ‰t          ¦  «        r‰|         n‰‰	dœ‰¤Ž‘ŒKS )r,   r   )rT   r´   rU   rV   r(   rµ   r¶   r·   r¸   rW   rY   rX   r¹   r»   rÀ   )r²   Úoutput_resolutionÚ
isinstanceÚlist)rÛ   r  r¶   rX   rq   r¹   r·   rV   r¸   r»   rU   rý   rY   rW   rp   rµ   r(   s     €€€€€€€€€€€€€€€r:   rÝ   z1SwinTransformerStage.__init__.<locals>.<listcomp>d  s�   ø€ ð &#ð &#ð &#ð$ õ# !ð ØØ!%Ô!7Ø#Ø!Ø'Ø!" Q¡¨!¢ ˜1˜1°*Ø!1Ø)Ø#Ø!Ø#Ø#Ý*4°YÅÑ*EÔ*EÐT˜) Aœ,˜,È9Ø%ðð ð ðð ð&#ð &#ð &#r<   NrÀ   )r`   ra   rT   r´   râ   r  r  Úgrad_checkpointingr   rü   r  rd   rÄ   Ú
SequentialÚrangeÚblocks)rp   rT   rý   r´   r  r  rU   rV   r(   r¶   r·   r¸   rW   rY   rX   r¹   r»   rC   rD   rq   rµ   rs   s   ` `   ```````````  @@€r:   ra   zSwinTransformerStage.__init__'  sˆ  øøøøøøøøøøøøøøøø€ ðL ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØˆŒØ 0ˆÔØLVÐ!l¥Ð&HÐ&HÐ7GÐ&HÑ&HÔ&HÑ!HÔ!HÐ!HÐ\lˆÔØˆŒ
Ø"'ˆÔÝ Ñ,Ô,ˆÝÐ8Ð8¨KÐ8Ñ8Ô8Ñ9Ô9ˆ
ð ð 		,Ý*ð ØØØ%ðð ð ð	ð ˆDŒOˆOð ˜'’>�>�>�>Ý œk™mœmˆDŒOõ ”mð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#õ$ ˜5‘\”\ð%&#ñ &#ô &#ð $ˆŒˆˆr<   rã   c                 óì   — || _         t          | j        t          j        ¦  «        r|| _        nt          d„ |D ¦   «         ¦  «        | _        | j        D ]}|                     | j        ||¬¦  «         Œ dS )a   Updates the resolution, window size and so the pair-wise relative positions.

        Args:
            feat_size: New input (feature) resolution
            window_size: New window size
            always_partition: Always partition / shift the window
        c              3   ó    K  — | ]	}|d z  V — Œ
dS r	  rÀ   r
  s     r:   r  z6SwinTransformerStage.set_input_size.<locals>.<genexpr>‰  s&   è è € Ð*EÐ*E°a¨1°©6Ð*EÐ*EÐ*EÐ*EÐ*EÐ*Er<   ©rã   r(   r¶   N)	r´   r  r  rd   rÄ   r  râ   r  rå   )rp   rã   r(   r¶   Úblocks        r:   rå   z#SwinTransformerStage.set_input_sizex  s–   € ð !*ˆÔÝ�d”o¥r¤{Ñ3Ô3ð 	FØ%.ˆDÔ"Ð"å%*Ð*EÐ*E¸9Ð*EÑ*EÔ*EÑ%EÔ%EˆDÔ"Ø”[ð 	ð 	ˆEØ× Ò ØÔ0Ø'Ø!1ð !ñ ô ð ð ð	ð 	r<   r'   r)   c                 óÐ   — |                       |¦  «        }| j        r4t          j                             ¦   «         st          | j        |¦  «        }n|                      |¦  «        }|S )zsForward pass.

        Args:
            x: Input features.

        Returns:
            Output features.
        )r  r  rE   r§   Úis_scriptingr    r  ©rp   r'   s     r:   r›   zSwinTransformerStage.forward‘  s\   € ð �OŠO˜AÑÔˆàÔ"ð 	­5¬9×+AÒ+AÑ+CÔ+Cð 	Ý˜tœ{¨AÑ.Ô.ˆAˆAà—’˜A‘”ˆAØˆr<   r¢   )r£   r¤   r¥   r¦   rd   rù   r«   r	   r©   r   r¬   r­   r
   r   r   rú   ra   rå   rE   r®   r›   r¯   r°   s   @r:   r  r  !  s¼  ø€ € € € € ðð ð  $ØØ&*Ø-.Ø%*Ø!&Ø!Ø!Ø!Ø!Ø35Ø*,¬,ØØð'O$ð O$àðO$ð ðO$ð $ C¨ Hœoð	O$ð
 ðO$ð ðO$ð ðO$ð ˜s”mðO$ð +ðO$ð #ðO$ð ðO$ð ðO$ð ðO$ð ðO$ð ðO$ð  ˜T %œ[¨%Ð/Ô0ð!O$ð" ˜RœYœð#O$ð O$ð O$ð O$ð O$ð O$ðj 04ð	ð à˜S #˜X”ðð ðð ' tœnð	ð ð ð ð2˜œð ¨%¬,ð ð ð ð ð ð ð ð r<   r  c            ,       óf  ‡ — e Zd ZdZdddddddd	d
dddddddddeej        dd
d
fdedededede	dede
edf         de
edf         dee         dedededed ed!ed"ed#ed$ed%eej                 d&ee	eej                 f         d'e	f*ˆ fd(„Zej        j        dGd)e	d*ed+d
fd,„¦   «         Zej        j        d+ee	         fd-„¦   «         Z	 	 	 	 	 dHdee
eef                  dee
eef                  dee
eef                  d/edee         d+d
fd0„Zej        j        dId1ed+ee	ef         fd2„¦   «         Zej        j        dJd3ed+d
fd4„¦   «         Zej        j        d+ej        fd5„¦   «         ZdKdedee	         d+d
fd6„Z	 	 	 	 	 dLd8ej        d9eeee e         f                  d:ed;ed<e	d=ed+ee ej                 e
ej        e ej                 f         f         fd>„Z!	 	 	 dMd9eee e         f         d@edAed+e e         fdB„Z"d8ej        d+ej        fdC„Z#dId8ej        dDed+ej        fdE„Z$d8ej        d+ej        fdF„Z%ˆ xZ&S )Nr&   z§Swin Transformer.

    A PyTorch impl of : `Swin Transformer: Hierarchical Vision Transformer using Shifted Windows`  -
          https://arxiv.org/pdf/2103.14030
    éà   r-   r+   éè  Úavgé`   ©r,   r,   é   r,   ©r+   r$  é   é   NrR   FTr³   rS   gš™™™™™¹?Ú Úimg_sizeÚ
patch_sizeÚin_chansÚnum_classesÚglobal_poolÚ	embed_dimÚdepths.rU   rV   r(   r¶   Ústrict_img_sizer¸   rW   Ú	drop_rateÚproj_drop_rateÚattn_drop_rateÚdrop_path_rateÚembed_layerr»   Úweight_initc                 ó  •‡— t          ¦   «                              ¦   «          ||dœ}|dv sJ ‚|| _        || _        || _        d| _        t          |¦  «        | _        ‰| _        t          ‰d| j        dz
  z  z  ¦  «        x| _
        | _        g | _        t          ‰t          t          f¦  «        s ˆfd„t!          | j        ¦  «        D ¦   «         Š |d#|||‰d         ||ddœ|¤Ž| _        | j        j        } t'          | j        ¦  «        |	¦  «        }	t          |
t          t          f¦  «        s t'          | j        ¦  «        |
¦  «        }
nt          |
¦  «        dk    r|
f| j        z  }
t          |
¦  «        | j        k    sJ ‚ t'          | j        ¦  «        |¦  «        }t)          ||d	¬
¦  «        }g }‰d         }d}t!          | j        ¦  «        D ]»}‰|         } |t+          d#i d|“d| “d|d         |z  |d         |z  f“d||         “d|dk    “d||         “d|	|         “d|
|         “d|“d| “d||         “d|“d|“d|“d||         “d|“|¤Žgz  }| }|dk    r|dz  }| xj        t-          | ||z  d|› �¬¦  «        gz  c_        Œ¼t/          j        |Ž | _         || j
        fi |¤Ž| _        t7          | j
        |f||| j        dœ|¤Ž| _        |dk    rdn|| _        |dk    r|                      d ¬!¦  «         d"S d"S )$a~  
        Args:
            img_size: Input image size.
            patch_size: Patch size.
            in_chans: Number of input image channels.
            num_classes: Number of classes for classification head.
            embed_dim: Patch embedding dimension.
            depths: Depth of each Swin Transformer layer.
            num_heads: Number of attention heads in different layers.
            head_dim: Dimension of self-attention heads.
            window_size: Window size.
            mlp_ratio: Ratio of mlp hidden dim to embedding dim.
            qkv_bias: If True, add a learnable bias to query, key, value.
            drop_rate: Dropout rate.
            attn_drop_rate (float): Attention dropout rate.
            drop_path_rate (float): Stochastic depth rate.
            embed_layer: Patch embedding layer.
            norm_layer (nn.Module): Normalization layer.
        rB   )r(  r!  ÚNHWCr,   r   c                 ó:   •— g | ]}t          ‰d |z  z  ¦  «        ‘ŒS r  )r«   )rÛ   r  r.  s     €r:   rÝ   z,SwinTransformer.__init__.<locals>.<listcomp>æ  s*   ø€ ÐQÐQÐQ°Q�˜Y¨¨a©Ñ/Ñ0Ô0ÐQÐQÐQr<   r   )r)  r*  r+  r.  r»   r0  Ú
output_fmtT)Ú	stagewiserT   rý   r´   r  r  rU   rV   r(   r¶   r·   r¸   rW   rY   rX   r¹   r»   úlayers.)Únum_chsr   Úmodule)Ú	pool_typer1  Ú	input_fmtÚskipÚresetF)Úneeds_resetNrÀ   )r`   ra   r,  r+  r-  r:  ÚlenÚ
num_layersr.  r«   Únum_featuresÚhead_hidden_sizeÚfeature_infor  râ   r  r  Úpatch_embedÚ	grid_sizer   r   r  Údictrd   r  Úlayersrÿ   r   ÚheadÚweight_init_modeÚinit_weights)"rp   r)  r*  r+  r,  r-  r.  r/  rU   rV   r(   r¶   r0  r¸   rW   r1  r2  r3  r4  r5  r»   r6  rC   rD   Úkwargsrq   Ú
patch_gridÚdprrL  Úin_dimrc   r  rý   rs   s"         `                          €r:   ra   zSwinTransformer.__init__ª  s6  øø€ õ\ 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆØ˜kÐ)Ð)Ð)Ð)Ø&ˆÔØ ˆŒØ&ˆÔØ ˆŒå˜f™+œ+ˆŒØ"ˆŒÝ47¸	ÀAÈ$Ì/Ð\]ÑJ]ÑD^Ñ8^Ñ4_Ô4_Ð_ˆÔ˜DÔ1ØˆÔå˜)¥e­T ]Ñ3Ô3ð 	RØQÐQÐQÐQ½%ÀÄÑ:PÔ:PÐQÑQÔQˆIð '˜;ð 	
ØØ!ØØ ”lØ!Ø+Øð	
ð 	
ð ð	
ð 	
ˆÔð Ô%Ô/ˆ
ð .•9˜Tœ_Ñ-Ô-¨hÑ7Ô7ˆÝ˜+­­e }Ñ5Ô5ð 	;Ø4�) D¤OÑ4Ô4°[ÑAÔAˆKˆKÝ�ÑÔ Ò"Ð"Ø&˜.¨4¬?Ñ:ˆKÝ�;ÑÔ 4¤?Ò2Ð2Ð2Ð2Ø.•I˜dœoÑ.Ô.¨yÑ9Ô9ˆ	Ý'¨¸È$ÐOÑOÔOˆØˆØ˜1”ˆØˆÝ�t”Ñ'Ô'ð 	mð 	mˆAØ ”lˆGØÕ+ð ð ð Ø�Fðà˜ðð ˜q”M UÑ*Ø˜q”M UÑ*ð"ð "ðð ˜Q”i�iðð ˜qš5˜5ðð $ Aœ,˜,ðð " !œ˜ðð (¨œN˜Nðð "2Ð!1ðð "1Ð0Ð0ðð $ Aœ,˜,ðð "˜ðð  )˜.ð!ð" )˜.ð#ð$ ˜aœ&˜&ð%ð& &˜:Øð)ð ð ñ ˆFð, ˆFØ�1ŠuˆuØ˜‘
�ØÐÔ¥$¨wÀ*ÈuÑBTÐ]jÐghÐ]jÐ]jÐ"kÑ"kÔ"kÐ!lÑlÐÔÐÝ”m VÐ,ˆŒà�J˜tÔ0Ð7Ð7°BÐ7Ð7ˆŒ	Ý"ØÔØð
ð "ØØ”oð
ð 
ð ð
ð 
ˆŒ	ð ,7¸&Ò+@Ð+@  ÀkˆÔà˜&Ò Ð Ø×Ò¨%ÐÑ0Ô0Ð0Ð0Ð0ð !Ð r<   ÚmoderC  r)   c                 ó¤   — |p| j         }|dv sJ ‚d|v rt          j        | j        ¦  «         nd}t	          t          |||¬¦  «        | ¦  «         dS )a*  Initialize model weights.

        Args:
            mode: Weight initialization mode ('jax', 'jax_nlhb', 'moco', or '').
            needs_reset: If True, call reset_parameters() on modules that have it.
                Set to False when modules have already self-initialized in __init__.
        )ÚjaxÚjax_nlhbÚmocorB  r(  ÚnlhbrS   )Ú	head_biasrC  N)rN  rÏ   Úlogr,  r!   r%   )rp   rT  rC  rZ  s       r:   rO  zSwinTransformer.init_weights.  sm   € ð Ð,�tÔ,ˆØÐ?Ð?Ð?Ð?Ð?Ø39¸T°>°>•T”X˜dÔ.Ñ/Ô/Ð/Ð/Àrˆ	ÝÕ(¨¸ÐP[Ð\Ñ\Ô\Ð^bÑcÔcÐcÐcÐcr<   c                 óˆ   — t          ¦   «         }|                      ¦   «         D ]\  }}d|v r|                     |¦  «         Œ|S )z,Parameters that should not use weight decay.rg   )ÚsetÚnamed_parametersÚadd)rp   ÚnwdÚnrð   s       r:   Úno_weight_decayzSwinTransformer.no_weight_decay<  sL   € õ ‰eŒeˆØ×)Ò)Ñ+Ô+ð 	ð 	‰DˆAˆqØ-°Ð2Ð2Ø—’˜‘
”
�
øØˆ
r<   é   Úwindow_ratioc                 óR  ‡— |€|�(| j                              ||¬¦  «         | j         j        }|€t          ˆfd„|D ¦   «         ¦  «        }t	          | j        ¦  «        D ]G\  }}dt          |dz
  d¦  «        z  }	|                     |d         |	z  |d         |	z  f||¬¦  «         ŒHdS )aê  Update the image resolution and window size.

        Args:
            img_size: New input resolution, if None current resolution is used.
            patch_size: New patch size, if None use current patch size.
            window_size: New window size, if None based on new_img_size // window_div.
            window_ratio: Divisor for calculating window size from grid size.
            always_partition: Always partition into windows and shift (even if window size < feat size).
        N)r)  r*  c                 ó   •— g | ]}|‰z  ‘ŒS rÀ   rÀ   )rÛ   Úpgrd  s     €r:   rÝ   z2SwinTransformer.set_input_size.<locals>.<listcomp>[  s   ø€ Ð IÐ IÐ I¸  |Ñ!3Ð IÐ IÐ Ir<   r,   r   r   r  )rI  rå   rJ  râ   Ú	enumeraterL  Úmax)
rp   r)  r*  r(   rd  r¶   rQ  ÚindexÚstageÚstage_scales
       `     r:   rå   zSwinTransformer.set_input_sizeE  sà   ø€ ð" Ð :Ð#9ØÔ×+Ò+°XÈ*Ð+ÑUÔUÐUØÔ)Ô3ˆJàÐÝÐ IÐ IÐ IÐ I¸jÐ IÑ IÔ IÑJÔJˆKå% d¤kÑ2Ô2ð 	ð 	‰LˆE�5Ø�s 5¨1¡9¨aÑ0Ô0Ñ0ˆKØ× Ò Ø% aœ=¨KÑ7¸ÀA¼È+Ñ9UÐVØ'Ø!1ð !ñ ô ð ð ð	ð 	r<   Úcoarsec                 ó0   — t          d|rdng d¢¬¦  «        S )z"Group parameters for optimization.z^patch_embedz^layers\.(\d+)))z^layers\.(\d+).downsamplerß   )z^layers\.(\d+)\.\w+\.(\d+)N)z^norm)iŸ† )Ústemr  )rK  )rp   rm  s     r:   Úgroup_matcherzSwinTransformer.group_matchere  s;   € õ Ø Ø(.ð Ð$Ð$ð 5ð 5ð 5ð
ñ 
ô 
ð 	
r<   Úenablec                 ó(   — | j         D ]	}||_        Œ
dS )z)Enable or disable gradient checkpointing.N)rL  r  )rp   rq  Úls      r:   Úset_grad_checkpointingz&SwinTransformer.set_grad_checkpointingq  s*   € ð ”ð 	*ð 	*ˆAØ#)ˆAÔ Ð ð	*ð 	*r<   c                 ó   — | j         j        S )zGet the classifier head.)rM  Úfcrx   s    r:   Úget_classifierzSwinTransformer.get_classifierw  s   € ð ŒyŒ|Ðr<   c                 ó\   — t          | ¦  «        }|| _         | j        j        |fd|i|¤Ž dS )zŸReset the classifier head.

        Args:
            num_classes: Number of classes for new classifier.
            global_pool: Global pooling type.
        r?  N)r   r,  rM  rB  )rp   r,  r-  rq   s       r:   Úreset_classifierz SwinTransformer.reset_classifier|  s@   € õ ˜dÑ#Ô#ˆØ&ˆÔØˆŒ	Œ˜ÐAÐA¨{ÐA¸bÐAÐAÐAÐAÐAr<   ÚNCHWr'   Úindicesrÿ   Ú
stop_earlyr:  Úintermediates_onlyc                 óx  — |dv s
J d¦   «         ‚g }t          t          | j        ¦  «        |¦  «        \  }}	|                      |¦  «        }t          | j        ¦  «        }
t          j                             ¦   «         s|s| j        }n| j        d|	dz   …         }t          |¦  «        D ]v\  }} ||¦  «        }||v rb|r||
dz
  k    r|                      |¦  «        }n|}| 	                    dddd¦  «         
                    ¦   «         }|                     |¦  «         Œw|r|S |                      |¦  «        }||fS )aK  Forward features that returns intermediates.

        Args:
            x: Input image tensor.
            indices: Take last n blocks if int, all if None, select matching indices if sequence.
            norm: Apply norm layer to compatible intermediates.
            stop_early: Stop iterating over blocks when last desired intermediate hit.
            output_fmt: Shape of intermediate feature outputs.
            intermediates_only: Only return intermediate features.

        Returns:
            List of intermediate features or tuple of (final features, intermediates).
        )rz  zOutput shape must be NCHW.Nr   r   r+   r,   )r   rD  rL  rI  rE   r§   r  rh  rÿ   r3   r4   Úappend)rp   r'   r{  rÿ   r|  r:  r}  ÚintermediatesÚtake_indicesÚ	max_indexÚ
num_stagesÚstagesr  rk  Úx_inters                  r:   Úforward_intermediatesz%SwinTransformer.forward_intermediates‡  s^  € ð, ˜YÐ&Ð&Ð&Ð(DÑ&Ô&Ð&ØˆÝ"6µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�ið ×Ò˜QÑÔˆå˜œÑ%Ô%ˆ
ÝŒ9×!Ò!Ñ#Ô#ð 	1¨:ð 	1Ø”[ˆFˆFà”[  )¨a¡- Ô0ˆFÝ! &Ñ)Ô)ð 	.ð 	.‰HˆAˆuØ��a‘”ˆAØ�LÐ Ð Øð  ˜A ¨a¡Ò/Ð/Ø"Ÿiši¨™lœl�G�Gà�GØ!Ÿ/š/¨!¨Q°°1Ñ5Ô5×@Ò@ÑBÔB�Ø×$Ò$ WÑ-Ô-Ð-øàð 	!Ø Ð à�IŠI�a‰LŒLˆà�-ÐÐr<   r   Ú
prune_normÚ
prune_headc                 óâ   — t          t          | j        ¦  «        |¦  «        \  }}| j        d|dz   …         | _        |rt          j        ¦   «         | _        |r|                      dd¦  «         |S )aE  Prune layers not required for specified intermediates.

        Args:
            indices: Indices of intermediate layers to keep.
            prune_norm: Whether to prune normalization layer.
            prune_head: Whether to prune the classifier head.

        Returns:
            List of indices that were kept.
        Nr   r   r(  )r   rD  rL  rd   rÄ   rÿ   ry  )rp   r{  r‡  rˆ  r�  r‚  s         r:   Úprune_intermediate_layersz)SwinTransformer.prune_intermediate_layersº  sq   € õ  #7µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�iØ”k . 9¨q¡= .Ô1ˆŒØð 	&Ýœ™œˆDŒIØð 	)Ø×!Ò! ! RÑ(Ô(Ð(ØÐr<   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S )z/Forward pass through feature extraction layers.)rI  rL  rÿ   r  s     r:   Úforward_featuresz SwinTransformer.forward_featuresÒ  s8   € à×Ò˜QÑÔˆØ�KŠK˜‰NŒNˆØ�IŠI�a‰LŒLˆØˆr<   Ú
pre_logitsc                 ó^   — |r|                       |d¬¦  «        n|                       |¦  «        S )zÊForward pass through classifier head.

        Args:
            x: Feature tensor.
            pre_logits: Return features before final classifier.

        Returns:
            Output tensor.
        T)r�  )rM  )rp   r'   r�  s      r:   Úforward_headzSwinTransformer.forward_headÙ  s.   € ð 1;ÐLˆt�yŠy˜ tˆyÑ,Ô,Ð,ÀÇ	Â	È!ÁÄÐLr<   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S )zoForward pass.

        Args:
            x: Input tensor.

        Returns:
            Output logits.
        )rŒ  r�  r  s     r:   r›   zSwinTransformer.forwardå  s/   € ð ×!Ò! !Ñ$Ô$ˆØ×Ò˜aÑ Ô ˆØˆr<   )r(  T)NNNrc  N©F)Tr¢   )NFFrz  F)r   FT)'r£   r¤   r¥   r¦   r   rd   rù   r¬   r«   Ústrr	   r   r©   r­   r   rú   r
   ra   rE   r§   ÚignorerO  r   rb  rå   r   r   rp  rt  rw  ry  r®   r   r†  rŠ  rŒ  r�  r›   r¯   r°   s   @r:   r&   r&   £  s!  ø€ € € € € ðð ð +.ØØØ#Ø$ØØ&2Ø)7Ø&*Ø-.Ø%*Ø$(Ø!Ø!Ø!Ø$&Ø$&Ø$'Ø+5Ø68´lØ!ØØð1B1ð B1à'ðB1ð ðB1ð ð	B1ð
 ðB1ð ðB1ð ðB1ð ˜#˜s˜(”OðB1ð ˜S #˜X”ðB1ð ˜s”mðB1ð +ðB1ð #ðB1ð "ðB1ð ðB1ð ðB1ð  ð!B1ð" "ð#B1ð$ "ð%B1ð& "ð'B1ð( ˜bœiœð)B1ð* ˜c 4¨¬	¤?Ð2Ô3ð+B1ð, ð-B1ð B1ð B1ð B1ð B1ð B1ðH „YÔðdð d ð d¸ð dÈð dð dð dñ Ôðdð „YÔð  S¤ð ð ð ñ Ôðð 37Ø48Ø59Ø !Ø/3ðð à˜u S¨# XœÔ/ðð !  s¨C x¤Ô1ðð " %¨¨S¨¤/Ô2ð	ð
 ðð ' tœnðð 
ðð ð ð ð@ „YÔð	
ð 	
 Dð 	
°T¸#¸s¸(´^ð 	
ð 	
ð 	
ñ Ôð	
ð „YÔð*ð *¨Tð *¸Tð *ð *ð *ñ Ôð*ð
 „YÔð ¤	ð ð ð ñ Ôðð	Bð 	B¨Cð 	B¸hÀs¼mð 	BÐW[ð 	Bð 	Bð 	Bð 	Bð 8<ØØ$Ø$Ø',ð1 ð 1 àŒ|ð1 ð ˜e C¨¨c¬ NÔ3Ô4ð1 ð ð	1 ð
 ð1 ð ð1 ð !%ð1 ð 
ˆt�E”LÔ! 5¨¬°t¸E¼LÔ7IÐ)IÔ#JÐJÔ	Kð1 ð 1 ð 1 ð 1 ðj ./Ø$Ø#ð	ð à˜3  S¤	˜>Ô*ðð ðð ð	ð
 
ˆcŒðð ð ð ð0 %¤,ð °5´<ð ð ð ð ð
Mð 
M˜eœlð 
M¸ð 
MÈÌð 
Mð 
Mð 
Mð 
Mð˜œð ¨%¬,ð ð ð ð ð ð ð ð r<   Ú
state_dictÚmodelc                 ó  ‡
— d}d| v rd}ddl }i }|                      d| ¦  «        } |                      d| ¦  «        } |                      ¦   «         D �]2\  Š
}t          ˆ
fd„d	D ¦   «         ¦  «        rŒ"d
‰
v rS|j        j        j        j        \  }}}}|j        d         |k    s|j        d         |k    rt          |||fddd¬¦  «        }‰
 	                    d¦  «        ro| 
                    ‰
dd…         ¦  «        }	|j        |	j        j        k    s|	j        d         |	j        d         k    r!t          ||	j        |	j        j        ¬¦  «        }|r.|                     dd„ ‰
¦  «        Š
‰
                     dd¦  «        Š
||‰
<   �Œ4|S )zàConvert patch embedding weight from manual patchify + linear proj to conv.

    Args:
        state_dict: State dictionary from checkpoint.
        model: Model instance.

    Returns:
        Filtered state dictionary.
    Tzhead.fc.weightFr   Nr•  r”  c                 ó   •— g | ]}|‰v ‘ŒS rÀ   rÀ   )rÛ   ra  r—   s     €r:   rÝ   z(checkpoint_filter_fn.<locals>.<listcomp>  s   ø€ ÐIÐIÐI˜1��Q�ÐIÐIÐIr<   )r\   r‰   zpatch_embed.proj.weightr‹   r/   Úbicubic)ÚinterpolationÚ	antialiasÚverboserg   iãÿÿÿr   r~   zlayers.(\d+).downsamplec                 óT   — dt          |                      d¦  «        ¦  «        dz   › d�S )Nr<  r   z.downsample)r«   Úgroup)r'   s    r:   ú<lambda>z&checkpoint_filter_fn.<locals>.<lambda>  s)   € Ð=gÅsÈ1Ï7Ê7ÐSTÉ:Ì:ÁÄÐYZÑGZÐ=gÐ=gÐ=g€ r<   zhead.zhead.fc.)ÚreÚgetÚitemsrÎ   rI  rl   rË   r1   r   ÚendswithÚget_submodulerg   r(   r   ÚsubÚreplace)r”  r•  Úold_weightsrŸ  Úout_dictr˜   rð   r6   r7   Úmr—   s             @r:   Úcheckpoint_filter_fnr©  ó  sÐ  ø€ ð €KØ˜:Ð%Ð%ØˆØ€I€I€IØ€HØ—’ ¨Ñ4Ô4€JØ—’ ¨jÑ9Ô9€JØ× Ò Ñ"Ô"ð ñ ‰ˆˆ1ÝÐIÐIÐIÐIÐ HÐIÑIÔIÑJÔJð 	Øà$¨Ð)Ð)ØÔ*Ô/Ô6Ô<‰JˆAˆq�!�QØŒw�rŒ{˜aÒÐ 1¤7¨2¤;°!Ò#3Ð#3Ý(ØØ˜�FØ"+Ø"Ø ðñ ô �ð �:Š:Ð4Ñ5Ô5ð 	Ø×#Ò# A d s d¤GÑ,Ô,ˆAØŒw˜!Ô8Ô>Ò>Ð>À!Ä-ÐPQÔBRÐVWÔVcÐdeÔVfÒBfÐBfÝ-ØØ$%¤MØ#$Ô#AÔ#Gðñ ô �ð ð 	/Ø—’Ð1Ð3gÐ3gÐijÑkÔkˆAØ—	’	˜' :Ñ.Ô.ˆAàˆ�‰‰Ø€Or<   FÚvariantÚ
pretrainedc           	      óü   — t          d„ t          |                     dd¦  «        ¦  «        D ¦   «         ¦  «        }|                     d|¦  «        }t	          t
          | |ft          t          d|¬¦  «        dœ|¤Ž}|S )zæCreate a Swin Transformer model.

    Args:
        variant: Model variant name.
        pretrained: Load pretrained weights.
        **kwargs: Additional model arguments.

    Returns:
        SwinTransformer model instance.
    c              3   ó    K  — | ]	\  }}|V — Œ
d S r¢   rÀ   )rÛ   r  rð   s      r:   r  z+_create_swin_transformer.<locals>.<genexpr>/  s&   è è € Ð\Ð\¡d a¨ Ð\Ð\Ð\Ð\Ð\Ð\r<   r/  )r   r   r+   r   Úout_indicesT)Úflatten_sequentialr®  )Úpretrained_filter_fnÚfeature_cfg)râ   rh  r   Úpopr   r&   r©  rK  )rª  r«  rP  Údefault_out_indicesr®  r•  s         r:   Ú_create_swin_transformerr´  $  s�   € õ  Ð\Ð\­i¸¿
º
À8È\Ñ8ZÔ8ZÑ.[Ô.[Ð\Ñ\Ô\Ñ\Ô\ÐØ—*’*˜]Ð,?Ñ@Ô@€Kå Ý˜ *ðå1Ý¨D¸kÐJÑJÔJðð ð ð	ð €Eð €Lr<   r(  Úurlc                 ó8   — | ddddddt           t          ddd	d
œ|¥S )z9Create default configuration for Swin Transformer models.r   )r+   r  r  )rR   rR   gÍÌÌÌÌÌì?r˜  Tzpatch_embed.projzhead.fcÚmit)rµ  r,  Ú
input_sizeÚ	pool_sizeÚcrop_pctr™  Úfixed_input_sizeÚmeanrv   Ú
first_convÚ
classifierÚlicenser   )rµ  rP  s     r:   Ú_cfgrÀ  ;  s<   € ð Ø¨=ÀvØ¨ÈÝ%Õ.BØ(¸	Øðð ð #ðð r<   z.swin_small_patch4_window7_224.ms_in22k_ft_in1kztimm/zvhttps://github.com/SwinTransformer/storage/releases/download/v1.0.8/swin_small_patch4_window7_224_22kto1k_finetune.pth)Ú	hf_hub_idrµ  z-swin_base_patch4_window7_224.ms_in22k_ft_in1kzlhttps://github.com/SwinTransformer/storage/releases/download/v1.0.0/swin_base_patch4_window7_224_22kto1k.pthz.swin_base_patch4_window12_384.ms_in22k_ft_in1kzmhttps://github.com/SwinTransformer/storage/releases/download/v1.0.0/swin_base_patch4_window12_384_22kto1k.pth)r+   é€  rÂ  )r&  r&  g      ð?)rÁ  rµ  r¸  r¹  rº  z.swin_large_patch4_window7_224.ms_in22k_ft_in1kzmhttps://github.com/SwinTransformer/storage/releases/download/v1.0.0/swin_large_patch4_window7_224_22kto1k.pthz/swin_large_patch4_window12_384.ms_in22k_ft_in1kznhttps://github.com/SwinTransformer/storage/releases/download/v1.0.0/swin_large_patch4_window12_384_22kto1k.pthz$swin_tiny_patch4_window7_224.ms_in1kzdhttps://github.com/SwinTransformer/storage/releases/download/v1.0.0/swin_tiny_patch4_window7_224.pthz%swin_small_patch4_window7_224.ms_in1kzehttps://github.com/SwinTransformer/storage/releases/download/v1.0.0/swin_small_patch4_window7_224.pthz$swin_base_patch4_window7_224.ms_in1kzdhttps://github.com/SwinTransformer/storage/releases/download/v1.0.0/swin_base_patch4_window7_224.pthz%swin_base_patch4_window12_384.ms_in1kzehttps://github.com/SwinTransformer/storage/releases/download/v1.0.0/swin_base_patch4_window12_384.pthz-swin_tiny_patch4_window7_224.ms_in22k_ft_in1kzuhttps://github.com/SwinTransformer/storage/releases/download/v1.0.8/swin_tiny_patch4_window7_224_22kto1k_finetune.pthz%swin_tiny_patch4_window7_224.ms_in22kzhhttps://github.com/SwinTransformer/storage/releases/download/v1.0.8/swin_tiny_patch4_window7_224_22k.pthiQU  )rÁ  rµ  r,  z&swin_small_patch4_window7_224.ms_in22kzihttps://github.com/SwinTransformer/storage/releases/download/v1.0.8/swin_small_patch4_window7_224_22k.pthz%swin_base_patch4_window7_224.ms_in22kzhhttps://github.com/SwinTransformer/storage/releases/download/v1.0.0/swin_base_patch4_window7_224_22k.pthz&swin_base_patch4_window12_384.ms_in22kzihttps://github.com/SwinTransformer/storage/releases/download/v1.0.0/swin_base_patch4_window12_384_22k.pth)rÁ  rµ  r¸  r¹  rº  r,  z&swin_large_patch4_window7_224.ms_in22kzihttps://github.com/SwinTransformer/storage/releases/download/v1.0.0/swin_large_patch4_window7_224_22k.pthz'swin_large_patch4_window12_384.ms_in22kzjhttps://github.com/SwinTransformer/storage/releases/download/v1.0.0/swin_large_patch4_window12_384_22k.pthzswin_s3_tiny_224.ms_in1kzbhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/s3_t-1d53f6a8.pthzbhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/s3_s-3bb4c69d.pthzbhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/s3_b-a1e95db4.pth)zswin_s3_small_224.ms_in1kzswin_s3_base_224.ms_in1kc           	      ó^   — t          ddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤ŽS )
z+ Swin-T @ 224x224, trained ImageNet-1k
    r-   rR   r"  r#  r%  ©r*  r(   r.  r/  rU   Úswin_tiny_patch4_window7_224r«  )rÅ  ©rK  r´  ©r«  rP  Ú
model_argss      r:   rÅ  rÅ  ’  s^   € õ  °¸RÈÐ`nÐoÑoÔo€JÝ#Ø&ð]ð ]Ø3=ð]ÝAEÀjÐA[ÐA[ÐTZÐA[ÐA[ð]ð ]ð ]r<   c           	      ó^   — t          ddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤ŽS )
z Swin-S @ 224x224
    r-   rR   r"  ©r,   r,   é   r,   r%  rÄ  Úswin_small_patch4_window7_224r«  )rÌ  rÆ  rÇ  s      r:   rÌ  rÌ  ›  s^   € õ  °¸RÈÐaoÐpÑpÔp€JÝ#Ø'ð^ð ^Ø4>ð^ÝBFÀzÐB\ÐB\ÐU[ÐB\ÐB\ð^ð ^ð ^r<   c           	      ó^   — t          ddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤ŽS )
z Swin-B @ 224x224
    r-   rR   é€   rÊ  ©r-   rc  é   é    rÄ  Úswin_base_patch4_window7_224r«  )rÒ  rÆ  rÇ  s      r:   rÒ  rÒ  ¤  s^   € õ  °¸SÈÐbpÐqÑqÔq€JÝ#Ø&ð]ð ]Ø3=ð]ÝAEÀjÐA[ÐA[ÐTZÐA[ÐA[ð]ð ]ð ]r<   c           	      ó^   — t          ddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤ŽS )
z Swin-B @ 384x384
    r-   r&  rÎ  rÊ  rÏ  rÄ  Úswin_base_patch4_window12_384r«  )rÔ  rÆ  rÇ  s      r:   rÔ  rÔ  ­  s^   € õ  °¸cÈ-ÐcqÐrÑrÔr€JÝ#Ø'ð^ð ^Ø4>ð^ÝBFÀzÐB\ÐB\ÐU[ÐB\ÐB\ð^ð ^ð ^r<   c           	      ó^   — t          ddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤ŽS )
z Swin-L @ 224x224
    r-   rR   éÀ   rÊ  ©r$  r&  r'  é0   rÄ  Úswin_large_patch4_window7_224r«  )rÙ  rÆ  rÇ  s      r:   rÙ  rÙ  ¶  s^   € õ  °¸SÈÐbqÐrÑrÔr€JÝ#Ø'ð^ð ^Ø4>ð^ÝBFÀzÐB\ÐB\ÐU[ÐB\ÐB\ð^ð ^ð ^r<   c           	      ó^   — t          ddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤ŽS )
z Swin-L @ 384x384
    r-   r&  rÖ  rÊ  r×  rÄ  Úswin_large_patch4_window12_384r«  )rÛ  rÆ  rÇ  s      r:   rÛ  rÛ  ¿  s^   € õ  °¸cÈ-ÐcrÐsÑsÔs€JÝ#Ø(ð_ð _Ø5?ð_ÝCGÈ
ÐC]ÐC]ÐV\ÐC]ÐC]ð_ð _ð _r<   c           	      ó\   — t          ddddd¬¦  «        }t          d	d| it          |fi |¤Ž¤ŽS )
z; Swin-S3-T @ 224x224, https://arxiv.org/abs/2111.14725
    r-   ©rR   rR   é   rR   r"  r#  r%  rÄ  Úswin_s3_tiny_224r«  )rß  rÆ  rÇ  s      r:   rß  rß  È  sS   € õ Ø -¸2ÀlÐ^lðnñ nô n€Jå#ÐlÐlÀ:ÐlÕQUÐV`ÐQkÐQkÐdjÐQkÐQkÐlÐlÐlr<   c           	      ó\   — t          ddddd¬¦  «        }t          d	d| it          |fi |¤Ž¤ŽS )
z; Swin-S3-S @ 224x224, https://arxiv.org/abs/2111.14725
    r-   )rÞ  rÞ  rÞ  rR   r"  rÊ  r%  rÄ  Úswin_s3_small_224r«  )rá  rÆ  rÇ  s      r:   rá  rá  Ñ  sS   € õ Ø /¸RÈÐaoðqñ qô q€Jå#ÐmÐmÀJÐmÕRVÐWaÐRlÐRlÐekÐRlÐRlÐmÐmÐmr<   c           	      ó\   — t          ddddd¬¦  «        }t          d	d| it          |fi |¤Ž¤ŽS )
z; Swin-S3-B @ 224x224, https://arxiv.org/abs/2111.14725
    r-   rÝ  r"  )r,   r,   é   r,   r%  rÄ  Úswin_s3_base_224r«  )rä  rÆ  rÇ  s      r:   rä  rä  Ú  sS   € õ Ø -¸2ÀmÐ_mðoñ oô o€Jå#ÐlÐlÀ:ÐlÕQUÐV`ÐQkÐQkÐdjÐQkÐQkÐlÐlÐlr<   )Ú"swin_base_patch4_window7_224_in22kÚ#swin_base_patch4_window12_384_in22kÚ#swin_large_patch4_window7_224_in22kÚ$swin_large_patch4_window12_384_in22kr¢   r‘  )r(  )Pr¦   ÚloggingrÏ   Útypingr   r   r   r   r   r   r	   r
   r   rE   Útorch.nnrd   Ú	timm.datar   r   Útimm.layersr   r   r   r   r   r   r   r   r   r   r   r   r   Ú_builderr   Ú	_featuresr   Ú_features_fxr   Ú_manipulater    r!   Ú	_registryr"   r#   r$   Úvision_transformerr%   Ú__all__Ú	getLoggerr£   Ú_loggerr«   r¬   r®   r;   r>   rN   rú   rP   r²   rü   r  r&   rK  r’  r©  r©   r´  rÀ  Údefault_cfgsrÅ  rÌ  rÒ  rÔ  rÙ  rÛ  rß  rá  rä  rÀ   r<   r:   ú<module>rø     sŽ	  ðð
ð 
ð" €€€Ø €€€Ø OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ Oà €€€Ø Ð Ð Ð Ð Ð à AÐ AÐ AÐ AÐ AÐ AÐ AÐ Að^ð ^ð ^ð ^ð ^ð ^ð ^ð ^ð ^ð ^ð ^ð ^ð ^ð ^ð ^ð ^ð ^ð ^ð ^ð ^ð ^ð ^ð ^ð ^ð ^ð ^ð ^ð ^ð ^ð ^à *Ð *Ð *Ð *Ð *Ð *Ø +Ð +Ð +Ð +Ð +Ð +Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø 4Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4Ø YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YØ 4Ð 4Ð 4Ð 4Ð 4Ð 4àÐ
€à
ˆ'Ô
˜HÑ
%Ô
%€à˜#˜u S¨# XœÐ.Ô/Ð ðØŒ<ðà˜3 ˜8”_ðð „\ðð ð ð ð& ð˜EœLð °u¸SÀ#¸X´ð È3ð ÐSVð Ð[`Ô[gð ð ð ñ Ôðð$#ð # sð #°3ð #ÈÌð #ð #ð #ð #ð0Tð Tð Tð Tð T�b”iñ Tô Tð Tðnoð oð oð oð o˜2œ9ñ oô oð oðd-ð -ð -ð -ð -�2”9ñ -ô -ð -ð`ð ð ð ð ˜2œ9ñ ô ð ðDMð Mð Mð Mð M�b”iñ Mô Mð Mð`
. Tð .°"´)ð .ÀÀSÈ%Ì,ÐEVÔ@Wð .ð .ð .ð .ðbð  cð °tð ÐRað ð ð ð ð.	ð 	ˆcð 	 T¨#¨s¨(¤^ð 	ð 	ð 	ð 	ð %Ð$ð H&Ø4°d°dØð Eð7Hñ 7Hô 7HðH&ð 4°T°TØØzð6}ñ 6}ô 6}ð	H&ð 5°d°dØØ{Ø ¨H¸sð7Dñ 7Dô 7DðH&ð 5°d°dØØ{ð7~ñ 7~ô 7~ðH&ð 6°t°tØØ|Ø ¨H¸sð8Dñ 8Dô 8DðH&ð& +¨D¨DØØrð-uñ -uô -uð'H&ð, ,¨T¨TØØsð.vñ .vô .vð-H&ð2 +¨D¨DØØrð-uñ -uô -uð3H&ð8 ,¨T¨TØØsØ ¨H¸sð.Dñ .Dô .Dð9H&ðD 4°T°TØð Dð6Fñ 6Fô 6FðEH&ðL ,¨T¨TØØvØð.ñ .ô .ðMH&ðT -¨d¨dØØwØð/ñ /ô /ðUH&ð\ ,¨T¨TØØvØð.ñ .ô .ð]H&ðd -¨d¨dØØwØ ¨H¸sÐPUð/Wñ /Wô /WðeH&ðl -¨d¨dØØwØð/ñ /ô /ðmH&ðt .¨t¨tØØxØ ¨H¸sÐPUð0Wñ 0Wô 0WðuH&ð~   ØØpð!rñ !rô !rðH&ðD "& ØØpð"rñ "rô "rð !% ØØpð!rñ !rô !rðKH&ð H&ð H&ñ Hô H€ðV ð]ð ]Àð ]ð ]ð ]ñ „ð]ð ð^ð ^Àð ^ð ^ð ^ñ „ð^ð ð]ð ]Àð ]ð ]ð ]ñ „ð]ð ð^ð ^Àð ^ð ^ð ^ñ „ð^ð ð^ð ^Àð ^ð ^ð ^ñ „ð^ð ð_ð _À/ð _ð _ð _ñ „ð_ð ðmð m°Oð mð mð mñ „ðmð ðnð n°_ð nð nð nñ „ðnð ðmð m°Oð mð mð mñ „ðmð Ð ˜HØ*QØ+SØ+SØ,Uð	'ð 'ñ ô ð ð ð r<   