§
    ·¯¸jkÏ  ã                   óö  — d Z ddlZddlmZ ddlmZmZmZmZm	Z	m
Z
mZmZmZ ddlZddlmZ ddlmc mZ ddlmZmZ ddlmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z# ddl$m%Z% dd	l&m'Z' dd
l(m)Z) ddl*m+Z+ ddl,m-Z-m.Z.m/Z/ dgZ0ee1ee1e1f         f         Z2dej3        dee1e1f         dej3        fd„Z4e)dej3        dee1e1f         dee1e1f         dej3        fd„¦   «         Z5 G d„ dej6        ¦  «        Z7 G d„ dej6        ¦  «        Z8 G d„ dej6        ¦  «        Z9 G d„ dej6        ¦  «        Z: G d„ dej6        ¦  «        Z;dee<ej3        f         dej6        dee<ej3        f         fd „Z=dQd"e<d#e>de;fd$„Z?dRd&„Z@ e- e@d'd(¬)¦  «         e@d'd*d+d,d-¬.¦  «         e@d'd/¬)¦  «         e@d'd0d+d,d-¬.¦  «         e@d'd1¬)¦  «         e@d'd2¬)¦  «         e@d'd3¬)¦  «         e@d'd4¬)¦  «         e@d'd5¬)¦  «         e@d'd6¬)¦  «         e@d'd7d8d9d:¬;¦  «         e@d'd<d8d9d:¬;¦  «        d=œ¦  «        ZAe.dQd#e>de;fd>„¦   «         ZBe.dQd#e>de;fd?„¦   «         ZCe.dQd#e>de;fd@„¦   «         ZDe.dQd#e>de;fdA„¦   «         ZEe.dQd#e>de;fdB„¦   «         ZFe.dQd#e>de;fdC„¦   «         ZGe.dQd#e>de;fdD„¦   «         ZHe.dQd#e>de;fdE„¦   «         ZIe.dQd#e>de;fdF„¦   «         ZJe.dQd#e>de;fdG„¦   «         ZKe.dQd#e>de;fdH„¦   «         ZLe.dQd#e>de;fdI„¦   «         ZM e/eNdJdKdLdMdNdOdPœ¦  «         dS )SaK   Swin Transformer V2
A PyTorch impl of : `Swin Transformer V2: Scaling Up Capacity and Resolution`
    - https://arxiv.org/abs/2111.09883

Code/weights from https://github.com/microsoft/Swin-Transformer, original copyright/license info below

Modifications and additions for timm hacked together by / Copyright 2022, Ross Wightman
é    N)Úpartial)	ÚAnyÚCallableÚDictÚListÚOptionalÚSetÚTupleÚTypeÚUnion©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)Ú
PatchEmbedÚMlpÚDropPathÚcalculate_drop_path_ratesÚ	to_2tupleÚtrunc_normal_ÚClassifierHeadÚresample_patch_embedÚndgridÚget_act_layerÚ	LayerTypeÚget_device_dtypeé   )Úbuild_model_with_cfg)Úfeature_take_indices)Úregister_notrace_function)Ú
checkpoint)Úgenerate_default_cfgsÚregister_modelÚregister_model_deprecationsÚSwinTransformerV2ÚxÚwindow_sizeÚreturnc                 ó*  — | j         \  }}}}|                      |||d         z  |d         ||d         z  |d         |¦  «        } |                      dddddd¦  «                             ¦   «                              d|d         |d         |¦  «        }|S )zúPartition into non-overlapping windows.

    Args:
        x: Input tensor of shape (B, H, W, C).
        window_size: Window size (height, width).

    Returns:
        Windows tensor of shape (num_windows*B, window_size[0], window_size[1], C).
    r   r   é   é   é   é   éÿÿÿÿ©ÚshapeÚviewÚpermuteÚ
contiguous)r%   r&   ÚBÚHÚWÚCÚwindowss          úo/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/models/swin_transformer_v2.pyÚwindow_partitionr9   %   s–   € ð ”�J€A€qˆ!ˆQØ	�Šˆq�!�{ 1”~Ñ% {°1¤~°q¸KÈ¼NÑ7JÈKÐXYÌNÐ\]Ñ^Ô^€AØ�iŠi˜˜1˜a  A qÑ)Ô)×4Ò4Ñ6Ô6×;Ò;¸BÀÈAÄÐP[Ð\]ÔP^Ð`aÑbÔb€GØ€Nó    r7   Úimg_sizec                 ó  — |\  }}| j         d         }|                      d||d         z  ||d         z  |d         |d         |¦  «        }|                     dddddd¦  «                             ¦   «                              d|||¦  «        }|S )a1  Merge windows back to feature map.

    Args:
        windows: Windows tensor of shape (num_windows * B, window_size[0], window_size[1], C).
        window_size: Window size (height, width).
        img_size: Image size (height, width).

    Returns:
        Feature map tensor of shape (B, H, W, C).
    r-   r   r   r)   r*   r+   r,   r.   )r7   r&   r;   r4   r5   r6   r%   s          r8   Úwindow_reverser=   8   s’   € ð  �D€A€qØŒ�bÔ€AØ�Š�R˜˜k¨!œnÑ,¨a°;¸q´>Ñ.AÀ;ÈqÄ>ÐS^Ð_`ÔSaÐcdÑeÔe€AØ	�	Š	�!�Q˜˜1˜a Ñ#Ô#×.Ò.Ñ0Ô0×5Ò5°b¸!¸QÀÑBÔB€AØ€Hr:   c                   ó&  ‡ — e Zd ZdZ	 	 	 	 	 	 	 ddedeeef         d	ed
ededededeeef         ddfˆ fd„Zdd„Z	dd„Z
	 	 ddeej        ej        f         fd„Zdeeef         ddfd„Zdd„Zddej        deej                 dej        fd„Zˆ xZS )ÚWindowAttentionzÔWindow based multi-head self attention (W-MSA) module with relative position bias.

    Supports both shifted and non-shifted window attention with continuous relative
    position bias and cosine attention.
    TFç        ©r   r   NÚdimr&   Ú	num_headsÚqkv_biasÚqkv_bias_separateÚ	attn_dropÚ	proj_dropÚpretrained_window_sizer'   c           	      óú  •— |	|
dœ}t          ¦   «                              ¦   «          || _        || _        t	          |¦  «        | _        || _        || _        t          j	        t          j        |ddffi |¤Ž¦  «        | _        t          j        t          j        dddi|¤Žt          j        d¬¦  «        t          j        d|fddi|¤Ž¦  «        | _        t          j        ||d	z  fddi|¤Ž| _        |r{t          j	        t          j        |fi |¤Ž¦  «        | _        |                      d
t          j        |fi |¤Žd¬¦  «         t          j	        t          j        |fi |¤Ž¦  «        | _        nd| _        d| _        d| _        t          j        |¦  «        | _        t          j        ||fi |¤Ž| _        t          j        |¦  «        | _        t          j        d¬¦  «        | _        | j        \  }}|                      dt          j        dd|z  dz
  d|z  dz
  dfi |¤Žd¬¦  «         |                      dt          j        ||z  ||z  |	t          j        ¬¦  «        d¬¦  «         |                      ¦   «          dS )a4  Initialize window attention module.

        Args:
            dim: Number of input channels.
            window_size: The height and width of the window.
            num_heads: Number of attention heads.
            qkv_bias: If True, add a learnable bias to query, key, value.
            qkv_bias_separate: If True, use separate bias for q, k, v projections.
            attn_drop: Dropout ratio of attention weight.
            proj_drop: Dropout ratio of output.
            pretrained_window_size: The height and width of the window in pre-training.
        ©ÚdeviceÚdtyper   r*   é   ÚbiasT)ÚinplaceFr)   Úk_bias©Ú
persistentNr-   ©rB   Úrelative_coords_tableÚrelative_position_index)r*   rM   )ÚsuperÚ__init__rB   r&   r   rH   rC   rE   ÚnnÚ	ParameterÚtorchÚemptyÚlogit_scaleÚ
SequentialÚLinearÚReLUÚcpb_mlpÚqkvÚq_biasÚregister_bufferÚv_biasrP   ÚDropoutrF   ÚprojrG   ÚSoftmaxÚsoftmaxÚlongÚreset_parameters)ÚselfrB   r&   rC   rD   rE   rF   rG   rH   rK   rL   ÚddÚwin_hÚwin_wÚ	__class__s                 €r8   rW   zWindowAttention.__init__V   sŸ  ø€ ð2 ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØˆŒØ&ˆÔÝ&/Ð0FÑ&GÔ&GˆÔ#Ø"ˆŒØ!2ˆÔåœ<­¬°YÀÀ1Ð4EÐ(LÐ(LÈÐ(LÐ(LÑMÔMˆÔõ ”}ÝŒIÐ.Ð. 4Ð.¨2Ð.Ð.ÝŒG˜DÐ!Ñ!Ô!ÝŒI�c˜9Ð7Ð7¨5Ð7°BÐ7Ð7ñ
ô 
ˆŒõ ”9˜S #¨¡'Ð<Ð<°Ð<¸Ð<Ð<ˆŒØð 	Ýœ,¥u¤{°3Ð'=Ð'=¸"Ð'=Ð'=Ñ>Ô>ˆDŒKØ× Ò  ­5¬;°sÐ+AÐ+A¸bÐ+AÐ+AÈeÐ ÑTÔTÐTÝœ,¥u¤{°3Ð'=Ð'=¸"Ð'=Ð'=Ñ>Ô>ˆDŒKˆKàˆDŒKØˆDŒKØˆDŒKÝœ IÑ.Ô.ˆŒÝ”I˜c 3Ð-Ð-¨"Ð-Ð-ˆŒ	Ýœ IÑ.Ô.ˆŒÝ”z bÐ)Ñ)Ô)ˆŒð Ô'‰ˆˆuØ×ÒØ#ÝŒK˜˜1˜u™9 q™=¨!¨e©)°a©-¸ÐAÐA¸bÐAÐAØð 	ñ 	
ô 	
ð 	
ð
 	×ÒØ%ÝŒK˜ ™ u¨u¡}¸VÍ5Ì:ÐVÑVÔVØð 	ñ 	
ô 	
ð 	
ð 	×ÒÑÔÐÐÐr:   c                 ó:  — t           j                             | j        t	          j        d¦  «        ¦  «         | j        �Ht           j                             | j        ¦  «         t           j                             | j        ¦  «         |  	                    ¦   «          dS )z"Initialize parameters and buffers.é
   N)
rX   ÚinitÚ	constant_r\   ÚmathÚlogrb   Úzeros_rd   Ú_init_buffers©rk   s    r8   rj   z WindowAttention.reset_parametersž   sq   € å
Œ×Ò˜$Ô*­D¬H°R©L¬LÑ9Ô9Ð9ØŒ;Ð"ÝŒG�NŠN˜4œ;Ñ'Ô'Ð'ÝŒG�NŠN˜4œ;Ñ'Ô'Ð'Ø×ÒÑÔÐÐÐr:   c                 ó  — | j         �| j                              ¦   «          |                      | j        j        j        | j        j        j        ¬¦  «        \  }}| j                             |¦  «         | j	                             |¦  «         dS )z.Compute and fill non-persistent buffer values.NrJ   )
rP   Úzero_Ú"_make_pair_wise_relative_positionsrf   ÚweightrK   rL   rT   Úcopy_rU   )rk   rT   rU   s      r8   rw   zWindowAttention._init_buffers¦   s�   € àŒ;Ð"ØŒK×ÒÑÔÐØ9=×9`Ò9`Ø”9Ô#Ô*°$´)Ô2BÔ2Hð :añ :
ô :
Ñ6ÐÐ6ð 	Ô"×(Ò(Ð)>Ñ?Ô?Ð?ØÔ$×*Ò*Ð+BÑCÔCÐCÐCÐCr:   c                 ó²  — t          j        | j        d         dz
   | j        d         |t           j        ¬¦  «        }t          j        | j        d         dz
   | j        d         |t           j        ¬¦  «        }t          j        t          ||¦  «        ¦  «        }|                     ddd¦  «                             ¦   «                              d¦  «        }| j	        d         dk    rQ|dd…dd…dd…dfxx         | j	        d         dz
  z  cc<   |dd…dd…dd…dfxx         | j	        d         dz
  z  cc<   nP|dd…dd…dd…dfxx         | j        d         dz
  z  cc<   |dd…dd…dd…dfxx         | j        d         dz
  z  cc<   |dz  }t          j
        |¦  «        t          j        t          j        |¦  «        dz   ¦  «        z  t          j        d¦  «        z  }|                     |¬¦  «        }t          j        | j        d         |t           j        ¬¦  «        }t          j        | j        d         |t           j        ¬¦  «        }t          j        t          ||¦  «        ¦  «        }t          j        |d¦  «        }	|	dd…dd…df         |	dd…ddd…f         z
  }
|
                     ddd¦  «                             ¦   «         }
|
dd…dd…dfxx         | j        d         dz
  z  cc<   |
dd…dd…dfxx         | j        d         dz
  z  cc<   |
dd…dd…dfxx         d| j        d         z  dz
  z  cc<   |
                     d	¦  «        }||fS )
z¡Compute pair-wise relative position index and coordinates table.

        Returns:
            Tuple of (relative_coords_table, relative_position_index)
        r   r   rJ   r*   Né   ç      ð?)rL   r-   )rZ   Úaranger&   Úfloat32Ústackr   r1   r2   Ú	unsqueezerH   ÚsignÚlog2Úabsrt   Útori   ÚflattenÚsum)rk   rK   rL   Úrelative_coords_hÚrelative_coords_wrT   Úcoords_hÚcoords_wÚcoordsÚcoords_flattenÚrelative_coordsrU   s               r8   r{   z2WindowAttention._make_pair_wise_relative_positions°   sÒ  € õ "œLØÔ˜qÔ! AÑ%Ð&¨Ô(8¸Ô(;ÀFÕRWÔR_ðañ aô aÐå!œLØÔ˜qÔ! AÑ%Ð&¨Ô(8¸Ô(;ÀFÕRWÔR_ðañ aô aÐå %¤­FÐ3DÐFWÑ,XÔ,XÑ YÔ YÐØ 5× =Ò =¸aÀÀAÑ FÔ F× QÒ QÑ SÔ S× ]Ò ]Ð^_Ñ `Ô `ÐØÔ& qÔ)¨AÒ-Ð-Ø! ! ! ! Q Q Q¨¨¨¨1 *Ð-Ð-Ô-°$Ô2MÈaÔ2PÐSTÑ2TÑUÐ-Ð-Ñ-Ø! ! ! ! Q Q Q¨¨¨¨1 *Ð-Ð-Ô-°$Ô2MÈaÔ2PÐSTÑ2TÑUÐ-Ð-Ñ-Ð-à! ! ! ! Q Q Q¨¨¨¨1 *Ð-Ð-Ô-°$Ô2BÀ1Ô2EÈÑ2IÑJÐ-Ð-Ñ-Ø! ! ! ! Q Q Q¨¨¨¨1 *Ð-Ð-Ô-°$Ô2BÀ1Ô2EÈÑ2IÑJÐ-Ð-Ñ-Ø Ñ"ÐÝ %¤
Ð+@Ñ AÔ AÅEÄJÝŒIÐ+Ñ,Ô,¨sÑ2ñE4ô E4ñ !4Ý6:´iÀ±l´lñ!CÐà 5× 8Ò 8¸uÐ 8Ñ EÔ EÐõ ”< Ô 0°Ô 3¸FÍ%Ì*ÐUÑUÔUˆÝ”< Ô 0°Ô 3¸FÍ%Ì*ÐUÑUÔUˆÝ”�V H¨hÑ7Ô7Ñ8Ô8ˆÝœ v¨qÑ1Ô1ˆØ(¨¨¨¨A¨A¨A¨t¨Ô4°~ÀaÀaÀaÈÈqÈqÈqÀjÔ7QÑQˆØ)×1Ò1°!°Q¸Ñ:Ô:×EÒEÑGÔGˆØ˜˜˜˜1˜1˜1˜a˜Ð Ð Ô  DÔ$4°QÔ$7¸!Ñ$;Ñ;Ð Ð Ñ Ø˜˜˜˜1˜1˜1˜a˜Ð Ð Ô  DÔ$4°QÔ$7¸!Ñ$;Ñ;Ð Ð Ñ Ø˜˜˜˜1˜1˜1˜a˜Ð Ð Ô  A¨Ô(8¸Ô(;Ñ$;¸aÑ$?Ñ?Ð Ð Ñ Ø"1×"5Ò"5°bÑ"9Ô"9Ðà$Ð&=Ð=Ð=r:   c                 ó"  — t          |¦  «        }|| j        k    rt| j        €J ‚| j        j        }| j        j        }|| _        |                      ||¬¦  «        \  }}|                      d|d¬¦  «         |                      d|d¬¦  «         dS dS )z�Update window size and regenerate relative position tables.

        Args:
            window_size: New window size (height, width).
        NrJ   rT   FrQ   rU   )r   r&   rT   rK   rL   r{   rc   )rk   r&   rK   rL   rT   rU   s         r8   Úset_window_sizezWindowAttention.set_window_sizeÚ   s·   € õ   Ñ,Ô,ˆØ˜$Ô*Ò*Ð*ØÔ-Ð9Ð9Ð9ØÔ/Ô6ˆFØÔ.Ô4ˆEØ*ˆDÔà×7Ò7¸vÈUÐ7ÑSÔSñ ;Ð!Ð#:à× Ò Ð!8Ð:OÐ\aÐ ÑbÔbÐbØ× Ò Ð!:Ð<SÐ`eÐ ÑfÔfÐfÐfÐfð +Ð*r:   c                 ó.   — |                       ¦   «          dS )z"Initialize non-persistent buffers.N)rw   rx   s    r8   Úinit_non_persistent_buffersz+WindowAttention.init_non_persistent_buffersë   s   € à×ÒÑÔÐÐÐr:   r%   Úmaskc                 óÈ  — |j         \  }}}| j        €|                      |¦  «        }nit          j        | j        | j        | j        f¦  «        }| j        r|                      |¦  «        }||z  }n!t          j	        || j        j
        |¬¦  «        }|                     ||d| j        d¦  «                             ddddd¦  «        }|                     d¦  «        \  }}	}
t          j        |d¬	¦  «        t          j        |	d¬	¦  «                             d
d¦  «        z  }t          j        | j        t'          j        d¦  «        ¬¦  «                             ¦   «         }||z  }|                      | j        ¦  «                             d| j        ¦  «        }|| j                             d¦  «                                      | j        d         | j        d         z  | j        d         | j        d         z  d¦  «        }|                     ddd¦  «                             ¦   «         }dt          j        |¦  «        z  }||                     d¦  «        z   }|�‡|j         d         }|                     d|| j        ||¦  «        |                     d¦  «                             d¦  «        z   }|                     d| j        ||¦  «        }|                      |¦  «        }n|                      |¦  «        }|                      |¦  «        }||
z                       dd¦  «                             |||¦  «        }|                       |¦  «        }|  !                    |¦  «        }|S )a#  Forward pass of window attention.

        Args:
            x: Input features with shape of (num_windows*B, N, C).
            mask: Attention mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None.

        Returns:
            Output features with shape of (num_windows*B, N, C).
        N)r|   rN   r)   r-   r*   r   r   r+   rS   éþÿÿÿg      Y@)Úmaxé   )"r/   rb   ra   rZ   ÚcatrP   rd   rE   ÚFÚlinearr|   ÚreshaperC   r1   ÚunbindÚ	normalizeÚ	transposeÚclampr\   rt   ru   Úexpr`   rT   r0   rU   r&   r2   Úsigmoidr„   rh   rF   rf   rG   )rk   r%   r–   ÚB_ÚNr6   ra   rD   ÚqÚkÚvÚattnr\   Úrelative_position_bias_tableÚrelative_position_biasÚnum_wins                   r8   ÚforwardzWindowAttention.forwardï   s  € ð ”7‰ˆˆAˆqàŒ;ÐØ—(’(˜1‘+”+ˆCˆCå”y $¤+¨t¬{¸D¼KÐ!HÑIÔIˆHØÔ%ð IØ—h’h˜q‘k”k�Ø�x‘��å”h˜q¨¬¬¸xÐHÑHÔH�Ø�kŠk˜"˜a  D¤N°BÑ7Ô7×?Ò?ÀÀ1ÀaÈÈAÑNÔNˆØ—*’*˜Q‘-”-‰ˆˆ1ˆaõ ”˜A 2Ð&Ñ&Ô&­¬°Q¸BÐ)?Ñ)?Ô)?×)IÒ)IÈ"ÈbÑ)QÔ)QÑQˆÝ”k $Ô"2½¼ÀÑ8KÔ8KÐLÑLÔL×PÒPÑRÔRˆØ�kÑ!ˆà'+§|¢|°DÔ4NÑ'OÔ'O×'TÒ'TÐUWÐY]ÔYgÑ'hÔ'hÐ$Ø!=¸dÔ>Z×>_Ò>_Ð`bÑ>cÔ>cÔ!d×!iÒ!iØÔ˜QÔ $Ô"2°1Ô"5Ñ5°tÔ7GÈÔ7JÈTÔM]Ð^_ÔM`Ñ7`Ðbdñ"fô "fÐà!7×!?Ò!?ÀÀ1ÀaÑ!HÔ!H×!SÒ!SÑ!UÔ!UÐØ!#¥e¤mÐ4JÑ&KÔ&KÑ!KÐØÐ,×6Ò6°qÑ9Ô9Ñ9ˆàÐØ”j ”mˆGØ—9’9˜R ¨$¬.¸!¸QÑ?Ô?À$Ç.Â.ÐQRÑBSÔBS×B]ÒB]Ð^_ÑB`ÔB`Ñ`ˆDØ—9’9˜R ¤°°AÑ6Ô6ˆDØ—<’< Ñ%Ô%ˆDˆDà—<’< Ñ%Ô%ˆDà�~Š~˜dÑ#Ô#ˆà�A‰X× Ò   AÑ&Ô&×.Ò.¨r°1°aÑ8Ô8ˆØ�IŠI�a‰LŒLˆØ�NŠN˜1ÑÔˆØˆr:   )TFr@   r@   rA   NN©r'   N)NN©N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Úintr
   ÚboolÚfloatrW   rj   rw   rZ   ÚTensorr{   r“   r•   r   r®   Ú__classcell__©ro   s   @r8   r?   r?   O   s¿  ø€ € € € € ðð ð "Ø&+Ø!Ø!Ø6<ØØðF ð F àðF ð ˜s C˜xœðF ð ð	F ð
 ðF ð  $ðF ð ðF ð ðF ð %*¨#¨s¨(¤OðF ð 
ðF ð F ð F ð F ð F ð F ðPð ð ð ðDð Dð Dð Dð Øð(>ð (>ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð	(>ð (>ð (>ð (>ðTg¨5°°c°¬?ð g¸tð gð gð gð gð"ð ð ð ð1ð 1˜œð 1¨X°e´lÔ-Cð 1ÈuÌ|ð 1ð 1ð 1ð 1ð 1ð 1ð 1ð 1r:   r?   c                   óú  ‡ — e Zd ZdZddddddddddej        dd	d	fd
ededededededede	dede	de	de	de
deej                 defˆ fd„Z	 	 	 d&deej                 deej                 deej                 deej                 fd„Z	 d'ded ee         deeeef         eeef         f         fd!„Z	 d'd"eeef         deeef         dee         dd	fd#„Zdej        dej        fd$„Zdej        dej        fd%„Zˆ xZS )(ÚSwinTransformerV2BlockzªSwin Transformer V2 Block.

    A standard transformer block with window attention and shifted window attention
    for modeling long-range dependencies efficiently.
    é   r   Fç      @Tr@   ÚgeluNrB   Úinput_resolutionrC   r&   Ú
shift_sizeÚalways_partitionÚdynamic_maskÚ	mlp_ratiorD   rG   rF   Ú	drop_pathÚ	act_layerÚ
norm_layerrH   c                 óŒ  •— ||dœ}t          ¦   «                              ¦   «          || _        t          |¦  «        | _        || _        t          |¦  «        | _        || _        || _        |  	                    ||¦  «        \  | _
        | _        | j
        d         | j
        d         z  | _        || _        t          |¦  «        }t          |ft          | j
        ¦  «        ||	||
t          |¦  «        dœ|¤Ž| _         ||fi |¤Ž| _        |dk    rt%          |¦  «        nt'          j        ¦   «         | _        t-          d|t/          ||z  ¦  «        ||
dœ|¤Ž| _         ||fi |¤Ž| _        |dk    rt%          |¦  «        nt'          j        ¦   «         | _        |                      d| j        rdn | j        di |¤Žd	¬
¦  «         dS )aç  
        Args:
            dim: Number of input channels.
            input_resolution: Input resolution.
            num_heads: Number of attention heads.
            window_size: Window size.
            shift_size: Shift size for SW-MSA.
            always_partition: Always partition into full windows and shift
            mlp_ratio: Ratio of mlp hidden dim to embedding dim.
            qkv_bias: If True, add a learnable bias to query, key, value.
            proj_drop: Dropout rate.
            attn_drop: Attention dropout rate.
            drop_path: Stochastic depth rate.
            act_layer: Activation layer.
            norm_layer: Normalization layer.
            pretrained_window_size: Window size in pretraining.
        rJ   r   r   )r&   rC   rD   rF   rG   rH   r@   )Úin_featuresÚhidden_featuresrÆ   ÚdropÚ	attn_maskNFrQ   © )rV   rW   rB   r   rÀ   rC   Útarget_shift_sizerÂ   rÃ   Ú_calc_window_shiftr&   rÁ   Úwindow_arearÄ   r   r?   rª   Únorm1r   rX   ÚIdentityÚ
drop_path1r   rµ   ÚmlpÚnorm2Ú
drop_path2rc   Úget_attn_mask)rk   rB   rÀ   rC   r&   rÁ   rÂ   rÃ   rÄ   rD   rG   rF   rÅ   rÆ   rÇ   rH   rK   rL   rl   ro   s                      €r8   rW   zSwinTransformerV2Block.__init__*  s  ø€ ðJ ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØˆŒÝ )Ð*:Ñ ;Ô ;ˆÔØ"ˆŒÝ!*¨:Ñ!6Ô!6ˆÔØ 0ˆÔØ(ˆÔØ,0×,CÒ,CÀKÐQ[Ñ,\Ô,\Ñ)ˆÔ˜$œ/ØÔ+¨AÔ.°Ô1AÀ!Ô1DÑDˆÔØ"ˆŒÝ! )Ñ,Ô,ˆ	å#Øð	
å! $Ô"2Ñ3Ô3ØØØØÝ#,Ð-CÑ#DÔ#Dð	
ð 	
ð ð	
ð 	
ˆŒ	ð  �Z Ð*Ð* rÐ*Ð*ˆŒ
Ø1:¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒåð 
ØÝ  i¡Ñ0Ô0ØØð	
ð 
ð
 ð
ð 
ˆŒð  �Z Ð*Ð* rÐ*Ð*ˆŒ
Ø1:¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒà×ÒØØÔ%ÐCˆDˆDÐ+=¨4Ô+=Ð+CÐ+CÀÐ+CÐ+CØð 	ñ 	
ô 	
ð 	
ð 	
ð 	
r:   r%   rK   rL   r'   c           	      óœ  — t          | j        ¦  «        �r´|€#t          j        dg| j        ¢d‘R ||¬¦  «        }n;t          j        d|j        d         |j        d         df|j        |j        ¬¦  «        }d}d| j        d          f| j        d          | j        d          f| j        d          dffD ]n}d| j        d          f| j        d          | j        d          f| j        d          dffD ]0}||dd…|d         |d         …|d         |d         …dd…f<   |dz  }Œ1Œot          || j        ¦  «        }| 
                    d| j        ¦  «        }|                     d¦  «        |                     d¦  «        z
  }	|	                     |	dk    t          d¦  «        ¦  «                             |	dk    t          d¦  «        ¦  «        }	nd}	|	S )	zÊGenerate attention mask for shifted window attention.

        Args:
            x: Input tensor for dynamic shape calculation.

        Returns:
            Attention mask or None if no shift.
        Nr   rJ   r*   r   r-   g      YÀr@   )ÚanyrÁ   rZ   ÚzerosrÀ   r/   rK   rL   r&   r9   r0   rÐ   r„   Úmasked_fillr·   )
rk   r%   rK   rL   Úimg_maskÚcntÚhÚwÚmask_windowsrÌ   s
             r8   r×   z$SwinTransformerV2Block.get_attn_masky  s	  € õ ˆtŒÑÔñ 	àˆyÝ œ;¨Ð'E¨DÔ,AÐ'EÀ1Ð'EÐ'EÈfÐ\aÐbÑbÔb��å œ;¨¨1¬7°1¬:°q´w¸q´zÀ1Ð'EÈaÌhÐ^_Ô^eÐfÑfÔf�ØˆCà˜Ô)¨!Ô,Ð,Ð-ØÔ& qÔ)Ð)¨D¬O¸AÔ,>Ð+>Ð?Ø”o aÔ(Ð(¨$Ð/ðð ð �ð ˜TÔ-¨aÔ0Ð0Ð1ØÔ*¨1Ô-Ð-°´ÀÔ0BÐ/BÐCØœ/¨!Ô,Ð,¨dÐ3ðð ð �Að
 <?�H˜Q˜Q˜Q  !¤ Q q¤T 	¨1¨Q¬4°°!´¨9°a°a°aÐ7Ñ8Ø˜1‘H�C�Cðõ ,¨H°dÔ6FÑGÔGˆLØ'×,Ò,¨R°Ô1AÑBÔBˆLØ$×.Ò.¨qÑ1Ô1°L×4JÒ4JÈ1Ñ4MÔ4MÑMˆIØ!×-Ò-¨i¸1ªn½eÀF¹m¼mÑLÔL×XÒXÐYbÐfgÒYgÕinÐorÑisÔisÑtÔtˆIˆIàˆIØÐr:   Útarget_window_sizerÎ   c                 ó¨  — t          |¦  «        }|€-| j        }t          |¦  «        r|d         dz  |d         dz  f}nt          |¦  «        }| j        r||fS t          |¦  «        }t          |¦  «        }d„ t	          | j        |¦  «        D ¦   «         }d„ t	          | j        ||¦  «        D ¦   «         }t          |¦  «        t          |¦  «        fS )a  Calculate window size and shift size based on input resolution.

        Args:
            target_window_size: Target window size.
            target_shift_size: Target shift size.

        Returns:
            Tuple of (adjusted_window_size, adjusted_shift_size).
        Nr   r*   r   c                 ó(   — g | ]\  }}||k    r|n|‘ŒS rÍ   rÍ   )Ú.0Úrrß   s      r8   ú
<listcomp>z=SwinTransformerV2Block._calc_window_shift.<locals>.<listcomp>¿  s(   € ÐeÐeÐe©d¨a°˜A šF˜F�q�q¨ÐeÐeÐer:   c                 ó*   — g | ]\  }}}||k    rd n|‘ŒS ©r   rÍ   )rä   rå   rß   Úss       r8   ræ   z=SwinTransformerV2Block._calc_window_shift.<locals>.<listcomp>À  s*   € ÐsÐsÐs©W¨Q°°1˜1 š6˜6�a�a qÐsÐsÐsr:   )r   rÎ   rÙ   rÂ   ÚziprÀ   Útuple)rk   rá   rÎ   r&   rÁ   s        r8   rÏ   z)SwinTransformerV2Block._calc_window_shift¢  sû   € õ 'Ð'9Ñ:Ô:ÐØÐ$à $Ô 6ÐÝÐ$Ñ%Ô%ð ]à%7¸Ô%:¸aÑ%?ÐASÐTUÔAVÐZ[ÑA[Ð$\Ð!øå )Ð*;Ñ <Ô <ÐàÔ ð 	9Ø%Ð'8Ð8Ð8å&Ð'9Ñ:Ô:ÐÝ%Ð&7Ñ8Ô8ÐØeÐeµc¸$Ô:OÐQcÑ6dÔ6dÐeÑeÔeˆØsÐs½¸DÔ<QÐS^Ð`qÑ8rÔ8rÐsÑsÔsˆ
Ý�[Ñ!Ô!¥5¨Ñ#4Ô#4Ð4Ð4r:   Ú	feat_sizec                 óÂ  — || _         |�|| _        |                      t          |¦  «        ¦  «        \  | _        | _        | j        d         | j        d         z  | _        | j                             | j        ¦  «         | j	        �| j	        j
        nd}| j	        �| j	        j        nd}|                      d| j        rdn|                      ||¬¦  «        d¬¦  «         dS )zßSet input size and update window configuration.

        Args:
            feat_size: New feature map size.
            window_size: New window size.
            always_partition: Override always_partition setting.
        Nr   r   rÌ   rJ   FrQ   )rÀ   rÂ   rÏ   r   r&   rÁ   rÐ   rª   r“   rÌ   rK   rL   rc   rÃ   r×   )rk   rì   r&   rÂ   rK   rL   s         r8   Úset_input_sizez%SwinTransformerV2Block.set_input_sizeÃ  sñ   € ð !*ˆÔØÐ'Ø$4ˆDÔ!Ø,0×,CÒ,CÅIÈkÑDZÔDZÑ,[Ô,[Ñ)ˆÔ˜$œ/ØÔ+¨AÔ.°Ô1AÀ!Ô1DÑDˆÔØŒ	×!Ò! $Ô"2Ñ3Ô3Ð3Ø*.¬.Ð*D�”Ô&Ð&È$ˆØ(,¬Ð(B�”Ô$Ð$ÈˆØ×ÒØØÔ%ÐYˆDˆD¨4×+=Ò+=ÀVÐSXÐ+=Ñ+YÔ+YØð 	ñ 	
ô 	
ð 	
ð 	
ð 	
r:   c           	      óÖ  — |j         \  }}}}t          | j        ¦  «        }|r2t          j        || j        d          | j        d          fd¬¦  «        }n|}| j        d         || j        d         z  z
  | j        d         z  }| j        d         || j        d         z  z
  | j        d         z  }	t          j        j                             |ddd|	d|f¦  «        }|j         \  }
}}}
t          || j        ¦  «        }| 
                    d| j        |¦  «        }t          | dd¦  «        r|                      |¦  «        }n| j        }|                      ||¬¦  «        }| 
                    d| j        d         | j        d         |¦  «        }t!          || j        ||f¦  «        }|d	d	…d	|…d	|…d	d	…f                              ¦   «         }|rt          j        || j        d¬¦  «        }n|}|S )
z»Apply windowed attention with optional shift.

        Args:
            x: Input tensor of shape (B, H, W, C).

        Returns:
            Output tensor of shape (B, H, W, C).
        r   r   )r   r*   )ÚshiftsÚdimsr-   rÃ   F)r–   N)r/   rÙ   rÁ   rZ   Úrollr&   rX   Ú
functionalÚpadr9   r0   rÐ   Úgetattrr×   rÌ   rª   r=   r2   )rk   r%   r3   r4   r5   r6   Ú	has_shiftÚ	shifted_xÚpad_hÚpad_wÚ_ÚHpÚWpÚ	x_windowsrÌ   Úattn_windowss                   r8   Ú_attnzSwinTransformerV2Block._attnß  s  € ð ”W‰
ˆˆ1ˆa�õ ˜œÑ(Ô(ˆ	Øð 	Ýœ
 1¨t¬¸qÔ/AÐ.AÀDÄOÐTUÔDVÐCVÐ-WÐ^dÐeÑeÔeˆIˆIàˆIàÔ! !Ô$ q¨4Ô+;¸AÔ+>Ñ'>Ñ>À$ÔBRÐSTÔBUÑUˆØÔ! !Ô$ q¨4Ô+;¸AÔ+>Ñ'>Ñ>À$ÔBRÐSTÔBUÑUˆÝ”HÔ'×+Ò+¨I¸¸1¸aÀÈÈ5Ð7QÑRÔRˆ	Ø ”‰ˆˆ2ˆr�1õ % Y°Ô0@ÑAÔAˆ	Ø—N’N 2 tÔ'7¸Ñ;Ô;ˆ	õ �4˜¨Ñ/Ô/ð 	'Ø×*Ò*¨9Ñ5Ô5ˆIˆIàœˆIØ—y’y °�yÑ;Ô;ˆð $×(Ò(¨¨TÔ-=¸aÔ-@À$ÔBRÐSTÔBUÐWXÑYÔYˆÝ" <°Ô1AÀBÈÀ8ÑLÔLˆ	Ø˜a˜a˜a  !  R a R¨¨¨˜lÔ+×6Ò6Ñ8Ô8ˆ	ð ð 	Ý”
˜9¨T¬_À6ÐJÑJÔJˆAˆAàˆAØˆr:   c                 ót  — |j         \  }}}}||                      |                      |                      |¦  «        ¦  «        ¦  «        z   }|                     |d|¦  «        }||                      |                      |                      |¦  «        ¦  «        ¦  «        z   }|                     ||||¦  «        }|S )Nr-   )r/   rÓ   rÑ   rÿ   rž   rÖ   rÕ   rÔ   )rk   r%   r3   r4   r5   r6   s         r8   r®   zSwinTransformerV2Block.forward  sš   € Ø”W‰
ˆˆ1ˆa�Ø�—’ §
¢
¨4¯:ª:°a©=¬=Ñ 9Ô 9Ñ:Ô:Ñ:ˆØ�IŠI�a˜˜QÑÔˆØ�—’ §
¢
¨4¯8ª8°A©;¬;Ñ 7Ô 7Ñ8Ô8Ñ8ˆØ�IŠI�a˜˜A˜qÑ!Ô!ˆØˆr:   )NNNr°   )r±   r²   r³   r´   rX   Ú	LayerNormrµ   Ú_int_or_tuple_2_tr¶   r·   r   r   ÚModulerW   r   rZ   r¸   rK   rL   r×   r
   rÏ   rî   rÿ   r®   r¹   rº   s   @r8   r¼   r¼   #  s“  ø€ € € € € ðð ð ./Ø,-Ø%*Ø!&Ø!Ø!Ø!Ø!Ø!Ø#)Ø*,¬,Ø89ØØð%M
ð M
àðM
ð 0ðM
ð ð	M
ð
 +ðM
ð *ðM
ð #ðM
ð ðM
ð ðM
ð ðM
ð ðM
ð ðM
ð ðM
ð !ðM
ð ˜RœYœðM
ð  %6ð!M
ð M
ð M
ð M
ð M
ð M
ðb )-Ø-1Ø+/ð	'ð 'à˜œÔ%ð'ð ˜Uœ\Ô*ð'ð ˜EœKÔ(ð	'ð
 
�%”,Ô	ð'ð 'ð 'ð 'ðX >Bð5ð 5à 1ð5ð  (Ð(9Ô:ð5ð 
ˆu�S˜#�XŒ  c¨3 h¤Ð/Ô	0ð	5ð 5ð 5ð 5ðJ 04ð	
ð 
à˜S #˜X”ð
ð ˜s C˜xœð
ð ' tœnð	
ð
 
ð
ð 
ð 
ð 
ð8,�u”|ð ,¨¬ð ,ð ,ð ,ð ,ð\˜œð ¨%¬,ð ð ð ð ð ð ð ð r:   r¼   c                   óŠ   ‡ — e Zd ZdZdej        ddfdedee         deej	                 fˆ fd„Z
dej        dej        fd	„Zˆ xZS )
ÚPatchMergingzŽPatch Merging Layer.

    Merges 2x2 neighboring patches and projects to higher dimension,
    effectively downsampling the feature maps.
    NrB   Úout_dimrÇ   c                 óâ   •— ||dœ}t          ¦   «                              ¦   «          || _        |pd|z  | _        t	          j        d|z  | j        fddi|¤Ž| _         || j        fi |¤Ž| _        dS )zì
        Args:
            dim (int): Number of input channels.
            out_dim (int): Number of output channels (or 2 * dim if None)
            norm_layer (nn.Module, optional): Normalization layer.  Default: nn.LayerNorm
        rJ   r*   r+   rN   FN)rV   rW   rB   r  rX   r^   Ú	reductionÚnorm)rk   rB   r  rÇ   rK   rL   rl   ro   s          €r8   rW   zPatchMerging.__init__  s„   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØˆŒØÐ) ! c¡'ˆŒÝœ 1 s¡7¨D¬LÐKÐK¸uÐKÈÐKÐKˆŒØ�J˜tœ|Ð2Ð2¨rÐ2Ð2ˆŒ	ˆ	ˆ	r:   r%   r'   c                 ó|  — |j         \  }}}}ddd|dz  d|dz  f}t          j                             ||¦  «        }|j         \  }}}}|                     ||dz  d|dz  d|¦  «                             dddddd¦  «                             d¦  «        }|                      |¦  «        }|                      |¦  «        }|S )Nr   r*   r   r)   r+   r,   )	r/   rX   ró   rô   rž   r1   r‰   r  r	  )rk   r%   r3   r4   r5   r6   Ú
pad_valuesrú   s           r8   r®   zPatchMerging.forward2  sÀ   € Ø”W‰
ˆˆ1ˆa�à˜˜A˜q 1™u a¨¨Q©Ð/ˆ
ÝŒM×Ò˜a Ñ,Ô,ˆØ”W‰
ˆˆ1ˆa�à�IŠI�a˜˜a™  A¨¡F¨A¨qÑ1Ô1×9Ò9¸!¸QÀÀ1ÀaÈÑKÔK×SÒSÐTUÑVÔVˆØ�NŠN˜1ÑÔˆØ�IŠI�a‰LŒLˆØˆr:   )r±   r²   r³   r´   rX   r  rµ   r   r   r  rW   rZ   r¸   r®   r¹   rº   s   @r8   r  r    s¨   ø€ € € € € ðð ð &*Ø*,¬,ØØð3ð 3àð3ð ˜c”]ð3ð ˜RœYœð	3ð 3ð 3ð 3ð 3ð 3ð*
˜œð 
¨%¬,ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r:   r  c            '       ó8  ‡ — e Zd ZdZdddddddddej        ddddfd	ed
edededededededede	dede	de	de	de
eeej                 f         deej                 dededdf&ˆ fd„Z	 d"deeef         dedee         ddfd„Zdej        dej        fd „Zd#d!„Zˆ xZS )$ÚSwinTransformerV2Stagez•A Swin Transformer V2 Stage.

    A single stage consisting of multiple Swin Transformer blocks with
    optional downsampling at the beginning.
    Fr¾   Tr@   r¿   r   NrB   r  rÀ   ÚdepthrC   r&   rÂ   rÃ   Ú
downsamplerÄ   rD   rG   rF   rÅ   rÆ   rÇ   rH   Úoutput_nchwr'   c                 ó>  •‡ ‡‡‡‡‡‡
‡‡‡‡‡‡‡‡‡— ||dœŠt          ¦   «                              ¦   «          |‰ _        |‰ _        |	rt	          d„ |D ¦   «         ¦  «        n|‰ _        |‰ _        |‰ _        d‰ _        t          ‰¦  «        Št	          d„ ‰D ¦   «         ¦  «        Š|	rt          d|‰‰dœ‰¤Ž‰ _        n |‰k    sJ ‚t          j        ¦   «         ‰ _        t          j        ˆˆˆˆˆˆˆ
ˆˆˆˆˆˆˆ ˆˆfd„t          |¦  «        D ¦   «         ¦  «        ‰ _        dS )	a  
        Args:
            dim: Number of input channels.
            out_dim: Number of output channels.
            input_resolution: Input resolution.
            depth: Number of blocks.
            num_heads: Number of attention heads.
            window_size: Local window size.
            always_partition: Always partition into full windows and shift
            dynamic_mask: Create attention mask in forward based on current input size
            downsample: Use downsample layer at start of the block.
            mlp_ratio: Ratio of mlp hidden dim to embedding dim.
            qkv_bias: If True, add a learnable bias to query, key, value.
            proj_drop: Projection dropout rate
            attn_drop: Attention dropout rate.
            drop_path: Stochastic depth rate.
            act_layer: Activation layer type.
            norm_layer: Normalization layer.
            pretrained_window_size: Local window size in pretraining.
            output_nchw: Output tensors on NCHW format instead of NHWC.
        rJ   c              3   ó    K  — | ]	}|d z  V — Œ
dS ©r*   NrÍ   ©rä   Úis     r8   ú	<genexpr>z2SwinTransformerV2Stage.__init__.<locals>.<genexpr>v  s&   è è € Ð&HÐ&H°! q¨A¡vÐ&HÐ&HÐ&HÐ&HÐ&HÐ&Hr:   Fc                 ó   — g | ]}|d z  ‘ŒS ©r*   rÍ   )rä   rß   s     r8   ræ   z3SwinTransformerV2Stage.__init__.<locals>.<listcomp>{  s   € Ð8Ð8Ð8 q˜A ™FÐ8Ð8Ð8r:   )rB   r  rÇ   c                 ó¢   •— g | ]K}t          d‰‰j        ‰
‰|d z  dk    rdn‰‰‰‰‰‰‰t          ‰t          ¦  «        r‰|         n‰‰‰	‰dœ‰¤Ž‘ŒLS )r*   r   )rB   rÀ   rC   r&   rÁ   rÂ   rÃ   rÄ   rD   rG   rF   rÅ   rÆ   rÇ   rH   rÍ   )r¼   Úoutput_resolutionÚ
isinstanceÚlist)rä   r  rÆ   rÂ   rF   rl   rÅ   rÃ   rÄ   rÇ   rC   r  rH   rG   rD   rk   rÁ   r&   s     €€€€€€€€€€€€€€€€r8   ræ   z3SwinTransformerV2Stage.__init__.<locals>.<listcomp>…  s    ø€ ð %#ð %#ð %#ð& õ% #ð ØØ!%Ô!7Ø#Ø'Ø!" Q¡¨!¢ ˜1˜1°*Ø!1Ø)Ø#Ø!Ø#Ø#Ý*4°YÅÑ*EÔ*EÐT˜) Aœ,˜,È9Ø#Ø%Ø'=ðð ð  ð!ð ð%#ð %#ð %#r:   NrÍ   )rV   rW   rB   rÀ   rë   r  r  r  Úgrad_checkpointingr   r  r  rX   rÒ   Ú
ModuleListÚrangeÚblocks)rk   rB   r  rÀ   r  rC   r&   rÂ   rÃ   r  rÄ   rD   rG   rF   rÅ   rÆ   rÇ   rH   r  rK   rL   rl   rÁ   ro   s   ` `  ```` ````````   @@€r8   rW   zSwinTransformerV2Stage.__init__F  s‹  øøøøøøøøøøøøøøøøø€ ðX ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØˆŒØ 0ˆÔØLVÐ!l¥Ð&HÐ&HÐ7GÐ&HÑ&HÔ&HÑ!HÔ!HÐ!HÐ\lˆÔØˆŒ
Ø&ˆÔØ"'ˆÔÝ Ñ,Ô,ˆÝÐ8Ð8¨KÐ8Ñ8Ô8Ñ9Ô9ˆ
ð ð 	,Ý*Ða¨s¸GÐPZÐaÐaÐ^`ÐaÐaˆDŒOˆOà˜'’>�>�>�>Ý œk™mœmˆDŒOõ ”mð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#ð %#õ& ˜5‘\”\ð'%#ñ %#ô %#ñ $ô $ˆŒˆˆr:   rì   c                 ó$  — || _         t          | j        t          j        ¦  «        r|| _        n:t          | j        t          ¦  «        sJ ‚t          d„ |D ¦   «         ¦  «        | _        | j        D ]}| 	                    | j        ||¬¦  «         Œ dS )zòUpdate resolution, window size and relative positions.

        Args:
            feat_size: New input (feature) resolution.
            window_size: New window size.
            always_partition: Always partition / shift the window.
        c              3   ó    K  — | ]	}|d z  V — Œ
dS r  rÍ   r  s     r8   r  z8SwinTransformerV2Stage.set_input_size.<locals>.<genexpr>¬  s&   è è € Ð*EÐ*E°a¨1°©6Ð*EÐ*EÐ*EÐ*EÐ*EÐ*Er:   ©rì   r&   rÂ   N)
rÀ   r  r  rX   rÒ   r  r  rë   r   rî   )rk   rì   r&   rÂ   Úblocks        r8   rî   z%SwinTransformerV2Stage.set_input_sizeš  s®   € ð !*ˆÔÝ�d”o¥r¤{Ñ3Ô3ð 	FØ%.ˆDÔ"Ð"å˜dœo­|Ñ<Ô<Ð<Ð<Ð<Ý%*Ð*EÐ*E¸9Ð*EÑ*EÔ*EÑ%EÔ%EˆDÔ"Ø”[ð 	ð 	ˆEØ× Ò ØÔ0Ø'Ø!1ð !ñ ô ð ð ð	ð 	r:   r%   c                 óÆ   — |                       |¦  «        }| j        D ]C}| j        r/t          j                             ¦   «         st          ||¦  «        }Œ8 ||¦  «        }ŒD|S )z°Forward pass through the stage.

        Args:
            x: Input tensor of shape (B, H, W, C).

        Returns:
            Output tensor of shape (B, H', W', C').
        )r  r   r  rZ   ÚjitÚis_scriptingr    )rk   r%   Úblks      r8   r®   zSwinTransformerV2Stage.forward´  sk   € ð �OŠO˜AÑÔˆà”;ð 	ð 	ˆCØÔ&ð ­u¬y×/EÒ/EÑ/GÔ/Gð Ý˜s AÑ&Ô&��à�C˜‘F”F��Øˆr:   c                 ój  — | j         D ]ª}t          j                             |j        j        d¦  «         t          j                             |j        j        d¦  «         t          j                             |j        j        d¦  «         t          j                             |j        j        d¦  «         Œ«dS )z/Initialize residual post-normalization weights.r   N)r   rX   rr   rs   rÑ   rN   r|   rÕ   )rk   r(  s     r8   Ú_init_respostnormz(SwinTransformerV2Stage._init_respostnormÆ  s’   € à”;ð 	3ð 	3ˆCÝŒG×Ò˜cœiœn¨aÑ0Ô0Ð0ÝŒG×Ò˜cœiÔ.°Ñ2Ô2Ð2ÝŒG×Ò˜cœiœn¨aÑ0Ô0Ð0ÝŒG×Ò˜cœiÔ.°Ñ2Ô2Ð2Ð2ð		3ð 	3r:   r°   r¯   )r±   r²   r³   r´   rX   r  rµ   r  r¶   r·   r   Ústrr   r  rW   r
   r   rî   rZ   r¸   r®   r*  r¹   rº   s   @r8   r  r  ?  sñ  ø€ € € € € ðð ð &+Ø!&Ø$Ø!Ø!Ø!Ø!Ø!Ø5;Ø*,¬,Ø89Ø %ØØð+R$ð R$àðR$ð ðR$ð 0ð	R$ð
 ðR$ð ðR$ð +ðR$ð #ðR$ð ðR$ð ðR$ð ðR$ð ðR$ð ðR$ð ðR$ð ðR$ð  ˜S $ r¤y¤/Ð1Ô2ð!R$ð" ˜RœYœð#R$ð$ %6ð%R$ð& ð'R$ð, 
ð-R$ð R$ð R$ð R$ð R$ð R$ðp 04ð	ð à˜S #˜X”ðð ðð ' tœnð	ð
 
ðð ð ð ð4˜œð ¨%¬,ð ð ð ð ð$3ð 3ð 3ð 3ð 3ð 3ð 3ð 3r:   r  c            +       ó>  ‡ — e Zd ZdZdddddddd	d
dddddddddej        dddfdedededededede	edf         de	edf         dede
de
ded e
d!ed"ed#ed$ed%eeef         d&eej                 d'e	edf         f(ˆ fd(„ZdHd)e
d*dfd+„ZdHd,ej        d)e
d*dfd-„Z	 	 	 	 	 dIdee	eef                  dee	eef                  dee	eef                  d/ee         dee
         f
d0„Zej        j        d*ee         fd1„¦   «         Zej        j        dJd2e
d*eeef         fd3„¦   «         Zej        j        dHd4e
d*dfd5„¦   «         Zej        j        d*ej        fd6„¦   «         ZdKdedee         d*dfd7„Z	 	 	 	 	 dLd9ej         d:eeee!e         f                  d;e
d<e
d=ed>e
d*ee!ej                  e	ej         e!ej                  f         f         fd?„Z"	 	 	 dMd:eee!e         f         dAe
dBe
fdC„Z#d9ej         d*ej         fdD„Z$dJd9ej         dEe
d*ej         fdF„Z%d9ej         d*ej         fdG„Z&ˆ xZ'S )Nr$   a   Swin Transformer V2.

    A hierarchical vision transformer using shifted windows for efficient
    self-attention computation with continuous position bias.

    A PyTorch impl of : `Swin Transformer V2: Scaling Up Capacity and Resolution`
        - https://arxiv.org/abs/2111.09883
    éà   r+   r)   éè  Úavgé`   ©r*   r*   é   r*   ©r)   r2  é   é   r½   FTr¾   r@   gš™™™™™¹?r¿   )r   r   r   r   Nr;   Ú
patch_sizeÚin_chansÚnum_classesÚglobal_poolÚ	embed_dimÚdepths.rC   r&   rÂ   Ústrict_img_sizerÄ   rD   Ú	drop_rateÚproj_drop_rateÚattn_drop_rateÚdrop_path_raterÆ   rÇ   Úpretrained_window_sizesc                 ó€  •‡— t          ¦   «                              ¦   «          ||dœ}|| _        || _        |dv sJ ‚|| _        d| _        t          |¦  «        | _        ‰| _        t          ‰d| j        dz
  z  z  ¦  «        x| _
        | _        g | _        t          ‰t          t          f¦  «        s ˆfd„t!          | j        ¦  «        D ¦   «         Št#          d#|||‰d         ||ddœ|¤Ž| _        | j        j        }t)          ||d	¬
¦  «        }g }‰d         }d}t!          | j        ¦  «        D ]²}‰|         }|t+          d#i d|“d|“d|d         |z  |d         |z  f“d||         “d|dk    “d||         “d|	“d|
“d| “d|“d|“d|“d|“d||         “d|“d|“d||         “|¤Žgz  }|}|dk    r|dz  }| xj        t-          |d|z  d|› �¬¦  «        gz  c_        Œ³t/          j        |Ž | _         || j
        fi |¤Ž| _        t7          | j
        |f||| j        dœ|¤Ž| _        |                      d ¬!¦  «         d"S )$a]  
        Args:
            img_size: Input image size.
            patch_size: Patch size.
            in_chans: Number of input image channels.
            num_classes: Number of classes for classification head.
            embed_dim: Patch embedding dimension.
            depths: Depth of each Swin Transformer stage (layer).
            num_heads: Number of attention heads in different layers.
            window_size: Window size.
            mlp_ratio: Ratio of mlp hidden dim to embedding dim.
            qkv_bias: If True, add a learnable bias to query, key, value.
            drop_rate: Head dropout rate.
            proj_drop_rate: Projection dropout rate.
            attn_drop_rate: Attention dropout rate.
            drop_path_rate: Stochastic depth rate.
            norm_layer: Normalization layer.
            act_layer: Activation layer type.
            patch_norm: If True, add normalization after patch embedding.
            pretrained_window_sizes: Pretrained window sizes of each layer.
            output_fmt: Output tensor format if not None, otherwise output 'NHWC' by default.
        rJ   )Ú r/  ÚNHWCr*   r   c                 ó:   •— g | ]}t          ‰d |z  z  ¦  «        ‘ŒS r  )rµ   )rä   r  r:  s     €r8   ræ   z.SwinTransformerV2.__init__.<locals>.<listcomp>  s*   ø€ ÐQÐQÐQ°Q�˜Y¨¨a©Ñ/Ñ0Ô0ÐQÐQÐQr:   r   )r;   r6  r7  r:  rÇ   r<  Ú
output_fmtT)Ú	stagewiserB   r  rÀ   r  r  rC   r&   rÂ   rÃ   rÄ   rD   rG   rF   rÅ   rÆ   rÇ   rH   r+   úlayers.)Únum_chsr  Úmodule)Ú	pool_typer=  Ú	input_fmtF©Úneeds_resetNrÍ   )rV   rW   r8  r7  r9  rF  ÚlenÚ
num_layersr:  rµ   Únum_featuresÚhead_hidden_sizeÚfeature_infor  rë   r  r  r   Úpatch_embedÚ	grid_sizer   r  ÚdictrX   r]   Úlayersr	  r   ÚheadÚinit_weights)!rk   r;   r6  r7  r8  r9  r:  r;  rC   r&   rÂ   r<  rÄ   rD   r=  r>  r?  r@  rÆ   rÇ   rA  rK   rL   Úkwargsrl   rU  ÚdprrW  Úin_dimÚscaler  r  ro   s!         `                         €r8   rW   zSwinTransformerV2.__init__Ù  sj  øø€ õ` 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆà&ˆÔØ ˆŒØ˜kÐ)Ð)Ð)Ð)Ø&ˆÔØ ˆŒÝ˜f™+œ+ˆŒØ"ˆŒÝ47¸	ÀAÈ$Ì/Ð\]ÑJ]ÑD^Ñ8^Ñ4_Ô4_Ð_ˆÔ˜DÔ1ØˆÔå˜)¥e­T ]Ñ3Ô3ð 	RØQÐQÐQÐQ½%ÀÄÑ:PÔ:PÐQÑQÔQˆIõ &ð 	
ØØ!ØØ ”lØ!Ø+Øð	
ð 	
ð ð	
ð 	
ˆÔð Ô$Ô.ˆ	å'¨¸È$ÐOÑOÔOˆØˆØ˜1”ˆØˆÝ�t”Ñ'Ô'ð 	dð 	dˆAØ ”lˆGØÕ-ð ð ð Ø�Fðà˜ðð #,¨A¤,°%Ñ"7¸À1¼ÈÑ9NÐ!OÐ!Oðð ˜Q”i�ið	ð
 ˜qš5˜5ðð $ Aœ,˜,ðð (˜Kðð "2Ð!1ðð "1Ð0Ð0ðð $˜)ðð "˜ðð )˜.ðð )˜.ðð ˜aœ&˜&ðð $˜)ðð  &˜:ð!ð" (?¸qÔ'AÐ'AØð%ð ð ñ ˆFð( ˆFØ�1ŠuˆuØ˜‘
�ØÐÔ¥$¨wÀ!ÀeÁ)ÐTaÐ^_ÐTaÐTaÐ"bÑ"bÔ"bÐ!cÑcÐÔÐå”m VÐ,ˆŒØ�J˜tÔ0Ð7Ð7°BÐ7Ð7ˆŒ	Ý"ØÔØð
ð "ØØ”oð
ð 
ð ð
ð 
ˆŒ	ð 	×Ò eÐÑ,Ô,Ð,Ð,Ð,r:   rN  r'   c                 ó”   — |                       t          | j        |¬¦  «        ¦  «         | j        D ]}|                     ¦   «          ŒdS )zÿInitialize model weights.

        Args:
            needs_reset: If True, call reset_parameters() on modules (default for after to_empty()).
                If False, skip reset_parameters() (for __init__ where modules already self-initialized).
        rM  N)Úapplyr   Ú_init_weightsrW  r*  )rk   rN  Úblys      r8   rY  zSwinTransformerV2.init_weightsS  sV   € ð 	�
Š
•7˜4Ô-¸;ÐGÑGÔGÑHÔHÐHØ”;ð 	$ð 	$ˆCØ×!Ò!Ñ#Ô#Ð#Ð#ð	$ð 	$r:   Úmc                 ó  — t          |t          j        ¦  «        rFt          |j        d¬¦  «         |j        �'t          j                             |j        d¦  «         dS dS |r&t          |d¦  «        r| 	                    ¦   «          dS dS dS )zªInitialize weights for Linear layers.

        Args:
            m: Module to initialize.
            needs_reset: Whether to call reset_parameters() on modules.
        g{®Gáz”?)ÚstdNr   rj   )
r  rX   r^   r   r|   rN   rr   rs   Úhasattrrj   )rk   rb  rN  s      r8   r`  zSwinTransformerV2._init_weights^  s£   € õ �a�œÑ#Ô#ð 	!Ý˜!œ(¨Ð,Ñ,Ô,Ð,ØŒvÐ!Ý”×!Ò! !¤&¨!Ñ,Ô,Ð,Ð,Ð,ð "Ð!àð 	!�W QÐ(:Ñ;Ô;ð 	!Ø×ÒÑ Ô Ð Ð Ð ð	!ð 	!ð 	!ð 	!r:   r   Úwindow_ratioc                 óV  ‡— |€|�(| j                              ||¬¦  «         | j         j        }|€‰�t          ˆfd„|D ¦   «         ¦  «        }t	          | j        ¦  «        D ]G\  }}dt          |dz
  d¦  «        z  }	|                     |d         |	z  |d         |	z  f||¬¦  «         ŒHdS )aT  Updates the image resolution, window size, and so the pair-wise relative positions.

        Args:
            img_size (Optional[Tuple[int, int]]): New input resolution, if None current resolution is used
            patch_size (Optional[Tuple[int, int]): New patch size, if None use current patch size
            window_size (Optional[int]): New window size, if None based on new_img_size // window_div
            window_ratio (int): divisor for calculating window size from patch grid size
            always_partition: always partition / shift windows even if feat size is < window
        N)r;   r6  c                 ó   •— g | ]}|‰z  ‘ŒS rÍ   rÍ   )rä   ré   rf  s     €r8   ræ   z4SwinTransformerV2.set_input_size.<locals>.<listcomp>‚  s   ø€ Ð FÐ FÐ F°q  lÑ!2Ð FÐ FÐ Fr:   r*   r   r   r#  )rT  rî   rU  rë   Ú	enumeraterW  r™   )
rk   r;   r6  r&   rf  rÂ   rU  ÚindexÚstageÚstage_scales
       `     r8   rî   z SwinTransformerV2.set_input_sizel  så   ø€ ð" Ð :Ð#9ØÔ×+Ò+°XÈ*Ð+ÑUÔUÐUØÔ(Ô2ˆIàÐ <Ð#;ÝÐ FÐ FÐ FÐ F¸IÐ FÑ FÔ FÑGÔGˆKå% d¤kÑ2Ô2ð 	ð 	‰LˆE�5Ø�s 5¨1¡9¨aÑ0Ô0Ñ0ˆKØ× Ò Ø$ Qœ<¨;Ñ6¸	À!¼ÈÑ8SÐTØ'Ø!1ð !ñ ô ð ð ð	ð 	r:   c                 ó¸   ‡— t          ¦   «         }|                      ¦   «         D ]5\  Š}t          ˆfd„dD ¦   «         ¦  «        r|                     ‰¦  «         Œ6|S )z‘Get parameter names that should not use weight decay.

        Returns:
            Set of parameter names to exclude from weight decay.
        c                 ó   •— g | ]}|‰v ‘ŒS rÍ   rÍ   )rä   ÚkwÚns     €r8   ræ   z5SwinTransformerV2.no_weight_decay.<locals>.<listcomp>•  s   ø€ ÐAÐAÐA �B˜!�GÐAÐAÐAr:   )r`   r\   )ÚsetÚnamed_modulesrÙ   Úadd)rk   Únodrb  rp  s      @r8   Úno_weight_decayz!SwinTransformerV2.no_weight_decayŒ  sh   ø€ õ ‰eŒeˆØ×&Ò&Ñ(Ô(ð 	ð 	‰DˆAˆqÝÐAÐAÐAÐAÐ&@ÐAÑAÔAÑBÔBð Ø—’˜‘
”
�
øØˆ
r:   Úcoarsec                 ó0   — t          d|rdng d¢¬¦  «        S )zØCreate parameter group matcher for optimizer parameter groups.

        Args:
            coarse: If True, use coarse grouping.

        Returns:
            Dictionary mapping group names to regex patterns.
        z^absolute_pos_embed|patch_embedz^layers\.(\d+)))z^layers\.(\d+).downsamplerè   )z^layers\.(\d+)\.\w+\.(\d+)N)z^norm)iŸ† )Ústemr   )rV  )rk   rv  s     r8   Úgroup_matcherzSwinTransformerV2.group_matcher™  s;   € õ Ø3Ø(.ð Ð$Ð$ð 5ð 5ð 5ð
ñ 
ô 
ð 	
r:   Úenablec                 ó(   — | j         D ]	}||_        Œ
dS )z}Enable or disable gradient checkpointing.

        Args:
            enable: If True, enable gradient checkpointing.
        N)rW  r  )rk   rz  Úls      r8   Úset_grad_checkpointingz(SwinTransformerV2.set_grad_checkpointing¬  s*   € ð ”ð 	*ð 	*ˆAØ#)ˆAÔ Ð ð	*ð 	*r:   c                 ó   — | j         j        S )z_Get the classifier head.

        Returns:
            The classification head module.
        )rX  Úfcrx   s    r8   Úget_classifierz SwinTransformerV2.get_classifier¶  s   € ð ŒyŒ|Ðr:   c                 óZ   — t          | ¦  «        }|| _         | j        j        ||fi |¤Ž dS )z�Reset the classification head.

        Args:
            num_classes: Number of classes for new head.
            global_pool: Global pooling type.
        N)r   r8  rX  Úreset)rk   r8  r9  rl   s       r8   Úreset_classifierz"SwinTransformerV2.reset_classifier¿  s=   € õ ˜dÑ#Ô#ˆØ&ˆÔØˆŒ	Œ˜ [Ð7Ð7°BÐ7Ð7Ð7Ð7Ð7r:   ÚNCHWr%   Úindicesr	  Ú
stop_earlyrF  Úintermediates_onlyc                 óx  — |dv s
J d¦   «         ‚g }t          t          | j        ¦  «        |¦  «        \  }}	|                      |¦  «        }t          | j        ¦  «        }
t          j                             ¦   «         s|s| j        }n| j        d|	dz   …         }t          |¦  «        D ]v\  }} ||¦  «        }||v rb|r||
dz
  k    r|                      |¦  «        }n|}| 	                    dddd¦  «         
                    ¦   «         }|                     |¦  «         Œw|r|S |                      |¦  «        }||fS )aï   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )r„  zOutput shape must be NCHW.Nr   r   r)   r*   )r   rO  rW  rT  rZ   r&  r'  ri  r	  r1   r2   Úappend)rk   r%   r…  r	  r†  rF  r‡  ÚintermediatesÚtake_indicesÚ	max_indexÚ
num_stagesÚstagesr  rk  Úx_inters                  r8   Úforward_intermediatesz'SwinTransformerV2.forward_intermediatesÊ  s^  € ð* ˜YÐ&Ð&Ð&Ð(DÑ&Ô&Ð&ØˆÝ"6µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�ið ×Ò˜QÑÔˆå˜œÑ%Ô%ˆ
ÝŒ9×!Ò!Ñ#Ô#ð 	1¨:ð 	1Ø”[ˆFˆFà”[  )¨a¡- Ô0ˆFÝ! &Ñ)Ô)ð 	.ð 	.‰HˆAˆuØ��a‘”ˆAØ�LÐ Ð Øð  ˜A ¨a¡Ò/Ð/Ø"Ÿiši¨™lœl�G�Gà�GØ!Ÿ/š/¨!¨Q°°1Ñ5Ô5×@Ò@ÑBÔB�Ø×$Ò$ WÑ-Ô-Ð-øàð 	!Ø Ð à�IŠI�a‰LŒLˆà�-ÐÐr:   r   Ú
prune_normÚ
prune_headc                 óâ   — t          t          | j        ¦  «        |¦  «        \  }}| j        d|dz   …         | _        |rt          j        ¦   «         | _        |r|                      dd¦  «         |S )z@ Prune layers not required for specified intermediates.
        Nr   r   rC  )r   rO  rW  rX   rÒ   r	  rƒ  )rk   r…  r‘  r’  r‹  rŒ  s         r8   Úprune_intermediate_layersz+SwinTransformerV2.prune_intermediate_layersü  sq   € õ #7µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�iØ”k . 9¨q¡= .Ô1ˆŒØð 	&Ýœ™œˆDŒIØð 	)Ø×!Ò! ! RÑ(Ô(Ð(ØÐr:   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S )zÀForward pass through feature extraction layers.

        Args:
            x: Input tensor of shape (B, C, H, W).

        Returns:
            Feature tensor of shape (B, H', W', C).
        )rT  rW  r	  ©rk   r%   s     r8   Úforward_featuresz"SwinTransformerV2.forward_features  s:   € ð ×Ò˜QÑÔˆØ�KŠK˜‰NŒNˆØ�IŠI�a‰LŒLˆØˆr:   Ú
pre_logitsc                 ó^   — |r|                       |d¬¦  «        n|                       |¦  «        S )a  Forward pass through classification head.

        Args:
            x: Feature tensor of shape (B, H, W, C).
            pre_logits: If True, return features before final linear layer.

        Returns:
            Logits tensor of shape (B, num_classes) or pre-logits.
        T)r˜  )rX  )rk   r%   r˜  s      r8   Úforward_headzSwinTransformerV2.forward_head  s.   € ð 1;ÐLˆt�yŠy˜ tˆyÑ,Ô,Ð,ÀÇ	Â	È!ÁÄÐLr:   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S )z±Forward pass through the model.

        Args:
            x: Input tensor of shape (B, C, H, W).

        Returns:
            Logits tensor of shape (B, num_classes).
        )r—  rš  r–  s     r8   r®   zSwinTransformerV2.forward&  s/   € ð ×!Ò! !Ñ$Ô$ˆØ×Ò˜aÑ Ô ˆØˆr:   )T)NNNr   N©Fr°   )NFFr„  F)r   FT)(r±   r²   r³   r´   rX   r  r  rµ   r+  r
   r¶   r·   r   r   r   r  rW   rY  r`  r   rî   rZ   r&  Úignorer	   ru  r   r   ry  r}  r€  rƒ  r¸   r   r�  r”  r—  rš  r®   r¹   rº   s   @r8   r$   r$   Ï  s  ø€ € € € € ðð ð +.ØØØ#Ø$ØØ&2Ø)7Ø-.Ø%*Ø$(Ø!Ø!Ø!Ø$&Ø$&Ø$'Ø.4Ø*,¬,Ø7CØØð/x-ð x-à'ðx-ð ðx-ð ð	x-ð
 ðx-ð ðx-ð ðx-ð ˜#˜s˜(”Oðx-ð ˜S #˜X”ðx-ð +ðx-ð #ðx-ð "ðx-ð ðx-ð ðx-ð ðx-ð  "ð!x-ð" "ð#x-ð$ "ð%x-ð& ˜S (˜]Ô+ð'x-ð( ˜RœYœð)x-ð* &+¨3°¨8¤_ð+x-ð x-ð x-ð x-ð x-ð x-ðt	$ð 	$¨ð 	$¸ð 	$ð 	$ð 	$ð 	$ð!ð !˜rœyð !°tð !Àtð !ð !ð !ð !ð  37Ø48Ø59Ø*+Ø/3ðð à˜u S¨# XœÔ/ðð !  s¨C x¤Ô1ðð " %¨¨S¨¤/Ô2ð	ð
 # 3œ-ðð ' tœnðð ð ð ð@ „YÔð
  S¤ð 
ð 
ð 
ñ Ôð
ð „YÔð
ð 
 Dð 
°T¸#¸s¸(´^ð 
ð 
ð 
ñ Ôð
ð$ „YÔð*ð *¨Tð *¸Tð *ð *ð *ñ Ôð*ð „YÔð ¤	ð ð ð ñ Ôðð	8ð 	8¨Cð 	8¸hÀs¼mð 	8ÐW[ð 	8ð 	8ð 	8ð 	8ð 8<ØØ$Ø$Ø',ð0 ð 0 àŒ|ð0 ð ˜e C¨¨c¬ NÔ3Ô4ð0 ð ð	0 ð
 ð0 ð ð0 ð !%ð0 ð 
ˆt�E”LÔ! 5¨¬°t¸E¼LÔ7IÐ)IÔ#JÐJÔ	Kð0 ð 0 ð 0 ð 0 ðh ./Ø$Ø#ð	ð à˜3  S¤	˜>Ô*ðð ðð ð	ð ð ð ð  %¤,ð °5´<ð ð ð ð ð
Mð 
M˜eœlð 
M¸ð 
MÈÌð 
Mð 
Mð 
Mð 
Mð˜œð ¨%¬,ð ð ð ð ð ð ð ð r:   Ú
state_dictÚmodelc                 óø  ‡	— |                       d| ¦  «        } |                       d| ¦  «        } d| v }i }ddl}|                      ¦   «         D ]­\  Š	}t          ˆ	fd„dD ¦   «         ¦  «        rŒ!d‰	v rS|j        j        j        j        \  }}}}|j        d	         |k    s|j        d
         |k    rt          |||fddd¬¦  «        }|s.| 	                    dd„ ‰	¦  «        Š	‰	 
                    dd¦  «        Š	||‰	<   Œ®|S )aM  Filter and process checkpoint state dict for loading.

    Handles resizing of patch embeddings and relative position tables
    when model size differs from checkpoint.

    Args:
        state_dict: Checkpoint state dictionary.
        model: Target model to load weights into.

    Returns:
        Filtered state dictionary.
    rŸ  rž  zhead.fc.weightr   Nc                 ó   •— g | ]}|‰v ‘ŒS rÍ   rÍ   )rä   rp  r¨   s     €r8   ræ   z(checkpoint_filter_fn.<locals>.<listcomp>G  s   ø€ ÐbÐbÐb˜1��Q�ÐbÐbÐbr:   )rU   rT   rÌ   zpatch_embed.proj.weightr˜   r-   ÚbicubicT)ÚinterpolationÚ	antialiasÚverbosezlayers.(\d+).downsamplec                 óT   — dt          |                      d¦  «        ¦  «        dz   › d�S )NrH  r   z.downsample)rµ   Úgroup)r%   s    r8   ú<lambda>z&checkpoint_filter_fn.<locals>.<lambda>W  s)   € Ð=gÅsÈ1Ï7Ê7ÐSTÉ:Ì:ÁÄÐYZÑGZÐ=gÐ=gÐ=g€ r:   zhead.zhead.fc.)ÚgetÚreÚitemsrÙ   rT  rf   r|   r/   r   ÚsubÚreplace)
rž  rŸ  Únative_checkpointÚout_dictrª  r©   rú   r4   r5   r¨   s
            @r8   Úcheckpoint_filter_fnr°  4  s@  ø€ ð —’ ¨Ñ4Ô4€JØ—’ ¨jÑ9Ô9€JØ(¨JÐ6ÐØ€HØ€I€I€IØ× Ò Ñ"Ô"ð ð ‰ˆˆ1ÝÐbÐbÐbÐbÐ aÐbÑbÔbÑcÔcð 	Øà$¨Ð)Ð)ØÔ*Ô/Ô6Ô<‰JˆAˆq�!�QØŒw�rŒ{˜aÒÐ 1¤7¨2¤;°!Ò#3Ð#3Ý(ØØ˜�FØ"+Ø"Ø ðñ ô �ð !ð 	/à—’Ð1Ð3gÐ3gÐijÑkÔkˆAØ—	’	˜' :Ñ.Ô.ˆAØˆ�‰ˆà€Or:   FÚvariantÚ
pretrainedc           	      óü   — t          d„ t          |                     dd¦  «        ¦  «        D ¦   «         ¦  «        }|                     d|¦  «        }t	          t
          | |ft          t          d|¬¦  «        dœ|¤Ž}|S )zôCreate a Swin Transformer V2 model.

    Args:
        variant: Model variant name.
        pretrained: If True, load pretrained weights.
        **kwargs: Additional model arguments.

    Returns:
        SwinTransformerV2 model instance.
    c              3   ó    K  — | ]	\  }}|V — Œ
d S r°   rÍ   )rä   r  rú   s      r8   r  z._create_swin_transformer_v2.<locals>.<genexpr>i  s&   è è € Ð\Ð\¡d a¨ Ð\Ð\Ð\Ð\Ð\Ð\r:   r;  )r   r   r   r   Úout_indicesT)Úflatten_sequentialrµ  )Úpretrained_filter_fnÚfeature_cfg)rë   ri  r©  Úpopr   r$   r°  rV  )r±  r²  rZ  Údefault_out_indicesrµ  rŸ  s         r8   Ú_create_swin_transformer_v2r»  ^  s�   € õ  Ð\Ð\­i¸¿
º
À8È\Ñ8ZÔ8ZÑ.[Ô.[Ð\Ñ\Ô\Ñ\Ô\ÐØ—*’*˜]Ð,?Ñ@Ô@€Kå Ý˜7 Jðå1Ý¨D¸kÐJÑJÔJðð ð ð	ð €Eð
 €Lr:   rC  c                 ó8   — | ddddddt           t          ddd	d
œ|¥S )Nr.  )r)   é   r½  )r   r   gÍÌÌÌÌÌì?r¢  Tzpatch_embed.projzhead.fcÚmit)Úurlr8  Ú
input_sizeÚ	pool_sizeÚcrop_pctr£  Úfixed_input_sizeÚmeanrd  Ú
first_convÚ
classifierÚlicenser   )r¿  rZ  s     r8   Ú_cfgrÈ  t  s:   € àØ¨=ÀvØ¨ÈÝ%Õ.BØ(¸	Øðð ð #ðð r:   ztimm/z{https://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_base_patch4_window12to16_192to256_22kto1k_ft.pth)Ú	hf_hub_idr¿  z{https://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_base_patch4_window12to24_192to384_22kto1k_ft.pth)r)   é€  rÊ  )r4  r4  r€   )rÉ  r¿  rÀ  rÁ  rÂ  z|https://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_large_patch4_window12to16_192to256_22kto1k_ft.pthz|https://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_large_patch4_window12to24_192to384_22kto1k_ft.pthzfhttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_tiny_patch4_window8_256.pthzghttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_tiny_patch4_window16_256.pthzghttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_small_patch4_window8_256.pthzhhttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_small_patch4_window16_256.pthzfhttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_base_patch4_window8_256.pthzghttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_base_patch4_window16_256.pthzkhttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_base_patch4_window12_192_22k.pthiQU  )r)   éÀ   rË  )r2  r2  )rÉ  r¿  r8  rÀ  rÁ  zlhttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_large_patch4_window12_192_22k.pth)ú2swinv2_base_window12to16_192to256.ms_in22k_ft_in1kú2swinv2_base_window12to24_192to384.ms_in22k_ft_in1kú3swinv2_large_window12to16_192to256.ms_in22k_ft_in1kú3swinv2_large_window12to24_192to384.ms_in22k_ft_in1kzswinv2_tiny_window8_256.ms_in1kz swinv2_tiny_window16_256.ms_in1kz swinv2_small_window8_256.ms_in1kz!swinv2_small_window16_256.ms_in1kzswinv2_base_window8_256.ms_in1kz swinv2_base_window16_256.ms_in1kú!swinv2_base_window12_192.ms_in22kú"swinv2_large_window12_192.ms_in22kc           	      ó\   — t          dddd¬¦  «        }t          	 dd| it          |fi |¤Ž¤ŽS )	z"Swin-T V2 @ 256x256, window 16x16.rš   r0  r1  r3  ©r&   r:  r;  rC   Úswinv2_tiny_window16_256r²  )rÔ  ©rV  r»  ©r²  rZ  Ú
model_argss      r8   rÔ  rÔ  ¹  s\   € õ  "°¸<ÐSaÐbÑbÔb€JÝ&Ø"ðYð YØ/9ðYÝ=AÀ*Ð=WÐ=WÐPVÐ=WÐ=WðYð Yð Yr:   c           	      ó\   — t          dddd¬¦  «        }t          	 dd| it          |fi |¤Ž¤ŽS )	z Swin-T V2 @ 256x256, window 8x8.r   r0  r1  r3  rÓ  Úswinv2_tiny_window8_256r²  )rÙ  rÕ  rÖ  s      r8   rÙ  rÙ  Á  s[   € õ  !¨r¸,ÐR`ÐaÑaÔa€JÝ&Ø!ðXð XØ.8ðXÝ<@ÀÐ<VÐ<VÈvÐ<VÐ<VðXð Xð Xr:   c           	      ó\   — t          dddd¬¦  «        }t          	 dd| it          |fi |¤Ž¤ŽS )	z"Swin-S V2 @ 256x256, window 16x16.rš   r0  ©r*   r*   é   r*   r3  rÓ  Úswinv2_small_window16_256r²  )rÝ  rÕ  rÖ  s      r8   rÝ  rÝ  É  s\   € õ  "°¸=ÐTbÐcÑcÔc€JÝ&Ø#ðZð ZØ0:ðZÝ>BÀ:Ð>XÐ>XÐQWÐ>XÐ>XðZð Zð Zr:   c           	      ó\   — t          dddd¬¦  «        }t          	 dd| it          |fi |¤Ž¤ŽS )	z Swin-S V2 @ 256x256, window 8x8.r   r0  rÛ  r3  rÓ  Úswinv2_small_window8_256r²  )rß  rÕ  rÖ  s      r8   rß  rß  Ñ  s\   € õ  !¨r¸-ÐSaÐbÑbÔb€JÝ&Ø"ðYð YØ/9ðYÝ=AÀ*Ð=WÐ=WÐPVÐ=WÐ=WðYð Yð Yr:   c           	      ó\   — t          dddd¬¦  «        }t          	 dd| it          |fi |¤Ž¤ŽS )	z"Swin-B V2 @ 256x256, window 16x16.rš   é€   rÛ  ©r+   r   rš   é    rÓ  Úswinv2_base_window16_256r²  )rä  rÕ  rÖ  s      r8   rä  rä  Ù  ó\   € õ  "°¸MÐUcÐdÑdÔd€JÝ&Ø"ðYð YØ/9ðYÝ=AÀ*Ð=WÐ=WÐPVÐ=WÐ=WðYð Yð Yr:   c           	      ó\   — t          dddd¬¦  «        }t          	 dd| it          |fi |¤Ž¤ŽS )	z Swin-B V2 @ 256x256, window 8x8.r   rá  rÛ  râ  rÓ  Úswinv2_base_window8_256r²  )rç  rÕ  rÖ  s      r8   rç  rç  á  s[   € õ  !¨s¸=ÐTbÐcÑcÔc€JÝ&Ø!ðXð XØ.8ðXÝ<@ÀÐ<VÐ<VÈvÐ<VÐ<VðXð Xð Xr:   c           	      ó\   — t          dddd¬¦  «        }t          	 dd| it          |fi |¤Ž¤ŽS )	z"Swin-B V2 @ 192x192, window 12x12.r4  rá  rÛ  râ  rÓ  Úswinv2_base_window12_192r²  )ré  rÕ  rÖ  s      r8   ré  ré  é  rå  r:   c           	      ó^   — t          ddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤ŽS )
zQSwin-B V2 @ 192x192, trained at window 12x12, fine-tuned to 256x256 window 16x16.rš   rá  rÛ  râ  ©r4  r4  r4  r2  ©r&   r:  r;  rC   rA  Ú!swinv2_base_window12to16_192to256r²  )rí  rÕ  rÖ  s      r8   rí  rí  ñ  óg   € õ Ø #¨mÀ~Ø /ð1ñ 1ô 1€Jõ 'Ø+ðbð bØ8BðbÝFJÈ:ÐF`ÐF`ÐY_ÐF`ÐF`ðbð bð br:   c           	      ó^   — t          ddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤ŽS )
zQSwin-B V2 @ 192x192, trained at window 12x12, fine-tuned to 384x384 window 24x24.r5  rá  rÛ  râ  rë  rì  Ú!swinv2_base_window12to24_192to384r²  )rð  rÕ  rÖ  s      r8   rð  rð  û  rî  r:   c           	      ó\   — t          dddd¬¦  «        }t          	 dd| it          |fi |¤Ž¤ŽS )	z"Swin-L V2 @ 192x192, window 12x12.r4  rË  rÛ  ©r2  r4  r5  é0   rÓ  Úswinv2_large_window12_192r²  )rô  rÕ  rÖ  s      r8   rô  rô    s\   € õ  "°¸MÐUdÐeÑeÔe€JÝ&Ø#ðZð ZØ0:ðZÝ>BÀ:Ð>XÐ>XÐQWÐ>XÐ>XðZð Zð Zr:   c           	      ó^   — t          ddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤ŽS )
zQSwin-L V2 @ 192x192, trained at window 12x12, fine-tuned to 256x256 window 16x16.rš   rË  rÛ  rò  rë  rì  Ú"swinv2_large_window12to16_192to256r²  )rö  rÕ  rÖ  s      r8   rö  rö    óg   € õ Ø #¨mÀØ /ð1ñ 1ô 1€Jõ 'Ø,ðcð cØ9CðcÝGKÈJÐGaÐGaÐZ`ÐGaÐGaðcð cð cr:   c           	      ó^   — t          ddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤ŽS )
zQSwin-L V2 @ 192x192, trained at window 12x12, fine-tuned to 384x384 window 24x24.r5  rË  rÛ  rò  rë  rì  Ú"swinv2_large_window12to24_192to384r²  )rù  rÕ  rÖ  s      r8   rù  rù    r÷  r:   rÐ  rÌ  rÍ  rÑ  rÎ  rÏ  )Úswinv2_base_window12_192_22kÚ)swinv2_base_window12to16_192to256_22kft1kÚ)swinv2_base_window12to24_192to384_22kft1kÚswinv2_large_window12_192_22kÚ*swinv2_large_window12to16_192to256_22kft1kÚ*swinv2_large_window12to24_192to384_22kft1krœ  )rC  )Or´   rt   Ú	functoolsr   Útypingr   r   r   r   r   r	   r
   r   r   rZ   Útorch.nnrX   Útorch.nn.functionalró   rœ   Ú	timm.datar   r   Útimm.layersr   r   r   r   r   r   r   r   r   r   r   r   Ú_builderr   Ú	_featuresr   Ú_features_fxr   Ú_manipulater    Ú	_registryr!   r"   r#   Ú__all__rµ   r  r¸   r9   r=   r  r?   r¼   r  r  r$   r+  r°  r¶   r»  rÈ  Údefault_cfgsrÔ  rÙ  rÝ  rß  rä  rç  ré  rí  rð  rô  rö  rù  r±   rÍ   r:   r8   ú<module>r     s¤  ððð ð €€€Ø Ð Ð Ð Ð Ð Ø OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ Oà €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð à AÐ AÐ AÐ AÐ AÐ AÐ AÐ AðMð Mð Mð Mð Mð Mð Mð Mð Mð Mð Mð Mð Mð Mð Mð Mð Mð Mð Mð Mð Mð Mð Mð Mð Mð Mð Mð Mà *Ð *Ð *Ð *Ð *Ð *Ø +Ð +Ð +Ð +Ð +Ð +Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø #Ð #Ð #Ð #Ð #Ð #Ø YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YàÐ
€à˜#˜u S¨# XœÐ.Ô/Ð ðØŒ<ðà˜3 ˜8”_ðð „\ðð ð ð ð& ðØ”ðà˜3 ˜8”_ðð ˜˜S˜”/ðð „\ð	ð ð ñ Ôðð,Qð Qð Qð Qð Q�b”iñ Qô Qð Qðhpð pð pð pð p˜RœYñ pô pð pðf&ð &ð &ð &ð &�2”9ñ &ô &ð &ðRM3ð M3ð M3ð M3ð M3˜RœYñ M3ô M3ð M3ð`bð bð bð bð b˜œ	ñ bô bð bðJ' T¨#¨u¬|Ð*;Ô%<ð 'ÀRÄYð 'ÐSWÐX[Ð]bÔ]iÐXiÔSjð 'ð 'ð 'ð 'ðTð ¨ð ¸$ð ÐUfð ð ð ð ð,ð ð ð ð %Ð$Ø:>¸$Øð Jð;ñ ;ô ;ð ;?¸$Øð JØ ¨H¸sð;ñ ;ô ;ð
 <@¸4Øð Kð<ñ <ô <ð <@¸4Øð KØ ¨H¸sð<ñ <ô <ð (, tØØtð(ñ (ô (ð )-¨ØØuð)ñ )ô )ð )-¨ØØuð)ñ )ô )ð *.¨ØØvð*ñ *ô *ð (, tØØtð(ñ (ô (ð )-¨ØØuð)ñ )ô )ð
 *.¨ØØyØ m¸vð*ñ *ô *ð
 +/¨$ØØzØ m¸vð+ñ +ô +ðe7&ð 7&ñ 7ô 7€ðt ðYð Y¨ð YÐDUð Yð Yð Yñ „ðYð ðXð X¨ð XÐCTð Xð Xð Xñ „ðXð ðZð Z¨$ð ZÐEVð Zð Zð Zñ „ðZð ðYð Y¨ð YÐDUð Yð Yð Yñ „ðYð ðYð Y¨ð YÐDUð Yð Yð Yñ „ðYð ðXð X¨ð XÐCTð Xð Xð Xñ „ðXð ðYð Y¨ð YÐDUð Yð Yð Yñ „ðYð ðbð b°$ð bÐM^ð bð bð bñ „ðbð ðbð b°$ð bÐM^ð bð bð bñ „ðbð ðZð Z¨$ð ZÐEVð Zð Zð Zñ „ðZð ðcð c°4ð cÐN_ð cð cð cñ „ðcð ðcð c°4ð cÐN_ð cð cð cñ „ðcð Ð ˜HØ$GØ1eØ1eØ%IØ2gØ2gð'ð 'ñ ô ð ð ð r:   