§
    ·¯¸j>€  ã                   ó   — d Z ddlmZ ddlmZmZmZmZmZ ddl	Z	ddl
mZ ddlmc mZ ddl	mZ ddlmZmZ ddlmZmZmZmZmZmZmZmZmZ ddlmZmZ d	d
lm Z  d	dl!m"Z" d	dl#m$Z$ d	dl%m&Z&m'Z' d	dl(m)Z)m*Z* dgZ+ G d„ dej,        ¦  «        Z- G d„ dej,        ¦  «        Z. G d„ dej,        ¦  «        Z/ G d„ dej,        ¦  «        Z0 G d„ dej,        ¦  «        Z1 G d„ dej,        ¦  «        Z2dedee3e3f         fd„Z4e$dedee3e3f         d e3d!e3fd"„¦   «         Z5 G d#„ d$ej,        ¦  «        Z6 G d%„ d&ej,        ¦  «        Z7 G d'„ d(ej,        ¦  «        Z8 G d)„ dej,        ¦  «        Z9dAd+„Z:d,„ Z;dBd.„Z<dCd0„Z= e) e=d1¬2¦  «         e=d1¬2¦  «         e=d1¬2¦  «         e=¦   «          e=¦   «          e=¦   «          e=d3dd4¬5¦  «         e=d6dd4¬5¦  «        d7œ¦  «        Z>e*dBd8e9fd9„¦   «         Z?e*dBd8e9fd:„¦   «         Z@e*dBd8e9fd;„¦   «         ZAe*dBd8e9fd<„¦   «         ZBe*dBd8e9fd=„¦   «         ZCe*dBd8e9fd>„¦   «         ZDe*dBd8e9fd?„¦   «         ZEe*dBd8e9fd@„¦   «         ZFdS )Daf   DaViT: Dual Attention Vision Transformers

As described in https://arxiv.org/abs/2204.03645

Input size invariant transformer architecture that combines channel and spacial
attention in each block. The attention mechanisms used are linear in complexity.

DaViT model defs and weights adapted from https://github.com/dingmyu/davit, original copyright below

é    )Úpartial)ÚListÚOptionalÚTupleÚTypeÚUnionN)ÚTensor©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)	ÚDropPathÚcalculate_drop_path_ratesÚ	to_2tupleÚtrunc_normal_ÚMlpÚLayerNorm2dÚget_norm_layerÚuse_fused_attnÚget_device_dtype)ÚNormMlpClassifierHeadÚClassifierHeadé   )Úbuild_model_with_cfg)Úfeature_take_indices)Úregister_notrace_function)Ú
checkpointÚcheckpoint_seq)Úgenerate_default_cfgsÚregister_modelÚDaVitc                   óB   ‡ — e Zd Z	 	 	 	 d
dededefˆ fd„Zdefd	„Zˆ xZS )Ú
ConvPosEncé   FNÚdimÚkÚactc                 óì   •— ||dœ}t          ¦   «                              ¦   «          t          j        ||f|d|dz  |dœ|¤Ž| _        |rt          j        ¦   «         nt          j        ¦   «         | _        d S )N©ÚdeviceÚdtyper   é   )Úkernel_sizeÚstrideÚpaddingÚgroups)ÚsuperÚ__init__ÚnnÚConv2dÚprojÚGELUÚIdentityr&   )Úselfr$   r%   r&   r)   r*   ÚddÚ	__class__s          €úa/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/models/davit.pyr1   zConvPosEnc.__init__#   s‰   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐå”IØØð
ð ØØ˜‘FØð
ð 
ð ð
ð 
ˆŒ	ð !$Ð6•2”7‘9”9�9­¬©¬ˆŒˆˆó    Úxc                 ó`   — |                       |¦  «        }||                      |¦  «        z   }|S ©N)r4   r&   )r7   r<   Úfeats      r:   ÚforwardzConvPosEnc.forward9   s*   € Ø�yŠy˜‰|Œ|ˆØ�—’˜‘”ÑˆØˆr;   )r#   FNN©	Ú__name__Ú
__module__Ú__qualname__ÚintÚboolr1   r	   r@   Ú__classcell__©r9   s   @r:   r"   r"   "   s‡   ø€ € € € € ð ØØØð7ð 7àð7ð ð7ð ð	7ð 7ð 7ð 7ð 7ð 7ð,˜ð ð ð ð ð ð ð ð r;   r"   c            
       ód   ‡ — e Zd ZdZdddeddfdededed	eej                 fˆ fd
„Z	de
fd„Zˆ xZS )ÚStemzŒ Size-agnostic implementation of 2D image to patch embedding,
        allowing input size to be adjusted during model forward operation
    r#   é`   é   NÚin_chsÚout_chsr-   Ú
norm_layerc                 ó
  •— ||dœ}t          ¦   «                              ¦   «          t          |¦  «        }|| _        || _        || _        |d         dk    sJ ‚t          j        ||fd|ddœ|¤Ž| _         ||fi |¤Ž| _	        d S )Nr(   r   rL   é   r#   ©r,   r-   r.   )
r0   r1   r   r-   rM   rN   r2   r3   ÚconvÚnorm)	r7   rM   rN   r-   rO   r)   r*   r8   r9   s	           €r:   r1   zStem.__init__D   s²   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ˜6Ñ"Ô"ˆØˆŒØˆŒØˆŒØ�aŒy˜AŠ~ˆ~ˆ~ˆ~Ý”IØØð
ð ØØð
ð 
ð ð
ð 
ˆŒ	ð �J˜wÐ-Ð-¨"Ð-Ð-ˆŒ	ˆ	ˆ	r;   r<   c                 óT  — |j         \  }}}}| j        d         || j        d         z  z
  | j        d         z  }| j        d         || j        d         z  z
  | j        d         z  }t          j        |d|d|f¦  «        }|                      |¦  «        }|                      |¦  «        }|S )Nr   r   )Úshaper-   ÚFÚpadrS   rT   )r7   r<   ÚBÚCÚHÚWÚpad_rÚpad_bs           r:   r@   zStem.forward^   sš   € Ø”W‰
ˆˆ1ˆa�Ø”˜Q” ! d¤k°!¤nÑ"4Ñ4¸¼ÀA¼ÑFˆØ”˜Q” ! d¤k°!¤nÑ"4Ñ4¸¼ÀA¼ÑFˆÝŒE�!�a˜  5Ð)Ñ*Ô*ˆØ�IŠI�a‰LŒLˆØ�IŠI�a‰LŒLˆØˆr;   )rB   rC   rD   Ú__doc__r   rE   r   r2   ÚModuler1   r	   r@   rG   rH   s   @r:   rJ   rJ   ?   s§   ø€ € € € € ðð ð ØØØ*5ØØð.ð .àð.ð ð.ð ð	.ð
 ˜RœYœð.ð .ð .ð .ð .ð .ð4˜ð ð ð ð ð ð ð ð r;   rJ   c            
       ó\   ‡ — e Zd Zdeddfdedededeej                 fˆ fd„Zde	fd	„Z
ˆ xZS )
Ú
Downsampler#   NrM   rN   r,   rO   c                 óö   •— ||dœ}t          ¦   «                              ¦   «          || _        || _         ||fi |¤Ž| _        |dz  dk    | _        t          j        ||f|d| j        rdn|dz  dœ|¤Ž| _        d S )Nr(   r+   r   rR   )	r0   r1   rM   rN   rT   Úeven_kr2   r3   rS   )	r7   rM   rN   r,   rO   r)   r*   r8   r9   s	           €r:   r1   zDownsample.__init__i   s«   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØˆŒØˆŒà�J˜vÐ,Ð,¨Ð,Ð,ˆŒ	Ø! A‘o¨Ò*ˆŒÝ”IØØð
ð $ØØœÐ:�A�A¨+¸Ñ*:ð
ð 
ð ð
ð 
ˆŒ	ˆ	ˆ	r;   r<   c                 óü   — |j         \  }}}}|                      |¦  «        }| j        r>| j        j        \  }}|||z  z
  |z  }|||z  z
  |z  }	t          j        |d|d|	f¦  «        }|                      |¦  «        }|S )Nr   )rV   rT   rd   rS   r,   rW   rX   )
r7   r<   rY   rZ   r[   r\   Úk_hÚk_wr]   r^   s
             r:   r@   zDownsample.forward‚   sŠ   € Ø”W‰
ˆˆ1ˆa�Ø�IŠI�a‰LŒLˆØŒ;ð 	0Ø”yÔ,‰HˆC�Ø˜1˜s™7‘] cÑ)ˆEØ˜1˜s™7‘] cÑ)ˆEÝ”�a˜!˜U Q¨Ð.Ñ/Ô/ˆAØ�IŠI�a‰LŒLˆØˆr;   )rB   rC   rD   r   rE   r   r2   r`   r1   r	   r@   rG   rH   s   @r:   rb   rb   h   s—   ø€ € € € € ð
  !Ø*5ØØð
ð 
àð
ð ð
ð ð	
ð
 ˜RœYœð
ð 
ð 
ð 
ð 
ð 
ð2	˜ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	r;   rb   c            	       óB   ‡ — e Zd Z	 	 	 	 	 d
dedededefˆ fd„Zd	„ Zˆ xZS )ÚChannelAttentionV2é   TNr$   Ú	num_headsÚqkv_biasÚdynamic_scalec                 óì   •— ||dœ}t          ¦   «                              ¦   «          || _        ||z  | _        || _        t          j        ||dz  fd|i|¤Ž| _        t          j        ||fi |¤Ž| _        d S )Nr(   r#   Úbias)	r0   r1   r/   Úhead_dimrm   r2   ÚLinearÚqkvr4   )	r7   r$   rk   rl   rm   r)   r*   r8   r9   s	           €r:   r1   zChannelAttentionV2.__init__�   s†   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØˆŒØ˜yÑ(ˆŒØ*ˆÔå”9˜S #¨¡'Ð?Ð?°Ð?¸BÐ?Ð?ˆŒÝ”I˜c 3Ð-Ð-¨"Ð-Ð-ˆŒ	ˆ	ˆ	r;   c                 ó`  — |j         \  }}}|                      |¦  «                             ||d| j        || j        z  ¦  «                             ddddd¦  «        }|                     d¦  «        \  }}}| j        r	||dz  z  }n|| j        dz  z  }|                     dd¦  «        |z  }	|	 	                    d¬	¦  «        }	|	|                     dd¦  «        z                       dd¦  «        }|                     dd¦  «                             |||¦  «        }|  
                    |¦  «        }|S )
Nr#   r+   r   r   rL   ç      à¿éÿÿÿÿéþÿÿÿ©r$   )rV   rr   Úreshaper/   ÚpermuteÚunbindrm   rp   Ú	transposeÚsoftmaxr4   ©
r7   r<   rY   ÚNrZ   rr   Úqr%   ÚvÚattns
             r:   r@   zChannelAttentionV2.forward¢   s$  € Ø”'‰ˆˆ1ˆaà�hŠh�q‰kŒk×!Ò! ! Q¨¨4¬;¸¸T¼[Ñ8HÑIÔI×QÒQÐRSÐUVÐXYÐ[\Ð^_Ñ`Ô`ˆØ—*’*˜Q‘-”-‰ˆˆ1ˆaàÔð 	*Ø�A˜‘I‘ˆAˆAà�D”M TÑ)Ñ)ˆAØ�{Š{˜2˜rÑ"Ô" QÑ&ˆØ�|Š| ˆ|Ñ#Ô#ˆØ�A—K’K  BÑ'Ô'Ñ'×2Ò2°2°rÑ:Ô:ˆà�KŠK˜˜1ÑÔ×%Ò% a¨¨AÑ.Ô.ˆØ�IŠI�a‰LŒLˆØˆr;   )rj   TTNN)rB   rC   rD   rE   rF   r1   r@   rG   rH   s   @r:   ri   ri   Ž   sˆ   ø€ € € € € ð
 Ø!Ø"&ØØð.ð .àð.ð ð.ð ð	.ð
  ð.ð .ð .ð .ð .ð .ð$ð ð ð ð ð ð r;   ri   c                   óB   ‡ — e Zd Z	 	 	 	 d
dededefˆ fd„Zdefd	„Zˆ xZS )ÚChannelAttentionrj   FNr$   rk   rl   c                 óè   •— ||dœ}t          ¦   «                              ¦   «          || _        ||z  }|dz  | _        t	          j        ||dz  fd|i|¤Ž| _        t	          j        ||fi |¤Ž| _        d S )Nr(   rt   r#   ro   )r0   r1   rk   Úscaler2   rq   rr   r4   )	r7   r$   rk   rl   r)   r*   r8   rp   r9   s	           €r:   r1   zChannelAttention.__init__¸   sˆ   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ"ˆŒØ˜)Ñ#ˆØ Ñ%ˆŒ
å”9˜S #¨¡'Ð?Ð?°Ð?¸BÐ?Ð?ˆŒÝ”I˜c 3Ð-Ð-¨"Ð-Ð-ˆŒ	ˆ	ˆ	r;   r<   c                 ó:  — |j         \  }}}|                      |¦  «                             ||d| j        || j        z  ¦  «                             ddddd¦  «        }|                     d¦  «        \  }}}|| j        z  }|                     dd¦  «        |z  }	|	                     d¬¦  «        }	|	|                     dd¦  «        z                       dd¦  «        }|                     dd¦  «                             |||¦  «        }|  	                    |¦  «        }|S )	Nr#   r+   r   r   rL   ru   rv   rw   )
rV   rr   rx   rk   ry   rz   r…   r{   r|   r4   r}   s
             r:   r@   zChannelAttention.forwardÉ   s  € Ø”'‰ˆˆ1ˆaà�hŠh�q‰kŒk×!Ò! ! Q¨¨4¬>¸1ÀÄÑ;NÑOÔO×WÒWÐXYÐ[\Ð^_ÐabÐdeÑfÔfˆØ—*’*˜Q‘-”-‰ˆˆ1ˆaà�”
‰NˆØ�{Š{˜2˜rÑ"Ô" QÑ&ˆØ�|Š| ˆ|Ñ#Ô#ˆØ�A—K’K  BÑ'Ô'Ñ'×2Ò2°2°rÑ:Ô:ˆØ�KŠK˜˜1ÑÔ×%Ò% a¨¨AÑ.Ô.ˆØ�IŠI�a‰LŒLˆØˆr;   )rj   FNNrA   rH   s   @r:   rƒ   rƒ   ¶   s‡   ø€ € € € € ð
 Ø"ØØð.ð .àð.ð ð.ð ð	.ð .ð .ð .ð .ð .ð"˜ð ð ð ð ð ð ð ð r;   rƒ   c                   óª   ‡ — e Zd Zdddej        ej        dddddf
dededed	ed
ede	ej
                 de	ej
                 dededefˆ fd„Zdefd„Zˆ xZS )ÚChannelBlockç      @Fç        TNr$   rk   Ú	mlp_ratiorl   Ú	drop_pathÚ	act_layerrO   ÚffnÚcpe_actÚv2c                 óh  •— ||dœ}t          ¦   «                              ¦   «          t          d|d|	dœ|¤Ž| _        || _         ||fi |¤Ž| _        |
rt          nt          } ||f||dœ|¤Ž| _        |dk    rt          |¦  «        nt          j        ¦   «         | _        t          d|d|	dœ|¤Ž| _        | j        rb ||fi |¤Ž| _        t          d|t!          ||z  ¦  «        |dœ|¤Ž| _        |dk    rt          |¦  «        nt          j        ¦   «         | _        d S d | _        d | _        d | _        d S ©Nr(   r#   )r$   r%   r&   )rk   rl   rŠ   )Úin_featuresÚhidden_featuresr�   © )r0   r1   r"   Úcpe1rŽ   Únorm1ri   rƒ   r�   r   r2   r6   Ú
drop_path1Úcpe2Únorm2r   rE   ÚmlpÚ
drop_path2)r7   r$   rk   r‹   rl   rŒ   r�   rO   rŽ   r�   r�   r)   r*   r8   Ú
attn_layerr9   s                  €r:   r1   zChannelBlock.__init__Ú   s‡  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐåÐ? 3¨!°Ð?Ð?¸BÐ?Ð?ˆŒ	ØˆŒØ�Z Ð*Ð* rÐ*Ð*ˆŒ
Ø+-ÐCÕ'Ð'Õ3Cˆ
Ø�JØð
àØð
ð 
ð ð	
ð 
ˆŒ	ð 2;¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒÝÐ? 3¨!°Ð?Ð?¸BÐ?Ð?ˆŒ	àŒ8ð 	#Ø#˜ CÐ.Ð.¨2Ð.Ð.ˆDŒJÝð ØÝ # C¨)¡OÑ 4Ô 4Ø#ðð ð ð	ð ˆDŒHð 6?À²^°^�h yÑ1Ô1Ð1ÍÌÉÌˆDŒOˆOˆOàˆDŒJØˆDŒHØ"ˆDŒOˆOˆOr;   r<   c                 óÌ  — |j         \  }}}}|                      |¦  «                             d¦  «                             dd¦  «        }|                      |¦  «        }|                      |¦  «        }||                      |¦  «        z   }|                      |                     dd¦  «                             ||||¦  «        ¦  «        }| j	        �“|                     d¦  «                             dd¦  «        }||  
                    |  	                    |                      |¦  «        ¦  «        ¦  «        z   }|                     dd¦  «                             ||||¦  «        }|S )Nr+   r   )rV   r–   Úflattenr{   r—   r�   r˜   r™   Úviewr›   rœ   rš   )r7   r<   rY   rZ   r[   r\   Úcurs          r:   r@   zChannelBlock.forward  s$  € Ø”W‰
ˆˆ1ˆa�à�IŠI�a‰LŒL× Ò  Ñ#Ô#×-Ò-¨a°Ñ3Ô3ˆà�jŠj˜‰mŒmˆØ�iŠi˜‰nŒnˆØ�—’ Ñ$Ô$Ñ$ˆà�IŠI�a—k’k ! QÑ'Ô'×,Ò,¨Q°°1°aÑ8Ô8Ñ9Ô9ˆàŒ8ÐØ—	’	˜!‘”×&Ò& q¨!Ñ,Ô,ˆAØ�D—O’O D§H¢H¨T¯ZªZ¸©]¬]Ñ$;Ô$;Ñ<Ô<Ñ<ˆAØ—’˜A˜qÑ!Ô!×&Ò& q¨!¨Q°Ñ2Ô2ˆAàˆr;   )rB   rC   rD   r2   r5   Ú	LayerNormrE   ÚfloatrF   r   r`   r1   r	   r@   rG   rH   s   @r:   rˆ   rˆ   Ø   sð   ø€ € € € € ð  "Ø"Ø!Ø)+¬Ø*,¬,ØØ!ØØØð+#ð +#àð+#ð ð+#ð ð	+#ð
 ð+#ð ð+#ð ˜BœI”ð+#ð ˜RœYœð+#ð ð+#ð ð+#ð ð+#ð +#ð +#ð +#ð +#ð +#ðZ˜ð ð ð ð ð ð ð ð r;   rˆ   r<   Úwindow_sizec                 ó*  — | j         \  }}}}|                      |||d         z  |d         ||d         z  |d         |¦  «        } |                      dddddd¦  «                             ¦   «                              d|d         |d         |¦  «        }|S )z™
    Args:
        x: (B, H, W, C)
        window_size (int): window size
    Returns:
        windows: (num_windows*B, window_size, window_size, C)
    r   r   r#   r+   rL   é   ru   ©rV   r    ry   Ú
contiguous)r<   r¤   rY   r[   r\   rZ   Úwindowss          r:   Úwindow_partitionrª     s–   € ð ”�J€A€qˆ!ˆQØ	�Šˆq�!�{ 1”~Ñ% {°1¤~°q¸KÈ¼NÑ7JÈKÐXYÌNÐ\]Ñ^Ô^€AØ�iŠi˜˜1˜a  A qÑ)Ô)×4Ò4Ñ6Ô6×;Ò;¸BÀÈAÄÐP[Ð\]ÔP^Ð`aÑbÔb€GØ€Nr;   r©   r[   r\   c                 ó  — | j         d         }|                      d||d         z  ||d         z  |d         |d         |¦  «        }|                     dddddd¦  «                             ¦   «                              d|||¦  «        }|S )zÚ
    Args:
        windows: (num_windows*B, window_size, window_size, C)
        window_size (int): Window size
        H (int): Height of image
        W (int): Width of image
    Returns:
        x: (B, H, W, C)
    ru   r   r   r#   r+   rL   r¦   r§   )r©   r¤   r[   r\   rZ   r<   s         r:   Úwindow_reverser¬   (  s‰   € ð 	Œ�bÔ€AØ�Š�R˜˜k¨!œnÑ,¨a°;¸q´>Ñ.AÀ;ÈqÄ>ÐS^Ð_`ÔSaÐcdÑeÔe€AØ	�	Š	�!�Q˜˜1˜a Ñ#Ô#×.Ò.Ñ0Ô0×5Ò5°b¸!¸QÀÑBÔB€AØ€Hr;   c            	       ó„   ‡ — e Zd ZU dZej        j        e         ed<   	 	 	 dde	de
e	e	f         de	defˆ fd	„Zd
efd„Zˆ xZS )ÚWindowAttentiona§   Window based multi-head self attention (W-MSA) module with relative position bias.
    It supports both of shifted and non-shifted window.
    Args:
        dim (int): Number of input channels.
        window_size (tuple[int]): The height and width of the window.
        num_heads (int): Number of attention heads.
        qkv_bias (bool, optional):  If True, add a learnable bias to query, key, value. Default: True
    Ú
fused_attnTNr$   r¤   rk   rl   c                 ó^  •— ||dœ}t          ¦   «                              ¦   «          || _        || _        || _        ||z  }|dz  | _        t          ¦   «         | _        t          j	        ||dz  fd|i|¤Ž| _
        t          j	        ||fi |¤Ž| _        t          j        d¬¦  «        | _        d S )Nr(   rt   r#   ro   ru   rw   )r0   r1   r$   r¤   rk   r…   r   r¯   r2   rq   rr   r4   ÚSoftmaxr|   )
r7   r$   r¤   rk   rl   r)   r*   r8   rp   r9   s
            €r:   r1   zWindowAttention.__init__D  s¸   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØˆŒØ&ˆÔØ"ˆŒØ˜)Ñ#ˆØ Ñ%ˆŒ
Ý(Ñ*Ô*ˆŒå”9˜S #¨¡'Ð?Ð?°Ð?¸BÐ?Ð?ˆŒÝ”I˜c 3Ð-Ð-¨"Ð-Ð-ˆŒ	å”z bÐ)Ñ)Ô)ˆŒˆˆr;   r<   c                 ó$  — |j         \  }}}|                      |¦  «                             ||d| j        || j        z  ¦  «                             ddddd¦  «        }|                     d¦  «        \  }}}| j        rt          j        |||¦  «        }n=|| j	        z  }|| 
                    dd¦  «        z  }	|                      |	¦  «        }	|	|z  }| 
                    dd¦  «                             |||¦  «        }|                      |¦  «        }|S )Nr#   r+   r   r   rL   rv   ru   )rV   rr   rx   rk   ry   rz   r¯   rW   Úscaled_dot_product_attentionr…   r{   r|   r4   )
r7   r<   ÚB_r~   rZ   rr   r   r%   r€   r�   s
             r:   r@   zWindowAttention.forward[  s  € Ø”7‰ˆˆAˆqà�hŠh�q‰kŒk×!Ò! " a¨¨D¬N¸AÀÄÑ<OÑPÔP×XÒXÐYZÐ\]Ð_`ÐbcÐefÑgÔgˆØ—*’*˜Q‘-”-‰ˆˆ1ˆaàŒ?ð 	ÝÔ.¨q°!°QÑ7Ô7ˆAˆAà�D”J‘ˆAØ˜Ÿš B¨Ñ+Ô+Ñ+ˆDØ—<’< Ñ%Ô%ˆDØ�q‘ˆAà�KŠK˜˜1ÑÔ×%Ò% b¨!¨QÑ/Ô/ˆØ�IŠI�a‰LŒLˆØˆr;   )TNN)rB   rC   rD   r_   ÚtorchÚjitÚFinalrF   Ú__annotations__rE   r   r1   r	   r@   rG   rH   s   @r:   r®   r®   9  s¹   ø€ € € € € € ðð ð ”	” Ô%Ð%Ð%Ñ%ð "ØØð*ð *àð*ð ˜s C˜xœð*ð ð	*ð
 ð*ð *ð *ð *ð *ð *ð.˜ð ð ð ð ð ð ð ð r;   r®   c                   ó®   ‡ — e Zd ZdZddddej        ej        ddddf
ded	ed
edede	dede
ej                 de
ej                 de	de	fˆ fd„Zdefd„Zˆ xZS )ÚSpatialBlocka<   Windows Block.
    Args:
        dim (int): Number of input channels.
        num_heads (int): Number of attention heads.
        window_size (int): Window size.
        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
        qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True
        drop_path (float, optional): Stochastic depth rate. Default: 0.0
        act_layer (nn.Module, optional): Activation layer. Default: nn.GELU
        norm_layer (nn.Module, optional): Normalization layer.  Default: nn.LayerNorm
    rQ   r‰   TrŠ   FNr$   rk   r¤   r‹   rl   rŒ   r�   rO   rŽ   r�   c                 ó²  •— ||dœ}t          ¦   «                              ¦   «          || _        |	| _        || _        t          |¦  «        | _        || _        t          d|d|
dœ|¤Ž| _	         ||fi |¤Ž| _
        t          || j        f||dœ|¤Ž| _        |dk    rt          |¦  «        nt          j        ¦   «         | _        t          d|d|
dœ|¤Ž| _        | j        rd ||fi |¤Ž| _        t'          ||z  ¦  «        }t)          d|||dœ|¤Ž| _        |dk    rt          |¦  «        nt          j        ¦   «         | _        d S d | _        d | _        d | _        d S r’   )r0   r1   r$   rŽ   rk   r   r¤   r‹   r"   r–   r—   r®   r�   r   r2   r6   r˜   r™   rš   rE   r   r›   rœ   )r7   r$   rk   r¤   r‹   rl   rŒ   r�   rO   rŽ   r�   r)   r*   r8   Úmlp_hidden_dimr9   s                  €r:   r1   zSpatialBlock.__init__{  s¦  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØˆŒØˆŒØ"ˆŒÝ$ [Ñ1Ô1ˆÔØ"ˆŒåÐ? 3¨!°Ð?Ð?¸BÐ?Ð?ˆŒ	Ø�Z Ð*Ð* rÐ*Ð*ˆŒ
Ý#ØØÔð
ð  Øð	
ð 
ð
 ð
ð 
ˆŒ	ð 2;¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒåÐ? 3¨!°Ð?Ð?¸BÐ?Ð?ˆŒ	ØŒ8ð 	#Ø#˜ CÐ.Ð.¨2Ð.Ð.ˆDŒJÝ   y¡Ñ1Ô1ˆNÝð ØØ .Ø#ðð ð ð	ð ˆDŒHð 6?À²^°^�h yÑ1Ô1Ð1ÍÌÉÌˆDŒOˆOˆOàˆDŒJØˆDŒHØ"ˆDŒOˆOˆOr;   r<   c           	      ó–  — |j         \  }}}}|                      |¦  «                             d¦  «                             dd¦  «        }|                      |¦  «        }|                     ||||¦  «        }dx}}| j        d         || j        d         z  z
  | j        d         z  }	| j        d         || j        d         z  z
  | j        d         z  }
t          j        |dd||	||
f¦  «        }|j         \  }}}}t          || j        ¦  «        }|                     d| j        d         | j        d         z  |¦  «        }|  
                    |¦  «        }|                     d| j        d         | j        d         |¦  «        }t          || j        ||¦  «        }|d d …d |…d |…d d …f                              ¦   «         }|                     |||z  |¦  «        }||                      |¦  «        z   }|                      |                     dd¦  «                             ||||¦  «        ¦  «        }| j        �“|                     d¦  «                             dd¦  «        }||                      |                      |                      |¦  «        ¦  «        ¦  «        z   }|                     dd¦  «                             ||||¦  «        }|S )Nr+   r   r   ru   )rV   r–   rŸ   r{   r—   r    r¤   rW   rX   rª   r�   r¬   r¨   r˜   r™   r›   rœ   rš   )r7   r<   rY   rZ   r[   r\   ÚshortcutÚpad_lÚpad_tr]   r^   Ú_ÚHpÚWpÚ	x_windowsÚattn_windowss                   r:   r@   zSpatialBlock.forward­  sš  € Ø”W‰
ˆˆ1ˆa�à—9’9˜Q‘<”<×'Ò'¨Ñ*Ô*×4Ò4°Q¸Ñ:Ô:ˆà�JŠJ�xÑ Ô ˆØ�FŠF�1�a˜˜AÑÔˆàÐˆ�ØÔ! !Ô$ q¨4Ô+;¸AÔ+>Ñ'>Ñ>À$ÔBRÐSTÔBUÑUˆØÔ! !Ô$ q¨4Ô+;¸AÔ+>Ñ'>Ñ>À$ÔBRÐSTÔBUÑUˆÝŒE�!�a˜˜E 5¨%°Ð7Ñ8Ô8ˆØ”w‰ˆˆ2ˆr�1å$ Q¨Ô(8Ñ9Ô9ˆ	Ø—N’N 2 tÔ'7¸Ô':¸TÔ=MÈaÔ=PÑ'PÐRSÑTÔTˆ	ð —y’y Ñ+Ô+ˆð $×(Ò(¨¨TÔ-=¸aÔ-@À$ÔBRÐSTÔBUÐWXÑYÔYˆÝ˜<¨Ô)9¸2¸rÑBÔBˆð ˆaˆaˆa��!��R�a�R˜˜˜ˆlŒO×&Ò&Ñ(Ô(ˆà�FŠF�1�a˜!‘e˜QÑÔˆØ�t—’ qÑ)Ô)Ñ)ˆà�IŠI�a—k’k ! QÑ'Ô'×,Ò,¨Q°°1°aÑ8Ô8Ñ9Ô9ˆàŒ8ÐØ—	’	˜!‘”×&Ò& q¨!Ñ,Ô,ˆAØ�D—O’O D§H¢H¨T¯ZªZ¸©]¬]Ñ$;Ô$;Ñ<Ô<Ñ<ˆAØ—’˜A˜qÑ!Ô!×&Ò& q¨!¨Q°Ñ2Ô2ˆAàˆr;   )rB   rC   rD   r_   r2   r5   r¢   rE   r£   rF   r   r`   r1   r	   r@   rG   rH   s   @r:   rº   rº   n  sú   ø€ € € € € ð
ð 
ð   !Ø!Ø!Ø!Ø)+¬Ø*,¬,ØØ!ØØð0#ð 0#àð0#ð ð0#ð ð	0#ð
 ð0#ð ð0#ð ð0#ð ˜BœI”ð0#ð ˜RœYœð0#ð ð0#ð ð0#ð 0#ð 0#ð 0#ð 0#ð 0#ðd%˜ð %ð %ð %ð %ð %ð %ð %ð %r;   rº   c            #       ó  ‡ — e Zd Zddddddddeej        ddd	ddd
d
fdededededee	df         dedede
dedee
df         deej                 deej                 dededededef"ˆ fd„Zej        j        d!d„¦   «         Zdefd „Zˆ xZS )"Ú
DaVitStager   T©ÚspatialÚchannelr#   rQ   r‰   )r   r   Fr+   NrM   rN   ÚdepthÚ
downsampleÚ
attn_types.rk   r¤   r‹   rl   Údrop_path_ratesrO   Únorm_layer_clrŽ   r�   Údown_kernel_sizeÚnamed_blocksÚchannel_attn_v2c                 óæ  •— ||dœ}t          ¦   «                              ¦   «          d| _        |rt          ||f||dœ|¤Ž| _        nt          j        ¦   «         | _        	 g }t          |¦  «        D ]ì}ddlm	} g }t          |¦  «        D ]t\  }}|dk    r2|                     dt          d||||	|
|         ||||dœ	|¤Žf¦  «         Œ=|d	k    r1|                     d
t          d||||	|
|         ||||dœ	|¤Žf¦  «         Œu|r1|                     t          j         ||¦  «        ¦  «        ¦  «         ŒÁ|                     t          j        d„ |D ¦   «         Ž ¦  «         Œít          j        |Ž | _        d S )Nr(   F)r,   rO   r   )ÚOrderedDictrÉ   Úspatial_block)	r$   rk   r‹   rl   rŒ   rO   rŽ   r�   r¤   rÊ   Úchannel_block)	r$   rk   r‹   rl   rŒ   rO   rŽ   r�   r�   c                 ó   — g | ]
}|d          ‘ŒS )r   r•   )Ú.0Úbs     r:   ú
<listcomp>z'DaVitStage.__init__.<locals>.<listcomp>   s   € Ð3WÐ3WÐ3W¸Q°A°a´DÐ3WÐ3WÐ3Wr;   r•   )r0   r1   Úgrad_checkpointingrb   rÌ   r2   r6   ÚrangeÚcollectionsrÔ   Ú	enumerateÚappendrº   rˆ   Ú
SequentialÚblocks)r7   rM   rN   rË   rÌ   rÍ   rk   r¤   r‹   rl   rÎ   rO   rÏ   rŽ   r�   rÐ   rÑ   rÒ   r)   r*   r8   Ústage_blocksÚ	block_idxrÔ   Údual_attention_blockÚattn_idxÚ	attn_typer9   s                              €r:   r1   zDaVitStage.__init__Ö  s   ø€ ð, ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐà"'ˆÔð ð 	,Ý(¨°ÐtÐFVÐcmÐtÐtÐqsÐtÐtˆDŒOˆOå œk™mœmˆDŒOð	ð ˆÝ˜u™œð !	Zð !	ZˆIØ/Ð/Ð/Ð/Ð/Ð/Ø#%Ð Ý'0°Ñ'<Ô'<ð ð Ñ#�˜)Ø 	Ò)Ð)Ø(×/Ò/°Å,ð CØ#Ø"+Ø"+Ø!)Ø"1°)Ô"<Ø#0ØØ 'Ø$/ðCð Cð ðCð Cð 1ñ ô ð ð ð  )Ò+Ð+Ø(×/Ò/°Å,ð CØ#Ø"+Ø"+Ø!)Ø"1°)Ô"<Ø#0ØØ 'Ø*ðCð Cð ðCð Cð 1ñ ô ð øð ð ZØ×#Ò#¥B¤M°+°+Ð>RÑ2SÔ2SÑ$TÔ$TÑUÔUÐUÐUà×#Ò#¥B¤MÐ3WÐ3WÐBVÐ3WÑ3WÔ3WÐ$XÑYÔYÐYÐYÝ”m \Ð2ˆŒˆˆr;   c                 ó   — || _         d S r>   )rÛ   )r7   Úenables     r:   Úset_grad_checkpointingz!DaVitStage.set_grad_checkpointing#  s   € à"(ˆÔÐÐr;   r<   c                 óÐ   — |                       |¦  «        }| j        r4t          j                             ¦   «         st          | j        |¦  «        }n|                      |¦  «        }|S r>   )rÌ   rÛ   rµ   r¶   Úis_scriptingr   rá   ©r7   r<   s     r:   r@   zDaVitStage.forward'  sZ   € Ø�OŠO˜AÑÔˆØÔ"ð 	­5¬9×+AÒ+AÑ+CÔ+Cð 	Ý˜tœ{¨AÑ.Ô.ˆAˆAà—’˜A‘”ˆAØˆr;   ©T)rB   rC   rD   r   r2   r¢   rE   rF   r   Ústrr£   r   r`   r1   rµ   r¶   Úignoreré   r	   r@   rG   rH   s   @r:   rÇ   rÇ   Õ  s˜  ø€ € € € € ð
 Ø#Ø*@ØØ Ø!Ø!Ø17Ø*5Ø-/¬\ØØ!Ø$%Ø!&Ø$)ØØð)K3ð K3àðK3ð ðK3ð ð	K3ð
 ðK3ð ˜c 3˜hœðK3ð ðK3ð ðK3ð ðK3ð ðK3ð # 5¨# :Ô.ðK3ð ˜RœYœðK3ð   ¤	œ?ðK3ð ðK3ð ðK3ð  "ð!K3ð" ð#K3ð$ "ð%K3ð K3ð K3ð K3ð K3ð K3ðZ „YÔð)ð )ð )ñ Ôð)ð˜ð ð ð ð ð ð ð ð r;   rÇ   c            +       ó¨  ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d@dedeedf         deedf         deedf         dededededededeedf         ded ed!ed"ed#ed$ed%ed&ed'ed(ef*ˆ fd)„Z	d*„ Z
ej        j        dAd+„¦   «         Zej        j        dBd,„¦   «         Zej        j        d-ej        fd.„¦   «         ZdCd&ed'ee         fd/„Z	 	 	 	 	 dDd1ej        d2eeeee         f                  d3ed4ed5ed6ed-eeej                 eej        eej                 f         f         fd7„Z	 	 	 dEd2eeee         f         d9ed:efd;„Zd<„ ZdAd=efd>„Zd?„ Zˆ xZS )Fr    aè   DaViT
        A PyTorch implementation of `DaViT: Dual Attention Vision Transformers`  - https://arxiv.org/abs/2204.03645
        Supports arbitrary input sizes and pyramid feature extraction

    Args:
        in_chans (int): Number of input image channels. Default: 3
        num_classes (int): Number of classes for classification head. Default: 1000
        depths (tuple(int)): Number of blocks in each stage. Default: (1, 1, 3, 1)
        embed_dims (tuple(int)): Patch embedding dimension. Default: (96, 192, 384, 768)
        num_heads (tuple(int)): Number of attention heads in different layers. Default: (3, 6, 12, 24)
        window_size (int): Window size. Default: 7
        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4
        qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True
        drop_path_rate (float): Stochastic depth rate. Default: 0.1
        norm_layer (nn.Module): Normalization layer. Default: nn.LayerNorm.
    r#   ©r   r   r#   r   ©rK   éÀ   é€  é   ©r#   é   é   é   rQ   rL   TÚlayernorm2dÚ	layernormçñhãˆµøä>rÈ   Fr+   rŠ   éè  ÚavgNÚin_chansÚdepths.Ú
embed_dimsrk   r¤   r‹   rl   rO   rÏ   Únorm_epsrÍ   rŽ   r�   rÐ   rÒ   rÑ   Ú	drop_rateÚdrop_path_rateÚnum_classesÚglobal_poolÚhead_norm_firstc                 óp  •— t          ¦   «                              ¦   «          ||dœ}t          |¦  «        }|t          |¦  «        cxk    rt          |¦  «        k    sn J ‚t          t	          |¦  «        |
¬¦  «        }t          t	          |	¦  «        |
¬¦  «        }	|| _        || _        |d         x| _        | _        || _	        d| _
        g | _        t          ||d         fd|i|¤Ž| _        |d         }t          ||d¬¦  «        }g }t          |¦  «        D ]}||         }t!          ||f||         |dk    |||         |||||         ||	|||||d	œ|¤Ž}|}|                     |¦  «         | xj        t%          |d
|d
z   z  d|› �¬¦  «        gz  c_        Œ€t'          j        |Ž | _        |r4 || j        fi |¤Ž| _        t/          | j        |f|| j	        dœ|¤Ž| _        n9t'          j        ¦   «         | _        t5          | j        |f|| j	        |dœ|¤Ž| _        |                      | j        ¦  «         d S )Nr(   )Úepsru   Fr   rO   T)Ú	stagewise)rË   rÌ   rÍ   rk   r¤   r‹   rl   rÎ   rO   rÏ   rŽ   r�   rÐ   rÒ   rÑ   r+   zstages.)Únum_chsÚ	reductionÚmodule)Ú	pool_typer  )r  r  rO   )r0   r1   Úlenr   r   r  rÿ   Únum_featuresÚhead_hidden_sizer  rÛ   Úfeature_inforJ   Ústemr   rÜ   rÇ   rß   Údictr2   rà   ÚstagesÚnorm_prer   Úheadr6   r   ÚapplyÚ_init_weights)!r7   rÿ   r   r  rk   r¤   r‹   rl   rO   rÏ   r  rÍ   rŽ   r�   rÐ   rÒ   rÑ   r  r  r  r  r  r)   r*   r8   Ú
num_stagesrM   Údprr  ÚirN   Ústager9   s!                                   €r:   r1   zDaVit.__init__B  sÉ  ø€ õ4 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆÝ˜‘_”_ˆ
Ø�S ™^œ^Ð:Ð:Ò:Ð:­s°6©{¬{Ò:Ð:Ð:Ð:Ð:Ð:Ý�^¨JÑ7Ô7¸XÐFÑFÔFˆ
Ý¥¨}Ñ =Ô =À8ÐLÑLÔLˆØ&ˆÔØ ˆŒØ4>¸r´NÐBˆÔ˜DÔ1Ø"ˆŒØ"'ˆÔØˆÔå˜ :¨a¤=ÐNÐN¸ZÐNÈ2ÐNÐNˆŒ	Ø˜A”ˆå'¨¸È$ÐOÑOÔOˆØˆÝ�zÑ"Ô"ð 	cð 	cˆAØ  ”mˆGÝØØðð ˜Q”iØ˜qš5Ø%Ø# Aœ,Ø'Ø#Ø!Ø # A¤Ø%Ø+ØØØ!1Ø /Ø)ð#ð ð$ ð%ð ˆEð( ˆFØ�MŠM˜%Ñ Ô Ð ØÐÔ¥$¨wÀ!ÀaÈÁcÁ(ÐS`Ð]^ÐS`ÐS`Ð"aÑ"aÔ"aÐ!bÑbÐÔÐå”m VÐ,ˆŒð
 ð 	Ø&˜J tÔ'8Ð?Ð?¸BÐ?Ð?ˆDŒMÝ&ØÔ!Øðð &Øœ.ð	ð ð
 ðð ˆDŒIˆIõ œK™MœMˆDŒMÝ-ØÔ!Øðð &Øœ.Ø%ðð ð ðð ˆDŒIð 	�
Š
�4Ô%Ñ&Ô&Ð&Ð&Ð&r;   c                 óþ   — t          |t          j        ¦  «        r^t          |j        d¬¦  «         t          |t          j        ¦  «        r0|j        �+t          j                             |j        d¦  «         d S d S d S d S )Ng{®Gáz”?)Ústdr   )Ú
isinstancer2   rq   r   Úweightro   ÚinitÚ	constant_)r7   Úms     r:   r  zDaVit._init_weights¢  s€   € Ý�a�œÑ#Ô#ð 	-Ý˜!œ(¨Ð,Ñ,Ô,Ð,Ý˜!�RœYÑ'Ô'ð -¨A¬FÐ,>Ý”×!Ò! !¤&¨!Ñ,Ô,Ð,Ð,Ð,ð	-ð 	-ð-ð -Ð,>Ð,>r;   c                 ó0   — t          d|rdng d¢¬¦  «        S )Nz^stemz^stages\.(\d+)))z^stages\.(\d+).downsample)r   )z^stages\.(\d+)\.blocks\.(\d+)N)z	^norm_pre)iŸ† )r  rá   )r  )r7   Úcoarses     r:   Úgroup_matcherzDaVit.group_matcher¨  s9   € åØØ(.ð Ð$Ð$ð 5ð 5ð 5ð
ñ 
ô 
ð 	
r;   c                 óT   — || _         | j        D ]}|                     |¬¦  «         Œd S )N)rè   )rÛ   r  ré   )r7   rè   r  s      r:   ré   zDaVit.set_grad_checkpointing³  s?   € à"(ˆÔØ”[ð 	8ð 	8ˆEØ×(Ò(°Ð(Ñ7Ô7Ð7Ð7ð	8ð 	8r;   Úreturnc                 ó   — | j         j        S r>   )r  Úfc)r7   s    r:   Úget_classifierzDaVit.get_classifier¹  s   € àŒyŒ|Ðr;   c                 óZ   — t          | ¦  «        }|| _         | j        j        ||fi |¤Ž d S r>   )r   r  r  Úreset)r7   r  r  r8   s       r:   Úreset_classifierzDaVit.reset_classifier½  s;   € Ý˜dÑ#Ô#ˆØ&ˆÔØˆŒ	Œ˜ [Ð7Ð7°BÐ7Ð7Ð7Ð7Ð7r;   ÚNCHWr<   ÚindicesrT   Ú
stop_earlyÚ
output_fmtÚintermediates_onlyc                 óœ  — |dv s
J d¦   «         ‚g }t          t          | j        ¦  «        |¦  «        \  }}	|                      |¦  «        }t          | j        ¦  «        dz
  }
t          j                             ¦   «         s|s| j        }n| j        d|	dz   …         }t          |¦  «        D ]\  }}| j        r/t          j                             ¦   «         st          ||¦  «        }n ||¦  «        }||v r5|r||
k    r|  
                    |¦  «        }n|}|                     |¦  «         Œ€|r|S ||
k    r|  
                    |¦  «        }||fS )aï   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )r0  zOutput shape must be NCHW.r   N)r   r  r  r  rµ   r¶   rë   rÞ   rÛ   r   r  rß   )r7   r<   r1  rT   r2  r3  r4  ÚintermediatesÚtake_indicesÚ	max_indexÚlast_idxr  Úfeat_idxr  Úx_inters                  r:   Úforward_intermediateszDaVit.forward_intermediatesÂ  sx  € ð* ˜YÐ&Ð&Ð&Ð(DÑ&Ô&Ð&ØˆÝ"6µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�ið �IŠI�a‰LŒLˆÝ�t”{Ñ#Ô# aÑ'ˆÝŒ9×!Ò!Ñ#Ô#ð 	1¨:ð 	1Ø”[ˆFˆFà”[  )¨a¡- Ô0ˆFå(¨Ñ0Ô0ð 
	.ð 
	.‰OˆH�eØÔ&ð ­u¬y×/EÒ/EÑ/GÔ/Gð Ý˜u aÑ(Ô(��à�E˜!‘H”H�Ø˜<Ð'Ð'Øð  ˜H¨Ò0Ð0Ø"Ÿmšm¨AÑ.Ô.�G�Gà�GØ×$Ò$ WÑ-Ô-Ð-øàð 	!Ø Ð à�xÒÐØ—’˜aÑ Ô ˆAà�-ÐÐr;   r   Ú
prune_normÚ
prune_headc                 óâ   — t          t          | j        ¦  «        |¦  «        \  }}| j        d|dz   …         | _        |rt          j        ¦   «         | _        |r|                      dd¦  «         |S )z@ Prune layers not required for specified intermediates.
        Nr   r   Ú )r   r  r  r2   r6   r  r/  )r7   r1  r=  r>  r7  r8  s         r:   Úprune_intermediate_layerszDaVit.prune_intermediate_layers÷  sq   € õ #7µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�iØ”k . 9¨q¡= .Ô1ˆŒØð 	*ÝœK™MœMˆDŒMØð 	)Ø×!Ò! ! RÑ(Ô(Ð(ØÐr;   c                 óú   — |                       |¦  «        }| j        r4t          j                             ¦   «         st          | j        |¦  «        }n|                      |¦  «        }|                      |¦  «        }|S r>   )r  rÛ   rµ   r¶   rë   r   r  r  rì   s     r:   Úforward_featureszDaVit.forward_features  si   € Ø�IŠI�a‰LŒLˆØÔ"ð 	­5¬9×+AÒ+AÑ+CÔ+Cð 	Ý˜tœ{¨AÑ.Ô.ˆAˆAà—’˜A‘”ˆAØ�MŠM˜!ÑÔˆØˆr;   Ú
pre_logitsc                 ó^   — |r|                       |d¬¦  «        n|                       |¦  «        S )NT)rD  )r  )r7   r<   rD  s      r:   Úforward_headzDaVit.forward_head  s,   € Ø0:ÐLˆt�yŠy˜ tˆyÑ,Ô,Ð,ÀÇ	Â	È!ÁÄÐLr;   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S r>   )rC  rF  rì   s     r:   r@   zDaVit.forward  s-   € Ø×!Ò! !Ñ$Ô$ˆØ×Ò˜aÑ Ô ˆØˆr;   )r#   rñ   rò   rö   rQ   rL   Trú   rû   rü   rÈ   TFr+   FFrŠ   rŠ   rý   rþ   FNN©Frí   r>   )NFFr0  F)r   FT)rB   rC   rD   r_   rE   r   r£   rF   rî   r1   r  rµ   r¶   rï   r'  ré   r2   r`   r,  r   r/  r	   r   r   r<  rA  rC  rF  r@   rG   rH   s   @r:   r    r    0  s�  ø€ € € € € ðð ð& Ø&2Ø*=Ø)7Ø Ø Ø!Ø+Ø!,Ø"Ø*@ØØ!Ø$%Ø$)Ø!&Ø!Ø$&Ø#Ø$Ø$)ØØð1^'ð ^'àð^'ð ˜#˜s˜(”Oð^'ð ˜c 3˜hœð	^'ð
 ˜S #˜X”ð^'ð ð^'ð ð^'ð ð^'ð ð^'ð ð^'ð ð^'ð ˜c 3˜hœð^'ð ð^'ð ð^'ð "ð^'ð  "ð!^'ð" ð#^'ð$ ð%^'ð& "ð'^'ð( ð)^'ð* ð+^'ð, "ð-^'ð ^'ð ^'ð ^'ð ^'ð ^'ð@-ð -ð -ð „YÔð
ð 
ð 
ñ Ôð
ð „YÔð8ð 8ð 8ñ Ôð8ð
 „YÔð ¤	ð ð ð ñ Ôðð8ð 8¨Cð 8¸hÀs¼mð 8ð 8ð 8ð 8ð 8<ØØ$Ø$Ø',ð3 ð 3 àŒ|ð3 ð ˜e C¨¨c¬ NÔ3Ô4ð3 ð ð	3 ð
 ð3 ð ð3 ð !%ð3 ð 
ˆt�E”LÔ! 5¨¬°t¸E¼LÔ7IÐ)IÔ#JÐJÔ	Kð3 ð 3 ð 3 ð 3 ðn ./Ø$Ø#ð	ð à˜3  S¤	˜>Ô*ðð ðð ð	ð ð ð ð ð ð ðMð M¨$ð Mð Mð Mð Mðð ð ð ð ð ð r;   úvision_tower.c                 óÂ  — dd l }i }|                      ¦   «         D �]B\  }}|                     |¦  «        r|                     |d¦  «        }nŒ3|                     dd|¦  «        }|                     dd|¦  «        }|                     dd¦  «        }|                     d	d
¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|||<   �ŒD|S )Nr   r@  zconvs.([0-9]+)ústages.\1.downsamplezblocks.([0-9]+)ústages.\1.blocksúdownsample.projúdownsample.convústages.0.downsampler  zwindow_attn.norm.znorm1.zwindow_attn.fn.zattn.zchannel_attn.norm.zchannel_attn.fn.z	ffn.norm.znorm2.zffn.fn.net.zmlp.zconv1.fn.dwz	cpe1.projzconv2.fn.dwz	cpe2.proj)ÚreÚitemsÚ
startswithÚreplaceÚsub)Ú
state_dictÚmodelÚprefixrP  Úout_dictr%   r€   s          r:   Ú_convert_florence2rY    s`  € Ø€I€I€IØ€Hà× Ò Ñ"Ô"ð ñ ‰ˆˆ1Ø�<Š<˜ÑÔð 	Ø—	’	˜& "Ñ%Ô%ˆAˆAàØ�FŠFÐ$Ð&=¸qÑAÔAˆØ�FŠFÐ%Ð':¸AÑ>Ô>ˆØ�IŠIÐ'Ð):Ñ;Ô;ˆØ�IŠIÐ+¨VÑ4Ô4ˆð �IŠIÐ)¨8Ñ4Ô4ˆØ�IŠIÐ'¨Ñ1Ô1ˆØ�IŠIÐ*¨HÑ5Ô5ˆØ�IŠIÐ(¨'Ñ2Ô2ˆØ�IŠI�k 8Ñ,Ô,ˆØ�IŠI�m VÑ,Ô,ˆØ�IŠI�m [Ñ1Ô1ˆØ�IŠI�m [Ñ1Ô1ˆØˆ�‰‰à€Or;   c                 ó   — d| v r| S d| v r| d         } d| v rt          | |¦  «        S ddl}i }|                      ¦   «         D ]¼\  }}|                     dd|¦  «        }|                     dd	|¦  «        }|                     d
d¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|||<   Œ½|S )z  Remap MSFT checkpoints -> timm zhead.fc.weightrU  z vision_tower.convs.0.proj.weightr   Nzpatch_embeds.([0-9]+)rK  zmain_blocks.([0-9]+)rL  rM  rN  rO  r  zhead.zhead.fc.znorms.z
head.norm.zcpe.0r–   zcpe.1r™   )rY  rP  rQ  rT  rS  )rU  rV  rP  rX  r%   r€   s         r:   Úcheckpoint_filter_fnr[  5  s#  € à˜:Ð%Ð%ØÐà�zÐ!Ð!Ø Ô-ˆ
à)¨ZÐ7Ð7Ý! *¨eÑ4Ô4Ð4à€I€I€IØ€HØ× Ò Ñ"Ô"ð 	ð 	‰ˆˆ1Ø�FŠFÐ+Ð-DÀaÑHÔHˆØ�FŠFÐ*Ð,?ÀÑCÔCˆØ�IŠIÐ'Ð):Ñ;Ô;ˆØ�IŠIÐ+¨VÑ4Ô4ˆØ�IŠI�g˜zÑ*Ô*ˆØ�IŠI�h Ñ-Ô-ˆØ�IŠI�g˜vÑ&Ô&ˆØ�IŠI�g˜vÑ&Ô&ˆØˆ�‰ˆØ€Or;   Fc           	      óX  — t          d„ t          |                     dd¦  «        ¦  «        D ¦   «         ¦  «        }|                     d|¦  «        }|                     dd¦  «        }|                      d¦  «        rd}t          t          | |ft          t          d|¬	¦  «        |d
œ|¤Ž}|S )Nc              3   ó    K  — | ]	\  }}|V — Œ
d S r>   r•   )rØ   r  rÁ   s      r:   ú	<genexpr>z _create_davit.<locals>.<genexpr>P  s&   è è € Ð\Ð\¡d a¨ Ð\Ð\Ð\Ð\Ð\Ð\r;   r   rñ   Úout_indicesÚpretrained_strictTÚ_flF)Úflatten_sequentialr_  )Úpretrained_filter_fnÚfeature_cfgr`  )	ÚtuplerÞ   ÚgetÚpopÚendswithr   r    r[  r  )ÚvariantÚ
pretrainedÚkwargsÚdefault_out_indicesr_  ÚstrictrV  s          r:   Ú_create_davitrn  O  sÃ   € ÝÐ\Ð\­i¸¿
º
À8È\Ñ8ZÔ8ZÑ.[Ô.[Ð\Ñ\Ô\Ñ\Ô\ÐØ—*’*˜]Ð,?Ñ@Ô@€Kà�ZŠZÐ+¨TÑ2Ô2€FØ×Ò˜ÑÔð àˆå ÝØØðõ 2Ý¨D¸kÐJÑJÔJØ ðð ð ðð €Eð €Lr;   r@  c                 ó6   — | dddddt           t          dddd	œ|¥S )
Nrý   )r#   éà   rp  )rQ   rQ   gffffffî?Úbicubicz	stem.convzhead.fcz
apache-2.0)Úurlr  Ú
input_sizeÚ	pool_sizeÚcrop_pctÚinterpolationÚmeanr  Ú
first_convÚ
classifierÚlicenser
   )rr  rk  s     r:   Ú_cfgr{  d  s8   € àØ¨=ÀvØ¨9Ý%Õ.BØ!°Øðð ð ðð r;   ztimm/)Ú	hf_hub_idzmicrosoft/Florence-2-base)r#   rõ   rõ   )r|  r  rs  zmicrosoft/Florence-2-large)zdavit_tiny.msft_in1kzdavit_small.msft_in1kzdavit_base.msft_in1kÚdavit_largeÚ
davit_hugeÚdavit_giantzdavit_base_fl.msft_florence2zdavit_huge_fl.msft_florence2r)  c           	      óX   — t          ddd¬¦  «        }t          dd| it          |fi |¤Ž¤ŽS )Nrñ   rò   rö   ©r   r  rk   Ú
davit_tinyrj  )r‚  ©r  rn  ©rj  rk  Ú
model_argss      r:   r‚  r‚  …  sB   € å˜\Ð6IÐUcÐdÑdÔd€JÝÐ[Ð[°*Ð[ÅÀZÐ@ZÐ@ZÐSYÐ@ZÐ@ZÐ[Ð[Ð[r;   c           	      óX   — t          ddd¬¦  «        }t          dd| it          |fi |¤Ž¤ŽS )N©r   r   é	   r   rò   rö   r�  Údavit_smallrj  )r‰  rƒ  r„  s      r:   r‰  r‰  ‹  sB   € å˜\Ð6IÐUcÐdÑdÔd€JÝÐ\Ð\°:Ð\ÅÀjÐA[ÐA[ÐTZÐA[ÐA[Ð\Ð\Ð\r;   c           	      óX   — t          ddd¬¦  «        }t          dd| it          |fi |¤Ž¤ŽS )Nr‡  ©é€   é   é   é   ©rL   rj   é   é    r�  Ú
davit_baserj  )r“  rƒ  r„  s      r:   r“  r“  ‘  sB   € å˜\Ð6KÐWeÐfÑfÔf€JÝÐ[Ð[°*Ð[ÅÀZÐ@ZÐ@ZÐSYÐ@ZÐ@ZÐ[Ð[Ð[r;   c           	      óX   — t          ddd¬¦  «        }t          dd| it          |fi |¤Ž¤ŽS )Nr‡  )ró   rô   rõ   é   )r÷   rø   rù   é0   r�  r}  rj  )r}  rƒ  r„  s      r:   r}  r}  —  sB   € å˜\Ð6KÐWfÐgÑgÔg€JÝÐ\Ð\°:Ð\ÅÀjÐA[ÐA[ÐTZÐA[ÐA[Ð\Ð\Ð\r;   c           	      óX   — t          ddd¬¦  «        }t          dd| it          |fi |¤Ž¤ŽS )Nr‡  ©r�  rŽ  r�  i   ©rj   r‘  r’  é@   r�  r~  rj  )r~  rƒ  r„  s      r:   r~  r~  �  sB   € å˜\Ð6LÐXgÐhÑhÔh€JÝÐ[Ð[°*Ð[ÅÀZÐ@ZÐ@ZÐSYÐ@ZÐ@ZÐ[Ð[Ð[r;   c           	      óX   — t          ddd¬¦  «        }t          dd| it          |fi |¤Ž¤ŽS )N)r   r   rø   r#   )rô   rõ   r•  i   )rø   rù   r–  rK   r�  r  rj  )r  rƒ  r„  s      r:   r  r  £  sB   € å˜]Ð7MÐYiÐjÑjÔj€JÝÐ\Ð\°:Ð\ÅÀjÐA[ÐA[ÐTZÐA[ÐA[Ð\Ð\Ð\r;   c           	      ó`   — t          ddddddd¬¦  «        }t          d
d	| it          |fi |¤Ž¤ŽS )Nr‡  r‹  r�  rø   r#   T©r   r  rk   r¤   rÐ   rÒ   rÑ   Údavit_base_flrj  )rž  rƒ  r„  s      r:   rž  rž  ª  sS   € åØÐ(=ÈØ¨¸DÈtðñ ô €Jõ Ð^Ð^°ZÐ^Å4È
ÐC]ÐC]ÐV\ÐC]ÐC]Ð^Ð^Ð^r;   c           	      ó`   — t          ddddddd¬¦  «        }t          d
d	| it          |fi |¤Ž¤ŽS )Nr‡  r˜  r™  rø   r#   Tr�  Údavit_huge_flrj  )r   rƒ  r„  s      r:   r   r   ³  sU   € õ ØÐ(>È/Ø¨¸DÈtðñ ô €Jõ Ð^Ð^°ZÐ^Å4È
ÐC]ÐC]ÐV\ÐC]ÐC]Ð^Ð^Ð^r;   )rI  rH  )r@  )Gr_   Ú	functoolsr   Útypingr   r   r   r   r   rµ   Útorch.nnr2   Útorch.nn.functionalÚ
functionalrW   r	   Ú	timm.datar   r   Útimm.layersr   r   r   r   r   r   r   r   r   r   r   Ú_builderr   Ú	_featuresr   Ú_features_fxr   Ú_manipulater   r   Ú	_registryr   r   Ú__all__r`   r"   rJ   rb   ri   rƒ   rˆ   rE   rª   r¬   r®   rº   rÇ   r    rY  r[  rn  r{  Údefault_cfgsr‚  r‰  r“  r}  r~  r  rž  r   r•   r;   r:   ú<module>r¯     s�  ðð	ð 	ð Ð Ð Ð Ð Ð Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5à €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à AÐ AÐ AÐ AÐ AÐ AÐ AÐ Að Zð  Zð  Zð  Zð  Zð  Zð  Zð  Zð  Zð  Zð  Zð  Zð  Zð  Zð  Zð  Zð  Zð  Zð  Zð  Zð  Zð  ZØ =Ð =Ð =Ð =Ð =Ð =Ð =Ð =Ø *Ð *Ð *Ð *Ð *Ð *Ø +Ð +Ð +Ð +Ð +Ð +Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø <Ð <Ð <Ð <Ð <Ð <Ð <Ð <àˆ)€ðð ð ð ð �”ñ ô ð ð:&ð &ð &ð &ð &ˆ2Œ9ñ &ô &ð &ðR#ð #ð #ð #ð #�”ñ #ô #ð #ðL$ð $ð $ð $ð $˜œñ $ô $ð $ðPð ð ð ð �r”yñ ô ð ðD?ð ?ð ?ð ?ð ?�2”9ñ ?ô ?ð ?ðD˜ð ¨U°3¸°8¬_ð ð ð ð ð ð˜Fð °°s¸C°x´ð ÀSð ÈSð ð ð ñ Ôðð 2ð 2ð 2ð 2ð 2�b”iñ 2ô 2ð 2ðjdð dð dð dð d�2”9ñ dô dð dðNXð Xð Xð Xð X�”ñ Xô Xð Xðvfð fð fð fð fˆBŒIñ fô fð fðRð ð ð ð8ð ð ð4ð ð ð ð*	ð 	ð 	ð 	ð %Ð$à ˜DØðñ ô à!˜TØðñ ô à ˜DØðñ ô à�4‘6”6Ø�$‘&”&Ø�4‘6”6Ø$( DØ-Ø -ð%1ñ %1ô %1ð %) DØ.Ø -ð%1ñ %1ô %1ð&ð &ñ ô €ð( ð\ð \¨eð \ð \ð \ñ „ð\ð
 ð]ð ]¨uð ]ð ]ð ]ñ „ð]ð
 ð\ð \¨eð \ð \ð \ñ „ð\ð
 ð]ð ]¨uð ]ð ]ð ]ñ „ð]ð
 ð\ð \¨eð \ð \ð \ñ „ð\ð
 ð]ð ]¨uð ]ð ]ð ]ñ „ð]ð ð_ð _°ð _ð _ð _ñ „ð_ð ð_ð _°ð _ð _ð _ñ „ð_ð _ð _r;   