§
    ·¯¸jY  ã            
       ó¦  — d Z ddlZddlZddlmZmZ ddlZddlmZ ddl	mc m
Z ddlmZ ddlmZ ddlmZ ddlmZ  eej                             d	d¦  «        ¦  «        dk    Z	 	 	 d-deeef         deeeef                  dedej        fd„Zdeeef         deedf         fd„Z	 	 d.dedefd„Zdeeef         deedf         fd„Z G d„ dej        ¦  «        Z 	 	 	 	 d/deeef         deeef         fd „Z! G d!„ d"ej        ¦  «        Z"	 	 	 d0d#ed$ee         fd%„Z#d&ed'ed(ej        d)ej        dej        f
d*„Z$ G d+„ d,ej        ¦  «        Z%dS )1zf Relative position embedding modules and functions

Hacked together by / Copyright 2022 Ross Wightman
é    N)ÚOptionalÚTupleé   )Úndgrid)ÚRegularGridInterpolator)ÚMlp)Útrunc_normal_ÚTIMM_USE_SCIPY_INTERPFÚq_sizeÚk_sizeÚclass_tokenÚreturnc           	      ó  — |�
J d¦   «         ‚t          j        t          t          j        | d         |¬¦  «        t          j        | d         |¬¦  «        ¦  «        ¦  «                             d¦  «        }|d d …d d …d f         |d d …d d d …f         z
  }|                     ddd¦  «        }|d d …d d …dfxx         | d         dz
  z  cc<   |d d …d d …dfxx         | d         dz
  z  cc<   |d d …d d …dfxx         d| d         z  dz
  z  cc<   d| d         z  dz
  d| d         z  dz
  z  }|                     d¦  «        }|r4t          j        |g d¢¦  «        }||ddd …f<   |dz   |dd …df<   |dz   |d<   | 	                    ¦   «         S )	Nz-Different q & k sizes not currently supportedr   ©Údevicer   é   éÿÿÿÿ)r   r   r   r   ©r   r   )
ÚtorchÚstackr   ÚarangeÚflattenÚpermuteÚsumÚFÚpadÚ
contiguous)r   r   r   r   ÚcoordsÚrelative_coordsÚnum_relative_distanceÚrelative_position_indexs           úi/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/layers/pos_embed_rel.pyÚgen_relative_position_indexr#      sû  € ð ˆ>ˆ>ÐJ‰>Œ>ˆ>åŒ[�ÝŒ�V˜A”Y vÐ.Ñ.Ô.ÝŒ�V˜A”Y vÐ.Ñ.Ô.ñô ñ ô ÷ ‚wˆq�z„zð ð ˜Q˜Q˜Q    4˜ZÔ(¨6°!°!°!°T¸1¸1¸1°*Ô+=Ñ=€OØ%×-Ò-¨a°°AÑ6Ô6€OØ�A�A�A�q�q�q˜!�GÐÐÔ  q¤	¨A¡Ñ-ÐÐÑØ�A�A�A�q�q�q˜!�GÐÐÔ  q¤	¨A¡Ñ-ÐÐÑØ�A�A�A�q�q�q˜!�GÐÐÔ  F¨1¤I¡°Ñ 1Ñ1ÐÐÑØ ¨¤™]¨QÑ.°1°v¸a´y±=À1Ñ3DÑEÐð0 .×1Ò1°"Ñ5Ô5Ðàð Bõ #$¤%Ð(?ÀÀÀÑ"NÔ"NÐØ)>Ð  1 2 2 Ñ&Ø)>ÀÑ)BÐ    A Ñ&Ø(=ÀÑ(AÐ Ñ%à"×-Ò-Ñ/Ô/Ð/ó    Únew_window_sizeÚnew_bias_shape.c           	      óÊ  — |d         dz  dz
  |d         dz  dz
  f}| j         dk    r‡|\  }}}| j        \  }}}	||d         k    r||d         k    sJ ‚||k    s|	|k    rNt          j        j                             |                      d¦  «        |dd¬¦  «                             d¦  «        } �n4| j         dk    sJ ‚|\  }
}| j        \  }}|
|d         |d         z  z
  }t          ||z
  dz  ¦  «        }||f}|d         |d         k    s|d         |d         k    rÇ|r| | d …d d …f         }| d | …d d …f         } nd }t          j        j                             |  	                    dd¦  «         
                    dd	|d         |d         f¦  «        |dd¬¦  «                             d	|
|z
  ¦  «         	                    dd¦  «        } |�t          j        | |fd¬
¦  «        } | S )Nr   r   r   é   ÚbicubicF)ÚsizeÚmodeÚalign_cornersç      à?r   ©Údim)ÚndimÚshaper   ÚnnÚ
functionalÚinterpolateÚ	unsqueezeÚsqueezeÚintÚ	transposeÚreshapeÚviewÚcat)Úrel_pos_biasr%   r&   Údst_sizeÚ_Údst_hÚdst_wÚnum_attn_headsÚsrc_hÚsrc_wÚdst_num_posÚsrc_num_posÚnum_extra_tokensÚsrc_sizeÚextra_tokenss                  r"   Ú resize_rel_pos_bias_table_simplerI   M   sN  € ð
   Ô" QÑ&¨Ñ*¨O¸AÔ,>ÀÑ,BÀQÑ,FÐG€HØÔ˜AÒÐà(‰ˆˆ5�%Ø'3Ô'9Ñ$ˆ˜˜uØ˜ œÒ#Ð#¨°¸!´Ò(<Ð(<Ð(<Ð<Ø�EŠ>ˆ>˜U eš^˜^Ý œ8Ô.×:Ò:Ø×&Ò& qÑ)Ô)ØØØ#ð	 ;ñ ô ÷
 Šg�a‰jŒjð ùð Ô  AÒ%Ð%Ð%Ð%à'‰ˆ�QØ&2Ô&8Ñ#ˆ�^Ø&¨(°1¬+¸À¼Ñ*CÑDÐÝ˜Ð&6Ñ6¸3Ñ>Ñ?Ô?ˆØ˜hÐ'ˆà�AŒ;˜( 1œ+Ò%Ð%¨°!¬¸À¼Ò)CÐ)CØð $Ø+Ð-=Ð,=Ð,>Ð,>ÀÀÀÐ,AÔB�Ø+Ð,>Ð.>Ð->Ð,>ÀÀÀÐ,AÔB��à#�å œ8Ô.×:Ò:Ø×&Ò& q¨!Ñ,Ô,×4Ò4°a¸¸XÀa¼[È(ÐSTÌ+Ð5VÑWÔWØØØ#ð	 ;ñ ô ÷
 Šd�2�{Ð%5Ñ5Ñ6Ô6·y²yÀÀA±´ð ð Ð'Ý$œy¨,¸Ð)EÈ1ÐMÑMÔM�àÐr$   r)   TÚinterpolationÚ	antialiasc                 óâ  — |                       ¦   «         \  }}|\  }}||k    sJ ‚||k    rÄ| j        }|                      ¦   «         } t          |dz  ¦  «        }	t          |dz  ¦  «        }
t	          j        |                      dd¦  «                             d||	|	¦  «        |
|
f||¬¦  «        }|                     ||¦  «                             dd¦  «        }|                     |¦  «         |S | S )z’
    Resample relative position bias table suggested in LeVit
    Adapted from: https://github.com/microsoft/Cream/blob/main/TinyViT/utils.py
    r-   r   r   )r*   r+   rK   )	r*   ÚdtypeÚfloatr7   r   r4   r   r:   Úto)Úposition_bias_tableÚnew_sizerJ   rK   ÚL1ÚnH1ÚL2ÚnH2Ú
orig_dtypeÚS1ÚS2Ú$relative_position_bias_table_resizeds               r"   Úresize_rel_pos_bias_table_levitrZ   |   s  € ð "×&Ò&Ñ(Ô(�G€BˆØ�G€BˆØ�#Š:ˆ:ˆ:ˆ:Ø	ˆR‚x€xØ(Ô.ˆ
Ø1×7Ò7Ñ9Ô9Ðå��s‘‰^Œ^ˆÝ��s‘‰^Œ^ˆÝ/0¬}Ø×'Ò'¨¨1Ñ-Ô-×2Ò2°1°c¸2¸rÑBÔBØ�b�ØØð	0
ñ 0
ô 0
Ð,ð 0T×/XÒ/XÐY\Ð^`Ñ/aÔ/a×/iÒ/iÐjkÐmnÑ/oÔ/oÐ,Ø,×/Ò/°
Ñ;Ô;Ð;Ø3Ð3à"Ð"r$   c                 óÖ  ‡— t           rddlm} |d         dz  dz
  |d         dz  dz
  f}| j        dk    r4d}|\  }}}||d         k    r||d         k    sJ ‚| j        \  }	}
}|
|f}d}nK| j        dk    sJ ‚|\  }}| j        \  }}	||d         |d         z  z
  }t          ||z
  dz  ¦  «        }||f}d}|d         |d         k    s|d         |d         k    �r�|r| | d	…d	d	…f         }| d	| …d	d	…f         } nd	}d
„ Šˆfd„} ||d         |d         ¦  «        } ||d         |d         ¦  «        }t          j        |¦  «        t          j        |¦  «        g}|d         dz  }|d         dz  }t          j        | |dz   d¦  «        }t          j        | |dz   d¦  «        }t          ||¦  «        }g }t          |	¦  «        D �]R}|r?| d	d	…|f                              |d         |d         ¦  «                             ¦   «         }n!| |d	d	…d	d	…f                              ¦   «         }t           rt|                     |||                     ¦   «         d¬¦  «        }t          j         |||¦  «        ¦  «                             ¦   «                              | j        ¦  «        }nEt'          ||¦  «        } ||¦  «                             ¦   «                              | j        ¦  «        }|r|                     dd¦  «        }|                     |¦  «         �ŒT|rt          j        |d¬¦  «        } nt          j        |d¬¦  «        } |�|sJ ‚t          j        | |fd¬¦  «        } | S )a�   Resize relative position bias table using more advanced interpolation.

    Modified from code in Microsoft Unilm (https://github.com/microsoft/unilm) repo (BeiT, BeiT-v2, etc).

    https://github.com/microsoft/unilm/blob/5255d52de86dad642810f5849dd357769346c1d7/beit/run_class_finetuning.py#L351

    Args:
        rel_pos_bias:
        new_window_size:
        new_bias_shape:

    Returns:

    r   )r4   r   r   r(   Fr-   TNc                 ó$   — | d||z  z
  z  d|z
  z  S )Nç      ð?© )ÚaÚrÚns      r"   Úgeometric_progressionz8resize_rel_pos_bias_table.<locals>.geometric_progressionÍ   s   € Ø˜˜a 1™f™Ñ%¨¨q©Ñ1Ð1r$   c                 ó4  •— d\  }}||z
  dk    r/||z   dz  } ‰
d|| dz  ¦  «        }||dz  k    r|}n|}||z
  dk    °/g }d}t          | dz  ¦  «        D ]"}|                     |¦  «         |||dz   z  z  }Œ#d„ t          |¦  «        D ¦   «         }	|	dgz   |z   S )N)g)\�Âõ(ð?g      ø?g�íµ ÷Æ°>ç       @r   r   c                 ó   — g | ]}| ‘ŒS r^   r^   )Ú.0r>   s     r"   ú
<listcomp>z<resize_rel_pos_bias_table.<locals>._calc.<locals>.<listcomp>ß   s   € Ð/Ð/Ð/˜A�a�RÐ/Ð/Ð/r$   r   )ÚrangeÚappendÚreversed)ÚsrcÚdstÚleftÚrightÚqÚgpÚdisÚcurÚiÚr_idsrb   s             €r"   Ú_calcz(resize_rel_pos_bias_table.<locals>._calcÐ   sÝ   ø€ Ø#‰KˆD�%Ø˜$‘, Ò%Ð%Ø˜E‘\ SÑ(�Ø*Ð*¨1¨a°¸±Ñ:Ô:�Ø˜˜q™’=�=Ø�E�Eà�Dð ˜$‘, Ò%Ð%ð ˆCØˆCÝ˜3 !™8‘_”_ð $ð $�Ø—
’
˜3‘”�Ø�q˜Q ™U‘|Ñ#��Ø/Ð/¥¨#¡¤Ð/Ñ/Ô/ˆEØ˜A˜3‘; Ñ$Ð$r$   rd   gš™™™™™¹?r]   Úcubic)Úkindr   r.   )Ú
_USE_SCIPYÚscipyr4   r0   r1   r7   r   Útensorr   r   rh   r:   rN   Úinterp2dÚnumpyÚTensorr   rO   r   r   ri   r;   ) r<   r%   r&   r4   r=   rF   r>   r?   r@   rA   rB   rC   rG   Úhas_flat_shaperD   rE   rH   ru   ÚyÚxÚyxÚtyÚtxÚdyÚdxÚdyxÚall_rel_pos_biasrs   ÚzÚfr`   rb   s                                   @r"   Úresize_rel_pos_bias_tablerŠ   œ   s	  ø€ õ& ð &Ø%Ð%Ð%Ð%Ð%Ð%à Ô" QÑ&¨Ñ*¨O¸AÔ,>ÀÑ,BÀQÑ,FÐG€HØÔ˜AÒÐàÐØ(‰ˆˆ5�%Ø˜ œÒ#Ð#¨°¸!´Ò(<Ð(<Ð(<Ð<Ø'3Ô'9Ñ$ˆ˜˜uØ˜5�>ˆØˆˆàÔ  AÒ%Ð%Ð%Ð%à'‰ˆ�QØ&2Ô&8Ñ#ˆ�^Ø&¨(°1¬+¸À¼Ñ*CÑDÐÝ˜Ð&6Ñ6¸3Ñ>Ñ?Ô?ˆØ˜hÐ'ˆØˆà�„{�h˜q”kÒ!Ð! X¨a¤[°H¸Q´KÒ%?Ñ%?àð 	 Ø'Ð)9Ð(9Ð(:Ð(:¸A¸A¸AÐ(=Ô>ˆLØ'Ð(:Ð*:Ð):Ð(:¸A¸A¸AÐ(=Ô>ˆLˆLàˆLð	2ð 	2ð 	2ð	%ð 	%ð 	%ð 	%ð 	%ð$ ˆE�(˜1”+˜x¨œ{Ñ+Ô+ˆØˆE�(˜1”+˜x¨œ{Ñ+Ô+ˆÝŒl˜1‰oŒo�uœ|¨A™œÐ/ˆð �aŒ[˜CÑˆØ�aŒ[˜CÑˆÝŒ\˜2˜#˜r C™x¨Ñ-Ô-ˆÝŒ\˜2˜#˜r C™x¨Ñ-Ô-ˆÝ�R˜‰nŒnˆð ÐÝ�~Ñ&Ô&ð 	'ñ 	'ˆAØð 2Ø     A Ô&×+Ò+¨H°Q¬K¸À!¼ÑEÔE×KÒKÑMÔM��à   A A A q q q Ô)×/Ò/Ñ1Ô1�åð 	@à×(Ò(¨¨A¨q¯wªw©y¬y¸wÐ(ÑGÔG�Ý”L   2 r¡¤Ñ+Ô+×6Ò6Ñ8Ô8×;Ò;¸LÔ<OÑPÔP��õ
 ,¨B°Ñ2Ô2�Ø�A�c‘F”F×%Ò%Ñ'Ô'×*Ò*¨<Ô+>Ñ?Ô?�àð "Ø—F’F˜2˜q‘M”M�Ø×#Ò# AÑ&Ô&Ð&Ñ&àð 	>Ý œ9Ð%5¸2Ð>Ñ>Ô>ˆLˆLå œ9Ð%5¸1Ð=Ñ=Ô=ˆLàÐ#Ø!Ð!Ð!�>Ý œ9 l°LÐ%AÀqÐIÑIÔIˆLàÐr$   c                   óš   ‡ — e Zd ZdZ	 	 	 ddeeef         dedefˆ fd„Zdd	„Zdd
„Zde	j
        fd„Zddee	j
                 fd„Zdd„Zˆ xZS )Ú
RelPosBiasz_ Relative Position Bias
    Adapted from Swin-V1 relative position bias impl, modularized.
    r   NÚwindow_sizeÚ	num_headsÚprefix_tokensc                 ó"  •— ||dœ}t          ¦   «                              ¦   «          |dk    sJ ‚|| _        |d         |d         z  | _        || _        | j        |z   fdz  |fz   | _        d|d         z  dz
  d|d         z  dz
  z  d|z  z   }t          j        t          j	        ||fi |¤Ž¦  «        | _
        | j        |z   dz  }|                      dt          j	        ||t          j        ¬¦  «        d¬¦  «         |                      ¦   «          d S )	N©r   rM   r   r   r   r(   r!   F©Ú
persistent)ÚsuperÚ__init__r�   Úwindow_arear�   Ú
bias_shaper2   Ú	Parameterr   ÚemptyÚrelative_position_bias_tableÚregister_bufferÚlongÚreset_parameters)
Úselfr�   rŽ   r�   r   rM   Úddr    Ú
index_sizeÚ	__class__s
            €r"   r•   zRelPosBias.__init__  s>  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ Ò!Ð!Ð!Ð!Ø&ˆÔØ& qœ>¨K¸¬NÑ:ˆÔØ*ˆÔØÔ+¨mÑ;Ð=ÀÑAÀYÀLÑPˆŒà!" [°¤^Ñ!3°aÑ!7¸AÀÈAÄÑ<NÐQRÑ<RÑ SÐVWÐZgÑVgÑ gÐÝ,.¬L½¼ÐEZÐ\eÐ9lÐ9lÐikÐ9lÐ9lÑ,mÔ,mˆÔ)ØÔ&¨Ñ6¸1Ñ<ˆ
Ø×ÒØ%ÝŒK˜
¨6½¼ÐDÑDÔDØð 	ñ 	
ô 	
ð 	
ð 	×ÒÑÔÐÐÐr$   r   c                 óZ   — t          | j        d¬¦  «         |                      ¦   «          dS ©ú"Initialize parameters and buffers.g{®Gáz”?)ÚstdN)r	   rš   Ú_init_buffers©rž   s    r"   r�   zRelPosBias.reset_parameters1  s0   € å�dÔ7¸SÐAÑAÔAÐAØ×ÒÑÔÐÐÐr$   c                 ó°   — | j                              t          | j        | j        dk    | j         j        ¬¦  «                             d¦  «        ¦  «         dS )ú.Compute and fill non-persistent buffer values.r   )r   r   r   N)r!   Úcopy_r#   r�   r�   r   r:   r§   s    r"   r¦   zRelPosBias._init_buffers6  s`   € àÔ$×*Ò*Ý'ØÔ Ø Ô.°Ò2ØÔ3Ô:ðñ ô ÷ Šd�2‰hŒhñ	
ô 	
ð 	
ð 	
ð 	
r$   c                 óÒ   — | j         | j                 }|                     | j        ¦  «                             ddd¦  «        }|                     d¦  «                             ¦   «         S )Nr   r   r   )rš   r!   r:   r—   r   r5   r   ©rž   Úrelative_position_biass     r"   Úget_biaszRelPosBias.get_bias@  s^   € Ø!%Ô!BÀ4ÔC_Ô!`Ðà!7×!<Ò!<¸T¼_Ñ!MÔ!M×!UÒ!UÐVWÐYZÐ\]Ñ!^Ô!^ÐØ%×/Ò/°Ñ2Ô2×=Ò=Ñ?Ô?Ð?r$   Úshared_rel_posc                 ó0   — ||                       ¦   «         z   S ©N©r®   ©rž   Úattnr¯   s      r"   ÚforwardzRelPosBias.forwardF  ó   € Ø�d—m’m‘o”oÑ%Ð%r$   c                 ó.   — |                       ¦   «          dS ©z"Initialize non-persistent buffers.N©r¦   r§   s    r"   Úinit_non_persistent_buffersz&RelPosBias.init_non_persistent_buffersI  ó   € à×ÒÑÔÐÐÐr$   ©r   NN©r   Nr±   ©Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r7   r•   r�   r¦   r   r}   r®   r   rµ   rº   Ú__classcell__©r¡   s   @r"   rŒ   rŒ     s   ø€ € € € € ðð ð "#ØØð ð  à˜s C˜xœð ð ð ð ð	 ð  ð  ð  ð  ð  ð8ð ð ð ð

ð 
ð 
ð 
ð@˜%œ,ð @ð @ð @ð @ð&ð &¨H°U´\Ô,Bð &ð &ð &ð &ðð ð ð ð ð ð ð r$   rŒ   r   ÚswinÚwin_sizeÚpretrained_win_sizec                 ó2  — |dv sJ ‚t          j        | d         dz
   | d         |¬¦  «                             t           j        ¦  «        }t          j        | d         dz
   | d         |¬¦  «                             t           j        ¦  «        }t          j        t          ||¦  «        ¦  «        }|                     ddd¦  «                             ¦   «         }|dk    ræ|d         dk    rA|d d …d d …dfxx         |d         dz
  z  cc<   |d d …d d …dfxx         |d         dz
  z  cc<   n@|d d …d d …dfxx         | d         dz
  z  cc<   |d d …d d …dfxx         | d         dz
  z  cc<   |dz  }t          j        |¦  «        t          j	        d| 
                    ¦   «         z   ¦  «        z  t          j	        d¦  «        z  }n>t          j        |¦  «        t          j        d| 
                    ¦   «         z   ¦  «        z  }|                     |¦  «        S )	N)rÅ   Úcrr   r   r   r   rÅ   é   r]   )r   r   rO   Úfloat32r   r   r   r   ÚsignÚlog2ÚabsÚmathÚlog)rÆ   rÇ   r+   r   rM   Úrelative_coords_hÚrelative_coords_wÚrelative_coords_tables           r"   Úgen_relative_log_coordsrÔ   N  s[  € ð �>Ð!Ð!Ð!Ð!åœ x°¤{°Q¡Ð%7¸À!¼ÈVÐTÑTÔT×WÒWÕX]ÔXeÑfÔfÐÝœ x°¤{°Q¡Ð%7¸À!¼ÈVÐTÑTÔT×WÒWÕX]ÔXeÑfÔfÐÝ!œK­Ð/@ÐBSÑ(TÔ(TÑUÔUÐØ1×9Ò9¸!¸QÀÑBÔB×MÒMÑOÔOÐØˆv‚~€~Ø˜qÔ! AÒ%Ð%Ø! ! ! ! Q Q Q¨ 'Ð*Ð*Ô*Ð/BÀ1Ô/EÈÑ/IÑJÐ*Ð*Ñ*Ø! ! ! ! Q Q Q¨ 'Ð*Ð*Ô*Ð/BÀ1Ô/EÈÑ/IÑJÐ*Ð*Ñ*Ð*à! ! ! ! Q Q Q¨ 'Ð*Ð*Ô*¨x¸¬{¸Q©Ñ?Ð*Ð*Ñ*Ø! ! ! ! Q Q Q¨ 'Ð*Ð*Ô*¨x¸¬{¸Q©Ñ?Ð*Ð*Ñ*Ø Ñ"ÐÝ %¤
Ð+@Ñ AÔ AÅEÄJØÐ'×+Ò+Ñ-Ô-Ñ-ñE/ô E/ñ !/Ý15´¸1±´ñ!>ÐÐõ !&¤
Ð+@Ñ AÔ AÅEÄIØÐ'×+Ò+Ñ-Ô-Ñ-ñE/ô E/ñ !/Ðð !×#Ò# EÑ*Ô*Ð*r$   c                   ó¾   ‡ — e Zd ZdZ	 	 	 	 	 	 	 ddeeef         d	ed
edededeeef         fˆ fd„Zdej	        fd„Z
ddeej	                 fd„Zdd„Zdd„Zdd„Zˆ xZS )Ú	RelPosMlpzß Log-Coordinate Relative Position MLP
    Based on ideas presented in Swin-V2 paper (https://arxiv.org/abs/2111.09883)

    This impl covers the 'swin' implementation as well as two timm specific modes ('cr', and 'rw')
    rÊ   é€   r   rÉ   r   Nr�   rŽ   Ú
hidden_dimr�   r+   Úpretrained_window_sizec	           	      ó
  •— ||dœ}	t          ¦   «                              ¦   «          || _        | j        d         | j        d         z  | _        || _        || _        | j        fdz  |fz   | _        || _        || _        |dk    r"t          j
        ¦   «         | _        d| _        d}
n!t          j        ¦   «         | _        d | _        d}
t          	 d||t          j        |
d	d
œ|	¤Ž| _        | j        dz  }d|d         z  dz
  d|d         z  dz
  df}|                      dt%          j        ||t$          j        ¬¦  «        d¬¦  «         |                      dt%          j        |fi |	¤Žd¬¦  «         |                      ¦   «          d S )Nr‘   r   r   r   rÅ   é   )TFT)g      À?g        )Úhidden_featuresÚout_featuresÚ	act_layerÚbiasÚdropr!   Fr’   Úrel_coords_log)r   )r”   r•   r�   r–   r�   rŽ   r—   r+   rÙ   r2   ÚSigmoidÚbias_actÚ	bias_gainÚIdentityr   ÚReLUÚmlpr›   r   r™   rœ   r�   )rž   r�   rŽ   rØ   r�   r+   rÙ   r   rM   rŸ   Úmlp_biasr    Úrel_coords_shaper¡   s                €r"   r•   zRelPosMlp.__init__s  sÀ  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ&ˆÔØÔ+¨AÔ.°Ô1AÀ!Ô1DÑDˆÔØ*ˆÔØ"ˆŒØÔ+Ð-°Ñ1°Y°LÑ@ˆŒØˆŒ	Ø&<ˆÔ#Ø�6Š>ˆ>ÝœJ™LœLˆDŒMØˆDŒNØ$ˆHˆHåœK™MœMˆDŒMØ!ˆDŒNØˆHåØð
à&Ø"Ý”gØØð
ð 
ð ð
ð 
ˆŒð Ô%¨Ñ*ˆ
Ø ¨A¤Ñ.°Ñ2°A¸ÀA¼Ñ4FÈÑ4JÈAÐNÐØ×ÒØ%ÝŒK˜
¨6½¼ÐDÑDÔDØð 	ñ 	
ô 	
ð 	
ð
 	×ÒØÝŒKÐ(Ð/Ð/¨BÐ/Ð/Øð 	ñ 	
ô 	
ð 	
ð 	×ÒÑÔÐÐÐr$   r   c                 óà  — |                       | j        ¦  «        }| j        �@|                     d| j        ¦  «        | j                 }|                     | j        ¦  «        }|                     ddd¦  «        }|                      |¦  «        }| j        �
| j        |z  }| j	        r#t          j        || j	        d| j	        dg¦  «        }|                     d¦  «                             ¦   «         S )Nr   r   r   r   )rç   rá   r!   r:   rŽ   r—   r   rã   rä   r�   r   r   r5   r   r¬   s     r"   r®   zRelPosMlp.get_biasª  sé   € Ø!%§¢¨$Ô*=Ñ!>Ô!>ÐØÔ'Ð3Ø%;×%@Ò%@ÀÀTÄ^Ñ%TÔ%TÐUYÔUqÔ%rÐ"Ø%;×%@Ò%@ÀÄÑ%QÔ%QÐ"Ø!7×!?Ò!?ÀÀ1ÀaÑ!HÔ!HÐØ!%§¢Ð/EÑ!FÔ!FÐØŒ>Ð%Ø%)¤^Ð6LÑ%LÐ"ØÔð 	sÝ%&¤UÐ+AÀDÔDVÐXYÐ[_Ô[mÐopÐCqÑ%rÔ%rÐ"Ø%×/Ò/°Ñ2Ô2×=Ò=Ñ?Ô?Ð?r$   r¯   c                 ó0   — ||                       ¦   «         z   S r±   r²   r³   s      r"   rµ   zRelPosMlp.forward·  r¶   r$   c                 ó.   — |                       ¦   «          dS )r¤   Nr¹   r§   s    r"   r�   zRelPosMlp.reset_parametersº  r»   r$   c           	      ó.  — | j         j        }| j        j        }| j                              t          | j        |¬¦  «                             d¦  «        ¦  «         | j                             t          | j        | j	        | j
        ||¬¦  «        ¦  «         dS )r©   r   r   )r+   r   rM   N)r!   r   rá   rM   rª   r#   r�   r:   rÔ   rÙ   r+   ©rž   r   rM   s      r"   r¦   zRelPosMlp._init_buffers¾  s¤   € àÔ-Ô4ˆØÔ#Ô)ˆØÔ$×*Ò*Ý'¨Ô(8ÀÐHÑHÔH×MÒMÈbÑQÔQñ	
ô 	
ð 	
ð 	Ô×!Ò!Ý#ØÔ ØÔ+Ø”YØØðñ ô ñ	
ô 	
ð 	
ð 	
ð 	
r$   c                 ó.   — |                       ¦   «          dS r¸   r¹   r§   s    r"   rº   z%RelPosMlp.init_non_persistent_buffersÏ  r»   r$   )rÊ   r×   r   rÉ   r   NNr±   r½   )r¿   rÀ   rÁ   rÂ   r   r7   Ústrr•   r   r}   r®   r   rµ   r�   r¦   rº   rÃ   rÄ   s   @r"   rÖ   rÖ   m  s3  ø€ € € € € ðð ð Ø!Ø!"ØØ6<ØØð5 ð 5 à˜s C˜xœð5 ð ð5 ð ð	5 ð
 ð5 ð ð5 ð %*¨#¨s¨(¤Oð5 ð 5 ð 5 ð 5 ð 5 ð 5 ðn@˜%œ,ð @ð @ð @ð @ð&ð &¨H°U´\Ô,Bð &ð &ð &ð &ðð ð ð ð
ð 
ð 
ð 
ð"ð ð ð ð ð ð ð r$   rÖ   ÚlengthÚmax_relative_positionc                 óì   — |€| dz
  }d|z  dz   }t          j        | | |||¬¦  «        }t          | ¦  «        D ];}t          | ¦  «        D ])}||z
  |z   }t          ||z
  ¦  «        |k    rŒ!d||||f<   Œ*Œ<|S )aÏ  Generate a one_hot lookup tensor to reindex embeddings along one dimension.

    Args:
        length: the length to reindex to.
        max_relative_position: the maximum relative position to consider.
            Relative position embeddings for distances above this threshold
            are zeroed out.
    Returns:
        a lookup Tensor of size [length, length, vocab_size] that satisfies
            ret[n,m,v] = 1{m - n + max_relative_position = v}.
    Nr   r   r‘   )r   Úzerosrh   rÎ   )	rñ   rò   r   rM   Ú
vocab_sizeÚretrs   r€   Úvs	            r"   Úgenerate_lookup_tensorrø   Ô  s­   € ð" Ð$Ø &¨¡
ÐàÐ*Ñ*¨QÑ.€JÝ
Œ+�f˜f j¸ÀuÐ
MÑ
MÔ
M€CÝ�6‰]Œ]ð ð ˆÝ�v‘”ð 	ð 	ˆAØ�A‘Ð-Ñ-ˆAÝ�1�q‘5‰zŒzÐ1Ò1Ð1ØØˆC��1�a�‰LˆLð		ð
 €Jr$   ÚheightÚwidthÚheight_lookupÚwidth_lookupc                 ó¨   — t          j        d| |¦  «        }t          j        d||¦  «        }||z  }|                     | j        d         ||¦  «        S )a\  Reindex 2d relative position bias with 2 independent einsum lookups.

    Adapted from:
     https://github.com/google-research/maxvit/blob/2e06a7f1f70c76e64cd3dabe5cd1b8c1a23c9fb7/maxvit/models/attention_utils.py

    Args:
        relative_position_tensor: tensor of shape
            [..., vocab_height, vocab_width, ...].
        height: height to reindex to.
        width: width to reindex to.
        height_lookup: one-hot height lookup
        width_lookup: one-hot width lookup
    Returns:
        reindexed_tensor: a Tensor of shape
            [..., height * width, height * width, ...]
    znhw,ixh->nixwznixw,jyw->nijxyr   )r   Úeinsumr9   r1   )Úrelative_position_tensorrù   rú   rû   rü   Úreindexed_tensorÚareas          r"   Úreindex_2d_einsum_lookupr  ó  sY   € õ. ”| OÐ5MÈ}Ñ]Ô]ÐÝ”|Ð$5Ð7GÈÑVÔVÐØ�E‰>€DØ×#Ò#Ð$<Ô$BÀ1Ô$EÀtÈTÑRÔRÐRr$   c                   óš   ‡ — e Zd ZdZ	 	 	 ddeeef         dedefˆ fd„Zdd	„Zdd
„Zde	j
        fd„Zddee	j
                 fd„Zdd„Zˆ xZS )ÚRelPosBiasTfzÜ Relative Position Bias Impl (Compatible with Tensorflow MaxViT models)
    Adapted from:
     https://github.com/google-research/maxvit/blob/2e06a7f1f70c76e64cd3dabe5cd1b8c1a23c9fb7/maxvit/models/attention_utils.py
    r   Nr�   rŽ   r�   c                 óh  •— ||dœ}t          ¦   «                              ¦   «          |dk    sJ ‚|| _        |d         |d         z  | _        || _        d|d         z  dz
  }d|d         z  dz
  }| j        ||f| _        t          j        t          j	        | j        fi |¤Ž¦  «        | _
        |d         |d         |f}	|d         |d         |f}
|                      dt          j	        |	fi |¤Žd¬¦  «         |                      dt          j	        |
fi |¤Žd¬¦  «         |                      ¦   «          d S )	Nr‘   r   r   r   rû   Fr’   rü   )r”   r•   r�   r–   rŽ   r—   r2   r˜   r   r™   rš   r›   r�   )rž   r�   rŽ   r�   r   rM   rŸ   Úvocab_heightÚvocab_widthÚheight_lookup_shapeÚwidth_lookup_shaper¡   s              €r"   r•   zRelPosBiasTf.__init__  s]  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ Ò!Ð!Ð!Ð!Ø&ˆÔØ& qœ>¨K¸¬NÑ:ˆÔØ"ˆŒà˜; qœ>Ñ)¨AÑ-ˆØ˜+ aœ.Ñ(¨1Ñ,ˆØœ>¨<¸ÐEˆŒÝ,.¬L½¼ÀTÄ_Ð9[Ð9[ÐXZÐ9[Ð9[Ñ,\Ô,\ˆÔ)Ø*¨1œ~¨{¸1¬~¸|ÐLÐØ)¨!œn¨k¸!¬n¸kÐJÐØ×Ò˜_­e¬kÐ:MÐ.TÐ.TÐQSÐ.TÐ.TÐafÐÑgÔgÐgØ×Ò˜^­U¬[Ð9KÐ-RÐ-RÈrÐ-RÐ-RÐ_dÐÑeÔeÐeð 	×ÒÑÔÐÐÐr$   r   c                 óz   — t           j                             | j        d¬¦  «         |                      ¦   «          dS r£   )r2   ÚinitÚnormal_rš   r¦   r§   s    r"   r�   zRelPosBiasTf.reset_parameters0  s5   € å
Œ�Š˜Ô9¸sˆÑCÔCÐCØ×ÒÑÔÐÐÐr$   c                 ó
  — | j         j        }| j         j        }| j                              t	          | j        d         ||¬¦  «        ¦  «         | j                             t	          | j        d         ||¬¦  «        ¦  «         dS )r©   r   r‘   r   N)rû   r   rM   rª   rø   r�   rü   rî   s      r"   r¦   zRelPosBiasTf._init_buffers5  s€   € àÔ#Ô*ˆØÔ"Ô(ˆØÔ× Ò Õ!7¸Ô8HÈÔ8KÐTZÐbgÐ!hÑ!hÔ!hÑiÔiÐiØÔ×ÒÕ 6°tÔ7GÈÔ7JÐSYÐafÐ gÑ gÔ gÑhÔhÐhÐhÐhr$   c                 ór   — t          | j        | j        d         | j        d         | j        | j        ¦  «        S )Nr   r   )r  rš   r�   rû   rü   r§   s    r"   r®   zRelPosBiasTf.get_bias<  s<   € å'ØÔ-ØÔ˜QÔØÔ˜QÔØÔØÔñ
ô 
ð 	
r$   r¯   c                 ó0   — ||                       ¦   «         z   S r±   r²   r³   s      r"   rµ   zRelPosBiasTf.forwardF  r¶   r$   c                 ó.   — |                       ¦   «          dS r¸   r¹   r§   s    r"   rº   z(RelPosBiasTf.init_non_persistent_buffersI  r»   r$   r¼   r½   r±   r¾   rÄ   s   @r"   r  r    sÿ   ø€ € € € € ðð ð "#ØØð ð  à˜s C˜xœð ð ð ð ð	 ð  ð  ð  ð  ð  ð6ð ð ð ð
ið ið ið ið
˜%œ,ð 
ð 
ð 
ð 
ð&ð &¨H°U´\Ô,Bð &ð &ð &ð &ðð ð ð ð ð ð ð r$   r  )NFN)r)   T)r   rÅ   NN)NNN)&rÂ   rÏ   ÚosÚtypingr   r   r   Útorch.nnr2   Útorch.nn.functionalr3   r   Úgridr   r4   r   rç   r   Úweight_initr	   r7   ÚenvironÚgetrx   Úboolr}   r#   rI   rð   rZ   rŠ   ÚModulerŒ   rÔ   rÖ   rø   r  r  r^   r$   r"   ú<module>r     sR  ððð ð €€€Ø 	€	€	€	Ø "Ð "Ð "Ð "Ð "Ð "Ð "Ð "à €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð à Ð Ð Ð Ð Ð Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø Ð Ð Ð Ð Ð Ø &Ð &Ð &Ð &Ð &Ð &àˆS�”—’Ð 7¸Ñ;Ô;Ñ<Ô<¸qÒ@€
ð
 -1Ø!Øð	50ð 50Ø�c˜3�h”ð50à˜˜s C˜xœÔ)ð50ð ð50ð
 „\ð50ð 50ð 50ð 50ðp,à˜s C˜xœð,ð ˜c 3˜hœð,ð ,ð ,ð ,ðd 'Øð	#ð #ð ð#ð ð	#ð #ð #ð #ð@qà˜s C˜xœðqð ˜c 3˜hœðqð qð qð qðh;ð ;ð ;ð ;ð ;�”ñ ;ô ;ð ;ð@ 06ØØØð+ð +Ø˜˜S˜”/ð+à" 3¨ 8œ_ð+ð +ð +ð +ð>dð dð dð dð d�”	ñ dô dð dðR 04ØØð	ð Øðà'¨œ}ðð ð ð ð>SàðSð ðSð ”|ð	Sð
 ”lðSð „\ðSð Sð Sð Sð:;ð ;ð ;ð ;ð ;�2”9ñ ;ô ;ð ;ð ;ð ;r$   