§
    ·¯¸jìÐ  ã                   ó.  — d Z ddlZddlmZmZmZmZ ddlZddlmZ ddl	m
Z
 ddlmZ ddlmZ 	 	 	 d_dedededeej                 fd„Z	 	 	 d`dedededeej                 dej        f
d„Zdddddej        fdee         dededededeej                 dej        dej        fd„Zdee         dee         fd„Zddd dddd
dd!d"dej        fdee         d#eej                 ded$ededed%ed&ed'eee                  d(ed)edeej                 dej        deej                 fd*„Z G d+„ d,ej        ¦  «        Zd-„ Zd.ej        dej        fd/„Z	 dad.ej        d0ej        d1ej        d2edej        f
d3„Z 	 dad.eej                 d0ej        d1ej        d2edeej                 f
d4„Z!	 dad.ej        d5ej        d2edej        fd6„Z"	 dad.ej        d7ej        d8ej        d9edej        f
d:„Z#ddd ddd
dd!d"ddej        fdee         d#eej                 ded$ededed&ed'eee                  d(ed)ed;edeej                 dej        fd<„Z$ G d=„ d>ej        ¦  «        Z% G d?„ d@ej        ¦  «        Z&	 	 dbdej        dBœdCedDedEededFedej        fdG„Z'ej(        j)        e
d"dej        fdHee         d)edeej                 dej        deej        ej        f         f
dI„¦   «         ¦   «         Z*dJej        dKej        dLej        dej        fdM„Z+ G dN„ dOej        ¦  «        Z, G dP„ dQej        ¦  «        Z-ej(        j)        e
dRd"d!dSej        fdTedUedVed)ed(edej        dej        dej        fdW„¦   «         ¦   «         Z. G dX„ dYej        ¦  «        Z/	 	 	 dcd]ededEedej        fd^„Z0dS )dzv Sin-cos, fourier, rotary position embedding modules and functions

Hacked together by / Copyright 2022 Ross Wightman
é    N)ÚListÚTupleÚOptionalÚUnion)Únné   )Úregister_notrace_function)Úndgrid)Ú_assertç      l@TÚ	num_bandsÚmax_freqÚlinear_bandsÚdevicec                 óê   — |r't          j        d|dz  | t           j        |¬¦  «        }n<dt          j        dt          j        |d¦  «        dz
  | t           j        |¬¦  «        z  }|t           j        z  S )Nç      ð?é   ©Údtyper   r   r   )ÚtorchÚlinspaceÚfloat32ÚmathÚlogÚpi)r   r   r   r   Úbandss        úl/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/layers/pos_embed_sincos.pyÚpixel_freq_bandsr      sr   € ð ð qÝ”˜s H¨q¡L°)Å5Ä=ÐY_Ð`Ñ`Ô`ˆˆà•U”^ A¥t¤x°¸!Ñ'<Ô'<¸qÑ'@À)ÕSXÔS`ÐioÐpÑpÔpÑpˆØ•5”8ÑÐó    ç     ˆÃ@r   ÚtemperatureÚstepÚreturnc                 óœ   — t          j        d| |t           j        |¬¦  «                             t           j        ¦  «        | z  }d||z  z  }|S )Nr   r   r   ©r   ÚarangeÚint64Útor   )r   r!   r"   r   Úexpr   s         r   Ú
freq_bandsr*      sL   € õ Œ,�q˜) Tµ´ÀVÐ
LÑ
LÔ
L×
OÒ
OÕPUÔP]Ñ
^Ô
^ÐajÑ
j€CØ�+ Ñ$Ñ%€EØ€Lr   é@   FÚ
feat_shapeÚdimÚreverse_coordÚinterleave_sin_cosr   c                 óH  ‡— |dz  dk    s
J d¦   «         ‚|dz  }t          ||d‰¬¦  «        }|r| ddd…         } t          j        t          ˆfd„| D ¦   «         ¦  «        ¦  «                             d¦  «                             dd¦  «        }	|	                     d¦  «        |                     d¦  «        z  }
|rd	nd}t          j        t          j        |
¦  «        t          j        |
¦  «        g|¬
¦  «                             d¦  «        }| 	                    |¬¦  «        S )a  

    Args:
        feat_shape:
        dim:
        temperature:
        reverse_coord: stack grid order W, H instead of H, W
        interleave_sin_cos: sin, cos, sin, cos stack instead of sin, sin, cos, cos
        dtype:
        device:

    Returns:

    é   r   zHEmbed dimension must be divisible by 4 for sin-cos 2D position embeddingr   ©r!   r"   r   Néÿÿÿÿc                 óŒ   •— g | ]@}t          j        |‰t           j        ¬ ¦  «                             t           j        ¦  «        ‘ŒAS ©©r   r   r%   ©Ú.0Úsr   s     €r   ú
<listcomp>z,build_sincos2d_pos_embed.<locals>.<listcomp>D   sL   ø€ ð ð ð àõ 	Œ�Q˜v­U¬[Ð9Ñ9Ô9×<Ò<½U¼]ÑKÔKðð ð r   r   ©r-   ©r   )
r*   r   Ústackr
   ÚflattenÚ	transposeÚ	unsqueezeÚsinÚcosr(   )r,   r-   r!   r.   r/   r   r   Úpos_dimr   ÚgridÚpos2Ú	stack_dimÚpos_embs        `       r   Úbuild_sincos2d_pos_embedrH   '   s3  ø€ ð. �‰7�aŠ<ˆ<ˆ<Ðc‰<Œ<ˆ<Ø�Q‰h€GÝ�w¨K¸aÈÐOÑOÔO€Eàð &Ø   " Ô%ˆ
ÝŒ;•vð ð ð ð àðñ ô ñ ô ñ ô ÷ 	Š�‰
Œ
—9’9˜Q ‘?”?ð 	ð �>Š>˜"ÑÔ §¢°Ñ 2Ô 2Ñ2€Dð (Ð.��¨Q€IÝŒk�5œ9 T™?œ?­E¬I°d©O¬OÐ<À)ÐLÑLÔL×TÒTÐUVÑWÔW€GØ�:Š:˜Eˆ:Ñ"Ô"Ð"r   Úseqc                 ó|   — t          | ¦  «        dk     r| S | d         | d         gt          | dd …         ¦  «        z   S )Nr   r   r   )ÚlenÚlist)rI   s    r   Úswap_shape_xyrM   P   s<   € Ý
ˆ3�x„x�!‚|€|Øˆ
Ø�ŒF�C˜”FÐ�d 3 q r r¤7™mœmÑ+Ð+r   éà   ç        Úijr   Úmax_resÚinclude_gridÚ	in_pixelsÚref_feat_shapeÚgrid_offsetÚgrid_indexingc                 ó¤  ‡	‡— |€7|r!t          |t          |¦  «        |‰¬¦  «        }n&t          ||d‰¬¦  «        }n‰€|j        Š|€|j        }|
dk    r t          | ¦  «        } |�t          |¦  «        }|rˆfd„| D ¦   «         }nˆˆ	fd„| D ¦   «         }|�d„ t          || |¦  «        D ¦   «         }t          j        t          j	        ||
¬	¦  «        d
¬¦  «        }| 
                    d
¦  «        }||z  }|                     ¦   «                              |¬¦  «        |                     ¦   «                              |¬¦  «        }}|r|||gn||g}|S )aä  

    Args:
        feat_shape: Feature shape for embedding.
        bands: Pre-calculated frequency bands.
        num_bands: Number of frequency bands (determines output dim).
        max_res: Maximum resolution for pixel based freq.
        temperature: Temperature for non-pixel freq.
        linear_bands: Linear band spacing for pixel based freq.
        include_grid: Include the spatial grid in output.
        in_pixels: Output in pixel freq.
        ref_feat_shape: Reference feature shape for resize / fine-tune.
        grid_offset: Constant offset to add to grid for non-pixel freq.
        grid_indexing: Indexing mode for meshgrid ('ij' or 'xy')
        dtype: Output dtype.
        device: Output device.

    Returns:

    N)r   r   r   r2   Úxyc           	      óV   •— g | ]%}t          j        d d|‰t           j        ¬¦  «        ‘Œ&S )g      ð¿r   )Ústepsr   r   )r   r   r   r7   s     €r   r:   z+build_fourier_pos_embed.<locals>.<listcomp>”   s@   ø€ ð 
ð 
ð 
àõ ŒN˜3 ¨!°FÅ%Ä-ÐPÑPÔPð
ð 
ð 
r   c                 ó’   •— g | ]C}t          j        |‰t           j        ¬ ¦  «                             t           j        ¦  «        ‰z   ‘ŒDS r5   r%   )r8   r9   r   rU   s     €€r   r:   z+build_fourier_pos_embed.<locals>.<listcomp>™   sR   ø€ ð 
ð 
ð 
àõ ŒL˜ 6µ´Ð=Ñ=Ô=×@Ò@ÅÄÑOÔOÐR]Ñ]ð
ð 
ð 
r   c                 ó&   — g | ]\  }}}||z  |z  ‘ŒS © r]   )r8   ÚxÚfÚrs       r   r:   z+build_fourier_pos_embed.<locals>.<listcomp>    s&   € ÐIÐIÐI™7˜1˜a ˆQ�‰U�Q‰YÐIÐIÐIr   ©Úindexingr3   r;   r<   )r   Úfloatr*   r   r   rM   Úzipr   r=   Úmeshgridr@   rA   r(   rB   )r,   r   r   rQ   r!   r   rR   rS   rT   rU   rV   r   r   ÚtrD   ÚposÚpos_sinÚpos_cosÚouts            ` `       r   Úbuild_fourier_pos_embedrk   V   s½  øø€ ðF €}Øð 	Ý$ØÝ�g‘”Ø)Øð	ñ ô ˆEˆEõ ØØ'ØØð	ñ ô ˆEˆEð ˆ>Ø”\ˆFØˆ=Ø”KˆEà˜ÒÐÝ" :Ñ.Ô.ˆ
ØÐ%Ý*¨>Ñ:Ô:ˆNàð 	
ð
ð 
ð 
ð 
àð
ñ 
ô 
ˆˆð

ð 
ð 
ð 
ð 
àð
ñ 
ô 
ˆð
 Ð!àIÐI¥c¨!¨Z¸Ñ&HÔ&HÐIÑIÔIˆåŒ;•u”~ a°-Ð@Ñ@Ô@ÀbÐIÑIÔI€DØ�>Š>˜"ÑÔ€DØ
�‰,€Cà—w’w‘y”y—|’|¨%�|Ñ0Ô0°#·'²'±)´)·,²,ÀU°,Ñ2KÔ2KˆW€GØ&2Ð
Jˆ4�˜'Ð
"Ð
"¸À'Ð8J€CØ€Jr   c                   óT   ‡ — e Zd Z	 	 	 	 	 	 ddedefˆ fd„Zdd
„Zdd„Zdd„Zd„ Zˆ xZ	S )ÚFourierEmbedrN   r+   TFNrQ   r   c                 ó  •— t          ¦   «                              ¦   «          || _        || _        || _        || _        |                      dt          j        |||¬¦  «        d¬¦  «         |  	                    ¦   «          d S )Nr   r6   F©Ú
persistent)
ÚsuperÚ__init__rQ   r   Úconcat_gridÚkeep_spatialÚregister_bufferr   ÚemptyÚreset_parameters)ÚselfrQ   r   rs   rt   r   r   Ú	__class__s          €r   rr   zFourierEmbed.__init__­   s�   ø€ õ 	‰Œ×ÒÑÔÐØˆŒØ"ˆŒØ&ˆÔØ(ˆÔØ×Ò˜W¥e¤k°)ÀFÐRWÐ&XÑ&XÔ&XÐejÐÑkÔkÐkð 	×ÒÑÔÐÐÐr   r#   c                 ó.   — |                       ¦   «          dS ©z"Initialize parameters and buffers.N©Ú_init_buffers©rx   s    r   rw   zFourierEmbed.reset_parametersÀ   ó   € à×ÒÑÔÐÐÐr   c                 ój   — | j                              t          | j        | j        ¦  «        ¦  «         dS ©ú.Compute and fill non-persistent buffer values.N)r   Úcopy_r   r   rQ   r~   s    r   r}   zFourierEmbed._init_buffersÄ   s-   € àŒ
×ÒÕ)¨$¬.¸$¼,ÑGÔGÑHÔHÐHÐHÐHr   c                 ó.   — |                       ¦   «          dS ©z"Initialize non-persistent buffers.Nr|   r~   s    r   Úinit_non_persistent_buffersz(FourierEmbed.init_non_persistent_buffersÈ   r   r   c           	      ó  — |j         d d…         \  }}|j         dd …         }t          || j        | j        |j        |j        ¬¦  «        }t          j        |d¬¦  «        }|                     dd¦  «         	                    t          |¦  «        ¦  «        }|fd|j        dz
  z  z   }| j        rUt          j        ||                     d¦  «                             |¦  «                             dd	dd¦  «        gd¬¦  «        }n}t          j        |                     ddd	d¦  «        |                     d¦  «                             |¦  «        gd¬¦  «        }|                     ||                     ¦   «         d¦  «        }|S )
Nr   )rR   r   r   r3   r;   éþÿÿÿ©r3   r   r   é   )Úshaperk   r   rs   r   r   r   Úcatr?   r>   rK   Úndimrt   r@   ÚexpandÚpermuteÚreshapeÚnumel)rx   r^   ÚBÚCr,   ÚembÚbatch_expands          r   ÚforwardzFourierEmbed.forwardÌ   si  € ØŒw�r˜�rŒ{‰ˆˆ1Ø”W˜Q˜R˜R”[ˆ
Ý%ØØŒJØÔ)Ø”'Ø”8ð
ñ 
ô 
ˆõ Œi˜ Ð$Ñ$Ô$ˆØ�mŠm˜B Ñ#Ô#×+Ò+­C°
©O¬OÑ<Ô<ˆØ�t˜e q¤v°¡zÑ2Ñ2ˆð Ôð 	5Ý”	˜1˜cŸmšm¨AÑ.Ô.×5Ò5°lÑCÔC×KÒKÈAÈqÐRSÐUVÑWÔWÐXÐ^_Ð`Ñ`Ô`ˆAˆAå”	˜1Ÿ9š9 Q¨¨1¨aÑ0Ô0°#·-²-ÀÑ2BÔ2B×2IÒ2IÈ,Ñ2WÔ2WÐXÐ^`ÐaÑaÔaˆAØ—	’	˜!˜Z×-Ò-Ñ/Ô/°Ñ4Ô4ˆAàˆr   )rN   r+   TFNN©r#   N)
Ú__name__Ú
__module__Ú__qualname__Úintrr   rw   r}   r†   r–   Ú__classcell__©ry   s   @r   rm   rm   «   s·   ø€ € € € € ð ØØØØØð ð  àð ð ð ð  ð  ð  ð  ð  ð&ð ð ð ðIð Ið Ið Iðð ð ð ðð ð ð ð ð ð r   rm   c                 óŽ   — t          j        | ddd d…f          | dd d d…f         gd¦  «                             | j        ¦  «        S )N.r   r   r3   )r   r=   r�   r‹   )r^   s    r   ÚrotrŸ   ä   sI   € õ Œ;˜˜3   1 ˜9œ˜ q¨¨c¨c°¨c¨¤{Ð3°RÑ8Ô8×@Ò@ÀÄÑIÔIÐIr   r^   c                 óh   — |                       dd¬¦  «        \  }}t          j        | |gd¬¦  «        S )Nr   r3   r;   )Úchunkr   rŒ   )r^   Úx1Úx2s      r   Úrope_rotate_halfr¤   ê   s8   € ð �WŠW�Q˜BˆWÑÔ�F€BˆÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'r   Úsin_embÚcos_embÚhalfc                 óf   — |r| |z  t          | ¦  «        |z  z   S | |z  t          | ¦  «        |z  z   S ©N)r¤   rŸ   ©r^   r¥   r¦   r§   s       r   Úapply_rot_embedr«   ñ   sG   € ð ð 	.ð �7‰{Õ-¨aÑ0Ô0°7Ñ:Ñ:Ð:ð
 �7‰{�S ™VœV gÑ-Ñ-Ð-r   c                 ó€   ‡‡— t          | t          j        ¦  «        r| g} |rˆˆfd„| D ¦   «         S ˆˆfd„| D ¦   «         S )Nc                 ó@   •— g | ]}|‰z  t          |¦  «        ‰z  z   ‘ŒS r]   )r¤   ©r8   rf   r¦   r¥   s     €€r   r:   z(apply_rot_embed_list.<locals>.<listcomp>  s0   ø€ ÐGÐGÐGÀ��G‘Õ.¨qÑ1Ô1°GÑ;Ñ;ÐGÐGÐGr   c                 ó@   •— g | ]}|‰z  t          |¦  «        ‰z  z   ‘ŒS r]   )rŸ   r®   s     €€r   r:   z(apply_rot_embed_list.<locals>.<listcomp>  s-   ø€ Ð:Ð:Ð:°1��G‘�c !™fœf wÑ.Ñ.Ð:Ð:Ð:r   )Ú
isinstancer   ÚTensorrª   s    `` r   Úapply_rot_embed_listr²     sh   øø€ õ �!•U”\Ñ"Ô"ð ØˆCˆàð 	;ð HÐGÐGÐGÐGÀQÐGÑGÔGÐGð
 ;Ð:Ð:Ð:Ð:¸Ð:Ñ:Ô:Ð:r   r”   c                 ó˜   — |                      dd¦  «        \  }}|r| |z  t          | ¦  «        |z  z   S | |z  t          | ¦  «        |z  z   S )Nr   r3   )r¡   r¤   rŸ   )r^   r”   r§   r¥   r¦   s        r   Úapply_rot_embed_catr´     s_   € ð
 —y’y  BÑ'Ô'Ñ€GˆWàð 	.ð �7‰{Õ-¨aÑ0Ô0°7Ñ:Ñ:Ð:ð
 �7‰{�S ™VœV gÑ-Ñ-Ð-r   Ú	pos_embedÚkeep_indicesÚpos_embed_has_batchc                 óø  — |rt          |j        dk    d¦  «         nZt          |j        dk    d¦  «         | j        d         fd|j        z  z   }|                     d¦  «                             |¦  «        }|j        d         fd|j        dz
  z  z   |j        d         dfz   }|                     |¦  «        }t          |j        ¦  «        }d|d	<   |                     |¦  «        }|                     d	|¦  «        S )
aá   Apply keep indices to different ROPE shapes

    Expected pos_embed shapes:
    * [seq_len, pos_embed_dim] --> output [batch_size, seq_len, pos_embed_dim]
    * [num_heads, seq_len, pos_embed_dim] --> output [batch_size, num_heads, seq_len, pos_embed_dim]
    * [depth, num_heads, seq_len, pos_embed_dim] --> output [batch_size, depth, num_heads, seq_len, pos_embed_dim]

    And all of the above with leading batch dimension already present if `pos_embed_has_batch == True`

    rŠ   zIncorrect number of dimensionsr   r   r‰   )r   r   r3   rˆ   )r   r�   r‹   r@   rŽ   ÚviewrL   Úgather)r^   rµ   r¶   r·   Úexpand_shapeÚ
keep_shapeÚkeep_expands          r   Úapply_keep_indices_nlcr¾   ,  s	  € ð  ð @å�	” !Ò#Ð%EÑFÔFÐFÐFõ 	�	” !Ò#Ð%EÑFÔFÐFØœ œ
�} u¨y¬~Ñ'=Ñ=ˆØ×'Ò'¨Ñ*Ô*×1Ò1°,Ñ?Ô?ˆ	ð Ô$ QÔ'Ð)¨D°I´NÀQÑ4FÑ,GÑGÈ<ÔK]Ð^_ÔK`ÐbcÐJdÑd€JØ×$Ò$ ZÑ0Ô0€Lõ �y”Ñ'Ô'€KØ€K��OØ×&Ò& {Ñ3Ô3€Là×Ò˜B Ñ-Ô-Ð-r   Úrotate_halfc                 óz  — t          | ||dz  |||||||	||¬¦  «        \  }}d}| D ]}||z  }Œ|                     |d¦  «        }|                     |d¦  «        }|
r1t          j        ||gd¬¦  «        }t          j        ||gd¬¦  «        }n,|                     dd¦  «        }|                     dd¦  «        }||fS )a¦  

    Args:
        feat_shape: Spatial shape of the target tensor for embedding.
        bands: Optional pre-generated frequency bands
        dim: Output dimension of embedding tensor.
        max_res: Maximum resolution for pixel mode.
        temperature: Temperature (inv freq) for non-pixel mode
        linear_bands: Linearly (instead of log) spaced bands for pixel mode
        in_pixels: Pixel vs language (inv freq) mode.
        ref_feat_shape: Reference feature shape for resize / fine-tune.
        grid_offset: Constant offset to add to grid for non-pixel freq.
        grid_indexing: Indexing mode for meshgrid ('ij' or 'xy')
        rotate_half: Emit the 'half' layout ([f0, f1, .., f0, f1, ..], for apply_rot_embed(half=True)) instead
            of the default interleaved layout ([f0, f0, f1, f1, ..], for apply_rot_embed(half=False)).
        device: Output device.
        dtype: Output dtype.

    Returns:

    r1   )r   r   rQ   r!   r   rS   rT   rU   rV   r   r   r   r3   r;   r   )rk   r�   r   rŒ   Úrepeat_interleave)r,   r   r-   rQ   r!   r   rS   rT   rU   rV   r¿   r   r   r¥   r¦   Únum_spatial_dimr^   s                    r   Úbuild_rotary_pos_embedrÃ   Q  sÿ   € õH /ØØØ˜‘(ØØØ!ØØ%ØØ#ØØðñ ô Ñ€GˆWð €Oàð ð ˆØ˜1ÑˆˆØ�oŠo˜o¨rÑ2Ô2€GØ�oŠo˜o¨rÑ2Ô2€GØð 3Ý”)˜W gÐ.°BÐ7Ñ7Ô7ˆÝ”)˜W gÐ.°BÐ7Ñ7Ô7ˆˆà×+Ò+¨A¨rÑ2Ô2ˆØ×+Ò+¨A¨rÑ2Ô2ˆØ�GÐÐr   c                   ó
  ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 dd	ed
eee                  deee                  dede	f
ˆ fd„Z
dd„Zdd„Zdd„Zdej        fd
ee         fd„Zdd„Zd
ee         fd„Zddeee                  fd„Zd„ Zˆ xZS )ÚRotaryEmbeddinga®   Rotary position embedding

    NOTE: This is my initial attempt at impl rotary embedding for spatial use, it has not
    been well tested, and will likely change. It will be moved to its own file.

    The following impl/resources were referenced for this impl:
    * https://github.com/lucidrains/vit-pytorch/blob/6f3a5fcf0bca1c5ec33a35ef48d97213709df4ba/vit_pytorch/rvt.py
    * https://blog.eleuther.ai/rotary-embeddings/
    rN   é'  TFNrO   rP   r   r,   rT   rU   rV   c                 ój  •— t          ¦   «                              ¦   «          || _        || _        || _        || _        || _        || _        || _        || _	        |	| _
        |d u| _        |€B|dz  f}|                      dt          j        ||
|¬¦  «        d¬¦  «         d | _        d | _        nqd | _        d}|D ]}||z  }Œ||f}|                      dt          j        ||
|¬¦  «        d¬¦  «         |                      dt          j        ||
|¬¦  «        d¬¦  «         |                      ¦   «          d S )	Nr1   r   r6   Fro   r   Úpos_embed_sinÚpos_embed_cos)rq   rr   r-   rQ   r!   r   rS   r,   rT   rU   rV   Ú_use_cached_embedru   r   rv   rÈ   rÉ   r   rw   )rx   r-   rQ   r!   rS   r   r,   rT   rU   rV   r   r   Úbands_shapeÚnum_posr9   Ú	emb_shapery   s                   €r   rr   zRotaryEmbedding.__init__�  si  ø€ õ 	‰Œ×ÒÑÔÐØˆŒØˆŒØ&ˆÔØ(ˆÔØ"ˆŒØ$ˆŒØ,ˆÔØ&ˆÔØ*ˆÔð ",°4Ð!7ˆÔàÐà !™8˜+ˆKØ× Ò  ­%¬+°kÈ&ÐX]Ð*^Ñ*^Ô*^ÐkpÐ ÑqÔqÐqØ!%ˆDÔØ!%ˆDÔÐð ˆDŒJØˆGØð ð �Ø˜1‘��Ø  #˜ˆIØ× Ò  µ%´+¸iÐPVÐ^cÐ2dÑ2dÔ2dÐqvÐ ÑwÔwÐwØ× Ò  µ%´+¸iÐPVÐ^cÐ2dÑ2dÔ2dÐqvÐ ÑwÔwÐwð 	×ÒÑÔÐÐÐr   r#   c                 ó.   — |                       ¦   «          dS r{   r|   r~   s    r   rw   z RotaryEmbedding.reset_parametersÌ  r   r   c                 ó  — | j         s.| j                             |                      ¦   «         ¦  «         dS |                      | j        ¦  «        \  }}| j                             |¦  «         | j                             |¦  «         dS r�   )rÊ   r   rƒ   Ú_compute_bandsÚ_get_pos_embed_valuesr,   rÈ   rÉ   )rx   Úemb_sinÚemb_coss      r   r}   zRotaryEmbedding._init_buffersÐ  sƒ   € àÔ%ð 	.ØŒJ×Ò˜T×0Ò0Ñ2Ô2Ñ3Ô3Ð3Ð3Ð3à#×9Ò9¸$¼/ÑJÔJÑˆG�WØÔ×$Ò$ WÑ-Ô-Ð-ØÔ×$Ò$ WÑ-Ô-Ð-Ð-Ð-r   c                 óà   — | j         r2t          | j        dz  t          | j        ¦  «        | j        ¬¦  «        }nt          | j        dz  | j        d¬¦  «        }|                     ||¬¦  «        S ©zCompute frequency bands.r1   )r   r   )r!   r"   r6   ©	rS   r   r-   rc   rQ   r   r*   r!   r(   ©rx   r   r   r   s       r   rÐ   zRotaryEmbedding._compute_bandsÙ  ó~   € àŒ>ð 	Ý$Ø”˜A‘Ý�d”lÑ#Ô#Ø!Ô.ðñ ô ˆEˆEõ Ø”˜A‘Ø Ô,Øðñ ô ˆEð
 �xŠx˜v¨UˆxÑ3Ô3Ð3r   c                 ó”   — t          || j        | j        | j        | j        | j        | j        | j        | j        ||¬¦  «        \  }}||fS )N)r,   r-   rQ   r!   r   rS   rT   rU   rV   r   r   )	rÃ   r-   rQ   r!   r   rS   rT   rU   rV   )rx   r,   r   r   rÒ   rÓ   s         r   rÑ   z%RotaryEmbedding._get_pos_embed_valuesé  s^   € Ý1Ø!Ø”Ø”LØÔ(ØÔ*Ø”nØÔ.ØÔ(ØÔ,ØØð
ñ 
ô 
Ñˆ�ð ˜ÐÐr   c                 ó.   — |                       ¦   «          dS r…   r|   r~   s    r   r†   z+RotaryEmbedding.init_non_persistent_buffersù  r   r   c                 óÖ   — | j         �_|| j         k    rV| j        €J ‚| j        €J ‚|                      || j        j        | j        j        ¬¦  «        \  | _        | _        || _         d S d S d S ©Nr6   )r,   rÈ   rÉ   rÑ   r   r   ©rx   r,   s     r   Úupdate_feat_shapez!RotaryEmbedding.update_feat_shapeý  sŠ   € ØŒ?Ð&¨:¸¼Ò+HÐ+HàÔ%Ð1Ð1Ð1ØÔ%Ð1Ð1Ð1Ø59×5OÒ5OØØÔ)Ô0ØÔ(Ô.ð 6Pñ 6ô 6Ñ2ˆDÔ Ô 2ð
 )ˆDŒOˆOˆOð 'Ð&Ð+HÐ+Hr   r‹   c                 ó¼   — |�5| j         �.t          || j         | j        | j        | j        | j        ¬¦  «        S | j        �| j        �| j        | j        fS J d¦   «         ‚)N)rS   rT   rU   rV   FzQget_embed() requires pre-computed pos embeds or valid shape w/ pre-computed bands)r   rÃ   rS   rT   rU   rV   rÈ   rÉ   )rx   r‹   s     r   Ú	get_embedzRotaryEmbedding.get_embed	  sw   € ØÐ ¤Ð!7å)ØØ”
Øœ.Ø#Ô2Ø Ô,Ø"Ô0ðñ ô ð ð ÔÐ+°Ô0BÐ0NØÔ% tÔ'9Ð9Ð9àmÐmÑmÔm�5r   c                 ón   — |                       |j        dd …         ¦  «        \  }}t          |||¦  «        S ©Nr   )rà   r‹   r«   )rx   r^   r¥   r¦   s       r   r–   zRotaryEmbedding.forward  s4   € àŸ>š>¨!¬'°!°"°"¬+Ñ6Ô6Ñˆ�Ý˜q '¨7Ñ3Ô3Ð3r   )
rN   rÆ   TFNNrO   rP   NNr—   ©NNr©   )r˜   r™   rš   Ú__doc__Úboolr   r   r›   rc   Ústrrr   rw   r}   rÐ   r   r   rÑ   r†   rÞ   rà   r–   rœ   r�   s   @r   rÅ   rÅ   ’  s�  ø€ € € € € ðð ð ØØØ!&Ø.2Ø26Ø!#Ø!%ØØð- ð - ð ð- ð !  c¤Ô+ð- ð % T¨#¤YÔ/ð- ð ð- ð ð- ð - ð - ð - ð - ð - ð^ð ð ð ð.ð .ð .ð .ð4ð 4ð 4ð 4ð  CGÈeÌmð  ð  °°S´	ð  ð  ð  ð  ð ð ð ð ð
)¨D°¬Ið 
)ð 
)ð 
)ð 
)ðnð n˜x¨¨S¬	Ô2ð nð nð nð nð 4ð 4ð 4ð 4ð 4ð 4ð 4r   rÅ   c                   ó�  ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 d!d	ed
ededededeee                  deee                  dede	defˆ fd„Z
d"d„Zd"d„Zd#d„Zdej        fdee         fd„Zd"d„Zdee         fd„Zd$deee                  fd„Z	 d$deeeef                  dee         deej        eej                 f         fd„Zd „ Zˆ xZS )%ÚRotaryEmbeddingCata   Rotary position embedding w/ concatenatd sin & cos

    The following impl/resources were referenced for this impl:
    * https://github.com/lucidrains/vit-pytorch/blob/6f3a5fcf0bca1c5ec33a35ef48d97213709df4ba/vit_pytorch/rvt.py
    * https://blog.eleuther.ai/rotary-embeddings/
    rN   rÆ   TFNrO   rP   r-   rQ   r!   rS   r   r,   rT   rU   rV   r¿   c                 ó  •— t          ¦   «                              ¦   «          || _        || _        || _        || _        || _        || _        || _        || _	        |	| _
        |
| _        |d u| _        |€;|dz  f}|                      dt          j        |||¬¦  «        d¬¦  «         d | _        nGd | _        d}|D ]}||z  }Œ||dz  f}|                      dt          j        |||¬¦  «        d¬¦  «         |                      ¦   «          d S )	Nr1   r   r6   Fro   r   r   rµ   )rq   rr   r-   rQ   r!   rS   r   r,   rT   rU   rV   r¿   rÊ   ru   r   rv   rµ   r   rw   )rx   r-   rQ   r!   rS   r   r,   rT   rU   rV   r¿   r   r   rË   rÌ   r9   rÍ   ry   s                    €r   rr   zRotaryEmbeddingCat.__init__'  s<  ø€ õ 	‰Œ×ÒÑÔÐØˆŒØˆŒØ&ˆÔØ"ˆŒØ(ˆÔØ$ˆŒØ,ˆÔØ&ˆÔØ*ˆÔà&ˆÔð ",°4Ð!7ˆÔàÐà !™8˜+ˆKØ× Ò  ­%¬+°kÈ&ÐX]Ð*^Ñ*^Ô*^ÐkpÐ ÑqÔqÐqØ!ˆDŒNˆNð ˆDŒJØˆGØð ð �Ø˜1‘��Ø  #¨¡'Ð*ˆIØ× Ò  ­e¬k¸)ÈFÐZ_Ð.`Ñ.`Ô.`ÐmrÐ ÑsÔsÐsð 	×ÒÑÔÐÐÐr   r#   c                 ó.   — |                       ¦   «          dS r{   r|   r~   s    r   rw   z#RotaryEmbeddingCat.reset_parametersW  r   r   c                 óÔ   — | j         s.| j                             |                      ¦   «         ¦  «         dS | j                             |                      | j        ¦  «        ¦  «         dS r�   )rÊ   r   rƒ   rÐ   rµ   rÑ   r,   r~   s    r   r}   z RotaryEmbeddingCat._init_buffers[  sb   € àÔ%ð 	NØŒJ×Ò˜T×0Ò0Ñ2Ô2Ñ3Ô3Ð3Ð3Ð3àŒN× Ò  ×!;Ò!;¸D¼OÑ!LÔ!LÑMÔMÐMÐMÐMr   c                 óà   — | j         r2t          | j        dz  t          | j        ¦  «        | j        ¬¦  «        }nt          | j        dz  | j        d¬¦  «        }|                     ||¬¦  «        S rÕ   rÖ   r×   s       r   rÐ   z!RotaryEmbeddingCat._compute_bandsb  rØ   r   c                 ó¼   — t          || j        | j        | j        | j        | j        | j        | j        | j        | j	        ||¬¦  «        }t          j        |d¦  «        S )N)r,   r-   rQ   r!   r   rS   rT   rU   rV   r¿   r   r   r3   )rÃ   r-   rQ   r!   r   rS   rT   rU   rV   r¿   r   rŒ   )rx   r,   r   r   Úembedss        r   rÑ   z(RotaryEmbeddingCat._get_pos_embed_valuesr  sf   € Ý'Ø!Ø”Ø”LØÔ(ØÔ*Ø”nØÔ.ØÔ(ØÔ,ØÔ(ØØð
ñ 
ô 
ˆõ Œy˜ Ñ$Ô$Ð$r   c                 ó.   — |                       ¦   «          dS r…   r|   r~   s    r   r†   z.RotaryEmbeddingCat.init_non_persistent_buffersƒ  r   r   c                 ó´   — | j         �N|| j         k    rE| j        €J ‚|                      || j        j        | j        j        ¬¦  «        | _        || _         d S d S d S rÜ   )r,   rµ   rÑ   r   r   rÝ   s     r   rÞ   z$RotaryEmbeddingCat.update_feat_shape‡  sn   € ØŒ?Ð&¨:¸¼Ò+HÐ+Hà”>Ð-Ð-Ð-Ø!×7Ò7ØØ”~Ô,Ø”nÔ*ð 8ñ ô ˆDŒNð
 )ˆDŒOˆOˆOð 'Ð&Ð+HÐ+Hr   r‹   c           	      óÖ   — |�P| j         �It          || j         | j        | j        | j        | j        | j        ¬¦  «        }t          j        |d¦  «        S | j	        �| j	        S J d¦   «         ‚)N)rS   rT   rU   rV   r¿   r3   FzPget_embed() requires pre-computed pos embed or valid shape w/ pre-computed bands)
r   rÃ   rS   rT   rU   rV   r¿   r   rŒ   rµ   )rx   r‹   rî   s      r   rà   zRotaryEmbeddingCat.get_embed’  sz   € ØÐ ¤Ð!7å+ØØ”
Øœ.Ø#Ô2Ø Ô,Ø"Ô0Ø Ô,ðñ ô ˆFõ ”9˜V RÑ(Ô(Ð(ØŒ^Ð'Ø”>Ð!àlÐlÑlÔl�5r   ÚshapesÚseq_lenc           	      óž  ‡— |sg S | j         €t          d¦  «        ‚t          d„ |D ¦   «         ¦  «        }t          d„ |D ¦   «         ¦  «        }t          ||f| j         | j        | j        | j        | j        | j        ¬¦  «        \  }}t          j
        ||gd¬¦  «                             ||d¦  «        Š|�‰t          j        t          |¦  «        |‰j        d         ¦  «                             |¦  «        }t!          |¦  «        D ]6\  }\  }	}
|	|
z  }‰d|	…d|
…f                              |d¦  «        ||d|…f<   Œ7|S ˆfd„|D ¦   «         }|S )	a»  Generate ROPE embeddings for multiple grid shapes efficiently.

        Computes embeddings for the maximum grid size once, then extracts
        and flattens the relevant portions for each requested shape.

        Args:
            shapes: List of (H, W) tuples representing different grid sizes

        Returns:
            List of concatenated sin/cos embeddings for each shape,
            where each tensor has shape (H*W, dim)
        NzMBatch embedding generation requires cached bands, not pre-computed embeddingsc              3   ó    K  — | ]	\  }}|V — Œ
d S r©   r]   ©r8   ÚhÚws      r   ú	<genexpr>z6RotaryEmbeddingCat.get_batch_embeds.<locals>.<genexpr>¾  ó&   è è € Ð)Ð)™$˜!˜Q�AÐ)Ð)Ð)Ð)Ð)Ð)r   c              3   ó    K  — | ]	\  }}|V — Œ
d S r©   r]   rö   s      r   rù   z6RotaryEmbeddingCat.get_batch_embeds.<locals>.<genexpr>¿  rú   r   )r,   r   rS   rT   rU   rV   r¿   r3   r;   c                 ó`   •— g | ]*\  }}‰d |…d |…f                               ||z  d¦  «        ‘Œ+S )Nr3   )r�   )r8   r÷   rø   Úrope_embed_2ds      €r   r:   z7RotaryEmbeddingCat.get_batch_embeds.<locals>.<listcomp>×  sC   ø€ Ð\Ð\Ð\ÉTÈQÐPQ ¨b¨q¨b°"°1°"¨fÔ 5× =Ò =¸aÀ!¹eÀRÑ HÔ HÐ\Ð\Ð\r   )r   ÚRuntimeErrorÚmaxrÃ   rS   rT   rU   rV   r¿   r   rŒ   r¹   ÚzerosrK   r‹   Útype_asÚ	enumerater�   )rx   rò   ró   Úmax_hÚmax_wr¥   r¦   Úflat_embedsÚir÷   rø   Úsrc_lenÚflat_embeds_listrý   s                @r   Úget_batch_embedsz#RotaryEmbeddingCat.get_batch_embeds¤  s–  ø€ ð" ð 	ØˆIð Œ:ÐåÐnÑoÔoÐoõ Ð)Ð) &Ð)Ñ)Ô)Ñ)Ô)ˆÝÐ)Ð) &Ð)Ñ)Ô)Ñ)Ô)ˆõ 2Ø˜u�~Ø”*Ø”nØÔ.ØÔ(ØÔ,ØÔ(ð
ñ 
ô 
Ñˆ�õ œ	 7¨GÐ"4¸"Ð=Ñ=Ô=×BÒBÀ5È%ÐQSÑTÔTˆàÐÝœ+¥c¨&¡k¤k°7¸MÔ<OÐPRÔ<SÑTÔT×\Ò\Ð]dÑeÔeˆKÝ& vÑ.Ô.ð Vð V‘	�‘6�A�qØ˜a™%�Ø+8¸¸!¸¸R¸a¸R¸Ô+@×+HÒ+HÈÐRTÑ+UÔ+U�˜A˜x ˜x˜KÑ(Ð(ØÐà\Ð\Ð\Ð\ÐU[Ð\Ñ\Ô\ÐØ#Ð#r   c                 ót   — |                       |j        dd …         ¦  «        }t          ||| j        ¬¦  «        S )Nr   ©r§   ©rà   r‹   r´   r¿   ©rx   r^   rµ   s      r   r–   zRotaryEmbeddingCat.forwardÚ  s5   € à—N’N 1¤7¨1¨2¨2¤;Ñ/Ô/ˆ	Ý" 1 i°dÔ6FÐGÑGÔGÐGr   )rN   rÆ   TFNNrO   rP   FNNr—   rã   r©   )r˜   r™   rš   rä   r›   rc   rå   r   r   ræ   rr   rw   r}   rÐ   r   r   rÑ   r†   rÞ   rà   r   r   r±   r	  r–   rœ   r�   s   @r   rè   rè     s*  ø€ € € € € ðð ð Ø!&Ø"Ø!&Ø.2Ø26Ø!#Ø!%Ø %ØØð. ð . àð. ð ð. ð ð	. ð
 ð. ð ð. ð !  c¤Ô+ð. ð % T¨#¤YÔ/ð. ð ð. ð ð. ð ð. ð . ð . ð . ð . ð . ð`ð ð ð ðNð Nð Nð Nð4ð 4ð 4ð 4ð  CGÈeÌmð %ð %°°S´	ð %ð %ð %ð %ð"ð ð ð ð	)¨D°¬Ið 	)ð 	)ð 	)ð 	)ðmð m˜x¨¨S¬	Ô2ð mð mð mð mð* &*ð4$ð 4$à˜˜s C˜xœÔ)ð4$ð ˜c”]ð4$ð 
ˆuŒ|˜T %¤,Ô/Ð/Ô	0ð	4$ð 4$ð 4$ð 4$ðlHð Hð Hð Hð Hð Hð Hr   rè   ç      $@r6   Úhead_dimÚdepthÚ	num_headsÚrotatec          	      ó˜  — d|t          j        d| d||¬¦  «        | z  z  z  }|                     d¦  «                             d¦  «        }|r*t          j        ||d||¬¦  «        dz  t           j        z  }nt          j        ||d||¬¦  «        }t          j        |t          j        |¦  «        z  |t          j        |t           j        dz  z   ¦  «        z  gd¬¦  «        }	t          j        |t          j        |¦  «        z  |t          j        |t           j        dz  z   ¦  «        z  gd¬¦  «        }
t          j	        |	|
gd¬¦  «        S )	zŽ Vectorised 2D ROPE frequencies with random rotation for mixed mode ROPE.
    Returns:
         Tensor (2, depth, num_heads, head_dim//2)
    r   r   r1   r6   r   r   r3   r;   )
r   r&   r@   Úrandr   r   rŒ   rB   rA   r=   )r  r  r  r!   r  r   r   ÚmagÚanglesÚfxÚfys              r   Úinit_random_2d_freqsr  à  sD  € ð �¥¤¨a°¸1ÀVÐSXÐ!YÑ!YÔ!YÐ\dÑ!dÑeÑ
f€Cð �-Š-˜Ñ
Ô
×
$Ò
$ QÑ
'Ô
'€Cð ð NÝ”˜E 9¨a¸ÀeÐLÑLÔLÈqÑPÕSXÔS[Ñ[ˆˆå”˜U I¨q¸ÀuÐMÑMÔMˆõ 
Œ�C�%œ) FÑ+Ô+Ñ+¨Sµ5´9¸VÅeÄhÐQRÁlÑ=RÑ3SÔ3SÑ-SÐTÐZ\Ð	]Ñ	]Ô	]€BÝ	Œ�C�%œ) FÑ+Ô+Ñ+¨Sµ5´9¸VÅeÄhÐQRÁlÑ=RÑ3SÔ3SÑ-SÐTÐZ\Ð	]Ñ	]Ô	]€Bõ Œ;˜˜B�x QÐ'Ñ'Ô'Ð'r   r‹   c                 ó˜  — |dk    rt          | ¦  «        } t          j        t          j        | d         |t          j        ¬¦  «        t          j        | d         |t          j        ¬¦  «        |¬¦  «        \  }}|                     |¦  «                             ¦   «         }|                     |¦  «                             ¦   «         }||fS )NrX   r   r6   r   ra   )rM   r   re   r&   r   r(   r>   )r‹   rV   r   r   Úx_posÚy_posÚt_xÚt_ys           r   Úget_mixed_gridr    s®   € ð ˜ÒÐÝ˜eÑ$Ô$ˆÝ”>ÝŒ�U˜1”X fµE´MÐBÑBÔBÝŒ�U˜1”X fµE´MÐBÑBÔBØðñ ô �L€Eˆ5ð
 �(Š(�5‰/Œ/×
!Ò
!Ñ
#Ô
#€CØ
�(Š(�5‰/Œ/×
!Ò
!Ñ
#Ô
#€CØ�ˆ8€Or   Úfreqsr  r  c                 ó   — | j         }|                      ¦   «         } |                     d¦  «        | d                              d¦  «        z  }|                     d¦  «        | d                              d¦  «        z  }||z   }t          j        |¦  «                             dd¦  «        }t          j        |¦  «                             dd¦  «        }t          j        ||gd¬¦  «        }	|	                     |¦  «        S )z&Compute mixed (learnable) frequencies.r3   r   rˆ   r   r   r;   )	r   rc   r@   r   rA   rÁ   rB   rŒ   r(   )
r   r  r  r   Úfreqs_xÚfreqs_yÚcombinedr¥   r¦   Úrope_embedss
             r   Úget_mixed_freqsr&    sÛ   € ð ŒK€EØ�KŠK‰MŒM€EØ�}Š}˜RÑ Ô  5¨¤8×#5Ò#5°bÑ#9Ô#9Ñ9€GØ�}Š}˜RÑ Ô  5¨¤8×#5Ò#5°bÑ#9Ô#9Ñ9€GØ˜Ñ €HÝŒi˜Ñ!Ô!×3Ò3°A°rÑ:Ô:€GÝŒi˜Ñ!Ô!×3Ò3°A°rÑ:Ô:€GÝ”)˜W gÐ.°BÐ7Ñ7Ô7€KØ�>Š>˜%Ñ Ô Ð r   c            	       óx   ‡ — e Zd ZdZ	 	 	 	 	 ddedeeeef         ded	efˆ fd
„Zde	e         de
j        fd„Zˆ xZS )ÚRotaryEmbeddingMRopeaa  Interleaved multimodal RoPE (Qwen2-VL style) for vision, matching the reference GenLIP layout.

    Drop-in sibling of ``RotaryEmbeddingCat``: ``get_embed(shape) -> [N, 2*dim]``, consumed by
    ``apply_rot_embed_cat(..., half=True)`` (no new apply path / no separate sin/cos tensors). The ``dim // 2``
    frequency channels are assigned to height/width/temporal axes in a strided ``T,H,W,T,H,W,...`` interleave
    (the reference ``apply_interleaved_mrope``): channels ``1,4,7,...`` -> height, ``2,5,8,...`` -> width, and
    the remainder -> temporal. ``mrope_section`` sets the strided extent per axis; the actual per-axis channel
    *counts* equal ``mrope_section`` only for the standard equal-section configs that tile exactly
    (``3*section == dim // 2``, e.g. ``(12,12,12)`` -> 36 channels = 12/12/12), and are otherwise the clamped
    interleave (e.g. ``dim=64, (8,12,12)`` -> 11/11/10) -- this matches the reference, which also clamps.

    For an image encoder there is no text, so the temporal channels sit at position 0 (inert) and this reduces
    to a 2-axis ``(h, w)`` rope -- numerically identical to a checkpoint trained with the reference MRoPE.

    Only ``grid_indexing='ij'`` is supported (GenLIP / NaFlex ``(y, x)`` row-major patch order); ``'xy'`` would
    require mirroring the timm axial shape-swap and is intentionally not implemented here.
    ©é   é   r+  r    rP   Nr-   Úmrope_sectionr!   rV   c                 óX  •— t          ¦   «                              ¦   «          |dz  dk    s
J d¦   «         ‚t          d„ |D ¦   «         ¦  «        sJ d|› �¦   «         ‚|dk    s
J d¦   «         ‚|| _        || _        || _        || _        d|t          j        d|d|¬	¦  «         	                    ¦   «         |z  z  z  }|  
                    d
|d¬¦  «         |\  }}	}
t          j        |dz  t          j        |¬¦  «        }d|d|	dz  d…<   d|d|
dz  d…<   |  
                    d|d¬¦  «         d S )Nr   r   zdim (head_dim) must be evenc              3   ó"   K  — | ]
}|d k    V — ŒdS )r   Nr]   )r8   r9   s     r   rù   z0RotaryEmbeddingMRope.__init__.<locals>.<genexpr>H  s&   è è € Ð1Ð1˜a�1˜’6Ð1Ð1Ð1Ð1Ð1Ð1r   z0mrope_section entries must be non-negative, got rP   zXRotaryEmbeddingMRope supports grid_indexing='ij' only (GenLIP/NaFlex (y,x) patch order).r   ©r   Úinv_freqFro   r   r   rŠ   Úaxis)rq   rr   Úallr-   r,  r!   rV   r   r&   rc   ru   r   Úlong)rx   r-   r,  r!   rV   r   r   r0  Ú_sec_tÚsec_hÚsec_wr1  ry   s               €r   rr   zRotaryEmbeddingMRope.__init__;  st  ø€ õ 	‰Œ×ÒÑÔÐØ�Q‰w˜!Š|ˆ|ˆ|Ð:‰|Œ|ˆ|õ Ð1Ð1 =Ð1Ñ1Ô1Ñ1Ô1ð 	Oð 	OØN¸}ÐNÐNñ	Oô 	OÐ1à Ò$Ð$Ð$Øfñ %Ô$Ð$àˆŒØ*ˆÔØ&ˆÔØ*ˆÔð ˜+­%¬,°q¸#¸qÈÐ*PÑ*PÔ*P×*VÒ*VÑ*XÔ*XÐ[^Ñ*^Ñ_Ñ`ˆØ×Ò˜Z¨¸eÐÑDÔDÐDð
  -Ñˆ��uÝŒ{˜3 !™8­5¬:¸fÐEÑEÔEˆØˆˆQˆu�q‰y˜ˆ]ÑØˆˆQˆu�q‰y˜ˆ]ÑØ×Ò˜V T°eÐÑ<Ô<Ð<Ð<Ð<r   r‹   r#   c                 óÎ  — |\  }}| j         j        }t          j        t          j        ||¬¦  «        t          j        ||¬¦  «        | j        ¬¦  «        \  }}|                     d¦  «                             ¦   «         |                     d¦  «                             ¦   «         }}t          j        |j	        d         | j
        dz  |¬¦  «        }|dd…df         |dd…| j        dk    f<   |dd…df         |dd…| j        dk    f<   || j         z  }t          j        ||gd¬¦  «        }	t          j        |	                     ¦   «         |	                     ¦   «         gd¬¦  «        S )	z©Args:
            shape: ``(H, W)`` patch grid.

        Returns:
            Rope tensor of shape ``[H*W, 2*dim]`` for ``apply_rot_embed_cat(..., half=True)``.
        r/  ra   r3   r   r   Nr   r;   )r0  r   r   re   r&   rV   r�   rc   r   r‹   r-   r1  rŒ   rA   rB   )
rx   r‹   r÷   rø   r   ÚysÚxsrg   r  r”   s
             r   rà   zRotaryEmbeddingMRope.get_embed^  sO  € ð ‰ˆˆ1Ø”Ô%ˆÝ”ÝŒL˜ 6Ð*Ñ*Ô*ÝŒL˜ 6Ð*Ñ*Ô*ØÔ'ð
ñ 
ô 
‰ˆˆBð
 —’˜B‘”×%Ò%Ñ'Ô'¨¯ª°B©¬×)=Ò)=Ñ)?Ô)?ˆBˆåŒk˜"œ( 1œ+ t¤x°1¡}¸VÐDÑDÔDˆØ!# A A A t G¤ˆˆAˆAˆAˆtŒy˜AŠ~ÐÑØ!# A A A t G¤ˆˆAˆAˆAˆtŒy˜AŠ~ÐÑð �t”}Ñ$ˆÝŒi˜ Ð(¨bÐ1Ñ1Ô1ˆÝŒy˜#Ÿ'š'™)œ) S§W¢W¡Y¤YÐ/°RÐ8Ñ8Ô8Ð8r   )r)  r    rP   NN)r˜   r™   rš   rä   r›   r   rc   ræ   rr   r   r   r±   rà   rœ   r�   s   @r   r(  r(  (  s·   ø€ € € € € ðð ð* 3>Ø!'Ø!%ØØð!=ð !=àð!=ð !  c¨3 Ô/ð!=ð ð	!=ð
 ð!=ð !=ð !=ð !=ð !=ð !=ðF9˜t Cœyð 9¨U¬\ð 9ð 9ð 9ð 9ð 9ð 9ð 9ð 9r   r(  c                   óp  ‡ — e Zd ZdZ	 	 	 	 	 ddedededed	eee                  d
efˆ fd„Z	dd„Z
dd„Zd	eee                  fd„Zd	eee                  fd„Zdd„Zddeee                  dej        fd„Z	 ddeeeef                  dee         deej        eej                 f         fd„Zd„ Zd„ Zˆ xZS )ÚRotaryEmbeddingMixeda~  Rotary position embedding with depth-dependent learnable frequencies.

    This implementation supports mixed (learnable) ROPE. In mixed mode,
    each transformer block has its own set of learnable frequency parameters.

    Based on 'Rotary Position Embedding for Vision: https://arxiv.org/abs/2403.13298)'
    Compatible with original at https://github.com/naver-ai/rope-vit
    r  NrX   r-   r  r  r!   r,   rV   c	           	      óH  •— t          ¦   «                              ¦   «          || _        || _        || _        || _        || _        || _        ||z  }	|	dz  dk    sJ d|	› �¦   «         ‚t          |	|||d||¬¦  «        }
t          j
        |
¦  «        | _        |�|d}|D ]}||z  }Œ|                      dt          j        |||¬	¦  «        d
¬¦  «         |                      dt          j        |||¬	¦  «        d
¬¦  «         |                      ¦   «          dS dx| _        | _        dS )aš  Initialize rotary embeddings.

        Args:
            dim: Embedding dimension (should be divisible by 4)
            depth: Number of transformer blocks
            num_heads: Number of attention heads
            temperature: Base for frequency computation
            feat_shape: Spatial dimensions [H, W] if known in advance
            grid_indexing: How to index grid positions ('xy' or 'ij')
        r1   r   z%head_dim must be divisible by 4, got T)r!   r  r   r   Nr   r  r6   Fro   r  )rq   rr   r-   r  r  r!   r,   rV   r  r   Ú	Parameterr   ru   r   rv   r}   r  r  )rx   r-   r  r  r!   r,   rV   r   r   r  r   rÌ   r9   ry   s                €r   rr   zRotaryEmbeddingMixed.__init__�  s^  ø€ õ* 	‰Œ×ÒÑÔÐØˆŒØˆŒ
Ø"ˆŒØ&ˆÔØ$ˆŒØ*ˆÔà˜)Ñ#ˆØ˜!‰|˜qÒ Ð Ð Ð"TÈ(Ð"TÐ"TÑ Ô Ð å$ØØØØ#ØØØð
ñ 
ô 
ˆõ ”\ %Ñ(Ô(ˆŒ
àÐ!àˆGØð ð �Ø˜1‘��Ø× Ò  ­¬°GÀFÐRWÐ(XÑ(XÔ(XÐejÐ ÑkÔkÐkØ× Ò  ­¬°GÀFÐRWÐ(XÑ(XÔ(XÐejÐ ÑkÔkÐkà×ÒÑ Ô Ð Ð Ð à"&Ð&ˆDŒH�t”x�x�xr   r#   c                 óº   — | j         �S|                      | j         ¦  «        \  }}| j                             |¦  «         | j                             |¦  «         dS dS r�   )r,   Ú_get_grid_valuesr  rƒ   r  )rx   r  r  s      r   r}   z"RotaryEmbeddingMixed._init_buffers¸  sY   € àŒ?Ð&Ø×,Ò,¨T¬_Ñ=Ô=‰HˆC�ØŒH�NŠN˜3ÑÔÐØŒH�NŠN˜3ÑÔÐÐÐð 'Ð&r   c                 ó.   — |                       ¦   «          dS r{   r|   r~   s    r   rw   z%RotaryEmbeddingMixed.reset_parameters¿  r   r   c                 óR   — t          || j        | j        j        ¬¦  «        \  }}||fS )N©rV   r   )r  rV   r   r   ©rx   r,   r  r  s       r   r?  z%RotaryEmbeddingMixed._get_grid_valuesÃ  s6   € Ý!ØØÔ,Ø”:Ô$ð
ñ 
ô 
‰ˆˆSð
 �Cˆxˆr   c                 óP  — | j         �œ|| j         k    r“| j        €J ‚| j        €J ‚|                      |¦  «        \  }}|                     | j        j        | j        j        ¦  «        | _        |                     | j        j        | j        j        ¦  «        | _        || _         d S d S d S r©   )r,   r  r  r?  r(   r   r   rC  s       r   rÞ   z&RotaryEmbeddingMixed.update_feat_shapeË  sœ   € ØŒ?Ð&¨:¸¼Ò+HÐ+HØ”8Ð'Ð'Ð'Ø”8Ð'Ð'Ð'Ø×,Ò,¨ZÑ8Ô8‰HˆC�Ø—v’v˜dœhœo¨t¬x¬~Ñ>Ô>ˆDŒHØ—v’v˜dœhœo¨t¬x¬~Ñ>Ô>ˆDŒHØ(ˆDŒOˆOˆOð 'Ð&Ð+HÐ+Hr   c                 ó.   — |                       ¦   «          dS r…   r|   r~   s    r   r†   z0RotaryEmbeddingMixed.init_non_persistent_buffersÔ  r   r   r‹   c                 óÊ   — |�%t          || j        | j        j        ¬¦  «        \  }}n'| j        �| j        �| j        | j        }}n
J d¦   «         ‚t          | j        ||¦  «        S )zçGenerate rotary embeddings for the given spatial shape.

        Args:
            shape: Spatial dimensions [H, W]

        Returns:
            Tensor of shape (depth, H*W, dim) containing concatenated sin/cos embeddings
        NrB  Fz8get_embed() requires pre-computed t_x/t_y or valid shape)r  rV   r   r   r  r  r&  )rx   r‹   r  r  s       r   rà   zRotaryEmbeddingMixed.get_embedØ  sw   € ð ÐÝ%ØØ"Ô0Ø”zÔ(ðñ ô ‰HˆC��ð
 ŒXÐ! d¤hÐ&:Ø”x ¤�ˆCˆCàTÐTÑTÔT�5å˜tœz¨3°Ñ4Ô4Ð4r   rò   ró   c           	      ó  — |sg S t          d„ |D ¦   «         ¦  «        }t          d„ |D ¦   «         ¦  «        }t          ||g| j        | j        j        ¬¦  «        \  }}t          | j        ||¦  «        }|j        \  }}	}
}|                     ||	|||¦  «        }|�™t          |¦  «        }t          j
        |||	||| j        j        | j        j        ¬¦  «        }t          |¦  «        D ]I\  }\  }}|dd…dd…d|…d|…f                              ||	||z  |¦  «        }||z  }|||dd…dd…d|…f<   ŒJ|S g }|D ]G\  }}|dd…dd…d|…d|…f                              ||	||z  |¦  «        }|                     |¦  «         ŒH|S )ai  Generate ROPE embeddings for multiple grid shapes efficiently.

        Computes embeddings for the maximum grid size once, then extracts
        and flattens the relevant portions for each requested shape.

        Args:
            shapes: List of (H, W) tuples representing different grid sizes
            seq_len: If provided, return padded tensor of this length. Otherwise return list.

        Returns:
            If seq_len is provided: Padded tensor of shape (len(shapes), depth, num_heads, seq_len, dim)
            Otherwise: List of tensors with shape (depth, num_heads, H*W, dim) for each shape
        c              3   ó    K  — | ]	\  }}|V — Œ
d S r©   r]   rö   s      r   rù   z8RotaryEmbeddingMixed.get_batch_embeds.<locals>.<genexpr>  rú   r   c              3   ó    K  — | ]	\  }}|V — Œ
d S r©   r]   rö   s      r   rù   z8RotaryEmbeddingMixed.get_batch_embeds.<locals>.<genexpr>  rú   r   rB  Nr6   )rÿ   r  rV   r   r   r&  r‹   r¹   rK   r   r   r   r  r�   Úappend)rx   rò   ró   r  r  r  r  Ú	max_embedr  r  Ú_r-   Úmax_embed_2dr’   Úpaddedr  r÷   rø   Úembed_sliceÚ
actual_lenÚresultss                        r   r	  z%RotaryEmbeddingMixed.get_batch_embedsî  sô  € ð$ ð 	ØˆIõ Ð)Ð) &Ð)Ñ)Ô)Ñ)Ô)ˆÝÐ)Ð) &Ð)Ñ)Ô)Ñ)Ô)ˆõ "Ø�EˆNØÔ,Ø”:Ô$ð
ñ 
ô 
‰ˆˆSõ
 $ D¤J°°SÑ9Ô9ˆ	ð $-¤?Ñ ˆˆy˜!˜SØ —~’~ e¨Y¸¸uÀcÑJÔJˆàÐå�F‘”ˆAÝ”[  E¨9°g¸sÈ4Ì:ÔK\ÐdhÔdnÔdtÐuÑuÔuˆFÝ& vÑ.Ô.ð ;ð ;‘	�‘6�A�qà*¨1¨1¨1¨a¨a¨a°°!°°R°a°R¨<Ô8×@Ò@ÀÈ	ÐSTÐWXÑSXÐZ]Ñ^Ô^�Ø ™U�
Ø/:��q˜!˜!˜!˜Q˜Q˜Q   Ð+Ñ,Ð,ØˆMð ˆGØð ,ð ,‘��1à*¨1¨1¨1¨a¨a¨a°°!°°R°a°R¨<Ô8×@Ò@ÀÈ	ÐSTÐWXÑSXÐZ]Ñ^Ô^�Ø—’˜{Ñ+Ô+Ð+Ð+ØˆNr   c                 óf   — |                       |j        dd …         ¦  «        }t          ||¦  «        S râ   )rà   r‹   r´   r  s      r   r–   zRotaryEmbeddingMixed.forward&  s-   € à—N’N 1¤7¨1¨2¨2¤;Ñ/Ô/ˆ	Ý" 1 iÑ0Ô0Ð0r   c                 ó   — dhS )z/Exclude frequency parameters from weight decay.r   r]   r~   s    r   Úno_weight_decayz$RotaryEmbeddingMixed.no_weight_decay+  s
   € àˆyÐr   )r  NrX   NNr—   r©   )r˜   r™   rš   rä   r›   rc   r   r   ræ   rr   r}   rw   r?  rÞ   r†   r   r±   rà   r   r   r	  r–   rT  rœ   r�   s   @r   r;  r;  x  sÔ  ø€ € € € € ðð ð "&Ø.2Ø!%ØØð5'ð 5'àð5'ð ð5'ð ð	5'ð
 ð5'ð !  c¤Ô+ð5'ð ð5'ð 5'ð 5'ð 5'ð 5'ð 5'ðn ð  ð  ð  ðð ð ð ð¨8°D¸´IÔ+>ð ð ð ð ð)¨H°T¸#´YÔ,?ð )ð )ð )ð )ðð ð ð ð5ð 5˜x¨¨S¬	Ô2ð 5¸e¼lð 5ð 5ð 5ð 5ð2 &*ð6ð 6à˜˜s C˜xœÔ)ð6ð ˜c”]ð6ð 
ˆuŒ|˜T %¤,Ô/Ð/Ô	0ð	6ð 6ð 6ð 6ðp1ð 1ð 1ð
ð ð ð ð ð ð r   r;  ÚseparateÚcpuÚheightÚwidthÚnormalize_coordsc                 ó  — t          j        d| |t           j        ¬¦  «        |z   }t          j        d||t           j        ¬¦  «        |z   }|dk    r"t          t	          | |¦  «        ¦  «        }	|	}
|	}n_|dk    r"t          t          | |¦  «        ¦  «        }	|	}
|	}n7|dk    rt          | ¦  «        }
t          |¦  «        }nt          d|› �¦  «        ‚||
z  }||z  }|                     |¦  «        }|                     |¦  «        }|dk    r3t          j        ||d¬¦  «        \  }}t          j	        ||gd	¬
¦  «        }n+t          j	        t          j        ||d¬¦  «        d	¬
¦  «        }| 
                    dd¦  «        }d|z  dz
  }|S )z{Make coordinate grid matching offset and normalization of original.
    Returns: coords with shape (HW, 2) in [-1, 1].
    g      à?r6   rÿ   ÚminrU  zUnknown normalize_coords: rX   ra   r3   r;   rP   r   r   ç       @r   )r   r&   r   rc   rÿ   r[  Ú
ValueErrorr(   re   r=   r>   )rW  rX  rY  rV   rU   r   r   Úcoords_hÚcoords_wÚdenomÚh_denomÚw_denomÚgrid_wÚgrid_hÚcoordss                  r   Úmake_coords_dinov3rf  0  s¡  € õ Œ|˜C °½e¼mÐLÑLÔLÈ{ÑZ€HÝŒ|˜C ¨v½U¼]ÐKÑKÔKÈkÑY€Hð ˜5Ò Ð Ý•c˜& %Ñ(Ô(Ñ)Ô)ˆØˆØˆˆØ	˜UÒ	"Ð	"Ý•c˜& %Ñ(Ô(Ñ)Ô)ˆØˆØˆˆØ	˜ZÒ	'Ð	'Ý˜‘-”-ˆÝ˜‘,”,ˆˆåÐHÐ6FÐHÐHÑIÔIÐIð ˜'Ñ!€HØ˜'Ñ!€HØ�{Š{˜5Ñ!Ô!€HØ�{Š{˜5Ñ!Ô!€Hð ˜ÒÐÝœ¨°(ÀTÐJÑJÔJ‰ˆ�Ý”˜f fÐ-°2Ð6Ñ6Ô6ˆˆå”�Uœ^¨H°hÈÐNÑNÔNÐTVÐWÑWÔWˆØ�^Š^˜A˜qÑ!Ô!€FØ�6‰\˜CÑ€FØ€Mr   c                   óB  ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 d)ded	ee         d
ee         dee         deee                  dededede	dee         dee         dee         fˆ fd„Z
d*d„Zd*d„Zdej        fdej        dej        dej        fd„Zdej        dej        fd„Zdej        deej        ej        f         fd„Z	 d+dee         d e	dej        fd!„Zdee         fd"„Zdee         fd#„Zd*d$„Zd,d%eee                  dej        fd&„Zd'ej        dej        fd(„Zˆ xZS )-ÚRotaryEmbeddingDinoV3a  RoPE for timm DinoV3 port, numerically matching original.

    Math is aligned to original DinoV3 RopePositionEmbedding at https://github.com/facebookresearch/dinov3:
      - 0.5-centered coords normalized by H/W (or min/max), mapped to [-1,1]
      - training-time augmentations (shift/jitter/rescale)
      - periods schedule equals Rope's temperature (base) or min/max period
    ç      Y@NrU  rO   rP   Tr-   r!   Ú
min_periodÚ
max_periodr,   rY  rU   rV   r¿   Úshift_coordsÚjitter_coordsÚrescale_coordsc                 ó–  •— t          ¦   «                              ¦   «          || _        |	| _        t	          |¦  «        | _        || _        || _        || _        |
| _	        || _
        || _        t          d„ | j	        | j
        | j        fD ¦   «         ¦  «        | _        || _        || _        || _        |dz  f}|                      dt%          j        |||¬¦  «        d¬¦  «         |�F|d         |d         z  }||d	z  f}|                      d
t%          j        |||¬¦  «        d¬¦  «         nd | _        |                      ¦   «          d S )Nc                 ó   — g | ]}|d u‘ŒS r©   r]   )r8   Úas     r   r:   z2RotaryEmbeddingDinoV3.__init__.<locals>.<listcomp>Œ  s   € ÐsÐsÐs°˜q¨˜}ÐsÐsÐsr   r1   Úperiodsr6   Fro   r   r   r   Úpos_embed_cached)rq   rr   r-   r¿   rc   r!   rj  rk  rY  rl  rm  rn  ÚanyÚ
aug_activer,   rU   rV   ru   r   rv   rs  rw   )rx   r-   r!   rj  rk  r,   rY  rU   rV   r¿   rl  rm  rn  r   r   Úperiods_shaperÌ   rÍ   ry   s                     €r   rr   zRotaryEmbeddingDinoV3.__init__k  sl  ø€ õ" 	‰Œ×ÒÑÔÐð ˆŒØ&ˆÔõ ! Ñ-Ô-ˆÔØ$ˆŒØ$ˆŒð !1ˆÔØ(ˆÔØ*ˆÔØ,ˆÔÝÐsÐs°tÔ7HÈ$ÔJ\Ð^bÔ^qÐ6rÐsÑsÔsÑtÔtˆŒð %ˆŒØ&ˆÔØ*ˆÔð  ™˜ˆØ×Ò˜Y­¬°MÈ&ÐX]Ð(^Ñ(^Ô(^ÐkpÐÑqÔqÐqàÐ!à  ”m j°¤mÑ3ˆGØ  #¨¡'Ð*ˆIØ× Ò Ð!3µU´[ÀÐSYÐafÐ5gÑ5gÔ5gÐtyÐ ÑzÔzÐzÐzà$(ˆDÔ!ð 	×ÒÑÔÐÐÐr   r#   c                 ó.   — |                       ¦   «          dS r{   r|   r~   s    r   rw   z&RotaryEmbeddingDinoV3.reset_parameters¢  r   r   c                 óî   — | j                              |                      ¦   «         ¦  «         | j        �?| j        �:|                      | j        d¬¦  «        }| j                             |¦  «         dS dS dS )r‚   NT©Úno_aug)rr  rƒ   Ú_compute_periodsr,   rs  Ú_create_embed)rx   Ú
rope_embeds     r   r}   z#RotaryEmbeddingDinoV3._init_buffers¦  sy   € àŒ×Ò˜4×0Ò0Ñ2Ô2Ñ3Ô3Ð3ØŒ?Ð&¨4Ô+@Ð+LØ×+Ò+¨D¬OÀDÐ+ÑIÔIˆJØÔ!×'Ò'¨
Ñ3Ô3Ð3Ð3Ð3ð 'Ð&Ð+LÐ+Lr   rV  r   r   c                 óz  — | j         dz  }| j        �E| j        �>t          j        dd|dt          j        ¬¦  «        }| j        | j        | j        z  |z  z  }nO| j        €t          d¦  «        ‚dt          j        |dt          j        ¬¦  «        z  | j         d	z  z  }| j        |z  }| 	                    ||¬¦  «        S )
z5Construct periods from either min/max or temperature.r1   Nr   r   rV  r6   z0Provide either min/max periods or `temperature`.r\  r   )
r-   rj  rk  r   r   r   r!   r]  r&   r(   )rx   r   r   r-   Ú	exponentsrr  s         r   r{  z&RotaryEmbeddingDinoV3._compute_periods­  s¾   € àŒh˜!‰mˆàŒ?Ð&¨4¬?Ð+FÝœ q¨!¨S¸ÅeÄmÐTÑTÔTˆIØ”o¨$¬/¸D¼OÑ*KÐPYÑ)YÑZˆGˆGàÔÐ'Ý Ð!SÑTÔTÐTØ�eœl¨3°uÅEÄMÐRÑRÔRÑRÐVZÔV^ÐbcÑVcÑdˆIØÔ&¨)Ñ3ˆGð �zŠz ¨uˆzÑ5Ô5Ð5r   re  c                 ó  — | j         r| j        s|S |j        }|j        }| j        �Ot          | j        ¦  «        }t          j        d||¬¦  «                             | |¦  «        }||ddd…f         z   }| j	        �Št          | j	        ¦  «        }|dk    rt          d¦  «        ‚t          j        |¦  «        }t          j        d||¬¦  «                             | |¦  «                             ¦   «         }||ddd…f         z  }| j        �€t          | j        ¦  «        }	|	dk    rt          d¦  «        ‚t          j        |	¦  «        }
t          j        d||¬¦  «                             |
 |
¦  «                             ¦   «         }||z  }|S )z4Apply shift/jitter/rescale train time augmentations.Nr   r6   r   zAjitter_coords must be > 0 (interpreted as multiplicative factor).zBrescale_coords must be > 0 (interpreted as multiplicative factor).r   )Útrainingru  r   r   rl  rc   r   rv   Úuniform_rm  r]  r   r   r)   rn  )rx   re  r   r   ÚshiftÚshift_hwÚjitter_factorÚ
jitter_maxÚ	jitter_hwÚrescale_factorÚrescale_maxÚrescales               r   Ú_apply_coord_augsz'RotaryEmbeddingDinoV3._apply_coord_augs¾  s›  € àŒ}ð 	 D¤Oð 	ØˆMà”ˆØ”ˆð ÔÐ(Ý˜$Ô+Ñ,Ô,ˆEÝ”{ 1¨V¸5ÐAÑAÔA×JÒJÈEÈ6ÐSXÑYÔYˆHØ˜h t¨Q¨Q¨Q wÔ/Ñ/ˆFð ÔÐ)Ý! $Ô"4Ñ5Ô5ˆMØ Ò!Ð!Ý Ð!dÑeÔeÐeÝœ -Ñ0Ô0ˆJÝœ A¨f¸EÐBÑBÔB×KÒKÈZÈKÐYcÑdÔd×hÒhÑjÔjˆIØ˜i¨¨a¨a¨a¨Ô0Ñ0ˆFð ÔÐ*Ý" 4Ô#6Ñ7Ô7ˆNØ Ò"Ð"Ý Ð!eÑfÔfÐfÝœ( >Ñ2Ô2ˆKÝ”k !¨F¸%Ð@Ñ@Ô@×IÒIÈ;È,ÐXcÑdÔd×hÒhÑjÔjˆGØ˜gÑ%ˆFàˆr   c                 ó   — | j         dz  }| j        j        }| j        j        }| j                             ¦   «         |k    sJ ‚|dd…dd…df                              ||¬¦  «        }dt          j        z  |z  | j        dddd…f         z  }|                     d¦  «        }| j	        r| 
                    d¦  «        }n|                     dd¬¦  «        }t          j        |¦  «        }t          j        |¦  «        }||fS )zEReturn sin/cos embeddings with either 'half' or 'interleaved' layout.r1   Nr6   r   r   r3   r;   )r-   rr  r   r   r‘   r(   r   r   r>   r¿   ÚtilerÁ   r   rA   rB   )rx   re  r-   r   r   r  rA   rB   s           r   Ú_get_pos_embed_from_coordsz0RotaryEmbeddingDinoV3._get_pos_embed_from_coordsà  s  € ð Œh˜!‰mˆØ”Ô$ˆØ”Ô"ˆØŒ|×!Ò!Ñ#Ô# sÒ*Ð*Ð*Ð*ð ˜˜˜˜1˜1˜1˜d˜
Ô#×&Ò&¨f¸EÐ&ÑBÔBˆØ•T”W‘˜vÑ%¨¬°T¸4ÀÀÀ°]Ô(CÑCˆØ—’ Ñ"Ô"ˆàÔð 	9à—[’[ ‘^”^ˆFˆFð ×-Ò-¨a°RÐ-Ñ8Ô8ˆFåŒi˜ÑÔˆÝŒi˜ÑÔˆØ�Cˆxˆr   Frz  c                 óä   — |\  }}t          ||| j        | j        | j        ¬¦  «        }|s|                      |¦  «        }|                      |¦  «        \  }}t          j        ||gd¬¦  «        }|S )N)rY  rV   rU   r3   r;   )rf  rY  rV   rU   r‹  rŽ  r   rŒ   )	rx   r,   rz  ÚHÚWre  rA   rB   r}  s	            r   r|  z#RotaryEmbeddingDinoV3._create_embedø  sŠ   € ð
 ‰ˆˆ1Ý#ØˆqØ!Ô2ØÔ,ØÔ(ð	
ñ 
ô 
ˆð ð 	4Ø×+Ò+¨FÑ3Ô3ˆFØ×2Ò2°6Ñ:Ô:‰ˆˆSÝ”Y  S˜z¨rÐ2Ñ2Ô2ˆ
ØÐr   c                 ór   — |                       |d¬¦  «        }|                      d|d¬¦  «         || _        d S )NTry  rs  Fro   )r|  ru   r,   )rx   r,   r}  s      r   Ú_cache_embedz"RotaryEmbeddingDinoV3._cache_embed
  sA   € à×'Ò'¨
¸4Ð'Ñ@Ô@ˆ
Ø×ÒÐ/°ÈÐÑNÔNÐNØ$ˆŒˆˆr   c                 ó\   — | j         �"|| j         k    r|                      |¦  «         d S d S d S r©   )r,   r“  rÝ   s     r   rÞ   z'RotaryEmbeddingDinoV3.update_feat_shape  s>   € ØŒ?Ð&¨:¸¼Ò+HÐ+Hà×Ò˜jÑ)Ô)Ð)Ð)Ð)ð 'Ð&Ð+HÐ+Hr   c                 ó.   — |                       ¦   «          dS r…   r|   r~   s    r   r†   z1RotaryEmbeddingDinoV3.init_non_persistent_buffers  r   r   r‹   c                 óà   — |�|                       |¦  «        }nU| j        du p| j        o| j        }|r,| j        €
J d¦   «         ‚|                       | j        ¦  «        }n| j        €J ‚| j        }|S )zœGenerate rope_embed matching DINOv3 RopePositionEmbedding numerics.

        Returns: (HW, num_heads, 2 * head_dim) with last dim = [sin, cos] cat.
        Nz&feature shape must be cached on create)r|  rs  r�  ru  r,   )rx   r‹   r}  Úneed_creates       r   rà   zRotaryEmbeddingDinoV3.get_embed  s�   € ð
 ÐØ×+Ò+¨EÑ2Ô2ˆJˆJàÔ/°4Ð7Ð^¸D¼MÐ<]ÈdÌoˆKØð 3Ø”Ð2Ð2Ð4\Ñ2Ô2Ð2Ø!×/Ò/°´Ñ@Ô@�
�
àÔ,Ð8Ð8Ð8Ø!Ô2�
àÐr   r^   c                 ót   — |                       |j        dd…         ¦  «        }t          ||| j        ¬¦  «        S )z$Get and apply rotary embeddings to xr   Nr  r  r  s      r   r–   zRotaryEmbeddingDinoV3.forward+  s7   € ð —N’N 1¤7¨1¨2¨2¤;Ñ/Ô/ˆ	Ý" 1 i°dÔ6FÐGÑGÔGÐGr   )ri  NNNrU  rO   rP   TNNNNNr—   ©Fr©   )r˜   r™   rš   rä   r›   r   rc   r   ræ   rå   rr   rw   r}   r   r   r   r   r±   r{  r‹  r   rŽ  r|  r“  rÞ   r†   rà   r–   rœ   r�   s   @r   rh  rh  b  sÅ  ø€ € € € € ðð ð ,1Ø*.Ø*.Ø.2Ø$.Ø!$Ø!%Ø $Ø,0Ø-1Ø.2ØØð5 ð 5 àð5 ð " %œð5 ð ! œð	5 ð
 ! œð5 ð !  c¤Ô+ð5 ð "ð5 ð ð5 ð ð5 ð ð5 ð # 5œ/ð5 ð $ Eœ?ð5 ð % UœOð5 ð 5 ð 5 ð 5 ð 5 ð 5 ðnð ð ð ð4ð 4ð 4ð 4ð 7<ÐRWÔR_ð 6ð 6 u¤|ð 6ÀEÄKð 6ÐdiÔdpð 6ð 6ð 6ð 6ð" ¨¬ð  ¸¼ð  ð  ð  ð  ðD°´ð À%ÈÌÐV[ÔVbÐHbÔBcð ð ð ð ð6 !ðð à˜Sœ	ðð ðð 
Œð	ð ð ð ð$% t¨C¤yð %ð %ð %ð %ð*¨D°¬Ið *ð *ð *ð *ð
ð ð ð ðð ˜x¨¨S¬	Ô2ð ¸e¼lð ð ð ð ð$H˜œð H¨%¬,ð Hð Hð Hð Hð Hð Hð Hð Hr   rh  rŒ   é   r+  Ú	rope_typec                 ó  — | dk    r'|                      dd¦  «         t          dd||z  i|¤ŽS | dk    rt          dd||z  i|¤ŽS | dk    r;|                      dd¦  «         |                      dd¦  «         t          d||d	œ|¤ŽS | d
k    r=|                      dd¦  «         |                      dd¦  «         t	          dd||z  i|¤ŽS | dk    r,dD ]}|                      |d¦  «         Œt          dd||z  i|¤ŽS t          d| › �¦  «        ‚)a°  Factory function for creating rotary position embeddings.

    Args:
        rope_type: Type of RoPE to create. Options:
            - 'base': Basic RotaryEmbedding
            - 'cat': RotaryEmbeddingCat (concatenated sin/cos)
            - 'mixed': RotaryEmbeddingMixed (learnable per-depth frequencies)
            - 'dinov3': RotaryEmbeddingDinoV3 (with coordinate transforms)
            - 'mrope': RotaryEmbeddingMRope (interleaved multimodal RoPE; requires `mrope_section`)
        dim: Total embedding dimension
        num_heads: Number of attention heads
        **kwargs: Additional arguments passed to the specific RoPE class

    Returns:
        Rotary embedding module
    Úbaser¿   Nr-   rŒ   ÚmixedrS   rT   )r-   r  Údinov3Úmrope)rS   rT   r¿   zUnknown RoPE type: r]   )ÚpoprÅ   rè   r;  rh  r(  r]  )r›  r-   r  ÚkwargsÚks        r   Úcreate_rope_embedr¤  2  sq  € ð, �FÒÐØ�
Š
�= $Ñ'Ô'Ð'ÝÐ>Ð> 3¨)Ñ#3Ð>°vÐ>Ð>Ð>Ø	�eÒ	Ð	Ý!ÐAÐA c¨YÑ&6ÐA¸&ÐAÐAÐAØ	�gÒ	Ð	à�
Š
�; Ñ%Ô%Ð%Ø�
Š
Ð# TÑ*Ô*Ð*Ý#ÐK¨°yÐKÐKÀFÐKÐKÐKØ	�hÒ	Ð	Ø�
Š
�; Ñ%Ô%Ð%Ø�
Š
Ð# TÑ*Ô*Ð*Ý$ÐDÐD¨°	Ñ)9ÐD¸VÐDÐDÐDØ	�gÒ	Ð	Ø?ð 	 ð 	 ˆAØ�JŠJ�q˜$ÑÔÐÐÝ#ÐCÐC¨¨yÑ(8ÐC¸FÐCÐCÐCåÐ:¨yÐ:Ð:Ñ;Ô;Ð;r   )r   TN)r    r   Nr™  )r  T)rŒ   rš  r+  )1rä   r   Útypingr   r   r   r   r   r   Ú_fxr	   rD   r
   Útrace_utilsr   r›   rc   rå   r   r   r±   r*   r   r   rH   rM   ræ   rk   ÚModulerm   rŸ   r¤   r«   r²   r´   r¾   rÃ   rÅ   rè   r  r  Úwrapr  r&  r(  r;  rf  rh  r¤  r]   r   r   ú<module>rª     sÁ  ððð ð €€€Ø /Ð /Ð /Ð /Ð /Ð /Ð /Ð /Ð /Ð /Ð /Ð /à €€€Ø Ð Ð Ð Ð Ð à *Ð *Ð *Ð *Ð *Ð *Ø Ð Ð Ð Ð Ð Ø  Ð  Ð  Ð  Ð  Ð  ð Ø!Ø)-ð	
ð 
Øð
àð
ð ð
ð ˜œÔ&ð	
ð 
ð 
ð 
ð $ØØ)-ð	ð Øðàðð ðð ˜œÔ&ð	ð
 „\ðð ð ð ð Ø#Ø#Ø#(Ø)-Ø"œ]ð&#ð &#Ø˜”Ið&#àð&#ð ð&#ð ð	&#ð
 !ð&#ð ˜œÔ&ð&#ð Œ{ð&#ð „\ð&#ð &#ð &#ð &#ðR,�t˜C”yð , T¨#¤Yð ,ð ,ð ,ð ,ð )-ØØØ#Ø"Ø"ØØ.2ØØ!Ø)-Ø"œ]ðRð RØ˜”IðRà˜œÔ%ðRð ðRð ð	Rð
 ðRð ðRð ðRð ðRð !  c¤Ô+ðRð ðRð ðRð ˜œÔ&ðRð Œ{ðRð 
ˆ%Œ,ÔðRð Rð Rð Rðj6ð 6ð 6ð 6ð 6�2”9ñ 6ô 6ð 6ðrJð Jð Jð(˜œð (¨¬ð (ð (ð (ð (ð ð	.ð .ØŒ<ð.à”ð.ð ”ð.ð ð	.ð
 „\ð.ð .ð .ð .ð. ð	;ð ;Ø�”Ôð;à”ð;ð ”ð;ð ð	;ð
 
ˆ%Œ,Ôð;ð ;ð ;ð ;ð0 ð.ð .ØŒ<ð.àŒ\ð.ð ð.ð „\ð	.ð .ð .ð .ð. %*ð	".ð ".ØŒ<ð".à”<ð".ð ”lð".ð "ð	".ð
 „\ð".ð ".ð ".ð ".ðN )-ØØØ#Ø"ØØ.2ØØ!Ø!Ø)-Ø"œ]ð>ð >Ø˜”Ið>à˜œÔ%ð>ð ð>ð ð	>ð
 ð>ð ð>ð ð>ð !  c¤Ô+ð>ð ð>ð ð>ð ð>ð ˜œÔ&ð>ð Œ{ð>ð >ð >ð >ðBJ4ð J4ð J4ð J4ð J4�b”iñ J4ô J4ð J4ðZ~Hð ~Hð ~Hð ~Hð ~H˜œñ ~Hô ~Hð ~HðJ "Øð(ð ØŒmð(ð (ð (Øð(àð(ð ð(ð ð	(ð
 ð(ð „\ð(ð (ð (ð (ðD „„Øð "Ø)-Ø"œ]ð	ð Ø�CŒyðàðð ˜œÔ&ðð Œ{ð	ð
 ˆ5Œ<˜œÐ%Ô&ðð ð ñ Ôñ „ðð$!ØŒ|ð!àŒ\ð!ð Œ\ð!ð „\ð	!ð !ð !ð !ð$M9ð M9ð M9ð M9ð M9˜2œ9ñ M9ô M9ð M9ð`uð uð uð uð u˜2œ9ñ uô uð uðp „„Øð !+Ø!ØØ$Ø"œ]ð-ð -Øð-àð-ð ð-ð ð	-ð
 ð-ð ”ð-ð Œ{ð-ð „\ð-ð -ð -ñ Ôñ „ð-ð`MHð MHð MHð MHð MH˜BœIñ MHô MHð MHðb ØØð)<ð )<Øð)<àð)<ð ð)<ð
 „Yð)<ð )<ð )<ð )<ð )<ð )<r   