§
    ·¯¸j*� ã                   ó@  — d Z ddlZddlZddlmZmZmZ ddlmZ ddl	m
Z
mZmZmZmZmZmZmZmZmZ ddlZddlmZ ddlmc mZ ddlmZmZ ddlmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z* dd	l+m,Z, dd
l-m.Z. ddl/m0Z0m1Z1 ddl2m3Z3m4Z4 ddl5m6Z6m7Z7 ddl8m9Z9 ddl:m;Z;m<Z< ddgZ= ej>        e?¦  «        Z@e G d„ d¦  «        ¦   «         ZAdeAdeAfd„ZB	 	 dQdejC        deeDeDf         deEdeEdeejC        eeDeDf         f         f
d„ZFdejC        fd„ZG	 dRdejC        d eEdejC        fd!„ZH G d"„ d#¦  «        ZIdSdeAd$eDde
fd%„ZJe1 G d&„ d'ejK        ¦  «        ¦   «         ZLe0dddejM        fd(ejC        d)eDd*eEd+eeD         d,ejN        deejC                 fd-„¦   «         ZOe0	 	 	 	 dTdejC        d(eejC                 d/ePd)eDd0eEdejC        fd1„¦   «         ZQ G d2„ dejK        ¦  «        ZRd3„ ZSdUd6ePd7eTd8eEde
fd9„ZUd:eePef         d;eRdeePef         fd<„ZVdVd>ePdeePef         fd?„ZW e7 eWd@¬A¦  «         eWd@¬A¦  «         eWd@¬A¦  «         eW¦   «          eW¦   «          eW¦   «          eWd@d¬B¦  «         eWd@d¬B¦  «        dCœ¦  «        ZXdRdDePdEeEdeRfdF„ZY	 dRdDePdEeEdeRfdG„ZZ	 dRdDePdEeEdeRfdH„Z[e6dRdEeEdeRfdI„¦   «         Z\e6dRdEeEdeRfdJ„¦   «         Z]e6dRdEeEdeRfdK„¦   «         Z^e6dRdEeEdeRfdL„¦   «         Z_e6dRdEeEdeRfdM„¦   «         Z`e6dRdEeEdeRfdN„¦   «         Zae6dRdEeEdeRfdO„¦   «         Zbe6dRdEeEdeRfdP„¦   «         ZcdS )Waœ   NaFlex Vision Transformer

An improved version of the Vision Transformer with:
1. Encapsulated embedding and position encoding in a single module
2. Support for linear patch embedding on pre-patchified inputs
3. Support for NaFlex variable aspect, variable resolution
4. Support for FlexiViT variable patch size
5. Support for NaViT fractional/factorized position embedding

Based on ideas from:
- Original Vision Transformer: https://arxiv.org/abs/2010.11929
- FlexiViT: https://arxiv.org/abs/2212.08013
- NaViT: https://arxiv.org/abs/2307.06304
- NaFlex (SigLip-2): https://arxiv.org/abs/2502.14786

Hacked together by / Copyright 2025, Ross Wightman, Hugging Face
é    N)Ú	dataclassÚfieldsÚreplace)Úpartial)
ÚCallableÚDictÚIterableÚListÚOptionalÚSetÚTupleÚTypeÚUnionÚAny©ÚIMAGENET_INCEPTION_MEANÚIMAGENET_INCEPTION_STD)Úget_device_dtypeÚAttentionPoolLatentÚMlpÚ	LayerNormÚPatchDropoutWithIndicesÚPatchEmbedInterpolatorÚ_assertÚ	to_2tupleÚget_act_layerÚget_norm_layerÚapply_keep_indices_nlcÚdisable_compilerÚcalculate_drop_path_ratesé   )Úbuild_model_with_cfg)Úfeature_take_indices)Úregister_notrace_functionÚregister_notrace_module)Ú
checkpointÚnamed_apply)Úregister_modelÚgenerate_default_cfgs)ÚEvaBlock)ÚBlockÚglobal_pool_nlcÚNaFlexVitCfgÚ	NaFlexVitc                   ó´  — e Zd ZU dZdZeeeeef         f         ed<   dZ	eed<   dZ
eed<   dZeed<   d	Zeed
<   dZeed<   dZeeeeedf         f                  ed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZee         ed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed <   d!Z e!ed"<   d#Z"eeeef                  ed$<   d%Z#e!ed&<   dZ$eed'<   dZ%eed(<   d)Z&e!ed*<   d+Z'eed,<   dZ(eeeef                  ed-<   dZ)eed.<   d/Z*e!ed0<   dZ+eed1<   dZ,ee         ed2<   dZ-ee         ed3<   dZ.ee         ed4<   dZ/eeeeef                  ed5<   dZ0eed6<   dZ1eed7<   dZ2eed8<   dZ3eed9<   dZ4ee         ed:<   d;Z5e!ed<<   dZ6eed=<   dZ7ee         ed><   dZ8ee         ed?<   d)Z9e!ed@<   dZ:eedA<   dBZ;e!edC<   dZ<ee!         edD<   dZ=ee!         edE<   dZ>ee!         edF<   dZ?ee!         edG<   dZ@ee!         edH<   dZAee!         edI<   dZBee!         edJ<   dKZCe!edL<   dZDeedM<   dZEeedN<   dZFeedO<   dZGeedP<   dZHeedQ<   dS )Rr-   zôConfiguration for FlexVit model.

    This dataclass contains the bulk of model configuration parameters,
    with core parameters (img_size, in_chans, num_classes, etc.) remaining
    as direct constructor arguments for API compatibility.
    é   Ú
patch_sizeé   Ú	embed_dimé   ÚdepthÚ	num_headsg      @Ú	mlp_ratioFÚscale_mlp_normN.Únum_kv_headsTÚqkv_biasÚqk_normÚ	proj_biasç        Úattn_drop_rateÚscale_attn_inner_normÚuse_key_only_attn_maskÚinit_valuesÚ	drop_rateÚpos_drop_rateÚpatch_drop_rateÚproj_drop_rateÚdrop_path_rateÚclass_tokenr   Ú
reg_tokensÚlearnedÚ	pos_embed)r0   r0   Úpos_embed_grid_sizeÚbicubicÚpos_embed_interp_modeÚpos_embed_ar_preservingÚpos_embed_use_grid_sampleÚ Ú	rope_typeç     ˆÃ@Úrope_temperatureÚrope_ref_feat_shapeÚrope_grid_offsetÚijÚrope_grid_indexingÚrope_rotate_halfÚrope_shift_coordsÚrope_jitter_coordsÚrope_rescale_coordsÚrope_mrope_sectionÚdynamic_img_padÚpatchify_channels_lastÚpre_normÚ
final_normÚfc_normÚmapÚglobal_poolÚpool_include_prefixÚattn_pool_num_headsÚattn_pool_mlp_ratioÚweight_initÚfix_initÚlinearÚembed_proj_typeÚinput_norm_layerÚembed_norm_layerÚ
norm_layerÚ	act_layerÚblock_fnÚ	mlp_layerÚ
attn_layerÚstandardÚ	attn_typeÚ
attn_gatedÚ
swiglu_mlpÚ	qkv_fusedÚattn_only_layer_scaleÚenable_patch_interpolator)IÚ__name__Ú
__module__Ú__qualname__Ú__doc__r1   r   Úintr   Ú__annotations__r3   r5   r6   r7   Úfloatr8   Úboolr9   r   r:   r;   r<   r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rJ   ÚstrrK   rM   rN   rO   rQ   rS   rT   rU   rW   rX   rY   rZ   r[   r\   r]   r^   r_   r`   ra   rc   rd   re   rf   rg   rh   rj   rk   rl   rm   rn   ro   rp   rq   rs   rt   ru   rv   rw   rx   © ó    úe/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/models/naflexvit.pyr-   r-   ;   s§  € € € € € € ðð ð /1€J��c˜5  c œ?Ð*Ô+Ð0Ð0Ñ0Ø€IˆsÐÐÑØ€Eˆ3€O€O�OØ€IˆsÐÐÑØ€IˆuÐÐÑØ €N�DÐ Ð Ñ ð ;?€L�(˜5  e¨C°¨H¤oÐ!5Ô6Ô7Ð>Ð>Ñ>Ø€HˆdÐÐÑØ€GˆTÐÐÑØ€IˆtÐÐÑØ€N�EÐÐÑØ"'Ð˜4Ð'Ð'Ñ'Ø#(Ð˜DÐ(Ð(Ñ(ð $(€K�˜%”Ð'Ð'Ñ'Ø€IˆuÐÐÑØ€M�5ÐÐÑØ €O�UÐ Ð Ñ Ø€N�EÐÐÑØ€N�EÐÐÑð €K�ÐÐÑØ€J�ÐÐÑð €IˆsÐÐÑØ5=Ð˜ %¨¨S¨¤/Ô2Ð=Ð=Ñ=Ø!*Ð˜3Ð*Ð*Ñ*Ø$)Ð˜TÐ)Ð)Ñ)Ø&+Ð˜tÐ+Ð+Ñ+ð €IˆsÐÐÑØ%Ð�eÐ%Ð%Ñ%Ø59Ð˜ %¨¨S¨¤/Ô2Ð9Ð9Ñ9Ø Ð�eÐ Ð Ñ Ø"Ð˜Ð"Ð"Ñ"Ø"Ð�dÐ"Ð"Ñ"Ø)-Ð�x ”Ð-Ð-Ñ-Ø*.Ð˜ œÐ.Ð.Ñ.Ø+/Ð˜ %œÐ/Ð/Ñ/Ø9=Ð˜  s¨C° }Ô!5Ô6Ð=Ð=Ñ=ð "€O�TÐ!Ð!Ñ!Ø#'Ð˜DÐ'Ð'Ñ'ð €HˆdÐÐÑØ€J�ÐÐÑØ"€GˆX�dŒ^Ð"Ð"Ñ"ð €K�ÐÐÑØ %Ð˜Ð%Ð%Ñ%Ø)-Ð˜ #œÐ-Ð-Ñ-Ø+/Ð˜ %œÐ/Ð/Ñ/ð €K�ÐÐÑØ€HˆdÐÐÑð $€O�SÐ#Ð#Ñ#Ø&*Ð�h˜s”mÐ*Ð*Ñ*Ø&*Ð�h˜s”mÐ*Ð*Ñ*ð !%€J�˜”Ð$Ð$Ñ$Ø#€Iˆx˜Œ}Ð#Ð#Ñ#Ø"€Hˆh�sŒmÐ"Ð"Ñ"Ø#€Iˆx˜Œ}Ð#Ð#Ñ#Ø $€J�˜”Ð$Ð$Ñ$ð  €IˆsÐÐÑØ€J�ÐÐÑØ€J�ÐÐÑØ€IˆtÐÐÑØ"'Ð˜4Ð'Ð'Ñ'ð ',Ð˜tÐ+Ð+Ñ+Ð+Ð+rƒ   ÚcfgÚreturnc                 ó²   ‡— t          | j                             ¦   «         ¦  «        Šˆfd„|                     ¦   «         D ¦   «         }|rt	          | fi |¤Ž} | S )zIOverlay kwargs onto config, replacing config values with provided kwargs.c                 ó$   •— i | ]\  }}|‰v ¯	||“ŒS r‚   r‚   )Ú.0ÚkÚvÚconfig_fieldss      €r„   ú
<dictcomp>z#_overlay_kwargs.<locals>.<dictcomp>¡   s)   ø€ ÐKÐKÐK™d˜a ¸¸]Ð8JÐ8J�Q˜Ð8JÐ8JÐ8Jrƒ   )ÚsetÚ__dataclass_fields__ÚkeysÚitemsr   )r…   ÚkwargsÚconfig_kwargsrŒ   s      @r„   Ú_overlay_kwargsr”   �   sd   ø€ õ ˜Ô0×5Ò5Ñ7Ô7Ñ8Ô8€MØKÐKÐKÐK f§l¢l¡n¤nÐKÑKÔK€Màð ,Ý�cÐ+Ð+˜]Ð+Ð+ˆà€Jrƒ   TÚxr1   ÚpadÚchannels_lastc                 óð  — | j         \  }}}}|\  }}	|rM||z  dk    s	||	z  dk    r;|||z  z
  |z  }
|	||	z  z
  |	z  }t          j        | d|d|
f¦  «        } | j         \  }}}}||z  ||	z  }}|r3|                      ||||||	¦  «                             dddddd¦  «        }n2|                      ||||||	¦  «                             dddddd¦  «        }|                     |||z  ||	z  |z  ¦  «        }|||ffS )uÏ  Patchify a batch of images.

    Args:
        x: Input tensor of shape ``[B, C, H, W]``.
        patch_size: Patch dimensions ``(patch_h, patch_w)``.
        pad: Whether to pad images to be divisible by patch size.
        channels_last: Per-patch flat layout. ``True`` (default) yields ``P-P-C``
            flat (channel index varies fastest); ``False`` yields ``C-P-P`` flat
            (channel index varies slowest â€” HF / Gemma4 native layout).

    Returns:
        Tuple of ``(patches, grid_size)`` where ``patches`` has shape ``[B, N, P*P*C]``
        and ``grid_size`` is ``(num_patches_h, num_patches_w)``. Grid order within
        ``N`` is row-major over ``(y, x)`` regardless of ``channels_last``.
    r   é   é   é   é   r!   )ÚshapeÚFr–   ÚviewÚpermuteÚreshape)r•   r1   r–   r—   ÚBÚCÚHÚWÚphÚpwÚpad_hÚpad_wÚ_ÚnhÚnwÚpatchess                   r„   Úbatch_patchifyr®   ©   s?  € ð* ”�J€A€qˆ!ˆQØ�F€Bˆð ð ��B‘˜!’�˜q 2™v¨š{˜{Ø�a˜"‘f‘ Ñ"ˆØ�a˜"‘f‘ Ñ"ˆÝŒE�!�a˜  5Ð)Ñ*Ô*ˆØ”W‰
ˆˆ1ˆa�à�"‰W�a˜2‘gˆ€Bð ð IØ—&’&˜˜A˜r 2 r¨2Ñ.Ô.×6Ò6°q¸!¸QÀÀ1ÀaÑHÔHˆˆà—&’&˜˜A˜r 2 r¨2Ñ.Ô.×6Ò6°q¸!¸QÀÀ1ÀaÑHÔHˆà�oŠo˜a  b¡¨"¨r©'°A©+Ñ6Ô6€Gà�R˜�HÐÐrƒ   Ú_coordc                 óÎ   — | d d …d d …df                               d¬¦  «        dz   }| d d …d d …df                               d¬¦  «        dz   }d„ t          ||¦  «        D ¦   «         S )Nr   r!   ©Údimc                 ó˜   — g | ]G\  }}t          |                     ¦   «         ¦  «        t          |                     ¦   «         ¦  «        f‘ŒHS r‚   )r}   Úitem)r‰   ÚhÚws      r„   ú
<listcomp>z/calculate_naflex_grid_sizes.<locals>.<listcomp>Ú   s=   € ÐIÐIÐI©t¨q°!�S�—’‘”‰]Œ]�C §¢¡¤™MœMÐ*ÐIÐIÐIrƒ   )ÚamaxÚzip)r¯   Úmax_yÚmax_xs      r„   Úcalculate_naflex_grid_sizesr¼   Ö   sz   € à�1�1�1�a�a�a˜�7ŒO× Ò  QÐ Ñ'Ô'¨!Ñ+€EØ�1�1�1�a�a�a˜�7ŒO× Ò  QÐ Ñ'Ô'¨!Ñ+€EØIÐIµs¸5À%Ñ7HÔ7HÐIÑIÔIÐIrƒ   FÚpatch_coordÚar_preservingc                 óþ   — |                       d¬¦  «        dz   }|r*|                      dd¬¦  «                             |¦  «        }d|                      t          j        ¬¦  «        z  dz   |dd…ddd…f         z  dz
  S )	zTNormalize NaFlex patch coordinates for ``grid_sample`` with ``align_corners=False``.r!   r±   T©r²   Úkeepdimç       @©Údtypeç      ð?N)r¸   Ú	expand_asÚtoÚtorchÚfloat32)r½   r¾   Úshapess      r„   Ú$_normalize_naflex_grid_sample_coordsrË   Ý   s‰   € ð
 ×Ò !ÐÑ$Ô$ qÑ(€FØð DØ—’ ¨D�Ñ1Ô1×;Ò;¸FÑCÔCˆà�+—.’.¥u¤}�.Ñ5Ô5Ñ5¸Ñ;¸vÀaÀaÀaÈÈqÈqÈqÀjÔ?QÑQÐTWÑWÐWrƒ   c                   óª   — e Zd ZdZdeeeef         ee         f         deeeef                  dededej	        dej
        fd„Zd	„ Zed
„ ¦   «         ZdS )ÚNaFlexRopeIteratorzXIterator for generating batched ROPE embeddings for mixed mode with multiple grid sizes.Úsize_to_indicesÚunique_sizesÚ
batch_sizeÚseq_lenÚdevicerÄ   c                 ó,  — || _         || _        || _        || _        || _        || _        || _        |j        | _        |j        | _        d|j	        z  |j        z  | _
        d| _        i | _        |D ]"}|                     |¬¦  «        }	|	| j        |<   Œ#d S )Nr™   r   ©r�   )ÚroperÎ   rÏ   rÐ   rÑ   rÄ   rÒ   r5   r6   r²   Úhead_dimÚ
_depth_idxÚ_embeddings_per_sizeÚ	get_embed)
ÚselfÚrope_modulerÎ   rÏ   rÐ   rÑ   rÒ   rÄ   Ú	grid_sizeÚ
rope_embeds
             r„   Ú__init__zNaFlexRopeIterator.__init__ì   s±   € ð  ˆŒ	Ø.ˆÔØ(ˆÔØ$ˆŒØˆŒØˆŒ
ØˆŒØ Ô&ˆŒ
Ø$Ô.ˆŒØ˜KœOÑ+¨{Ô/DÑDˆŒØˆŒð %'ˆÔ!Ø%ð 	>ð 	>ˆIà$×.Ò.°YÐ.Ñ?Ô?ˆJØ3=ˆDÔ% iÑ0Ð0ð	>ð 	>rƒ   c                 ó   — d| _         | S ©Nr   )r×   ©rÚ   s    r„   Ú__iter__zNaFlexRopeIterator.__iter__	  s   € ØˆŒØˆrƒ   c           	      ó|  — | j         | j        k    rt          ‚t          j        | j        | j        | j        | j        | j	        | j
        ¬¦  «        }| j        D ]T}|\  }}||z  }| j        |         }| j        |         | j                  }|D ] }|d d …d |…d d …f         ||d d …d |…d d …f<   Œ!ŒU| xj         dz  c_         |S )N©rÄ   rÒ   r!   )r×   r5   ÚStopIterationrÈ   ÚzerosrÐ   r6   rÑ   rÖ   rÄ   rÒ   rÏ   rÎ   rØ   )	rÚ   Úbatch_embedrÜ   rµ   r¶   Ú
actual_lenÚbatch_indicesÚembedÚbis	            r„   Ú__next__zNaFlexRopeIterator.__next__  sÿ   € àŒ?˜dœjÒ(Ð(ÝÐõ ”kØŒO˜Tœ^¨T¬\¸4¼=Ø”* T¤[ð
ñ 
ô 
ˆð Ô*ð 
	Nð 
	NˆIØ‰DˆAˆqØ˜Q™ˆJØ Ô0°Ô;ˆMð Ô-¨iÔ8¸¼ÔIˆEð $ð Nð N�Ø5:¸1¸1¸1¸k¸z¸kÈ1È1È1Ð;LÔ5M�˜B    ; J ;°°°Ð1Ñ2Ð2ðNð 	ˆŒ˜1ÑˆŒØÐrƒ   N)ry   rz   r{   r|   r   r   r}   r
   rÈ   rÒ   rÄ   rÞ   râ   r   rì   r‚   rƒ   r„   rÍ   rÍ   é   s¼   € € € € € ØbÐbð>ð ˜e C¨ Hœo¨t°C¬yÐ8Ô9ð>ð ˜5  c œ?Ô+ð	>ð
 ð>ð ð>ð ”ð>ð Œ{ð>ð >ð >ð >ð:ð ð ð ðð ñ Ôðð ð rƒ   rÍ   Ú	block_idxc                 óJ  — | j         dv p&| j        dvp| j        p| j        p| j        p| j        du}|rž| j         }|dk    r| j        dvrd}| j        rdnd| j        z   }| j        }|�t          |t          ¦  «        s||         }t          t          || j        | j        | j        | j        || j        || j        p
| j        dk    | j        ¬	¦  «        S | j        pt"          }i }| j        s| j        r| j        |d
<   | j        |d<   | j        r
| j        |d<   |rt          |fi |¤Ž}|S )zÒGet appropriate block function based on configuration.

    Returns a partially applied block constructor with EVA-specific
    or conflicting parameters pre-configured for the given block index if needed.
    )ÚevarÕ   )rP   ÚnoneNrr   rÕ   r!   r   Úmrope)
rs   ru   Ú	scale_mlpÚscale_attn_innerrv   r9   rw   Únum_prefix_tokensÚrotate_halfÚ
gated_attnr8   Úscale_attn_normrq   )rs   rQ   ru   rt   rw   r9   rG   rH   Ú
isinstancer}   r   r*   r8   r?   rv   rX   ro   r+   rq   )r…   rí   Úuse_eva_featuresrs   rô   r9   ro   Úblock_kwargss           r„   Úget_block_fnrû   )  s—  € ð 	Œ˜Ð(ð 	%ØŒ˜\Ð)ð	%àŒð	%ð 	Œð	%ð 	Ô!ð		%ð
 	Ô Ð$ð ð ð #à”Mˆ	Ø˜
Ò"Ð" s¤}¸LÐ'HÐ'HØˆIà"%¤/Ð8˜Q˜Q°q¸C¼NÑJÐØÔ'ˆØÐ#­J°|ÅSÑ,IÔ,IÐ#Ø'¨	Ô2ˆLÝÝØØ”~ØÔ(Ø Ô6Ø”mØ%Ø"%Ô";Ø/ØÔ,ÐH°´ÀÒ0HØ”~ð
ñ 
ô 
ð 	
ð ”<Ð(¥5ˆØˆØÔð 	H Ô!:ð 	Hà-0Ô-?ˆLÐ)Ñ*Ø.1Ô.GˆLÐ*Ñ+ØŒ>ð 	8Ø),¬ˆL˜Ñ&Øð 	9Ý˜xÐ8Ð8¨<Ð8Ð8ˆHØˆrƒ   c            +       ó$  ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d6deeeeef         f         dededee         de	de	dede	deeeeeef         f                  dedeeeef                  dede	de	dee
ej                          dee	ee
ej                          f         dee
ej                          dede	d e	d!df*ˆ fd"„Zd7d#„Zej        j        d$eeeeeef         f                  d!dfd%„¦   «         Zd!eeef         fd&„Zd8d'e	d!eeeeef         f         fd(„Zd)eeef         d!eeef         fd*„Zed+ej        d,ej        d!dfd-„¦   «         Zd+ej        d,ej        d!dfd.„Zd+ej        d/ee         d!dfd0„Zed+ej        d,ej        d!dfd1„¦   «         Z ed+ej        d,ej        d!dfd2„¦   «         Z!d+ej        d/ee         d!dfd3„Z"	 	 d9d+ej        d,eej                 d4eej                 d!eej        eeeef                  f         fd5„Z#ˆ xZ$S ):ÚNaFlexEmbedsaõ  NaFlex Embedding module for Vision Transformers.

    This module encapsulates the complete embedding process for Vision Transformers,
    supporting both standard and NaFlex (NaViT + FlexiViT) functionality:

    1. Patch embedding (via Conv2d or Linear)
    2. Class and register token preparation
    3. Position embedding addition with interpolation support
    4. Pre-normalization (if requested)
    5. Dropout application

    NaFlex capabilities include:
    - Variable aspect ratio and resolution via patch coordinates
    - Patch type indicators for handling padding tokens in attention
    - Flexible position embedding interpolation for arbitrary grid sizes
    - Support for factorized position embeddings

    The patch embedding can be one of two types:
    - Conv2d-based (default): For standard image inputs [B, C, H, W]
    - Linear-based: For pre-patchified inputs [B, N, P*P*C]

    Args:
        patch_size: Size of patches for patch embedding
        in_chans: Number of input image channels
        embed_dim: Dimensionality of patch embedding
        proj_type: Type of embedding projection layer ('conv' or 'linear')
        input_norm_layer: Normalization layer applied to input (linear mode only)
        proj_norm_layer: Normalization layer applied after projection
        pos_embed: Type of position embedding ('learned', 'factorized', 'none')
        pos_drop_rate: Dropout rate for position embeddings
        class_token: Whether to include a class token
        reg_tokens: Number of register tokens to include
        bias: Whether to use bias in projection layers
        dynamic_img_pad: Whether to enable dynamic padding for variable resolution
        pos_embed_grid_size: Grid size for position embedding initialization
        pos_embed_interp_mode: Interpolation mode for position embedding resizing
        pos_embed_ar_preserving: Whether to preserve aspect ratio during position embedding interpolation
        default_img_size: Default image size for position embedding grid calculation
    r0   r›   r2   NTr   FrI   ©é   rÿ   rL   r=   r1   Úin_chansr3   Ú	proj_typer<   rG   rH   r]   Údefault_img_sizerJ   rK   rM   rN   rO   rk   Úproj_norm_layerrm   rC   rx   r—   r†   c           	      ó¦  •— ||dœ}t          ¦   «                              ¦   «          || _        || _        || _        || _        || _        t          |¦  «        | _        || _	        || _
        || _        || _        || _        |rdnd| _        | xj        |z  c_        |r&t          j        t#          j        dd|fi |¤Ž¦  «        nd| _        |r&t          j        t#          j        d||fi |¤Ž¦  «        nd| _        d| _        d| _        |�|| _        nL|	�Jt          |	¦  «        | _        t/          d„ t1          | j        | j        ¦  «        D ¦   «         ¦  «        | _        |dk    ru| j        d         | j        d         z  |z  }|du r|€
J d¦   «         ‚|du r|n|pd}|r ||¦  «        nd| _        t          j        ||fd	|i|¤Ž| _        d
| _        d| _        n5|rJ ‚d| _        t          j        ||f|||dœ|¤Ž| _        d| _        d
| _        | j        r!t?          | j        |||d||¬¦  «        | _         nd| _         tC          | j        o| j         duo| j        du ¦  «        | _"        |du r|€
J d¦   «         ‚|du r|n|pd}|r ||¦  «        nt          j#        ¦   «         | _$        |
dv r| j        €tK          d¦  «        ‚d| _&        d| _'        d| _(        |
r|
dk    rd| _)        n½|
dk    rq| j        €J ‚| j        \  }}d| _)        t          j        t#          j        d||fi |¤Ž¦  «        | _'        t          j        t#          j        d||fi |¤Ž¦  «        | _(        nF| j        €J ‚| j        \  }}t          j        t#          j        d|||fi |¤Ž¦  «        | _&        d| _)        t          j*        |¬¦  «        | _+        |  ,                    ¦   «          dS )aÀ  Initialize NaFlexEmbeds module.

        Args:
            patch_size: Size of patches for patch embedding.
            in_chans: Number of input image channels.
            embed_dim: Dimensionality of patch embedding.
            proj_type: Type of embedding projection layer ('conv' or 'linear').
            proj_bias: Whether to use bias in projection layers.
            class_token: Whether to include a class token.
            reg_tokens: Number of register tokens to include.
            dynamic_img_pad: Whether to enable dynamic padding for variable resolution.
            default_img_size: Default image size for position embedding grid calculation.
            pos_embed: Type of position embedding ('learned', 'factorized', 'none').
            pos_embed_grid_size: Grid size for position embedding initialization.
            pos_embed_interp_mode: Interpolation mode for position embedding resizing.
            pos_embed_ar_preserving: Whether to preserve aspect ratio during interpolation.
            input_norm_layer: Normalization layer applied to input (linear mode only).
            proj_norm_layer: Normalization layer applied after projection.
            norm_layer: Default normalization layer.
            pos_drop_rate: Dropout rate for position embeddings.
            enable_patch_interpolator: Enable dynamic patch size support.
            channels_last: Per-patch flat layout: True=(ph,pw,C) [NaFlex default], False=(C,ph,pw).
        ©rÒ   rÄ   r!   r   Nc                 ó   — g | ]
\  }}||z  ‘ŒS r‚   r‚   )r‰   ÚsÚps      r„   r·   z)NaFlexEmbeds.__init__.<locals>.<listcomp>Ø  s    € Ð-mÐ-mÐ-m¹¸¸A¨a°1©fÐ-mÐ-mÐ-mrƒ   ri   Tz5`norm_layer` must be given when input_norm_layer=TrueÚbiasF)Úkernel_sizeÚstrider	  )Úbase_patch_sizer   r3   ÚinterpolationÚ	antialiasr—   rÒ   z4`norm_layer` must be given when proj_norm_layer=True)Ú
factorizedrI   zgCannot initialize position embeddings without grid_size.Please provide img_size or pos_embed_grid_size.rð   r  rI   )r  )-ÚsuperrÞ   Úhas_class_tokenÚnum_reg_tokensrM   rN   rO   r   r1   r   r3   r]   rx   r—   rô   ÚnnÚ	ParameterrÈ   ÚemptyÚ	cls_tokenÚ	reg_tokenr  rK   Útupler¹   Ú
norm_inputÚLinearÚprojÚflattenÚ	is_linearÚConv2dr   Úpatch_interpolatorr€   Úsupports_patch_interpolationÚIdentityÚnormÚ
ValueErrorrJ   Úpos_embed_yÚpos_embed_xÚpos_embed_typeÚDropoutÚpos_dropÚreset_parameters)rÚ   r1   r   r3   r  r<   rG   rH   r]   r  rJ   rK   rM   rN   rO   rk   r  rm   rC   rx   r—   rÒ   rÄ   ÚddÚ	patch_dimrµ   r¶   Ú	__class__s                              €r„   rÞ   zNaFlexEmbeds.__init__‰  sÔ  ø€ ð` ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ*ˆÔØ(ˆÔØ%:ˆÔ"Ø'>ˆÔ$Ø)BˆÔ&Ý# JÑ/Ô/ˆŒØ ˆŒØ"ˆŒØ.ˆÔØ)BˆÔ&Ø*ˆÔð '2Ð!8  °qˆÔØÐÔ *Ñ,ÐÔð NYÐb�œ¥e¤k°!°Q¸	Ð&HÐ&HÀRÐ&HÐ&HÑIÔIÐIÐ^bˆŒØV`Ðj�œ¥e¤k°!°ZÀÐ&QÐ&QÈbÐ&QÐ&QÑRÔRÐRÐfjˆŒð <@ˆÔØ>BˆÔ ØÐ*à':ˆDÔ$Ð$ØÐ)å$-Ð.>Ñ$?Ô$?ˆDÔ!Ý',Ð-mÐ-mÅÀTÔEZÐ\`Ô\kÑAlÔAlÐ-mÑ-mÔ-mÑ'nÔ'nˆDÔ$ð ˜Ò Ð ð œ¨Ô*¨T¬_¸QÔ-?Ñ?À(ÑJˆIØ(¨DÐ0Ð0°ZÐ5GÐ5GØGñ 6HÔ5GÐHà-=ÀÐ-EÐ-E˜z˜zÐL\ÐLdÐ`dÐØ=MÐWÐ.Ð.¨yÑ9Ô9Ð9ÐSWˆDŒOÝœ	 )¨YÐMÐM¸YÐMÈ"ÐMÐMˆDŒIØ ˆDŒLØ!ˆDŒNˆNð (Ð'Ð'Ð'Ø"ˆDŒOÝœ	ØØðð 'Ø!Øðð ð ðð ˆDŒIð  ˆDŒLØ"ˆDŒNð Ô)ð 	+Ý&<Ø $¤Ø!Ø#Ø3ØØ+Øð'ñ 'ô 'ˆDÔ#Ð#ð '+ˆDÔ#å,0ØŒNð (ØÔ'¨tÐ3ð(à” 4Ð'ñ-
ô -
ˆÔ)ð $ tÐ+Ð+°
Ð0BÐ0BØBñ 1CÔ0BÐCà(7¸4Ð(?Ð(?˜*˜*ÀoÐF]ÐY]ˆØ2AÐT�O�O IÑ.Ô.Ð.ÅrÄ{Á}Ä}ˆŒ	ð Ð1Ð1Ð1°dÔ6NÐ6VÝðBñCô Cð Cð 26ˆŒØ37ˆÔØ37ˆÔØð 	,˜I¨Ò/Ð/Ø"(ˆDÔÐØ˜,Ò&Ð&ØÔ+Ð7Ð7Ð7ØÔ+‰DˆAˆqØ".ˆDÔÝ!œ|­E¬K¸¸1¸iÐ,NÐ,NÈ2Ð,NÐ,NÑOÔOˆDÔÝ!œ|­E¬K¸¸1¸iÐ,NÐ,NÈ2Ð,NÐ,NÑOÔOˆDÔÐàÔ+Ð7Ð7Ð7ØÔ+‰DˆAˆqÝœ\­%¬+°a¸¸A¸yÐ*OÐ*OÈBÐ*OÐ*OÑPÔPˆDŒNØ"+ˆDÔõ œ
 ]Ð3Ñ3Ô3ˆŒð 	×ÒÑÔÐÐÐrƒ   c                 óÌ  — | j         �&t          j                             | j         d¬¦  «         | j        �&t          j                             | j        d¬¦  «         | j        �&t          j                             | j        d¬¦  «         | j        �&t          j                             | j        d¬¦  «         | j        �(t          j                             | j        d¬¦  «         d S d S )Ng�íµ ÷Æ°>)Ústdg{®Gáz”?)r  r  ÚinitÚnormal_r  rJ   r$  r%  rá   s    r„   r)  zNaFlexEmbeds.reset_parameters+  s¾   € ØŒ>Ð%ÝŒG�OŠO˜DœN°ˆOÑ5Ô5Ð5ØŒ>Ð%ÝŒG�OŠO˜DœN°ˆOÑ5Ô5Ð5ØŒ>Ð%ÝŒG�OŠO˜DœN°ˆOÑ4Ô4Ð4ØÔÐ'ÝŒG�OŠO˜DÔ,°#ˆOÑ6Ô6Ð6ØÔÐ'ÝŒG�OŠO˜DÔ,°#ˆOÑ6Ô6Ð6Ð6Ð6ð (Ð'rƒ   Úpatch_sizesc                 óˆ   — | j         st          d¦  «        ‚| j                             || j        j        j        ¬¦  «         dS )aF  Precompute patch interpolation matrices on the projection device.

        The cache is cleared by any subsequent ``.to()`` / dtype conversion of the model,
        so prewarm after the model has been moved to its execution device.

        Args:
            patch_sizes: Iterable of target patch sizes to precompute.
        zEPatch interpolation is not supported by this embedding configuration.©rÒ   N)r   ÚRuntimeErrorr  Úprewarmr  ÚweightrÒ   ©rÚ   r1  s     r„   Úprewarm_patch_interpolatorz'NaFlexEmbeds.prewarm_patch_interpolator7  sI   € ð Ô0ð 	hÝÐfÑgÔgÐgØÔ×'Ò'¨¸D¼IÔ<LÔ<SÐ'ÑTÔTÐTÐTÐTrƒ   c                 ó8   — t          | j        | j        ¬¦  «        S )zãGet feature information for feature extraction.

        Args:
            location: Feature extraction location identifier

        Returns:
            Dictionary containing feature channel count and reduction factor
        )Únum_chsÚ	reduction)Údictr3   r1   )rÚ   Úlocations     r„   Úfeature_infozNaFlexEmbeds.feature_infoH  s   € õ ˜DœN°d´oÐFÑFÔFÐFrƒ   Ú	as_scalarc                 ó<   — |rt          | j        ¦  «        S | j        S )zìGet the feature reduction ratio (stride) of the patch embedding.

        Args:
            as_scalar: Whether to return the maximum dimension as a scalar

        Returns:
            Feature reduction ratio as scalar or tuple
        )Úmaxr1   )rÚ   r?  s     r„   Ú
feat_ratiozNaFlexEmbeds.feat_ratioS  s$   € ð ð 	#Ý�t”Ñ'Ô'Ð'à”?Ð"rƒ   Úimg_sizec                 ó  — | j         rPt          j        |d         | j        d         z  ¦  «        t          j        |d         | j        d         z  ¦  «        fS |d         | j        d         z  |d         | j        d         z  fS )a  Calculate grid (feature) size for given image size.

        Takes into account dynamic padding when enabled.

        Args:
            img_size: Input image size as (height, width)

        Returns:
            Grid size as (grid_height, grid_width)
        r   r!   )r]   ÚmathÚceilr1   )rÚ   rC  s     r„   Údynamic_feat_sizezNaFlexEmbeds.dynamic_feat_sizea  s   € ð Ôð 	XÝ”9˜X aœ[¨4¬?¸1Ô+=Ñ=Ñ>Ô>ÅÄ	È(ÐSTÌ+ÐX\ÔXgÐhiÔXjÑJjÑ@kÔ@kÐkÐkà˜A”; $¤/°!Ô"4Ñ4°h¸q´kÀTÄ_ÐUVÔEWÑ6WÐWÐWrƒ   r•   r½   c           
      ó¶  ‡ ‡‡‡‡— t          |¦  «        }‰ j        j        dd…         \  ŠŠ‰ j                             dddd¦  «                             ¦   «         Šˆˆˆˆ ˆfd„}i }t          |¦  «        D ].\  }}|                     |g ¦  «                             |¦  «         Œ/|                     ¦   «         D ] \  }} ||¦  «        }	t          ‰j        d         |	j        d         ¦  «        }
‰dd…d|
…f          
                    dt          j        |‰j        ¬¦  «        |	dd…d|
…f                              t          |¦  «        dd¦  «        ¦  «         Œ¡dS )	a_  Apply learned position embeddings to NaFlex batch in-place.

        Interpolates learned 2D position embeddings for each sample in the batch
        based on their individual grid sizes.

        Args:
            x: Input tensor to add position embeddings to [B, N, C]
            patch_coord: Patch coordinates [B, N, 2] with (y, x) values
        r!   r›   r   r™   c                 óº  •— | d         ‰k    r,| d         ‰k    r ‰j                              d‰‰z  d¦  «        }nˆ‰j        rt          t	          | ¦  «        ¦  «        n| }t          j        ‰|‰j        dd¬¦  «        dd…dd…d| d         …d| d         …f                              d¦  «         	                    dd¦  «        }| 
                    ‰j        ¬	¦  «        S )
a  
            Return a flattened positional-embedding grid at an arbitrary spatial resolution.

            Converts the learned 2-D table stored in NCHW format (pos_embed_nchw) into
            a (1, H*W, C) sequence that matches the requested size.
            r   r!   éÿÿÿÿFT©ÚsizeÚmodeÚalign_cornersr  Nr™   rÃ   )rJ   r¡   rN   r   rA  rž   ÚinterpolaterM   r  Ú	transposerÇ   rÄ   )rL  Úpos_embed_flatÚ_interp_sizeÚorig_hÚorig_wÚpos_embed_nchwrÚ   r•   s      €€€€€r„   Ú	_interp2dz?NaFlexEmbeds._apply_learned_naflex_pos_embed.<locals>._interp2d…  sì   ø€ ð �Q”˜6Ò!Ð!¨¨Q¬°6Ò(9Ð(9Ø!%¤×!7Ò!7¸¸6ÀF¹?ÈBÑ!OÔ!O��à7;Ô7SÐ]�y­¨T©¬Ñ3Ô3Ð3ÐY]�Ý!"¤Ø"Ø%ØÔ3Ø"'Ø"ð"ñ "ô "ð �!�!�Q�Q�Q˜˜˜aœ˜ ( 4¨¤7 (Ð*ô",÷ -4ªG°A©J¬J·y²yÀÀA±´ð ð "×$Ò$¨1¬7Ð$Ñ3Ô3Ð3rƒ   Nr3  rJ  )r¼   rJ   r�   r    r   Ú	enumerateÚ
setdefaultÚappendr‘   ÚminÚ
index_add_rÈ   Ú	as_tensorrÒ   ÚexpandÚlen)rÚ   r•   r½   Únaflex_grid_sizesrV  rÎ   rë   rŠ   ré   rQ  rÑ   rS  rT  rU  s   ``         @@@r„   Ú_apply_learned_naflex_pos_embedz,NaFlexEmbeds._apply_learned_naflex_pos_embedq  s”  øøøøø€ õ  8¸ÑDÔDÐØœÔ-¨a°¨cÔ2‰ˆ�Øœ×/Ò/°°1°a¸Ñ;Ô;×AÒAÑCÔCˆð	4ð 	4ð 	4ð 	4ð 	4ð 	4ð 	4ð 	4ð 	4ð* =?ˆÝÐ0Ñ1Ô1ð 	9ð 	9‰EˆB�à×&Ò& q¨"Ñ-Ô-×4Ò4°RÑ8Ô8Ð8Ð8à /× 5Ò 5Ñ 7Ô 7ð 		ð 		ÑˆAˆ}ð '˜Y q™\œ\ˆNÝ˜!œ' !œ* nÔ&:¸1Ô&=Ñ>Ô>ˆGØˆaˆaˆa��'�ˆkŒN×%Ò%ØÝ” °a´hÐ?Ñ?Ô?Ø˜q˜q˜q ( 7 (˜{Ô+×2Ò2µ3°}Ñ3EÔ3EÀrÈ2ÑNÔNñô ð ð ð		ð 		rƒ   c                 óä  — |j         d         }t          || j        ¬¦  «                             d¦  «                             d¦  «        }t          j        | j                             dddd¦  «         	                    |ddd¦  «         
                    ¦   «         || j        dd¬	¦  «                             |j        ¬
¦  «        }||                     d¦  «                             dd¦  «        z  }dS )aÉ  Apply learned position embeddings to NaFlex batch using grid_sample.

        Directly samples the learned 2D position embeddings at each patch coordinate,
        avoiding a dense grid sized by the largest height and width in the batch.
        Based on proposal by https://github.com/stas-sl

        Args:
            x: Input tensor to add position embeddings to [B, N, C]
            patch_coord: Patch coordinates [B, N, 2] with (y, x) values
        r   )r¾   rJ  r™   r›   r!   FÚborder©rM  rN  Úpadding_moderÃ   N)r�   rË   rN   ÚflipÚ	unsqueezerž   Úgrid_samplerJ   r    r]  r   rM   rÇ   rÄ   ÚsqueezerP  )rÚ   r•   r½   r¢   ÚgridrJ   s         r„   Ú+_apply_learned_naflex_pos_embed_grid_samplez8NaFlexEmbeds._apply_learned_naflex_pos_embed_grid_sampleª  sê   € ð ŒG�AŒJˆÝ3ØØÔ6ð
ñ 
ô 
÷ Š$ˆr‰(Œ(—9’9˜Q‘<”<ð 	õ ”MØŒN×"Ò" 1 a¨¨AÑ.Ô.×5Ò5°a¸¸RÀÑDÔD×JÒJÑLÔLØØÔ+ØØ!ð
ñ 
ô 
÷ Š"�1”7ˆ"Ñ
Ô
ð 	ð 	
ˆY×Ò˜rÑ"Ô"×,Ò,¨Q°Ñ2Ô2Ñ2ˆˆˆrƒ   rÜ   c                 ó^  — | j         j        dd…         \  }}|d         |k    r,|d         |k    r | j                              d||z  d¦  «        }n­| j        rt	          |¦  «        }||f}n|}t          j        | j                              dddd¦  «                             ¦   «         || j	        dd¬¦  «        d	d	…d	d	…d	|d         …d	|d         …f          
                    d¦  «                             dd¦  «        }|                     |j        ¬
¦  «        }|                     |¦  «         d	S )a0  Apply learned position embeddings to standard 2D batch in-place.

        Interpolates learned 2D position embeddings to match the specified grid size.

        Args:
            x: Input tensor to add position embeddings to [B, H*W, C]
            grid_size: Target grid size as [height, width]
        r!   r›   r   rJ  r™   FTrK  NrÃ   )rJ   r�   r¡   rN   rA  rž   rO  r    r   rM   r  rP  rÇ   rÄ   Úadd_)rÚ   r•   rÜ   rS  rT  rQ  ÚLrR  s           r„   Ú_apply_learned_pos_embedz%NaFlexEmbeds._apply_learned_pos_embedÈ  sB  € ð œÔ-¨a°¨cÔ2‰ˆ�Ø�QŒ<˜6Ò!Ð! i°¤l°fÒ&<Ð&<à!œ^×3Ò3°A°vÀ±ÈÑKÔKˆNˆNð Ô+ð )Ý˜	‘N”N�Ø  !˜t��à(�Ýœ]Ø”×&Ò& q¨!¨Q°Ñ2Ô2×8Ò8Ñ:Ô:Ø!ØÔ/Ø#Øðñ ô ð ˆaˆa����M�Y˜q”\�M = I¨a¤L =Ð0ô2÷ 3:²'¸!±*´*¿YºYÀqÈ!¹_¼_ð ð (×*Ò*°´Ð*Ñ9Ô9ˆà	�Šˆ~ÑÔÐÐÐrƒ   c           
      ó.  ‡— t          |¦  «        }t          |¦  «        ‰                     d¦  «        k    sJ ‚| j        j        d         | j        j        d         }}i }t          |¦  «        D ].\  }}|                     |g ¦  «                             |¦  «         Œ/dt          j
        dt          dt          dt          j
        fˆfd„}	|                     ¦   «         D �]6\  }}
|\  }}| j        rt          ||¦  «        x}}n||}} |	| j        ||¦  «        dd…d|…f         } |	| j        ||¦  «        dd…d|…f         }|                     d	¦  «        |                     d¦  «        z   }|                     dd	¦  «        }t#          ‰j        d         |j        d         ¦  «        }‰dd…d|…f                              dt          j        |
‰j        ¬
¦  «        |dd…d|…f                              t          |
¦  «        dd¦  «        ¦  «         �Œ8dS )ac  Apply factorized position embeddings to NaFlex batch in-place.

        Uses separate Y and X position embedding tables that are interpolated
        and combined for each sample's grid size.

        Args:
            x: Input tensor to add position embeddings to [B, N, C]
            patch_coord: Patch coordinates [B, N, 2] with (y, x) values
        r   r!   ÚtableÚ
new_lengthÚorig_lengthr†   c                 ó   •— ||k    r|                       ‰j        ¬¦  «        S t          j        |                      ddd¦  «                             ¦   «         |dd¬¦  «                             ddd¦  «                              ‰j        ¬¦  «        S )z˜
            Resample a 1-D positional-embedding table to specified length
            and return it in (1, L, C) layout, dtype matching x.
            rÃ   r   r™   r!   ri   F©rL  rM  rN  ©rÇ   rÄ   rž   rO  r    r   ©rp  rq  rr  r•   s      €r„   Ú	_interp1dzBNaFlexEmbeds._apply_factorized_naflex_pos_embed.<locals>._interp1d  sŠ   ø€ ð
 ˜[Ò(Ð(Ø—x’x a¤g�xÑ.Ô.Ð.Ý”=Ø—’˜a  AÑ&Ô&×,Ò,Ñ.Ô.ØØØ#ð	ñ ô ÷
 Šg�a˜˜AÑÔŸršr¨¬˜rÑ0Ô0ð1rƒ   Nr™   r3  rJ  )r¼   r^  rL  r$  r�   r%  rW  rX  rY  rÈ   ÚTensorr}   r‘   rN   rA  rf  r  rZ  r[  r\  rÒ   r]  )rÚ   r•   r½   r_  rS  rT  rÎ   rë   rŠ   rw  ré   Útarget_hÚtarget_wÚlen_yÚlen_xÚpe_yÚpe_xÚposrÑ   s    `                 r„   Ú"_apply_factorized_naflex_pos_embedz/NaFlexEmbeds._apply_factorized_naflex_pos_embedë  sT  ø€ õ  8¸ÑDÔDÐÝÐ$Ñ%Ô%¨¯ª°©¬Ò2Ð2Ð2Ð2ð Ô)Ô/°Ô2°DÔ4DÔ4JÈ1Ô4M�ˆð =?ˆÝÐ0Ñ1Ô1ð 	9ð 	9‰EˆB�Ø×&Ò& q¨"Ñ-Ô-×4Ò4°RÑ8Ô8Ð8Ð8ð	1�Uœ\ð 	1µsð 	1Íð 	1ÕQVÔQ]ð 	1ð 	1ð 	1ð 	1ð 	1ð 	1ð !0× 5Ò 5Ñ 7Ô 7ð 	ñ 	ÑˆAˆ}Ø!"ÑˆH�hØÔ+ð 2Ý # H¨hÑ 7Ô 7Ð7�˜˜à'¨�u�à�9˜TÔ-¨u°fÑ=Ô=¸a¸a¸aÀÀ(À¸lÔKˆDØ�9˜TÔ-¨u°fÑ=Ô=¸a¸a¸aÀÀ(À¸lÔKˆDð —.’. Ñ#Ô# d§n¢n°QÑ&7Ô&7Ñ7ˆCØ—+’+˜a Ñ#Ô#ˆCå˜!œ' !œ* c¤i°¤lÑ3Ô3ˆGØˆaˆaˆa��'�ˆkŒN×%Ò%ØÝ” °a´hÐ?Ñ?Ô?Ø�A�A�A�x˜�x�KÔ ×'Ò'­¨MÑ(:Ô(:¸BÀÑCÔCñô ð ñ ð	ð 	rƒ   c                 óÄ  ‡‡‡— ‰j         }‰j        \  Š}Š|                     d¬¦  «        dz   }| j        r6|                     d¬¦  «        }|                     ¦   «         }|x}}	||z  x}
}n6|                     d¦  «        \  }}	|	|dd…df         z  }
||dd…df         z  }dt          j        dt          j        dt          j        dt          j        fˆˆˆfd	„} || j        |
|	¬
¦  «        } || j        ||¬
¦  «        }t	          j        ‰|t          j	        ¬¦  «         
                    d¦  «        }‰||dd…d|d         f         ||dd…d|d         f         z   z  ŠdS )a¥  Apply factorized position embeddings to NaFlex batch using grid_sample.

        Uses F.grid_sample for efficient interpolation of separate Y and X position
        embedding tables based on patch coordinates. Based on proposal by https://github.com/stas-sl

        Args:
            x: Input tensor to add position embeddings to [B, N, C]
            patch_coord: Patch coordinates [B, N, 2] with (y, x) values
        r!   r±   r   Nrp  ÚscaleÚ
out_lengthr†   c                 óÀ  •— |                       ddd¦  «                             d¦  «                             ‰ddd¦  «                             ¦   «         }t	          j        ‰dd‰j        ¬¦  «        }||d d …ddf<   |dz
  |d d …ddf<   d|d d …ddf<   t          j        |‰‰d|fd¬¦  «        }t          j	        ||d	dd
¬¦  «        }| 
                    ‰j        ¦  «        S )Nr   r™   r!   rJ  r›   r3  F)rN  Úbilinearrb  rc  )r    rf  r]  r   rÈ   ræ   rÒ   rž   Úaffine_gridrg  rÇ   rÄ   )	rp  r‚  rƒ  ÚpeÚthetari  r¢   r£   r•   s	         €€€r„   rw  zNNaFlexEmbeds._apply_factorized_naflex_pos_embed_grid_sample.<locals>._interp1dH  sé   ø€ Ø—’˜q ! QÑ'Ô'×1Ò1°!Ñ4Ô4×;Ò;¸A¸rÀ2ÀrÑJÔJ×PÒPÑRÔRˆBÝ”K  1 a°´Ð9Ñ9Ô9ˆEØ"ˆE�!�!�!�Q˜�'‰NØ" Q™YˆE�!�!�!�Q˜�'‰NØˆE�!�!�!�Q˜�'‰NÝ”= ¨¨A¨q°*Ð(=ÈUÐSÑSÔSˆDÝ”˜r 4¨jÈÐ\dÐeÑeÔeˆBØ—5’5˜œ‘>”>Ð!rƒ   )r‚  rƒ  r  ).r!   ).r   )rÒ   r�   r¸   rN   rÈ   rx  r%  r$  ÚarangeÚlongrf  )rÚ   r•   r½   rÒ   rª   rÊ   ÚL_iÚL_globalÚgrid_size_yÚgrid_size_xÚscale_xÚscale_yrw  r~  r}  rë   r¢   r£   s    `              @@r„   Ú._apply_factorized_naflex_pos_embed_grid_samplez;NaFlexEmbeds._apply_factorized_naflex_pos_embed_grid_sample)  s¦  øøø€ ð ”ˆØ”'‰ˆˆ1ˆaØ×!Ò! aÐ!Ñ(Ô(¨1Ñ,ˆàÔ'ð 
	1à—+’+ !�+Ñ$Ô$ˆCØ—x’x‘z”zˆHØ(0Ð0ˆK˜+Ø (¨3¡Ð.ˆG�g�gð (.§{¢{°1¡~¤~Ñ$ˆK˜Ø! F¨1¨1¨1¨a¨4¤LÑ0ˆGØ! F¨1¨1¨1¨a¨4¤LÑ0ˆGð	"�Uœ\ð 	"µ%´,ð 	"ÍEÌLð 	"Õ]bÔ]ið 	"ð 	"ð 	"ð 	"ð 	"ð 	"ð 	"ð 	"ð ˆy˜Ô)°À[ÐQÑQÔQˆØˆy˜Ô)°À[ÐQÑQÔQˆåŒ\˜! Fµ%´*Ð=Ñ=Ô=×GÒGÈÑJÔJˆØ	ˆT�"�a�a�a˜˜K¨Ô/Ð/Ô0°4¸¸A¸A¸A¸qÀ+ÈfÔBUÐ8UÔ3VÑVÑVˆˆˆrƒ   c                 ó  ‡— | j         j        d         | j        j        d         }}|\  }}| j        rt	          ||¦  «        x}}n||}}dt
          j        dt          dt          dt
          j        fˆfd„}	 |	| j         ||¦  «        dd…d|…f         }
 |	| j        ||¦  «        dd…d|…f         }|
                     d¦  «        |                     d¦  «        z   }| 	                    dd¦  «        }‰ 
                    |¦  «         dS )	a]  Apply factorized position embeddings to standard 2D batch in-place.

        Uses separate Y and X position embedding tables that are interpolated
        and combined for the specified grid size.

        Args:
            x: Input tensor to add position embeddings to [B, H*W, C]
            grid_size: Target grid size as [height, width]
        r!   rp  rq  rr  r†   c                 ó   •— ||k    r|                       ‰j        ¬¦  «        S t          j        |                      ddd¦  «                             ¦   «         |dd¬¦  «                             ddd¦  «                              ‰j        ¬¦  «        S )NrÃ   r   r™   r!   ri   Frt  ru  rv  s      €r„   rw  z;NaFlexEmbeds._apply_factorized_pos_embed.<locals>._interp1do  sˆ   ø€ Ø˜[Ò(Ð(Ø—x’x a¤g�xÑ.Ô.Ð.Ý”=Ø—’˜a  AÑ&Ô&×,Ò,Ñ.Ô.ØØØ#ð	ñ ô ÷
 Šg�a˜˜AÑÔŸršr¨¬˜rÑ0Ô0ð1rƒ   Nr™   )r$  r�   r%  rN   rA  rÈ   rx  r}   rf  r  rl  )rÚ   r•   rÜ   rS  rT  ry  rz  r{  r|  rw  r}  r~  rJ   rQ  s    `            r„   Ú_apply_factorized_pos_embedz(NaFlexEmbeds._apply_factorized_pos_embedY  s9  ø€ ð Ô)Ô/°Ô2°DÔ4DÔ4JÈ1Ô4M�ˆØ&Ñˆ�(àÔ'ð 	.Ý ¨(Ñ3Ô3Ð3ˆE�E�Eà# X�5ˆEð	1�Uœ\ð 	1µsð 	1Íð 	1ÕQVÔQ]ð 	1ð 	1ð 	1ð 	1ð 	1ð 	1ð ˆy˜Ô)¨5°&Ñ9Ô9¸!¸!¸!¸Y¸h¸Y¸,ÔGˆØˆy˜Ô)¨5°&Ñ9Ô9¸!¸!¸!¸Y¸h¸Y¸,ÔGˆð —N’N 1Ñ%Ô%¨¯ª°qÑ(9Ô(9Ñ9ˆ	Ø"×*Ò*¨1¨aÑ0Ô0ˆà	�Šˆ~ÑÔÐÐÐrƒ   Úpatch_validc                 ój  — d}|j         d         }| j        �r7|€?t          |j        dk    d¦  «         t	          || j        | j        | j        ¬¦  «        \  }}n$t          |j        dk    p
|j        dk    d¦  «         | j        r„|j        dk    ryt          | j	        du d	¦  «         t          | j        r|j         d
d…         n|j         dd…         ¦  «        }|                      || j        j        | j        j        |d¬¦  «        }�n>|                     d
¦  «        }| j	        �|  	                    |¦  «        }|                      |¦  «        }n÷t          |j        dk    d¦  «         | j        rƒ|j         dd…         \  }}| j        d         || j        d         z  z
  | j        d         z  }	| j        d         || j        d         z  z
  | j        d         z  }
t!          j        |d|
d|	f¦  «        }|                      |¦  «        }|j         dd…         }| j        r)|                     d
¦  «                             dd
¦  «        }|                      |¦  «        }| j        dk    rQ|�|                      ||¬¦  «         n’| j        r|                      ||¬¦  «         ns|                      ||¬¦  «         n[| j        dk    rP|�|                      ||¬¦  «         n6| j        r|                      ||¬¦  «         n|                      ||¬¦  «         g }| j        �/|                     | j                             |dd¦  «        ¦  «         | j        �/|                     | j                             |dd¦  «        ¦  «         |rtA          j!        ||gz   d¬¦  «        }|  "                    |¦  «        }||fS )a×  Forward pass for patch embedding with position encoding.

        Args:
            x: Input tensor. Supported formats:
                - [B, C, H, W] for conv mode
                - [B, N, P*P*C] for pre-patchified linear mode (normal)
                - [B, N, Ph, Pw, C] for pre-patchified linear mode (variable patch size)
            patch_coord: Optional patch coordinates [B, N, 2] for NaFlex mode.
            patch_valid: Optional validity mask for patches [B, N] for NaFlex mode.

        Returns:
            Tuple of (embedded_tensor, grid_size) where:
                - embedded_tensor: [B, num_prefix_tokens + N, embed_dim]
                - grid_size: (H, W) tuple for standard mode, None for NaFlex mode
        Nr   rš   z-Expecting 2D image input with input ndim == 4)r–   r—   rœ   r›   z/Expecting patchified input with ndim == 3 or 5.z,input norm not supported with patch resizingr™   T)r1   r  zConvolutional input must be 4Déþÿÿÿr!   rI   )rÜ   )r½   r  rJ  r±   )#r�   r  r   Úndimr®   r1   r]   r—   rx   r  r  r  r  r6  r	  r  rž   r–   rP  r"  r&  rn  rO   rj  r`  r”  r‘  r€  r  rY  r]  r  rÈ   Úcatr(  )rÚ   r•   r½   r•  rÜ   r¢   Úpatch_hwr¤   r¥   r¨   r©   Úto_cats               r„   ÚforwardzNaFlexEmbeds.forwardƒ  sæ  € ð* 04ˆ	ØŒG�AŒJˆØŒ>ñ ,	1àÐ"å˜œ !šÐ%TÑUÔUÐUÝ-Ø�t”¨DÔ,@ÐPTÔPbð dñ  dô  d‘��9�9õ
 ˜œ !šÐ2 q¤v°¢{Ð4eÑfÔfÐfð Ô-ð !°!´&¸A²+°+Ý˜œ¨4Ð/Ð1_Ñ`Ô`Ð`õ !°Ô1CÐ!U ¤¨¨1¨¤ ÈÌÐQRÐSTÐQTÌÑVÔV�à×+Ò+ØØ”IÔ$Ø”I”NØ'Ø"ð ,ñ ô �‘ð —I’I˜a‘L”L�Ø”?Ð.ØŸš¨Ñ*Ô*�AØ—I’I˜a‘L”L��å�A”F˜a’KÐ!AÑBÔBÐBØÔ#ð 3Ø”w˜r˜s˜s”|‘��1Øœ¨Ô+¨a°$´/À!Ô2DÑ.DÑDÈÌÐXYÔHZÑZ�Øœ¨Ô+¨a°$´/À!Ô2DÑ.DÑDÈÌÐXYÔHZÑZ�Ý”E˜!˜a ¨¨5Ð1Ñ2Ô2�à—	’	˜!‘”ˆAàœ   œˆIØŒ|ð 1Ø—I’I˜a‘L”L×*Ò*¨1¨aÑ0Ô0�ð �IŠI�a‰LŒLˆàÔ )Ò+Ð+ØÐ$à×-Ò-¨a¸9Ð-ÑEÔEÐEÐEð Ô1ð UØ×DÒDÀQÐT_ÐDÑ`Ô`Ð`Ð`à×8Ò8¸ÈÐ8ÑTÔTÐTÐTØÔ  LÒ0Ð0ØÐ$à×0Ò0°¸iÐ0ÑHÔHÐHÐHð Ô1ð XØ×GÒGÈÐWbÐGÑcÔcÐcÐcà×;Ò;¸AÈ;Ð;ÑWÔWÐWð ˆØŒ>Ð%Ø�MŠM˜$œ.×/Ò/°°2°rÑ:Ô:Ñ;Ô;Ð;ØŒ>Ð%Ø�MŠM˜$œ.×/Ò/°°2°rÑ:Ô:Ñ;Ô;Ð;àð 	/Ý”	˜& A 3™,¨AÐ.Ñ.Ô.ˆAð �MŠM˜!ÑÔˆà�)ˆ|Ðrƒ   )r0   r›   r2   NTTr   FNrI   rþ   rL   FFNNNr=   FTNN©r†   N©T©NN)%ry   rz   r{   r|   r   r}   r   r   r�   r€   r   r  ÚModuler   rÞ   r)  rÈ   ÚjitÚignorer	   r8  r   r   r>  rB  rG  r   rx  r`  rj  r
   rn  r€  r‘  r”  rœ  Ú__classcell__©r,  s   @r„   rý   rý   _  sÑ  ø€ € € € € ð&ð &ðT 79ØØ Ø'+Ø"Ø $ØØ$)ØFJØ&Ø=EØ)2Ø,1Ø.3Ø:>ØFJØ48Ø#%Ø.3Ø"&ØØð/` ð ` à˜c 5¨¨c¨¤?Ð2Ô3ð` ð ð` ð ð	` ð
   ”}ð` ð ð` ð ð` ð ð` ð "ð` ð ' u¨S°%¸¸S¸´/Ð-AÔ'BÔCð` ð ð` ð "*¨%°°S°¬/Ô!:ð` ð $'ð` ð &*ð` ð (,ð` ð  ' t¨B¬I¤Ô7ð!` ð" # 4¨°$°r´y´/Ô)BÐ#BÔCð#` ð$ !  b¤i¤Ô1ð%` ð& !ð'` ð( (,ð)` ð*  ð+` ð0 
ð1` ð ` ð ` ð ` ð ` ð ` ðD
7ð 
7ð 
7ð 
7ð „YÔðUà! %¨¨U°3¸°8¬_Ð(<Ô"=Ô>ðUð 
ðUð Uð Uñ ÔðUð 	G¨¨S°#¨X¬ð 	Gð 	Gð 	Gð 	Gð#ð # Dð #°E¸#¸uÀSÈ#ÀX¼Ð:NÔ4Oð #ð #ð #ð #ðX¨%°°S°¬/ð X¸eÀCÈÀH¼oð Xð Xð Xð Xð  ð6àŒ|ð6ð œð6ð 
ð	6ð 6ð 6ñ Ôð6ðp3àŒ|ð3ð œð3ð 
ð	3ð 3ð 3ð 3ð<!àŒ|ð!ð ˜C”yð!ð 
ð	!ð !ð !ð !ðF ð;àŒ|ð;ð œð;ð 
ð	;ð ;ð ;ñ Ôð;ðz ð-WàŒ|ð-Wð œð-Wð 
ð	-Wð -Wð -Wñ Ôð-Wð^(àŒ|ð(ð ˜C”yð(ð 
ð	(ð (ð (ð (ðZ 37Ø26ð	jð jàŒ|ðjð " %¤,Ô/ðjð " %¤,Ô/ð	jð
 
ˆuŒ|˜X e¨C°¨H¤oÔ6Ð6Ô	7ðjð jð jð jð jð jð jð jrƒ   rý   r•  rô   Ú	symmetricÚq_lenrÄ   c                 ó<  — | €dS |                       ¦   «         } | j        \  }}|}|dk    r@|                      ||ft          j         ¬¦  «        }t          j        || gd¬¦  «        } ||z  }|rA|                      d¦  «        |                      d¦  «        z  }	|	                     d¦  «        }	n*|p|}| dd…dddd…f                              |d||¦  «        }	t          j        |	|¬¦  «        }
|
                     |	 t          j	        |¦  «        j
        ¦  «         |
S )a3  Creates an attention mask from patch validity information.

    Supports two modes controlled by `symmetric`:
    1. `symmetric=True` (default): Creates a symmetric mask of shape
       [B, 1, seq_len, seq_len]. An attention pair (i, j) is allowed only if
       both token i and token j are valid. Suitable for standard self-attention.
    2. `symmetric=False`: Creates a potentially non-square mask of shape
       [B, 1, q_len, kv_len]. An attention pair (q, k) is allowed only if
       the key/value token k is valid. Query token validity is not checked
       in the mask itself. Useful for cross-attention or specific self-attention
       implementations `q_len` can be specified.

    Used for NaFlex mode to handle variable token counts and padding tokens.

    Args:
        patch_valid: Tensor of shape [B, N] with True for valid patches, False for padding.
        num_prefix_tokens: Number of prefix tokens (class token, register tokens)
            to prepend, which are always considered valid.
        symmetric: If True, create a symmetric mask.
            If False, create an expanded mask based only on key/value validity.
        q_len: Query sequence length override. Only used when `symmetric` is False.
            Defaults to the key/value sequence length (`kv_len`) if None.
        dtype: Dtype of the output attention mask (e.g., torch.float32).

    Returns:
        Attention mask tensor. Additive mask (-inf for masked, 0 for unmasked).
        Shape is [B, 1, seq_len, seq_len] if symmetric=True,
        or [B, 1, q_len, kv_len] if symmetric=False.
    Nr   rÃ   r!   r±   rJ  )r€   r�   Únew_onesrÈ   r™  rf  r]  Ú
zeros_likeÚmasked_fill_ÚfinforZ  )r•  rô   r¥  r¦  rÄ   r¢   ÚNÚkv_lenÚprefix_validÚ	mask_boolÚ
mask_floats              r„   Úcreate_attention_maskr±  ð  sC  € ðJ ÐØˆtà×"Ò"Ñ$Ô$€KØÔ�D€A€qØ€Fð ˜1ÒÐà"×+Ò+¨QÐ0AÐ,BÍ%Ì*Ð+ÑUÔUˆå”i ¨{Ð ;ÀÐCÑCÔCˆØÐ#Ñ#ˆàð Nà×)Ò)¨"Ñ-Ô-°×0EÒ0EÀaÑ0HÔ0HÑHˆ	Ø×'Ò'¨Ñ*Ô*ˆ	ˆ	ð �˜ˆØ    4¨¨q¨q¨qÐ 0Ô1×8Ò8¸¸A¸uÀfÑMÔMˆ	õ Ô! )°5Ð9Ñ9Ô9€Jà×Ò˜Y˜J­¬°EÑ(:Ô(:Ô(>Ñ?Ô?Ð?àÐrƒ   ÚtokenÚ	pool_typeÚreduce_include_prefixc                 óè  — |�|dvrt          | |||¬¦  «        } | S |dk    rJ|r:|                     | j        d         |¦  «        }t          j        ||gd¬¦  «        }n| dd…|d…f         } |                     | j        ¦  «        }|dk    r^| |                     d¦  «        z                       d¬¦  «        }|                     dd	¬
¦  «         	                    d¬¦  «        }||z  }	|	S |dk    r°| |                     d¦  «        z                       d¬¦  «        }|                     dd	¬
¦  «         	                    d¬¦  «        }||z  }
|  
                    ¦   «         }t          j        |j        ¦  «        j        || <   |                     d¬¦  «        }d|
|z   z  S |dk    rL|  
                    ¦   «         }t          j        |j        ¦  «        j        || <   |                     d¬¦  «        S J ‚)aP  Global pooling with NaFlex support for masked tokens.

    Applies global pooling while respecting patch validity masks to exclude
    padding tokens from pooling operations.

    Args:
        x: Input tensor with shape [B, N, C]
        patch_valid: Optional validity mask for patches [B, N-num_prefix_tokens]
        pool_type: Type of pooling ('token', 'avg', 'avgmax', 'max')
        num_prefix_tokens: Number of prefix tokens (class/register)
        reduce_include_prefix: Whether to include prefix tokens in pooling reduction

    Returns:
        Pooled tensor with shape [B, C]
    N)ÚavgÚavgmaxrA  ©r³  rô   r´  r   r!   r±   r¶  rJ  TrÀ   )rZ  r·  g      à?rA  )r,   r¨  r�   rÈ   r™  rÇ   rÄ   rf  ÚsumÚclampÚcloner«  rZ  r¸   )r•   r•  r³  rô   r´  r®  Úpatch_valid_floatÚmasked_sumsÚvalid_countsÚpooledÚ
masked_avgÚmasked_xÚ
masked_maxs                r„   Úglobal_pool_naflexrÃ  5  s*  € ð. Ð˜iÐ/GÐGÐGåØØØ/Ø"7ð	
ñ 
ô 
ˆð ˆð ˜1ÒÐØ ð 	)ð '×/Ò/°´¸´
Ð<MÑNÔNˆLÝœ) \°;Ð$?ÀQÐGÑGÔGˆKˆKð �!�!�!Ð&Ð'Ð'Ð'Ô(ˆAà#Ÿš q¤wÑ/Ô/ÐØ�EÒÐàÐ,×6Ò6°rÑ:Ô:Ñ:×?Ò?ÀAÐ?ÑFÔFˆØ(×,Ò,°¸DÐ,ÑAÔA×GÒGÈAÐGÑNÔNˆØ˜|Ñ+ˆØˆØ	�hÒ	Ð	àÐ,×6Ò6°rÑ:Ô:Ñ:×?Ò?ÀAÐ?ÑFÔFˆØ(×,Ò,°¸DÐ,ÑAÔA×GÒGÈAÐGÑNÔNˆØ  <Ñ/ˆ
ð —7’7‘9”9ˆÝ!&¤¨X¬^Ñ!<Ô!<Ô!@ˆ�+�ÑØ—]’] q�]Ñ)Ô)ˆ
ð �j :Ñ-Ñ.Ð.Ø	�eÒ	Ð	à—7’7‘9”9ˆÝ!&¤¨X¬^Ñ!<Ô!<Ô!@ˆ�+�ÑØ�}Š} ˆ}Ñ#Ô#Ð#àˆurƒ   c                   óP  ‡ — e Zd ZdZ	 	 	 	 	 	 d6dee         dededeeeeeef         f                  d	df
ˆ fd
„Z	d7d„Z
d8deded	dfd„Zej                             ¦   «         d9deded	dfd„¦   «         Zej        j        d	efd„¦   «         Zej        j        d	eeef         fd„¦   «         Zej        j        deeeeeef         f                  d	dfd„¦   «         Zej        j        d:ded	efd„¦   «         Zej        j        d;ded	dfd„¦   «         Zej        j        d	ej        fd„¦   «         Zedej        dej        d	eej        eej                 e f         fd „¦   «         Z!d<ded!ee         d	dfd"„Z"d	eeej        f         fd#„Z#	 	 	 	 	 	 	 	 	 	 d=deej        eeej        f         f         d%eeeee         f                  d&ed'ed(ed)ed*ed+edeej                 d,eej                 d-eej                 d	eeej                 eej        eej                 f         eee f         f         fd.„Z$	 	 	 d>d/ej        deej                 d,eej                 d-eej                 d	eej        eeej        f         f         f
d0„Z%	 	 d?dej        d1ee         d,eej                 d	ej        fd2„Z&	 	 d@d/ej        d3ed,eej                 d	ej        fd4„Z'	 	 	 d>deej        eeej        f         f         deej                 d,eej                 d-eej                 d	ej        f
d5„Z(ˆ xZ)S )Ar.   az  NaFlexVit: Vision Transformer with NaFlex support for flexible input handling.

    A flexible implementation of Vision Transformer that supports:
    - Standard image classification with various pooling strategies
    - NaFlex functionality for variable aspect ratios and resolutions
    - Linear patch embedding for pre-patchified inputs
    - Multiple position embedding strategies (learned, factorized, rope)
    - Comprehensive attention masking for efficient batch processing
    - Encapsulated embedding and position encoding in FlexEmbeds module
    - Compatible with standard ViT checkpoints through checkpoint filtering
    Nr›   éè  r…   r   Únum_classesrC  r†   c                 ó0  •‡‡‡‡‡‡‡— t          ¦   «                              ¦   «          ||dœŠ‰pt          ¦   «         Š|rt          ‰fi |¤ŽŠ‰j        dv sJ ‚‰j        s‰j        dk    sJ ‚‰j        dv sJ ‚‰j        �At          ‰j        t          ¦  «        s't          ‰j        ¦  «        ‰j        k    s
J d¦   «         ‚t          ‰j        ¦  «        pt          Št          ‰j        ¦  «        }t!          ‰j        ¦  «        pt$          j        Š‰j        pt*          Š|| _        || _        ‰j        | _        ‰j        x| _        x| _        | _        ‰j        rdnd| _        | xj        ‰j        z  c_        ‰j        | _        ‰j        | _        ‰j        | _        ‰j         | _         d	| _!        tE          d2i d
‰j#        “d|“d‰j        “d‰j$        “d‰j%         “d‰j        “d‰j        “d|“d‰j&        “d‰j        “d‰j'        “d‰j(        “d‰j)        “d‰j*        “d|“d‰j+        “dtY          ‰dd	¦  «        “dtY          ‰dd¦  «        “‰¤Ž| _-        | j-        j.        | _.        ‰j%        r ‰‰j        fi ‰¤Žnt%          j/        ¦   «         | _0        d| _1        d	| _2        ‰j3        �re‰j3        dk    �rYddl4m5}	m6}
m7}m8} ‰j3        d k    r6 |‰j        f‰j        ‰j9        ‰j:        d‰j;        d!œ‰¤Ž| _1        d| _2        �n‰j3        d"k    r> |	‰j        ‰j9        z  f‰j:        d	d‰j<        ‰j=        ‰j;        d#œ‰¤Ž| _1        d	| _2        nÃ‰j3        d$k    rO |
‰j        ‰j9        z  f‰j:        d‰j=        ‰j;        ‰j>        ‰j?        ‰j@        ‰jA        d%œ‰¤Ž| _1        d	| _2        ni‰j3        d&k    rG‰jB        €
J d'¦   «         ‚ |‰j        ‰j9        z  f‰jB        ‰j:        ‰j;        d(œ‰¤Ž| _1        d	| _2        nt‡          d)‰j3        › �¦  «        ‚‰jD        dk    r!t‹          ‰jD        | j        ¬*¦  «        | _F        nd| _F        t�          ‰jH        ‰j        ¦  «        Št%          jI        ˆˆˆˆˆˆfd+„t•          ‰j        ¦  «        D ¦   «         Ž | _K        | j-         L                    d¬,¦  «        Šˆˆfd-„t•          ‰j        ¦  «        D ¦   «         | _M        ‰jN        r‰jO        s ‰‰j        fi ‰¤Žnt%          j/        ¦   «         | _P        ‰j        d.k    r5t£          | j        f‰jR        p‰j9        ‰jS        p‰jT        ‰‰d/œ‰¤Ž| _U        nd| _U        ‰jO        }|€‰j        d0k    }‰jN        r|r ‰‰j        fi ‰¤Žnt%          j/        ¦   «         | _O        t%          jV        ‰jW        ¦  «        | _X        |dk    rt%          jY        | j        |fi ‰¤Žnt%          j/        ¦   «         | _Z        ‰j[        | _\        ‰j]        | _]        |  ^                    ‰j[        d	¬1¦  «         dS )3a�  Initialize NaFlexVit model.

        Args:
            cfg: Model configuration. If None, uses default NaFlexVitCfg.
            in_chans: Number of input image channels.
            num_classes: Number of classification classes.
            img_size: Input image size (for backwards compatibility with classic vit).
            **kwargs: Additional config parameters to override cfg values.
        r  ©rP   r¶  r·  rA  r²  rb   r²  )rP   rð   rI   r  Nz3num_kv_heads sequence must have one entry per blockr!   r   Fr1   r   r3   r  r<   rG   rH   r  r]   rJ   rK   rM   rN   rO   r  rC   rx   r—   r^   Trð   )ÚRotaryEmbeddingCatÚRotaryEmbeddingDinoV3ÚRotaryEmbeddingMixedÚRotaryEmbeddingMRopeÚmixed)r5   r6   ÚtemperatureÚ
feat_shapeÚgrid_indexingÚaxial)rÎ  Ú	in_pixelsrÏ  Úref_feat_shapeÚgrid_offsetrÐ  Údinov3)rÎ  rÏ  rÔ  rÐ  rõ   Úshift_coordsÚjitter_coordsÚrescale_coordsrñ   z1rope_type='mrope' requires cfg.rope_mrope_section)Úmrope_sectionrÎ  rÐ  zUnknown rope_type: ©rô   c                 óÀ   •— g | ]Z} t          ‰|¦  «        d‰j        ‰j        ‰j        ‰j        ‰j        ‰j        ‰j        ‰j        ‰j	        ‰|         ‰‰‰|d œ‰¤Ž‘Œ[S ))r²   r6   r7   r:   r;   r<   rA   Ú	proj_dropÚ	attn_dropÚ	drop_pathrm   rn   rp   r5   r‚   )
rû   r3   r6   r7   r:   r;   r<   rA   rE   r>   )r‰   Úirn   r…   r*  Údprrp   rm   s     €€€€€€r„   r·   z&NaFlexVit.__init__.<locals>.<listcomp>  s—   ø€ ð &
ð &
ð &
ð$ ð# !�L˜˜aÑ Ô ð Ø”MØœ-Øœ-ØœØœØœ-ØœOØÔ,ØÔ,Ø˜aœ&Ø%Ø#Ø#Øðð ð ðð ð&
ð &
ð &
rƒ   )r?  c                 óD   •— g | ]}t          d |› �‰j        ‰¬¦  «        ‘ŒS )zblocks.)Úmoduler:  r;  )r<  r3   )r‰   rß  r…   Úpatch_reductions     €€r„   r·   z&NaFlexVit.__init__.<locals>.<listcomp>6  s@   ø€ ð 
ð 
ð 
àõ ˜ !˜˜¨s¬}ÈÐXÑXÔXð
ð 
ð 
rƒ   rb   )r6   r7   rm   rn   r¶  ©Úneeds_resetr‚   )_r  rÞ   r-   r”   rc   rG   rJ   r9   rø   r}   r^  r5   r   rm   r   rl   r   rn   r  ÚGELUrp   r   rÆ  r   r3   Únum_featuresÚhead_hidden_sizerô   rH   r  r  rd   r@   Úgrad_checkpointingrý   r1   rj   r_   r]   rK   rM   rN   rO   rC   ÚgetattrÚembedsr   r!  Únorm_prerÕ   Úrope_is_mixedrQ   Útimm.layers.pos_embed_sincosrÉ  rÊ  rË  rÌ  r6   rS   rW   rT   rU   rX   rY   rZ   r[   r\   r#  rD   r   Ú
patch_dropr    rF   Ú
SequentialÚrangeÚblocksrB  r>  r`   ra   r"  r   re   rf   r7   Ú	attn_poolr'  rB   Ú	head_dropr  Úheadrg   Úweight_init_moderh   Úinit_weights)rÚ   r…   r   rÆ  rC  rÒ   rÄ   r’   rl   rÉ  rÊ  rË  rÌ  ra   rn   r*  rà  rp   rm   rã  r,  s    `            @@@@@@€r„   rÞ   zNaFlexVit.__init__‹  sÜ  øøøøøøøø€ õ& 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆð Ð#•\‘^”^ˆØð 	1Ý! #Ð0Ð0¨Ð0Ð0ˆCð ŒÐ"NÐNÐNÐNÐNØŒÐ< #¤/°WÒ"<Ð"<Ð"<Ð<ØŒ}Ð EÐEÐEÐEÐEØÔÐ'µ
¸3Ô;KÍSÑ0QÔ0QÐ'Ý�sÔ'Ñ(Ô(¨C¬IÒ5Ð5Ð5Ð7lÑ5Ô5Ð5õ $ C¤NÑ3Ô3Ð@µyˆ
Ý)¨#Ô*>Ñ?Ô?ÐÝ! #¤-Ñ0Ô0Ð;µB´Gˆ	Ø”MÐ(¥Sˆ	ð 'ˆÔØ ˆŒØœ?ˆÔØEHÄ]ÐRˆÔÐR˜DÔ1°D´NØ&)¤oÐ!<  ¸1ˆÔØÐÔ #¤.Ñ0ÐÔØ!œnˆÔØ"œˆÔØ#&Ô#:ˆÔ Ø&)Ô&@ˆÔ#Ø"'ˆÔõ #ð 
ð 
ð 
Ø”~�~ð
à�Xð
ð ”m�mð
ð Ô)Ð)ð	
ð
 œ,Ð&Ð&ð
ð œ˜ð
ð ”~�~ð
ð &˜Xð
ð  Ô/Ð/ð
ð ”m�mð
ð !$Ô 7Ð 7ð
ð #&Ô";Ð";ð
ð %(Ô$?Ð$?ð
ð '*Ô&CÐ&Cð
ð -Ð,ð
ð  Ô+Ð+ð!
õ" '.¨cÐ3NÐPUÑ&VÔ&VÐ&Vð#
õ$ " #Ð'?ÀÑFÔFÐFØð'
ð 
ˆŒð* -1¬KÔ,TˆÔ)Ø;>¼<ÐZ˜
˜
 3¤=Ð7Ð7°BÐ7Ð7Ð7ÍRÌ[É]Ì]ˆŒð *.ˆŒ	Ø"ˆÔØŒ=ñ 4	H˜Sœ]¨fÒ4Ñ4ðð ð ð ð ð ð ð ð ð ð ð ð Œ} Ò'Ð'Ø0Ð0Ø”Mðàœ)Ø!œmØ #Ô 4Ø#Ø"%Ô"8ðð ð ðð �”	ð &*�Ô"Ñ"Ø” 'Ò)Ð)Ø.Ð.Ø”M S¤]Ñ2ð	à #Ô 4Ø#Ø#Ø#&Ô#:Ø #Ô 4Ø"%Ô"8ð	ð 	ð ð	ð 	�”	ð &+�Ô"Ð"Ø” (Ò*Ð*Ø1Ð1Ø”M S¤]Ñ2ðà #Ô 4Ø#Ø #Ô 4Ø"%Ô"8Ø #Ô 4Ø!$Ô!6Ø"%Ô"8Ø#&Ô#:ðð ð ðð �”	ð &+�Ô"Ð"Ø” 'Ò)Ð)ØÔ-Ð9Ð9Ð;nÑ9Ô9Ð9Ø0Ð0Ø”M S¤]Ñ2ðà"%Ô"8Ø #Ô 4Ø"%Ô"8ð	ð ð
 ðð �”	ð &+�Ô"Ð"å Ð!F°s´}Ð!FÐ!FÑGÔGÐGð Ô Ò"Ð"Ý5ØÔ#Ø"&Ô"8ðñ ô ˆDŒOˆOð
 #ˆDŒOõ (¨Ô(:¸C¼IÑFÔFˆå”mð &
ð &
ð &
ð &
ð &
ð &
ð &
ð &
ð &
õ$ ˜3œ9Ñ%Ô%ð%&
ñ &
ô &
ð ˆŒð, œ+×0Ò0¸4Ð0Ñ@Ô@ˆð
ð 
ð 
ð 
ð 
å˜3œ9Ñ%Ô%ð
ñ 
ô 
ˆÔð
 8;´~ÐlÈcÌkÐl�J�J˜sœ}Ð3Ð3°Ð3Ð3Ð3Õ_aÔ_jÑ_lÔ_lˆŒ	ð Œ?˜eÒ#Ð#Ý0Ø”ðàÔ1ÐB°S´]ØÔ1ÐB°S´]Ø%Ø#ðð ð ðð ˆDŒNˆNð "ˆDŒNð ”+ˆØˆ?Ø”o¨Ò.ˆGØ:=¼.ÐgÈWÐg�z�z #¤-Ð6Ð6°2Ð6Ð6Ð6ÕZ\ÔZeÑZgÔZgˆŒÝœ C¤MÑ2Ô2ˆŒØDOÐRSÂOÀO•B”I˜dœn¨kÐ@Ð@¸RÐ@Ð@Ð@ÕY[ÔYdÑYfÔYfˆŒ	à #¤ˆÔØœˆŒà×Ò˜#œ/°uÐÑ=Ô=Ð=Ð=Ð=rƒ   c                 óÈ  — dt           j        dt          ddfd„}t          | j        ¦  «        D ]³\  }}t          |d¦  «        r ||j        j        j        |dz   ¦  «         t          |d¦  «        r ||j	        j
        j        |dz   ¦  «         t          |d	¦  «        r ||j        j        |dz   ¦  «         t          |d
¦  «        r ||j        j        |dz   ¦  «         Œ´dS )z8Apply initialization weight fix with layer-wise scaling.ÚparamÚ	_layer_idr†   Nc                 ó²   — t          j        ¦   «         5  |                      t          j        d|z  ¦  «        ¦  «         d d d ¦  «         d S # 1 swxY w Y   d S )NrÂ   )rÈ   Úno_gradÚdiv_rE  Úsqrt)rù  rú  s     r„   Úrescalez*NaFlexVit.fix_init_weight.<locals>.rescaleY  s’   € Ý”‘”ð 7ð 7Ø—
’
�4œ9 S¨9¡_Ñ5Ô5Ñ6Ô6Ð6ð7ð 7ð 7ñ 7ô 7ð 7ð 7ð 7ð 7ð 7ð 7ð 7øøøð 7ð 7ð 7ð 7ð 7ð 7s   ”+AÁAÁAÚattnr!   ÚmlpÚattn_out_projÚmlp_out_proj)rÈ   rx  r}   rW  rò  Úhasattrr   r  r6  r  Úfc2r  r  )rÚ   rÿ  Úlayer_idÚlayers       r„   Úfix_init_weightzNaFlexVit.fix_init_weightW  s  € ð	7�5œ<ð 	7µCð 	7¸Dð 	7ð 	7ð 	7ð 	7õ  )¨¬Ñ5Ô5ð 	Að 	A‰OˆH�eÝ�u˜fÑ%Ô%ð >Ø�˜œ
œÔ.°¸1±Ñ=Ô=Ð=Ý�u˜eÑ$Ô$ð <Ø�˜œ	œÔ,¨h¸©lÑ;Ô;Ð;Ý�u˜oÑ.Ô.ð BØ�˜Ô+Ô2°H¸q±LÑAÔAÐAÝ�u˜nÑ-Ô-ð AØ�˜Ô*Ô1°8¸a±<Ñ@Ô@Ð@øð	Að 	Arƒ   rP   TrM  rå  c                 óÞ   — |p| j         }|dv sJ ‚d|v rt          j        | j        ¦  «         nd}t	          t          |||¬¦  «        | ¦  «         | j        r|                      ¦   «          dS dS )af  Initialize model weights according to specified scheme.

        Args:
            mode: Initialization mode ('jax', 'jax_nlhb', 'moco', or '')
            needs_reset: If True, call reset_parameters() on modules (default for after to_empty()).
                If False, skip reset_parameters() (for __init__ where modules already self-initialized).
        )ÚjaxÚjax_nlhbÚmocorP   Únlhbr=   rä  N)rö  rE  ÚlogrÆ  r'   Úget_init_weights_vitrh   r  )rÚ   rM  rå  Ú	head_biass       r„   r÷  zNaFlexVit.init_weightsh  s’   € ð Ð,�tÔ,ˆØÐ6Ð6Ð6Ð6Ð6Ø39¸T°>°>•T”X˜dÔ.Ñ/Ô/Ð/Ð/Àrˆ	åÕ(¨¨yÀkÐRÑRÔRÐTXÑYÔYÐYàŒ=ð 	#Ø× Ò Ñ"Ô"Ð"Ð"Ð"ð	#ð 	#rƒ   Úcheckpoint_pathÚprefixc                 ó,   — ddl m}  || ||¦  «         dS )a¹  Load model-specific weights from an original JAX/Flax NumPy checkpoint.

        This helper handles foreign ``.npz`` checkpoint layouts, including the
        Big Vision SigLIP2 NaFlex checkpoints. Native PyTorch state dictionaries
        are loaded by the generic factory/checkpoint helpers.

        Args:
            checkpoint_path: Path to a NumPy checkpoint.
            prefix: Prefix for parameter names in the checkpoint.
        r!   )Ú_load_weightsN)Úvision_transformerr  )rÚ   r  r  r  s       r„   Úload_pretrainedzNaFlexVit.load_pretrainedy  s0   € ð 	6Ð5Ð5Ð5Ð5Ð5àˆ�d˜O¨VÑ4Ô4Ð4Ð4Ð4rƒ   c                 óž   — h d£}| j         rAt          | j         d¦  «        r,|                     | j                              ¦   «         ¦  «         |S )zŸGet set of parameter names that should not have weight decay applied.

        Returns:
            Set of parameter names to skip during weight decay
        >   úembeds.cls_tokenúembeds.pos_embedúembeds.reg_tokenÚno_weight_decay)rÕ   r  Úupdater  )rÚ   Ú	skip_lists     r„   r  zNaFlexVit.no_weight_decay‰  sV   € ð QÐPÐPˆ	ØŒ9ð 	:� ¤Ð,=Ñ>Ô>ð 	:Ø×Ò˜TœY×6Ò6Ñ8Ô8Ñ9Ô9Ð9ØÐrƒ   c                 ó   — | j         j        S )zHReturn the 2-tuple patch size. For NaFlex dataloader / transform wiring.)rë  r1   rá   s    r„   Úget_patch_sizezNaFlexVit.get_patch_size•  s   € ð Œ{Ô%Ð%rƒ   r1  c                 ó:   — | j                              |¦  «         dS )a:  Precompute patch interpolation matrices before captured execution.

        Useful before ``torch.compile`` so the first forward at a non-base patch size
        does not build a cache entry inside the captured graph. The cache is cleared by
        any subsequent ``.to()`` / dtype conversion, so prewarm after moving the model
        to its execution device.

        Args:
            patch_sizes: Iterable of target patch sizes to precompute.

        Raises:
            RuntimeError: If this model's embedding config does not support patch interpolation.
        N)rë  r8  r7  s     r„   r8  z$NaFlexVit.prewarm_patch_interpolatorš  s    € ð$ 	Œ×.Ò.¨{Ñ;Ô;Ð;Ð;Ð;rƒ   FÚcoarsec                 ó(   — t          dddg¬¦  «        S )zßGet parameter group matcher for optimizer parameter grouping.

        Args:
            coarse: Whether to use coarse-grained grouping

        Returns:
            Dictionary mapping group names to regex patterns
        z^embeds)z^blocks\.(\d+)N)z^norm)iŸ† )Ústemrò  )r<  )rÚ   r!  s     r„   Úgroup_matcherzNaFlexVit.group_matcher®  s'   € õ ØØ-Ð/CÐDð
ñ 
ô 
ð 	
rƒ   Úenablec                 ó¸   — || _         t          | j        d¦  «        r;t          | j        j        d¦  «        r#| j        j                             |¦  «         dS dS dS )z”Enable or disable gradient checkpointing for memory efficiency.

        Args:
            enable: Whether to enable gradient checkpointing
        Úpatch_embedÚset_grad_checkpointingN)ré  r  rë  r'  r(  )rÚ   r%  s     r„   r(  z NaFlexVit.set_grad_checkpointing½  sp   € ð #)ˆÔÝ�4”; Ñ.Ô.ð 	Cµ7¸4¼;Ô;RÐTlÑ3mÔ3mð 	CØŒKÔ#×:Ò:¸6ÑBÔBÐBÐBÐBð	Cð 	Cð 	Cð 	Crƒ   c                 ó   — | j         S )zeGet the classification head module.

        Returns:
            Classification head module
        )rõ  rá   s    r„   Úget_classifierzNaFlexVit.get_classifierÈ  s   € ð ŒyÐrƒ   r•   r½   c           	      óf  — t          |¦  «        }i }g }t          |¦  «        D ]>\  }}||vrg ||<   |                     |¦  «         ||                              |¦  «         Œ?|j        \  }}	}
|	| j        z
  }| j        r$t          | j        |||||j        |j	        ¦  «        S t          j        ||| j        j        dz  |j        |j	        ¬¦  «        }t          | j        d¦  «        re| j                             |¦  «        }t          |||                     ¦   «         ¦  «        D ]&\  }}}|\  }}||z  }|D ]}|d|…         ||d|…f<   ŒŒ'nP|                     ¦   «         D ];\  }}| j                             |¬¦  «        }|\  }}||z  }|d|…         ||d|…f<   Œ<|                     d¦  «        }|S )aí  Generate ROPE position embeddings for NaFlex batch with variable grid sizes.

        Args:
            x: Input tensor [B, N, C]
            patch_coord: Patch coordinates [B, N, 2] with (y, x) values

        Returns:
            ROPE embeddings:
            - Axial mode: Tensor of shape [B, 1, N, dim*2]
            - Mixed mode: List of tensors, each of shape [B, num_heads, N, dim], one per depth layer
            - Mixed mode with iterator: Iterator yielding tensors per depth
        r™   rä   Úget_batch_embedsNrÔ   r!   )r¼   rW  rY  r�   rô   rí  rÍ   rÕ   rÄ   rÒ   rÈ   ræ   r²   r  r,  r¹   Úvaluesr‘   rÙ   rf  )rÚ   r•   r½   r_  rÎ   rÏ   rë   rÜ   r¢   r¬  r£   rÑ   Úrope_embedsÚunique_embedsrê   ré   rµ   r¶   rè   rÝ   s                       r„   Ú_generate_rope_naflexzNaFlexVit._generate_rope_naflexÑ  s)  € õ& 8¸ÑDÔDÐð ˆØˆÝ&Ð'8Ñ9Ô9ð 	2ð 	2‰MˆB�	Ø Ð/Ð/Ø-/� 	Ñ*Ø×#Ò# IÑ.Ô.Ð.Ø˜IÔ&×-Ò-¨bÑ1Ô1Ð1Ð1à”'‰ˆˆ1ˆaØ�dÔ,Ñ,ˆàÔð 
	å%Ø”	ØØØØØ”Ø”ñô ð õ ”k ! W¨d¬i¬m¸aÑ.?ÀqÄwÐWXÔW_Ð`Ñ`Ô`ˆå�4”9Ð0Ñ1Ô1ð 	Gà œI×6Ò6°|ÑDÔDˆMÝ36°|À]ÐTc×TjÒTjÑTlÔTlÑ3mÔ3mð Fð FÑ/�	˜5 -Ø ‘��1Ø ™U�
Ø'ð Fð F�BØ38¸¸*¸Ô3E�K  K Z K Ñ0Ð0ðFðFð "1×!6Ò!6Ñ!8Ô!8ð Gð G‘�	˜2Ø!œY×0Ò0°yÐ0ÑAÔA�
Ø ‘��1Ø ™U�
Ø/9¸+¸:¸+Ô/F�˜B   ˜OÑ,Ð,à!×+Ò+¨AÑ.Ô.ˆàÐrƒ   rc   c                 óP  — t          | ¦  «        }|| _        |�8|dv sJ ‚|dk    r| j        €
J d¦   «         ‚|dk    r| j        �d| _        || _        |dk    rt	          j        | j        |fi |¤Žnt	          j        ¦   «         | _        | j         	                    | j
        ¦  «         dS )zÞReset the classification head with new number of classes and pooling.

        Args:
            num_classes: Number of classes for new classification head
            global_pool: Optional new global pooling type
        NrÈ  rb   Fz=Cannot currently add attention pooling in reset_classifier().r   )r   rÆ  ró  rc   r  r  r3   r!  rõ  ÚtrainÚtraining)rÚ   rÆ  rc   r*  s       r„   Úreset_classifierzNaFlexVit.reset_classifier  sÃ   € õ ˜dÑ#Ô#ˆØ&ˆÔØÐ"ØÐ"NÐNÐNÐNÐNØ˜eÒ#Ð#¨¬Ð(>Ø]Ð]Ñ]Ô]�uØ Ò%Ð%¨$¬.Ð*DØ!%�”Ø*ˆDÔØDOÐRSÂOÀO•B”I˜dœn¨kÐ@Ð@¸RÐ@Ð@Ð@ÕY[ÔYdÑYfÔYfˆŒ	ØŒ	�Š˜œÑ&Ô&Ð&Ð&Ð&rƒ   c                 óR  — |du}|                       |||¬¦  «        \  }}d}| j        �A|�|                      ||¦  «        }n(|�| j                             |¬¦  «        }n
J d¦   «         ‚d}| j        rj| j        �c|                      |¦  «        \  }}|�|                     d|¦  «        }|�1| j        s*t          ||||¬¦  «        }|s| 	                    d¦  «        }|€-t          || j        | j         | j        rdnd|j        ¬¦  «        }|                      |¦  «        }|||||d	œS )
zR Forward pass through patch / abs pos / rope pos embeds and patch dropout
        N)r½   r•  rÔ   Fz4Expected one of patch_coord or grid_size to be validr!   ©Úpos_embed_has_batch©rô   r¥  r¦  rÄ   )r­   r•  r.  Ú	attn_maskÚkeep_indices)rë  rÕ   r0  rÙ   r3  rï  Úgatherrí  r   rf  r±  rô   r@   rÄ   rì  )	rÚ   r•   r½   r•  r9  Únaflex_moderÜ   r.  r:  s	            r„   Ú_forward_embedszNaFlexVit._forward_embeds)  sŒ  € ð "¨Ð-ˆð —{’{ØØ#Ø#ð #ñ 
ô 
‰ˆˆ9ð ˆØŒ9Ð ØÐ&à"×8Ò8¸¸KÑHÔH��ØÐ&à"œi×1Ò1¸	Ð1ÑBÔB��àTÐTÑTÔT�uð 04ˆØŒ=ð 	;˜Tœ_Ð8Ø"Ÿošo¨aÑ0Ô0‰OˆAˆ|àÐ&Ø)×0Ò0°°LÑAÔA�ØÐ&¨tÔ/AÐ&õ 5°Q¸À\ÐgrÐsÑsÔs�Ø"ð ;à"-×"7Ò"7¸Ñ":Ô":�Kð ÐÝ-ØØ"&Ô"8Ø"Ô9Ð9ØÔ6Ð@�a�a¸DØ”gðñ ô ˆIð �MŠM˜!ÑÔˆàØ&Ø&Ø"Ø(ð
ð 
ð 	
rƒ   ÚNCHWÚindicesÚreturn_prefix_tokensr"  Ú
stop_earlyÚ
output_fmtÚintermediates_onlyÚoutput_dictr•  r9  c           
      óÒ  ‡ ‡‡— |dv s
J d¦   «         ‚|dk    }g }t          t          ‰ j        ¦  «        |¦  «        \  }}t          |t          ¦  «        rq|rt          d¦  «        ‚|d         }	|                     d|
¦  «        }
|                     d|¦  «        }|d         }d	xŠŠ|s‰ j        r‰ j        �t          d
¦  «        ‚n3|}|j	        dd	…         \  }}‰ j
                             ||f¦  «        \  ŠŠ‰                      ||	|
|¬¦  «        }|d         }|                     dd	¦  «        }|                     dd	¦  «        }|                     dd	¦  «        }|                     d|
¦  «        }
t          j                             ¦   «         s|s‰ j        }n‰ j        d	|dz   …         }‰ j        ot          j                             ¦   «          }‰ j        r¹|�·t%          t'          ‰ j        |¦  «        ¦  «        D ]“\  }\  }}‰ j        r0‰ j        �)|�'t)          ||||                     dd¦  «        ¬¦  «        }|rt+          ||||¬¦  «        }n ||||¬¦  «        }||v r,|                     |r‰                      |¦  «        n|¦  «         Œ”nŽt%          |¦  «        D ]~\  }}|�%|rt+          ||||¬¦  «        }n1 ||||¬¦  «        }n"|rt+          |||¬¦  «        }n |||¬¦  «        }||v r,|                     |r‰                      |¦  «        n|¦  «         Œ‰ j        rˆ fd„|D ¦   «         }ˆ fd„|D ¦   «         }nd	}|rˆˆfd„|D ¦   «         }|r5i }||d<   |�|r||d<   |
�|
|d<   |s‰                      |¦  «        }||d<   |S t          j                             ¦   «         s!|r|�t3          t'          ||¦  «        ¦  «        }|r|S ‰                      |¦  «        }||fS )a¨   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            return_prefix_tokens: Return both prefix and spatial intermediate tokens
            norm: Apply norm layer to all intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
            output_dict: Return outputs as a dictionary with 'image_features' and 'image_intermediates' keys
            patch_coord: Optional patch coordinates [B, N, 2] for NaFlex mode
            patch_valid: Optional patch type indicators (1=patch, 0=padding) for NaFlex
            attn_mask: Optional attention mask for masked attention
        Returns:
            A tuple with (final_features, intermediates), a list of intermediate features, or a dictionary containing
            'image_features' and 'image_intermediates' (and optionally 'image_intermediates_prefix').

        NaFlex (dict / pre-patchified) inputs: NLC output only (per-sample grids are variable, a single
        spatial reshape is undefined); with ``output_dict=True`` the result also carries 'patch_valid'
        aligned with the spatial intermediates so consumers can mask padding or scatter via patch_coord.
        )r>  ÚNLCz)Output format must be one of NCHW or NLC.r>  z“output_fmt="NCHW" is not supported for NaFlex (dict) inputs, use "NLC". Per-sample grids vary; reconstruct spatial maps downstream via patch_coord.r½   r•  r9  r­   NztNaFlex forward_intermediates with active patch dropout requires output_dict=True to return the gathered patch_valid.r—  ©r½   r•  r9  r.  r:  r!   r<  Fr6  ©rÕ   r9  ©r9  c                 ó6   •— g | ]}|d d …d‰j         …f         ‘ŒS rà   rÚ  ©r‰   ÚyrÚ   s     €r„   r·   z3NaFlexVit.forward_intermediates.<locals>.<listcomp>ç  s/   ø€ ÐSÐSÐSÀ˜Q˜q˜q˜q ! DÔ$:Ð":Ð:Ô;ÐSÐSÐSrƒ   c                 ó6   •— g | ]}|d d …‰j         d …f         ‘ŒS ©NrÚ  rK  s     €r„   r·   z3NaFlexVit.forward_intermediates.<locals>.<listcomp>è  s0   ø€ ÐRÐRÐR¸q˜Q˜q˜q˜q $Ô"8Ð"9Ð"9Ð9Ô:ÐRÐRÐRrƒ   c                 ó¦   •— g | ]M}|                      |j        d          ‰‰d¦  «                             d ddd¦  «                             ¦   «         ‘ŒNS )r   rJ  r›   r!   r™   )r¡   r�   r    Ú
contiguous)r‰   rL  r¤   r¥   s     €€r„   r·   z3NaFlexVit.forward_intermediates.<locals>.<listcomp>î  sa   ø€ ð ð ð àð —	’	˜!œ' !œ* a¨¨BÑ/Ô/×7Ò7¸¸1¸aÀÑCÔC×NÒNÑPÔPðð ð rƒ   Úimage_intermediatesÚimage_intermediates_prefixÚimage_features)r#   r^  rò  rø   r<  r#  Úgetr3  rï  r�   rë  rG  r=  rÈ   r¡  Úis_scriptingré  rí  rW  r¹   r   r&   rY  r"  rô   Úlist) rÚ   r•   r?  r@  r"  rA  rB  rC  rD  r½   r•  r9  r¡   ÚintermediatesÚtake_indicesÚ	max_indexr­   ÚheightÚwidthrë  r.  r:  rò  Údo_checkpointingrß  ÚblkrÝ   Úprefix_tokensÚresult_dictÚx_finalr¤   r¥   s    `                             @@r„   Úforward_intermediateszNaFlexVit.forward_intermediatesi  s  øøø€ ðJ ˜_Ð,Ð,Ð,Ð.YÑ,Ô,Ð,Ø Ò&ˆØˆÝ"6µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�iÝ�a�ÑÔð 	Bð ð cÝ ðbñcô cð cð ˜MÔ*ˆKØŸ%š% ¨{Ñ;Ô;ˆKØŸš˜k¨9Ñ5Ô5ˆIØ˜	”lˆGØˆLˆA�Øð L 4¤=ð L°T´_Ð5Põ !ðKñLô Lð Løð ˆGØœG B C CœL‰MˆF�EØ”;×0Ò0°&¸%°ÑAÔA‰DˆAˆqð ×%Ò%ØØ#Ø#Øð	 &ñ 
ô 
ˆð �9ÔˆØ—j’j °Ñ5Ô5ˆØ—z’z .°$Ñ7Ô7ˆØ—J’J˜{¨DÑ1Ô1ˆ	à—j’j °Ñ<Ô<ˆõ Œ9×!Ò!Ñ#Ô#ð 	1¨:ð 	1Ø”[ˆFˆFà”[  )¨a¡- Ô0ˆFàÔ2ÐS½5¼9×;QÒ;QÑ;SÔ;SÐ7SÐØÔð "	F +Ð"9å(1µ#°d´kÀ;Ñ2OÔ2OÑ(PÔ(Pð Fð FÑ$�Ñ$�C˜à”=ð  T¤_Ð%@À\ÐE]å!7ØØ"Ø$Ø,2¯JªJ°}ÀeÑ,LÔ,Lð	"ñ "ô "�Jð $ð EÝ" 3¨°
ÀiÐPÑPÔP�A�Aà˜˜A J¸)ÐDÑDÔD�AØ˜Ð$Ð$à!×(Ò(¸Ð)D¨¯ª°1©¬¨À1ÑEÔEÐEøð!Fõ$ $ FÑ+Ô+ð Fð F‘��3àÐ*Ø'ð JÝ& s¨A°KÈ9ÐUÑUÔU˜˜à˜C ¨¸yÐIÑIÔI˜˜à'ð 8Ý& s¨A¸ÐCÑCÔC˜˜à˜C ¨YÐ7Ñ7Ô7˜Ø˜Ð$Ð$à!×(Ò(¸Ð)D¨¯ª°1©¬¨À1ÑEÔEÐEøð Ô!ð 	!àSÐSÐSÐSÀ]ÐSÑSÔSˆMØRÐRÐRÐRÀMÐRÑRÔRˆMˆMà ˆMàð 	ðð ð ð ð à&ðñ ô ˆMð ð 	ØˆKà1>ˆKÐ-Ñ.ØÐ(Ð-AÐ(Ø<I�Ð8Ñ9ØÐ&ð .9�˜MÑ*ð &ð 8ØŸ)š) A™,œ,�Ø07�Ð,Ñ-àÐõ Œy×%Ò%Ñ'Ô'ð 	DÐ,@ð 	DÀ]ÐE^å ¥ ]°MÑ!BÔ!BÑCÔCˆMàð 	!Ø Ð à�IŠI�a‰LŒLˆà�-ÐÐrƒ   r­   c                 ó2  — |du}|                       ||||¬¦  «        }|d         }|                     dd¦  «        }|                     dd¦  «        }	|                     dd¦  «        }| j        ot          j                             ¦   «          }
| j        ru|�st          t          | j	        |¦  «        ¦  «        D ]O\  }\  }}| j
        r| j        �|	�t          |||	|¬¦  «        }|
rt          ||||¬¦  «        }ŒA ||||¬¦  «        }ŒPn]|�/| j	        D ]&}|
rt          ||||¬¦  «        }Œ ||||¬¦  «        }Œ'n,| j	        D ]$}|
rt          |||¬	¦  «        }Œ |||¬	¦  «        }Œ%|                      |¦  «        }|r||                     d
d¦  «        dœS |S )z	
        NrG  r­   r.  r:  r9  r6  rH  rI  r•  )r­   r•  )r=  rT  ré  rÈ   r¡  rU  rí  rW  r¹   rò  r3  rï  r   r&   r"  )rÚ   r­   r½   r•  r9  r<  rë  r•   r.  r:  r\  rß  r]  rÝ   s                 r„   Úforward_featureszNaFlexVit.forward_features  s4  € ð "¨Ð-ˆð ×%Ò%ØØ#Ø#Øð	 &ñ 
ô 
ˆð �9ÔˆØ—j’j °Ñ5Ô5ˆØ—z’z .°$Ñ7Ô7ˆØ—J’J˜{¨DÑ1Ô1ˆ	ð  Ô2ÐS½5¼9×;QÒ;QÑ;SÔ;SÐ7SÐØÔð 	4 +Ð"9å(1µ#°d´kÀ;Ñ2OÔ2OÑ(PÔ(Pð Eð EÑ$�Ñ$�C˜Ø”=ð  T¤_Ð%@À\ÐE]å!7ØØ"Ø$Ø,7ð	"ñ "ô "�Jð $ð EÝ" 3¨°
ÀiÐPÑPÔP�A�Aà˜˜A J¸)ÐDÑDÔD�A�AðEð Ð$à”{ð Fð F�Ø#ð FÝ" 3¨°ÀyÐQÑQÔQ�A�Aà˜˜A K¸9ÐEÑEÔE�A�Að	Fð ”{ð 4ð 4�Ø#ð 4Ý" 3¨°YÐ?Ñ?Ô?�A�Aà˜˜A¨Ð3Ñ3Ô3�A�Aà�IŠI�a‰LŒLˆàð 	àØ%Ÿzšz¨-¸Ñ>Ô>ðð ð ð
 ˆrƒ   r³  c                 ó  — | j         �Zt          || j        r| j        nddd|j        ¬¦  «        }| j        s|d d …| j        d …f         }|                       ||¬¦  «        }|S |€| j        n|}t          |||| j        | j        ¬¦  «        }|S )Nr   Fr!   r8  rI  r¸  )ró  r±  rd   rô   rÄ   rc   rÃ  )rÚ   r•   r³  r•  r9  s        r„   Ú_poolzNaFlexVit._poolR  sÇ   € ð Œ>Ð%Ý-ØØ<@Ô<TÐ"[ $Ô"8Ð"8ÐZ[ØØØ”gðñ ô ˆIð Ô+ð 2Ø�a�a�a˜Ô/Ð0Ð0Ð0Ô1�Ø—’˜q¨I�Ñ6Ô6ˆAØˆHà(1Ð(9�DÔ$Ð$¸yˆ	åØØØØ"Ô4Ø"&Ô":ð
ñ 
ô 
ˆð ˆrƒ   Ú
pre_logitsc                 ó¶   — |                       ||¬¦  «        }|                      |¦  «        }|                      |¦  «        }|r|n|                      |¦  «        S )N)r•  )re  ra   rô  rõ  )rÚ   r­   rf  r•  r•   s        r„   Úforward_headzNaFlexVit.forward_headp  sR   € ð �JŠJ�w¨KˆJÑ8Ô8ˆØ�LŠL˜‰OŒOˆØ�NŠN˜1ÑÔˆØÐ0ˆqˆq D§I¢I¨a¡L¤LÐ0rƒ   c                 óÈ  — t          |t          ¦  «        }|p|du}|rš|rK|d         }|                     d|¦  «        }|                     d|¦  «        }|                     d|¦  «        }n|}t          |dud¦  «         t          |dud¦  «         |                      ||||¬¦  «        } | j        d	i |¤Ž}n*|                      |¦  «        }|                      |¦  «        }|S )
a�  Forward pass with optional NaFlex support.

        Args:
            x: Input tensor. Supported formats:
                - [B, C, H, W] standard image input
                - [B, N, P*P*C] pre-patchified tensor (flattened patches)
                - [B, N, Ph, Pw, C] pre-patchified tensor (variable patch size)
                - Dict from NaFlex collator
            patch_coord: Optional patch coordinates [B, N, 2] for NaFlex mode.
            patch_valid: Optional patch validity indicators for NaFlex.
            attn_mask: Optional attn mask to override defaults generated from patch_valid

        Returns:
            Model output tensor.
        Nr­   r•  r½   r9  z&patch_coord is required in naflex modez&patch_valid is required in naflex mode)r­   r•  r½   r9  r‚   )rø   r<  rT  r   rc  rh  )	rÚ   r•   r½   r•  r9  Úinput_is_dictr<  r­   Úfeaturess	            r„   rœ  zNaFlexVit.forward{  s  € õ, # 1¥dÑ+Ô+ˆØ#Ð> {¸$Ð'>ˆØð 	%Øð à˜Iœ,�ØŸeše M°;Ñ?Ô?�ØŸeše M°;Ñ?Ô?�ØŸEšE +¨yÑ9Ô9�	�	à�Ý�K tÐ+Ð-UÑVÔVÐVÝ�K tÐ+Ð-UÑVÔVÐVà×,Ò,ØØ'Ø'Ø#ð	 -ñ ô ˆHð "�Ô!Ð-Ð- HÐ-Ð-ˆAˆAà×%Ò% aÑ(Ô(ˆAØ×!Ò! !Ñ$Ô$ˆAØˆrƒ   )Nr›   rÅ  NNNr�  )rP   T©rP   ©Frž  rN  )
NFFFr>  FFNNN)NNNrŸ  )FN)*ry   rz   r{   r|   r   r-   r}   r   r   rÞ   r  r�   r€   r÷  rÈ   r¡  r¢  r  r   r  r  r	   r8  r   r$  r(  r  r   r*  r   rx  r
   r   r0  r4  r=  ra  rc  re  rh  rœ  r£  r¤  s   @r„   r.   r.   ~  s>  ø€ € € € € ð
ð 
ð +/ØØ#Ø>BØØðJ>ð J>à˜,Ô'ðJ>ð ðJ>ð ð	J>ð
 ˜u S¨%°°S°¬/Ð%9Ô:Ô;ðJ>ð 
ðJ>ð J>ð J>ð J>ð J>ð J>ðXAð Að Að Að"#ð # ð #¸ð #Èð #ð #ð #ð #ð" „Y×ÒÑÔð5ð 5¨sð 5¸Cð 5Èð 5ð 5ð 5ñ Ôð5ð „YÔð	 ð 	ð 	ð 	ñ Ôð	ð „YÔð&  c¨3 h¤ð &ð &ð &ñ Ôð&ð „YÔð<à! %¨¨U°3¸°8¬_Ð(<Ô"=Ô>ð<ð 
ð<ð <ð <ñ Ôð<ð& „YÔð
ð 
 Dð 
°Tð 
ð 
ð 
ñ Ôð
ð „YÔðCð C¨Tð C¸Tð Cð Cð Cñ ÔðCð „YÔð ¤	ð ð ð ñ Ôðð ðBàŒ|ðBð œðBð 
ˆuŒ|˜T %¤,Ô/°Ð4Ô	5ð	Bð Bð Bñ ÔðBðH'ð '¨Cð '¸hÀs¼mð 'ÐW[ð 'ð 'ð 'ð 'ð&>
ð 
ˆc�5”<ÐÔ	 ð>
ð >
ð >
ð >
ðF 8<Ø).ØØ$Ø$Ø',Ø %Ø26Ø26Ø04ðg ð g à�U”\ 4¨¨U¬\Ð(9Ô#:Ð:Ô;ðg ð ˜e C¨¨c¬ NÔ3Ô4ðg ð #'ð	g ð
 ðg ð ðg ð ðg ð !%ðg ð ðg ð " %¤,Ô/ðg ð " %¤,Ô/ðg ð   ¤Ô-ðg ð 
ˆt�E”LÔ! 5¨¬°t¸E¼LÔ7IÐ)IÔ#JÈDÐQTÐVYÐQYÌNÐZÔ	[ðg ð g ð g ð g ðX 37Ø26Ø04ð>ð >à”\ð>ð " %¤,Ô/ð>ð " %¤,Ô/ð	>ð
   ¤Ô-ð>ð 
ˆuŒ|˜T # u¤|Ð"3Ô4Ð4Ô	5ð>ð >ð >ð >ðF (,Ø26ð	ð àŒ|ðð   ”}ðð " %¤,Ô/ð	ð
 
Œðð ð ð ðB  %Ø26ð		1ð 	1à”\ð	1ð ð	1ð " %¤,Ô/ð		1ð
 
Œð	1ð 	1ð 	1ð 	1ð 37Ø26Ø04ð0ð 0à�U”\ 4¨¨U¬\Ð(9Ô#:Ð:Ô;ð0ð " %¤,Ô/ð0ð " %¤,Ô/ð	0ð
   ¤Ô-ð0ð 
Œð0ð 0ð 0ð 0ð 0ð 0ð 0ð 0rƒ   c                 ó6  — | d         }| d         }| d         }t          t          |¦  «        ¦  «        D ]ã}||         ||                  }||d d …df                              ¦   «         dz                        ¦   «         }||d d …df                              ¦   «         dz                        ¦   «         }|                     ||ddd¦  «                             ddd	dd¦  «        }|                     d|dz  |dz  ¦  «        }dd
lm}  ||d|› d�d¬¦  «         Œäd S )Nr½   r•  r­   r   r!   r0   r›   rš   r™   )Ú
save_imageÚpatch_z.jpgT)Ú	normalize)rñ  r^  rA  r´   r¡   r    Útorchvision.utilsro  )	r•   r½   r•  r­   rß  Úpatchrµ   r¶   ro  s	            r„   Ú_debug_dump_patchesrt  ®  s>  € à�MÔ"€KØ�MÔ"€KØ�	Œl€GÝ•3�w‘<”<Ñ Ô ð <ð <ˆØ˜”
˜; qœ>Ô*ˆØ˜˜A˜A˜A˜q˜Ô!×%Ò%Ñ'Ô'¨!Ñ+×1Ò1Ñ3Ô3ˆØ˜˜A˜A˜A˜q˜Ô!×%Ò%Ñ'Ô'¨!Ñ+×1Ò1Ñ3Ô3ˆØ—’˜a  B¨¨AÑ.Ô.×6Ò6°q¸!¸QÀÀ1ÑEÔEˆØ—’˜a  2¡ q¨¡tÑ,Ô,ˆØ0Ð0Ð0Ð0Ð0Ð0Øˆ
�5Ð* 1Ð*Ð*Ð*°dÐ;Ñ;Ô;Ð;Ð;ð<ð <rƒ   r
  r=   rM  r  rå  c                 ó’   — ddl m}m}m}m} d| v rt          |||¬¦  «        S d| v rt          ||¬¦  «        S t          ||¬¦  «        S )zFFunction imported from vision_transformer.py to maintain compatibilityr!   )Úinit_weights_vit_jaxÚinit_weights_vit_mocoÚinit_weights_vit_timmÚinit_weights_reset_parametersr
  )r  rå  r  rä  )r  rv  rw  rx  ry  r   )rM  r  rå  rv  rw  rx  ry  s          r„   r  r  ½  s�   € ðð ð ð ð ð ð ð ð ð ð ð ð �€}€}ÝÐ+°yÈkÐZÑZÔZÐZØ	�4ˆˆÝÐ,¸+ÐFÑFÔFÐFåÐ,¸+ÐFÑFÔFÐFrƒ   Ú
state_dictÚmodelc                 ó8  — i }|                       ¦   «         D �]�\  }}|dk    �rùt          |j        d¦  «        �rÜ|j        dk    �rÐd}d}d| v r| d         j        d         }d| v r| d         j        d         }||z   }|j        d         }||z
  }	t          j        |	¦  «        }
t          j        |¦  «        }|
|k    rž|
                     ¦   «         rŠ|                     ¦   «         sv|	}|dd…d|…f         }|                     ¦   «         r| dxx         |z  cc<   |dd…||…f         }|                     ¦   «         r| dxx         |z  cc<   |dd…|d…f         }|
}t          |¦  «        }||z  |k    r$| 
                    d|||j        d         ¦  «        }nt          |j        j        d	¦  «        re|j        j        j        \  }}||z  |k    r$| 
                    d|||j        d         ¦  «        }n$t                               d
|› d||z  › d�¦  «         ||d<   �Œ|dk    r||d<   �Œ|dk    r||d<   �Œ |                     d¦  «        rG|dd…         }|dk    r+|                     dddd¦  «                             d¦  «        }d|z   }|||<   �Œ||||<   �Œƒ|S )zZHandle state dict conversion from original ViT to the new version with combined embedding.rJ   r›   r   r  r!   r  Nr™   rÜ   z-Position embedding size mismatch: checkpoint=z, model=z?. Using default initialization and will resize in forward pass.r  r  r  zpatch_embed.r4   zproj.weightzembeds.)r‘   r  rë  r˜  r�   rE  rþ  Ú
is_integerÚnumelr}   r¡   r'  rÜ   Ú_loggerÚwarningÚ
startswithr    r  )rz  r{  Úout_dictrŠ   r‹   Únum_cls_tokenÚnum_reg_tokenrô   Únum_patchesÚnum_patches_no_prefixÚgrid_size_no_prefixrÜ   Úcls_token_embÚreg_token_embrµ   r¶   ÚsuffixÚnew_keys                     r„   Úcheckpoint_filter_fnrŒ  Î  s[  € ð €HØ× Ò Ñ"Ô"ð Añ A‰ˆˆ1à�ÒÑå�u”| [Ñ1Ô1ñ -°a´fÀ²k±kØ !�Ø !�Ø *Ð,Ð,Ø$.¨{Ô$;Ô$AÀ!Ô$D�MØ *Ð,Ð,Ø$.¨{Ô$;Ô$AÀ!Ô$D�MØ$1°MÑ$AÐ!ð  œg aœj�Ø(3Ð6GÑ(GÐ%Ý&*¤iÐ0EÑ&FÔ&FÐ#Ý œI kÑ2Ô2�	Ø'¨9Ò4Ð4Ø0×;Ò;Ñ=Ô=ð 5ØFO×FZÒFZÑF\ÔF\ð 5ð #8�KØ$% a a a¨¨=¨Ð&8Ô$9�MØ$×*Ò*Ñ,Ô,ð AØ" ;Ð/Ð/Ô/°=Ñ@Ð/Ð/Ñ/Ø$% a a a¨°}Ð)DÐ&DÔ$E�MØ$×*Ò*Ñ,Ô,ð AØ" ;Ð/Ð/Ô/°=Ñ@Ð/Ð/Ñ/Ø˜!˜!˜!Ð.Ð/Ð/Ð/Ô0�AØ 3�IÝ 	™NœN�	ð ˜yÑ(¨KÒ7Ð7àŸ	š	 ! Y°	¸1¼7À1¼:ÑFÔF�A�Aõ ˜uœ|Ô7¸ÑEÔEð 
Ø$œ|Ô7ÔA™˜˜1Ø˜q™5 KÒ/Ð/à !§	¢	¨!¨Q°°1´7¸1´:Ñ >Ô >˜A˜Aõ $ŸOšOð!aÐP[ð !að !aÐfgÐjkÑfkð !að !að !añô ð ð ,-ˆHÐ'Ñ(Ñ(Ø�+ÒÐØ+,ˆHÐ'Ñ(Ñ(Ø�+ÒÐØ+,ˆHÐ'Ñ(Ñ(à�\Š\˜.Ñ)Ô)ð 	Ø�r�s�s”VˆFØ˜Ò&Ð&Ø—I’I˜a  A qÑ)Ô)×1Ò1°!Ñ4Ô4�Ø &Ñ(ˆGØ !ˆH�WÑÑàˆH�Q‰K‰Kà€Orƒ   rP   Úurlc                 ó6   — | ddd ddt           t          ddddœ|¥S )	NrÅ  )r›   é€  r�  rÅ   rL   zembeds.projrõ  z
apache-2.0)r�  rÆ  Ú
input_sizeÚ	pool_sizeÚcrop_pctr  Úmeanr.  Ú
first_convÚ
classifierÚlicenser   )r�  r’   s     r„   Ú_cfgr—    s<   € àØØ#ØØØ"Ý'Ý%Ø#ØØðð ð ðð rƒ   ztimm/)Ú	hf_hub_id)r˜  rÆ  )z)naflexvit_base_patch16_gap.e300_s576_in1kz-naflexvit_base_patch16_par_gap.e300_s576_in1kz0naflexvit_base_patch16_parfac_gap.e300_s576_in1kz$naflexvit_base_patch16_map.untrainedz,naflexvit_so150m2_patch16_reg1_gap.untrainedz,naflexvit_so150m2_patch16_reg1_map.untrainedz&naflexvit_base_patch16_siglip.v2_webliz(naflexvit_so400m_patch16_siglip.v2_webliÚvariantÚ
pretrainedc           	      ód  ‡‡— ‰                      dd¦  «        }‰                      dt          ¦   «         ¦  «        }d„ t          t          ¦  «        D ¦   «         Šˆˆfd„t          ‰¦  «        D ¦   «         }|rt	          |fi |¤Ž}t          t          | |ft          |t          |d¬¦  «        dœ‰¤Ž}|S )	NÚout_indicesr›   r…   c                 ó   — h | ]	}|j         ’Œ
S r‚   )Úname)r‰   Úfs     r„   ú	<setcomp>z$_create_naflexvit.<locals>.<setcomp>E  s   € Ð<Ð<Ð< !�q”vÐ<Ð<Ð<rƒ   c                 óD   •— i | ]}|‰v ¯|‰                      |¦  «        “ŒS r‚   )Úpop)r‰   rŠ   Úcfg_field_namesr’   s     €€r„   r�   z%_create_naflexvit.<locals>.<dictcomp>G  s/   ø€ ÐRÐRÐR¨¸QÀ/Ð=QÐ=Q�1�f—j’j ‘m”mÐ=QÐ=QÐ=Qrƒ   Úgetter)rœ  Úfeature_cls)Úpretrained_filter_fnr…   Úfeature_cfg)	r¢  r-   r   rV  r”   r"   r.   rŒ  r<  )r™  rš  r’   rœ  r…   Úcfg_updatesr{  r£  s     `    @r„   Ú_create_naflexvitr©  B  sÏ   øø€ Ø—*’*˜]¨AÑ.Ô.€KØ
�*Š*�U�L™NœNÑ
+Ô
+€CØ<Ð<¥v­lÑ';Ô';Ð<Ñ<Ô<€OàRÐRÐRÐRÐR­T°&©\¬\ÐRÑRÔR€KØð 2Ý˜cÐ1Ð1 [Ð1Ð1ˆå Ý�7˜Jðå1ØÝ [¸hÐGÑGÔGð	ð ð
 ðð €Eð €Lrƒ   c           	      óp  — |                      dd¦  «         |                      dd¦  «         |                      dd¦  «        }|                      dd¦  «        }|€|dk    rd}d|                     d	d¦  «        |||                      d
d¦  «        |                      dd¦  «        dœ|¥}t          | |fi |¤ŽS )aÜ  Create FlexVit model from classic VisionTransformer configuration.

    This function handles the parameter mapping and configuration logic needed
    to create FlexVit models that are compatible with classic VisionTransformer
    configurations and pretrained weights.

    Args:
        variant: Model variant name
        pretrained: Whether to load pretrained weights
        **kwargs: Classic VisionTransformer parameters

    Returns:
        FlexVit model instance
    Úno_embed_classNÚdynamic_img_sizerc   r²  ra   r¶  TrG   r8   Fr÷   )rK   rG   rc   ra   r8   r?   ©r¢  rT  r©  )r™  rš  r’   Úgpra   Úflex_kwargss         r„   Ú_create_naflexvit_from_classicr°  U  sØ   € ð( ‡J‚JÐ Ñ&Ô&Ð&Ø
‡J‚JÐ! 4Ñ(Ô(Ð(ð 
�Š�M 7Ñ	+Ô	+€BØ�jŠj˜ DÑ)Ô)€GØ€˜2 š;˜;Øˆð  $Ø—z’z -°Ñ6Ô6ØØØ Ÿ*š*Ð%5°uÑ=Ô=Ø!'§¢Ð,=¸uÑ!EÔ!Eðð ð ð€Kõ ˜W jÐ@Ð@°KÐ@Ð@Ð@rƒ   c           	      óv  — |                      dd¦  «         |                      dd¦  «        }|                      dd¦  «        }|                      dd¦  «        }|                      dd	¦  «        }|                      d
d¦  «        }|                      dd¦  «        }|r	|dk    r|}	n|r|rdnd}	nd}	|                      dd¦  «        }
|                      dd¦  «        }|                      dd¦  «        }|                      dd¦  «        }|€|
dk    }i dd“d|                     dd¦  «        “d|                      d|                     dd¦  «        ¦  «        “d|
“d|“d|“d|“d|                      dd¦  «        rd nd“d|	“d|“d|“d
|“d!|                      d!d¦  «        “d"|                     d#d¦  «        “d$|                      d$d%¦  «        “d&|                      d&d¦  «        “d'|                      d'd¦  «        “|                      d(d¦  «        |                      d)d¦  «        d*œ¥|¥}t          | |fi |¤ŽS )+a¦  Create NaFlexVit model from EVA configuration.

    This function handles the parameter mapping and configuration logic needed
    to create NaFlexVit models that are compatible with EVA configurations
    and pretrained weights.

    Args:
        variant: Model variant name
        pretrained: Whether to load pretrained weights
        **kwargs: EVA model parameters

    Returns:
        NaFlexVit model instance
    r«  NÚuse_rot_pos_embFÚrope_mixed_moderS   rR   rU   r=   rW   rV   rQ   rð   rÍ  rÑ  rc   r¶  Úuse_pre_transformer_normÚuse_post_transformer_normTÚuse_fc_normrK   rG   rH   r  r   r_   r`   ra   rJ   Úuse_abs_pos_embrI   rX   rT   rÓ  rs   rï   ru   rv   rò   ró   )r8   r?   r­  )r™  rš  r’   r²  r³  rS   rU   rW   Úexplicit_rope_typerQ   r®  r´  rµ  r¶  Únaflex_kwargss                  r„   Ú_create_naflexvit_from_evarº  €  så  € ð( ‡J‚JÐ Ñ&Ô&Ð&ð —j’jÐ!2°EÑ:Ô:€OØ—j’jÐ!2°EÑ:Ô:€OØ—z’zÐ"4°fÑ=Ô=ÐØ—z’zÐ"4°bÑ9Ô9ÐØŸšÐ$8¸$Ñ?Ô?ÐàŸš K°Ñ6Ô6ÐØð Ð0°FÒ:Ð:Ø&ˆ	ˆ	Ø	ð Ø.Ð;�G�G°Gˆ	ˆ	àˆ	ð 
�Š�M 5Ñ	)Ô	)€BØ%ŸzšzÐ*DÀeÑLÔLÐØ &§
¢
Ð+FÈÑ MÔ MÐØ—*’*˜]¨DÑ1Ô1€KØÐØ˜E’kˆðØ˜tðà�v—z’z -°Ñ6Ô6ðð 	�v—z’zÐ"2°F·J²J¸|ÈQÑ4OÔ4OÑPÔPðð 	�rð	ð
 	Ð,ðð 	Ð/ðð 	�;ðð 	 &§*¢*Ð->ÀÑ"EÔ"EÐQ�Y�YÈ6ðð 	�Yðð 	Ð,ðð 	Ð,ðð 	Ð0ðð 	˜FŸJšJÐ'9¸5ÑAÔAðð 	˜vŸzšzÐ*:¸DÑAÔAðð 	�V—Z’Z ¨UÑ3Ô3ðð  	�f—j’j ¨uÑ5Ô5ð!ð" 	�V—Z’Z ¨TÑ2Ô2ð#ð$ !Ÿ*š* [°%Ñ8Ô8Ø!'§¢Ð,>ÀÑ!FÔ!Fð'ð ð ð( ð)€Mõ. ˜W jÐBÐB°MÐBÐBÐBrƒ   c           
      óR   — t          dddddddd¬¦  «        }t          d| |d
œ|¤Ž}|S )zCViT-Base with NaFlex functionality and global average pooling.
    r0   r2   r4   çñhãˆµøä>r¶  rš   T)r1   r3   r5   r6   rA   rc   rH   ra   Únaflexvit_base_patch16_gap©rš  r…   )r½  ©r-   r©  ©rš  r’   r…   r{  s       r„   r½  r½  È  sS   € õ ØØØØØØØØð	ñ 	ô 	€Cõ ÐeÀzÐWZÐeÐeÐ^dÐeÐe€EØ€Lrƒ   c                 óT   — t          ddddddddd¬¦	  «	        }t          d| |d
œ|¤Ž}|S )z]ViT-Base with NaFlex functionality, aspect preserving pos embed, global average pooling.
    r0   r2   r4   r¼  Tr¶  rš   )	r1   r3   r5   r6   rA   rN   rc   rH   ra   Únaflexvit_base_patch16_par_gapr¾  )rÂ  r¿  rÀ  s       r„   rÂ  rÂ  Ú  sV   € õ ØØØØØØ $ØØØð
ñ 
ô 
€Cõ ÐiÈ:Ð[^ÐiÐiÐbhÐiÐi€EØ€Lrƒ   c                 óV   — t          dddddddddd¬	¦
  «
        }t          d| |dœ|¤Ž}|S )zjViT-Base with NaFlex functionality, aspect preserving & factorized pos embed, global average pooling.
    r0   r2   r4   r¼  Tr  r¶  rš   )
r1   r3   r5   r6   rA   rN   rJ   rc   rH   ra   Ú!naflexvit_base_patch16_parfac_gapr¾  )rÄ  r¿  rÀ  s       r„   rÄ  rÄ  í  sY   € õ ØØØØØØ $ØØØØðñ ô €Cõ ÐlÈjÐ^aÐlÐlÐekÐlÐl€EØ€Lrƒ   c           	      óP   — t          ddddddd¬¦  «        }t          d
| |d	œ|¤Ž}|S )zBViT-Base with NaFlex functionality and MAP attention pooling.
    r0   r2   r4   r¼  rb   r!   )r1   r3   r5   r6   rA   rc   rH   Únaflexvit_base_patch16_mapr¾  )rÆ  r¿  rÀ  s       r„   rÆ  rÆ  	  sP   € õ ØØØØØØØðñ ô €Cõ ÐeÀzÐWZÐeÐeÐ^dÐeÐe€EØ€Lrƒ   c                 óV   — t          ddddddddd	d
¬¦
  «
        }t          d| |dœ|¤Ž}|S )á/  ViT-SO150M2 with NaFlex functionality for variable aspect ratios and resolutions.

    This model supports:
    1. Variable aspect ratios and resolutions via patch coordinates
    2. Position embedding interpolation for arbitrary grid sizes
    3. Explicit patch coordinates and valid token masking
    r0   é@  é   é   çÅNìÄNì@r¼  Fr!   r¶  T)
r1   r3   r5   r6   r7   rA   r:   rH   rc   ra   Ú"naflexvit_so150m2_patch16_reg1_gapr¾  )rÍ  r¿  rÀ  s       r„   rÍ  rÍ  	  sY   € õ ØØØØØØØØØØðñ ô €Cõ ÐmÈzÐ_bÐmÐmÐflÐmÐm€EØ€Lrƒ   c                 óT   — t          ddddddddd	¬
¦	  «	        }t          d| |dœ|¤Ž}|S )rÈ  r0   rÉ  rÊ  rË  rÌ  r¼  Fr!   rb   )	r1   r3   r5   r6   r7   rA   r:   rH   rc   Ú"naflexvit_so150m2_patch16_reg1_mapr¾  )rÏ  r¿  rÀ  s       r„   rÏ  rÏ  +	  sV   € õ ØØØØØØØØØð
ñ 
ô 
€Cõ ÐmÈzÐ_bÐmÐmÐflÐmÐm€EØ€Lrƒ   c           	      óP   — t          ddddddd¬¦  «        }t          d
| |d	œ|¤Ž}|S )zGViT-Base with NaFlex functionality and SigLIP-style configuration.
    r0   r2   r4   Ú	gelu_tanhrb   r…  )r1   r3   r5   r6   rn   rc   rM   Únaflexvit_base_patch16_siglipr¾  )rÒ  r¿  rÀ  s       r„   rÒ  rÒ  C	  sP   € õ ØØØØØØØ(ðñ ô €Cõ ÐhÈ*ÐZ]ÐhÐhÐagÐhÐh€EØ€Lrƒ   c           
      óR   — t          dddddddd¬¦  «        }t          d| |d
œ|¤Ž}|S )zUViT-SO400M with NaFlex functionality for variable aspect ratios and resolutions.
    r0   i€  é   gˆ…ZÓ¼ã@rÑ  rb   r…  )r1   r3   r5   r6   r7   rn   rc   rM   Únaflexvit_so400m_patch16_siglipr¾  )rÕ  r¿  rÀ  s       r„   rÕ  rÕ  T	  sS   € õ ØØØØØØØØ(ð	ñ 	ô 	€Cõ ÐjÈJÐ\_ÐjÐjÐciÐjÐj€EØ€Lrƒ   )TTrm  )r   )Nr²  r!   F)r
  r=   Trl  )dr|   ÚloggingrE  Údataclassesr   r   r   Ú	functoolsr   Útypingr   r   r	   r
   r   r   r   r   r   r   rÈ   Útorch.nnr  Útorch.nn.functionalÚ
functionalrž   Ú	timm.datar   r   Útimm.layersr   r   r   r   r   r   r   r   r   r   r   r   r    Ú_builderr"   Ú	_featuresr#   Ú_features_fxr$   r%   Ú_manipulater&   r'   Ú	_registryr(   r)   rï   r*   r  r+   r,   Ú__all__Ú	getLoggerry   r  r-   r”   rx  r}   r€   r®   r¼   rË   rÍ   rû   r   rý   rÉ   rÄ   r±  r�   rÃ  r.   rt  r   r  rŒ  r—  Údefault_cfgsr©  r°  rº  r½  rÂ  rÄ  rÆ  rÍ  rÏ  rÒ  rÕ  r‚   rƒ   r„   ú<module>rç     s	  ððð ð$ €€€Ø €€€Ø 2Ð 2Ð 2Ð 2Ð 2Ð 2Ð 2Ð 2Ð 2Ð 2Ø Ð Ð Ð Ð Ð Ø YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ Yà €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð à EÐ EÐ EÐ EÐ EÐ EÐ EÐ Eðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð +Ð *Ð *Ð *Ð *Ð *Ø +Ð +Ð +Ð +Ð +Ð +Ø LÐ LÐ LÐ LÐ LÐ LÐ LÐ LØ 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø <Ð <Ð <Ð <Ð <Ð <Ð <Ð <Ø Ð Ð Ð Ð Ð Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ð 6Ð 6à˜;Ð
'€ð ˆ'Ô
˜HÑ
%Ô
%€ð ð^,ð ^,ð ^,ð ^,ð ^,ñ ^,ô ^,ñ „ð^,ðB	˜ð 	°Lð 	ð 	ð 	ð 	ð Ø"ð	*ð *ØŒ<ð*à˜#˜s˜(”Oð*ð ð*ð ð	*ð
 ˆ5Œ<˜˜s C˜xœÐ(Ô)ð*ð *ð *ð *ðZJ¨¬ð Jð Jð Jð Jð $ð	Xð 	XØ”\ð	Xàð	Xð „\ð	Xð 	Xð 	Xð 	Xð=ð =ð =ð =ð =ñ =ô =ð =ð@3ð 3�lð 3¨sð 3¸8ð 3ð 3ð 3ð 3ðl ðM
ð M
ð M
ð M
ð M
�2”9ñ M
ô M
ñ ÔðM
ð` ð "#ØØ#Ø"œ]ðAð AØ”\ðAàðAð ðAð ˜Œ}ð	Að
 Œ{ðAð ˆeŒlÔðAð Að Añ ÔðAðH ð /3Ø Ø!"Ø&+ðEð EØŒ<ðEà˜eœlÔ+ðEð ðEð ð	Eð
  $ðEð „\ðEð Eð Eñ ÔðEðPmð mð mð mð m�”	ñ mô mð mð`<ð <ð <ðGð G˜sð G°uð GÐQUð GÐaið Gð Gð Gð Gð"H T¨#¨s¨(¤^ð H¸Ið HÈ$ÈsÐTWÈxÌ.ð Hð Hð Hð HðVð ˆcð  T¨#¨s¨(¤^ð ð ð ð ð" %Ð$Ø15°Øð2ñ 2ô 2ð 6:°TØð6ñ 6ô 6ð 9=¸Øð9ñ 9ô 9ð -1¨D©F¬FØ48°D±F´FØ48°D±F´Fð /3¨dØØð/ñ /ô /ð 15°ØØð1ñ 1ô 1ð%&ð &ñ ô €ð0ð ˜sð °ð È9ð ð ð ð ð* !ð(Að (AØð(Aàð(Að ð	(Að (Að (Að (AðZ !ðECð ECØðECàðECð ð	ECð ECð ECð ECðP ðð ¨4ð Àið ð ð ñ „ðð" ðð ¨tð È)ð ð ð ñ „ðð$ ðð °$ð ÈYð ð ð ñ „ðð& ðð ¨4ð Àið ð ð ñ „ðð  ðð °4ð Èið ð ð ñ „ðð0 ðð °4ð Èið ð ð ñ „ðð. ðð ¨dð Èð ð ð ñ „ðð  ðð °ð È9ð ð ð ñ „ðð ð rƒ   