§
    ·¯¸j_ë  ã                   ó^  — d Z ddlZddlmZ ddlmZmZmZmZm	Z	m
Z
mZmZ ddlZddlmZ ddlmc mZ ddlmZmZ ddlmZmZmZmZmZmZmZ ddlmZ dd	l m!Z! dd
l"m#Z#m$Z$ ddl%m&Z&m'Z' ddl(m)Z) ddgZ* G d„ dej+        ¦  «        Z,dej-        dej-        fd„Z.	 dTdej-        dej-        dej-        de/dej-        f
d„Z0	 	 dUdej-        dej-        dej-        de/de/dej-        fd„Z1 G d„ dej+        ¦  «        Z2 G d„ dej+        ¦  «        Z3 G d„ d ej+        ¦  «        Z4 G d!„ d"ej+        ¦  «        Z5 G d#„ d$ej+        ¦  «        Z6 G d%„ d&ej+        ¦  «        Z7 G d'„ dej+        ¦  «        Z8 G d(„ dej+        ¦  «        Z9dVd+ej+        d,e:d-e;ddfd.„Z<dVd/e:d-e;defd0„Z=d1ee:ej-        f         d2e8dee:ej-        f         fd3„Z>d1ee:ej-        f         d2ddee:ej-        f         fd4„Z?dWd6e:d7e;de8fd8„Z@dWd6e:d7e;de9fd9„ZAdXd:e:dee:ef         fd;„ZB e& eBd<d=d>¬?¦  «         eBd@d=d>¬?¦  «         eBdAd>¬B¦  «         eBdCd>¬B¦  «        dDœ¦  «        ZC eDdEdFdGdHdId5d*¬J¦  «        ZE eDdKdLdFdMdNd*¬O¦  «        ZFe'dWd7e;de9fdP„¦   «         ZGe'dWd7e;de8fdQ„¦   «         ZHe'dWd7e;de9fdR„¦   «         ZIe'dWd7e;de8fdS„¦   «         ZJdS )YaM  Gemma4 Vision Transformer

Vision encoder from Google's Gemma 4 multimodal model.
Custom ViT with 2D RoPE, Gated MLP, QKV normalization, and 4-norm sandwich blocks.

Paper: https://ai.google.dev/gemma/docs/core/model_card_4
Reference impl: https://github.com/huggingface/transformers (Gemma4VisionModel)

Copyright 2025 Yonghye Kwon
é    N)Úpartial)ÚAnyÚCallableÚDictÚListÚOptionalÚSetÚTupleÚUnion)ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)ÚRmsNormÚDropPathÚ	to_2tupleÚtrunc_normal_tf_Úuse_fused_attnÚget_device_dtypeÚcalculate_drop_path_ratesé   )Úbuild_model_with_cfg)Úfeature_take_indices)Ú
checkpointÚnamed_apply)Úgenerate_default_cfgsÚregister_model)Úbatch_patchifyÚGemma4VitEncoderÚGemma4VitClassifierc            	       óh   ‡ — e Zd ZdZ	 	 	 ddedededdfˆ fd„Zdd	„Zd
ej	        dej	        fd„Z
ˆ xZS )ÚGemma4ClippableLinearzÜLinear layer with optional input/output clamping.

    Used in Gemma4 E4B variant where clamp values are finite and affect output.
    When use_clipped=False, behaves as a standard nn.Linear (no buffers registered).
    FNÚin_featuresÚout_featuresÚuse_clippedÚreturnc                 óì  •— ||dœ}t          ¦   «                              ¦   «          || _        t          j        ||fddi|¤Ž| _        |rª|                      dt          j        d	i |¤Ž¦  «         |                      dt          j        d	i |¤Ž¦  «         |                      dt          j        d	i |¤Ž¦  «         |                      dt          j        d	i |¤Ž¦  «         |  	                    ¦   «          d S d S )
N©ÚdeviceÚdtypeÚbiasFÚ	input_min© Ú	input_maxÚ
output_minÚ
output_max)r+   )
ÚsuperÚ__init__r#   ÚnnÚLinearÚlinearÚregister_bufferÚtorchÚemptyÚreset_parameters)Úselfr!   r"   r#   r'   r(   ÚddÚ	__class__s          €úf/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/models/gemma4_vit.pyr0   zGemma4ClippableLinear.__init__/   s  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ&ˆÔÝ”i ¨\ÐLÐLÀÐLÈÐLÐLˆŒØð 	$Ø× Ò  ­e¬kÐ.CÐ.CÀÐ.CÐ.CÑDÔDÐDØ× Ò  ­e¬kÐ.CÐ.CÀÐ.CÐ.CÑDÔDÐDØ× Ò  ­u¬{Ð/DÐ/DÀÐ/DÐ/DÑEÔEÐEØ× Ò  ­u¬{Ð/DÐ/DÀÐ/DÐ/DÑEÔEÐEà×!Ò!Ñ#Ô#Ð#Ð#Ð#ð	$ð 	$ó    c                 óT  — | j         r | j                             t          d¦  «         ¦  «         | j                             t          d¦  «        ¦  «         | j                             t          d¦  «         ¦  «         | j                             t          d¦  «        ¦  «         d S d S )NÚinf)r#   r*   Úfill_Úfloatr,   r-   r.   ©r8   s    r;   r7   z&Gemma4ClippableLinear.reset_parametersC   s“   € ð Ôð 	0ØŒN× Ò ¥%¨¡,¤, Ñ/Ô/Ð/ØŒN× Ò ¥ u¡¤Ñ.Ô.Ð.ØŒO×!Ò!¥5¨¡<¤< -Ñ0Ô0Ð0ØŒO×!Ò!¥%¨¡,¤,Ñ/Ô/Ð/Ð/Ð/ð		0ð 	0r<   Úxc                 óÌ   — | j         r t          j        || j        | j        ¦  «        }|                      |¦  «        }| j         r t          j        || j        | j        ¦  «        }|S ©N)r#   r5   Úclampr*   r,   r3   r-   r.   ©r8   rB   s     r;   ÚforwardzGemma4ClippableLinear.forwardL   s[   € ØÔð 	?Ý”˜A˜tœ~¨t¬~Ñ>Ô>ˆAØ�KŠK˜‰NŒNˆØÔð 	AÝ”˜A˜tœ°´Ñ@Ô@ˆAØˆr<   )FNN©r$   N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚintÚboolr0   r7   r5   ÚTensorrG   Ú__classcell__©r:   s   @r;   r    r    (   s·   ø€ € € € € ðð ð !&ØØð$ð $àð$ð ð$ð ð	$ð 
ð$ð $ð $ð $ð $ð $ð(0ð 0ð 0ð 0ð˜œð ¨%¬,ð ð ð ð ð ð ð ð r<   r    rB   r$   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..Néÿÿÿÿé   ©Údim)Úshaper5   Úcat)rB   Úx1Úx2s      r;   Úrotate_halfr[   U   s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'r<   rT   ÚcosÚsinÚunsqueeze_dimc                 ó†   — |                      |¦  «        }|                      |¦  «        }| |z  t          | ¦  «        |z  z   S )z0Apply rotary position embedding to input tensor.)Ú	unsqueezer[   )rB   r\   r]   r^   s       r;   Úapply_rotary_pos_embra   \   s@   € ð �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�‰G� A™œ¨Ñ,Ñ-Ð-r<   Úndimc                 ó8  ‡‡	‡
‡— | j         d         }d|d|z  z  z  }|g|z  }t          j        | |d¬¦  «        Št          j        ||d¬¦  «        Š	t          j        ||d¬¦  «        Š
ˆ	ˆ
ˆˆfd„t          |¦  «        D ¦   «         }t          j        |d¬¦  «        S )z•Apply multidimensional RoPE to input tensor.

    Splits input along head_dim into ndim parts, applies RoPE to each,
    then concatenates back.
    rS   rT   rU   c                 óZ   •— g | ]'}t          ‰|         ‰|         ‰|         ‰¬ ¦  «        ‘Œ(S ))r^   )ra   )Ú.0ÚkÚ	cos_partsÚ	sin_partsr^   Úx_partss     €€€€r;   ú
<listcomp>z/apply_multidimensional_rope.<locals>.<listcomp>|   sE   ø€ ð ð ð ØfgÕ˜W QœZ¨°1¬°yÀ´|ÐS`ÐaÑaÔaðð ð r<   )rW   r5   ÚsplitÚrangerX   )rB   r\   r]   rb   r^   Únum_input_channelsÚnum_rotated_channels_per_dimÚsplit_sizesÚy_partsrg   rh   ri   s       `    @@@r;   Úapply_multidimensional_roperq   h   sÏ   øøøø€ ð œ œÐØ#$Ð(:¸qÀ4¹xÑ(HÑ#IÐ Ø/Ð0°4Ñ7€KåŒk˜!˜[¨bÐ1Ñ1Ô1€GÝ”˜C °"Ð5Ñ5Ô5€IÝ”˜C °"Ð5Ñ5Ô5€Iðð ð ð ð ð ð ÝkpÐquÑkvÔkvðñ ô €Gõ Œ9�W "Ð%Ñ%Ô%Ð%r<   c                   óœ   ‡ — e Zd ZdZ	 	 	 ddededdfˆ fd„Zdd„Zdd	„Zdd
„Z	de
j        de
j        dee
j        e
j        f         fd„Zˆ xZS )ÚGemma4RotaryEmbedding2Dz´2D Rotary Position Embedding for Gemma4 vision encoder.

    Computes RoPE independently for each spatial dimension (x, y),
    using theta=100.0 and the partitioned head_dim.
    ç      Y@NÚhead_dimÚ
rope_thetar$   c                 ó
  •— t          ¦   «                              ¦   «          || _        || _        |dz  dz  }|                      dt          j        ||t
          j        ¬¦  «        d¬¦  «         |                      ¦   «          d S )NrT   Úinv_freqr&   F)Ú
persistent)	r/   r0   ru   rv   r4   r5   r6   r@   Ú_init_buffers)r8   ru   rv   r'   r(   Ú	num_freqsr:   s         €r;   r0   z Gemma4RotaryEmbedding2D.__init__‰   sŒ   ø€ õ 	‰Œ×ÒÑÔÐØ ˆŒØ$ˆŒð  ‘] qÑ(ˆ	Ø×ÒØÝŒK˜	¨&½¼ÐDÑDÔDØð 	ñ 	
ô 	
ð 	
ð 	×ÒÑÔÐÐÐr<   c           	      óÄ   — | j         dz  }d| j        t          j        d|dt          j        | j        j        ¬¦  «        |z  z  z  }| j                             |¦  «         dS )z.Compute and fill non-persistent buffer values.rT   ç      ð?r   ©r(   r'   N)ru   rv   r5   Úaranger@   rx   r'   Úcopy_)r8   Úspatial_dimrx   s      r;   rz   z%Gemma4RotaryEmbedding2D._init_buffersŸ   sf   € à”m qÑ(ˆØØŒOÝ”˜Q ¨Qµe´kÈ$Ì-ÔJ^Ð_Ñ_Ô_ÐbmÑmñoñ
ˆð 	Œ×Ò˜HÑ%Ô%Ð%Ð%Ð%r<   c                 ó.   — |                       ¦   «          d S rD   ©rz   rA   s    r;   r7   z(Gemma4RotaryEmbedding2D.reset_parameters¨   s   € Ø×ÒÑÔÐÐÐr<   c                 ó.   — |                       ¦   «          dS )z"Initialize non-persistent buffers.Nrƒ   rA   s    r;   Úinit_non_persistent_buffersz3Gemma4RotaryEmbedding2D.init_non_persistent_buffers«   s   € à×ÒÑÔÐÐÐr<   rB   Úposition_idsc                 ó¦  — t          j        ¦   «         5  | j        ddd…df                              ¦   «                              |j        d         dd¦  «                             |j        ¦  «        }g }g }t          d¦  «        D ]Õ}|dd…dd…|f         }|dd…ddd…f                              ¦   «         }|                     ¦   «         |                     ¦   «         z   	                    dd¦  «        }	t          j
        |	|	fd¬¦  «        }
|                     |
                     ¦   «         ¦  «         |                     |
                     ¦   «         ¦  «         ŒÖt          j
        |d¬¦  «                             |j        ¬¦  «        }t          j
        |d¬¦  «                             |j        ¬¦  «        }ddd¦  «         n# 1 swxY w Y   ||fS )zü
        Args:
            x: Hidden states tensor, used only for dtype/device.
            position_ids: (B, N, 2) Gemma4-internal ``(x, y)`` patch coords.

        Returns:
            cos, sin: (B, N, head_dim) tensors for RoPE application.
        Nr   rS   r   rT   rU   )r(   )r5   Úno_gradrx   r@   ÚexpandrW   Útor'   rl   Ú	transposerX   Úappendr\   r]   r(   )r8   rB   r†   Úinv_freq_expandedÚall_cosÚall_sinÚiÚdim_posÚdim_pos_expandedÚfreqsÚembr\   r]   s                r;   rG   zGemma4RotaryEmbedding2D.forward¯   sð  € õ  Œ]‰_Œ_ð 	?ð 	?Ø $¤¨d°A°A°A°t¨mÔ <× BÒ BÑ DÔ D× KÒ KÈLÔL^Ð_`ÔLaÐceÐghÑ iÔ i× lÒ lÐmnÔmuÑ vÔ vÐà*,ˆGØ*,ˆGÝ˜1‘X”Xð *ð *�Ø& q q q¨!¨!¨!¨Q wÔ/�Ø#*¨1¨1¨1¨d°A°A°A¨:Ô#6×#<Ò#<Ñ#>Ô#>Ð à*×0Ò0Ñ2Ô2Ð5E×5KÒ5KÑ5MÔ5MÑM×XÒXÐYZÐ\]Ñ^Ô^�Ý”i ¨ °BÐ7Ñ7Ô7�Ø—’˜sŸwšw™yœyÑ)Ô)Ð)Ø—’˜sŸwšw™yœyÑ)Ô)Ð)Ð)å”)˜G¨Ð,Ñ,Ô,×/Ò/°a´gÐ/Ñ>Ô>ˆCÝ”)˜G¨Ð,Ñ,Ô,×/Ò/°a´gÐ/Ñ>Ô>ˆCð	?ð 	?ð 	?ñ 	?ô 	?ð 	?ð 	?ð 	?ð 	?ð 	?ð 	?øøøð 	?ð 	?ð 	?ð 	?ð  �Cˆxˆs   ”F$GÇGÇG)rt   NNrH   )rI   rJ   rK   rL   rM   r@   r0   rz   r7   r…   r5   rO   r
   rG   rP   rQ   s   @r;   rs   rs   ‚   sñ   ø€ € € € € ðð ð !&ØØðð àðð ðð 
ðð ð ð ð ð ð,&ð &ð &ð &ðð ð ð ðð ð ð ð àŒ|ð ð  œ,ð ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð	 ð  ð  ð  ð  ð  ð  ð  r<   rs   c                   ó˜  ‡ — e Zd ZdZ	 	 	 	 	 	 ddeeeeef         f         ded	ed
eddf
ˆ fd„Zdd„Zdededede	j
        dee	j        e	j        f         f
d„Zde	j        de	j        de	j        fd„Z	 	 ddee	j        eee	j        f         f         dee	j                 dee	j                 dee	j        e	j        e	j        f         fd„Zˆ xZS )ÚGemma4PatchEmbeduï  Linear patch embedding with a 2D position-embedding table.

    Unlike the standard ViT PatchEmbed (Conv2d), Gemma4 uses a Linear projection
    on flattened patches plus a separate 2D position-embedding table applied via
    one-hot lookup. Inputs may be supplied as:

      * ``(B, C, H, W)`` raw images â€” patchified inline in P-P-C flat order.
      * ``(B, N, P*P*C)`` pre-patchified P-P-C patches (NaFlex loader output).
      * ``(B, N, Ph, Pw, C)`` pre-patchified unflattened patches.
      * ``dict`` with keys ``patches`` / ``patch_coord`` / ``patch_valid``.

    All input-dispatch logic is owned by this module; the parent model simply
    calls ``self.patch_embed(x, patch_coord, patch_valid)`` and unpacks the
    returned tensors.
    é   é   é   é (  NÚ
patch_sizeÚin_chansÚ	embed_dimÚposition_embedding_sizer$   c                 óh  •— ||dœ}t          ¦   «                              ¦   «          t          |¦  «        | _        || _        || _        | j        \  }}	t          j        ||z  |	z  |fddi|¤Ž| _        t          j	        t          j        d||fi |¤Ž¦  «        | _        |                      ¦   «          d S )Nr&   r)   FrT   )r/   r0   r   r›   r�   rž   r1   r2   Ú
input_projÚ	Parameterr5   r6   Úposition_embedding_tabler7   )r8   r›   rœ   r�   rž   r'   r(   r9   ÚphÚpwr:   s             €r;   r0   zGemma4PatchEmbed.__init__ã   sÃ   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ# JÑ/Ô/ˆŒØ"ˆŒØ'>ˆÔ$à”‰ˆˆBÝœ) H¨r¡M°BÑ$6¸	ÐTÐTÈÐTÐQSÐTÐTˆŒÝ(*¬ÝŒK˜Ð2°IÐDÐDÀÐDÐDñ)
ô )
ˆÔ%ð 	×ÒÑÔÐÐÐr<   c                 ó2   — t          | j        d¬¦  «         d S )Nç{®Gáz”?©Ústd)r   r¢   rA   s    r;   r7   z!Gemma4PatchEmbed.reset_parametersú   s   € Ý˜Ô6¸DÐAÑAÔAÐAÐAÐAr<   Ú
batch_sizeÚpHÚpWr'   c                 ó¬  — t          j        ||¬¦  «        }t          j        ||¬¦  «        }t          j        ||d¬¦  «        \  }}t          j        |                     ¦   «         |                     ¦   «         gd¬¦  «        }	|	                     d¦  «                             |dd¦  «        }	t          j        |||z  t           j        |¬¦  «        }
|	|
fS )z:Row-major grid coords in external NaFlex ``(y, x)`` order.)r'   Úij)ÚindexingrS   rU   r   r~   )	r5   r   ÚmeshgridÚstackÚflattenr`   r‰   ÚonesrN   )r8   r©   rª   r«   r'   ÚysÚxsÚgrid_yÚgrid_xÚpatch_coordÚpatch_valids              r;   Ú_default_patch_coordz%Gemma4PatchEmbed._default_patch_coordý   sÁ   € õ Œ\˜" VÐ,Ñ,Ô,ˆÝŒ\˜" VÐ,Ñ,Ô,ˆÝœ¨¨B¸Ð>Ñ>Ô>‰ˆ�Ý”k 6§>¢>Ñ#3Ô#3°V·^²^Ñ5EÔ5EÐ"FÈBÐOÑOÔOˆØ!×+Ò+¨AÑ.Ô.×5Ò5°jÀ"ÀbÑIÔIˆÝ”j ¨R°"©W½E¼JÈvÐVÑVÔVˆØ˜KÐ'Ð'r<   r†   Úpadding_positionsc                 óZ  — |                      d¬¦  «        }t          j        || j        ¬¦  «        }|                     dddd¦  «                             | j        ¦  «        }|| j        z  }|                     d¬¦  «        }t          j	        | 
                    d¦  «        d	|¦  «        }|S )
zšCompute position embeddings via one-hot matmul against the 2D table.

        ``position_ids`` follows the Gemma4-internal ``(x, y)`` convention.
        r   ©Úmin©Únum_classesrT   r   r˜   rU   rS   ç        )rE   ÚFÚone_hotrž   ÚpermuterŠ   r¢   Úsumr5   Úwherer`   )r8   r†   rº   Úclamped_positionsrÂ   Úposition_embeddingss         r;   Ú_position_embeddingsz%Gemma4PatchEmbed._position_embeddings  s°   € ð )×.Ò.°1Ð.Ñ5Ô5ÐÝ”)Ð-¸4Ô;WÐXÑXÔXˆØ—/’/ ! Q¨¨1Ñ-Ô-×0Ò0°Ô1NÑOÔOˆà%¨Ô(EÑEÐØ1×5Ò5¸!Ð5Ñ<Ô<ÐÝ#œkØ×'Ò'¨Ñ+Ô+¨SÐ2Eñ
ô 
Ðð #Ð"r<   rB   r·   r¸   c                 óÌ  — t          |t          ¦  «        r4|                     d|¦  «        }|                     d|¦  «        }|d         }| j        \  }}|j        dk    rM|j        \  }}}}	|€&|                      |||z  |	|z  |j        ¦  «        \  }}t          |||fdd¬¦  «        \  }}nÌ|j        dk    rE| 	                    d	d
ddd¦  «         
                    |j        d	         |j        d
         d¦  «        }n||j        dk    rY|j        \  }}
}|||z  z  }|                     ||
|||¦  «         	                    d	d
ddd¦  «         
                    ||
|¦  «        }nt          d|j        › d�¦  «        ‚|€t          d¦  «        ‚|€e|dk                         d¬¦  «        }|                     ¦   «         r| }n3t          j        |j        dd…         t          j        |j        ¬¦  «        }d|dz
  z  }|                      |                     | j        j        j        ¦  «        ¦  «        }|                     d¬¦  «        }| }||                      ||¦  «        z   }|||fS )aÅ  Normalize inputs, patchify / project / position-embed.

        Accepts external NaFlex ``patch_coord`` (y, x) / ``patch_valid`` inputs
        and returns the internal (x, y) form used by the rest of the model.

        Returns:
            embeddings: (B, N, embed_dim) patch embeddings with position info added.
            position_ids: (B, N, 2) Gemma4-internal (x, y) coords.
            padding_positions: (B, N) True for padding tokens.
        r·   r¸   Úpatchesé   NF)ÚpadÚchannels_lasté   r   r   rT   r˜   rS   z&Expected input ndim in (3, 4, 5); got ú.z1patch_coord is required for pre-patchified input.rU   r~   ç      à?)rS   )Údims)Ú
isinstanceÚdictÚgetr›   rb   rW   r¹   r'   r   rÃ   ÚreshapeÚviewÚ
ValueErrorÚallÚanyr5   r²   rN   r    rŠ   Úweightr(   ÚfliprÈ   )r8   rB   r·   r¸   r£   r¤   ÚBÚ_ÚHÚWÚNÚPPCÚCÚsentinelr†   rº   s                   r;   rG   zGemma4PatchEmbed.forward!  sŒ  € õ  �a�ÑÔð 	ØŸ%š% ¨{Ñ;Ô;ˆKØŸ%š% ¨{Ñ;Ô;ˆKØ�)”ˆAà”‰ˆˆBØŒ6�QŠ;ˆ;àœ‰JˆAˆq�!�QØÐ"Ø+/×+DÒ+DÀQÈÈRÉÐQRÐVXÑQXÐZ[ÔZbÑ+cÔ+cÑ(�˜[Ý! ! b¨" X°5ÈÐNÑNÔN‰DˆAˆqˆqØŒV�qŠ[ˆ[ð —	’	˜!˜Q  1 aÑ(Ô(×0Ò0°´¸´¸Q¼WÀQ¼ZÈÑLÔLˆAˆAØŒV�qŠ[ˆ[ð œ‰IˆAˆq�#Ø˜˜R™Ñ ˆAØ—’�q˜!˜R  QÑ'Ô'×/Ò/°°1°a¸¸AÑ>Ô>×FÒFÀqÈ!ÈSÑQÔQˆAˆAåØB¸¼ÐBÐBÐBñô ð ð ÐÝÐPÑQÔQÐQàÐØ# rÒ)×.Ò.°2Ð.Ñ6Ô6ˆHØ�|Š|‰~Œ~ð Ø'˜i��å#œjØÔ% b q bÔ)µ´ÀKÔDVðñ ô �ð
 ��S‘‰MˆØ�OŠO˜AŸDšD ¤Ô!7Ô!=Ñ>Ô>Ñ?Ô?ˆð #×'Ò'¨UÐ'Ñ3Ô3ˆØ(˜LÐØ�×)Ò)¨,Ð8IÑJÔJÑJˆà�,Ð 1Ð1Ð1r<   )r—   r˜   r™   rš   NNrH   ©NN)rI   rJ   rK   rL   r   rM   r
   r0   r7   r5   r'   rO   r¹   rÈ   r   Ústrr   rG   rP   rQ   s   @r;   r–   r–   Ò   sÐ  ø€ € € € € ðð ð$ 79ØØ Ø+0ØØð ð  à˜c 5¨¨c¨¤?Ð2Ô3ð ð ð ð ð	 ð
 &)ð ð 
ð ð  ð  ð  ð  ð  ð.Bð Bð Bð Bð(àð(ð ð(ð ð	(ð
 ”Lð(ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð(ð (ð (ð (ð #àœ,ð#ð  %œ|ð#ð 
Œð	#ð #ð #ð #ð. 37Ø26ð	@2ð @2à�U”\ 4¨¨U¬\Ð(9Ô#:Ð:Ô;ð@2ð " %¤,Ô/ð@2ð " %¤,Ô/ð	@2ð
 
ˆuŒ|˜Uœ\¨5¬<Ð7Ô	8ð@2ð @2ð @2ð @2ð @2ð @2ð @2ð @2r<   r–   c                   óò   ‡ — e Zd ZU dZej        j        e         ed<   	 	 	 	 	 	 	 	 	 dd	e	d
e	de	de
e	         dededededdfˆ fd„Z	 ddej        dej        dej        de
ej                 dej        f
d„Zˆ xZS )ÚGemma4Attentiona  Gemma4 Vision Attention with QKV normalization and 2D RoPE.

    Key features:
    - Separate Q, K, V projections (not fused)
    - RMSNorm on Q, K (with scale) and V (without scale)
    - 2D RoPE applied after normalization
    - Attention scale = 1.0 (since QK are normalized)
    Ú
fused_attné   é@   NrÀ   ç�íµ ÷Æ°>FrV   Ú	num_headsru   Únum_kv_headsÚ	attn_dropÚ	proj_dropÚnorm_epsÚuse_clipped_linearsr$   c                 óz  •— |	|
dœ}t          ¦   «                              ¦   «          || _        || _        |p|| _        || j        z  | _        t          ¦   «         | _        t          |||z  fd|i|¤Ž| _	        t          || j        |z  fd|i|¤Ž| _
        t          || j        |z  fd|i|¤Ž| _        t          ||z  |fd|i|¤Ž| _        t          |f|ddœ|¤Ž| _        t          |f|ddœ|¤Ž| _        t          |f|ddœ|¤Ž| _        t#          j        |¦  «        | _        t#          j        |¦  «        | _        d S )Nr&   r#   T©ÚepsÚaffineF)r/   r0   rì   ru   rí   Únum_kv_groupsr   rè   r    Úq_projÚk_projÚv_projÚo_projr   Úq_normÚk_normÚv_normr1   ÚDropoutrî   rï   )r8   rV   rì   ru   rí   rî   rï   rð   rñ   r'   r(   r9   r:   s               €r;   r0   zGemma4Attention.__init__p  sz  ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ"ˆŒØ ˆŒØ(Ð5¨IˆÔØ&¨$Ô*;Ñ;ˆÔÝ(Ñ*Ô*ˆŒå+¨C°¸XÑ1EÐmÐmÐSfÐmÐjlÐmÐmˆŒÝ+¨C°Ô1BÀXÑ1MÐuÐuÐ[nÐuÐrtÐuÐuˆŒÝ+¨C°Ô1BÀXÑ1MÐuÐuÐ[nÐuÐrtÐuÐuˆŒÝ+¨I¸Ñ,@À#ÐmÐmÐSfÐmÐjlÐmÐmˆŒå˜hÐH¨H¸TÐHÐHÀRÐHÐHˆŒÝ˜hÐH¨H¸TÐHÐHÀRÐHÐHˆŒå˜hÐI¨H¸UÐIÐIÀbÐIÐIˆŒåœ IÑ.Ô.ˆŒÝœ IÑ.Ô.ˆŒˆˆr<   rB   Úrope_cosÚrope_sinÚ	attn_maskc                 óæ  — |j         \  }}}|                      |¦  «                             ||| j        | j        ¦  «        }|                      |¦  «                             ||| j        | j        ¦  «        }	|                      |¦  «                             ||| j        | j        ¦  «        }
|                      |¦  «        }|  	                    |	¦  «        }	|  
                    |
¦  «        }
t          |||dd¬¦  «        }t          |	||dd¬¦  «        }	|                     dd¦  «        }|	                     dd¦  «        }	|
                     dd¦  «        }
| j        dk    r8|	                     | j        d¬¦  «        }	|
                     | j        d¬¦  «        }
| j        r.t!          j        ||	|
|| j        r| j        j        ndd¬¦  «        }nP||	                     dd	¦  «        z  }|�||z   }|                     d	¬¦  «        }|                      |¦  «        }||
z  }|                     dd¦  «                             ||d	¦  «        }|                      |¦  «        }|                      |¦  «        }|S )
NrT   )rb   r^   r   rU   rÀ   r}   )r  Ú	dropout_pÚscaleéþÿÿÿrS   )rW   r÷   rÖ   rì   ru   rø   rí   rù   rû   rü   rý   rq   r‹   rö   Úrepeat_interleaverè   rÁ   Úscaled_dot_product_attentionÚtrainingrî   ÚpÚsoftmaxrÕ   rú   rï   )r8   rB   rÿ   r   r  rÜ   rà   râ   Úqrf   ÚvÚattns               r;   rG   zGemma4Attention.forward’  sS  € ð ”'‰ˆˆ1ˆaà�KŠK˜‰NŒN×Ò  1 d¤n°d´mÑDÔDˆØ�KŠK˜‰NŒN×Ò  1 dÔ&7¸¼ÑGÔGˆØ�KŠK˜‰NŒN×Ò  1 dÔ&7¸¼ÑGÔGˆð �KŠK˜‰NŒNˆØ�KŠK˜‰NŒNˆØ�KŠK˜‰NŒNˆõ (¨¨8°XÀAÐUVÐWÑWÔWˆÝ'¨¨8°XÀAÐUVÐWÑWÔWˆð �KŠK˜˜1ÑÔˆØ�KŠK˜˜1ÑÔˆØ�KŠK˜˜1ÑÔˆð Ô Ò!Ð!Ø×#Ò# DÔ$6¸AÐ#Ñ>Ô>ˆAØ×#Ò# DÔ$6¸AÐ#Ñ>Ô>ˆAàŒ?ð 	ÝÔ.ØØØØ#Ø.2¬mÐD˜$œ.Ô*Ð*ÀØðñ ô ˆAˆAð �q—{’{ 2 rÑ*Ô*Ñ*ˆDØÐ$Ø˜iÑ'�Ø—<’< B�<Ñ'Ô'ˆDØ—>’> $Ñ'Ô'ˆDØ�q‘ˆAà�KŠK˜˜1ÑÔ×%Ò% a¨¨BÑ/Ô/ˆØ�KŠK˜‰NŒNˆØ�NŠN˜1ÑÔˆØˆr<   )	ré   rê   NrÀ   rÀ   rë   FNNrD   )rI   rJ   rK   rL   r5   ÚjitÚFinalrN   Ú__annotations__rM   r   r@   r0   rO   rG   rP   rQ   s   @r;   rç   rç   d  s<  ø€ € € € € € ðð ð ”	” Ô%Ð%Ð%Ñ%ð
  ØØ*.Ø"Ø"Ø"Ø(-ØØð /ð  /àð /ð ð /ð ð	 /ð
 # 3œ-ð /ð ð /ð ð /ð ð /ð "&ð /ð 
ð /ð  /ð  /ð  /ð  /ð  /ðN 15ð5ð 5àŒ|ð5ð ”lð5ð ”lð	5ð
   ¤Ô-ð5ð 
Œð5ð 5ð 5ð 5ð 5ð 5ð 5ð 5r<   rç   c                   óx   ‡ — e Zd ZdZ	 	 	 	 	 ddededee         ded	ed
dfˆ fd„Z	de
j        d
e
j        fd„Zˆ xZS )ÚGemma4GatedMlpz�Gated MLP for Gemma4 Vision Encoder.

    Uses GELUTanh activation: output = down_proj(gelu_tanh(gate_proj(x)) * up_proj(x))
    NrÀ   Fr!   Úhidden_featuresÚ	act_layerÚdroprñ   r$   c                 óN  •— ||dœ}t          ¦   «                              ¦   «          t          ||fd|i|¤Ž| _        t          ||fd|i|¤Ž| _        t          ||fd|i|¤Ž| _        |�
 |¦   «         nt          j        d¬¦  «        | _        t          j	        |¦  «        | _
        d S )Nr&   r#   Útanh©Úapproximate)r/   r0   r    Ú	gate_projÚup_projÚ	down_projr1   ÚGELUÚactrþ   r  )
r8   r!   r  r  r  rñ   r'   r(   r9   r:   s
            €r;   r0   zGemma4GatedMlp.__init__Ð  sÁ   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ.¨{¸OÐsÐsÐYlÐsÐprÐsÐsˆŒÝ,¨[¸/ÐqÐqÐWjÐqÐnpÐqÐqˆŒÝ.¨ÀÐsÐsÐYlÐsÐprÐsÐsˆŒØ"+Ð"7�9�9‘;”;�;½R¼WÐQWÐ=XÑ=XÔ=XˆŒÝ”J˜tÑ$Ô$ˆŒ	ˆ	ˆ	r<   rB   c           	      óÊ   — |                       |                      |                      |                      |¦  «        ¦  «        |                      |¦  «        z  ¦  «        ¦  «        S rD   )r  r  r  r  r  rF   s     r;   rG   zGemma4GatedMlp.forwardâ  sH   € Ø�yŠy˜Ÿš¨¯ª°·²ÀÑ1BÔ1BÑ(CÔ(CÀdÇlÂlÐSTÁoÄoÑ(UÑVÔVÑWÔWÐWr<   )NrÀ   FNN)rI   rJ   rK   rL   rM   r   r   r@   rN   r0   r5   rO   rG   rP   rQ   s   @r;   r  r  Ê  sÌ   ø€ € € € € ðð ð -1ØØ(-ØØð%ð %àð%ð !ð%ð   Ô)ð	%ð
 ð%ð "&ð%ð 
ð%ð %ð %ð %ð %ð %ð$X˜œð X¨%¬,ð Xð Xð Xð Xð Xð Xð Xð Xr<   r  c                   óÞ   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 ddededed	ed
ee         dededededee         deddfˆ fd„Z		 dde
j        de
j        de
j        dee
j                 de
j        f
d„Zˆ xZS )ÚGemma4Blocka  Gemma4 Vision Encoder Block with 4-norm sandwich pattern.

    Unlike standard ViT (pre-norm with 2 norms), Gemma4 uses:
    - input_layernorm (norm1) + post_attention_layernorm (norm2)
    - pre_feedforward_layernorm (norm3) + post_feedforward_layernorm (norm4)
    Nrë   rÀ   FrV   rì   ru   Úintermediate_sizerí   rð   rî   rï   Ú	drop_pathr  rñ   r$   c                 ó¬  •— ||dœ}t          ¦   «                              ¦   «          t          |fd|i|¤Ž| _        t	          d||||||||dœ|¤Ž| _        t          |fd|i|¤Ž| _        t          |fd|i|¤Ž| _        t          d|||
|dœ|¤Ž| _	        t          |fd|i|¤Ž| _
        |	dk    rt          |	¦  «        nt          j        ¦   «         | _        d S )Nr&   rô   )rV   rì   ru   rí   rî   rï   rð   rñ   )r!   r  r  rñ   rÀ   r+   )r/   r0   r   Únorm1rç   r  Únorm2Únorm3r  ÚmlpÚnorm4r   r1   ÚIdentityr#  )r8   rV   rì   ru   r"  rí   rð   rî   rï   r#  r  rñ   r'   r(   r9   r:   s                  €r;   r0   zGemma4Block.__init__î  s0  ø€ ð  ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ˜SÐ5Ð5 hÐ5°"Ð5Ð5ˆŒ
Ý#ð 

ØØØØ%ØØØØ 3ð

ð 

ð ð

ð 

ˆŒ	õ ˜SÐ5Ð5 hÐ5°"Ð5Ð5ˆŒ
Ý˜SÐ5Ð5 hÐ5°"Ð5Ð5ˆŒ
Ý!ð 
ØØ-ØØ 3ð	
ð 
ð
 ð
ð 
ˆŒõ ˜SÐ5Ð5 hÐ5°"Ð5Ð5ˆŒ
Ø09¸C²°� )Ñ,Ô,Ð,ÅRÄ[Á]Ä]ˆŒˆˆr<   rB   rÿ   r   r  c                 ór  — |}|                       |¦  «        }|                      ||||¬¦  «        }|                      |¦  «        }||                      |¦  «        z   }|}|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }||                      |¦  «        z   }|S )N©r  )r%  r  r&  r#  r'  r(  r)  )r8   rB   rÿ   r   r  Úresiduals         r;   rG   zGemma4Block.forward  s¥   € ð ˆØ�JŠJ�q‰MŒMˆØ�IŠI�a˜ 8°yˆIÑAÔAˆØ�JŠJ�q‰MŒMˆØ�t—~’~ aÑ(Ô(Ñ(ˆð ˆØ�JŠJ�q‰MŒMˆØ�HŠH�Q‰KŒKˆØ�JŠJ�q‰MŒMˆØ�t—~’~ aÑ(Ô(Ñ(ˆØˆr<   )	Nrë   rÀ   rÀ   rÀ   NFNNrD   )rI   rJ   rK   rL   rM   r   r@   r   rN   r0   r5   rO   rG   rP   rQ   s   @r;   r!  r!  æ  sY  ø€ € € € € ðð ð +/Ø"Ø"Ø"Ø"Ø,0Ø(-ØØð(Sð (Sàð(Sð ð(Sð ð	(Sð
  #ð(Sð # 3œ-ð(Sð ð(Sð ð(Sð ð(Sð ð(Sð   Ô)ð(Sð "&ð(Sð 
ð(Sð (Sð (Sð (Sð (Sð (Sð^ 15ðð àŒ|ðð ”lðð ”lð	ð
   ¤Ô-ðð 
Œðð ð ð ð ð ð ð r<   r!  c            
       óÔ   ‡ — e Zd ZdZddedefˆ fd„Zdej        dej        deej        ej        f         fd	„Z	dej        dej        d
ej        deej        ej        f         fd„Z
ˆ xZS )ÚGemma4VisionPooleru·   Spatial pooling for Gemma4 vision encoder output.

    Pools patches by averaging within kÃ—k grid cells based on position coordinates.
    Output is scaled by sqrt(hidden_size).
    r˜   Úhidden_sizeÚpooling_kernel_sizec                 óx   •— t          ¦   «                              ¦   «          || _        |dz  | _        || _        d S )NrÐ   )r/   r0   r0  Úroot_hidden_sizer1  )r8   r0  r1  r:   s      €r;   r0   zGemma4VisionPooler.__init__6  s<   ø€ Ý‰Œ×ÒÑÔÐØ&ˆÔØ +¨SÑ 0ˆÔØ#6ˆÔ Ð Ð r<   Úhidden_statesr†   r$   c           	      ó´  — |j         d         }| j        }||z  }||z  dk    rt          d|› d|› d|› d�¦  «        ‚||z  }|                     d¬¦  «        }|d                              d	d
¬¦  «        d         dz   }t          j        ||d¬¦  «        }	|	d         ||z  |	d         z  z   }	t          j        |	 	                    ¦   «         |¦  «         
                    ¦   «         |z  }
|
                     dd¦  «        | 
                    ¦   «         z  }t          j        |
dk                         d¬¦  «        ¦  «        }|                     |j        ¦  «        |fS )u@  2D spatial pooling on a ``k Ã— k`` grid (k = ``self.pooling_kernel_size``).

        ``N`` patches are binned into ``k^2``-sized cells, so the pool requires
        ``N % k^2 == 0`` (caller ensures both grid dims divide by k upstream).
        ``position_ids`` follows the Gemma4-internal ``(x, y)`` convention.
        r   r   zCannot pool z tokens with k=z: N must be divisible by k^2=z.. Both grid dimensions must be divisible by k.r¼   ©.r   rS   T©rV   ÚkeepdimÚfloor)Úrounding_mode©.r   rT   rU   )rW   r1  r×   rE   Úmaxr5   ÚdivrÁ   rÂ   Úlongr@   r‹   Úlogical_notrØ   rŠ   r(   )r8   r4  r†   rà   rf   Ú	k_squaredÚoutput_lengthrÆ   Úmax_xÚkernel_idxsÚweightsÚoutputÚmasks                r;   Ú_avg_pool_by_positionsz)Gemma4VisionPooler._avg_pool_by_positions<  sŠ  € ð Ô Ô"ˆØÔ$ˆØ˜‘Eˆ	Øˆy‰=˜AÒÐÝð@˜qð @ð @°ð @ð @ÐQZð @ð @ð @ñô ð ð ˜Y™ˆà(×.Ò.°1Ð.Ñ5Ô5ÐØ! &Ô)×-Ò-°"¸dÐ-ÑCÔCÀAÔFÈÑJˆÝ”iÐ 1°1ÀGÐLÑLÔLˆØ! &Ô)¨U°a©Z¸;ÀvÔ;NÑ,NÑNˆå”)˜K×,Ò,Ñ.Ô.°Ñ>Ô>×DÒDÑFÔFÈÑRˆØ×"Ò" 1 aÑ(Ô(¨=×+>Ò+>Ñ+@Ô+@Ñ@ˆÝÔ  '¨Q¢,×!3Ò!3¸Ð!3Ñ!:Ô!:Ñ;Ô;ˆØ�yŠy˜Ô,Ñ-Ô-¨tÐ3Ð3r<   rº   c                 ó¢   — |                      |                     d¦  «        d¦  «        }|                      ||¦  «        \  }}|| j        z  }||fS )u  Spatial pool with ``pooling_kernel_size Ã— pooling_kernel_size`` cells.

        Args:
            hidden_states: (B, N, D) encoder output.
            position_ids: (B, N, 2) Gemma4-internal ``(x, y)`` coords.
            padding_positions: (B, N) True for padding tokens.

        Returns:
            pooled hidden states (B, N // k^2, D) and validity mask (B, N // k^2).
        rS   rÀ   )Úmasked_fillr`   rG  r3  )r8   r4  r†   rº   Úpooler_masks        r;   rG   zGemma4VisionPooler.forward[  s\   € ð" &×1Ò1Ð2C×2MÒ2MÈbÑ2QÔ2QÐSVÑWÔWˆØ%)×%@Ò%@ÀÐP\Ñ%]Ô%]Ñ"ˆ�{Ø%¨Ô(=Ñ=ˆØ˜kÐ)Ð)r<   )r˜   )rI   rJ   rK   rL   rM   r0   r5   rO   r
   rG  rG   rP   rQ   s   @r;   r/  r/  /  sê   ø€ € € € € ðð ð7ð 7 Cð 7¸cð 7ð 7ð 7ð 7ð 7ð 7ð4à œ<ð4ð  œ,ð4ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð	4ð 4ð 4ð 4ð>*à œ<ð*ð  œ,ð*ð  %œ|ð	*ð
 
ˆuŒ|˜Uœ\Ð)Ô	*ð*ð *ð *ð *ð *ð *ð *ð *r<   r/  c            -       óž  ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dLdeeeeef         f         dedededededededee         dede	de	dedede
de
d e	d!e	d"e	d#ee         d$ed%df,ˆ fd&„Zej        j        dMd(ed)e
d%dfd*„¦   «         Zej        j        d%ee         fd+„¦   «         Zej        j        d%eeef         fd,„¦   «         Zej        j        dNd-e
d%eeef         fd.„¦   «         Zej        j        dOd/e
d%dfd0„¦   «         Zej        j        dOd1e
d%dfd2„¦   «         Zd3ej        d%dfd4„Z	 	 dPd3ej        d5ej        d6ej        d7eeeej        gdf                  d8ee         d%ej        fd9„Z	 	 dPd3eej        eeej        f         f         d:eej                 d;eej                 d%ej        fd<„ZdNd3ej        d=e
d%ej        fd>„Z	 	 dPd3eej        eeej        f         f         d:eej                 d;eej                 d%ej        fd?„Z	 	 	 	 	 	 	 	 dQd3eej        eeej        f         f         d:eej                 d;eej                 dAeeeee         f                  dBe
dCe
dDedEe
dFe
d%eeej                 eej        eej                 f         eeef         f         fdG„Z 	 	 	 dRdAeeee         f         dIe
dJe
d%ee         fdK„Z!ˆ xZ"S )Sr   u#  Gemma4 Vision Encoder.

    The pure encoder from Google's Gemma 4 multimodal model. Custom ViT with 2D
    RoPE, gated MLP, QKV normalization, and 4-norm sandwich blocks.

    When ``standardize=True`` (31B variant), ``std_bias/std_scale`` are applied
    after the soft-token pooler (original contract). Other pool modes don't apply it.
    Output shape depends on ``global_pool``:

    - ``'soft'`` (default): spatial ``kÃ—k`` pooler + ``âˆšD`` scale + optional
      standardization. Output ``(B, num_soft_tokens, embed_dim)``
    - ``'avg'``: masked mean over patch tokens; skips the pooler's ``âˆšD`` scale
      and standardization. Output ``(B, embed_dim)``.
    - ``'none'`` / ``''``: no pool â€” returns raw patch tokens
      ``(B, N, embed_dim)``. Useful for building custom pool heads.
    r™   r—   r˜   Úsoftré   rê   Né   rë   rt   rš   FrÀ   Ú Úimg_sizer›   rœ   Úglobal_poolr�   Údepthrì   ru   rí   r"  rð   rv   rž   r1  Ústandardizerñ   Úproj_drop_rateÚattn_drop_rateÚdrop_path_rater  Úweight_initr$   c                 ó  •‡‡‡‡	‡
‡‡‡‡‡‡‡‡— ||dœŠt          ¦   «                              ¦   «          |dv sJ d|›�¦   «         ‚|| _        ‰x| _        x| _        | _        d| _        d| _        d| _        d| _	        t          |¦  «        | _        || _        ‰| _        ‰pt          t          j        d¬¦  «        Št#          d| j        |‰|d	œ‰¤Ž| _        t'          d‰|d
œ‰¤Ž| _        t+          ||¦  «        Št          j        ˆˆˆˆˆˆˆ
ˆˆˆ	ˆˆfd„t/          |¦  «        D ¦   «         ¦  «        | _        t3          ‰|¬¦  «        | _        |rM|                      dt9          j        ‰fi ‰¤Ž¦  «         |                      dt9          j        ‰fi ‰¤Ž¦  «         nd | _        d | _        tA          | j        ¦  «        Šˆˆfd„t/          |¦  «        D ¦   «         | _!        |dk    rdn|| _"        |dk    r|  #                    d¬¦  «         d S d S )Nr&   )rL  ÚavgÚnonerN  z>global_pool must be one of 'soft', 'avg', 'none' (or ''); got r   ÚNLCFr  r  )r›   rœ   r�   rž   )ru   rv   c                 óL   •— g | ] }t          d‰‰
‰‰‰‰	‰‰‰|         ‰‰d œ‰¤Ž‘Œ!S ))rV   rì   ru   rí   r"  rð   rî   rï   r#  r  rñ   r+   )r!  )re   r�   r  rT  r9   Údprr�   ru   r"  rð   rì   rí   rS  rñ   s     €€€€€€€€€€€€r;   rj   z-Gemma4VitEncoder.__init__.<locals>.<listcomp>Æ  sn   ø€ ð %
ð %
ð %
ð õ ð ØØ#Ø!Ø)Ø"3Ø!Ø(Ø(Ø˜aœ&Ø#Ø$7ðð ð ðð ð%
ð %
ð %
r<   )r0  r1  Ústd_biasÚ	std_scalec                 ó:   •— g | ]}t          ‰‰d |› �¬¦  «        ‘ŒS )úblocks.)Únum_chsÚ	reductionÚmodule©rÓ   )re   r�   Ú_redr�   s     €€r;   rj   z-Gemma4VitEncoder.__init__.<locals>.<listcomp>ì  s0   ø€ ÐqÐqÐqÐ_`�T¨)¸tÈMÐVWÈMÈMÐZÑZÔZÐqÐqÐqr<   ÚskipÚreset©Úneeds_resetr+   )$r/   r0   rP  Únum_featuresÚhead_hidden_sizer�   r¿   Ú
output_fmtÚnum_prefix_tokensÚgrad_checkpointingr   r›   r1  rñ   r   r1   r  r–   Úpatch_embedrs   Ú
rotary_embr   Ú
ModuleListrl   Úblocksr/  Úpoolerr4   r5   r6   r]  r^  r<  Úfeature_infoÚweight_init_modeÚinit_weights)r8   rO  r›   rœ   rP  r�   rQ  rì   ru   rí   r"  rð   rv   rž   r1  rR  rñ   rS  rT  rU  r  rV  r'   r(   re  r9   r\  r:   s        ` `````    ``` `   @@@€r;   r0   zGemma4VitEncoder.__init__„  sÉ  øøøøøøøøøøøøøø€ ð4 ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØÐ9Ð9Ð9Ð9Ø\È[Ð\Ð\ñ :Ô9Ð9à&ˆÔØENÐNˆÔÐN˜DÔ1°D´NàˆÔð  ˆŒØ!"ˆÔØ"'ˆÔõ $ JÑ/Ô/ˆŒØ#6ˆÔ Ø#6ˆÔ àÐE¥­¬¸fÐ!EÑ!EÔ!Eˆ	õ ,ð 
Ø”ØØØ$;ð	
ð 
ð
 ð
ð 
ˆÔõ 2ð 
ØØ!ð
ð 
ð ð
ð 
ˆŒõ (¨¸Ñ>Ô>ˆÝ”mð %
ð %
ð %
ð %
ð %
ð %
ð %
ð %
ð %
ð %
ð %
ð %
ð %
ð %
ð %
õ ˜5‘\”\ð%
ñ %
ô %
ñ ô ˆŒõ& )Ø!Ø 3ð
ñ 
ô 
ˆŒð ð 	"Ø× Ò  ­U¬[¸Ð-IÐ-IÀbÐ-IÐ-IÑJÔJÐJØ× Ò  ­e¬k¸)Ð.JÐ.JÀrÐ.JÐ.JÑKÔKÐKÐKà ˆDŒMØ!ˆDŒNõ
 �4”?Ñ#Ô#ˆØqÐqÐqÐqÐqÕdiÐjoÑdpÔdpÐqÑqÔqˆÔà+6¸&Ò+@Ð+@  ÀkˆÔà˜&Ò Ð Ø×Ò¨%ÐÑ0Ô0Ð0Ð0Ð0ð !Ð r<   TÚmoderi  c                 ó  — |p| j         }|dv sJ ‚| j        �$t          j                             | j        ¦  «         | j        �$t          j                             | j        ¦  «         t          t          ||¬¦  «        | ¦  «         dS )a§  Initialize model weights.

        Args:
            mode: Init mode. '' applies trunc-normal-TF Linear init; 'reset'
                only calls ``reset_parameters`` on each sub-module.
            needs_reset: If True, call ``reset_parameters`` on modules that
                have one (for post-``to_empty()`` reinit). Set to False during
                ``__init__`` since modules already self-initialize there.
        )rN  rg  Nrh  )	ru  r]  r1   ÚinitÚzeros_r^  Úones_r   Úget_init_weights_gemma4_vit©r8   rw  ri  s      r;   rv  zGemma4VitEncoder.init_weightsó  s…   € ð Ð,�tÔ,ˆØ�}Ð$Ð$Ð$Ð$ð Œ=Ð$ÝŒG�NŠN˜4œ=Ñ)Ô)Ð)ØŒ>Ð%ÝŒG�MŠM˜$œ.Ñ)Ô)Ð)åÕ/°À+ÐNÑNÔNÐPTÑUÔUÐUÐUÐUr<   c                 ó   — dhS )Nz$patch_embed.position_embedding_tabler+   rA   s    r;   Úno_weight_decayz Gemma4VitEncoder.no_weight_decay
  s   € à6Ð7Ð7r<   c                 ó   — | j         S ©zHReturn the 2-tuple patch size. For NaFlex dataloader / transform wiring.)r›   rA   s    r;   Úget_patch_sizezGemma4VitEncoder.get_patch_size  s   € ð ŒÐr<   Úcoarsec                 ó(   — t          dddg¬¦  «        S )Nz^patch_embed|^rotary_emb)z^blocks\.(\d+)N)z^pooler|^std_)iŸ† ©Ústemrr  rd  ©r8   rƒ  s     r;   Úgroup_matcherzGemma4VitEncoder.group_matcher  s%   € åØ,Ø-Ð/KÐLð
ñ 
ô 
ð 	
r<   Úenablec                 ó   — || _         d S rD   )rn  ©r8   r‰  s     r;   Úset_grad_checkpointingz'Gemma4VitEncoder.set_grad_checkpointing  s   € à"(ˆÔÐÐr<   Úenabledc                 ól   — |                       ¦   «         D ]}t          |t          ¦  «        r||_        ŒdS )aŸ  Toggle the ``Gemma4ClippableLinear`` clamp ops.

        Gemma4's pretrained E4B checkpoint ships finite clamp buffers on every
        projection, which can saturate and stall gradient flow during classifier
        fine-tuning. Setting ``enabled=False`` skips both the input and output
        clamps in the forward pass (buffers are left untouched so the call is
        reversible and checkpoint-safe).
        N)ÚmodulesrÒ   r    r#   )r8   r�  Úmods      r;   Úset_clamp_enabledz"Gemma4VitEncoder.set_clamp_enabled  s@   € ð —<’<‘>”>ð 	*ð 	*ˆCÝ˜#Õ4Ñ5Ô5ð *Ø")�”øð	*ð 	*r<   rB   c                 óø   — |j         dk    s| j        dk    rdS |j        dd…         \  }}| j        \  }}| j        }||z  ||z  }}||z  dk    s	||z  dk    r"t          d|› d|› d|› d|› d	|› d|› d
�¦  «        ‚dS )zäWhen using the soft-token pooler, raw-image H/W must divide
        by ``patch_size * pooling_kernel_size`` so the pool cell grid is integral.
        Pre-patchified / NaFlex inputs are assumed to be conformant already.
        rË   rL  Nr  r   zImage size (z, z=) must be divisible by (patch_size * pooling_kernel_size) = (z3) when global_pool='soft'. Resize to multiples of (z#), or use global_pool='avg'/'none'.)rb   rP  rW   r›   r1  r×   )	r8   rB   rÞ   rß   r£   r¤   rf   Úcell_hÚcell_ws	            r;   Ú_assert_raw_img_conformantz+Gemma4VitEncoder._assert_raw_img_conformant,  sî   € ð
 Œ6�QŠ;ˆ;˜$Ô*¨fÒ4Ð4ØˆFØŒw�r�s�sŒ|‰ˆˆ1Ø”‰ˆˆBØÔ$ˆØ˜a™  a¡�ˆØˆv‰:˜Š?ˆ?˜a &™j¨Ašo˜oÝða˜qð að a Að að aØ9?ðað aØCIðað aà+1ðað aà5;ðað að añô ð ð .˜or<   r†   rº   Úblock_callbackÚmax_block_indexc                 ó@  — |j         dd…         \  }}|                      ||¦  «        \  }}	d}
|                     ¦   «         rUt          j        |dd||j        |j        ¬¦  «        }
|
                     |dd…dddd…f         t          d¦  «        ¦  «         |€| j	        }n| j	        d|dz   …         }t          |¦  «        D ][\  }}| j        r2t          j                             ¦   «         st          ||||	|
¦  «        }n ||||	|
¬¦  «        }|� |||¦  «         Œ\|S )z?RoPE + transformer-block pipeline over already-embedded tokens.NrT   r   r&   z-infr,  )rW   rp  rÙ   r5   Úzerosr'   r(   Úmasked_fill_r@   rr  Ú	enumeratern  r  Úis_scriptingr   )r8   rB   r†   rº   r–  r—  rÜ   rà   rÿ   r   r  rr  r�   Úblks                 r;   Ú_encodezGemma4VitEncoder._encode>  sL  € ð Œw�r˜�rŒ{‰ˆˆ1Ø!Ÿ_š_¨Q°Ñ=Ô=Ñˆ�(à,0ˆ	Ø× Ò Ñ"Ô"ð 	Wåœ A q¨!¨Q°q´xÀqÄwÐOÑOÔOˆIØ×"Ò"Ð#4°Q°Q°Q¸¸dÀAÀAÀAÐ5EÔ#FÍÈfÉÌÑVÔVÐVàÐ"Ø”[ˆFˆFà”[Ð!5 /°AÑ"5Ð!5Ô6ˆFå Ñ'Ô'ð 	%ð 	%‰FˆAˆsØÔ&ð D­u¬y×/EÒ/EÑ/GÔ/Gð DÝ˜s A x°¸9ÑEÔE��à�C˜˜8 X¸ÐCÑCÔC�ØÐ)Ø�˜q !Ñ$Ô$Ð$øàˆr<   r·   r¸   c                 óÊ   — |                       t          |t          ¦  «        s|n|d         ¦  «         |                      |||¦  «        \  }}}|                      |||¦  «        S )z9Raw patch tokens pre-pool. Returns ``(B, N, embed_dim)``.rÊ   )r•  rÒ   rÓ   ro  rž  )r8   rB   r·   r¸   r†   rº   s         r;   Úforward_featuresz!Gemma4VitEncoder.forward_features_  se   € ð 	×'Ò'µ¸A½tÑ1DÔ1DÐ(V¨¨È!ÈIÌ,ÑWÔWÐWØ-1×-=Ò-=¸aÀÈkÑ-ZÔ-ZÑ*ˆˆ<Ð*Ø�|Š|˜A˜|Ð->Ñ?Ô?Ð?r<   Ú
pre_logitsc                 ó    — t          d¦  «        ‚)Nz;Gemma4VitEncoder does not support classification use cases.)ÚNotImplementedError)r8   rB   r¡  s      r;   Úforward_headzGemma4VitEncoder.forward_headj  s   € Ý!Ð"_Ñ`Ô`Ð`r<   c                 ó�  — |                       t          |t          ¦  «        s|n|d         ¦  «         |                      |||¦  «        \  }}}|                      |||¦  «        }| j        dk    r4|                      |||¦  «        \  }}| j        �|| j        z
  | j        z  }n¢| j        dk    r—| 	                    ¦   «         rm| 
                    |                     d¦  «        d¦  «        }|                     d¬¦  «        |                      dd	¬
¦  «                             d¬¦  «        z  }n|                     d¬¦  «        }|S )u  Encode + apply the configured pool.

        Output shape depends on ``self.global_pool``:
          ``'soft'`` â†’ ``(B, num_soft_tokens, D)``
          ``'avg'``  â†’ ``(B, D)``
          ``'none'`` â†’ ``(B, N, D)`` (raw patch tokens, identical to forward_features)
        rÊ   rL  NrX  rS   rÀ   r   rU   Tr7  r¼   )r•  rÒ   rÓ   ro  rž  rP  rs  r]  r^  rÙ   rI  r`   rÄ   rE   Úmean)r8   rB   r·   r¸   r†   rº   rÝ   s          r;   rG   zGemma4VitEncoder.forwardm  sE  € ð 	×'Ò'µ¸A½tÑ1DÔ1DÐ(V¨¨È!ÈIÌ,ÑWÔWÐWØ-1×-=Ò-=¸aÀÈkÑ-ZÔ-ZÑ*ˆˆ<Ð*Ø�LŠL˜˜LÐ*;Ñ<Ô<ˆàÔ˜vÒ%Ð%Ø—;’;˜q ,Ð0AÑBÔB‰DˆAˆqàŒ}Ð(Ø˜œÑ&¨$¬.Ñ8�øàÔ Ò&Ð&à ×$Ò$Ñ&Ô&ð "Ø—M’MÐ"3×"=Ò"=¸bÑ"AÔ"AÀ3ÑGÔG�Ø—E’E˜a�E‘L”LÐ%6Ð$6×#;Ò#;ÀÈ4Ð#;Ñ#PÔ#P×#VÒ#VÐ[\Ð#VÑ#]Ô#]Ñ]��à—F’F˜q�F‘M”M�ð ˆr<   ÚNCHWÚindicesÚnormÚ
stop_earlyrl  Úintermediates_onlyÚoutput_dictc
                 ód  ‡‡‡‡‡— |dv sJ ‚|dk    }
t          t          | j        ¦  «        |¦  «        \  Š}t          |t          ¦  «        s|n|d         }|j        }|                      |¦  «         |                      |||¦  «        \  }}}g Šdt          dt          j
        ddfˆˆfd„}d}|r t          j                             ¦   «         s|}|                      |||||¬	¦  «        }|
rª|d
k    rt          d¦  «        ‚|j        d         Št          |d                              ¦   «                              ¦   «         ¦  «        dz   Št          |d                              ¦   «                              ¦   «         ¦  «        dz   Šˆˆˆfd„‰D ¦   «         Š|	rd‰i}|s||d<   | |d<   |S |r‰S |‰fS )a  Forward features returning intermediates.

        Args:
            x: Input tensor ``(B, C, H, W)`` or NaFlex pre-patchified tensor/dict.
            patch_coord: ``(B, N, 2)`` patch coords in NaFlex external (y, x) convention.
            patch_valid: ``(B, N)`` boolean mask, True for valid tokens.
            indices: Block indices to return intermediates for.
            norm: Not used (no final norm in Gemma4 encoder).
            stop_early: Stop iterating after last needed intermediate.
            output_fmt: Output format ('NCHW' or 'NLC'). NCHW requires a fixed full grid.
            intermediates_only: Only return intermediate features.
            output_dict: Return a dict matching the NaFlexVit intermediates contract:
                'image_intermediates' (+ 'image_features' unless ``intermediates_only``) and
                'patch_valid' aligned with the token sequence (all-True for dense image input;
                gemma4 has no patch dropout, so it always equals the input validity).
        )r§  rZ  r§  rÊ   r�   Úyr$   Nc                 ó>   •— | ‰v r‰                      |¦  «         d S d S rD   )rŒ   )r�   r®  ÚintermediatesÚtake_indicess     €€r;   Ú_cbz3Gemma4VitEncoder.forward_intermediates.<locals>._cb¶  s0   ø€ Ø�LÐ Ð Ø×$Ò$ QÑ'Ô'Ð'Ð'Ð'ð !Ð r<   )r–  r—  rË   z:output_fmt='NCHW' requires a raw image (B, C, H, W) input.r   r6  r   r;  c                 ó�   •— g | ]B}|                      ‰‰‰d ¦  «                             dddd¦  «                             ¦   «         ‘ŒCS )rS   r   r˜   r   rT   )rÕ   rÃ   Ú
contiguous)re   r®  rÜ   rª   r«   s     €€€r;   rj   z:Gemma4VitEncoder.forward_intermediates.<locals>.<listcomp>Í  sL   ø€ ÐnÐnÐnÐ[\˜QŸYšY q¨"¨b°"Ñ5Ô5×=Ò=¸aÀÀAÀqÑIÔI×TÒTÑVÔVÐnÐnÐnr<   Úimage_intermediatesÚimage_featuresr¸   )r   Úlenrr  rÒ   rÓ   rb   r•  ro  rM   r5   rO   r  rœ  rž  r×   rW   r<  Úitem)r8   rB   r·   r¸   r¨  r©  rª  rl  r«  r¬  rÕ   Ú	max_indexÚrawÚraw_input_ndimr†   rº   r²  r—  Úresult_dictrÜ   r°  rª   r«   r±  s                      @@@@@r;   Úforward_intermediatesz&Gemma4VitEncoder.forward_intermediates�  s(  øøøøø€ ð8 ˜_Ð,Ð,Ð,Ð,Ø Ò&ˆÝ"6µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�iå! !¥TÑ*Ô*Ð<ˆaˆa°°)´ˆØœˆØ×'Ò'¨Ñ,Ô,Ð,Ø-1×-=Ò-=¸aÀÈkÑ-ZÔ-ZÑ*ˆˆ<Ð*à,.ˆð	(•3ð 	(�5œ<ð 	(¨Dð 	(ð 	(ð 	(ð 	(ð 	(ð 	(ð 	(ð ˆØð 	(�eœi×4Ò4Ñ6Ô6ð 	(Ø'ˆOà�LŠLØØØØØ+ð ñ 
ô 
ˆð ð 	oØ Ò"Ð"Ý Ð!]Ñ^Ô^Ð^àÔ" 1Ô%ˆAÝ�\ &Ô)×-Ò-Ñ/Ô/×4Ò4Ñ6Ô6Ñ7Ô7¸!Ñ;ˆBÝ�\ &Ô)×-Ò-Ñ/Ô/×4Ò4Ñ6Ô6Ñ7Ô7¸!Ñ;ˆBØnÐnÐnÐnÐnÐnÐ`mÐnÑnÔnˆMàð 	Ø+@À-Ð*PˆKØ%ð 2Ø01�Ð,Ñ-Ø*;Ð);ˆK˜Ñ&ØÐàð 	!Ø Ð à�-ÐÐr<   r   Ú
prune_normÚ
prune_headc                 ó~   — t          t          | j        ¦  «        |¦  «        \  }}| j        d|dz   …         | _        |S )a  Prune layers not required for specified intermediates.

        ``prune_head`` is accepted for API compatibility; the encoder has no
        classifier head so it's a no-op here. (The wrapping ``Gemma4VitClassifier``
        handles its own head pruning.)
        Nr   )r   r·  rr  )r8   r¨  r¾  r¿  r±  r¹  s         r;   Úprune_intermediate_layersz*Gemma4VitEncoder.prune_intermediate_layersÛ  s>   € õ #7µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�iØ”k . 9¨q¡= .Ô1ˆŒØÐr<   )r™   r—   r˜   rL  r™   r—   ré   rê   NrM  rë   rt   rš   r˜   FFrÀ   rÀ   rÀ   NrN  NN©rN  T©F©Trä   )NNNFFr§  FF©r   FT)#rI   rJ   rK   rL   r   rM   r
   rå   r   r@   rN   r   r0   r5   r  Úignorerv  r	   r  r‚  r   r   rˆ  rŒ  r‘  rO   r•  rž  r   r¤  rG   r   r½  rÁ  rP   rQ   s   @r;   r   r   r  s6  ø€ € € € € ðð ð& 58Ø ØØ%Ø ØØØØ*.Ø%)Ø"Ø %Ø+0Ø'(Ø %Ø(-Ø$'Ø$'Ø$'Ø,0Ø!ØØð1m1ð m1à˜C  s¨C x¤Ð0Ô1ðm1ð ðm1ð ð	m1ð
 ðm1ð ðm1ð ðm1ð ðm1ð ðm1ð # 3œ-ðm1ð  #ðm1ð ðm1ð ðm1ð &)ðm1ð "%ðm1ð  ð!m1ð" "&ð#m1ð$ "ð%m1ð& "ð'm1ð( "ð)m1ð*   Ô)ð+m1ð, ð-m1ð2 
ð3m1ð m1ð m1ð m1ð m1ð m1ð^ „YÔðVð V ð V¸ð VÈð Vð Vð Vñ ÔðVð, „YÔð8  S¤ð 8ð 8ð 8ñ Ôð8ð „YÔð  c¨3 h¤ð ð ð ñ Ôðð „YÔð
ð 
 Dð 
°T¸#¸s¸(´^ð 
ð 
ð 
ñ Ôð
ð „YÔð)ð )¨Tð )¸Tð )ð )ð )ñ Ôð)ð „YÔð*ð *¨ð *¸ð *ð *ð *ñ Ôð*ð¨E¬Lð ¸Tð ð ð ð ð. MQØ-1ðð àŒ|ðð  œ,ðð  %œ|ð	ð
 % X¨s°E´LÐ.AÀ4Ð.GÔ%HÔIðð & cœ]ðð 
Œðð ð ð ðH 37Ø26ð		@ð 	@à�U”\ 4¨¨U¬\Ð(9Ô#:Ð:Ô;ð	@ð " %¤,Ô/ð	@ð " %¤,Ô/ð		@ð
 
Œð	@ð 	@ð 	@ð 	@ðað a˜eœlð a¸ð aÈÌð að að að að 37Ø26ð	 ð  à�U”\ 4¨¨U¬\Ð(9Ô#:Ð:Ô;ð ð " %¤,Ô/ð ð " %¤,Ô/ð	 ð
 
Œð ð  ð  ð  ðJ 37Ø26Ø7;ØØ$Ø$Ø',Ø %ðJ ð J à�U”\ 4¨¨U¬\Ð(9Ô#:Ð:Ô;ðJ ð " %¤,Ô/ðJ ð " %¤,Ô/ð	J ð
 ˜e C¨¨c¬ NÔ3Ô4ðJ ð ðJ ð ðJ ð ðJ ð !%ðJ ð ðJ ð 
ˆt�E”LÔ! 5¨¬°t¸E¼LÔ7IÐ)IÔ#JÈDÐQTÐVYÐQYÌNÐZÔ	[ðJ ð J ð J ð J ð\ ./Ø$Ø#ð	ð à˜3  S¤	˜>Ô*ðð ðð ð	ð
 
ˆcŒðð ð ð ð ð ð ð r<   c            5       óz  ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dLdeeeeef         f         dedededededededededee         dede	de	d ed!ed"e
d#e
d$e
d%e	d&e	d'e	d(e	d)ee         d*ed+d
f4ˆ fd,„Zej        j        dMd-ed.e
d+d
fd/„¦   «         Zej        j        d+ee         fd0„¦   «         Zej        j        dNd1e
d+eeef         fd2„¦   «         Zej        j        dOd3e
d+d
fd4„¦   «         Zej        j        dOd5e
d+d
fd6„¦   «         Zej        j        d+eeef         fd7„¦   «         Zej        j        d+ej        fd8„¦   «         ZdPdedee         d+d
fd9„Z	 	 dQd:eej        eeej        f         f         d;eej                 d<eej                 d+ej        fd=„Z	 	 dRd:ej        d<eej                 d>e
d+ej        fd?„Z	 	 dQd:eej        eeej        f         f         d;eej                 d<eej                 d+ej        fd@„Z 	 	 	 	 	 	 	 dSd:eej        eeej        f         f         d;eej                 d<eej                 dBeeee!e         f                  dCe
dDe
dEedFe
d+ee!ej                 eej        e!ej                 f         f         fdG„Z"	 	 	 dTdBeee!e         f         dIe
dJe
d+e!e         fdK„Z#ˆ xZ$S )Ur   uú  Classification wrapper around ``Gemma4VitEncoder``.

    Holds:
        - ``encoder``: a ``Gemma4VitEncoder``. By default constructed with
          ``global_pool=''`` so the classifier does its own masked-mean pool
          over raw patch tokens in ``forward_head``. Pass
          ``encoder_pool='soft'`` to turn on the native VLM spatial ``kÃ—k``
          soft-token pool inside the encoder; ``forward_features`` then
          returns ``(B, num_soft_tokens, D)`` already pooled (plus the ``âˆšD``
          scale and, for the 31B variant, ``std_bias/std_scale``).
        - ``norm``: optional ``RmsNorm`` after pooling.
        - ``head``: linear classifier.

    Input/output contract matches timm convention: ``forward_features`` returns
    pre-head features (pre-pool ``(B, N, D)`` patch tokens by default, or
    ``(B, num_soft_tokens, D)`` post-soft-pool when ``encoder_pool='soft'``);
    ``forward_head`` does classifier pool + norm + head; ``forward`` =
    ``forward_features`` + ``forward_head``.
    r™   r—   r˜   éè  rX  rN  ré   rê   NrM  rë   rt   rš   FTrÀ   rO  r›   rœ   r¿   rP  Úencoder_poolr�   rQ  rì   ru   rí   r"  rð   rv   rž   r1  rR  rñ   Ú
final_normÚ	drop_raterS  rT  rU  r  rV  r$   c                 óÄ  •— ||dœ}t          ¦   «                              ¦   «          |dv sJ d|›d�¦   «         ‚|dv sJ d|›d�¦   «         ‚|| _        || _        || _        t          d!i d|“d	|“d
|“d|“d|“d|“d|	“d|
“d|“d|“d|“d|“d|“d|“d|“d|“d|“d|“d|“d|“d|“|¤Ž| _        |rt          |f|ddœ|¤Žnt          j	        ¦   «         | _
        t          j        |¦  «        | _        |dk    rt          j        ||fi |¤Žnt          j	        ¦   «         | _        | j        j        x| _        | _        | j        j        | _        | j        j        | _        | j        j        | _        | j        j        | _        |d k    rVt+          | j        t          j        ¦  «        r9| j        j        �/t          j                             | j        j        ¦  «         d S d S d S d S )"Nr&   ©rX  rY  rN  úBGemma4VitClassifier global_pool must be 'avg', 'none', or '' (got z>); use Gemma4VitEncoder directly for 'soft' VLM-style pooling.)rN  rY  rL  zDGemma4VitClassifier encoder_pool must be '', 'none', or 'soft' (got z).rO  r›   rœ   rP  r�   rQ  rì   ru   rí   r"  rð   rv   rž   r1  rR  rñ   rS  rT  rU  r  rV  Fró   r   rf  r+   )r/   r0   r¿   rP  rÉ  r   Úencoderr   r1   r*  r©  rþ   Ú	head_dropr2   Úheadrj  rk  r�   r›   rt  ru  rÒ   r)   ry  rz  )r8   rO  r›   rœ   r¿   rP  rÉ  r�   rQ  rì   ru   rí   r"  rð   rv   rž   r1  rR  rñ   rÊ  rË  rS  rT  rU  r  rV  r'   r(   r9   r:   s                                €r;   r0   zGemma4VitClassifier.__init__  sÞ  ø€ ð< ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØÐ1Ð1Ð1Ð1ðKÐQ\ð Kð Kð Kñ 2Ô1Ð1ð Ð3Ð3Ð3Ð3ØeÐS_ÐeÐeÐeñ 4Ô3Ð3à&ˆÔØ&ˆÔØ(ˆÔõ (ð 
ð 
ð 
Ø�Xð
à!�zð
ð �Xð
ð %˜ð	
ð
  �ið
ð �%ð
ð  �ið
ð �Xð
ð &˜ð
ð 0Ð/ð
ð �Xð
ð "�zð
ð %<Ð$;ð
ð !4Ð 3ð
ð $˜ð
ð  !4Ð 3ð!
ð" *˜>ð#
ð$ *˜>ð%
ð& *˜>ð'
ð(  �ið)
ð* $˜Øð-
ð 
ˆŒð0 MWÐi•G˜IÐH¨8¸EÐHÐHÀRÐHÐHÐHÕ\^Ô\gÑ\iÔ\iˆŒ	Ýœ IÑ.Ô.ˆŒØ?JÈQº¸•B”I˜i¨Ð;Ð;¸Ð;Ð;Ð;ÕTVÔT_ÑTaÔTaˆŒ	ð 59´LÔ4MÐMˆÔ˜DÔ1ØœÔ/ˆŒØœ,Ô1ˆŒØ œLÔ5ˆÔà $¤Ô =ˆÔØ˜&Ò Ð å˜$œ)¥R¤YÑ/Ô/ð /°D´I´NÐ4NÝ”—’˜tœyœ~Ñ.Ô.Ð.Ð.Ð.ð !Ð ð/ð /Ð4NÐ4Nr<   rw  ri  c                 óî   — | j                              ||¬¦  «         t          | j        t          j        ¦  «        r7| j        j        �-t          j                             | j        j        ¦  «         dS dS dS )z>Init encoder + classifier-specific pieces (head bias to zero).©rw  ri  N)	rÏ  rv  rÒ   rÑ  r1   r2   r)   ry  rz  r}  s      r;   rv  z Gemma4VitClassifier.init_weightsT  sl   € ð 	Œ×!Ò! t¸Ð!ÑEÔEÐEÝ�d”i¥¤Ñ+Ô+ð 	+°´	´Ð0JÝŒG�NŠN˜4œ9œ>Ñ*Ô*Ð*Ð*Ð*ð	+ð 	+Ð0JÐ0Jr<   c                 óH   — d„ | j                              ¦   «         D ¦   «         S )Nc                 ó   — h | ]}d |› �’ŒS ©zencoder.r+   )re   rf   s     r;   ú	<setcomp>z6Gemma4VitClassifier.no_weight_decay.<locals>.<setcomp>]  s   € ÐGÐGÐG 1�˜1��ÐGÐGÐGr<   )rÏ  r  rA   s    r;   r  z#Gemma4VitClassifier.no_weight_decay[  s$   € àGÐG¨¬×(DÒ(DÑ(FÔ(FÐGÑGÔGÐGr<   rƒ  c                 ó&   — t          ddg¬¦  «        S )Nz*^encoder\.patch_embed|^encoder\.rotary_emb)z^encoder\.blocks\.(\d+)Nr…  rd  r‡  s     r;   rˆ  z!Gemma4VitClassifier.group_matcher_  s$   € õ Ø>Ø6Ð7ð
ñ 
ô 
ð 	
r<   r‰  c                 ó:   — | j                              |¦  «         d S rD   )rÏ  rŒ  r‹  s     r;   rŒ  z*Gemma4VitClassifier.set_grad_checkpointingg  s   € àŒ×+Ò+¨FÑ3Ô3Ð3Ð3Ð3r<   r�  c                 ó:   — | j                              |¦  «         d S rD   )rÏ  r‘  )r8   r�  s     r;   r‘  z%Gemma4VitClassifier.set_clamp_enabledk  s   € àŒ×&Ò& wÑ/Ô/Ð/Ð/Ð/r<   c                 ó4   — | j                              ¦   «         S r�  )rÏ  r‚  rA   s    r;   r‚  z"Gemma4VitClassifier.get_patch_sizeo  s   € ð Œ|×*Ò*Ñ,Ô,Ð,r<   c                 ó   — | j         S rD   )rÑ  rA   s    r;   Úget_classifierz"Gemma4VitClassifier.get_classifiert  s
   € àŒyÐr<   c                 óÂ  — t          | ¦  «        }|| _        |�|dv sJ d|›d�¦   «         ‚|| _        |dk    rt          j        | j        |fi |¤Žnt          j        ¦   «         | _        | j                             | j	        ¦  «         t          | j        t          j        ¦  «        r7| j        j        �-t          j                             | j        j        ¦  «         d S d S d S )NrÍ  rÎ  ú)r   )r   r¿   rP  r1   r2   r�   r*  rÑ  Útrainr  rÒ   r)   ry  rz  )r8   r¿   rP  r9   s       r;   Úreset_classifierz$Gemma4VitClassifier.reset_classifierx  sæ   € Ý˜dÑ#Ô#ˆØ&ˆÔØÐ"ØÐ"5Ð5Ð5Ð5ØeÐU`ÐeÐeÐeñ 6Ô5Ð5à*ˆDÔØDOÐRSÂOÀO•B”I˜dœn¨kÐ@Ð@¸RÐ@Ð@Ð@ÕY[ÔYdÑYfÔYfˆŒ	ØŒ	�Š˜œÑ&Ô&Ð&Ý�d”i¥¤Ñ+Ô+ð 	+°´	´Ð0JÝŒG�NŠN˜4œ9œ>Ñ*Ô*Ð*Ð*Ð*ð	+ð 	+Ð0JÐ0Jr<   rB   r·   r¸   c                 ó2   — |                       |||¬¦  «        S )u]  Pre-head features.

        Shape depends on ``encoder_pool``:
          ``''`` / ``'none'``: raw patch tokens ``(B, N, embed_dim)``.
          ``'soft'``: spatial ``kÃ—k`` soft-token pool output
              ``(B, num_soft_tokens, embed_dim)`` (with ``âˆšD`` scale + optional
              ``std_bias/std_scale`` baked in by the encoder).
        ©r·   r¸   )rÏ  )r8   rB   r·   r¸   s       r;   r   z$Gemma4VitClassifier.forward_features„  s   € ð �|Š|˜A¨;ÀKˆ|ÑPÔPÐPr<   r¡  c                 ó¾  — | j         dk    r�| j        dk    s|€|                     d¬¦  «        }nl|                     |                      d¦  «        d¦  «        }|                     d¬¦  «        |                     dd¬	¦  «                             d¬
¦  «        z  }|                      |¦  «        }|                      |¦  «        }|r|n|  	                    |¦  «        S )uì  Pool (if configured) â†’ norm â†’ head_drop â†’ head.

        Args:
            x: pre-head features from ``forward_features``.
            patch_valid: ``(B, N)`` valid mask for masked mean over raw patch
                tokens. Ignored when ``encoder_pool='soft'`` (the soft pooler
                has already collapsed padding tokens, and the new token count
                no longer aligns with ``patch_valid``).
            pre_logits: If True, return pre-classifier features.
        rX  rL  Nr   rU   rS   rÀ   Tr7  r¼   )
rP  rÉ  r¦  rI  r`   rÄ   rE   r©  rÐ  rÑ  )r8   rB   r¸   r¡  s       r;   r¤  z Gemma4VitClassifier.forward_head”  sÌ   € ð  Ô˜uÒ$Ð$ØÔ  FÒ*Ð*¨kÐ.AØ—F’F˜q�F‘M”M��à—M’M K <×":Ò":¸2Ñ">Ô">ÀÑDÔD�Ø—E’E˜a�E‘L”L ;§?¢?°qÀ$ ?Ñ#GÔ#G×#MÒ#MÐRSÐ#MÑ#TÔ#TÑT�à�IŠI�a‰LŒLˆØ�NŠN˜1ÑÔˆØÐ0ˆqˆq D§I¢I¨a¡L¤LÐ0r<   c                 ó\  — t          |t          ¦  «        r4|                     d|¦  «        }|                     d|¦  «        }|d         }|€3|�1|dk                         d¬¦  «        }|                     ¦   «         r| }|                      |||¬¦  «        }|                      ||¬¦  «        S )Nr·   r¸   rÊ   rS   rU   rã  )r¸   )rÒ   rÓ   rÔ   rØ   rÙ   r   r¤  )r8   rB   r·   r¸   rã   Úfeatss         r;   rG   zGemma4VitClassifier.forward¯  sµ   € õ �a�ÑÔð 	ØŸ%š% ¨{Ñ;Ô;ˆKØŸ%š% ¨{Ñ;Ô;ˆKØ�)”ˆAØÐ ;Ð#:Ø# rÒ)×.Ò.°2Ð.Ñ6Ô6ˆHØ�|Š|‰~Œ~ð (Ø'˜i�à×%Ò% a°[ÈkÐ%ÑZÔZˆØ× Ò  °KÐ Ñ@Ô@Ð@r<   r§  r¨  r©  rª  rl  r«  c	           
      óF   — | j                              ||||||||¬¦  «        S )N)r·   r¸   r¨  r©  rª  rl  r«  )rÏ  r½  )	r8   rB   r·   r¸   r¨  r©  rª  rl  r«  s	            r;   r½  z)Gemma4VitClassifier.forward_intermediatesÈ  s;   € ð Œ|×1Ò1ØØ#Ø#ØØØ!Ø!Ø1ð 2ñ 	
ô 	
ð 		
r<   r   r¾  r¿  c                 ón   — | j                              ||d¬¦  «        }|r|                      d¦  «         |S )NF)r¾  r¿  r   )rÏ  rÁ  rá  )r8   r¨  r¾  r¿  r±  s        r;   rÁ  z-Gemma4VitClassifier.prune_intermediate_layersÞ  sD   € ð ”|×=Ò=¸gÐR\ÐinÐ=ÑoÔoˆØð 	%Ø×!Ò! !Ñ$Ô$Ð$ØÐr<   )r™   r—   r˜   rÈ  rX  rN  r™   r—   ré   rê   NrM  rë   rt   rš   r˜   FFTrÀ   rÀ   rÀ   rÀ   NrN  NNrÂ  rÃ  rÄ  rD   rä   )NF)NNNFFr§  FrÅ  )%rI   rJ   rK   rL   r   rM   r
   rå   r   r@   rN   r   r0   r5   r  rÆ  rv  r	   r  r   r   rˆ  rŒ  r‘  r‚  r1   ÚModulerÝ  rá  rO   r   r¤  rG   r   r½  rÁ  rP   rQ   s   @r;   r   r   ì  s  ø€ € € € € ðð ð, 58Ø ØØ#Ø$Ø "Ø ØØØØ*.Ø%)Ø"Ø %Ø+0Ø'(Ø %Ø(-Ø#Ø"Ø$'Ø$'Ø$'Ø,0Ø!ØØð9Q/ð Q/à˜C  s¨C x¤Ð0Ô1ðQ/ð ðQ/ð ð	Q/ð
 ðQ/ð ðQ/ð ðQ/ð ðQ/ð ðQ/ð ðQ/ð ðQ/ð # 3œ-ðQ/ð  #ðQ/ð ðQ/ð ðQ/ð  &)ð!Q/ð" "%ð#Q/ð$ ð%Q/ð& "&ð'Q/ð( ð)Q/ð* ð+Q/ð, "ð-Q/ð. "ð/Q/ð0 "ð1Q/ð2   Ô)ð3Q/ð4 ð5Q/ð: 
ð;Q/ð Q/ð Q/ð Q/ð Q/ð Q/ðf „YÔð+ð + ð +¸ð +Èð +ð +ð +ñ Ôð+ð „YÔðH  S¤ð Hð Hð Hñ ÔðHð „YÔð
ð 
 Dð 
°T¸#¸s¸(´^ð 
ð 
ð 
ñ Ôð
ð „YÔð4ð 4¨Tð 4¸Tð 4ð 4ð 4ñ Ôð4ð „YÔð0ð 0¨ð 0¸ð 0ð 0ð 0ñ Ôð0ð „YÔð-  c¨3 h¤ð -ð -ð -ñ Ôð-ð „YÔð ¤	ð ð ð ñ Ôðð
+ð 
+¨Cð 
+¸hÀs¼mð 
+ÐW[ð 
+ð 
+ð 
+ð 
+ð 37Ø26ð	Qð Qà�U”\ 4¨¨U¬\Ð(9Ô#:Ð:Ô;ðQð " %¤,Ô/ðQð " %¤,Ô/ð	Qð
 
ŒðQð Qð Qð Qð& 37Ø$ð	1ð 1àŒ|ð1ð " %¤,Ô/ð1ð ð	1ð
 
Œð1ð 1ð 1ð 1ð< 37Ø26ð	Að Aà�U”\ 4¨¨U¬\Ð(9Ô#:Ð:Ô;ðAð " %¤,Ô/ðAð " %¤,Ô/ð	Að
 
ŒðAð Að Að Að8 37Ø26Ø7;ØØ$Ø$Ø',ð
ð 
à�U”\ 4¨¨U¬\Ð(9Ô#:Ð:Ô;ð
ð " %¤,Ô/ð
ð " %¤,Ô/ð	
ð
 ˜e C¨¨c¬ NÔ3Ô4ð
ð ð
ð ð
ð ð
ð !%ð
ð 
ˆt�E”LÔ! 5¨¬°t¸E¼LÔ7IÐ)IÔ#JÐJÔ	Kð
ð 
ð 
ð 
ð0 ./Ø$Ø#ð		ð 	à˜3  S¤	˜>Ô*ð	ð ð	ð ð		ð
 
ˆcŒð	ð 	ð 	ð 	ð 	ð 	ð 	ð 	r<   rN  Trc  Únameri  c                 ód  — t          | t          j        ¦  «        rEt          | j        d¬¦  «         | j        �&t          j                             | j        ¦  «         dS dS t          | d¦  «        r|  	                    ¦   «          dS |r&t          | d¦  «        r|  
                    ¦   «          dS dS dS )a  Per-module init for Gemma4Vit (trunc-normal-TF for Linear weights).

    Args:
        module: Module to initialize.
        name: Dotted module name (from ``named_apply``).
        needs_reset: If True, call ``reset_parameters`` on modules that define one.
    r¦   r§   Nrv  r7   )rÒ   r1   r2   r   rÚ   r)   ry  rz  Úhasattrrv  r7   )rc  rê  ri  s      r;   Úinit_weights_gemma4_vitrí  ê  sÉ   € õ �&�"œ)Ñ$Ô$ð "Ý˜œ¨DÐ1Ñ1Ô1Ð1ØŒ;Ð"ÝŒG�NŠN˜6œ;Ñ'Ô'Ð'Ð'Ð'ð #Ð"å	�˜Ñ	(Ô	(ð "Ø×ÒÑÔÐÐÐØ	ð "� Ð);Ñ<Ô<ð "Ø×ÒÑ!Ô!Ð!Ð!Ð!ð"ð "ð "ð "r<   rw  c                 ó.   — t          t          |¬¦  «        S )Nrh  )r   rí  rÓ  s     r;   r|  r|  ü  s   € åÕ*¸ÐDÑDÔDÐDr<   Ú
state_dictÚmodelc                 ó2  — i }d}|                       ¦   «         D ]ý\  }}d}|D ]}|                     |¦  «        r|} nŒ|€|                     d¦  «        r|||<   ŒB|t          |¦  «        d…         }d|v rŒ^|                     dd¦  «        }|                     dd¦  «        }|                     d	d
¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|||<   Œþ|S )u¿  Convert HuggingFace Gemma4 vision encoder weights â†’ ``Gemma4VitEncoder`` keys.

    Pure key remapping â€” no value transforms. The encoder consumes patches in
    HF's native C-P-P flat layout (via ``batch_patchify(channels_last=False)``
    in ``Gemma4PatchEmbed``), so ``input_proj.weight`` passes through unchanged.
    ``Gemma4ClippableLinear`` preserves HF's ``.linear.weight`` + clamp-buffer
    structure, so those pass through too.
    )zmodel.vision_tower.zmodel.vision_model.zvision_model.zvision_tower.N)úpatch_embed.r`  Ústd_zpooler.zrotary_emb.rp  zpatch_embedder.rò  zencoder.layers.r`  z.input_layernorm.z.norm1.z.post_attention_layernorm.z.norm2.z.pre_feedforward_layernorm.z.norm3.z.post_feedforward_layernorm.z.norm4.z.self_attn.z.attn.)ÚitemsÚ
startswithr·  Úreplace)	rï  rð  Úout_dictÚhf_prefixesrf   r  Úmatched_prefixÚprefixÚnew_ks	            r;   Úcheckpoint_filter_fn_encoderrü    sP  € ð €HØb€Kà× Ò Ñ"Ô"ð ð ‰ˆˆ1ØˆØ!ð 	ð 	ˆFØ�|Š|˜FÑ#Ô#ð Ø!'�Ø�ðð Ð!à�|Š|ÐYÑZÔZð  Ø�˜‘Øà•#�nÑ%Ô%Ð&Ð&Ô'ˆð ˜5Ð Ð Øà—’Ð/°Ñ@Ô@ˆØ—’Ð/°Ñ;Ô;ˆØ—’Ð1°9Ñ=Ô=ˆØ—’Ð:¸IÑFÔFˆØ—’Ð;¸YÑGÔGˆØ—’Ð<¸iÑHÔHˆØ—’˜m¨XÑ6Ô6ˆàˆ�‰ˆà€Or<   c                 ó  ‡‡— dŠˆfd„|                       ¦   «         D ¦   «         Šˆfd„|                       ¦   «         D ¦   «         }t          ||j        ¦  «        }d„ |                      ¦   «         D ¦   «         }|                     ‰¦  «         |S )uZ  Convert HF or timm-encoder state dict â†’ ``Gemma4VitClassifier`` keys.

    Runs the encoder filter, then prefixes every encoder-owned key with
    ``encoder.``. Top-level keys that belong to the classifier (``norm.*``,
    ``head.*``) pass through unchanged â€” they're absent from HF checkpoints, so
    the model's own init values stand.
    )znorm.zhead.c                 óp   •— i | ]2\  }}|                      ‰¦  «        s|                      d ¦  «        ¯/||“Œ3S rÖ  )rõ  )re   rf   r  Úclassifier_local_prefixess      €r;   ú
<dictcomp>z3checkpoint_filter_fn_classifier.<locals>.<dictcomp>>  s]   ø€ ð `ð `ð `¡  AØŸ<š<Ð(AÑBÔBð`ØFGÇlÂlÐS]ÑF^ÔF^ð`˜˜1ð `ð `ð `r<   c                 ó$   •— i | ]\  }}|‰v¯	||“ŒS r+   r+   )re   rf   r  Úclassifier_locals      €r;   r   z3checkpoint_filter_fn_classifier.<locals>.<dictcomp>@  s*   ø€ ÐRÐRÐR™$˜!˜Q¸ÐAQÐ8QÐ8Q��AÐ8QÐ8QÐ8Qr<   c                 ó    — i | ]\  }}d |› �|“ŒS rÖ  r+   )re   rf   r  s      r;   r   z3checkpoint_filter_fn_classifier.<locals>.<dictcomp>C  s$   € ÐCÐCÐC¡d a¨�˜1�� ÐCÐCÐCr<   )rô  rü  rÏ  Úupdate)rï  rð  Ú	to_filterÚencoder_dictÚprefixedr  rÿ  s        @@r;   Úcheckpoint_filter_fn_classifierr  0  s´   øø€ ð !3Ðð`ð `ð `ð `¨×)9Ò)9Ñ);Ô);ð `ñ `ô `ÐàRÐRÐRÐR *×"2Ò"2Ñ"4Ô"4ÐRÑRÔR€Iå/°	¸5¼=ÑIÔI€LØCÐC¨l×.@Ò.@Ñ.BÔ.BÐCÑCÔC€HØ‡O‚OÐ$Ñ%Ô%Ð%Ø€Or<   FÚvariantÚ
pretrainedc                 ó†   — |                      dd¦  «        }t          t          | |ft          t	          |d¬¦  «        ddœ|¤ŽS )NÚout_indicesr˜   Úgetter©r  Úfeature_clsr¾   )Úpretrained_filter_fnÚfeature_cfgÚkwargs_filter)Úpopr   r   rü  rÓ   ©r	  r
  Úkwargsr  s       r;   Ú_create_gemma4_vit_encoderr  H  s\   € Ø—*’*˜]¨AÑ.Ô.€KÝÝØØðõ :Ý [¸hÐGÑGÔGØ&ðð ð ðð ð r<   c                 ó„   — |                      dd¦  «        }t          t          | |ft          t	          |d¬¦  «        dœ|¤ŽS )Nr  r˜   r  r  )r  r  )r  r   r   r  rÓ   r  s       r;   Ú_create_gemma4_vit_classifierr  U  sY   € Ø—*’*˜]¨AÑ.Ô.€KåÝØØðõ =Ý [¸hÐGÑGÔGðð ð ðð ð r<   Úurlc                 ó$   — | dddd dddddd	d
dœ|¥S )Nr   )r˜   r™   r™   )r˜   é`   r  r}   ÚbicubicF)rÀ   rÀ   rÀ   )r}   r}   r}   zpatch_embed.input_projrÑ  )r  r¿   Ú
input_sizeÚmin_input_sizeÚ	pool_sizeÚcrop_pctÚinterpolationÚfixed_input_sizer¦  r¨   Ú
first_convÚ
classifierr+   )r  r  s     r;   Ú_cfgr%  b  sC   € àØØ#ð &ØØØ"Ø!ð
  ØØ.Øð'ð ð( ð)ð r<   z"timm/gemma4_vit_167m.gemma4_e4b_itzencoder.patch_embed.input_projz
apache-2.0)Ú	hf_hub_idr#  Úlicensez"timm/gemma4_vit_570m.gemma4_31b_itz&timm/gemma4_vit_167m_enc.gemma4_e4b_it)r&  r'  z&timm/gemma4_vit_570m_enc.gemma4_31b_it)zgemma4_vit_167m.gemma4_e4b_itzgemma4_vit_570m.gemma4_31b_itz!gemma4_vit_167m_enc.gemma4_e4b_itz!gemma4_vit_570m_enc.gemma4_31b_itr™   r—   ré   rê   rM  )r�   rQ  rì   ru   r"  rR  rñ   i€  é   éH   iÐ  )r�   rQ  rì   ru   r"  rR  c           	      ó`   — t          t          d¬¦  «        }t          dd| it          |fi |¤Ž¤ŽS )zöGemma4 ~167M (E2B/E4B  vision tower) classifier.

    Masked mean pool over patch tokens + norm + linear classifier. Output:
    ``(B, num_classes)``. For the native VLM encoder interface (soft-token
    output), use ``gemma4_vit_167m_enc``.
    T©rÊ  Úgemma4_vit_167mr
  )r,  )rÓ   Ú
_167M_ARCHr  ©r
  r  Ú
model_argss      r;   r,  r,  §  sB   € õ •j¨TÐ2Ñ2Ô2€JÝ(ÐpÐpÀzÐpÕUYÐZdÐUoÐUoÐhnÐUoÐUoÐpÐpÐpr<   c           	      ó`   — t          t          d¬¦  «        }t          dd| it          |fi |¤Ž¤ŽS )u  Gemma4 ~167M (E2B/E4B vision tower) â€” native VLM encoder.

    ``global_pool='soft'`` applies the spatial ``kÃ—k`` soft-token pool + âˆšD
    scale; output: ``(B, num_soft_tokens, embed_dim)``. Bit-perfect with HF
    ``Gemma4VisionModel`` on matching weights.
    rL  ©rP  Úgemma4_vit_167m_encr
  )r2  )rÓ   r-  r  r.  s      r;   r2  r2  ³  óB   € õ •j¨fÐ5Ñ5Ô5€JÝ%ÐqÐqÈ
ÐqÕVZÐ[eÐVpÐVpÐioÐVpÐVpÐqÐqÐqr<   c           	      ó`   — t          t          d¬¦  «        }t          dd| it          |fi |¤Ž¤ŽS )ag  Gemma4 ~570M (26B/31B vision tower) classifier.

    Masked mean pool over patch tokens + norm + linear classifier. Output:
    ``(B, num_classes)``. The classifier uses ``encoder.forward_features`` and
    does its own pool + norm; the encoder's ``std_bias/std_scale`` (which only
    applies in ``'soft'`` pool mode) is not used on the classifier path.
    Tr+  Úgemma4_vit_570mr
  )r5  )rÓ   Ú
_570M_ARCHr  r.  s      r;   r5  r5  ¿  sB   € õ •j¨TÐ2Ñ2Ô2€JÝ(ÐpÐpÀzÐpÕUYÐZdÐUoÐUoÐhnÐUoÐUoÐpÐpÐpr<   c           	      ó`   — t          t          d¬¦  «        }t          dd| it          |fi |¤Ž¤ŽS )u  Gemma4 ~570M (26B/31B vision tower) â€” native VLM encoder.

    ``global_pool='soft'`` output ``(B, num_soft_tokens, embed_dim)`` with
    ``std_bias/std_scale`` standardization applied post-pool (HF-native
    ordering â€” bit-perfect with ``Gemma4VisionModel``).
    rL  r1  Úgemma4_vit_570m_encr
  )r8  )rÓ   r6  r  r.  s      r;   r8  r8  Ì  r3  r<   )rT   )rT   rT   rÂ  rÃ  )rN  )KrL   ÚmathÚ	functoolsr   Útypingr   r   r   r   r   r	   r
   r   r5   Útorch.nnr1   Útorch.nn.functionalÚ
functionalrÁ   Ú	timm.datar   r   Útimm.layersr   r   r   r   r   r   r   Ú_builderr   Ú	_featuresr   Ú_manipulater   r   Ú	_registryr   r   Ú	naflexvitr   Ú__all__ré  r    rO   r[   rM   ra   rq   rs   r–   rç   r  r!  r/  r   r   rå   rN   rí  r|  rü  r  r  r  r%  Údefault_cfgsrÓ   r-  r6  r,  r2  r5  r8  r+   r<   r;   ú<module>rH     s›  ðð	ð 	ð €€€Ø Ð Ð Ð Ð Ð Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ Ià €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð à AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð +Ð *Ð *Ð *Ð *Ð *Ø +Ð +Ð +Ð +Ð +Ð +Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø <Ð <Ð <Ð <Ð <Ð <Ð <Ð <Ø %Ð %Ð %Ð %Ð %Ð %àÐ4Ð
5€ð*ð *ð *ð *ð *˜BœIñ *ô *ð *ðZ(�5”<ð ( E¤Lð (ð (ð (ð (ð ð		.ð 	.ØŒ<ð	.àŒ\ð	.ð Œ\ð	.ð ð		.ð
 „\ð	.ð 	.ð 	.ð 	.ð  Øð&ð &ØŒ<ð&àŒ\ð&ð Œ\ð&ð ð	&ð
 ð&ð „\ð&ð &ð &ð &ð4Mð Mð Mð Mð M˜bœiñ Mô Mð Mð`O2ð O2ð O2ð O2ð O2�r”yñ O2ô O2ð O2ðdcð cð cð cð c�b”iñ cô cð cðLXð Xð Xð Xð X�R”Yñ Xô Xð Xð8Fð Fð Fð Fð F�"”)ñ Fô Fð FðR@*ð @*ð @*ð @*ð @*˜œñ @*ô @*ð @*ðFwð wð wð wð w�r”yñ wô wð wðt{ð {ð {ð {ð {˜"œ)ñ {ô {ð {ð|"ð " B¤Ið "°Sð "ÈDð "Ð\`ð "ð "ð "ð "ð$Eð E cð E¸Tð EÈXð Eð Eð Eð Eð
,Ø˜˜eœlÐ*Ô+ð,àð,ð 
ˆ#ˆuŒ|Ð
Ôð,ð ,ð ,ð ,ð^Ø˜˜eœlÐ*Ô+ðà$ðð 
ˆ#ˆuŒ|Ð
Ôðð ð ð ð0
ð 
¨ð 
¸ð 
ÐTdð 
ð 
ð 
ð 
ð
ð 
¨3ð 
¸Dð 
ÐWjð 
ð 
ð 
ð 
ðð ˆcð  T¨#¨s¨(¤^ð ð ð ð ð2 %Ð$à%) TØ6Ø3Øð&ñ &ô &ð
 &* TØ6Ø3Øð&ñ &ô &ð *.¨Ø:Øð*ñ *ô *ð *.¨Ø:Øð*ñ *ô *ð#&ð &ñ ô €ð0 ˆTØØ
ØØØØØðñ ô €
ð ˆTØØ
ØØØØðñ ô €
ð ðqð q ð qÐ;Nð qð qð qñ „ðqð ðrð r Dð rÐ?Oð rð rð rñ „ðrð ð	qð 	q ð 	qÐ;Nð 	qð 	qð 	qñ „ð	qð ðrð r Dð rÐ?Oð rð rð rñ „ðrð rð rr<   