§
    ·¯¸jÀ–  ã            
       óx  — d Z ddlZddlmZ ddlmZmZmZmZm	Z	m
Z
 ddlZddlmZ ddlmc mZ ddlmZmZ ddlmZmZmZmZmZmZmZmZmZmZmZm Z  ddl!m"Z"m#Z# dd	l$m%Z% dd
l&m'Z' ddl(m)Z) ddl*m+Z+m,Z, dgZ-de.de	ej/                 fd„Z0e)dee.         dej1        dej1        fd„¦   «         Z2dej1        dee.         dee.         dej1        fd„Z3 G d„ dej/        ¦  «        Z4 G d„ dej/        ¦  «        Z5 G d„ dej/        ¦  «        Z6 G d„ dej/        ¦  «        Z7 G d „ d!ej/        ¦  «        Z8 G d"„ dej/        ¦  «        Z9dMd$„Z: e"i d% e:d&d'¬(¦  «        “d) e:d&d'd¬*¦  «        “d+ e:d&d'¬(¦  «        “d, e:d&d'd¬*¦  «        “d- e:d&d'¬(¦  «        “d. e:d&d'd¬*¦  «        “d/ e:d&d'¬(¦  «        “d0 e:d&d'd¬*¦  «        “d1 e:d&d'¬(¦  «        “d2 e:d&d'd¬*¦  «        “d3 e:d&d'¬(¦  «        “d4 e:d&d'd¬*¦  «        “d5 e:d&d6d7¬8¦  «        “d9 e:d&d6d7¬8¦  «        “d: e:d&d;d6d7¬<¦  «        “d= e:d&d;d6d7¬<¦  «        “d> e:d6d7¬?¦  «        “¦  «        Z;dNd@„Z<dOdBe=dCe>de9fdD„Z?e#dOdE„¦   «         Z@e#dOdF„¦   «         ZAe#dOdG„¦   «         ZBe#dOdH„¦   «         ZCe#dOdI„¦   «         ZDe#dOdJ„¦   «         ZEe#dOdK„¦   «         ZFe#dOdL„¦   «         ZGdS )Pzr An PyTorch implementation of Hiera

Adapted for timm from originals at https://github.com/facebookresearch/hiera
é    N)Úpartial)ÚDictÚListÚOptionalÚTupleÚTypeÚUnion©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)Úget_device_dtypeÚDropPathÚcalculate_drop_path_ratesÚMlpÚ
LayerScaleÚClNormMlpClassifierHeadÚuse_fused_attnÚ_assertÚget_norm_layerÚ	to_2tupleÚinit_weight_vitÚinit_weight_jaxé   )Úgenerate_default_cfgsÚregister_model)Úbuild_model_with_cfg)Úfeature_take_indices)Úregister_notrace_function)Únamed_applyÚ
checkpointÚHieraÚnÚreturnc                 ój   — t           j        t           j        t           j        t           j        g|          S )z¤
    Returns a conv with nd (e.g., Conv2d for n=2). Work up to n=3.
    If you wanted a 4d Hiera, you could probably just implement this for n=4. (no promises)
    )ÚnnÚIdentityÚConv1dÚConv2dÚConv3d)r"   s    úa/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/models/hiera.pyÚconv_ndr+   ;   s    € õ
 ŒK�œ¥B¤I­r¬yÐ9¸!Ô<Ð<ó    Útarget_sizeÚmaskc                 óú   — |€|S t          t          |j        dd …         ¦  «        t          | ¦  «        k    d¦  «         |j        dd …         | k    r(t          j        |                     ¦   «         | ¬¦  «        S |S )Né   z.mask spatial shape and target_size must match.)Úsize)r   ÚlenÚshapeÚFÚinterpolateÚfloat©r-   r.   s     r*   Úget_resized_maskr8   C   su   € ð €|Øˆå�C�”
˜1˜2˜2”ÑÔ¥3 {Ñ#3Ô#3Ò3Ð5eÑfÔfÐfØ„z�!�"�"„~˜Ò$Ð$ÝŒ}˜TŸZšZ™\œ\°Ð<Ñ<Ô<Ð<Ø€Kr,   Úxr3   Úmu_shapec                 óÐ  — t          |¦  «        }| j        d         | j        d         }}d„ t          ||¦  «        D ¦   «         } | j        |g|¢|¢|‘R Ž } dgt	          d„ t          t          dd|z   ¦  «        t          d|z   dd|z  z   ¦  «        ¦  «        D ¦   «         g ¦  «        z   t          | j        ¦  «        dz
  gz   } |                      |¦  «        j        |g|¢|‘R Ž } | S )a¾  
    Restore spatial organization by undoing windowed organization of mask units.

    Args:
        x: organized by mask units windows, e.g. in 2d [B, #MUy*#MUx, MUy, MUx, C]
        shape: current spatial shape, if it were not organized into mask unit
            windows, e.g. in 2d [B, #MUy*MUy, #MUx*MUx, C].
        mu_shape: current mask unit shape, e.g. in 2d [MUy, MUx]
    Returns:
        x: e.g. in 2d, [B, #MUy*MUy, #MUx*MUx, C]
    r   éÿÿÿÿc                 ó   — g | ]
\  }}||z  ‘ŒS © r>   )Ú.0ÚsÚmus      r*   ú
<listcomp>z"undo_windowing.<locals>.<listcomp>c   s    € Ð9Ð9Ð9™5˜1˜bˆq�B‰wÐ9Ð9Ð9r,   c                 ó,   — g | ]}t          |¦  «        ‘ŒS r>   ©Úlist©r?   Úps     r*   rB   z"undo_windowing.<locals>.<listcomp>i   s   € ÐNÐNÐN˜1�t�A‰wŒwÐNÐNÐNr,   r   r0   )r2   r3   ÚzipÚviewÚsumÚrangeÚpermuteÚreshape)r9   r3   r:   ÚDÚBÚCÚnum_MUsrL   s           r*   Úundo_windowingrR   P   s	  € õ  	ˆE‰
Œ
€AØŒ7�1Œ:�q”w˜r”{€q€Aà9Ð9¥C¨¨xÑ$8Ô$8Ð9Ñ9Ô9€GØˆŒˆqÐ)�7Ð)˜XÐ) qÐ)Ð)Ð)€Að 
ˆÝ
ÐNÐN¥¥E¨!¨Q°©U¡O¤OµU¸1¸q¹5À!ÀaÈ!ÁeÁ)Ñ5LÔ5LÑ MÔ MÐNÑNÔNÐPRÑ
SÔ
Sñ	TåˆqŒw‰<Œ<˜!ÑÐ
ñ	ð ð
 	#ˆ�	Š	�'ÑÔÔ" 1Ð0 uÐ0¨aÐ0Ð0Ð0€Aà€Hr,   c            	       ó�   ‡ — e Zd ZdZdeedf         deedf         deeedf                  fˆ fd„Zdej	        dej	        fd	„Z
ˆ xZS )
ÚUnrolla>  
    Reorders the tokens such that patches are contiguous in memory.
    E.g., given [B, (H, W), C] and stride of (Sy, Sx), this will re-order the tokens as
                           [B, (Sy, Sx, H // Sy, W // Sx), C]

    This allows operations like Max2d to be computed as x.view(B, Sx*Sy, -1, C).max(dim=1).
    Not only is this faster, but it also makes it easy to support inputs of arbitrary
    dimensions in addition to patch-wise sparsity.

    Performing this operation multiple times in sequence puts entire windows as contiguous
    in memory. For instance, if you applied the stride (2, 2) 3 times, entire windows of
    size 8x8 would be contiguous in memory, allowing operations like mask unit attention
    computed easily and efficiently, while also allowing max to be applied sequentially.

    Note: This means that intermediate values of the model are not in HxW order, so they
    need to be re-rolled if you want to use the intermediate values as a HxW feature map.
    The last block of the network is fine though, since by then the strides are all consumed.
    Ú
input_size.Úpatch_strideÚunroll_schedulec                 ó”   •— t          ¦   «                              ¦   «          d„ t          ||¦  «        D ¦   «         | _        || _        d S )Nc                 ó   — g | ]
\  }}||z  ‘ŒS r>   r>   ©r?   Úir@   s      r*   rB   z#Unroll.__init__.<locals>.<listcomp>Œ   ó    € ÐFÐFÐF¡  1�Q˜!‘VÐFÐFÐFr,   )ÚsuperÚ__init__rH   r1   Úschedule)ÚselfrU   rV   rW   Ú	__class__s       €r*   r^   zUnroll.__init__…   sE   ø€ õ 	‰Œ×ÒÑÔÐØFÐF­¨J¸Ñ(EÔ(EÐFÑFÔFˆŒ	Ø'ˆŒˆˆr,   r9   r#   c           	      óØ  — |j         \  }}}| j        } |j        |g|z   |gz   Ž }| j        D �]}d„ t	          ||¦  «        D ¦   «         }|gt          d„ t	          ||¦  «        D ¦   «         g ¦  «        z   |gz   }|                     |¦  «        }t          |¦  «        }dgt          t          d|dz
  d¦  «        ¦  «        z   t          t          d|dz
  d¦  «        ¦  «        z   |dz
  gz   }	| 	                    |	¦  «        }| 
                    dt          |¦  «        ¦  «        }|t          j        |¦  «        z  }�Œ|                     dt          j        | j        ¦  «        |¦  «        }|S )zª
        Input: Flattened patch embeddings [B, N, C]
        Output: Patch embeddings [B, N, C] permuted such that [B, 4, N//4, C].max(1) etc. performs MaxPoolNd
        c                 ó   — g | ]
\  }}||z  ‘ŒS r>   r>   rZ   s      r*   rB   z"Unroll.forward.<locals>.<listcomp>�   s    € ÐBÐBÐB¡4 1 a˜˜Q™ÐBÐBÐBr,   c                 ó   — g | ]	\  }}||g‘Œ
S r>   r>   rZ   s      r*   rB   z"Unroll.forward.<locals>.<listcomp>ž   s    € Ð"MÐ"MÐ"M©d¨a° A q 6Ð"MÐ"MÐ"Mr,   r   r0   r   r<   )r3   r1   rI   r_   rH   rJ   r2   rE   rK   rL   ÚflattenÚmathÚprodrM   )
r`   r9   rO   Ú_rP   Úcur_sizeÚstridesÚ	new_shapeÚLrL   s
             r*   ÚforwardzUnroll.forward�   sr  € ð
 ”'‰ˆˆ1ˆaØ”9ˆØˆAŒF�a�S˜8‘^ q cÑ)Ð+ˆà”}ð 	$ñ 	$ˆGð
 CÐB­3¨x¸Ñ+AÔ+AÐBÑBÔBˆHØ˜�cÐ"MÐ"Mµc¸(ÀGÑ6LÔ6LÐ"MÑ"MÔ"MÈrÑRÔRÑRÐVWÐUXÑXˆIØ—’�yÑ!Ô!ˆAõ �I‘”ˆAØ�c�D¥ q¨!¨a©%°Ñ!3Ô!3Ñ4Ô4Ñ4µt½EÀ!ÀQÈÁUÈAÑ<NÔ<NÑ7OÔ7OÑOÐSTÐWXÑSXÐRYÑYˆGØ—	’	˜'Ñ"Ô"ˆAð —	’	˜!�S ™\œ\Ñ*Ô*ˆAØ•”˜7Ñ#Ô#Ñ#ˆA‰Aà�IŠI�b�$œ) D¤IÑ.Ô.°Ñ2Ô2ˆØˆr,   ©Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Úintr   r^   ÚtorchÚTensorrm   Ú__classcell__©ra   s   @r*   rT   rT   q   s¦   ø€ € € € € ðð ð&(à˜c 3˜hœð(ð    S œ/ð(ð " %¨¨S¨¤/Ô2ð	(ð (ð (ð (ð (ð (ð˜œð ¨%¬,ð ð ð ð ð ð ð ð r,   rT   c            
       óº   ‡ — e Zd ZdZdeedf         deedf         deeedf                  dee         def
ˆ fd„Z	 dd
ej	        dedej	        dej	        fd„Z
ˆ xZS )ÚRerollzQ
    Undos the "unroll" operation so that you can use intermediate features.
    rU   .rV   rW   Ú
stage_endsÚq_poolc                 ó‚  •— t          ¦   «                              ¦   «          d„ t          ||¦  «        D ¦   «         | _        i | _        | j        }t          |d         dz   ¦  «        D ]W}||f| j        |<   ||d |…         v r=t          |¦  «        dk    r d„ t          ||d         ¦  «        D ¦   «         }|dd …         }ŒXd S )Nc                 ó   — g | ]
\  }}||z  ‘ŒS r>   r>   rZ   s      r*   rB   z#Reroll.__init__.<locals>.<listcomp>½   r\   r,   r<   r   r   c                 ó   — g | ]
\  }}||z  ‘ŒS r>   r>   )r?   r"   r@   s      r*   rB   z#Reroll.__init__.<locals>.<listcomp>È   s    € ÐMÐMÐM¡t q¨!˜A ™FÐMÐMÐMr,   )r]   r^   rH   r1   r_   rK   r2   )	r`   rU   rV   rW   rz   r{   r1   r[   ra   s	           €r*   r^   zReroll.__init__´   sÝ   ø€ õ 	‰Œ×ÒÑÔÐØFÐF­¨J¸Ñ(EÔ(EÐFÑFÔFˆŒ	ð ˆŒØŒyˆÝ�z "”~¨Ñ)Ñ*Ô*ð 	6ð 	6ˆAØ.°Ð4ˆDŒM˜!Ñà�J˜w ˜wÔ'Ð'Ð'Ý�Ñ'Ô'¨!Ò+Ð+ØMÐM­s°4¸ÈÔ9KÑ/LÔ/LÐMÑMÔM�DØ"1°!°"°"Ô"5�øð	6ð 	6r,   Nr9   Ú	block_idxr.   r#   c                 óª  — | j         |         \  }}|j        \  }}}t          |¦  «        }	dg|	z  }
|D ]ù} |j        |g|¢|t	          j        |¦  «        z  ‘|
¢|‘R Ž }t          |j        ¦  «        }dd|	z   gt          d„ t          t          dd|	z   ¦  «        t          d|	z   dz   |dz
  ¦  «        ¦  «        D ¦   «         g ¦  «        z   |dz
  gz   }| 	                    |¦  «        }t          |	¦  «        D ]}|
|xx         ||         z  cc<   Œ |j
        |dg|
¢|‘R Ž }|j        d         }Œú |j        ||g|
¢|‘R Ž }|�|S t          |||
¦  «        }|S )a&  
        Roll the given tensor back up to spatial order assuming it's from the given block.

        If no mask is provided:
            - Returns [B, H, W, C] for 2d, [B, T, H, W, C] for 3d, etc.
        If a mask is provided:
            - Returns [B, #MUs, MUy, MUx, C] for 2d, etc.
        r   r   c                 ó,   — g | ]}t          |¦  «        ‘ŒS r>   rD   rF   s     r*   rB   z"Reroll.forward.<locals>.<listcomp>è   s   € ÐVÐVÐV 1•t˜A‘w”wÐVÐVÐVr,   r<   )r_   r3   r2   rI   rf   rg   rJ   rH   rK   rL   rM   rR   )r`   r9   r   r.   r_   r1   rO   ÚNrP   rN   Úcur_mu_shaperj   rl   rL   r[   s                  r*   rm   zReroll.forwardË   s´  € ð œ yÔ1‰ˆ�$Ø”'‰ˆˆ1ˆaå�‰IŒIˆØ�s˜Q‘wˆàð 	ð 	ˆGà�”�qÐN˜7ÐN A­¬°7Ñ);Ô);Ñ$;ÐN¸lÐNÈAÐNÐNÐNˆAõ �A”G‘”ˆAà�A˜‘E�
ÝÐVÐV­­E°!°Q¸±U©O¬O½UÀ1ÀqÁ5È1Á9ÈaÐRSÉeÑ=TÔ=TÑ(UÔ(UÐVÑVÔVÐXZÑ[Ô[ñ\à�q‘5�'ñð ð
 —	’	˜'Ñ"Ô"ˆAõ ˜1‘X”Xð .ð .�Ø˜Q��” 7¨1¤:Ñ-��‘�Ø�”	˜!˜RÐ2 ,Ð2°Ð2Ð2Ð2ˆAØ”˜”
ˆAˆAð ˆAŒF�1�aÐ*˜,Ð*¨Ð*Ð*Ð*ˆð ÐØˆHõ ˜1˜d LÑ1Ô1ˆàˆr,   ©Nrn   rw   s   @r*   ry   ry   ¯   sâ   ø€ € € € € ðð ð6à˜c 3˜hœð6ð    S œ/ð6ð " %¨¨S¨¤/Ô2ð	6ð
 ˜Sœ	ð6ð ð6ð 6ð 6ð 6ð 6ð 6ð6 "&ð	2ð 2àŒ|ð2ð ð2ð ”,ð	2ð
 
Œð2ð 2ð 2ð 2ð 2ð 2ð 2ð 2r,   ry   c                   ó˜   ‡ — e Zd ZU dZej        j        e         ed<   	 	 	 	 	 dde	de	d	e	d
e	de	defˆ fd„Z
dej        dej        fd„Zˆ xZS )ÚMaskUnitAttentionzÛ
    Computes either Mask Unit or Global Attention. Also is able to perform q pooling.

    Note: this assumes the tokens have already been flattened and unrolled into mask units.
    See `Unroll` for more details.
    Ú
fused_attnr   r   FNÚdimÚdim_outÚheadsÚq_strideÚwindow_sizeÚuse_mask_unit_attnc	                 ód  •— ||dœ}	t          ¦   «                              ¦   «          || _        || _        || _        || _        ||z  | _        | j        dz  | _        t          ¦   «         | _	        t          j        |d|z  fi |	¤Ž| _        t          j        ||fi |	¤Ž| _        || _        || _        dS )a–  
        Args:
        - dim, dim_out: The input and output feature dimensions.
        - heads: The number of attention heads.
        - q_stride: If greater than 1, pool q with this stride. The stride should be flattened (e.g., 2x2 = 4).
        - window_size: The current (flattened) size of a mask unit *after* pooling (if any).
        - use_mask_unit_attn: Use Mask Unit or Global Attention.
        ©ÚdeviceÚdtypeg      à¿é   N)r]   r^   rˆ   r‰   rŠ   r‹   Úhead_dimÚscaler   r‡   r%   ÚLinearÚqkvÚprojrŒ   r�   )r`   rˆ   r‰   rŠ   r‹   rŒ   r�   r�   r‘   Úddra   s             €r*   r^   zMaskUnitAttention.__init__	  s»   ø€ ð& ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØˆŒØˆŒØˆŒ
Ø ˆŒØ 5Ñ(ˆŒØ”] dÑ*ˆŒ
Ý(Ñ*Ô*ˆŒå”9˜S ! g¡+Ð4Ð4°Ð4Ð4ˆŒÝ”I˜g wÐ5Ð5°"Ð5Ð5ˆŒ	à&ˆÔØ"4ˆÔÐÐr,   r9   r#   c                 ó0  — |j         \  }}}| j        rÃ|| j        | j        z  z  }|                      |¦  «                             |d|d| j        | j        ¦  «                             dddddd¦  «        }| 	                    d¦  «        \  }}}	| j        dk    r=| 
                    || j        || j        d| j        ¦  «                             d¬¦  «        }né|                      |¦  «                             ||d| j        | j        ¦  «                             ddddd¦  «        }| 	                    d¦  «        \  }}}	| j        dk    r<| 
                    || j        | j        d| j        ¦  «                             d¬¦  «        }|                     ¦   «         |                     ¦   «         |	                     ¦   «         }	}}| j        rt          j        |||	¦  «        }n<|| j        z  |                     dd	¦  «        z  }
|
                     d¬¦  «        }
|
|	z  }| j        r1|                     dd¦  «                             |d| j        ¦  «        }n0|                     dd¦  «                             |d| j        ¦  «        }|                      |¦  «        }|S )
z5 Input should be of shape [batch, tokens, channels]. r<   r’   r   é   r0   r   é   ©rˆ   éþÿÿÿ)r3   r�   r‹   rŒ   r–   rM   rŠ   r“   rL   ÚunbindrI   ÚamaxÚ
contiguousr‡   r4   Úscaled_dot_product_attentionr”   Ú	transposeÚsoftmaxr‰   r—   )r`   r9   rO   r‚   rh   Únum_windowsr–   ÚqÚkÚvÚattns              r*   rm   zMaskUnitAttention.forward,  s\  € à”'‰ˆˆ1ˆaàÔ"ð 	Eà ¤°Ô0@Ñ @ÑAˆKØ—(’(˜1‘+”+×%Ò%Ø�2�{ A t¤z°4´=ñô çŠg�a˜˜A˜q ! QÑ'Ô'ð ð —j’j ‘m”m‰GˆAˆq�!àŒ}˜qÒ Ð à—F’F˜1˜dœj¨+°t´}ÀbÈ$Ì-ÑXÔX×]Ò]ÐbcÐ]ÑdÔd�øð —(’(˜1‘+”+×%Ò% a¨¨A¨t¬z¸4¼=ÑIÔI×QÒQÐRSÐUVÐXYÐ[\Ð^_Ñ`Ô`ˆCØ—j’j ‘m”m‰GˆAˆq�!àŒ}˜qÒ Ð à—F’F˜1˜dœj¨$¬-¸¸T¼]ÑKÔK×PÒPÐUVÐPÑWÔW�ð —l’l‘n”n a§l¢l¡n¤n°a·l²l±n´n�!ˆqˆAàŒ?ð 	åÔ.¨q°!°QÑ7Ô7ˆAˆAà˜œ
‘N a§k¢k°"°bÑ&9Ô&9Ñ9ˆDØ—<’< B�<Ñ'Ô'ˆDØ�q‘ˆAð Ô"ð 	?Ø—’˜A˜qÑ!Ô!×)Ò)¨!¨R°´Ñ>Ô>ˆAˆAà—’˜A˜qÑ!Ô!×)Ò)¨!¨R°´Ñ>Ô>ˆAà�IŠI�a‰LŒLˆØˆr,   )r   r   FNN)ro   rp   rq   rr   rt   ÚjitÚFinalÚboolÚ__annotations__rs   r^   ru   rm   rv   rw   s   @r*   r†   r†      s×   ø€ € € € € € ðð ð ”	” Ô%Ð%Ð%Ñ%ð Ø Ø',ØØð!5ð !5àð!5ð ð!5ð ð	!5ð
 ð!5ð ð!5ð !%ð!5ð !5ð !5ð !5ð !5ð !5ðF,˜œð ,¨%¬,ð ,ð ,ð ,ð ,ð ,ð ,ð ,ð ,r,   r†   c                   óØ   ‡ — e Zd Zdddej        ej        ddddddfded	ed
edededee         de	ej
                 de	ej
                 dedededefˆ fd„Zdej        dej        fd„Zˆ xZS )Ú
HieraBlockç      @ç        Nr   r   TFrˆ   r‰   rŠ   Ú	mlp_ratioÚ	drop_pathÚinit_valuesÚ
norm_layerÚ	act_layerr‹   rŒ   Úuse_expand_projr�   c                 ó  •— ||dœ}t          ¦   «                              ¦   «          || _        || _         ||fi |¤Ž| _        ||k    r5d| _        |rt          j        ||fi |¤Ž| _        n!||dz  k    sJ ‚d | _        nd| _        d | _        t          ||||	|
|fi |¤Ž| _
        |�t          |fd|i|¤Žnt          j        ¦   «         | _        |dk    rt          |¦  «        nt          j        ¦   «         | _         ||fi |¤Ž| _        t#          |t%          ||z  ¦  «        fd|i|¤Ž| _        |�t          |fd|i|¤Žnt          j        ¦   «         | _        |dk    rt          |¦  «        nt          j        ¦   «         | _        d S )Nr�   Tr0   Fr³   r   rµ   )r]   r^   rˆ   r‰   Únorm1Ú	do_expandr%   r•   r—   r†   r¨   r   r&   Úls1r   Ú
drop_path1Únorm2r   rs   ÚmlpÚls2Ú
drop_path2)r`   rˆ   r‰   rŠ   r±   r²   r³   r´   rµ   r‹   rŒ   r¶   r�   r�   r‘   r˜   ra   s                   €r*   r^   zHieraBlock.__init__\  sÓ  ø€ ð" ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØˆŒØˆŒà�Z Ð*Ð* rÐ*Ð*ˆŒ
Ø�'Š>ˆ>Ø!ˆDŒNØð !ÝœI c¨7Ð9Ð9°bÐ9Ð9�”	�	à #¨¡'Ò)Ð)Ð)Ð)Ø �”	�	à"ˆDŒNØˆDŒIÝ%ØØØØØØð
ð 
ð ð
ð 
ˆŒ	ð JUÐI`•:˜gÐEÐE°;ÐEÀ"ÐEÐEÐEÕfhÔfqÑfsÔfsˆŒØ1:¸Q²°�( 9Ñ-Ô-Ð-ÅBÄKÁMÄMˆŒà�Z Ð.Ð.¨2Ð.Ð.ˆŒ
Ý�w¥ G¨iÑ$7Ñ 8Ô 8ÐTÐTÀIÐTÐQSÐTÐTˆŒØITÐI`•:˜gÐEÐE°;ÐEÀ"ÐEÐEÐEÕfhÔfqÑfsÔfsˆŒØ1:¸Q²°�( 9Ñ-Ô-Ð-ÅBÄKÁMÄMˆŒˆˆr,   r9   r#   c           
      óˆ  — |                       |¦  «        }| j        �r| j        �b|                      |¦  «        }|                     |j        d         | j        j        d|j        d         ¦  «                             d¬¦  «        }n¬t          j	        |                     |j        d         | j        j        d|j        d         ¦  «                             d¬¦  «        |                     |j        d         | j        j        d|j        d         ¦  «         
                    d¬¦  «        gd¬¦  «        }||                      |                      |                      |¦  «        ¦  «        ¦  «        z   }||                      |                      |                      |                      |¦  «        ¦  «        ¦  «        ¦  «        z   }|S )Nr   r<   r   rœ   )r¸   r¹   r—   rI   r3   r¨   r‹   rŸ   rt   ÚcatÚmeanr»   rº   r¿   r¾   r½   r¼   )r`   r9   Úx_norms      r*   rm   zHieraBlock.forwardŽ  so  € à—’˜A‘”ˆØŒ>ñ 
	ØŒyÐ$Ø—I’I˜fÑ%Ô%�Ø—F’F˜1œ7 1œ: t¤yÔ'9¸2¸q¼wÀr¼{ÑKÔK×PÒPÐUVÐPÑWÔW��å”IØ—F’F˜1œ7 1œ: t¤yÔ'9¸2¸q¼wÀr¼{ÑKÔK×PÒPÐUVÐPÑWÔWØ—F’F˜1œ7 1œ: t¤yÔ'9¸2¸q¼wÀr¼{ÑKÔK×PÒPÐUVÐPÑWÔWðð ð	ñ ô �ð �—’ §¢¨¯ª°6Ñ):Ô):Ñ ;Ô ;Ñ<Ô<Ñ<ˆð �—’ §¢¨¯ª°$·*²*¸Q±-´-Ñ)@Ô)@Ñ AÔ AÑBÔBÑBˆØˆr,   )ro   rp   rq   r%   Ú	LayerNormÚGELUrs   r6   r   r   ÚModuler«   r^   rt   ru   rm   rv   rw   s   @r*   r®   r®   [  s)  ø€ € € € € ð  #Ø"Ø+/Ø*,¬,Ø)+¬ØØ Ø$(Ø',ØØð0Rð 0Ràð0Rð ð0Rð ð	0Rð
 ð0Rð ð0Rð " %œð0Rð ˜RœYœð0Rð ˜BœI”ð0Rð ð0Rð ð0Rð "ð0Rð !%ð0Rð 0Rð 0Rð 0Rð 0Rð 0Rðd˜œð ¨%¬,ð ð ð ð ð ð ð ð r,   r®   c                   ó¶   ‡ — e Zd ZdZ	 	 	 ddededeedf         deedf         d	eedf         d
efˆ fd„Z	 ddej	        de
ej	                 dej	        fd„Zˆ xZS )Ú
PatchEmbedzHPatch embed that supports any number of spatial dimensions (1d, 2d, 3d).TNÚdim_inr‰   Úkernel.ÚstrideÚpaddingrM   c	                 óÒ   •— ||dœ}	t          ¦   «                              ¦   «          t          |¦  «        | _        || _         t          | j        ¦  «        ||f|||dœ|	¤Ž| _        d S )Nr�   )Úkernel_sizerË   rÌ   )r]   r^   r2   Úspatial_dimsrM   r+   r—   )r`   rÉ   r‰   rÊ   rË   rÌ   rM   r�   r‘   r˜   ra   s             €r*   r^   zPatchEmbed.__init__¦  s„   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐå ™KœKˆÔØˆŒØ.•G˜DÔ-Ñ.Ô.ØØð
ð ØØð
ð 
ð ð
ð 
ˆŒ	ˆ	ˆ	r,   r9   r.   r#   c                 ól  — |�Tt          |j        dd …         |¬¦  «        }|                      ||                     t          j        ¦  «        z  ¦  «        }n|                      |¦  «        }| j        rA|                     |j        d         |j        d         d¦  «                             dd¦  «        }|S )Nr0   r7   r   r   r<   )r8   r3   r—   Útort   r«   rM   r¢   ©r`   r9   r.   s      r*   rm   zPatchEmbed.forward¿  sœ   € ð
 ÐÝ#°´¸¸¸´À$ÐGÑGÔGˆDØ—	’	˜!˜dŸgšg¥e¤jÑ1Ô1Ñ1Ñ2Ô2ˆAˆAà—	’	˜!‘”ˆAØŒ<ð 	FØ—	’	˜!œ' !œ* a¤g¨a¤j°"Ñ5Ô5×?Ò?ÀÀ1ÑEÔEˆAØˆr,   )TNNr„   )ro   rp   rq   rr   rs   r   r«   r^   rt   ru   r   rm   rv   rw   s   @r*   rÈ   rÈ   £  sê   ø€ € € € € ØRÐRð !ØØð
ð 
àð
ð ð
ð ˜#˜s˜(”Oð	
ð
 ˜#˜s˜(”Oð
ð ˜3 ˜8”_ð
ð ð
ð 
ð 
ð 
ð 
ð 
ð8 ,0ðð àŒ|ðð ˜5œ<Ô(ðð 
Œð	ð ð ð ð ð ð ð r,   rÈ   c            =       óz  ‡ — e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dUdeedf         dedededededeedf         d ed!eedf         d"eedf         d#eedf         d$ed%ed&ed'eedf         d(eedf         d)eedf         d*ed+ed,ee         d-ed.ed/e	ee
ej                 f         d0ed1ed2ed3ed4ed5eeef         f:ˆ fd6„Zd7„ Zej        j        d8„ ¦   «         Zej        j        dVd9ed:efd;„¦   «         Zej        j        dWd<ed:dfd=„¦   «         Zej        j        d>„ ¦   «         ZdXdedee         d?efd@„ZdAej        dBed:ej        fdC„Zd:ej        fdD„Z	 	 	 	 	 	 	 dYdAej        dFeej                 dGee	eee         f                  dHedIedJedKed9ed:e	eej                 eej        eej                 f         f         fdL„Z	 	 	 	 dZdGe	eee         f         dMedNed9efdO„Z	 	 dXdAej        dFeej                 dPed:ej        fdQ„ZdVdRed:ej        fdS„Z	 d[dAej        dFeej                 d:ej        fdT„Z ˆ xZ!S )\r!   ©éà   rÕ   r’   é`   r   éè  Úavg©r0   r’   é   r’   ©r0   r0   ©é   rÝ   ©TTFFTç       @©é   rá   ©rš   rš   ©r’   r’   r¯   r°   NÚ rÄ   çü©ñÒMbP?F©é   rç   Úimg_size.Úin_chansÚ	embed_dimÚ	num_headsÚnum_classesÚglobal_poolÚstagesr{   r‹   Úmask_unit_sizeÚmask_unit_attnr¶   Údim_mulÚhead_mulÚpatch_kernelrV   Úpatch_paddingr±   Údrop_path_rater³   Úfix_initÚweight_initr´   Ú	drop_rateÚpatch_drop_rateÚhead_init_scaleÚsep_pos_embedÚabs_win_pos_embedÚglobal_pos_sizec                  ó¸  •‡— t          ¦   «                              ¦   «          ||dœ} || _        || _        d| _        t          |¦  «        }t          |t          ¦  «        rt          |¦  «        }|| _	        d„ t          ||¦  «        D ¦   «         | _        t          j        | j        ¦  «        }!t          j        |
¦  «        }"t          j        |	¦  «        }#|t          ‰¦  «        k     sJ ‚||	c| _        | _        |"|
c| _        | _        d„ t          | j        | j        ¦  «        D ¦   «         | _        ˆfd„t)          dt          ‰¦  «        dz   ¦  «        D ¦   «         | _        || _        t/          |||||fi | ¤Ž| _        d | _        d | _        d | _        d | _        |r{t;          j        t?          j         d| j        d         | j        d         z  |fi | ¤Ž¦  «        | _        t;          j        t?          j         d| j        d         |fi | ¤Ž¦  «        | _        nˆ|r[t;          j        t?          j         d|g|¢R i | ¤Ž¦  «        | _        t;          j        t?          j         d|g|
¢R i | ¤Ž¦  «        | _        n+t;          j        t?          j         d|!|fi | ¤Ž¦  «        | _        tC          |||	gt          | j        d d	…         ¦  «        z  ¦  «        | _"        tG          |||	gt          | j        d d	…         ¦  «        z  | j        |¦  «        | _$        d
„ | j        d |…         D ¦   «         }$d}%tK          ‰¦  «        }&tM          ||&¦  «        }'t;          j'        ¦   «         | _(        g | _)        t)          |&¦  «        D ]È}(|})||%         }*|(dz
  | j        v r2t          ||z  ¦  «        })t          ||z  ¦  «        }|%dz  }%|(|$v r|"|#z  }"tU          d||)|||'|(         |||(|$v r|#nd|"||*dœ| ¤Ž}+|)}|(| j        v r5| xj)        tW          |)d|%dz   z  d| j        |%         › �¬¦  «        gz  c_)        | j(         ,                    |+¦  «         ŒÉ|x| _-        | _.        t_          ||f|||ddœ| ¤Ž| _0        |rMt:          j1         2                    | j        d¬¦  «         t:          j1         2                    | j        d¬¦  «         nZ| j        �&t:          j1         2                    | j        d¬¦  «         | j        �&t:          j1         2                    | j        d¬¦  «         |dk    r5|dk    rtf          nth          },tk          |,d¬¦  «        },tm          |,| ¦  «         |r|  7                    ¦   «          t          | j0        j8        t:          j9        ¦  «        rT| j0        j8        j:        j;         <                    |¦  «         | j0        j8        j=        j;         <                    |¦  «         d S d S )Nr�   Fc                 ó   — g | ]
\  }}||z  ‘ŒS r>   r>   rZ   s      r*   rB   z"Hiera.__init__.<locals>.<listcomp>ý  s    € Ð$TÐ$TÐ$T±°°1 Q¨!¡VÐ$TÐ$TÐ$Tr,   c                 ó   — g | ]
\  }}||z  ‘ŒS r>   r>   rZ   s      r*   rB   z"Hiera.__init__.<locals>.<listcomp>  s    € Ð"jÐ"jÐ"j©d¨a° 1¨¡6Ð"jÐ"jÐ"jr,   c                 óD   •— g | ]}t          ‰d |…         ¦  «        dz
  ‘ŒS )Nr   )rJ   )r?   r[   rî   s     €r*   rB   z"Hiera.__init__.<locals>.<listcomp>  s,   ø€ ÐRÐRÐR°1�3˜v b q bœz™?œ?¨QÑ.ÐRÐRÐRr,   r   r0   r   r<   c                 ó   — g | ]}|d z   ‘ŒS )r   r>   )r?   r9   s     r*   rB   z"Hiera.__init__.<locals>.<listcomp>2  s   € ÐAÐAÐA 1˜˜Q™ÐAÐAÐAr,   )rˆ   r‰   rŠ   r±   r²   r³   r´   r‹   rŒ   r¶   r�   zblocks.)Únum_chsÚ	reductionÚmoduleÚNLC)Ú	pool_typerø   r´   Ú	input_fmtg{®Gáz”?)ÚstdÚskipÚjaxúhead.fc)Úclassifier_namer>   )>r]   r^   rì   ré   Úgrad_checkpointingr   Ú
isinstancers   r   rV   rH   Útokens_spatial_shaperf   rg   r2   r{   r‹   Úmu_sizerï   Úmask_spatial_shaperK   rz   rù   rÈ   Úpatch_embedÚ	pos_embedÚpos_embed_winÚpos_embed_spatialÚpos_embed_temporalr%   Ú	Parameterrt   ÚzerosrT   Úunrollry   ÚrerollrJ   r   Ú
ModuleListÚblocksÚfeature_infor®   ÚdictÚappendÚnum_featuresÚhead_hidden_sizer   ÚheadÚinitÚtrunc_normal_r   r   r   r   Úfix_init_weightÚfcr•   ÚweightÚdataÚmul_Úbias).r`   rè   ré   rê   rë   rì   rí   rî   r{   r‹   rï   rð   r¶   rñ   rò   ró   rV   rô   r±   rõ   r³   rö   r÷   r´   rø   rù   rú   rû   rü   rý   r�   r‘   r˜   Ú
num_tokensÚflat_mu_sizeÚflat_q_strideÚq_pool_blocksÚ	cur_stageÚdepthÚdprr[   r‰   r�   ÚblockÚinit_fnra   s.          `                                     €r*   r^   zHiera.__init__Ð  su  øø€ õF 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆØ&ˆÔØ ˆŒØ"'ˆÔÝ# JÑ/Ô/ˆ
Ý�h¥Ñ$Ô$ð 	+Ý  Ñ*Ô*ˆHà(ˆÔØ$TÐ$T½¸HÀlÑ8SÔ8SÐ$TÑ$TÔ$TˆÔ!Ý”Y˜tÔ8Ñ9Ô9ˆ
Ý”y Ñ0Ô0ˆÝœ	 (Ñ+Ô+ˆØ�˜F™œÒ#Ð#Ð#Ð#Ø%+¨XÐ"ˆŒ�T”]Ø,8¸.Ð)ˆŒ�dÔ)Ø"jÐ"jµc¸$Ô:SÐUYÔUhÑ6iÔ6iÐ"jÑ"jÔ"jˆÔØRÐRÐRÐR½¸aÅÀVÁÄÈqÁÑ8QÔ8QÐRÑRÔRˆŒØ.ˆÔå%ØØØØØð
ð 
ð ð
ð 
ˆÔð 26ˆŒØ59ˆÔØ9=ˆÔØ:>ˆÔØð 	[Ý%'¤\Ý”˜A˜tÔ8¸Ô;¸dÔ>WÐXYÔ>ZÑZÐ\eÐlÐlÐikÐlÐlñ&ô &ˆDÔ"õ ')¤lÝ”˜A˜tÔ8¸Ô;¸YÐMÐMÈ"ÐMÐMñ'ô 'ˆDÔ#Ð#ð !ð [å!#¤­e¬k¸!¸YÐ._ÈÐ._Ð._Ð._Ð\^Ð._Ð._Ñ!`Ô!`�”Ý%'¤\µ%´+¸aÀÐ2bÈ^Ð2bÐ2bÐ2bÐ_aÐ2bÐ2bÑ%cÔ%c�Ô"Ð"å!#¤­e¬k¸!¸ZÈÐ.YÐ.YÐVXÐ.YÐ.YÑ!ZÔ!Z�”õ ØØØˆJ�˜Tœ_¨S¨b¨SÔ1Ñ2Ô2Ñ2ñ
ô 
ˆŒõ
 ØØØˆJ�˜Tœ_¨S¨b¨SÔ1Ñ2Ô2Ñ2ØŒOØñ
ô 
ˆŒð BÐA¨¬¸¸¸Ô(@ÐAÑAÔAˆð ˆ	Ý�F‘”ˆÝ'¨¸Ñ>Ô>ˆÝ”m‘o”oˆŒØˆÔÝ�u‘”ð  	&ð  	&ˆAØˆGð "0°	Ô!:Ðà�1‰u˜œÐ'Ð'Ý˜i¨'Ñ1Ñ2Ô2�Ý 	¨HÑ 4Ñ5Ô5�	Ø˜Q‘�	Ø˜Ð%Ð%Ø  ]Ñ2�Låð ØØØØ#Ø˜aœ&Ø'Ø%Ø+,°Ð+=Ð+=˜-˜-À1Ø(Ø /Ø#5ðð ð ðð ˆEð  ˆIØ�D”OÐ#Ð#ØÐ!Ô!Ý °A¸	À!¹Ñ4DÐMsÐW[ÔWfÐgpÔWqÐMsÐMsÐtÑtÔtð&vñ vÐ!Ô!àŒK×Ò˜uÑ%Ô%Ð%Ð%à4=Ð=ˆÔ˜DÔ1Ý+ØØð
ð "ØØ!Øð
ð 
ð ð
ð 
ˆŒ	ð ð 	DÝŒG×!Ò! $Ô"8¸dÐ!ÑCÔCÐCÝŒG×!Ò! $Ô"9¸tÐ!ÑDÔDÐDÐDàŒ~Ð)Ý”×%Ò% d¤n¸$Ð%Ñ?Ô?Ð?ØÔ!Ð-Ý”×%Ò% dÔ&8¸dÐ%ÑCÔCÐCà˜&Ò Ð Ø)4¸Ò)=Ð)=•o�oÅ?ˆGÝ˜g°yÐAÑAÔAˆGÝ˜ Ñ&Ô&Ð&Øð 	#Ø× Ò Ñ"Ô"Ð"Ý�d”i”l¥B¤IÑ.Ô.ð 	9ØŒIŒLÔÔ$×)Ò)¨/Ñ:Ô:Ð:ØŒIŒLÔÔ"×'Ò'¨Ñ8Ô8Ð8Ð8Ð8ð	9ð 	9r,   c                 óÌ   — d„ }t          | j        ¦  «        D ]K\  }} ||j        j        j        j        |dz   ¦  «          ||j        j        j        j        |dz   ¦  «         ŒLd S )Nc                 óZ   — |                       t          j        d|z  ¦  «        ¦  «         d S )Nrß   )Údiv_rf   Úsqrt)ÚparamÚ	_layer_ids     r*   Úrescalez&Hiera.fix_init_weight.<locals>.rescale|  s(   € Ø�JŠJ•t”y  y¡Ñ1Ô1Ñ2Ô2Ð2Ð2Ð2r,   r   )Ú	enumerater  r¨   r—   r(  r)  r½   Úfc2)r`   r;  Úlayer_idÚlayers       r*   r&  zHiera.fix_init_weight{  s}   € ð	3ð 	3ð 	3õ  )¨¬Ñ5Ô5ð 	=ð 	=‰OˆH�eØˆG�E”J”OÔ*Ô/°¸A±Ñ>Ô>Ð>ØˆG�E”I”MÔ(Ô-¨x¸!©|Ñ<Ô<Ð<Ð<ð	=ð 	=r,   c                 ó4   — | j         �dgS | j        �ddgS ddgS )Nr  Úpos_embed_absr  r  r  )r  rA  ©r`   s    r*   Úno_weight_decayzHiera.no_weight_decayƒ  s2   € àŒ>Ð%Ø�=Ð ØÔÐ+Ø# _Ð5Ð5à'Ð)=Ð>Ð>r,   Úcoarser#   c                 ó(   — t          dddg¬¦  «        S )NzW^pos_embed|pos_embed_spatial|pos_embed_temporal|pos_embed_abs|pos_embed_win|patch_embed)z^blocks\.(\d+)N)z^norm)iŸ† )Ústemr  )r  )r`   rD  s     r*   Úgroup_matcherzHiera.group_matcherŒ  s%   € åØkØ-Ð/CÐDð
ñ 
ô 
ð 	
r,   Úenablec                 ó   — || _         d S r„   )r  )r`   rH  s     r*   Úset_grad_checkpointingzHiera.set_grad_checkpointing“  s   € à"(ˆÔÐÐr,   c                 ó   — | j         j        S r„   )r#  r'  rB  s    r*   Úget_classifierzHiera.get_classifier—  s   € àŒyŒ|Ðr,   Úreset_otherc                 ó^   — t          | ¦  «        }|| _         | j        j        ||fd|i|¤Ž d S )NrM  )r   rì   r#  Úreset)r`   rì   rí   rM  r˜   s        r*   Úreset_classifierzHiera.reset_classifier›  s@   € Ý˜dÑ#Ô#ˆØ&ˆÔØˆŒ	Œ˜ [ÐPÐP¸kÐPÈRÐPÐPÐPÐPÐPr,   r9   Ú
mask_ratioc                 ó®  — |j         d         }t          j        | j        ¦  «        }t	          |d|z
  z  ¦  «        }t          j        |||j        ¬¦  «        }t          j        |d¬¦  «        }t          j        |d¬¦  «        }t          j	        ||g|j        ¬¦  «        }	d|	dd…d|…f<   t          j
        |	d|¬¦  «        }	|	                     ¦   «         S )zŠ
        Generates a random mask, mask_ratio fraction are dropped.
        1 is *keep*, 0 is *remove*. Useful for MAE, FLIP, etc.
        r   r   )r�   rœ   N)rˆ   Úindex)r3   rf   rg   r  rs   rt   Úrandr�   Úargsortr  Úgatherr«   )
r`   r9   rQ  rO   r¤   Úlen_keepÚnoiseÚids_shuffleÚids_restorer.   s
             r*   Úget_random_maskzHiera.get_random_mask   sÐ   € ð
 ŒG�AŒJˆå”i Ô 7Ñ8Ô8ˆÝ�{ a¨*¡nÑ5Ñ6Ô6ˆÝ”
˜1˜k°!´(Ð;Ñ;Ô;ˆõ ”m E¨qÐ1Ñ1Ô1ˆÝ”m K°QÐ7Ñ7Ô7ˆõ Œ{˜A˜{Ð+°A´HÐ=Ñ=Ô=ˆØˆˆQˆQˆQ�	��	ˆ\ÑåŒ|˜D a¨{Ð;Ñ;Ô;ˆà�yŠy‰{Œ{Ðr,   c                 óæ  — | j         �x| j                              | j        ¦  «        }t          j        | j        |j        dd …         dd¬¦  «        }||z   }|                     d¦  «                             dd¦  «        }nl| j        �| j        }n]| j	         
                    d| j        d         d¦  «        t          j        | j        | j        d         | j        d         z  d¬¦  «        z   }||z   }|S )	Nr�   ÚbicubicT)r1   ÚmodeÚ	antialiasr0   r   r   rœ   )r  Útiler  r4   r5   r  r3   re   r¢   r  Úrepeatr  rt   Úrepeat_interleaver  )r`   r9   r  r  s       r*   Ú
_pos_embedzHiera._pos_embed¸  s
  € ØÔÐ)ð !Ô.×3Ò3°DÔ4KÑLÔLˆMÝœØ”Ø"Ô(¨¨¨Ô-ØØð	ñ ô ˆIð " MÑ1ˆIØ!×)Ò)¨!Ñ,Ô,×6Ò6°q¸!Ñ<Ô<ˆIˆIØŒ^Ð'ØœˆIˆIð Ô&×-Ò-¨a°Ô1JÈ1Ô1MÈqÑQÔQåÔ'ØÔ+ØÔ-¨aÔ0°4Ô3LÈQÔ3OÑOØðñ ô ñð ð �	‰MˆØˆr,   ÚNCHWr.   ÚindicesÚnormÚ
stop_earlyÚ
output_fmtÚintermediates_onlyc	           	      ód  ‡ — |r
J d¦   «         ‚|dv s
J d¦   «         ‚|rAt          t          ‰ j        ¦  «        |¦  «        \  }	}
ˆ fd„|	D ¦   «         }	‰ j        |
         }
n%t          t          ‰ j        ¦  «        |¦  «        \  }	}
|�  |j        |j        d         dg‰ j        ¢R Ž }nd}‰                      ||¬¦  «        }‰                      |¦  «        }‰  	                    |¦  «        }|�^||d	          
                    d‰ j        |j        d
         ¦  «                                      |j        d         d|j        d         ¦  «        }g }t          j                             ¦   «         s|s‰ j        }n‰ j        d|
dz   …         }t          |¦  «        D ]•\  }}‰ j        r/t          j                             ¦   «         st#          ||¦  «        }n ||¦  «        }||	v rK‰                      |||¬¦  «        }|                     |dk    r|                     dddd
¦  «        n|¦  «         Œ–|r|S ||fS )aè   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to all intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        z'normalization of features not supported)rd  ÚNHWCz(Output format must be one of NCHW, NHWC.c                 ó*   •— g | ]}‰j         |         ‘ŒS r>   )rz   )r?   r[   r`   s     €r*   rB   z/Hiera.forward_intermediates.<locals>.<listcomp>ï  s    ø€ ÐEÐEÐE°1˜DœO¨AÔ.ÐEÐEÐEr,   Nr   r   ©r.   ©.Nr0   r<   rd  r’   )r   r2   rz   r  rI   r3   r  r  rc  r  r`  r  rt   r©   Úis_scriptingr<  r  r    r  r   rL   )r`   r9   r.   re  rf  rg  rh  ri  rD  Útake_indicesÚ	max_indexÚ
patch_maskÚintermediatesr  r[   ÚblkÚx_ints   `                r*   Úforward_intermediateszHiera.forward_intermediatesÔ  sk  ø€ ð. ÐBÐBÐBÑBÔBˆxØÐ-Ð-Ð-Ð-Ð/YÑ-Ô-Ð-Øð 	VÝ&:½3¸t¼Ñ;OÔ;OÐQXÑ&YÔ&YÑ#ˆL˜)ØEÐEÐEÐE¸ÐEÑEÔEˆLØœ¨	Ô2ˆIˆIå&:½3¸t¼{Ñ;KÔ;KÈWÑ&UÔ&UÑ#ˆL˜)àÐØ"˜œ 1¤7¨1¤:¨qÐK°4Ô3JÐKÐKÐKˆJˆJàˆJØ×Ò˜Q ZÐÑ0Ô0ˆØ�OŠO˜AÑÔˆØ�KŠK˜‰NŒNˆð ÐØ�$�y”/×&Ò& q¨$¬,¸¼À¼
ÑCÔCÔD×IÒIÈ!Ì'ÐRSÌ*ÐVXÐZ[ÔZaÐbdÔZeÑfÔfˆAàˆÝŒ9×!Ò!Ñ#Ô#ð 	1¨:ð 	1Ø”[ˆFˆFà”[  )¨a¡- Ô0ˆFÝ Ñ'Ô'ð 	cð 	c‰FˆAˆsØÔ&ð ­u¬y×/EÒ/EÑ/GÔ/Gð Ý˜s AÑ&Ô&��à�C˜‘F”F�Ø�LÐ Ð ØŸš A q¨t˜Ñ4Ô4�Ø×$Ò$À*ÐPVÒBVÐBV U§]¢]°1°a¸¸AÑ%>Ô%>Ð%>Ð\aÑbÔbÐbøàð 	!Ø Ð à�-ÐÐr,   Ú
prune_normÚ
prune_headc                 ó$  — |r3t          t          | j        ¦  «        |¦  «        \  }}| j        |         }n%t          t          | j        ¦  «        |¦  «        \  }}| j        d|dz   …         | _        |r| j                             dd¬¦  «         |S )z@ Prune layers not required for specified intermediates.
        Nr   r   T)rM  )r   r2   rz   r  r#  rO  )r`   re  rw  rx  rD  rp  rq  s          r*   Úprune_intermediate_layerszHiera.prune_intermediate_layers  s”   € ð ð 	VÝ&:½3¸t¼Ñ;OÔ;OÐQXÑ&YÔ&YÑ#ˆL˜)Øœ¨	Ô2ˆIˆIå&:½3¸t¼{Ñ;KÔ;KÈWÑ&UÔ&UÑ#ˆL˜)Ø”k . 9¨q¡= .Ô1ˆŒØð 	1ØŒI�OŠO˜A¨4ˆOÑ0Ô0Ð0ØÐr,   Úreturn_intermediatesc                 ó&  — | j         r+| j        dk    r |�J ‚|                      || j        ¬¦  «        }|�  |j        |j        d         dg| j        ¢R Ž }nd}|                      ||¬¦  «        }|                      |¦  «        }|                      |¦  «        }|�^||d          	                    d| j
        |j        d         ¦  «                                      |j        d         d|j        d         ¦  «        }g }t          | j        ¦  «        D ]|\  }}| j        r/t          j                             ¦   «         st#          ||¦  «        }n ||¦  «        }|r4|| j        v r+|                     |                      |||¬¦  «        ¦  «         Œ}|r||fS |S )	zê
        mask should be a boolean tensor of shape [B, #MUt*#MUy*#MUx] where #MU are the number of mask units in that dim.
        Note: 1 in mask is *keep*, 0 is *remove*; mask.sum(dim=-1) should be the same across the batch.
        r   N)rQ  r   rm  rn  r0   r<   )Útrainingrù   r[  rI   r3   r  r  rc  r  r`  r  r<  r  r  rt   r©   ro  r    rz   r   r  )r`   r9   r.   r{  rr  rs  r[   rt  s           r*   Úforward_featureszHiera.forward_features&  sµ  € ð Œ=ð 	L˜TÔ1°AÒ5Ð5à�<�<�<Ø×'Ò'¨°dÔ6JÐ'ÑKÔKˆDàÐØ"˜œ 1¤7¨1¤:¨qÐK°4Ô3JÐKÐKÐKˆJˆJàˆJØ×Ò˜Q ZÐÑ0Ô0ˆØ�OŠO˜AÑÔˆØ�KŠK˜‰NŒNˆð ÐØ�$�y”/×&Ò& q¨$¬,¸¼À¼
ÑCÔCÔD×IÒIÈ!Ì'ÐRSÌ*ÐVXÐZ[ÔZaÐbdÔZeÑfÔfˆAàˆÝ ¤Ñ,Ô,ð 	Cð 	C‰FˆAˆsØÔ&ð ­u¬y×/EÒ/EÑ/GÔ/Gð Ý˜s AÑ&Ô&��à�C˜‘F”F�Ø#ð C¨¨T¬_Ð(<Ð(<Ø×$Ò$ T§[¢[°°A¸D [Ñ%AÔ%AÑBÔBÐBøð  ð 	$Ø�mÐ#Ð#àˆr,   Ú
pre_logitsc                 ób   — |r|                       ||¬¦  «        n|                       |¦  «        }|S )N)r  )r#  )r`   r9   r  s      r*   Úforward_headzHiera.forward_headS  s0   € Ø3=ÐOˆD�IŠI�a JˆIÑ/Ô/Ð/À4Ç9Â9ÈQÁ<Ä<ˆØˆr,   c                 ób   — |                       ||¬¦  «        }|€|                      |¦  «        }|S )Nrm  )r~  r�  rÒ   s      r*   rm   zHiera.forwardW  s9   € ð
 ×!Ò! !¨$Ð!Ñ/Ô/ˆØˆ<Ø×!Ò! !Ñ$Ô$ˆAØˆr,   )rÔ   r’   rÖ   r   r×   rØ   rÙ   r’   rÛ   rÜ   rÞ   Trß   rß   rà   râ   rã   r¯   r°   NTrä   rÄ   r°   r°   rå   FFræ   NN©F)T)NF)NNFTrd  FT)r   FTTr„   )"ro   rp   rq   r   rs   Ústrr«   r6   r   r	   r   r%   rÆ   r^   r&  rt   r©   ÚignorerC  r   rG  rJ  rL  rP  ru   r[  rc  r   rv  rz  r~  r�  rm   rv   rw   s   @r*   r!   r!   Î  so  ø€ € € € € ð )3ØØØØ#Ø$Ø&3ØØ(.Ø.4à/IØ$(Ø Ø!Ø,2Ø,2Ø-3Ø"Ø$'Ø+/Ø!Ø!Ø6AØ"Ø%(Ø%*Ø"'Ø&+Ø/7ØØðCi9ð i9à˜C ˜H”oði9ð ði9ð ð	i9ð
 ði9ð ði9ð ði9ð ˜#˜s˜(”Oði9ð ði9ð ˜C ˜H”oði9ð " # s (œOði9ð " $¨ )Ô,ði9ð "ði9ð ði9ð  ð!i9ð"    S œ/ð#i9ð$    S œ/ð%i9ð& !  c œ?ð'i9ð( ð)i9ð* "ð+i9ð, " %œð-i9ð. ð/i9ð0 ð1i9ð2 ˜c 4¨¬	¤?Ð2Ô3ð3i9ð4 ð5i9ð6 #ð7i9ð8 #ð9i9ð:  ð;i9ð<  $ð=i9ð> # 3¨ 8œ_ð?i9ð i9ð i9ð i9ð i9ð i9ðV=ð =ð =ð „YÔð?ð ?ñ Ôð?ð „YÔð
ð 
 Dð 
°Tð 
ð 
ð 
ñ Ôð
ð „YÔð)ð )¨Tð )¸Tð )ð )ð )ñ Ôð)ð „YÔðð ñ ÔððQð Q¨Cð Q¸hÀs¼mð QÐaeð Qð Qð Qð Qð
 ¤ð ¸5ð ÀUÄ\ð ð ð ð ð0˜uœ|ð ð ð ð ð> ,0Ø7;ØØ#Ø$Ø',Øð= ð = àŒ|ð= ð ˜5œ<Ô(ð= ð ˜e C¨¨c¬ NÔ3Ô4ð	= ð
 ð= ð ð= ð ð= ð !%ð= ð ð= ð 
ˆt�E”LÔ! 5¨¬°t¸E¼LÔ7IÐ)IÔ#JÐJÔ	Kð= ð = ð = ð = ðB ./Ø$Ø#Øðð à˜3  S¤	˜>Ô*ðð ðð ð	ð
 ðð ð ð ð, ,0Ø).ð	+ð +àŒ|ð+ð ˜5œ<Ô(ð+ð #'ð	+ð
 
Œð+ð +ð +ð +ðZð ¨$ð ¸5¼<ð ð ð ð ð ,0ðð àŒ|ðð ˜5œ<Ô(ðð 
Œð	ð ð ð ð ð ð ð r,   rä   c                 ó8   — | ddd dddt           t          dddd	œ|¥S )
Nr×   )r’   rÕ   rÕ   gÍÌÌÌÌÌì?r]  Tzpatch_embed.projr  z
apache-2.0)Úurlrì   rU   Ú	pool_sizeÚcrop_pctÚinterpolationÚfixed_input_sizerÂ   r	  Ú
first_convÚ
classifierÚlicenser
   )r‡  Úkwargss     r*   Ú_cfgr�  b  s:   € àØ¨=ÀtØ¨ÈÝ%Õ.BØ(¸	Øðð ð ðð r,   zhiera_tiny_224.mae_in1k_ft_in1kztimm/zcc-by-nc-4.0)Ú	hf_hub_idrŽ  zhiera_tiny_224.mae)r‘  rŽ  rì   z hiera_small_224.mae_in1k_ft_in1kzhiera_small_224.maezhiera_base_224.mae_in1k_ft_in1kzhiera_base_224.maez$hiera_base_plus_224.mae_in1k_ft_in1kzhiera_base_plus_224.maez hiera_large_224.mae_in1k_ft_in1kzhiera_large_224.maezhiera_huge_224.mae_in1k_ft_in1kzhiera_huge_224.maez.hiera_small_abswin_256.sbb2_e200_in12k_ft_in1k)r’   é   r’  gffffffî?)r‘  rU   r‰  z1hiera_small_abswin_256.sbb2_pd_e200_in12k_ft_in1kz&hiera_small_abswin_256.sbb2_e200_in12ki-.  )r‘  rì   rU   r‰  z)hiera_small_abswin_256.sbb2_pd_e200_in12kzhiera_base_abswin_256.untrained)rU   r‰  c                 óf  — |                       d| ¦  «        } i }|                      ¦   «         D ]ƒ\  }}d|v r|                     dd¦  «        }|                     d¦  «        r|                     dd¦  «        }n+|                     d¦  «        r|                     dd¦  «        }|dk    rd}|||<   Œ„|S )	NÚmodel_statezhead.projection.zhead.fc.zencoder_norm.z
head.norm.znorm.rA  r  )ÚgetÚitemsÚreplaceÚ
startswith)Ú
state_dictÚmodelÚoutputr¦   r§   s        r*   Úcheckpoint_filter_fnrœ  Ä  sÊ   € Ø—’ ¨zÑ:Ô:€JØ€FØ× Ò Ñ"Ô"ð ð ‰ˆˆ1ð  Ð"Ð"Ø—	’	Ð,¨jÑ9Ô9ˆAØ�<Š<˜Ñ(Ô(ð 	1Ø—	’	˜/¨<Ñ8Ô8ˆAˆAØ�\Š\˜'Ñ"Ô"ð 	1Ø—	’	˜' <Ñ0Ô0ˆAØ�ÒÐØˆAØˆˆq‰	ˆ	Ø€Mr,   FÚvariantÚ
pretrainedc                 ó„   — |                      dd¦  «        }t          t          | |ft          t	          |d¬¦  «        dœ|¤ŽS )NÚout_indicesrš   Úgetter)r   Úfeature_cls)Úpretrained_filter_fnÚfeature_cfg)Úpopr   r!   rœ  r  )r�  rž  r�  r   s       r*   Ú_create_hierar¦  Ý  sY   € Ø—*’*˜]¨AÑ.Ô.€KåÝØØðõ 2Ý [¸hÐGÑGÔGðð ð ðð ð r,   c           	      óX   — t          ddd¬¦  «        }t          dd| it          |fi |¤Ž¤ŽS )NrÖ   r   )r   r0   rá   r0   ©rê   rë   rî   Úhiera_tiny_224rž  )r©  ©r  r¦  ©rž  r�  Ú
model_argss      r*   r©  r©  ê  s@   € å ¨a¸ÐEÑEÔE€JÝÐ_Ð_°jÐ_ÅDÈÐD^ÐD^ÐW]ÐD^ÐD^Ð_Ð_Ð_r,   c           	      óX   — t          ddd¬¦  «        }t          dd| it          |fi |¤Ž¤ŽS )NrÖ   r   ©r   r0   é   r0   r¨  Úhiera_small_224rž  )r°  rª  r«  s      r*   r°  r°  ð  s@   € å ¨a¸ÐFÑFÔF€JÝÐ`Ð`°zÐ`ÅTÈ*ÐE_ÐE_ÐX^ÐE_ÐE_Ð`Ð`Ð`r,   c           	      óX   — t          ddd¬¦  «        }t          dd| it          |fi |¤Ž¤ŽS )NrÖ   r   rÙ   r¨  Úhiera_base_224rž  )r²  rª  r«  s      r*   r²  r²  ö  s@   € å ¨a¸ÐFÑFÔF€JÝÐ_Ð_°jÐ_ÅDÈÐD^ÐD^ÐW]ÐD^ÐD^Ð_Ð_Ð_r,   c           	      óX   — t          ddd¬¦  «        }t          dd| it          |fi |¤Ž¤ŽS )Nép   r0   rÙ   r¨  Úhiera_base_plus_224rž  )rµ  rª  r«  s      r*   rµ  rµ  ü  s@   € å ¨q¸ÐGÑGÔG€JÝÐdÐd¸:ÐdÍÈjÐIcÐIcÐ\bÐIcÐIcÐdÐdÐdr,   c           	      óX   — t          ddd¬¦  «        }t          dd| it          |fi |¤Ž¤ŽS )Né�   r0   ©r0   é   é$   rš   r¨  Úhiera_large_224rž  )r»  rª  r«  s      r*   r»  r»    s@   € å ¨q¸ÐGÑGÔG€JÝÐ`Ð`°zÐ`ÅTÈ*ÐE_ÐE_ÐX^ÐE_ÐE_Ð`Ð`Ð`r,   c           	      óX   — t          ddd¬¦  «        }t          dd| it          |fi |¤Ž¤ŽS )Nr’  rš   r¸  r¨  Úhiera_huge_224rž  )r½  rª  r«  s      r*   r½  r½    s@   € å ¨q¸ÐGÑGÔG€JÝÐ_Ð_°jÐ_ÅDÈÐD^ÐD^ÐW]ÐD^ÐD^Ð_Ð_Ð_r,   c           
      ób   — t          dddddddd¬	¦  «        }t          dd| it          |fi |¤Ž¤ŽS )NrÖ   r   r®  T)rÚ   rÚ   çñhãˆµøä>r  F)rê   rë   rî   rü   rý   r³   r÷   r¶   Úhiera_small_abswin_256rž  )rÀ  rª  r«  s      r*   rÀ  rÀ    sV   € åØ ¨-È4ÐaiØ e¸Uðñ ô €Jõ ÐgÐg¸jÐgÍDÐQ[ÐLfÐLfÐ_eÐLfÐLfÐgÐgÐgr,   c           	      ó^   — t          dddddd¬¦  «        }t          d
d	| it          |fi |¤Ž¤ŽS )NrÖ   r   rÙ   Tr¿  r  )rê   rë   rî   rü   r³   r÷   Úhiera_base_abswin_256rž  )rÂ  rª  r«  s      r*   rÂ  rÂ    sS   € åØ ¨-È4Ð]aÐotðvñ vô v€JåÐfÐf¸ZÐfÍ4ÐPZÐKeÐKeÐ^dÐKeÐKeÐfÐfÐfr,   )rä   r„   rƒ  )Hrr   rf   Ú	functoolsr   Útypingr   r   r   r   r   r	   rt   Útorch.nnr%   Útorch.nn.functionalÚ
functionalr4   Ú	timm.datar   r   Útimm.layersr   r   r   r   r   r   r   r   r   r   r   r   Ú	_registryr   r   Ú_builderr   Ú	_featuresr   Ú_features_fxr   Ú_manipulater   r    Ú__all__rs   rÆ   r+   ru   r8   rR   rT   ry   r†   r®   rÈ   r!   r�  Údefault_cfgsrœ  r„  r«   r¦  r©  r°  r²  rµ  r»  r½  rÀ  rÂ  r>   r,   r*   ú<module>rÑ     s´  ððð ð0 €€€Ø Ð Ð Ð Ð Ð Ø ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;à €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð à AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð =Ð <Ð <Ð <Ð <Ð <Ð <Ð <Ø *Ð *Ð *Ð *Ð *Ð *Ø +Ð +Ð +Ð +Ð +Ð +Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0ð ˆ)€ð=ˆsð =�t˜BœI”ð =ð =ð =ð =ð ð	 $ s¤)ð 	°5´<ð 	ÀEÄLð 	ð 	ð 	ñ Ôð	ðØŒ<ðà�CŒyðð �s”)ðð „\ð	ð ð ð ðB;ð ;ð ;ð ;ð ;ˆRŒYñ ;ô ;ð ;ð|Nð Nð Nð Nð NˆRŒYñ Nô Nð NðbXð Xð Xð Xð X˜œ	ñ Xô Xð XðvEð Eð Eð Eð E�”ñ Eô Eð EðP(ð (ð (ð (ð (�”ñ (ô (ð (ðVQð Qð Qð Qð QˆBŒIñ Qô Qð Qðh	ð 	ð 	ð 	ð %Ð$ð S&Ø% t tØØð(ñ (ô (ðS&ð
 ˜$˜$ØØØðñ ô ðS&ð '¨¨ØØð)ñ )ô )ðS&ð ˜4˜4ØØØðñ ô ðS&ð* & t tØØð(ñ (ô (ð+S&ð2 ˜$˜$ØØØðñ ô ð3S&ð> +¨D¨DØØð-ñ -ô -ð?S&ðF ˜t˜tØØØð ñ  ô  ðGS&ðR '¨¨ØØð)ñ )ô )ðSS&ðZ ˜4˜4ØØØðñ ô ð[S&ðf & t tØØð(ñ (ô (ðgS&ðn ˜$˜$ØØØðñ ô ðoS&ðz 5°d°dØØ ¨4ð7ñ 7ô 7ð{S&ðB 8¸¸ØØ ¨4ð:ñ :ô :ðCS&ðJ -¨d¨dØØØ ¨4ð/ñ /ô /ðKS&ðT 0°°ØØØ ¨4ð2ñ 2ô 2ðUS&ð^ & t tà ¨4ð(ñ (ô (ð_S&ñ Sô S€ðlð ð ð ð2
ð 
˜3ð 
¨Dð 
Àuð 
ð 
ð 
ð 
ð ð`ð `ð `ñ „ð`ð
 ðað að añ „ðað
 ð`ð `ð `ñ „ð`ð
 ðeð eð eñ „ðeð
 ðað að añ „ðað
 ð`ð `ð `ñ „ð`ð
 ðhð hð hñ „ðhð ðgð gð gñ „ðgð gð gr,   