§
    ·¯¸j‘X  ã                   ó  — d Z ddlZddlmZmZmZmZmZmZ ddl	Z	ddl
mZ ddlmZmZ ddlmZmZmZmZmZmZmZmZ ddlmZ ddlmZ dd	lmZ dd
lmZm Z  dgZ! G d„ dej"        ¦  «        Z# G d„ dej"        ¦  «        Z$ G d„ dej"        ¦  «        Z% G d„ dej"        ¦  «        Z&dd„Z' e e'd¬¦  «         e'd¬¦  «         e'd¬¦  «        dœ¦  «        Z(d„ Z)d d„Z*e d de&fd„¦   «         Z+e d de&fd„¦   «         Z,e d de&fd„¦   «         Z-dS )!a¥   Transformer in Transformer (TNT) in PyTorch

A PyTorch implement of TNT as described in
'Transformer in Transformer' - https://arxiv.org/abs/2103.00112

The official mindspore code is released and available at
https://gitee.com/mindspore/mindspore/tree/master/model_zoo/research/cv/TNT

The official pytorch code is released and available at
https://github.com/huawei-noah/Efficient-AI-Backbones/tree/master/tnt_pytorch
é    N)ÚListÚOptionalÚTupleÚUnionÚTypeÚAny©ÚIMAGENET_INCEPTION_MEANÚIMAGENET_INCEPTION_STD)ÚMlpÚDropPathÚcalculate_drop_path_ratesÚtrunc_normal_Ú_assertÚ	to_2tupleÚresample_abs_pos_embedÚget_device_dtypeé   )Úbuild_model_with_cfg)Úfeature_take_indices)Ú
checkpoint)Úgenerate_default_cfgsÚregister_modelÚTNTc                   óP   ‡ — e Zd ZdZ	 	 	 	 	 	 ddededed	ed
edefˆ fd„Zd„ Zˆ xZ	S )Ú	Attentionz Multi-Head Attention
    é   Fç        NÚdimÚ
hidden_dimÚ	num_headsÚqkv_biasÚ	attn_dropÚ	proj_dropc	                 ó¤  •— ||dœ}	t          ¦   «                              ¦   «          || _        || _        ||z  }
|
| _        |
dz  | _        t          j        ||dz  fd|i|	¤Ž| _        t          j        ||fd|i|	¤Ž| _	        t          j
        |d¬¦  «        | _        t          j        ||fi |	¤Ž| _        t          j
        |d¬¦  «        | _        d S )N©ÚdeviceÚdtypeg      à¿é   ÚbiasT)Úinplace)ÚsuperÚ__init__r    r!   Úhead_dimÚscaleÚnnÚLinearÚqkÚvÚDropoutr#   Úprojr$   )Úselfr   r    r!   r"   r#   r$   r'   r(   Úddr.   Ú	__class__s              €ú_/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/models/tnt.pyr-   zAttention.__init__    sâ   ø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ$ˆŒØ"ˆŒØ Ñ*ˆØ ˆŒØ Ñ%ˆŒ
å”)˜C ¨a¡ÐEÐE°hÐEÀ"ÐEÐEˆŒÝ”˜3 Ð9Ð9¨(Ð9°bÐ9Ð9ˆŒÝœ I°tÐ<Ñ<Ô<ˆŒÝ”I˜c 3Ð-Ð-¨"Ð-Ð-ˆŒ	Ýœ I°tÐ<Ñ<Ô<ˆŒˆˆó    c                 óº  — |j         \  }}}|                      |¦  «                             ||d| j        | j        ¦  «                             ddddd¦  «        }|                     d¦  «        \  }}|                      |¦  «                             ||| j        d¦  «                             dddd¦  «        }||                     dd¦  «        z  | j	        z  }	|	 
                    d¬¦  «        }	|                      |	¦  «        }	|	|z                       dd¦  «                             ||d¦  «        }|                      |¦  «        }|                      |¦  «        }|S )	Nr)   r   é   r   é   éÿÿÿÿéþÿÿÿ©r   )Úshaper2   Úreshaper!   r.   ÚpermuteÚunbindr3   Ú	transposer/   Úsoftmaxr#   r5   r$   )
r6   ÚxÚBÚNÚCr2   ÚqÚkr3   Úattns
             r9   ÚforwardzAttention.forward9   s2  € Ø”'‰ˆˆ1ˆaØ�WŠW�Q‰ZŒZ×Ò  1 a¨¬¸¼ÑGÔG×OÒOÐPQÐSTÐVWÐYZÐ\]Ñ^Ô^ˆØ�yŠy˜‰|Œ|‰ˆˆ1Ø�FŠF�1‰IŒI×Ò˜a  D¤N°BÑ7Ô7×?Ò?ÀÀ1ÀaÈÑKÔKˆà�A—K’K  BÑ'Ô'Ñ'¨4¬:Ñ5ˆØ�|Š| ˆ|Ñ#Ô#ˆØ�~Š~˜dÑ#Ô#ˆà�A‰X× Ò   AÑ&Ô&×.Ò.¨q°!°RÑ8Ô8ˆØ�IŠI�a‰LŒLˆØ�NŠN˜1ÑÔˆØˆr:   )r   Fr   r   NN)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚintÚboolÚfloatr-   rN   Ú__classcell__©r8   s   @r9   r   r      s©   ø€ € € € € ðð ð Ø"Ø!Ø!ØØð=ð =àð=ð ð=ð ð	=ð
 ð=ð ð=ð ð=ð =ð =ð =ð =ð =ð2ð ð ð ð ð ð r:   r   c                   ó¸   ‡ — e Zd ZdZdddddddej        ej        dddfded	ed
edededede	dededede
ej                 de
ej                 de	fˆ fd„Zd„ Zˆ xZS )ÚBlockz TNT Block
    r=   é   ç      @Fr   Nr   Údim_outÚ	num_pixelÚnum_heads_inÚnum_heads_outÚ	mlp_ratior"   r$   r#   Ú	drop_pathÚ	act_layerÚ
norm_layerÚlegacyc           	      ó  •— ||dœ}t          ¦   «                              ¦   «           ||fi |¤Ž| _        t          ||f|||	|dœ|¤Ž| _         ||fi |¤Ž| _        t          d	|t          |dz  ¦  «        |||dœ|¤Ž| _        || _	        | j	        r3 ||fi |¤Ž| _
        t          j        ||z  |fddi|¤Ž| _        d | _        n< |||z  fi |¤Ž| _
        t          j        ||z  |fddi|¤Ž| _         ||fi |¤Ž| _         ||fi |¤Ž| _        t          ||f|||	|dœ|¤Ž| _        |
dk    rt#          |
¦  «        nt          j        ¦   «         | _         ||fi |¤Ž| _        t          d	|t          ||z  ¦  «        |||dœ|¤Ž| _        d S )
Nr&   )r!   r"   r#   r$   r=   )Úin_featuresÚhidden_featuresÚout_featuresrb   Údropr*   TFr   © )r,   r-   Únorm_inr   Úattn_inÚnorm_mlp_inr   rS   Úmlp_inrd   Ú
norm1_projr0   r1   r5   Ú
norm2_projÚnorm_outÚattn_outr   ÚIdentityra   Únorm_mlpÚmlp)r6   r   r\   r]   r^   r_   r`   r"   r$   r#   ra   rb   rc   rd   r'   r(   r7   r8   s                    €r9   r-   zBlock.__init__M   s]  ø€ ð$ ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐà!�z #Ð,Ð,¨Ð,Ð,ˆŒÝ ØØð
ð #ØØØð
ð 
ð ð
ð 
ˆŒð &˜: cÐ0Ð0¨RÐ0Ð0ˆÔÝð 
ØÝ  a¡™LœLØØØð
ð 
ð ð
ð 
ˆŒð ˆŒØŒ;ð 	8Ø(˜j¨Ð3Ð3°Ð3Ð3ˆDŒOÝœ	 #¨	¡/°7ÐLÐLÀÐLÈÐLÐLˆDŒIØ"ˆDŒOˆOà(˜j¨¨y©Ð?Ð?¸BÐ?Ð?ˆDŒOÝœ	 #¨	¡/°7ÐMÐMÀÐMÈ"ÐMÐMˆDŒIØ(˜j¨Ð7Ð7°BÐ7Ð7ˆDŒOð #˜
 7Ð1Ð1¨bÐ1Ð1ˆŒÝ!ØØð
ð $ØØØð
ð 
ð ð
ð 
ˆŒð 1:¸B²°� )Ñ,Ô,Ð,ÅBÄKÁMÄMˆŒà"˜
 7Ð1Ð1¨bÐ1Ð1ˆŒÝð 
ØÝ ¨)Ñ 3Ñ4Ô4Ø ØØð
ð 
ð ð
ð 
ˆŒˆˆr:   c                 ó$  — ||                       |                      |                      |¦  «        ¦  «        ¦  «        z   }||                       |                      |                      |¦  «        ¦  «        ¦  «        z   }|                     ¦   «         \  }}}| j        €rt          j        |d d …dd…f         |d d …dd …f         |  	                    |  
                    |¦  «                             ||dz
  d¦  «        ¦  «        z   gd¬¦  «        }n„t          j        |d d …dd…f         |d d …dd …f         |                      |  	                    |  
                    |                     ||dz
  d¦  «        ¦  «        ¦  «        ¦  «        z   gd¬¦  «        }||                       |                      |                      |¦  «        ¦  «        ¦  «        z   }||                       |                      |                      |¦  «        ¦  «        ¦  «        z   }||fS )Nr   r   r>   r@   )ra   rl   rk   rn   rm   Úsizerp   ÚtorchÚcatr5   ro   rB   rr   rq   ru   rt   )r6   Úpixel_embedÚpatch_embedrH   rI   rJ   s         r9   rN   zBlock.forward—   sù  € à! D§N¢N°4·<²<ÀÇÂÈ[Ñ@YÔ@YÑ3ZÔ3ZÑ$[Ô$[Ñ[ˆØ! D§N¢N°4·;²;¸t×?OÒ?OÐP[Ñ?\Ô?\Ñ3]Ô3]Ñ$^Ô$^Ñ^ˆà×"Ò"Ñ$Ô$‰ˆˆ1ˆaØŒ?Ð"Ýœ)Ø˜A˜A˜A˜q ˜s˜FÔ#Ø˜A˜A˜A˜q˜r˜r˜EÔ" T§Y¢Y¨t¯ª¸{Ñ/KÔ/K×/SÒ/SÐTUÐWXÐ[\ÑW\Ð^`Ñ/aÔ/aÑ%bÔ%bÑbð%ð ðñ ô ˆKˆKõ
  œ)Ø˜A˜A˜A˜q ˜s˜FÔ#Ø˜A˜A˜A˜q˜r˜r˜EÔ" T§_¢_°T·Y²Y¸t¿ºÈ{×ObÒObÐcdÐfgÐjkÑfkÐmoÑOpÔOpÑ?qÔ?qÑ5rÔ5rÑ%sÔ%sÑsð%ð ðñ ô ˆKð " D§N¢N°4·=²=ÀÇÂÈ{ÑA[ÔA[Ñ3\Ô3\Ñ$]Ô$]Ñ]ˆØ! D§N¢N°4·8²8¸D¿MºMÈ+Ñ<VÔ<VÑ3WÔ3WÑ$XÔ$XÑXˆØ˜KÐ'Ð'r:   )rO   rP   rQ   rR   r0   ÚGELUÚ	LayerNormrS   rU   rT   r   ÚModuler-   rN   rV   rW   s   @r9   rY   rY   I   s&  ø€ € € € € ðð ð !"Ø!#Ø!Ø"Ø!Ø!Ø!Ø)+¬Ø*,¬,Ø ØØð!H
ð H
àðH
ð ðH
ð ð	H
ð
 ðH
ð ðH
ð ðH
ð ðH
ð ðH
ð ðH
ð ðH
ð ˜BœI”ðH
ð ˜RœYœðH
ð ðH
ð H
ð H
ð H
ð H
ð H
ðT(ð (ð (ð (ð (ð (ð (r:   rY   c                   ó  ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 dd	eeeeef         f         d
eeeeef         f         dedededefˆ fd„Zddeeeef         ef         fd„Z	d	eeef         deeef         fd„Z
dej        dej        dej        fd„Zˆ xZS )Ú
PixelEmbedz Image to Pixel Embedding
    éà   é   r<   é0   r=   FNÚimg_sizeÚ
patch_sizeÚin_chansÚin_dimÚstriderd   c	                 óD  •‡— ||dœ}	t          ¦   «                              ¦   «          t          |¦  «        }t          |¦  «        }|d         |d         z  |d         |d         z  f| _        | j        d         | j        d         z  }
|| _        || _        || _        |
| _        || _        ˆfd„|D ¦   «         }|| _	        t          j        || j        fdd‰dœ|	¤Ž| _        | j        rt          j        ||¬¦  «        | _        d S t          j        ||¬¦  «        | _        d S )	Nr&   r   r   c                 ó>   •— g | ]}t          j        |‰z  ¦  «        ‘ŒS rj   )ÚmathÚceil)Ú.0Úpsrˆ   s     €r9   ú
<listcomp>z'PixelEmbed.__init__.<locals>.<listcomp>Ç   s'   ø€ ÐFÐFÐF°R�$œ) B¨¡KÑ0Ô0ÐFÐFÐFr:   é   r<   )Úkernel_sizeÚpaddingrˆ   )r‘   rˆ   )r,   r-   r   Ú	grid_sizer„   r…   rd   Únum_patchesr‡   Únew_patch_sizer0   ÚConv2dr5   ÚUnfoldÚunfold)r6   r„   r…   r†   r‡   rˆ   rd   r'   r(   r7   r”   r•   r8   s        `      €r9   r-   zPixelEmbed.__init__°   s.  øø€ ð ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐÝ˜XÑ&Ô&ˆÝ˜zÑ*Ô*ˆ
à" 1œ+¨°A¬Ñ6¸À¼ÀzÐRSÄ}Ñ8TÐUˆŒØ”~ aÔ(¨T¬^¸AÔ->Ñ?ˆØ ˆŒØ$ˆŒØˆŒØ&ˆÔØˆŒØFÐFÐFÐF¸:ÐFÑFÔFˆØ,ˆÔå”I˜h¨¬ÐcÀÈAÐV\ÐcÐcÐ`bÐcÐcˆŒ	ØŒ;ð 	OÝœ)°À~ÐVÑVÔVˆDŒKˆKˆKåœ)°
À:ÐNÑNÔNˆDŒKˆKˆKr:   TÚreturnc                 ó<   — |rt          | j        ¦  «        S | j        S ©N)Úmaxr…   )r6   Ú	as_scalars     r9   Ú
feat_ratiozPixelEmbed.feat_ratioÐ   s"   € Øð 	#Ý�t”Ñ'Ô'Ð'à”?Ð"r:   c                 óZ   — |d         | j         d         z  |d         | j         d         z  fS )Nr   r   )r…   )r6   r„   s     r9   Údynamic_feat_sizezPixelEmbed.dynamic_feat_sizeÖ   s.   € Ø˜Œ{˜dœo¨aÔ0Ñ0°(¸1´+ÀÄÐQRÔASÑ2SÐSÐSr:   rG   Ú	pixel_posc                 óŠ  — |j         \  }}}}t          || j        d         k    d|› d|› d| j        d         › d| j        d         › d�	¦  «         t          || j        d         k    d|› d|› d| j        d         › d| j        d         › d�	¦  «         | j        rz|                      |¦  «        }|                      |¦  «        }|                     dd¦  «                             || j        z  | j	        | j
        d         | j
        d         ¦  «        }nt|                      |¦  «        }|                     dd¦  «                             || j        z  || j        d         | j        d         ¦  «        }|                      |¦  «        }||z   }|                     || j        z  | j	        d¦  «                             dd¦  «        }|S )	Nr   zInput image size (Ú*z) doesn't match model (r   z).r)   r>   )rA   r   r„   rd   r5   r˜   rE   rB   r”   r‡   r•   r…   )r6   rG   r¡   rH   rJ   ÚHÚWs          r9   rN   zPixelEmbed.forwardÙ   sÜ  € Ø”W‰
ˆˆ1ˆa�ÝØ�”˜qÔ!Ò!Øf ÐfÐf QÐfÐf¸t¼}ÈQÔ?OÐfÐfÐRVÔR_Ð`aÔRbÐfÐfÐfñ	hô 	hð 	hõ 	Ø�”˜qÔ!Ò!Øf ÐfÐf QÐfÐf¸t¼}ÈQÔ?OÐfÐfÐRVÔR_Ð`aÔRbÐfÐfÐfñ	hô 	hð 	hð Œ;ð 	Ø—	’	˜!‘”ˆAØ—’˜A‘”ˆAØ—’˜A˜qÑ!Ô!×)Ò)Ø�DÔ$Ñ$ d¤k°4Ô3FÀqÔ3IÈ4ÔK^Ð_`ÔKañcô cˆAˆAð —’˜A‘”ˆAØ—’˜A˜qÑ!Ô!×)Ò)¨!¨dÔ.>Ñ*>ÀÀ4Ä?ÐSTÔCUÐW[ÔWfÐghÔWiÑjÔjˆAØ—	’	˜!‘”ˆAØ�	‰MˆØ�IŠI�a˜$Ô*Ñ*¨D¬K¸Ñ<Ô<×FÒFÀqÈ!ÑLÔLˆØˆr:   )r�   r‚   r<   rƒ   r=   FNN©T)rO   rP   rQ   rR   r   rS   r   rT   r-   rž   r    rx   ÚTensorrN   rV   rW   s   @r9   r€   r€   ¬   sk  ø€ € € € € ðð ð
 58Ø68ØØØØ ØØðOð Oà˜C  s¨C x¤Ð0Ô1ðOð ˜c 5¨¨c¨¤?Ð2Ô3ðOð ð	Oð
 ðOð ðOð ðOð Oð Oð Oð Oð Oð@#ð #¨E°%¸¸S¸´/À3Ð2FÔ,Gð #ð #ð #ð #ðT¨%°°S°¬/ð T¸eÀCÈÀH¼oð Tð Tð Tð Tð˜œð °%´,ð À5Ä<ð ð ð ð ð ð ð ð r:   r€   c            )       óò  ‡ — e Zd ZdZdddddddd	d
d	dddddddej        d
dddfdeeeeef         f         deeeeef         f         dedede	dededededede
dede
de
de
de
de
d eej                 d!ed"ef(ˆ fd#„Zd$„ Zej        j        d%„ ¦   «         Zej        j        d=d&„¦   «         Zej        j        d>d(„¦   «         Zej        j        d)ej        fd*„¦   «         Zd?dedee	         fd+„Z	 	 	 	 	 	 d@d-ej        d.eeeee         f                  d/ed0ed1ed2e	d3ed)eeej                 eej        eej                 f         f         fd4„Z	 	 	 dAd.eeee         f         d6ed7efd8„Zd9„ Zd=d:efd;„Zd<„ Zˆ xZ S )Br   zC Transformer in Transformer - https://arxiv.org/abs/2103.00112
    r�   r‚   r<   éè  Útokeni   rƒ   rZ   r=   r[   Fr   Nr„   r…   r†   Únum_classesÚglobal_poolÚ	embed_dimÚ	inner_dimÚdepthÚnum_heads_innerÚnum_heads_outerr`   r"   Ú	drop_rateÚpos_drop_rateÚproj_drop_rateÚattn_drop_rateÚdrop_path_raterc   Úfirst_striderd   c                 ó  •‡‡— t          ¦   «                              ¦   «          ||dœ}|dv sJ ‚|| _        || _        || _        ‰x| _        x| _        | _        d| _        d| _	        t          d||||||dœ|¤Ž| _        | j        j        }t          | j        d¦  «        r| j                             ¦   «         n|Š|| _        | j        j        }|d         |d         z  } |||z  fi |¤Ž| _        t#          j        ||z  ‰fi |¤Ž| _         |‰fi |¤Ž| _        t#          j        t-          j        dd‰fi |¤Ž¦  «        | _        t#          j        t-          j        d|dz   ‰fi |¤Ž¦  «        | _        t#          j        t-          j        d||d         |d         fi |¤Ž¦  «        | _        t#          j        |¬¦  «        | _        t;          ||¦  «        }g }t=          |¦  «        D ]4}|                     tA          d|‰||	|
||||||         ||d	œ|¤Ž¦  «         Œ5t#          j!        |¦  «        | _"        ˆˆfd
„t=          |¦  «        D ¦   «         | _#         |‰fi |¤Ž| _$        t#          j        |¦  «        | _%        |dk    rt#          j        ‰|fi |¤Žnt#          j&        ¦   «         | _'        tQ          | j        d¬¦  «         tQ          | j        d¬¦  «         tQ          | j        d¬¦  «         |  )                    | j*        ¦  «         d S )Nr&   ©Ú rª   Úavgr   F)r„   r…   r†   r‡   rˆ   rd   rž   r   )Úp)r   r\   r]   r^   r_   r`   r"   r$   r#   ra   rc   rd   c                 ó:   •— g | ]}t          d |› �‰‰¬¦  «        ‘ŒS )zblocks.)ÚmoduleÚnum_chsÚ	reduction©Údict)r�   Úir­   Úrs     €€r9   r�   z TNT.__init__.<locals>.<listcomp>A  sA   ø€ ð ^ð ^ð ^ØKL�D˜ !˜˜¨yÀAÐFÑFÔFð^ð ^ð ^r:   ç{®Gáz”?©Ústdrj   )+r,   r-   r«   r†   r¬   Únum_featuresÚhead_hidden_sizer­   Únum_prefix_tokensÚgrad_checkpointingr€   rz   r”   Úhasattrrž   r•   ro   r0   r1   r5   rp   Ú	Parameterrx   ÚzerosÚ	cls_tokenÚ	patch_posr¡   r4   Úpos_dropr   ÚrangeÚappendrY   Ú
ModuleListÚblocksÚfeature_infoÚnormÚ	head_droprs   Úheadr   ÚapplyÚ_init_weights) r6   r„   r…   r†   r«   r¬   r­   r®   r¯   r°   r±   r`   r"   r²   r³   r´   rµ   r¶   rc   r·   rd   r'   r(   r7   r”   r•   r]   ÚdprrÕ   rÃ   rÄ   r8   s          `                       @€r9   r-   zTNT.__init__ó   sž  øøø€ õ2 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆØÐ2Ð2Ð2Ð2Ð2Ø&ˆÔØ ˆŒØ&ˆÔØENÐNˆÔÐN˜DÔ1°D´NØ!"ˆÔØ"'ˆÔå%ð 
ØØ!ØØØØð
ð 
ð ð
ð 
ˆÔð Ô&Ô2ˆÝ-4°TÔ5EÀ|Ñ-TÔ-TÐdˆDÔ×'Ò'Ñ)Ô)Ð)ÐZdˆØ&ˆÔØÔ)Ô8ˆØ" 1Ô%¨°qÔ(9Ñ9ˆ	à$˜* Y°Ñ%:ÐAÐA¸bÐAÐAˆŒÝ”I˜i¨)Ñ3°YÐEÐEÀ"ÐEÐEˆŒ	Ø$˜* YÐ5Ð5°"Ð5Ð5ˆŒåœ¥e¤k°!°Q¸	Ð&HÐ&HÀRÐ&HÐ&HÑIÔIˆŒÝœ¥e¤k°!°[À1±_ÀiÐ&VÐ&VÐSUÐ&VÐ&VÑWÔWˆŒÝœ¥e¤k°!°YÀÈqÔ@QÐSaÐbcÔSdÐ&kÐ&kÐhjÐ&kÐ&kÑlÔlˆŒÝœ
 ]Ð3Ñ3Ô3ˆŒå'¨¸Ñ>Ô>ˆØˆÝ�u‘”ð 	ð 	ˆAØ�MŠM�%ð ØØ!Ø#Ø,Ø-Ø#Ø!Ø(Ø(Ø˜aœ&Ø%Øðð ð ðð ñ ô ð ð õ ”m FÑ+Ô+ˆŒð^ð ^ð ^ð ^ð ^ÝPUÐV[ÑP\ÔP\ð^ñ ^ô ^ˆÔð �J˜yÐ/Ð/¨BÐ/Ð/ˆŒ	Ýœ IÑ.Ô.ˆŒØ?JÈQº¸•B”I˜i¨Ð;Ð;¸Ð;Ð;Ð;ÕTVÔT_ÑTaÔTaˆŒ	å�d”n¨#Ð.Ñ.Ô.Ð.Ý�d”n¨#Ð.Ñ.Ô.Ð.Ý�d”n¨#Ð.Ñ.Ô.Ð.Ø�
Š
�4Ô%Ñ&Ô&Ð&Ð&Ð&r:   c                 óÊ  — t          |t          j        ¦  «        rbt          |j        d¬¦  «         t          |t          j        ¦  «        r.|j        �)t          j                             |j        d¦  «         d S d S d S t          |t          j        ¦  «        rLt          j                             |j        d¦  «         t          j                             |j        d¦  «         d S d S )NrÅ   rÆ   r   g      ð?)	Ú
isinstancer0   r1   r   Úweightr*   ÚinitÚ	constant_r}   )r6   Úms     r9   rÛ   zTNT._init_weightsM  sÎ   € Ý�a�œÑ#Ô#ð 	-Ý˜!œ(¨Ð,Ñ,Ô,Ð,Ý˜!�RœYÑ'Ô'ð -¨A¬FÐ,>Ý”×!Ò! !¤&¨!Ñ,Ô,Ð,Ð,Ð,ð-ð -Ð,>Ð,>å˜�2œ<Ñ(Ô(ð 	-ÝŒG×Ò˜aœf aÑ(Ô(Ð(ÝŒG×Ò˜aœh¨Ñ,Ô,Ð,Ð,Ð,ð	-ð 	-r:   c                 ó
   — h d£S )N>   rÏ   rÐ   r¡   rj   ©r6   s    r9   Úno_weight_decayzTNT.no_weight_decayV  s   € à6Ð6Ð6Ð6r:   c                 ó,   — t          dddg¬¦  «        }|S )Nz=^cls_token|patch_pos|pixel_pos|pixel_embed|norm[12]_proj|proj)z^blocks\.(\d+)N)z^norm)iŸ† )ÚstemrÕ   rÁ   )r6   ÚcoarseÚmatchers      r9   Úgroup_matcherzTNT.group_matcherZ  s+   € åØQà)Ø$ðð
ñ 
ô 
ˆð ˆr:   Tc                 ó   — || _         d S r›   )rË   )r6   Úenables     r9   Úset_grad_checkpointingzTNT.set_grad_checkpointinge  s   € à"(ˆÔÐÐr:   r™   c                 ó   — | j         S r›   )rÙ   rä   s    r9   Úget_classifierzTNT.get_classifieri  s
   € àŒyÐr:   c                 óú   — t          | ¦  «        }|| _        |�|dv sJ ‚|| _        |dk    rt          j        | j        |fi |¤Žnt          j        ¦   «         | _        | j                             | j	        ¦  «         d S )Nr¹   r   )
r   r«   r¬   r0   r1   r­   rs   rÙ   ÚtrainÚtraining)r6   r«   r¬   r7   s       r9   Úreset_classifierzTNT.reset_classifierm  sˆ   € Ý˜dÑ#Ô#ˆØ&ˆÔØÐ"ØÐ"6Ð6Ð6Ð6Ð6Ø*ˆDÔØDOÐRSÂOÀO•B”I˜dœn¨kÐ@Ð@¸RÐ@Ð@Ð@ÕY[ÔYdÑYfÔYfˆŒ	ØŒ	�Š˜œÑ&Ô&Ð&Ð&Ð&r:   ÚNCHWrG   ÚindicesÚreturn_prefix_tokensr×   Ú
stop_earlyÚ
output_fmtÚintermediates_onlyc                 ó  ‡ ‡‡‡— |dv s
J d¦   «         ‚|dk    }g }	t          t          ‰ j        ¦  «        |¦  «        \  }
}|j        \  Š}}}‰                      |‰ j        ¦  «        }‰                      ‰                      ‰                      | 	                    ‰‰ j
        d¦  «        ¦  «        ¦  «        ¦  «        }t          j        ‰ j                             ‰dd¦  «        |fd¬¦  «        }|‰ j        z   }‰                      |¦  «        }t          j                             ¦   «         s|s‰ j        }n‰ j        d|dz   …         }t'          |¦  «        D ]~\  }}‰ j        r3t          j                             ¦   «         st+          |||¦  «        \  }}n |||¦  «        \  }}||
v r,|	                     |r‰                      |¦  «        n|¦  «         Œ‰ j        rˆ fd„|	D ¦   «         }ˆ fd	„|	D ¦   «         }	|r/‰ j                             ||f¦  «        \  ŠŠˆˆˆfd
„|	D ¦   «         }	t          j                             ¦   «         s|rt5          t7          |	|¦  «        ¦  «        }	|r|	S ‰                      |¦  «        }||	fS )a<   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if an int, if is a sequence, select by matching indices
            return_prefix_tokens: Return both prefix and spatial intermediate tokens
            norm: Apply norm layer to all intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )rô   ÚNLCz)Output format must be one of NCHW or NLC.rô   r>   r   r@   Nc                 ó6   •— g | ]}|d d …d‰j         …f         ‘ŒS )Nr   ©rÊ   ©r�   Úyr6   s     €r9   r�   z-TNT.forward_intermediates.<locals>.<listcomp>­  s/   ø€ ÐSÐSÐSÀ˜Q˜q˜q˜q ! DÔ$:Ð":Ð:Ô;ÐSÐSÐSr:   c                 ó6   •— g | ]}|d d …‰j         d …f         ‘ŒS r›   rý   rþ   s     €r9   r�   z-TNT.forward_intermediates.<locals>.<listcomp>®  s0   ø€ ÐRÐRÐR¸q˜Q˜q˜q˜q $Ô"8Ð"9Ð"9Ð9Ô:ÐRÐRÐRr:   c                 ó�   •— g | ]B}|                      ‰‰‰d ¦  «                             dddd¦  «                             ¦   «         ‘ŒCS )r>   r   r<   r   r)   )rB   rC   Ú
contiguous)r�   rÿ   rH   r¤   r¥   s     €€€r9   r�   z-TNT.forward_intermediates.<locals>.<listcomp>³  sL   ø€ ÐlÐlÐlÐYZ˜QŸYšY q¨!¨Q°Ñ3Ô3×;Ò;¸A¸qÀ!ÀQÑGÔG×RÒRÑTÔTÐlÐlÐlr:   )r   ÚlenrÕ   rA   rz   r¡   rp   r5   ro   rB   r”   rx   ry   rÏ   ÚexpandrÐ   rÑ   ÚjitÚis_scriptingÚ	enumeraterË   r   rÓ   r×   rÊ   r    ÚlistÚzip)r6   rG   rõ   rö   r×   r÷   rø   rù   rB   ÚintermediatesÚtake_indicesÚ	max_indexÚ_ÚheightÚwidthrz   r{   rÕ   rÃ   ÚblkÚprefix_tokensrH   r¤   r¥   s   `                    @@@r9   Úforward_intermediateszTNT.forward_intermediatesv  sÉ  øøøø€ ð. ˜_Ð,Ð,Ð,Ð.YÑ,Ô,Ð,Ø Ò&ˆØˆÝ"6µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�ið  œgÑˆˆ1ˆf�eà×&Ò& q¨$¬.Ñ9Ô9ˆà—o’o d§i¢i°·²À×@SÒ@SÐTUÐW[ÔWgÐikÑ@lÔ@lÑ0mÔ0mÑ&nÔ&nÑoÔoˆÝ”i ¤×!6Ò!6°q¸"¸bÑ!AÔ!AÀ;Ð OÐUVÐWÑWÔWˆØ! D¤NÑ2ˆØ—m’m KÑ0Ô0ˆåŒ9×!Ò!Ñ#Ô#ð 	1¨:ð 	1Ø”[ˆFˆFà”[  )¨a¡- Ô0ˆFå Ñ'Ô'ð 	Vð 	V‰FˆAˆsØÔ&ð I­u¬y×/EÒ/EÑ/GÔ/Gð IÝ+5°c¸;ÈÑ+TÔ+TÑ(�˜[˜[à+.¨3¨{¸KÑ+HÔ+HÑ(�˜[Ø�LÐ Ð à×$Ò$¸tÐ%T T§Y¢Y¨{Ñ%;Ô%;Ð%;ÈÑUÔUÐUøð Ô!ð 	SàSÐSÐSÐSÀ]ÐSÑSÔSˆMØRÐRÐRÐRÀMÐRÑRÔRˆMàð 	màÔ#×5Ò5°v¸u°oÑFÔF‰DˆAˆqØlÐlÐlÐlÐlÐlÐ^kÐlÑlÔlˆMÝŒy×%Ò%Ñ'Ô'ð 	DÐ,@ð 	Då ¥ ]°MÑ!BÔ!BÑCÔCˆMàð 	!Ø Ð à—i’i Ñ,Ô,ˆà˜MÐ)Ð)r:   r   Ú
prune_normÚ
prune_headc                 óâ   — t          t          | j        ¦  «        |¦  «        \  }}| j        d|dz   …         | _        |rt          j        ¦   «         | _        |r|                      dd¦  «         |S )z@ Prune layers not required for specified intermediates.
        Nr   r   rº   )r   r  rÕ   r0   rs   r×   ró   )r6   rõ   r  r  r  r  s         r9   Úprune_intermediate_layerszTNT.prune_intermediate_layers¿  sq   € õ #7µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�iØ”k . 9¨q¡= .Ô1ˆŒØð 	&Ýœ™œˆDŒIØð 	)Ø×!Ò! ! RÑ(Ô(Ð(ØÐr:   c                 ór  — |j         d         }|                      || j        ¦  «        }|                      |                      |                      |                     || j        d¦  «        ¦  «        ¦  «        ¦  «        }t          j	        | j
                             |dd¦  «        |fd¬¦  «        }|| j        z   }|                      |¦  «        }| j        D ]K}| j        r3t          j                             ¦   «         st%          |||¦  «        \  }}Œ< |||¦  «        \  }}ŒL|                      |¦  «        }|S )Nr   r>   r   r@   )rA   rz   r¡   rp   r5   ro   rB   r”   rx   ry   rÏ   r  rÐ   rÑ   rÕ   rË   r  r  r   r×   )r6   rG   rH   rz   r{   r  s         r9   Úforward_featureszTNT.forward_featuresÏ  s,  € ØŒG�AŒJˆØ×&Ò& q¨$¬.Ñ9Ô9ˆà—o’o d§i¢i°·²À×@SÒ@SÐTUÐW[ÔWgÐikÑ@lÔ@lÑ0mÔ0mÑ&nÔ&nÑoÔoˆÝ”i ¤×!6Ò!6°q¸"¸bÑ!AÔ!AÀ;Ð OÐUVÐWÑWÔWˆØ! D¤NÑ2ˆØ—m’m KÑ0Ô0ˆà”;ð 	Ið 	IˆCØÔ&ð I­u¬y×/EÒ/EÑ/GÔ/Gð IÝ+5°c¸;ÈÑ+TÔ+TÑ(�˜[˜[à+.¨3¨{¸KÑ+HÔ+HÑ(�˜[˜[à—i’i Ñ,Ô,ˆØÐr:   Ú
pre_logitsc                 óè   — | j         r>| j         dk    r'|d d …| j        d …f                              d¬¦  «        n|d d …df         }|                      |¦  «        }|r|n|                      |¦  «        S )Nr»   r   r@   r   )r¬   rÊ   ÚmeanrØ   rÙ   )r6   rG   r  s      r9   Úforward_headzTNT.forward_headá  s‹   € ØÔð 	dØ=AÔ=MÐQVÒ=VÐ=V��!�!�!�TÔ+Ð,Ð,Ð,Ô-×2Ò2°qÐ2Ñ9Ô9Ð9Ð\]Ð^_Ð^_Ð^_ÐabÐ^bÔ\cˆAØ�NŠN˜1ÑÔˆØÐ0ˆqˆq D§I¢I¨a¡L¤LÐ0r:   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S r›   )r  r  )r6   rG   s     r9   rN   zTNT.forwardç  s-   € Ø×!Ò! !Ñ$Ô$ˆØ×Ò˜aÑ Ô ˆØˆr:   ©Fr¦   r›   )NFFFrô   F)r   FT)!rO   rP   rQ   rR   r0   r}   r   rS   r   ÚstrrU   rT   r   r~   r-   rÛ   rx   r  Úignorerå   rê   rí   rï   r   ró   r§   r   r  r  r  r  rN   rV   rW   s   @r9   r   r   ï   sÀ  ø€ € € € € ðð ð
 58Ø68ØØ#Ø&Ø ØØØ#$Ø#%Ø!Ø"Ø!Ø#%Ø$&Ø$&Ø$&Ø*,¬,Ø !Ø ØØð/X'ð X'à˜C  s¨C x¤Ð0Ô1ðX'ð ˜c 5¨¨c¨¤?Ð2Ô3ðX'ð ð	X'ð
 ðX'ð ðX'ð ðX'ð ðX'ð ðX'ð !ðX'ð !ðX'ð ðX'ð ðX'ð ðX'ð !ðX'ð  "ð!X'ð" "ð#X'ð$ "ð%X'ð& ˜RœYœð'X'ð( ð)X'ð* ð+X'ð X'ð X'ð X'ð X'ð X'ðt-ð -ð -ð „YÔð7ð 7ñ Ôð7ð „YÔðð ð ñ Ôðð „YÔð)ð )ð )ñ Ôð)ð „YÔð ¤	ð ð ð ñ Ôðð'ð '¨Cð '¸hÀs¼mð 'ð 'ð 'ð 'ð 8<Ø).ØØ$Ø$Ø',ðG*ð G*àŒ|ðG*ð ˜e C¨¨c¬ NÔ3Ô4ðG*ð #'ð	G*ð
 ðG*ð ðG*ð ðG*ð !%ðG*ð 
ˆt�E”LÔ! 5¨¬°t¸E¼LÔ7IÐ)IÔ#JÐJÔ	KðG*ð G*ð G*ð G*ðV ./Ø$Ø#ð	ð à˜3  S¤	˜>Ô*ðð ðð ð	ð ð ð ð ð ð ð$1ð 1¨$ð 1ð 1ð 1ð 1ðð ð ð ð ð ð r:   rº   c                 ó>   — | ddd dddt           t          dddd	d
ddœ|¥S )Nr©   )r<   r�   r�   gÍÌÌÌÌÌì?ÚbicubicTzpixel_embed.projrÙ   zarXiv:2103.00112zTransformer in TransformerzMhttps://github.com/huawei-noah/Efficient-AI-Backbones/tree/master/tnt_pytorchz
apache-2.0)Úurlr«   Ú
input_sizeÚ	pool_sizeÚcrop_pctÚinterpolationÚfixed_input_sizer  rÇ   Ú
first_convÚ
classifierÚ	paper_idsÚ
paper_nameÚ
origin_urlÚlicenser	   )r#  Úkwargss     r9   Ú_cfgr0  í  sC   € àØ¨=ÀtØ¨ÈÝ'Õ0FØ(¸Ø'Ø2ØeØðð ð ðð r:   ztimm/)Ú	hf_hub_id)ztnt_s_legacy_patch16_224.in1kztnt_s_patch16_224.in1kztnt_b_patch16_224.in1kc                 ó   — |                       dd ¦  «         d| v r| }�n®i }|                      ¦   «         D �]–\  }}|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd	¦  «        }|                     d
d¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|dk    rg|j        j        dk    rW|j        \  }}}t          |dz  ¦  «        x}}	||	z  |k    sJ ‚|                     ddd ¦  «                             ||||	¦  «        }|||<   �Œ˜	 |d         j        |j	        j        k    r%t          |d         |j        j        d ¬!¦  «        |d<   |S )"NÚouter_tokensrÐ   Ú	outer_posÚ	inner_posr¡   r{   rz   Ú
proj_norm1ro   Ú
proj_norm2rp   Úinner_norm1rk   Ú
inner_attnrl   Úinner_norm2rm   Ú	inner_mlprn   Úouter_norm1rq   Ú
outer_attnrr   Úouter_norm2rt   Ú	outer_mlpru   Fg      à?r   r)   r   )Únew_sizerÊ   )ÚpopÚitemsÚreplacerz   rd   rA   rS   rC   rB   rÐ   r   r“   )
Ú
state_dictÚmodelÚout_dictrL   r3   rH   rI   rJ   r¤   r¥   s
             r9   Úcheckpoint_filter_fnrG    s  € Ø‡N‚N�> 4Ñ(Ô(Ð(Ø�jÐ Ð Øˆ‰àˆØ×$Ò$Ñ&Ô&ð 	ñ 	‰DˆAˆqØ—	’	˜+ {Ñ3Ô3ˆAØ—	’	˜+ {Ñ3Ô3ˆAØ—	’	˜-¨Ñ7Ô7ˆAØ—	’	˜,¨Ñ5Ô5ˆAØ—	’	˜,¨Ñ5Ô5ˆAØ—	’	˜-¨Ñ3Ô3ˆAØ—	’	˜,¨	Ñ2Ô2ˆAØ—	’	˜-¨Ñ7Ô7ˆAØ—	’	˜+ xÑ0Ô0ˆAØ—	’	˜-¨Ñ4Ô4ˆAØ—	’	˜,¨
Ñ3Ô3ˆAØ—	’	˜-¨Ñ4Ô4ˆAØ—	’	˜+ uÑ-Ô-ˆAØ�KÒÐ EÔ$5Ô$<ÀÒ$EÐ$EØœ'‘��1�aÝ˜A ™H™œÐ%��AØ˜1‘u ’z�z�z�zØ—I’I˜a  AÑ&Ô&×.Ò.¨q°!°Q¸Ñ:Ô:�ØˆH�Q‰K‰KàTØ�ÔÔ" e¤oÔ&;Ò;Ð;Ý 6Ø�[Ô!ØÔ&Ô0Øð!
ñ !
ô !
ˆ�Ñð
 €Or:   Fc                 óˆ   — |                      dd¦  «        }t          t          | |ft          t	          |d¬¦  «        dœ|¤Ž}|S )NÚout_indicesr<   Úgetter)rI  Úfeature_cls)Úpretrained_filter_fnÚfeature_cfg)rA  r   r   rG  rÂ   )ÚvariantÚ
pretrainedr/  rI  rE  s        r9   Ú_create_tntrP  1  sY   € Ø—*’*˜]¨AÑ.Ô.€KÝ ÝˆW�jðå1Ý [¸hÐGÑGÔGðð ð ð	ð €Eð
 €Lr:   r™   c           	      ód   — t          ddddddd¬¦  «        }t          dd
| it          |fi |¤Ž¤Ž}|S )Nr‚   é€  é   rZ   é   FT)r…   r­   r®   r¯   r±   r"   rd   Útnt_s_legacy_patch16_224rO  )rU  ©rÂ   rP  ©rO  r/  Ú	model_cfgrE  s       r9   rU  rU  ;  sW   € åØ °¸"ÈaØ˜tð%ñ %ô %€Iõ ÐgÐg¸zÐgÍTÐR[ÐMfÐMfÐ_eÐMfÐMfÐgÐg€EØ€Lr:   c           	      ób   — t          dddddd¬¦  «        }t          d
d	| it          |fi |¤Ž¤Ž}|S )Nr‚   rR  rS  rZ   rT  F©r…   r­   r®   r¯   r±   r"   Útnt_s_patch16_224rO  )r[  rV  rW  s       r9   r[  r[  D  sT   € åØ °¸"ÈaØðñ ô €Iõ Ð`Ð`¸
Ð`ÅdÈ9ÐF_ÐF_ÐX^ÐF_ÐF_Ð`Ð`€EØ€Lr:   c           	      ób   — t          dddddd¬¦  «        }t          d
d	| it          |fi |¤Ž¤Ž}|S )Nr‚   i€  é(   rZ   é
   FrZ  Útnt_b_patch16_224rO  )r_  rV  rW  s       r9   r_  r_  M  sT   € åØ °¸"ÈbØðñ ô €Iõ Ð`Ð`¸
Ð`ÅdÈ9ÐF_ÐF_ÐX^ÐF_ÐF_Ð`Ð`€EØ€Lr:   )rº   r  ).rR   r‹   Útypingr   r   r   r   r   r   rx   Útorch.nnr0   Ú	timm.datar
   r   Útimm.layersr   r   r   r   r   r   r   r   Ú_builderr   Ú	_featuresr   Ú_manipulater   Ú	_registryr   r   Ú__all__r~   r   rY   r€   r   r0  Údefault_cfgsrG  rP  rU  r[  r_  rj   r:   r9   ú<module>rj     sC  ðð
ð 
ð €€€Ø :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ð :à €€€Ø Ð Ð Ð Ð Ð à EÐ EÐ EÐ EÐ EÐ EÐ EÐ Eð Nð  Nð  Nð  Nð  Nð  Nð  Nð  Nð  Nð  Nð  Nð  Nð  Nð  Nð  Nð  Nð  Nð  Nð  Nð  NØ *Ð *Ð *Ð *Ð *Ð *Ø +Ð +Ð +Ð +Ð +Ð +Ø #Ð #Ð #Ð #Ð #Ð #Ø <Ð <Ð <Ð <Ð <Ð <Ð <Ð <àˆ'€ð*ð *ð *ð *ð *�”	ñ *ô *ð *ðZ`(ð `(ð `(ð `(ð `(ˆBŒIñ `(ô `(ð `(ðF@ð @ð @ð @ð @�”ñ @ô @ð @ðF{ð {ð {ð {ð {ˆ"Œ)ñ {ô {ð {ð|ð ð ð ð %Ð$Ø%) TØð&ñ &ô &ð #˜dØðñ ô ð #˜dØðñ ô ð&ð &ñ ô €ð "ð "ð "ðJð ð ð ð ðð ¸Cð ð ð ñ „ðð ðð °Sð ð ð ñ „ðð ðð °Sð ð ð ñ „ðð ð r:   