§
    ·¯¸jt4  ã            '       óî  — d Z ddlmZmZmZmZ ddlZddlmZ ddlm	Z	 ddl
mZmZmZ ddlmZ d	eed
f         dededeeeef                  fd„Z G d„ de	¦  «        Zdee         dee         deee                  deee                  deee                  deee                  dee         dededededededee         deeej        f         dee         d ed!ed"ee         f&d#„Zdee         dee         deee                  deee                  deee                  deee                  dee         dededededededee         deeej        f         dee         d ed!ed"ee         f&d$„ZdS )%aÌ   Adafactor (Big Vision variant) for PyTorch

Adapted from the implementation in big vision: https://github.com/google-research/big_vision

Described in 'Scaling Vision Transformers': https://arxiv.org/abs/2106.04560

References for added functionality:
    Cautious Optimizers: https://arxiv.org/abs/2411.16085
    Why Gradients Rapidly Increase Near the End of Training: https://arxiv.org/abs/2506.02285

Adaptation and PyTorch modifications by Ross Wightman
é    )ÚListÚOptionalÚTupleÚUnionN)ÚTensor)Ú	Optimizeré   )Ú
_get_valueÚ_init_scalarÚ_validate_scalar)ÚParamsTÚshape.ÚfactoredÚmin_dim_size_to_factorÚreturnc                 ó  — |rt          | ¦  «        dk     rdS t          d„ t          | ¦  «        D ¦   «         ¦  «        }| |d         d                  |k     rdS t          |d         d         ¦  «        t          |d         d         ¦  «        fS )aØ  Whether to use a factored second moment estimator.

    This function returns a tuple with the two largest axes to reduce over.
    If no two dimensions have size >= min_dim_size_to_factor, return None.

    Args:
      shape: an input shape
      factored: whether to use factored second-moment estimator for > 2d vars.
      min_dim_size_to_factor: only factor accumulator if two array dimensions have at least this size.

    Returns:
      None or a tuple of ints
    é   Nc              3   ó$   K  — | ]\  }}||fV — Œd S ©N© )Ú.0ÚiÚxs      úg/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/optim/adafactor_bv.pyú	<genexpr>z!_factored_dims.<locals>.<genexpr>+   s*   è è € Ð>Ð>¡T Q¨˜1˜a˜&Ð>Ð>Ð>Ð>Ð>Ð>ó    éþÿÿÿr	   éÿÿÿÿ)ÚlenÚsortedÚ	enumerateÚint)r   r   r   Úsorted_dimss       r   Ú_factored_dimsr$      s�   € ð$ ð •s˜5‘z”z A’~�~ØˆtÝÐ>Ð>­Y°uÑ-=Ô-=Ð>Ñ>Ô>Ñ?Ô?€KØˆ[˜Œ_˜QÔÔ Ð#9Ò9Ð9ØˆtÝˆ{˜2Œ˜qÔ!Ñ"Ô"¥C¨°B¬¸Ô(:Ñ$;Ô$;Ð;Ð;r   c            !       ó  ‡ — e Zd ZdZddddddej        dd	dd
d
d
fd
dœdedededededede	e         de
eej        f         de	e         dede	e         dededede	e         fˆ fd„Zˆ fd„Z ej        ¦   «         dd„¦   «         Zˆ xZS )ÚAdafactorBigVisionzÅ
    PyTorch implementation of BigVision's Adafactor variant with both single and multi tensor implementations.

    Adapted from https://github.com/google-research/big_vision by Ross Wightman
    ç      ð?é   gš™™™™™é?r   g+‡ÙÎ÷ï?gÍÌÌÌÌÌì?Ng        F)ÚforeachÚparamsÚlrr   Ú
decay_rateÚdecay_offsetÚ	beta2_capÚmomentumÚmomentum_dtypeÚepsÚweight_decayÚclipping_thresholdÚunscaled_wdÚcautionÚcorrected_weight_decayr)   c                óz  •— t          |t          ¦  «        rE|dk    rt          j        }n2|dk    rt          j        }n|dk    sJ |› d�¦   «         ‚t          j        }t          d|¦  «         t          d|
¦  «         t          ||||||||	|
|||||¬¦  «        }t          ¦   «          	                    ||¦  «         d S )NÚfloat16Úbfloat16Úfloat32z dtype not supportedzlearning rater2   )r+   r   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r)   )
Ú
isinstanceÚstrÚtorchr8   r9   r:   r   ÚdictÚsuperÚ__init__)Úselfr*   r+   r   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r)   ÚdefaultsÚ	__class__s                    €r   r@   zAdafactorBigVision.__init__8   sè   ø€ õ& �n¥cÑ*Ô*ð 	/Ø Ò*Ð*Ý!&¤��Ø :Ò-Ð-Ý!&¤��à%¨Ò2Ð2Ð2°~Ð4[Ð4[Ð4[Ñ2Ô2Ð2Ý!&¤�å˜¨"Ñ-Ô-Ð-Ý˜¨Ñ6Ô6Ð6åØØ#9Ø!Ø%ØØØ)ØØ%Ø1Ø#ØØ#9Øð
ñ 
ô 
ˆõ  	‰Œ×Ò˜ Ñ*Ô*Ð*Ð*Ð*r   c                 ó6  •— t          ¦   «                              |¦  «         | j        D ]î}|                     dd¦  «         |                     dd¦  «         |                     dd ¦  «         |d         D ]¡}| j                             |i ¦  «        }t          |¦  «        dk    r)d|v r%t          |d         dt          j	        ¬	¦  «        |d<   d
|v rDt          j
        |d
         ¦  «        r*|d
                              | j        d         ¬¦  «        |d
<   Œ¢Œïd S )Nr5   Fr6   r)   r*   r   ÚstepÚcpu©ÚdeviceÚdtypeÚexp_avgr0   ©rI   )r?   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetr   r   r=   Úfloat64Ú	is_tensorÚtorB   )rA   rO   ÚgroupÚpÚp_staterC   s        €r   rL   zAdafactorBigVision.__setstate__i   s3  ø€ Ý‰Œ×Ò˜UÑ#Ô#Ð#ØÔ&ð 	fð 	fˆEØ×Ò˜Y¨Ñ.Ô.Ð.Ø×ÒÐ5°uÑ=Ô=Ð=Ø×Ò˜Y¨Ñ-Ô-Ð-Ø˜8”_ð 	fð 	f�Øœ*Ÿ.š.¨¨BÑ/Ô/�Ý�w‘<”< 1Ò$Ð$¨°7Ð):Ð):Ý&2°7¸6´?È5ÕX]ÔXeÐ&fÑ&fÔ&f�G˜F‘Oà Ð'Ð'­E¬O¸GÀIÔ<NÑ,OÔ,OÐ'ð *1°Ô);×)>Ò)>ÀTÄ]ÐScÔEdÐ)>Ñ)eÔ)e�G˜IÑ&øð	fð		fð 	fr   c                 óÊ  — d }|�5t          j        ¦   «         5   |¦   «         }d d d ¦  «         n# 1 swxY w Y   | j        D �] }g }g }g }g }g }g }	g }
|d         D �]X}|j        €Œ|j        j        rt          d¦  «        ‚|                     |¦  «         |                     |j        ¦  «         | j        |         }t          |¦  «        dk    �r#t          dt           j
        ¬¦  «        |d<   |j        j        }t          |d| j        d         ¬	¦  «        }|�||\  }}t          |j        j        ¦  «        }d
||<   t          |j        j        ¦  «        }d
||<   |j                             |¦  «        |d<   |j                             |¦  «        |d<   n(t          j        |j        t           j        ¬¦  «        |d<   | j        d         �)t          j        |j        | j        d         ¬¦  «        |d<   |	                     |d         ¦  «         |                     |                     dd ¦  «        ¦  «         |                     |                     dd ¦  «        ¦  «         |                     |                     dd ¦  «        ¦  «         |
                     |                     dd ¦  «        ¦  «         �ŒZ|d         rt&          }nt(          } |d%i d|“d|“d|“d|“d|“d|
“d|	“d|d         “d|d         “d|d         “d|d         “d|d         “d|d         “d|d         “d|d         “d |d          “d!|d!         “d"|d"         “d#|d$         r| j        d         nd “Ž �Œ"|S )&Nr*   zSparse gradients not supportedr   rF   rG   rE   Tr   )r   r   r	   Úexp_avg_sq_rÚexp_avg_sq_c)Úmemory_formatÚ
exp_avg_sqr/   r0   rK   rJ   r)   ÚgradsÚexp_avg_sq_rsÚexp_avg_sq_csÚexp_avg_sqsÚexp_avgsÚstate_stepsÚbeta2_decayr,   r.   r1   r+   r2   r3   r4   r5   Úmax_lrr6   r   )r=   Úenable_gradrM   ÚgradÚ	is_sparseÚRuntimeErrorÚappendrO   r   r   rQ   r   r$   rB   ÚlistÚ	new_zerosÚ
zeros_likeÚpreserve_formatrP   Ú_multi_tensor_adafactorÚ_single_tensor_adafactor)rA   ÚclosureÚlossrT   Úparams_with_gradr\   r]   r^   r_   ra   r`   rU   rO   r   Úfactored_dimsÚdcÚdrÚ	row_shapeÚ	col_shapeÚfuncs                       r   rE   zAdafactorBigVision.stepz   s]  € àˆØÐÝÔ"Ñ$Ô$ð !ð !Ø�w‘y”y�ð!ð !ð !ñ !ô !ð !ð !ð !ð !ð !ð !øøøð !ð !ð !ð !ð Ô&ð L	ñ L	ˆEØ!ÐØˆEØˆMØˆMØˆKØˆKØˆHà˜8”_ð (<ñ (<�Ø”6�>Øà”6Ô#ð IÝ&Ð'GÑHÔHÐHà ×'Ò'¨Ñ*Ô*Ð*Ø—’˜QœVÑ$Ô$Ð$àœ
 1œ�å�u‘:”: ’?‘?Ý$0¸ÅUÄ]Ð$SÑ$SÔ$S�E˜&‘MàœFœL�EÝ$2ØØ!%Ø/3¬}Ð=UÔ/Vð%ñ %ô %�Mð %Ð0Ø!.™˜˜BÝ$(¨¬¬Ñ$6Ô$6˜	Ø()˜	 "™Ý$(¨¬¬Ñ$6Ô$6˜	Ø()˜	 "™Ø01´×0@Ò0@ÀÑ0KÔ0K˜˜nÑ-Ø01´×0@Ò0@ÀÑ0KÔ0K˜˜nÑ-Ð-å.3Ô.>¸q¼vÕUZÔUjÐ.kÑ.kÔ.k˜˜lÑ+à”} ZÔ0Ð<Ý+0Ô+;¸A¼FÈ$Ì-ÐXhÔJiÐ+jÑ+jÔ+j˜˜iÑ(à×"Ò" 5¨¤=Ñ1Ô1Ð1Ø×$Ò$ U§Y¢Y¨~¸tÑ%DÔ%DÑEÔEÐEØ×$Ò$ U§Y¢Y¨~¸tÑ%DÔ%DÑEÔEÐEØ×"Ò" 5§9¢9¨\¸4Ñ#@Ô#@ÑAÔAÐAØ—’ §	¢	¨)°TÑ :Ô :Ñ;Ô;Ð;Ñ;à�YÔð 0Ý.��å/�àˆDð ð ð Ø'Ð'ðà�eðð ,˜mðð ,˜mð	ð
 (˜Kðð "˜ðð (˜Kðð " ,Ô/Ð/ðð   Ô,Ð,ðð (-Ð-EÔ'FÐ'Fðð ˜%”L�Lðð ˜”;�;ðð # >Ô2Ð2ðð ˜zÔ*Ð*ðð  %Ð%5Ô6Ð6ðð  $)Ð)=Ô#>Ð#>ð!ð" " -Ô0Ð0ð#ð$ ˜iÔ(Ð(ð%ð& /4Ð4LÔ.MÐW�t”} TÔ*Ð*ÐSWøð'ð ñ ð, ˆs   ˜/¯3¶3r   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r=   r9   r   Úfloatr"   r   r   r<   rI   Úboolr@   rL   Úno_gradrE   Ú__classcell__)rC   s   @r   r&   r&   1   s…  ø€ € € € € ðð ð Ø*,Ø #Ø !Ø$Ø(+Ø6;´nØ#'Ø"%Ø26Ø %Ø!Ø+0ð/+ð" ',ð#/+ð /+ð /+àð/+ð ð/+ð %(ð	/+ð
 ð/+ð ð/+ð ð/+ð ˜u”oð/+ð " # u¤{Ð"2Ô3ð/+ð ˜%”ð/+ð  ð/+ð !)¨¤ð/+ð ð/+ð ð/+ð %)ð/+ð" ˜d”^ð#/+ð /+ð /+ð /+ð /+ð /+ðbfð fð fð fð fð" €U„]�_„_ðTð Tð Tñ „_ðTð Tð Tð Tð Tr   r&   r*   r\   r]   r^   r_   r`   ra   rb   r.   r1   r+   r2   r/   r0   r3   r4   r5   rc   c          	      ó  — t          | ¦  «        D �]ø\  }}||         }||         }||         }||         }||         }||         }|
€|j        t          j        k    rdnd}
|                     d¦  «         t          |¦  «        }t          j        |¦  «        rNt          j        ||j        |j        ¬¦  «        }t          j	        |dt          j
        || ¦  «        z
  ¦  «        }nt          |d|| z  z
  ¦  «        }d|z
  }t          j        |¦  «        |
z   }|€»t          |j        d|	¬¦  «        \  } }!|                     |                     |!d¬¦  «        |¦  «         |                     |                     | d¬¦  «        |¦  «         |                     | d¬¦  «        }"||"z                       ¦   «         }#|                     ¦   «         }$||#z  |$z  }%n3|€|�J ‚|                     ||¦  «         ||                     ¦   «         z  }%|�Y|%                     d	¦  «        |%                     ¦   «         d
z  z  }&|%                     |&|z                       d¬¦  «        ¦  «         |�ô|�ò||j        k    rG|                     |%                     |¦  «        d|z
  ¦  «         |                     |j        ¦  «        }%n-|                     |%d|z
  ¦  «         |                     ¦   «         }%|rq|%|z  dk                         |j        ¦  «        }'|'                     |'                     ¦   «                              d¬¦  «        ¦  «         |%                     |'¦  «         |%                     |¦  «         |dk    r{|r:|€|                     d|z
  ¦  «         n^|                     d||z  |z  z
  ¦  «         n?|€|                     d||z  z
  ¦  «         n!|                     d|d	z  |z  |z  z
  ¦  «         |                     |%d¬¦  «         �Œúd S )NgH¯¼šò×z>g ÂëþKH´9r	   )rI   rH   r'   T)r   )ÚdimÚkeepdimr   g      à?)Úminr   gü©ñÒMbP?g      ð¿)Úalpha)r!   rI   r=   r8   Úadd_r
   rR   ÚtensorrH   ÚminimumÚpowrƒ   Úsquarer$   r   Úlerp_ÚmeanÚrsqrtÚnormÚnumelÚdiv_Úclamp_rS   ÚcloneÚmul_)(r*   r\   r]   r^   r_   r`   ra   rb   r.   r   r1   r+   r2   r/   r0   r3   r4   r5   rc   r   Úparamre   rX   rY   r[   rJ   Ústep_trE   Úbeta2_cap_tÚbeta2_tÚone_minus_beta2_tÚgrad_sqrrs   rt   Úrow_col_meanÚ
row_factorÚ
col_factorÚupdateÚ
update_rmsÚmasks(                                           r   rn   rn   Ò   s-  € õ, ˜fÑ%Ô%ð T'ñ T'‰ˆˆ5Ø�QŒxˆØ$ QÔ'ˆØ$ QÔ'ˆØ  ”^ˆ
Ø˜1”+ˆØ˜Q”ˆØˆ;àœ*­¬Ò5Ð5�$�$¸5ˆCð 	�Š�A‰ŒˆÝ˜&Ñ!Ô!ˆÝŒ?˜4Ñ Ô ð 	CÝœ, y¸¼
È4Ì;ÐWÑWÔWˆKÝ”m K°µu´yÀÈÀ|Ñ7TÔ7TÑ1TÑUÔUˆGˆGå˜) S¨4°[°LÑ+AÑ%AÑBÔBˆGØ ™KÐå”< Ñ%Ô%¨Ñ+ˆàÐå# D¤J°ÐMcÐdÑdÔd‰FˆB�Ø×Ò˜xŸ}š}°¸T˜}ÑBÔBÐDUÑVÔVÐVØ×Ò˜xŸ}š}°¸T˜}ÑBÔBÐDUÑVÔVÐVð
 (×,Ò,°¸TÐ,ÑBÔBˆLØ&¨Ñ5×<Ò<Ñ>Ô>ˆJØ%×+Ò+Ñ-Ô-ˆJà˜JÑ&¨Ñ3ˆFˆFð  Ð'¨LÐ,@Ð,@Ð@Ø×Ò˜XÐ'8Ñ9Ô9Ð9Ø˜J×,Ò,Ñ.Ô.Ñ.ˆFð Ð)ØŸš Q™œ¨6¯<ª<©>¬>¸SÑ+@ÑAˆJØ�KŠK˜Ð&8Ñ8×@Ò@ÀSÐ@ÑIÔIÑJÔJÐJð Ð GÐ$7Ø ¤Ò+Ð+Ø—’˜fŸiši¨Ñ7Ô7¸¸X¹ÑFÔFÐFØ Ÿš D¤JÑ/Ô/��à—’˜f a¨(¡lÑ3Ô3Ð3Ø Ÿš™œ�àð "à ™¨Ò)×-Ò-¨d¬jÑ9Ô9�Ø—	’	˜$Ÿ)š)™+œ+×,Ò,°Ð,Ñ6Ô6Ñ7Ô7Ð7Ø—’˜DÑ!Ô!Ð!ð 	�Š�B‰Œˆð ˜1ÒÐØð Gà�>Ø—J’J˜r LÑ0Ñ1Ô1Ð1Ð1ð —J’J˜r R¨&¡[°LÑ$@Ñ@ÑAÔAÐAÐAð �>Ø—J’J˜r B¨Ñ$5Ñ5Ñ6Ô6Ð6Ð6ð —J’J˜r R¨1¡W¨vÑ%5¸Ñ$EÑEÑFÔFÐFð 	�
Š
�6 ˆ
Ñ&Ô&Ð&Ñ&ðiT'ð T'r   c                ó   — J d¦   «         ‚)NFz2multi-tensor fn (foreach=True) not implemented yetr   )r*   r\   r]   r^   r_   r`   ra   rb   r.   r   r1   r+   r2   r/   r0   r3   r4   r5   rc   s                      r   rm   rm   ?  s   € ð. GÐFÑFÔFˆ5r   )r{   Útypingr   r   r   r   r=   r   Útorch.optimr   Ú_helpersr
   r   r   Ú_typesr   r"   r}   r$   r&   r|   r<   rI   rn   rm   r   r   r   ú<module>r¤      s�  ððð ð 0Ð /Ð /Ð /Ð /Ð /Ð /Ð /Ð /Ð /Ð /Ð /à €€€Ø Ð Ð Ð Ð Ð Ø !Ð !Ð !Ð !Ð !Ð !à @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ø Ð Ð Ð Ð Ð ð<Ø�S˜#�XŒð<àð<ð !$ð<ð ˆe�C˜�HŒoÔð	<ð <ð <ð <ð4^ð ^ð ^ð ^ð ^˜ñ ^ô ^ð ^ðBj'Ø�V”ðj'à�FŒ|ðj'ð ˜H VÔ,Ô-ðj'ð ˜H VÔ,Ô-ð	j'ð
 ˜( 6Ô*Ô+ðj'ð �x Ô'Ô(ðj'ð ˜&”\ðj'ð ðj'ð ðj'ð !$ðj'ð ðj'ð ðj'ð ðj'ð ˜5”/ðj'ð  ˜c 5¤;Ð.Ô/ð!j'ð" % UœOð#j'ð$ ð%j'ð& ð'j'ð( ˜”ð)j'ð j'ð j'ð j'ðZGØ�V”ðGà�FŒ|ðGð ˜H VÔ,Ô-ðGð ˜H VÔ,Ô-ð	Gð
 ˜( 6Ô*Ô+ðGð �x Ô'Ô(ðGð ˜&”\ðGð ðGð ðGð !$ðGð ðGð ðGð ðGð ˜5”/ðGð  ˜c 5¤;Ð.Ô/ð!Gð" % UœOð#Gð$ ð%Gð& ð'Gð( ˜”ð)Gð Gð Gð Gð Gð Gr   