§
    ·¯¸j  ã                   óf   — d Z ddlmZ ddlmZ ddlZddlmZmZm	Z	 ddl
mZ  G d„ d	e¦  «        ZdS )
an   PyTorch impl of LaProp optimizer

Code simplified from https://github.com/Z-T-WANG/LaProp-Optimizer, MIT License

Paper: LaProp: Separating Momentum and Adaptivity in Adam, https://arxiv.org/abs/2002.04839

@article{ziyin2020laprop,
  title={LaProp: a Better Way to Combine Momentum with Adaptive Gradient},
  author={Ziyin, Liu and Wang, Zhikang T and Ueda, Masahito},
  journal={arXiv preprint arXiv:2002.04839},
  year={2020}
}

References for added functionality:
    Cautious Optimizers: https://arxiv.org/abs/2411.16085
    Why Gradients Rapidly Increase Near the End of Training: https://arxiv.org/abs/2506.02285

é    )ÚTuple)Ú	OptimizerNé   )Ú_add_scaled_Ú_init_scalarÚ_validate_scalar)ÚParamsTc                   óš   ‡ — e Zd ZdZ	 	 	 	 	 	 ddeded	eeef         d
edededefˆ fd„Zˆ fd„Z	 e
j        ¦   «         dd„¦   «         Zˆ xZS )ÚLaPropzw LaProp Optimizer

    Paper: LaProp: Separating Momentum and Adaptivity in Adam, https://arxiv.org/abs/2002.04839
    ç-Cëâ6:?©gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?çVçž¯Ò<ç        FÚparamsÚlrÚbetasÚepsÚweight_decayÚcautionÚcorrected_weight_decayc                 ó¸  •— t          d|¦  «         t          d|¦  «         d|d         cxk    rdk     s*n t          d                     |d         ¦  «        ¦  «        ‚d|d         cxk    rdk     s*n t          d                     |d         ¦  «        ¦  «        ‚t          ||||||¬	¦  «        }t	          t
          | ¦  «                             ||¦  «         d S )
Nzlearning rateÚepsilonr   r   ç      ð?z%Invalid beta parameter at index 0: {}r   z%Invalid beta parameter at index 1: {})r   r   r   r   r   r   )r   Ú
ValueErrorÚformatÚdictÚsuperr   Ú__init__)
Úselfr   r   r   r   r   r   r   ÚdefaultsÚ	__class__s
            €úa/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/optim/laprop.pyr   zLaProp.__init__!   sý   ø€ õ 	˜¨"Ñ-Ô-Ð-Ý˜ CÑ(Ô(Ð(Ø�e˜A”hÐ$Ð$Ò$Ð$ Ò$Ð$Ð$Ð$ÝÐD×KÒKÈEÐRSÌHÑUÔUÑVÔVÐVØ�e˜A”hÐ$Ð$Ò$Ð$ Ò$Ð$Ð$Ð$ÝÐD×KÒKÈEÐRSÌHÑUÔUÑVÔVÐVÝØØØØ%ØØ#9ð
ñ 
ô 
ˆõ 	�f�dÑÔ×$Ò$ V¨XÑ6Ô6Ð6Ð6Ð6ó    c                 ó  •— t          ¦   «                              |¦  «         | j        D ]â}|                     dd¦  «         |                     dd¦  «         |d         D ]«}| j                             |i ¦  «        }|sŒ d|v rt          |d         d¬¦  «        |d<   d|v rKt          j        |d	         ¦  «        r1t          |d         |d	         j	        |d	         j
        ¬
¦  «        |d<   d|v rt          |d         d¬¦  «        |d<   Œ¬Œãd S )Nr   Fr   r   ÚstepÚcpu©ÚdeviceÚexp_avg_lr_1r   )Údtyper(   Úexp_avg_lr_2)r   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetr   ÚtorchÚ	is_tensorr*   r(   )r   r/   ÚgroupÚpÚp_stater!   s        €r"   r,   zLaProp.__setstate__;   sF  ø€ Ý‰Œ×Ò˜UÑ#Ô#Ð#ØÔ&ð 	bð 	bˆEØ×Ò˜Y¨Ñ.Ô.Ð.Ø×ÒÐ5°uÑ=Ô=Ð=Ø˜8”_ð bð b�Øœ*Ÿ.š.¨¨BÑ/Ô/�Øð ØØ˜WÐ$Ð$Ý&2°7¸6´?È5Ð&QÑ&QÔ&Q�G˜F‘OØ! WÐ,Ð,µ´ÀÀtÄÑ1MÔ1MÐ,Ý.:Ø Ô/Ø# DœkÔ/Ø$ Tœ{Ô1ð/ñ /ô /�G˜NÑ+ð
 " WÐ,Ð,Ý.:¸7À>Ô;RÐ[`Ð.aÑ.aÔ.a�G˜NÑ+øðbð	bð 	br#   Nc           
      ó†  — d}|�5t          j        ¦   «         5   |¦   «         }ddd¦  «         n# 1 swxY w Y   | j        D �]~}|d         D �]q}|j        €Œ|j        }|j        rt          d¦  «        ‚| j        |         }t          |¦  «        dk    r�t          d¬¦  «        |d<   t          j	        |¦  «        |d<   t          j
        |d	         ¦  «        rt          j	        |d	         ¦  «        nd
|d<   t          d¬¦  «        |d<   t          j	        |¦  «        |d<   |d         |d         }}|d         \  }	}
|d                              d¦  «         d|
z
  }d|	z
  }|                     |
¦  «                             |||¬¦  «         |d         |	z  ||d	         z  z   |d<   |d         |
z  |z   |d<   t          j
        |d	         ¦  «        r€t          j        |d	         d
k    |d	         t          j        |d	         ¦  «        ¦  «        }t          j        |d	         d
k    |d         |z  t          j        |d	         ¦  «        ¦  «        }n|d	         d
k    r|d         |d	         z  nd}|d         }d|z  }|                     |¦  «                             ¦   «                              |d         ¦  «        }||z  }|                     |	¦  «         t%          |||d	         |z  ¦  «         |d         ra||z  dk                         |j        ¦  «        }|                     |                     ¦   «                              d¬¦  «        ¦  «         ||z  }t%          ||| ¦  «         |d         dk    rE|d         r|d	         dz  | j        d	         z  }n|d	         }t%          ||| |d         z  ¦  «         �Œs�Œ€|S )z¶Performs a single optimization step.

        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   z(LaProp does not support sparse gradientsr   r&   r'   r%   Úexp_avgr   r   r)   r+   Ú
exp_avg_sqr   r   )Úvaluer   r   r   gü©ñÒMbP?)Úminr   r   é   )r1   Úenable_gradr-   ÚgradÚ	is_sparseÚRuntimeErrorr/   Úlenr   Ú
zeros_liker2   Úadd_Úmul_Úaddcmul_ÚwhereÚ	ones_likeÚdivÚsqrt_r   Útor*   Údiv_ÚmeanÚclamp_r    )r   ÚclosureÚlossr3   r4   r=   r/   r7   r8   Úbeta1Úbeta2Úone_minus_beta2Úone_minus_beta1Úlr_safeÚbias_correction1Úbias_correction2Ú	step_sizeÚdenomÚstep_of_this_gradÚmaskÚwd_scales                        r"   r%   zLaProp.stepO   sG  € ð ˆØÐÝÔ"Ñ$Ô$ð !ð !Ø�w‘y”y�ð!ð !ð !ñ !ô !ð !ð !ð !ð !ð !ð !øøøð !ð !ð !ð !ð Ô&ð A	Jñ A	JˆEØ˜8”_ð @Jñ @J�Ø”6�>ØØ”v�Ø”>ð SÝ&Ð'QÑRÔRÐRàœ
 1œ�õ �u‘:”: ’?�?Ý$0¸Ð$>Ñ$>Ô$>�E˜&‘Må',Ô'7¸Ñ':Ô':�E˜)Ñ$åMRÌ_Ð]bÐcgÔ]hÑMiÔMiÐ,q­EÔ,<¸UÀ4¼[Ñ,IÔ,IÐ,IÐoq�E˜.Ñ)Ý,8ÀÐ,FÑ,FÔ,F�E˜.Ñ)å*/Ô*:¸1Ñ*=Ô*=�E˜,Ñ'à&+¨IÔ&6¸¸lÔ8K˜�Ø$ Wœ~‘��uà�f”×"Ò" 1Ñ%Ô%Ð%Ø"# e¡)�Ø"# e¡)�ð —’ Ñ&Ô&×/Ò/°°dÀ/Ð/ÑRÔRÐRà(-¨nÔ(=ÀÑ(EÈÐZ_Ð`dÔZeÑHeÑ(e��nÑ%Ø(-¨nÔ(=ÀÑ(EÈÑ(W��nÑ%õ ”? 5¨¤;Ñ/Ô/ð hÝ#œk¨%°¬+¸Ò*;¸UÀ4¼[Í%Ì/ÐZ_Ð`dÔZeÑJfÔJfÑgÔg�GÝ',¤{Ø˜dœ rÒ)Ø˜nÔ-°Ñ7Ýœ¨¨d¬Ñ4Ô4ñ(ô (Ð$Ð$ð OTÐTXÌkÐ]_ÒN_ÐN_ u¨^Ô'<¸uÀT¼{Ñ'JÐ'JÐegÐ$Ø#(¨Ô#8Ð ØÐ 0Ñ0�	à"ŸšÐ'7Ñ8Ô8×>Ò>Ñ@Ô@×EÒEÀeÈEÄlÑSÔS�Ø$(¨5¡LÐ!Ø—’˜UÑ#Ô#Ð#Ý˜WÐ&7¸¸t¼ÀÑ9VÑWÔWÐWà˜Ô#ð -à# d™N¨QÒ.×2Ò2°4´:Ñ>Ô>�DØ—I’I˜dŸiši™kœk×0Ò0°TÐ0Ñ:Ô:Ñ;Ô;Ð;Ø%¨™n�Gå˜Q ¨)¨Ñ4Ô4Ð4à˜Ô(¨AÒ-Ð-ØÐ5Ô6ð /Ø#(¨¤;°!Ñ#3°d´mÀDÔ6IÑ#I˜˜à#(¨¤;˜Ý   A¨ y°5¸Ô3HÑ'HÑIÔIÐIùñA@JðD ˆs   ˜/¯3¶3)r   r   r   r   FF)N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r	   Úfloatr   Úboolr   r,   r1   Úno_gradr%   Ú__classcell__)r!   s   @r"   r   r      sû   ø€ € € € € ðð ð Ø)5ØØ"$Ø!Ø+0ð7ð 7àð7ð ð7ð ˜ ˜Ô&ð	7ð
 ð7ð  ð7ð ð7ð %)ð7ð 7ð 7ð 7ð 7ð 7ð4bð bð bð bð bð( €U„]�_„_ðOð Oð Oñ „_ðOð Oð Oð Oð Or#   r   )r^   Útypingr   Útorch.optimr   r1   Ú_helpersr   r   r   Ú_typesr	   r   © r#   r"   ú<module>rh      s¬   ððð ð$ Ð Ð Ð Ð Ð à !Ð !Ð !Ð !Ð !Ð !Ø €€€à BÐ BÐ BÐ BÐ BÐ BÐ BÐ BÐ BÐ BØ Ð Ð Ð Ð Ð ðCð Cð Cð Cð CˆYñ Cô Cð Cð Cð Cr#   