§
    ·¯¸jY  ã                   óB   — d Z ddlZddlmZ ddlZ G d„ de¦  «        ZdS )a6   Nvidia NovoGrad Optimizer.
Original impl by Nvidia from Jasper example:
    - https://github.com/NVIDIA/DeepLearningExamples/blob/master/PyTorch/SpeechRecognition/Jasper
Paper: `Stochastic Gradient Methods with Layer-wise Adaptive Moments for Training of Deep Networks`
    - https://arxiv.org/abs/1905.11286
é    N)Ú	Optimizerc                   ól   ‡ — e Zd ZdZ	 	 	 	 	 	 dˆ fd„	Zˆ fd„Z ej        ¦   «         dd
„¦   «         Zˆ xZ	S )Ú
NvNovoGrada(  
    Implements Novograd algorithm.

    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 1e-3)
        betas (Tuple[float, float], optional): coefficients used for computing
            running averages of gradient and its square (default: (0.95, 0.98))
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        grad_averaging: gradient averaging
        amsgrad (boolean, optional): whether to use the AMSGrad variant of this
            algorithm from the paper `On the Convergence of Adam and Beyond`_
            (default: False)
    çü©ñÒMbP?©gffffffî?g\�Âõ(\ï?ç:Œ0âŽyE>r   Fc                 ó  •— d|k    s"t          d                     |¦  «        ¦  «        ‚d|k    s"t          d                     |¦  «        ¦  «        ‚d|d         cxk    rdk     s*n t          d                     |d         ¦  «        ¦  «        ‚d|d         cxk    rdk     s*n t          d                     |d         ¦  «        ¦  «        ‚t          ||||||¬	¦  «        }t          t          | ¦  «                             ||¦  «         d S )
Ng        zInvalid learning rate: {}zInvalid epsilon value: {}r   g      ð?z%Invalid beta parameter at index 0: {}é   z%Invalid beta parameter at index 1: {})ÚlrÚbetasÚepsÚweight_decayÚgrad_averagingÚamsgrad)Ú
ValueErrorÚformatÚdictÚsuperr   Ú__init__)
ÚselfÚparamsr   r   r   r   r   r   ÚdefaultsÚ	__class__s
            €úe/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/optim/nvnovograd.pyr   zNvNovoGrad.__init__    s)  ø€ ð �bŠyˆyÝÐ8×?Ò?ÀÑCÔCÑDÔDÐDØ�cŠzˆzÝÐ8×?Ò?ÀÑDÔDÑEÔEÐEØ�e˜A”hÐ$Ð$Ò$Ð$ Ò$Ð$Ð$Ð$ÝÐD×KÒKÈEÐRSÌHÑUÔUÑVÔVÐVØ�e˜A”hÐ$Ð$Ò$Ð$ Ò$Ð$Ð$Ð$ÝÐD×KÒKÈEÐRSÌHÑUÔUÑVÔVÐVÝØØØØ%Ø)Øð
ñ 
ô 
ˆõ 	�j˜$ÑÔ×(Ò(¨°Ñ:Ô:Ð:Ð:Ð:ó    c                 ó˜   •— t          t          | ¦  «                             |¦  «         | j        D ]}|                     dd¦  «         Œd S )Nr   F)r   r   Ú__setstate__Úparam_groupsÚ
setdefault)r   ÚstateÚgroupr   s      €r   r   zNvNovoGrad.__setstate__=   sV   ø€ Ý�j˜$ÑÔ×,Ò,¨UÑ3Ô3Ð3ØÔ&ð 	/ð 	/ˆEØ×Ò˜Y¨Ñ.Ô.Ð.Ð.ð	/ð 	/r   Nc                 óÎ  — d}|�5t          j        ¦   «         5   |¦   «         }ddd¦  «         n# 1 swxY w Y   | j        D �]¢}|d         D �]•}|j        €Œ|j        }|j        rt          d¦  «        ‚|d         }| j        |         }t          |¦  «        dk    rˆd|d<   t          j        |¦  «        |d<   t          j	        g ¦  «         
                    |d         j        ¦  «        |d<   |r5t          j	        g ¦  «         
                    |d         j        ¦  «        |d	<   |d         |d         }	}|r|d	         }
|d
         \  }}|dxx         dz  cc<   t          j        t          j        |d¦  «        ¦  «        }|	dk    r|	                     |¦  «         n-|	                     |¦  «                             |d|z
  ¬¦  «         |rEt          j        |
|	|
¬¦  «         |
                     ¦   «                              |d         ¦  «        }n-|	                     ¦   «                              |d         ¦  «        }|                     |¦  «         |d         dk    r|                     ||d         ¬¦  «         |d         r|                     d|z
  ¦  «         |                     |¦  «                             |¦  «         |                     ||d          ¬¦  «         �Œ—�Œ¤|S )z²Performs a single optimization step.

        Arguments:
            closure (callable, optional): A closure that reevaluates the model
            and returns the loss.
        Nr   z#Sparse gradients are not supported.r   r   ÚstepÚexp_avgÚ
exp_avg_sqÚmax_exp_avg_sqr   r
   é   )Úalpha)Úoutr   r   r   r   )ÚtorchÚenable_gradr   ÚgradÚ	is_sparseÚRuntimeErrorr    ÚlenÚ
zeros_likeÚzerosÚtoÚdeviceÚsumÚpowÚcopy_Úmul_Úadd_ÚmaxÚsqrtÚdiv_)r   ÚclosureÚlossr!   Úpr,   r   r    r$   r%   r&   Úbeta1Úbeta2ÚnormÚdenoms                  r   r#   zNvNovoGrad.stepB   s/  € ð ˆØÐÝÔ"Ñ$Ô$ð !ð !Ø�w‘y”y�ð!ð !ð !ñ !ô !ð !ð !ð !ð !ð !ð !øøøð !ð !ð !ð !ð Ô&ð 3	4ñ 3	4ˆEØ˜8”_ð 24ñ 24�Ø”6�>ØØ”v�Ø”>ð NÝ&Ð'LÑMÔMÐMØ 	Ô*�àœ
 1œ�õ �u‘:”: ’?�?Ø$%�E˜&‘Må',Ô'7¸Ñ':Ô':�E˜)Ñ$å*/¬+°b©/¬/×*<Ò*<¸UÀ9Ô=MÔ=TÑ*UÔ*U�E˜,Ñ'Øð ^å27´+¸b±/´/×2DÒ2DÀUÈ9ÔEUÔE\Ñ2]Ô2]˜Ð.Ñ/à&+¨IÔ&6¸¸lÔ8K˜�Øð =Ø%*Ð+;Ô%<�NØ$ Wœ~‘��uà�f��” Ñ"��‘å”y¥¤¨4°Ñ!3Ô!3Ñ4Ô4�à ’?�?Ø×$Ò$ TÑ*Ô*Ð*Ð*à—O’O EÑ*Ô*×/Ò/°¸AÀ¹IÐ/ÑFÔFÐFàð Aå”I˜n¨j¸nÐMÑMÔMÐMà*×/Ò/Ñ1Ô1×6Ò6°u¸U´|ÑDÔD�E�Eà&ŸOšOÑ-Ô-×2Ò2°5¸´<Ñ@Ô@�Eà—	’	˜%Ñ Ô Ð Ø˜Ô(¨AÒ-Ð-Ø—I’I˜a u¨^Ô'<�IÑ=Ô=Ð=ØÐ)Ô*ð )Ø—I’I˜a %™iÑ(Ô(Ð(Ø—’˜UÑ#Ô#×(Ò(¨Ñ.Ô.Ð.à—’�w u¨T¤{ l�Ñ3Ô3Ð3Ñ3ñe24ðh ˆs   ˜/¯3¶3)r   r   r   r   FF)N)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r*   Úno_gradr#   Ú__classcell__)r   s   @r   r   r      s¤   ø€ € € € € ðð ð* ØØØØ Øð;ð ;ð ;ð ;ð ;ð ;ð:/ð /ð /ð /ð /ð
 €U„]�_„_ðAð Að Añ „_ðAð Að Að Að Ar   r   )rF   r*   Útorch.optim.optimizerr   Úmathr   © r   r   ú<module>rL      ss   ððð ð €€€Ø +Ð +Ð +Ð +Ð +Ð +Ø €€€ðwð wð wð wð w�ñ wô wð wð wð wr   