§
    ·¯¸j+  ã                   ón   — d Z ddlmZmZ ddlZddlmZ ddlmZm	Z	m
Z
mZ ddlmZ  G d„ d	e¦  «        ZdS )
a   PyTorch Lamb optimizer w/ behaviour similar to NVIDIA FusedLamb

This optimizer code was adapted from the following (starting with latest)
* https://github.com/HabanaAI/Model-References/blob/2b435114fe8e31f159b1d3063b8280ae37af7423/PyTorch/nlp/bert/pretraining/lamb.py
* https://github.com/NVIDIA/DeepLearningExamples/blob/master/PyTorch/LanguageModeling/Transformer-XL/pytorch/lamb.py
* https://github.com/cybertronai/pytorch-lamb

Use FusedLamb if you can (GPU). The reason for including this variant of Lamb is to have a version that is
similar in behaviour to APEX FusedLamb if you aren't using NVIDIA GPUs or cannot install/use APEX.

In addition to some cleanup, this Lamb impl has been modified to support PyTorch XLA and has been tested on TPU.

References for added functionality:
    Cautious Optimizers: https://arxiv.org/abs/2411.16085
    Why Gradients Rapidly Increase Near the End of Training: https://arxiv.org/abs/2506.02285

Original copyrights for above sources are below.

Modifications Copyright 2021 Ross Wightman
é    )ÚOptionalÚTupleN)Ú	Optimizeré   )Ú_add_scaled_Ú
_get_valueÚ_init_scalarÚ_validate_scalar)ÚParamsTc                   óÐ   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 dd	ed
ededeeef         dedededee         dededededefˆ fd„Z	ˆ fd„Z
d„ Z ej        ¦   «         dd„¦   «         Zˆ xZS )ÚLambaû  Implements a pure pytorch variant of FuseLAMB (NvLamb variant) optimizer from apex.optimizers.FusedLAMB
    reference: https://github.com/NVIDIA/DeepLearningExamples/blob/master/PyTorch/LanguageModeling/Transformer-XL/pytorch/lamb.py

    LAMB was proposed in:
    - Large Batch Optimization for Deep Learning - Training BERT in 76 minutes:  https://arxiv.org/abs/1904.00962
    - On the Convergence of Adam and Beyond: https://openreview.net/forum?id=ryQu7f-RZ

    Args:
        params: Iterable of parameters to optimize or dicts defining parameter groups.
        lr: Learning rate
        betas: Coefficients used for computing running averages of gradient and its norm.
        eps: Term added to the denominator to improve numerical stability.
        weight_decay: Weight decay
        grad_averaging: Whether apply (1-beta2) to grad when calculating running averages of gradient.
        max_grad_norm: Value used to clip global grad norm.
        trust_clip: Enable LAMBC trust ratio clipping.
        always_adapt: Apply adaptive learning rate to 0.0 weight decay parameter.
        caution: Apply caution.
        decoupled: apply decoupled weight decay
        corrected_weight_decay: apply corrected weight decay (lr**2 / max_lr) when using decoupled_decay
    çü©ñÒMbP?T©gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?ç�íµ ÷Æ°>ç{®Gáz„?ç      ð?FÚparamsÚlrÚbias_correctionÚbetasÚepsÚweight_decayÚgrad_averagingÚmax_grad_normÚ
trust_clipÚalways_adaptÚcautionÚdecoupled_decayÚcorrected_weight_decayc                 óâ   •— t          d|¦  «         t          d|¦  «         t          d|¦  «         t          ||||||||	|
|||¬¦  «        }t          ¦   «                              ||¦  «         d S )Nzlearning rateÚepsilonr   )r   r   r   r   r   r   r   r   r   r   r   r   )r
   ÚdictÚsuperÚ__init__)Úselfr   r   r   r   r   r   r   r   r   r   r   r   r   ÚdefaultsÚ	__class__s                  €ú_/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/optim/lamb.pyr$   zLamb.__init__Z   s�   ø€ õ  	˜¨"Ñ-Ô-Ð-Ý˜ CÑ(Ô(Ð(Ý˜¨Ñ6Ô6Ð6ÝØØ+ØØØ%Ø)Ø'Ø!Ø%ØØ+Ø#9ð
ñ 
ô 
ˆõ 	‰Œ×Ò˜ Ñ*Ô*Ð*Ð*Ð*ó    c                 ó  •— t          ¦   «                              |¦  «         | j        D ]b}|                     dd¦  «         |                     dd¦  «         |                     dd¦  «         d|v rt	          |d         d¬¦  «        |d<   Œcd S )Nr   Fr   r   ÚstepÚcpu©Údevice)r#   Ú__setstate__Úparam_groupsÚ
setdefaultr	   )r%   ÚstateÚgroupr'   s      €r(   r/   zLamb.__setstate__}   s£   ø€ Ý‰Œ×Ò˜UÑ#Ô#Ð#ØÔ&ð 	Jð 	JˆEØ×Ò˜Y¨Ñ.Ô.Ð.Ø×ÒÐ.°Ñ6Ô6Ð6Ø×ÒÐ5°uÑ=Ô=Ð=Ø˜ˆˆÝ ,¨U°6¬]À5Ð IÑ IÔ I��f‘øð	Jð 	Jr)   c                 ó˜  — | j         d         }|€d S g }| j        D ]d}|d         D ]Y}|j        €Œ
|j        }|j        rt	          d¦  «        ‚|                     t          j                             |¦  «        ¦  «         ŒZŒet          j                             t          j	        |¦  «        ¦  «        }||z   
                    d¬¦  «        }|S )Nr   r   zDLamb does not support sparse gradients, consider SparseAdam instead.r   ©Úmin)r&   r0   ÚgradÚ	is_sparseÚRuntimeErrorÚappendÚtorchÚlinalgÚvector_normÚstackÚclamp_)r%   r   Únormsr3   Úpr7   Úglobal_normÚclip_global_norms           r(   Ú_get_clip_grad_normzLamb._get_clip_grad_norm†   sÛ   € Øœ oÔ6ˆØÐ Ø�4àˆØÔ&ð 	=ð 	=ˆEØ˜8”_ð =ð =�Ø”6�>ØØ”v�Ø”>ð oÝ&Ð'mÑnÔnÐnØ—’�Uœ\×5Ò5°dÑ;Ô;Ñ<Ô<Ð<Ð<ð=õ ”l×.Ò.­u¬{¸5Ñ/AÔ/AÑBÔBˆØ'¨-Ñ7×?Ò?ÀCÐ?ÑHÔHÐØÐr)   Nc           
      óÒ  — d}|�5t          j        ¦   «         5   |¦   «         }ddd¦  «         n# 1 swxY w Y   |                      ¦   «         }| j        D �]�}|d         rdnd}|d         \  }}|d         rdnd}|rd|z
  nd}	d|vrt	          d	¬
¦  «        |d<   |d                              d¦  «         t          d„ |d         D ¦   «         ¦  «        sŒ�t          |d         ¦  «        }
|rd||
z  z
  }d||
z  z
  }nd\  }}|d         D �]Ø}|j        €Œ|j        }|�| 	                    |¦  «         | j
        |         }t          |¦  «        dk    r.t          j        |¦  «        |d<   t          j        |¦  «        |d<   |d         |d         }}|                     |¦  «                             ||	¬¦  «         |                     |¦  «                             ||d|z
  ¬¦  «         |                     ¦   «         |dz  z                       |d         ¦  «        }||z   	                    |¦  «        }|d         rq||z  dk                         |j        ¦  «        }| 	                    |                     ¦   «                              d¬¦  «        ¦  «         |                     |¦  «         |d         }|dk    rm|                     dd¦  «        r@|d         r|d         dz  | j        d         z  }n|d         }t-          ||| |z  ¦  «         n|                     ||¬¦  «         |dk    s|d         r”|                     d¦  «        }|                     d¦  «        }||z  }t          j        |dk    t          j        |dk    |d¦  «        d¦  «        }|d         rt          j        |d¬ ¦  «        }|                     |¦  «         t-          |||d          ¦  «         �ŒÚ�Œ’|S )!zµPerforms a single optimization step.
        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   r   r   r   r   r   r+   r,   r-   c              3   ó(   K  — | ]}|j         d uV — Œd S ©N)r7   )Ú.0rA   s     r(   ú	<genexpr>zLamb.step.<locals>.<genexpr>²   s)   è è € ÐCÐC¨a�q”v TÐ)ÐCÐCÐCÐCÐCÐCr)   r   )r   r   Úexp_avgÚ
exp_avg_sq)Úalpha)Úvalueg      à?r   r   r   r5   r   r   Fr   r   é   r   g       @r   )Úmax)r;   Úenable_gradrD   r0   r	   Úadd_Úanyr   r7   Údiv_r2   ÚlenÚ
zeros_likeÚmul_Úaddcmul_ÚsqrtÚtoÚdtypeÚmeanr?   Úgetr&   r   ÚnormÚwhereÚclamp)r%   ÚclosureÚlossÚclip_grad_normr3   r   Úbeta1Úbeta2r   Úbeta3r+   Úbias_correction1Úbias_correction2rA   r7   r2   rJ   rK   ÚdenomÚupdateÚmaskr   Úwd_scaleÚw_normÚg_normÚtrust_ratios                             r(   r+   z	Lamb.step˜   sˆ  € ð ˆØÐÝÔ"Ñ$Ô$ð !ð !Ø�w‘y”y�ð!ð !ð !ñ !ô !ð !ð !ð !ð !ð !ð !øøøð !ð !ð !ð !ð ×1Ò1Ñ3Ô3ˆàÔ&ð T	6ñ T	6ˆEØ#(Ð):Ô#;ÐB˜a˜aÀˆOØ  œ>‰LˆE�5Ø"'Ð(8Ô"9Ð@˜Q˜Q¸qˆNØ!/Ð8�A˜‘I�I°SˆEð ˜UÐ"Ð"Ý ,°EÐ :Ñ :Ô :��f‘Ø�&ŒM×Ò˜qÑ!Ô!Ð!åÐCÐC°5¸´?ÐCÑCÔCÑCÔCð Øå˜e FœmÑ,Ô,ˆDàð >Ø#$ u°¡}Ñ#4Ð Ø#$ u°¡}Ñ#4Ð Ð à5=Ñ2Ð Ð"2à˜8”_ð =6ñ =6�Ø”6�>ØØ”v�à!Ð-Ø—I’I˜nÑ-Ô-Ð-àœ
 1œ�õ �u‘:”: ’?�?å',Ô'7¸Ñ':Ô':�E˜)Ñ$å*/Ô*:¸1Ñ*=Ô*=�E˜,Ñ'à&+¨IÔ&6¸¸lÔ8K˜�ð —’˜UÑ#Ô#×(Ò(¨°UÐ(Ñ;Ô;Ð;Ø—’ Ñ&Ô&×/Ò/°°dÀ!ÀeÁ)Ð/ÑLÔLÐLà#ŸšÑ*Ô*Ð.>À#Ñ.EÑF×LÒLÈUÐSXÌ\ÑZÔZ�Ø!Ð$4Ñ4×:Ò:¸5ÑAÔA�à˜Ô#ð &à" T™M¨AÒ-×1Ò1°$´*Ñ=Ô=�DØ—I’I˜dŸiši™kœk×0Ò0°TÐ0Ñ:Ô:Ñ;Ô;Ð;Ø—K’K Ñ%Ô%Ð%à$ ^Ô4�Ø 1Ò$Ð$Ø—y’yÐ!2°EÑ:Ô:ð ;Ø Ð!9Ô:ð 3Ø',¨T¤{°aÑ'7¸$¼-ÈÔ:MÑ'M˜H˜Hà',¨T¤{˜HÝ$ Q¨¨H¨9°|Ñ+CÑDÔDÐDÐDàŸš A¨\˜Ñ:Ô:Ð:à 1Ò$Ð$¨¨nÔ(=Ð$ð ŸVšV C™[œ[�FØ#Ÿ[š[¨Ñ-Ô-�FØ"(¨6¡/�Kõ #(¤+Ø š
Ýœ F¨Q¢J°¸SÑAÔAØñ#ô #�Kð
 ˜\Ô*ð Hå&+¤k°+À3Ð&GÑ&GÔ&G˜Ø—K’K Ñ,Ô,Ð,å˜Q ¨¨t¬¨Ñ5Ô5Ð5Ñ5ñ{=6ð~ ˆs   ˜/¯3¶3)r   Tr   r   r   Tr   FFFFFrG   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚfloatÚboolr   r   r$   r/   rD   r;   Úno_gradr+   Ú__classcell__)r'   s   @r(   r   r   C   s]  ø€ € € € € ðð ð2 Ø$(Ø)5ØØ"&Ø#'Ø-0Ø$Ø!&Ø!Ø$)Ø+0ð!+ð !+àð!+ð ð!+ð "ð	!+ð
 ˜ ˜Ô&ð!+ð ð!+ð  ð!+ð !ð!+ð $ Eœ?ð!+ð ð!+ð ð!+ð ð!+ð "ð!+ð %)ð!+ð !+ð !+ð !+ð !+ð !+ðFJð Jð Jð Jð Jð ð  ð  ð$ €U„]�_„_ðcð cð cñ „_ðcð cð cð cð cr)   r   )rr   Útypingr   r   r;   Útorch.optimr   Ú_helpersr   r   r	   r
   Ú_typesr   r   © r)   r(   ú<module>r|      s¹   ððð ðr #Ð "Ð "Ð "Ð "Ð "Ð "Ð "à €€€Ø !Ð !Ð !Ð !Ð !Ð !à NÐ NÐ NÐ NÐ NÐ NÐ NÐ NÐ NÐ NÐ NÐ NØ Ð Ð Ð Ð Ð ðyð yð yð yð yˆ9ñ yô yð yð yð yr)   