§
    ·¯¸j.  ã                   ó:   — d Z ddlZddlmZ  G d„ de¦  «        ZdS )an   PyTorch LARS / LARC Optimizer

An implementation of LARS (SGD) + LARC in PyTorch

Based on:
  * PyTorch SGD: https://github.com/pytorch/pytorch/blob/1.7/torch/optim/sgd.py#L100
  * NVIDIA APEX LARC: https://github.com/NVIDIA/apex/blob/master/apex/parallel/LARC.py

Additional cleanup and modifications to properly support PyTorch XLA.

Copyright 2021 Ross Wightman
é    N)Ú	Optimizerc                   ór   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 dˆ fd„	Zˆ fd„Z ej        ¦   «         dd
„¦   «         Zˆ xZ	S )ÚLarsa»   LARS for PyTorch
    
    Paper: `Large batch training of Convolutional Networks` - https://arxiv.org/pdf/1708.03888.pdf

    Args:
        params (iterable): iterable of parameters to optimize or dicts defining parameter groups.
        lr (float, optional): learning rate (default: 1.0).
        momentum (float, optional): momentum factor (default: 0)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        dampening (float, optional): dampening for momentum (default: 0)
        nesterov (bool, optional): enables Nesterov momentum (default: False)
        trust_coeff (float): trust coefficient for computing adaptive lr / trust_ratio (default: 0.001)
        eps (float): eps for division denominator (default: 1e-8)
        trust_clip (bool): enable LARC trust ratio clipping (default: False)
        always_adapt (bool): always apply LARS LR adapt, otherwise only when group weight_decay != 0 (default: False)
    ç      ð?r   Fçü©ñÒMbP?ç:Œ0âŽyE>c                 óF  •— |dk     rt          d|› �¦  «        ‚|dk     rt          d|› �¦  «        ‚|dk     rt          d|› �¦  «        ‚|r|dk    s|dk    rt          d¦  «        ‚t          ||||||||	|
¬¦	  «	        }t          ¦   «                              ||¦  «         d S )Ng        zInvalid learning rate: zInvalid momentum value: zInvalid weight_decay value: r   z8Nesterov momentum requires a momentum and zero dampening)	ÚlrÚmomentumÚ	dampeningÚweight_decayÚnesterovÚtrust_coeffÚepsÚ
trust_clipÚalways_adapt)Ú
ValueErrorÚdictÚsuperÚ__init__)ÚselfÚparamsr
   r   r   r   r   r   r   r   r   ÚdefaultsÚ	__class__s               €ú_/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/optim/lars.pyr   zLars.__init__#   sà   ø€ ð �Š8ˆ8ÝÐ;°rÐ;Ð;Ñ<Ô<Ð<Ø�cŠ>ˆ>ÝÐB¸ÐBÐBÑCÔCÐCØ˜#ÒÐÝÐJ¸LÐJÐJÑKÔKÐKØð 	Y˜ Qš˜¨)°qª.¨.ÝÐWÑXÔXÐXåØØØØ%ØØ#ØØ!Ø%ð

ñ 

ô 

ˆõ 	‰Œ×Ò˜ Ñ*Ô*Ð*Ð*Ð*ó    c                 óŠ   •— t          ¦   «                              |¦  «         | j        D ]}|                     dd¦  «         Œd S )Nr   F)r   Ú__setstate__Úparam_groupsÚ
setdefault)r   ÚstateÚgroupr   s      €r   r   zLars.__setstate__F   sP   ø€ Ý‰Œ×Ò˜UÑ#Ô#Ð#ØÔ&ð 	0ð 	0ˆEØ×Ò˜Z¨Ñ/Ô/Ð/Ð/ð	0ð 	0r   Nc           
      ó(  — d}|�5t          j        ¦   «         5   |¦   «         }ddd¦  «         n# 1 swxY w Y   | j        D �]Ï}|d         }|d         }|d         }|d         }|d         }|d         }	|d         D �]’}
|
j        €Œ|
j        }|d	k    s|d
         rÀ|
                     d¦  «        }|                     d¦  «        }||z  |||z  z   |	z   z  }t          j        |d	k    t          j        |d	k    |d¦  «        d¦  «        }|d         rt          j        ||d         z  d¬¦  «        }|                     |
|¬¦  «         |                     |¦  «         |d	k    rŽ| j	        |
         }d|vr,t          j
        |¦  «                             ¦   «         x}|d<   n5|d         }|                     |¦  «                             |d|z
  ¬¦  «         |r|                     ||¬¦  «        }n|}|
                     ||d          ¬¦  «         �Œ”�ŒÑ|S )z¡Performs a single optimization step.

        Args:
            closure (callable, optional): A closure that reevaluates the model and returns the loss.
        Nr   r   r   r   r   r   r   r   r   g       @r   r   r
   )Úmax)ÚalphaÚmomentum_buffer)ÚtorchÚenable_gradr   ÚgradÚnormÚwhereÚclampÚadd_Úmul_r!   ÚcloneÚdetachÚadd)r   ÚclosureÚlossr"   r   r   r   r   r   r   Úpr)   Úw_normÚg_normÚtrust_ratioÚparam_stateÚbufs                    r   Ústepz	Lars.stepK   sœ  € ð ˆØÐÝÔ"Ñ$Ô$ð !ð !Ø�w‘y”y�ð!ð !ð !ñ !ô !ð !ð !ð !ð !ð !ð !øøøð !ð !ð !ð !ð Ô&ð ,	1ñ ,	1ˆEØ  Ô0ˆLØ˜ZÔ(ˆHØ˜kÔ*ˆIØ˜ZÔ(ˆHØ Ô.ˆKØ˜”,ˆCà˜8”_ð $1ñ $1�Ø”6�>ØØ”v�ð   1Ò$Ð$¨¨nÔ(=Ð$ØŸVšV C™[œ[�FØ!ŸYšY s™^œ^�FØ"-°Ñ"6¸&À6ÈLÑCXÑ:XÐ[^Ñ:^Ñ"_�Kõ #(¤+Ø š
Ýœ F¨Q¢J°¸SÑAÔAØñ#ô #�Kð
 ˜\Ô*ð VÝ&+¤k°+ÀÀdÄÑ2KÐQTÐ&UÑ&UÔ&U˜Ø—I’I˜a |�IÑ4Ô4Ð4Ø—I’I˜kÑ*Ô*Ð*ð ˜q’=�=Ø"&¤*¨Q¤-�KØ(°Ð;Ð;Ý?D¼{È4Ñ?PÔ?P×?WÒ?WÑ?YÔ?YÐY˜˜kÐ*;Ñ<Ð<à)Ð*;Ô<˜ØŸš Ñ*Ô*×/Ò/°¸BÀ¹NÐ/ÑKÔKÐKØð #Ø#Ÿxšx¨°8˜xÑ<Ô<˜˜à"˜à—’�t E¨$¤K <�Ñ0Ô0Ð0Ñ0ñI$1ðL ˆs   ˜/¯3¶3)	r   r   r   r   Fr   r   FF)N)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r'   Úno_gradr:   Ú__classcell__)r   s   @r   r   r      s¦   ø€ € € € € ðð ð( ØØØØØØØØð!+ð !+ð !+ð !+ð !+ð !+ðF0ð 0ð 0ð 0ð 0ð
 €U„]�_„_ð9ð 9ð 9ñ „_ð9ð 9ð 9ð 9ð 9r   r   )r>   r'   Útorch.optim.optimizerr   r   © r   r   ú<module>rC      sj   ððð ð €€€Ø +Ð +Ð +Ð +Ð +Ð +ðtð tð tð tð tˆ9ñ tô tð tð tð tr   